Include notification and accessList in CSV

I omitted the "/" on the initial commit for the SPAWN error on extensions

.eslintignore

@@ -0,0 +1,2 @@
+!/bin/.*.ts
+!/tools/.*.ts

.eslintrc.json

@@ -0,0 +1,27 @@
+{
+  "env": {
+    "es6": true,
+    "node": true
+  },
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "project": "./tsconfig.json"
+  },
+  "plugins": ["@typescript-eslint"],
+  "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended", "prettier", "prettier/@typescript-eslint"],
+  "rules": {
+    "no-shadow": ["error", { "allow": ["err"] }],
+    "handle-callback-err": "error",
+    "prefer-arrow-callback": "error",
+    "curly": ["error", "multi-or-nest", "consistent"],
+    "no-buffer-constructor": "error",
+    "prefer-const": ["error", {"destructuring": "all"}],
+    "eqeqeq": ["error", "always", {"null": "ignore"}],
+    "no-return-await": "error",
+    "@typescript-eslint/no-explicit-any": 0,
+    "@typescript-eslint/no-use-before-define": ["error", { "functions": false }],
+    "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
+    "require-atomic-updates": 0,
+    "no-prototype-builtins": 0
+  }
+}

.gitignore

@@ -1,27 +1,6 @@
 *~
-lib/*.map
-lib/api-functions.js
-lib/auth.js
-lib/common.js
-lib/config.js
-lib/cwmp.js
-lib/db.js
-lib/fs.js
-lib/nbi.js
-lib/query.js
-lib/server.js
-lib/session.js
 node_modules
-debug
-config/auth.js
-config/config.json
-config/cwmp.key
-config/cwmp.crt
-config/cwmp.ca-bundle
-config/nbi.key
-config/nbi.crt
-config/nbi.ca-bundle
-config/fs.key
-config/fs.crt
-config/fs.ca-bundle
-config/ext
+dist
+bin/.*.ts
+tools/.*.ts
+docs/_build

.npmignore

@@ -1,7 +1,6 @@
 *~
 .npmignore
 
-config/auth.js
 config/config.json
 config/cwmp.key
 config/cwmp.crt
@@ -17,5 +16,10 @@ config/ext
 debug/*
 !debug/.gitkeep
 
-lib/*.coffee
-lib/*.map
+dist
+.eslintrc.json
+.eslintignore
+ava.config.js
+tsconfig.json
+bin/.*
+tools/.*

CHANGELOG.md

@@ -1,5 +1,52 @@
 # Change Log
 
+## 1.2.0-beta.0 (2019-07-30)
+
+- A brand new UI superseding genieacs-gui.
+- New initialization wizard on first run.
+- New expression/query language used in search filters and preset
+preconditions.
+- CPE -> ACS authentication is now supported.
+- New config option (CWMP_KEEP_ALIVE_TIMEOUT) to specify how long to wait for a
+reply from the CPE before closing the TCP connection.
+- Debug logging has been reimplemented utilizing YAML format for logs.
+- Handle 9005 faults (Invalid Parameter Name) gracefully by attempting to
+rediscover the path of the missing parameter recursively.
+- declare() statements not followed by an explicit commit() are now deferred
+until all currently active scripts have been executed.
+- FS_HOSTNAME now defaults to the server's hostname or IP.
+- The API now validates the structure of task objects before saving.
+- New XML parser implementation for better performance. You can revert to the
+old parser by enabling the config option XML_LIBXMLJS. Requires Node.js v11 or
+v10.
+- Performance optimizations. While performance has improved for the majority of
+use cases, there may be situations where performance has degraded. It's
+recommended to revisit your hardware requirements.
+- Connection request authentication no longer uses 'auth.js' file. Instead, the
+connection request authentication behavior can now be customized using an
+'expression'.
+- The config file (config.json) has been deprecated. System configuration (e.g.
+listen ports, worker count) are now recommended to be passed as environment.
+variables. Other general configuration options are stored in the database so as
+to not require service restart for changes to take effect.
+- Optional redis dependency has been removed completely.
+- Tags now allow only alphanumeric characters and underscore.
+- Supported versions of NodeJs and MongoDB are 10.x and up and 2.6 and up
+respectively.
+
+## 1.1.3 (2018-10-23)
+
+- New config option (MAX_COMMIT_ITERATIONS) to avoid max commit iterations
+faults for more complex scripts.
+- Support base64 and hexBinary parameter types.
+- Strict parsing of number values in queries (e.g. "123abc" no longer accepted
+as 123).
+- Mixing $ne and $not operators is not allowed. Now it throws an error instead
+of returning incorrect results.
+- When a task expires, any associated fault is also deleted.
+- API now accepts 'timeout' argument when posting a task.
+- A number of stability fixes.
+
 ## 1.1.2 (2018-02-24)
 
 - A large number of bug fixes as well as stability and performance improvements.

README.md

@@ -1,80 +1,86 @@
 # GenieACS
 
-GenieACS is a fast and lightweight TR-069 auto configuration server (ACS). This
-is the core back end component. A GUI front end is available at
-https://github.com/genieacs/genieacs-gui.
-
-## Features
-
-- **Battle-tested at scale**: Can scale to manage hundreds of thousands and
-potentially millions of devices
-- **Powerful provisioning system**: Scriptable configuration to handle complex
-device provisioning scenarios and other automated operations such as firmware
-upgrade
-- **Extensive API**: A rich HTTP-based API allows easy integration with other
-systems
+GenieACS is a high performance Auto Configuration Server (ACS) for remote
+management of TR-069 enabled devices. It utilizes a declarative and fault
+tolerant configuration engine for automating complex provisioning scenarios at
+scale. It's battle-tested to handle hundreds of thousands and potentially
+millions of concurrent devices.
 
 ## Quick Start
 
+*Important: This is a pre-release branch. Use
+[v1.1](https://github.com/genieacs/genieacs/tree/v1.1) for production
+deployments.*
+
 Install [Node.js](http://nodejs.org/) and [MongoDB](http://www.mongodb.org/).
 Refer to their corresponding documentation for installation instructions. The
 supported versions are:
 
-- Node.js: 6.x and 8.x (8.x recommended)
-- MongoDB: 2.6 through 3.4
+- Node.js: 10.x through 12.x
+- MongoDB: 2.6 through 4.1
 
-Install build tools and libxml2 development files from your system's package
-manager.
+Clone this repo or download the source archive then *cd* into the source
+directory and install the required dependencies:
 
-Then install GenieACS using NPM:
-
-    npm install -g genieacs
-
-Alternatively, you can install from source by cloning the git repository:
-
-    git clone https://github.com/genieacs/genieacs.git
-    cd genieacs
-    git checkout $(git tag -l v1.1.* --sort=-v:refname | head -n 1)
     npm install
-    npm run compile
 
-Before proceeding, find and review the file "config.json" in "config" directory
-where GenieACS is downloaded.
+Now build:
 
-Finally, run the following (from bin directory if installing from source):
+    npm run build
 
-    genieacs-cwmp
+Finally, run the following services from the build that's generated under
+'dist' directory:
 
-This is the service that the CPEs will communicate with. It listens to port 7547
-by default (see config/config.json). Configure the ACS URL of your devices
-accordingly.
+### genieacs-cwmp
 
-    genieacs-nbi
+This is the service that the CPEs will communicate with. It listens on port
+7547 by default. Configure the ACS URL in your devices accordingly.
+
+    ./dist/bin/genieacs-cwmp
+
+You may optionally use [genieacs-sim](https://github.com/genieacs/genieacs-sim)
+as a dummy TR-069 simulator if you don't have a CPE at hand.
+
+### genieacs-nbi
 
 This is the northbound interface module. It exposes a REST API on port 7557 by
-default. This must be running for the GUI front end to work.
+default. This one is only required if you have an external system integrating
+with GenieACS using this API.
 
-    genieacs-fs
+    ./dist/bin/genieacs-nbi
+
+### genieacs-fs
 
 This is the file server from which the CPEs will download firmware images and
 such.
 
-Note: For production deployment make sure to run those as background services.
+    ./dist/bin/genieacs-fs
 
-For further details about installation and configuration, refer to the
-[wiki section](https://github.com/genieacs/genieacs/wiki).
+### genieacs-ui
+
+This serves the web based user interface. It listens on port 3000 by default.
+
+    ./dist/bin/genieacs-ui --ui-jwt-secret secret
+
+The argument *--ui-jwt-secret* supplies the key used for signing browser
+cookies.
+
+The UI has plenty of configuration options. When you open GenieACS's UI in a
+browser you'll be greeted with a database initialization wizard to help you
+populate some initial configuration.
 
 ## Support
 
-The [Users mailing list](http://lists.genieacs.com) is a good place to get
-guidance and help from the community. Head on over and join the conversation!
-In addition, the [wiki](https://github.com/genieacs/genieacs/wiki) provides useful
+The [forum](https://forum.genieacs.com) is a good place to get guidance and
+help from the community. Head on over and join the conversation! In addition,
+the [wiki](https://github.com/genieacs/genieacs/wiki) provides useful
 documentation and tips from GenieACS users.
 
 For commercial support options and professional services, please visit
-[genieacs.com](https://genieacs.com).
+[genieacs.com](https://genieacs.com/support/).
 
 ## License
 
-Copyright 2013-2018 GenieACS Inc. GenieACS is released under the
-[AGPLv3 license terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).
+Copyright 2013-2019 GenieACS Inc. GenieACS is released under the [AGPLv3
+license
+terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

ava.config.js

@@ -0,0 +1,6 @@
+export default {
+  "compileEnhancements": false,
+  "extensions": ["ts"],
+  "require": ["esm", "ts-node/register/transpile-only"]
+};
+

bin/genieacs-cwmp

@@ -1,9 +1,134 @@
-#!/usr/bin/env node
-var path = require('path');
-var fs = require('fs');
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
 
-var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.chdir(dir);
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
 
-var cluster = require(path.resolve(dir, 'lib/cluster'));
-cluster.start('cwmp');
+import * as config from "../lib/config";
+import * as logger from "../lib/logger";
+import * as cluster from "../lib/cluster";
+import * as server from "../lib/server";
+import * as cwmp from "../lib/cwmp";
+import * as db from "../lib/db";
+import * as extensions from "../lib/extensions";
+import * as cache from "../lib/cache";
+import { version as VERSION } from "../package.json";
+
+logger.init("cwmp", VERSION);
+
+const SERVICE_ADDRESS = config.get("CWMP_INTERFACE");
+const SERVICE_PORT = config.get("CWMP_PORT");
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    cache.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect()
+  ]).catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  extensions.killAll().then(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("CWMP_WORKER_PROCESSES");
+
+  logger.info({
+    message: `genieacs-cwmp starting`,
+    pid: process.pid,
+    version: VERSION
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+} else {
+  const ssl = {
+    key: config.get("CWMP_SSL_KEY"),
+    cert: config.get("CWMP_SSL_CERT")
+  };
+
+  process.on("uncaughtException", err => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid
+    });
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  const initPromise = Promise.all([db.connect(), cache.connect()])
+    .then(() => {
+      server.start(
+        SERVICE_PORT,
+        SERVICE_ADDRESS,
+        ssl,
+        cwmp.listener,
+        cwmp.onConnection,
+        0
+      );
+    })
+    .catch(err => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+}

bin/genieacs-ext

@@ -1,5 +1,7 @@
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
+
 /**
- * Copyright 2013-2017  Zaid Abdulla
+ * Copyright 2013-2019  GenieACS Inc.
  *
  * This file is part of GenieACS.
  *
@@ -16,20 +18,19 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
-"use strict";
+
+import { Fault } from "../lib/types";
 
 const jobs = new Set();
 const fileName = process.argv[2];
 let script;
 
-function errorToFault(err) {
-  if (!err)
-    return null;
+function errorToFault(err: Error): Fault {
+  if (!err) return null;
 
-  if (!err.name)
-    return {code: "ext", message: `${err}`};
+  if (!err.name) return { code: "ext", message: `${err}` };
 
-  let fault = {
+  const fault: Fault = {
     code: `ext.${err.name}`,
     message: err.message,
     detail: {
@@ -39,30 +40,36 @@ function errorToFault(err) {
   };
 
   if (err.stack) {
-    fault.detail.stack = err.stack;
+    fault.detail["stack"] = err.stack;
     // Trim the stack trace
-    let stackTrimIndex = fault.detail.stack.match(/\s+at\s[^\s]+\s\(.*extension-wrapper\.js:.+\)/);
-    if (stackTrimIndex)
-      fault.detail.stack = fault.detail.stack.slice(0, stackTrimIndex.index);
+    const stackTrimIndex = fault.detail["stack"].match(
+      /\s+at\s[^\s]+\s\(.*genieacs-ext:.+\)/
+    );
+    if (stackTrimIndex) {
+      fault.detail["stack"] = fault.detail["stack"].slice(
+        0,
+        stackTrimIndex.index
+      );
+    }
   }
 
   return fault;
 }
 
-process.on("uncaughtException", function(err) {
-  let fault = errorToFault(err);
-  jobs.forEach(function(jobId) {
-    process.send([jobId, fault]);
+process.on("uncaughtException", err => {
+  const fault = errorToFault(err);
+  jobs.forEach(jobId => {
+    process.send([jobId, fault, null]);
   });
   jobs.clear();
   process.disconnect();
 });
 
-process.on("message", function(message) {
+process.on("message", message => {
   jobs.add(message[0]);
 
   if (!script) {
-    let cwd = process.env["GENIEACS_CONFIG_DIR"] + "/ext";
+    const cwd = process.env["GENIEACS_EXT_DIR"];
     process.chdir(cwd);
     script = require(`${cwd}/${fileName}`);
   }
@@ -70,21 +77,20 @@ process.on("message", function(message) {
   const funcName = message[1][0];
 
   if (!script[funcName]) {
-    let fault = {
+    const fault = {
       code: "ext",
       message: `No such function '${funcName}' in extension '${fileName}'`
     };
-    process.send([message[0], fault]);
+    process.send([message[0], fault, null]);
     return;
   }
 
-  script[funcName](message[1].slice(1), function(err, res) {
-    if (!jobs.delete(message[0]))
-      return;
+  script[funcName](message[1].slice(1), (err, res) => {
+    if (!jobs.delete(message[0])) return;
 
     process.send([message[0], errorToFault(err), res]);
   });
 });
 
 // Ignore SIGINT
-process.on('SIGINT', function() {});
+process.on("SIGINT", () => {});

bin/genieacs-fs

@@ -1,9 +1,123 @@
-#!/usr/bin/env node
-var path = require('path');
-var fs = require('fs');
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
 
-var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.chdir(dir);
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
 
-var cluster = require(path.resolve(dir, 'lib/cluster'));
-cluster.start('fs');
+import * as config from "../lib/config";
+import * as logger from "../lib/logger";
+import * as cluster from "../lib/cluster";
+import * as server from "../lib/server";
+import { listener } from "../lib/fs";
+import * as db from "../lib/db";
+import * as cache from "../lib/cache";
+import { version as VERSION } from "../package.json";
+
+logger.init("fs", VERSION);
+
+const SERVICE_ADDRESS = config.get("FS_INTERFACE");
+const SERVICE_PORT = config.get("FS_PORT");
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    cache.disconnect(),
+    cluster.worker.disconnect()
+  ]).catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  process.exit(1);
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("FS_WORKER_PROCESSES");
+
+  logger.info({
+    message: `genieacs-fs starting`,
+    pid: process.pid,
+    version: VERSION
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+} else {
+  const ssl = {
+    key: config.get("FS_SSL_KEY"),
+    cert: config.get("FS_SSL_CERT")
+  };
+
+  process.on("uncaughtException", err => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid
+    });
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  const initPromise = Promise.all([db.connect(), cache.connect()])
+    .then(() => {
+      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
+    })
+    .catch(err => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+}

bin/genieacs-nbi

@@ -1,9 +1,127 @@
-#!/usr/bin/env node
-var path = require('path');
-var fs = require('fs');
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
 
-var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.chdir(dir);
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
 
-var cluster = require(path.resolve(dir, 'lib/cluster'));
-cluster.start('nbi');
+import * as config from "../lib/config";
+import * as logger from "../lib/logger";
+import * as cluster from "../lib/cluster";
+import * as server from "../lib/server";
+import { listener } from "../lib/nbi";
+import * as db from "../lib/db";
+import * as extensions from "../lib/extensions";
+import * as cache from "../lib/cache";
+import { version as VERSION } from "../package.json";
+
+logger.init("nbi", VERSION);
+
+const SERVICE_ADDRESS = config.get("NBI_INTERFACE");
+const SERVICE_PORT = config.get("NBI_PORT");
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    cache.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect()
+  ]).catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  extensions.killAll().then(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("NBI_WORKER_PROCESSES");
+
+  logger.info({
+    message: `genieacs-nbi starting`,
+    pid: process.pid,
+    version: VERSION
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+} else {
+  const ssl = {
+    key: config.get("NBI_SSL_KEY"),
+    cert: config.get("NBI_SSL_CERT")
+  };
+
+  process.on("uncaughtException", err => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid
+    });
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  const initPromise = Promise.all([db.connect(), cache.connect()])
+    .then(() => {
+      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
+    })
+    .catch(err => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+}

bin/genieacs-ui

@@ -0,0 +1,139 @@
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
+
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as config from "../lib/config";
+import * as logger from "../lib/logger";
+import * as cluster from "../lib/cluster";
+import * as server from "../lib/server";
+import { listener } from "../lib/ui";
+import * as extensions from "../lib/extensions";
+import * as db from "../lib/ui/db";
+import * as db2 from "../lib/db";
+import * as cache from "../lib/cache";
+import { version as VERSION } from "../package.json";
+
+logger.init("ui", VERSION);
+
+const SERVICE_ADDRESS = config.get("UI_INTERFACE");
+const SERVICE_PORT = config.get("UI_PORT");
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    db2.disconnect(),
+    cache.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect()
+  ]).catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  extensions.killAll().then(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("UI_WORKER_PROCESSES");
+
+  logger.info({
+    message: `genieacs-ui starting`,
+    pid: process.pid,
+    version: VERSION
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+} else {
+  const ssl = {
+    key: config.get("UI_SSL_KEY"),
+    cert: config.get("UI_SSL_CERT")
+  };
+
+  let stopping = false;
+
+  process.on("uncaughtException", err => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid
+    });
+    stopping = true;
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  const _listener = (req, res): void => {
+    if (stopping) res.setHeader("Connection", "close");
+    listener(req, res);
+  };
+
+  const initPromise = Promise.all([db2.connect(), cache.connect()])
+    .then(() => {
+      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
+    })
+    .catch(err => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+}

build/build.ts

@@ -0,0 +1,391 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as path from "path";
+import * as fs from "fs";
+import { promisify } from "util";
+import { rollup, WarningHandler } from "rollup";
+import rollupReplace from "rollup-plugin-replace";
+import rollupJson from "rollup-plugin-json";
+import typescript from "rollup-plugin-typescript";
+import { terser } from "rollup-plugin-terser";
+import webpack from "webpack";
+import postcss from "postcss";
+import postcssImport from "postcss-import";
+import postcssPresetEnv from "postcss-preset-env";
+import cssnano from "cssnano";
+import SVGO from "svgo";
+import * as xmlParser from "../lib/xml-parser";
+
+const MODE = process.env["NODE_ENV"] || "production";
+
+const BUILD_METADATA = new Date()
+  .toISOString()
+  .split(".")[0]
+  .replace(/[^0-9]/g, "");
+
+const INPUT_DIR = path.resolve(__dirname, "..");
+const OUTPUT_DIR = path.resolve(__dirname, "../dist");
+
+const externals = [
+  "path",
+  "fs",
+  "cluster",
+  "os",
+  "tls",
+  "http",
+  "https",
+  "zlib",
+  "crypto",
+  "util",
+  "mongodb",
+  "libxmljs",
+  "vm",
+  "later",
+  "parsimmon",
+  "seedrandom",
+  "querystring",
+  "child_process",
+  "dgram",
+  "url",
+  "iconv-lite",
+  "koa",
+  "koa-router",
+  "koa-compress",
+  "koa-bodyparser",
+  "koa-jwt",
+  "koa-static",
+  "jsonwebtoken",
+  "stream",
+  "mithril",
+  "parsimmon",
+  "yaml",
+  "codemirror",
+  "codemirror/mode/javascript/javascript",
+  "codemirror/mode/yaml/yaml"
+];
+
+function rmDirSync(dirPath): void {
+  if (!fs.existsSync(dirPath)) return;
+  const files = fs.readdirSync(dirPath);
+
+  for (const file of files) {
+    const filePath = `${dirPath}/${file}`;
+    if (fs.statSync(filePath).isFile()) fs.unlinkSync(filePath);
+    else rmDirSync(filePath);
+  }
+  fs.rmdirSync(dirPath);
+}
+
+function stripDevDeps(deps): void {
+  if (!deps["dependencies"]) return;
+  for (const [k, v] of Object.entries(deps["dependencies"])) {
+    if (v["dev"]) delete deps["dependencies"][k];
+    else stripDevDeps(v);
+  }
+  if (!Object.keys(deps["dependencies"]).length) delete deps["dependencies"];
+}
+
+function xmlTostring(xml): string {
+  const children = [];
+  for (const c of xml.children || []) children.push(xmlTostring(c));
+
+  return xml.name === "root" && xml.bodyIndex === 0
+    ? children.join("")
+    : `<${xml.name} ${xml.attrs}>${children.join("")}</${xml.name}>`;
+}
+
+function generateSymbol(id: string, svgStr: string): string {
+  const xml = xmlParser.parseXml(svgStr);
+  const svg = xml.children[0];
+  const svgAttrs = xmlParser.parseAttrs(svg.attrs);
+  let viewBox = "";
+  for (const a of svgAttrs) {
+    if (a.name === "viewBox") {
+      viewBox = `viewBox="${a.value}"`;
+      break;
+    }
+  }
+  const symbolBody = xml.children[0].children
+    .map(c => {
+      return xmlTostring(c);
+    })
+    .join("");
+  return `<symbol id="icon-${id}" ${viewBox}>${symbolBody}</symbol>`;
+}
+
+async function init(): Promise<void> {
+  // Delete any old output directory
+  rmDirSync(OUTPUT_DIR);
+
+  // Create output directory layout
+  fs.mkdirSync(OUTPUT_DIR);
+  fs.mkdirSync(OUTPUT_DIR + "/bin");
+  fs.mkdirSync(OUTPUT_DIR + "/public");
+  fs.mkdirSync(OUTPUT_DIR + "/tools");
+
+  // Create package.json
+  const packageJson = JSON.parse(
+    fs.readFileSync(path.resolve(INPUT_DIR, "package.json")).toString()
+  );
+  delete packageJson["devDependencies"];
+  packageJson["scripts"] = {
+    install: packageJson["scripts"].install,
+    configure: packageJson["scripts"].configure
+  };
+  packageJson["version"] = `${packageJson["version"]}+${BUILD_METADATA}`;
+  fs.writeFileSync(
+    path.resolve(OUTPUT_DIR, "package.json"),
+    JSON.stringify(packageJson, null, 2)
+  );
+
+  // Create npm-shrinkwrap.json
+  const npmShrinkwrapJson = JSON.parse(
+    fs.readFileSync(path.resolve(INPUT_DIR, "npm-shrinkwrap.json")).toString()
+  );
+  npmShrinkwrapJson["version"] = packageJson["version"];
+  stripDevDeps(npmShrinkwrapJson);
+  fs.writeFileSync(
+    path.resolve(OUTPUT_DIR, "npm-shrinkwrap.json"),
+    JSON.stringify(npmShrinkwrapJson, null, 2)
+  );
+}
+
+async function copyStatic(): Promise<void> {
+  const files = [
+    "LICENSE",
+    "README.md",
+    "CHANGELOG.md",
+    "public/logo.svg",
+    "public/favicon.png"
+  ];
+
+  for (const file of files) {
+    fs.copyFileSync(
+      path.resolve(INPUT_DIR, file),
+      path.resolve(OUTPUT_DIR, file)
+    );
+  }
+}
+
+async function generateCss(): Promise<void> {
+  const cssInPath = path.resolve(INPUT_DIR, "ui/css/app.css");
+  const cssOutPath = path.resolve(OUTPUT_DIR, "public/app.css");
+  const cssIn = fs.readFileSync(cssInPath);
+  const cssOut = await postcss([
+    postcssImport,
+    postcssPresetEnv({
+      stage: 3,
+      features: {
+        "nesting-rules": true,
+        "color-mod-function": true
+      }
+    }),
+    cssnano
+  ]).process(cssIn, { from: cssInPath, to: cssOutPath });
+  fs.writeFileSync(cssOutPath, cssOut.css);
+}
+
+async function generateToolsJs(): Promise<void> {
+  for (const bin of ["dump-data-model"]) {
+    const inputFile = path.resolve(INPUT_DIR, `tools/${bin}`);
+    const outputFile = path.resolve(OUTPUT_DIR, `tools/${bin}`);
+    const bundle = await rollup({
+      input: inputFile,
+      external: externals,
+      acorn: {
+        allowHashBang: true
+      },
+      plugins: [
+        rollupReplace({
+          delimiters: ["", ""],
+          "#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only": ""
+        }),
+        typescript({
+          tsconfig: "./tsconfig.json",
+          include: [`tools/${bin}`, "lib/**/*.ts"]
+        }),
+        MODE === "production" ? terser() : null
+      ]
+    });
+
+    await bundle.write({
+      format: "cjs",
+      preferConst: true,
+      sourcemap: "inline",
+      sourcemapExcludeSources: true,
+      banner: "#!/usr/bin/env node",
+      file: outputFile
+    });
+
+    // Mark as executable
+    const mode = fs.statSync(outputFile).mode;
+    fs.chmodSync(outputFile, mode | 73);
+  }
+}
+
+async function generateBackendJs(): Promise<void> {
+  for (const bin of [
+    "genieacs-cwmp",
+    "genieacs-ext",
+    "genieacs-nbi",
+    "genieacs-fs",
+    "genieacs-ui"
+  ]) {
+    const inputFile = path.resolve(INPUT_DIR, `bin/${bin}`);
+    const outputFile = path.resolve(OUTPUT_DIR, `bin/${bin}`);
+    const bundle = await rollup({
+      input: inputFile,
+      external: externals,
+      acorn: {
+        allowHashBang: true
+      },
+      treeshake: {
+        propertyReadSideEffects: false,
+        pureExternalModules: true
+      },
+      plugins: [
+        rollupReplace({
+          delimiters: ["", ""],
+          "#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only": ""
+        }),
+        rollupJson({ preferConst: true }),
+        {
+          resolveId: (importee, importer) => {
+            if (importee.endsWith("/package.json")) {
+              const p = path.resolve(path.dirname(importer), importee);
+              if (p === path.resolve(INPUT_DIR, "package.json"))
+                return path.resolve(OUTPUT_DIR, "package.json");
+            }
+            return null;
+          }
+        },
+        typescript({
+          tsconfig: "./tsconfig.json",
+          include: [`bin/${bin}`, "lib/**/*.ts"]
+        }),
+        MODE === "production" ? terser() : null
+      ]
+    });
+
+    await bundle.write({
+      format: "cjs",
+      preferConst: true,
+      sourcemap: "inline",
+      sourcemapExcludeSources: true,
+      banner: "#!/usr/bin/env node",
+      file: outputFile
+    });
+
+    // Mark as executable
+    const mode = fs.statSync(outputFile).mode;
+    fs.chmodSync(outputFile, mode | 73);
+  }
+}
+
+async function generateFrontendJs(): Promise<void> {
+  const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
+  const outputFile = path.resolve(OUTPUT_DIR, "public/app.js");
+
+  const bundle = await rollup({
+    input: inputFile,
+    external: externals,
+    plugins: [
+      rollupJson({ preferConst: true }),
+      typescript({ tsconfig: "./tsconfig.json" })
+    ],
+    inlineDynamicImports: true,
+    treeshake: {
+      propertyReadSideEffects: false,
+      pureExternalModules: true
+    },
+    onwarn: ((warning, warn) => {
+      // Ignore circular dependency warnings
+      if (warning.code !== "CIRCULAR_DEPENDENCY") warn(warning);
+    }) as WarningHandler
+  });
+
+  await bundle.write({
+    preferConst: true,
+    format: "esm",
+    sourcemap: "inline",
+    sourcemapExcludeSources: true,
+    file: outputFile
+  });
+
+  const webpackConf = {
+    mode: MODE,
+    entry: outputFile,
+    resolve: {
+      aliasFields: ["module"]
+    },
+    devtool: "nosources-source-map",
+    module: {
+      rules: [
+        {
+          test: /\.js$/,
+          use: ["source-map-loader"],
+          enforce: "pre"
+        }
+      ]
+    },
+    output: {
+      path: path.resolve(OUTPUT_DIR, "public"),
+      filename: "app.js"
+    }
+  };
+
+  const stats = await promisify(webpack)(webpackConf);
+  process.stdout.write(stats.toString({ colors: true }) + "\n");
+}
+
+async function generateIconsSprite(): Promise<void> {
+  const svgo = new SVGO({ plugins: [{ removeViewBox: false }] });
+  const symbols = [];
+  const iconsDir = path.resolve(INPUT_DIR, "ui/icons");
+  for (const file of fs.readdirSync(iconsDir)) {
+    const id = path.parse(file).name;
+    const filePath = path.join(iconsDir, file);
+    const { data } = await svgo.optimize(fs.readFileSync(filePath).toString());
+    symbols.push(generateSymbol(id, data));
+  }
+  fs.writeFileSync(
+    path.resolve(OUTPUT_DIR, "public/icons.svg"),
+    `<svg xmlns="http://www.w3.org/2000/svg">${symbols.join("")}</svg>`
+  );
+}
+
+init()
+  .then(() => {
+    Promise.all([
+      copyStatic(),
+      generateCss(),
+      generateIconsSprite(),
+      generateToolsJs(),
+      generateBackendJs(),
+      generateFrontendJs()
+    ])
+      .then(() => {})
+      .catch(err => {
+        process.stderr.write(err.stack + "\n");
+      });
+  })
+  .catch(err => {
+    process.stderr.write(err.stack + "\n");
+  });

config/auth-sample.js

@@ -1,7 +0,0 @@
-"use strict";
-
-function connectionRequest(deviceId, url, username, password, callback) {
-  return callback(username || deviceId, password || "");
-}
-
-exports.connectionRequest = connectionRequest;

config/config-sample.json

@@ -1,12 +0,0 @@
-{
-  "MONGODB_CONNECTION_URL" : "mongodb://127.0.0.1/genieacs",
-  "CWMP_INTERFACE" : "0.0.0.0",
-  "CWMP_PORT" : 7547,
-  "CWMP_SSL" : false,
-  "NBI_INTERFACE" : "0.0.0.0",
-  "NBI_PORT" : 7557,
-  "FS_INTERFACE" : "0.0.0.0",
-  "FS_PORT" : 7567,
-  "FS_HOSTNAME" : "acs.example.com",
-  "DEBUG" : false
-}

docs/administration-faq.rst

@@ -0,0 +1,64 @@
+.. _administration-faq:
+
+Administration FAQ
+==================
+
+.. _administration-faq-duplicate-log-entries:
+
+Duplicate log entries when using :func:`log()` function
+-------------------------------------------------------
+
+Because GenieACS uses a full fledged scripting language for device
+configuration, the only way to guarantee that it has satisfied the 'desired
+state' is by repeatedly executing the script until there's no more
+discrepancies with the current device state. Though it may seem like this will
+cause duplicate requests going to the device, this isn't actually the case
+because device configuration are stated declaratively and that the scripts
+themselves are pure functions in the context of a session (e.g. Date.now()
+always returns the same value within the session).
+
+To illustrate with an example, consider the following script:
+
+.. code:: javascript
+
+  log("Executing script");
+  declare("Device.param", null, {value: 1});
+  commit();
+  declare("Device.param", null, {value: 2});
+
+This will set the value of the 'Device.param' to 1, then to 2. Then as the
+script is run again the value is set back to 1 and so on. A stable state will
+never be reached so GenieACS will execute the script a few times until it gives
+up and throws a fault. This is an edge case that should be avoided. A more
+typical case is where the script is run once or twice. Essentially if an
+execution doesn't result in any request to the CPE or a change in the data
+model then a stable state is deemed to have been reached.
+
+Configurations not pushed to device after factory reset
+---------------------------------------------------------
+
+After a device is reset to its factory default state, the cached data model in
+GenieACS's database needs to be invalidated to force rediscovery. Ensure the
+following lines are called on ``0 BOOTSTRAP`` event:
+
+.. code:: javascript
+
+  const now = Date.now();
+
+  // Clear cached data model to force a refresh
+  clear("Device", now);
+  clear("InternetGatewayDevice", now);
+
+
+Most device parameters are missing
+----------------------------------
+
+For performance reasons (server, client, and network), GenieACS by default only
+fetches parts of the data model that are necessary to satisfy the declarations
+in your provision scripts. Create declarations for any parameters you need
+fetched by default.
+
+If you're unsure and want to explore the available parameters exposed by the
+device, refresh the root parameter (e.g. ``InternetGatewayDevice``) from
+GenieACS's UI. You typically only need to do that one time for a given CPE
+model.

docs/api-reference.rst

@@ -0,0 +1,538 @@
+API Reference
+=============
+
+GenieACS expoeses a rich RESTful API through its NBI component. This document
+serves as a reference for the available APIs.
+
+This API makes use of MongoDB's query language in some of its endpoints. Refer
+to MongoDB's documentation for details.
+
+.. note::
+
+  The examples below use ``curl`` command for simplicity and ease of testing.
+  Query parameters are URL-encoded, but the original pre-encoding values are
+  shown for reference. These examples assume genieacs-nbi is running locally
+  and listening on the default NBI port (7557).
+
+.. warning::
+
+  A common pitfll is not properly percent-encoding special characters in the
+  device ID or query in the URL.
+
+Endpoints
+---------
+
+GET /\<collection\>/?query=\<query\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Search for records in the database (e.g. devices, tasks, presets, files).
+Returns a JSON representation of all items in the given collection that match
+the search criteria.
+
+*collection*: The data collection to search. Could be one of: tasks, devices,
+presets, objects.
+
+*query*: Search query. Refer to MongoDB queries for reference.
+
+Examples
+^^^^^^^^
+
+- Find a device by its ID:
+
+.. code:: javascript
+
+  query = {"_id": "202BC1-BM632w-0000000"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%22202BC1-BM632w-0000000%22%7D'
+
+- Find a device by its MAC address:
+
+.. code:: javascript
+
+  query = {
+    "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress": "20:2B:C1:E0:06:65"
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress%22%3A%2220:2B:C1:E0:06:65%22%7D'
+
+- Search for devices that have not initiated an inform in the last 7 days.
+
+.. code:: javascript
+
+  query = {
+    "_lastInform": {
+      "$lt" : "2017-12-11 13:16:23 +0000"
+    }
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_lastInform%22%3A%7B%22%24lt%22%3A%222017-12-11%2013%3A16%3A23%20%2B0000%22%7D%7D'
+
+- Show pending tasks for a given device:
+
+.. code:: javascript
+
+  query = {"device": "202BC1-BM632w-0000000"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/tasks/?query=%7B%22device%22%3A%22202BC1-BM632w-0000000%22%7D'
+
+- Return specific parameters for a given device:
+
+.. code:: javascript
+
+  query = {"_id": "202BC1-BM632w-0000000"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices?query=%7B%22_id%22%3A%22202BC1-BM632w-0000000%22%7D&projection=InternetGatewayDevice.DeviceInfo.ModelName,InternetGatewayDevice.DeviceInfo.Manufacturer'
+
+The ``projection`` URL param is a comma-separated list of the parameters to receive.
+
+POST /devices/\<device_id\>/tasks?[connection_request]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Enqueue task(s) and optionally trigger a connection request to the device.
+Refer to :ref:`tasks` section for information about the task object format.
+Returns status code 200 if the tasks have been successfully executed, and 202
+if the tasks have been queued to be executed at the next inform.
+
+*device_id*: The ID of the device.
+
+*connection_request*: Indicates that a connection request will be triggered to
+execute the tasks immediatly. Otherwise, the tasks will be queued and be
+processed at the next inform.
+
+The response body is the task object as it is inserted in the database. The
+object will include ``_id`` property which you can use to look up the task
+later.
+
+Examples
+^^^^^^^^
+
+- Refresh all device parameters now:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tasks?connection_request' \
+  -X POST \
+  --data '{"name": "refreshObject", "objectName": ""}'
+
+- Change WiFi SSID and password:
+
+.. code:: javascript
+
+  {
+    "name": "setParameterValues",
+    "parameterValues": [
+      ["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],
+      ["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]
+    ]
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tasks?connection_request' \
+  -X POST \
+  --data '{"name":"setParameterValues", "parameterValues": [["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]]}'
+
+POST /tasks/\<task_id\>/retry
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Retry a faulty task at the next inform.
+
+*task_id*: The ID of the task as returned by 'GET /tasks' request.
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/tasks/5403908ef28ea3a25c138adc/retry' -X POST
+
+DELETE /tasks/\<task_id\>
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete the given task.
+
+*task_id*: The ID of the task as returned by 'GET /tasks' request.
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/tasks/5403908ef28ea3a25c138adc' -X DELETE
+
+DELETE /faults/\<fault_id\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete the given fault.
+
+*fault_id*: The ID of the fault as returned by 'GET /faults' request. The ID
+format is "\<device_id\>:\<channel\>".
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/faults/202BC1-BM632w-0000000:default' -X DELETE
+
+DELETE /devices/\<device_id\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete the given device from the database.
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -X DELETE -i 'http://localhost:7557/devices/202BC1-BM632w-000001'
+
+.. note::
+
+  Note that the device will be registered again when/if it contacts the ACS
+  again (e.g. on the next periodic inform).
+
+POST /devices/\<device_id\>/tags/\<tag\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Assign a tag to a device. Has no effect if such tag already exists.
+
+*device_id*: The ID of the device.
+
+*tag*: The tag to be assigned.
+
+Example
+^^^^^^^
+
+Assign the tag "testing" to a device:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tags/testing' -X POST
+
+DELETE /devices/\<device_id\>/tags/\<tag\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Remove a tag from a device.
+
+*device_id*: The ID of the device.
+
+*tag*: The tag to be removed.
+
+Example
+^^^^^^^
+
+Remove the tag "testing" from a device:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tags/testing' -X DELETE
+
+PUT /presets/\<preset_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Create or update a preset. Returns status code 200 if the preset has been
+added/updated successfully. The body of the request is a JSON representation of
+the preset. Refer to :ref:`presets` section below for details about its format.
+
+*preset_name*: The name of the preset.
+
+Example
+^^^^^^^
+
+Create a preset to set 5 minutes inform interval for all devices tagged with
+"test":
+
+.. code:: javascript
+
+  query = {
+    "weight": 0,
+    "precondition": "{\"_tags\": \"test\"}"
+    "configurations": [
+      {
+        "type": "value",
+        "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable",
+        "value": "true"
+      },
+      {
+        "type": "value",
+        "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval",
+        "value": "300"
+      }
+    ]
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/presets/inform' \
+  -X PUT \
+  --data '{"weight": 0, "precondition": "{\"_tags\": \"test\"}", "configurations": [{"type": "value", "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable", "value": "true"}, {"type": "value", "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval", "value": "300"}]}'
+
+DELETE /presets/\<preset_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+	curl -i 'http://localhost:7557/presets/inform' -X DELETE
+
+PUT /files/\<file_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Upload a new file or overwrite an existing one. Returns status code 200 if the
+file has been added/updated successfully. The file content should be sent as
+the request body.
+
+*file_name*: The name of the uploaded file.
+
+The following file metadata may be sent as request headers:
+
+- ``fileType``: For firmware images it should be "1 Firmware Upgrade Image".
+  Other common types are "2 Web Content" and "3 Vendor Configuration File".
+
+- ``oui``: The OUI of the device model that this file belogs to.
+
+- ``productClass``: The product class of the device.
+
+- ``version``: In case of firmware images, this refer to the firmware version.
+
+Example
+^^^^^^^
+
+Upload a firmware image file:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/files/new_firmware_v1.0.bin' \
+  -X PUT \
+  --data-binary @"./new_firmware_v1.0.bin" \
+  --header "fileType: 1 Firmware Upgrade Image" \
+  --header "oui: 123456" \
+  --header "productClass: ABC" \
+  --header "version: 1.0"
+
+DELETE /files/\<file_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete a previously uploaded file:
+
+.. code:: bash
+
+	curl -i 'http://localhost:7557/files/new_firmware_v1.0.bin' -X DELETE
+
+GET /files/
+~~~~~~~~~~~
+
+Gets all previously uploaded files.
+
+GET /files/?query={"filename":"\<filename\>"}
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Find files using a query.
+
+.. _tasks:
+
+Tasks
+-----
+
+Find the different availabe tasks and their object structure.
+
+``getParameterValues``
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  query = {
+    "name": "getParameterValues",
+    "parameterNames": [
+      "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnectionNumberOfEntries",
+      "InternetGatewayDevice.Time.NTPServer1", "InternetGatewayDevice.Time.Status"
+    ]
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-96318REF-SR360NA0A4%252D0003196/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "getParameterValues", "parameterNames": ["InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnectionNumberOfEntries", "InternetGatewayDevice.Time.NTPServer1", "InternetGatewayDevice.Time.Status"] }'
+
+You may request a single or multiple parameters at once.
+
+After the task has been executed successfully you can then fetch the CPE object
+and read the parameters from the JSON object.
+
+.. code:: javascript
+
+  query = {"_id": "00236a-96318REF-SR360NA0A4%2D0003196"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%2200236a-96318REF-SR360NA0A4%252D0003196%22%7D'
+
+``refreshObject``
+~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "refreshObject", "objectName": "InternetGatewayDevice.WANDevice.1.WANConnectionDevice"}'
+
+``setParameterValues``
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "setParameterValues", "parameterValues": [["InternetGatewayDevice.ManagementServer.UpgradesManaged",false]]}'
+
+Multiple values can be set at once by adding multiple arrays to the
+parameterValues key. For example:
+
+.. code:: javascript
+
+  {
+    name: "setParameterValues",
+    parameterValues: [["InternetGatewayDevice.ManagementServer.UpgradesManaged", false], ["InternetGatewayDevice.Time.Enable", true], ["InternetGatewayDevice.Time.NTPServer1", "pool.ntp.org"]]
+  }
+
+``addObject``
+~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name":"addObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection"}'
+
+``deleteObject``
+~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name":"deleteObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection.1"}'
+
+``reboot``
+~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "reboot"}'
+
+``factoryReset``
+~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "factoryReset"}'
+
+``download``
+~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "download", "file": "mipsbe-6-42-lite.xml"}'
+
+.. _presets:
+
+Presets
+-------
+
+Presets assign a set of configuration or a Provision script to devices based on
+a precondition (search filter), schedule (cron expression), and events.
+
+Precondition
+~~~~~~~~~~~~
+
+The ``precondition`` property is a JSON string representation of the search
+filter to test if the preset applies to a given device. Examples preconditions
+are:
+
+- ``{"param": "value"}``
+- ``{"param": value", "param2": {"$ne": "value2"}}``
+
+Other operators that can be used are ``$gt``, ``$lt``, ``$gte`` and ``$lte``.
+
+Configuration
+~~~~~~~~~~~~~
+
+The configuration property is an array containing the different configurations
+to be applied to a device, as shown below:
+
+.. code:: javascript
+
+  [
+    {
+      "type": "value",
+      "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable",
+      "value": "true"
+    },
+    {
+      "type": "value",
+      "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval",
+      "value": "300"
+    },
+    {
+      "type": "delete_object",
+      "name": "object_parent",
+      "object": "object_name"
+    },
+    {
+      "type": "add_object",
+      "name": "object_parent",
+      "object": "object_name"
+    },
+    {
+      "type": "provision",
+      "name": "YourProvisionName"
+    },
+  ] 
+
+The configuration type ``provision`` triggers a Provision script. In the
+example above, the provision named "YourProvisionName" will be executed.
+
+Provisions
+----------
+
+Create a provision
+~~~~~~~~~~~~~~~~~~
+
+The Provision's JavaScript code is the body of the HTTP PUT request.
+
+.. code:: bash
+
+  curl -X PUT -i 'http://localhost:7557/provisions/mynewprovision' --data 'log("Provision started at " + now);'
+
+Delete a provision
+~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -X DELETE -i 'http://localhost:7557/provisions/mynewprovision'
+
+Get provisions
+~~~~~~~~~~~~~~
+
+Get all provisions:
+
+.. code:: bash
+
+  curl -X GET -i 'http://localhost:7557/provisions/'

docs/conf.py

@@ -0,0 +1,65 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+import json
+
+# -- Project information -----------------------------------------------------
+
+project = 'GenieACS Documentation'
+copyright = '2019, GenieACS Inc.'
+author = 'GenieACS Inc.'
+
+# The full version, including alpha/beta/rc tags
+release = json.load(open("../package.json"))["version"]
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+# html_theme = 'alabaster'
+html_theme = 'sphinx_rtd_theme'
+html_logo = "logo.svg"
+html_theme_options = {
+  "logo_only": True
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+master_doc = 'index'
+
+highlight_language = 'javascript'

docs/cpe-authentication.rst

@@ -0,0 +1,6 @@
+.. _cpe-authentication:
+
+CPE Authentication
+==================
+
+TODO

docs/environment-variables.rst

@@ -0,0 +1,216 @@
+.. _environment-variables:
+
+Environment Variables
+=====================
+
+Configuring GenieACS services can be done through the following environment
+variables:
+
+.. attention::
+
+  All GenieACS environment variables must be prefixed with ``GENIEACS_``.
+
+MONGODB_CONNECTION_URL
+  MongoDB connection string.
+
+  Default: ``mongodb://127.0.0.1/genieacs``
+
+EXT_DIR
+  The directory from which to look up extension scripts.
+
+  Default: ``<installation dir>/config/ext``
+
+EXT_TIMEOUT
+  Timeout (in milliseconds) to allow for calls to extensions to return a
+  response.
+
+  Default: ``3000``
+
+DEBUG_FILE
+  File to dump CPE debug log.
+
+  Default: unset
+
+DEBUG_FORMAT
+  Debug log format. Valid values are 'yaml' and 'json'.
+
+  Default: ``yaml``
+
+LOG_FORMAT
+  The format used for the log entries in ``CWMP_LOG_FILE``, ``NBI_LOG_FILE``,
+  ``FS_LOG_FILE``, and ``UI_LOG_FILE``. Possible values are ``simple`` and
+  ``json``.
+
+  Default: ``simple``
+
+ACCESS_LOG_FORMAT
+  The format used for the log entries in ``CWMP_ACCESS_LOG_FILE``,
+  ``NBI_ACCESS_LOG_FILE``, ``FS_ACCESS_LOG_FILE``, and ``UI_ACCESS_LOG_FILE``.
+  Possible values are ``simple`` and ``json``.
+
+  Default: ``simple``
+
+CWMP_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-cwmp. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+CWMP_PORT
+  The TCP port that genieacs-cwmp listens on.
+
+  Default: ``7547``
+
+CWMP_INTERFACE
+  The network interface that genieacs-cwmp binds to.
+
+  Default: ``0.0.0.0``
+
+CWMP_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+CWMP_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+CWMP_LOG_FILE
+  File to log process related events for genieacs-cwmp. If omitted, logs will
+  go to stderr.
+
+  Default: unset
+
+CWMP_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-cwmp. If omitted, logs will go to
+  stdout.
+
+  Default: unest
+
+NBI_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-nbi. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+NBI_PORT
+  The TCP port that genieacs-nbi listens on.
+
+  Default: ``7557``
+
+NBI_INTERFACE
+  The network interface that genieacs-nbi binds to.
+
+  Default: ``0.0.0.0``
+
+NBI_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unest
+
+NBI_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+NBI_LOG_FILE
+  File to log process related events for genieacs-nbi. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+NBI_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-nbi. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+FS_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-fs. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+FS_PORT
+  The TCP port that genieacs-fs listens on.
+
+  Default: ``7567``
+
+FS_INTERFACE
+  The network interface that genieacs-fs binds to.
+
+  Default: ``0.0.0.0``
+
+FS_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+FS_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+FS_LOG_FILE
+  File to log process related events for genieacs-fs. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+FS_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-fs. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+FS_HOSTNAME
+  The hostname to use when generating the file URL for TR-069 Download
+  requests. Set this if the hostname where genieacs-fs is listening differs
+  from the hostname in the ACS URL.
+
+  Default: the hostname from the ACS URL
+
+UI_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-ui. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+UI_PORT
+  The TCP port that genieacs-ui listens on.
+
+  Default: ``3000``
+
+UI_INTERFACE
+  The network interface that genieacs-ui binds to.
+
+  Default: ``0.0.0.0``
+
+UI_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+UI_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+UI_LOG_FILE
+  File to log process related events for genieacs-ui. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+UI_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-ui. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+UI_JWT_SECRET
+  The key used for signing JWT tokens that are stored in browser cookies. The
+  string can be up to 64 characters in length.
+
+  Default: unset

docs/extensions.rst

@@ -0,0 +1,33 @@
+.. _extensions:
+
+Extensions
+==========
+
+Given that :ref:`provisions` and :ref:`virtual-parameters` are executed in a
+sandbox environment, it is not possible to interact with external sources or
+execute any action that requires OS, file system, or network access. Extensions
+exist to bridge that gap.
+
+Extensions are fully-privileged Node.js modules and as such have access to
+standard Node libraries and 3rd party packages. Functions exposed by the
+extension can be called from Provision scripts using the ``ext()`` function. A
+typical use case for extensions is fetching credentials from a database to have
+that pushed to the device during provisioning.
+
+By default, the extension JS code must be placed under ``config/ext``
+directory. You may need to create that directory if it doesn't already exist.
+
+The example extension below fetches data from an external REST API and returns
+that to the caller:
+
+.. literalinclude:: ext-sample.js
+  :language: javascript
+
+To call this extension from a Provision or a Virtual Parameter script:
+
+.. code:: javascript
+
+  // The arguments "arg1" and "arg2" are passed to the latlong. Though they are
+  // unused in this particular example.
+  const res = ext("ext-sample", "latlong", "arg1", "arg2");
+  log(JSON.stringify(res));

docs/https.rst

@@ -0,0 +1,6 @@
+.. _https:
+
+HTTPS
+=====
+
+TODO

docs/index.rst

@@ -0,0 +1,56 @@
+.. GenieACS documentation master file, created by
+   sphinx-quickstart on Wed Jun  5 13:47:06 2019.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to GenieACS's documentation!
+====================================
+
+.. toctree::
+  :caption: Table of Contents
+
+.. raw:: latex
+
+   \part{Installation}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Installation
+
+  installation-guide
+  environment-variables
+
+.. raw:: latex
+
+   \part{Administration}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Administration
+
+  provisions
+  virtual-parameters
+  administration-faq
+
+.. raw:: latex
+
+   \part{Integration}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Integration
+
+  extensions
+  api-reference
+
+.. raw:: latex
+
+   \part{Security}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Security
+
+  https
+  cpe-authentication
+  roles-and-permissions

docs/installation-guide.rst

@@ -0,0 +1,232 @@
+Installation Guide
+==================
+
+This guide is for installing GenieACS on a single server on any Linux distro
+that uses *systemd* as its init system.
+
+The various GenieACS services are independant of each other and may be
+installed on different servers. You may also run multiple instances of each in
+a load-balancing/failover setup.
+
+.. attention::
+
+  For production deployments make sure to configure TLS and change
+  ``UI_JWT_SECRET`` to a unique and secure string. Refer to :ref:`https`
+  section for how to enable TLS to encrypt traffic.
+
+Prerequisites
+-------------
+
+.. topic:: Node.js
+
+  GenieACS requires Node.js 10.x and up. Refer to https://nodejs.org/ for
+  instructions.
+
+.. topic:: MongoDB
+
+  GenieACS requires MongoDB 3.6 and up. Refer to https://www.mongodb.com/ for
+  instructions.
+
+Install GenieACS
+-------------------
+
+.. topic:: Installing from NPM:
+
+  .. parsed-literal::
+
+    sudo npm install -g --unsafe-perm genieacs@\ |release|
+
+.. topic:: Installing from source
+
+  If you prefer installing from source, such as when running a GenieACS copy
+  with custom patches, refer to README.md file in the source package. Adjust
+  the next steps below accordingly.
+
+Configure systemd
+-----------------
+
+.. topic:: Create a system user to run GenieACS daemons
+
+  .. code:: bash
+
+    sudo useradd --system --no-create-home --user-group genieacs
+
+.. topic:: Create directory to save extensions and environment file
+
+  We'll use :file:`/opt/genieacs/ext/` directory to store extension scripts (if any).
+
+  .. code:: bash
+    
+    mkdir /opt/genieacs
+    mkdir /opt/genieacs/ext
+    chown genieacs:genieacs /opt/genieacs/ext
+
+  Create the file :file:`/opt/genieacs/genieacs.env` to hold our configuration
+  options which we pass to GenieACS as environment variables. See
+  :ref:`environment-variables` section for a list of all available
+  configuration options.
+
+  .. code:: bash
+
+    GENIEACS_CWMP_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-cwmp-access.log
+    GENIEACS_NBI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-nbi-access.log
+    GENIEACS_FS_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-fs-access.log
+    GENIEACS_UI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-ui-access.log
+    GENIEACS_DEBUG_FILE=/var/log/genieacs/genieacs-debug.yaml
+    GENIEACS_EXT_DIR=/opt/genieacs/ext
+    GENIEACS_UI_JWT_SECRET=secret
+  
+  Set file ownership and permissions:
+
+  .. code:: bash
+
+    sudo chown genieacs:genieacs /opt/genieacs/genieacs.env
+    sudo chmod 600 /opt/genieacs/genieacs.env
+
+.. topic:: Create logs directory
+
+  .. code:: bash
+    
+    mkdir /var/log/genieacs
+    chown genieacs:genieacs /var/log/genieacs
+
+.. topic:: Create systemd unit files
+
+  Create a systemd unit file for each of the four GenieACS services. Note that
+  we're using EnvironmentFile directive to read the environment variables from
+  the file we created earlier.
+
+  Each service has two streams of logs: access log and process log. Access logs
+  are configured here to be dumped in a log file under
+  :file:`/var/log/genieacs/` while process logs go to *journald*. Use
+  ``journalctl`` command to view process logs.
+
+  .. attention::
+
+    If the command :command:`systemctl edit --force --full` fails, you can
+    create the unit file manually.
+
+  1. Run the following command to create ``genieacs-cwmp`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force --full genieacs-cwmp
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS CWMP
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=/usr/bin/genieacs-cwmp
+
+      [Install]
+      WantedBy=default.target
+
+  2. Run the following command to create ``genieacs-nbi`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force --full genieacs-nbi
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS NBI
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=/usr/bin/genieacs-nbi
+
+      [Install]
+      WantedBy=default.target
+
+  3. Run the following command to create ``genieacs-fs`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force --full genieacs-fs
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS FS
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=/usr/bin/genieacs-fs
+
+      [Install]
+      WantedBy=default.target
+
+  4. Run the following command to create ``genieacs-ui`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force --full genieacs-ui
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS UI
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=/usr/bin/genieacs-ui
+
+      [Install]
+      WantedBy=default.target
+
+.. topic:: Configure log file rotation using logrotate
+
+  Save the following as :file:`/etc/logrotate.d/genieacs`
+
+  .. code::
+  
+    /var/log/genieacs/*.log /var/log/genieacs/*.yaml {
+        daily
+        rotate 30
+        compress
+        delaycompress
+        dateext
+    }
+
+.. topic:: Enable and start services
+
+  .. code:: bash
+
+    sudo systemctl enable genieacs-cwmp
+    sudo systemctl start genieacs-cwmp
+    sudo systemctl status genieacs-cwmp
+
+    sudo systemctl enable genieacs-nbi
+    sudo systemctl start genieacs-nbi
+    sudo systemctl status genieacs-nbi
+
+    sudo systemctl enable genieacs-fs
+    sudo systemctl start genieacs-fs
+    sudo systemctl status genieacs-fs
+
+    sudo systemctl enable genieacs-ui
+    sudo systemctl start genieacs-ui
+    sudo systemctl status genieacs-ui
+
+  Review the status message for each to verify that the services are running
+  successfully.

docs/logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="310" height="64" viewBox="0 0 82.021 16.933"><path d="M7.341 9.892l.687 1.307c-.305.53-.718.99-1.238 1.378-.851.503-1.543.421-2.076-.244 1.03-.466 1.907-1.28 2.627-2.44M4.676 5.97a2.144 2.144 0 0 1 .237 1.228 8.358 8.358 0 0 1-.518 2.21 7.386 7.386 0 0 0-.424 1.708 2.38 2.38 0 0 0 .168 1.205c.163.393.457.73.883 1.01-.458.138-.859.175-1.202.112-.6-.114-.992-.476-1.175-1.084a3.571 3.571 0 0 1-.093-1.787 11.742 11.742 0 0 1 .435-1.644 11.606 11.606 0 0 0 .432-1.708c.037-.22.041-.45.01-.687a1.66 1.66 0 0 0-.273-.706 1.582 1.582 0 0 0-.6-.506c.245-.046.487-.083.728-.113a1.45 1.45 0 0 1 .728.123c.295.13.517.343.665.639m7.901-1.802l.222.218-1.847 1.678.555.555 1.678-1.846.218.22-1.843 1.679.521.521L13.76 5.35l.222.218-1.847 1.678.556.556 1.764-1.765.375.368-1.584 1.584h.004l-2.065 2.061-1.253.515-1.795-1.348-.007-.008-.008.008.004.007 1.194 1.592-.398.161-1.75-1.75.444-1.08 1.636 1.228.004.003.011-.007-.003-.004-1.39-1.85.225-.553 1.694-1.689h-.004l1.956-1.952.372.368-1.769 1.768.552.552 1.678-1.843m1.456 9.91c1.55-1.549 2.324-3.42 2.324-5.612 0-2.187-.774-4.058-2.323-5.612C12.48 1.304 10.61.53 8.423.53 6.23.53 4.36 1.305 2.81 2.854c-.95.953-1.609 2.025-1.974 3.213l.158-.086c.788-.415 1.486-.203 2.094.638-.55.113-.992.232-1.325.357a1.923 1.923 0 0 0-.503.289c-.337.268-.55.608-.638 1.02a3.98 3.98 0 0 0 .011 1.761c.285 1.527 1.011 2.87 2.178 4.032 1.549 1.552 3.42 2.328 5.612 2.328 2.19 0 4.06-.776 5.612-2.328" fill="white"/><path d="M47.104 1.767c-.456 0-.816.388-.816.849 0 .46.355.816.816.816a.82.82 0 0 0 .833-.816.85.85 0 0 0-.833-.849zM25.602 3.171c-.205 0-.382.046-.5.13-.118.085-.187.201-.187.4v.981l-.151-.031a9.89 9.89 0 0 0-1.872-.173c-1.288 0-2.148.066-2.657.374a1.264 1.264 0 0 0-.548.676c-.119.313-.177.73-.177 1.28v6.123c0 .982.25 1.508.787 1.824.539.315 1.407.41 2.628.41 1.308 0 2.178-.13 2.701-.484.524-.354.745-.932.745-1.942 0-.821-.147-1.37-.543-1.722-.395-.352-1.075-.528-2.186-.528h-2.787V9.255l.148.03c.478.092 1.094.11 1.89.11 1.292 0 2.148-.112 2.658-.453.51-.34.724-.915.724-1.958V3.7c0-.198-.07-.314-.186-.399a.843.843 0 0 0-.487-.13zm6.112 1.29c-1.226 0-2.038.137-2.526.495-.488.357-.697.945-.697 1.947v2.84c0 .935.23 1.54.743 1.932.512.39 1.337.571 2.544.571 1.392 0 2.17-.058 2.567-.196.199-.069.298-.15.355-.246.058-.097.078-.227.078-.405 0-.208-.04-.378-.117-.485-.076-.108-.186-.171-.396-.171-.794 0-1.057.19-2.408.19-.593 0-1.06-.078-1.395-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.717.717 0 0 0 .539-.213c.13-.138.213-.342.213-.62V6.903c0-1.037-.179-1.623-.654-1.97-.476-.348-1.3-.472-2.617-.472zm8.83 0c-1.303 0-2.117.086-2.59.49-.472.403-.665 1.168-.665 2.623v4.002c0 .215.074.338.197.426.123.088.31.134.524.134.223 0 .412-.046.537-.134.125-.088.198-.211.198-.426v-3.81c0-.424.006-.754.04-1.019.036-.265.102-.47.236-.623.135-.152.328-.236.574-.28.246-.044.552-.053.949-.053.4 0 .712.01.963.053.25.044.447.126.587.278.139.151.211.358.25.623.038.265.046.597.046 1.02v3.81c0 .217.07.34.19.427.122.088.308.134.53.134.215 0 .401-.046.525-.134.123-.088.196-.211.196-.426V7.574c0-1.455-.201-2.219-.682-2.623-.48-.404-1.303-.49-2.605-.49zm12.996 0c-1.227 0-2.038.137-2.527.495-.488.357-.696.945-.696 1.947v2.84c0 .935.23 1.54.742 1.932.513.39 1.338.571 2.545.571 1.392 0 2.17-.058 2.567-.196.2-.069.298-.15.355-.246.058-.097.079-.227.079-.405 0-.208-.04-.378-.117-.485-.077-.108-.188-.171-.398-.171-.793 0-1.056.19-2.406.19-.594 0-1.062-.078-1.396-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.716.716 0 0 0 .539-.213c.13-.138.213-.342.213-.62V6.903c0-1.037-.18-1.623-.655-1.97-.475-.348-1.3-.472-2.616-.472zm-6.436.13c-.212 0-.398.047-.522.14-.124.09-.198.22-.198.435v6.41c0 .215.073.338.196.426.123.088.31.134.524.134.223 0 .414-.046.54-.134.124-.088.198-.211.198-.426v-6.41c0-.215-.075-.342-.201-.434-.126-.092-.316-.142-.537-.142zM22.892 5.726c.662 0 1.4.065 1.918.146l.105.016v.92c0 .246-.01.453-.058.632a.757.757 0 0 1-.292.441c-.305.216-.798.263-1.673.263h-.123v-.013c-.761 0-1.26-.002-1.55-.16a.637.637 0 0 1-.307-.395c-.049-.17-.057-.37-.057-.625v-.062c0-.227.03-.418.108-.58a.811.811 0 0 1 .373-.373c.337-.171.826-.21 1.556-.21zm8.839.049c.677 0 1.116.054 1.412.237a.775.775 0 0 1 .316.373c.063.154.085.33.085.535v.811h-3.611V6.92c0-.437.135-.755.439-.928.304-.174.739-.217 1.359-.217zm21.825 0c.677 0 1.117.054 1.413.237a.779.779 0 0 1 .318.373c.062.154.084.33.084.535v.811h-3.613V6.92c0-.437.136-.755.44-.928.304-.174.738-.217 1.358-.217zm-32.701 5.867h2.388c.613 0 1.05.045 1.355.226.306.18.443.505.443.95 0 .429-.179.743-.533.907-.354.164-.861.206-1.568.206-.652 0-1.151-.045-1.51-.186a.952.952 0 0 1-.425-.3.789.789 0 0 1-.15-.483v-1.195z" fill="white"/><path d="M62.339 4.462c-1.399 0-2.18.078-2.58.225-.2.073-.3.157-.357.254-.058.097-.079.22-.079.386 0 .199.043.362.122.469.079.107.19.172.392.172.38 0 .616-.048.947-.096a9.457 9.457 0 0 1 1.475-.098c.58 0 .966.043 1.233.192a.685.685 0 0 1 .292.312c.06.13.082.281.082.452v.857h-2.484c-1.134 0-1.763.19-2.11.559-.346.367-.443.95-.443 1.802 0Line truncated

docs/provisions.rst

@@ -0,0 +1,279 @@
+.. _provisions:
+
+Provisions
+==========
+
+A Provision is a piece of JavaScript code that is executed on the server on a
+per-device basis. It enables implementing complex provisioning scenarios and
+other operations such as automated firmware upgrade rollout. Apart from a few
+special functions, the script is essentially a standard ES6 code executed in
+strict mode.
+
+Provisions are mapped to devices using presets. Note that the added performance
+overhead when using Provisions as opposed to simple preset configuration
+entries is relatively small. Anything that can be done via preset
+configurations can be done using a Provision script. In fact, the now
+deprecated configuration format is still supported primarily for backward
+compatibility and it is recommended to use Provision scripts for all
+configuration.
+
+When assigning a Provision script to a preset, you may pass arguments to the
+script. The arguments can be accessed from the script through the global
+``args`` variable.
+
+.. note::
+
+  Provision scripts may get executed multiple times in a given session.
+  Although all data model-mutating operations are idempotent, a script as a
+  whole may not be. It is, therefore, necessary to repeatedly run the script
+  until there are no more side effects and a stable state is reached.
+
+Built-in functions
+------------------
+
+``declare(path, timestamps, values)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This function is for declaring parameter values to be set, as well as specify
+constraints on how recent you'd like the parameter value (or other attributes)
+to have been refreshed from the device. If the given timestamp is lower than
+the timestamp of the last refresh from the device, then this function will
+return the last known value. Otherwise, the value will be fetched from the
+device before being returned to the caller.
+
+The timestamp argument is an object where the key is the attribute name (e.g.
+``value``, ``object``, ``writable``, ``path``) and the value is an integer
+representing a Unix timestamp.
+
+The values argument is an object similar to the timestamp argument but its
+property values being the parameter values to be set.
+
+The possible attributes in 'timestamps' and 'values' arguments are:
+
+- ``value``: a [<value>, <type>] pair
+
+This attribute is not available for objects or object instances. If the value
+is not a [<value>, <type>] array then it'll assumed to be a value without a
+type and therefore the type will be inferred from the parameter's type.
+
+- ``writable``: boolean
+
+The meaning of this attribute can vary depending on the type of the parameter.
+In the case of regular parameters, it indicates if its value is writable. In
+the case of objects, it's whether or not it's possible to add new object
+instances. In the case of object instances, it indicates whether or not this
+instance can be deleted.
+
+- ``object``: boolean
+
+True if this is an object or object instance, false otherwise.
+
+- ``path``: string
+
+This attribute is special in that it's not a parameter attribute per se, but it
+refers to the presence of parameters matching the given path. For example,
+given the following wildcard path:
+
+``InternetGatewayDevice.LANDevice.1.Hosts.Host.*.MACAddress``
+
+Using a recent timestamp for path in ``declare()`` will result in a sync with
+the device to rediscover all Host instances (``Host.*``). The path attribute
+can also be used to create or delete object instances as described in
+:ref:`path-format` section.
+
+The return value of ``declare()`` is an iterator to access parameters that
+match the given path. Each item in the iterator has the attribute 'path' in
+addition to any other attribute given in the ``declare()`` call. The iterator
+object itself has convenience attribute accessors which come in handy when
+you're expecting a single parameter (e.g. when path does not contain wildcards
+or aliases).
+
+.. code:: javascript
+
+  // Example: Setting the SSID as the last 6 characters of the serial number
+  let serial = declare("Device.DeviceInfo.SerialNumber", {value: 1});
+  declare("Device.LANDevice.1.WLANConfiguration.1.SSID", null, {value: serial.value[0]});
+
+``clear(path, timestamp)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This function invalidates the database copy of parameters (and their child
+parameters) that match the given path and have a last refresh timestamp that is
+less than the given timestamp. The most obvious use for this function is to
+invalidate the database copy of the entire data model after the device has been
+factory reset:
+
+.. code:: javascript
+
+  // Example: Clear cached device data model Note
+  // Make sure to apply only on "0 BOOTSTRAP" event
+  clear("Device", Date.now());
+  clear("InternetGatewayDevice", Date.now());
+
+``commit()``
+~~~~~~~~~~~~
+
+This function commits the pending declarations and performs any necessary sync
+with the device. It's usually not required to call this function as it called
+implicitly at the end of the script and when accessing any property of the
+promise-like object returned by the ``declare()`` function. Calling this
+explicitly is only necessary if you want to control the order in which
+parameters are configured.
+
+``ext(file, function, arg1, arg2, ...)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Execute an extension script and return the result. The first argument is the
+script filename while second argument is the function name within that script.
+Any remaining arguments will be passed to that function. See :ref:`extensions`
+for more details.
+
+``log(message)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Prints out a string in genieacs-cwmp's access log. It's meant to be used for
+debugging. Note that you may see multiple log entries as the script can be
+executed multiple times in a session. See :ref:`this FAQ
+<administration-faq-duplicate-log-entries>`.
+
+.. _path-format:
+
+Path format
+-----------
+
+A parameter path may contain a wildcard (``*``) or an alias filter
+(``[name:value]``). A wildcard segment in a parameter path will apply the
+declared configuration to zero or more parameters that match the given path
+where the wildcard segment can be anything.
+
+An alias filter is like a wildcard, but additionally performs filtering on the
+child parameters based on the key-value pairs provided. For example, the
+following path:
+
+``Device.WANDevice.1.WANConnectionDevice.1.WANIPConnection.[AddressingType:DHCP].ExternalIPAddress``
+
+will return a list of ExternalIPAddress parameters (0 or more) where the
+sibling parameter AddressingType is assigned the value "DHCP".
+
+This can be useful when the exact instance numbers may be different from one
+device to another. It is possible to use more than one key-value pair in the
+alias filter. It's also possible to use multiple filters or use a combination
+of filters and wildcards.
+
+Creating/deleting object instances
+----------------------------------
+
+Given the declarative nature of provisions, we cannot explicitly tell the
+device to create or delete an instance under a given object. Instead, we
+specify the number of instances we want there to be, and based on that GenieACS
+will determine whether or not it needs to create or delete instances. For
+example, the following declaration will ensure we have one and only one
+WANIPConnection object:
+
+.. code:: javascript
+
+  // Example: Ensure we have one and only one WANIPConnection object
+  declare("InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.*", null, {path: 1});
+
+Note the wildcard at the end of the parameter path.
+
+It is also possible to use alias filters as the last path segment which will
+ensure that the declared number of instances is satisfied given the alias
+filter:
+
+.. code:: javascript
+
+  // Ensure that *all* other instances are deleted
+  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[]", null, {path: 0});
+
+  // Add the two entries we care about
+  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[SourceIPAddress:192.168.1.0,SourceNetMask:255.255.255.0]",  {path: now}, {path: 1});
+  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[SourceIPAddress:172.16.12.0,SourceNetMask:255.255.0.0]", {path: now}, {path: 1});
+
+Special GenieACS parameters
+---------------------------
+
+In addition to the parameters exposed in the device's data model through
+TR-069, GenieACS has its own set of special parameters:
+
+``DeviceID``
+~~~~~~~~~~~~
+
+This parameter sub-tree includes the following read-only parameters:
+
+- ``DeviceID.ID``
+- ``DeviceID.SerialNumber``
+- ``DeviceID.ProductClass``
+- ``DeviceID.OUI``
+- ``DeviceID.Manufacturer``
+
+``Tags``
+~~~~~~~~
+
+The ``Tags`` root parameter is used to expose device tags in the data model.
+Tags appear as child parameters that are writable and have boolean value.
+Setting a tag to ``false`` will delete that tag, and setting the value of a
+non-existing tag parameter to ``true`` will create it.
+
+.. code:: javascript
+
+  // Example: Remove "tag1", add "tag2", and read "tag3"
+  declare("Tags.tag1", null, {value: false});
+  declare("Tags.tag2", null, {value: true});
+  let tag3 = declare("Tags.tag3", {value: 1});
+
+``Reboot``
+~~~~~~~~~~
+
+The ``Reboot`` root parameter hold the timestamp of the last reboot command.
+The parameter value is writable and declaring a timestamp value that is larger
+than the current value will trigger a reboot.
+
+.. code:: javascript
+
+  // Example: Reboot the device only if it hasn't been rebooted in the past 300 seconds
+  declare("Reboot", null, {value: Date.now() - (300 * 1000)});
+
+``FactoryReset``
+~~~~~~~~~~~~~~~~
+
+Works like ``Reboot`` parameter but for factory reset.
+
+.. code:: javascript
+
+  // Example: Default the device to factory settings
+  declare("FactoryReset", null, {value: Date.now()});
+
+``Downloads``
+~~~~~~~~~~~~~
+
+The ``Downloads`` sub-tree holds information about the last download
+command(s). A download command is represented as an instance (e.g.
+``Downloads.1``) containing parameters such as ``Download`` (timestamp),
+``LastFileType``, ``LastFileName``. The parameters ``FileType``, ``FileName``,
+``TargetFileName`` and ``Download`` are writable and can be used to trigger a
+new download.
+
+.. code:: javascript
+
+  declare("Downloads.[FileType:1 Firmware Upgrade Image]", {path: 1}, {path: 1});
+  declare("Downloads.[FileType:1 Firmware Upgrade Image].FileName", {value: 1}, {value: "firmware-2017.01.tar"});
+  declare("Downloads.[FileType:1 Firmware Upgrade Image].Download", {value: 1}, {value: Date.now()});
+
+Common file types are:
+
+- ``1 Firmware Upgrade Image``
+- ``2 Web Content``
+- ``3 Vendor Configuration File``
+- ``4 Tone File``
+- ``5 Ringer File``
+
+.. warning::
+
+  Pushing a file to the device is often a service-interrupting operation. It's
+  recommended to only trigger it on certain events such as ``1 BOOT`` or during
+  a predetermined maintenance window).
+
+After the CPE had finished downloading and applying the config file, it will
+send a ``7 TRANSFER COMPLETE`` event. You may use that to trigger a reboot
+after the firmware image or configuration file had been applied.

docs/roles-and-permissions.rst

@@ -0,0 +1,6 @@
+.. _roles-and-permissions:
+
+Roles and Permissions
+=====================
+
+TODO

docs/virtual-parameters.rst

@@ -0,0 +1,105 @@
+.. _virtual-parameters:
+
+Virtual Parameters
+==================
+
+Virtual parameters are user-defined parameters whose values are generated using
+a custom Javascript code. Virtual parameters behave just like regular
+parameters and appear in the data model under ``VirtualParameters.`` path.
+Virtual parameter names cannot contain a period (``.``).
+
+The execution environment for virtual parameters is almost identical to that of
+provisions. See :ref:`provisions` for more details and examples. The only
+differences between the scripts of provisions and virtual parameters are:
+
+- You can't pass custom arguments to virtual parameter scripts. Instead, the
+  variable ``args`` will hold the current vparam timestamps and values as well
+  as the declared timestamps and values. Like this:
+
+.. code:: javascript
+
+  // [<current attr timestamps>, <current attr values>, <declared attr timestamps, declared attr values>]
+  [{path: 1559840000000, value: 1559840000000}, {value: ["cur val", "xsd:string"]}, {path: 1559849387191, value: 1559849387191}, {value: ["new val", "xsd:string"]}]
+
+- Virtual parameter scripts must return an object containing the attributes of
+  this parameter.
+
+.. note::
+
+  Just like a regular parameter, creating a virtual parameter does not
+  automatically add it to the parameter list for a device. It needs to fetched
+  (manually or via a preset) before you can see it in the data model.
+
+Examples
+--------
+
+Unified MAC parameter across different device models
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  // Example: Unified MAC parameter across different device models
+  let m = "00:00:00:00:00:00";
+  let d = declare("Device.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress", {value: Date.now()});
+  let igd = declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANPPPConnection.*.MACAddress", {value: Date.now()});
+
+  if (d.size) {
+    for (let p of d) {
+      if (p.value[0]) {
+        m = p.value[0];
+        break;
+      }
+    }  
+  }
+  else if (igd.size) {
+    for (let p of igd) {
+      if (p.value[0]) {
+        m = p.value[0];
+        break;
+      }
+    }  
+  }
+
+  return {writable: false, value: [m, "xsd:string"]};
+
+Expose an external value as a virtual parameter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  // Example: Expose an external value as a virtual parameter
+  let serial = declare("DeviceID.SerialNumber", {value: 1});
+  if (values.value) {
+    ext("example-ext", "set", serial.value[0], values.value[0]);
+    return {writable: true, value: [values.value[0], "xsd:string"]};
+  }
+  else {
+    let v = ext("example-ext", "get", serial.value[0]);
+    return {writable: true, value: [v, "xsd:string"]};
+  }
+
+Create an editable virtual parameter for WPA passphrase
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  // Example: Create an editable virtual parameter for WPA passphrase
+  let m = "";
+  if (VALUES && VALUES.value) {
+    m = VALUES.value[0];
+    declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", null, {value: m});
+    declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", null, {value: m});
+  }
+  else {
+    let d = declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", {value: Date.now()});
+    let igd = declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", {value: Date.now()});
+
+    if (d.size) {
+      m = d.value[0];
+    }
+    else if (igd.size) {
+      m = igd.value[0];  
+    }
+  }
+
+  return {writable: true, value: [m, "xsd:string"]};

lib/api-functions.coffee

@@ -1,265 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-crypto = require 'crypto'
-dgram = require 'dgram'
-
-config = require './config'
-db = require './db'
-http = require 'http'
-common = require './common'
-util = require 'util'
-URL = require 'url'
-auth = require './auth'
-cache = require './cache'
-
-
-udpConReq = (address, un, key, callback) ->
-  return false if not address
-  [host, port] = address.split(':', 2)
-  if port is undefined
-    port = 80
-  else
-    port = parseInt(port)
-    return false if isNaN(port)
-
-  ts = Math.trunc(Date.now() / 1000)
-  id = Math.trunc(Math.random() * 4294967295)
-  cn = crypto.randomBytes(8).toString('hex')
-  sig = crypto.createHmac('sha1', key).update("#{ts}#{id}#{un}#{cn}").digest('hex')
-  uri = "http://#{address}?ts=#{ts}&id=#{id}&un=#{un}&cn=#{cn}&sig=#{sig}"
-
-  message = Buffer.from("GET #{uri} HTTP/1.1\r\nHost: #{address}\r\n\r\n")
-
-  client = dgram.createSocket({type: 'udp4', reuseAddr: true})
-  UDP_CONNECTION_REQUEST_PORT = config.get('UDP_CONNECTION_REQUEST_PORT')
-  if UDP_CONNECTION_REQUEST_PORT
-    # When a device is NAT'ed, the UDP Connection Request must originate from the same address and port used by the STUN server, in order to traverse the firewall.
-    # This does require that the Genieacs NBI and STUN server are allowed to bind to the same address and port.
-    # The STUN server needs to open its UDP port with the SO_REUSEADDR option, allowing the NBI to also bind to the same port.
-    client.bind({port: UDP_CONNECTION_REQUEST_PORT, exclusive: true})
-
-  count = 3
-  client.send(message, 0, message.length, port, host, f = (err) ->
-    if err or -- count <= 0
-      client.close()
-      return callback(err)
-    client.send(message, 0, message.length, port, host, f)
-  )
-  return true
-
-
-httpConReq = (url, username, password, allowBasicAuth, timeout, callback) ->
-  options = URL.parse(url)
-
-  if options.protocol != 'http:'
-    return callback(new Error('Invalid connection request URL or protocol'))
-
-  # Ensure socket is reused in case of digest authentication
-  options.agent = new http.Agent({maxSockets : 1})
-
-  statusToError = (statusCode) ->
-    switch statusCode
-      when 200, 204
-        null
-      when 401
-        new Error('Incorrect connection request credentials')
-      when 0
-        new Error('Device is offline')
-      else
-        new Error("Unexpected response code from device: #{statusCode}")
-
-  request = http.get(options, (res) ->
-    if res.statusCode == 401 and res.headers['www-authenticate']?
-      authHeader = auth.parseAuthHeader(res.headers['www-authenticate'])
-      if authHeader.method is 'Basic'
-        if not allowBasicAuth
-          request.abort()
-          callback(new Error('Basic HTTP authentication not allowed'))
-          return
-        options.headers = {'Authorization' : auth.basic(username or '', password or '')}
-      else if authHeader.method is 'Digest'
-        options.headers = {
-          'Authorization' : auth.digest(username or '', password or '', options.path, 'GET', null, authHeader)
-        }
-
-      request = http.get(options, (res) ->
-        if res.statusCode == 0
-          # Workaround for some devices unexpectedly closing the connection
-          request = http.get(options, (res) ->
-            callback(statusToError(res.statusCode))
-            res.resume()
-          ).on('error', (err) ->
-            request.abort()
-            callback(statusToError(0))
-          ).on('socket', (socket) ->
-            socket.setTimeout(timeout)
-            socket.on('timeout', () -> request.abort())
-          )
-        else
-          callback(statusToError(res.statusCode))
-        res.resume()
-      ).on('error', (err) ->
-        request.abort()
-        callback(statusToError(0))
-      ).on('socket', (socket) ->
-        socket.setTimeout(timeout)
-        socket.on('timeout', () -> request.abort())
-      )
-    else
-      callback(statusToError(res.statusCode))
-
-    # No listener for data so emit resume
-    res.resume()
-  ).on('error', (err) ->
-    request.abort()
-    callback(statusToError(0))
-  ).on('socket', (socket) ->
-    socket.setTimeout(timeout)
-    socket.on('timeout', () -> request.abort())
-  )
-
-
-connectionRequest = (deviceId, callback) ->
-  CONNECTION_REQUEST_TIMEOUT = config.get('CONNECTION_REQUEST_TIMEOUT', deviceId)
-  CONNECTION_REQUEST_ALLOW_BASIC_AUTH = config.get('CONNECTION_REQUEST_ALLOW_BASIC_AUTH', deviceId)
-  proj = {
-    'Device.ManagementServer.ConnectionRequestURL._value' : 1,
-    'Device.ManagementServer.UDPConnectionRequestAddress._value' : 1,
-    'Device.ManagementServer.ConnectionRequestUsername._value' : 1,
-    'Device.ManagementServer.ConnectionRequestPassword._value' : 1,
-    'InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value' : 1,
-    'InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress._value' : 1,
-    'InternetGatewayDevice.ManagementServer.ConnectionRequestUsername._value' : 1,
-    'InternetGatewayDevice.ManagementServer.ConnectionRequestPassword._value' : 1
-  }
-
-  db.devicesCollection.findOne({_id : deviceId}, proj, (err, device)->
-    return callback(err) if err
-
-    if device.Device? # TR-181 data model
-      connectionRequestUrl = device.Device.ManagementServer.ConnectionRequestURL._value
-      udpConnectionRequestAddress = device.Device.ManagementServer?.UDPConnectionRequestAddress?._value
-      username = device.Device.ManagementServer.ConnectionRequestUsername?._value
-      password = device.Device.ManagementServer.ConnectionRequestPassword?._value
-    else # TR-098 data model
-      connectionRequestUrl = device.InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value
-      udpConnectionRequestAddress = device.InternetGatewayDevice.ManagementServer?.UDPConnectionRequestAddress?._value
-      username = device.InternetGatewayDevice.ManagementServer.ConnectionRequestUsername?._value
-      password = device.InternetGatewayDevice.ManagementServer.ConnectionRequestPassword?._value
-
-    conReq = () ->
-      udpSent = udpConReq(udpConnectionRequestAddress, username, password, (err) -> throw err if err)
-
-      httpConReq(connectionRequestUrl, username, password, CONNECTION_REQUEST_ALLOW_BASIC_AUTH, CONNECTION_REQUEST_TIMEOUT, (err) ->
-        if udpSent
-          return callback()
-        callback(err)
-      )
-
-    if config.auth?.connectionRequest?
-      # Callback is optional for backward compatibility
-      if config.auth.connectionRequest.length > 4
-        return config.auth.connectionRequest(deviceId, connectionRequestUrl, username, password, (u, p) ->
-          username = u
-          password = p
-          conReq()
-        )
-      [username, password] = config.auth.connectionRequest(deviceId, connectionRequestUrl, username, password)
-    conReq()
-  )
-
-
-watchTask = (deviceId, taskId, timeout, callback) ->
-  setTimeout(() ->
-    db.tasksCollection.findOne({_id : taskId}, {'_id' : 1}, (err, task) ->
-      return callback(err) if err
-
-      if not task
-        return callback(null, 'completed')
-
-      db.faultsCollection.findOne({_id : "#{deviceId}:task_#{taskId}"}, {'_id' : 1}, (err, fault) ->
-        return callback(err) if err
-
-        if fault
-          return callback(null, 'fault')
-
-        if (timeout -= 500) <= 0
-          return callback(null, 'timeout')
-
-        watchTask(deviceId, taskId, timeout, callback)
-      )
-    )
-  , 500)
-
-
-sanitizeTask = (task, callback) ->
-  task.timestamp = new Date(task.timestamp ? Date.now())
-  if task.expiry?
-    if common.typeOf(task.expiry) is common.DATE_TYPE or isNaN(task.expiry)
-      task.expiry = new Date(task.expiry)
-    else
-      task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000)
-
-  callback(task)
-
-
-insertTasks = (tasks, callback) ->
-  if tasks? and common.typeOf(tasks) isnt common.ARRAY_TYPE
-    tasks = [tasks]
-  else if not tasks? or tasks.length == 0
-    return callback(null, tasks or [])
-
-  counter = tasks.length
-
-  for task in tasks
-    sanitizeTask(task, (t) ->
-      if t.uniqueKey?
-        db.tasksCollection.remove({device : t.device, uniqueKey : t.uniqueKey}, (err) ->
-        )
-
-      --counter
-      if counter == 0
-        db.tasksCollection.insert(tasks, (err, _tasks) ->
-          #util.log("#{_task.device}: Added task #{_task.name}(#{_task._id})") for _task in _tasks
-          callback(err, _tasks)
-        )
-    )
-
-
-deleteDevice = (deviceId, callback) ->
-  db.tasksCollection.remove({'device' : deviceId}, (err) ->
-    return callback(err) if err
-    db.devicesCollection.remove({'_id' : deviceId}, (err) ->
-      return callback(err) if err
-      db.faultsCollection.remove({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}, (err) ->
-        return callback(err) if err
-        db.operationsCollection.remove({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}, (err) ->
-          return callback(err) if err
-          cache.del("#{deviceId}_tasks_faults_operations", callback)
-        )
-      )
-    )
-  )
-
-
-exports.connectionRequest = connectionRequest
-exports.watchTask = watchTask
-exports.insertTasks = insertTasks
-exports.deleteDevice = deleteDevice

lib/api-functions.ts

@@ -0,0 +1,294 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { parse } from "url";
+import * as db from "./db";
+import * as common from "./common";
+import * as cache from "./cache";
+import {
+  getCurrentSnapshot,
+  getConfig,
+  getConfigExpression
+} from "./local-cache";
+import {
+  httpConnectionRequest,
+  udpConnectionRequest
+} from "./connection-request";
+import { Expression, Task } from "./types";
+
+export async function connectionRequest(deviceId): Promise<void> {
+  const options = {
+    projection: {
+      _deviceId: 1,
+      "Device.ManagementServer.ConnectionRequestURL._value": 1,
+      "Device.ManagementServer.UDPConnectionRequestAddress._value": 1,
+      "Device.ManagementServer.ConnectionRequestUsername._value": 1,
+      "Device.ManagementServer.ConnectionRequestPassword._value": 1,
+      "InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value": 1,
+      "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress._value": 1,
+      "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername._value": 1,
+      "InternetGatewayDevice.ManagementServer.ConnectionRequestPassword._value": 1
+    }
+  };
+
+  const device = await db.devicesCollection.findOne({ _id: deviceId }, options);
+  if (!device) throw new Error("No such device");
+
+  let managementServer,
+    connectionRequestUrl,
+    udpConnectionRequestAddress,
+    username,
+    password;
+  if (device.Device)
+    // TR-181 data model
+    managementServer = device.Device.ManagementServer;
+  // TR-098 data model
+  else managementServer = device.InternetGatewayDevice.ManagementServer;
+
+  if (managementServer.ConnectionRequestURL)
+    connectionRequestUrl = managementServer.ConnectionRequestURL._value;
+  if (managementServer.UDPConnectionRequestAddress) {
+    udpConnectionRequestAddress =
+      managementServer.UDPConnectionRequestAddress._value;
+  }
+  if (managementServer.ConnectionRequestUsername)
+    username = managementServer.ConnectionRequestUsername._value;
+  if (managementServer.ConnectionRequestPassword)
+    password = managementServer.ConnectionRequestPassword._value;
+
+  const context = {
+    id: device["_id"],
+    serialNumber: device["_deviceId"]["SerialNumber"],
+    productClass: device["_deviceId"]["ProductClass"],
+    oui: device["_deviceId"]["OUI"],
+    remoteAddress: connectionRequestUrl
+      ? parse(connectionRequestUrl).host
+      : null,
+    username: username || "",
+    password: password || ""
+  };
+
+  const snapshot = await getCurrentSnapshot();
+  const now = Date.now();
+  const UDP_CONNECTION_REQUEST_PORT = +getConfig(
+    snapshot,
+    "cwmp.udpConnectionRequestPort",
+    context,
+    now
+  );
+  const CONNECTION_REQUEST_TIMEOUT = +getConfig(
+    snapshot,
+    "cwmp.connectionRequestTimeout",
+    context,
+    now
+  );
+  const CONNECTION_REQUEST_ALLOW_BASIC_AUTH = !!getConfig(
+    snapshot,
+    "cwmp.connectionRequestAllowBasicAuth",
+    context,
+    now
+  );
+  let authExp: Expression = getConfigExpression(
+    snapshot,
+    "cwmp.connectionRequestAuth"
+  );
+
+  if (authExp === undefined) {
+    authExp = [
+      "FUNC",
+      "AUTH",
+      ["PARAM", "username"],
+      ["PARAM", "password"]
+    ] as Expression;
+  }
+
+  const debug = !!getConfig(snapshot, "cwmp.debug", context, now);
+
+  let udpProm;
+  if (udpConnectionRequestAddress) {
+    udpProm = udpConnectionRequest(
+      udpConnectionRequestAddress,
+      authExp,
+      context,
+      UDP_CONNECTION_REQUEST_PORT,
+      debug,
+      deviceId
+    );
+  }
+
+  try {
+    await httpConnectionRequest(
+      connectionRequestUrl,
+      authExp,
+      context,
+      CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
+      CONNECTION_REQUEST_TIMEOUT,
+      debug,
+      deviceId
+    );
+  } catch (err) {
+    if (!udpProm) throw err;
+    await udpProm;
+  }
+}
+
+export async function watchTask(deviceId, taskId, timeout): Promise<string> {
+  await new Promise(resolve => setTimeout(resolve, 500));
+
+  const task = await db.tasksCollection.findOne(
+    { _id: taskId },
+    { projection: { _id: 1 } }
+  );
+  if (!task) return "completed";
+
+  const q = { _id: `${deviceId}:task_${taskId}` };
+  const fault = await db.faultsCollection.findOne(q, {
+    projection: { _id: 1 }
+  });
+  if (fault) return "fault";
+
+  if ((timeout -= 500) <= 0) return "timeout";
+
+  return watchTask(deviceId, taskId, timeout);
+}
+
+function sanitizeTask(task): void {
+  task.timestamp = new Date(task.timestamp || Date.now());
+  if (task.expiry) {
+    if (common.typeOf(task.expiry) === common.DATE_TYPE || isNaN(task.expiry))
+      task.expiry = new Date(task.expiry);
+    else task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000);
+  }
+
+  const validParamValue = (p): boolean => {
+    if (
+      !Array.isArray(p) ||
+      p.length < 2 ||
+      typeof p[0] !== "string" ||
+      !p[0].length ||
+      !["string", "boolean", "number"].includes(typeof p[1]) ||
+      (p[2] != null && typeof p[2] !== "string")
+    )
+      return false;
+    return true;
+  };
+
+  switch (task.name) {
+    case "getParameterValues":
+      if (!Array.isArray(task.parameterNames) || !task.parameterNames.length)
+        throw new Error("Missing 'parameterNames' property");
+      for (const p of task.parameterNames) {
+        if (typeof p !== "string" || !p.length)
+          throw new Error(`Invalid parameter name '${p}'`);
+      }
+      break;
+
+    case "setParameterValues":
+      if (!Array.isArray(task.parameterValues) || !task.parameterValues.length)
+        throw new Error("Missing 'parameterValues' property");
+      for (const p of task.parameterValues) {
+        if (!validParamValue(p))
+          throw new Error(`Invalid parameter value '${p}'`);
+      }
+      break;
+
+    case "refreshObject":
+    case "deleteObject":
+      if (typeof task.objectName !== "string" || !task.objectName.length)
+        throw new Error("Missing 'objectName' property");
+      break;
+
+    case "addObject":
+      if (task.parameterValues != null) {
+        if (!Array.isArray(task.parameterValues))
+          throw new Error("Invalid 'parameterValues' property");
+        for (const p of task.parameterValues) {
+          if (!validParamValue(p))
+            throw new Error(`Invalid parameter value '${p}'`);
+        }
+      }
+      break;
+
+    case "download":
+      // genieacs-gui sends file ID instead of fileName and fileType
+      if (!task.file) {
+        if (typeof task.fileType !== "string" || !task.fileType.length)
+          throw new Error("Missing 'fileType' property");
+
+        if (typeof task.fileName !== "string" || !task.fileName.length)
+          throw new Error("Missing 'fileName' property");
+      }
+
+      if (
+        task.targetFileName != null &&
+        typeof task.targetFileName !== "string"
+      )
+        throw new Error("Invalid 'targetFileName' property");
+      break;
+
+    case "provisions":
+      if (
+        !Array.isArray(task.provisions) ||
+        !task.provisions.every(arr =>
+          arr.every(
+            s => s == null || ["boolean", "number", "string"].includes(typeof s)
+          )
+        )
+      )
+        throw new Error("Invalid 'provisions' property");
+      break;
+  }
+
+  return task;
+}
+
+export async function insertTasks(tasks): Promise<Task[]> {
+  if (tasks && common.typeOf(tasks) !== common.ARRAY_TYPE) tasks = [tasks];
+  else if (!tasks || tasks.length === 0) return tasks || [];
+
+  for (const task of tasks) {
+    sanitizeTask(task);
+    if (task.uniqueKey) {
+      await db.tasksCollection.deleteOne({
+        device: task.device,
+        uniqueKey: task.uniqueKey
+      });
+    }
+  }
+  await db.tasksCollection.insertMany(tasks);
+  return tasks;
+}
+
+export async function deleteDevice(deviceId): Promise<void> {
+  await Promise.all([
+    db.tasksCollection.deleteMany({ device: deviceId }),
+    db.devicesCollection.deleteOne({ _id: deviceId }),
+    db.faultsCollection.deleteMany({
+      _id: {
+        $regex: `^${common.escapeRegExp(deviceId)}\\:`
+      }
+    }),
+    db.operationsCollection.deleteMany({
+      _id: {
+        $regex: `^${common.escapeRegExp(deviceId)}\\:`
+      }
+    }),
+    cache.del(`${deviceId}_tasks_faults_operations`)
+  ]);
+}

lib/auth.coffee

@@ -1,92 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-crypto = require 'crypto'
-
-
-exports.parseAuthHeader = (authHeader) ->
-  authHeader = authHeader.trim()
-  method = authHeader.split(' ', 1)[0]
-  res = {method: method}
-  parts = authHeader.slice(method.length + 1).split(',')
-
-  while (part = parts.shift())?
-    name = part.split('=', 1)[0]
-    throw new Error('Unable to parse auth header') if name.length == part.length
-    value = part.slice(name.length + 1)
-    if not /^\s*"/.test(value)
-      value = value.trim()
-    else
-      while not /[^\\]"\s*$/.test(value)
-        p = parts.shift()
-        throw new Error('Unable to parse auth header') if not p?
-        value += ',' + p
-
-      try
-        value = JSON.parse(value)
-      catch
-        throw new Error('Unable to parse auth header')
-
-    res[name.trim()] = value
-
-  return res
-
-
-exports.basic = (username, password) ->
-  "Basic #{new Buffer("#{username}:#{password}").toString('base64')}"
-
-
-exports.digest = (username, password, uri, httpMethod, body, authHeader) ->
-  cnonce = '0a4f113b'
-  nc = '00000001'
-
-  if authHeader.qop?
-    if authHeader.qop.indexOf(',') != -1
-      qop = 'auth' # either auth or auth-int. prefer auth
-    else
-      qop = authHeader.qop
-
-  ha1 = crypto.createHash('md5')
-  ha1.update(username).update(':').update(authHeader.realm).update(':').update(password)
-  # TODO support "MD5-sess" algorithm directive
-  ha1 = ha1.digest('hex')
-
-  ha2 = crypto.createHash('md5')
-  ha2.update(httpMethod).update(':').update(uri)
-  if qop == 'auth-int'
-    ha2.update(':').update(body)
-  ha2 = ha2.digest('hex')
-
-  response = crypto.createHash('md5')
-  response.update(ha1).update(':').update(authHeader.nonce)
-
-  if qop?
-    response.update(':').update(nc).update(':').update(cnonce).update(':').update(qop)
-  response.update(':').update(ha2)
-  response = response.digest('hex')
-
-  authString = "Digest username=\"#{username}\""
-  authString += ",realm=\"#{authHeader.realm}\""
-  authString += ",nonce=\"#{authHeader.nonce}\""
-  authString += ",uri=\"#{uri}\""
-  authString += ",algorithm=#{authHeader.algorithm}" if authHeader.algorithm?
-  authString += ",qop=#{qop},nc=#{nc},cnonce=\"#{cnonce}\"" if qop?
-  authString += ",response=\"#{response}\""
-  authString += ",opaque=\"#{authHeader.opaque}\"" if authHeader.opaque?
-  return authString

lib/auth.ts

@@ -0,0 +1,202 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { createHash, randomBytes, pbkdf2 } from "crypto";
+
+function parseHeaderFeilds(str: string): {} {
+  const res = {};
+  const parts = str.split(",");
+
+  let part;
+  while ((part = parts.shift()) != null) {
+    const name = part.split("=", 1)[0];
+    if (name.length === part.length)
+      throw new Error("Unable to parse auth header");
+
+    let value = part.slice(name.length + 1);
+    if (!/^\s*"/.test(value)) {
+      value = value.trim();
+    } else {
+      while (!/[^\\]"\s*$/.test(value)) {
+        const p = parts.shift();
+        if (p == null) throw new Error("Unable to parse auth header");
+        value += "," + p;
+      }
+
+      try {
+        value = JSON.parse(value);
+      } catch (error) {
+        throw new Error("Unable to parse auth header");
+      }
+    }
+    res[name.trim()] = value;
+  }
+  return res;
+}
+
+export function parseAuthorizationHeader(authHeader): { method: string } {
+  authHeader = authHeader.trim();
+  const method = authHeader.split(" ", 1)[0];
+  const res = { method: method };
+
+  if (method === "Basic") {
+    // Inspired by https://github.com/jshttp/basic-auth
+    const USER_PASS_REGEX = /^([^:]*):(.*)$/;
+    const creds = USER_PASS_REGEX.exec(
+      Buffer.from(authHeader.slice(method.length + 1), "base64").toString()
+    );
+
+    if (!creds) throw new Error("Unable to parse auth header");
+    res["username"] = creds[1];
+    res["password"] = creds[2];
+  } else if (method === "Digest") {
+    Object.assign(res, parseHeaderFeilds(authHeader.slice(method.length + 1)));
+  }
+
+  return res;
+}
+
+export function parseWwwAuthenticateHeader(authHeader): {} {
+  authHeader = authHeader.trim();
+  const method = authHeader.split(" ", 1)[0];
+  const res = { method: method };
+  Object.assign(res, parseHeaderFeilds(authHeader.slice(method.length + 1)));
+  return res;
+}
+
+export function basic(username, password): string {
+  return "Basic " + Buffer.from(`${username}:${password}`).toString("base64");
+}
+
+export function digest(
+  username: string | Buffer,
+  realm: string | Buffer,
+  password: string | Buffer,
+  nonce: string | Buffer,
+  httpMethod: string | Buffer,
+  uri: string | Buffer,
+  qop?: string | Buffer,
+  body?: string | Buffer,
+  cnonce?: string | Buffer,
+  nc?: string | Buffer
+): string {
+  const ha1 = createHash("md5");
+  ha1
+    .update(username)
+    .update(":")
+    .update(realm)
+    .update(":")
+    .update(password);
+  // TODO support "MD5-sess" algorithm directive
+  const ha1d = ha1.digest("hex");
+
+  const ha2 = createHash("md5");
+  ha2
+    .update(httpMethod)
+    .update(":")
+    .update(uri);
+
+  if (qop === "auth-int") {
+    const bodyHash = createHash("md5")
+      .update(body || "")
+      .digest("hex");
+    ha2.update(":").update(bodyHash);
+  }
+
+  const ha2d = ha2.digest("hex");
+
+  const hash = createHash("md5");
+  hash
+    .update(ha1d)
+    .update(":")
+    .update(nonce);
+  if (qop) {
+    hash
+      .update(":")
+      .update(nc)
+      .update(":")
+      .update(cnonce)
+      .update(":")
+      .update(qop);
+  }
+  hash.update(":").update(ha2d);
+
+  return hash.digest("hex");
+}
+
+export function solveDigest(
+  username: string | Buffer,
+  password: string | Buffer,
+  uri: string | Buffer,
+  httpMethod: string | Buffer,
+  body: string | Buffer,
+  authHeader
+): string {
+  const cnonce = randomBytes(8).toString("hex");
+  const nc = "00000001";
+
+  let qop;
+  if (authHeader.qop) {
+    if (authHeader.qop.indexOf(",") !== -1) qop = "auth";
+    // Either auth or auth-int, prefer auth
+    else qop = authHeader.qop;
+  }
+
+  const hash = digest(
+    username,
+    authHeader.realm,
+    password,
+    authHeader.nonce,
+    httpMethod,
+    uri,
+    qop,
+    body,
+    cnonce,
+    nc
+  );
+
+  let authString = `Digest username="${username}"`;
+  authString += `,realm="${authHeader.realm}"`;
+  authString += `,nonce="${authHeader.nonce}"`;
+  authString += `,uri="${uri}"`;
+  if (authHeader.algorithm) authString += `,algorithm=${authHeader.algorithm}`;
+  if (qop) authString += `,qop=${qop},nc=${nc},cnonce="${cnonce}"`;
+  authString += `,response="${hash}"`;
+  if (authHeader.opaque) authString += `,opaque="${authHeader.opaque}"`;
+
+  return authString;
+}
+
+export function generateSalt(length): Promise<string> {
+  return new Promise((resolve, reject) => {
+    randomBytes(length, (err, rand) => {
+      if (err) return void reject(err);
+      resolve(rand.toString("hex"));
+    });
+  });
+}
+
+export function hashPassword(pass, salt): Promise<string> {
+  return new Promise((resolve, reject) => {
+    pbkdf2(pass, salt, 10000, 128, "sha512", (err, hash) => {
+      if (err) return void reject(err);
+      resolve(hash.toString("hex"));
+    });
+  });
+}

lib/cache.js

@@ -1,257 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const config = require("./config");
-
-const MAX_CACHE_TTL = config.get("MAX_CACHE_TTL");
-
-let redisClient;
-let mongoDb;
-let mongoCollection;
-let mongoTimeOffset = 0;
-
-
-function connect(callback) {
-  const REDIS_HOST = config.get("REDIS_HOST");
-
-  if (REDIS_HOST) {
-    const redis = require("redis");
-    const REDIS_PORT = config.get("REDIS_PORT");
-    const REDIS_DB = config.get("REDIS_DB");
-    redisClient = redis.createClient(REDIS_PORT, REDIS_HOST);
-    redisClient.select(REDIS_DB, function(err) {
-      exports.get = redisGet;
-      exports.set = redisSet;
-      exports.del = redisDel;
-      exports.pop = redisPop;
-      exports.lock = redisLock;
-      callback(err);
-    });
-  }
-  else {
-    const mongodb = require("mongodb");
-    const MONGODB_CONNECTION_URL = config.get("MONGODB_CONNECTION_URL");
-
-    mongodb.MongoClient.connect(MONGODB_CONNECTION_URL, function(err, db) {
-      if (err)
-        return callback(err);
-
-      mongoDb = db;
-      mongoCollection = db.collection("cache");
-      mongoCollection.ensureIndex({"expire": 1}, {expireAfterSeconds: 0});
-
-      const now = Date.now();
-      mongoDb.command({hostInfo: 1}, function(err, res) {
-        if (err)
-          return callback(err);
-        mongoTimeOffset = res.system.currentTime.getTime() - now;
-        exports.get = mongoGet;
-        exports.set = mongoSet;
-        exports.del = mongoDel;
-        exports.pop = mongoPop;
-        exports.lock = mongoLock;
-        callback();
-      });
-    });
-  }
-}
-
-
-function disconnect() {
-  if (redisClient)
-    redisClient.quit();
-
-  if (mongoDb)
-    mongoDb.close();
-}
-
-
-function redisGet(key, callback) {
-  if (Array.isArray(key))
-    redisClient.mget(key, callback);
-  else
-    redisClient.get(key, callback);
-}
-
-
-function redisDel(key, callback) {
-  redisClient.del(key, callback);
-}
-
-
-function redisSet(key, value, ttl, callback) {
-  if (!callback && typeof v === "function") {
-    callback = ttl;
-    ttl = null;
-  }
-
-  if (!ttl)
-    ttl = MAX_CACHE_TTL;
-
-  redisClient.setex(key, ttl, value, callback);
-}
-
-
-function redisPop(key, callback) {
-  const POP_SCRIPT = "local v=redis.call(\"get\",KEYS[1]);redis.call(\"del\",KEYS[1]);return v;";
-  redisClient.eval(POP_SCRIPT, 1, key, callback);
-}
-
-
-function redisLock(lockName, ttl, callback) {
-  const UNLOCK_SCRIPT = "if redis.call(\"get\", KEYS[1]) == ARGV[1] then return redis.call(\"del\", KEYS[1]) else return 0 end";
-  const EXTEND_SCRIPT = "if redis.call(\"get\", KEYS[1]) == ARGV[1] then return redis.call(\"pexpire\", KEYS[1], ARGV[2]) else return 0 end";
-  const token = Math.random().toString(36).slice(2);
-
-  function unlockOrExtend(ttl) {
-    if (!ttl) {
-      redisClient.eval(UNLOCK_SCRIPT, 1, lockName, token, function(err, res) {
-        if (err || !res)
-          throw err || new Error("Lock expired");
-      });
-    }
-    else {
-      redisClient.eval(EXTEND_SCRIPT, 1, lockName, token, ttl, function(err, res) {
-        if (err || !res)
-          throw err || new Error("Lock expired");
-      });
-    }
-  }
-
-  redisClient.set(lockName, token, "NX", "EX", ttl, function(err, res) {
-    if (err || res)
-      return callback(err, unlockOrExtend);
-
-    setTimeout(function() {
-      redisLock(lockName, ttl, callback);
-    }, 200);
-  });
-}
-
-
-function mongoGet(key, callback) {
-  let expire = new Date(Date.now() - mongoTimeOffset);
-  if (Array.isArray(key))
-    mongoCollection.find({"_id": {"$in": key}}).toArray(function(err, res) {
-      if (err)
-        return callback(err);
-
-      let indices = {};
-      key.forEach(function(v, i) {
-        indices[v] = i;
-      });
-
-      let values = [];
-      res.forEach(function(r) {
-        if (r["expire"] > expire)
-          values[indices[r["_id"]]] = r["value"];
-      });
-      callback(null, values);
-    });
-  else
-    mongoCollection.findOne({"_id": {"$in": [key]}}, function(err, res) {
-      if (err || !res)
-        return callback(err);
-
-      if (res["expire"] > expire)
-        return callback(null, res["value"]);
-
-      callback();
-    });
-}
-
-
-function mongoDel(key, callback) {
-  if (Array.isArray(key))
-    mongoCollection.remove({"_id": {"$in": key}}, callback);
-  else
-    mongoCollection.remove({"_id": key}, callback);
-}
-
-
-function mongoSet(key, value, ttl, callback) {
-  if (!callback && typeof v === "function") {
-    callback = ttl;
-    ttl = null;
-  }
-
-  if (!ttl)
-    ttl = MAX_CACHE_TTL;
-
-  let expire = new Date((Date.now() - mongoTimeOffset) + (ttl * 1000));
-  mongoCollection.save({"_id": key, "value": value, "expire": expire}, callback);
-}
-
-
-function mongoPop(key, callback) {
-  mongoCollection.findAndModify({"_id": key}, null, null, {remove: true}, function(err, res) {
-    if (err || !res["value"])
-      return callback(err);
-
-    if (res["value"]["expire"] > new Date(Date.now() - mongoTimeOffset))
-      return callback(null, res["value"]["value"]);
-
-    callback();
-  });
-}
-
-
-function mongoLock(lockName, ttl, callback) {
-  const token = Math.random().toString(36).slice(2);
-
-  function unlockOrExtend(ttl) {
-    if (!ttl) {
-      mongoCollection.remove({"_id": lockName, "value": token}, function(err, res) {
-        if (err || res["result"]["n"] !== 1)
-          throw err || new Error("Lock expired");
-      });
-    }
-    else {
-      let expire = new Date((Date.now() - mongoTimeOffset) + (ttl * 1000));
-      mongoCollection.update(
-        {"_id": lockName, "value": token},
-        {"expire": expire},
-        function(err, res) {
-          if (err || res["result"]["n"] !== 1)
-            throw err | new Error("Lock expired");
-        }
-      );
-    }
-  }
-
-  let expireTest = new Date(Date.now() - mongoTimeOffset);
-  let expireSet = new Date((Date.now() - mongoTimeOffset) + (ttl * 1000));
-
-  mongoCollection.update(
-    {"_id": lockName, "expire": {"$lte": expireTest}},
-    {"$set": {"value" : token, "expire": expireSet}},
-    {upsert: true},
-    function(err, res) {
-      if (err && err.code === 11000)
-        return setTimeout(function() {mongoLock(lockName, ttl, callback)}, 200);
-
-      return callback(err, unlockOrExtend);
-    }
-  );
-}
-
-
-exports.connect = connect;
-exports.disconnect = disconnect;

lib/cache.ts

@@ -0,0 +1,140 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { MongoClient, Collection } from "mongodb";
+import * as config from "./config";
+
+const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
+
+let clientPromise: Promise<MongoClient>;
+let mongoCollection: Collection;
+let mongoTimeOffset = 0;
+
+export async function connect(): Promise<void> {
+  const MONGODB_CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
+  clientPromise = MongoClient.connect(MONGODB_CONNECTION_URL, {
+    useNewUrlParser: true
+  });
+  const db = (await clientPromise).db();
+  mongoCollection = db.collection("cache");
+  await mongoCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
+  const now = Date.now();
+  const res = await db.command({ hostInfo: 1 });
+  mongoTimeOffset = res.system.currentTime.getTime() - now;
+}
+
+export async function disconnect(): Promise<void> {
+  if (clientPromise) await (await clientPromise).close();
+}
+
+export async function get(key): Promise<any> {
+  const expire = new Date(Date.now() - mongoTimeOffset);
+  if (Array.isArray(key)) {
+    const res = await mongoCollection.find({ _id: { $in: key } }).toArray();
+
+    const indices = {};
+    key.forEach((v, i) => {
+      indices[v] = i;
+    });
+
+    const values = [];
+    res.forEach(r => {
+      if (r["expire"] > expire) values[indices[r["_id"]]] = r["value"];
+    });
+
+    return values;
+  } else {
+    const res = await mongoCollection.findOne({ _id: { $in: [key] } });
+    if (res && res["expire"] > expire) return res["value"];
+    return null;
+  }
+}
+
+export async function del(key): Promise<void> {
+  if (Array.isArray(key))
+    await mongoCollection.deleteMany({ _id: { $in: key } });
+  else await mongoCollection.deleteOne({ _id: key });
+}
+
+export async function set(
+  key: string,
+  value: string | number,
+  ttl: number = MAX_CACHE_TTL
+): Promise<void> {
+  const expire = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
+  await mongoCollection.replaceOne(
+    { _id: key },
+    { _id: key, value: value, expire: expire },
+    { upsert: true }
+  );
+}
+
+export async function pop(key): Promise<any> {
+  const res = await mongoCollection.findOneAndDelete({ _id: key });
+
+  if (
+    res &&
+    res["value"] &&
+    +res["value"]["expire"] - (Date.now() - mongoTimeOffset)
+  )
+    return res["value"]["value"];
+
+  return null;
+}
+
+export async function lock(lockName, ttl): Promise<Function> {
+  const token = Math.random()
+    .toString(36)
+    .slice(2);
+
+  async function unlockOrExtend(extendTtl): Promise<void> {
+    if (!extendTtl) {
+      const res = await mongoCollection.deleteOne({
+        _id: lockName,
+        value: token
+      });
+      if (res["result"]["n"] !== 1) throw new Error("Lock expired");
+    } else {
+      const expire = new Date(Date.now() - mongoTimeOffset + extendTtl * 1000);
+      const res = await mongoCollection.updateOne(
+        { _id: lockName, value: token },
+        { expire: expire }
+      );
+      if (res["result"]["n"] !== 1) throw new Error("Lock expired");
+    }
+  }
+
+  const expireTest = new Date(Date.now() - mongoTimeOffset);
+  const expireSet = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
+
+  try {
+    await mongoCollection.updateOne(
+      { _id: lockName, expire: { $lte: expireTest } },
+      { $set: { value: token, expire: expireSet } },
+      { upsert: true }
+    );
+  } catch (err) {
+    if (err && err.code === 11000) {
+      await new Promise(resolve => setTimeout(resolve, 200));
+      return lock(lockName, ttl);
+    }
+  }
+
+  return unlockOrExtend;
+}

lib/cluster.js

@@ -1,177 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const cluster = require("cluster");
-const logger = require("./logger");
-const config = require("./config");
-
-const VERSION = require("../package.json").version;
-
-let respawnTimestamp = 0;
-let tooManyCrashesTimestamp = 0;
-
-function getDependencyVersions() {
-  let dependencies = [`node@${process.versions.node}`];
-  try {
-    let npmls = JSON.parse(require("child_process").execSync("npm ls --prod --json --depth 0"));
-
-    for (let k in npmls.dependencies)
-      dependencies.push(`${k}@${npmls.dependencies[k].version}`);
-  } catch (err) {}
-
-  return dependencies.join(",");
-}
-
-
-function getConfig() {
-  let conf = [];
-  const keys = Object.keys(process.env);
-  for (let k of keys)
-    if (k.startsWith("GENIEACS_")) {
-      const name = k.slice(9);
-      const value = process.env[k];
-      const def = config.getDefault(name);
-      if (def == null || value !== `${def}`)
-        conf.push(`${name}=${value}`);
-    }
-
-  return conf.join(",");
-}
-
-
-function restartWorker(worker, code, signal) {
-  let msg = {
-    message: "Worker died",
-    pid: worker.process.pid
-  };
-
-  if (code != null)
-    msg.exitCode = code;
-
-  if (signal != null)
-    msg.signal = signal;
-
-  logger.error(msg);
-
-  const now = Date.now();
-  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
-  if (respawnTimestamp === now) {
-    tooManyCrashesTimestamp = now;
-    cluster.fork();
-    return
-  }
-
-  if (now - tooManyCrashesTimestamp > 60000) {
-    process.exitCode = 1;
-    cluster.removeListener("exit", restartWorker);
-    for (let pid in cluster.workers)
-      cluster.workers[pid].kill();
-
-    logger.error({
-      message: "Too many crashes, exiting",
-      pid: process.pid
-    });
-    return;
-  }
-
-  setTimeout(function() {
-    if (process.exitCode) return;
-    cluster.fork();
-  }, respawnTimestamp - now);
-}
-
-
-function start(service) {
-  logger.init(service, VERSION);
-  const config = require("./config");
-  const SERVICE_ADDRESS = config.get(`${service.toUpperCase()}_INTERFACE`)
-  const SERVICE_PORT = config.get(`${service.toUpperCase()}_PORT`)
-
-  logger.info({
-    message: `GenieACS (genieacs-${service}) starting`,
-    pid: process.pid,
-    version: VERSION,
-    dependencies: getDependencyVersions(),
-    config: getConfig()
-  });
-
-  cluster.on("listening", function(worker, address) {
-    if ((address.addressType === 4 || address.addressType === 6) &&
-        address.address === SERVICE_ADDRESS && address.port === SERVICE_PORT) {
-      logger.info({
-        message: "Worker listening",
-        pid: worker.process.pid,
-        address: address.address,
-        port: address.port
-      });
-    }
-  });
-
-  cluster.on("exit", restartWorker);
-
-  process.on("SIGINT", function() {
-    logger.info({
-      message: "Received signal SIGINT, exiting",
-      pid: process.pid
-    });
-
-    cluster.removeListener("exit", restartWorker);
-  });
-
-  process.on("SIGTERM", function() {
-    logger.info({
-      message: "Received signal SIGTERM, exiting",
-      pid: process.pid
-    });
-
-    cluster.removeListener("exit", restartWorker);
-    for (let pid in cluster.workers)
-      cluster.workers[pid].kill();
-  });
-
-  cluster.setupMaster({
-    exec: "lib/server",
-    args: [service]
-  });
-
-  let workerCount = config.get(`${service.toUpperCase()}_WORKER_PROCESSES`);
-
-  if (!workerCount)
-    workerCount = Math.max(2, require("os").cpus().length);
-
-  for (let i = 0; i < workerCount; ++ i)
-    cluster.fork();
-}
-
-
-process.on("uncaughtException", function(err) {
-  logger.error({
-    message: "Uncaught exception in master process, exiting",
-    exception: err,
-    pid: process.pid
-  });
-  process.exitCode = 1;
-  cluster.removeListener("exit", restartWorker);
-  for (let pid in cluster.workers)
-    cluster.workers[pid].kill();
-});
-
-
-exports.start = start;

lib/cluster.ts

@@ -0,0 +1,120 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import cluster from "cluster";
+import { cpus } from "os";
+import * as logger from "./logger";
+
+let respawnTimestamp = 0;
+let crashes: number[] = [];
+
+function fork(): cluster.Worker {
+  const w = cluster.fork();
+  w.on("error", (err: NodeJS.ErrnoException) => {
+    // Avoid exception when attempting to kill the process just as it's exiting
+    if (err.code !== "EPIPE") throw err;
+    setTimeout(() => {
+      if (!w.isDead()) throw err;
+    }, 50);
+  });
+  return w;
+}
+
+function restartWorker(worker, code, signal): void {
+  const msg = {
+    message: "Worker died",
+    pid: worker.process.pid,
+    exitCode: null,
+    signal: null
+  };
+
+  if (code != null) msg.exitCode = code;
+
+  if (signal != null) msg.signal = signal;
+
+  logger.error(msg);
+
+  const now = Date.now();
+  crashes.push(now);
+
+  let min1 = 0,
+    min2 = 0,
+    min3 = 0;
+
+  crashes = crashes.filter(n => {
+    if (n > now - 60000) ++min1;
+    else if (n > now - 120000) ++min2;
+    else if (n > now - 180000) ++min3;
+    else return false;
+    return true;
+  });
+
+  if (min1 > 5 && min2 > 5 && min3 > 5) {
+    process.exitCode = 1;
+    cluster.removeListener("exit", restartWorker);
+    for (const pid in cluster.workers) cluster.workers[pid].kill();
+
+    logger.error({
+      message: "Too many crashes, exiting",
+      pid: process.pid
+    });
+    return;
+  }
+
+  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  if (respawnTimestamp === now) {
+    fork();
+    return;
+  }
+
+  setTimeout(() => {
+    if (process.exitCode) return;
+    fork();
+  }, respawnTimestamp - now);
+}
+
+export function start(workerCount, servicePort, serviceAddress): void {
+  cluster.on("listening", (worker, address) => {
+    if (
+      (address.addressType === 4 || address.addressType === 6) &&
+      address.address === serviceAddress &&
+      address.port === servicePort
+    ) {
+      logger.info({
+        message: "Worker listening",
+        pid: worker.process.pid,
+        address: address.address,
+        port: address.port
+      });
+    }
+  });
+
+  cluster.on("exit", restartWorker);
+
+  if (!workerCount) workerCount = Math.max(2, cpus().length);
+
+  for (let i = 0; i < workerCount; ++i) fork();
+}
+
+export function stop(): void {
+  cluster.removeListener("exit", restartWorker);
+  for (const pid in cluster.workers) cluster.workers[pid].kill();
+}
+
+export const worker = cluster.worker;

lib/common.coffee

@@ -1,229 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-buffer = require 'buffer'
-
-UNDEFINED_TYPE = '[object Undefined]'
-NULL_TYPE = '[object Null]'
-BOOLEAN_TYPE = '[object Boolean]'
-NUMBER_TYPE = '[object Number]'
-STRING_TYPE = '[object String]'
-ARRAY_TYPE = '[object Array]'
-OBJECT_TYPE = '[object Object]'
-REGEXP_TYPE = '[object RegExp]'
-DATE_TYPE = '[object Date]'
-
-
-typeOf = (obj) ->
-  Object.prototype.toString.call(obj)
-
-
-exports.generateDeviceId = (deviceIdStruct) ->
-  # Percent escaping function. Escapes everything except alphanumerics and underscore
-  esc = (str) ->
-    str.replace(/[^A-Za-z0-9_]/g, (chr) ->
-      buf = new buffer.Buffer(chr, 'utf8')
-      rep = ''
-      rep += "%#{b.toString(16).toUpperCase()}" for b in buf
-      return rep
-    )
-
-  # Guaranteeing globally unique id as defined in TR-069
-  if deviceIdStruct['ProductClass']
-    return "#{esc(deviceIdStruct['OUI'])}-#{esc(deviceIdStruct['ProductClass'])}-#{esc(deviceIdStruct['SerialNumber'])}"
-
-  return "#{esc(deviceIdStruct['OUI'])}-#{esc(deviceIdStruct['SerialNumber'])}"
-
-
-exports.getParamValueFromPath = (obj, path) ->
-  pp = path.split('.')
-  ref = obj
-  try
-    for p in pp
-      ref = ref[p]
-    return ref
-  catch err
-    return undefined
-
-
-pathOverlap = (a, b, start) ->
-  if a.length == b.length
-    res = 3
-  else if a.length > b.length
-    res = 2
-  else
-    res = 1
-
-  for i in [(start ? 0)...Math.min(a.length, b.length)] by 1
-    if a[i]?
-      if not b[i]?
-        res &= 2
-      else if a[i] != b[i]
-        return 0
-    else if b[i]?
-      res &= 1
-
-    return 0 if not res
-
-  return res
-
-
-parseAlias = (pattern, start, res) ->
-  aliases = []
-  i = start
-  while i < pattern.length and pattern[i] != ']'
-    alias = []
-    i = j = parsePath(pattern, i, alias) + 1
-
-    while pattern[j] not in [']', ',']
-      if pattern[j] == '"' and i == j
-        ++ j
-        while pattern[j] != '"' or pattern[j - 1] == '\\'
-          if ++ j >= pattern.length
-            throw new Error('Invalid alias expression')
-      if ++ j >= pattern.length
-        throw new Error('Invalid alias expression')
-    value = pattern[i...j].trim()
-    i = j
-    if value[0] == '"'
-      try
-        value = JSON.parse(value)
-      catch err
-        throw new Error('Invalid alias expression')
-    alias.push(value)
-    aliases.push(alias)
-    ++ i if pattern[i] == ','
-
-  # Need to sort to ensure identical expressions have idential string representation
-  srt = (a, b) ->
-    for j in [0...Math.min(a.length, b.length)] by 2
-      for k in [0...Math.min(a[j].length, b[j].length)] by 1
-        if Array.isArray(a[j][k])
-          if Array.isArray(b[j][k])
-            return srt(a[j][k], b[j][k])
-          else if not b[j][k]?
-            return -1
-          else
-            return 1
-        else if not a[j][k]?
-          if not b[j][k]?
-            return 0
-          else
-            return 1
-        else if not b[j][k]? or Array.isArray(b[j][k])
-            return -1
-        else if a[j][k] > b[j][k]
-          return 1
-        else if a[j][k] < b[j][k]
-          return -1
-
-      if a[j].length > b[j].length
-        return -1
-      else if a[j].length < b[j].length
-        return 1
-
-      if a[j + 1] > b[j + 1]
-        return -1
-      else if a[j + 1] < b[j + 1]
-        return 1
-
-    if a.length > b.length
-      return -1
-    else if a.length < b.length
-      return 1
-
-    return 0
-
-  aliases.sort(srt)
-  res.push([].concat.apply([], aliases))
-  return i
-
-
-parsePath = (pattern, start, res) ->
-  path = []
-  path.wildcard = 0
-  path.alias = 0
-  i = start ? 0
-
-  # Colon separator is needed for parseAlias
-  if i < pattern.length and pattern[i] != ':'
-    while true
-      if pattern[i] == '['
-        path.alias |= 1 << path.length
-        i = parseAlias(pattern, i + 1, path) + 1
-      else
-        j = i
-        while i < pattern.length and pattern[i] != ':' and pattern[i] != '.'
-          ++ i
-        n = pattern.slice(j, i)
-        path.wildcard |= 1 << path.length if n == '*'
-        path.push(n)
-
-      if i >= pattern.length or pattern[i] == ':'
-        break
-      else if pattern[i] != '.'
-        throw new Error('Invalid alias expression')
-
-      ++ i
-
-  Object.freeze(path)
-
-  if not res?
-    return path
-
-  res.push(path)
-  return i
-
-
-addPathMeta = (path) ->
-  return path if path.alias? or path.wildcard?
-
-  path.alias = 0
-  path.wildcard = 0
-
-  for p, i in path
-    if typeOf(p) is ARRAY_TYPE
-      path.alias |= 1 << i
-      for j in [0...p.length] by 2
-        addPathMeta(p[j])
-    else if p == '*'
-      path.wildcard |= 1 << i
-
-  Object.freeze(path)
-  return path
-
-
-# Source: http://stackoverflow.com/a/6969486
-escapeRegExp = (str) ->
-  return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&")
-
-
-exports.UNDEFINED_TYPE = UNDEFINED_TYPE
-exports.NULL_TYPE = NULL_TYPE
-exports.NUMBER_TYPE = NUMBER_TYPE
-exports.STRING_TYPE = STRING_TYPE
-exports.ARRAY_TYPE = ARRAY_TYPE
-exports.OBJECT_TYPE = OBJECT_TYPE
-exports.REGEXP_TYPE = REGEXP_TYPE
-
-exports.typeOf = typeOf
-exports.pathOverlap = pathOverlap
-exports.parsePath = parsePath
-exports.addPathMeta = addPathMeta
-exports.escapeRegExp = escapeRegExp

lib/common.ts

@@ -0,0 +1,59 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+export const UNDEFINED_TYPE = "[object Undefined]";
+export const NULL_TYPE = "[object Null]";
+export const BOOLEAN_TYPE = "[object Boolean]";
+export const NUMBER_TYPE = "[object Number]";
+export const STRING_TYPE = "[object String]";
+export const ARRAY_TYPE = "[object Array]";
+export const OBJECT_TYPE = "[object Object]";
+export const REGEXP_TYPE = "[object RegExp]";
+export const DATE_TYPE = "[object Date]";
+
+export const typeOf = (obj): string => Object.prototype.toString.call(obj);
+
+export function generateDeviceId(deviceIdStruct): string {
+  // Escapes everything except alphanumerics and underscore
+  function esc(str): string {
+    return str.replace(/[^A-Za-z0-9_]/g, chr => {
+      const buf = Buffer.from(chr, "utf8");
+      let rep = "";
+      for (const b of buf) rep += "%" + b.toString(16).toUpperCase();
+      return rep;
+    });
+  }
+
+  // Guaranteeing globally unique id as defined in TR-069
+  if (deviceIdStruct["ProductClass"]) {
+    return (
+      esc(deviceIdStruct["OUI"]) +
+      "-" +
+      esc(deviceIdStruct["ProductClass"]) +
+      "-" +
+      esc(deviceIdStruct["SerialNumber"])
+    );
+  }
+  return esc(deviceIdStruct["OUI"]) + "-" + esc(deviceIdStruct["SerialNumber"]);
+}
+
+// Source: http://stackoverflow.com/a/6969486
+export function escapeRegExp(str): string {
+  return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
+}

lib/common/authorizer.ts

@@ -0,0 +1,148 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { PermissionSet, Expression } from "../types";
+import { evaluate, or } from "./expression";
+
+export default class Authorizer {
+  private permissionSets: PermissionSet[];
+  private validatorCache: WeakMap<
+    object,
+    (mutationType, mutation, any) => boolean
+  >;
+  private hasAccessCache: Map<string, boolean>;
+  private getFilterCache: Map<string, Expression>;
+
+  public constructor(permissionSets) {
+    this.permissionSets = permissionSets;
+    this.validatorCache = new WeakMap();
+    this.hasAccessCache = new Map();
+    this.getFilterCache = new Map();
+  }
+
+  public hasAccess(resourceType: string, access: number): boolean {
+    const cacheKey = `${resourceType}-${access}`;
+    if (this.hasAccessCache.has(cacheKey))
+      return this.hasAccessCache.get(cacheKey);
+
+    let has = false;
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (perm[resourceType]) {
+          if (perm[resourceType].access >= access) {
+            has = true;
+            break;
+          }
+        }
+      }
+    }
+
+    this.hasAccessCache.set(cacheKey, has);
+    return has;
+  }
+
+  public getFilter(resourceType: string, access: number): Expression {
+    const cacheKey = `${resourceType}-${access}`;
+    if (this.getFilterCache.has(cacheKey))
+      return this.getFilterCache.get(cacheKey);
+
+    let filter: Expression = null;
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (perm[resourceType]) {
+          if (perm[resourceType].access >= access)
+            filter = or(filter, perm[resourceType].filter);
+        }
+      }
+    }
+
+    this.getFilterCache.set(cacheKey, filter);
+    return filter;
+  }
+
+  public getValidator(
+    resourceType,
+    resource
+  ): (mutationType: string, mutation?: any, args?: any) => boolean {
+    if (this.validatorCache.has(resource))
+      return this.validatorCache.get(resource);
+
+    const validators: Expression[] = [];
+
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (
+          perm[resourceType] &&
+          perm[resourceType].access >= 3 &&
+          perm[resourceType].validate
+        )
+          validators.push(perm[resourceType].validate);
+      }
+    }
+
+    const validator = (
+      mutationType: string,
+      mutation: any,
+      any: any
+    ): boolean => {
+      if (!validators.length) return false;
+
+      const object = {
+        mutationType,
+        mutation,
+        resourceType,
+        object: resource,
+        options: any
+      };
+
+      const valueFunction = (paramName): any => {
+        const entry = paramName.split(".", 1)[0];
+        paramName = paramName.slice(entry.length + 1);
+        let value = null;
+        if (["mutation", "options"].includes(entry)) {
+          value = object[entry];
+          for (const seg of paramName.split(".")) {
+            // typeof null is "object"
+            if (value != null && typeof value !== "object") value = null;
+            else value = value[seg];
+            if (value == null) break;
+          }
+        } else if (object[entry]) {
+          value = object[entry][paramName];
+        }
+
+        return value;
+      };
+
+      const res = evaluate(
+        validators.length > 1 ? ["OR", validators] : validators[0],
+        valueFunction,
+        Date.now()
+      );
+      return !Array.isArray(res) && !!res;
+    };
+
+    this.validatorCache.set(resource, validator);
+    return validator;
+  }
+
+  public getPermissionSets(): PermissionSet[] {
+    return this.permissionSets;
+  }
+}

lib/common/expression-cnf.ts

@@ -0,0 +1,285 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { map, parseLikePattern } from "./expression-parser";
+import { Expression } from "../types";
+
+type CNF = number[][];
+
+function or(cnf1: CNF, cnf2: CNF): CNF {
+  if (cnf1.length === 0) return cnf2;
+  else if (cnf2.length === 0) return cnf1;
+
+  if (cnf1.length > cnf2.length) {
+    const c = cnf1;
+    cnf1 = cnf2;
+    cnf2 = c;
+  }
+
+  const res = [];
+  for (let i = 0; i < cnf1.length; ++i)
+    for (let j = i; j < cnf2.length; ++j) res.push(cnf1[i].concat(cnf2[j]));
+
+  return res;
+}
+
+function likePatternIncludes(pat1, pat2, idx1 = 0, idx2 = 0): boolean {
+  while (idx1 < pat1.length && idx2 < pat2.length) {
+    if (pat1[idx1] === "\\%") {
+      for (let i = idx2; i <= pat2.length; ++i)
+        if (likePatternIncludes(pat1, pat2, idx1 + 1, i)) return true;
+      return false;
+    } else if (pat2[idx2] === "\\%") {
+      if (pat1[idx1] !== "\\%") return false;
+    } else if (pat1[idx1] === "\\_") {
+      // Ignore
+    } else if (pat2[idx2] === "\\_") {
+      return false;
+    } else if (pat2[idx2] !== pat1[idx1]) {
+      return false;
+    }
+    ++idx1;
+    ++idx2;
+  }
+
+  if (idx1 === pat1.length && idx2 === pat2.length) return true;
+  return false;
+}
+
+export function booleanCnf(exp: Expression): { vars: number; clauses: CNF } {
+  const ranges = new Map();
+  const likePatterns = new Map();
+  map(exp, e => {
+    if (!Array.isArray(e)) return e;
+    if (e[0] === "LIKE" || e[0] === "NOT LIKE") {
+      if (
+        typeof e[2] === "string" &&
+        (e.length >= 4 || (e[3] != null && !Array.isArray(e[3])))
+      ) {
+        const key = JSON.stringify(e[1]);
+        let s = likePatterns.get(key);
+        if (!s) likePatterns.set(key, (s = new Map()));
+        s.set(`${e[3] || ""}:${e[2]}`, parseLikePattern(e[2], e[3]));
+      }
+    } else if ([">", ">=", "<", "<=", "=", "<>"].includes(e[0])) {
+      if (!Array.isArray(e[2])) {
+        const key = `${JSON.stringify(e[1])}:${typeof e[2]}`;
+        let s = ranges.get(key);
+        if (!s) ranges.set(key, (s = new Set()));
+        s.add(e[2]);
+      } else if (!Array.isArray(e[1])) {
+        const key = `${JSON.stringify(e[2])}:${typeof e[1]}`;
+        let s = ranges.get(key);
+        if (!s) ranges.set(key, (s = new Set()));
+        s.add(e[2]);
+      }
+    }
+    return e;
+  });
+
+  for (const [k, v] of ranges) {
+    if (k.endsWith(":number")) {
+      ranges.set(
+        k,
+        Array.from(v as number[]).sort((a: number, b: number) => a - b)
+      );
+    } else {
+      ranges.set(k, Array.from(v).sort());
+    }
+  }
+
+  const expressions: Map<string, number> = new Map();
+  const mutuallyExclusive: Map<string, Set<number>> = new Map();
+
+  function getVariable(lhs, op, rhs = ""): number {
+    const key = `${lhs}:${op}:${rhs}`;
+    let f = expressions.get(key);
+    if (!f) {
+      expressions.set(key, (f = expressions.size + 1));
+      let m = mutuallyExclusive.get(lhs);
+      if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
+      m.add(f);
+      if (rhs || op !== "null") m.add(getVariable(lhs, "null"));
+    }
+    return f;
+  }
+
+  function getComparisonVariables(
+    lhs,
+    from,
+    fromInclusive,
+    to,
+    toInclusive
+  ): number[] {
+    const type = from != null ? typeof from : typeof to;
+    const range = ranges.get(`${lhs}:${type}`);
+    const vars = [];
+
+    if (from == null) vars.push(`${lhs}:--${range[0]}`);
+
+    for (let i = 0; i < range.length; ++i) {
+      if (
+        i > 0 &&
+        (from == null || range[i] > from) &&
+        (to == null || range[i] <= to)
+      )
+        vars.push(`${lhs}:${range[i - 1]}--${range[i]}`);
+
+      if (
+        (range[i] > from && range[i] < to) ||
+        (range[i] === from && fromInclusive) ||
+        (range[i] === to && toInclusive)
+      )
+        vars.push(`${lhs}:${range[i]}`);
+    }
+
+    if (to == null) vars.push(`${lhs}:${range[range.length - 1]}--`);
+
+    return vars.map(key => {
+      let f = expressions.get(key);
+      if (!f) {
+        expressions.set(key, (f = expressions.size + 1));
+        let m = mutuallyExclusive.get(lhs);
+        if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
+        m.add(f);
+      }
+      return f;
+    });
+  }
+
+  function getLikeVariables(lhs, pat, esc): number[] {
+    const vars = [];
+    const pats = likePatterns.get(lhs);
+    pat = pats.get(`${esc || ""}:${pat}`);
+    for (const p of pats.values()) {
+      if (pat === p || likePatternIncludes(pat, p))
+        vars.push(`${lhs}:like:${JSON.stringify(p)}`);
+    }
+
+    return vars.map(key => {
+      let f = expressions.get(key);
+      if (!f) expressions.set(key, (f = expressions.size + 1));
+      return f;
+    });
+  }
+
+  interface ComparisonOperators {
+    [name: string]: (v: string) => [string, boolean, string, boolean];
+  }
+
+  const comparisonOperators: ComparisonOperators = {
+    "=": v => [v, true, v, true],
+    ">": v => [v, false, null, null],
+    ">=": v => [v, true, null, null],
+    "<": v => [null, null, v, false],
+    "<=": v => [null, null, v, true]
+  };
+
+  function recursive(clause, negate): CNF {
+    let op = clause[0];
+
+    if ((op === "AND" && !negate) || (op === "OR" && negate)) {
+      let res = [];
+      for (let i = 1; i < clause.length; ++i)
+        res = res.concat(recursive(clause[i], negate));
+      return res;
+    } else if ((op === "OR" && !negate) || (op === "AND" && negate)) {
+      let res = [];
+      for (let i = 1; i < clause.length; ++i)
+        res = or(res, recursive(clause[i], negate));
+      return res;
+    } else if (op === "NOT") {
+      return recursive(clause[1], !negate);
+    } else if (op === "<>") {
+      const vars = getComparisonVariables(
+        JSON.stringify(clause[1]),
+        ...comparisonOperators["="](clause[2])
+      );
+      if (!negate) return vars.map(x => [0 - x]);
+      else return [vars];
+    } else if (op === "IS NULL") {
+      const v = getVariable(JSON.stringify(clause[1]), "null");
+      if (negate) return [[0 - v]];
+      else return [[v]];
+    } else if (op === "IS NOT NULL") {
+      const v = getVariable(JSON.stringify(clause[1]), "null");
+      if (!negate) return [[0 - v]];
+      else return [[v]];
+    } else if (op in comparisonOperators) {
+      if (!Array.isArray(clause[2])) {
+        const vars = getComparisonVariables(
+          JSON.stringify(clause[1]),
+          ...comparisonOperators[op](clause[2])
+        );
+        if (negate) return vars.map(x => [0 - x]);
+        else return [vars];
+      } else if (!Array.isArray(clause[1])) {
+        op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
+        const vars = getComparisonVariables(
+          JSON.stringify(clause[2]),
+          ...comparisonOperators[op](clause[1])
+        );
+        if (negate) return vars.map(x => [0 - x]);
+        else return [vars];
+      } else {
+        let lhs = JSON.stringify(clause[1]);
+        let rhs = JSON.stringify(clause[2]);
+        // For consistency
+        if (rhs > lhs) {
+          const t = rhs;
+          rhs = lhs;
+          lhs = t;
+          op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
+        }
+
+        const v = getVariable(lhs, op, rhs);
+        if (negate) return [[0 - v]];
+        else return [[v]];
+      }
+    } else if (
+      (op === "LIKE" || op === "NOT LIKE") &&
+      !Array.isArray(clause[2]) &&
+      clause[2] != null &&
+      (clause.length >= 4 || clause[3] != null)
+    ) {
+      if (op === "NOT LIKE") negate = !negate;
+      const vars = getLikeVariables(
+        JSON.stringify(clause[1]),
+        clause[2],
+        clause[3]
+      );
+      if (negate) return vars.map(x => [0 - x]);
+      else return [vars];
+    } else {
+      const v = getVariable(JSON.stringify(clause), "");
+      if (negate) return [[0 - v]];
+      else return [[v]];
+    }
+  }
+
+  const cnf = recursive(exp, false);
+
+  for (const m of mutuallyExclusive.values()) {
+    const ar: number[] = Array.from(m);
+    for (let i = 0; i < ar.length; ++i)
+      for (let j = i + 1; j < ar.length; ++j) cnf.push([0 - ar[i], 0 - ar[j]]);
+  }
+
+  return { vars: expressions.size, clauses: cnf };
+}

lib/common/expression-parser.ts

@@ -0,0 +1,412 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as parsimmon from "parsimmon";
+import { Expression } from "../types";
+
+// Turn escaped characters into real ones (e.g. "\\n" becomes "\n").
+function interpretEscapes(str): string {
+  const escapes = {
+    b: "\b",
+    f: "\f",
+    n: "\n",
+    r: "\r",
+    t: "\t"
+  };
+  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
+    const type = escape.charAt(0);
+    const hex = escape.slice(1);
+    if (type === "u") return String.fromCharCode(parseInt(hex, 16));
+
+    if (escapes.hasOwnProperty(type)) return escapes[type];
+
+    return type;
+  });
+}
+
+export function map(exp, callback): Expression {
+  if (!Array.isArray(exp)) return callback(exp);
+
+  let clone;
+  for (let i = 1; i < exp.length; ++i) {
+    const sub = map(exp[i], callback);
+    if (sub !== exp[i]) {
+      clone = clone || exp.slice();
+      clone[i] = sub;
+    }
+  }
+
+  return callback(clone || exp);
+}
+
+export async function mapAsync(exp, callback): Promise<Expression> {
+  if (!Array.isArray(exp)) return callback(exp);
+
+  let clone;
+  for (let i = 1; i < exp.length; ++i) {
+    const sub = await mapAsync(exp[i], callback);
+    if (sub !== exp[i]) {
+      clone = clone || exp.slice();
+      clone[i] = sub;
+    }
+  }
+
+  return callback(clone || exp);
+}
+
+function binaryLeft(operatorsParser, nextParser): parsimmon.Parser<{}> {
+  return parsimmon.seqMap(
+    nextParser,
+    parsimmon.seq(operatorsParser, nextParser).many(),
+    (first, rest) =>
+      rest.reduce((acc, ch) => {
+        const [op, another] = ch;
+        if (Array.isArray(acc) && op === acc[0]) return acc.concat([another]);
+        if (Array.isArray(another) && op === another[0])
+          return [op, acc].concat(another.slice(1));
+        return [op, acc, another];
+      }, first)
+  );
+}
+
+const lang = parsimmon.createLanguage({
+  ComparisonOperator: function() {
+    return parsimmon
+      .alt(
+        parsimmon.string(">="),
+        parsimmon.string("<>"),
+        parsimmon.string("<="),
+        parsimmon.string("="),
+        parsimmon.string(">"),
+        parsimmon.string("<")
+      )
+      .skip(parsimmon.optWhitespace);
+  },
+  LikeOperator: function() {
+    return parsimmon
+      .alt(
+        parsimmon
+          .regexp(/like/i)
+          .result("LIKE")
+          .desc("LIKE"),
+        parsimmon
+          .regexp(/not\s+like/i)
+          .result("NOT LIKE")
+          .desc("NOT LIKE")
+      )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace);
+  },
+  IsNullOperator: function() {
+    return parsimmon
+      .alt(
+        parsimmon
+          .regexp(/is\s+null/i)
+          .result("IS NULL")
+          .desc("IS NULL"),
+        parsimmon
+          .regexp(/is\s+not\s+null/i)
+          .result("IS NOT NULL")
+          .desc("IS NOT NULL")
+      )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace);
+  },
+  NotOperator: function() {
+    return parsimmon
+      .regexp(/not/i)
+      .result("NOT")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .desc("NOT");
+  },
+  AndOperator: function() {
+    return parsimmon
+      .regexp(/and/i)
+      .result("AND")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .desc("AND");
+  },
+  OrOperator: function() {
+    return parsimmon
+      .regexp(/or/i)
+      .result("OR")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .desc("OR");
+  },
+  Parameter: function(r) {
+    return parsimmon
+      .alt(
+        parsimmon.regexp(/[a-zA-Z0-9_.*-]+/),
+        r.Expression.wrap(
+          parsimmon.string("{").skip(parsimmon.optWhitespace),
+          parsimmon.string("}")
+        )
+      )
+      .atLeast(1)
+      .map(x => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
+      .skip(parsimmon.optWhitespace)
+      .desc("parameter");
+  },
+  StringValueSql: function() {
+    return parsimmon
+      .regexp(/'([^']*)'/, 1)
+      .atLeast(1)
+      .skip(parsimmon.optWhitespace)
+      .map(s => s.join("'"))
+      .desc("string");
+  },
+  StringValueJs: function() {
+    return parsimmon
+      .regexp(/"((?:\\.|.)*?)"/, 1)
+      .skip(parsimmon.optWhitespace)
+      .map(interpretEscapes)
+      .desc("string");
+  },
+  NumberValue: function() {
+    return parsimmon
+      .regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/)
+      .skip(parsimmon.optWhitespace)
+      .map(Number)
+      .desc("number");
+  },
+  BooleanValue: function() {
+    return parsimmon
+      .alt(
+        parsimmon
+          .regexp(/true/i)
+          .result(true)
+          .desc("TRUE"),
+        parsimmon
+          .regexp(/false/i)
+          .result(false)
+          .desc("FALSE")
+      )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace);
+  },
+  NullValue: function() {
+    return parsimmon
+      .regexp(/null/i)
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .result(null)
+      .desc("NULL");
+  },
+  FuncValue: function(r) {
+    return parsimmon.seqMap(
+      parsimmon
+        .regexp(/([a-zA-Z0-9_]+)/, 1)
+        .skip(parsimmon.optWhitespace)
+        .desc("function"),
+      r.ValueExpression.sepBy(
+        parsimmon.string(",").skip(parsimmon.optWhitespace)
+      ).wrap(
+        parsimmon.string("(").skip(parsimmon.optWhitespace),
+        parsimmon.string(")").skip(parsimmon.optWhitespace)
+      ),
+      (f, args) => ["FUNC", f.toUpperCase()].concat(args)
+    );
+  },
+  Value: function(r) {
+    return parsimmon.alt(
+      r.NullValue,
+      r.BooleanValue,
+      r.NumberValue,
+      r.StringValueSql,
+      r.StringValueJs,
+      r.FuncValue
+    );
+  },
+  ValueExpression: function(r) {
+    return binaryLeft(
+      parsimmon.string("||").skip(parsimmon.optWhitespace),
+      binaryLeft(
+        parsimmon
+          .alt(parsimmon.string("+"), parsimmon.string("-"))
+          .skip(parsimmon.optWhitespace),
+        binaryLeft(
+          parsimmon
+            .alt(parsimmon.string("*"), parsimmon.string("/"))
+            .skip(parsimmon.optWhitespace),
+          parsimmon.alt(
+            r.Value,
+            r.Parameter,
+            r.Expression.wrap(
+              parsimmon.string("(").skip(parsimmon.optWhitespace),
+              parsimmon.string(")").skip(parsimmon.optWhitespace)
+            )
+          )
+        )
+      )
+    );
+  },
+  Comparison: function(r) {
+    return parsimmon.alt(
+      parsimmon.seqMap(r.ValueExpression, r.IsNullOperator, (p, o) => [o, p]),
+      parsimmon.seqMap(
+        r.ValueExpression,
+        r.ComparisonOperator,
+        r.ValueExpression,
+        (p, o, v) => [o, p, v]
+      ),
+      parsimmon.seqMap(
+        r.ValueExpression,
+        r.LikeOperator,
+        r.ValueExpression.skip(
+          parsimmon
+            .regexp(/escape/i)
+            .result("ESCAPE")
+            .skip(parsimmon.whitespace)
+            .desc("ESCAPE")
+        ),
+        r.ValueExpression,
+        (a, b, c, d) => [b, a, c, d]
+      ),
+      parsimmon.seqMap(
+        r.ValueExpression,
+        r.LikeOperator,
+        r.ValueExpression,
+        (a, b, c) => [b, a, c]
+      )
+    );
+  },
+  Expression: function(r) {
+    function unary(operatorsParser, nextParser): parsimmon.Parser<{}> {
+      return parsimmon.seq(operatorsParser, nextParser).or(nextParser);
+    }
+
+    return binaryLeft(
+      r.OrOperator,
+      binaryLeft(
+        r.AndOperator,
+        unary(r.NotOperator, r.Comparison.or(r.ValueExpression))
+      )
+    ).trim(parsimmon.optWhitespace);
+  }
+});
+
+export function parse(str: string): Expression {
+  if (!str) return null;
+  return lang.Expression.tryParse(str);
+}
+
+export function stringify(exp, level = 0): string {
+  if (!Array.isArray(exp)) return JSON.stringify(exp);
+
+  const opLevels = {
+    OR: 10,
+    AND: 11,
+    NOT: 12,
+    "=": 20,
+    "<>": 20,
+    ">": 20,
+    ">=": 20,
+    "<": 20,
+    "<=": 20,
+    LIKE: 20,
+    "NOT LIKE": 20,
+    "IS NULL": 20,
+    "IS NOT NULL": 20,
+    "||": 30,
+    "+": 31,
+    "-": 31,
+    "*": 32,
+    "/": 32
+  };
+
+  const op = exp[0].toUpperCase();
+
+  function wrap(e): string {
+    if (opLevels[op] <= level) return `(${e})`;
+    else return e;
+  }
+
+  if (op === "FUNC") {
+    return wrap(
+      `${exp[1]}(${exp
+        .slice(2)
+        .map(e => stringify(e))
+        .join(", ")})`
+    );
+  } else if (op === "PARAM") {
+    if (typeof exp[1] === "string") {
+      return wrap(exp[1]);
+    } else if (Array.isArray(exp[1]) && exp[1][0] === "||") {
+      return wrap(
+        exp[1]
+          .slice(1)
+          .map(p => {
+            if (typeof p === "string") return p;
+            else return `{${stringify(p)}}`;
+          })
+          .join("")
+      );
+    } else {
+      return wrap(`{${stringify(exp[1])}}`);
+    }
+  } else if (op === "IS NULL" || op === "IS NOT NULL") {
+    return wrap(`${stringify(exp[1], opLevels[op])} ${op}`);
+  } else if (op === "LIKE" || op === "NOT LIKE") {
+    if (exp[3]) {
+      return wrap(
+        `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
+          exp[2],
+          opLevels[op]
+        )} ESCAPE ${stringify(exp[3], opLevels[op])}`
+      );
+    } else {
+      return wrap(
+        `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
+          exp[2],
+          opLevels[op]
+        )}`
+      );
+    }
+  } else if (op in opLevels) {
+    const parts = exp.slice(1).map((e, i) => {
+      return stringify(e, opLevels[exp[0]] + Math.min(i - 1, 0));
+    });
+
+    if (op === "NOT") return wrap(`${op} ${parts[0]}`);
+    else return wrap(parts.join(` ${op} `));
+  } else {
+    throw new Error(`Unrecognized operator ${exp[0]}`);
+  }
+}
+
+export function parseLikePattern(pat, esc): string[] {
+  const chars = pat.split("");
+
+  for (let i = 0; i < chars.length; ++i) {
+    const c = chars[i];
+    if (c === esc) {
+      chars[i] = chars[i + 1] || "";
+      chars[i + 1] = "";
+    } else if (c === "_") {
+      chars[i] = "\\_";
+    } else if (c === "%") {
+      chars[i] = "\\%";
+      while (chars[i + 1] === "%") chars[++i] = "";
+    }
+  }
+  return chars.filter(c => c);
+}

lib/common/expression.ts

@@ -0,0 +1,329 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { map, mapAsync, parseLikePattern } from "./expression-parser";
+import { booleanCnf } from "./expression-cnf";
+import { naiveDpll } from "./sat-solver";
+import { Expression } from "../types";
+
+const isArray = Array.isArray;
+
+const regExpCache = new WeakMap<object, RegExp>();
+
+const REDUCE_SKIP = {};
+function reduce(exp, callback): Expression {
+  let loop = true;
+  while (loop) {
+    loop = false;
+    for (let i = 2; i < exp.length; ++i) {
+      const res = callback(exp[i - 1], exp[i], i - 2);
+      if (res !== REDUCE_SKIP) {
+        loop = true;
+        exp = exp.slice();
+        exp.splice(i - 1, 2, res);
+      }
+    }
+  }
+  if (exp.length === 2) return exp[1];
+  return exp;
+}
+
+export function likePatternToRegExp(pat, esc = "", flags = ""): RegExp {
+  const convChars = {
+    "-": "\\-",
+    "/": "\\/",
+    "\\": "\\/",
+    "^": "\\^",
+    $: "\\$",
+    "*": "\\*",
+    "+": "\\+",
+    "?": "\\?",
+    ".": "\\.",
+    "(": "\\(",
+    ")": "\\)",
+    "|": "\\|",
+    "[": "\\[",
+    "]": "\\]",
+    "{": "\\{",
+    "}": "\\}",
+    "\\%": ".*",
+    "\\_": "."
+  };
+  let chars = parseLikePattern(pat, esc);
+  if (!chars.length) return new RegExp("^$", flags);
+  chars = chars.map(c => convChars[c] || c);
+  chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
+  const l = chars.length - 1;
+  chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
+  return new RegExp(chars.join(""), flags);
+}
+
+function evalExp(e: Expression): Expression {
+  if (e[0] === "AND") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a)) return a ? b : a;
+      if (!isArray(b)) return b ? a : b;
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "OR") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a)) return a ? a : b;
+      if (!isArray(b)) return b ? b : a;
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "NOT") {
+    if (!isArray(e[1])) return !e[1];
+    else if (e[1][0] === "NOT") return e[1][1];
+  } else if (e[0] === "IS NULL") {
+    if (isArray(e[1])) return e;
+    else if (e[1] == null) return true;
+    else return null;
+  } else if (e[0] === "IS NOT NULL") {
+    if (isArray(e[1])) return e;
+    else if (e[1] != null) return true;
+    else return null;
+  } else if (e[0] === "LIKE") {
+    if (isArray(e[1]) || isArray(e[2]) || isArray(e[3])) return e;
+    else if (
+      e[1] == null ||
+      e[2] == null ||
+      ((e as any[]).length >= 4 && e[3] == null)
+    )
+      return null;
+    let r = regExpCache.get(e as any[]);
+    if (!r) {
+      r = likePatternToRegExp(e[2], e[3]);
+      regExpCache.set(e as any[], r);
+    }
+    return r.test(e[1]);
+  } else if (e[0] === "NOT LIKE") {
+    if (isArray(e[1]) || isArray(e[2]) || isArray(e[3])) return e;
+    else if (
+      e[1] == null ||
+      e[2] == null ||
+      ((e as any[]).length >= 4 && e[3] == null)
+    )
+      return null;
+    let r = regExpCache.get(e as any[]);
+    if (!r) {
+      r = likePatternToRegExp(e[2], e[3]);
+      regExpCache.set(e as any[], r);
+    }
+    return !r.test(e[1]);
+  } else if (e[0] === "=") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] === e[2];
+  } else if (e[0] === "<>") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] !== e[2];
+  } else if (e[0] === ">") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] > e[2];
+  } else if (e[0] === ">=") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] >= e[2];
+  } else if (e[0] === "<") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] < e[2];
+  } else if (e[0] === "<=") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] <= e[2];
+  } else if (e[0] === "*") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a) && !isArray(b)) {
+        if (a == null || b == null) return null;
+        return a * b;
+      }
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "/") {
+    return reduce(e, (a, b, i) => {
+      if (!isArray(a) && !isArray(b)) {
+        if (a == null || b == null) return null;
+        return i === 0 ? a / b : a * b;
+      }
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "+") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a) && !isArray(b)) {
+        if (a == null || b == null) return null;
+        return a + b;
+      }
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "-") {
+    return reduce(e, (a, b, i) => {
+      if (!isArray(a) && !isArray(b)) {
+        if (a == null || b == null) return null;
+        return i === 0 ? a - b : a + b;
+      }
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "||") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a) && !isArray(b)) {
+        if (a == null || b == null) return null;
+        return `${a}${b}`;
+      }
+      return REDUCE_SKIP;
+    });
+  }
+  return e;
+}
+
+export function evaluate(
+  exp,
+  obj,
+  now: number,
+  cb?: Function
+): string | number | boolean | null;
+export function evaluate(exp, obj?, now?: number, cb?: Function): Expression;
+export function evaluate(exp, obj?, now?: number, cb?: Function): Expression {
+  return map(exp, e => {
+    if (cb) e = cb(e);
+    if (!isArray(e)) return e;
+
+    if (e[0] === "FUNC") {
+      if (e[1] === "NOW") {
+        if (now) return now;
+      } else if (e[1] === "UPPER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
+      } else if (e[1] === "LOWER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
+      }
+    } else if (e[0] === "PARAM") {
+      if (e[1] == null) return null;
+      if (obj && !isArray(e[1])) {
+        let v;
+        if (typeof obj === "function") v = obj(e[1]);
+        else v = obj[e[1]];
+
+        if (v == null) return null;
+        if (typeof v === "object") v = v.value ? v.value[0] : null;
+        return v;
+      }
+    }
+    return evalExp(e);
+  });
+}
+
+export async function evaluateAsync(
+  exp,
+  obj,
+  now: number,
+  cb?: Function
+): Promise<string | number | boolean | null>;
+export async function evaluateAsync(
+  exp,
+  obj?,
+  now?: number,
+  cb?: Function
+): Promise<Expression>;
+export async function evaluateAsync(
+  exp,
+  obj?,
+  now?: number,
+  cb?: Function
+): Promise<Expression> {
+  return mapAsync(exp, async e => {
+    if (cb) e = await cb(e);
+    if (!isArray(e)) return e;
+
+    if (e[0] === "FUNC") {
+      if (e[1] === "NOW") {
+        if (now) return now;
+      } else if (e[1] === "UPPER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
+      } else if (e[1] === "LOWER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
+      }
+    } else if (e[0] === "PARAM") {
+      if (e[1] == null) return null;
+      if (obj && !isArray(e[1])) {
+        let v = obj[e[1]];
+        if (v == null) return null;
+        if (typeof v === "object") v = v.value ? v.value[0] : null;
+        return v;
+      }
+    }
+    return evalExp(e);
+  });
+}
+
+export function and(exp1: Expression, exp2: Expression): Expression {
+  if (!isArray(exp1)) return exp1 ? exp2 : exp1;
+  if (!isArray(exp2)) return exp2 ? exp1 : exp2;
+
+  let res: Expression = ["AND"];
+
+  if (exp1[0] === "AND") res = res.concat(exp1.slice(1));
+  else res.push(exp1);
+
+  if (exp2[0] === "AND") res = res.concat(exp2.slice(1));
+  else res.push(exp2);
+
+  return res;
+}
+
+export function or(exp1: Expression, exp2: Expression): Expression {
+  if (!isArray(exp1)) return exp1 ? exp1 : exp2;
+  if (!isArray(exp2)) return exp2 ? exp2 : exp1;
+
+  let res: Expression = ["OR"];
+
+  if (exp1[0] === "OR") res = res.concat(exp1.slice(1));
+  else res.push(exp1);
+
+  if (exp2[0] === "OR") res = res.concat(exp2.slice(1));
+  else res.push(exp2);
+
+  return res;
+}
+
+export function not(exp): Expression {
+  if (isArray(exp) && exp[0] === "NOT") return exp[1];
+  return ["NOT", exp];
+}
+
+export function subset(exp1, exp2): boolean {
+  const e = evaluate(["NOT", ["OR", ["NOT", exp1], exp2]]);
+  if (!isArray(e)) return !e;
+  const { vars, clauses } = booleanCnf(e);
+  return !naiveDpll(clauses, vars);
+}
+
+export function extractParams(exp): Expression[] {
+  const params = new Set<Expression>();
+  map(exp, e => {
+    if (isArray(e) && e[0] === "PARAM") params.add(e[1]);
+    return e;
+  });
+  return Array.from(params);
+}

lib/common/memoize.ts

@@ -0,0 +1,63 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+let cache1 = new Map();
+let cache2 = new Map();
+const keys = new WeakMap();
+
+function getKey(obj): string {
+  if (obj === null) return "null";
+  else if (obj === undefined) return "undefined";
+
+  const t = typeof obj;
+  if (t === "number" || t === "boolean" || t === "string") return `${t}:${obj}`;
+  if (t !== "function" && t !== "object")
+    throw new Error(`Cannot memoize ${t} arguments`);
+
+  let k = keys.get(obj);
+  if (!k) {
+    const rnd = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER);
+    k = `${t}:${rnd.toString(36)}`;
+    keys.set(obj, k);
+  }
+  return k;
+}
+
+export default function memoize<T extends Function>(func: T): T {
+  const funcKey = getKey(func);
+  return ((...args) => {
+    const key = JSON.stringify(args.map(getKey)) + funcKey;
+
+    if (cache1.has(key)) return cache1.get(key);
+
+    let r;
+    if (cache2.has(key)) r = cache2.get(key);
+    else r = func(...args);
+    cache1.set(key, r);
+    return r;
+  }) as any;
+}
+
+const interval = setInterval(() => {
+  cache2 = cache1;
+  cache1 = new Map();
+}, 120000);
+
+// Don't hold Node.js process
+if (interval.unref) interval.unref();

lib/common/path-set.ts

@@ -0,0 +1,130 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Path from "./path";
+
+export default class PathSet {
+  private lengthIndex: Set<Path>[];
+  private fragmentIndex: Map<string, Set<Path>>[];
+  private stringIndex: Map<string, Path>;
+
+  public constructor() {
+    this.lengthIndex = [];
+    this.fragmentIndex = [];
+    this.stringIndex = new Map();
+  }
+
+  public get depth(): number {
+    return this.lengthIndex.length;
+  }
+
+  public add(path: Path): Path {
+    if (path.alias) throw new Error("PathSet does not support aliased paths");
+    const p = this.get(path);
+
+    if (p) return p;
+
+    this.stringIndex.set(path.toString(), path);
+
+    while (this.lengthIndex.length <= path.length) {
+      this.lengthIndex.push(new Set());
+      // fragmentIndex is one less than lengthIndex
+      if (this.lengthIndex.length > 1) this.fragmentIndex.push(new Map());
+    }
+
+    const lengthIndex = this.lengthIndex[path.length];
+    lengthIndex.add(path);
+
+    for (let i = 0; i < path.length; ++i) {
+      const fragment = path.segments[i] as string;
+      const fragmentIndex = this.fragmentIndex[i];
+
+      let fragmentIndexSet = fragmentIndex.get(fragment);
+      if (!fragmentIndexSet) {
+        fragmentIndexSet = new Set<Path>();
+        fragmentIndex.set(fragment, fragmentIndexSet);
+      }
+
+      fragmentIndexSet.add(path);
+    }
+
+    return path;
+  }
+
+  public get(path: Path): Path {
+    return this.stringIndex.get(path.toString()) || null;
+  }
+
+  public find(
+    path: Path,
+    superset: boolean = false,
+    subset: boolean = false,
+    depth: number = path.length
+  ): Path[] {
+    if (path.alias) throw new Error("PathSet does not support aliased paths");
+
+    const len = path.length;
+
+    if (!superset && depth === len && (!subset || !path.wildcard)) {
+      const p = this.get(path);
+      return p ? [p] : [];
+    }
+
+    const lengthIndex = this.lengthIndex.slice(len, depth + 1);
+    if (!lengthIndex.length) return [];
+
+    let res;
+    for (let i = len - 1; i >= 0; --i) {
+      let fragmentIndexSet2;
+      const fragmentIndex = this.fragmentIndex[i];
+
+      if ((path.wildcard >> i) & 1) {
+        if (subset) continue;
+      } else if (superset) {
+        fragmentIndexSet2 = fragmentIndex.get("*");
+      }
+
+      const fragment = path.segments[i] as string;
+      const fragmentIndexSet1 = fragmentIndex.get(fragment);
+
+      if (!fragmentIndexSet1) {
+        if (!fragmentIndexSet2) return [];
+        if (!res) res = [...fragmentIndexSet2];
+        else res = res.filter(r => fragmentIndexSet2.has(r));
+      } else if (!fragmentIndexSet2) {
+        if (!res) res = [...fragmentIndexSet1];
+        else res = res.filter(r => fragmentIndexSet1.has(r));
+      } else {
+        if (!res) {
+          res = [...fragmentIndexSet1, ...fragmentIndexSet2];
+        } else {
+          res = res.filter(
+            r => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r)
+          );
+        }
+      }
+      if (!res.length) return res;
+    }
+
+    if (!res) res = [].concat(...lengthIndex.map(a => [...a]));
+    else res = res.filter(r => lengthIndex.some(a => a.has(r)));
+
+    return res;
+  }
+}

lib/common/path.ts

@@ -0,0 +1,237 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+type Segments = (string | Alias)[];
+type Alias = [Path, string][];
+
+let cache1 = new Map<string, Path>();
+let cache2 = new Map<string, Path>();
+
+export default class Path {
+  public readonly segments: Segments;
+  public readonly wildcard: number;
+  public readonly alias: number;
+  protected _string: string;
+  protected _stringIndex: number[];
+
+  protected static parseAlias(
+    pattern: string,
+    index: number
+  ): { index: number; alias: Alias } {
+    const alias: Alias = [];
+    while (index < pattern.length && pattern[index] !== "]") {
+      const { index: idx, segments } = Path.parsePath(pattern, index);
+      let j = (index = idx + 1);
+      while (pattern[j] !== "]" && pattern[j] !== ",") {
+        if (pattern[j] === '"' && index === j) {
+          ++j;
+          while (pattern[j] !== '"' || pattern[j - 1] === "\\") {
+            if (++j >= pattern.length)
+              throw new Error("Invalid alias expression");
+          }
+        }
+        if (++j >= pattern.length) throw new Error("Invalid alias expression");
+      }
+
+      let value = pattern.slice(index, j).trim();
+      index = j;
+      if (value[0] === '"') {
+        try {
+          value = JSON.parse(value);
+        } catch (error) {
+          throw new Error("Invalid alias expression");
+        }
+      }
+
+      alias.push([new Path(segments), value]);
+      if (pattern[index] === ",") ++index;
+    }
+
+    // Ensure identical expressions have idential string representation
+    alias.sort((a, b) => {
+      if (a[0].toString() > b[0].toString()) return 1;
+      else if (a[0].toString() < b[0].toString()) return -1;
+      else if (a[1] > b[1]) return 1;
+      else if (a[1] < b[1]) return -1;
+      else return 0;
+    });
+
+    Object.freeze(alias);
+    return { index, alias };
+  }
+
+  protected static parsePath(
+    pattern: string,
+    index: number
+  ): { index: number; segments: Segments } {
+    const segments = [];
+    // Colon separator is needed for parseAlias
+    if (index < pattern.length && pattern[index] !== ":") {
+      for (;;) {
+        if (pattern[index] === "[") {
+          const { index: idx, alias } = Path.parseAlias(pattern, index + 1);
+          index = idx + 1;
+          segments.push(alias);
+        } else {
+          const j = index;
+          while (
+            index < pattern.length &&
+            pattern[index] !== ":" &&
+            pattern[index] !== "."
+          )
+            ++index;
+          const s = pattern.slice(j, index).trim();
+          segments.push(s);
+        }
+
+        if (index >= pattern.length || pattern[index] === ":") break;
+        else if (pattern[index] !== ".")
+          throw new Error("Invalid alias expression");
+        ++index;
+      }
+    }
+
+    Object.freeze(segments);
+    return { index, segments };
+  }
+
+  protected constructor(segments: Segments) {
+    let alias = 0;
+    let wildcard = 0;
+    const arr = segments.map((s, i) => {
+      if (Array.isArray(s)) {
+        alias |= 1 << i;
+        const parts = s.map(
+          al => `${al[0].toString()}:${JSON.stringify(al[1])}`
+        );
+        return `[${parts.join(",")}]`;
+      } else if (s === "*") {
+        wildcard |= 1 << i;
+      }
+      return s;
+    });
+
+    let offset = 0;
+    const stringIndex = arr.map((s, i) => (offset += s.length) + i);
+
+    this.segments = segments;
+    this.wildcard = wildcard;
+    this.alias = alias;
+    this._string = arr.join(".");
+    this._stringIndex = stringIndex;
+  }
+
+  public static parse(str: string): Path {
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        const { segments } = Path.parsePath(str, 0);
+        path = new Path(segments);
+        if (path.toString() !== str) cache1.set(path.toString(), path);
+      }
+      cache1.set(str, path);
+    }
+    return path;
+  }
+
+  public get length(): number {
+    return this.segments.length;
+  }
+
+  public toString(): string {
+    return this._string;
+  }
+
+  public slice(start: number = 0, end: number = this.segments.length): Path {
+    if (start < 0) start = Math.max(0, this.segments.length + start);
+    if (end < 0) end = Math.max(0, this.segments.length + end);
+
+    let str;
+    if (start >= end) {
+      str = "";
+    } else {
+      const i1 = start > 0 ? this._stringIndex[start - 1] + 1 : 0;
+      const i2 =
+        end <= this.segments.length
+          ? this._stringIndex[end - 1]
+          : this._string.length;
+      str = this._string.slice(i1, i2);
+    }
+
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        const segments = this.segments.slice(start, end);
+        Object.freeze(segments);
+        path = new Path(segments);
+      }
+      cache1.set(str, path);
+    }
+
+    return path;
+  }
+
+  public concat(path2: Path): Path {
+    if (!path2._string) return this;
+    else if (!this._string) return path2;
+
+    const str = `${this._string}.${path2._string}`;
+
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        const segments = this.segments.concat(path2.segments);
+        Object.freeze(segments);
+        path = new Path(segments);
+      }
+      cache1.set(str, path);
+    }
+
+    return path;
+  }
+
+  public stripAlias(): Path {
+    if (!this.alias) return this;
+    const segments = this.segments.map(s => (Array.isArray(s) ? "*" : s));
+    const str = segments.join(".");
+
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        Object.freeze(segments);
+        path = new Path(segments);
+      }
+      cache1.set(str, path);
+    }
+
+    return path;
+  }
+}
+
+const interval = setInterval(() => {
+  cache2 = cache1;
+  cache1 = new Map();
+}, 120000);
+
+// Don't hold Node.js process
+if (interval.unref) interval.unref();

lib/common/sat-solver.ts

@@ -0,0 +1,175 @@
+/**
+ * Naive DPLL solver for propositional logic adapted from
+ * https://github.com/tammet/logictools
+ *
+ * Origianl code by Tanel Tammet released under the MIT license.
+ *
+ * Check whether a clause set is satisfiable or contradictory, using a naive
+ * implementation of the dpll (Davis-Putnam-Logemann-Loveland) procedure
+ * combining truth table search with the resolution-like unit propagation.
+ *
+ * The standard pure-literal rule is not implemented in this naive version,
+ * since it is not really crucial for the algorithm.
+ *
+ * The code is very similar to the pure table solver, except we use a
+ * unit-propagating, derived-varlist-returing satisfiableAt.
+ */
+
+/* eslint-disable @typescript-eslint/no-use-before-define */
+
+function satisfiableAt(clauses, varVals, varnr, val): boolean {
+  if (varnr !== 0) varVals[varnr] = val;
+
+  const propRes = unitPropagate(clauses, varVals);
+  if (propRes === 1) {
+    varVals[varnr] = 0;
+    return true;
+  }
+
+  // if result false (-1) unitPropagate restores the varVals state itself
+  if (propRes === -1) {
+    varVals[varnr] = 0;
+    return false;
+  }
+
+  // find next unassigned var to split
+  let nextVar = 0;
+  for (let i = 1; i < varVals.length; i++) {
+    if (varVals[i] === 0) {
+      nextVar = i;
+      break;
+    }
+  }
+
+  if (nextVar === 0) throw new Error("Error in satisfiableAt");
+
+  if (
+    satisfiableAt(clauses, varVals, nextVar, 1) ||
+    satisfiableAt(clauses, varVals, nextVar, -1)
+  )
+    // not necessary to restore split var: search is halted anyway
+    return true;
+
+  // restore split var
+  varVals[nextVar] = 0;
+
+  // restore derived vars
+  for (let i = 0; i < propRes.length; i++) varVals[propRes[i]] = 0;
+
+  return false;
+}
+
+/**
+ * Check whether clauses are satisfiable under the varVals assignment, without
+ * search.
+ *
+ * Iterate through all clauses and check whether
+ *
+ *  - all literals in the clause are assigned false: undo derived units (set
+ *    values in varVals to 0) and return false immediately
+ *
+ *  - a clause contains a literal assigned true: skip further actions on this
+ *    clause
+ *
+ *  - a single literal in the clause is unassigned, all the rest are assigned
+ *    false: derive this literal as a unit clause.
+ *
+ * In case at least one literal is derived during the process above, iterate
+ * the whole process until no more new literals are derived.
+ *
+ * Returns:
+ *
+ *   1: all clauses were assigned true, hence the clause set is satisfiable.
+ *   The whole search will terminate after that.
+ *
+ *   -1: clause set is contradictory. Derived units have been restored.
+ *
+ *   a list of derived variables:  the status of the clause set is
+ *   undetermined. The values of variables on the list will be restored (set to
+ *   0) during backtracking.
+ */
+function unitPropagate(clauses, varVals): number[] | 1 | -1 {
+  // all vars with a value derived in the forthcoming main loop
+  const derivedVars = [];
+
+  // the main loop is run, deriving new unit clauses, until either:
+  // - the clause set is detected to be unsatisfiable
+  // - the clause set is detected to be satisfiable
+  // - there are no more derived unit clauses
+
+  for (;;) {
+    let allClausesFound = true; // set to false if we find an undetermined clause
+    const derivedQueue = []; // derived units (literals) during one iteration
+    for (let i = 0; i < clauses.length; i++) {
+      const clause = clauses[i];
+      let clauseVal = 0;
+      let unassignedCount = 0; // count unassigned vars in clause
+      let unassignedLit = 0; // 0 means none found for this clause
+
+      for (let j = 0; j < clause.length; j++) {
+        const lit = clause[j];
+        let nr, polarity;
+        if (lit < 0) {
+          nr = 0 - lit;
+          polarity = -1;
+        } else {
+          nr = lit;
+          polarity = 1;
+        }
+
+        if (varVals[nr] === polarity) {
+          clauseVal = 1;
+          break;
+        } else if (varVals[nr] === 0) {
+          unassignedCount++;
+          unassignedLit = lit;
+        }
+      }
+
+      if (clauseVal === 1) continue; // clause is subsumed
+
+      // clause is not subsumed by varVals
+      if (unassignedCount === 0) {
+        for (let j = 0; j < derivedVars.length; j++)
+          varVals[derivedVars[j]] = 0; // restore derived
+        return -1;
+      } else if (unassignedCount === 1) {
+        // unassignedLit is a derived literal
+        derivedQueue.push(unassignedLit);
+      }
+
+      if (unassignedCount !== 0) allClausesFound = false;
+    }
+
+    // if all clauses were subsumed or all the variables in a clause had a value
+    // no need to restore varVals: whole search finished
+    if (allClausesFound) return 1;
+
+    // in case there are any derived units present, store to varVals and
+    // iterate the main loop
+    if (derivedQueue.length === 0) break;
+
+    // some units were derived: iterate the main loop
+    for (let j = 0; j < derivedQueue.length; j++) {
+      const lit = derivedQueue[j];
+      let nr, polarity;
+      if (lit < 0) {
+        nr = 0 - lit;
+        polarity = -1;
+      } else {
+        nr = lit;
+        polarity = 1;
+      }
+      varVals[nr] = polarity;
+      derivedVars.push(nr);
+    }
+  }
+
+  return derivedVars;
+}
+
+export function naiveDpll(clauses, maxvarnr): boolean {
+  // variable values are 0 if not set, 1 if positive, -1 if negative
+  const varVals = new Int32Array(maxvarnr + 1);
+  return satisfiableAt(clauses, varVals, 0, 0);
+}

lib/config.coffee

@@ -1,209 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-path = require 'path'
-common = require './common'
-
-options = {
-  CONFIG_DIR : {type : 'path', default : 'config'},
-  MONGODB_CONNECTION_URL : {type : 'string', default : 'mongodb://127.0.0.1/genieacs'},
-  REDIS_PORT : {type : 'int', default : 6379},
-  REDIS_HOST : {type : 'string', default : ''},
-  REDIS_DB : {type : 'int', default : 0},
-
-  CWMP_WORKER_PROCESSES : {type : 'int', default : 0},
-  CWMP_PORT : {type : 'int', default : 7547},
-  CWMP_INTERFACE : {type : 'string', default : '0.0.0.0'},
-  CWMP_SSL : {type : 'bool', default : false},
-  CWMP_LOG_FILE : {type: 'path', default : ''},
-  CWMP_ACCESS_LOG_FILE : {type : 'path', default : ''},
-
-  NBI_WORKER_PROCESSES : {type : 'int', default : 0},
-  NBI_PORT : {type : 'int', default : 7557},
-  NBI_INTERFACE : {type : 'string', default : '0.0.0.0'},
-  NBI_SSL : {type : 'bool', default : false},
-  NBI_LOG_FILE : {type: 'path', default : ''},
-  NBI_ACCESS_LOG_FILE : {type : 'path', default : ''},
-
-  FS_WORKER_PROCESSES : {type : 'int', default : 0},
-  FS_PORT : {type : 'int', default : 7567},
-  FS_INTERFACE : {type : 'string', default : '0.0.0.0'},
-  FS_SSL : {type : 'bool', default : false},
-  FS_HOSTNAME : {type : 'string', default : 'acs.example.com'},
-  FS_LOG_FILE : {type: 'path', default : ''},
-  FS_ACCESS_LOG_FILE : {type : 'path', default : ''},
-
-  UDP_CONNECTION_REQUEST_PORT : {type : 'int', default : 0},
-
-  DOWNLOAD_TIMEOUT: {type : 'int', default : 3600},
-  EXT_TIMEOUT: {type: 'int', default: 3000},
-  MAX_CACHE_TTL : {type : 'int', default : 86400},
-  DEBUG : {type : 'bool', default : false},
-  RETRY_DELAY : {type : 'int', default : 300},
-  SESSION_TIMEOUT : {type : 'int', default : 30},
-  CONNECTION_REQUEST_TIMEOUT : {type : 'int', default: 2000},
-  GPN_NEXT_LEVEL : {type : 'int', default : 0},
-  GPV_BATCH_SIZE : {type : 'int', default : 32},
-  MAX_DEPTH : {type: 'int', default : 16},
-  COOKIES_PATH : {type : 'string'},
-  LOG_FORMAT : {type : 'string', default : 'simple'},
-  ACCESS_LOG_FORMAT : {type : 'string', default : ''},
-  MAX_CONCURRENT_REQUESTS : {type : 'int', default: 20},
-  DATETIME_MILLISECONDS : {type: 'bool', default: true},
-  BOOLEAN_LITERAL : {type: 'bool', default: true},
-  CONNECTION_REQUEST_ALLOW_BASIC_AUTH: {type: 'bool', default: false},
-  MAX_COMMIT_ITERATIONS : {type : 'int', default: 32},
-
-  # XML configuration
-  XML_RECOVER : {type : 'bool', default : false},
-  XML_IGNORE_ENC : {type : 'bool', default : false},
-  XML_FORMAT : {type : 'bool', default : false},
-  XML_NO_DECL : {type : 'bool', default : false},
-  XML_NO_EMPTY : {type : 'bool', default : false},
-  XML_IGNORE_NAMESPACE : {type : 'bool', default : false},
-
-  # Should probably never be changed
-  DEVICE_ONLINE_THRESHOLD : {type : 'int', default : 4000}
-}
-
-allConfig = {}
-
-
-setConfig = (name, value, commandLineArgument) ->
-  return true if allConfig[name]?
-
-  # For compatibility with v1.0
-  if name in ['PRESETS_CACHE_DURATION', 'presets-cache-duration']
-    setConfig('MAX_CACHE_TTL', value)
-
-  if name in ['GET_PARAMETER_NAMES_DEPTH_THRESHOLD', 'get-parameter-names-depth-threshold']
-    setConfig('GPN_NEXT_LEVEL', value)
-
-  if name in ['TASK_PARAMETERS_BATCH_SIZE', 'task-parameters-batch-size']
-    setConfig('GPV_BATCH_SIZE', value)
-
-  if name in ['IGNORE_XML_NAMESPACES', 'ignore-xml-namespaces']
-    setConfig('XML_IGNORE_NAMESPACE', value)
-
-  if name in ['XML_PARSE_IGNORE_ENC', 'xml-parse-ignore-enc']
-    setConfig('XML_IGNORE_ENC', value)
-
-  if name in ['XML_PARSE_RECOVER', 'xml-parse-recover']
-    setConfig('XML_RECOVER', value)
-
-  if name in ['FS_IP', 'fs-ip']
-    setConfig('FS_HOSTNAME', value)
-
-  cast = (val, type) ->
-    switch type
-      when 'int'
-        Number(val)
-      when 'bool'
-        String(val).trim().toLowerCase() in ['true', 'on', 'yes', '1']
-      when 'string'
-        String(val)
-      when 'path'
-        if val then path.resolve(val) else ''
-      else
-        null
-
-  _value = null
-
-  for optionName, optionDetails of options
-    n = optionName
-    if commandLineArgument
-      n = n.toLowerCase().replace(/_/g, '-')
-
-    if name == n
-      _value = cast(value, optionDetails.type)
-      n = optionName
-    else if name.startsWith("#{n}-")
-      _value = cast(value, optionDetails.type)
-      n = "#{optionName}-#{name[optionName.length+1..]}"
-
-    if _value?
-      allConfig[n] = _value
-      # Save as environmnet variable to pass on to any child processes
-      process.env["GENIEACS_#{n}"] = _value
-      return true
-
-  return false
-
-
-# Command line arguments
-argv = process.argv[2..]
-while argv.length
-  arg = argv.shift()
-  if arg[0] == '-'
-    v = argv.shift()
-    setConfig(arg[2..], v, true)
-
-
-# Environment variable
-for k, v of process.env
-  continue if k.lastIndexOf('GENIEACS_', 0) != 0
-  k = k[9..] # remove "GENIEACS_" prefix
-  setConfig(k, v)
-
-
-# Use default config dir if none defined
-setConfig('CONFIG_DIR', options['CONFIG_DIR'].default)
-
-# Configuration file
-for k, v of require(path.resolve(allConfig.CONFIG_DIR, 'config'))
-  if not setConfig(k, v)
-    # Pass as environment variable to be accessable by extensions
-    process.env["GENIEACS_#{k}"] = v
-
-
-# Defaults
-for k, v of options
-  setConfig(k, v.default) if v.default?
-
-
-get = (option, deviceId) ->
-  if deviceId?
-    name = "#{option}-#{deviceId}"
-    v = allConfig[name]
-    return v if v?
-    i = name.lastIndexOf('-')
-    v = allConfig[name[0...i]]
-    return v if v?
-    i = name.lastIndexOf('-', i-1)
-    v = allConfig[name[0...i]]
-    return v if v? or i == -1
-
-  return allConfig[option]
-
-
-getDefault = (optionName) ->
-  option = options[optionName]
-  return null if not option
-  val = option.default
-  if val and option.type == 'path'
-    val = path.resolve(val)
-  return val
-
-
-# load authentication scripts
-try
-  exports.auth = require(path.resolve(allConfig.CONFIG_DIR, 'auth'))
-
-exports.get = get
-exports.getDefault = getDefault

lib/config.ts

@@ -0,0 +1,320 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { resolve } from "path";
+import { readFileSync, existsSync } from "fs";
+
+// Find project root directory
+export let ROOT_DIR = resolve(__dirname, "..");
+while (!existsSync(`${ROOT_DIR}/package.json`)) {
+  const d = resolve(ROOT_DIR, "..");
+  if (d === ROOT_DIR) {
+    ROOT_DIR = process.cwd();
+    break;
+  }
+  ROOT_DIR = d;
+}
+
+// For compatibility with v1.1
+let configDir, cwmpSsl, nbiSsl, fsSsl, uiSsl;
+
+const options = {
+  EXT_DIR: { type: "path", default: resolve(ROOT_DIR, "config/ext") },
+  MONGODB_CONNECTION_URL: {
+    type: "string",
+    default: "mongodb://127.0.0.1/genieacs"
+  },
+
+  CWMP_WORKER_PROCESSES: { type: "int", default: 0 },
+  CWMP_PORT: { type: "int", default: 7547 },
+  CWMP_INTERFACE: { type: "string", default: "0.0.0.0" },
+  CWMP_SSL_CERT: { type: "string", default: "" },
+  CWMP_SSL_KEY: { type: "string", default: "" },
+  CWMP_LOG_FILE: { type: "path", default: "" },
+  CWMP_ACCESS_LOG_FILE: { type: "path", default: "" },
+
+  NBI_WORKER_PROCESSES: { type: "int", default: 0 },
+  NBI_PORT: { type: "int", default: 7557 },
+  NBI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  NBI_SSL_CERT: { type: "string", default: "" },
+  NBI_SSL_KEY: { type: "string", default: "" },
+  NBI_LOG_FILE: { type: "path", default: "" },
+  NBI_ACCESS_LOG_FILE: { type: "path", default: "" },
+
+  FS_WORKER_PROCESSES: { type: "int", default: 0 },
+  FS_PORT: { type: "int", default: 7567 },
+  FS_INTERFACE: { type: "string", default: "0.0.0.0" },
+  FS_SSL_CERT: { type: "string", default: "" },
+  FS_SSL_KEY: { type: "string", default: "" },
+  FS_HOSTNAME: { type: "string", default: "" },
+  FS_LOG_FILE: { type: "path", default: "" },
+  FS_ACCESS_LOG_FILE: { type: "path", default: "" },
+
+  UI_WORKER_PROCESSES: { type: "int", default: 0 },
+  UI_PORT: { type: "int", default: 3000 },
+  UI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  UI_SSL_CERT: { type: "string", default: "" },
+  UI_SSL_KEY: { type: "string", default: "" },
+  UI_LOG_FILE: { type: "path", default: "" },
+  UI_ACCESS_LOG_FILE: { type: "path", default: "" },
+  UI_JWT_SECRET: { type: "string", default: "" },
+
+  UDP_CONNECTION_REQUEST_PORT: { type: "int", default: 0 },
+
+  DOWNLOAD_TIMEOUT: { type: "int", default: 3600 },
+  EXT_TIMEOUT: { type: "int", default: 3000 },
+  MAX_CACHE_TTL: { type: "int", default: 86400 },
+  DEBUG_FILE: { type: "path", default: "" },
+  DEBUG_FORMAT: { type: "string", default: "yaml" },
+  DEBUG: { type: "bool", default: false },
+  RETRY_DELAY: { type: "int", default: 300 },
+  SESSION_TIMEOUT: { type: "int", default: 30 },
+  CONNECTION_REQUEST_TIMEOUT: { type: "int", default: 2000 },
+  GPN_NEXT_LEVEL: { type: "int", default: 0 },
+  GPV_BATCH_SIZE: { type: "int", default: 32 },
+  MAX_DEPTH: { type: "int", default: 16 },
+  COOKIES_PATH: { type: "string" },
+  LOG_FORMAT: { type: "string", default: "simple" },
+  ACCESS_LOG_FORMAT: { type: "string", default: "" },
+  MAX_CONCURRENT_REQUESTS: { type: "int", default: 20 },
+  DATETIME_MILLISECONDS: { type: "bool", default: true },
+  BOOLEAN_LITERAL: { type: "bool", default: true },
+  CONNECTION_REQUEST_ALLOW_BASIC_AUTH: { type: "bool", default: false },
+  MAX_COMMIT_ITERATIONS: { type: "int", default: 32 },
+
+  // XML configuration
+  XML_RECOVER: { type: "bool", default: false },
+  XML_IGNORE_ENC: { type: "bool", default: false },
+  XML_FORMAT: { type: "bool", default: false },
+  XML_NO_DECL: { type: "bool", default: false },
+  XML_NO_EMPTY: { type: "bool", default: false },
+  XML_LIBXMLJS: { type: "bool", default: false },
+
+  // Should probably never be changed
+  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 }
+};
+
+const allConfig: { [name: string]: string | number } = {};
+
+function setConfig(name, value, commandLineArgument = false): boolean {
+  if (allConfig[name] != null) return true;
+
+  // For compatibility with v1.1
+  if (name === "CONFIG_DIR" || name === "config-dir")
+    configDir = configDir || resolve(ROOT_DIR, value);
+
+  if (name === "CWMP_SSL" || name === "cwmp-ssl") {
+    cwmpSsl =
+      cwmpSsl ||
+      String(value)
+        .toLowerCase()
+        .trim();
+  }
+
+  if (name === "NBI_SSL" || name === "nbi-ssl") {
+    nbiSsl =
+      nbiSsl ||
+      String(value)
+        .toLowerCase()
+        .trim();
+  }
+
+  if (name === "FS_SSL" || name === "fs-ssl") {
+    fsSsl =
+      fsSsl ||
+      String(value)
+        .toLowerCase()
+        .trim();
+  }
+
+  if (name === "UI_SSL" || name === "ui-ssl") {
+    uiSsl =
+      uiSsl ||
+      String(value)
+        .toLowerCase()
+        .trim();
+  }
+
+  // For compatibility with v1.0
+  if (name === "PRESETS_CACHE_DURATION" || name === "presets-cache-duration")
+    setConfig("MAX_CACHE_TTL", value);
+
+  if (
+    name === "GET_PARAMETER_NAMES_DEPTH_THRESHOLD" ||
+    name === "get-parameter-names-depth-threshold"
+  )
+    setConfig("GPN_NEXT_LEVEL", value);
+
+  if (
+    name === "TASK_PARAMETERS_BATCH_SIZE" ||
+    name === "task-parameters-batch-size"
+  )
+    setConfig("GPV_BATCH_SIZE", value);
+
+  if (name === "XML_PARSE_IGNORE_ENC" || name === "xml-parse-ignore-enc")
+    setConfig("XML_IGNORE_ENC", value);
+
+  if (name === "XML_PARSE_RECOVER" || name === "xml-parse-recover")
+    setConfig("XML_RECOVER", value);
+
+  if (name === "FS_IP" || name === "fs-ip") setConfig("FS_HOSTNAME", value);
+
+  function cast(val, type): string | number | boolean {
+    switch (type) {
+      case "int":
+        return Number(val);
+      case "bool":
+        return ["true", "1"].includes(
+          String(val)
+            .trim()
+            .toLowerCase()
+        );
+      case "string":
+        return String(val);
+      case "path":
+        if (!val) return "";
+        return resolve(val);
+      default:
+        return null;
+    }
+  }
+
+  let _value = null;
+  for (const [optionName, optionDetails] of Object.entries(options)) {
+    let n = optionName;
+    if (commandLineArgument) n = n.toLowerCase().replace(/_/g, "-");
+
+    if (name === n) {
+      _value = cast(value, optionDetails.type);
+      n = optionName;
+    } else if (name.startsWith(`${n}-`)) {
+      _value = cast(value, optionDetails.type);
+      n = `${optionName}-${name.slice(optionName.length + 1)}`;
+    }
+
+    if (_value != null) {
+      allConfig[n] = _value;
+      // Save as environmnet variable to pass on to any child process
+      process.env[`GENIEACS_${n}`] = _value;
+      return true;
+    }
+  }
+
+  return false;
+}
+
+// Command line arguments
+const argv = process.argv.slice(2);
+while (argv.length) {
+  const arg = argv.shift();
+  if (arg[0] === "-") {
+    const v = argv.shift();
+    setConfig(arg.slice(2), v, true);
+  }
+}
+
+// Environment variable
+for (const [k, v] of Object.entries(process.env))
+  if (k.startsWith("GENIEACS_")) setConfig(k.slice(9), v);
+
+// Configuration file
+const configFilename = configDir
+  ? `${configDir}/config.json`
+  : `${ROOT_DIR}/config/config.json`;
+
+if (existsSync(configFilename)) {
+  const configFile = JSON.parse(readFileSync(configFilename).toString());
+
+  for (const [k, v] of Object.entries(configFile)) {
+    if (!setConfig(k, v))
+      // Pass as environment variable to be accessable by extensions
+      process.env[`GENIEACS_${k}`] = `${v}`;
+  }
+}
+
+if (configDir) setConfig("EXT_DIR", `${configDir}/ext`);
+
+if (["true", "1"].includes(cwmpSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("CWMP_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("CWMP_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (["true", "1"].includes(nbiSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("NBI_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("NBI_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (["true", "1"].includes(fsSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("FS_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("FS_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (["true", "1"].includes(uiSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("UI_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("UI_SSL_KEY", `${d}/cwmp.key`);
+}
+
+// Defaults
+for (const [k, v] of Object.entries(options))
+  if (v["default"] != null) setConfig(k, v["default"]);
+
+export function get(
+  optionName: string,
+  deviceId?: string
+): string | number | boolean {
+  if (!deviceId) return allConfig[optionName];
+
+  optionName = `${optionName}-${deviceId}`;
+  let v = allConfig[optionName];
+  if (v != null) return v;
+
+  let i = optionName.lastIndexOf("-");
+  v = allConfig[optionName.slice(0, i)];
+  if (v != null) return v;
+
+  i = optionName.lastIndexOf("-", i - 1);
+  v = allConfig[optionName.slice(0, i)];
+  if (v != null) return v;
+
+  i = optionName.lastIndexOf("-", i - 1);
+  v = allConfig[optionName.slice(0, i)];
+  if (v != null) return v;
+
+  i = optionName.lastIndexOf("-", i - 1);
+  if (i > 0) {
+    v = allConfig[optionName.slice(0, i)];
+    if (v != null) return v;
+  }
+
+  return null;
+}
+
+export function getDefault(optionName): string | number | boolean {
+  const option = options[optionName];
+  if (!option) return null;
+
+  let val = option["default"];
+  if (val && option.type === "path") val = resolve(val);
+
+  return val;
+}

lib/connection-request.ts

@@ -0,0 +1,244 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as crypto from "crypto";
+import * as dgram from "dgram";
+import { parse } from "url";
+import * as http from "http";
+import { evaluateAsync } from "./common/expression";
+import { Expression } from "./types";
+import * as auth from "./auth";
+import * as extensions from "./extensions";
+import * as debug from "./debug";
+
+async function extractAuth(
+  exp: Expression,
+  context: {},
+  now: number,
+  dflt: any
+): Promise<[string, string, Expression]> {
+  let username, password;
+  const _exp = await evaluateAsync(
+    exp,
+    context,
+    now,
+    async (e: Expression): Promise<Expression> => {
+      if (!username && Array.isArray(e) && e[0] === "FUNC") {
+        if (e[1] === "EXT") {
+          if (typeof e[2] !== "string" || typeof e[3] !== "string") return null;
+
+          for (let i = 4; i < e.length; i++)
+            if (Array.isArray(e[i])) return null;
+
+          const { fault, value } = await extensions.run(e.slice(2));
+          return fault ? null : value;
+        } else if (e[1] === "AUTH") {
+          if (!Array.isArray(e[2]) && !Array.isArray(e[3])) {
+            username = e[2];
+            password = e[3];
+          }
+          return dflt;
+        }
+      }
+      return e;
+    }
+  );
+  return [username, password, _exp];
+}
+
+function httpGet(
+  options: http.RequestOptions,
+  timeout,
+  _debug: boolean,
+  deviceId: string
+): Promise<{ statusCode: number; headers: {} }> {
+  return new Promise((resolve, reject) => {
+    const req = http
+      .get(options, res => {
+        res.resume();
+        resolve({ statusCode: res.statusCode, headers: res.headers });
+        if (_debug) {
+          debug.outgoingHttpRequest(req, deviceId, options, null);
+          debug.incomingHttpResponse(res, deviceId, null);
+        }
+      })
+      .on("error", err => {
+        req.abort();
+        reject(new Error("Device is offline"));
+        if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
+      })
+      .on("socket", socket => {
+        socket.setTimeout(timeout);
+        socket.on("timeout", () => {
+          req.abort();
+        });
+      });
+  });
+}
+
+export async function httpConnectionRequest(
+  address: string,
+  authExp: Expression,
+  context: {},
+  allowBasicAuth: boolean,
+  timeout: number,
+  _debug: boolean,
+  deviceId: string
+): Promise<void> {
+  const now = Date.now();
+  const options: http.RequestOptions = parse(address);
+  if (options.protocol !== "http:")
+    throw new Error("Invalid connection request URL or protocol");
+
+  options.agent = new http.Agent({
+    maxSockets: 1,
+    keepAlive: true
+  });
+
+  let authHeader: {};
+  let username: string;
+  let password: string;
+
+  while (!authHeader || (username != null && password != null)) {
+    let opts = options;
+    if (authHeader) {
+      if (authHeader["method"] === "Basic") {
+        if (!allowBasicAuth)
+          throw new Error("Basic HTTP authentication not allowed");
+
+        opts = Object.assign(
+          {
+            headers: {
+              Authorization: auth.basic(username || "", password || "")
+            }
+          },
+          options
+        );
+      } else if (authHeader["method"] === "Digest") {
+        opts = Object.assign(
+          {
+            headers: {
+              Authorization: auth.solveDigest(
+                username,
+                password,
+                options.path,
+                "GET",
+                null,
+                authHeader
+              )
+            }
+          },
+          options
+        );
+      } else {
+        throw new Error("Unrecognized auth method");
+      }
+    }
+
+    let res = await httpGet(opts, timeout, _debug, deviceId);
+
+    // Workaround for some devices unexpectedly closing the connection
+    if (res.statusCode === 0 && authHeader)
+      res = await httpGet(opts, timeout, _debug, deviceId);
+    if (res.statusCode === 0) throw new Error("Device is offline");
+    if (res.statusCode === 200 || res.statusCode === 204) return;
+
+    if (res.statusCode === 401 && res.headers["www-authenticate"]) {
+      authHeader = auth.parseWwwAuthenticateHeader(
+        res.headers["www-authenticate"]
+      );
+      [username, password, authExp] = await extractAuth(
+        authExp,
+        context,
+        now,
+        false
+      );
+    } else {
+      throw new Error(
+        `Unexpected response code from device: ${res.statusCode}`
+      );
+    }
+  }
+  throw new Error("Incorrect connection request credentials");
+}
+
+export async function udpConnectionRequest(
+  address: string,
+  authExp: Expression,
+  context: {},
+  sourcePort: number = 0,
+  _debug: boolean,
+  deviceId: string
+): Promise<void> {
+  const [host, portStr] = address.split(":", 2);
+  const port = portStr ? parseInt(portStr) : 80;
+  const now = Date.now();
+
+  const client = dgram.createSocket({ type: "udp4", reuseAddr: true });
+  // When a device is NAT'ed, the UDP Connection Request must originate from
+  // the same address and port used by the STUN server, in order to traverse
+  // the firewall. This does require that the Genieacs NBI and STUN server
+  // are allowed to bind to the same address and port. The STUN server needs
+  // to open its UDP port with the SO_REUSEADDR option, allowing the NBI to
+  // also bind to the same port.
+  if (sourcePort) client.bind({ port: sourcePort, exclusive: true });
+
+  let username: string;
+  let password: string;
+
+  [username, password, authExp] = await extractAuth(
+    authExp,
+    context,
+    now,
+    null
+  );
+
+  if (username == null) username = "";
+  if (password == null) password = "";
+  while (username != null && password != null) {
+    const ts = Math.trunc(now / 1000);
+    const id = Math.trunc(Math.random() * 4294967295);
+    const cn = crypto.randomBytes(8).toString("hex");
+    const sig = crypto
+      .createHmac("sha1", password)
+      .update(`${ts}${id}${username}${cn}`)
+      .digest("hex");
+    const uri = `http://${address}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
+    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${address}\r\n\r\n`;
+    const message = Buffer.from(msg);
+
+    for (let i = 0; i < 3; ++i) {
+      await new Promise((resolve, reject) => {
+        client.send(message, 0, message.length, port, host, (err: Error) => {
+          if (err) reject(err);
+          else resolve();
+          if (_debug) debug.outgoingUdpMessage(host, deviceId, port, msg);
+        });
+      });
+    }
+
+    [username, password, authExp] = await extractAuth(
+      authExp,
+      context,
+      now,
+      null
+    );
+  }
+  client.close();
+}

lib/cwmp.coffee

@@ -1,965 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-util = require 'util'
-zlib = require 'zlib'
-crypto = require 'crypto'
-
-config = require './config'
-common = require './common'
-soap = require './soap'
-session = require './session'
-query = require './query'
-device = require './device'
-cache = require './cache'
-localCache = require './local-cache'
-db = require './db'
-logger = require './logger'
-scheduling = require './scheduling'
-
-
-MAX_CYCLES = 4
-
-MAX_CONCURRENT_REQUESTS = config.get('MAX_CONCURRENT_REQUESTS')
-stats = {
-  concurrentRequests: 0
-  totalRequests: 0
-  droppedRequests: 0
-  initiatedSessions: 0
-}
-
-throwError = (err, httpResponse) ->
-  if httpResponse
-    currentSessions.delete(httpResponse.connection)
-    httpResponse.writeHead(500, {'Connection' : 'close'})
-    httpResponse.end("#{err.name}: #{err.message}")
-    stats.concurrentRequests -= 1
-
-  throw err
-
-
-writeResponse = (sessionContext, res, close) ->
-  # Close connection after last request in session
-  res.headers['Connection'] = 'close' if close
-
-  if config.get('DEBUG', sessionContext.deviceId)
-    dump = "# RESPONSE #{new Date(Date.now())}\n" + JSON.stringify(res.headers) + "\n#{res.data}\n\n"
-    require('fs').appendFile("./debug/#{sessionContext.deviceId}.dump", dump, (err) ->
-      throwError(err) if err
-    )
-
-  finish = () ->
-    stats.concurrentRequests -= 1
-    if sessionContext.httpRequest.connection.destroyed
-      logger.accessError({
-        sessionContext: sessionContext
-        message: 'Connection dropped'
-      })
-    else if close
-      endSession(sessionContext, (err, isNew) ->
-        return throwError(err) if err
-
-        if isNew
-          logger.accessInfo({
-            sessionContext: sessionContext
-            message: 'New device registered'
-          })
-      )
-    else
-      sessionContext.lastActivity = Date.now()
-      currentSessions.set(sessionContext.httpRequest.connection, sessionContext)
-
-  # respond using the same content-encoding as the request
-  if sessionContext.httpRequest.headers['content-encoding']? and res.data.length > 0
-    switch sessionContext.httpRequest.headers['content-encoding']
-      when 'gzip'
-        res.headers['Content-Encoding'] = 'gzip'
-        compress = zlib.gzip
-      when 'deflate'
-        res.headers['Content-Encoding'] = 'deflate'
-        compress = zlib.deflate
-
-  if compress?
-    compress(res.data, (err, data) ->
-      return throwError(err, sessionContext.httpResponse) if err
-      res.headers['Content-Length'] = data.length
-      sessionContext.httpResponse.writeHead(res.code, res.headers)
-      sessionContext.httpResponse.end(data)
-      finish()
-    )
-  else
-    res.headers['Content-Length'] = res.data.length
-    sessionContext.httpResponse.writeHead(res.code, res.headers)
-    sessionContext.httpResponse.end(res.data)
-    finish()
-
-
-recordFault = (sessionContext, fault, provisions, channels) ->
-  if not provisions
-    provisions = sessionContext.provisions
-    channels = sessionContext.channels
-
-  faults = sessionContext.faults
-
-  for channel of channels
-    provs = sessionContext.faults[channel]?.provisions or []
-    faults[channel] = Object.assign({provisions: provs}, fault)
-    if channel.startsWith('task_')
-      taskId = channel.slice(5)
-      for t in sessionContext.tasks
-        faults[channel].expiry = t.expiry if t._id == taskId and t.expiry
-
-    if sessionContext.retries[channel]?
-      ++ sessionContext.retries[channel]
-    else
-      sessionContext.retries[channel] = 0
-      if Object.keys(channels).length != 1
-        faults[channel].retryNow = true
-
-    if channels[channel] == 0
-      faults[channel].precondition = true
-
-    sessionContext.faultsTouched ?= {}
-    sessionContext.faultsTouched[channel] = true
-
-    logger.accessWarn({
-      sessionContext: sessionContext
-      message: 'Channel has faulted'
-      fault: fault
-      channel: channel
-      retries: sessionContext.retries[channel]
-    })
-
-  for provision, i in provisions
-    for channel of channels
-      if (channels[channel] >> i) & 1
-        faults[channel].provisions.push(provision)
-
-  for channel of channels
-    provs = faults[channel].provisions
-    faults[channel].provisions = []
-    appendProvisions(faults[channel].provisions, provs)
-
-  session.clearProvisions(sessionContext)
-  return
-
-
-inform = (sessionContext, rpc) ->
-  session.inform(sessionContext, rpc.cpeRequest, (err, acsResponse) ->
-    return throwError(err, sessionContext.httpResponse) if err
-    res = soap.response({
-      id: rpc.id,
-      acsResponse: acsResponse,
-      cwmpVersion : sessionContext.cwmpVersion
-    })
-
-    if !!cookiesPath = config.get('COOKIES_PATH', sessionContext.deviceId)
-      res.headers['Set-Cookie'] = "session=#{sessionContext.sessionId}; Path=#{cookiesPath}"
-    else
-      res.headers['Set-Cookie'] = "session=#{sessionContext.sessionId}"
-
-    writeResponse(sessionContext, res)
-  )
-
-
-transferComplete = (sessionContext, rpc) ->
-  session.transferComplete(sessionContext, rpc.cpeRequest, (err, acsResponse, operation, fault) ->
-    return throwError(err, sessionContext.httpResponse) if err
-
-    if not operation?
-      logger.accessWarn({
-        sessionContext: sessionContext
-        message: 'Unrecognized command key'
-        rpc: rpc
-      })
-
-    if fault
-      for k, v of operation.retries
-        sessionContext.retries[k] = v
-
-      recordFault(sessionContext, fault, operation.provisions, operation.channels)
-
-    res = soap.response({
-      id : rpc.id,
-      acsResponse : acsResponse,
-      cwmpVersion : sessionContext.cwmpVersion
-    })
-    writeResponse(sessionContext, res)
-  )
-
-
-# Append providions and remove duplicates
-appendProvisions = (original, toAppend) ->
-  modified = false
-  stringified = new WeakMap()
-  for p, i in original
-    stringified.set(p, JSON.stringify(p))
-
-  for p, i in toAppend by -1
-    s = JSON.stringify(p)
-    for pp, j in original by -1
-      ss = stringified.get(pp)
-      if s == ss
-        if (not p) or j >= original.length - (toAppend.length - i)
-          p = null
-        else
-          original.splice(j, 1)
-          modified = true
-    if p
-      original.splice(original.length - (toAppend.length - i) + 1, 0, p)
-      stringified.set(p, s)
-      modified = true
-
-  return modified
-
-
-applyPresets = (sessionContext) ->
-  localCache.getPresets((err, presetsHash, presets) ->
-    return throwError(err, sessionContext.httpResponse) if err
-
-    # Filter presets based on existing faults
-    blackList = {}
-    whiteList = null
-    whiteListProvisions = null
-
-    RETRY_DELAY = config.get('RETRY_DELAY', sessionContext.deviceId)
-
-    if sessionContext.faults?
-      for channel, fault of sessionContext.faults
-        if fault.retryNow
-          retryTimestamp = 0
-        else
-          retryTimestamp = fault.timestamp + (RETRY_DELAY * Math.pow(2, sessionContext.retries[channel])) * 1000
-
-        if retryTimestamp <= sessionContext.timestamp
-          whiteList = channel
-          whiteListProvisions = fault.provisions
-          break
-
-        blackList[channel] = if fault.precondition then 1 else 2
-
-    sessionContext.deviceData.timestamps.revision = 1
-    sessionContext.deviceData.attributes.revision = 1
-
-    deviceEvents = {}
-    for p in sessionContext.deviceData.paths.find(['Events', '*'], false, true)
-      if sessionContext.timestamp <= sessionContext.deviceData.attributes.get(p)?.value?[1][0]
-        deviceEvents[p[1]] = true
-
-    parameters = {}
-    filteredPresets = []
-
-    for preset in presets
-      if whiteList?
-        continue if preset.channel != whiteList
-      else if blackList[preset.channel] == 1
-        continue
-
-      eventsMatch = true
-      for k, v of preset.events
-        if (!v) != (!deviceEvents[k.replace(' ', '_')])
-          eventsMatch = false
-          break
-
-      continue if not eventsMatch
-
-      if preset.schedule
-        continue if not preset.schedule.schedule
-        r = scheduling.cron(sessionContext.timestamp, preset.schedule.schedule)
-        if not (r[0] + preset.schedule.duration > sessionContext.timestamp)
-          continue
-
-      filteredPresets.push(preset)
-      for k of preset.precondition
-        sessionContext.channels[preset.channel] = 0
-        p = k.split(/([^a-zA-Z0-9\-\_\.].*)/, 1)[0]
-        parameters[p] = common.parsePath(p)
-
-    declarations = []
-    for k, v of parameters
-      declarations.push([v, 1, {value: 1}])
-
-    session.rpcRequest(sessionContext, declarations, (err, fault, id, acsRequest) ->
-      return throwError(err, sessionContext.httpResponse) if err
-
-      if fault
-        recordFault(sessionContext, fault)
-        session.clearProvisions(sessionContext)
-        return applyPresets(sessionContext)
-
-      if acsRequest
-        return sendAcsRequest(sessionContext, id, acsRequest)
-
-      session.clearProvisions(sessionContext)
-
-      for k, v of parameters
-        unpacked = device.unpack(sessionContext.deviceData, v)
-        if unpacked[0] and (vv = sessionContext.deviceData.attributes.get(unpacked[0]).value?[1])?
-          parameters[k] = vv[0]
-        else
-          delete parameters[k]
-
-      if whiteList?
-        session.addProvisions(sessionContext, whiteList, whiteListProvisions)
-
-      appendProvisionsToFaults = {}
-      for p in filteredPresets
-        if query.testFilter(parameters, p.precondition)
-          if blackList[p.channel] == 2
-            appendProvisionsToFaults[p.channel] =
-              (appendProvisionsToFaults[p.channel] or []).concat(p.provisions)
-          else
-            session.addProvisions(sessionContext, p.channel, p.provisions)
-
-      for channel, provisions of appendProvisionsToFaults
-        if appendProvisions(sessionContext.faults[channel].provisions, provisions)
-          sessionContext.faultsTouched ?= {}
-          sessionContext.faultsTouched[channel] = true
-
-      # Don't increment when processing individual channels (e.g. after fault)
-      sessionContext.presetCycles = (sessionContext.presetCycles or 0) + 1 if not whiteList?
-
-      if sessionContext.presetCycles > MAX_CYCLES
-        fault = {
-          code: 'preset_loop'
-          message: 'The presets are stuck in an endless configuration loop'
-          timestamp: sessionContext.timestamp
-        }
-        recordFault(sessionContext, fault)
-        # No need to save retryNow
-        for c of sessionContext.faults
-          delete sessionContext.faults[c].retryNow
-        session.clearProvisions(sessionContext)
-        return sendAcsRequest(sessionContext)
-
-      sessionContext.deviceData.timestamps.dirty = 0;
-      sessionContext.deviceData.attributes.dirty = 0;
-
-      session.rpcRequest(sessionContext, null, (err, fault, id, acsRequest) ->
-        return throwError(err, sessionContext.httpResponse) if err
-
-        if fault
-          recordFault(sessionContext, fault)
-          session.clearProvisions(sessionContext)
-          return applyPresets(sessionContext)
-
-        if not acsRequest
-          for channel, flags of sessionContext.channels
-            if channel of sessionContext.faults
-              delete sessionContext.faults[channel]
-              sessionContext.faultsTouched ?= {}
-              sessionContext.faultsTouched[channel] = true
-
-          if whiteList?
-            return applyPresets(sessionContext)
-
-          if sessionContext.deviceData.timestamps.dirty > 1 or
-              sessionContext.deviceData.attributes.dirty > 1
-            return applyPresets(sessionContext)
-
-        sendAcsRequest(sessionContext, id, acsRequest)
-      )
-    )
-  )
-
-
-nextRpc = (sessionContext) ->
-  session.rpcRequest(sessionContext, null, (err, fault, id, acsRequest) ->
-    return throwError(err, sessionContext.httpResponse) if err
-
-    if fault
-      recordFault(sessionContext, fault)
-      session.clearProvisions(sessionContext)
-      return nextRpc(sessionContext)
-
-    if acsRequest
-      return sendAcsRequest(sessionContext, id, acsRequest)
-
-    for channel, flags of sessionContext.channels
-      if flags and channel of sessionContext.faults
-        delete sessionContext.faults[channel]
-        sessionContext.faultsTouched ?= {}
-        sessionContext.faultsTouched[channel] = true
-
-      if channel.startsWith('task_')
-        taskId = channel.slice(5)
-        sessionContext.doneTasks ?= []
-        sessionContext.doneTasks.push(taskId)
-        for t, j in sessionContext.tasks
-          if t._id == taskId
-            sessionContext.tasks.splice(j, 1)
-            break
-
-    session.clearProvisions(sessionContext)
-
-    # Clear expired tasks
-    sessionContext.tasks = sessionContext.tasks.filter((task) ->
-      return true if not (task.expiry <= sessionContext.timestamp)
-
-      logger.accessInfo({
-        sessionContext: sessionContext
-        message: 'Task expired'
-        task: task
-      })
-      sessionContext.doneTasks ?= []
-      sessionContext.doneTasks.push(task._id)
-      channel = "task_#{task._id}"
-      if channel of sessionContext.faults
-        delete sessionContext.faults[channel]
-        sessionContext.faultsTouched ?= {}
-        sessionContext.faultsTouched[channel] = true
-
-      return false
-    )
-
-    task = sessionContext.tasks.find((task) -> !sessionContext.faults["task_#{task._id}"])
-
-    return applyPresets(sessionContext) if not task
-
-    switch task.name
-      when 'getParameterValues'
-        # Set channel in case params array is empty
-        sessionContext.channels["task_#{task._id}"] = 0
-        for p in task.parameterNames
-          session.addProvisions(sessionContext, "task_#{task._id}",
-            [['refresh', p]])
-      when 'setParameterValues'
-        # Set channel in case params array is empty
-        sessionContext.channels["task_#{task._id}"] = 0
-        for p in task.parameterValues
-          session.addProvisions(sessionContext, "task_#{task._id}",
-            [['value', p[0], p[1]]])
-      when 'refreshObject'
-        session.addProvisions(sessionContext, "task_#{task._id}",
-          [['refresh', task.objectName]])
-      when 'reboot'
-        session.addProvisions(sessionContext, "task_#{task._id}",
-          [['reboot']])
-      when 'factoryReset'
-        session.addProvisions(sessionContext, "task_#{task._id}",
-          [['reset']])
-      when 'download'
-        session.addProvisions(sessionContext, "task_#{task._id}",
-          [['download', task.fileType, task.fileName, task.targetFileName]])
-      when 'addObject'
-        alias = ("#{p[0]}:#{JSON.stringify(p[1])}" for p in task.parameterValues or []).join(',')
-        session.addProvisions(sessionContext, "task_#{task._id}",
-          [['instances', "#{task.objectName}.[#{alias}]", "+1"]]);
-      when 'deleteObject'
-        session.addProvisions(sessionContext, "task_#{task._id}",
-          [['instances', task.objectName, 0]]);
-      else
-        return throwError(new Error('Task name not recognized'), sessionContext.httpResponse)
-
-    return nextRpc(sessionContext)
-  )
-
-
-endSession = (sessionContext, callback) ->
-  saveCache = sessionContext.cacheUntil?
-
-  counter = 3
-
-  counter += 2
-  db.saveDevice(sessionContext.deviceId, sessionContext.deviceData, sessionContext.new, sessionContext.timestamp, (err) ->
-    if err
-      callback(err) if counter & 1
-      return counter = 0
-
-    return callback(null, sessionContext.new) if (counter -= 2) == 1
-  )
-
-  for k of sessionContext.operationsTouched
-    counter += 2
-    saveCache = true
-    if sessionContext.operations[k]?
-      db.saveOperation(sessionContext.deviceId, k, sessionContext.operations[k], (err) ->
-        if err
-          callback(err) if counter & 1
-          return counter = 0
-
-        return callback(null, sessionContext.new) if (counter -= 2) == 1
-      )
-    else
-      db.deleteOperation(sessionContext.deviceId, k, (err) ->
-        if err
-          callback(err) if counter & 1
-          return counter = 0
-
-        return callback(null, sessionContext.new) if (counter -= 2) == 1
-      )
-
-  if sessionContext.doneTasks?.length
-    counter += 2
-    saveCache = true
-    db.clearTasks(sessionContext.deviceId, sessionContext.doneTasks, (err) ->
-      if err
-        callback(err) if counter & 1
-        return counter = 0
-
-      return callback(null, sessionContext.new) if (counter -= 2) == 1
-    )
-
-  for k of sessionContext.faultsTouched
-    counter += 2
-    saveCache = true
-    if sessionContext.faults[k]
-      sessionContext.faults[k].retries = sessionContext.retries[k]
-      db.saveFault(sessionContext.deviceId, k, sessionContext.faults[k], (err) ->
-        if err
-          callback(err) if counter & 1
-          return counter = 0
-
-        return callback(null, sessionContext.new) if (counter -= 2) == 1
-      )
-    else
-      db.deleteFault(sessionContext.deviceId, k, (err) ->
-        if err
-          callback(err) if counter & 1
-          return counter = 0
-
-        return callback(null, sessionContext.new) if (counter -= 2) == 1
-      )
-
-  if saveCache
-    counter += 2
-    cacheDueTasksAndFaultsAndOperations(sessionContext.deviceId, sessionContext.tasks, sessionContext.faults, sessionContext.operations, sessionContext.cacheUntil, (err) ->
-      if err
-        callback(err) if counter & 1
-        return counter = 0
-
-      return callback(null, sessionContext.new) if (counter -= 2) == 1
-    )
-
-  callback(null, sessionContext.new) if (counter -= 2) == 1
-
-
-sendAcsRequest = (sessionContext, id, acsRequest) ->
-  if not acsRequest?
-    return writeResponse(sessionContext, soap.response(null), true)
-
-  if acsRequest.name is 'Download'
-    if not acsRequest.url?
-      FS_PORT = config.get('FS_PORT')
-      FS_HOSTNAME = config.get('FS_HOSTNAME')
-      FS_SSL = config.get('FS_SSL')
-      acsRequest.url = if FS_SSL then 'https://' else 'http://'
-      acsRequest.url += FS_HOSTNAME
-      acsRequest.url += ":#{FS_PORT}" if FS_PORT != 80
-      acsRequest.url += "/#{encodeURI(acsRequest.fileName)}"
-
-    if not acsRequest.fileSize?
-      return localCache.getFiles((err, hash, files) ->
-        return throwError(err, sessionContext.httpResponse) if err
-
-        if acsRequest.fileName of files
-          acsRequest.fileSize = files[acsRequest.fileName].length
-        else
-          acsRequest.fileSize = 0
-
-        return sendAcsRequest(sessionContext, id, acsRequest)
-      )
-
-  rpc = {
-    id : id,
-    acsRequest : acsRequest,
-    cwmpVersion : sessionContext.cwmpVersion
-  }
-
-  logger.accessInfo({
-    sessionContext: sessionContext
-    message: 'ACS request'
-    rpc: rpc
-  })
-
-  res = soap.response(rpc)
-
-  writeResponse(sessionContext, res)
-
-
-getSession = (connection, sessionId, callback) ->
-  sessionContext = currentSessions.get(connection)
-
-  if sessionContext
-    currentSessions.delete(connection)
-    return callback(null, sessionContext)
-
-  return callback() if not sessionId
-
-  setTimeout(() ->
-    cache.pop("session_#{sessionId}", (err, sessionContextString) ->
-      return callback(err) if err or not sessionContextString
-      session.deserialize(sessionContextString, callback)
-    )
-  , 100)
-
-
-currentSessions = new WeakMap()
-
-# When socket closes, store active sessions in cache
-onConnection = (socket) ->
-  # The property remoteAddress may be undefined after the connection is
-  # closed, unless we read it at least once (caching?)
-  socket.remoteAddress
-
-  socket.on('close', () ->
-    sessionContext = currentSessions.get(socket)
-    return if not sessionContext
-    currentSessions.delete(socket)
-    now = Date.now()
-
-    lastActivity = sessionContext.lastActivity
-    timeoutMsg = logger.flatten({
-      sessionContext: sessionContext
-      message: 'Session timeout'
-      sessionTimestamp: sessionContext.timestamp
-    })
-
-    timeout = (sessionContext.lastActivity + sessionContext.timeout * 1000) - now
-    if timeout <= 0
-      return logger.accessError(timeoutMsg)
-
-    setTimeout(() ->
-      cache.get("session_#{sessionContext.sessionId}", (err, sessionContextString) ->
-        return throwError(err) if err
-        return if not sessionContextString
-        session.deserialize(sessionContextString, (err, sessionContext) ->
-          return throwError(err) if err
-          if sessionContext.lastActivity == lastActivity
-            logger.accessError(timeoutMsg)
-        )
-      )
-    , timeout + 1000).unref()
-
-    session.serialize(sessionContext, (err, sessionContextString) ->
-      return throwError(err) if err
-      cache.set("session_#{sessionContext.sessionId}", sessionContextString, Math.ceil(timeout / 1000) + 3, (err) ->
-        return throwError(err) if err
-      )
-    )
-  )
-
-setInterval(() ->
-  if stats.droppedRequests
-    logger.warn({
-      message: 'Worker overloaded'
-      droppedRequests: stats.droppedRequests
-      totalRequests: stats.totalRequests
-      initiatedSessions: stats.initiatedSessions
-      pid: process.pid
-    })
-
-  stats.totalRequests = 0
-  stats.droppedRequests = 0
-  stats.initiatedSessions = 0
-, 10000).unref()
-
-
-getDueTasksAndFaultsAndOperations = (deviceId, timestamp, callback) ->
-  cache.get("#{deviceId}_tasks_faults_operations", (err, res) ->
-    return callback(err) if err
-
-    if res
-      res = JSON.parse(res)
-      return callback(null, res.tasks or [], res.faults or {}, res.operations or {})
-
-    faults = tasks = operations = cacheUntil = null
-
-    db.getFaults(deviceId, (err, _faults) ->
-      if err
-        callback?(err)
-        return callback = null
-
-      faults = _faults
-      if tasks? and operations?
-        return callback(null, tasks, faults, operations, cacheUntil)
-    )
-
-    db.getDueTasks(deviceId, timestamp, (err, dueTasks, nextTimestamp) ->
-      if err
-        callback?(err)
-        return callback = null
-
-      tasks = dueTasks
-      cacheUntil = nextTimestamp or 0
-
-      if faults? and operations?
-        return callback(null, tasks, faults, operations, cacheUntil)
-    )
-
-    db.getOperations(deviceId, (err, _operations) ->
-      if err
-        callback?(err)
-        return callback = null
-
-      operations = _operations
-
-      if faults? and tasks?
-        return callback(null, tasks, faults, operations, cacheUntil)
-    )
-
-    if faults? and tasks? and operations?
-      return callback(null, tasks, faults, operations, cacheUntil)
-  )
-
-
-cacheDueTasksAndFaultsAndOperations = (deviceId, tasks, faults, operations, cacheUntil, callback) ->
-  v = {}
-  v.tasks = tasks if tasks.length
-  v.faults = faults if Object.keys(faults).length
-  v.operations = operations if Object.keys(operations).length
-
-  if cacheUntil
-    ttl = Math.trunc((Date.now() - cacheUntil) / 1000)
-  else
-    ttl = config.get('MAX_CACHE_TTL', deviceId)
-
-  cache.set("#{deviceId}_tasks_faults_operations", JSON.stringify(v), ttl, callback)
-
-
-processRequest = (sessionContext, rpc) ->
-  if rpc.cpeRequest?
-    if rpc.cpeRequest.name is 'Inform'
-      logger.accessInfo({
-        sessionContext: sessionContext
-        message: 'Inform'
-        rpc: rpc
-      })
-      inform(sessionContext, rpc)
-    else if rpc.cpeRequest.name is 'TransferComplete'
-      logger.accessInfo({
-        sessionContext: sessionContext
-        message: 'CPE request'
-        rpc: rpc
-      })
-      transferComplete(sessionContext, rpc)
-    else if rpc.cpeRequest.name is 'GetRPCMethods'
-      logger.accessInfo({
-        sessionContext: sessionContext
-        message: 'CPE request'
-        rpc: rpc
-      })
-      res = soap.response({
-        id : rpc.id
-        acsResponse : {name: 'GetRPCMethodsResponse', methodList: ['Inform', 'GetRPCMethods', 'TransferComplete']}
-        cwmpVersion : sessionContext.cwmpVersion
-      })
-      writeResponse(sessionContext, res)
-    else
-      return throwError(new Error('ACS method not supported'), sessionContext.httpResponse)
-  else if rpc.cpeResponse
-    session.rpcResponse(sessionContext, rpc.id, rpc.cpeResponse, (err) ->
-      return throwError(err, sessionContext.httpResponse) if err
-      nextRpc(sessionContext)
-    )
-  else if rpc.cpeFault
-    logger.accessWarn({
-      sessionContext: sessionContext
-      message: 'CPE fault'
-      rpc:rpc
-    })
-    session.rpcFault(sessionContext, rpc.id, rpc.cpeFault, (err, fault) ->
-      return throwError(err, sessionContext.httpResponse) if err
-
-      if fault
-        recordFault(sessionContext, fault)
-        session.clearProvisions(sessionContext)
-      nextRpc(sessionContext)
-    )
-  else # CPE sent empty response
-    session.timeoutOperations(sessionContext, (err, faults, operations) ->
-      return throwError(err, sessionContext.httpResponse) if err
-
-      for fault, i in faults
-        for k, v of operations[i].retries
-          sessionContext.retries[k] = v
-
-        recordFault(sessionContext, fault, operations[i].provisions, operations[i].channels)
-
-      nextRpc(sessionContext)
-    )
-
-
-listener = (httpRequest, httpResponse) ->
-  stats.totalRequests += 1
-  if httpRequest.method != 'POST'
-    httpResponse.writeHead 405, {'Allow': 'POST', 'Connection': 'close'}
-    httpResponse.end('405 Method Not Allowed')
-    return
-
-  sessionId = null
-  # Separation by comma is important as some devices don't comform to standard
-  COOKIE_REGEX = /\s*([a-zA-Z0-9\-_]+?)\s*=\s*"?([a-zA-Z0-9\-_]*?)"?\s*(,|;|$)/g
-  while match = COOKIE_REGEX.exec(httpRequest.headers.cookie)
-    sessionId = match[2] if match[1] == 'session'
-
-  # If overloaded, ask CPE to retry in 60 seconds
-  if not sessionId and stats.concurrentRequests > MAX_CONCURRENT_REQUESTS
-    httpResponse.writeHead(503, {'Retry-after': 60, 'Connection': 'close'})
-    httpResponse.end('503 Service Unavailable')
-    stats.droppedRequests += 1
-    return
-
-  if httpRequest.headers['content-encoding']?
-    switch httpRequest.headers['content-encoding']
-      when 'gzip'
-        stream = httpRequest.pipe(zlib.createGunzip())
-      when 'deflate'
-        stream = httpRequest.pipe(zlib.createInflate())
-      else
-        httpResponse.writeHead(415, {'Connection': 'close'})
-        httpResponse.end('415 Unsupported Media Type')
-        return
-  else
-    stream = httpRequest
-
-  stats.concurrentRequests += 1
-  httpRequest.on('aborted', () ->
-    stats.concurrentRequests -= 1
-    # In some cases event end can be emitted after aborted event
-    httpRequest.removeAllListeners('end')
-  )
-
-  chunks = []
-  bytes = 0
-
-  stream.on('data', (chunk) ->
-    chunks.push(chunk)
-    bytes += chunk.length
-  )
-
-  stream.on('end', () ->
-    body = new Buffer(bytes)
-    offset = 0
-    chunks.forEach((chunk) ->
-      chunk.copy(body, offset, 0, chunk.length)
-      offset += chunk.length
-    )
-
-    parsedRpc = (sessionContext, rpc, parseWarnings) ->
-      for w in parseWarnings
-        w.sessionContext = sessionContext
-        w.rpc = rpc;
-        logger.accessWarn(w)
-
-      if config.get('DEBUG', sessionContext.deviceId)
-        dump = "# REQUEST #{new Date(Date.now())}\n" + JSON.stringify(httpRequest.headers) + "\n#{body}\n\n"
-        require('fs').appendFile("./debug/#{sessionContext.deviceId}.dump", dump, (err) ->
-          return throwError(err) if err
-        )
-
-      processRequest(sessionContext, rpc)
-
-    getSession(httpRequest.connection, sessionId, (err, sessionContext) ->
-      return throwError(err, httpResponse) if err
-
-      if sessionContext
-        sessionContext.httpRequest = httpRequest
-        sessionContext.httpResponse = httpResponse
-
-        if sessionContext.sessionId != sessionId or
-            sessionContext.lastActivity + sessionContext.timeout * 1000 < Date.now()
-          logger.accessError({message: 'Invalid session', sessionContext: sessionContext})
-          httpResponse.writeHead(400, {'Connection': 'close'})
-          httpResponse.end('Invalid session')
-          stats.concurrentRequests -= 1
-          return
-      else if stats.concurrentRequests > MAX_CONCURRENT_REQUESTS
-        # Check again just in case device included old session ID from the previous session
-        httpResponse.writeHead(503, {'Retry-after': 60, 'Connection': 'close'})
-        httpResponse.end('503 Service Unavailable')
-        stats.droppedRequests += 1
-        stats.concurrentRequests -= 1
-        return
-
-      parseWarnings = []
-      try
-        rpc = soap.request(body, sessionContext?.cwmpVersion, parseWarnings)
-      catch err
-        logger.accessError({
-          message: 'XML parse error'
-          parseError: err.message.trim()
-          sessionContext: sessionContext or {httpRequest: httpRequest, httpResponse: httpResponse}
-        })
-        httpResponse.writeHead(400, {'Connection': 'close'})
-        httpResponse.end(err.message)
-        stats.concurrentRequests -= 1
-        return
-
-      if sessionContext
-        if rpc.cpeRequest?.name is 'Inform' or
-            (!sessionContext.rpcRequest ^ !(rpc.cpeResponse or rpc.cpeFault))
-          logger.accessError({
-            message: 'Bad session state'
-            sessionContext: sessionContext
-          })
-          httpResponse.writeHead(400, {'Connection': 'close'})
-          httpResponse.end('Bad session state')
-          stats.concurrentRequests -= 1
-          return
-
-        return parsedRpc(sessionContext, rpc, parseWarnings)
-
-      if rpc.cpeRequest?.name isnt 'Inform'
-        logger.accessError({
-          message: 'Invalid session'
-          sessionContext: sessionContext or {httpRequest: httpRequest, httpResponse: httpResponse}
-        })
-        httpResponse.writeHead(400, {'Connection': 'close'})
-        httpResponse.end('Invalid session')
-        stats.concurrentRequests -= 1
-        return
-
-      stats.initiatedSessions += 1
-      deviceId = common.generateDeviceId(rpc.cpeRequest.deviceId)
-      return session.init(deviceId, rpc.cwmpVersion, rpc.sessionTimeout ? config.get('SESSION_TIMEOUT', deviceId), (err, sessionContext) ->
-        return throwError(err, httpResponse) if err
-
-        sessionContext.httpRequest = httpRequest
-        sessionContext.httpResponse = httpResponse
-        sessionContext.sessionId = crypto.randomBytes(8).toString('hex')
-        httpRequest.connection.setTimeout(sessionContext.timeout * 1000)
-
-        getDueTasksAndFaultsAndOperations(deviceId, sessionContext.timestamp, (err, dueTasks, faults, operations, cacheUntil) ->
-          return throwError(err, httpResponse) if err
-
-          sessionContext.tasks = dueTasks
-          sessionContext.faults = faults
-          sessionContext.retries = {}
-          for k, v of faults
-            sessionContext.retries[k] = v.retries
-          sessionContext.operations = operations
-          sessionContext.cacheUntil = cacheUntil
-
-          # Delete expired faults
-          for k, v of sessionContext.faults
-            if v.expiry >= sessionContext.timestamp
-              delete sessionContext.faults[k]
-              sessionContext.faultsTouched ?= {}
-              sessionContext.faultsTouched[k] = true
-
-          parsedRpc(sessionContext, rpc, parseWarnings)
-        )
-      )
-    )
-  )
-
-
-exports.listener = listener
-exports.onConnection = onConnection

lib/db.coffee

@@ -1,652 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-config = require './config'
-mongodb = require 'mongodb'
-common = require './common'
-
-tasksCollection = null
-devicesCollection = null
-presetsCollection = null
-objectsCollection = null
-provisionsCollection = null
-virtualParametersCollection = null
-faultsCollection = null
-filesCollection = null
-operationsCollection = null
-
-
-connect = (callback) ->
-  callbackCounter = 9
-  mongodb.MongoClient.connect(config.get('MONGODB_CONNECTION_URL'), (err, db) ->
-    return callback(err) if err
-    exports.mongoDb = db
-    db.collection('tasks', (err, collection) ->
-      exports.tasksCollection = tasksCollection = collection
-      collection?.ensureIndex({device: 1, timestamp: 1}, (err) ->
-      )
-
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('devices', (err, collection) ->
-      exports.devicesCollection  = devicesCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('presets', (err, collection) ->
-      exports.presetsCollection = presetsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('objects', (err, collection) ->
-      exports.objectsCollection = objectsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('fs.files', (err, collection) ->
-      exports.filesCollection = filesCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('provisions', (err, collection) ->
-      exports.provisionsCollection = provisionsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('virtualParameters', (err, collection) ->
-      exports.virtualParametersCollection = virtualParametersCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('faults', (err, collection) ->
-      exports.faultsCollection = faultsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('operations', (err, collection) ->
-      exports.operationsCollection = operationsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-  )
-
-
-disconnect = () ->
-  exports.mongoDb?.close()
-
-
-# Optimize projection by removing overlaps
-# This can modify the object
-optimizeProjection = (obj) ->
-  if obj['']
-    return {'': obj['']}
-
-  keys = Object.keys(obj).sort()
-  return obj if keys.length <= 1
-  i = 1
-  while i < keys.length
-    a = keys[i-1]
-    b = keys[i]
-    if b.startsWith(a)
-      if b.charAt(a.length) == '.' or b.charAt(a.length - 1) == '.'
-        delete obj[b]
-        keys.splice(i--, 1)
-    ++ i
-  return obj
-
-
-fetchDevice = (id, timestamp, patterns, callback) ->
-  MAX_DEPTH = config.get('MAX_DEPTH', id)
-
-  if not patterns?.length
-    patterns = [[[], (1 << MAX_DEPTH) - 1]]
-
-  projection = {}
-  projectionTree = {}
-  func = (path, pats, projTree) ->
-    children = {}
-    for pat in pats
-      fragment = pat[0][path.length] or '*'
-
-      if fragment == '*'
-        projection[path.join('.')] = 1 if pat[1] << path.length
-        return
-
-      projTree[fragment] ?= {}
-
-      if pat[1] & (1 << path.length)
-        projection[path.concat('_timestamp').join('.')] = 1
-        projection[path.concat('_object').join('.')] = 1
-        projection[path.concat('_instance').join('.')] = 1
-        projection[path.concat([fragment, '_timestamp']).join('.')] = 1
-        projection[path.concat([fragment, '_value']).join('.')] = 1
-        projection[path.concat([fragment, '_type']).join('.')] = 1
-        projection[path.concat([fragment, '_object']).join('.')] = 1
-        projection[path.concat([fragment, '_instance']).join('.')] = 1
-        projection[path.concat([fragment, '_writable']).join('.')] = 1
-        projection[path.concat([fragment, '_orig']).join('.')] = 1
-
-      if pat[1] >> (path.length + 1)
-        children[fragment] ?= []
-        children[fragment].push(pat)
-
-    for k, v of children
-      func(path.concat(k), v, projTree[k])
-
-  func([], patterns, projectionTree)
-
-  delete projectionTree['DeviceID']
-  delete projectionTree['Events']
-  delete projectionTree['Tags']
-
-  res = []
-  loaded = []
-
-  projection = optimizeProjection(projection)
-
-  for k, v of projection
-    if k == '' or k == 'Events' or k.startsWith('Events.')
-      if k == '' or k == 'Events' or k == 'Events._writable'
-        res.push([['Events'], timestamp,
-          {object: [timestamp, 1], writable: [timestamp, 0]}])
-        loaded.push([['Events'], 1]) if k == 'Events._writable'
-      if k == 'Events'
-        projection['_lastInform'] = 1
-        projection['_lastBoot'] = 1
-        projection['_lastBootstrap'] = 1
-        projection['_registered'] = 1
-        loaded.push([['Events'], ((1 << MAX_DEPTH) - 1)])
-      else if k == 'Events.Inform._writable' or k == 'Events.Inform'
-        projection['_lastInform'] = 1
-        loaded.push([['Events', 'Inform'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'Events.1_BOOT._writable' or k == 'Events.1_BOOT'
-        projection['_lastBoot'] = 1
-        loaded.push([['Events', '1_BOOT'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'Events.0_BOOTSTRAP._writable' or k == 'Events.0_BOOTSTRAP'
-        projection['_lastBootstrap'] = 1
-        loaded.push([['Events', '0_BOOTSTRAP'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'Events.Registered._writable' or k == 'Events.Registered'
-        projection['_registered'] = 1
-        loaded.push([['Events', 'Registered'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k.endsWith('._writable') and k != 'Events._writable'
-        loaded.push([k.split('.').slice(0, 2), 1 ^ ((1 << MAX_DEPTH) - 1)])
-      delete projection[k] if k != ''
-
-    if k == '' or k == 'DeviceID' or k.startsWith('DeviceID.')
-      if k == '' or k == 'DeviceID' or k == 'DeviceID._writable'
-        res.push([['DeviceID'], timestamp,
-          {object: [timestamp, 1], writable: [timestamp, 0]}])
-        loaded.push([['DeviceID'], 1]) if k == 'DeviceID._writable'
-      if k == 'DeviceID'
-        projection['_id'] = 1
-        projection['_deviceId'] = 1
-        loaded.push([['DeviceID'], ((1 << MAX_DEPTH) - 1)])
-      else if k == 'DeviceID.ID._writable' or k == 'DeviceID.ID'
-        projection['_id'] = 1
-        loaded.push([['DeviceID', 'ID'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'DeviceID.Manufacturer._writable' or k == 'DeviceID.DeManufacturer'
-        projection['_deviceId._Manufacturer'] = 1
-        loaded.push([['DeviceID', 'Manufacturer'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'DeviceID.ProductClass._writable' or k == 'DeviceID.ProductClass'
-        projection['_deviceId._ProductClass'] = 1
-        loaded.push([['DeviceID', 'ProductClass'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'DeviceID.OUI._writable' or k == 'DeviceID.OUI'
-        projection['_deviceId._OUI'] = 1
-        loaded.push([['DeviceID', 'ProductClass'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'DeviceID.SerialNumber._writable' or k == 'DeviceID.SerialNumber'
-        projection['_deviceId._SerialNumber'] = 1
-        loaded.push([['DeviceID', 'SerialNumber'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k.endsWith('._writable') and k != 'DeviceID._writable'
-        loaded.push([k.split('.').slice(0, 2), 1 ^ ((1 << MAX_DEPTH) - 1)])
-      delete projection[k] if k != ''
-
-    if k == 'Tags' or k.startsWith('Tags.')
-      if not projection['_tags']
-        projection['_tags'] = 1
-        loaded.push([['Tags'], ((1 << MAX_DEPTH) - 1)])
-      delete projection[k]
-
-  if projection['']
-    proj = {}
-  else if Object.keys(projection).length == 0
-    proj = {'_id': 1}
-  else
-    proj = projection
-
-  devicesCollection.findOne({'_id' : id}, proj, (err, device) ->
-    return callback(err) if err or not device?
-
-    storeParams = (obj, path, timestamp, descendantsFetched, projTree) ->
-      if descendantsFetched
-        thisFetched = true
-      else
-        if projection[path.join('.')]
-          descendantsFetched = true
-          if path.length and projection[path.slice(0, -1).concat("_timestamp").join('.')]
-            thisFetched = true
-            loaded.push([path, ((1 << (path.length - 1)) - 1) ^ ((1 << MAX_DEPTH) - 1)])
-          else
-            loaded.push([path, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])
-        else if projection[path.concat('_writable').join('.')]
-          loaded.push([path, 1 << (path.length - 1)])
-          thisFetched = true
-
-      if obj['_timestamp']?
-        obj['_timestamp'] = +obj['_timestamp']
-
-      # For compatibility with v1.0 database
-      if obj['_instance'] and not obj['_object']?
-        obj['_object'] = true
-
-      if thisFetched
-        attrs = {}
-
-        t = obj['_timestamp'] or 1
-        t = timestamp if timestamp > t
-
-        if obj['_value']?
-          attrs.value = [obj['_timestamp'] or 1, [obj['_value'], obj['_type']]]
-          attrs.value[1][0] = +attrs.value[1][0] if obj['_type'] == 'xsd:dateTime'
-          obj['_object'] = false
-
-        if obj['_writable']?
-          attrs.writable = [timestamp or 1, if obj['_writable'] then 1 else 0]
-
-        if obj['_object']?
-          attrs.object = [t, if obj['_object'] then 1 else 0]
-
-        res.push([path, t, attrs])
-
-      for k, v of obj
-        if not k.startsWith('_')
-          kk = k
-          obj['_object'] = true
-          storeParams(v, path.concat(k), obj['_timestamp'], descendantsFetched, projTree?[k])
-          delete projTree[kk] if projTree
-
-      if not descendantsFetched
-        for k, v of projTree
-          p = path.concat(k)
-          loaded.push([p, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])
-          if (obj['_object'] or path.length == 0) and obj['_timestamp']
-            res.push([p, obj['_timestamp']])
-      else if (obj['_object'] or path.length == 0) and obj['_timestamp']
-          res.push([path.concat('*'), obj['_timestamp']])
-
-    for k, v of device
-      switch k
-        when '_lastInform'
-          res.push([['Events', 'Inform'], +v,
-            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
-          delete device[k]
-        when '_lastBoot'
-          res.push([['Events', '1_BOOT'], +v,
-            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
-          delete device[k]
-        when '_lastBootstrap'
-          res.push([['Events', '0_BOOTSTRAP'], +v,
-            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
-          delete device[k]
-        when '_registered'
-          # Use current timestamp for registered event attribute timestamps
-          res.push([['Events', 'Registered'], timestamp,
-            {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [+v, 'xsd:dateTime']]}])
-          delete device[k]
-        when '_id'
-          if projection[''] or projection['_id']
-            res.push([['DeviceID', 'ID'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v, 'xsd:string']]}])
-          delete device[k]
-        when '_tags'
-          if v.length
-            res.push([['Tags'], timestamp,
-              {object: [timestamp, 1], writable: [timestamp, 0]}])
-          for t in v
-            t = t.replace(/[^a-zA-Z0-9\-]+/g, '_')
-            res.push([['Tags', t], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 1], value: [timestamp, [true, 'xsd:boolean']]}])
-          delete device[k]
-        when '_deviceId'
-          if v['_Manufacturer']?
-            res.push([['DeviceID', 'Manufacturer'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_Manufacturer'], 'xsd:string']]}])
-          if v['_OUI']?
-            res.push([['DeviceID', 'OUI'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_OUI'], 'xsd:string']]}])
-          if v['_ProductClass']?
-            res.push([['DeviceID', 'ProductClass'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_ProductClass'], 'xsd:string']]}])
-          if v['_SerialNumber']?
-            res.push([['DeviceID', 'SerialNumber'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_SerialNumber'], 'xsd:string']]}])
-          delete device[k]
-
-    storeParams(device, [], 0, false, projectionTree)
-
-    return callback(null, res, loaded)
-  )
-
-
-saveDevice = (deviceId, deviceData, isNew, sessionTimestamp, callback) ->
-  update = {'$set' : {}, '$unset' : {}, '$addToSet' : {}, '$pull' : {}}
-
-  iter = deviceData.timestamps.diff()
-  while diff = iter.next().value
-    continue if diff[0].wildcard != (1 << (diff[0].length - 1))
-    continue if diff[0][0] in ['Events', 'DeviceID', 'Tags']
-
-    # Param timestamps may be greater than session timestamp to track revisions
-    if diff[2] > sessionTimestamp
-      diff[2] = sessionTimestamp
-
-    if not diff[2]? and diff[1]?
-      update['$unset'][diff[0].slice(0, -1).concat('_timestamp').join('.')] = 1
-    else if diff[2] != diff[1]
-      parent = deviceData.paths.get(diff[0].slice(0, -1))
-      if parent and (parent.length == 0 or deviceData.attributes.has(parent))
-        update['$set'][diff[0].slice(0, -1).concat('_timestamp').join('.')] = new Date(diff[2])
-
-  iter = deviceData.attributes.diff()
-  while diff = iter.next().value
-    continue if diff[1] == diff[2]
-    path = diff[0]
-    switch path[0]
-      when 'Events'
-        if diff[0].length == 2 and diff[2]?.value?[1][0] != diff[1]?.value?[1][0]
-          if not diff[2]
-            switch path[1]
-              when 'Inform'
-                update['$unset']['_lastInform'] = 1
-              when '1_BOOT'
-                update['$unset']['_lastBoot'] = 1
-              when '0_BOOTSTRAP'
-                update['$unset']['_lastBootstrap'] = 1
-              when 'Registered'
-                update['$unset']['_registered'] = 1
-          else
-            t = new Date(diff[2].value[1][0])
-            switch path[1]
-              when 'Inform'
-                update['$set']['_lastInform'] = t
-              when '1_BOOT'
-                update['$set']['_lastBoot'] = t
-              when '0_BOOTSTRAP'
-                update['$set']['_lastBootstrap'] = t
-              when 'Registered'
-                update['$set']['_registered'] = t
-      when 'DeviceID'
-        if diff[2].value?[1]?[0] != diff[1]?.value?[1]?[0]
-          v = diff[2].value[1][0]
-          switch path[1]
-            when 'ID'
-              update['$set']['_id'] = v
-            when 'Manufacturer'
-              update['$set']['_deviceId._Manufacturer'] = v
-            when 'OUI'
-              update['$set']['_deviceId._OUI'] = v
-            when 'ProductClass'
-              update['$set']['_deviceId._ProductClass'] = v
-            when 'SerialNumber'
-              update['$set']['_deviceId._SerialNumber'] = v
-      when 'Tags'
-        if diff[2]?.value?[1]?[0] != diff[1]?.value?[1]?[0]
-          v = diff[2]?.value?[1][0]
-          if v?
-            update['$addToSet']['_tags'] ?= {'$each' : []}
-            update['$addToSet']['_tags']['$each'].push(diff[0][1])
-          else
-            update['$pull']['_tags'] ?= {'$in' : []}
-            update['$pull']['_tags']['$in'].push(diff[0][1])
-      else
-        if not diff[2]
-          update['$unset'][diff[0].join('.')] = 1
-          continue
-
-        for attrName of diff[2]
-          # Param timestamps may be greater than session timestamp to track revisions
-          if diff[2][attrName][0] > sessionTimestamp
-            diff[2][attrName][0] = sessionTimestamp
-
-          if diff[2][attrName][1]?
-            switch attrName
-              when 'value'
-                if diff[2].value[1][0] != diff[1]?.value?[1]?[0]
-                  if diff[2].value[1][1] == 'xsd:dateTime' and Number.isInteger(diff[2].value[1][0])
-                    update['$set'][path.concat('_value').join('.')] = new Date(diff[2].value[1][0])
-                  else
-                    update['$set'][path.concat('_value').join('.')] = diff[2].value[1][0]
-                if diff[2].value[1][1] != diff[1]?.value?[1]?[1]
-                  update['$set'][path.concat('_type').join('.')] = diff[2].value[1][1]
-                if diff[2].value[0] != diff[1]?.value?[0]
-                  update['$set'][path.concat('_timestamp').join('.')] = new Date(diff[2].value[0])
-              when 'object'
-                if not diff[1]?.object or diff[2].object[1] != diff[1].object?[1]
-                  update['$set'][path.concat('_object').join('.')] = !!diff[2].object[1]
-              when 'writable'
-                if not diff[1]?.writable or diff[2].writable[1] != diff[1].writable?[1]
-                  update['$set'][path.concat('_writable').join('.')] = !!diff[2].writable[1]
-
-        if diff[1]
-          for attrName of diff[1]
-            if diff[1][attrName][1]? and not diff[2]?[attrName]?[1]?
-              update['$unset'][path.concat("_#{attrName}").join('.')] = 1
-              if attrName is 'value'
-                update['$unset'][path.concat('_type').join('.')] = 1
-                update['$unset'][path.concat('_timestamp').join('.')] = 1
-
-  update['$unset'] = optimizeProjection(update['$unset'])
-
-  # Remove overlap possibly caused by parameters changing from objects
-  # to regular parameters or vice versa. Reason being that _timestamp
-  # represents two different things depending on whether the parameter
-  # is an object or not.
-  for k of update['$unset']
-    delete update['$unset'][k] if update['$set'][k]?
-
-  # Remove empty keys
-  for k of update
-    if k == '$addToSet'
-      for kk of update[k]
-        delete update[k][kk] if update[k][kk]['$each'].length == 0
-    else if k == '$pull'
-      for kk of update[k]
-        delete update[k][kk] if update[k][kk]['$in'].length == 0
-    delete update[k] if Object.keys(update[k]).length == 0
-
-  return callback() if Object.keys(update).length == 0
-
-  if update['$addToSet'] and update['$pull']
-    # Mongo doesn't allow $addToSet and $pull at the same time
-    update2 = {'$pull': update['$pull']}
-    delete update['$pull']
-
-  devicesCollection.update({'_id' : deviceId}, update, {upsert: isNew}, (err, result) ->
-    if not err and result.result.n != 1
-      return callback(new Error("Device #{deviceId} not found in database"))
-
-    if update2
-      return devicesCollection.update({'_id': deviceId}, update2, callback)
-
-    return callback(err)
-  )
-
-
-getFaults = (deviceId, callback) ->
-  faultsCollection.find({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}).toArray((err, res) ->
-    return callback(err) if err
-
-    faults = {}
-    for r in res
-      channel = r._id.slice(deviceId.length + 1)
-      delete r._id
-      delete r.channel
-      delete r.device
-      r.timestamp = +r.timestamp
-      r.provisions = JSON.parse(r.provisions)
-      faults[channel] = r
-
-    callback(err, faults)
-  )
-
-
-saveFault = (deviceId, channel, fault, callback) ->
-  fault = Object.assign({}, fault)
-  fault._id = "#{deviceId}:#{channel}"
-  fault.device = deviceId
-  fault.channel = channel
-  fault.timestamp = new Date(fault.timestamp)
-  fault.provisions = JSON.stringify(fault.provisions)
-  faultsCollection.save(fault, callback)
-
-
-deleteFault = (deviceId, channel, callback) ->
-  faultsCollection.remove({_id: "#{deviceId}:#{channel}"}, callback)
-
-
-getDueTasks = (deviceId, timestamp, callback) ->
-  cur = tasksCollection.find({'device' : deviceId}).sort(['timestamp'])
-
-  tasks = []
-
-  cur.nextObject(f = (err, task) ->
-    return callback(err) if err
-
-    if not task?
-      return callback(null, tasks, null)
-
-    task.timestamp = +task.timestamp if task.timestamp?
-    task.expiry = +task.expiry if task.expiry?
-
-    if task.timestamp >= timestamp
-      return callback(null, tasks, +task.timestamp)
-
-    task._id = String(task._id)
-    tasks.push(task)
-
-    # For API compatibility
-    if task.name is 'download' and task.file?
-      if mongodb.ObjectID.isValid(task.file)
-        q = {_id: {'$in' : [task.file, new mongodb.ObjectID(task.file)]}}
-      else
-        q = {_id: task.file}
-
-      filesCollection.find(q).toArray((err, res) ->
-        return callback(err) if err
-        if res[0]?
-          task.fileType ?= res[0].metadata.fileType
-          task.fileName ?= res[0].filename or res[0]._id.toString()
-        cur.nextObject(f)
-      )
-    else
-      cur.nextObject(f)
-  )
-
-
-clearTasks = (deviceId, taskIds, callback) ->
-  tasksCollection.remove({'_id' : {'$in' : (new mongodb.ObjectID(id) for id in taskIds)}}, callback)
-
-
-getOperations = (deviceId, callback) ->
-  operationsCollection.find({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}).toArray((err, res) ->
-    return callback(err) if err
-
-    operations = {}
-    for r in res
-      commandKey = r._id.slice(deviceId.length + 1)
-      delete r._id
-      r.timestamp = +r.timestamp
-      r.args = JSON.parse(r.args) if r.args
-      r.provisions = JSON.parse(r.provisions)
-      r.retries = JSON.parse(r.retries)
-      operations[commandKey] = r
-
-    callback(err, operations)
-  )
-
-
-saveOperation = (deviceId, commandKey, operation, callback) ->
-  operation = Object.assign({}, operation)
-  operation._id = "#{deviceId}:#{commandKey}"
-  operation.timestamp = new Date(operation.timestamp)
-  operation.provisions = JSON.stringify(operation.provisions)
-  operation.retries = JSON.stringify(operation.retries)
-  operation.args = JSON.stringify(operation.args)
-  operationsCollection.save(operation, callback)
-
-
-deleteOperation = (deviceId, commandKey, callback) ->
-  operationsCollection.remove({_id: "#{deviceId}:#{commandKey}"}, callback)
-
-
-getPresets = (callback) ->
-  presetsCollection.find().toArray(callback)
-
-
-getObjects = (callback) ->
-  objectsCollection.find().toArray(callback)
-
-
-getProvisions = (callback) ->
-  provisionsCollection.find().toArray(callback)
-
-
-getVirtualParameters = (callback) ->
-  virtualParametersCollection.find().toArray(callback)
-
-
-getFiles = (callback) ->
-  filesCollection.find().toArray(callback)
-
-
-exports.connect = connect
-exports.disconnect = disconnect
-exports.fetchDevice = fetchDevice
-exports.saveDevice = saveDevice
-exports.getFaults = getFaults
-exports.saveFault = saveFault
-exports.deleteFault = deleteFault
-exports.clearTasks = clearTasks
-exports.saveOperation = saveOperation
-exports.deleteOperation = deleteOperation
-exports.getPresets = getPresets
-exports.getObjects = getObjects
-exports.getProvisions = getProvisions
-exports.getVirtualParameters = getVirtualParameters
-exports.getFiles = getFiles
-exports.getDueTasks = getDueTasks
-exports.getOperations = getOperations

lib/db.ts

@@ -0,0 +1,764 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { MongoClient, ObjectID, Collection } from "mongodb";
+import { get } from "./config";
+import { escapeRegExp } from "./common";
+import { parse } from "./common/expression-parser";
+import {
+  DeviceData,
+  Attributes,
+  SessionFault,
+  Task,
+  Operation,
+  Expression
+} from "./types";
+import Path from "./common/path";
+
+export let tasksCollection: Collection,
+  devicesCollection: Collection,
+  presetsCollection: Collection,
+  objectsCollection: Collection,
+  provisionsCollection: Collection,
+  virtualParametersCollection: Collection,
+  faultsCollection: Collection,
+  filesCollection: Collection,
+  operationsCollection: Collection,
+  permissionsCollection: Collection,
+  usersCollection: Collection,
+  configCollection: Collection;
+
+let clientPromise: Promise<MongoClient>;
+export let client: MongoClient;
+
+function compareAccessLists(list1: string[], list2: string[]): boolean {
+  if (list1.length !== list2.length) return false;
+  for (const [i, v] of list1.entries()) if (v !== list2[i]) return false;
+  return true;
+}
+
+export async function connect(): Promise<void> {
+  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
+    useNewUrlParser: true
+  });
+
+  client = await clientPromise;
+  const db = client.db();
+
+  tasksCollection = db.collection("tasks");
+  await tasksCollection.createIndex({ device: 1, timestamp: 1 });
+
+  devicesCollection = db.collection("devices");
+  presetsCollection = db.collection("presets");
+  objectsCollection = db.collection("objects");
+  filesCollection = db.collection("fs.files");
+  provisionsCollection = db.collection("provisions");
+  virtualParametersCollection = db.collection("virtualParameters");
+  faultsCollection = db.collection("faults");
+  operationsCollection = db.collection("operations");
+  permissionsCollection = db.collection("permissions");
+  usersCollection = db.collection("users");
+  configCollection = db.collection("config");
+}
+
+export async function disconnect(): Promise<void> {
+  if (clientPromise) await (await clientPromise).close();
+}
+
+// Optimize projection by removing overlaps
+// This can modify the object
+function optimizeProjection(obj: {}): {} {
+  if (obj[""]) return { "": obj[""] };
+
+  const keys = Object.keys(obj).sort();
+  if (keys.length <= 1) return obj;
+
+  for (let i = 1; i < keys.length; ++i) {
+    const a = keys[i - 1];
+    const b = keys[i];
+    if (b.startsWith(a)) {
+      if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") {
+        delete obj[b];
+        keys.splice(i--, 1);
+      }
+    }
+  }
+  return obj;
+}
+
+export async function fetchDevice(
+  id: string,
+  timestamp: number
+): Promise<[Path, number, Attributes?][]> {
+  const res: [Path, number, Attributes?][] = [
+    [
+      Path.parse("Events"),
+      timestamp,
+      { object: [timestamp, 1], writable: [timestamp, 0] }
+    ],
+    [
+      Path.parse("DeviceID"),
+      timestamp,
+      { object: [timestamp, 1], writable: [timestamp, 0] }
+    ]
+  ];
+
+  const device = await devicesCollection.findOne({ _id: id });
+  if (!device) return null;
+
+  function storeParams(obj, path: string, pathLength: number, ts): void {
+    if (obj["_timestamp"]) obj["_timestamp"] = +obj["_timestamp"];
+    if (obj["_attributesTimestamp"])
+      obj["_attributesTimestamp"] = +obj["_attributesTimestamp"];
+
+    const attrs: Attributes = {};
+    let t = obj["_timestamp"] || 1;
+    if (ts > t) t = ts;
+
+    if (obj["_value"] != null) {
+      attrs.value = [obj["_timestamp"] || 1, [obj["_value"], obj["_type"]]];
+      if (obj["_type"] === "xsd:dateTime")
+        attrs.value[1][0] = +attrs.value[1][0];
+
+      obj["_object"] = false;
+    }
+    if (obj["_writable"] != null)
+      attrs.writable = [ts || 1, obj["_writable"] ? 1 : 0];
+
+    if (obj["_object"] != null) attrs.object = [t, obj["_object"] ? 1 : 0];
+
+    if (obj["_notification"] != null) {
+      attrs.notification = [
+        obj["_attributesTimestamp"] || 1,
+        obj["_notification"]
+      ];
+    }
+
+    if (obj["_accessList"] != null)
+      attrs.accessList = [obj["_attributesTimestamp"] || 1, obj["_accessList"]];
+
+    res.push([Path.parse(path.slice(0, -1)), t, attrs]);
+
+    for (const [k, v] of Object.entries(obj)) {
+      if (!k.startsWith("_")) {
+        obj["_object"] = true;
+        storeParams(v, path + k + ".", pathLength + 1, obj["_timestamp"]);
+      }
+    }
+
+    if ((obj["_object"] || !pathLength) && obj["_timestamp"])
+      res.push([Path.parse(path + "*"), obj["_timestamp"]]);
+  }
+
+  for (const [k, v] of Object.entries(device)) {
+    switch (k) {
+      case "_lastInform":
+        res.push([
+          Path.parse("Events.Inform"),
+          +v,
+          {
+            object: [+v, 0],
+            writable: [+v, 0],
+            value: [+v, [+v, "xsd:dateTime"]]
+          }
+        ]);
+        break;
+      case "_lastBoot":
+        res.push([
+          Path.parse("Events.1_BOOT"),
+          +v,
+          {
+            object: [+v, 0],
+            writable: [+v, 0],
+            value: [+v, [+v, "xsd:dateTime"]]
+          }
+        ]);
+        break;
+      case "_lastBootstrap":
+        res.push([
+          Path.parse("Events.0_BOOTSTRAP"),
+          +v,
+          {
+            object: [+v, 0],
+            writable: [+v, 0],
+            value: [+v, [+v, "xsd:dateTime"]]
+          }
+        ]);
+        break;
+      case "_registered":
+        // Use current timestamp for registered event attribute timestamps
+        res.push([
+          Path.parse("Events.Registered"),
+          timestamp,
+          {
+            object: [timestamp, 0],
+            writable: [timestamp, 0],
+            value: [timestamp, [+v, "xsd:dateTime"]]
+          }
+        ]);
+        break;
+      case "_id":
+        res.push([
+          Path.parse("DeviceID.ID"),
+          timestamp,
+          {
+            object: [timestamp, 0],
+            writable: [timestamp, 0],
+            value: [timestamp, [v as string, "xsd:string"]]
+          }
+        ]);
+        break;
+      case "_tags":
+        if ((v as string[]).length) {
+          res.push([
+            Path.parse("Tags"),
+            timestamp,
+            { object: [timestamp, 1], writable: [timestamp, 0] }
+          ]);
+        }
+
+        for (let t of v as string[]) {
+          t = t.replace(/[^a-zA-Z0-9-]+/g, "_");
+          res.push([
+            Path.parse("Tags." + t),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 1],
+              value: [timestamp, [true, "xsd:boolean"]]
+            }
+          ]);
+        }
+        break;
+      case "_deviceId":
+        if (v["_Manufacturer"] != null) {
+          res.push([
+            Path.parse("DeviceID.Manufacturer"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_Manufacturer"], "xsd:string"]]
+            }
+          ]);
+        }
+
+        if (v["_OUI"] != null) {
+          res.push([
+            Path.parse("DeviceID.OUI"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_OUI"], "xsd:string"]]
+            }
+          ]);
+        }
+
+        if (v["_ProductClass"] != null) {
+          res.push([
+            Path.parse("DeviceID.ProductClass"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_ProductClass"], "xsd:string"]]
+            }
+          ]);
+        }
+
+        if (v["_SerialNumber"] != null) {
+          res.push([
+            Path.parse("DeviceID.SerialNumber"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_SerialNumber"], "xsd:string"]]
+            }
+          ]);
+        }
+    }
+  }
+
+  storeParams(device, "", 0, 0);
+  return res;
+}
+
+export async function saveDevice(
+  deviceId: string,
+  deviceData: DeviceData,
+  isNew: boolean,
+  sessionTimestamp: number
+): Promise<void> {
+  const update = { $set: {}, $unset: {}, $addToSet: {}, $pull: {} };
+
+  for (const diff of deviceData.timestamps.diff()) {
+    if (diff[0].wildcard !== 1 << (diff[0].length - 1)) continue;
+
+    if (
+      diff[0].segments[0] === "Events" ||
+      diff[0].segments[0] === "DeviceID" ||
+      diff[0].segments[0] === "Tags"
+    )
+      continue;
+
+    const parent = deviceData.paths.get(diff[0].slice(0, -1));
+
+    // Param timestamps may be greater than session timestamp to track revisions
+    if (diff[2] > sessionTimestamp) diff[2] = sessionTimestamp;
+
+    if (diff[2] == null && diff[1] != null) {
+      update["$unset"][
+        parent.length ? parent.toString() + "._timestamp" : "_timestamp"
+      ] = 1;
+    } else {
+      if (parent && (!parent.length || deviceData.attributes.has(parent))) {
+        update["$set"][
+          parent.length ? parent.toString() + "._timestamp" : "_timestamp"
+        ] = new Date(diff[2]);
+      }
+    }
+  }
+
+  for (const diff of deviceData.attributes.diff()) {
+    const path = diff[0];
+    const value1 = (((diff[1] || {}).value || [])[1] || [])[0];
+    const value2 = (((diff[2] || {}).value || [])[1] || [])[0];
+    const valueType1 = (((diff[1] || {}).value || [])[1] || [])[1];
+    const valueType2 = (((diff[2] || {}).value || [])[1] || [])[1];
+    const valueTimestamp1 = ((diff[1] || {}).value || [])[0];
+    const valueTimestamp2 = ((diff[2] || {}).value || [])[0];
+    const object1 = ((diff[1] || {}).object || [])[1];
+    const object2 = ((diff[2] || {}).object || [])[1];
+    const writable2 = ((diff[2] || {}).writable || [])[1];
+    const writable1 = ((diff[1] || {}).writable || [])[1];
+    const attributesTimestamp1 = ((diff[1] || {}).notification || [])[0];
+    const attributesTimestamp2 = ((diff[2] || {}).notification || [])[0];
+    const notification1 = ((diff[1] || {}).notification || [])[1];
+    const notification2 = ((diff[2] || {}).notification || [])[1];
+    const accessList1 = ((diff[1] || {}).accessList || [])[1];
+    const accessList2 = ((diff[2] || {}).accessList || [])[1];
+
+    switch (path.segments[0]) {
+      case "Events":
+        if (path.length === 2 && value2 !== value1) {
+          if (!diff[2]) {
+            switch (path.segments[1]) {
+              case "Inform":
+                update["$unset"]["_lastInform"] = 1;
+                break;
+              case "1_BOOT":
+                update["$unset"]["_lastBoot"] = 1;
+                break;
+              case "0_BOOTSTRAP":
+                update["$unset"]["_lastBootstrap"] = 1;
+                break;
+              case "Registered":
+                update["$unset"]["_registered"] = 1;
+            }
+          } else {
+            const t = new Date(diff[2].value[1][0] as number);
+            switch (path.segments[1]) {
+              case "Inform":
+                update["$set"]["_lastInform"] = t;
+                break;
+              case "1_BOOT":
+                update["$set"]["_lastBoot"] = t;
+                break;
+              case "0_BOOTSTRAP":
+                update["$set"]["_lastBootstrap"] = t;
+                break;
+              case "Registered":
+                update["$set"]["_registered"] = t;
+            }
+          }
+        }
+
+        break;
+      case "DeviceID":
+        if (value2 !== value1) {
+          const v = diff[2].value[1][0];
+          switch (path.segments[1]) {
+            case "ID":
+              update["$set"]["_id"] = v;
+              break;
+            case "Manufacturer":
+              update["$set"]["_deviceId._Manufacturer"] = v;
+              break;
+            case "OUI":
+              update["$set"]["_deviceId._OUI"] = v;
+              break;
+            case "ProductClass":
+              update["$set"]["_deviceId._ProductClass"] = v;
+              break;
+            case "SerialNumber":
+              update["$set"]["_deviceId._SerialNumber"] = v;
+          }
+        }
+        break;
+      case "Tags":
+        if (value2 !== value1) {
+          if (value2 != null) {
+            if (!update["$addToSet"]["_tags"])
+              update["$addToSet"]["_tags"] = { $each: [] };
+            update["$addToSet"]["_tags"]["$each"].push(path.segments[1]);
+          } else {
+            if (!update["$pull"]["_tags"]) {
+              update["$pull"]["_tags"] = {
+                $in: []
+              };
+            }
+            update["$pull"]["_tags"]["$in"].push(path.segments[1]);
+          }
+        }
+
+        break;
+      default:
+        if (!diff[2]) {
+          update["$unset"][path.toString()] = 1;
+          continue;
+        }
+
+        for (const attrName of Object.keys(diff[2])) {
+          // Param timestamps may be greater than session timestamp to track revisions
+          if (diff[2][attrName][0] > sessionTimestamp)
+            diff[2][attrName][0] = sessionTimestamp;
+
+          if (diff[2][attrName][1] != null) {
+            switch (attrName) {
+              case "value":
+                if (value2 !== value1) {
+                  if (
+                    valueType2 === "xsd:dateTime" &&
+                    Number.isInteger(value2 as number)
+                  ) {
+                    update["$set"][path.toString() + "._value"] = new Date(
+                      value2 as number
+                    );
+                  } else {
+                    update["$set"][path.toString() + "._value"] = value2;
+                  }
+                }
+
+                if (valueType2 !== valueType1)
+                  update["$set"][path.toString() + "._type"] = valueType2;
+
+                if (valueTimestamp2 !== valueTimestamp1) {
+                  update["$set"][path.toString() + "._timestamp"] = new Date(
+                    valueTimestamp2
+                  );
+                }
+
+                break;
+              case "object":
+                if (!diff[1] || !diff[1].object || object2 !== object1) {
+                  update["$set"][
+                    path.length ? path.toString() + "._object" : "_object"
+                  ] = !!object2;
+                }
+
+                break;
+              case "writable":
+                if (!diff[1] || !diff[1].writable || writable2 !== writable1) {
+                  update["$set"][
+                    path.length ? path.toString() + "._writable" : "_writable"
+                  ] = !!writable2;
+                }
+
+                break;
+              case "notification":
+                if (
+                  !diff[1] ||
+                  !diff[1].notification ||
+                  notification2 !== notification1
+                ) {
+                  update["$set"][
+                    path.length
+                      ? path.toString() + "._notification"
+                      : "_notification"
+                  ] = notification2;
+                }
+
+                if (attributesTimestamp2 !== attributesTimestamp1) {
+                  update["$set"][
+                    path.toString() + "._attributesTimestamp"
+                  ] = new Date(attributesTimestamp2);
+                }
+
+                break;
+              case "accessList":
+                if (
+                  !diff[1] ||
+                  !diff[1].accessList ||
+                  !compareAccessLists(accessList2, accessList1)
+                ) {
+                  update["$set"][
+                    path.length
+                      ? path.toString() + "._accessList"
+                      : "_accessList"
+                  ] = accessList2;
+                }
+
+                if (attributesTimestamp2 !== attributesTimestamp1) {
+                  update["$set"][
+                    path.toString() + "._attributesTimestamp"
+                  ] = new Date(attributesTimestamp2);
+                }
+            }
+          }
+        }
+
+        if (diff[1]) {
+          for (const attrName of Object.keys(diff[1])) {
+            if (
+              diff[1][attrName][1] != null &&
+              (!diff[2] || !diff[2][attrName] || diff[2][attrName][1] == null)
+            ) {
+              const p = path.length ? path.toString() + "." : "";
+              update["$unset"][`${p}_${attrName}`] = 1;
+              if (attrName === "value") {
+                update["$unset"][p + "_type"] = 1;
+                update["$unset"][p + "_timestamp"] = 1;
+              } else if (attrName === "notification") {
+                if (accessList2 == null)
+                  update["$unset"][`${p}_attributesTimestamp`] = 1;
+              } else if (attrName === "accessList") {
+                if (notification2 == null)
+                  update["$unset"][`${p}_attributesTimestamp`] = 1;
+              }
+            }
+          }
+        }
+    }
+  }
+
+  update["$unset"] = optimizeProjection(update["$unset"]);
+
+  // Remove overlap possibly caused by parameters changing from objects
+  // to regular parameters or vice versa. Reason being that _timestamp
+  // represents two different things depending on whether the parameter
+  // is an object or not.
+  for (const k of Object.keys(update["$unset"]))
+    if (update["$set"][k] != null) delete update["$unset"][k];
+
+  // Remove empty keys
+  for (const [k, v] of Object.entries(update)) {
+    if (k === "$addToSet") {
+      for (const [kk, vv] of Object.entries(v))
+        if (!vv["$each"].length) delete v[kk];
+    } else if (k === "$pull") {
+      for (const [kk, vv] of Object.entries(v))
+        if (!vv["$in"].length) delete v[kk];
+    }
+    if (!Object.keys(v).length) delete update[k];
+  }
+
+  if (!Object.keys(update).length) return;
+
+  // Mongo doesn't allow $addToSet and $pull at the same time
+  let update2;
+  if (update["$addToSet"] && update["$pull"]) {
+    update2 = { $pull: update["$pull"] };
+    delete update["$pull"];
+  }
+
+  const result = await devicesCollection.updateOne({ _id: deviceId }, update, {
+    upsert: isNew
+  });
+
+  if (result.result.n !== 1)
+    throw new Error(`Device ${deviceId} not found in database`);
+
+  if (update2) {
+    await devicesCollection.updateOne({ _id: deviceId }, update2);
+    return;
+  }
+}
+
+export async function getFaults(
+  deviceId
+): Promise<{ [channel: string]: SessionFault }> {
+  const res = await faultsCollection
+    .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
+    .toArray();
+
+  const faults: { [channel: string]: SessionFault } = {};
+  for (const r of res) {
+    const channel = r._id.slice(deviceId.length + 1);
+    delete r._id;
+    delete r.channel;
+    delete r.device;
+    r.timestamp = +r.timestamp;
+    r.provisions = JSON.parse(r.provisions);
+    faults[channel] = r;
+  }
+
+  return faults;
+}
+
+export async function saveFault(deviceId, channel, fault): Promise<void> {
+  const id = `${deviceId}:${channel}`;
+  fault = Object.assign({}, fault);
+  fault._id = id;
+  fault.device = deviceId;
+  fault.channel = channel;
+  fault.timestamp = new Date(fault.timestamp);
+  fault.provisions = JSON.stringify(fault.provisions);
+  await faultsCollection.replaceOne({ _id: id }, fault, { upsert: true });
+}
+
+export async function deleteFault(deviceId, channel): Promise<void> {
+  await faultsCollection.deleteOne({ _id: `${deviceId}:${channel}` });
+}
+
+export async function getDueTasks(
+  deviceId,
+  timestamp
+): Promise<[Task[], number]> {
+  const cur = tasksCollection.find({ device: deviceId }).sort(["timestamp"]);
+  const tasks = [] as Task[];
+
+  for await (const task of cur) {
+    if (task.timestamp) task.timestamp = +task.timestamp;
+    if (task.expiry) task.expiry = +task.expiry;
+    if (task.timestamp >= timestamp) return [tasks, +task.timestamp];
+    task._id = String(task._id);
+
+    tasks.push(task);
+
+    // For API compatibility
+    if (task.name === "download" && task.file) {
+      let q;
+      if (ObjectID.isValid(task.file))
+        q = { _id: { $in: [task.file, new ObjectID(task.file)] } };
+      else q = { _id: task.file };
+
+      const res = await filesCollection.find(q).toArray();
+
+      if (res[0]) {
+        if (!task.fileType) task.fileType = res[0].metadata.fileType;
+
+        if (!task.fileName)
+          task.fileName = res[0].filename || res[0]._id.toString();
+      }
+    }
+  }
+  return [tasks, null];
+}
+
+export async function clearTasks(deviceId, taskIds): Promise<void> {
+  await tasksCollection.deleteMany({
+    _id: { $in: taskIds.map(id => new ObjectID(id)) }
+  });
+}
+
+export async function getOperations(
+  deviceId
+): Promise<{ [commandKey: string]: Operation }> {
+  const res = await operationsCollection
+    .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
+    .toArray();
+
+  const operations: { [commandKey: string]: Operation } = {};
+  for (const r of res) {
+    const commandKey = r._id.slice(deviceId.length + 1);
+    delete r._id;
+    r.timestamp = +r.timestamp;
+    if (r.args) r.args = JSON.parse(r.args);
+    r.provisions = JSON.parse(r.provisions);
+    r.retries = JSON.parse(r.retries);
+    operations[commandKey] = r;
+  }
+  return operations;
+}
+
+export async function saveOperation(
+  deviceId,
+  commandKey,
+  operation
+): Promise<void> {
+  const id = `${deviceId}:${commandKey}`;
+  operation = Object.assign({}, operation);
+  operation._id = id;
+  operation.timestamp = new Date(operation.timestamp);
+  operation.provisions = JSON.stringify(operation.provisions);
+  operation.retries = JSON.stringify(operation.retries);
+  operation.args = JSON.stringify(operation.args);
+  await operationsCollection.replaceOne({ _id: id }, operation, {
+    upsert: true
+  });
+}
+
+export async function deleteOperation(deviceId, commandKey): Promise<void> {
+  await operationsCollection.deleteOne({ _id: `${deviceId}:${commandKey}` });
+}
+
+export async function getPresets(): Promise<{}[]> {
+  return presetsCollection.find().toArray();
+}
+
+export async function getObjects(): Promise<{}[]> {
+  return objectsCollection.find().toArray();
+}
+
+export async function getProvisions(): Promise<{}[]> {
+  return provisionsCollection.find().toArray();
+}
+
+export async function getVirtualParameters(): Promise<{}[]> {
+  return virtualParametersCollection.find().toArray();
+}
+
+export function getFiles(): Promise<{}[]> {
+  return filesCollection.find().toArray();
+}
+
+export async function getConfig(): Promise<
+  { id: string; value: Expression }[]
+> {
+  const res = await configCollection.find().toArray();
+  return res.map(c => ({
+    id: c["_id"],
+    value: parse(c["value"])
+  }));
+}
+
+interface Permission {
+  role: string;
+  resource: string;
+  access: number;
+  filter: string;
+  validate: string;
+}
+
+export async function getPermissions(): Promise<Permission[]> {
+  return permissionsCollection.find().toArray();
+}
+
+interface User {
+  _id: string;
+  password: string;
+  salt: string;
+  roles: string;
+}
+
+export async function getUsers(): Promise<User[]> {
+  return usersCollection.find().toArray();
+}

lib/debug.ts

@@ -0,0 +1,204 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import {
+  IncomingMessage,
+  ServerResponse,
+  ClientRequest,
+  RequestOptions
+} from "http";
+import { Socket } from "net";
+import { appendFileSync } from "fs";
+import * as yaml from "yaml";
+import * as config from "./config";
+
+const DEBUG_FILE = "" + config.get("DEBUG_FILE");
+const DEBUG_FORMAT = "" + config.get("DEBUG_FORMAT");
+
+const connectionTimestamps = new WeakMap<Socket, Date>();
+
+function getConnectionTimestamp(connection: Socket): Date {
+  let t = connectionTimestamps.get(connection);
+  if (!t) {
+    t = new Date();
+    connectionTimestamps.set(connection, t);
+  }
+  return t;
+}
+
+export function incomingHttpRequest(
+  httpRequest: IncomingMessage,
+  deviceId: string,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpRequest.connection;
+  const msg = {
+    event: "incoming HTTP request",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(con),
+    localPort: con.localPort,
+    method: httpRequest.method,
+    url: httpRequest.url,
+    headers: httpRequest.headers,
+    body: body
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingHttpResponse(
+  httpResponse: ServerResponse,
+  deviceId: string,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpResponse.connection;
+  const msg = {
+    event: "outgoing HTTP response",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(con),
+    statusCode: httpResponse.statusCode,
+    headers: httpResponse.getHeaders(),
+    body: body
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingHttpRequest(
+  httpRequest: ClientRequest,
+  deviceId: string,
+  options: RequestOptions,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpRequest.connection;
+  const msg = {
+    event: "outgoing HTTP request",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(con),
+    remotePort: options.port,
+    method: options.method || "GET",
+    url: options.path,
+    headers: httpRequest.getHeaders(),
+    body: body
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingHttpRequestError(
+  httpRequest: ClientRequest,
+  deviceId: string,
+  options: RequestOptions,
+  err: Error
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing HTTP request",
+    timestamp: now,
+    remoteAddress: options.hostname,
+    deviceId: deviceId,
+    connection: null,
+    remotePort: options.port,
+    method: options.method,
+    url: options.path,
+    headers: httpRequest.getHeaders(),
+    error: err.message
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function incomingHttpResponse(
+  httpResponse: IncomingMessage,
+  deviceId: string,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpResponse.connection;
+  const msg = {
+    event: "incoming HTTP response",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(httpResponse.connection),
+    statusCode: httpResponse.statusCode,
+    headers: httpResponse.headers,
+    body: body
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingUdpMessage(
+  remoteAddress: string,
+  deviceId: string,
+  remotePort: number,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing UDP message",
+    timestamp: now,
+    remoteAddress: remoteAddress,
+    deviceId: deviceId,
+    remotePort: remotePort,
+    body: body
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}

lib/default-provisions.js

@@ -1,117 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const config = require("./config");
-const common = require("./common");
-const device = require("./device");
-const scheduling = require("./scheduling");
-
-const MAX_DEPTH = config.get("MAX_DEPTH");
-
-
-function refresh(sessionContext, provision, declarations, startRevision, endRevision) {
-  let path = common.parsePath(provision[1]).slice();
-  let l = path.length;
-  path.length = MAX_DEPTH;
-  path.fill("*", l);
-  let every = 1000 * (provision[2] || 1);
-  let offset = scheduling.variance(sessionContext.deviceId, every);
-  let t = scheduling.interval(sessionContext.timestamp, every, offset);
-
-  for (let i = l; i < path.length; ++ i)
-    declarations.push([path.slice(0, i), t, {object: 1, writable: 1, value: t}]);
-
-  return true;
-}
-
-
-function value(sessionContext, provision, declarations, startRevision, endRevision) {
-  let path = common.parsePath(provision[1]);
-  declarations.push([path, 1, {value: 1}, null, {value: [provision[2]]}]);
-
-  return true;
-}
-
-
-function tag(sessionContext, provision, declarations, startRevision, endRevision) {
-  let path = ["Tags", provision[1]];
-  declarations.push([path, 1, {value: 1}, null, {value: [provision[2]]}]);
-
-  return true;
-}
-
-
-function reboot(sessionContext, provision, declarations, startRevision, endRevision) {
-  declarations.push([["Reboot"], 1, {value: 1}, null, {value: [sessionContext.timestamp]}]);
-
-  return true;
-}
-
-
-function reset(sessionContext, provision, declarations, startRevision, endRevision) {
-  declarations.push([["FactoryReset"], 1, {value: 1}, null, {value: [sessionContext.timestamp]}]);
-
-  return true;
-}
-
-
-function download(sessionContext, provision, declarations, startRevision, endRevision) {
-  let alias = [["FileType"], provision[1] || "", ["FileName"], provision[2] || "",
-    ["TargetFileName"], provision[3] || ""];
-
-  declarations.push([["Downloads", alias], 1, {}, 1]);
-  declarations.push([["Downloads", alias, "Download"], 1, {value: 1}, null,
-    {value: [sessionContext.timestamp]}]);
-
-  return true;
-}
-
-
-function instances(sessionContext, provision, declarations, startRevision, endRevision) {
-  let count = Number(provision[2]);
-
-  if (Number.isNaN(count))
-    return true;
-
-  let path = common.parsePath(provision[1]);
-
-  if (provision[2][0] === "+" || provision[2][0] === "-") {
-    declarations.push([path, 1]);
-
-    if (endRevision === startRevision)
-      return false;
-
-    let unpacked = device.unpack(sessionContext.deviceData, path, startRevision + 1);
-    count = Math.max(0, unpacked.length + count);
-  }
-
-  declarations.push([path, 1, null, count]);
-
-  return true;
-}
-
-
-exports.refresh = refresh;
-exports.value = value;
-exports.tag = tag;
-exports.reboot = reboot;
-exports.reset = reset;
-exports.download = download;
-exports.instances = instances;

lib/default-provisions.ts

@@ -0,0 +1,253 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Path from "./common/path";
+import * as config from "./config";
+import * as device from "./device";
+import * as scheduling from "./scheduling";
+
+const MAX_DEPTH = +config.get("MAX_DEPTH");
+
+export function refresh(sessionContext, provision, declarations): boolean {
+  if (
+    (provision.length !== 2 || typeof provision[1] !== "string") &&
+    (provision.length !== 3 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "number") &&
+    (provision.length < 4 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "number" ||
+      typeof provision[3] !== "boolean")
+  )
+    throw new Error("Invalid arguments");
+
+  const every = 1000 * (provision[2] || 1);
+  const offset = scheduling.variance(sessionContext.deviceId, every);
+  const t = scheduling.interval(sessionContext.timestamp, every, offset);
+
+  let attrGet;
+  let refreshChildren;
+  if (provision[3] == null) {
+    refreshChildren = true;
+    attrGet = { object: 1, writable: 1, value: t };
+  } else {
+    attrGet = {};
+    refreshChildren = !!provision[3];
+    for (const a of provision.slice(4)) attrGet[a] = t;
+  }
+
+  let path = Path.parse(provision[1]);
+  let l = path.length;
+  if (refreshChildren) {
+    const segments = path.segments.slice();
+    l = segments.length;
+    segments.length = MAX_DEPTH;
+    segments.fill("*", l);
+    path = Path.parse(segments.join("."));
+  }
+
+  for (let i = l; i <= path.length; ++i) {
+    declarations.push({
+      path: path.slice(0, i),
+      pathGet: t,
+      pathSet: null,
+      attrGet: attrGet,
+      attrSet: null,
+      defer: true
+    });
+  }
+
+  return true;
+}
+
+export function value(sessionContext, provision, declarations): boolean {
+  if (
+    provision.length < 3 ||
+    provision.length > 4 ||
+    typeof provision[1] !== "string"
+  )
+    throw new Error("Invalid arguments");
+
+  let attr: string, val: any;
+
+  if (provision.length === 3) {
+    attr = "value";
+    val = provision[2];
+  } else {
+    attr = provision[2] || "";
+    val = provision[3];
+  }
+
+  if (attr === "accessList") {
+    val = (val || "")
+      .split(",")
+      .map(s => s.trim())
+      .filter(s => !!s);
+  } else if (attr === "value") {
+    val = [val];
+  }
+
+  declarations.push({
+    path: Path.parse(provision[1]),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { [attr]: 1 },
+    attrSet: { [attr]: val },
+    defer: true
+  });
+
+  return true;
+}
+
+export function tag(sessionContext, provision, declarations): boolean {
+  if (
+    provision.length !== 3 ||
+    typeof provision[1] !== "string" ||
+    typeof provision[2] !== "boolean"
+  )
+    throw new Error("Invalid arguments");
+
+  declarations.push({
+    path: Path.parse(`Tags.${provision[1]}`),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [provision[2]] },
+    defer: true
+  });
+
+  return true;
+}
+
+export function reboot(sessionContext, provision, declarations): boolean {
+  if (provision.length !== 1) throw new Error("Invalid arguments");
+
+  declarations.push({
+    path: Path.parse("Reboot"),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [sessionContext.timestamp] },
+    defer: true
+  });
+
+  return true;
+}
+
+export function reset(sessionContext, provision, declarations): boolean {
+  if (provision.length !== 1) throw new Error("Invalid arguments");
+
+  declarations.push({
+    path: Path.parse("FactoryReset"),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [sessionContext.timestamp] },
+    defer: true
+  });
+
+  return true;
+}
+
+export function download(sessionContext, provision, declarations): boolean {
+  if (
+    (provision.length !== 3 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "string") &&
+    (provision.length !== 4 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "string" ||
+      typeof provision[3] !== "string")
+  )
+    throw new Error("Invalid arguments");
+
+  const alias = [
+    `FileType:${JSON.stringify(provision[1] || "")}`,
+    `FileName:${JSON.stringify(provision[2] || "")}`,
+    `TargetFileName:${JSON.stringify(provision[3] || "")}`
+  ].join(",");
+
+  declarations.push({
+    path: Path.parse(`Downloads.[${alias}]`),
+    pathGet: 1,
+    pathSet: 1,
+    attrGet: null,
+    attrSet: null,
+    defer: true
+  });
+
+  declarations.push({
+    path: Path.parse(`Downloads.[${alias}].Download`),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [sessionContext.timestamp] },
+    defer: true
+  });
+
+  return true;
+}
+
+export function instances(
+  sessionContext,
+  provision,
+  declarations,
+  startRevision,
+  endRevision
+): boolean {
+  if (provision.length !== 3 || typeof provision[1] !== "string")
+    throw new Error("Invalid arguments");
+
+  let count = Number(provision[2]);
+
+  if (Number.isNaN(count)) throw new Error("Invalid arguments");
+
+  const path = Path.parse(provision[1]);
+
+  if (provision[2][0] === "+" || provision[2][0] === "-") {
+    declarations.push({
+      path: path,
+      pathGet: 1,
+      pathSet: null,
+      attrGet: null,
+      attrSet: null,
+      defer: true
+    });
+
+    if (endRevision === startRevision) return false;
+
+    const unpacked = device.unpack(
+      sessionContext.deviceData,
+      path,
+      startRevision + 1
+    );
+    count = Math.max(0, unpacked.length + count);
+  }
+
+  declarations.push({
+    path: path,
+    pathGet: 1,
+    pathSet: count,
+    attrGet: null,
+    attrSet: null,
+    defer: true
+  });
+
+  return true;
+}

lib/device.js

@@ -1,395 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const common = require('./common');
-
-
-const CHANGE_FLAGS = {
-  object: 2,
-  writable: 4,
-  value: 8
-};
-
-
-function parseBool(v) {
-  v = "" + v;
-  if (v === "true" || v === "TRUE" || v === "True" || v === "1")
-    return true;
-  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
-    return false;
-  else
-    return null;
-}
-
-
-function sanitizeParameterValue(parameterValue) {
-  if (parameterValue[0] != null) {
-    switch (parameterValue[1]) {
-      case 'xsd:boolean':
-        if (typeof parameterValue[0] !== 'boolean') {
-          let v = parseBool(parameterValue[0]);
-          if (v == null)
-            v = '' + parameterValue[0];
-          parameterValue[0] = v;
-        }
-        break;
-      case 'xsd:string':
-      case 'xsd:base64':
-      case 'xsd:hexBinary':
-        if (typeof parameterValue[0] !== 'string') {
-          parameterValue[0] = '' + parameterValue[0];
-        }
-        break;
-      case 'xsd:int':
-      case 'xsd:unsignedInt':
-        if (typeof parameterValue[0] !== 'number') {
-          let v = parseInt(parameterValue[0]);
-          if (isNaN(v))
-            v = '' + parameterValue[0];
-          parameterValue[0] = v;
-        }
-        break;
-      case 'xsd:dateTime':
-        if (typeof parameterValue[0] !== 'number') {
-          // Don't use parseInt because it reads date string as a number
-          let v = +parameterValue[0];
-          if (isNaN(v)) {
-            v = Date.parse(parameterValue[0]);
-            if (isNaN(v))
-              v = '' + parameterValue[0];
-          }
-          parameterValue[0] = v;
-        }
-        break;
-      default:
-        if (parameterValue[1] != null)
-          throw new Error(`Parameter value type "${parameterValue[1]}" not recognized.`);
-
-        parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
-        break;
-    }
-  }
-  return parameterValue;
-}
-
-
-function getAliasDeclarations(path, timestamp) {
-  var decs = [];
-
-  function recursive(pattern, prefix) {
-    var pattern = prefix.concat(pattern);
-
-    for (let i = prefix.length; i < pattern.length; ++ i) {
-      if (Array.isArray(pattern[i])) {
-        let pat = pattern[i];
-        pattern[i] = '*';
-        for (let j = 0; j < pat.length; j += 2)
-          recursive(pat[j], pattern.slice(0, i + 1));
-      }
-    }
-
-    if (prefix.length)
-      decs.push([pattern, timestamp, {value: timestamp}]);
-    else
-      decs.push([pattern, timestamp]);
-  }
-
-  recursive(path, []);
-
-  return decs;
-}
-
-
-function unpack(deviceData, path, revision) {
-  let allMatches = [];
-  if (!path.alias) {
-    for (let p of deviceData.paths.find(path, false, true))
-      if (deviceData.attributes.has(p, revision))
-        allMatches.push(p);
-  }
-  else {
-    let wildcardPath = path.slice();
-    wildcardPath.wildcard = path.wildcard;
-    for (let i = 0; i < wildcardPath.length; ++ i)
-      if (Array.isArray(wildcardPath[i])) {
-        wildcardPath[i] = '*';
-        wildcardPath.wildcard |= 1 << i;
-      }
-
-    for (let p of deviceData.paths.find(wildcardPath, false, true))
-      if (deviceData.attributes.has(p, revision))
-        allMatches.push(p);
-
-    for (let i = path.length - 1; i >= 0; -- i) {
-      if (Array.isArray(path[i])) {
-        for (let j = 0; j < path[i].length; j += 2) {
-          let p = wildcardPath.slice(0, i + 1).concat(path[i][j]);
-          p.alias = (wildcardPath.alias & ((1 << (i + 1)) - 1)) + path[i][j].alias << (i + 1);
-          p.wildcard = (wildcardPath.wildcard & ((1 << (i + 1)) - 1)) + path[i][j].wildcard << (i + 1);
-          let unpacked = unpack(deviceData, p, revision);
-          let filtered = [];
-          for (let up of unpacked) {
-            let attributes = deviceData.attributes.get(up, revision);
-            if (attributes && attributes.value && attributes.value[1] &&
-                sanitizeParameterValue([path[i][j + 1], attributes.value[1][1]])[0] === attributes.value[1][0]) {
-              for (let m = 0; m < allMatches.length; ++ m) {
-                let k;
-                for (k = i; k >= 0; -- k)
-                  if (allMatches[m][k] != up[k])
-                    break
-
-                if (k < 0) {
-                  filtered.push(allMatches[m]);
-                  allMatches[m] = [];
-                }
-              }
-            }
-          }
-          allMatches = filtered;
-        }
-      }
-    }
-  }
-
-  allMatches.sort(function(a, b) {
-    for (let i = 0; i < a.length; ++ i) {
-      if (a === b)
-        continue;
-
-      // Use numeric sorting for numbers
-      let ia = parseInt(a), ib = parseInt(b);
-      if (ia.toString() === a && ib.toString() === b) {
-        a = ia;
-        b = ib;
-      }
-
-      if (a < b)
-        return -1;
-      else
-        return 1;
-    }
-    return 0;
-  });
-
-  return allMatches;
-}
-
-
-function clear(deviceData, path, timestamp, attributes, changeFlags = 0) {
-  let changeTrackers = {};
-
-  timestamp = timestamp || 0;
-
-  let descendantsTimestamp = timestamp;
-  if (attributes && attributes.object) {
-    if (attributes.object > descendantsTimestamp)
-      descendantsTimestamp = attributes.object;
-    if (!(attributes.object <= attributes.value))
-      attributes.value = attributes.object;
-  }
-
-  for (let p of deviceData.paths.find(path, true, true, descendantsTimestamp ? 99 : path.length)) {
-    let tracker = deviceData.trackers.get(p);
-    for (let k in tracker)
-      changeTrackers[k] |= tracker[k];
-
-    let currentTimestamp = deviceData.timestamps.get(p);
-    if (currentTimestamp === undefined)
-      continue;
-
-    if (timestamp > currentTimestamp ||
-      (descendantsTimestamp > currentTimestamp && p.length > path.length)) {
-      deviceData.timestamps.delete(p);
-      deviceData.attributes.delete(p);
-      changeFlags |= 1;
-    }
-    else if (attributes && p.length == path.length) {
-      let currentAttributes = deviceData.attributes.get(p);
-      if (currentAttributes) {
-        let newAttrs;
-        for (let attrName in attributes) {
-          if (attrName in currentAttributes &&
-              attributes[attrName] > currentAttributes[attrName][0]) {
-            changeFlags |= CHANGE_FLAGS[attrName];
-            if (!newAttrs) {
-              newAttrs = Object.assign({}, currentAttributes);
-              deviceData.attributes.set(p, newAttrs);
-            }
-            delete newAttrs[attrName];
-          }
-        }
-      }
-    }
-  }
-
-  // Note: For performance, we're merging all changes together rather than
-  // mark changes based the exact parameters affected.
-  for (let k in changeTrackers) {
-    if (changeTrackers[k] & changeFlags)
-      deviceData.changes.add(k);
-  }
-}
-
-
-function compareEquality(a, b) {
-  var t = typeof a;
-  if (a === null ||
-      a === undefined ||
-      t === 'number' ||
-      t === 'boolean' ||
-      t === 'string' ||
-      t === 'symbol')
-    return a === b;
-
-  return JSON.stringify(a) === JSON.stringify(b);
-}
-
-
-function set(deviceData, path, timestamp, attributes, toClear) {
-  path = deviceData.paths.add(path);
-
-  let currentTimestamp = deviceData.timestamps.get(path);
-
-  let currentAttributes;
-
-  if (path.wildcard)
-    attributes = undefined;
-  else if (currentTimestamp)
-    currentAttributes = deviceData.attributes.get(path);
-
-  let changeFlags = 0;
-
-  if (attributes) {
-    if (attributes.value && attributes.value[1]
-        && attributes.value[0] >= (attributes.object ? attributes.object[0] : 0))
-      attributes.object = [attributes.value[0], 0];
-
-    if (attributes.object && attributes.object[1]
-      && attributes.object[0] >= (attributes.value ? attributes.value[0] : 0))
-      attributes.value = [attributes.object[0]];
-
-    let newAttributes = Object.assign({}, currentAttributes, attributes);
-
-    if (currentAttributes) {
-      for (let attrName in attributes) {
-        timestamp = Math.max(timestamp, attributes[attrName][0]);
-        if (!(attrName in currentAttributes))
-          changeFlags |= CHANGE_FLAGS[attrName];
-        else if (attributes[attrName][0] <= currentAttributes[attrName][0])
-          newAttributes[attrName] = currentAttributes[attrName];
-        else if (!compareEquality(attributes[attrName][1], currentAttributes[attrName][1]))
-          changeFlags |= CHANGE_FLAGS[attrName];
-      }
-    }
-    else
-      changeFlags |= 1;
-
-    deviceData.attributes.set(path, newAttributes);
-
-    if (!(timestamp <= currentTimestamp)) {
-      deviceData.timestamps.set(path, timestamp);
-      if (path.length > 1)
-        toClear = set(deviceData, path.slice(0, path.length - 1), timestamp,
-          {object: [timestamp, 1]}, toClear);
-    }
-  }
-  else if (!(timestamp <= currentTimestamp)) {
-    deviceData.timestamps.set(path, timestamp);
-
-    if (currentAttributes) {
-      deviceData.attributes.delete(path);
-      changeFlags |= 1;
-    }
-    else if (path.wildcard) {
-      for (let p of deviceData.paths.find(path, false, true, path.length))
-        if (timestamp > deviceData.timestamps.get(p)) {
-          toClear = toClear || [];
-          toClear.push([p, timestamp]);
-        }
-    }
-  }
-
-  if (changeFlags) {
-    if (changeFlags & 1) {
-      toClear = toClear || [];
-      toClear.push([path, timestamp, null, changeFlags]);
-    }
-    else if (changeFlags & CHANGE_FLAGS.object) {
-      toClear = toClear || [];
-      toClear.push([path, 0, {object: attributes.object[0]}, changeFlags]);
-    }
-    else {
-      for (let p of deviceData.paths.find(path, true, false, path.length)) {
-        let tracker = deviceData.trackers.get(p);
-        for (let k in tracker) {
-          if (tracker[k] & changeFlags)
-            deviceData.changes.add(k);
-        }
-      }
-    }
-  }
-
-  return toClear;
-}
-
-
-function track(deviceData, path, marker, attributes) {
-  path = deviceData.paths.add(path);
-  let f = 1;
-
-  if (attributes)
-    for (let attrName of attributes)
-      f |= CHANGE_FLAGS[attrName];
-
-  let cur = deviceData.trackers.get(path);
-  if (!cur) {
-    cur = {};
-    deviceData.trackers.set(path, cur);
-  }
-
-  cur[marker] |= f;
-}
-
-
-function clearTrackers(deviceData, tracker) {
-  if (Array.isArray(tracker)) {
-    deviceData.trackers.forEach(function(v, k) {
-      for (let t of tracker)
-        delete v[t];
-    });
-    for (let t of tracker)
-      deviceData.changes.delete(t);
-  }
-  else {
-    deviceData.trackers.forEach(function(v, k) {
-      delete v[tracker];
-    });
-    deviceData.changes.delete(tracker);
-  }
-}
-
-
-exports.sanitizeParameterValue = sanitizeParameterValue;
-exports.getAliasDeclarations = getAliasDeclarations;
-exports.unpack = unpack;
-exports.set = set;
-exports.clear = clear;
-exports.track = track;
-exports.clearTrackers = clearTrackers;

lib/device.ts

@@ -0,0 +1,407 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Path from "./common/path";
+import { DeviceData, Declaration, Clear } from "./types";
+
+const CHANGE_FLAGS = {
+  object: 2,
+  writable: 4,
+  value: 8,
+  notification: 16,
+  accessList: 32
+};
+
+function parseBool(v): boolean {
+  v = "" + v;
+  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
+  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
+    return false;
+  else return null;
+}
+
+export function sanitizeParameterValue(
+  parameterValue: [string | number | boolean, string]
+): [string | number | boolean, string?] {
+  if (parameterValue[0] != null) {
+    switch (parameterValue[1]) {
+      case "xsd:boolean":
+        if (typeof parameterValue[0] !== "boolean") {
+          const b = parseBool(parameterValue[0]);
+          if (b == null) parameterValue[0] = "" + parameterValue[0];
+          else parameterValue[0] = b;
+        }
+        break;
+      case "xsd:string":
+      case "xsd:base64":
+      case "xsd:hexBinary":
+        if (typeof parameterValue[0] !== "string")
+          parameterValue[0] = "" + parameterValue[0];
+
+        break;
+      case "xsd:int":
+      case "xsd:unsignedInt":
+        if (typeof parameterValue[0] !== "number") {
+          const i = parseInt(parameterValue[0] as string);
+          if (isNaN(i)) parameterValue[0] = "" + parameterValue[0];
+          else parameterValue[0] = i;
+        }
+        break;
+      case "xsd:dateTime":
+        if (typeof parameterValue[0] !== "number") {
+          // Don't use parseInt because it reads date string as a number
+          let i = +parameterValue[0];
+          if (isNaN(i)) {
+            i = Date.parse(parameterValue[0] as string);
+            if (isNaN(i)) parameterValue[0] = "" + parameterValue[0];
+            else parameterValue[0] = i;
+          } else {
+            parameterValue[0] = i;
+          }
+        }
+        break;
+      default:
+        parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
+        break;
+    }
+  }
+
+  return parameterValue;
+}
+
+export function getAliasDeclarations(
+  path: Path,
+  timestamp: number,
+  attrGet = null
+): Declaration[] {
+  const stripped = path.stripAlias();
+  let decs: Declaration[] = [
+    {
+      path: stripped,
+      pathGet: timestamp,
+      pathSet: null,
+      attrGet: attrGet,
+      attrSet: null,
+      defer: true
+    }
+  ];
+
+  if (path.alias) {
+    for (const [i, alias] of path.segments.entries()) {
+      if (Array.isArray(alias)) {
+        const parent = stripped.slice(0, i + 1);
+        for (const [p] of alias as [Path, string][]) {
+          decs = decs.concat(
+            getAliasDeclarations(parent.concat(p), timestamp, {
+              value: timestamp
+            })
+          );
+        }
+      }
+    }
+  }
+
+  return decs;
+}
+
+export function unpack(
+  deviceData: DeviceData,
+  path: Path,
+  revision?: number
+): Path[] {
+  let allMatches = [] as Path[];
+  if (!path.alias) {
+    for (const p of deviceData.paths.find(path, false, true))
+      if (deviceData.attributes.has(p, revision)) allMatches.push(p);
+  } else {
+    const wildcardPath = path.stripAlias();
+
+    for (const p of deviceData.paths.find(wildcardPath, false, true))
+      if (deviceData.attributes.has(p, revision)) allMatches.push(p);
+
+    for (let i = path.length - 1; i >= 0; --i) {
+      if (path.alias & (1 << i)) {
+        for (const [param, val] of path.segments[i] as [Path, string][]) {
+          const p = wildcardPath.slice(0, i + 1).concat(param);
+          const unpacked = unpack(deviceData, p, revision);
+          const filtered: Path[] = [];
+          for (const up of unpacked) {
+            const attributes = deviceData.attributes.get(up, revision);
+            if (
+              attributes &&
+              attributes.value &&
+              attributes.value[1] &&
+              sanitizeParameterValue([val, attributes.value[1][1]])[0] ===
+                attributes.value[1][0]
+            ) {
+              for (let m = 0; m < allMatches.length; ++m) {
+                let k;
+                const match = allMatches[m];
+                if (!match) continue;
+                for (k = i; k >= 0; --k)
+                  if (match.segments[k] !== up.segments[k]) break;
+
+                if (k < 0) {
+                  filtered.push(match);
+                  allMatches[m] = null;
+                }
+              }
+            }
+          }
+          allMatches = filtered;
+        }
+      }
+    }
+  }
+
+  allMatches.sort((p1, p2) => {
+    for (let i = 0; i < p1.length; ++i) {
+      const a = p1.segments[i] as string;
+      const b = p2.segments[i] as string;
+      if (a !== b) {
+        // Use numeric sorting for numbers
+        const ia = parseInt(a);
+        const ib = parseInt(b);
+
+        if (ia === +a && ib === +b) return ia - ib;
+        else if (a < b) return -1;
+        else return 1;
+      }
+    }
+    return 0;
+  });
+
+  return allMatches;
+}
+
+export function clear(
+  deviceData,
+  path,
+  timestamp,
+  attributes,
+  changeFlags = 0
+): void {
+  const changeTrackers = {};
+
+  timestamp = timestamp || 0;
+
+  let descendantsTimestamp = timestamp;
+  if (attributes && attributes.object) {
+    if (attributes.object > descendantsTimestamp)
+      descendantsTimestamp = attributes.object;
+    if (!(attributes.object <= attributes.value))
+      attributes.value = attributes.object;
+  }
+
+  for (const p of deviceData.paths.find(
+    path,
+    true,
+    true,
+    descendantsTimestamp ? 99 : path.length
+  )) {
+    const tracker = deviceData.trackers.get(p);
+    for (const k in tracker) changeTrackers[k] |= tracker[k];
+
+    const currentTimestamp = deviceData.timestamps.get(p);
+    if (currentTimestamp === undefined) continue;
+
+    if (
+      timestamp > currentTimestamp ||
+      (descendantsTimestamp > currentTimestamp && p.length > path.length)
+    ) {
+      deviceData.timestamps.delete(p);
+      deviceData.attributes.delete(p);
+      changeFlags |= 1;
+    } else if (attributes && p.length === path.length) {
+      const currentAttributes = deviceData.attributes.get(p);
+      if (currentAttributes) {
+        let newAttrs;
+        for (const attrName in attributes) {
+          if (
+            attrName in currentAttributes &&
+            attributes[attrName] > currentAttributes[attrName][0]
+          ) {
+            changeFlags |= CHANGE_FLAGS[attrName];
+            if (!newAttrs) {
+              newAttrs = Object.assign({}, currentAttributes);
+              deviceData.attributes.set(p, newAttrs);
+            }
+            delete newAttrs[attrName];
+          }
+        }
+      }
+    }
+  }
+
+  // Note: For performance, we're merging all changes together rather than
+  // mark changes based the exact parameters affected.
+  for (const k in changeTrackers)
+    if (changeTrackers[k] & changeFlags) deviceData.changes.add(k);
+}
+
+function compareEquality(a, b): boolean {
+  const t = typeof a;
+  if (
+    a === null ||
+    a === undefined ||
+    t === "number" ||
+    t === "boolean" ||
+    t === "string" ||
+    t === "symbol"
+  )
+    return a === b;
+
+  return JSON.stringify(a) === JSON.stringify(b);
+}
+
+export function set(
+  deviceData: DeviceData,
+  path: Path,
+  timestamp,
+  attributes,
+  toClear?: Clear[]
+): Clear[] {
+  path = deviceData.paths.add(path);
+
+  const currentTimestamp = deviceData.timestamps.get(path);
+
+  let currentAttributes;
+
+  if (path.wildcard) attributes = undefined;
+  else if (currentTimestamp)
+    currentAttributes = deviceData.attributes.get(path);
+
+  let changeFlags = 0;
+
+  if (attributes) {
+    if (
+      attributes.value &&
+      attributes.value[1] &&
+      attributes.value[0] >= (attributes.object ? attributes.object[0] : 0)
+    )
+      attributes.object = [attributes.value[0], 0];
+
+    if (
+      attributes.object &&
+      attributes.object[1] &&
+      attributes.object[0] >= (attributes.value ? attributes.value[0] : 0)
+    )
+      attributes.value = [attributes.object[0]];
+
+    const newAttributes = Object.assign({}, currentAttributes, attributes);
+
+    if (currentAttributes) {
+      for (const attrName in attributes) {
+        timestamp = Math.max(timestamp, attributes[attrName][0]);
+        if (!(attrName in currentAttributes))
+          changeFlags |= CHANGE_FLAGS[attrName];
+        else if (attributes[attrName][0] <= currentAttributes[attrName][0])
+          newAttributes[attrName] = currentAttributes[attrName];
+        else if (
+          !compareEquality(
+            attributes[attrName][1],
+            currentAttributes[attrName][1]
+          )
+        )
+          changeFlags |= CHANGE_FLAGS[attrName];
+      }
+    } else {
+      changeFlags |= 1;
+    }
+
+    deviceData.attributes.set(path, newAttributes);
+
+    if (!(timestamp <= currentTimestamp)) {
+      deviceData.timestamps.set(path, timestamp);
+      if (path.length > 1) {
+        toClear = set(
+          deviceData,
+          path.slice(0, path.length - 1),
+          timestamp,
+          { object: [timestamp, 1] },
+          toClear
+        );
+      }
+    }
+  } else if (!(timestamp <= currentTimestamp)) {
+    deviceData.timestamps.set(path, timestamp);
+
+    if (currentAttributes) {
+      deviceData.attributes.delete(path);
+      changeFlags |= 1;
+    } else if (path.wildcard) {
+      for (const p of deviceData.paths.find(path, false, true, path.length)) {
+        if (timestamp > deviceData.timestamps.get(p)) {
+          toClear = toClear || [];
+          toClear.push([p, timestamp]);
+        }
+      }
+    }
+  }
+
+  if (changeFlags) {
+    if (changeFlags & 1) {
+      toClear = toClear || [];
+      toClear.push([path, timestamp, null, changeFlags]);
+    } else if (changeFlags & CHANGE_FLAGS.object) {
+      toClear = toClear || [];
+      toClear.push([path, 0, { object: attributes.object[0] }, changeFlags]);
+    } else {
+      for (const p of deviceData.paths.find(path, true, false, path.length)) {
+        const tracker = deviceData.trackers.get(p);
+        for (const k in tracker)
+          if (tracker[k] & changeFlags) deviceData.changes.add(k);
+      }
+    }
+  }
+
+  return toClear;
+}
+
+export function track(
+  deviceData: DeviceData,
+  path: Path,
+  marker: string,
+  attributes?
+): void {
+  path = deviceData.paths.add(path);
+  let f = 1;
+
+  if (attributes)
+    for (const attrName of attributes) f |= CHANGE_FLAGS[attrName];
+
+  let cur = deviceData.trackers.get(path);
+  if (!cur) {
+    cur = {};
+    deviceData.trackers.set(path, cur);
+  }
+
+  cur[marker] |= f;
+}
+
+export function clearTrackers(deviceData: DeviceData, tracker): void {
+  if (Array.isArray(tracker)) {
+    for (const v of deviceData.trackers.values())
+      for (const t of tracker) delete v[t];
+    for (const t of tracker) deviceData.changes.delete(t);
+  } else {
+    for (const v of deviceData.trackers.values()) delete v[tracker];
+    deviceData.changes.delete(tracker);
+  }
+}

lib/extensions.js

@@ -1,117 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const childProcess = require("child_process");
-const crypto = require ("crypto");
-
-const config = require("./config");
-
-const TIMEOUT = config.get('EXT_TIMEOUT');
-
-const processes = {};
-const jobs = {};
-
-function messageHandler(message) {
-  let func = jobs[message[0]];
-  if (func) {
-    delete jobs[message[0]];
-    func(null, message[1], message[2]);
-  }
-}
-
-
-function run(args, callback) {
-  let scriptName = args[0];
-
-  if (!processes[scriptName]) {
-    let p = childProcess.fork('lib/extension-wrapper', [scriptName]);
-    processes[scriptName] = p;
-
-    p.on('error', function() {
-      if (processes[scriptName] === p) {
-        kill(processes[scriptName]);
-        delete processes[scriptName];
-      }
-    });
-
-    p.on('disconnect', function() {
-      if (processes[scriptName] === p) {
-        kill(processes[scriptName]);
-        delete processes[scriptName];
-      }
-    });
-
-    p.on('message', messageHandler);
-  }
-
-  let id = crypto.randomBytes(8).toString('hex');
-  jobs[id] = callback;
-  setTimeout(function() {
-    if (id in jobs) {
-      delete jobs[id];
-      return callback(null, {code: "timeout", message: "Extension timed out"});
-    }
-  }, TIMEOUT);
-
-  return processes[scriptName].send([id, args.slice(1)]);
-};
-
-
-function kill(process, callback) {
-  if (process.signalCode !== null || process.exitCode !== null)
-    return;
-
-  let timeToKill = Date.now() + 5000;
-
-  process.kill();
-
-  let t = setInterval(function() {
-    if (process.signalCode !== null || process.exitCode !== null) {
-      clearInterval(t);
-      if (callback)
-        callback();
-    }
-    else if (Date.now() > timeToKill) {
-      process.kill('SIGKILL');
-      clearInterval(t);
-      if (callback)
-        callback();
-    }
-  }, 100);
-}
-
-
-function killAll(callback) {
-  let counter = 1;
-  for (let n in processes) {
-    ++counter;
-    kill(processes[n], function() {
-      if (--counter == 0 && callback)
-        callback();
-    });
-    delete processes[n];
-  }
-  if (--counter == 0 && callback)
-    callback();
-}
-
-
-exports.run = run;
-exports.killAll = killAll;

lib/extensions.ts

@@ -0,0 +1,114 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { spawn, ChildProcess } from "child_process";
+import * as crypto from "crypto";
+import * as config from "./config";
+import { Fault } from "./types";
+import { ROOT_DIR } from "./config";
+
+const TIMEOUT = +config.get("EXT_TIMEOUT");
+
+const processes: { [script: string]: ChildProcess } = {};
+const jobs = new Map();
+
+function messageHandler(message): void {
+  const func = jobs.get(message[0]);
+  if (func) {
+    jobs.delete(message[0]);
+    func({ fault: message[1], value: message[2] });
+  }
+}
+
+export function run(args): Promise<{ fault: Fault; value: any }> {
+  return new Promise(resolve => {
+    const scriptName = args[0];
+
+    const id = crypto.randomBytes(8).toString("hex");
+    jobs.set(id, resolve);
+
+    if (!processes[scriptName]) {
+      const p = spawn(ROOT_DIR + "/bin/genieacs-ext", [scriptName], {
+        stdio: ["inherit", "inherit", "inherit", "ipc"]
+      });
+      processes[scriptName] = p;
+
+      p.on("error", err => {
+        if (processes[scriptName] === p) {
+          if (jobs.delete(id)) {
+            resolve({
+              fault: { code: err.name, message: err.message },
+              value: null
+            });
+          }
+
+          kill(processes[scriptName]);
+          delete processes[scriptName];
+        }
+      });
+
+      p.on("disconnect", () => {
+        if (processes[scriptName] === p) delete processes[scriptName];
+      });
+
+      p.on("message", messageHandler);
+    }
+
+    setTimeout(() => {
+      if (jobs.delete(id)) {
+        resolve({
+          fault: { code: "timeout", message: "Extension timed out" },
+          value: null
+        });
+      }
+    }, TIMEOUT);
+
+    if (!processes[scriptName].connected) return false;
+
+    return processes[scriptName].send([id, args.slice(1)]);
+  });
+}
+
+function kill(process: ChildProcess): Promise<void> {
+  return new Promise(resolve => {
+    const timeToKill = Date.now() + 5000;
+
+    process.kill();
+
+    const t = setInterval(() => {
+      if (!process.connected) {
+        clearInterval(t);
+        resolve();
+      } else if (Date.now() > timeToKill) {
+        process.kill("SIGKILL");
+        clearInterval(t);
+        resolve();
+      }
+    }, 100);
+  });
+}
+
+export async function killAll(): Promise<void> {
+  await Promise.all(
+    Object.entries(processes).map(([k, p]) => {
+      delete processes[k];
+      return kill(p);
+    })
+  );
+}

lib/fs.coffee

@@ -1,49 +0,0 @@
-###
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-config = require './config'
-url = require 'url'
-mongodb = require 'mongodb'
-querystring = require 'querystring'
-db = require './db'
-
-
-listener = (request, response) ->
-  urlParts = url.parse(request.url, true)
-  if request.method == 'GET'
-    filename = querystring.unescape(urlParts.pathname.substring(1))
-    gs = new mongodb.GridStore(db.mongoDb, filename, 'r', {})
-    gs.open((err, gs) ->
-      if err
-        response.writeHead(404)
-        response.end()
-        return
-      stream = gs.stream(true)
-      response.writeHead(200, {'Content-Type' : 'application/octet-stream', 'Content-Length' : gs.length})
-      stream.pipe(response)
-    )
-  else
-    response.writeHead(405, {'Allow': 'GET'})
-    response.end('405 Method Not Allowed')
-
-
-exports.listener = listener

lib/fs.ts

@@ -0,0 +1,63 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as url from "url";
+import * as querystring from "querystring";
+import { GridFSBucket } from "mongodb";
+import * as db from "./db";
+import * as logger from "./logger";
+
+export function listener(request, response): void {
+  const urlParts = url.parse(request.url, true);
+  if (request.method === "GET") {
+    const filename = querystring.unescape(urlParts.pathname.substring(1));
+
+    const log = {
+      message: "Fetch file",
+      filename: filename,
+      remoteAddress: request.connection.remoteAddress
+    };
+
+    db.filesCollection.findOne({ _id: filename }, (err, file) => {
+      if (err) throw err;
+
+      if (!file) {
+        response.writeHead(404);
+        response.end();
+        log.message += " not found";
+        logger.accessError(log);
+        return;
+      }
+
+      response.writeHead(200, {
+        "Content-Type": file.contentType || "application/octet-stream",
+        "Content-Length": file.length
+      });
+
+      const bucket = new GridFSBucket(db.client.db());
+      const downloadStream = bucket.openDownloadStreamByName(filename);
+      downloadStream.pipe(response);
+
+      logger.accessInfo(log);
+    });
+  } else {
+    response.writeHead(405, { Allow: "GET" });
+    response.end("405 Method Not Allowed");
+  }
+}

lib/gpn-heuristic.ts

@@ -1,5 +1,7 @@
+import Path from "./common/path";
+
 /**
- * Copyright 2013-2017  Zaid Abdulla
+ * Copyright 2013-2019  GenieACS Inc.
  *
  * This file is part of GenieACS.
  *
@@ -16,7 +18,6 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
-"use strict";
 
 const WILDCARD_MULTIPLIER = 2;
 const UNDISCOVERED_DEPTH = 7;
@@ -28,48 +29,50 @@ const UNDISCOVERED_DEPTH = 7;
 // flags is an int where its bits mark the segments in the pattern that
 // need refreshing. Leading 0s indicate that the pattern up to that
 // point has been discovered.
-function estimateGpnCount(gpnPatterns, depth = 0) {
-  const children = {};
-  const wildcardChildren = [];
+export function estimateGpnCount(
+  gpnPatterns: [Path, number][],
+  depth: number = 0
+): number {
+  const children: { [segment: string]: [Path, number][] } = {};
+  const wildcardChildren: [Path, number][] = [];
   let wildcardDiscovered = false;
   let gpnCount = 0;
 
-  for (let pattern of gpnPatterns) {
+  for (const pattern of gpnPatterns) {
     const path = pattern[0];
     const flags = pattern[1] >> depth;
 
-    let k = path[depth];
+    const k = path.segments[depth] as string;
 
     if (!k) {
-      if (flags & 1)
-        gpnCount = 1;
+      if (flags & 1) gpnCount = 1;
       continue;
     }
 
     if (flags & 1) {
       gpnCount = 1;
 
-      if (depth > UNDISCOVERED_DEPTH)
-        continue;
-    }
-    else if (k === "*")
+      if (depth > UNDISCOVERED_DEPTH) continue;
+    } else if (k === "*") {
       wildcardDiscovered = true;
+    }
 
     if (k === "*") {
       wildcardChildren.push(pattern);
-    }
-    else {
+    } else {
       children[k] = children[k] || [];
       children[k].push(pattern);
     }
   }
 
   let wildcardGpnCount = 0;
-  if (!wildcardDiscovered && wildcardChildren.length)
-      wildcardGpnCount += estimateGpnCount(wildcardChildren, depth + 1) * WILDCARD_MULTIPLIER;
+  if (!wildcardDiscovered && wildcardChildren.length) {
+    wildcardGpnCount +=
+      estimateGpnCount(wildcardChildren, depth + 1) * WILDCARD_MULTIPLIER;
+  }
 
-  for (let k of Object.keys(children)) {
-    let c = estimateGpnCount(children[k].concat(wildcardChildren), depth + 1);
+  for (const k of Object.keys(children)) {
+    const c = estimateGpnCount(children[k].concat(wildcardChildren), depth + 1);
     wildcardGpnCount -= c;
     gpnCount += c;
   }
@@ -78,5 +81,3 @@ function estimateGpnCount(gpnPatterns, depth = 0) {
 
   return gpnCount;
 }
-
-exports.estimateGpnCount = estimateGpnCount;

lib/init.ts

@@ -0,0 +1,425 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as localCache from "./local-cache";
+import { generateSalt, hashPassword } from "./auth";
+import * as db from "./ui/db";
+import { del } from "./cache";
+
+interface Status {
+  users: boolean;
+  presets: boolean;
+  filters: boolean;
+  device: boolean;
+  index: boolean;
+  overview: boolean;
+}
+
+const BOOTSTRAP_SCRIPT = `
+const now = Date.now();
+
+// Clear cached data model to force a refresh
+clear("Device", now);
+clear("InternetGatewayDevice", now);
+`.trim();
+
+const DEFAULT_SCRIPT = `
+const hourly = Date.now(3600000);
+
+// Refresh basic parameters hourly
+declare("InternetGatewayDevice.DeviceInfo.HardwareVersion", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.DeviceInfo.SoftwareVersion", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID", {path: hourly, value: hourly});
+// Don't refresh password field periodically because CPEs always report blank passowrds for security reasons
+declare("InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase", {path: hourly, value: 1});
+declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress", {path: hourly, value: hourly});
+`.trim();
+
+const INFORM_SCRIPT = `
+// Device ID as user name
+const username = declare("DeviceID.ID", {value: 1}).value[0]
+
+// Password will be fixed a given device because Math.random() is seeded with devcie ID by default.
+const password = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER).toString(36);
+
+const informInterval = 300;
+
+// Refresh values daily
+const daily = Date.now(86400000);
+
+// Unique inform offset per device for better load distribution
+const informTime = daily % 86400000;
+
+declare("InternetGatewayDevice.ManagementServer.ConnectionRequestUsername", {value: daily}, {value: username});
+declare("InternetGatewayDevice.ManagementServer.ConnectionRequestPassword", {value: daily}, {value: password});
+declare("InternetGatewayDevice.ManagementServer.PeriodicInformEnable", {value: daily}, {value: true});
+declare("InternetGatewayDevice.ManagementServer.PeriodicInformInterval", {value: daily}, {value: informInterval});
+declare("InternetGatewayDevice.ManagementServer.PeriodicInformTime", {value: daily}, {value: informTime});
+
+declare("Device.ManagementServer.ConnectionRequestUsername", {value: daily}, {value: username});
+declare("Device.ManagementServer.ConnectionRequestPassword", {value: daily}, {value: password});
+declare("Device.ManagementServer.PeriodicInformEnable", {value: daily}, {value: true});
+declare("Device.ManagementServer.PeriodicInformInterval", {value: daily}, {value: informInterval});
+declare("Device.ManagementServer.PeriodicInformTime", {value: daily}, {value: informTime});
+`.trim();
+
+export async function getStatus(): Promise<Status> {
+  const configSnapshot = await localCache.getCurrentSnapshot();
+  const users = localCache.getUsers(configSnapshot);
+  const presets = localCache.getPresets(configSnapshot);
+  const ui = localCache.getUiConfig(configSnapshot);
+
+  return {
+    users: !Object.keys(users).length,
+    presets: !presets.length,
+    filters: !Object.keys(ui["filters"]).length,
+    device: !Object.keys(ui["device"]).length,
+    index: !Object.keys(ui["index"]).length,
+    overview: !Object.keys(ui["overview"]).length
+  };
+}
+
+export async function seed(options): Promise<void> {
+  const resources = {};
+  const proms = [];
+
+  if (options.users) {
+    resources["permissions"] = [
+      { role: "admin", resource: "devices", access: 3, validate: "true" },
+      { role: "admin", resource: "faults", access: 3, validate: "true" },
+      { role: "admin", resource: "files", access: 3, validate: "true" },
+      { role: "admin", resource: "presets", access: 3, validate: "true" },
+      { role: "admin", resource: "provisions", access: 3, validate: "true" },
+      { role: "admin", resource: "config", access: 3, validate: "true" },
+      { role: "admin", resource: "permissions", access: 3, validate: "true" },
+      { role: "admin", resource: "users", access: 3, validate: "true" },
+      {
+        role: "admin",
+        resource: "virtualParameters",
+        access: 3,
+        validate: "true"
+      }
+    ];
+
+    resources["users"] = [
+      { username: "admin", password: "admin", roles: ["admin"] }
+    ];
+  }
+
+  if (options.filters) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.filters.0.label", value: "'Serial number'" },
+      { _id: "ui.filters.0.parameter", value: "DeviceID.SerialNumber" },
+      { _id: "ui.filters.0.type", value: "'string'" },
+      { _id: "ui.filters.1.label", value: "'Product class'" },
+      { _id: "ui.filters.1.parameter", value: "DeviceID.ProductClass" },
+      { _id: "ui.filters.1.type", value: "'string'" },
+      { _id: "ui.filters.2.label", value: "'Tag'" },
+      { _id: "ui.filters.2.type", value: "'tag'" }
+    ]);
+  }
+
+  if (options.device) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.device.0.type", value: "'tags'" },
+      { _id: "ui.device.1.type", value: "'ping'" },
+      { _id: "ui.device.2.type", value: "'parameter-list'" },
+      { _id: "ui.device.2.parameters.0.type", value: "'container'" },
+      { _id: "ui.device.2.parameters.0.element", value: "'span.inform'" },
+      { _id: "ui.device.2.parameters.0.label", value: "'Last inform'" },
+      {
+        _id: "ui.device.2.parameters.0.components.0.type",
+        value: "'parameter'"
+      },
+      { _id: "ui.device.2.parameters.0.components.1.chart", value: "'online'" },
+      {
+        _id: "ui.device.2.parameters.0.components.1.type",
+        value: "'overview-dot'"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.type",
+        value: "'summon-button'"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.0",
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.1",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.2",
+        value:
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.3",
+        value:
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.4",
+        value: "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.5",
+        value:
+          "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.6",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.7",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.8",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress"
+      },
+      {
+        _id: "ui.device.2.parameters.0.parameter",
+        value: "DATE_STRING(Events.Inform)"
+      },
+      { _id: "ui.device.2.parameters.1.label", value: "'Serial number'" },
+      {
+        _id: "ui.device.2.parameters.1.parameter",
+        value: "DeviceID.SerialNumber"
+      },
+      { _id: "ui.device.2.parameters.2.label", value: "'Product class'" },
+      {
+        _id: "ui.device.2.parameters.2.parameter",
+        value: "DeviceID.ProductClass"
+      },
+      { _id: "ui.device.2.parameters.3.label", value: "'OUI'" },
+      {
+        _id: "ui.device.2.parameters.3.parameter",
+        value: "DeviceID.OUI"
+      },
+      { _id: "ui.device.2.parameters.4.label", value: "'Manufacturer'" },
+      {
+        _id: "ui.device.2.parameters.4.parameter",
+        value: "DeviceID.Manufacturer"
+      },
+      { _id: "ui.device.2.parameters.5.label", value: "'Hardware version'" },
+      {
+        _id: "ui.device.2.parameters.5.parameter",
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion"
+      },
+      { _id: "ui.device.2.parameters.6.label", value: "'Software version'" },
+      {
+        _id: "ui.device.2.parameters.6.parameter",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+      },
+      { _id: "ui.device.2.parameters.7.label", value: "'MAC'" },
+      {
+        _id: "ui.device.2.parameters.7.parameter",
+        value:
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress"
+      },
+      { _id: "ui.device.2.parameters.8.label", value: "'IP'" },
+      {
+        _id: "ui.device.2.parameters.8.parameter",
+        value:
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress"
+      },
+      { _id: "ui.device.2.parameters.9.label", value: "'WLAN SSID'" },
+      {
+        _id: "ui.device.2.parameters.9.parameter",
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID"
+      },
+      { _id: "ui.device.2.parameters.10.label", value: "'WLAN passphrase'" },
+      {
+        _id: "ui.device.2.parameters.10.parameter",
+        value:
+          "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase"
+      },
+      { _id: "ui.device.3.type", value: "'parameter-table'" },
+      { _id: "ui.device.3.label", value: "'WLAN hosts'" },
+      {
+        _id: "ui.device.3.parameter",
+        value: "InternetGatewayDevice.LANDevice.1.Hosts.Host"
+      },
+      { _id: "ui.device.3.childParameters.0.label", value: "'Host name'" },
+      { _id: "ui.device.3.childParameters.0.parameter", value: "HostName" },
+      { _id: "ui.device.3.childParameters.1.label", value: "'IP address'" },
+      { _id: "ui.device.3.childParameters.1.parameter", value: "IPAddress" },
+      { _id: "ui.device.3.childParameters.2.label", value: "'MAC address'" },
+      { _id: "ui.device.3.childParameters.2.parameter", value: "MACAddress" },
+      { _id: "ui.device.4.type", value: "'container'" },
+      { _id: "ui.device.4.element", value: "'div'" },
+      { _id: "ui.device.4.components.0.components.0", value: "'Faults'" },
+      { _id: "ui.device.4.components.0.element", value: "'h3'" },
+      { _id: "ui.device.4.components.0.type", value: "'container'" },
+      { _id: "ui.device.4.components.1.type", value: "'device-faults'" },
+      { _id: "ui.device.5.type", value: "'container'" },
+      { _id: "ui.device.5.element", value: "'div.container-full-width'" },
+      {
+        _id: "ui.device.5.components.0.components.0",
+        value: "'All parameters'"
+      },
+      { _id: "ui.device.5.components.0.element", value: "'h3'" },
+      { _id: "ui.device.5.components.0.type", value: "'container'" },
+      { _id: "ui.device.5.components.1.type", value: "'all-parameters'" },
+      { _id: "ui.device.6.type", value: "'device-actions'" }
+    ]);
+  }
+
+  if (options.index) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.index.0.type", value: "'device-link'" },
+      { _id: "ui.index.0.label", value: "'Serial number'" },
+      { _id: "ui.index.0.parameter", value: "DeviceID.SerialNumber" },
+      { _id: "ui.index.0.components.0.type", value: "'parameter'" },
+      { _id: "ui.index.1.label", value: "'Product class'" },
+      { _id: "ui.index.1.parameter", value: "DeviceID.ProductClass" },
+      { _id: "ui.index.2.label", value: "'Software version'" },
+      {
+        _id: "ui.index.2.parameter",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+      },
+      { _id: "ui.index.3.label", value: "'IP'" },
+      {
+        _id: "ui.index.3.parameter",
+        value:
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress"
+      },
+      { _id: "ui.index.4.label", value: "'SSID'" },
+      {
+        _id: "ui.index.4.parameter",
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID"
+      },
+      { _id: "ui.index.5.type", value: "'container'" },
+      { _id: "ui.index.5.label", value: "'Last inform'" },
+      { _id: "ui.index.5.element", value: "'span.inform'" },
+      { _id: "ui.index.5.parameter", value: "DATE_STRING(Events.Inform)" },
+      { _id: "ui.index.5.components.0.type", value: "'parameter'" },
+      { _id: "ui.index.5.components.1.chart", value: "'online'" },
+      { _id: "ui.index.5.components.1.type", value: "'overview-dot'" },
+      { _id: "ui.index.6.type", value: "'tags'" },
+      { _id: "ui.index.6.label", value: "'Tags'" },
+      { _id: "ui.index.6.parameter", value: "Tags" },
+      { _id: "ui.index.6.unsortable", value: "true" },
+      { _id: "ui.index.6.writable", value: "false" }
+    ]);
+  }
+
+  if (options.overview) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.overview.charts.online.label", value: "'Online status'" },
+      {
+        _id: "ui.overview.charts.online.slices.1_onlineNow.color",
+        value: "'#31a354'"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.1_onlineNow.filter",
+        value: "Events.Inform > NOW() - 5 * 60 * 1000"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.1_onlineNow.label",
+        value: "'Online now'"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.2_past24.color",
+        value: "'#a1d99b'"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.2_past24.filter",
+        value:
+          "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.2_past24.label",
+        value: "'Past 24 hours'"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.3_others.color",
+        value: "'#e5f5e0'"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.3_others.filter",
+        value: "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.3_others.label",
+        value: "'Others'"
+      },
+      { _id: "ui.overview.groups.online.label", value: "''" },
+      { _id: "ui.overview.groups.online.charts.0", value: "'online'" }
+    ]);
+  }
+
+  if (options.presets) {
+    resources["presets"] = [
+      {
+        _id: "bootstrap",
+        weight: 0,
+        channel: "bootstrap",
+        events: "0 BOOTSTRAP",
+        provision: "bootstrap"
+      },
+      { _id: "default", weight: 0, channel: "default", provision: "default" },
+      { _id: "inform", weight: 0, channel: "inform", provision: "inform" }
+    ];
+
+    resources["provisions"] = [
+      { _id: "bootstrap", script: BOOTSTRAP_SCRIPT },
+      { _id: "default", script: DEFAULT_SCRIPT },
+      { _id: "inform", script: INFORM_SCRIPT }
+    ];
+  }
+
+  if (resources["permissions"]) {
+    for (const p of resources["permissions"]) {
+      p["_id"] = `${p["role"]}:${p["resource"]}:${p["access"]}`;
+      proms.push(db.putPermission(p["_id"], p));
+    }
+  }
+
+  if (resources["users"]) {
+    for (const u of resources["users"]) {
+      u["salt"] = await generateSalt(64);
+      u["password"] = await hashPassword(u["password"], u["salt"]);
+      u["roles"] = (u["roles"] || []).join(",");
+      u["_id"] = u["username"];
+      delete u["username"];
+      proms.push(db.putUser(u["_id"], u));
+    }
+  }
+
+  if (resources["provisions"]) {
+    for (const p of resources["provisions"])
+      proms.push(db.putProvision(p["_id"], p));
+  }
+
+  if (resources["presets"])
+    for (const p of resources["presets"]) proms.push(db.putPreset(p["_id"], p));
+
+  if (resources["config"])
+    for (const c of resources["config"]) proms.push(db.putConfig(c["_id"], c));
+
+  await proms;
+  return del("presets_hash");
+}

lib/instance-set.js

@@ -1,138 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-
-class InstanceSet {
-
-  constructor() {
-    this.set = new Set();
-  }
-
-  add(instance) {
-    this.set.add(instance);
-  }
-
-  delete(instance) {
-    this.set.delete(instance);
-  }
-
-  superset(instance) {
-    let res = [];
-    for (let inst of this.set) {
-      let match = true;
-      for (let k in instance) {
-        if (inst[k] !== instance[k]) {
-          match = false;
-          break;
-        }
-      }
-
-      if (match)
-        res.push(inst);
-    }
-
-    res.sort(function(a, b) {
-      let keysA = Object.keys(a);
-      let keysB = Object.keys(b);
-
-      if (keysA.length != keysB.length)
-        return keysB.length - keysA.length;
-
-      keysA.sort();
-      keysB.sort();
-
-      for (let i = 0; i < keysA.length; ++ i) {
-        if (keysA[i] > keysB[i])
-          return 1;
-        else if (keysA[i] < keysB[i])
-          return -1;
-        else if (a[keysA[i]] > b[keysB[i]])
-          return 1;
-        else if (a[keysA[i]] < b[keysB[i]])
-          return -1;
-      }
-
-      return 0;
-    });
-
-    return res;
-  }
-
-  subset(instance) {
-    let res = [];
-
-    for (let inst of this.set) {
-      let match = true;
-      for (let k in inst) {
-        if (inst[k] !== instance[k]) {
-          match = false;
-          break;
-        }
-      }
-
-      if (match)
-        res.push(inst);
-    }
-
-    res.sort(function(a, b) {
-      let keysA = Object.keys(a);
-      let keysB = Object.keys(b);
-
-      if (keysA.length != keysB.length)
-        return keysA.length - keysB.length;
-
-      keysA.sort();
-      keysB.sort();
-
-      for (let i = 0; i < keysA.length; ++ i) {
-        if (keysA[i] > keysB[i])
-          return 1;
-        else if (keysA[i] < keysB[i])
-          return -1;
-        else if (a[keysA[i]] > b[keysB[i]])
-          return 1;
-        else if (a[keysA[i]] < b[keysB[i]])
-          return -1;
-      }
-
-      return 0;
-    });
-
-    return res;
-  }
-
-  forEach(callback) {
-    return this.set.forEach(callback);
-  }
-
-  values() {
-    return this.set.values();
-  }
-
-  clear() {
-    this.set.clear();
-  }
-
-  get size() {
-    return this.set.size;
-  }
-}
-
-module.exports = InstanceSet;

lib/instance-set.ts

@@ -0,0 +1,131 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+interface Instance {
+  [name: string]: string;
+}
+
+export default class InstanceSet {
+  private set: Set<Instance>;
+
+  public constructor() {
+    this.set = new Set();
+  }
+
+  public add(instance: Instance): void {
+    this.set.add(instance);
+  }
+
+  public delete(instance: Instance): void {
+    this.set.delete(instance);
+  }
+
+  public superset(instance: Instance): Instance[] {
+    const res = [];
+    for (const inst of this.set) {
+      let match = true;
+      for (const k in instance) {
+        if (inst[k] !== instance[k]) {
+          match = false;
+          break;
+        }
+      }
+
+      if (match) res.push(inst);
+    }
+
+    res.sort((a, b) => {
+      const keysA = Object.keys(a);
+      const keysB = Object.keys(b);
+
+      if (keysA.length !== keysB.length) return keysB.length - keysA.length;
+
+      keysA.sort();
+      keysB.sort();
+
+      for (let i = 0; i < keysA.length; ++i) {
+        if (keysA[i] > keysB[i]) return 1;
+        else if (keysA[i] < keysB[i]) return -1;
+        else if (a[keysA[i]] > b[keysB[i]]) return 1;
+        else if (a[keysA[i]] < b[keysB[i]]) return -1;
+      }
+
+      return 0;
+    });
+
+    return res;
+  }
+
+  public subset(instance: Instance): Instance[] {
+    const res = [];
+
+    for (const inst of this.set) {
+      let match = true;
+      for (const k in inst) {
+        if (inst[k] !== instance[k]) {
+          match = false;
+          break;
+        }
+      }
+
+      if (match) res.push(inst);
+    }
+
+    res.sort((a, b) => {
+      const keysA = Object.keys(a);
+      const keysB = Object.keys(b);
+
+      if (keysA.length !== keysB.length) return keysA.length - keysB.length;
+
+      keysA.sort();
+      keysB.sort();
+
+      for (let i = 0; i < keysA.length; ++i) {
+        if (keysA[i] > keysB[i]) return 1;
+        else if (keysA[i] < keysB[i]) return -1;
+        else if (a[keysA[i]] > b[keysB[i]]) return 1;
+        else if (a[keysA[i]] < b[keysB[i]]) return -1;
+      }
+
+      return 0;
+    });
+
+    return res;
+  }
+
+  public [Symbol.iterator](): IterableIterator<Instance> {
+    return this.set.values();
+  }
+
+  public forEach(callback: (instance: Instance) => void): void {
+    this.set.forEach(callback);
+  }
+
+  public values(): IterableIterator<Instance> {
+    return this.set.values();
+  }
+
+  public clear(): void {
+    this.set.clear();
+  }
+
+  public get size(): number {
+    return this.set.size;
+  }
+}

lib/local-cache.js

@@ -1,356 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const vm = require("vm");
-const crypto = require("crypto");
-
-const db = require("./db");
-const cache = require("./cache");
-const query = require("./query");
-const logger = require("./logger");
-const scheduling = require("./scheduling");
-
-const REFRESH = 3000;
-
-let nextRefresh = 1;
-let hash = null;
-let presets, provisions, virtualParameters, files;
-
-
-function computeHash() {
-  // MD5 hash for presets, provisions, virtual parameters for detecting changes
-  let h = crypto.createHash("md5");
-  for (let p of presets) {
-    h.update(JSON.stringify(p.name));
-    h.update(JSON.stringify(p.channel));
-    h.update(JSON.stringify(p.schedule));
-    h.update(JSON.stringify(p.events));
-    h.update(JSON.stringify(p.precondition));
-    h.update(JSON.stringify(p.provisions));
-  }
-
-  let keys;
-
-  keys = Object.keys(provisions).sort();
-  h.update(JSON.stringify(keys));
-  for (let k of keys)
-    h.update(provisions[k].md5);
-
-  keys = Object.keys(virtualParameters).sort();
-  h.update(JSON.stringify(keys));
-  for (let k of keys)
-    h.update(virtualParameters[k].md5);
-
-  hash = h.digest("hex");
-}
-
-function flattenObject(src, prefix = "", dst = {}) {
-  for (let k of Object.keys(src)) {
-    let v = src[k];
-    if (typeof v === "object" && !Array.isArray(v))
-      flattenObject(v, `${prefix}${k}.`, dst);
-    else
-      dst[`${prefix}${k}`] = v;
-  }
-  return dst;
-}
-
-function refresh(callback) {
-  if (!nextRefresh)
-    return setTimeout(function() {refresh(callback);}, 20);
-
-  nextRefresh = 0;
-  const now = Date.now();
-
-  cache.get("presets_hash", function(err, res) {
-    if (err)
-      return callback(err);
-
-    if (hash && res === hash) {
-      nextRefresh = now + (REFRESH - (now % REFRESH));
-      return callback();
-    }
-
-    cache.lock("presets_hash_lock", 3, function(err, unlockOrExtend) {
-      if (err)
-        return callback(err);
-
-      let counter = 3;
-
-      counter += 2;
-      db.getPresets(function(err, res) {
-        if (err) {
-          if (counter & 1)
-            callback(err);
-          return counter = 0;
-        }
-
-        db.getObjects(function(err, objects) {
-          if (err) {
-            if (counter & 1)
-              callback(err);
-            return counter = 0;
-          }
-
-          objects = objects.map(obj => {
-            // Flatten object
-            obj = flattenObject(obj);
-
-            // If no keys are defined, consider all parameters as keys to keep the
-            // same behavior from v1.0
-            if (!obj["_keys"] || !obj["_keys"].length)
-              obj["_keys"] = Object.keys(obj).filter(k => !k.startsWith("_"));
-
-            return obj
-          })
-
-          res.sort((a, b) => {
-            if (a.weight === b.weight)
-              return a._id > b._id;
-            else
-              return a.weight - b.weight;
-          });
-
-          presets = [];
-          for (let preset of res) {
-            let schedule = null;
-            if (preset.schedule) {
-              let parts = preset.schedule.trim().split(/\s+/);
-              schedule = {
-                md5: crypto.createHash("md5").update(preset.schedule).digest("hex")
-              };
-
-              try {
-                schedule.duration = +(parts.shift()) * 1000;
-                schedule.schedule = scheduling.parseCron(parts.join(" "));
-              }
-              catch (err) {
-                logger.warn({
-                  message: "Invalid preset schedule",
-                  preset: preset._id,
-                  schedule: preset.schedule
-                });
-                schedule.schedule = false;
-              }
-            }
-
-            let events = preset.events || {};
-            let precondition = query.convertMongoQueryToFilters(JSON.parse(preset.precondition));
-            let _provisions = preset.provisions || [];
-
-            // Generate provisions from the old configuration format
-            for (let c of preset.configurations) {
-              switch (c.type) {
-                case "age":
-                  _provisions.push(["refresh", c.name, c.age]);
-                  break;
-
-                case "value":
-                  _provisions.push(["value", c.name, c.value]);
-                  break;
-
-                case "add_tag":
-                  _provisions.push(["tag", c.tag, true]);
-                  break;
-
-                case "delete_tag":
-                  _provisions.push(["tag", c.tag, false]);
-                  break;
-
-                case "provision":
-                  _provisions.push([c.name].concat(c.args || []));
-                  break;
-
-                case "add_object":
-                  for (let obj of objects) {
-                    if (obj["_id"] === c.object) {
-                      let alias = obj["_keys"].map((k) => `${k}:${JSON.stringify(obj[k])}`).join(",");
-                      let p = `${c.name}.[${alias}]`;
-                      _provisions.push(["instances", p, 1]);
-
-                      for (let k in obj)
-                        if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
-                          _provisions.push(["value", `${p}.${k}`, obj[k]]);
-                    }
-                  }
-                  break;
-
-                case "delete_object":
-                  for (let obj of objects) {
-                    if (obj["_id"] === c.object) {
-                      let alias = obj["_keys"].map((k) => `${k}:${JSON.stringify(obj[k])}`).join(",");
-                      let p = `${c.name}.[${alias}]`;
-                      _provisions.push(["instances", p, 0]);
-                    }
-                  }
-                  break;
-
-                default:
-                  if (counter & 1)
-                    callback(new Error(`Unknown configuration type ${c.type}`));
-                  return counter = 0;
-                  break;
-              }
-            }
-
-            presets.push({
-              name: preset._id,
-              channel: preset.channel || "default",
-              schedule: schedule,
-              events: events,
-              precondition: precondition,
-              provisions: _provisions
-            });
-          }
-
-          if ((counter -= 2) === 1) {
-            computeHash();
-            cache.set("presets_hash", hash, 300, function(err) {
-              unlockOrExtend(0);
-              nextRefresh = now + (REFRESH - (now % REFRESH));
-              callback(err);
-            });
-          }
-        });
-      });
-
-      counter += 2;
-      db.getProvisions(function(err, res) {
-        if (err) {
-          if (counter & 1)
-            callback(err);
-          return counter = 0;
-        }
-
-        provisions = {};
-        for (let r of res) {
-          provisions[r._id] = {};
-          provisions[r._id].md5 = crypto.createHash("md5").update(r.script).digest("hex");
-          provisions[r._id].script = new vm.Script(`"use strict";(function(){\n${r.script}\n})();`, {filename: r._id, lineOffset: -1, timeout: 50});
-        }
-
-        if ((counter -= 2) === 1) {
-          computeHash();
-          cache.set("presets_hash", hash, 300, function(err) {
-            unlockOrExtend(0);
-            nextRefresh = now + (REFRESH - (now % REFRESH));
-            callback(err);
-          });
-        }
-      });
-
-      counter += 2;
-      db.getVirtualParameters(function(err, res) {
-        if (err) {
-          if (counter & 1)
-            callback(err);
-          return counter = 0;
-        }
-
-        virtualParameters = {};
-        for (let r of res) {
-          virtualParameters[r._id] = {}
-          virtualParameters[r._id].md5 = crypto.createHash("md5").update(r.script).digest("hex");
-          virtualParameters[r._id].script = new vm.Script(`"use strict";(function(){\n${r.script}\n})();`, {filename: r._id, lineOffset: -1, timeout: 50});
-        }
-
-        if ((counter -= 2) === 1) {
-          computeHash();
-          cache.set("presets_hash", hash, 300, function(err) {
-            unlockOrExtend(0);
-            nextRefresh = now + (REFRESH - (now % REFRESH));
-            callback(err);
-          });
-        }
-      });
-
-      counter += 2;
-      db.getFiles(function(err, res) {
-        if (err) {
-          if (counter & 1)
-            callback(err);
-          return counter = 0;
-        }
-
-        files = {};
-        for (let r of res) {
-          const id = r.filename || r._id.toString();
-          files[id] = {};
-          files[id].length = r.length;
-          files[id].md5 = r.md5;
-          files[id].contentType = r.contentType;
-        }
-
-        if ((counter -= 2) === 1) {
-          computeHash();
-          cache.set("presets_hash", hash, 300, function(err) {
-            unlockOrExtend(0);
-            nextRefresh = now + (REFRESH - (now % REFRESH));
-            callback(err);
-          });
-        }
-      });
-
-      if ((counter -= 2) === 1) {
-        computeHash();
-        cache.set("presets_hash", hash, 300, function(err) {
-          unlockOrExtend(0);
-          nextRefresh = now + (REFRESH - (now % REFRESH));
-          callback(err);
-        });
-      }
-    });
-  });
-}
-
-
-function getPresets(callback) {
-  if (Date.now() < nextRefresh)
-    return callback(null, hash, presets);
-
-  refresh(function(err) {
-    callback(err, hash, presets);
-  });
-}
-
-
-function getFiles(callback) {
-  if (Date.now() < nextRefresh)
-    return callback(null, hash, files);
-
-  refresh(function(err) {
-    callback(err, hash, files);
-  })
-}
-
-
-function getProvisionsAndVirtualParameters(callback) {
-  if (Date.now() < nextRefresh)
-    return callback(null, hash, provisions, virtualParameters);
-
-  refresh(function(err) {
-    callback(err, hash, provisions, virtualParameters);
-  });
-}
-
-
-exports.getPresets = getPresets;
-exports.getFiles = getFiles;
-exports.getProvisionsAndVirtualParameters = getProvisionsAndVirtualParameters;

lib/local-cache.ts

@@ -0,0 +1,524 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as vm from "vm";
+import * as crypto from "crypto";
+import * as config from "./config";
+import * as db from "./db";
+import * as cache from "./cache";
+import { mongoQueryToFilter } from "./mongodb-functions";
+import * as logger from "./logger";
+import * as scheduling from "./scheduling";
+import * as expression from "./common/expression";
+import { parse } from "./common/expression-parser";
+import { Preset, Expression } from "./types";
+
+interface Permissions {
+  [role: string]: {
+    [access: number]: {
+      [resource: string]: {
+        access: number;
+        filter: Expression;
+        validate?: Expression;
+      };
+    };
+  };
+}
+
+const REFRESH = 3000;
+const EVICT_TIMEOUT = 60000;
+
+const snapshots = new Map();
+let currentSnapshot = null;
+let nextRefresh = 1;
+
+function computeHash(snapshot): string {
+  // MD5 hash for presets, provisions, virtual parameters for detecting changes
+  const h = crypto.createHash("md5");
+  for (const p of snapshot.presets) {
+    h.update(JSON.stringify(p.name));
+    h.update(JSON.stringify(p.channel));
+    h.update(JSON.stringify(p.schedule));
+    h.update(JSON.stringify(p.events));
+    h.update(JSON.stringify(p.precondition));
+    h.update(JSON.stringify(p.provisions));
+  }
+
+  let keys;
+
+  keys = Object.keys(snapshot.provisions).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(snapshot.provisions[k].md5);
+
+  keys = Object.keys(snapshot.virtualParameters).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(snapshot.virtualParameters[k].md5);
+
+  keys = Object.keys(snapshot.config).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(JSON.stringify(snapshot.config[k]));
+
+  keys = Object.keys(snapshot.files).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(JSON.stringify(snapshot.files[k]));
+
+  keys = Object.keys(snapshot.users).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(JSON.stringify(snapshot.users[k]));
+
+  const roles = Object.keys(snapshot.permissions).sort();
+  h.update(JSON.stringify(roles));
+  for (const r of roles) {
+    const levels = Object.keys(snapshot.permissions[r]).sort();
+    h.update(JSON.stringify(levels));
+    for (const l of levels) {
+      keys = Object.keys(snapshot.permissions[r][l]).sort();
+      h.update(JSON.stringify(keys));
+      for (const k of keys)
+        h.update(JSON.stringify(snapshot.permissions[r][l][k]));
+    }
+  }
+
+  return h.digest("hex");
+}
+
+function flattenObject(src, prefix = "", dst = {}): {} {
+  for (const k of Object.keys(src)) {
+    const v = src[k];
+    if (typeof v === "object" && !Array.isArray(v))
+      flattenObject(v, `${prefix}${k}.`, dst);
+    else dst[`${prefix}${k}`] = v;
+  }
+  return dst;
+}
+
+async function fetchPresets(): Promise<Preset[]> {
+  const res = await db.getPresets();
+  let objects = await db.getObjects();
+
+  objects = objects.map(obj => {
+    // Flatten object
+    obj = flattenObject(obj);
+
+    // If no keys are defined, consider all parameters as keys to keep the
+    // same behavior from v1.0
+    if (!obj["_keys"] || !obj["_keys"].length)
+      obj["_keys"] = Object.keys(obj).filter(k => !k.startsWith("_"));
+
+    return obj;
+  });
+
+  res.sort((a, b) => {
+    if (a["weight"] === b["weight"])
+      return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
+    else return a["weight"] - b["weight"];
+  });
+
+  const presets = [] as Preset[];
+  for (const preset of res) {
+    let schedule: { md5: string; duration: number; schedule: any } = null;
+    if (preset["schedule"]) {
+      const parts = preset["schedule"].trim().split(/\s+/);
+      schedule = {
+        md5: crypto
+          .createHash("md5")
+          .update(preset["schedule"])
+          .digest("hex"),
+        duration: null,
+        schedule: null
+      };
+
+      try {
+        schedule.duration = +parts.shift() * 1000;
+        schedule.schedule = scheduling.parseCron(parts.join(" "));
+      } catch (err) {
+        logger.warn({
+          message: "Invalid preset schedule",
+          preset: preset["_id"],
+          schedule: preset["schedule"]
+        });
+        schedule.schedule = false;
+      }
+    }
+
+    const events = preset["events"] || {};
+    let precondition = true as Expression;
+    if (preset["precondition"]) {
+      try {
+        precondition = parse(preset["precondition"]);
+      } catch (error) {
+        precondition = mongoQueryToFilter(JSON.parse(preset["precondition"]));
+      }
+
+      // Simplify expression
+      precondition = expression.evaluate(precondition);
+    }
+
+    const _provisions = preset["provisions"] || [];
+
+    // Generate provisions from the old configuration format
+    for (const c of preset["configurations"]) {
+      switch (c.type) {
+        case "age":
+          _provisions.push(["refresh", c.name, c.age]);
+          break;
+
+        case "value":
+          _provisions.push(["value", c.name, c.value]);
+          break;
+
+        case "add_tag":
+          _provisions.push(["tag", c.tag, true]);
+          break;
+
+        case "delete_tag":
+          _provisions.push(["tag", c.tag, false]);
+          break;
+
+        case "provision":
+          _provisions.push([c.name].concat(c.args || []));
+          break;
+
+        case "add_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map(k => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 1]);
+
+              for (const k in obj) {
+                if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
+                  _provisions.push(["value", `${p}.${k}`, obj[k]]);
+              }
+            }
+          }
+
+          break;
+
+        case "delete_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map(k => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 0]);
+            }
+          }
+
+          break;
+
+        default:
+          throw new Error(`Unknown configuration type ${c.type}`);
+      }
+    }
+
+    presets.push({
+      name: preset["_id"],
+      channel: (preset["channel"] as string) || "default",
+      schedule: schedule,
+      events: events,
+      precondition: precondition,
+      provisions: _provisions
+    });
+  }
+
+  return presets;
+}
+
+async function fetchProvisions(): Promise<{}> {
+  const res = await db.getProvisions();
+
+  const provisions = {};
+  for (const r of res) {
+    provisions[r["_id"]] = {};
+    provisions[r["_id"]].md5 = crypto
+      .createHash("md5")
+      .update(r["script"])
+      .digest("hex");
+    provisions[r["_id"]].script = new vm.Script(
+      `"use strict";(function(){\n${r["script"]}\n})();`,
+      { filename: r["_id"], lineOffset: -1, timeout: 50 }
+    );
+  }
+
+  return provisions;
+}
+
+async function fetchVirtualParameters(): Promise<{}> {
+  const res = await db.getVirtualParameters();
+
+  const virtualParameters = {};
+  for (const r of res) {
+    virtualParameters[r["_id"]] = {};
+    virtualParameters[r["_id"]].md5 = crypto
+      .createHash("md5")
+      .update(r["script"])
+      .digest("hex");
+    virtualParameters[r["_id"]].script = new vm.Script(
+      `"use strict";(function(){\n${r["script"]}\n})();`,
+      { filename: r["_id"], lineOffset: -1, timeout: 50 }
+    );
+  }
+
+  return virtualParameters;
+}
+
+async function fetchPermissions(): Promise<Permissions> {
+  const perms = await db.getPermissions();
+  const permissions: Permissions = {};
+
+  for (const p of perms) {
+    if (!permissions[p.role]) permissions[p.role] = {};
+    if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {};
+
+    permissions[p.role][p.access][p.resource] = {
+      access: p.access,
+      filter: parse(p.filter || "true")
+    };
+    if (p.validate)
+      permissions[p.role][p.access][p.resource].validate = parse(p.validate);
+  }
+
+  return permissions;
+}
+
+async function fetchFiles(): Promise<{}> {
+  const res = await db.getFiles();
+  const files = {};
+
+  for (const r of res) {
+    const id = r["filename"] || r["_id"].toString();
+    files[id] = {};
+    files[id].length = r["length"];
+    files[id].md5 = r["md5"];
+    files[id].contentType = r["contentType"];
+  }
+
+  return files;
+}
+
+async function fetchUsers(): Promise<{}> {
+  const _users = await db.getUsers();
+  const users = {};
+
+  for (const user of _users) {
+    users[user._id] = {
+      password: user.password,
+      salt: user.salt,
+      roles: user.roles.split(",").map(s => s.trim())
+    };
+  }
+
+  return users;
+}
+
+async function fetchConfig(): Promise<[{}, {}]> {
+  const conf = await db.getConfig();
+
+  conf.sort((a, b) => (a.id > b.id ? 1 : a.id < b.id ? -1 : 0));
+
+  const ui = {
+    filters: {},
+    device: {},
+    index: {},
+    overview: {}
+  };
+  const _config = {};
+
+  for (const c of conf) {
+    // Evaluate expressions to simplify them
+    const val = expression.evaluate(c.value);
+    _config[c.id] = val;
+    if (c.id.startsWith("ui.")) {
+      const keys = c.id.split(".");
+      // remove the first key(ui)
+      keys.shift();
+      let ref = ui;
+      while (keys.length > 1) {
+        const k = keys.shift();
+        if (typeof ref[k] !== "object") ref[k] = {};
+        ref = ref[k];
+      }
+      ref[keys[0]] = val;
+    }
+  }
+
+  return [_config, ui];
+}
+
+async function refresh(): Promise<void> {
+  if (!nextRefresh) {
+    await new Promise(resolve => setTimeout(resolve, 20));
+    await refresh();
+    return;
+  }
+
+  nextRefresh = 0;
+  const now = Date.now();
+
+  const dbHash = await cache.get("presets_hash");
+
+  if (currentSnapshot && dbHash === currentSnapshot) {
+    nextRefresh = now + (REFRESH - (now % REFRESH));
+    return;
+  }
+
+  const unlockOrExtend = await cache.lock("presets_hash_lock", 3);
+
+  const res = await Promise.all([
+    fetchPresets(),
+    fetchProvisions(),
+    fetchVirtualParameters(),
+    fetchFiles(),
+    fetchPermissions(),
+    fetchUsers(),
+    fetchConfig()
+  ]);
+
+  const snapshot = {
+    presets: res[0],
+    provisions: res[1],
+    virtualParameters: res[2],
+    files: res[3],
+    permissions: res[4],
+    users: res[5],
+    config: res[6][0],
+    ui: res[6][1]
+  };
+
+  if (currentSnapshot) {
+    const h = currentSnapshot;
+    const s = snapshots.get(h);
+    setTimeout(() => {
+      if (snapshots.get(h) === s) snapshots.delete(h);
+    }, EVICT_TIMEOUT).unref();
+  }
+
+  currentSnapshot = computeHash(snapshot);
+  snapshots.set(currentSnapshot, snapshot);
+  await cache.set("presets_hash", currentSnapshot, 300);
+  await unlockOrExtend(0);
+
+  nextRefresh = now + (REFRESH - (now % REFRESH));
+}
+
+export async function getCurrentSnapshot(): Promise<string> {
+  if (Date.now() > nextRefresh) await refresh();
+  return currentSnapshot;
+}
+
+export function hasSnapshot(hash): boolean {
+  return snapshots.has(hash);
+}
+
+export function getPresets(snapshotKey): Preset[] {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+  return snapshot.presets;
+}
+
+export function getProvisions(
+  snapshotKey
+): { [name: string]: { md5: string; script: vm.Script } } {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+  return snapshot.provisions;
+}
+
+export function getVirtualParameters(
+  snapshotKey
+): { [name: string]: { md5: string; script: vm.Script } } {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+  return snapshot.virtualParameters;
+}
+
+export function getFiles(
+  snapshotKey
+): { [name: string]: { length: number; md5: string; contentType: string } } {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+  return snapshot.files;
+}
+
+export function getConfig(
+  snapshotKey,
+  key,
+  context?,
+  now?
+): string | number | boolean | null {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  const oldOpts = {
+    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
+    "cwmp.debug": "DEBUG",
+    "cwmp.retryDelay": "RETRY_DELAY",
+    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
+    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
+    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
+    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
+    "cwmp.cookiesPath": "COOKIES_PATH",
+    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
+    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
+    "cwmp.connectionRequestAllowBasicAuth":
+      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
+    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
+    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT"
+  };
+
+  if (!(key in snapshot.config)) {
+    if (key in oldOpts)
+      return config.get(oldOpts[key], context ? context.id : null);
+    else return null;
+  }
+  return expression.evaluate(snapshot.config[key], context, now || Date.now());
+}
+
+export function getConfigExpression(snapshotKey, key): Expression {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  return snapshot.config[key];
+}
+
+export function getUsers(snapshotKey): {} {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  return snapshot.users;
+}
+
+export function getPermissions(snapshotKey): {} {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  return snapshot.permissions;
+}
+
+export function getUiConfig(snapshotKey): {} {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  return snapshot.ui;
+}

lib/logger.ts

@@ -1,5 +1,5 @@
 /**
- * Copyright 2013-2017  Zaid Abdulla
+ * Copyright 2013-2019  GenieACS Inc.
  *
  * This file is part of GenieACS.
  *
@@ -16,77 +16,95 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
-"use strict";
 
-const fs = require("fs");
-const os = require("os");
+import * as fs from "fs";
+import * as os from "os";
 
-const config = require("./config");
+import * as config from "./config";
 
 const REOPEN_EVERY = 60000;
 
 const LOG_FORMAT = config.get("LOG_FORMAT");
 const ACCESS_LOG_FORMAT = config.get("ACCESS_LOG_FORMAT") || LOG_FORMAT;
 
-const defaultMeta = {};
+const defaultMeta: { [name: string]: any } = {};
 
 let LOG_SYSTEMD = false;
 let ACCESS_LOG_SYSTEMD = false;
 
 let LOG_FILE, ACCESS_LOG_FILE;
 
-let logStream = fs.createWriteStream(null, {fd: process.stderr.fd});
+declare global {
+  /* eslint-disable-next-line @typescript-eslint/no-namespace */
+  namespace NodeJS {
+    export interface WritableStream {
+      fd?: number;
+    }
+  }
+}
+
+declare module "fs" {
+  interface WriteStream {
+    fd?: number;
+  }
+}
+
+let logStream = fs.createWriteStream(null, { fd: process.stderr.fd });
 let logStat = fs.fstatSync(logStream.fd);
-let accessLogStream = fs.createWriteStream(null, {fd: process.stdout.fd});
+let accessLogStream = fs.createWriteStream(null, { fd: process.stdout.fd });
 let accessLogStat = fs.fstatSync(accessLogStream.fd);
 
-
 // Reopen if original files have been moved (e.g. logrotate)
-function reopen() {
+function reopen(): void {
   let counter = 1;
 
   if (LOG_FILE) {
-    ++ counter;
-    fs.stat(LOG_FILE, function(err, stat) {
-      if (err && !err.message.startsWith("ENOENT:"))
-        throw err;
+    ++counter;
+    fs.stat(LOG_FILE, (err, stat) => {
+      if (err && !err.message.startsWith("ENOENT:")) throw err;
 
       if (!(stat && stat.dev === logStat.dev && stat.ino === logStat.ino)) {
         logStream.end();
-        logStream = fs.createWriteStream(null,
-          {fd: fs.openSync(LOG_FILE, "a")});
+        logStream = fs.createWriteStream(null, {
+          fd: fs.openSync(LOG_FILE, "a")
+        });
         logStat = fs.fstatSync(logStream.fd);
       }
 
-      if (-- counter === 0)
+      if (--counter === 0)
         setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
     });
   }
 
   if (ACCESS_LOG_FILE) {
-    ++ counter;
-    fs.stat(ACCESS_LOG_FILE, function(err, stat) {
-      if (err && !err.message.startsWith("ENOENT:"))
-        throw err;
+    ++counter;
+    fs.stat(ACCESS_LOG_FILE, (err, stat) => {
+      if (err && !err.message.startsWith("ENOENT:")) throw err;
 
-      if (!(stat && stat.dev === accessLogStat.dev && stat.ino === accessLogStat.ino)) {
+      if (
+        !(
+          stat &&
+          stat.dev === accessLogStat.dev &&
+          stat.ino === accessLogStat.ino
+        )
+      ) {
         accessLogStream.end();
-        accessLogStream = fs.createWriteStream(null,
-          {fd: fs.openSync(ACCESS_LOG_FILE, "a")});
+        accessLogStream = fs.createWriteStream(null, {
+          fd: fs.openSync(ACCESS_LOG_FILE, "a")
+        });
         accessLogStat = fs.fstatSync(accessLogStream.fd);
       }
 
-      if (-- counter === 0)
+      if (--counter === 0)
         setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
     });
   }
 
-  if (-- counter === 0)
+  if (--counter === 0)
     setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
 }
 
-
-function init(service, version) {
+export function init(service, version): void {
   defaultMeta.hostname = os.hostname();
   defaultMeta.pid = process.pid;
   defaultMeta.name = `genieacs-${service}`;
@@ -96,14 +114,14 @@ function init(service, version) {
   ACCESS_LOG_FILE = config.get(`${service.toUpperCase()}_ACCESS_LOG_FILE`);
 
   if (LOG_FILE) {
-    logStream = fs.createWriteStream(null,
-      {fd: fs.openSync(LOG_FILE, "a")});
+    logStream = fs.createWriteStream(null, { fd: fs.openSync(LOG_FILE, "a") });
     logStat = fs.fstatSync(logStream.fd);
   }
 
   if (ACCESS_LOG_FILE) {
-    accessLogStream = fs.createWriteStream(null,
-      {fd: fs.openSync(ACCESS_LOG_FILE, "a")});
+    accessLogStream = fs.createWriteStream(null, {
+      fd: fs.openSync(ACCESS_LOG_FILE, "a")
+    });
     accessLogStat = fs.fstatSync(accessLogStream.fd);
   }
 
@@ -111,31 +129,28 @@ function init(service, version) {
   const JOURNAL_STREAM = process.env["JOURNAL_STREAM"];
 
   if (JOURNAL_STREAM) {
-    let [dev, inode] = JOURNAL_STREAM.split(":");
-    dev = parseInt(dev);
-    inode = parseInt(inode);
+    const [dev, inode] = JOURNAL_STREAM.split(":").map(parseInt);
 
-    LOG_SYSTEMD = (logStat.dev === dev && logStat.ino === inode);
-    ACCESS_LOG_SYSTEMD = (accessLogStat.dev === dev && accessLogStat.ino === inode);
+    LOG_SYSTEMD = logStat.dev === dev && logStat.ino === inode;
+    ACCESS_LOG_SYSTEMD =
+      accessLogStat.dev === dev && accessLogStat.ino === inode;
   }
 
-  if (LOG_FILE || ACCESS_LOG_FILE) {
+  if (LOG_FILE || ACCESS_LOG_FILE)
     // Can't use setInterval as we need all workers to cehck at the same time
     setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
-  }
 }
 
-
-function close() {
+export function close(): void {
   accessLogStream.end();
   logStream.end();
 }
 
-
-function flatten(details) {
+export function flatten(details): {} {
   if (details.sessionContext) {
     details.deviceId = details.sessionContext.deviceId;
-    details.remoteAddress = details.sessionContext.httpRequest.connection.remoteAddress;
+    details.remoteAddress =
+      details.sessionContext.httpRequest.connection.remoteAddress;
     delete details.sessionContext;
   }
 
@@ -157,20 +172,17 @@ function flatten(details) {
       details.acsRequestName = details.rpc.acsRequest.name;
       if (details.rpc.acsRequest.commandKey)
         details.acsRequestCommandKey = details.rpc.acsRequest.commandKey;
-    }
-    else if (details.rpc.cpeRequest) {
+    } else if (details.rpc.cpeRequest) {
       details.cpeRequestId = details.rpc.id;
       if (details.rpc.cpeRequest.name === "Inform") {
         details.informEvent = details.rpc.cpeRequest.event.join(",");
         details.informRetryCount = details.rpc.cpeRequest.retryCount;
-      }
-      else {
+      } else {
         details.cpeRequestName = details.rpc.cpeRequest.name;
         if (details.rpc.cpeRequest.commandKey)
           details.cpeRequestCommandKey = details.rpc.cpeRequest.commandKey;
       }
-    }
-    else if (details.rpc.cpeFault) {
+    } else if (details.rpc.cpeFault) {
       details.acsRequestId = details.rpc.id;
       details.cpeFaultCode = details.rpc.cpeFault.detail.faultCode;
       details.cpeFaultString = details.rpc.cpeFault.detail.faultString;
@@ -184,19 +196,26 @@ function flatten(details) {
     delete details.fault;
   }
 
+  // For genieacs-ui
+  if (details.context) {
+    details.remoteAddress = details.context.request.ip;
+    if (details.context.state.user)
+      details.user = details.context.state.user.username;
+    delete details.context;
+  }
+
+  for (const [k, v] of Object.entries(details))
+    if (v == null) delete details[k];
+
   return details;
 }
 
-
-function formatJson(details, systemd) {
+function formatJson(details, systemd): string {
   if (systemd) {
     let severity = "";
-    if (details.severity === "info")
-      severity = "<6>";
-    else if (details.severity === "warn")
-      severity = "<4>";
-    else if (details.severity === "error")
-      severity = "<3>";
+    if (details.severity === "info") severity = "<6>";
+    else if (details.severity === "warn") severity = "<4>";
+    else if (details.severity === "error") severity = "<3>";
 
     return `${severity}${JSON.stringify(flatten(details))}${os.EOL}`;
   }
@@ -204,54 +223,52 @@ function formatJson(details, systemd) {
   return `${JSON.stringify(flatten(details))}${os.EOL}`;
 }
 
+function formatSimple(details, systemd): string {
+  const skip = {
+    user: true,
+    remoteAddress: true,
+    severity: true,
+    timestamp: true,
+    message: true,
+    deviceId: !!details.sessionContext
+  };
 
-function formatSimple(details, systemd) {
   flatten(details);
 
   let remote = "";
   if (details.remoteAddress) {
-    if (details.deviceId)
+    if (details.deviceId && skip["deviceId"])
       remote = `${details.remoteAddress} ${details.deviceId}: `;
-    else
-      remote = `${details.remoteAddress}: `;
+    else if (details.user)
+      remote = `${details.user}@${details.remoteAddress}: `;
+    else remote = `${details.remoteAddress}: `;
   }
 
   const keys = Object.keys(details);
-  const skip = {
-    deviceId: 1,
-    remoteAddress: 1,
-    severity: 1,
-    timestamp: 1,
-    message: 1
-  };
 
   let meta = "";
 
-  let kv = [];
-  for (let k of keys)
-    if (!skip[k])
-      kv.push(`${k}=${JSON.stringify(details[k])}`);
+  const kv = [];
+  for (const k of keys)
+    if (!skip[k]) kv.push(`${k}=${JSON.stringify(details[k])}`);
 
-  if (kv.length)
-    meta = `; ${kv.join(" ")}`;
+  if (kv.length) meta = `; ${kv.join(" ")}`;
 
   if (systemd) {
     let severity = "";
-    if (details.severity === "info")
-      severity = "<6>";
-    else if (details.severity === "warn")
-      severity = "<4>";
-    else if (details.severity === "error")
-      severity = "<3>";
+    if (details.severity === "info") severity = "<6>";
+    else if (details.severity === "warn") severity = "<4>";
+    else if (details.severity === "error") severity = "<3>";
 
     return `${severity}${remote}${details.message}${meta}${os.EOL}`;
   }
 
-  return `${details.timestamp} [${details.severity.toUpperCase()}] ${remote}${details.message}${meta}${os.EOL}`;
+  return `${details.timestamp} [${details.severity.toUpperCase()}] ${remote}${
+    details.message
+  }${meta}${os.EOL}`;
 }
 
-
-function log(details) {
+function log(details): void {
   details.timestamp = new Date().toISOString();
   if (LOG_FORMAT === "json") {
     details = Object.assign({}, defaultMeta, details);
@@ -261,26 +278,22 @@ function log(details) {
   }
 }
 
-
-function info(details) {
+export function info(details): void {
   details.severity = "info";
   log(details);
 }
 
-
-function warn(details) {
+export function warn(details): void {
   details.severity = "warn";
   log(details);
 }
 
-
-function error(details) {
+export function error(details): void {
   details.severity = "error";
   log(details);
 }
 
-
-function accessLog(details) {
+export function accessLog(details): void {
   details.timestamp = new Date().toISOString();
   if (ACCESS_LOG_FORMAT === "json") {
     Object.assign(details, defaultMeta);
@@ -290,31 +303,17 @@ function accessLog(details) {
   }
 }
 
-
-function accessInfo(details) {
+export function accessInfo(details): void {
   details.severity = "info";
   accessLog(details);
 }
 
-
-function accessWarn(details) {
+export function accessWarn(details): void {
   details.severity = "warn";
   accessLog(details);
 }
 
-
-function accessError(details) {
+export function accessError(details): void {
   details.severity = "error";
   accessLog(details);
 }
-
-
-exports.init = init;
-exports.close = close;
-exports.flatten = flatten;
-exports.info = info;
-exports.warn = warn;
-exports.error = error;
-exports.accessInfo = accessInfo;
-exports.accessWarn = accessWarn;
-exports.accessError = accessError;

lib/mongodb-functions.ts

@@ -0,0 +1,653 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { ObjectID } from "mongodb";
+import { map, parse, stringify } from "./common/expression-parser";
+import { likePatternToRegExp, evaluate } from "./common/expression";
+import { Expression, Fault, Task } from "./types";
+
+const isArray = Array.isArray;
+
+export function processDeviceFilter(filter): Expression {
+  return map(filter, exp => {
+    if (!isArray(exp)) return exp;
+
+    if (exp[0] === "PARAM") {
+      const p = exp[1];
+      if (p === "DeviceID.ID") return ["PARAM", "_id"];
+      else if (p === "DeviceID") return ["PARAM", "_deviceId"];
+      else if (p.startsWith("DeviceID."))
+        return ["PARAM", "_deviceId._" + p.slice(9)];
+      else if (p === "Events.Inform") return ["PARAM", "_lastInform"];
+      else if (p === "Events.Registered") return ["PARAM", "_registered"];
+      else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"];
+      else if (p === "Events.1_BOOT") return ["PARAM", "_lastBoot"];
+    } else if (
+      isArray(exp[1]) &&
+      exp[1][0] === "PARAM" &&
+      exp[1][1].startsWith("Tags.")
+    ) {
+      const t = exp[1][1].slice(5);
+      if (exp[0] === "IS NULL") return ["<>", ["PARAM", "_tags"], t];
+      else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t];
+      else if (exp[0] === "=" && exp[2] === true)
+        return ["=", ["PARAM", "_tags"], t];
+      else if (exp[0] === "<>" && exp[2] !== true)
+        return ["=", ["PARAM", "_tags"], t];
+    } else if (
+      ["=", "<>", ">", ">=", "<", "<=", "LIKE", "NOT LIKE"].includes(exp[0])
+    ) {
+      let e = map(exp, ee => {
+        if (
+          isArray(ee) &&
+          ee[0] === "PARAM" &&
+          typeof ee[1] === "string" &&
+          !ee[1].startsWith("_")
+        )
+          return ["PARAM", `${ee[1]}._value`];
+        return ee;
+      });
+
+      if (typeof e[2] === "number") {
+        const alt = (e as any[]).slice();
+        alt[2] = new Date(e[2]);
+        e = ["OR", e, alt];
+      }
+      return e;
+    }
+    return exp;
+  });
+}
+
+export function processTasksFilter(filter): Expression {
+  return map(filter, exp => {
+    if (!isArray(exp)) return exp;
+    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
+      const e = exp.slice();
+      if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectID(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "timestamp")
+        e[2] = new Date(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
+        e[2] = new Date(e[2]);
+      return e;
+    }
+    return exp;
+  });
+}
+
+export function processFaultsFilter(filter): Expression {
+  return map(filter, exp => {
+    if (!isArray(exp)) return exp;
+    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
+      const e = exp.slice();
+      if (e[1][0] === "PARAM" && e[1][1] === "timestamp") e[2] = new Date(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
+        e[2] = new Date(e[2]);
+      return e;
+    }
+    return exp;
+  });
+}
+
+export function filterToMongoQuery(exp: Expression): {} {
+  const ops = {
+    OR: 0,
+    AND: 0,
+    NOT: 0,
+    "=": 1,
+    "<>": 1,
+    ">": 1,
+    ">=": 1,
+    "<": 1,
+    "<=": 1,
+    LIKE: 1,
+    "NOT LIKE": 1,
+    "IS NULL": 1,
+    "IS NOT NULL": 1
+  };
+
+  function recursive(filter, negate, res = {}): {} {
+    const op = filter[0];
+
+    if (ops[op] === 0) {
+      for (let i = 1; i < filter.length; ++i) {
+        if (!isArray(filter[i]) || ops[filter[i][0]] == null)
+          throw new Error(`Invalid expression in ${op} clause`);
+      }
+
+      if ((!negate && op === "AND") || (negate && op === "OR")) {
+        res["$and"] = res["$and"] || [];
+        for (let i = 1; i < filter.length; ++i)
+          res["$and"].push(recursive(filter[i], negate));
+      } else if ((!negate && op === "OR") || (negate && op === "AND")) {
+        res["$or"] = res["$or"] || [];
+
+        for (let i = 1; i < filter.length; ++i)
+          res["$or"].push(recursive(filter[i], negate));
+      } else if (op === "NOT") {
+        recursive(filter[1], !negate, res);
+      }
+    } else if (ops[op] === 1) {
+      if (isArray(filter[2])) {
+        throw new Error(`Right operand of ${op} clause is not a primitive`);
+      } else if (
+        ["LIKE", "NOT LIKE"].includes(op) &&
+        isArray(filter[1]) &&
+        filter[1][0] === "FUNC" &&
+        ["UPPER", "LOWER"].includes(filter[1][1])
+      ) {
+        if (
+          (filter[1][1] === "UPPER" && filter[2] !== filter[2].toUpperCase()) ||
+          (filter[1][1] === "LOWER" && filter[2] !== filter[2].toLowerCase())
+        ) {
+          throw new Error(
+            `Cannot compare ${
+              filter[1][1]
+            }() against non-${filter[1][1].toLowerCase()}case pattern`
+          );
+        }
+      } else if (!isArray(filter[1]) || filter[1][0] !== "PARAM") {
+        throw new Error(`Left operand of ${op} clause is not a parameter`);
+      }
+
+      if (op === "=") {
+        const param = filter[1][1];
+        const p = (res[param] = res[param] || {});
+        if (negate) p["$ne"] = filter[2];
+        else p["$eq"] = filter[2];
+      } else if (op === "<>") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$ne"] = filter[2];
+        if (param !== "_tags") p["$exists"] = true;
+      } else if (op === ">") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$gt"] = filter[2];
+      } else if (op === ">=") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$gte"] = filter[2];
+      } else if (op === "<") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$lt"] = filter[2];
+      } else if (op === "<=") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$lte"] = filter[2];
+      } else if (op === "IS NULL") {
+        const param = filter[1][1];
+        res[param] = { $exists: negate };
+      } else if (op === "IS NOT NULL") {
+        const param = filter[1][1];
+        res[param] = { $exists: !negate };
+      } else {
+        if (op === "NOT LIKE") negate = !negate;
+        let param;
+        let flags;
+        if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
+          param = filter[1][2][1];
+          flags = "i";
+        } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
+          param = filter[1][2][1];
+          flags = "i";
+        } else {
+          param = filter[1][1];
+          flags = "";
+        }
+        const r = likePatternToRegExp(filter[2], filter[3], flags);
+        if (negate) res[param] = { $not: r };
+        else res[param] = r;
+      }
+    } else {
+      throw new Error(`Unrecognized operator ${op}`);
+    }
+
+    return res;
+  }
+
+  const _exp = evaluate(exp, null, Date.now());
+
+  if (!isArray(_exp)) {
+    if (_exp === true) return {};
+    throw new Error("Primitives are not valid queries");
+  }
+
+  return recursive(_exp, false);
+}
+
+export function processDeviceProjection(projection: {}): {} {
+  if (!projection) return projection;
+  const p = {};
+  for (const [k, v] of Object.entries(projection)) {
+    if (k === "DeviceID.ID") {
+      p["_id"] = 1;
+    } else if (k.startsWith("DeviceID")) {
+      p["_deviceId._SerialNumber"] = v;
+      p["_deviceId._OUI"] = v;
+      p["_deviceId._ProductClass"] = v;
+      p["_deviceId._Manufacturer"] = v;
+    } else if (k.startsWith("Tags")) {
+      p["_tags"] = v;
+    } else if (k.startsWith("Events")) {
+      p["_lastInform"] = v;
+      p["_registered"] = v;
+      p["_lastBoot"] = v;
+      p["_lastBootstrap"] = v;
+    } else {
+      p[k] = v;
+    }
+  }
+
+  return p;
+}
+
+export function processDeviceSort(sort: {}): {} {
+  if (!sort) return sort;
+  const s = {};
+  for (const [k, v] of Object.entries(sort)) {
+    if (k === "DeviceID.ID") s["_id"] = v;
+    else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v;
+    else if (k === "Events.Inform") s["_lastInform"] = v;
+    else if (k === "Events.Registered") s["_registered"] = v;
+    else if (k === "Events.1_BOOT") s["_lastBoot"] = v;
+    else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v;
+    else s[`${k}._value`] = v;
+  }
+
+  return s;
+}
+
+function parseDate(d): number | string {
+  const n = +d;
+  return isNaN(n) ? "" + d : n;
+}
+
+interface FlatAttributes {
+  object?: boolean;
+  objectTimestamp?: number;
+  writable?: boolean;
+  writableTimestamp?: number;
+  value?: [string | number | boolean, string];
+  valueTimestamp?: number;
+  notification?: number;
+  notificationTimestamp?: number;
+  accessList?: string[];
+  accessListTimestamp?: number;
+}
+
+interface FlatDevice {
+  [param: string]: FlatAttributes;
+}
+
+export function flattenDevice(device): FlatDevice {
+  function recursive(
+    input,
+    root: string,
+    output: FlatDevice,
+    timestamp: number
+  ): void {
+    for (const [name, tree] of Object.entries(input)) {
+      if (!root) {
+        if (name === "_lastInform") {
+          output["Events.Inform"] = {
+            value: [parseDate(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_registered") {
+          output["Events.Registered"] = {
+            value: [parseDate(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_lastBoot") {
+          output["Events.1_BOOT"] = {
+            value: [parseDate(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_lastBootstrap") {
+          output["Events.0_BOOTSTRAP"] = {
+            value: [parseDate(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_id") {
+          output["DeviceID.ID"] = {
+            value: [tree as string, "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_deviceId") {
+          output["DeviceID.Manufacturer"] = {
+            value: [tree["_Manufacturer"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+          output["DeviceID.OUI"] = {
+            value: [tree["_OUI"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+          output["DeviceID.ProductClass"] = {
+            value: [tree["_ProductClass"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+          output["DeviceID.SerialNumber"] = {
+            value: [tree["_SerialNumber"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_tags") {
+          output["Tags"] = {
+            writable: false,
+            writableTimestamp: timestamp,
+            object: true,
+            objectTimestamp: timestamp
+          };
+
+          for (const t of tree as string[]) {
+            output[`Tags.${t}`] = {
+              value: [true, "xsd:boolean"],
+              valueTimestamp: timestamp,
+              writable: true,
+              writableTimestamp: timestamp,
+              object: false,
+              objectTimestamp: timestamp
+            };
+          }
+        }
+      }
+
+      if (name.startsWith("_")) continue;
+
+      let childrenTimestamp = timestamp;
+
+      if (!root) childrenTimestamp = +(input["_timestamp"] || 1);
+      else if (+input["_timestamp"] > timestamp)
+        childrenTimestamp = +input["_timestamp"];
+
+      const attrs: FlatAttributes = {};
+      if (tree["_value"] != null) {
+        attrs.value = [
+          tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
+          tree["_type"]
+        ];
+        attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
+        attrs.object = false;
+        attrs.objectTimestamp = childrenTimestamp;
+      } else if (tree["_object"] != null) {
+        attrs.object = tree["_object"];
+        attrs.objectTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_writable"] != null) {
+        attrs.writable = tree["_writable"];
+        attrs.writableTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_notification"] != null) {
+        attrs.notification = tree["_notification"];
+        attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      if (tree["_accessList"] != null) {
+        attrs.accessList = tree["_accessList"];
+        attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      const r = root ? `${root}.${name}` : name;
+      output[r] = attrs;
+
+      if (attrs.object || tree["object"] == null)
+        recursive(tree, r, output, childrenTimestamp);
+    }
+  }
+
+  const newDevice: FlatDevice = {};
+  const timestamp = new Date(device["_lastInform"] || 1).getTime();
+  recursive(device, "", newDevice, timestamp);
+  return newDevice;
+}
+
+export function flattenFault(fault): Fault {
+  const f = Object.assign({}, fault);
+  if (f.timestamp) f.timestamp = +f.timestamp;
+  if (f.expiry) f.expiry = +f.expiry;
+  return f;
+}
+
+export function flattenTask(task): Task {
+  const t = Object.assign({}, task);
+  t._id = "" + t._id;
+  if (t.timestamp) t.timestamp = +t.timestamp;
+  if (t.expiry) t.expiry = +t.expiry;
+  return t;
+}
+
+export function mongoQueryToFilter(query): Expression {
+  function recursive(_query): Expression {
+    const expressions: Expression[] = [];
+    for (const [k, v] of Object.entries(_query)) {
+      if (k[0] === "$") {
+        if (k === "$and") {
+          const and: Expression = ["AND"];
+          for (const vv of Object.values(v)) and.push(recursive(vv));
+          expressions.push(and);
+        } else if (k === "$or") {
+          const or: Expression = ["OR"];
+          for (const vv of Object.values(v)) or.push(recursive(vv));
+          expressions.push(or);
+        } else {
+          throw new Error(`Operator ${k} not supported`);
+        }
+      } else if (k === "_tags") {
+        if (typeof v === "object") {
+          if (isArray(v)) throw new Error(`Invalid type`);
+
+          if (v.hasOwnProperty("$ne"))
+            expressions.push(["IS NULL", ["PARAM", `Tags.${v["$ne"]}`]]);
+          else if (v.hasOwnProperty("$eq"))
+            expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v["$eq"]}`]]);
+          else throw new Error(`Invalid tag query`);
+        } else {
+          expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v}`]]);
+        }
+      } else if (k.startsWith("Tags.")) {
+        let exists: boolean;
+        if (typeof v === "boolean") exists = v;
+        else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"];
+        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
+        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
+        else throw new Error(`Invalid tag query`);
+
+        expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
+      } else if (typeof v === "object") {
+        if (isArray(v)) throw new Error(`Invalid type`);
+
+        const exps: Expression[] = [];
+        for (const [kk, vv] of Object.entries(v)) {
+          let op;
+          switch (kk) {
+            case "$eq":
+              op = "=";
+              break;
+            case "$ne":
+              op = "<>";
+              break;
+            case "$lt":
+              op = "<";
+              break;
+            case "$lte":
+              op = "<=";
+              break;
+            case "$gt":
+              op = ">";
+              break;
+            case "$gte":
+              op = ">=";
+              break;
+            default:
+              throw new Error(`Operator ${kk} not supported`);
+          }
+          exps.push([op, ["PARAM", k], vv]);
+        }
+        if (exps.length === 1) {
+          expressions.push(exps[0]);
+        } else {
+          const and: Expression = ["AND"];
+          expressions.push(and.concat(exps));
+        }
+      } else {
+        expressions.push(["=", ["PARAM", k], v]);
+      }
+    }
+    if (expressions.length === 1) return expressions[0];
+    const and: Expression = ["AND"];
+    return and.concat(expressions);
+  }
+
+  // empty filter
+  if (!Object.keys(query).length) return true;
+
+  return recursive(query);
+}
+
+export function flattenPreset(preset): {} {
+  const p = Object.assign({}, preset);
+  if (p.precondition) {
+    try {
+      // Try parse to check expression validity
+      parse(p.precondition);
+    } catch (error) {
+      p.precondition = mongoQueryToFilter(JSON.parse(p.precondition));
+      p.precondition = p.precondition.length ? stringify(p.precondition) : "";
+    }
+  }
+
+  if (p.events) {
+    const e = [];
+    for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`);
+    p.events = e.join(", ");
+  }
+
+  const provision = p.configurations[0];
+  if (
+    p.configurations.length === 1 &&
+    provision.type === "provision" &&
+    provision.name &&
+    provision.name.length
+  ) {
+    p.provision = provision.name;
+    p.provisionArgs = provision.args
+      ? JSON.stringify(provision.args).slice(1, -1)
+      : "";
+  }
+
+  delete p.configurations;
+  return p;
+}
+
+export function flattenFile(file): {} {
+  const f = {};
+  f["_id"] = file["_id"];
+  if (file.metadata) {
+    f["metadata.fileType"] = file.metadata.fileType || "";
+    f["metadata.oui"] = file.metadata.oui || "";
+    f["metadata.productClass"] = file.metadata.productClass || "";
+    f["metadata.version"] = file.metadata.version || "";
+  }
+  return f;
+}
+
+export function preProcessPreset(data): {} {
+  const preset = Object.assign({}, data);
+
+  if (!preset.precondition) preset.precondition = "";
+  // Try parse to check expression validity
+  parse(preset.precondition);
+
+  preset.weight = parseInt(preset.weight) || 0;
+
+  const events = {};
+  if (preset.events) {
+    for (let e of preset.events.split(",")) {
+      let v = true;
+      e = e.trim();
+      if (e.startsWith("-")) {
+        v = false;
+        e = e.slice(1).trim();
+      }
+      if (e) events[e] = v;
+    }
+  }
+
+  preset.events = events;
+
+  if (!preset.provision) throw new Error("Invalid preset provision");
+
+  const configuration = {
+    type: "provision",
+    name: preset.provision,
+    args: null
+  };
+
+  if (preset.provisionArgs)
+    configuration.args = JSON.parse(`[${preset.provisionArgs}]`);
+
+  delete preset.provision;
+  delete preset.provisionArgs;
+  preset.configurations = [configuration];
+  return preset;
+}

lib/nbi.coffee

@@ -1,516 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This file incorporates work covered by the following copyright and
-# permission notice:
-#
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-url = require 'url'
-mongodb = require 'mongodb'
-querystring = require 'querystring'
-vm = require 'vm'
-
-config = require './config'
-common = require './common'
-db = require './db'
-query = require './query'
-apiFunctions = require './api-functions'
-cache = require './cache'
-
-VERSION = require('../package.json').version
-
-# regular expression objects
-DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/tasks\/?$/
-TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-\_\%]+)(\/[a-zA-Z_]*)?$/
-TAGS_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/tags\/([a-zA-Z0-9\-\_\%]+)\/?$/
-PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-\_\%]+)\/?$/
-OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-\_\%]+)\/?$/
-FILES_REGEX = /^\/files\/([a-zA-Z0-9\%\!\*\'\(\)\;\:\@\&\=\+\$\,\?\#\[\]\-\_\.\~]+)\/?$/
-PING_REGEX = /^\/ping\/([a-zA-Z0-9\-\_\.\:]+)\/?$/
-QUERY_REGEX = /^\/([a-zA-Z0-9_]+)\/?$/
-DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/?$/
-PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-\_\%]+)\/?$/
-VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-\_\%]+)\/?$/
-FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-\_\%\:]+)\/?$/
-
-
-
-throwError = (err, httpResponse) ->
-  if httpResponse
-    httpResponse.writeHead(500, {'Connection' : 'close'})
-    httpResponse.end("#{err.name}: #{err.message}")
-
-  throw err
-
-
-listener = (request, response) ->
-  chunks = []
-  bytes = 0
-
-  response.setHeader('GenieACS-Version', VERSION)
-
-  request.addListener 'data', (chunk) ->
-    chunks.push(chunk)
-    bytes += chunk.length
-
-  request.getBody = () ->
-    # Write all chunks into a Buffer
-    body = new Buffer(bytes)
-    offset = 0
-    chunks.forEach((chunk) ->
-      chunk.copy(body, offset, 0, chunk.length)
-      offset += chunk.length
-    )
-    return body
-
-  request.addListener 'end', () ->
-    body = request.getBody()
-    urlParts = url.parse(request.url, true)
-    if PRESETS_REGEX.test(urlParts.pathname)
-      presetName = querystring.unescape(PRESETS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        preset = JSON.parse(body)
-        preset._id = presetName
-
-        db.presetsCollection.save(preset, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else if request.method == 'DELETE'
-        db.presetsCollection.remove({'_id' : presetName}, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if OBJECTS_REGEX.test(urlParts.pathname)
-      objectName = querystring.unescape(OBJECTS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        object = JSON.parse(body)
-        object._id = objectName
-
-        db.objectsCollection.save(object, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else if request.method == 'DELETE'
-        db.objectsCollection.remove({'_id' : objectName}, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if PROVISIONS_REGEX.test(urlParts.pathname)
-      provisionName = querystring.unescape(PROVISIONS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        object = {
-          _id: provisionName
-          script: body.toString()
-        }
-        try
-          new vm.Script("\"use strict\";(function(){\n#{object.script}\n})();")
-        catch err
-          response.writeHead(400)
-          response.end("#{err.name}: #{err.message}")
-          return
-
-        db.provisionsCollection.save(object, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else if request.method == 'DELETE'
-        db.provisionsCollection.remove({'_id' : provisionName}, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if VIRTUAL_PARAMETERS_REGEX.test(urlParts.pathname)
-      virtualParameterName = querystring.unescape(VIRTUAL_PARAMETERS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        object = {
-          _id: virtualParameterName
-          script: body.toString()
-        }
-        try
-          new vm.Script("\"use strict\";(function(){\n#{object.script}\n})();")
-        catch err
-          response.writeHead(400)
-          response.end("#{err.name}: #{err.message}")
-          return
-
-        db.virtualParametersCollection.save(object, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else if request.method == 'DELETE'
-        db.virtualParametersCollection.remove({'_id' : virtualParameterName}, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if TAGS_REGEX.test(urlParts.pathname)
-      r = TAGS_REGEX.exec(urlParts.pathname)
-      deviceId = querystring.unescape(r[1])
-      tag = querystring.unescape(r[2])
-      if request.method == 'POST'
-        db.devicesCollection.update({'_id' : deviceId}, {'$addToSet' : {'_tags' : tag}}, {safe: true}, (err) ->
-          return throwError(err, response) if err
-          response.writeHead(200)
-          response.end()
-        )
-      else if request.method == 'DELETE'
-        db.devicesCollection.update({'_id' : deviceId}, {'$pull' : {'_tags' : tag}}, {safe: true}, (err) ->
-          return throwError(err, response) if err
-          response.writeHead(200)
-          response.end()
-        )
-      else
-        response.writeHead 405, {'Allow': 'POST, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if FAULTS_REGEX.test(urlParts.pathname)
-      if request.method == 'DELETE'
-        faultId = querystring.unescape(FAULTS_REGEX.exec(urlParts.pathname)[1])
-        deviceId = faultId.split(':', 1)[0]
-        channel = faultId.slice(deviceId.length + 1)
-        db.faultsCollection.remove({_id : faultId}, (err) ->
-          return throwError(err, response) if err
-          if channel.startsWith('task_')
-            return db.tasksCollection.remove({_id : new mongodb.ObjectID(channel.slice(5))}, (err) ->
-              return throwError(err, response) if err
-              cache.del("#{deviceId}_tasks_faults_operations", (err) ->
-                return throwError(err, response) if err
-                response.writeHead(200)
-                response.end()
-              )
-            )
-
-          cache.del("#{deviceId}_tasks_faults_operations", (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'DELETE'}
-        response.end('405 Method Not Allowed')
-    else if DEVICE_TASKS_REGEX.test(urlParts.pathname)
-      if request.method == 'POST'
-        deviceId = querystring.unescape(DEVICE_TASKS_REGEX.exec(urlParts.pathname)[1])
-        if body.length
-          task = JSON.parse(body)
-          task.device = deviceId
-          apiFunctions.insertTasks(task, (err) ->
-            return throwError(err, response) if err
-            cache.del("#{deviceId}_tasks_faults_operations", (err) ->
-              return throwError(err, response) if err
-
-              if urlParts.query.connection_request?
-                apiFunctions.connectionRequest(deviceId, (err) ->
-                  if err
-                    response.writeHead(202, err.message, {'Content-Type' : 'application/json'})
-                    response.end(JSON.stringify(task))
-                  else
-                    apiFunctions.watchTask(deviceId, task._id, config.get('DEVICE_ONLINE_THRESHOLD', deviceId), (err, status) ->
-                      return throwError(err, response) if err
-
-                      if status is 'timeout'
-                        response.writeHead(202, 'Task queued but not processed', {'Content-Type' : 'application/json'})
-                        response.end(JSON.stringify(task))
-                      else if status is 'fault'
-                        db.tasksCollection.findOne({_id : task._id}, (err, task) ->
-                          return throwError(err, response) if err
-
-                          response.writeHead(202, 'Task faulted', {'Content-Type' : 'application/json'})
-                          response.end(JSON.stringify(task))
-                        )
-                      else
-                        response.writeHead(200, {'Content-Type' : 'application/json'})
-                        response.end(JSON.stringify(task))
-                    )
-                )
-              else
-                response.writeHead(202, {'Content-Type' : 'application/json'})
-                response.end(JSON.stringify(task))
-            )
-          )
-        else if urlParts.query.connection_request?
-          # no task, send connection request only
-          apiFunctions.connectionRequest(deviceId, (err) ->
-            if err
-              response.writeHead 504
-              response.end("#{err.name}: #{err.message}")
-              return
-            response.writeHead 200
-            response.end()
-          )
-        else
-          response.writeHead(400)
-          response.end()
-      else
-        response.writeHead 405, {'Allow': 'POST'}
-        response.end('405 Method Not Allowed')
-    else if TASKS_REGEX.test(urlParts.pathname)
-      r = TASKS_REGEX.exec(urlParts.pathname)
-      taskId = new mongodb.ObjectID(querystring.unescape(r[1]))
-      action = r[2]
-      if not action? or action is '/'
-        if request.method == 'DELETE'
-          db.tasksCollection.findOne({'_id' : taskId}, {'device' : 1}, (err, task) ->
-            return throwError(err, response) if err
-            if not task?
-              response.writeHead(404)
-              response.end("Task not found")
-              return
-
-            deviceId = task.device
-            db.tasksCollection.remove({'_id' : taskId}, (err) ->
-              return throwError(err, response) if err
-              db.faultsCollection.remove({_id : "#{deviceId}:task_#{String(taskId)}"}, (err) ->
-                return throwError(err, response) if err
-                cache.del("#{deviceId}_tasks_faults_operations", (err) ->
-                  return throwError(err, response) if err
-                  response.writeHead(200)
-                  response.end()
-                )
-              )
-            )
-          )
-        else
-          response.writeHead 405, {'Allow': 'PUT DELETE'}
-          response.end('405 Method Not Allowed')
-      else if action is '/retry'
-        if request.method == 'POST'
-          db.tasksCollection.findOne({'_id' : taskId}, {'device' : 1}, (err, task) ->
-            return throwError(err, response) if err
-            deviceId = task.device
-            db.faultsCollection.remove({_id : "#{deviceId}:task_#{String(taskId)}"}, (err) ->
-              return throwError(err, response) if err
-              cache.del("#{deviceId}_tasks_faults_operations", (err) ->
-                return throwError(err, response) if err
-                response.writeHead(200)
-                response.end()
-              )
-            )
-          )
-        else
-          response.writeHead 405, {'Allow': 'POST'}
-          response.end('405 Method Not Allowed')
-      else
-        response.writeHead(404)
-        response.end()
-    else if FILES_REGEX.test(urlParts.pathname)
-      filename = querystring.unescape(FILES_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        metadata = {
-          fileType : request.headers.filetype,
-          oui : request.headers.oui,
-          productClass : request.headers.productclass,
-          version : request.headers.version,
-        }
-
-        gs = new mongodb.GridStore(db.mongoDb, filename, filename, 'w', {metadata : metadata})
-        gs.open((err, gs) ->
-          gs.write(body, (err, res) ->
-            return throwError(err, response) if err
-            gs.close((err) ->
-              return throwError(err, response) if err
-              response.writeHead(201)
-              response.end()
-            )
-          )
-        )
-      else if request.method == 'DELETE'
-        mongodb.GridStore.unlink(db.mongoDb, filename, (err) ->
-          response.writeHead(200)
-          response.end()
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if PING_REGEX.test(urlParts.pathname)
-      host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1])
-      require('child_process').exec("ping -w 1 -i 0.2 -c 3 #{host}", (err, stdout, stderr) ->
-        if err
-          response.writeHead(404, {'Cache-Control' : 'no-cache'})
-          response.end("#{err.name}: #{err.message}")
-          return
-        response.writeHead(200, {'Content-Type' : 'text/plain', 'Cache-Control' : 'no-cache'})
-        response.end(stdout)
-      )
-    else if DELETE_DEVICE_REGEX.test(urlParts.pathname)
-      if request.method isnt 'DELETE'
-        response.writeHead(405, {'Allow' : 'DELETE'})
-        response.end('405 Method Not Allowed')
-        return
-
-      deviceId = querystring.unescape(DELETE_DEVICE_REGEX.exec(urlParts.pathname)[1])
-      apiFunctions.deleteDevice(deviceId, (err) ->
-        return throwError(err, response) if err
-        response.writeHead(200)
-        response.end()
-      )
-    else if QUERY_REGEX.test(urlParts.pathname)
-      collectionName = QUERY_REGEX.exec(urlParts.pathname)[1]
-
-      # Convert to camel case
-      i = collectionName.indexOf('_')
-      while i >= 0
-        ++ i
-        up = if i < collectionName.length then collectionName[i].toUpperCase() else ''
-        collectionName = collectionName.slice(0, i - 1) + up + collectionName.slice(i + 1)
-        i = collectionName.indexOf('_', i)
-
-      if request.method not in ['GET', 'HEAD']
-        response.writeHead 405, {'Allow' : 'GET, HEAD'}
-        response.end('405 Method Not Allowed')
-        return
-      collection = db["#{collectionName}Collection"]
-      if not collection?
-        response.writeHead 404
-        response.end('404 Not Found')
-        return
-
-      if urlParts.query.query?
-        try
-          q = JSON.parse(urlParts.query.query)
-        catch err
-          response.writeHead(400)
-          response.end("#{err.name}: #{err.message}")
-          return
-      else
-        q = {}
-
-      switch collectionName
-        when 'devices'
-          q = query.expand(q)
-        when 'tasks'
-          q = query.sanitizeQueryTypes(q, {
-            _id: ((v) -> return new mongodb.ObjectID(v))
-            timestamp: ((v) -> return new Date(v))
-            retries: Number
-          })
-        when 'faults'
-          q = query.sanitizeQueryTypes(q, {
-            timestamp: ((v) -> return new Date(v))
-            retries: Number
-          })
-
-      if urlParts.query.projection?
-        projection = {}
-        for p in urlParts.query.projection.split(',')
-          p = p.trim()
-          projection[p] = 1
-
-      cur = collection.find(q, projection, {batchSize : 50})
-      if urlParts.query.sort?
-        s = JSON.parse(urlParts.query.sort)
-        sort = {}
-        for k,v of s
-          if k[k.lastIndexOf('.') + 1] != '_' and collectionName is 'devices'
-            sort["#{k}._value"] = v
-          else
-            sort[k] = v
-        cur.sort(sort)
-
-      cur.skip(parseInt(urlParts.query.skip)) if urlParts.query.skip?
-      cur.limit(limit = parseInt(urlParts.query.limit)) if urlParts.query.limit?
-      cur.count(false, (err, total) ->
-        response.writeHead(200, {'Content-Type' : 'application/json', 'total' : total})
-        if request.method is 'HEAD'
-          response.end()
-          return
-        response.write("[\n")
-        i = 0
-        cur.each((err, item) ->
-          if err
-            throwError(err)
-            return false
-
-          if item?
-            response.write(",\n") if i++
-            response.write(JSON.stringify(item))
-
-          if not item? or (limit? and i >= limit)
-            response.end("\n]")
-        )
-      )
-
-    else
-      response.writeHead 404
-      response.end('404 Not Found')
-
-
-exports.listener = listener

lib/nbi.ts

@@ -0,0 +1,742 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as url from "url";
+import * as mongodb from "mongodb";
+import * as querystring from "querystring";
+import * as vm from "vm";
+import * as config from "./config";
+import * as db from "./db";
+import * as query from "./query";
+import * as apiFunctions from "./api-functions";
+import * as cache from "./cache";
+import { version as VERSION } from "../package.json";
+import { ping } from "./ping";
+
+const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
+const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
+const TAGS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tags\/([a-zA-Z0-9\-_%]+)\/?$/;
+const PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-_%]+)\/?$/;
+const OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-_%]+)\/?$/;
+const FILES_REGEX = /^\/files\/([a-zA-Z0-9%!*'();:@&=+$,?#[\]\-_.~]+)\/?$/;
+const PING_REGEX = /^\/ping\/([a-zA-Z0-9\-_.:]+)\/?$/;
+const QUERY_REGEX = /^\/([a-zA-Z0-9_]+)\/?$/;
+const DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/?$/;
+const PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-_%]+)\/?$/;
+const VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
+const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
+
+function throwError(err, httpResponse?): never {
+  if (httpResponse) {
+    httpResponse.writeHead(500, { Connection: "close" });
+    httpResponse.end(`${err.name}: ${err.message}`);
+  }
+  throw err;
+}
+
+export function listener(request, response): void {
+  const chunks = [];
+  let bytes = 0;
+  response.setHeader("GenieACS-Version", VERSION);
+
+  request.addListener("data", (chunk): void => {
+    chunks.push(chunk);
+    bytes += chunk.length;
+  });
+
+  function getBody(): Buffer {
+    // Write all chunks into a Buffer
+    const body = Buffer.allocUnsafe(bytes);
+    let offset = 0;
+    for (const chunk of chunks) {
+      chunk.copy(body, offset, 0, chunk.length);
+      offset += chunk.length;
+    }
+    return body;
+  }
+
+  request.addListener("end", () => {
+    const body = getBody();
+    const urlParts = url.parse(request.url, true);
+    if (PRESETS_REGEX.test(urlParts.pathname)) {
+      const presetName = querystring.unescape(
+        PRESETS_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const preset = JSON.parse(body.toString());
+        preset._id = presetName;
+        db.presetsCollection.replaceOne(
+          { _id: presetName },
+          preset,
+          { upsert: true },
+          err => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch(err => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else if (request.method === "DELETE") {
+        db.presetsCollection.deleteOne({ _id: presetName }, err => {
+          if (err) return void throwError(err, response);
+
+          cache
+            .del("presets_hash")
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch(err => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        });
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (OBJECTS_REGEX.test(urlParts.pathname)) {
+      const objectName = querystring.unescape(
+        OBJECTS_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const object = JSON.parse(body.toString());
+        object._id = objectName;
+        db.objectsCollection.replaceOne(
+          { _id: objectName },
+          object,
+          { upsert: true },
+          err => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch(err => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else if (request.method === "DELETE") {
+        db.objectsCollection.deleteOne({ _id: objectName }, err => {
+          if (err) return void throwError(err, response);
+
+          cache
+            .del("presets_hash")
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch(err => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        });
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (PROVISIONS_REGEX.test(urlParts.pathname)) {
+      const provisionName = querystring.unescape(
+        PROVISIONS_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const object = {
+          _id: provisionName,
+          script: body.toString()
+        };
+
+        try {
+          new vm.Script(`"use strict";(function(){\n${object.script}\n})();`);
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+
+        db.provisionsCollection.replaceOne(
+          { _id: provisionName },
+          object,
+          { upsert: true },
+          err => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch(err => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else if (request.method === "DELETE") {
+        db.provisionsCollection.deleteOne({ _id: provisionName }, err => {
+          if (err) return void throwError(err, response);
+
+          cache
+            .del("presets_hash")
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch(err => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        });
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (VIRTUAL_PARAMETERS_REGEX.test(urlParts.pathname)) {
+      const virtualParameterName = querystring.unescape(
+        VIRTUAL_PARAMETERS_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const object = {
+          _id: virtualParameterName,
+          script: body.toString()
+        };
+
+        try {
+          new vm.Script(`"use strict";(function(){\n${object.script}\n})();`);
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+
+        db.virtualParametersCollection.replaceOne(
+          { _id: virtualParameterName },
+          object,
+          { upsert: true },
+          err => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch(err => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else if (request.method === "DELETE") {
+        db.virtualParametersCollection.deleteOne(
+          { _id: virtualParameterName },
+          err => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch(err => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (TAGS_REGEX.test(urlParts.pathname)) {
+      const r = TAGS_REGEX.exec(urlParts.pathname);
+      const deviceId = querystring.unescape(r[1]);
+      const tag = querystring.unescape(r[2]);
+      if (request.method === "POST") {
+        db.devicesCollection.updateOne(
+          { _id: deviceId },
+          { $addToSet: { _tags: tag } },
+          err => {
+            if (err) return void throwError(err, response);
+            response.writeHead(200);
+            response.end();
+          }
+        );
+      } else if (request.method === "DELETE") {
+        db.devicesCollection.updateOne(
+          { _id: deviceId },
+          { $pull: { _tags: tag } },
+          err => {
+            if (err) return void throwError(err, response);
+
+            response.writeHead(200);
+            response.end();
+          }
+        );
+      } else {
+        response.writeHead(405, { Allow: "POST, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (FAULTS_REGEX.test(urlParts.pathname)) {
+      if (request.method === "DELETE") {
+        const faultId = querystring.unescape(
+          FAULTS_REGEX.exec(urlParts.pathname)[1]
+        );
+        const deviceId = faultId.split(":", 1)[0];
+        const channel = faultId.slice(deviceId.length + 1);
+        db.faultsCollection.deleteOne({ _id: faultId }, err => {
+          if (err) return void throwError(err, response);
+
+          if (channel.startsWith("task_")) {
+            const objId = new mongodb.ObjectID(channel.slice(5));
+            return void db.tasksCollection.deleteOne({ _id: objId }, err => {
+              if (err) return void throwError(err, response);
+
+              cache
+                .del(`${deviceId}_tasks_faults_operations`)
+                .then(() => {
+                  response.writeHead(200);
+                  response.end();
+                })
+                .catch(err => {
+                  setTimeout(() => {
+                    throwError(err, response);
+                  });
+                });
+            });
+          }
+
+          cache
+            .del(`${deviceId}_tasks_faults_operations`)
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch(err => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        });
+      } else {
+        response.writeHead(405, { Allow: "DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (DEVICE_TASKS_REGEX.test(urlParts.pathname)) {
+      if (request.method === "POST") {
+        const deviceId = querystring.unescape(
+          DEVICE_TASKS_REGEX.exec(urlParts.pathname)[1]
+        );
+        if (body.length) {
+          const task = JSON.parse(body.toString());
+          task.device = deviceId;
+          apiFunctions
+            .insertTasks(task)
+            .then(() => {
+              cache
+                .del(`${deviceId}_tasks_faults_operations`)
+                .then(() => {
+                  if (urlParts.query.connection_request != null) {
+                    apiFunctions
+                      .connectionRequest(deviceId)
+                      .then(() => {
+                        const taskTimeout =
+                          (urlParts.query.timeout &&
+                            parseInt(urlParts.query.timeout as string)) ||
+                          config.get("DEVICE_ONLINE_THRESHOLD", deviceId);
+
+                        apiFunctions
+                          .watchTask(deviceId, task._id, taskTimeout)
+                          .then(status => {
+                            if (status === "timeout") {
+                              response.writeHead(
+                                202,
+                                "Task queued but not processed",
+                                {
+                                  "Content-Type": "application/json"
+                                }
+                              );
+                              response.end(JSON.stringify(task));
+                            } else if (status === "fault") {
+                              db.tasksCollection.findOne(
+                                { _id: task._id },
+                                (err, task2) => {
+                                  if (err)
+                                    return void throwError(err, response);
+
+                                  response.writeHead(202, "Task faulted", {
+                                    "Content-Type": "application/json"
+                                  });
+                                  response.end(JSON.stringify(task2));
+                                }
+                              );
+                            } else {
+                              response.writeHead(200, {
+                                "Content-Type": "application/json"
+                              });
+                              response.end(JSON.stringify(task));
+                            }
+                          })
+                          .catch(err => {
+                            setTimeout(() => {
+                              throwError(err, response);
+                            });
+                          });
+                      })
+                      .catch(err => {
+                        response.writeHead(202, err.message, {
+                          "Content-Type": "application/json"
+                        });
+                        response.end(JSON.stringify(task));
+                      });
+                  } else {
+                    response.writeHead(202, {
+                      "Content-Type": "application/json"
+                    });
+                    response.end(JSON.stringify(task));
+                  }
+                })
+                .catch(err => {
+                  setTimeout(() => {
+                    throwError(err, response);
+                  });
+                });
+            })
+            .catch(err => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        } else if (urlParts.query.connection_request != null) {
+          // No task, send connection request only
+          apiFunctions
+            .connectionRequest(deviceId)
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch(err => {
+              response.writeHead(504);
+              response.end(`${err.name}: ${err.message}`);
+            });
+        } else {
+          response.writeHead(400);
+          response.end();
+        }
+      } else {
+        response.writeHead(405, { Allow: "POST" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (TASKS_REGEX.test(urlParts.pathname)) {
+      const r = TASKS_REGEX.exec(urlParts.pathname);
+      const taskId = querystring.unescape(r[1]);
+      const action = r[2];
+      if (!action || action === "/") {
+        if (request.method === "DELETE") {
+          db.tasksCollection.findOne(
+            { _id: new mongodb.ObjectID(taskId) },
+            { projection: { device: 1 } },
+            (err, task) => {
+              if (err) return void throwError(err, response);
+
+              if (!task) {
+                response.writeHead(404);
+                response.end("Task not found");
+                return;
+              }
+
+              const deviceId = task.device;
+              db.tasksCollection.deleteOne(
+                { _id: new mongodb.ObjectID(taskId) },
+                err => {
+                  if (err) return void throwError(err, response);
+
+                  db.faultsCollection.deleteOne(
+                    { _id: `${deviceId}:task_${taskId}` },
+                    err => {
+                      if (err) return void throwError(err, response);
+
+                      cache
+                        .del(`${deviceId}_tasks_faults_operations`)
+                        .then(() => {
+                          response.writeHead(200);
+                          response.end();
+                        })
+                        .catch(err => {
+                          setTimeout(() => {
+                            throwError(err, response);
+                          });
+                        });
+                    }
+                  );
+                }
+              );
+            }
+          );
+        } else {
+          response.writeHead(405, { Allow: "PUT DELETE" });
+          response.end("405 Method Not Allowed");
+        }
+      } else if (action === "/retry") {
+        if (request.method === "POST") {
+          db.tasksCollection.findOne(
+            { _id: new mongodb.ObjectID(taskId) },
+            { projection: { device: 1 } },
+            (err, task) => {
+              if (err) return void throwError(err, response);
+
+              const deviceId = task.device;
+              db.faultsCollection.deleteOne(
+                { _id: `${deviceId}:task_${taskId}` },
+                err => {
+                  if (err) return void throwError(err, response);
+
+                  cache
+                    .del(`${deviceId}_tasks_faults_operations`)
+                    .then(() => {
+                      response.writeHead(200);
+                      response.end();
+                    })
+                    .catch(err => {
+                      setTimeout(() => {
+                        throwError(err, response);
+                      });
+                    });
+                }
+              );
+            }
+          );
+        } else {
+          response.writeHead(405, { Allow: "POST" });
+          response.end("405 Method Not Allowed");
+        }
+      } else {
+        response.writeHead(404);
+        response.end();
+      }
+    } else if (FILES_REGEX.test(urlParts.pathname)) {
+      const filename = querystring.unescape(
+        FILES_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const metadata = {
+          fileType: request.headers.filetype,
+          oui: request.headers.oui,
+          productClass: request.headers.productclass,
+          version: request.headers.version
+        };
+        const bucket = new mongodb.GridFSBucket(db.client.db());
+        const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
+          metadata: metadata
+        });
+
+        uploadStream.on("error", err => {
+          throwError(err, response);
+        });
+
+        uploadStream.end(body, () => {
+          response.writeHead(201);
+          response.end();
+        });
+      } else if (request.method === "DELETE") {
+        const bucket = new mongodb.GridFSBucket(db.client.db());
+        bucket.delete((filename as unknown) as mongodb.ObjectId, err => {
+          if (err) return void throwError(err, response);
+
+          response.writeHead(200);
+          response.end();
+        });
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (PING_REGEX.test(urlParts.pathname)) {
+      const host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1]);
+      ping(host, (err, res, stdout) => {
+        if (err) {
+          if (!res) throwError(err, response);
+          response.writeHead(404, { "Cache-Control": "no-cache" });
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+
+        response.writeHead(200, {
+          "Content-Type": "text/plain",
+          "Cache-Control": "no-cache"
+        });
+        response.end(stdout);
+      });
+    } else if (DELETE_DEVICE_REGEX.test(urlParts.pathname)) {
+      if (request.method !== "DELETE") {
+        response.writeHead(405, { Allow: "DELETE" });
+        response.end("405 Method Not Allowed");
+        return;
+      }
+
+      const deviceId = querystring.unescape(
+        DELETE_DEVICE_REGEX.exec(urlParts.pathname)[1]
+      );
+      apiFunctions
+        .deleteDevice(deviceId)
+        .then(() => {
+          response.writeHead(200);
+          response.end();
+        })
+        .catch(err => {
+          setTimeout(() => {
+            throwError(err, response);
+          });
+        });
+    } else if (QUERY_REGEX.test(urlParts.pathname)) {
+      let collectionName = QUERY_REGEX.exec(urlParts.pathname)[1];
+
+      // Convert to camel case
+      let i = collectionName.indexOf("_");
+      while (i++ >= 0) {
+        const up =
+          i < collectionName.length ? collectionName[i].toUpperCase() : "";
+        collectionName =
+          collectionName.slice(0, i - 1) + up + collectionName.slice(i + 1);
+        i = collectionName.indexOf("_", i);
+      }
+
+      if (request.method !== "GET" && request.method !== "HEAD") {
+        response.writeHead(405, { Allow: "GET, HEAD" });
+        response.end("405 Method Not Allowed");
+        return;
+      }
+
+      const collection = db[`${collectionName}Collection`];
+      if (!collection) {
+        response.writeHead(404);
+        response.end("404 Not Found");
+        return;
+      }
+
+      let q = {};
+      if (urlParts.query.query) {
+        try {
+          q = JSON.parse(urlParts.query.query as string);
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+      }
+
+      switch (collectionName) {
+        case "devices":
+          q = query.expand(q);
+          break;
+        case "tasks":
+          q = query.sanitizeQueryTypes(q, {
+            _id: v => new mongodb.ObjectID(v),
+            timestamp: v => new Date(v),
+            retries: Number
+          });
+          break;
+        case "faults":
+          q = query.sanitizeQueryTypes(q, {
+            timestamp: v => new Date(v),
+            retries: Number
+          });
+      }
+
+      let projection = null;
+      if (urlParts.query.projection) {
+        projection = {};
+        for (const p of (urlParts.query.projection as string).split(","))
+          projection[p.trim()] = 1;
+      }
+
+      const cur = collection.find(q, { projection: projection });
+
+      if (urlParts.query.sort) {
+        const s = JSON.parse(urlParts.query.sort as string);
+        const sort = {};
+        for (const [k, v] of Object.entries(s)) {
+          if (k[k.lastIndexOf(".") + 1] !== "_" && collectionName === "devices")
+            sort[`${k}._value`] = v;
+          else sort[k] = v;
+        }
+
+        cur.sort(sort);
+      }
+
+      if (urlParts.query.skip)
+        cur.skip(parseInt(urlParts.query.skip as string));
+
+      if (urlParts.query.limit)
+        cur.limit(parseInt(urlParts.query.limit as string));
+
+      cur.count(false, (err, total) => {
+        if (err) return void throwError(err);
+
+        response.writeHead(200, {
+          "Content-Type": "application/json",
+          total: total
+        });
+
+        if (request.method === "HEAD") {
+          response.end();
+          return;
+        }
+
+        response.write("[\n");
+        i = 0;
+        cur.forEach(
+          item => {
+            if (i++) response.write(",\n");
+            response.write(JSON.stringify(item));
+          },
+          err => {
+            if (err) return void throwError(err);
+            response.end("\n]");
+          }
+        );
+      });
+    } else {
+      response.writeHead(404);
+      response.end("404 Not Found");
+    }
+  });
+}

lib/path-set.js

@@ -1,179 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-
-class PathSet {
-
-  constructor() {
-    this.lengthIndex = [];
-    this.fragmentIndex = [];
-  }
-
-
-  get depth() {
-    return this.lengthIndex.length;
-  }
-
-
-  add(path) {
-    let p = this.get(path);
-
-    if (p)
-      return p;
-
-    if (!this.lengthIndex[path.length])
-      this.lengthIndex[path.length] = new Set();
-
-    this.lengthIndex[path.length].add(path);
-
-    let wildcard = 0;
-    for (let i = 0; i < path.length; ++ i) {
-      let fragment = path[i];
-      if (fragment === "*")
-        wildcard = wildcard | (1 << i);
-
-      if (!this.fragmentIndex[i])
-        this.fragmentIndex[i] = {};
-
-      if (!this.fragmentIndex[i][fragment])
-        this.fragmentIndex[i][fragment] = new Set();
-
-      this.fragmentIndex[i][fragment].add(path);
-    }
-
-    if (path.wildcard == null) {
-      path.alias = 0;
-      path.wildcard = wildcard;
-      Object.freeze(path);
-    }
-    return path;
-  }
-
-
-  get(pattern) {
-    if (!this.lengthIndex[pattern.length])
-      return null;
-
-    if (this.lengthIndex[pattern.length].has(pattern))
-      return pattern;
-
-    let sets = [];
-
-    sets.push(this.lengthIndex[pattern.length]);
-
-    for (let i = 0; i < pattern.length; ++ i) {
-      if (!this.fragmentIndex[i])
-        return null;
-
-      let fragment = pattern[i];
-
-      if (!this.fragmentIndex[i][fragment])
-        return null;
-
-      sets.push(this.fragmentIndex[i][fragment]);
-    }
-
-    sets.sort(function(a, b) {
-      return a.size - b.size;
-    });
-
-    let smallestSet = sets.shift();
-    for (let path of smallestSet) {
-      for (let s of sets) {
-        if (!s.has(path)) {
-          path = null;
-          break
-        }
-      }
-      if (path)
-        return path;
-    }
-
-    return null;
-  }
-
-
-  find(pattern, superset = false, subset = false, depth = pattern.length) {
-    let res = [];
-    let groups = [[]];
-
-    for (let i = pattern.length; i <= depth; ++ i)
-      if (this.lengthIndex[i])
-        groups[0].push(this.lengthIndex[i]);
-
-    if (!groups[0].length)
-      return res;
-
-    for (let i = 0; i < pattern.length; ++ i) {
-      if (!this.fragmentIndex[i])
-        return res;
-
-      let fragment = pattern[i];
-
-      if (fragment === "*" && subset)
-        continue;
-
-      let g = [];
-
-      if (this.fragmentIndex[i][fragment])
-        g.push(this.fragmentIndex[i][fragment]);
-
-      if (fragment !== "*" && superset && this.fragmentIndex[i]["*"])
-        g.push(this.fragmentIndex[i]["*"]);
-
-      if (!g.length)
-        return res;
-
-      groups.push(g);
-    }
-
-    groups.sort(function(a, b) {
-      return a.reduce(function(prev, cur) {return prev + cur.size;}, 0) -
-        b.reduce(function(prev, cur) {return prev + cur.size;}, 0);
-    });
-
-    let smallestGroup = groups.shift();
-    for (let set of smallestGroup) {
-      for (let path of set) {
-        for (let group of groups) {
-          let found = false;
-          for (let s of group) {
-            if (s.has(path)) {
-              found = true;
-              break;
-            }
-          }
-          if (!found) {
-            path = null;
-            break;
-          }
-        }
-        if (path)
-          res.push(path);
-      }
-    }
-
-    return res;
-  }
-
-}
-
-
-module.exports = PathSet;

lib/ping.ts

@@ -0,0 +1,89 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { platform } from "os";
+import { exec } from "child_process";
+
+interface Ping {
+  packetsTransmitted: number;
+  packetsReceived: number;
+  packetLoss: number;
+  min: number;
+  avg: number;
+  max: number;
+  mdev: number;
+}
+
+export function ping(
+  host: string,
+  callback: (err, res?, stdout?) => void
+): void {
+  let cmd: string, parseRegExp1: RegExp, parseRegExp2: RegExp;
+  switch (platform()) {
+    case "linux":
+      cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
+      parseRegExp1 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
+      parseRegExp2 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss/;
+      break;
+
+    case "freebsd":
+      // Send a single packet because on FreeBSD only superuser can send
+      // packets that are only 200 ms apart.
+      cmd = `ping -t 1 -c 3 ${host}`;
+      parseRegExp1 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
+      parseRegExp2 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
+      break;
+
+    default:
+      return callback(new Error("Platform not supported"));
+  }
+
+  exec(cmd, (err, stdout) => {
+    let parsed: Ping;
+    if (stdout) {
+      const m1 = stdout.match(parseRegExp1);
+      if (m1) {
+        parsed = {
+          packetsTransmitted: +m1[1],
+          packetsReceived: +m1[2],
+          packetLoss: +m1[3],
+          min: +m1[4],
+          avg: +m1[5],
+          max: +m1[6],
+          mdev: +m1[7]
+        };
+      } else {
+        const m2 = stdout.match(parseRegExp2);
+        if (m2) {
+          parsed = {
+            packetsTransmitted: +m2[1],
+            packetsReceived: +m2[2],
+            packetLoss: +m2[3],
+            min: null,
+            avg: null,
+            max: null,
+            mdev: null
+          };
+        }
+      }
+    }
+
+    callback(err, parsed, stdout);
+  });
+}

lib/query.coffee

@@ -1,372 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This file incorporates work covered by the following copyright and
-# permission notice:
-#
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-common = require './common'
-mongodb = require 'mongodb'
-
-
-stringToRegexp = (input, flags) ->
-  if (input.indexOf('*') == -1)
-    return false
-
-  output = input.replace(/[\[\]\\\^\$\.\|\?\+\(\)]/, "\\$&")
-  if output[0] == '*'
-    output = output.replace(/^\*+/g, '')
-  else
-    output = '^' + output
-
-  if output[output.length - 1] == '*'
-    output = output.replace(/\*+$/g, '')
-  else
-    output = output + '$'
-
-  output = output.replace(/[\*]/, '.*')
-
-  return new RegExp(output, flags)
-
-
-normalize = (input) ->
-  if common.typeOf(input) is common.STRING_TYPE
-    vals = [input]
-    if (m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input))
-      vals.push({'$regex' : new RegExp(m[1], m[2])})
-
-    if +input == parseFloat(input)
-      vals.push(+input)
-
-    d = new Date(input)
-    if input.length >= 8 and d.getFullYear() > 1983
-      vals.push(d)
-
-    r = stringToRegexp(input)
-    if r isnt false
-      vals.push({'$regex' : r})
-
-    return vals
-  return input
-
-
-expandValue = (value) ->
-  if common.typeOf(value) is common.ARRAY_TYPE
-    a = []
-    for j in value
-      a = a.concat(expandValue(j))
-    return [a]
-  else if common.typeOf(value) isnt common.OBJECT_TYPE
-    n = normalize(value)
-    if common.typeOf(n) isnt common.ARRAY_TYPE
-      return [n]
-    else
-      return n
-
-  objs = []
-  indices = []
-  keys = []
-  values = []
-  for k,v of value
-    keys.push(k)
-    values.push(expandValue(v))
-    indices.push(0)
-
-  i = 0
-  while i < indices.length
-    obj = {}
-    for i in [0...keys.length]
-      obj[keys[i]] = values[i][indices[i]]
-    objs.push(obj)
-
-    for i in [0...indices.length]
-      indices[i] += 1
-      if indices[i] < values[i].length
-        break
-      indices[i] = 0
-  return objs
-
-
-permute = (param, val) ->
-  conditions = []
-
-  values = expandValue(val)
-  if param[param.lastIndexOf('.') + 1] != '_'
-    param += '._value'
-
-  for v in values
-    obj = {}
-    obj[param] = v
-    conditions.push(obj)
-
-  return conditions
-
-
-expand = (query) ->
-  new_query = {}
-  for k,v of query
-    if k[0] == '$' # operator
-      expressions = []
-      for e in v
-        expressions.push(expand(e))
-      new_query[k] = expressions
-    else
-      conditions = permute(k, v)
-      if conditions.length > 1
-        new_query['$and'] ?= []
-        if v?['$ne']? or v?['$not']?
-          if Object.keys(v) > 1
-            throw new Error("Cannot mix $ne or $not with other operators")
-          for c in conditions
-            new_query['$and'].push(c)
-        else
-          new_query['$and'].push({'$or' : conditions})
-      else
-        Object.assign(new_query, conditions[0])
-
-  return new_query
-
-
-# Generate parameter projection from a query
-# If second arg is given, it's edited and returned
-queryProjection = (query, proj) ->
-  proj ?= {}
-  for k,v of query
-    if k.charAt(0) == '$' # this is a logical operator
-      for q in v
-        queryProjection(q, proj)
-    else
-      proj[k] = 1
-  return proj
-
-
-testExpressions = (params, expressions, lop) ->
-  for f in expressions
-    res = test(params, f)
-
-    switch lop
-      when '$and'
-        return false if not res
-      when '$or'
-        return true if res
-      when '$nor'
-        return false if res
-
-  return switch lop
-    when '$and' then true
-    when '$or' then false
-    when '$nor' then true
-    else throw new Error('Unknown logical operator')
-
-
-test = (params, query) ->
-  for k,v of query
-    if k.charAt(0) == '$' # this is a logical operator
-      res = testExpressions(params, v, k)
-    else
-      value = params[k]
-
-      if common.typeOf(v) isnt common.OBJECT_TYPE
-        # TODO comparing array to regex, array to array, and object to object
-        if common.typeOf(value) is common.ARRAY_TYPE
-          res = value.indexOf(v) != -1
-        else
-          if common.typeOf(v) is common.REGEXP_TYPE
-            res = v.test(value)
-          else
-            res = v == value
-      else
-        for k2,v2 of v
-          switch k2
-            when '$ne'
-              if common.typeOf(value) is common.ARRAY_TYPE
-                res = value.indexOf(v2) == -1
-              else
-                res = value != v2
-            when '$lt'
-              res = value < v2
-            when '$lte'
-              res = value <= v2
-            when '$gt'
-              res = value > v2
-            when '$gte'
-              res = value >= v2
-            when '$regex'
-              res = v2.test(value)
-            when '$in'
-              throw new Error('Operator not supported')
-            when '$nin'
-              throw new Error('Operator not supported')
-            when '$all'
-              throw new Error('Operator not supported')
-            when '$exists'
-              throw new Error('Operator not supported')
-            else
-              throw new Error('Operator not supported')
-
-    if not res
-      return false
-
-  return true
-
-
-matchType = (src, dst) ->
-  if typeof src is 'string'
-    return String(dst)
-  else if typeof src is 'number'
-    if +dst == parseFloat(dst)
-      return +dst
-    else if not isNaN(Date.parse(dst))
-      return Date.parse(dst)
-  else if typeof src is 'boolean'
-    v = String(dst).trim()
-    if v is 'true' or v is 'TRUE' or v is 'True' or v is '1'
-      return true
-    else if v is 'false' or v is 'FALSE' or v is 'False' or v is '0'
-      return false
-
-  return dst
-
-
-testFilter = (obj, filter) ->
-  for k, v of filter
-    [param, op] = k.split(/([^a-zA-Z0-9\-\_\.].*)/, 2)
-    val = matchType(obj[param], v)
-    switch op
-      when '=', undefined
-        return false if obj[param] != val
-      when '>'
-        return false if not (obj[param] > val)
-      when '>='
-        return false if not (obj[param] >= val)
-      when '<'
-        return false if not (obj[param] < val)
-      when '<='
-        return false if not (obj[param] <= val)
-      when '!'
-        return false if obj[param] == val
-      else
-        throw new Error("Unrecognized operator #{op}")
-
-  return true
-
-
-convertMongoQueryToFilters = (query, filters) ->
-  filters ?= {}
-  for k, v of query
-    if k[0] == '$'
-      if k == '$and'
-        for vv in v
-          convertMongoQueryToFilters(vv, filters)
-      else
-        throw new Error("Operator #{k} not supported")
-    else if k == '_tags'
-      if common.typeOf(v) is common.OBJECT_TYPE
-        for kk, vv of v
-          vv = vv.replace(/[^a-zA-Z0-9\-]+/g, '_')
-          switch kk
-            when '$ne'
-              filters["Tags.#{vv}!"] = true
-            else
-              throw new Error("Operator #{kk} not supported")
-      else
-        v = v.replace(/[^a-zA-Z0-9\-]+/g, '_')
-        filters["Tags.#{v}"] = true
-    else
-      switch k
-        when '_id' then k = 'DeviceID.ID'
-        when '_deviceId._Manufacturer' then k = 'DeviceID.Manufacturer'
-        when '_deviceId._OUI' then k = 'DeviceID.OUI'
-        when '_deviceId._ProductClass' then k = 'DeviceID.ProductClass'
-        when '_deviceId._SerialNumber' then k = 'DeviceID.SerialNumber'
-        when '_lastInform' then k = 'Events.Inform'
-        when '_lastBootstrap' then k = 'Events.0_BOOTSTRAP'
-        when '_lastBoot' then k = 'Events.1_BOOT'
-        when '_registered' then k = 'Events.Registered'
-
-      if common.typeOf(v) is common.OBJECT_TYPE
-        for kk, vv of v
-          switch kk
-            when '$eq'
-              filters[k] = vv
-            when '$ne'
-              filters["#{k}!"] = vv
-            when '$lt'
-              filters["#{k}<"] = vv
-            when '$lte'
-              filters["#{k}<="] = vv
-            when '$gt'
-              filters["#{k}>"] = vv
-            when '$gte'
-              filters["#{k}>="] = vv
-            else
-              throw new Error("Oprator #{kk} not supported")
-      else
-        filters[k] = v
-
-  return filters
-
-
-sanitizeQueryTypes = (query, types) ->
-  for k, v of query
-    if k[0] == '$' # logical operator
-      sanitizeQueryTypes(vv, types) for vv in v
-    else if k of types
-      if common.typeOf(v) is common.OBJECT_TYPE
-        for kk, vv of v
-          switch kk # operator
-            when '$in', '$nin'
-              vv[i] = types[k](vv[i]) for i in [0...vv.length]
-            when '$eq', '$gt', '$gte', '$lt', '$lte', '$ne'
-              v[kk] = types[k](vv)
-            when '$exists', '$type' then
-              # ignore
-            else
-              throw new Error('Operator not supported')
-      else
-        query[k] = types[k](query[k])
-
-  return query
-
-
-exports.expand = expand
-exports.queryProjection = queryProjection
-exports.test = test
-exports.sanitizeQueryTypes = sanitizeQueryTypes
-exports.convertMongoQueryToFilters = convertMongoQueryToFilters
-exports.testFilter = testFilter

lib/query.ts

@@ -0,0 +1,255 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as common from "./common";
+
+function stringToRegexp(input, flags?): RegExp | false {
+  if (input.indexOf("*") === -1) return false;
+
+  let output = input.replace(/[[\]\\^$.|?+()]/, "\\$&");
+  if (output[0] === "*") output = output.replace(/^\*+/g, "");
+  else output = "^" + output;
+
+  if (output[output.length - 1] === "*") output = output.replace(/\*+$/g, "");
+  else output = output + "$";
+
+  output = output.replace(/[*]/, ".*");
+  return new RegExp(output, flags);
+}
+
+function normalize(input): any {
+  if (common.typeOf(input) === common.STRING_TYPE) {
+    const vals = [input];
+    const m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input);
+    if (m) vals.push({ $regex: new RegExp(m[1], m[2]) });
+
+    if (+input === parseFloat(input)) vals.push(+input);
+
+    const d = new Date(input);
+    if (input.length >= 8 && d.getFullYear() > 1983) vals.push(d);
+
+    const r = stringToRegexp(input);
+    if (r !== false) vals.push({ $regex: r });
+
+    return vals;
+  }
+  return input;
+}
+
+export function expandValue(value): any[] {
+  if (common.typeOf(value) === common.ARRAY_TYPE) {
+    let a = [];
+    for (const j of value) a = a.concat(expandValue(j));
+    return [a];
+  } else if (common.typeOf(value) !== common.OBJECT_TYPE) {
+    const n = normalize(value);
+    if (common.typeOf(n) !== common.ARRAY_TYPE) return [n];
+    else return n;
+  }
+
+  const objs = [];
+  const indices = [];
+  const keys = [];
+  const values = [];
+  for (const [k, v] of Object.entries(value)) {
+    keys.push(k);
+    values.push(expandValue(v));
+    indices.push(0);
+  }
+
+  let i = 0;
+  while (i < indices.length) {
+    const obj = {};
+    for (let j = 0; j < keys.length; ++j) obj[keys[j]] = values[j][indices[j]];
+    objs.push(obj);
+
+    for (i = 0; i < indices.length; ++i) {
+      indices[i] += 1;
+      if (indices[i] < values[i].length) break;
+      indices[i] = 0;
+    }
+  }
+  return objs;
+}
+
+function permute(param, val): any[] {
+  const conditions = [];
+  const values = expandValue(val);
+
+  if (param[param.lastIndexOf(".") + 1] !== "_") param += "._value";
+
+  for (const v of values) {
+    const obj = {};
+    obj[param] = v;
+    conditions.push(obj);
+  }
+
+  return conditions;
+}
+
+export function expand(query): {} {
+  const newQuery = {};
+  for (const [k, v] of Object.entries(query)) {
+    if (k[0] === "$") {
+      // Operator
+      newQuery[k] = (v as any[]).map(e => expand(e));
+    } else {
+      const conditions = permute(k, v);
+      if (conditions.length > 1) {
+        newQuery["$and"] = newQuery["$and"] || [];
+        if (v && (v["$ne"] != null || v["$not"] != null)) {
+          if (Object.keys(v).length > 1)
+            throw new Error("Cannot mix $ne or $not with other operators");
+          for (const c of conditions) newQuery["$an"].push(c);
+        } else {
+          newQuery["$and"].push({ $or: conditions });
+        }
+      } else {
+        Object.assign(newQuery, conditions[0]);
+      }
+    }
+  }
+
+  return newQuery;
+}
+
+function testExpressions(params, expressions, lop): boolean {
+  for (const f of expressions) {
+    const res = test(params, f);
+    switch (lop) {
+      case "$and":
+        if (!res) return false;
+        break;
+      case "$or":
+        if (res) return true;
+        break;
+      case "$nor":
+        if (res) return false;
+    }
+  }
+
+  switch (lop) {
+    case "$and":
+      return true;
+    case "$or":
+      return false;
+    case "$nor":
+      return true;
+    default:
+      throw new Error("Unknown logical operator");
+  }
+}
+
+export function test(params, query): boolean {
+  let res;
+  for (const [k, v] of Object.entries(query)) {
+    if (k.charAt(0) === "$") {
+      // Logical operator
+      res = testExpressions(params, v, k);
+    } else {
+      const value = params[k];
+      if (common.typeOf(v) !== common.OBJECT_TYPE) {
+        if (common.typeOf(value) === common.ARRAY_TYPE) {
+          // TODO comparing array to regex, array to array, and object to object
+          res = value.indexOf(v) !== -1;
+        } else {
+          if (common.typeOf(v) === common.REGEXP_TYPE)
+            res = (v as RegExp).test(value);
+          else res = v === value;
+        }
+      } else {
+        for (const [k2, v2] of Object.entries(v)) {
+          switch (k2) {
+            case "$ne":
+              if (common.typeOf(value) === common.ARRAY_TYPE)
+                res = value.indexOf(v2) === -1;
+              else res = value !== v2;
+              break;
+            case "$lt":
+              res = value < v2;
+              break;
+            case "$lte":
+              res = value <= v2;
+              break;
+            case "$gt":
+              res = value > v2;
+              break;
+            case "$gte":
+              res = value >= v2;
+              break;
+            case "$regex":
+              res = v2.test(value);
+              break;
+            case "$in":
+              throw new Error("Operator not supported");
+            case "$nin":
+              throw new Error("Operator not supported");
+            case "$all":
+              throw new Error("Operator not supported");
+            case "$exists":
+              throw new Error("Operator not supported");
+            default:
+              throw new Error("Operator not supported");
+          }
+        }
+      }
+    }
+
+    if (!res) return false;
+  }
+  return true;
+}
+
+export function sanitizeQueryTypes(query, types): {} {
+  for (const [k, v] of Object.entries(query)) {
+    if (k[0] === "$") {
+      // Logical operator
+      for (const vv of v as any[]) sanitizeQueryTypes(vv, types);
+    } else if (k in types) {
+      if (common.typeOf(v) === common.OBJECT_TYPE) {
+        for (const [kk, vv] of Object.entries(v)) {
+          switch (kk) {
+            case "$in":
+            case "$nin":
+              for (let i = 0; i < vv.length; ++i) vv[i] = types[k](vv[i]);
+              break;
+            case "$eq":
+            case "$gt":
+            case "$gte":
+            case "$lt":
+            case "$lte":
+            case "$ne":
+              v[kk] = types[k](vv);
+              break;
+            case "$exists":
+            case "$type":
+              // Ignore
+              break;
+            default:
+              throw new Error("Operator not supported");
+          }
+        }
+      } else {
+        query[k] = types[k](query[k]);
+      }
+    }
+  }
+
+  return query;
+}

lib/sandbox.js

@@ -1,388 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const vm = require("vm");
-
-const seedrandom = require("seedrandom");
-
-const common = require("./common");
-const device = require("./device");
-const extensions = require("./extensions");
-const logger = require("./logger");
-const scheduling = require("./scheduling");
-
-
-// Used for throwing to exit user script and commit
-const COMMIT = Symbol();
-
-// Used to execute extensions and restart
-const EXT = Symbol();
-
-const context = vm.createContext();
-
-let state;
-
-const runningExtensions = new WeakMap();
-function runExtension(sessionContext, key, extCall, callback) {
-  let re = runningExtensions.get(sessionContext);
-  if (!re) {
-    re = {};
-    runningExtensions.set(sessionContext, re);
-  }
-
-  if (re[key]) {
-    re[key].push(callback);
-  }
-  else {
-    re[key] = [callback];
-    extensions.run(extCall, function(err, fault, res) {
-      const callbacks = re[key];
-      delete re[key];
-      if (!err && !fault)
-        sessionContext.extensionsCache[key] = res;
-      for (let c of callbacks)
-        c(err, fault);
-    });
-  }
-}
-
-class SandboxDate {
-  constructor() {
-    if (arguments.length)
-      return new Date(...arguments);
-
-    return new Date(state.sessionContext.timestamp);
-  }
-
-  static now(intervalOrCron, variance) {
-    let t = state.sessionContext.timestamp;
-
-    if (typeof intervalOrCron === "number") {
-
-      if (variance == null)
-        variance = intervalOrCron;
-
-      let offset = 0;
-      if (variance)
-        offset = scheduling.variance(state.sessionContext.deviceId, variance);
-
-      t = scheduling.interval(t, intervalOrCron, offset);
-    }
-    else if (typeof intervalOrCron === "string") {
-      let offset = 0;
-      if (variance)
-        offset = scheduling.variance(state.sessionContext.deviceId, variance);
-      let cron = scheduling.parseCron(intervalOrCron);
-      t = scheduling.cron(t, cron, offset)[0];
-    }
-    else if (intervalOrCron) {
-      throw new Error("Invalid Date.now() argument");
-    }
-
-    return t;
-  }
-};
-
-function random() {
-  if (!state.rng)
-    state.rng = seedrandom(state.sessionContext.deviceId);
-
-  return state.rng();
-}
-
-random.seed = function(s) {
-  state.rng = seedrandom(s);
-}
-
-
-class ParameterWrapper {
-  constructor(path, attributes, unpacked, unpackedRevision) {
-    for (let attrName of attributes) {
-      Object.defineProperty(this, attrName, {get: function() {
-        if (state.uncommitted)
-          commit();
-
-        if (state.revision !== unpackedRevision) {
-          unpackedRevision = state.revision;
-          unpacked = device.unpack(state.sessionContext.deviceData, path, state.revision);
-        }
-
-        if (!unpacked.length)
-          return undefined;
-
-        let attr = state.sessionContext.deviceData.attributes.get(unpacked[0], state.revision)[attrName];
-
-        if (!attr)
-          return undefined;
-
-        return attr[1];
-      }});
-    }
-
-    Object.defineProperty(this, "path", {get: function() {
-      if (state.uncommitted)
-        commit();
-
-      if (state.revision !== unpackedRevision) {
-        unpackedRevision = state.revision;
-        unpacked = device.unpack(state.sessionContext.deviceData, path, state.revision);
-      }
-
-      if (!unpacked.length)
-        return undefined;
-
-      return unpacked[0].join(".");
-    }});
-
-    Object.defineProperty(this, "size", {get: function() {
-      if (state.uncommitted)
-        commit();
-
-      if (state.revision !== unpackedRevision) {
-        unpackedRevision = state.revision;
-        unpacked = device.unpack(state.sessionContext.deviceData, path, state.revision);
-      }
-
-      if (!unpacked.length)
-        return undefined;
-
-      return unpacked.length;
-    }});
-
-    this[Symbol.iterator] = function*() {
-      if (state.uncommitted)
-        commit();
-
-      if (state.revision !== unpackedRevision) {
-        unpackedRevision = state.revision;
-        unpacked = device.unpack(state.sessionContext.deviceData, path, state.revision);
-      }
-
-      for (let p of unpacked)
-        yield new ParameterWrapper(p, attributes, [p], state.revision);
-    }
-  }
-}
-
-
-function declare(path, timestamps, values) {
-  state.uncommitted = true;
-  if (!timestamps)
-    timestamps = {};
-
-  if (!values)
-    values = {};
-
-  let parsedPath = common.parsePath(path);
-
-  let declaration = [parsedPath, 1];
-  let attrs = {};
-
-  for (let attrName in values) {
-    if (!timestamps[attrName])
-      timestamps[attrName] = 1;
-
-    if (attrName === "path")
-      declaration[3] = values.path;
-    else {
-      attrs[attrName] = 1;
-      declaration[4] = declaration[4] || {};
-      if (attrName === "value" && !Array.isArray(values.value))
-        declaration[4].value = [values.value];
-      else
-        declaration[4][attrName] = values[attrName];
-    }
-  }
-
-  for (let attrName in timestamps) {
-    if (attrName === "path")
-      declaration[1] = timestamps.path;
-    else {
-      attrs[attrName] = 1;
-      declaration[2] = declaration[2] || {};
-      declaration[2][attrName] = timestamps[attrName];
-    }
-  }
-
-  state.declarations.push(declaration);
-
-  return new ParameterWrapper(parsedPath, Object.keys(attrs));
-}
-
-
-function clear(path, timestamp, attributes) {
-  state.uncommitted = true;
-
-  if (state.revision === state.maxRevision)
-    state.clear.push([common.parsePath(path), timestamp, attributes]);
-}
-
-
-function commit() {
-  ++ state.revision;
-  state.uncommitted = false;
-
-  if (state.revision === state.maxRevision + 1)
-    throw COMMIT;
-  else if (state.revision > state.maxRevision + 1)
-    throw new Error("Declare function should not be called from within a try/catch block");
-}
-
-
-function ext() {
-  ++ state.extCounter;
-  let extCall = Array.from(arguments).map(String);
-  let key = `${state.revision}: ${JSON.stringify(extCall)}`;
-
-  if (key in state.sessionContext.extensionsCache)
-    return state.sessionContext.extensionsCache[key];
-
-  state.extensions[key] = extCall;
-  throw EXT;
-}
-
-
-function log(msg, meta) {
-  if (state.revision === state.maxRevision && state.extCounter >= 0) {
-    const details = Object.assign({}, meta, {
-      sessionContext: state.sessionContext,
-      message: `Script: ${msg}`
-    });
-
-    delete details.hostname;
-    delete details.pid;
-    delete details.name;
-    delete details.version;
-    delete details.deviceId;
-    delete details.remoteAddress;
-
-    logger.accessInfo(details);
-  }
-}
-
-
-Object.defineProperty(context, "Date", {value: SandboxDate});
-Object.defineProperty(context, "declare", {value: declare});
-Object.defineProperty(context, "clear", {value: clear});
-Object.defineProperty(context, "commit", {value: commit});
-Object.defineProperty(context, "ext", {value: ext});
-Object.defineProperty(context, "log", {value: log});
-
-// Monkey-patch Math.random() to make it deterministic
-context.random = random;
-vm.runInContext("Math.random = random;", context);
-delete context.random;
-
-
-function errorToFault(err) {
-  if (!err)
-    return null;
-
-  if (!err.name)
-    return {code: "script", message: `${err}`};
-
-  let fault = {
-    code: `script.${err.name}`,
-    message: err.message,
-    detail: {
-      name: err.name,
-      message: err.message
-    }
-  };
-
-  if (err.stack) {
-    fault.detail.stack = err.stack;
-    // Trim the stack trace at the self-executing anonymous wrapper function
-    let stackTrimIndex = fault.detail.stack.match(/\s+at\s[^\s]+\s+at\s[^\s]+\s\(vm\.js.+\)/);
-    if (stackTrimIndex)
-      fault.detail.stack = fault.detail.stack.slice(0, stackTrimIndex.index);
-  }
-
-  return fault;
-}
-
-
-function run(script, globals, sessionContext, startRevision, maxRevision, callback, extCounter = 0) {
-  state = {
-    sessionContext: sessionContext,
-    revision: startRevision,
-    maxRevision: maxRevision,
-    uncommitted: false,
-    declarations: [],
-    extensions: {},
-    clear: [],
-    rng: null,
-    extCounter: extCounter
-  };
-
-  for (let n of Object.keys(context))
-    delete context[n];
-
-  Object.assign(context, globals);
-
-  let ret, status;
-
-  try {
-    ret = script.runInContext(context, {displayErrors: false});
-    status = 0;
-  }
-  catch (err) {
-    if (err === COMMIT)
-      status = 1;
-    else if (err === EXT)
-      status = 2;
-    else
-      return callback(null, errorToFault(err));
-  }
-
-  let _state = state;
-  let args = Array.from(arguments);
-
-  // Need to maintain a counter of ext() calls to avoid calling certain
-  // functions (e.g. log()) multiple times as calling ext() results in
-  // re-execution of script without revision being incremented.
-  args[6] = 0 - (state.extCounter - extCounter);
-
-  let counter = 3;
-  for (let key of Object.keys(_state.extensions)) {
-    counter += 2;
-    runExtension(_state.sessionContext, key, _state.extensions[key], function(err, fault) {
-      if (err || fault) {
-        if (counter & 1)
-          callback(err, fault);
-        return counter = 0;
-      }
-      if ((counter -= 2) === 1)
-        if (status === 2)
-          return run.apply(null, args);
-        else
-          return callback(null, null, _state.clear, _state.declarations, status === 0, ret);
-    });
-  }
-
-  if ((counter -= 2) === 1)
-    if (status === 2)
-      return run.apply(null, args);
-    else
-      return callback(null, null, _state.clear, _state.declarations, status === 0, ret);
-}
-
-
-exports.run = run;

lib/sandbox.ts

@@ -0,0 +1,438 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as vm from "vm";
+import seedrandom from "seedrandom";
+import * as device from "./device";
+import * as extensions from "./extensions";
+import * as logger from "./logger";
+import * as scheduling from "./scheduling";
+import Path from "./common/path";
+import { Fault, SessionContext, ScriptResult } from "./types";
+
+// Used for throwing to exit user script and commit
+const COMMIT = Symbol();
+
+// Used to execute extensions and restart
+const EXT = Symbol();
+
+const UNDEFINED = undefined;
+
+const context = vm.createContext();
+
+let state;
+
+const runningExtensions = new WeakMap<
+  SessionContext,
+  Map<string, Promise<Fault>>
+>();
+function runExtension(sessionContext, key, extCall): Promise<Fault> {
+  let re = runningExtensions.get(sessionContext);
+  if (!re) {
+    re = new Map<string, Promise<Fault>>();
+    runningExtensions.set(sessionContext, re);
+  }
+
+  let prom = re.get(key);
+  if (!prom) {
+    re.set(
+      key,
+      (prom = new Promise((resolve, reject) => {
+        extensions
+          .run(extCall)
+          .then(({ fault, value }) => {
+            re.delete(key);
+            if (!fault) sessionContext.extensionsCache[key] = value;
+            resolve(fault);
+          })
+          .catch(reject);
+      }))
+    );
+  }
+
+  return prom;
+}
+
+class SandboxDate {
+  public constructor(
+    ...argumentList: [
+      number?,
+      number?,
+      number?,
+      number?,
+      number?,
+      number?,
+      number?
+    ]
+  ) {
+    if (argumentList.length) return new Date(...argumentList);
+
+    return new Date(state.sessionContext.timestamp);
+  }
+
+  public static now(intervalOrCron, variance): number {
+    let t = state.sessionContext.timestamp;
+
+    if (typeof intervalOrCron === "number") {
+      if (variance == null) variance = intervalOrCron;
+
+      let offset = 0;
+      if (variance)
+        offset = scheduling.variance(state.sessionContext.deviceId, variance);
+
+      t = scheduling.interval(t, intervalOrCron, offset);
+    } else if (typeof intervalOrCron === "string") {
+      let offset = 0;
+      if (variance)
+        offset = scheduling.variance(state.sessionContext.deviceId, variance);
+      const cron = scheduling.parseCron(intervalOrCron);
+      t = scheduling.cron(t, cron, offset)[0];
+    } else if (intervalOrCron) {
+      throw new Error("Invalid Date.now() argument");
+    }
+
+    return t;
+  }
+}
+
+function random(): number {
+  if (!state.rng) state.rng = seedrandom(state.sessionContext.deviceId);
+
+  return state.rng();
+}
+
+random.seed = function(s) {
+  state.rng = seedrandom(s);
+};
+
+class ParameterWrapper {
+  public constructor(path: Path, attributes, unpacked?, unpackedRevision?) {
+    for (const attrName of attributes) {
+      Object.defineProperty(this, attrName, {
+        get: function() {
+          if (state.uncommitted) commit();
+
+          if (state.revision !== unpackedRevision) {
+            unpackedRevision = state.revision;
+            unpacked = device.unpack(
+              state.sessionContext.deviceData,
+              path,
+              state.revision
+            );
+          }
+
+          if (!unpacked.length) return UNDEFINED;
+
+          const attr = state.sessionContext.deviceData.attributes.get(
+            unpacked[0],
+            state.revision
+          )[attrName];
+
+          if (!attr) return UNDEFINED;
+
+          return attr[1];
+        }
+      });
+    }
+
+    Object.defineProperty(this, "path", {
+      get: function() {
+        if (state.uncommitted) commit();
+
+        if (state.revision !== unpackedRevision) {
+          unpackedRevision = state.revision;
+          unpacked = device.unpack(
+            state.sessionContext.deviceData,
+            path,
+            state.revision
+          );
+        }
+
+        if (!unpacked.length) return UNDEFINED;
+
+        return unpacked[0].toString();
+      }
+    });
+
+    Object.defineProperty(this, "size", {
+      get: function() {
+        if (state.uncommitted) commit();
+
+        if (state.revision !== unpackedRevision) {
+          unpackedRevision = state.revision;
+          unpacked = device.unpack(
+            state.sessionContext.deviceData,
+            path,
+            state.revision
+          );
+        }
+
+        if (!unpacked.length) return UNDEFINED;
+
+        return unpacked.length;
+      }
+    });
+
+    this[Symbol.iterator] = function*() {
+      if (state.uncommitted) commit();
+
+      if (state.revision !== unpackedRevision) {
+        unpackedRevision = state.revision;
+        unpacked = device.unpack(
+          state.sessionContext.deviceData,
+          path,
+          state.revision
+        );
+      }
+
+      for (const p of unpacked)
+        yield new ParameterWrapper(p, attributes, [p], state.revision);
+    };
+  }
+}
+
+function declare(
+  path: string,
+  timestamps: { [attr: string]: number },
+  values: { [attr: string]: any }
+): ParameterWrapper {
+  state.uncommitted = true;
+  if (!timestamps) timestamps = {};
+
+  if (!values) values = {};
+
+  const parsedPath = Path.parse(path);
+
+  const declaration = {
+    path: parsedPath,
+    pathGet: 1,
+    pathSet: null,
+    attrGet: null,
+    attrSet: null,
+    defer: true
+  };
+
+  const attrs = new Set();
+
+  for (const [attrName, attrValue] of Object.entries(values)) {
+    if (attrName === "path") {
+      declaration.pathSet = attrValue;
+    } else {
+      attrs.add(attrName);
+      if (!declaration.attrGet) declaration.attrGet = {};
+      if (!declaration.attrSet) declaration.attrSet = {};
+      declaration.attrGet[attrName] = 1;
+      if (attrName === "value" && !Array.isArray(values.value))
+        declaration.attrSet.value = [values.value];
+      else declaration.attrSet[attrName] = values[attrName];
+    }
+  }
+
+  for (const [attrName, attrTimestamp] of Object.entries(timestamps)) {
+    if (attrName === "path") {
+      declaration.pathGet = attrTimestamp;
+    } else {
+      attrs.add(attrName);
+      if (!declaration.attrGet) declaration.attrGet = {};
+      declaration.attrGet[attrName] = attrTimestamp;
+    }
+  }
+
+  state.declarations.push(declaration);
+
+  return new ParameterWrapper(parsedPath, attrs);
+}
+
+function clear(path: string, timestamp: number, attributes?): void {
+  state.uncommitted = true;
+
+  if (state.revision === state.maxRevision)
+    state.clear.push([Path.parse(path), timestamp, attributes]);
+}
+
+function commit(): void {
+  ++state.revision;
+  state.uncommitted = false;
+
+  if (state.revision === state.maxRevision + 1) {
+    for (const d of state.declarations) d.defer = false;
+    throw COMMIT;
+  } else if (state.revision > state.maxRevision + 1) {
+    throw new Error(
+      "Declare function should not be called from within a try/catch block"
+    );
+  }
+}
+
+function ext(): any {
+  ++state.extCounter;
+  const extCall = Array.from(arguments).map(String);
+  const key = `${state.revision}: ${JSON.stringify(extCall)}`;
+
+  if (key in state.sessionContext.extensionsCache)
+    return state.sessionContext.extensionsCache[key];
+
+  state.extensions[key] = extCall;
+  throw EXT;
+}
+
+function log(msg: string, meta: {}): void {
+  if (state.revision === state.maxRevision && state.extCounter >= 0) {
+    const details = Object.assign({}, meta, {
+      sessionContext: state.sessionContext,
+      message: `Script: ${msg}`
+    });
+
+    delete details["hostname"];
+    delete details["pid"];
+    delete details["name"];
+    delete details["version"];
+    delete details["deviceId"];
+    delete details["remoteAddress"];
+
+    logger.accessInfo(details);
+  }
+}
+
+Object.defineProperty(context, "Date", { value: SandboxDate });
+Object.defineProperty(context, "declare", { value: declare });
+Object.defineProperty(context, "clear", { value: clear });
+Object.defineProperty(context, "commit", { value: commit });
+Object.defineProperty(context, "ext", { value: ext });
+Object.defineProperty(context, "log", { value: log });
+
+// Monkey-patch Math.random() to make it deterministic
+context.random = random;
+vm.runInContext("Math.random = random;", context);
+delete context.random;
+
+function errorToFault(err: Error): Fault {
+  if (!err) return null;
+
+  if (!err.name) return { code: "script", message: `${err}` };
+
+  const fault: Fault = {
+    code: `script.${err.name}`,
+    message: err.message,
+    detail: {
+      name: err.name,
+      message: err.message
+    }
+  };
+
+  if (err.stack) {
+    fault.detail["stack"] = err.stack;
+    // Trim the stack trace at the self-executing anonymous wrapper function
+    const stackTrimIndex = fault.detail["stack"].match(
+      /\s+at\s[^\s]+\s+at\s[^\s]+\s\(vm\.js.+\)/
+    );
+    if (stackTrimIndex) {
+      fault.detail["stack"] = fault.detail["stack"].slice(
+        0,
+        stackTrimIndex.index
+      );
+    }
+  }
+
+  return fault;
+}
+
+export async function run(
+  script: vm.Script,
+  globals: {},
+  sessionContext: SessionContext,
+  startRevision: number,
+  maxRevision: number,
+  extCounter = 0
+): Promise<ScriptResult> {
+  state = {
+    sessionContext: sessionContext,
+    revision: startRevision,
+    maxRevision: maxRevision,
+    uncommitted: false,
+    declarations: [],
+    extensions: {},
+    clear: [],
+    rng: null,
+    extCounter: extCounter
+  };
+
+  for (const n of Object.keys(context)) delete context[n];
+
+  Object.assign(context, globals);
+
+  let ret, status;
+
+  try {
+    ret = script.runInContext(context, { displayErrors: false });
+    status = 0;
+  } catch (err) {
+    if (err === COMMIT) {
+      status = 1;
+    } else if (err === EXT) {
+      status = 2;
+    } else {
+      return {
+        fault: errorToFault(err),
+        clear: null,
+        declare: null,
+        done: false,
+        returnValue: null
+      };
+    }
+  }
+
+  const _state = state;
+  let fault;
+
+  await Promise.all(
+    Object.entries(_state.extensions).map(async ([k, v]) => {
+      fault = (await runExtension(_state.sessionContext, k, v)) || fault;
+    })
+  );
+
+  if (fault) {
+    return {
+      fault: fault,
+      clear: null,
+      declare: null,
+      done: false,
+      returnValue: null
+    };
+  }
+
+  if (status === 2) {
+    return run(
+      script,
+      globals,
+      sessionContext,
+      startRevision,
+      maxRevision,
+      extCounter - _state.extCounter
+    );
+  }
+
+  return {
+    fault: null,
+    clear: _state.clear,
+    declare: _state.declarations,
+    done: status === 0,
+    returnValue: ret
+  };
+}

lib/scheduling.ts

@@ -1,5 +1,5 @@
 /**
- * Copyright 2013-2017  Zaid Abdulla
+ * Copyright 2013-2019  GenieACS Inc.
  *
  * This file is part of GenieACS.
  *
@@ -16,14 +16,11 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
-"use strict";
 
-const crypto = require("crypto");
+import * as crypto from "crypto";
+import * as later from "later";
 
-const later = require("later");
-
-
-function md532(str) {
+function md532(str): number {
   const digest = crypto
     .createHash("md5")
     .update(str)
@@ -36,43 +33,30 @@ function md532(str) {
   );
 }
 
-
-function variance(deviceId, variance) {
-  return (md532(deviceId) >>> 0) % variance;
+export function variance(deviceId, vrnc): number {
+  return (md532(deviceId) >>> 0) % vrnc;
 }
 
-
-function interval(timestamp, interval, offset = 0) {
-  return Math.trunc((timestamp + offset) / interval) * interval - offset;
+export function interval(timestamp, intrvl, offset = 0): number {
+  return Math.trunc((timestamp + offset) / intrvl) * intrvl - offset;
 }
 
-
-function parseCron(cronExp) {
-  let parts = cronExp.trim().split(/\s+/);
-  if (parts.length === 5)
-    parts.unshift("*");
+export function parseCron(cronExp): any {
+  const parts = cronExp.trim().split(/\s+/);
+  if (parts.length === 5) parts.unshift("*");
 
   return later.schedule(later.parse.cron(parts.join(" "), true));
 }
 
-
-function cron(timestamp, schedule, offset = 0) {
+export function cron(timestamp, schedule, offset = 0): number[] {
   // TODO later.js doesn't throw erorr if expression is invalid!
   const ret = [0, 0];
 
-  let prev = schedule.prev(1, new Date(timestamp + offset));
-  if (prev)
-    ret[0] = prev.setMilliseconds(0) - offset;
+  const prev = schedule.prev(1, new Date(timestamp + offset));
+  if (prev) ret[0] = prev.setMilliseconds(0) - offset;
 
-  let next = schedule.next(1, new Date(timestamp + offset + 1000));
-  if (next)
-    ret[1] = next.setMilliseconds(0) - offset;
+  const next = schedule.next(1, new Date(timestamp + offset + 1000));
+  if (next) ret[1] = next.setMilliseconds(0) - offset;
 
   return ret;
 }
-
-
-exports.variance = variance;
-exports.interval = interval;
-exports.parseCron = parseCron;
-exports.cron = cron;

lib/server.coffee

@@ -1,123 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-service = process.argv[2]
-if not service?
-  throw new Error('Missing argument cwmp, fs, or nbi')
-
-util = require 'util'
-cluster = require 'cluster'
-logger = require './logger'
-logger.init(service, require('../package.json').version)
-config = require './config'
-db = require './db'
-cache = require './cache'
-extensions = require './extensions'
-
-networkInterface = config.get("#{service.toUpperCase()}_INTERFACE")
-port = config.get("#{service.toUpperCase()}_PORT")
-useHttps = config.get("#{service.toUpperCase()}_SSL")
-
-listener = require("./#{service}").listener
-onConnection = require("./#{service}").onConnection
-
-server = null
-
-closeServer = (timeout, callback) ->
-  return callback() if not server
-
-  setTimeout(() ->
-    return if not callback
-
-    # Ignore HTTP requests from connection that may still be open
-    server.removeListener('request', listener)
-    server.setTimeout(1)
-
-    cb = callback
-    callback = null
-    setTimeout(cb, 1000)
-  , timeout).unref()
-
-  server.close(() ->
-    return if not callback
-
-    cb = callback
-    callback = null
-    cb()
-  )
-
-
-exit = () ->
-  setTimeout(() ->
-    extensions.killAll(() ->
-      process.exit(1)
-    )
-  , 30000).unref()
-
-  closeServer(20000, () ->
-    db.disconnect()
-    cache.disconnect()
-    extensions.killAll()
-    cluster.worker?.disconnect()
-  )
-
-
-process.on('uncaughtException', (err) ->
-  logger.error({
-    message: 'Uncaught exception'
-    exception: err,
-    pid: process.pid
-  })
-  exit()
-)
-
-
-if useHttps
-  path = require 'path'
-  fs = require 'fs'
-  httpsKey = path.resolve(config.get('CONFIG_DIR'), "#{service}.key")
-  httpsCert = path.resolve(config.get('CONFIG_DIR'), "#{service}.crt")
-  httpsCa = path.resolve(config.get('CONFIG_DIR'), "#{service}.ca-bundle")
-  options = {
-    key: fs.readFileSync(httpsKey),
-    cert: fs.readFileSync(httpsCert)
-  }
-
-  try
-    # Use intermediate certificates if available
-    options.ca = fs.readFileSync(httpsCa).toString()
-      .match(/\-+BEGIN CERTIFICATE\-+[0-9a-zA-Z\+\-\/\=\s]+?\-+END CERTIFICATE\-+/g)
-
-  server = require('https').createServer(options, listener)
-  server.on('secureConnection', onConnection) if onConnection?
-else
-  server = require('http').createServer(listener)
-  server.on('connection', onConnection) if onConnection?
-
-db.connect((err) ->
-  throw err if err
-  cache.connect((err) ->
-    throw err if err
-    server.listen(port, networkInterface)
-  )
-)
-
-
-process.on('SIGINT', exit)
-process.on('SIGTERM', exit)

lib/server.ts

@@ -0,0 +1,92 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as fs from "fs";
+import * as http from "http";
+import * as https from "https";
+import * as path from "path";
+import { ROOT_DIR } from "./config";
+
+let server: http.Server | https.Server;
+let listener: (...args) => void;
+
+function closeServer(timeout, callback): void {
+  if (!server) return void callback();
+
+  setTimeout(() => {
+    if (!callback) return;
+
+    // Ignore HTTP requests from connection that may still be open
+    server.removeListener("request", listener);
+    server.setTimeout(1);
+
+    const cb = callback;
+    callback = null;
+    setTimeout(cb, 1000);
+  }, timeout).unref();
+
+  server.close(() => {
+    if (!callback) return;
+
+    const cb = callback;
+    callback = null;
+    // Allow some time for connection close events to fire
+    setTimeout(cb, 50);
+  });
+}
+
+export function start(
+  port,
+  networkInterface,
+  ssl,
+  _listener,
+  onConnection?,
+  keepAliveTimeout: number = -1
+): void {
+  listener = _listener;
+
+  if (ssl && ssl.key && ssl.cert) {
+    const options = {
+      key: ssl.key
+        .split(":")
+        .map(f => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
+      cert: ssl.cert
+        .split(":")
+        .map(f => fs.readFileSync(path.resolve(ROOT_DIR, f.trim())))
+    };
+
+    server = https.createServer(options, listener);
+    if (onConnection != null) server.on("secureConnection", onConnection);
+  } else {
+    server = http.createServer(listener);
+    if (onConnection != null) server.on("connection", onConnection);
+  }
+
+  if (keepAliveTimeout >= 0) server.keepAliveTimeout = keepAliveTimeout;
+  server.listen(port, networkInterface);
+}
+
+export function stop(): Promise<void> {
+  return new Promise((resolve, reject) => {
+    setTimeout(() => {
+      reject(new Error("Could not close server in a timely manner"));
+    }, 30000).unref();
+    closeServer(20000, resolve);
+  });
+}

lib/soap-libxmljs.ts

@@ -0,0 +1,745 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { parseXml, Document, Element, ParserOptions } from "libxmljs";
+import * as config from "./config";
+import { version as VERSION } from "../package.json";
+import {
+  CpeGetResponse,
+  CpeSetResponse,
+  InformRequest,
+  AcsResponse,
+  CpeRequest,
+  FaultStruct,
+  SpvFault,
+  CpeFault,
+  SoapMessage,
+  TransferCompleteRequest
+} from "./types";
+
+const SERVER_NAME = `GenieACS/${VERSION}`;
+
+const NAMESPACES = {
+  "1.0": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-0"
+  },
+  "1.1": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-1"
+  },
+  "1.2": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-2"
+  },
+  "1.3": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-2"
+  },
+  "1.4": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-3"
+  }
+};
+
+const LIBXMLJS_PARSE_OPTIONS: ParserOptions = {
+  nocdata: true,
+  recover: !!config.get("XML_RECOVER"),
+  /* eslint-disable-next-line @typescript-eslint/camelcase */
+  ignore_enc: !!config.get("XML_IGNORE_ENC")
+};
+
+interface WriterOptions {
+  declaration?: boolean;
+  format?: boolean;
+  selfCloseEmpty?: boolean;
+  whitespace?: boolean;
+  type?: "XML" | "HTML" | "XHTML";
+}
+
+declare module "libxmljs" {
+  interface Document {
+    toString(formatting: boolean | WriterOptions): string;
+  }
+}
+
+const LIBXMLJS_SAVE_OPTIONS: WriterOptions = {
+  format: !!config.get("XML_FORMAT"),
+  declaration: !config.get("XML_NO_DECL"),
+  selfCloseEmpty: !config.get("XML_NO_EMPTY")
+};
+
+let warnings;
+
+function parseBool(v): boolean {
+  v = "" + v;
+  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
+  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
+    return false;
+  else return null;
+}
+
+function event(xml): string[] {
+  return xml
+    .childNodes()
+    .filter(n => n.name() === "EventStruct")
+    .map(c =>
+      c
+        .childNodes()
+        .find(n => n.name() === "EventCode")
+        .text()
+        .trim()
+    );
+}
+
+function parameterInfoList(xml): [string, boolean][] {
+  return xml
+    .childNodes()
+    .filter(e => e.name() === "ParameterInfoStruct")
+    .map(e => {
+      let param, value;
+      for (const c of e.childNodes()) {
+        switch (c.name()) {
+          case "Name":
+            param = c.text();
+            break;
+          case "Writable":
+            value = c.text();
+            break;
+        }
+      }
+
+      let parsed = parseBool(value);
+
+      if (parsed === null) {
+        warnings.push({
+          message: "Invalid writable attribute",
+          parameter: param
+        });
+        parsed = false;
+      }
+
+      return [param, parsed];
+    });
+}
+
+function parameterValueList(
+  xml
+): [string, string | number | boolean, string][] {
+  return xml
+    .childNodes()
+    .filter(e => e.name() === "ParameterValueStruct")
+    .map(e => {
+      let valueElement, param;
+      for (const c of e.childNodes()) {
+        switch (c.name()) {
+          case "Name":
+            param = c.text();
+            break;
+          case "Value":
+            valueElement = c;
+            break;
+        }
+      }
+
+      const valueType = valueElement.attr("type").value();
+      const value = valueElement.text();
+      let parsed = value;
+      if (valueType === "xsd:boolean") {
+        parsed = parseBool(value);
+        if (parsed === null) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      } else if (valueType === "xsd:int" || valueType === "xsd:unsignedInt") {
+        parsed = parseInt(value);
+        if (isNaN(parsed)) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      } else if (valueType === "xsd:dateTime") {
+        parsed = Date.parse(value);
+        if (isNaN(parsed)) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      }
+
+      return [param, parsed, valueType];
+    });
+}
+
+function GetParameterNames(xml, methodRequest): void {
+  const el = xml.node("cwmp:GetParameterNames");
+  el.node("ParameterPath").text(methodRequest.parameterPath);
+  el.node("NextLevel").text(+methodRequest.nextLevel);
+}
+
+function GetParameterNamesResponse(xml): CpeGetResponse {
+  return {
+    name: "GetParameterNamesResponse",
+    parameterList: parameterInfoList(
+      xml.childNodes().find(n => n.name() === "ParameterList")
+    )
+  };
+}
+
+function GetParameterValues(xml, methodRequest): void {
+  const el = xml.node("cwmp:GetParameterValues").node("ParameterNames");
+  el.attr({
+    "soap-enc:arrayType": `xsd:string[${methodRequest.parameterNames.length}]`
+  });
+
+  for (const p of methodRequest.parameterNames) el.node("string").text(p);
+}
+
+function GetParameterValuesResponse(xml): CpeGetResponse {
+  return {
+    name: "GetParameterValuesResponse",
+    parameterList: parameterValueList(
+      xml.childNodes().find(n => n.name() === "ParameterList")
+    )
+  };
+}
+
+function SetParameterValues(xml, methodRequest): void {
+  const el = xml.node("cwmp:SetParameterValues");
+  const paramList = el.node("ParameterList");
+  paramList.attr({
+    "soap-enc:arrayType": `cwmp:ParameterValueStruct[${methodRequest.parameterList.length}]`
+  });
+
+  for (const p of methodRequest.parameterList) {
+    const pvs = paramList.node("ParameterValueStruct");
+    pvs.node("Name").text(p[0]);
+    let val = p[1];
+    if (p[2] === "xsd:dateTime" && typeof val === "number") {
+      val = new Date(val).toISOString();
+      if (methodRequest.DATETIME_MILLISECONDS === false)
+        val = val.replace(".000", "");
+    }
+    if (p[2] === "xsd:boolean" && typeof val === "boolean")
+      if (methodRequest.BOOLEAN_LITERAL === false) val = +val;
+
+    pvs
+      .node("Value")
+      .attr({ "xsi:type": p[2] })
+      .text(val);
+  }
+
+  el.node("ParameterKey").text(methodRequest.parameterKey || "");
+}
+
+function SetParameterValuesResponse(xml): CpeSetResponse {
+  return {
+    name: "SetParameterValuesResponse",
+    status: parseInt(
+      xml
+        .childNodes()
+        .find(n => n.name() === "Status")
+        .text()
+    )
+  };
+}
+
+function AddObject(xml, methodRequest): void {
+  const el = xml.node("cwmp:AddObject");
+  el.node("ObjectName").text(methodRequest.objectName);
+  el.node("ParameterKey").text(methodRequest.parameterKey || "");
+}
+
+function AddObjectResponse(xml): CpeSetResponse {
+  let instanceNumber, status;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "InstanceNumber":
+        instanceNumber = parseInt(c.text());
+        break;
+      case "Status":
+        status = parseInt(c.text());
+        break;
+    }
+  }
+
+  return {
+    name: "AddObjectResponse",
+    instanceNumber: instanceNumber,
+    status: status
+  };
+}
+
+function DeleteObject(xml, methodRequest): void {
+  const el = xml.node("cwmp:DeleteObject");
+  el.node("ObjectName").text(methodRequest.objectName);
+  el.node("ParameterKey").text(methodRequest.parameterKey || "");
+}
+
+function DeleteObjectResponse(xml): CpeSetResponse {
+  return {
+    name: "DeleteObjectResponse",
+    status: parseInt(
+      xml
+        .childNodes()
+        .find(n => n.name() === "Status")
+        .text()
+    )
+  };
+}
+
+function Reboot(xml, methodRequest): void {
+  const el = xml.node("cwmp:Reboot");
+  el.node("CommandKey").text(methodRequest.commandKey || "");
+}
+
+function RebootResponse(): CpeSetResponse {
+  return {
+    name: "RebootResponse"
+  };
+}
+
+function FactoryReset(xml): void {
+  xml.node("cwmp:FactoryReset");
+}
+
+function FactoryResetResponse(): CpeSetResponse {
+  return {
+    name: "FactoryResetResponse"
+  };
+}
+
+function Download(xml, methodRequest): void {
+  const el = xml.node("cwmp:Download");
+  el.node("CommandKey").text(methodRequest.commandKey || "");
+  el.node("FileType").text(methodRequest.fileType);
+  el.node("URL").text(methodRequest.url);
+  el.node("Username").text(methodRequest.username || "");
+  el.node("Password").text(methodRequest.password || "");
+  el.node("FileSize").text(methodRequest.fileSize || "0");
+  el.node("TargetFileName").text(methodRequest.targetFileName || "");
+  el.node("DelaySeconds").text(methodRequest.delaySeconds || "0");
+  el.node("SuccessURL").text(methodRequest.successUrl || "");
+  el.node("FailureURL").text(methodRequest.failureUrl || "");
+}
+
+function DownloadResponse(xml): CpeSetResponse {
+  let status, startTime, completeTime;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "Status":
+        status = parseInt(c.text());
+        break;
+      case "StartTime":
+        startTime = Date.parse(c.text());
+        break;
+      case "CompleteTime":
+        completeTime = Date.parse(c.text());
+        break;
+    }
+  }
+
+  return {
+    name: "DownloadResponse",
+    status: status,
+    startTime: startTime,
+    completeTime: completeTime
+  };
+}
+
+function Inform(xml): InformRequest {
+  let retryCount, evnt, parameterList;
+  const deviceId = {
+    Manufacturer: null,
+    OUI: null,
+    ProductClass: null,
+    SerialNumber: null
+  };
+
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "ParameterList":
+        parameterList = parameterValueList(c);
+        break;
+      case "DeviceId":
+        for (const cc of c.childNodes()) {
+          const n = cc.name();
+          if (n in deviceId) deviceId[n] = cc.text();
+        }
+        break;
+      case "Event":
+        evnt = event(c);
+        break;
+      case "RetryCount":
+        retryCount = parseInt(c.text());
+        break;
+    }
+  }
+
+  return {
+    name: "Inform",
+    parameterList: parameterList,
+    deviceId: deviceId,
+    event: evnt,
+    retryCount: retryCount
+  };
+}
+
+function InformResponse(xml): void {
+  xml
+    .node("cwmp:InformResponse")
+    .node("MaxEnvelopes")
+    .text(1);
+}
+
+function GetRPCMethods(): AcsResponse {
+  return { name: "GetRPCMethods" };
+}
+
+function GetRPCMethodsResponse(xml, methodResponse): void {
+  const el = xml.node("cwmp:GetRPCMethodsResponse").node("MethodList");
+  el.attr({
+    "soap-enc:arrayType": `xsd:string[${methodResponse.methodList.length}]`
+  });
+
+  for (const m of methodResponse.methodList) el.node("string").text(m);
+}
+
+function TransferComplete(xml): TransferCompleteRequest {
+  let commandKey, _faultStruct, startTime, completeTime;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "CommandKey":
+        commandKey = c.text();
+        break;
+      case "FaultStruct":
+        _faultStruct = faultStruct(c);
+        break;
+      case "StartTime":
+        startTime = Date.parse(c.text());
+        break;
+      case "CompleteTime":
+        completeTime = Date.parse(c.text());
+        break;
+    }
+  }
+
+  return {
+    name: "TransferComplete",
+    commandKey: commandKey,
+    faultStruct: _faultStruct,
+    startTime: startTime,
+    completeTime: completeTime
+  };
+}
+
+function TransferCompleteResponse(xml): void {
+  xml.node("cwmp:TransferCompleteResponse").text("");
+}
+
+function RequestDownload(xml): CpeRequest {
+  return {
+    name: "RequestDownload",
+    fileType: xml
+      .childNodes()
+      .find(n => n.name() === "FileType")
+      .text()
+  };
+}
+
+function RequestDownloadResponse(xml): void {
+  xml.node("cwmp:RequestDownloadResponse").text("");
+}
+
+function faultStruct(xml): FaultStruct {
+  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "FaultCode":
+        faultCode = c.text();
+        break;
+      case "FaultString":
+        faultString = c.text();
+        break;
+      case "SetParameterValuesFault":
+        setParameterValuesFault = setParameterValuesFault || [];
+        for (const cc of c.childNodes()) {
+          switch (cc.name()) {
+            case "ParameterName":
+              pn = cc.text();
+              break;
+            case "FaultCode":
+              fc = cc.text();
+              break;
+            case "FaultString":
+              fs = cc.text();
+              break;
+          }
+        }
+        setParameterValuesFault.push({
+          parameterName: pn,
+          faultCode: fc,
+          faultString: fs
+        });
+    }
+  }
+
+  return { faultCode, faultString, setParameterValuesFault };
+}
+
+function fault(xml): CpeFault {
+  let faultCode, faultString, detail;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "faultcode":
+        faultCode = c.text();
+        break;
+      case "faultstring":
+        faultString = c.text();
+        break;
+      case "detail":
+        detail = faultStruct(c.childNodes().find(n => n.name() === "Fault"));
+        break;
+    }
+  }
+
+  return { faultCode, faultString, detail };
+}
+
+export function request(data, cwmpVersion, warn): SoapMessage {
+  warnings = warn;
+
+  const rpc = {
+    id: null,
+    cwmpVersion: cwmpVersion,
+    sessionTimeout: null,
+    cpeRequest: null,
+    cpeFault: null,
+    cpeResponse: null
+  };
+
+  if (!data.length) return rpc;
+
+  const xml = parseXml(data, LIBXMLJS_PARSE_OPTIONS);
+
+  let headerElement, bodyElement;
+
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "Header":
+        headerElement = c;
+        break;
+      case "Body":
+        bodyElement = c;
+        break;
+    }
+  }
+
+  if (headerElement) {
+    for (const c of headerElement.childNodes()) {
+      switch (c.name()) {
+        case "ID":
+          rpc.id = c.text();
+          break;
+        case "sessionTimeout":
+          rpc.sessionTimeout = parseInt(c.text());
+          break;
+      }
+    }
+  }
+
+  for (const methodElement of bodyElement.childNodes()) {
+    const methodName = methodElement.name();
+    if (methodName === "text") continue;
+
+    if (!rpc.cwmpVersion && methodName !== "Fault") {
+      switch (methodElement.namespace().href()) {
+        case "urn:dslforum-org:cwmp-1-0":
+          rpc.cwmpVersion = "1.0";
+          break;
+        case "urn:dslforum-org:cwmp-1-1":
+          rpc.cwmpVersion = "1.1";
+          break;
+        case "urn:dslforum-org:cwmp-1-2":
+          if (rpc.sessionTimeout) rpc.cwmpVersion = "1.3";
+          else rpc.cwmpVersion = "1.2";
+
+          break;
+        case "urn:dslforum-org:cwmp-1-3":
+          rpc.cwmpVersion = "1.4";
+          break;
+        default:
+          throw new Error("Unrecognized CWMP version");
+      }
+    }
+
+    switch (methodName) {
+      case "Inform":
+        rpc.cpeRequest = Inform(methodElement);
+        break;
+      case "GetRPCMethods":
+        rpc.cpeRequest = GetRPCMethods();
+        break;
+      case "TransferComplete":
+        rpc.cpeRequest = TransferComplete(methodElement);
+        break;
+      case "RequestDownload":
+        rpc.cpeRequest = RequestDownload(methodElement);
+        break;
+      case "GetParameterNamesResponse":
+        rpc.cpeResponse = GetParameterNamesResponse(methodElement);
+        break;
+      case "GetParameterValuesResponse":
+        rpc.cpeResponse = GetParameterValuesResponse(methodElement);
+        break;
+      case "SetParameterValuesResponse":
+        rpc.cpeResponse = SetParameterValuesResponse(methodElement);
+        break;
+      case "AddObjectResponse":
+        rpc.cpeResponse = AddObjectResponse(methodElement);
+        break;
+      case "DeleteObjectResponse":
+        rpc.cpeResponse = DeleteObjectResponse(methodElement);
+        break;
+      case "RebootResponse":
+        rpc.cpeResponse = RebootResponse();
+        break;
+      case "FactoryResetResponse":
+        rpc.cpeResponse = FactoryResetResponse();
+        break;
+      case "DownloadResponse":
+        rpc.cpeResponse = DownloadResponse(methodElement);
+        break;
+      case "Fault":
+        rpc.cpeFault = fault(methodElement);
+        break;
+      default:
+        throw new Error(`8000 Method not supported ${methodName}`);
+    }
+  }
+
+  return rpc;
+}
+
+function createSoapEnv(cwmpVersion: string): Element {
+  const xml = new Document();
+  const env = xml.node("soap-env:Envelope");
+
+  for (const prefix of Object.keys(NAMESPACES[cwmpVersion]))
+    env.defineNamespace(prefix, NAMESPACES[cwmpVersion][prefix]);
+
+  return env;
+}
+
+export function response(rpc): { code: number; headers: {}; data: string } {
+  const headers = {
+    Server: SERVER_NAME,
+    SOAPServer: SERVER_NAME
+  };
+
+  if (!rpc) return { code: 204, headers: headers, data: "" };
+
+  const env = createSoapEnv(rpc.cwmpVersion);
+  const header = env.node("soap-env:Header");
+  header
+    .node("cwmp:ID")
+    .attr({
+      "soap-env:mustUnderstand": "1"
+    })
+    .text(rpc.id);
+
+  const body = env.node("soap-env:Body");
+
+  if (rpc.acsResponse) {
+    switch (rpc.acsResponse.name) {
+      case "InformResponse":
+        InformResponse(body);
+        break;
+      case "GetRPCMethodsResponse":
+        GetRPCMethodsResponse(body, rpc.acsResponse);
+        break;
+      case "TransferCompleteResponse":
+        TransferCompleteResponse(body);
+        break;
+      case "RequestDownloadResponse":
+        RequestDownloadResponse(body);
+        break;
+      default:
+        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
+    }
+  } else if (rpc.acsRequest) {
+    switch (rpc.acsRequest.name) {
+      case "GetParameterNames":
+        GetParameterNames(body, rpc.acsRequest);
+        break;
+      case "GetParameterValues":
+        GetParameterValues(body, rpc.acsRequest);
+        break;
+      case "SetParameterValues":
+        SetParameterValues(body, rpc.acsRequest);
+        break;
+      case "AddObject":
+        AddObject(body, rpc.acsRequest);
+        break;
+      case "DeleteObject":
+        DeleteObject(body, rpc.acsRequest);
+        break;
+      case "Reboot":
+        Reboot(body, rpc.acsRequest);
+        break;
+      case "FactoryReset":
+        FactoryReset(body);
+        break;
+      case "Download":
+        Download(body, rpc.acsRequest);
+        break;
+      default:
+        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
+    }
+  }
+
+  headers["Content-Type"] = 'text/xml; charset="utf-8"';
+  return {
+    code: 200,
+    headers: headers,
+    data: env.doc().toString(LIBXMLJS_SAVE_OPTIONS)
+  };
+}

lib/soap.js

@@ -1,612 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const libxmljs = require("libxmljs");
-
-const config = require("./config");
-
-const SERVER_NAME = `GenieACS/${require("../package.json").version}`;
-
-const NAMESPACES = {
-  "1.0": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    "xsd": "http://www.w3.org/2001/XMLSchema",
-    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
-    "cwmp": "urn:dslforum-org:cwmp-1-0"
-  },
-  "1.1": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    "xsd": "http://www.w3.org/2001/XMLSchema",
-    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
-    "cwmp": "urn:dslforum-org:cwmp-1-1"
-  },
-  "1.2": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    "xsd": "http://www.w3.org/2001/XMLSchema",
-    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
-    "cwmp": "urn:dslforum-org:cwmp-1-2"
-  },
-  "1.3": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    "xsd": "http://www.w3.org/2001/XMLSchema",
-    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
-    "cwmp": "urn:dslforum-org:cwmp-1-2"
-  },
-  "1.4": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    "xsd": "http://www.w3.org/2001/XMLSchema",
-    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
-    "cwmp": "urn:dslforum-org:cwmp-1-3"
-  }
-};
-
-const LIBXMLJS_PARSE_OPTIONS = {
-  "nocdata": true,
-  "recover": config.get("XML_RECOVER"),
-  "ignore_enc": config.get("XML_IGNORE_ENC")
-};
-
-const LIBXMLJS_SAVE_OPTIONS = {
-  "format": config.get("XML_FORMAT"),
-  "declaration": !config.get("XML_NO_DECL"),
-  "selfCloseEmpty": !config.get("XML_NO_EMPTY")
-};
-
-const XML_IGNORE_NAMESPACE = config.get("XML_IGNORE_NAMESPACE");
-
-if (XML_IGNORE_NAMESPACE) {
-  libxmljs.Element.prototype.__find = libxmljs.Element.prototype.find;
-  libxmljs.Element.prototype.find = function(xpath, namespaces) {
-    const p = xpath.replace(/([^\/:]+:)?([^\/]+)(\/|$)/g, "$2$3").replace(/([a-zA-Z0-9_-]+)([^\/]*)(\/|$)/g, "*[local-name()=\"$1\"]$2$3");
-    return libxmljs.Element.prototype.__find.call(this, p, namespaces);
-  };
-}
-
-let warnings;
-
-
-function parseBool(v) {
-  v = "" + v;
-  if (v === "true" || v === "TRUE" || v === "True" || v === "1")
-    return true;
-  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
-    return false;
-  else
-    return null;
-}
-
-
-function event(xml) {
-  return xml.find("EventStruct/EventCode").map(e => e.text().trim());
-}
-
-
-function parameterInfoList(xml) {
-  return xml.find("ParameterInfoStruct").map(function(e) {
-    let param = e.get("Name").text();
-    let value = e.get("Writable").text();
-    let parsed = parseBool(value);
-
-    if (parsed === null) {
-      warnings.push({
-        message: "Invalid writable attribute",
-        parameter: param
-      });
-      parsed = false;
-    }
-
-    return [param, parsed];
-  });
-}
-
-
-function parameterValueList(xml) {
-  return xml.find("ParameterValueStruct").map(function(e) {
-    const valueElement = e.get("Value");
-    const valueType = valueElement.attr("type").value();
-    let param = e.get("Name").text();
-    let value = valueElement.text();
-    let parsed = value;
-    if (valueType === "xsd:boolean") {
-      parsed = parseBool(value);
-      if (parsed === null) {
-        warnings.push({
-          message: "Invalid value attribute",
-          parameter: param
-        });
-        parsed = value;
-      }
-    }
-    else if (valueType === "xsd:int" || valueType === "xsd:unsignedInt") {
-      parsed = parseInt(value);
-      if (isNaN(parsed)) {
-        warnings.push({
-          message: "Invalid value attribute",
-          parameter: param
-        });
-        parsed = value;
-      }
-    }
-    else if (valueType === "xsd:dateTime") {
-      parsed = Date.parse(value);
-      if (isNaN(parsed)) {
-        warnings.push({
-          message: "Invalid value attribute",
-          parameter: param
-        });
-        parsed = value;
-      }
-    }
-
-   return [param, parsed, valueType];
-  });
-}
-
-
-function GetParameterNames(xml, methodRequest) {
-  const el = xml.node("cwmp:GetParameterNames");
-  el.node("ParameterPath").text(methodRequest.parameterPath);
-  el.node("NextLevel").text(+methodRequest.nextLevel);
-}
-
-
-function GetParameterNamesResponse(xml) {
-  return {
-    name: "GetParameterNamesResponse",
-    parameterList: parameterInfoList(xml.get("ParameterList"))
-  };
-}
-
-
-function GetParameterValues(xml, methodRequest) {
-  const el = xml.node("cwmp:GetParameterValues").node("ParameterNames");
-  el.attr({
-    "soap-enc:arrayType": `xsd:string[${methodRequest.parameterNames.length}]`
-  });
-
-  for (const p of methodRequest.parameterNames)
-    el.node("string").text(p);
-}
-
-
-function GetParameterValuesResponse(xml) {
-  return {
-    name: "GetParameterValuesResponse",
-    parameterList: parameterValueList(xml.get("ParameterList"))
-  };
-}
-
-
-function SetParameterValues(xml, methodRequest) {
-  const el = xml.node("cwmp:SetParameterValues");
-  const paramList = el.node("ParameterList");
-  paramList.attr({
-    "soap-enc:arrayType": `cwmp:ParameterValueStruct[${methodRequest.parameterList.length}]`
-  });
-
-  for (const p of methodRequest.parameterList) {
-    const pvs = paramList.node("ParameterValueStruct");
-    pvs.node("Name").text(p[0]);
-    let val = p[1];
-    if (p[2] === 'xsd:dateTime' && typeof val === 'number') {
-      val = (new Date(val)).toISOString();
-      if (methodRequest.DATETIME_MILLISECONDS === false)
-        val = val.replace('.000', '');
-    }
-    if (p[2] === 'xsd:boolean' && typeof val === 'boolean') {
-      if (methodRequest.BOOLEAN_LITERAL === false)
-        val = +val;
-    }
-    pvs.node("Value").attr({"xsi:type": p[2]}).text(val);
-  }
-
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-
-function SetParameterValuesResponse(xml) {
-  return {
-    name: "SetParameterValuesResponse",
-    status: parseInt(xml.get("Status").text())
-  };
-}
-
-
-function AddObject(xml, methodRequest) {
-  const el = xml.node("cwmp:AddObject");
-  el.node("ObjectName").text(methodRequest.objectName);
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-
-function AddObjectResponse(xml) {
-  return {
-    name: "AddObjectResponse",
-    instanceNumber: parseInt(xml.get("InstanceNumber").text()),
-    status: parseInt(xml.get("Status").text())
-  };
-}
-
-
-function DeleteObject(xml, methodRequest) {
-  const el = xml.node("cwmp:DeleteObject");
-  el.node("ObjectName").text(methodRequest.objectName);
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-
-function DeleteObjectResponse(xml) {
-  return {
-    name: "DeleteObjectResponse",
-    status: parseInt(xml.get("Status").text())
-  };
-}
-
-
-function Reboot(xml, methodRequest) {
-  const el = xml.node("cwmp:Reboot");
-  el.node("CommandKey").text(methodRequest.commandKey || "");
-}
-
-
-function RebootResponse(xml) {
-  return {
-    name: "RebootResponse"
-  };
-}
-
-
-function FactoryReset(xml, methodRequest) {
-  xml.node("cwmp:FactoryReset");
-}
-
-
-function FactoryResetResponse(xml, methodRequest) {
-  return {
-    name: "FactoryResetResponse"
-  };
-}
-
-
-function Download(xml, methodRequest) {
-  const el = xml.node("cwmp:Download");
-  el.node("CommandKey").text(methodRequest.commandKey || "");
-  el.node("FileType").text(methodRequest.fileType);
-  el.node("URL").text(methodRequest.url);
-  el.node("Username").text(methodRequest.username || "");
-  el.node("Password").text(methodRequest.password || "");
-  el.node("FileSize").text(methodRequest.fileSize || "0");
-  el.node("TargetFileName").text(methodRequest.targetFileName || "");
-  el.node("DelaySeconds").text(methodRequest.delaySeconds || "0");
-  el.node("SuccessURL").text(methodRequest.successUrl || "");
-  el.node("FailureURL").text(methodRequest.failureUrl || "");
-}
-
-
-function DownloadResponse(xml) {
-  const res = {
-    name: "DownloadResponse",
-    status: parseInt(xml.get("Status").text())
-  };
-
-  if (res.status === 0) {
-    res.startTime = Date.parse(xml.get("StartTime").text());
-    res.completeTime = Date.parse(xml.get("CompleteTime").text());
-  }
-
-  return res;
-}
-
-
-function Inform(xml) {
-  return {
-    name: "Inform",
-    parameterList: parameterValueList(xml.get("ParameterList")),
-    deviceId: {
-      "Manufacturer": xml.get("DeviceId/Manufacturer").text(),
-      "OUI": xml.get("DeviceId/OUI").text(),
-      "ProductClass": xml.get("DeviceId/ProductClass").text(),
-      "SerialNumber": xml.get("DeviceId/SerialNumber").text()
-    },
-    event: event(xml.get("Event")),
-    retryCount: parseInt(xml.get("RetryCount").text())
-  };
-}
-
-
-function InformResponse(xml, methodResponse) {
-  xml.node("cwmp:InformResponse").node("MaxEnvelopes").text(1);
-}
-
-
-function GetRPCMethods(xml) {
-  return {name: "GetRPCMethods"};
-}
-
-
-function GetRPCMethodsResponse(xml, methodResponse) {
-  const el = xml.node("cwmp:GetRPCMethodsResponse").node("MethodList");
-  el.attr({
-    "soap-enc:arrayType": `xsd:string[${methodResponse.methodList.length}]`
-  });
-
-  for (const m of methodResponse.methodList)
-    el.node("string").text(m);
-}
-
-
-function TransferComplete(xml) {
-  return {
-    name: "TransferComplete",
-    commandKey: xml.get("CommandKey").text(),
-    faultStruct: faultStruct(xml.get("FaultStruct")),
-    startTime: Date.parse(xml.get("StartTime").text()),
-    completeTime: Date.parse(xml.get("CompleteTime").text())
-  };
-}
-
-
-function TransferCompleteResponse(xml, methodResponse) {
-  xml.node("cwmp:TransferCompleteResponse").text("");
-}
-
-
-function RequestDownload(xml) {
-  return {
-    name: "RequestDownload",
-    fileType: xml.get("FileType").text()
-  };
-}
-
-
-function RequestDownloadResponse(xml, methodResponse) {
-  xml.node("cwmp:RequestDownloadResponse").text("");
-}
-
-
-function faultStruct(xml) {
-  const f = {
-    faultCode: xml.get("FaultCode").text(),
-    faultString: xml.get("FaultString").text()
-  };
-
-  for (const e of xml.find("SetParameterValuesFault")) {
-    if (!f.setParameterValuesFault)
-      f.setParameterValuesFault = [];
-
-    f.setParameterValuesFault.push({
-      parameterName: e.get("ParameterName").text(),
-      faultCode: e.get("FaultCode").text(),
-      faultString: e.get("FaultString").text()
-    });
-  }
-
-  return f;
-}
-
-
-function fault(xml, cwmpVersion) {
-  return {
-    faultCode: xml.get("faultcode").text(),
-    faultString: xml.get("faultstring").text(),
-    detail: faultStruct(xml.get("detail/cwmp:Fault", NAMESPACES[cwmpVersion]))
-  };
-}
-
-
-function request(data, cwmpVersion, warn) {
-  warnings = warn;
-
-  const rpc = {
-    cwmpVersion: cwmpVersion
-  };
-
-  if (!data.length)
-    return rpc;
-
-  const xml = libxmljs.parseXml(data, LIBXMLJS_PARSE_OPTIONS);
-
-  let methodElement;
-
-  if (rpc.cwmpVersion) {
-    methodElement = xml.get("/soap-env:Envelope/soap-env:Body/cwmp:*", NAMESPACES[rpc.cwmpVersion]);
-  }
-  else {
-    // cwmpVersion not passed, thus it's an inform request
-    methodElement = xml.get("/soap-env:Envelope/soap-env:Body/*", NAMESPACES["1.0"]);
-    switch (methodElement.namespace().href()) {
-      case "urn:dslforum-org:cwmp-1-0":
-        rpc.cwmpVersion = "1.0";
-        break;
-      case "urn:dslforum-org:cwmp-1-1":
-        rpc.cwmpVersion = "1.1";
-        break;
-      case "urn:dslforum-org:cwmp-1-2":
-        const timeoutElement = xml.get("/soap-env:Envelope/soap-env:Header/cwmp:sessionTimeout", NAMESPACES["1.2"]);
-        if (timeoutElement) {
-          rpc.sessionTimeout = parseInt(timeoutElement.text());
-          rpc.cwmpVersion = "1.3";
-        }
-        else {
-          rpc.cwmpVersion = "1.2";
-        }
-        break;
-      case "urn:dslforum-org:cwmp-1-3":
-        rpc.cwmpVersion = "1.4";
-        break;
-      default:
-        throw new Error("Unrecognized CWMP version");
-    }
-
-    if (!rpc.sessionTimeout) {
-      const timeoutElement = xml.get("/soap-env:Envelope/soap-env:Header/cwmp:sessionTimeout", NAMESPACES[rpc.cwmpVersion]);
-      if (timeoutElement)
-        rpc.sessionTimeout = parseInt(timeoutElement.text());
-    }
-  }
-
-  let idElement = xml.get("/soap-env:Envelope/soap-env:Header/cwmp:ID", NAMESPACES[rpc.cwmpVersion]);
-  if (idElement)
-    rpc.id = idElement.text();
-
-  if (methodElement && !(XML_IGNORE_NAMESPACE && methodElement.name() === "Fault")) {
-    switch (methodElement.name()) {
-      case "Inform":
-        rpc.cpeRequest = Inform(methodElement);
-        break;
-      case "GetRPCMethods":
-        rpc.cpeRequest = GetRPCMethods(methodElement);
-        break;
-      case "TransferComplete":
-        rpc.cpeRequest = TransferComplete(methodElement);
-        break;
-      case "RequestDownload":
-        rpc.cpeRequest = RequestDownload(methodElement);
-        break;
-      case "GetParameterNamesResponse":
-        rpc.cpeResponse = GetParameterNamesResponse(methodElement);
-        break;
-      case "GetParameterValuesResponse":
-        rpc.cpeResponse = GetParameterValuesResponse(methodElement);
-        break;
-      case "SetParameterValuesResponse":
-        rpc.cpeResponse = SetParameterValuesResponse(methodElement);
-        break;
-      case "AddObjectResponse":
-        rpc.cpeResponse = AddObjectResponse(methodElement);
-        break;
-      case "DeleteObjectResponse":
-        rpc.cpeResponse = DeleteObjectResponse(methodElement);
-        break;
-      case "RebootResponse":
-        rpc.cpeResponse = RebootResponse(methodElement);
-        break;
-      case "FactoryResetResponse":
-        rpc.cpeResponse = FactoryResetResponse(methodElement);
-        break;
-      case "DownloadResponse":
-        rpc.cpeResponse = DownloadResponse(methodElement);
-        break;
-      default:
-        throw new Error(`8000 Method not supported ${methodElement.name()}`);
-    }
-  }
-  else {
-    const faultElement = xml.get("/soap-env:Envelope/soap-env:Body/soap-env:Fault", NAMESPACES[rpc.cwmpVersion]);
-    rpc.cpeFault = fault(faultElement, rpc.cwmpVersion);
-  }
-
-  return rpc;
-}
-
-
-function createSoapEnv(cwmpVersion) {
-  const xml = libxmljs.Document();
-  const env = xml.node("soap-env:Envelope");
-
-  for (const prefix of Object.keys(NAMESPACES[cwmpVersion]))
-    env.defineNamespace(prefix, NAMESPACES[cwmpVersion][prefix]);
-
-  return env;
-}
-
-
-function response(rpc) {
-  const headers = {
-    "Server": SERVER_NAME,
-    "SOAPServer": SERVER_NAME
-  };
-
-  if (!rpc)
-    return {code: 204, headers: headers, data: new Buffer(0)};
-
-  const env = createSoapEnv(rpc.cwmpVersion);
-  const header = env.node("soap-env:Header");
-  header.node("cwmp:ID").attr({
-    "soap-env:mustUnderstand": 1
-  }).text(rpc.id);
-
-  const body = env.node("soap-env:Body");
-
-  if (rpc.acsResponse) {
-    switch (rpc.acsResponse.name) {
-      case "InformResponse":
-        InformResponse(body, rpc.acsResponse);
-        break;
-      case "GetRPCMethodsResponse":
-        GetRPCMethodsResponse(body, rpc.acsResponse);
-        break;
-      case "TransferCompleteResponse":
-        TransferCompleteResponse(body, rpc.acsResponse);
-        break;
-      case "RequestDownloadResponse":
-        RequestDownloadResponse(body, rpc.acsResponse);
-        break;
-      default:
-        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
-    }
-  }
-  else if (rpc.acsRequest) {
-    switch (rpc.acsRequest.name) {
-      case "GetParameterNames":
-        GetParameterNames(body, rpc.acsRequest);
-        break;
-      case "GetParameterValues":
-        GetParameterValues(body, rpc.acsRequest);
-        break;
-      case "SetParameterValues":
-        SetParameterValues(body, rpc.acsRequest);
-        break;
-      case "AddObject":
-        AddObject(body, rpc.acsRequest);
-        break;
-      case "DeleteObject":
-        DeleteObject(body, rpc.acsRequest);
-        break;
-      case "Reboot":
-        Reboot(body, rpc.acsRequest);
-        break;
-      case "FactoryReset":
-        FactoryReset(body, rpc.acsRequest);
-        break;
-      case "Download":
-        Download(body, rpc.acsRequest);
-        break;
-      default:
-        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
-    }
-  }
-
-  headers["Content-Type"] = "text/xml; charset=\"utf-8\"";
-  return {
-    code: 200,
-    headers: headers,
-    data: new Buffer(env.doc().toString(LIBXMLJS_SAVE_OPTIONS))
-  };
-}
-
-
-exports.request = request;
-exports.response = response;

lib/soap.ts

@@ -0,0 +1,824 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import {
+  parseXml,
+  Element,
+  parseAttrs,
+  encodeEntities,
+  decodeEntities
+} from "./xml-parser";
+import memoize from "./common/memoize";
+import { version as VERSION } from "../package.json";
+import {
+  CpeGetResponse,
+  CpeSetResponse,
+  InformRequest,
+  AcsResponse,
+  CpeRequest,
+  FaultStruct,
+  SpvFault,
+  CpeFault,
+  SoapMessage,
+  TransferCompleteRequest
+} from "./types";
+
+const SERVER_NAME = `GenieACS/${VERSION}`;
+
+const NAMESPACES = {
+  "1.0": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-0"
+  },
+  "1.1": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-1"
+  },
+  "1.2": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-2"
+  },
+  "1.3": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-2"
+  },
+  "1.4": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-3"
+  }
+};
+
+let warnings;
+
+const memoizedParseAttrs = memoize(parseAttrs);
+
+function parseBool(v): boolean {
+  v = "" + v;
+  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
+  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
+    return false;
+  else return null;
+}
+
+function event(xml: Element): string[] {
+  return xml.children
+    .filter(n => n.localName === "EventStruct")
+    .map(c => c.children.find(n => n.localName === "EventCode").text.trim());
+}
+
+function parameterInfoList(xml: Element): [string, boolean][] {
+  return xml.children
+    .filter(e => e.localName === "ParameterInfoStruct")
+    .map<[string, boolean]>(e => {
+      let param: string, value: string;
+      for (const c of e.children) {
+        switch (c.localName) {
+          case "Name":
+            param = c.text;
+            break;
+          case "Writable":
+            value = c.text;
+            break;
+        }
+      }
+
+      let parsed: boolean = parseBool(value);
+
+      if (parsed === null) {
+        warnings.push({
+          message: "Invalid writable attribute",
+          parameter: param
+        });
+        parsed = false;
+      }
+
+      return [param, parsed];
+    });
+}
+
+const getValueType = memoize((str: string) =>
+  parseAttrs(str)
+    .find(s => s.localName === "type")
+    .value.trim()
+);
+
+function parameterValueList(
+  xml: Element
+): [string, string | number | boolean, string][] {
+  return xml.children
+    .filter(e => e.localName === "ParameterValueStruct")
+    .map<[string, string | number | boolean, string]>(e => {
+      let valueElement: Element, param: string;
+      for (const c of e.children) {
+        switch (c.localName) {
+          case "Name":
+            param = c.text;
+            break;
+          case "Value":
+            valueElement = c;
+            break;
+        }
+      }
+
+      const valueType = getValueType(valueElement.attrs);
+
+      const value = decodeEntities(valueElement.text);
+      let parsed: string | number | boolean = value;
+      if (valueType === "xsd:boolean") {
+        parsed = parseBool(value);
+        if (parsed === null) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      } else if (valueType === "xsd:int" || valueType === "xsd:unsignedInt") {
+        parsed = parseInt(value);
+        if (isNaN(parsed)) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      } else if (valueType === "xsd:dateTime") {
+        parsed = Date.parse(value);
+        if (isNaN(parsed)) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      }
+
+      return [param, parsed, valueType];
+    });
+}
+
+function parameterAttributeList(xml: Element): [string, number, string[]][] {
+  return xml.children
+    .filter(e => e.localName === "ParameterAttributeStruct")
+    .map<[string, number, string[]]>(e => {
+      let notificationElement: Element,
+        accessListElement: Element,
+        param: string;
+      for (const c of e.children) {
+        switch (c.localName) {
+          case "Name":
+            param = c.text;
+            break;
+          case "Notification":
+            notificationElement = c;
+            break;
+          case "AccessList":
+            accessListElement = c;
+            break;
+        }
+      }
+
+      const notification = decodeEntities(notificationElement.text);
+      const notif = +notification;
+
+      if (notif !== parseInt(notification)) {
+        warnings.push({
+          message: "Invalid notification attribute",
+          parameter: param
+        });
+      }
+
+      const accessList = accessListElement.children
+        .filter(c => c.localName === "string")
+        .map(c => decodeEntities(c.text));
+
+      return [param, notif, accessList];
+    });
+}
+
+function GetParameterNames(methodRequest): string {
+  return `<cwmp:GetParameterNames><ParameterPath>${
+    methodRequest.parameterPath
+  }</ParameterPath><NextLevel>${+methodRequest.nextLevel}</NextLevel></cwmp:GetParameterNames>`;
+}
+
+function GetParameterNamesResponse(xml): CpeGetResponse {
+  return {
+    name: "GetParameterNamesResponse",
+    parameterList: parameterInfoList(
+      xml.children.find(n => n.localName === "ParameterList")
+    )
+  };
+}
+
+function GetParameterValues(methodRequest): string {
+  return `<cwmp:GetParameterValues><ParameterNames soap-enc:arrayType="xsd:string[${
+    methodRequest.parameterNames.length
+  }]">${methodRequest.parameterNames
+    .map(p => `<string>${p}</string>`)
+    .join("")}</ParameterNames></cwmp:GetParameterValues>`;
+}
+
+function GetParameterValuesResponse(xml: Element): CpeGetResponse {
+  return {
+    name: "GetParameterValuesResponse",
+    parameterList: parameterValueList(
+      xml.children.find(n => n.localName === "ParameterList")
+    )
+  };
+}
+
+function GetParameterAttributes(methodRequest): string {
+  return `<cwmp:GetParameterAttributes><ParameterNames soap-enc:arrayType="xsd:string[${
+    methodRequest.parameterNames.length
+  }]">${methodRequest.parameterNames
+    .map(p => `<string>${p}</string>`)
+    .join("")}</ParameterNames></cwmp:GetParameterAttributes>`;
+}
+
+function GetParameterAttributesResponse(xml: Element): CpeGetResponse {
+  return {
+    name: "GetParameterAttributesResponse",
+    parameterList: parameterAttributeList(
+      xml.children.find(n => n.localName === "ParameterList")
+    )
+  };
+}
+
+function SetParameterValues(methodRequest): string {
+  const params = methodRequest.parameterList.map(p => {
+    let val = p[1];
+    if (p[2] === "xsd:dateTime" && typeof val === "number") {
+      val = new Date(val).toISOString();
+      if (methodRequest.DATETIME_MILLISECONDS === false)
+        val = val.replace(".000", "");
+    }
+    if (p[2] === "xsd:boolean" && typeof val === "boolean")
+      if (methodRequest.BOOLEAN_LITERAL === false) val = +val;
+    return `<ParameterValueStruct><Name>${p[0]}</Name><Value xsi:type="${
+      p[2]
+    }">${encodeEntities("" + val)}</Value></ParameterValueStruct>`;
+  });
+
+  return `<cwmp:SetParameterValues><ParameterList soap-enc:arrayType="cwmp:ParameterValueStruct[${
+    methodRequest.parameterList.length
+  }]">${params.join(
+    ""
+  )}</ParameterList><ParameterKey>${methodRequest.parameterKey ||
+    ""}</ParameterKey></cwmp:SetParameterValues>`;
+}
+
+function SetParameterValuesResponse(xml: Element): CpeSetResponse {
+  return {
+    name: "SetParameterValuesResponse",
+    status: parseInt(xml.children.find(n => n.localName === "Status").text)
+  };
+}
+
+function SetParameterAttributes(methodRequest): string {
+  const params = methodRequest.parameterList.map(p => {
+    return `<SetParameterAttributesStruct><Name>${
+      p[0]
+    }</Name><NotificationChange>${
+      p[1] == null ? "false" : "true"
+    }</NotificationChange><Notification>${
+      p[1] == null ? "" : p[1]
+    }</Notification><AccessListChange>${
+      p[2] == null ? "false" : "true"
+    }</AccessListChange><AccessList soap-enc:arrayType="xsd:string[${
+      (p[2] || []).length
+    }]">${
+      p[2] == null
+        ? ""
+        : p[2].map(s => `<string>${encodeEntities(s)}</string>`).join("")
+    }</AccessList></SetParameterAttributesStruct>`;
+  });
+
+  return `<cwmp:SetParameterAttributes><ParameterList soap-enc:arrayType="cwmp:SetParameterAttributesStruct[${
+    methodRequest.parameterList.length
+  }]">${params.join("")}</ParameterList></cwmp:SetParameterAttributes>`;
+}
+
+function SetParameterAttributesResponse(): CpeSetResponse {
+  return {
+    name: "SetParameterAttributesResponse"
+  };
+}
+
+function AddObject(methodRequest): string {
+  return `<cwmp:AddObject><ObjectName>${
+    methodRequest.objectName
+  }</ObjectName><ParameterKey>${methodRequest.parameterKey ||
+    ""}</ParameterKey></cwmp:AddObject>`;
+}
+
+function AddObjectResponse(xml: Element): CpeSetResponse {
+  let instanceNumber, status;
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "InstanceNumber":
+        instanceNumber = parseInt(c.text);
+        break;
+      case "Status":
+        status = parseInt(c.text);
+        break;
+    }
+  }
+
+  return {
+    name: "AddObjectResponse",
+    instanceNumber: instanceNumber,
+    status: status
+  };
+}
+
+function DeleteObject(methodRequest): string {
+  return `<cwmp:DeleteObject><ObjectName>${
+    methodRequest.objectName
+  }</ObjectName><ParameterKey>${methodRequest.parameterKey ||
+    ""}</ParameterKey></cwmp:DeleteObject>`;
+}
+
+function DeleteObjectResponse(xml: Element): CpeSetResponse {
+  return {
+    name: "DeleteObjectResponse",
+    status: parseInt(xml.children.find(n => n.localName === "Status").text)
+  };
+}
+
+function Reboot(methodRequest): string {
+  return `<cwmp:Reboot><CommandKey>${methodRequest.commandKey ||
+    ""}</CommandKey></cwmp:Reboot>`;
+}
+
+function RebootResponse(): CpeSetResponse {
+  return {
+    name: "RebootResponse"
+  };
+}
+
+function FactoryReset(): string {
+  return "<cwmp:FactoryReset></cwmp:FactoryReset>";
+}
+
+function FactoryResetResponse(): CpeSetResponse {
+  return {
+    name: "FactoryResetResponse"
+  };
+}
+
+function Download(methodRequest): string {
+  return `<cwmp:Download><CommandKey>${methodRequest.commandKey ||
+    ""}</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
+    methodRequest.url
+  }</URL><Username>${encodeEntities(
+    methodRequest.username || ""
+  )}</Username><Password>${encodeEntities(
+    methodRequest.password || ""
+  )}</Password><FileSize>${methodRequest.fileSize ||
+    "0"}</FileSize><TargetFileName>${encodeEntities(
+    methodRequest.targetFileName || ""
+  )}</TargetFileName><DelaySeconds>${methodRequest.delaySeconds ||
+    "0"}</DelaySeconds><SuccessURL>${encodeEntities(
+    methodRequest.successUrl || ""
+  )}</SuccessURL><FailureURL>${encodeEntities(
+    methodRequest.failureUrl || ""
+  )}</FailureURL></cwmp:Download>`;
+}
+
+function DownloadResponse(xml: Element): CpeSetResponse {
+  let status, startTime, completeTime;
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "Status":
+        status = parseInt(c.text);
+        break;
+      case "StartTime":
+        startTime = Date.parse(c.text);
+        break;
+      case "CompleteTime":
+        completeTime = Date.parse(c.text);
+        break;
+    }
+  }
+
+  return {
+    name: "DownloadResponse",
+    status: status,
+    startTime: startTime,
+    completeTime: completeTime
+  };
+}
+
+function Inform(xml: Element): InformRequest {
+  let retryCount, evnt, parameterList;
+  const deviceId = {
+    Manufacturer: null,
+    OUI: null,
+    ProductClass: null,
+    SerialNumber: null
+  };
+
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "ParameterList":
+        parameterList = parameterValueList(c);
+        break;
+      case "DeviceId":
+        for (const cc of c.children) {
+          const n = cc.localName;
+          if (n in deviceId) deviceId[n] = decodeEntities(cc.text);
+        }
+        break;
+      case "Event":
+        evnt = event(c);
+        break;
+      case "RetryCount":
+        retryCount = parseInt(c.text);
+        break;
+    }
+  }
+
+  return {
+    name: "Inform",
+    parameterList: parameterList,
+    deviceId: deviceId,
+    event: evnt,
+    retryCount: retryCount
+  };
+}
+
+function InformResponse(): string {
+  return "<cwmp:InformResponse><MaxEnvelopes>1</MaxEnvelopes></cwmp:InformResponse>";
+}
+
+function GetRPCMethods(): AcsResponse {
+  return { name: "GetRPCMethods" };
+}
+
+function GetRPCMethodsResponse(methodResponse): string {
+  return `<cwmp:GetRPCMethodsResponse><MethodList soap-enc:arrayType="xsd:string[${
+    methodResponse.methodList.length
+  }]">${methodResponse.methodList
+    .map(m => `<string>${m}</string>`)
+    .join("")}</MethodList></cwmp:GetRPCMethodsResponse>`;
+}
+
+function TransferComplete(xml: Element): TransferCompleteRequest {
+  let commandKey, _faultStruct, startTime, completeTime;
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "CommandKey":
+        commandKey = c.text;
+        break;
+      case "FaultStruct":
+        _faultStruct = faultStruct(c);
+        break;
+      case "StartTime":
+        startTime = Date.parse(c.text);
+        break;
+      case "CompleteTime":
+        completeTime = Date.parse(c.text);
+        break;
+    }
+  }
+
+  return {
+    name: "TransferComplete",
+    commandKey: commandKey,
+    faultStruct: _faultStruct,
+    startTime: startTime,
+    completeTime: completeTime
+  };
+}
+
+function TransferCompleteResponse(): string {
+  return "<cwmp:TransferCompleteResponse></cwmp:TransferCompleteResponse>";
+}
+
+function RequestDownload(xml: Element): CpeRequest {
+  return {
+    name: "RequestDownload",
+    fileType: xml.children.find(n => n.localName === "FileType").text
+  };
+}
+
+function RequestDownloadResponse(): string {
+  return "<cwmp:RequestDownloadResponse></cwmp:RequestDownloadResponse>";
+}
+
+function faultStruct(xml: Element): FaultStruct {
+  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "FaultCode":
+        faultCode = c.text;
+        break;
+      case "FaultString":
+        faultString = decodeEntities(c.text);
+        break;
+      case "SetParameterValuesFault":
+        setParameterValuesFault = setParameterValuesFault || [];
+        for (const cc of c.children) {
+          switch (cc.localName) {
+            case "ParameterName":
+              pn = cc.text;
+              break;
+            case "FaultCode":
+              fc = cc.text;
+              break;
+            case "FaultString":
+              fs = decodeEntities(cc.text);
+              break;
+          }
+        }
+        setParameterValuesFault.push({
+          parameterName: pn,
+          faultCode: fc,
+          faultString: fs
+        });
+    }
+  }
+
+  return { faultCode, faultString, setParameterValuesFault };
+}
+
+function fault(xml: Element): CpeFault {
+  let faultCode, faultString, detail;
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "faultcode":
+        faultCode = c.text;
+        break;
+      case "faultstring":
+        faultString = decodeEntities(c.text);
+        break;
+      case "detail":
+        detail = faultStruct(c.children.find(n => n.localName === "Fault"));
+        break;
+    }
+  }
+
+  return { faultCode, faultString, detail };
+}
+
+export function request(body: string, cwmpVersion, warn): SoapMessage {
+  warnings = warn;
+
+  const rpc = {
+    id: null,
+    cwmpVersion: cwmpVersion,
+    sessionTimeout: null,
+    cpeRequest: null,
+    cpeFault: null,
+    cpeResponse: null
+  };
+
+  if (!body.length) return rpc;
+
+  const xml = parseXml(body);
+
+  if (!xml.children.length) return rpc;
+
+  const envelope = xml.children[0];
+
+  let headerElement: Element, bodyElement: Element;
+
+  for (const c of envelope.children) {
+    switch (c.localName) {
+      case "Header":
+        headerElement = c;
+        break;
+      case "Body":
+        bodyElement = c;
+        break;
+    }
+  }
+
+  if (headerElement) {
+    for (const c of headerElement.children) {
+      switch (c.localName) {
+        case "ID":
+          rpc.id = decodeEntities(c.text);
+          break;
+        case "sessionTimeout":
+          rpc.sessionTimeout = parseInt(c.text);
+          break;
+      }
+    }
+  }
+
+  const methodElement = bodyElement.children[0];
+
+  if (!rpc.cwmpVersion && methodElement.localName !== "Fault") {
+    let namespace, namespaceHref;
+    for (const e of [methodElement, bodyElement, envelope]) {
+      namespace = namespace || e.namespace;
+      if (e.attrs) {
+        const attrs = memoizedParseAttrs(e.attrs);
+        const attr = namespace
+          ? attrs.find(
+              s => s.namespace === "xmlns" && s.localName === namespace
+            )
+          : attrs.find(s => s.name === "xmlns");
+
+        if (attr) namespaceHref = attr.value;
+      }
+    }
+
+    switch (namespaceHref) {
+      case "urn:dslforum-org:cwmp-1-0":
+        rpc.cwmpVersion = "1.0";
+        break;
+      case "urn:dslforum-org:cwmp-1-1":
+        rpc.cwmpVersion = "1.1";
+        break;
+      case "urn:dslforum-org:cwmp-1-2":
+        if (rpc.sessionTimeout) rpc.cwmpVersion = "1.3";
+        else rpc.cwmpVersion = "1.2";
+
+        break;
+      case "urn:dslforum-org:cwmp-1-3":
+        rpc.cwmpVersion = "1.4";
+        break;
+      default:
+        throw new Error("Unrecognized CWMP version");
+    }
+  }
+
+  switch (methodElement.localName) {
+    case "Inform":
+      rpc.cpeRequest = Inform(methodElement);
+      break;
+    case "GetRPCMethods":
+      rpc.cpeRequest = GetRPCMethods();
+      break;
+    case "TransferComplete":
+      rpc.cpeRequest = TransferComplete(methodElement);
+      break;
+    case "RequestDownload":
+      rpc.cpeRequest = RequestDownload(methodElement);
+      break;
+    case "GetParameterNamesResponse":
+      rpc.cpeResponse = GetParameterNamesResponse(methodElement);
+      break;
+    case "GetParameterValuesResponse":
+      rpc.cpeResponse = GetParameterValuesResponse(methodElement);
+      break;
+    case "GetParameterAttributesResponse":
+      rpc.cpeResponse = GetParameterAttributesResponse(methodElement);
+      break;
+    case "SetParameterValuesResponse":
+      rpc.cpeResponse = SetParameterValuesResponse(methodElement);
+      break;
+    case "SetParameterAttributesResponse":
+      rpc.cpeResponse = SetParameterAttributesResponse();
+      break;
+    case "AddObjectResponse":
+      rpc.cpeResponse = AddObjectResponse(methodElement);
+      break;
+    case "DeleteObjectResponse":
+      rpc.cpeResponse = DeleteObjectResponse(methodElement);
+      break;
+    case "RebootResponse":
+      rpc.cpeResponse = RebootResponse();
+      break;
+    case "FactoryResetResponse":
+      rpc.cpeResponse = FactoryResetResponse();
+      break;
+    case "DownloadResponse":
+      rpc.cpeResponse = DownloadResponse(methodElement);
+      break;
+    case "Fault":
+      rpc.cpeFault = fault(methodElement);
+      break;
+    default:
+      throw new Error(`8000 Method not supported ${methodElement.localName}`);
+  }
+
+  return rpc;
+}
+
+const namespacesAttrs = {
+  "1.0": Object.entries(NAMESPACES["1.0"])
+    .map(([k, v]) => `xmlns:${k}="${v}"`)
+    .join(" "),
+  "1.1": Object.entries(NAMESPACES["1.1"])
+    .map(([k, v]) => `xmlns:${k}="${v}"`)
+    .join(" "),
+  "1.2": Object.entries(NAMESPACES["1.2"])
+    .map(([k, v]) => `xmlns:${k}="${v}"`)
+    .join(" "),
+  "1.3": Object.entries(NAMESPACES["1.3"])
+    .map(([k, v]) => `xmlns:${k}="${v}"`)
+    .join(" "),
+  "1.4": Object.entries(NAMESPACES["1.4"])
+    .map(([k, v]) => `xmlns:${k}="${v}"`)
+    .join(" ")
+};
+
+export function response(rpc): { code: number; headers: {}; data: string } {
+  const headers = {
+    Server: SERVER_NAME,
+    SOAPServer: SERVER_NAME
+  };
+
+  if (!rpc) return { code: 204, headers: headers, data: "" };
+
+  let body;
+  if (rpc.acsResponse) {
+    switch (rpc.acsResponse.name) {
+      case "InformResponse":
+        body = InformResponse();
+        break;
+      case "GetRPCMethodsResponse":
+        body = GetRPCMethodsResponse(rpc.acsResponse);
+        break;
+      case "TransferCompleteResponse":
+        body = TransferCompleteResponse();
+        break;
+      case "RequestDownloadResponse":
+        body = RequestDownloadResponse();
+        break;
+      default:
+        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
+    }
+  } else if (rpc.acsRequest) {
+    switch (rpc.acsRequest.name) {
+      case "GetParameterNames":
+        body = GetParameterNames(rpc.acsRequest);
+        break;
+      case "GetParameterValues":
+        body = GetParameterValues(rpc.acsRequest);
+        break;
+      case "GetParameterAttributes":
+        body = GetParameterAttributes(rpc.acsRequest);
+        break;
+      case "SetParameterValues":
+        body = SetParameterValues(rpc.acsRequest);
+        break;
+      case "SetParameterAttributes":
+        body = SetParameterAttributes(rpc.acsRequest);
+        break;
+      case "AddObject":
+        body = AddObject(rpc.acsRequest);
+        break;
+      case "DeleteObject":
+        body = DeleteObject(rpc.acsRequest);
+        break;
+      case "Reboot":
+        body = Reboot(rpc.acsRequest);
+        break;
+      case "FactoryReset":
+        body = FactoryReset();
+        break;
+      case "Download":
+        body = Download(rpc.acsRequest);
+        break;
+      default:
+        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
+    }
+  }
+
+  headers["Content-Type"] = 'text/xml; charset="utf-8"';
+  return {
+    code: 200,
+    headers: headers,
+    data: `<?xml version="1.0" encoding="UTF-8"?>\n<soap-env:Envelope ${
+      namespacesAttrs[rpc.cwmpVersion]
+    }><soap-env:Header><cwmp:ID soap-env:mustUnderstand="1">${
+      rpc.id
+    }</cwmp:ID></soap-env:Header><soap-env:Body>${body}</soap-env:Body></soap-env:Envelope>`
+  };
+}

lib/types.ts

@@ -0,0 +1,392 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Path from "./common/path";
+import PathSet from "./common/path-set";
+import VersionedMap from "./versioned-map";
+import InstanceSet from "./instance-set";
+import { IncomingMessage, ServerResponse } from "http";
+
+export type Expression = string | number | boolean | null | any[];
+
+export interface Fault {
+  code: string;
+  message: string;
+  detail?:
+    | FaultStruct
+    | {
+        name: string;
+        message: string;
+        stack?: string;
+      };
+  timestamp?: number;
+}
+
+export interface SessionFault extends Fault {
+  timestamp: number;
+  provisions: string[][];
+  retryNow?: boolean;
+  precondition?: boolean;
+  retries?: number;
+  expiry?: number;
+}
+
+export interface Attributes {
+  object?: [number, 1 | 0];
+  writable?: [number, 1 | 0];
+  value?: [number, [string | number | boolean, string]];
+  notification?: [number, number];
+  accessList?: [number, string[]];
+}
+
+export interface AttributeTimestamps {
+  object?: number;
+  writable?: number;
+  value?: number;
+  notification?: number;
+  accessList?: number;
+}
+
+export interface AttributeValues {
+  object?: boolean;
+  writable?: boolean;
+  value?: [string | number | boolean, string?];
+  notification?: number;
+  accessList?: string[];
+}
+
+export interface DeviceData {
+  paths: PathSet;
+  timestamps: VersionedMap<Path, number>;
+  attributes: VersionedMap<Path, Attributes>;
+  trackers: Map<Path, { [name: string]: number }>;
+  changes: Set<string>;
+}
+
+export type VirtualParameterDeclaration = [
+  Path,
+  {
+    path?: number;
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  }?,
+  {
+    path?: [number, number];
+    object?: boolean;
+    writable?: boolean;
+    value?: [string | number | boolean, string?];
+    notification?: number;
+    accessList?: string[];
+  }?
+];
+
+export interface SyncState {
+  refreshAttributes: {
+    exist: Set<Path>;
+    object: Set<Path>;
+    writable: Set<Path>;
+    value: Set<Path>;
+    notification: Set<Path>;
+    accessList: Set<Path>;
+  };
+  spv: Map<Path, [string | number | boolean, string]>;
+  spa: Map<Path, { notification: number; accessList: string[] }>;
+  gpn: Set<Path>;
+  gpnPatterns: Map<Path, number>;
+  tags: Map<Path, boolean>;
+  virtualParameterDeclarations: VirtualParameterDeclaration[][];
+  instancesToDelete: Map<Path, Set<Path>>;
+  instancesToCreate: Map<Path, InstanceSet>;
+  downloadsToDelete: Set<Path>;
+  downloadsToCreate: InstanceSet;
+  downloadsValues: Map<Path, string | number>;
+  downloadsDownload: Map<Path, number>;
+  reboot: number;
+  factoryReset: number;
+}
+
+export interface SessionContext {
+  sessionId?: string;
+  timestamp: number;
+  deviceId: string;
+  deviceData: DeviceData;
+  cwmpVersion: string;
+  timeout: number;
+  provisions: any[];
+  channels: { [channel: string]: number };
+  virtualParameters: [
+    string,
+    AttributeTimestamps,
+    AttributeValues,
+    AttributeTimestamps,
+    AttributeValues
+  ][][];
+  revisions: number[];
+  rpcCount: number;
+  iteration: number;
+  cycle: number;
+  extensionsCache: any;
+  declarations: Declaration[][];
+  faults?: { [channel: string]: SessionFault };
+  retries?: { [channel: string]: number };
+  cacheSnapshot?: string;
+  configContext?: { [name: string]: any };
+  httpResponse?: ServerResponse;
+  httpRequest?: IncomingMessage;
+  faultsTouched?: { [channel: string]: boolean };
+  presetCycles?: number;
+  new?: boolean;
+  debug?: boolean;
+  state: number;
+  tasks?: Task[];
+  operations?: { [commandKey: string]: Operation };
+  cacheUntil?: number;
+  syncState?: SyncState;
+  lastActivity?: number;
+  rpcRequest?: AcsRequest;
+  operationsTouched?: { [commandKey: string]: 1 | 0 };
+  provisionsRet?: any[];
+  doneTasks?: string[];
+}
+
+export interface Task {
+  _id: string;
+  name: string;
+  parameterNames?: string[];
+  parameterValues?: [string, string | number | boolean, string?][];
+  objectName?: string;
+  fileType?: string;
+  fileName?: string;
+  targetFileName?: string;
+  expiry?: number;
+  provisions?: (string | number | boolean)[][];
+}
+
+export interface Operation {
+  name: string;
+  timestamp: number;
+  provisions: string[][];
+  channels: { [channel: string]: number };
+  retries: { [channel: string]: number };
+  args: {
+    instance: string;
+    fileType: string;
+    fileName: string;
+    targetFileName: string;
+  };
+}
+
+export interface AcsRequest {
+  name: string;
+  next?: string;
+}
+
+export interface GetAcsRequest extends AcsRequest {
+  name: "GetParameterNames" | "GetParameterValues" | "GetParameterAttributes";
+  objectName?: string;
+  parameterNames?: string[];
+  parameterPath?: string;
+  nextLevel?: boolean;
+  instanceValues?: { [name: string]: string };
+}
+
+export interface SetAcsRequest extends AcsRequest {
+  name:
+    | "SetParameterValues"
+    | "SetParameterAttributes"
+    | "AddObject"
+    | "DeleteObject"
+    | "FactoryReset"
+    | "Reboot"
+    | "Download";
+  parameterList?:
+    | [string, string | number | boolean, string][]
+    | [string, number, string[]][];
+  instanceValues?: { [param: string]: string | number | boolean };
+  objectName?: string;
+  DATETIME_MILLISECONDS?: boolean;
+  BOOLEAN_LITERAL?: boolean;
+  commandKey?: string;
+  instance?: string;
+  fileType?: string;
+  fileSize?: number;
+  url?: string;
+  fileName?: string;
+  targetFileName?: string;
+}
+
+export interface CpeResponse {
+  name: string;
+}
+
+export interface SpvFault {
+  parameterName: string;
+  faultCode: string;
+  faultString: string;
+}
+
+export interface FaultStruct {
+  faultCode: string;
+  faultString: string;
+  setParameterValuesFault?: SpvFault[];
+}
+
+export interface CpeFault {
+  faultCode: string;
+  faultString: string;
+  detail?: FaultStruct;
+}
+
+export interface CpeGetResponse extends CpeResponse {
+  name:
+    | "GetParameterNamesResponse"
+    | "GetParameterValuesResponse"
+    | "GetParameterAttributesResponse";
+  parameterList?:
+    | [string, boolean][]
+    | [string, string | number | boolean, string][]
+    | [string, number, string[]][];
+}
+
+export interface CpeSetResponse extends CpeResponse {
+  name:
+    | "SetParameterValuesResponse"
+    | "SetParameterAttributesResponse"
+    | "AddObjectResponse"
+    | "DeleteObjectResponse"
+    | "RebootResponse"
+    | "FactoryResetResponse"
+    | "DownloadResponse";
+  status?: number;
+  instanceNumber?: string;
+  startTime?: number;
+  completeTime?: number;
+}
+
+export interface CpeRequest {
+  name: string;
+  fileType?: string;
+}
+
+export interface InformRequest extends CpeRequest {
+  name: "Inform";
+  deviceId: {
+    Manufacturer: string;
+    OUI: string;
+    ProductClass?: string;
+    SerialNumber: string;
+  };
+  event: string[];
+  retryCount: number;
+  parameterList: [string, string | number | boolean, string][];
+}
+
+export interface TransferCompleteRequest extends CpeRequest {
+  name: "TransferComplete";
+  commandKey?: string;
+  faultStruct?: FaultStruct;
+  startTime?: number;
+  completeTime?: number;
+}
+
+export interface AcsResponse {
+  name: string;
+  commandKey?: string;
+  faultStruct?: FaultStruct;
+}
+
+export interface QueryOptions {
+  projection?: any;
+  skip?: number;
+  limit?: number;
+  sort?: {
+    [param: string]: number;
+  };
+}
+
+export interface Declaration {
+  path: Path;
+  pathGet: number;
+  pathSet?: [number, number];
+  attrGet?: {
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  };
+  attrSet?: {
+    object?: boolean;
+    writable?: boolean;
+    value?: [string | number | boolean, string?];
+    notification?: number;
+    accessList?: string[];
+  };
+  defer: boolean;
+}
+
+export type Clear = [
+  Path,
+  number,
+  {
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  }?,
+  number?
+];
+
+export interface Preset {
+  name: string;
+  channel: string;
+  schedule?: { md5: string; duration: number; schedule: any };
+  events?: { [event: string]: boolean };
+  precondition?: {};
+  provisions: string[][];
+}
+
+export type PermissionSet = {
+  [resource: string]: {
+    access: number;
+    validate?: Expression;
+    filter: Expression;
+  };
+}[];
+
+export interface SoapMessage {
+  id: string;
+  cwmpVersion: string;
+  sessionTimeout: number;
+  cpeRequest?: CpeRequest;
+  cpeFault?: CpeFault;
+  cpeResponse?: CpeResponse;
+}
+
+export interface ScriptResult {
+  fault: Fault;
+  clear: Clear[];
+  declare: Declaration[];
+  done: boolean;
+  returnValue: any;
+}

lib/ui.ts

@@ -0,0 +1,250 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { Z_SYNC_FLUSH } from "zlib";
+import Koa from "koa";
+import Router from "koa-router";
+import * as jwt from "jsonwebtoken";
+import koaStatic from "koa-static";
+import koaCompress from "koa-compress";
+import koaBodyParser from "koa-bodyparser";
+import koaJwt from "koa-jwt";
+import * as config from "./config";
+import api from "./ui/api";
+import Authorizer from "./common/authorizer";
+import * as logger from "./logger";
+import * as localCache from "./local-cache";
+import { PermissionSet } from "./types";
+import { authLocal } from "./ui/api-functions";
+import * as init from "./init";
+import { version as VERSION } from "../package.json";
+
+declare module "koa" {
+  interface Request {
+    body: any;
+  }
+}
+
+const koa = new Koa();
+const router = new Router();
+
+const JWT_SECRET = "" + config.get("UI_JWT_SECRET");
+const JWT_COOKIE = "genieacs-ui-jwt";
+
+koa.on("error", async err => {
+  throw err;
+});
+
+koa.use(async (ctx, next) => {
+  const configSnapshot = await localCache.getCurrentSnapshot();
+  ctx.state.configSnapshot = configSnapshot;
+  ctx.set("X-Config-Snapshot", configSnapshot);
+  ctx.set("GenieACS-Version", VERSION);
+  return next();
+});
+
+koa.use(
+  koaJwt({
+    secret: JWT_SECRET,
+    passthrough: true,
+    cookie: JWT_COOKIE,
+    isRevoked: async (ctx, token) => {
+      if (token["authMethod"] === "local") {
+        return !localCache.getUsers(ctx.state.configSnapshot)[
+          token["username"]
+        ];
+      }
+
+      return true;
+    }
+  })
+);
+
+koa.use(async (ctx, next) => {
+  const permissionSets: PermissionSet[] = [];
+
+  if (ctx.state.user && ctx.state.user.username) {
+    const allPermissions = localCache.getPermissions(ctx.state.configSnapshot);
+
+    let user;
+    if (ctx.state.user.authMethod === "local") {
+      user = localCache.getUsers(ctx.state.configSnapshot)[
+        ctx.state.user.username
+      ];
+    } else {
+      throw new Error("Invalid auth method");
+    }
+
+    for (const role of (user.roles || []).sort())
+      permissionSets.push(Object.values(allPermissions[role] || {}));
+  }
+
+  ctx.state.authorizer = new Authorizer(permissionSets);
+
+  return next();
+});
+
+router.post("/login", async ctx => {
+  if (!JWT_SECRET) {
+    ctx.status = 500;
+    ctx.body = "UI_JWT_SECRET is not set";
+    logger.error({ message: "UI_JWT_SECRET is not set" });
+    return;
+  }
+
+  const username = ctx.request.body.username;
+  const password = ctx.request.body.password;
+
+  const log = {
+    message: "Log in",
+    context: ctx,
+    username: username,
+    method: null
+  };
+
+  function success(authMethod): void {
+    log.method = authMethod;
+    const token = jwt.sign({ username, authMethod }, JWT_SECRET);
+    ctx.cookies.set(JWT_COOKIE, token, { sameSite: "lax" });
+    ctx.body = JSON.stringify(token);
+    logger.accessInfo(log);
+  }
+
+  function failure(): void {
+    ctx.status = 400;
+    ctx.body = "Incorrect username or password";
+    log.message += " failed";
+    logger.accessWarn(log);
+  }
+
+  if (await authLocal(ctx.state.configSnapshot, username, password))
+    return void success("local");
+
+  failure();
+});
+
+router.post("/logout", async ctx => {
+  ctx.cookies.set(JWT_COOKIE); // Delete cookie
+  ctx.body = "";
+
+  logger.accessInfo({
+    message: "Log out",
+    context: ctx
+  });
+});
+
+koa.use(async (ctx, next) => {
+  if (ctx.request.type === "application/octet-stream")
+    ctx.disableBodyParser = true;
+
+  return next();
+});
+
+koa.use(koaBodyParser());
+router.use("/api", api.routes(), api.allowedMethods());
+
+router.get("/status", ctx => {
+  ctx.body = "OK";
+});
+
+router.get("/init", async ctx => {
+  const status = await init.getStatus();
+  if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
+    if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
+    if (!ctx.state.authorizer.hasAccess("permissions", 3))
+      status["users"] = false;
+    if (!ctx.state.authorizer.hasAccess("config", 3)) {
+      status["filters"] = false;
+      status["device"] = false;
+      status["index"] = false;
+      status["overview"] = false;
+    }
+    if (!ctx.state.authorizer.hasAccess("presets", 3))
+      status["presets"] = false;
+    if (!ctx.state.authorizer.hasAccess("provisions", 3))
+      status["presets"] = false;
+  }
+
+  ctx.body = status;
+});
+
+router.post("/init", async ctx => {
+  const status = ctx.request.body;
+  if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
+    if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
+    if (!ctx.state.authorizer.hasAccess("permissions", 3))
+      status["users"] = false;
+    if (!ctx.state.authorizer.hasAccess("config", 3)) {
+      status["filters"] = false;
+      status["device"] = false;
+      status["index"] = false;
+      status["overview"] = false;
+    }
+    if (!ctx.state.authorizer.hasAccess("presets", 3))
+      status["presets"] = false;
+    if (!ctx.state.authorizer.hasAccess("provisions", 3))
+      status["presets"] = false;
+  }
+  await init.seed(status);
+  ctx.body = "";
+});
+
+router.get("/", async ctx => {
+  const permissionSets = ctx.state.authorizer.getPermissionSets();
+
+  let wizard = "";
+  if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)
+    wizard = '<script>window.location.hash = "#!/wizard";</script>';
+
+  ctx.body = `
+  <html>
+    <head>
+      <title>GenieACS</title>
+      <link rel="shortcut icon" type="image/png" href="favicon.png" />
+      <link rel="stylesheet" href="app.css">
+    </head>
+    <body>
+    <noscript>GenieACS UI requires JavaScript to work. Please enable JavaScript in your browser.</noscript>
+      <script>
+        window.clientConfig = ${JSON.stringify({
+          ui: localCache.getUiConfig(ctx.state.configSnapshot)
+        })};
+        window.configSnapshot = ${JSON.stringify(ctx.state.configSnapshot)};
+        window.genieacsVersion = ${JSON.stringify(VERSION)};
+        window.username = ${JSON.stringify(
+          ctx.state.user ? ctx.state.user.username : ""
+        )};
+        window.permissionSets = ${JSON.stringify(permissionSets)};
+      </script>
+      <script src="app.js"></script>${wizard} 
+    </body>
+  </html>
+  `;
+});
+
+koa.use(
+  koaCompress({
+    flush: Z_SYNC_FLUSH
+  })
+);
+
+koa.use(router.routes());
+koa.use(koaStatic(config.ROOT_DIR + "/public"));
+
+export const listener = koa.callback();

lib/ui/api-functions.ts

@@ -0,0 +1,183 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { ObjectID } from "mongodb";
+import * as db from "./db";
+import { del } from "../cache";
+import { getUsers } from "../local-cache";
+import { hashPassword } from "../auth";
+import {
+  insertTasks,
+  watchTask,
+  connectionRequest,
+  deleteDevice
+} from "../api-functions";
+
+async function deleteFault(id): Promise<void> {
+  const deviceId = id.split(":", 1)[0];
+  const channel = id.slice(deviceId.length + 1);
+
+  await Promise.all([
+    db.deleteFault(id),
+    channel.startsWith("task_")
+      ? db.deleteTask(new ObjectID(channel.slice(5)))
+      : null
+  ]);
+
+  await del(`${deviceId}_tasks_faults_operations`);
+}
+
+export async function deleteResource(resource, id): Promise<void> {
+  switch (resource) {
+    case "devices":
+      await deleteDevice(id);
+      break;
+
+    case "files":
+      await db.deleteFile(id);
+      break;
+
+    case "faults":
+      await deleteFault(id);
+      break;
+
+    case "provisions":
+      await db.deleteProvision(id);
+      break;
+
+    case "presets":
+      await db.deletePreset(id);
+      break;
+
+    case "virtualParameters":
+      await db.deleteVirtualParameter(id);
+      break;
+
+    case "config":
+      await db.deleteConfig(id);
+      break;
+
+    case "permissions":
+      await db.deletePermission(id);
+      break;
+
+    case "users":
+      await db.deleteUser(id);
+      break;
+  }
+
+  await del("presets_hash");
+}
+
+export async function postTasks(
+  deviceId,
+  tasks,
+  timeout
+): Promise<{ connectionRequest: string; tasks: any[] }> {
+  for (const task of tasks) {
+    delete task._id;
+    task.device = deviceId;
+  }
+
+  tasks = await insertTasks(tasks);
+  const statuses = tasks.map(t => {
+    return { _id: t._id, status: "pending" };
+  });
+
+  await del(`${deviceId}_tasks_faults_operations`);
+
+  try {
+    await connectionRequest(deviceId);
+  } catch (err) {
+    return {
+      connectionRequest: err.message,
+      tasks: statuses
+    };
+  }
+
+  const sample = tasks[tasks.length - 1];
+
+  // Waiting for session to finish or timeout
+  await watchTask(deviceId, sample._id, timeout);
+
+  const promises = [];
+  for (const s of statuses) {
+    promises.push(db.query("tasks", ["=", ["PARAM", "_id"], s._id]));
+    promises.push(
+      db.query("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
+    );
+  }
+
+  const res = await Promise.all(promises);
+  for (const [i, r] of statuses.entries()) {
+    if (res[i * 2].length === 0) {
+      r.status = "done";
+    } else if (res[i * 2 + 1].length === 1) {
+      r.status = "fault";
+      r.fault = res[i * 2 + 1][0];
+    }
+    db.deleteTask(r._id);
+  }
+
+  return { connectionRequest: "OK", tasks: statuses };
+}
+
+interface PingResponse {
+  packetsTransmitted: number;
+  packetsReceived: number;
+  packetLoss: string;
+  min: number;
+  avg: number;
+  max: number;
+  mdev: number;
+}
+
+export async function putResource(resource, id, data): Promise<void> {
+  if (resource === "presets") {
+    await db.putPreset(id, data);
+  } else if (resource === "provisions") {
+    await db.putProvision(id, data);
+  } else if (resource === "virtualParameters") {
+    await db.putVirtualParameter(id, data);
+  } else if (resource === "config") {
+    await db.putConfig(id, data);
+  } else if (resource === "permissions") {
+    await db.putPermission(id, data);
+  } else if (resource === "users") {
+    delete data.password;
+    delete data.salt;
+    await db.putUser(id, data);
+  }
+
+  await del("presets_hash");
+}
+
+export function authLocal(snapshot, username, password): Promise<boolean> {
+  return new Promise((resolve, reject) => {
+    const users = getUsers(snapshot);
+    const user = users[username];
+    if (!user || !user.password) return void resolve(null);
+    hashPassword(password, user.salt)
+      .then(hash => {
+        if (hash === user.password) resolve(true);
+        else resolve(false);
+      })
+      .catch(reject);
+  });
+}

lib/ui/api.ts

@@ -0,0 +1,625 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as stream from "stream";
+import Router from "koa-router";
+import * as db from "./db";
+import * as apiFunctions from "./api-functions";
+import { evaluate, and, extractParams } from "../common/expression";
+import { parse } from "../common/expression-parser";
+import * as logger from "../logger";
+import { getConfig } from "../local-cache";
+import { QueryOptions, Expression } from "../types";
+import { generateSalt, hashPassword } from "../auth";
+import { del } from "../cache";
+import Authorizer from "../common/authorizer";
+import { ping } from "../ping";
+
+const router = new Router();
+export default router;
+
+function logUnauthorizedWarning(log): void {
+  log.message += " not authorized";
+  logger.accessWarn(log);
+}
+
+const RESOURCE_DELETE = 1 << 0;
+const RESOURCE_PUT = 1 << 1;
+
+const RESOURCE_IDS = {
+  devices: "DeviceID.ID",
+  presets: "_id",
+  provisions: "_id",
+  files: "_id",
+  virtualParameters: "_id",
+  config: "_id",
+  permissions: "_id",
+  users: "_id",
+  faults: "_id",
+  tasks: "_id"
+};
+
+const resources = {
+  devices: 0 | RESOURCE_DELETE,
+  presets: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  provisions: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  files: 0 | RESOURCE_DELETE,
+  virtualParameters: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  config: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  permissions: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  users: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  faults: 0 | RESOURCE_DELETE,
+  tasks: 0
+};
+
+router.get(`/devices/:id.csv`, async ctx => {
+  const authorizer: Authorizer = ctx.state.authorizer;
+  const log = {
+    message: "Query device (CSV)",
+    context: ctx,
+    id: ctx.params.id
+  };
+
+  const filter = and(authorizer.getFilter("devices", 2), [
+    "=",
+    ["PARAM", RESOURCE_IDS.devices],
+    ctx.params.id
+  ]);
+
+  if (!authorizer.hasAccess("devices", 2)) {
+    logUnauthorizedWarning(log);
+    return void (ctx.status = 404);
+  }
+
+  const res = await db.query("devices", filter);
+  if (!res[0]) return void (ctx.status = 404);
+
+  ctx.type = "text/csv";
+  ctx.attachment(
+    `device-${ctx.params.id}-${new Date()
+      .toISOString()
+      .replace(/[:.]/g, "")}.csv`
+  );
+
+  ctx.body = new stream.PassThrough();
+  ctx.body.write(
+    "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp,Notification,Notification timestamp,Access list,Access list timestamp\n"
+  );
+
+  for (const k of Object.keys(res[0]).sort()) {
+    const p = res[0][k];
+    const row = [
+      k,
+      p.object,
+      p.objectTimestamp,
+      p.writable,
+      p.writableTimestamp,
+      p.value != null ? `"${p.value[0].toString().replace(/"/g, '""')}"` : "",
+      p.value != null ? p.value[1] : "",
+      p.valueTimestamp,
+      p.notification,
+      p.notificationTimestamp,
+      p.accessList ? p.accessList.join(", ") : "",
+      p.accessListTimestamp
+    ];
+    ctx.body.write(row.map(r => (r != null ? r : "")).join(",") + "\n");
+  }
+  ctx.body.end();
+  logger.accessInfo(log);
+});
+
+for (const [resource, flags] of Object.entries(resources)) {
+  router.head(`/${resource}`, async (ctx, next) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
+    let filter: Expression = authorizer.getFilter(resource, 1);
+    if (ctx.request.query.filter)
+      filter = and(filter, parse(ctx.request.query.filter));
+
+    const log = {
+      message: `Count ${resource}`,
+      context: ctx,
+      filter: ctx.request.query.filter,
+      count: null
+    };
+
+    if (!authorizer.hasAccess(resource, 1)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    // Exclude temporary tasks and faults
+    if (resource === "tasks" || resource === "faults") {
+      filter = and(filter, [
+        "NOT",
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+      ]);
+    }
+
+    const count = await db.count(resource, filter);
+
+    ctx.set("X-Total-Count", `${count}`);
+    ctx.body = "";
+
+    log.count = count;
+    logger.accessInfo(log);
+  });
+
+  router.get(`/${resource}`, async (ctx, next) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
+    const options: QueryOptions = {};
+    let filter: Expression = authorizer.getFilter(resource, 2);
+    if (ctx.request.query.filter)
+      filter = and(filter, parse(ctx.request.query.filter));
+    if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
+    if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
+    if (ctx.request.query.sort)
+      options.sort = JSON.parse(ctx.request.query.sort);
+    if (ctx.request.query.projection) {
+      options.projection = ctx.request.query.projection
+        .split(",")
+        .reduce((obj, k) => Object.assign(obj, { [k]: 1 }), {});
+    }
+
+    const log = {
+      message: `Query ${resource}`,
+      context: ctx,
+      filter: ctx.request.query.filter,
+      limit: options.limit,
+      skip: options.skip,
+      sort: options.sort,
+      projection: options.projection
+    };
+
+    if (!authorizer.hasAccess(resource, 2)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    // Exclude temporary tasks and faults
+    if (resource === "tasks" || resource === "faults") {
+      filter = and(filter, [
+        "NOT",
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+      ]);
+    }
+
+    ctx.body = new stream.PassThrough();
+    ctx.type = "application/json";
+
+    let c = 0;
+    ctx.body.write("[\n");
+    await db.query(resource, filter, options, obj => {
+      ctx.body.write((c++ ? "," : "") + JSON.stringify(obj) + "\n");
+    });
+    ctx.body.end("]");
+
+    logger.accessInfo(log);
+  });
+
+  // CSV download
+  router.get(`/${resource}.csv`, async (ctx, next) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
+    const options: QueryOptions = { projection: {} };
+    let filter: Expression = authorizer.getFilter(resource, 2);
+    if (ctx.request.query.filter)
+      filter = and(filter, parse(ctx.request.query.filter));
+    if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
+    if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
+    if (ctx.request.query.sort)
+      options.sort = JSON.parse(ctx.request.query.sort);
+
+    const log = {
+      message: `Query ${resource} (CSV)`,
+      context: ctx,
+      filter: ctx.request.query.filter,
+      limit: options.limit,
+      skip: options.skip,
+      sort: options.sort
+    };
+
+    if (!authorizer.hasAccess(resource, 2)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    const columns = JSON.parse(ctx.request.query.columns);
+    const now = Date.now();
+
+    for (const [k, v] of Object.entries(columns)) {
+      const e = evaluate(parse(v as string), null, now);
+      columns[k] = e;
+      for (const p of extractParams(e))
+        if (typeof p === "string") options.projection[p] = 1;
+    }
+
+    // Exclude temporary tasks and faults
+    if (resource === "tasks" || resource === "faults") {
+      filter = and(filter, [
+        "NOT",
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+      ]);
+    }
+
+    ctx.body = new stream.PassThrough();
+    ctx.type = "text/csv";
+    ctx.attachment(
+      `${resource}-${new Date(now).toISOString().replace(/[:.]/g, "")}.csv`
+    );
+
+    ctx.body.write(
+      Object.keys(columns).map(k => `"${k.replace(/"/, '""')}"`) + "\n"
+    );
+    await db.query(resource, filter, options, obj => {
+      const arr = Object.values(columns).map(exp => {
+        const v = evaluate(exp, obj, null, e => {
+          if (Array.isArray(e)) {
+            if (e[0] === "PARAM") {
+              if (resource === "devices") {
+                if (e[1] === "Tags") {
+                  const tags = [];
+                  for (const p in obj)
+                    if (p.startsWith("Tags.")) tags.push(p.slice(5));
+
+                  return tags.join(", ");
+                }
+              }
+            } else if (e[0] === "FUNC") {
+              if (e[1] === "DATE_STRING") {
+                if (e[2] && !Array.isArray(e[2]))
+                  return new Date(e[2]).toISOString();
+              }
+            }
+          }
+
+          return e;
+        });
+
+        if (Array.isArray(v) || v == null) return "";
+        if (typeof v === "string") return `"${v.replace(/"/g, '""')}"`;
+        return v;
+      });
+      ctx.body.write(arr.join(",") + "\n");
+    });
+    ctx.body.end();
+
+    logger.accessInfo(log);
+  });
+
+  router.head(`/${resource}/:id`, async (ctx, next) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
+    const log = {
+      message: `Count ${resource}`,
+      context: ctx,
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
+    };
+
+    const filter = and(authorizer.getFilter(resource, 2), [
+      "=",
+      ["PARAM", RESOURCE_IDS[resource]],
+      ctx.params.id
+    ]);
+    if (!authorizer.hasAccess(resource, 2)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    const res = await db.query(resource, filter);
+
+    if (!res.length) return void next();
+
+    logger.accessInfo(log);
+    ctx.body = "";
+  });
+
+  router.get(`/${resource}/:id`, async (ctx, next) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
+    const log = {
+      message: `Query ${resource}`,
+      context: ctx,
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
+    };
+
+    const filter = and(authorizer.getFilter(resource, 2), [
+      "=",
+      ["PARAM", RESOURCE_IDS[resource]],
+      ctx.params.id
+    ]);
+    if (!authorizer.hasAccess(resource, 2)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    const res = await db.query(resource, filter);
+
+    if (!res.length) return void next();
+
+    logger.accessInfo(log);
+    ctx.body = res[0];
+  });
+
+  if (flags & RESOURCE_DELETE) {
+    router.delete(`/${resource}/:id`, async (ctx, next) => {
+      const authorizer: Authorizer = ctx.state.authorizer;
+      const log = {
+        message: `Delete ${resource}`,
+        context: ctx,
+        id: ctx.params.id
+      };
+
+      const filter = and(authorizer.getFilter(resource, 3), [
+        "=",
+        ["PARAM", RESOURCE_IDS[resource]],
+        ctx.params.id
+      ]);
+      if (!authorizer.hasAccess(resource, 3)) {
+        logUnauthorizedWarning(log);
+        return void next();
+      }
+      const res = await db.query(resource, filter);
+      if (!res.length) return void next();
+
+      const validate = authorizer.getValidator(resource, res[0]);
+      if (!validate("delete")) {
+        logUnauthorizedWarning(log);
+        return void (ctx.status = 403);
+      }
+
+      await apiFunctions.deleteResource(resource, ctx.params.id);
+
+      logger.accessInfo(log);
+
+      ctx.body = "";
+    });
+  }
+
+  if (flags & RESOURCE_PUT) {
+    router.put(`/${resource}/:id`, async (ctx, next) => {
+      const authorizer: Authorizer = ctx.state.authorizer;
+      const id = ctx.params.id;
+
+      const log = {
+        message: `Put ${resource}`,
+        context: ctx,
+        id: id
+      };
+
+      if (!authorizer.hasAccess(resource, 3)) {
+        logUnauthorizedWarning(log);
+        return void next();
+      }
+
+      const obj = ctx.request.body;
+
+      const validate = authorizer.getValidator(resource, obj);
+      if (!validate("put")) {
+        logUnauthorizedWarning(log);
+        return void (ctx.status = 403);
+      }
+
+      try {
+        await apiFunctions.putResource(resource, id, obj);
+      } catch (err) {
+        log.message += " failed";
+        logger.accessWarn(log);
+        ctx.body = `${err.name}: ${err.message}`;
+        return void (ctx.status = 400);
+      }
+
+      logger.accessInfo(log);
+
+      ctx.body = "";
+    });
+  }
+}
+
+router.put("/files/:id", async (ctx, next) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
+  const resource = "files";
+  const id = ctx.params.id;
+
+  const log = {
+    message: `Upload ${resource}`,
+    context: ctx,
+    id: id,
+    metadata: null
+  };
+
+  if (!authorizer.hasAccess(resource, 3)) {
+    logUnauthorizedWarning(log);
+    return void next();
+  }
+
+  const metadata = {
+    fileType: ctx.request.headers["metadata.filetype"] || "",
+    oui: ctx.request.headers["metadata.oui"] || "",
+    productClass: ctx.request.headers["metadata.productclass"] || "",
+    version: ctx.request.headers["metadata.version"] || ""
+  };
+
+  const validate = authorizer.getValidator(resource, metadata);
+  if (!validate("put")) {
+    logUnauthorizedWarning(log);
+    return void (ctx.status = 403);
+  }
+
+  await db.putFile(id, metadata, ctx.req);
+  log.metadata = metadata;
+  logger.accessInfo(log);
+
+  ctx.body = "";
+});
+
+router.post("/devices/:id/tasks", async (ctx, next) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
+  const log = {
+    message: "Commit tasks",
+    context: ctx,
+    deviceId: ctx.params.id,
+    tasks: null
+  };
+
+  const filter = and(authorizer.getFilter("devices", 3), [
+    "=",
+    ["PARAM", "DeviceID.ID"],
+    ctx.params.id
+  ]);
+  if (!authorizer.hasAccess("devices", 3)) {
+    logUnauthorizedWarning(log);
+    return void next();
+  }
+  const devices = await db.query("devices", filter);
+  if (!devices.length) return void next();
+
+  const validate = authorizer.getValidator("devices", devices[0]);
+  for (const t of ctx.request.body) {
+    if (!validate("task", t)) {
+      logUnauthorizedWarning(log);
+      return void (ctx.status = 403);
+    }
+  }
+
+  // TODO Pass context and timestamp
+  const onlineThreshold = getConfig(
+    ctx.state.configSnapshot,
+    "cwmp.deviceOnlineThreshold"
+  );
+
+  const res = await apiFunctions.postTasks(
+    ctx.params.id,
+    ctx.request.body,
+    onlineThreshold
+  );
+
+  log.tasks = res.tasks.map(t => t._id).join(",");
+
+  logger.accessInfo(log);
+
+  ctx.set("Connection-Request", res.connectionRequest);
+  ctx.body = res.tasks;
+});
+
+router.post("/devices/:id/tags", async (ctx, next) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
+  const log = {
+    message: "Update tags",
+    context: ctx,
+    deviceId: ctx.params.id,
+    tags: ctx.request.body
+  };
+
+  const filter = and(authorizer.getFilter("devices", 3), [
+    "=",
+    ["PARAM", "DeviceID.ID"],
+    ctx.params.id
+  ]);
+  if (!authorizer.hasAccess("devices", 3)) {
+    logUnauthorizedWarning(log);
+    return void next();
+  }
+  const res = await db.query("devices", filter);
+  if (!res.length) return void next();
+
+  const validate = authorizer.getValidator("devices", res[0]);
+  if (!validate("tags", ctx.request.body)) {
+    logUnauthorizedWarning(log);
+    return void (ctx.status = 403);
+  }
+
+  try {
+    await db.updateDeviceTags(ctx.params.id, ctx.request.body);
+  } catch (error) {
+    log.message += " failed";
+    logger.accessWarn(log);
+    ctx.body = error.message;
+    return void (ctx.status = 400);
+  }
+
+  logger.accessInfo(log);
+
+  ctx.body = "";
+});
+
+router.get("/ping/:host", async ctx => {
+  return new Promise(resolve => {
+    ping(ctx.params.host, (err, parsed) => {
+      if (parsed) {
+        ctx.body = parsed;
+      } else {
+        ctx.status = 500;
+        ctx.body = `${err.name}: ${err.message}`;
+      }
+      resolve();
+    });
+  });
+});
+
+router.put("/users/:id/password", async (ctx, next) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
+  const username = ctx.params.id;
+  const log = {
+    message: "Change password",
+    context: ctx,
+    username: username
+  };
+
+  if (!ctx.state.user) {
+    // User not logged in
+    if (
+      !(await apiFunctions.authLocal(
+        ctx.state.configSnapshot,
+        username,
+        ctx.request.body.authPassword
+      ))
+    ) {
+      logUnauthorizedWarning(log);
+      ctx.status = 401;
+      ctx.body = "Authentication failed, check your username and password";
+      return;
+    }
+  } else if (!authorizer.hasAccess("users", 3)) {
+    logUnauthorizedWarning(log);
+    return void next();
+  }
+
+  const filter = and(authorizer.getFilter("users", 3), [
+    "=",
+    ["PARAM", RESOURCE_IDS.users],
+    username
+  ]);
+  const res = await db.query("users", filter);
+  if (!res.length) return void next();
+
+  const newPassword = ctx.request.body.newPassword;
+  if (ctx.state.user) {
+    const validate = authorizer.getValidator("users", res[0]);
+    if (!validate("password", { password: newPassword })) {
+      logUnauthorizedWarning(log);
+      return void (ctx.status = 403);
+    }
+  }
+
+  const salt = await generateSalt(64);
+  const password = await hashPassword(newPassword, salt);
+  await db.putUser(username, { password, salt });
+
+  await del("presets_hash");
+
+  logger.accessInfo(log);
+  ctx.body = "";
+});

lib/ui/db.ts

@@ -0,0 +1,410 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { MongoClient, GridFSBucket } from "mongodb";
+import { Script } from "vm";
+import * as config from "../config";
+import * as mongodbFunctions from "../mongodb-functions";
+import * as expression from "../common/expression";
+import { QueryOptions } from "../types";
+
+const CACHE_TTL = 300000;
+
+let clientPromise: Promise<MongoClient>;
+
+const RESOURCE_COLLECTION = {
+  files: "fs.files"
+};
+
+function ensureIndexes(client): void {
+  client
+    .db()
+    .collection("cache")
+    .createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
+}
+
+function getClient(): Promise<MongoClient> {
+  if (!clientPromise) {
+    clientPromise = new Promise((resolve, reject) => {
+      const CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
+      MongoClient.connect(
+        CONNECTION_URL,
+        { useNewUrlParser: true },
+        (err, client) => {
+          if (err) return void reject(err);
+          ensureIndexes(client);
+          resolve(client);
+        }
+      );
+    });
+  }
+
+  return clientPromise;
+}
+
+export function cache<T>(key, valueGetter: () => Promise<T>, ttl): Promise<T> {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const collection = client.db().collection("cache");
+        collection.findOne({ _id: key }, (err, doc) => {
+          if (err) return void reject(err);
+          if (doc != null) return void resolve(JSON.parse(doc.value) as T);
+          valueGetter()
+            .then(res => {
+              const expire = Date.now() + (ttl || CACHE_TTL);
+              const cacheDoc = {
+                _id: key,
+                value: JSON.stringify(res),
+                expire: new Date(expire)
+              };
+              collection.updateOne(
+                { _id: key },
+                { $set: cacheDoc },
+                { upsert: true },
+                err => {
+                  if (err) reject(err);
+                  else resolve(res);
+                }
+              );
+            })
+            .catch(reject);
+        });
+      })
+      .catch(reject);
+  });
+}
+
+export function query(
+  resource,
+  filter,
+  options?: QueryOptions,
+  callback?: (doc: any) => void
+): Promise<any[]> {
+  options = options || {};
+  let q;
+  filter = expression.evaluate(filter, null, Date.now());
+
+  if (Array.isArray(filter)) {
+    if (resource === "devices") {
+      filter = mongodbFunctions.processDeviceFilter(filter);
+    } else if (resource === "tasks") {
+      filter = mongodbFunctions.processTasksFilter(filter);
+    } else if (resource === "faults") {
+      filter = mongodbFunctions.processFaultsFilter(filter);
+    } else if (resource === "users") {
+      // Protect against brute force, and dictionary attacks
+      const params = expression.extractParams(filter);
+      if (params.includes("password") || params.includes("salt"))
+        return Promise.reject(new Error("Invalid users filter"));
+    }
+
+    q = mongodbFunctions.filterToMongoQuery(filter);
+  } else if (filter != null && !filter) {
+    return Promise.resolve([]);
+  }
+
+  return new Promise((resolve, reject) => {
+    getClient().then(client => {
+      const collection = client
+        .db()
+        .collection(RESOURCE_COLLECTION[resource] || resource);
+      const cursor = collection.find(q);
+      if (options.projection) {
+        let projection = options.projection;
+        if (resource === "devices") {
+          projection = mongodbFunctions.processDeviceProjection(
+            options.projection
+          );
+        }
+
+        if (resource === "presets") projection.configurations = 1;
+        cursor.project(projection);
+      }
+
+      if (resource === "users") cursor.project({ password: 0, salt: 0 });
+
+      if (options.skip) cursor.skip(options.skip);
+      if (options.limit) cursor.limit(options.limit);
+
+      if (options.sort) {
+        let s = Object.entries(options.sort)
+          .sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
+          .reduce(
+            (obj, [k, v]) =>
+              Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
+            {}
+          );
+
+        if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
+        cursor.sort(s);
+      }
+
+      if (!callback) {
+        cursor.toArray((err, docs) => {
+          if (err) return reject(err);
+          if (resource === "devices")
+            docs = docs.map(d => mongodbFunctions.flattenDevice(d));
+          else if (resource === "faults")
+            docs = docs.map(d => mongodbFunctions.flattenFault(d));
+          else if (resource === "tasks")
+            docs = docs.map(d => mongodbFunctions.flattenTask(d));
+          else if (resource === "presets")
+            docs = docs.map(d => mongodbFunctions.flattenPreset(d));
+          else if (resource === "files")
+            docs = docs.map(d => mongodbFunctions.flattenFile(d));
+          return resolve(docs);
+        });
+      } else {
+        cursor.forEach(
+          doc => {
+            if (resource === "devices")
+              doc = mongodbFunctions.flattenDevice(doc);
+            else if (resource === "faults")
+              doc = mongodbFunctions.flattenFault(doc);
+            else if (resource === "tasks")
+              doc = mongodbFunctions.flattenTask(doc);
+            else if (resource === "presets")
+              doc = mongodbFunctions.flattenPreset(doc);
+            else if (resource === "files")
+              doc = mongodbFunctions.flattenFile(doc);
+            callback(doc);
+          },
+          err => {
+            if (err) reject(err);
+            else resolve();
+          }
+        );
+      }
+    });
+  });
+}
+
+export function count(resource, filter): Promise<number> {
+  let q;
+  filter = expression.evaluate(filter, null, Date.now());
+
+  if (Array.isArray(filter)) {
+    if (resource === "devices")
+      filter = mongodbFunctions.processDeviceFilter(filter);
+    else if (resource === "tasks")
+      filter = mongodbFunctions.processTasksFilter(filter);
+    else if (resource === "faults")
+      filter = mongodbFunctions.processFaultsFilter(filter);
+    q = mongodbFunctions.filterToMongoQuery(filter);
+  } else if (filter != null && !filter) {
+    return Promise.resolve(0);
+  }
+
+  return new Promise((resolve, reject) => {
+    getClient().then(client => {
+      const collection = client
+        .db()
+        .collection(RESOURCE_COLLECTION[resource] || resource);
+      collection.find(q).count((err, c) => {
+        if (err) reject(err);
+        else resolve(c);
+      });
+    });
+  });
+}
+
+export function updateDeviceTags(deviceId, tags): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const add = [];
+    const pull = [];
+
+    const regex = /^[0-9a-zA-Z_]+$/;
+    for (let [tag, onOff] of Object.entries(tags)) {
+      tag = tag.trim();
+      if (onOff) {
+        if (!tag.match(regex))
+          return void reject(new Error(`Invalid tag '${tag}'`));
+        add.push(tag);
+      } else {
+        pull.push(tag);
+      }
+    }
+    getClient()
+      .then(client => {
+        const collection = client.db().collection("devices");
+
+        const object = {};
+        if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
+        if (pull && pull.length) object["$pullAll"] = { _tags: pull };
+        collection.updateOne({ _id: deviceId }, object, err => {
+          if (err) return void reject(err);
+          resolve();
+        });
+      })
+      .catch(reject);
+  });
+}
+
+function putResource(resource, id, object): Promise<void> {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const collection = client
+          .db()
+          .collection(RESOURCE_COLLECTION[resource] || resource);
+        collection.replaceOne({ _id: id }, object, { upsert: true }, err => {
+          if (err) return void reject(err);
+          resolve();
+        });
+      })
+      .catch(reject);
+  });
+}
+
+function deleteResource(resource, id): Promise<void> {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const collection = client
+          .db()
+          .collection(RESOURCE_COLLECTION[resource] || resource);
+        collection.deleteOne({ _id: id }, err => {
+          if (err) return void reject(err);
+          resolve();
+        });
+      })
+      .catch(reject);
+  });
+}
+
+export function putPreset(id, object): Promise<void> {
+  object = mongodbFunctions.preProcessPreset(object);
+  return putResource("presets", id, object);
+}
+
+export function deletePreset(id): Promise<void> {
+  return deleteResource("presets", id);
+}
+
+export function putProvision(id, object): Promise<void> {
+  if (!object.script) object.script = "";
+  try {
+    new Script(`"use strict";(function(){\n${object.script}\n})();`);
+  } catch (error) {
+    return Promise.reject(error);
+  }
+  return putResource("provisions", id, object);
+}
+
+export function deleteProvision(id): Promise<void> {
+  return deleteResource("provisions", id);
+}
+
+export function putVirtualParameter(id, object): Promise<void> {
+  if (!object.script) object.script = "";
+  try {
+    new Script(`"use strict";(function(){\n${object.script}\n})();`);
+  } catch (error) {
+    return Promise.reject(error);
+  }
+  return putResource("virtualParameters", id, object);
+}
+
+export function deleteVirtualParameter(id): Promise<void> {
+  return deleteResource("virtualParameters", id);
+}
+
+export function putConfig(id, object): Promise<void> {
+  return putResource("config", id, object);
+}
+
+export function deleteConfig(id): Promise<void> {
+  return deleteResource("config", id);
+}
+
+export function putPermission(id, object): Promise<void> {
+  return putResource("permissions", id, object);
+}
+
+export function deletePermission(id): Promise<void> {
+  return deleteResource("permissions", id);
+}
+
+export function putUser(id, object): Promise<void> {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const collection = client.db().collection("users");
+        // update instead of replace to keep the password if not set by user
+        collection.updateOne(
+          { _id: id },
+          { $set: object },
+          { upsert: true },
+          err => {
+            if (err) return void reject(err);
+            resolve();
+          }
+        );
+      })
+      .catch(reject);
+  });
+}
+
+export function deleteUser(id): Promise<void> {
+  return deleteResource("users", id);
+}
+
+export function putFile(filename, metadata, contentStream): Promise<void> {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const bucket = new GridFSBucket(client.db());
+        const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
+          metadata: metadata
+        });
+        uploadStream.on("error", reject);
+        contentStream.on("error", reject);
+        uploadStream.on("finish", resolve);
+        contentStream.pipe(uploadStream);
+      })
+      .catch(reject);
+  });
+}
+
+export function deleteFile(filename): Promise<void> {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const bucket = new GridFSBucket(client.db());
+        bucket.delete(filename, err => {
+          if (err) return void reject(err);
+          resolve();
+        });
+      })
+      .catch(reject);
+  });
+}
+
+export function deleteFault(id): Promise<void> {
+  return deleteResource("faults", id);
+}
+
+export function deleteTask(id): Promise<void> {
+  return deleteResource("tasks", id);
+}
+
+export async function disconnect(): Promise<void> {
+  if (clientPromise) await (await clientPromise).close();
+}

lib/versioned-map.js

@@ -1,267 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const NONEXISTENT = new Object();
-
-
-function compareEquality(a, b) {
-  var t = typeof a;
-  if (t === 'number' || (a == null) || t === 'boolean' || t === 'string' || t === 'symbol')
-    return a === b;
-
-  return JSON.stringify(a) === JSON.stringify(b);
-}
-
-
-class VersionedMap {
-
-  constructor() {
-    this._sizeDiff = [0];
-    this._revision = 0;
-    this.map = new Map();
-    this.dirty = 0;
-  }
-
-
-  get size() {
-    return this.map.size + this._sizeDiff[this.revision];
-  }
-
-
-  get revision() {
-    return this._revision;
-  }
-
-
-  set revision(rev) {
-    for (let i = this._sizeDiff.length; i <= rev; ++ i)
-      this._sizeDiff[i] = this._sizeDiff[i - 1];
-
-    this._revision = rev;
-  }
-
-
-  get(key, rev) {
-    if (rev == null)
-      rev = this._revision
-
-    var v = this.map.get(key);
-    if (!v)
-      return;
-
-    for (let i = Math.min(v.length - 1, rev); i >= 0; -- i)
-      if (i in v)
-        if (v[i] === NONEXISTENT)
-          return undefined;
-        else
-          return v[i];
-  }
-
-
-  has(key, rev) {
-    if (rev == null)
-      rev = this._revision
-
-    var v = this.map.get(key);
-
-    if (v == null)
-      return false;
-
-    for (let i = Math.min(v.length - 1, rev); i >= 0; -- i)
-      if (i in v)
-        if (v[i] === NONEXISTENT)
-          return false;
-        else
-          return true;
-
-    return false;
-  }
-
-
-  set(key, value, rev) {
-    if (rev == null)
-      rev = this._revision;
-
-    var v = this.map.get(key);
-    if (!v) {
-      this.dirty |= 1 << rev;
-      for (let i = 0; i < rev; ++ i)
-        this._sizeDiff[i] -= 1;
-
-      v = [];
-      v[rev] = value;
-      this.map.set(key, v);
-      return this;
-    }
-
-    var old = v[Math.min(rev, v.length - 1)];
-
-    if (rev < v.length - 1) {
-      if (compareEquality(value, old))
-        return
-      else
-        throw new Error('Cannot modify old revisions');
-    }
-
-    this.dirty |= 1 << rev;
-    if (old === NONEXISTENT)
-      ++ this._sizeDiff[rev];
-
-    v[rev] = value;
-  }
-
-
-  delete(key, rev) {
-    if (rev == null)
-      rev = this._revision;
-
-    var v = this.map.get(key);
-    if (!v)
-      return false;
-
-    if (rev < v.length - 1)
-      throw new Error('Cannot modify old revisions');
-
-    var old = v[v.length - 1];
-    if (old === NONEXISTENT)
-      return false;
-
-    this.dirty |= 1 << rev;
-    -- this._sizeDiff[rev];
-    v[rev] = NONEXISTENT;
-
-    return true;
-  }
-
-
-  getRevisions(key) {
-    var v = this.map.get(key);
-    if (!v)
-      return null;
-
-    var res = {};
-
-    for (let i in v)
-      if (v[i] === NONEXISTENT)
-        res.delete |= 1 << i;
-      else
-        res[i] = v[i];
-
-    return res;
-  }
-
-
-  setRevisions(key, revisions) {
-    var del = 0;
-    var rev = [];
-    var minKey = 999;
-    for (let k in revisions) {
-      if (k === 'delete')
-        del = revisions[k];
-      else {
-        let r = parseInt(k);
-        minKey = Math.min(minKey, r);
-        for (let i = this._sizeDiff.length; i <= r; ++ i)
-          this._sizeDiff[i] = this._sizeDiff[i - 1];
-
-        this.dirty |= 1 << r;
-        rev[r] = revisions[k];
-      }
-    }
-
-    for (let i = 0; i < minKey; ++ i)
-      -- this._sizeDiff[i];
-
-    this.dirty |= del;
-
-    for (let i = 0; del > 0; del >>= 1, ++ i)
-      if (del & 1)
-        rev[i] = NONEXISTENT;
-
-    this.map.set(key, rev);
-  }
-
-
-  getDiff(key) {
-    var revisions = this.map.get(key);
-    if (!revisions)
-      return [];
-
-    var current = revisions[revisions.length - 1];
-    if (current === NONEXISTENT) {
-      if (0 in revisions)
-        return [key, revisions[0]];
-      else
-        return [key];
-    } else if (0 in revisions)
-      return [key, revisions[0], current];
-    else
-      return [key, , current];
-  }
-
-
-  *diff() {
-    for (let pair of this.map) {
-      let current = pair[1][pair[1].length - 1];
-      if (current === NONEXISTENT) {
-        if (0 in pair[1])
-          yield [pair[0], pair[1][0]];
-        else
-          yield [pair[0]];
-      } else if (0 in pair[1])
-        yield [pair[0], pair[1][0], current];
-      else
-        yield [pair[0], , current];
-    }
-  }
-
-
-  collapse(revision) {
-    if (this._sizeDiff.length <= revision)
-      return;
-
-    this._sizeDiff[revision] = this._sizeDiff[this._sizeDiff.length - 1];
-    this._sizeDiff.splice(revision + 1, this._sizeDiff.length);
-
-    let d = this.dirty >> revision;
-    this.dirty = this.dirty ^ (d << revision);
-    this.dirty |= (+!!d) << revision;
-
-    for (let pair of this.map) {
-      let k = pair[0];
-      let v = pair[1];
-      let last = v[v.length - 1];
-      if (last === NONEXISTENT) {
-        v.splice(revision, v.length);
-        if (v.some(function(val) { return true; }))
-          v[revision] = NONEXISTENT;
-        else
-          this.map.delete(k);
-      }
-      else {
-        v[revision] = last;
-        v.splice(revision + 1, v.length);
-      }
-    }
-  }
-
-}
-
-module.exports = VersionedMap;

lib/versioned-map.ts

@@ -0,0 +1,200 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+const NONEXISTENT = Symbol();
+const UNDEFINED = undefined;
+
+interface Revisions<V> {
+  [rev: number]: V;
+  delete?: number;
+}
+
+export default class VersionedMap<K, V> {
+  private _sizeDiff: number[];
+  private _revision: number;
+  private map: Map<K, (V | symbol)[]>;
+  public dirty: number;
+
+  public constructor() {
+    this._sizeDiff = [0];
+    this._revision = 0;
+    this.map = new Map();
+    this.dirty = 0;
+  }
+
+  public get size(): number {
+    return this.map.size + this._sizeDiff[this.revision];
+  }
+
+  public get revision(): number {
+    return this._revision;
+  }
+
+  public set revision(rev) {
+    for (let i = this._sizeDiff.length; i <= rev; ++i)
+      this._sizeDiff[i] = this._sizeDiff[i - 1];
+
+    this._revision = rev;
+  }
+
+  public get(key: K, rev = this._revision): V {
+    const revisions = this.map.get(key);
+    if (!revisions) return UNDEFINED;
+
+    const v = revisions[Math.min(revisions.length - 1, rev)];
+    if (v === NONEXISTENT) return UNDEFINED;
+    return v as V;
+  }
+
+  public has(key: K, rev = this._revision): boolean {
+    const revisions = this.map.get(key);
+    if (!revisions) return false;
+
+    const v = revisions[Math.min(revisions.length - 1, rev)];
+    if (v === NONEXISTENT) return false;
+    return true;
+  }
+
+  public set(key: K, value: V, rev = this._revision): this {
+    let revisions = this.map.get(key);
+    if (!revisions) {
+      this.dirty |= 1 << rev;
+      for (let i = 0; i < rev; ++i) this._sizeDiff[i] -= 1;
+
+      revisions = [];
+      for (let i = 0; i < rev; ++i) revisions[i] = NONEXISTENT;
+      revisions[rev] = value;
+      this.map.set(key, revisions);
+      return this;
+    }
+
+    // Can't modify old revisions
+    if (rev < revisions.length - 1) return null;
+
+    const old = revisions[revisions.length - 1];
+
+    this.dirty |= 1 << rev;
+    if (old === NONEXISTENT) ++this._sizeDiff[rev];
+
+    for (let i = revisions.length; i < rev; ++i) revisions[i] = old;
+    revisions[rev] = value;
+
+    return this;
+  }
+
+  public delete(key: K, rev = this._revision): boolean {
+    const revisions = this.map.get(key);
+    if (!revisions) return false;
+
+    // Can't modify old revisions
+    if (rev < revisions.length - 1) return null;
+
+    const old = revisions[revisions.length - 1];
+    if (old === NONEXISTENT) return false;
+
+    this.dirty |= 1 << rev;
+    --this._sizeDiff[rev];
+
+    for (let i = revisions.length; i < rev; ++i) revisions[i] = old;
+    revisions[rev] = NONEXISTENT;
+
+    return true;
+  }
+
+  public getRevisions(key: K): Revisions<V> {
+    const revisions = this.map.get(key);
+    if (!revisions) return null;
+
+    const res: Revisions<V> = {};
+
+    let prev: V | symbol = NONEXISTENT;
+    for (const [i, v] of revisions.entries()) {
+      if (v === prev) continue;
+      if (v === NONEXISTENT) res.delete |= 1 << i;
+      else res[i] = v as V;
+      prev = v;
+    }
+
+    return res;
+  }
+
+  public setRevisions(key: K, revisionsObj: Revisions<V>): void {
+    const del = revisionsObj.delete || 0;
+    const mutations = Object.keys(revisionsObj).reduce(
+      (acc, cur) => (cur === "delete" ? acc : acc | (1 << +cur)),
+      del
+    );
+
+    const revisions = [];
+
+    let prev: V | symbol = NONEXISTENT;
+    for (let i = 0; mutations >> i; ++i) {
+      let v = prev;
+      if (del & (1 << i)) v = NONEXISTENT;
+      else if (i in revisionsObj) v = revisionsObj[i];
+      if (v !== prev) this.dirty |= 1 << i;
+      revisions[i] = v;
+      prev = v;
+    }
+
+    this.map.set(key, revisions);
+  }
+
+  public getDiff(key: K): [V, V] {
+    const revisions = this.map.get(key);
+    if (!revisions) return [UNDEFINED, UNDEFINED];
+    let first = revisions[0];
+    if (first === NONEXISTENT) first = UNDEFINED;
+    let last = revisions[revisions.length - 1];
+    if (last === NONEXISTENT) last = UNDEFINED;
+    return [first as V, last as V];
+  }
+
+  public *diff(): IterableIterator<[K, V, V]> {
+    for (const [key, revisions] of this.map) {
+      let first = revisions[0];
+      let last = revisions[revisions.length - 1];
+      if (first === NONEXISTENT && last === NONEXISTENT) continue;
+      if (first === NONEXISTENT) first = UNDEFINED;
+      if (last === NONEXISTENT) last = UNDEFINED;
+      yield [key, first as V, last as V];
+    }
+  }
+
+  public collapse(revision: number): void {
+    if (this._sizeDiff.length <= revision) return;
+
+    this._sizeDiff[revision] = this._sizeDiff[this._sizeDiff.length - 1];
+    this._sizeDiff.splice(revision + 1, this._sizeDiff.length);
+
+    const d = this.dirty >> revision;
+    this.dirty = this.dirty ^ (d << revision);
+    this.dirty |= +!!d << revision;
+
+    for (const [k, v] of this.map) {
+      const l = v.length - 1;
+      if (l <= revision) continue;
+      const last = v[l];
+      v.splice(revision, l - revision);
+
+      if (last === NONEXISTENT && !v.some(vv => vv !== NONEXISTENT))
+        this.map.delete(k);
+    }
+  }
+}

lib/xml-parser.ts

@@ -0,0 +1,357 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+const CHAR_SINGLE_QUOTE = 39;
+const CHAR_DOUBLE_QUOTE = 34;
+const CHAR_LESS_THAN = 60;
+const CHAR_GREATER_THAN = 62;
+const CHAR_COLON = 58;
+const CHAR_SPACE = 32;
+const CHAR_TAB = 9;
+const CHAR_CR = 13;
+const CHAR_LF = 10;
+const CHAR_SLASH = 47;
+const CHAR_EXMARK = 33;
+const CHAR_QMARK = 63;
+const CHAR_EQUAL = 61;
+
+const STATE_LESS_THAN = 1;
+const STATE_SINGLE_QUOTE = 2;
+const STATE_DOUBLE_QUOTE = 3;
+
+export interface Attribute {
+  name: string;
+  namespace: string;
+  localName: string;
+  value: string;
+}
+
+export interface Element {
+  name: string;
+  namespace: string;
+  localName: string;
+  attrs: string;
+  text: string;
+  bodyIndex: number;
+  children: Element[];
+}
+
+export function parseXmlDeclaration(buffer: Buffer): Attribute[] {
+  for (const enc of ["utf16le", "utf8", "latin1", "ascii"]) {
+    let str = buffer.toString(enc, 0, 150);
+    if (str.startsWith("<?xml")) {
+      str = str.split("\n")[0].trim();
+      try {
+        return parseAttrs(str.slice(5, -2));
+      } catch (err) {
+        // Ignore
+      }
+    }
+  }
+  return null;
+}
+
+export function parseAttrs(string: string): Attribute[] {
+  const attrs: Attribute[] = [];
+  const len = string.length;
+
+  let state = 0;
+  let name = "";
+  let namespace = "";
+  let localName = "";
+  let idx = 0;
+  let colonIdx = 0;
+  for (let i = 0; i < len; ++i) {
+    const c = string.charCodeAt(i);
+    switch (c) {
+      case CHAR_SINGLE_QUOTE:
+      case CHAR_DOUBLE_QUOTE:
+        if (state === c) {
+          state = 0;
+          if (name) {
+            const value = string.slice(idx + 1, i);
+            const e = {
+              name: name,
+              namespace: namespace,
+              localName: localName,
+              value: value
+            };
+            attrs.push(e);
+            name = "";
+            idx = i + 1;
+          }
+        } else {
+          state = c;
+          idx = i;
+        }
+        continue;
+
+      case CHAR_COLON:
+        if (idx >= colonIdx) colonIdx = i;
+        continue;
+
+      case CHAR_EQUAL:
+        if (name) throw new Error(`Unexpected character at ${i}`);
+        name = string.slice(idx, i).trim();
+        // TODO validate name
+        if (colonIdx > idx) {
+          namespace = string.slice(idx, colonIdx).trim();
+          localName = string.slice(colonIdx + 1, i).trim();
+        } else {
+          namespace = "";
+          localName = name;
+        }
+    }
+  }
+
+  if (name) throw new Error(`Attribute must have value at ${idx}`);
+
+  const tail = string.slice(idx);
+  if (tail.trim()) throw new Error(`Unexpected string at ${len - tail.length}`);
+
+  return attrs;
+}
+
+export function decodeEntities(string): string {
+  return string.replace(/&[0-9a-z#]+;/gi, match => {
+    switch (match) {
+      case "&quot;":
+        return '"';
+
+      case "&amp;":
+        return "&";
+
+      case "&apos;":
+        return "'";
+
+      case "&lt;":
+        return "<";
+
+      case "&gt;":
+        return ">";
+
+      default:
+        if (match.startsWith("&#x")) {
+          const str = match.slice(3, -1).toLowerCase();
+          const n = parseInt(str, 16);
+          if (str.endsWith(n.toString(16))) return String.fromCharCode(n);
+        } else if (match.startsWith("&#")) {
+          const str = match.slice(2, -1);
+          const n = parseInt(str);
+          if (str.endsWith(n.toString())) return String.fromCharCode(n);
+        }
+    }
+    return match;
+  });
+}
+
+export function encodeEntities(string): string {
+  const entities = {
+    "&": "&amp;",
+    '"': "&quot;",
+    "'": "&apos;",
+    "<": "&lt;",
+    ">": "&gt;"
+  };
+  return string.replace(/[&"'<>]/g, m => entities[m]);
+}
+
+export function parseXml(string: string): Element {
+  const len = string.length;
+  let state1 = 0;
+  let state1Index = 0;
+  let state2 = 0;
+  let state2Index = 0;
+
+  const root: Element = {
+    name: "root",
+    namespace: "",
+    localName: "root",
+    attrs: "",
+    text: "",
+    bodyIndex: 0,
+    children: []
+  };
+
+  const stack: Element[] = [root];
+
+  for (let i = 0; i < len; ++i) {
+    switch (string.charCodeAt(i)) {
+      case CHAR_SINGLE_QUOTE:
+        switch (state1 & 0xff) {
+          case STATE_SINGLE_QUOTE:
+            state1 = state2;
+            state1Index = state2Index;
+            state2 = 0;
+            continue;
+
+          case STATE_LESS_THAN:
+            state2 = state1;
+            state2Index = state1Index;
+            state1 = STATE_SINGLE_QUOTE;
+            state1Index = i;
+            continue;
+        }
+        continue;
+
+      case CHAR_DOUBLE_QUOTE:
+        switch (state1 & 0xff) {
+          case STATE_DOUBLE_QUOTE:
+            state1 = state2;
+            state1Index = state2Index;
+            state2 = 0;
+            continue;
+
+          case STATE_LESS_THAN:
+            state2 = state1;
+            state2Index = state1Index;
+            state1 = STATE_DOUBLE_QUOTE;
+            state1Index = i;
+            continue;
+        }
+        continue;
+
+      case CHAR_LESS_THAN:
+        if ((state1 & 0xff) === 0) {
+          state2 = state1;
+          state2Index = state1Index;
+          state1 = STATE_LESS_THAN;
+          state1Index = i;
+        }
+        continue;
+
+      case CHAR_COLON:
+        if ((state1 & 0xff) === STATE_LESS_THAN) {
+          const colonIndex = (state1 >> 8) & 0xff;
+          if (colonIndex === 0) state1 ^= ((i - state1Index) & 0xff) << 8;
+        }
+        continue;
+
+      case CHAR_SPACE:
+      case CHAR_TAB:
+      case CHAR_CR:
+      case CHAR_LF:
+        if ((state1 & 0xff) === STATE_LESS_THAN) {
+          const wsIndex = (state1 >> 16) & 0xff;
+          if (wsIndex === 0) state1 ^= ((i - state1Index) & 0xff) << 16;
+        }
+        continue;
+
+      case CHAR_GREATER_THAN:
+        if ((state1 & 0xff) === STATE_LESS_THAN) {
+          const secondChar = string.charCodeAt(state1Index + 1);
+          const wsIndex: number = (state1 >> 16) & 0xff;
+          let name: string,
+            colonIndex: number,
+            e: Element,
+            parent: Element,
+            selfClosing: number,
+            localName: string,
+            namespace: string;
+
+          switch (secondChar) {
+            case CHAR_SLASH:
+              e = stack.pop();
+              name =
+                wsIndex === 0
+                  ? string.slice(state1Index + 2, i)
+                  : string.slice(state1Index + 2, state1Index + wsIndex);
+              if (e.name !== name)
+                throw new Error(`Unmatched closing tag at ${i}`);
+              if (!e.children.length)
+                e.text = string.slice(e.bodyIndex, state1Index);
+              state1 = state2;
+              state1Index = state2Index;
+              state2 = 0;
+              continue;
+
+            case CHAR_EXMARK:
+              if (string.startsWith("![CDATA[", state1Index + 1)) {
+                if (string.endsWith("]]", i))
+                  throw new Error(`CDATA nodes are not supported at ${i}`);
+              } else if (string.startsWith("!--", state1Index + 1)) {
+                // Comment node, ignore
+                if (string.endsWith("--", i)) {
+                  state1 = state2;
+                  state1Index = state2Index;
+                  state2 = 0;
+                }
+              }
+              continue;
+
+            case CHAR_QMARK:
+              if (string.charCodeAt(i - 1) === CHAR_QMARK) {
+                // XML declaration node, ignore
+                state1 = state2;
+                state1Index = state2Index;
+                state2 = 0;
+              }
+              continue;
+
+            default:
+              selfClosing = +(string.charCodeAt(i - 1) === CHAR_SLASH);
+              parent = stack[stack.length - 1];
+              colonIndex = (state1 >> 8) & 0xff;
+
+              name =
+                wsIndex === 0
+                  ? string.slice(state1Index + 1, i - selfClosing)
+                  : string.slice(state1Index + 1, state1Index + wsIndex);
+              if (colonIndex && (!wsIndex || colonIndex < wsIndex)) {
+                localName = name.slice(colonIndex);
+                namespace = name.slice(0, colonIndex - 1);
+              } else {
+                localName = name;
+                namespace = "";
+              }
+
+              e = {
+                name: name,
+                namespace: namespace,
+                localName: localName,
+                attrs: wsIndex
+                  ? string.slice(state1Index + wsIndex + 1, i - selfClosing)
+                  : "",
+                text: "",
+                bodyIndex: i + 1,
+                children: []
+              };
+              parent.children.push(e);
+              if (!selfClosing) stack.push(e);
+
+              state1 = state2;
+              state1Index = state2Index;
+              state2 = 0;
+              continue;
+          }
+        }
+        continue;
+    }
+  }
+
+  if (state1) throw new Error(`Unclosed token at ${state1Index}`);
+
+  if (stack.length > 1) {
+    const e = stack[stack.length - 1];
+    throw new Error(`Unclosed XML element at ${e.bodyIndex}`);
+  }
+
+  if (!root.children.length) root.text = string;
+  return root;
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.1.2",
+  "version": "1.2.0-beta.0",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -16,28 +16,73 @@
   "license": "AGPL-3.0",
   "preferGlobal": true,
   "directories": {
-    "bin": "bin"
+    "bin": "bin",
+    "tools": "tools"
   },
   "dependencies": {
+    "iconv-lite": "^0.5.0",
+    "jsonwebtoken": "^8.5.1",
+    "koa": "^2.7.0",
+    "koa-bodyparser": "^4.2.1",
+    "koa-compress": "^3.0.0",
+    "koa-jwt": "^3.6.0",
+    "koa-router": "^7.4.0",
+    "koa-static": "^5.0.0",
     "later": "^1.2.0",
-    "libxmljs": "^0.18.7",
-    "mongodb": "^2.2.34",
-    "seedrandom": "^2.4.3"
+    "mongodb": "^3.2.6",
+    "parsimmon": "^1.13.0",
+    "seedrandom": "^3.0.1",
+    "yaml": "^1.6.0"
   },
   "optionalDependencies": {
-    "redis": "^2.6.5"
+    "libxmljs": "^0.19.5"
   },
   "devDependencies": {
-    "coffeescript": "^1.12.7"
+    "@types/jsonwebtoken": "^8.3.2",
+    "@types/koa": "^2.0.49",
+    "@types/koa-compress": "^2.0.9",
+    "@types/koa-router": "^7.0.42",
+    "@types/koa-static": "^4.0.1",
+    "@types/libxmljs": "^0.18.3",
+    "@types/mithril": "^2.0.0",
+    "@types/mongodb": "^3.1.31",
+    "@types/parsimmon": "^1.10.0",
+    "@types/seedrandom": "^2.4.28",
+    "@types/svgo": "^1.2.1",
+    "@typescript-eslint/eslint-plugin": "^1.13.0",
+    "@typescript-eslint/parser": "^1.13.0",
+    "ava": "^2.2.0",
+    "codemirror": "^5.48.2",
+    "cssnano": "^4.1.10",
+    "eslint": "^6.1.0",
+    "eslint-config-prettier": "^6.0.0",
+    "esm": "^3.2.25",
+    "mithril": "^2.0.3",
+    "postcss": "^7.0.17",
+    "postcss-import": "^12.0.1",
+    "postcss-preset-env": "^6.7.0",
+    "prettier": "^1.18.2",
+    "rollup": "^1.17.0",
+    "rollup-plugin-json": "^4.0.0",
+    "rollup-plugin-replace": "^2.2.0",
+    "rollup-plugin-terser": "^5.1.1",
+    "rollup-plugin-typescript": "^1.0.1",
+    "source-map-loader": "^0.2.4",
+    "svgo": "^1.3.0",
+    "ts-node": "^8.3.0",
+    "typescript": "^3.5.3",
+    "webpack": "^4.38.0"
   },
   "engines": {
-    "node": ">=6.9.2 <9.0"
+    "node": ">=8.14.0"
   },
   "scripts": {
-    "prepublish": "npm run compile",
-    "compile": "coffee -cb lib/*.coffee",
-    "configure": "cp -n ./config/config-sample.json ./config/config.json",
-    "install": "npm run configure",
-    "test": "node test"
+    "test": "ava --verbose --color",
+    "lint-ln": "find bin -type f -exec sh -c 'ln -fs \"$(sh -c pwd)/$1\" \"bin/.${1#bin/}.ts\"' _ {} \\;; find tools -type f -exec sh -c 'ln -fs \"$(sh -c pwd)/$1\" \"tools/.${1#tools/}.ts\"' _ {} \\;",
+    "lint-eslint": "eslint --fix \"bin/**/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/**/*.ts\"",
+    "lint-prettier": "prettier --parser typescript --write \"bin/**/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/**/*.ts\"",
+    "lint-tsc": "tsc --noEmit",
+    "lint": "npm run lint-ln && npm run lint-eslint && npm run lint-prettier && npm run lint-tsc",
+    "build": "node -r esm -r ts-node/register/transpile-only build/build"
   }
 }

public/logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="310" height="64" viewBox="0 0 82.020832 16.933334"><defs><linearGradient id="b"><stop offset="0" stop-color="#737373"/><stop offset="1" stop-color="#4b4b4b"/></linearGradient><linearGradient id="a"><stop offset="0" stop-color="#b72f5f"/><stop offset="1" stop-color="#872346"/></linearGradient><linearGradient xlink:href="#a" id="c" gradientUnits="userSpaceOnUse" gradientTransform="translate(173.233 -461.003) scale(.21343)" x1="16.045" y1="132.803" x2="16.045" y2="-81.164"/><linearGradient xlink:href="#a" id="d" gradientUnits="userSpaceOnUse" x1="46.459" y1="281.163" x2="46.459" y2="296.66" gradientTransform="translate(1.09 16.9) scale(.94143)"/><linearGradient xlink:href="#b" id="e" gradientUnits="userSpaceOnUse" gradientTransform="translate(11.105 -994.648) scale(3.55816)" x1="46.459" y1="281.163" x2="46.459" y2="296.66"/></defs><path d="M173.233-461.003l1.953-3.714c-.868-1.509-2.042-2.814-3.522-3.917-2.419-1.43-4.386-1.199-5.901.694 2.931 1.323 5.421 3.635 7.47 6.937m-7.577 11.151a6.095 6.095 0 0 0 .673-3.49 23.76 23.76 0 0 0-1.473-6.285 20.998 20.998 0 0 1-1.206-4.855 6.766 6.766 0 0 1 .48-3.426c.463-1.117 1.298-2.074 2.508-2.87-1.302-.392-2.44-.499-3.415-.32-1.707.326-2.82 1.354-3.34 3.083a10.152 10.152 0 0 0-.267 5.08 33.38 33.38 0 0 0 1.238 4.674 32.993 32.993 0 0 1 1.227 4.856c.107.626.118 1.277.032 1.953a4.72 4.72 0 0 1-.779 2.006 4.498 4.498 0 0 1-1.707 1.44c.697.129 1.387.235 2.07.32a4.124 4.124 0 0 0 2.07-.351c.84-.37 1.47-.975 1.89-1.815m22.463 5.123l.63-.62-5.25-4.77 1.579-1.579 4.77 5.25.619-.629-5.24-4.77 1.483-1.483 4.77 5.24.63-.62-5.25-4.77 1.58-1.58 5.015 5.016 1.067-1.045-4.503-4.504h.01l-5.87-5.859-3.563-1.462-5.102 3.831-.02.022-.022-.022.01-.02 3.394-4.526-1.131-.458-4.973 4.973 1.26 3.073 4.652-3.49.01-.01.033.02-.01.012-3.95 5.26.64 1.57 4.814 4.802h-.01l5.559 5.549 1.057-1.046-5.027-5.026 1.569-1.569 4.77 5.24m4.14-28.173c4.405 4.403 6.607 9.722 6.607 15.954 0 6.218-2.202 11.536-6.606 15.954-4.418 4.404-9.736 6.606-15.954 6.606-6.233 0-11.55-2.202-15.955-6.606-2.703-2.71-4.574-5.756-5.613-9.135l.448.245c2.241 1.181 4.226.577 5.955-1.814-1.565-.32-2.82-.658-3.767-1.014a5.467 5.467 0 0 1-1.43-.821c-.96-.762-1.565-1.73-1.814-2.903a11.315 11.315 0 0 1 .032-5.005c.81-4.34 2.874-8.16 6.19-11.461 4.403-4.411 9.721-6.617 15.954-6.617 6.225 0 11.543 2.206 15.954 6.617" fill="url(#c)" transform="matrix(.35176 0 0 -.35176 -53.595 -152.27)"/><path style="line-height:6.61458302px;-inkscape-font-specification:Norwegian" d="M47.104 281.834c-.456 0-.816.388-.816.849 0 .46.355.816.816.816a.82.82 0 0 0 .833-.816.85.85 0 0 0-.833-.849zm-21.502 1.404c-.205 0-.382.046-.5.13-.118.085-.187.201-.187.4v.981l-.151-.031a9.89 9.89 0 0 0-1.872-.173c-1.288 0-2.148.066-2.657.374a1.264 1.264 0 0 0-.548.676c-.119.313-.177.73-.177 1.28v6.123c0 .982.25 1.508.787 1.824.539.315 1.407.41 2.628.41 1.308 0 2.178-.13 2.701-.484.524-.354.745-.932.745-1.942 0-.821-.147-1.37-.543-1.722-.395-.352-1.075-.528-2.186-.528h-2.787v-1.234l.148.03c.478.092 1.094.11 1.89.11 1.292 0 2.148-.112 2.658-.453.51-.34.724-.915.724-1.958v-3.284c0-.198-.07-.314-.186-.399a.843.843 0 0 0-.487-.13zm6.112 1.29c-1.226 0-2.038.137-2.526.495-.488.357-.697.945-.697 1.947v2.84c0 .935.23 1.54.743 1.932.512.39 1.337.571 2.544.571 1.392 0 2.17-.058 2.567-.196.199-.069.298-.15.355-.246.058-.097.078-.227.078-.405 0-.208-.04-.378-.117-.485-.076-.108-.186-.171-.396-.171-.794 0-1.057.19-2.408.19-.593 0-1.06-.078-1.395-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.717.717 0 0 0 .539-.213c.13-.138.213-.342.213-.62v-1.212c0-1.037-.179-1.623-.654-1.97-.476-.348-1.3-.472-2.617-.472zm8.83 0c-1.303 0-2.117.086-2.59.49-.472.403-.665 1.168-.665 2.623v4.002c0 .215.074.338.197.426.123.088.31.134.524.134.223 0 .412-.046.537-.134.125-.088.198-.211.198-.426v-3.81c0-.424.006-.754.04-1.019.036-.265.102-.47.236-.623.135-.152.328-.236.574-.28.246-.044.552-.053.949-.053.4 0 .712.01.963.053.25.044.447.126.587.278.139.151.211.358.25.623.038.265.046.597.046 1.02v3.81c0 .217.07.34.19.427.122.088.308.134.53.134.215 0 .401-.046.525-.134.123-.088.196-.211.196-.426v-4.002c0-1.455-.201-2.219-.682-2.623-.48-.404-1.303-.49-2.605-.49zm12.996 0c-1.227 0-2.038.137-2.527.495-.488.357-.696.945-.696 1.947v2.84c0 .935.23 1.54.742 1.932.513.39 1.338.571 2.545.571 1.392 0 2.17-.058 2.567-.196.2-.069.298-.15.355-.246.058-.097.079-.227.079-.405 0-.208-.04-.378-.117-.485-.077-.108-.188-.171-.398-.171-.793 0-1.056.19-2.406.19-.594 0-1.062-.078-1.396-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.716.716 0 0 0 .539-.213c.13-.138.213-.342.213-.62v-1.212c0-1.037-.18-1.623-.655-1.97-.475-.348-1.3-.472-2.616-.472zm-6.436.13c-.212 0-.398.047-.522.14-.124.09-.198.22-.198.435v6.41c0 .215.073.338.196.426.123.088.31.134.524.134.223 0 .414-.046.54-.134.124-.088.198-.211.198-.426v-6.41c0-.215-.075-.342-.201-.434-.126-.092-.316-.142-.537-.142zm-24.212 1.135c.662 0 1.4.065 1.918.146l.105.016v.92c0 .246-.01.453-.058.632a.757.757 0 0 1-Line truncated