1.2.0-beta.0

Long text component text was not being updated when the value was
changed.

.eslintignore

@@ -0,0 +1,2 @@
+!/bin/.*.ts
+!/tools/.*.ts

.eslintrc.json

@@ -0,0 +1,27 @@
+{
+  "env": {
+    "es6": true,
+    "node": true
+  },
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "project": "./tsconfig.json"
+  },
+  "plugins": ["@typescript-eslint"],
+  "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended", "prettier", "prettier/@typescript-eslint"],
+  "rules": {
+    "no-shadow": ["error", { "allow": ["err"] }],
+    "handle-callback-err": "error",
+    "prefer-arrow-callback": "error",
+    "curly": ["error", "multi-or-nest", "consistent"],
+    "no-buffer-constructor": "error",
+    "prefer-const": ["error", {"destructuring": "all"}],
+    "eqeqeq": ["error", "always", {"null": "ignore"}],
+    "no-return-await": "error",
+    "@typescript-eslint/no-explicit-any": 0,
+    "@typescript-eslint/no-use-before-define": ["error", { "functions": false }],
+    "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
+    "require-atomic-updates": 0,
+    "no-prototype-builtins": 0
+  }
+}

.gitignore

@@ -1,3 +1,6 @@
 *~
-*.js
 node_modules
+dist
+bin/.*.ts
+tools/.*.ts
+docs/_build

.npmignore

@@ -1,14 +1,25 @@
 *~
 .npmignore
 
-config/*
-!config/*-sample.*
-!config/custom_commands
+config/config.json
+config/cwmp.key
+config/cwmp.crt
+config/cwmp.ca-bundle
+config/nbi.key
+config/nbi.crt
+config/nbi.ca-bundle
+config/fs.key
+config/fs.crt
+config/fs.ca-bundle
+config/ext
 
 debug/*
 !debug/.gitkeep
 
-lib/*.coffee
-tools/*.coffee
-test/*.coffee
-
+dist
+.eslintrc.json
+.eslintignore
+ava.config.js
+tsconfig.json
+bin/.*
+tools/.*

CHANGELOG.md

@@ -0,0 +1,138 @@
+# Change Log
+
+## 1.2.0-beta.0 (2019-07-30)
+
+- A brand new UI superseding genieacs-gui.
+- New initialization wizard on first run.
+- New expression/query language used in search filters and preset
+preconditions.
+- CPE -> ACS authentication is now supported.
+- New config option (CWMP_KEEP_ALIVE_TIMEOUT) to specify how long to wait for a
+reply from the CPE before closing the TCP connection.
+- Debug logging has been reimplemented utilizing YAML format for logs.
+- Handle 9005 faults (Invalid Parameter Name) gracefully by attempting to
+rediscover the path of the missing parameter recursively.
+- declare() statements not followed by an explicit commit() are now deferred
+until all currently active scripts have been executed.
+- FS_HOSTNAME now defaults to the server's hostname or IP.
+- The API now validates the structure of task objects before saving.
+- New XML parser implementation for better performance. You can revert to the
+old parser by enabling the config option XML_LIBXMLJS. Requires Node.js v11 or
+v10.
+- Performance optimizations. While performance has improved for the majority of
+use cases, there may be situations where performance has degraded. It's
+recommended to revisit your hardware requirements.
+- Connection request authentication no longer uses 'auth.js' file. Instead, the
+connection request authentication behavior can now be customized using an
+'expression'.
+- The config file (config.json) has been deprecated. System configuration (e.g.
+listen ports, worker count) are now recommended to be passed as environment.
+variables. Other general configuration options are stored in the database so as
+to not require service restart for changes to take effect.
+- Optional redis dependency has been removed completely.
+- Tags now allow only alphanumeric characters and underscore.
+- Supported versions of NodeJs and MongoDB are 10.x and up and 2.6 and up
+respectively.
+
+## 1.1.3 (2018-10-23)
+
+- New config option (MAX_COMMIT_ITERATIONS) to avoid max commit iterations
+faults for more complex scripts.
+- Support base64 and hexBinary parameter types.
+- Strict parsing of number values in queries (e.g. "123abc" no longer accepted
+as 123).
+- Mixing $ne and $not operators is not allowed. Now it throws an error instead
+of returning incorrect results.
+- When a task expires, any associated fault is also deleted.
+- API now accepts 'timeout' argument when posting a task.
+- A number of stability fixes.
+
+## 1.1.2 (2018-02-24)
+
+- A large number of bug fixes as well as stability and performance improvements.
+- Three security vulnerabilities disclosed by Maximilian Hils have been patched.
+- New config option UDP_CONNECTION_REQUEST_PORT to specify binding port for UDP
+connection requests.
+- New config option DATETIME_MILLISECONDS to strip milliseconds from dateTime
+values.
+- New config option BOOLEAN_LITERAL to use 1/0 or true/false for boolean values.
+- Parameter values that cannot be parsed according to the reported type now show
+a warning message.
+- Virtual parameter scripts now use the variable 'args' instead of the special
+'TIMESTAMPS' and 'VALUES' variables. The content of the args array is: {declare
+timestamps}, {declare values}, {current timestamps}, {current values}.
+- Virtual parameter value types are now inferred from the JavaScript type if the
+returned value attribute is not a value-type pair.
+- Show a fault when a virtual parameter script doesn't return the required
+attributes.
+- Redis is now optional (and disabled by default), reducing the complexity of
+scalable deployments.
+- Better detection of cyclical presets resulting in fewer faults for complex
+provisioning scripts.
+- Math.random() is now deterministic on per-device basis. A function has been
+added to allow specifying a seed value (e.g. Math.random.seed(Date.now())).
+- Overload spikes are now handled gracefully by refusing to accept new sessions
+temporarily when under abnormal load.
+- Added log messages for session timeouts, connection drops, and XML parsing
+errors.
+- Date.now() now takes an optional argument to specify "time steps" (in
+milliseconds). This can be used to ensure a group of parameters are all
+refreshed at the same time intervals.
+- Only the non-default configuration options are now logged at process start.
+- Faults caused by errors from extensions now show a cleaner stack trace.
+- Exit main process if there are too many worker crashes (e.g. when DB is down).
+- Updated dependencies and included a lockfile to ensure installations get the
+exact dependencies it was tested against.
+
+## 1.1.1 (2017-03-23)
+
+- Avoid crashing when connection request credentials are missing.
+- Show a warning instead of crashing when failing to parse parameter values
+according to the expected value type.
+- Add missing "Registered" event.
+- Fix bug where in certain cases many more instances than declared are created.
+- Fix parameter discovery bug when declared path timestamp is 1 or is not set.
+- Fix preset precondition failing when testing against datetime parameters and
+certain other parameters like \_deviceId.\_ProductClass.
+
+## 1.1.0 (2017-03-10)
+
+- Provisions enable implementing dynamic device configuration or complex device
+provisioning work flow using arbitrary scripts.
+- Virtual parameters are user-defined parameters whose values are evaluated from
+a custom script.
+- Extensions are sandboxed Node.js scripts that are accessible from provision
+and virtual parameter scripts to facilitate integration with external entities.
+- Support for UDP/STUN based connection requests for reaching devices behind
+NAT (TR-069 Annex G).
+- Presets can now be scheduled using a cron-like expression.
+- Presets can now be tied to specific device events (e.g. boot).
+- Presets precondition queries no longer support "$or" or other MongoDB logical
+operators.
+- Faults are no longer a part of tasks but are now first class objects.
+- Presets are now assigned to channels. A fault in one channel only blocks
+presets in that channel.
+- New API CRUD functions for provisions, virtual parameters, and faults.
+- New config options for XML output.
+- API responses now include "GenieACS-Version" header.
+- Graceful shutdown when receiving SIGINT and SIGTERM events.
+- Support SSL intermediate certificate chains.
+- Supported Node.js versions are 6.x and 7.x.
+- Supported MongoDB versions are 2.6 through 3.4.
+- Expect performance differences due to major under the hood changes. Some
+operations are faster and some are slower. Overall performance is improved.
+- GenieACS will no longer fetch the entire device data model upon first contact
+but will instead only fetch the parameters it needs to fulfill the presets.
+- Logs have been overhauled and split into two streams: process log (stderr) and
+access log (stdout). Also added config options to dump logs to files rather than
+standard streams.
+- Connection request authentication credentials are picked up from the device
+data model if available. config/auth.js is still supported as a fallback and now
+supports an optional callback argument.
+- Custom commands have been removed. Use virtual parameters and/or extensions.
+- Aliases and value normalizers (config/parameters.json) have been removed. Use
+virtual parameters.
+- The API /devices/<device_id>/preset has been removed.
+- Rarely used RequestDownload method no longer supported.
+- The TR-069 client simulator has moved to its own repo at
+https://github.com/zaidka/genieacs-sim

CONTRIBUTING.md

@@ -0,0 +1,39 @@
+# Have a question?
+
+Please ask in the [Users mailing list][mailing-lists] instead of opening a
+Github Issue. The Github issue tracker is strictly for bug reports.
+
+# Issues
+
+We use GitHub issues to track bugs. Please ensure your bug description is
+clear and has sufficient instructions to be able to reproduce the issue.
+
+If you face interoperability issues with your device, more often than not it's
+a device related issue. Please consult the [Users mailing list][mailing-lists]
+before submitting an Issue.
+
+# Pull Requests
+
+Pull requests are welcome. If you have a fix for a bug, go ahead and create a
+pull request. For other kinds of enhancements, please discuss in the [Users
+mailing list][mailing-lists] first.
+
+Please follow the guidelines [here][commit-guidelines] when committing your
+changes.
+
+## Contributor License Agreement
+
+By submitting a pull request to this repository, you acknowledge that, while
+maintaining copyright, you grant Zaid Abdulla a perpetual, worldwide, non-
+exclusive, no-charge, royalty-free, irrevocable license to reproduce, prepare
+derivative works of, publicly display, publicly perform, sublicense, and
+distribute your contributions and such derivative works under the AGPLv3
+license or any other license terms, including, but not limited to, proprietary
+or commercial license terms.
+
+You confirm that you own or have rights to distribute and sublicense the
+source code contained therein, and that your content does not infringe upon
+the intellectual property rights of a third party.
+
+[commit-guidelines]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
+[mailing-lists]: http://lists.genieacs.com

README.md

@@ -1,66 +1,86 @@
 # GenieACS
 
-GenieACS is a *blazing fast* TR-069 auto configuration server (ACS) built with Node.js, Redis, and MongoDB. It's technology neutral and configurable to fit any service provider needs. This is the core back end component. Official front end GUI is available at https://github.com/zaidka/genieacs-gui.
+GenieACS is a high performance Auto Configuration Server (ACS) for remote
+management of TR-069 enabled devices. It utilizes a declarative and fault
+tolerant configuration engine for automating complex provisioning scenarios at
+scale. It's battle-tested to handle hundreds of thousands and potentially
+millions of concurrent devices.
 
-## Features
+## Quick Start
 
-* **Massively concurrent**: Can handle hundreds of thousands of connected devices on a single server even with low inform interval.
-* **Preset-based configuration**: Define sets of configurations that devices will pick up and apply as needed based on given preconditions.
-* **Tagging**: Use tags to group devices for more manageable presets.
-* **Searching**: Query for devices on any parameter (supports all common operators including regular expressions).
-* **Parameter aliasing**: Define aliases to unify parameter paths from different types of devices. Aliases behave like normal parameters (i.e. can be queried or modified).
-* **Extensive API**: A simple yet rich HTTP-based API allows easy integration with other system.
+*Important: This is a pre-release branch. Use
+[v1.1](https://github.com/genieacs/genieacs/tree/v1.1) for production
+deployments.*
 
-## Getting started
+Install [Node.js](http://nodejs.org/) and [MongoDB](http://www.mongodb.org/).
+Refer to their corresponding documentation for installation instructions. The
+supported versions are:
 
-Install [Node.js](http://nodejs.org/), [Redis](http://redis.io/), and [MongoDB](http://www.mongodb.org/). Refer to the corresponding documentation for installation guides. Then use NPM to install GenieACS by typing:
+- Node.js: 10.x through 12.x
+- MongoDB: 2.6 through 4.1
 
-    npm install -g genieacs
+Clone this repo or download the source archive then *cd* into the source
+directory and install the required dependencies:
 
-You may need to modify the configuration files under "config" directory (in /lib/node_modules/genieacs/config) depending on your setup.
-
-Alternatively, you can use git to get the latest development version (not recommended for production use):
-
-    cd /opt
-    git clone https://github.com/zaidka/genieacs.git
-    cd genieacs
     npm install
-    npm run configure
-    npm run compile
 
-Finally, run the following in GNU Screen session or something similar:
+Now build:
 
-    genieacs-cwmp
+    npm run build
 
-This is the service that the CPEs will communicate with. It listens to port 7547 by default (see config/config.json). Configure the ACS URL of your devices accordingly.
+Finally, run the following services from the build that's generated under
+'dist' directory:
 
-    genieacs-nbi
+### genieacs-cwmp
 
-This is the northbound interface module. It exposes a REST API on port 7557 by default. This is needed for the GUI front end to communicate with.
+This is the service that the CPEs will communicate with. It listens on port
+7547 by default. Configure the ACS URL in your devices accordingly.
 
-    genieacs-fs
+    ./dist/bin/genieacs-cwmp
 
-This is the file server from which the CPEs will download firmware images and other types of files.
+You may optionally use [genieacs-sim](https://github.com/genieacs/genieacs-sim)
+as a dummy TR-069 simulator if you don't have a CPE at hand.
 
-You can use stream redirection to output to log files:
+### genieacs-nbi
 
-    genieacs-cwmp >> /var/log/genieacs-cwmp.log 2>> /var/log/genieacs-cwmp-err.log
-    genieacs-nbi >> /var/log/genieacs-nbi.log 2>> /var/log/genieacs-nbi-err.log
-    genieacs-fs >> /var/log/genieacs-fs.log 2>> /var/log/genieacs-fs-err.log
+This is the northbound interface module. It exposes a REST API on port 7557 by
+default. This one is only required if you have an external system integrating
+with GenieACS using this API.
 
-For further details about installation and configuration, refer to the [wiki section](https://github.com/zaidka/genieacs/wiki).
+    ./dist/bin/genieacs-nbi
 
-You may now proceed with installing [GenieACS GUI front end](https://github.com/zaidka/genieacs-gui).
+### genieacs-fs
+
+This is the file server from which the CPEs will download firmware images and
+such.
+
+    ./dist/bin/genieacs-fs
+
+### genieacs-ui
+
+This serves the web based user interface. It listens on port 3000 by default.
+
+    ./dist/bin/genieacs-ui --ui-jwt-secret secret
+
+The argument *--ui-jwt-secret* supplies the key used for signing browser
+cookies.
+
+The UI has plenty of configuration options. When you open GenieACS's UI in a
+browser you'll be greeted with a database initialization wizard to help you
+populate some initial configuration.
 
 ## Support
 
-Documentation is still work in progress. Feel free to contact me if you require any assistance in installation, configuration, or using the APIs.
+The [forum](https://forum.genieacs.com) is a good place to get guidance and
+help from the community. Head on over and join the conversation! In addition,
+the [wiki](https://github.com/genieacs/genieacs/wiki) provides useful
+documentation and tips from GenieACS users.
 
-You may submit bug reports or feature requests [here](https://github.com/zaidka/genieacs/issues).
+For commercial support options and professional services, please visit
+[genieacs.com](https://genieacs.com/support/).
 
-For commercial support options, please contact me.
-
-## Contributing
-
-Contributions are welcome. Fork this repo and open a pull request and wait for feedback. You can also contribute by enhancing the documentation in the [wiki section](https://github.com/zaidka/genieacs/wiki).
+## License
 
+Copyright 2013-2019 GenieACS Inc. GenieACS is released under the [AGPLv3
+license
+terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

ava.config.js

@@ -0,0 +1,6 @@
+export default {
+  "compileEnhancements": false,
+  "extensions": ["ts"],
+  "require": ["esm", "ts-node/register/transpile-only"]
+};
+

bin/genieacs-cwmp

@@ -1,9 +1,129 @@
-#!/usr/bin/env node
-var path = require('path');
-var fs = require('fs');
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
 
-var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.chdir(dir);
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
 
-cluster = require(path.resolve(dir, 'lib/cluster'));
-cluster.start('cwmp');
+import * as config from "../lib/config";
+import * as logger from "../lib/logger";
+import * as cluster from "../lib/cluster";
+import * as server from "../lib/server";
+import * as cwmp from "../lib/cwmp";
+import * as db from "../lib/db";
+import * as extensions from "../lib/extensions";
+import * as cache from "../lib/cache";
+import { version as VERSION } from "../package.json";
+
+logger.init("cwmp", VERSION);
+
+const SERVICE_ADDRESS = config.get("CWMP_INTERFACE");
+const SERVICE_PORT = config.get("CWMP_PORT");
+
+function exitWorkerGracefully(): void {
+  db.disconnect();
+  cache.disconnect();
+  extensions.killAll();
+  cluster.worker.disconnect();
+}
+
+function exitWorkerUngracefully(): void {
+  extensions.killAll().then(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("CWMP_WORKER_PROCESSES");
+
+  logger.info({
+    message: `genieacs-cwmp starting`,
+    pid: process.pid,
+    version: VERSION
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+} else {
+  const ssl = {
+    key: config.get("CWMP_SSL_KEY"),
+    cert: config.get("CWMP_SSL_CERT")
+  };
+
+  const keepAliveTimeout = config.get("CWMP_KEEP_ALIVE_TIMEOUT") as number;
+
+  process.on("uncaughtException", err => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid
+    });
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  process.on("SIGINT", () => {
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  process.on("SIGTERM", () => {
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  Promise.all([db.connect(), cache.connect()])
+    .then(() => {
+      server.start(
+        SERVICE_PORT,
+        SERVICE_ADDRESS,
+        ssl,
+        cwmp.listener,
+        cwmp.onConnection,
+        keepAliveTimeout
+      );
+    })
+    .catch(err => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+}

bin/genieacs-ext

@@ -0,0 +1,96 @@
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
+
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { Fault } from "../lib/types";
+
+const jobs = new Set();
+const fileName = process.argv[2];
+let script;
+
+function errorToFault(err: Error): Fault {
+  if (!err) return null;
+
+  if (!err.name) return { code: "ext", message: `${err}` };
+
+  const fault: Fault = {
+    code: `ext.${err.name}`,
+    message: err.message,
+    detail: {
+      name: err.name,
+      message: err.message
+    }
+  };
+
+  if (err.stack) {
+    fault.detail["stack"] = err.stack;
+    // Trim the stack trace
+    const stackTrimIndex = fault.detail["stack"].match(
+      /\s+at\s[^\s]+\s\(.*genieacs-ext:.+\)/
+    );
+    if (stackTrimIndex) {
+      fault.detail["stack"] = fault.detail["stack"].slice(
+        0,
+        stackTrimIndex.index
+      );
+    }
+  }
+
+  return fault;
+}
+
+process.on("uncaughtException", err => {
+  const fault = errorToFault(err);
+  jobs.forEach(jobId => {
+    process.send([jobId, fault, null]);
+  });
+  jobs.clear();
+  process.disconnect();
+});
+
+process.on("message", message => {
+  jobs.add(message[0]);
+
+  if (!script) {
+    const cwd = process.env["GENIEACS_EXT_DIR"];
+    process.chdir(cwd);
+    script = require(`${cwd}/${fileName}`);
+  }
+
+  const funcName = message[1][0];
+
+  if (!script[funcName]) {
+    const fault = {
+      code: "ext",
+      message: `No such function '${funcName}' in extension '${fileName}'`
+    };
+    process.send([message[0], fault, null]);
+    return;
+  }
+
+  script[funcName](message[1].slice(1), (err, res) => {
+    if (!jobs.delete(message[0])) return;
+
+    process.send([message[0], errorToFault(err), res]);
+  });
+});
+
+// Ignore SIGINT
+process.on("SIGINT", () => {});

bin/genieacs-fs

@@ -1,9 +1,116 @@
-#!/usr/bin/env node
-var path = require('path');
-var fs = require('fs');
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
 
-var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.chdir(dir);
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
 
-cluster = require(path.resolve(dir, 'lib/cluster'));
-cluster.start('fs');
+import * as config from "../lib/config";
+import * as logger from "../lib/logger";
+import * as cluster from "../lib/cluster";
+import * as server from "../lib/server";
+import { listener } from "../lib/fs";
+import * as db from "../lib/db";
+import * as cache from "../lib/cache";
+import { version as VERSION } from "../package.json";
+
+logger.init("fs", VERSION);
+
+const SERVICE_ADDRESS = config.get("FS_INTERFACE");
+const SERVICE_PORT = config.get("FS_PORT");
+
+function exitWorkerGracefully(): void {
+  db.disconnect();
+  cache.disconnect();
+  cluster.worker.disconnect();
+}
+
+function exitWorkerUngracefully(): void {
+  process.exit(1);
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("FS_WORKER_PROCESSES");
+
+  logger.info({
+    message: `genieacs-fs starting`,
+    pid: process.pid,
+    version: VERSION
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+} else {
+  const ssl = {
+    key: config.get("FS_SSL_KEY"),
+    cert: config.get("FS_SSL_CERT")
+  };
+
+  process.on("uncaughtException", err => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid
+    });
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  process.on("SIGINT", () => {
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  process.on("SIGTERM", () => {
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  Promise.all([db.connect(), cache.connect()])
+    .then(() => {
+      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
+    })
+    .catch(err => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+}

bin/genieacs-nbi

@@ -1,9 +1,120 @@
-#!/usr/bin/env node
-var path = require('path');
-var fs = require('fs');
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
 
-var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.chdir(dir);
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
 
-cluster = require(path.resolve(dir, 'lib/cluster'));
-cluster.start('nbi');
+import * as config from "../lib/config";
+import * as logger from "../lib/logger";
+import * as cluster from "../lib/cluster";
+import * as server from "../lib/server";
+import { listener } from "../lib/nbi";
+import * as db from "../lib/db";
+import * as extensions from "../lib/extensions";
+import * as cache from "../lib/cache";
+import { version as VERSION } from "../package.json";
+
+logger.init("nbi", VERSION);
+
+const SERVICE_ADDRESS = config.get("NBI_INTERFACE");
+const SERVICE_PORT = config.get("NBI_PORT");
+
+function exitWorkerGracefully(): void {
+  db.disconnect();
+  cache.disconnect();
+  extensions.killAll();
+  cluster.worker.disconnect();
+}
+
+function exitWorkerUngracefully(): void {
+  extensions.killAll().then(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("NBI_WORKER_PROCESSES");
+
+  logger.info({
+    message: `genieacs-nbi starting`,
+    pid: process.pid,
+    version: VERSION
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+} else {
+  const ssl = {
+    key: config.get("NBI_SSL_KEY"),
+    cert: config.get("NBI_SSL_CERT")
+  };
+
+  process.on("uncaughtException", err => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid
+    });
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  process.on("SIGINT", () => {
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  process.on("SIGTERM", () => {
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  Promise.all([db.connect(), cache.connect()])
+    .then(() => {
+      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
+    })
+    .catch(err => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+}

bin/genieacs-ui

@@ -0,0 +1,132 @@
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
+
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as config from "../lib/config";
+import * as logger from "../lib/logger";
+import * as cluster from "../lib/cluster";
+import * as server from "../lib/server";
+import { listener } from "../lib/ui";
+import * as extensions from "../lib/extensions";
+import * as db from "../lib/ui/db";
+import * as db2 from "../lib/db";
+import * as cache from "../lib/cache";
+import { version as VERSION } from "../package.json";
+
+logger.init("ui", VERSION);
+
+const SERVICE_ADDRESS = config.get("UI_INTERFACE");
+const SERVICE_PORT = config.get("UI_PORT");
+
+function exitWorkerGracefully(): void {
+  db.disconnect();
+  db2.disconnect();
+  cache.disconnect();
+  extensions.killAll();
+  cluster.worker.disconnect();
+}
+
+function exitWorkerUngracefully(): void {
+  extensions.killAll().then(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("UI_WORKER_PROCESSES");
+
+  logger.info({
+    message: `genieacs-ui starting`,
+    pid: process.pid,
+    version: VERSION
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid
+    });
+
+    cluster.stop();
+  });
+} else {
+  const ssl = {
+    key: config.get("UI_SSL_KEY"),
+    cert: config.get("UI_SSL_CERT")
+  };
+
+  let stopping = false;
+
+  process.on("uncaughtException", err => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid
+    });
+    stopping = true;
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  process.on("SIGINT", () => {
+    stopping = true;
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  process.on("SIGTERM", () => {
+    stopping = true;
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
+  });
+
+  const _listener = (req, res): void => {
+    if (stopping) res.setHeader("Connection", "close");
+    listener(req, res);
+  };
+
+  Promise.all([db2.connect(), cache.connect()])
+    .then(() => {
+      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
+    })
+    .catch(err => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+}

build/build.ts

@@ -0,0 +1,390 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as path from "path";
+import * as fs from "fs";
+import { promisify } from "util";
+import { rollup, WarningHandler } from "rollup";
+import rollupReplace from "rollup-plugin-replace";
+import rollupJson from "rollup-plugin-json";
+import typescript from "rollup-plugin-typescript";
+import { terser } from "rollup-plugin-terser";
+import webpack from "webpack";
+import postcss from "postcss";
+import postcssImport from "postcss-import";
+import postcssPresetEnv from "postcss-preset-env";
+import cssnano from "cssnano";
+import SVGO from "svgo";
+import * as xmlParser from "../lib/xml-parser";
+
+const MODE = process.env["NODE_ENV"] || "production";
+
+const BUILD_METADATA = new Date()
+  .toISOString()
+  .split(".")[0]
+  .replace(/[^0-9]/g, "");
+
+const INPUT_DIR = path.resolve(__dirname, "..");
+const OUTPUT_DIR = path.resolve(__dirname, "../dist");
+
+const externals = [
+  "path",
+  "fs",
+  "cluster",
+  "os",
+  "tls",
+  "http",
+  "https",
+  "zlib",
+  "crypto",
+  "util",
+  "mongodb",
+  "libxmljs",
+  "vm",
+  "later",
+  "parsimmon",
+  "seedrandom",
+  "querystring",
+  "child_process",
+  "dgram",
+  "url",
+  "koa",
+  "koa-router",
+  "koa-compress",
+  "koa-bodyparser",
+  "koa-jwt",
+  "koa-static",
+  "jsonwebtoken",
+  "stream",
+  "mithril",
+  "parsimmon",
+  "yaml",
+  "codemirror",
+  "codemirror/mode/javascript/javascript",
+  "codemirror/mode/yaml/yaml"
+];
+
+function rmDirSync(dirPath): void {
+  if (!fs.existsSync(dirPath)) return;
+  const files = fs.readdirSync(dirPath);
+
+  for (const file of files) {
+    const filePath = `${dirPath}/${file}`;
+    if (fs.statSync(filePath).isFile()) fs.unlinkSync(filePath);
+    else rmDirSync(filePath);
+  }
+  fs.rmdirSync(dirPath);
+}
+
+function stripDevDeps(deps): void {
+  if (!deps["dependencies"]) return;
+  for (const [k, v] of Object.entries(deps["dependencies"])) {
+    if (v["dev"]) delete deps["dependencies"][k];
+    else stripDevDeps(v);
+  }
+  if (!Object.keys(deps["dependencies"]).length) delete deps["dependencies"];
+}
+
+function xmlTostring(xml): string {
+  const children = [];
+  for (const c of xml.children || []) children.push(xmlTostring(c));
+
+  return xml.name === "root" && xml.bodyIndex === 0
+    ? children.join("")
+    : `<${xml.name} ${xml.attrs}>${children.join("")}</${xml.name}>`;
+}
+
+function generateSymbol(id: string, svgStr: string): string {
+  const xml = xmlParser.parseXml(svgStr);
+  const svg = xml.children[0];
+  const svgAttrs = xmlParser.parseAttrs(svg.attrs);
+  let viewBox = "";
+  for (const a of svgAttrs) {
+    if (a.name === "viewBox") {
+      viewBox = `viewBox="${a.value}"`;
+      break;
+    }
+  }
+  const symbolBody = xml.children[0].children
+    .map(c => {
+      return xmlTostring(c);
+    })
+    .join("");
+  return `<symbol id="icon-${id}" ${viewBox}>${symbolBody}</symbol>`;
+}
+
+async function init(): Promise<void> {
+  // Delete any old output directory
+  rmDirSync(OUTPUT_DIR);
+
+  // Create output directory layout
+  fs.mkdirSync(OUTPUT_DIR);
+  fs.mkdirSync(OUTPUT_DIR + "/bin");
+  fs.mkdirSync(OUTPUT_DIR + "/public");
+  fs.mkdirSync(OUTPUT_DIR + "/tools");
+
+  // Create package.json
+  const packageJson = JSON.parse(
+    fs.readFileSync(path.resolve(INPUT_DIR, "package.json")).toString()
+  );
+  delete packageJson["devDependencies"];
+  packageJson["scripts"] = {
+    install: packageJson["scripts"].install,
+    configure: packageJson["scripts"].configure
+  };
+  packageJson["version"] = `${packageJson["version"]}+${BUILD_METADATA}`;
+  fs.writeFileSync(
+    path.resolve(OUTPUT_DIR, "package.json"),
+    JSON.stringify(packageJson, null, 2)
+  );
+
+  // Create npm-shrinkwrap.json
+  const npmShrinkwrapJson = JSON.parse(
+    fs.readFileSync(path.resolve(INPUT_DIR, "npm-shrinkwrap.json")).toString()
+  );
+  npmShrinkwrapJson["version"] = packageJson["version"];
+  stripDevDeps(npmShrinkwrapJson);
+  fs.writeFileSync(
+    path.resolve(OUTPUT_DIR, "npm-shrinkwrap.json"),
+    JSON.stringify(npmShrinkwrapJson, null, 2)
+  );
+}
+
+async function copyStatic(): Promise<void> {
+  const files = [
+    "LICENSE",
+    "README.md",
+    "CHANGELOG.md",
+    "public/logo.svg",
+    "public/favicon.png"
+  ];
+
+  for (const file of files) {
+    fs.copyFileSync(
+      path.resolve(INPUT_DIR, file),
+      path.resolve(OUTPUT_DIR, file)
+    );
+  }
+}
+
+async function generateCss(): Promise<void> {
+  const cssInPath = path.resolve(INPUT_DIR, "ui/css/app.css");
+  const cssOutPath = path.resolve(OUTPUT_DIR, "public/app.css");
+  const cssIn = fs.readFileSync(cssInPath);
+  const cssOut = await postcss([
+    postcssImport,
+    postcssPresetEnv({
+      stage: 3,
+      features: {
+        "nesting-rules": true,
+        "color-mod-function": true
+      }
+    }),
+    cssnano
+  ]).process(cssIn, { from: cssInPath, to: cssOutPath });
+  fs.writeFileSync(cssOutPath, cssOut.css);
+}
+
+async function generateToolsJs(): Promise<void> {
+  for (const bin of ["dump-data-model"]) {
+    const inputFile = path.resolve(INPUT_DIR, `tools/${bin}`);
+    const outputFile = path.resolve(OUTPUT_DIR, `tools/${bin}`);
+    const bundle = await rollup({
+      input: inputFile,
+      external: externals,
+      acorn: {
+        allowHashBang: true
+      },
+      plugins: [
+        rollupReplace({
+          delimiters: ["", ""],
+          "#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only": ""
+        }),
+        typescript({
+          tsconfig: "./tsconfig.json",
+          include: [`tools/${bin}`, "lib/**/*.ts"]
+        }),
+        MODE === "production" ? terser() : null
+      ]
+    });
+
+    await bundle.write({
+      format: "cjs",
+      preferConst: true,
+      sourcemap: "inline",
+      sourcemapExcludeSources: true,
+      banner: "#!/usr/bin/env node",
+      file: outputFile
+    });
+
+    // Mark as executable
+    const mode = fs.statSync(outputFile).mode;
+    fs.chmodSync(outputFile, mode | 73);
+  }
+}
+
+async function generateBackendJs(): Promise<void> {
+  for (const bin of [
+    "genieacs-cwmp",
+    "genieacs-ext",
+    "genieacs-nbi",
+    "genieacs-fs",
+    "genieacs-ui"
+  ]) {
+    const inputFile = path.resolve(INPUT_DIR, `bin/${bin}`);
+    const outputFile = path.resolve(OUTPUT_DIR, `bin/${bin}`);
+    const bundle = await rollup({
+      input: inputFile,
+      external: externals,
+      acorn: {
+        allowHashBang: true
+      },
+      treeshake: {
+        propertyReadSideEffects: false,
+        pureExternalModules: true
+      },
+      plugins: [
+        rollupReplace({
+          delimiters: ["", ""],
+          "#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only": ""
+        }),
+        rollupJson({ preferConst: true }),
+        {
+          resolveId: (importee, importer) => {
+            if (importee.endsWith("/package.json")) {
+              const p = path.resolve(path.dirname(importer), importee);
+              if (p === path.resolve(INPUT_DIR, "package.json"))
+                return path.resolve(OUTPUT_DIR, "package.json");
+            }
+            return null;
+          }
+        },
+        typescript({
+          tsconfig: "./tsconfig.json",
+          include: [`bin/${bin}`, "lib/**/*.ts"]
+        }),
+        MODE === "production" ? terser() : null
+      ]
+    });
+
+    await bundle.write({
+      format: "cjs",
+      preferConst: true,
+      sourcemap: "inline",
+      sourcemapExcludeSources: true,
+      banner: "#!/usr/bin/env node",
+      file: outputFile
+    });
+
+    // Mark as executable
+    const mode = fs.statSync(outputFile).mode;
+    fs.chmodSync(outputFile, mode | 73);
+  }
+}
+
+async function generateFrontendJs(): Promise<void> {
+  const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
+  const outputFile = path.resolve(OUTPUT_DIR, "public/app.js");
+
+  const bundle = await rollup({
+    input: inputFile,
+    external: externals,
+    plugins: [
+      rollupJson({ preferConst: true }),
+      typescript({ tsconfig: "./tsconfig.json" })
+    ],
+    inlineDynamicImports: true,
+    treeshake: {
+      propertyReadSideEffects: false,
+      pureExternalModules: true
+    },
+    onwarn: ((warning, warn) => {
+      // Ignore circular dependency warnings
+      if (warning.code !== "CIRCULAR_DEPENDENCY") warn(warning);
+    }) as WarningHandler
+  });
+
+  await bundle.write({
+    preferConst: true,
+    format: "esm",
+    sourcemap: "inline",
+    sourcemapExcludeSources: true,
+    file: outputFile
+  });
+
+  const webpackConf = {
+    mode: MODE,
+    entry: outputFile,
+    resolve: {
+      aliasFields: ["module"]
+    },
+    devtool: "nosources-source-map",
+    module: {
+      rules: [
+        {
+          test: /\.js$/,
+          use: ["source-map-loader"],
+          enforce: "pre"
+        }
+      ]
+    },
+    output: {
+      path: path.resolve(OUTPUT_DIR, "public"),
+      filename: "app.js"
+    }
+  };
+
+  const stats = await promisify(webpack)(webpackConf);
+  process.stdout.write(stats.toString({ colors: true }) + "\n");
+}
+
+async function generateIconsSprite(): Promise<void> {
+  const svgo = new SVGO({ plugins: [{ removeViewBox: false }] });
+  const symbols = [];
+  const iconsDir = path.resolve(INPUT_DIR, "ui/icons");
+  for (const file of fs.readdirSync(iconsDir)) {
+    const id = path.parse(file).name;
+    const filePath = path.join(iconsDir, file);
+    const { data } = await svgo.optimize(fs.readFileSync(filePath).toString());
+    symbols.push(generateSymbol(id, data));
+  }
+  fs.writeFileSync(
+    path.resolve(OUTPUT_DIR, "public/icons.svg"),
+    `<svg xmlns="http://www.w3.org/2000/svg">${symbols.join("")}</svg>`
+  );
+}
+
+init()
+  .then(() => {
+    Promise.all([
+      copyStatic(),
+      generateCss(),
+      generateIconsSprite(),
+      generateToolsJs(),
+      generateBackendJs(),
+      generateFrontendJs()
+    ])
+      .then(() => {})
+      .catch(err => {
+        process.stderr.write(err.stack + "\n");
+      });
+  })
+  .catch(err => {
+    process.stderr.write(err.stack + "\n");
+  });

config/auth-sample.js

@@ -1,4 +0,0 @@
-exports.connectionRequest = function(deviceId) {
-  // return username/password pair for a given device
-  return ["", ""];
-}

config/config-sample.json

@@ -1,15 +0,0 @@
-{
-  "MONGODB_CONNECTION_URL" : "mongodb://127.0.0.1/genieacs",
-  "REDIS_PORT" : "6379",
-  "REDIS_HOST" : "127.0.0.1",
-  "CWMP_INTERFACE" : "0.0.0.0",
-  "CWMP_PORT" : 7547,
-  "CWMP_SSL" : false,
-  "NBI_INTERFACE" : "0.0.0.0",
-  "NBI_PORT" : 7557,
-  "FS_INTERFACE" : "0.0.0.0",
-  "FS_PORT" : 7567,
-  "FS_IP" : "192.168.0.1",
-  "LOG_INFORMS" : true,
-  "DEBUG" : false
-}

config/custom_commands-sample.json

@@ -1,4 +0,0 @@
-{
-  "seowon_frequencies" : "/^[0-9]+\\-SWU\\%2D9200\\-[0-9A-Z]+$/",
-  "greenpacket_frequencies" : "/^[0-9A-Z]+\\-DX350\\-[0-9A-Z]+$/"
-}

config/custom_commands/greenpacket_frequencies.js

@@ -1,210 +0,0 @@
-/*
- * Copyright 2013 Fanoos Telecom
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-var net = require('net');
-var PORT = 23;
-var USERNAME = 'mt7109';
-var PASSWORD = 'wimax';
-var TIMEOUT = 5000;
-
-var FREQUENCIES_REGEX = /([0-9,]+)/
-var BANDWIDTHS_REGEX = /[0-9]+\-([0-9,]+)/
-var DEFAULT_BW_REGEX = /([0-9]+)\-[0-9,]+/
-var telnetConnect = function(host, callback) {
-  var client = net.connect(PORT, host, function() {
-    // enter username
-    telnetExecute(USERNAME, client, "Password: ", function(err, response) {
-      if (err) return callback(err);
-      // enter password
-      telnetExecute(PASSWORD, client, "\n# ", function(err, response) {
-        return callback(err);
-      });
-    });
-  });
-  return client;
-}
-
-var telnetExecute = function(command, connection, prompt, callback) {
-  var response = '';
-  command += "\n";
-  var listener = function(chunk) {
-    response += chunk.toString();
-    if (response.slice(0 - prompt.length) == prompt) {
-      connection.removeListener('data', listener);
-      return callback(null, response.slice(command.length, 0 - prompt.length));
-    }
-  };
-
-  var end = function() {
-    connection.removeListener('end', end);
-    return callback(null, response.slice(command.length));
-  };
-
-  if (prompt)
-    connection.on('data', listener);
-  else
-    connection.on('end', end);
-
-  if (command)
-    connection.write(command);
-}
-
-var getDeviceIp = function(deviceId, callback) {
-  var db = require('../../lib/db');
-  var URL = require('url');
-
-  db.devicesCollection.findOne({_id : deviceId}, {'InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value' : 1}, function(err, device) {
-    var connectionRequestUrl = device.InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value;
-    var url = URL.parse(connectionRequestUrl);
-    return callback(url.hostname);
-  });
-};
-
-exports.init = function(deviceId, args, callback) {
-  return exports.get(deviceId, args, callback);
-}
-
-exports.get = function(deviceId, args, callback) {
-  var currentFrequencies = [];
-  var currentBandwidths = [];
-  var defaultBandwidth = '';
-  var final = [];
-
-  getDeviceIp(deviceId, function(ip) {
-    var client = telnetConnect(ip, function(err) {
-      // get frequencies
-      telnetExecute("sncfg get WMX_FREQ_LIST", client, "\n# ", function(err, response) {
-        if (err) return callback(err);
-        currentFrequencies = FREQUENCIES_REGEX.exec(response)[1].split(',')
-        // get bandwidths
-        telnetExecute("sncfg get WMX_FREQ_BANDWDITH_FL", client, "\n# ", function(err, response) {
-          if (err) return callback(err);
-          currentBandwidths = BANDWIDTHS_REGEX.exec(response)[1].split(',')
-          defaultBandwidth = DEFAULT_BW_REGEX.exec(response)[1]
-          // generate freq/bw pairs string
-          for (var i = 0; i < currentFrequencies.length; i++) {
-            final.push(currentFrequencies[i] + ' ' + currentBandwidths[i])
-          }
-          // log out
-          telnetExecute("exit", client, null, function(err, response) {
-            client.end();
-            return callback(err, defaultBandwidth + '-' + final.join(','));
-          });
-        });
-      });
-    });
-
-    client.on("error", function(err) {
-      client.destroy();
-      callback(err);
-    });
-
-    client.setTimeout(TIMEOUT, function() {
-      client.destroy();
-      callback("timeout");
-    });
-  });
-};
-
-var parsePairs = function(pairsString) {
-  var a = pairsString.split(',');
-  var f = [];
-  var b = [];
-  var l = [];
-  for (i in a) {
-    var sp = a[i].trim().split(' ');
-    f.push(parseInt(sp[0]));
-    b.push(parseFloat(sp[1]));
-  }
-  l.push(f.join(','));
-  l.push(b.join(','));
-  return l;
-}
-
-exports.set = function(deviceId, args, callback) {
-  executeSet(deviceId, args, ['set'], callback);
-};
-
-exports.dset = function(deviceId, args, callback) {
-  executeSet(deviceId, args, ['dset'], callback);
-};
-
-exports.setdset = function(deviceId, args, callback) {
-  executeSet(deviceId, args, ['dset', 'set'], callback);
-};
-
-var executeSet = function(deviceId, args, method, callback) {
-  var tmp = args.split('-');
-  var defaultBandwidth = tmp[0];
-  var pairs = parsePairs(tmp[1]);
-  var frequenciesString = pairs[0];
-  var bandwidthsString = pairs[1];
-
-  getDeviceIp(deviceId, function(ip) {
-    var client = telnetConnect(ip, function(err) {
-      var telnet_cmd = "sncfg " + method[0];
-      telnetExecute(telnet_cmd + " WMX_FREQ_BANDWDITH_FL '" + defaultBandwidth + "-" + bandwidthsString + "'", client, "\n# ", function(err, response) {
-        if (err) return callback(err);
-        telnetExecute(telnet_cmd + " WMX_FREQ_LIST '" + frequenciesString + "'", client, "\n# ", function(err, response) {
-          if (err) return callback(err);
-          if (method.length > 1) {
-            telnet_cmd = "sncfg " + method[1];
-            telnetExecute(telnet_cmd + " WMX_FREQ_BANDWDITH_FL '" + defaultBandwidth + "-" + bandwidthsString + "'", client, "\n# ", function(err, response) {
-              if (err) return callback(err);
-              telnetExecute(telnet_cmd + " WMX_FREQ_LIST '" + frequenciesString + "'", client, "\n# ", function(err, response) {
-                if (err) return callback(err);
-                telnetExecute("sncfg commit", client, "\n# ", function(err, response) {
-                  if (err) return callback(err);
-                  // log out
-                  telnetExecute("exit", client, null, function(err, response) {
-                    client.end();
-                    return callback(null, args);
-                  });
-                });
-              });
-            });
-          }
-          else {
-            telnetExecute("sncfg commit", client, "\n# ", function(err, response) {
-            if (err) return callback(err);
-              // log out
-              telnetExecute("exit", client, null, function(err, response) {
-                client.end();
-                return callback(null, args);
-              });
-            });
-          }
-        });
-      });
-    });
-
-    client.on("error", function(err) {
-      client.destroy();
-      callback(err);
-    });
-
-    client.setTimeout(TIMEOUT, function() {
-      client.destroy();
-      callback("timeout");
-    });
-  });
-};

config/custom_commands/seowon_frequencies.js

@@ -1,230 +0,0 @@
-/*
- * Copyright 2013 Fanoos Telecom
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-// Change frequency plans for Seowon devices.
-// Args format: 3532500 5000,3537500 5000,3542500 5000,3547500 5000,3552500 5000,3557500 5000,3505000 10000,3515000 10000,3525000 10000,3535000 10000,3545000 10000,3555000 10000
-var net = require('net');
-var PORT = 23;
-var USERNAME = 'admin';
-var PASSWORD = 'admin';
-var TIMEOUT = 5000;
-
-var FREQUENCIES_REGEX = /\s*([\d]+)\.\s*Frequency\s*:\s*([\d]+)\s\(KHz\),\s*Bandwidth\s*:\s*([\d\.]+)\s*\(MHz\)/gm
-
-var telnetConnect = function(host, callback) {
-  var client = net.connect(PORT, host, function() {
-    // enter username
-    telnetExecute(USERNAME, client, "Password: ", function(err, response) {
-      if (err) return callback(err);
-      // enter password
-      telnetExecute(PASSWORD, client, "MT7109> ", function(err, response) {
-        return callback(err);
-      });
-    });
-  });
-  return client;
-}
-
-var telnetExecute = function(command, connection, prompt, callback) {
-  var response = '';
-  command += "\n";
-  var listener = function(chunk) {
-    response += chunk.toString();
-    if (response.slice(0 - prompt.length) == prompt) {
-      connection.removeListener('data', listener);
-      return callback(null, response.slice(command.length, 0 - prompt.length));
-    }
-  };
-
-  var end = function() {
-    connection.removeListener('end', end);
-    return callback(null, response.slice(command.length));
-  };
-
-  if (prompt)
-    connection.on('data', listener);
-  else
-    connection.on('end', end);
-
-  if (command)
-    connection.write(command);
-}
-
-var parseFrequencies = function(frequenciesString) {
-  var a = frequenciesString.split(',');
-  var l = [];
-  for (i in a) {
-    var sp = a[i].trim().split(' ');
-    var frequency = parseInt(sp[0]);
-    var bandwidth = parseFloat(sp[1]);
-    l.push([frequency, bandwidth]);
-  }
-  return l;
-}
-
-var findFrequency = function(haystack, needle) {
-  for (i in haystack) {
-    if (haystack[i][0] == needle[0] && haystack[i][1] == needle[1])
-      return i;
-  }
-  return -1;
-}
-
-var getDeviceIp = function(deviceId, callback) {
-  var db = require('../../lib/db');
-  var URL = require('url');
-
-  db.devicesCollection.findOne({_id : deviceId}, {'InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value' : 1}, function(err, device) {
-    var connectionRequestUrl = device.InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value;
-    var url = URL.parse(connectionRequestUrl);
-    return callback(url.hostname);
-  });
-};
-
-exports.init = function(deviceId, args, callback) {
-  return exports.get(deviceId, args, callback);
-};
-
-exports.get = function(deviceId, args, callback) {
-  var currentFrequencies = [];
-
-  getDeviceIp(deviceId, function(ip) {
-    var client = telnetConnect(ip, function(err) {
-      // enter privileged mode
-      telnetExecute("enable", client, "MT7109# ", function(err, response) {
-        if (err) return callback(err);
-        // show frequencies
-        telnetExecute("show wmx freq", client, "MT7109# ", function(err, response) {
-          if (err) return callback(err);
-          while (f = FREQUENCIES_REGEX.exec(response)) {
-            var frequency = parseInt(f[2]);
-            var bandwidth = Math.floor(parseFloat(f[3]) * 1000);
-            var freqStr = String(frequency) + ' ' + String(bandwidth);
-            currentFrequencies.push(freqStr);
-          }
-          // log out
-          telnetExecute("logout", client, null, function(err, response) {
-            client.end();
-            return callback(err, currentFrequencies.join(','));
-          });
-        });
-      });
-    });
-
-    client.on("error", function(err) {
-      client.destroy();
-      callback(err);
-    });
-
-    client.setTimeout(TIMEOUT, function() {
-      client.destroy();
-      callback("timeout");
-    });
-  });
-};
-
-exports.set = function(deviceId, args, callback) {
-  var newFrequencies = parseFrequencies(args);
-  var addFrequencies = [];
-  var delFrequencies = [];
-
-  getDeviceIp(deviceId, function(ip) {
-    var client = telnetConnect(ip, function(err) {
-      // enter privileged mode
-      telnetExecute("enable", client, "MT7109# ", function(err, response) {
-        if (err) return callback(err);
-        // show frequencies
-        telnetExecute("show wmx freq", client, "MT7109# ", function(err, response) {
-          if (err) return callback(err);
-          while (f = FREQUENCIES_REGEX.exec(response)) {
-            var index = parseInt(f[1]);
-            var frequency = parseInt(f[2]);
-            var bandwidth = Math.floor(parseFloat(f[3]));
-
-            var i = findFrequency(newFrequencies, [frequency, bandwidth * 1000]);
-            if (i == -1)
-              delFrequencies.push(index);
-            else
-              newFrequencies.splice(i, 1);
-          }
-          for (f in newFrequencies)
-            addFrequencies.push(newFrequencies[f]);
-
-          if (addFrequencies.length == 0 && delFrequencies.length == 0) {
-            // log out
-            telnetExecute("logout", client, null, function(err, response) {
-              client.end();
-              return callback(err, args);
-            });
-            return;
-          }
-
-          telnetExecute("wimax", client, "MT7109 (WiMax)# ", function(err, response) {
-            if (err) return callback(err);
-            var add, del;
-            del = function() {
-              if (delFrequencies.length == 0) {
-                add();
-                return;
-              }
-              var i = delFrequencies.pop();
-              telnetExecute("wmx freq del " + i, client, "MT7109 (WiMax)# ", function(err, response) {
-                if (err) return callback(err);
-                del();
-              });
-            };
-            add = function() {
-              if (addFrequencies.length == 0) {
-                // commit
-                telnetExecute("commit", client, "MT7109 (WiMax)# ", function(err, response) {
-                  if (err) return callback(err);
-                  // log out
-                  telnetExecute("logout", client, null, function(err, response) {
-                    client.end();
-                    return callback(null, args);
-                  });
-                });
-                return;
-              }
-              var i = addFrequencies.pop();
-              telnetExecute("wmx freq add " + i[0] + ' ' + i[1]/1000, client, "MT7109 (WiMax)# ", function(err, response) {
-                if (err) return callback(err);
-                add();
-              });
-            };
-            del();
-          });
-        });
-      });
-    });
-
-    client.on("error", function(err) {
-      client.destroy();
-      callback(err);
-    });
-
-    client.setTimeout(TIMEOUT, function() {
-      client.destroy();
-      callback("timeout");
-    });
-  });
-};

config/parameters-sample.json

@@ -1,19 +0,0 @@
-{
-  "_lastInform" : {"alias" : "summary.lastInform", "type" : "date"},
-  "_lastBoot" : {"alias" : "summary.lastBoot", "type" : "date"},
-  "_lastBootstrap" : {"alias" : "summary.lastBootstrap", "type" : "date"},
-  "_deviceId._SerialNumber" : {"alias" : "summary.serialNumber"},
-  "_deviceId._ProductClass" : {"alias" : "summary.productClass"},
-  "_deviceId._OUI" : {"alias" : "summary.oui"},
-  "_deviceId._Manufacturer" : {"alias" : "summary.manufacturer"},
-  "InternetGatewayDevice.DeviceInfo.HardwareVersion" : {"alias" : "summary.hardwareVersion"},
-  "InternetGatewayDevice.DeviceInfo.SoftwareVersion" : {"alias" : "summary.softwareVersion"},
-
-  "InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
-  "InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
-  "InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
-  "InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
-
-  "InternetGatewayDevice.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.SSID" : {"alias" : "summary.wlanSsid"},
-  "InternetGatewayDevice.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.PreSharedKey.1.KeyPassphrase" : {"alias" : "summary.wlanPassphrase"}
-}

docs/api-reference.rst

@@ -0,0 +1,529 @@
+API Reference
+=============
+
+GenieACS expoeses a rich RESTful API through its NBI component. This document
+serves as a reference for the available APIs.
+
+This API makes use of MongoDB's query language in some of its endpoints. Refer
+to MongoDB's documentation for details.
+
+.. note::
+
+  The examples below use ``curl`` command for simplicity and ease of testing.
+  Query parameters are URL-encoded, but the original pre-encoding values are
+  shown for reference. These examples assume genieacs-nbi is running locally
+  and listening on the default NBI port (7557).
+
+.. warning::
+
+  A common pitfll is not properly percent-encoding special characters in the
+  device ID or query in the URL.
+
+Endpoints
+---------
+
+GET /\<collection\>/?query=\<query\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Search for records in the database (e.g. devices, tasks, presets, files).
+Returns a JSON representation of all items in the given collection that match
+the search criteria.
+
+*collection*: The data collection to search. Could be one of: tasks, devices,
+presets, objects.
+
+*query*: Search query. Refer to MongoDB queries for reference.
+
+Examples
+^^^^^^^^
+
+- Find a device by its ID:
+
+.. code:: javascript
+
+  query = {"_id": "202BC1-BM632w-0000000"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%22202BC1-BM632w-0000000%22%7D'
+
+- Find a device by its MAC address:
+
+.. code:: javascript
+
+  query = {
+    "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress": "20:2B:C1:E0:06:65"
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress%22%3A%2220:2B:C1:E0:06:65%22%7D'
+
+- Search for devices that have not initiated an inform in the last 7 days.
+
+.. code:: javascript
+
+  query = {
+    "_lastInform": {
+      "$lt" : "2017-12-11 13:16:23 +0000"
+    }
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_lastInform%22%3A%7B%22%24lt%22%3A%222017-12-11%2013%3A16%3A23%20%2B0000%22%7D%7D'
+
+- Show pending tasks for a given device:
+
+.. code:: javascript
+
+  query = {"device": "202BC1-BM632w-0000000"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/tasks/?query=%7B%22device%22%3A%22202BC1-BM632w-0000000%22%7D'
+
+- Return specific parameters for a given device:
+
+.. code:: javascript
+
+  query = {"_id": "202BC1-BM632w-0000000"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices?query=%7B%22_id%22%3A%22202BC1-BM632w-0000000%22%7D&projection=InternetGatewayDevice.DeviceInfo.ModelName,InternetGatewayDevice.DeviceInfo.Manufacturer'
+
+The ``projection`` URL param is a comma-separated list of the parameters to receive.
+
+POST /devices/\<device_id\>/tasks?[connection_request]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Enqueue task(s) and optionally trigger a connection request to the device.
+Refer to :ref:`tasks` section for information about the task object format.
+Returns status code 200 if the tasks have been successfully executed, and 202
+if the tasks have been queued to be executed at the next inform.
+
+*device_id*: The ID of the device.
+
+*connection_request*: Indicates that a connection request will be triggered to
+execute the tasks immediatly. Otherwise, the tasks will be queued and be
+processed at the next inform.
+
+The response body is the task object as it is inserted in the database. The
+object will include ``_id`` property which you can use to look up the task
+later.
+
+Examples
+^^^^^^^^
+
+- Refresh all device parameters now:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tasks?connection_request' \
+  -X POST \
+  --data '{"name": "refreshObject", "objectName": ""}'
+
+- Change WiFi SSID and password:
+
+.. code:: javascript
+
+  {
+    "name": "setParameterValues",
+    "parameterValues": [
+      ["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],
+      ["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]
+    ]
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tasks?connection_request' \
+  -X POST \
+  --data '{"name":"setParameterValues", "parameterValues": [["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]]}'
+
+POST /tasks/\<task_id\>/retry
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Retry a faulty task at the next inform.
+
+*task_id*: The ID of the task as returned by 'GET /tasks' request.
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/tasks/5403908ef28ea3a25c138adc/retry' -X POST
+
+DELETE /tasks/\<task_id\>
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete the given task.
+
+*task_id*: The ID of the task as returned by 'GET /tasks' request.
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/tasks/5403908ef28ea3a25c138adc' -X DELETE
+
+DELETE /faults/\<fault_id\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete the given fault.
+
+*fault_id*: The ID of the fault as returned by 'GET /faults' request. The ID
+format is "\<device_id\>:\<channel\>".
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/faults/202BC1-BM632w-0000000:default' -X DELETE
+
+DELETE /devices/\<device_id\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete the given device from the database.
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -X DELETE -i 'http://localhost:7557/devices/202BC1-BM632w-000001'
+
+.. note::
+
+  Note that the device will be registered again when/if it contacts the ACS
+  again (e.g. on the next periodic inform).
+
+POST /devices/\<device_id\>/tags/\<tag\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Assign a tag to a device. Has no effect if such tag already exists.
+
+*device_id*: The ID of the device.
+
+*tag*: The tag to be assigned.
+
+Example
+^^^^^^^
+
+Assign the tag "testing" to a device:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tags/testing' -X POST
+
+DELETE /devices/\<device_id\>/tags/\<tag\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Remove a tag from a device.
+
+*device_id*: The ID of the device.
+
+*tag*: The tag to be removed.
+
+Example
+^^^^^^^
+
+Remove the tag "testing" from a device:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tags/testing' -X DELETE
+
+PUT /presets/\<preset_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Create or update a preset. Returns status code 200 if the preset has been
+added/updated successfully. The body of the request is a JSON representation of
+the preset. Refer to :ref:`presets` section below for details about its format.
+
+*preset_name*: The name of the preset.
+
+Example
+^^^^^^^
+
+Create a preset to set 5 minutes inform interval for all devices tagged with
+"test":
+
+.. code:: javascript
+
+  query = {
+    "weight": 0,
+    "precondition": "{\"_tags\": \"test\"}"
+    "configurations": [
+      {
+        "type": "value",
+        "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable",
+        "value": "true"
+      },
+      {
+        "type": "value",
+        "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval",
+        "value": "300"
+      }
+    ]
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/presets/inform' \
+  -X PUT \
+  --data '{"weight": 0, "precondition": "{\"_tags\": \"test\"}", "configurations": [{"type": "value", "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable", "value": "true"}, {"type": "value", "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval", "value": "300"}]}'
+
+DELETE /presets/\<preset_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+	curl -i 'http://localhost:7557/presets/inform' -X DELETE
+
+PUT /files/\<file_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Upload a new file or overwrite an existing one. Returns status code 200 if the
+file has been added/updated successfully. The file content should be sent as
+the request body.
+
+*file_name*: The name of the uploaded file.
+
+The following file metadata may be sent as request headers:
+
+- ``fileType``: For firmware images it should be "1 Firmware Upgrade Image".
+  Other common types are "2 Web Content" and "3 Vendor Configuration File".
+
+- ``oui``: The OUI of the device model that this file belogs to.
+
+- ``productClass``: The product class of the device.
+
+- ``version``: In case of firmware images, this refer to the firmware version.
+
+Example
+^^^^^^^
+
+Upload a firmware image file:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/files/new_firmware_v1.0.bin' \
+  -X PUT \
+  --data-binary @"./new_firmware_v1.0.bin" \
+  --header "fileType: 1 Firmware Upgrade Image" \
+  --header "oui: 123456" \
+  --header "productClass: ABC" \
+  --header "version: 1.0"
+
+DELETE /files/\<file_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete a previously uploaded file:
+
+.. code:: bash
+
+	curl -i 'http://localhost:7557/files/new_firmware_v1.0.bin' -X DELETE
+
+GET /files/
+~~~~~~~~~~~
+
+Gets all previously uploaded files.
+
+GET /files/?query={"filename":"\<filename\>"}
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Find files using a query.
+
+.. _tasks:
+
+Tasks
+-----
+
+Find the different availabe tasks and their object structure.
+
+``getParameterValues``
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  query = {
+    "name": "getParameterValues",
+    "parameterNames": [
+      "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnectionNumberOfEntries",
+      "InternetGatewayDevice.Time.NTPServer1", "InternetGatewayDevice.Time.Status"
+    ]
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-96318REF-SR360NA0A4%252D0003196/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "getParameterValues", "parameterNames": ["InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnectionNumberOfEntries", "InternetGatewayDevice.Time.NTPServer1", "InternetGatewayDevice.Time.Status"] }'
+
+You may request a single or multiple parameters at once.
+
+After the task has been executed successfully you can then fetch the CPE object
+and read the parameters from the JSON object.
+
+.. code:: javascript
+
+  query = {"_id": "00236a-96318REF-SR360NA0A4%2D0003196"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%2200236a-96318REF-SR360NA0A4%252D0003196%22%7D'
+
+``refreshObject``
+~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "refreshObject", "objectName": "InternetGatewayDevice.WANDevice.1.WANConnectionDevice"}'
+
+``setParameterValues``
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "setParameterValues", "parameterValues": [["InternetGatewayDevice.ManagementServer.UpgradesManaged",false]]}'
+
+Multiple values can be set at once by adding multiple arrays to the
+parameterValues key. For example:
+
+.. code:: javascript
+
+  {
+    name: "setParameterValues",
+    parameterValues: [["InternetGatewayDevice.ManagementServer.UpgradesManaged", false], ["InternetGatewayDevice.Time.Enable", true], ["InternetGatewayDevice.Time.NTPServer1", "pool.ntp.org"]]
+  }
+
+``addObject``
+~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name":"addObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection"}'
+
+``reboot``
+~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "reboot"}'
+
+``factoryReset``
+~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "factoryReset"}'
+
+``download``
+~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "download", "file": "mipsbe-6-42-lite.xml"}'
+
+.. _presets:
+
+Presets
+-------
+
+Presets assign a set of configuration or a Provision script to devices based on
+a precondition (search filter), schedule (cron expression), and events.
+
+Precondition
+~~~~~~~~~~~~
+
+The ``precondition`` property is a JSON string representation of the search
+filter to test if the preset applies to a given device. Examples preconditions
+are:
+
+- ``{"param": "value"}``
+- ``{"param": value", "param2": {"$ne": "value2"}}``
+
+Other operators that can be used are ``$gt``, ``$lt``, ``$gte`` and ``$lte``.
+
+Configuration
+~~~~~~~~~~~~~
+
+The configuration property is an array containing the different configurations
+to be applied to a device, as shown below:
+
+.. code:: javascript
+
+  [
+    {
+      "type": "value",
+      "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable",
+      "value": "true"
+    },
+    {
+      "type": "value",
+      "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval",
+      "value": "300"
+    },
+    {
+      "type": "delete_object",
+      "name": "object_parent",
+      "object": "object_name"
+    },
+    {
+      "type": "add_object",
+      "name": "object_parent",
+      "object": "object_name"
+    },
+    {
+      "type": "provision",
+      "name": "YourProvisionName"
+    },
+  ] 
+
+The configuration type ``provision`` triggers a Provision script. In the
+example above, the provision named "YourProvisionName" will be executed.
+
+Provisions
+----------
+
+Create a provision
+~~~~~~~~~~~~~~~~~~
+
+The Provision's JavaScript code is the body of the HTTP PUT request.
+
+.. code:: bash
+
+  curl -X PUT -i 'http://localhost:7557/provisions/mynewprovision' --data 'log("Provision started at " + now);'
+
+Delete a provision
+~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -X DELETE -i 'http://localhost:7557/provisions/mynewprovision'
+
+Get provisions
+~~~~~~~~~~~~~~
+
+Get all provisions:
+
+.. code:: bash
+
+  curl -X GET -i 'http://localhost:7557/provisions/'

docs/conf.py

@@ -0,0 +1,65 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+import json
+
+# -- Project information -----------------------------------------------------
+
+project = 'GenieACS Documentation'
+copyright = '2019, GenieACS Inc.'
+author = 'GenieACS Inc.'
+
+# The full version, including alpha/beta/rc tags
+release = json.load(open("../package.json"))["version"]
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+# html_theme = 'alabaster'
+html_theme = 'sphinx_rtd_theme'
+html_logo = "logo.svg"
+html_theme_options = {
+  "logo_only": True
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+master_doc = 'index'
+
+highlight_language = 'javascript'

docs/cpe-authentication.rst

@@ -0,0 +1,6 @@
+.. _cpe-authentication:
+
+CPE Authentication
+==================
+
+TODO

docs/environment-variables.rst

@@ -0,0 +1,6 @@
+.. _environment-variables:
+
+Environment Variables
+=====================
+
+TODO

docs/ext-sample.js

@@ -0,0 +1,34 @@
+// This is an exmaple GenieACS extension to get the current latitude/longitude
+// of the International Space Station. Why, you ask? Because why not.
+// To install, copy this file to config/ext/iss.js.
+
+"use strict";
+
+const http = require("http");
+
+let cache = null;
+let cacheExpire = 0;
+
+function latlong(args, callback) {
+  if (Date.now() < cacheExpire)
+    return callback(null, cache);
+
+  http.get("http://api.open-notify.org/iss-now.json", (res) => {
+    if (res.statusCode !== 200)
+      return callback(new Error(`Request failed (status code: ${res.statusCode})`));
+
+    let rawData = "";
+    res.on("data", (chunk) => rawData += chunk);
+
+    res.on("end", () => {
+      let pos = JSON.parse(rawData)["iss_position"];
+      cache = [+pos["latitude"], +pos["longitude"]];
+      cacheExpire = Date.now() + 10000;
+      callback(null, cache);
+    });
+  }).on("error", (err) => {
+    callback(err);
+  });
+}
+
+exports.latlong = latlong;

docs/extensions.rst

@@ -0,0 +1,33 @@
+.. _extensions:
+
+Extensions
+==========
+
+Given that :ref:`provisions` and :ref:`virtual-parameters` are executed in a
+sandbox environment, it is not possible to interact with external sources or
+execute any action that requires OS, file system, or network access. Extensions
+exist to bridge that gap.
+
+Extensions are fully-privileged Node.js modules and as such have access to
+standard Node libraries and 3rd party packages. Functions exposed by the
+extension can be called from Provision scripts using the ``ext()`` function. A
+typical use case for extensions is fetching credentials from a database to have
+that pushed to the device during provisioning.
+
+By default, the extension JS code must be placed under ``config/ext``
+directory. You may need to create that directory if it doesn't already exist.
+
+The example extension below fetches data from an external REST API and returns
+that to the caller:
+
+.. literalinclude:: ext-sample.js
+  :language: javascript
+
+To call this extension from a Provision or a Virtual Parameter script:
+
+.. code:: javascript
+
+  // The arguments "arg1" and "arg2" are passed to the latlong. Though they are
+  // unused in this particular example.
+  const res = ext("ext-sample", "latlong", "arg1", "arg2");
+  log(JSON.stringify(res));

docs/https.rst

@@ -0,0 +1,6 @@
+.. _https:
+
+HTTPS
+=====
+
+TODO

docs/index.rst

@@ -0,0 +1,55 @@
+.. GenieACS documentation master file, created by
+   sphinx-quickstart on Wed Jun  5 13:47:06 2019.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to GenieACS's documentation!
+====================================
+
+.. toctree::
+  :caption: Table of Contents
+
+.. raw:: latex
+
+   \part{Installation}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Installation
+
+  installation-guide
+  environment-variables
+
+.. raw:: latex
+
+   \part{Administration}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Administration
+
+  provisions
+  virtual-parameters
+
+.. raw:: latex
+
+   \part{Integration}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Integration
+
+  extensions
+  api-reference
+
+.. raw:: latex
+
+   \part{Security}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Security
+
+  https
+  cpe-authentication
+  roles-and-permissions

docs/installation-guide.rst

@@ -0,0 +1,230 @@
+Installation Guide
+==================
+
+This guide is for installing GenieACS on a single server on any Linux distro
+that uses *systemd* as its init system.
+
+The various GenieACS services are independant of each other and may be
+installed on different servers. You may also run multiple instances of each in
+a load-balancing/failover setup.
+
+.. attention::
+
+  For production deployments make sure to configure TLS and change
+  ``UI_JWT_SECRET`` to a unique and secure string. Refer to :ref:`https`
+  section for how to enable TLS to encrypt traffic.
+
+Prerequisites
+-------------
+
+.. topic:: Node.js
+
+  GenieACS requires Node.js 10.x and up. Refer to https://nodejs.org/ for
+  instructions.
+
+.. topic:: MongoDB
+
+  GenieACS requires MongoDB 3.6 and up. Refer to https://www.mongodb.com/ for
+  instructions.
+
+Install GenieACS
+-------------------
+
+.. topic:: Installing from NPM:
+
+  .. parsed-literal::
+
+    sudo npm install -g genieacs@\ |release|
+
+.. topic:: Installing from source
+
+  If you prefer installing from source, such as when running a GenieACS copy
+  with custom patches, refer to README.md file in the source package. Adjust
+  the next steps below accordingly.
+
+Configure systemd
+-----------------
+
+.. topic:: Create a system user to run GenieACS daemons
+
+  .. code:: bash
+
+    sudo useradd --system --no-create-home --user-group genieacs
+
+.. topic:: Create directory to save extensions and environment file
+
+  We'll use :file:`/opt/genieacs/ext/` directory to store extension scripts (if any).
+
+  .. code:: bash
+    
+    mkdir /opt/genieacs
+    mkdir /opt/genieacs/ext
+    chown genieacs:genieacs /opt/genieacs/ext
+
+  Create the file :file:`/opt/genieacs/genieacs.env` to hold our configuration
+  options which we pass to GenieACS as environment variables. See
+  :ref:`environment-variables` section for a list of all available
+  configuration options.
+
+  .. code:: bash
+
+    GENIEACS_CWMP_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-cmwp-access.log
+    GENIEACS_NBI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-nbi-access.log
+    GENIEACS_FS_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-fs-access.log
+    GENIEACS_UI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-ui-access.log
+    GENIEACS_DEBUG_FILE=/var/log/genieacs/genieacs-debug.yaml
+    GENIEACS_UI_JWT_SECRET=secret
+  
+  Set file ownership and permissions:
+
+  .. code:: bash
+
+    sudo chown genieacs:genieacs /opt/genieacs/genieacs.env
+    sudo chmod 600 /opt/genieacs/genieacs.env
+
+.. topic:: Create logs directory  
+
+  .. code:: bash
+    
+    mkdir /var/log/genieacs
+    chown genieacs:genieacs /var/log/genieacs
+
+.. topic:: Create systemd unit files
+
+  Create a systemd unit file for each of the four GenieACS services. Note that
+  we're using EnvironmentFile directive to read the environment variables from
+  the file we created earlier.
+
+  Each service has two streams of logs: access log and process log. Access logs
+  are configured here to be dumped in a log file under
+  :file:`/var/log/genieacs/` while process logs go to *journald*. Use
+  ``journalctl`` command to view process logs.
+
+  1. Run the following command to create ``genieacs-cwmp`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force genieacs-cwmp
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS CWMP
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=genieacs-cwmp
+      KillMode=process
+
+      [Install]
+      WantedBy=default.target
+
+  2. Run the following command to create ``genieacs-nbi`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force genieacs-nbi
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS NBI
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=genieacs-nbi
+      KillMode=process
+
+      [Install]
+      WantedBy=default.target
+
+  3. Run the following command to create ``genieacs-fs`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force genieacs-fs
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS FS
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=genieacs-fs
+      KillMode=process
+
+      [Install]
+      WantedBy=default.target
+
+  4. Run the following command to create ``genieacs-ui`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force genieacs-ui
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS UI
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=genieacs-ui
+      KillMode=process
+
+      [Install]
+      WantedBy=default.target
+
+.. topic:: Configure log file rotation using logrotate
+
+  Save the following under :file:`/etc/logrotate.d/genieacs`
+
+  .. code::
+  
+    /var/log/genieacs/*.{log,yaml} {
+        daily
+        rotate 30
+        compress
+        delaycompress
+        dateext
+    }
+
+.. topic:: Enable and start services
+
+  .. code:: bash
+
+    sudo systemctl enable genieacs-cwmp
+    sudo systemctl start genieacs-cwmp
+    sudo systemctl status genieacs-cwmp
+
+    sudo systemctl enable genieacs-nbi
+    sudo systemctl start genieacs-nbi
+    sudo systemctl status genieacs-nbi
+
+    sudo systemctl enable genieacs-fs
+    sudo systemctl start genieacs-fs
+    sudo systemctl status genieacs-fs
+
+    sudo systemctl enable genieacs-ui
+    sudo systemctl start genieacs-ui
+    sudo systemctl status genieacs-ui
+
+  Review the status message for each to verify that the services are running
+  successfully.

docs/logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="310" height="64" viewBox="0 0 82.021 16.933"><path d="M7.341 9.892l.687 1.307c-.305.53-.718.99-1.238 1.378-.851.503-1.543.421-2.076-.244 1.03-.466 1.907-1.28 2.627-2.44M4.676 5.97a2.144 2.144 0 0 1 .237 1.228 8.358 8.358 0 0 1-.518 2.21 7.386 7.386 0 0 0-.424 1.708 2.38 2.38 0 0 0 .168 1.205c.163.393.457.73.883 1.01-.458.138-.859.175-1.202.112-.6-.114-.992-.476-1.175-1.084a3.571 3.571 0 0 1-.093-1.787 11.742 11.742 0 0 1 .435-1.644 11.606 11.606 0 0 0 .432-1.708c.037-.22.041-.45.01-.687a1.66 1.66 0 0 0-.273-.706 1.582 1.582 0 0 0-.6-.506c.245-.046.487-.083.728-.113a1.45 1.45 0 0 1 .728.123c.295.13.517.343.665.639m7.901-1.802l.222.218-1.847 1.678.555.555 1.678-1.846.218.22-1.843 1.679.521.521L13.76 5.35l.222.218-1.847 1.678.556.556 1.764-1.765.375.368-1.584 1.584h.004l-2.065 2.061-1.253.515-1.795-1.348-.007-.008-.008.008.004.007 1.194 1.592-.398.161-1.75-1.75.444-1.08 1.636 1.228.004.003.011-.007-.003-.004-1.39-1.85.225-.553 1.694-1.689h-.004l1.956-1.952.372.368-1.769 1.768.552.552 1.678-1.843m1.456 9.91c1.55-1.549 2.324-3.42 2.324-5.612 0-2.187-.774-4.058-2.323-5.612C12.48 1.304 10.61.53 8.423.53 6.23.53 4.36 1.305 2.81 2.854c-.95.953-1.609 2.025-1.974 3.213l.158-.086c.788-.415 1.486-.203 2.094.638-.55.113-.992.232-1.325.357a1.923 1.923 0 0 0-.503.289c-.337.268-.55.608-.638 1.02a3.98 3.98 0 0 0 .011 1.761c.285 1.527 1.011 2.87 2.178 4.032 1.549 1.552 3.42 2.328 5.612 2.328 2.19 0 4.06-.776 5.612-2.328" fill="white"/><path d="M47.104 1.767c-.456 0-.816.388-.816.849 0 .46.355.816.816.816a.82.82 0 0 0 .833-.816.85.85 0 0 0-.833-.849zM25.602 3.171c-.205 0-.382.046-.5.13-.118.085-.187.201-.187.4v.981l-.151-.031a9.89 9.89 0 0 0-1.872-.173c-1.288 0-2.148.066-2.657.374a1.264 1.264 0 0 0-.548.676c-.119.313-.177.73-.177 1.28v6.123c0 .982.25 1.508.787 1.824.539.315 1.407.41 2.628.41 1.308 0 2.178-.13 2.701-.484.524-.354.745-.932.745-1.942 0-.821-.147-1.37-.543-1.722-.395-.352-1.075-.528-2.186-.528h-2.787V9.255l.148.03c.478.092 1.094.11 1.89.11 1.292 0 2.148-.112 2.658-.453.51-.34.724-.915.724-1.958V3.7c0-.198-.07-.314-.186-.399a.843.843 0 0 0-.487-.13zm6.112 1.29c-1.226 0-2.038.137-2.526.495-.488.357-.697.945-.697 1.947v2.84c0 .935.23 1.54.743 1.932.512.39 1.337.571 2.544.571 1.392 0 2.17-.058 2.567-.196.199-.069.298-.15.355-.246.058-.097.078-.227.078-.405 0-.208-.04-.378-.117-.485-.076-.108-.186-.171-.396-.171-.794 0-1.057.19-2.408.19-.593 0-1.06-.078-1.395-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.717.717 0 0 0 .539-.213c.13-.138.213-.342.213-.62V6.903c0-1.037-.179-1.623-.654-1.97-.476-.348-1.3-.472-2.617-.472zm8.83 0c-1.303 0-2.117.086-2.59.49-.472.403-.665 1.168-.665 2.623v4.002c0 .215.074.338.197.426.123.088.31.134.524.134.223 0 .412-.046.537-.134.125-.088.198-.211.198-.426v-3.81c0-.424.006-.754.04-1.019.036-.265.102-.47.236-.623.135-.152.328-.236.574-.28.246-.044.552-.053.949-.053.4 0 .712.01.963.053.25.044.447.126.587.278.139.151.211.358.25.623.038.265.046.597.046 1.02v3.81c0 .217.07.34.19.427.122.088.308.134.53.134.215 0 .401-.046.525-.134.123-.088.196-.211.196-.426V7.574c0-1.455-.201-2.219-.682-2.623-.48-.404-1.303-.49-2.605-.49zm12.996 0c-1.227 0-2.038.137-2.527.495-.488.357-.696.945-.696 1.947v2.84c0 .935.23 1.54.742 1.932.513.39 1.338.571 2.545.571 1.392 0 2.17-.058 2.567-.196.2-.069.298-.15.355-.246.058-.097.079-.227.079-.405 0-.208-.04-.378-.117-.485-.077-.108-.188-.171-.398-.171-.793 0-1.056.19-2.406.19-.594 0-1.062-.078-1.396-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.716.716 0 0 0 .539-.213c.13-.138.213-.342.213-.62V6.903c0-1.037-.18-1.623-.655-1.97-.475-.348-1.3-.472-2.616-.472zm-6.436.13c-.212 0-.398.047-.522.14-.124.09-.198.22-.198.435v6.41c0 .215.073.338.196.426.123.088.31.134.524.134.223 0 .414-.046.54-.134.124-.088.198-.211.198-.426v-6.41c0-.215-.075-.342-.201-.434-.126-.092-.316-.142-.537-.142zM22.892 5.726c.662 0 1.4.065 1.918.146l.105.016v.92c0 .246-.01.453-.058.632a.757.757 0 0 1-.292.441c-.305.216-.798.263-1.673.263h-.123v-.013c-.761 0-1.26-.002-1.55-.16a.637.637 0 0 1-.307-.395c-.049-.17-.057-.37-.057-.625v-.062c0-.227.03-.418.108-.58a.811.811 0 0 1 .373-.373c.337-.171.826-.21 1.556-.21zm8.839.049c.677 0 1.116.054 1.412.237a.775.775 0 0 1 .316.373c.063.154.085.33.085.535v.811h-3.611V6.92c0-.437.135-.755.439-.928.304-.174.739-.217 1.359-.217zm21.825 0c.677 0 1.117.054 1.413.237a.779.779 0 0 1 .318.373c.062.154.084.33.084.535v.811h-3.613V6.92c0-.437.136-.755.44-.928.304-.174.738-.217 1.358-.217zm-32.701 5.867h2.388c.613 0 1.05.045 1.355.226.306.18.443.505.443.95 0 .429-.179.743-.533.907-.354.164-.861.206-1.568.206-.652 0-1.151-.045-1.51-.186a.952.952 0 0 1-.425-.3.789.789 0 0 1-.15-.483v-1.195z" fill="white"/><path d="M62.339 4.462c-1.399 0-2.18.078-2.58.225-.2.073-.3.157-.357.254-.058.097-.079.22-.079.386 0 .199.043.362.122.469.079.107.19.172.392.172.38 0 .616-.048.947-.096a9.457 9.457 0 0 1 1.475-.098c.58 0 .966.043 1.233.192a.685.685 0 0 1 .292.312c.06.13.082.281.082.452v.857h-2.484c-1.134 0-1.763.19-2.11.559-.346.367-.443.95-.443 1.802 0Line truncated

docs/provisions.rst

@@ -0,0 +1,271 @@
+.. _provisions:
+
+Provisions
+==========
+
+A Provision is a piece of JavaScript code that is executed on the server on a
+per-device basis. It enables implementing complex provisioning scenarios and
+other operations such as automated firmware upgrade rollout. Apart from a few
+special functions, the script is essentially a standard ES6 code executed in
+strict mode.
+
+Provisions are mapped to devices using presets. Note that the added performance
+overhead when using Provisions as opposed to simple preset configuration
+entries is relatively small. Anything that can be done via preset
+configurations can be done using a Provision script. In fact, the now
+deprecated configuration format is still supported primarily for backward
+compatibility and it is recommended to use Provision scripts for all
+configuration.
+
+When assigning a Provision script to a preset, you may pass arguments to the
+script. The arguments can be accessed from the script through the global
+``args`` variable.
+
+.. note::
+
+  Provision scripts may get executed multiple times in a given session.
+  Although all data model-mutating operations are idempotent, a script as a
+  whole may not be. It is, therefore, necessary to repeatedly run the script
+  until there are no more side effects and a stable state is reached.
+
+Built-in functions
+------------------
+
+``declare(path, timestamps, values)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This function is for declaring parameter values to be set, as well as specify
+constraints on how recent you'd like the parameter value (or other attributes)
+to have been refreshed from the device. If the given timestamp is lower than
+the timestamp of the last refresh from the device, then this function will
+return the last known value. Otherwise, the value will be fetched from the
+device before being returned to the caller.
+
+The timestamp argument is an object where the key is the attribute name (e.g.
+``value``, ``object``, ``writable``, ``path``) and the value is an integer
+representing a Unix timestamp.
+
+The values argument is an object similar to the timestamp argument but its
+property values being the parameter values to be set.
+
+The possible attributes in 'timestamps' and 'values' arguments are:
+
+- ``value``: a [<value>, <type>] pair
+
+This attribute is not available for objects or object instances. If the value
+is not a [<value>, <type>] array then it'll assumed to be a value without a
+type and therefore the type will be inferred from the parameter's type.
+
+- ``writable``: boolean
+
+The meaning of this attribute can vary depending on the type of the parameter.
+In the case of regular parameters, it indicates if its value is writable. In
+the case of objects, it's whether or not it's possible to add new object
+instances. In the case of object instances, it indicates whether or not this
+instance can be deleted.
+
+- ``object``: boolean
+
+True if this is an object or object instance, false otherwise.
+
+- ``path``: string
+
+This attribute is special in that it's not a parameter attribute per se, but it
+refers to the presence of parameters matching the given path. For example,
+given the following wildcard path:
+
+``InternetGatewayDevice.LANDevice.1.Hosts.Host.*.MACAddress``
+
+Using a recent timestamp for path in ``declare()`` will result in a sync with
+the device to rediscover all Host instances (``Host.*``). The path attribute
+can also be used to create or delete object instances as described in
+:ref:`path-format` section.
+
+The return value of ``declare()`` is an iterator to access parameters that
+match the given path. Each item in the iterator has the attribute 'path' in
+addition to any other attribute given in the ``declare()`` call. The iterator
+object itself has convenience attribute accessors which come in handy when
+you're expecting a single parameter (e.g. when path does not contain wildcards
+or aliases).
+
+.. code:: javascript
+
+  // Example: Setting the SSID as the last 6 characters of the serial number
+  let serial = declare("Device.DeviceInfo.SerialNumber", {value: 1});
+  declare("Device.LANDevice.1.WLANConfiguration.1.SSID", null, {value: serial.value[0]});
+
+``clear(path, timestamp)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This function invalidates the database copy of parameters (and their child
+parameters) that match the given path and have a last refresh timestamp that is
+less than the given timestamp. The most obvious use for this function is to
+invalidate the database copy of the entire data model after the device has been
+factory reset:
+
+.. code:: javascript
+
+  // Example: Clear cached device data model Note
+  // Make sure to apply only on "0 BOOTSTRAP" event
+  clear("Device", Date.now());
+  clear("InternetGatewayDevice", Date.now());
+
+``commit()``
+~~~~~~~~~~~~
+
+This function commits the pending declarations and performs any necessary sync
+with the device. It's usually not required to call this function as it called
+implicitly at the end of the script and when accessing any property of the
+promise-like object returned by the ``declare()`` function. Calling this
+explicitly is only necessary if you want to control the order in which
+parameters are configured.
+
+``ext(file, function, arg1, arg2, ...)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Execute an extension script and return the result. The first argument is the
+script filename while second argument is the function name within that script.
+Any remaining arguments will be passed to that function. See :ref:`extensions`
+for more details.
+
+.. _path-format:
+
+Path format
+-----------
+
+A parameter path may contain a wildcard (``*``) or an alias filter
+(``[name:value]``). A wildcard segment in a parameter path will apply the
+declared configuration to zero or more parameters that match the given path
+where the wildcard segment can be anything.
+
+An alias filter is like a wildcard, but additionally performs filtering on the
+child parameters based on the key-value pairs provided. For example, the
+following path:
+
+``Device.WANDevice.1.WANConnectionDevice.1.WANIPConnection.[AddressingType:DHCP].ExternalIPAddress``
+
+will return a list of ExternalIPAddress parameters (0 or more) where the
+sibling parameter AddressingType is assigned the value "DHCP".
+
+This can be useful when the exact instance numbers may be different from one
+device to another. It is possible to use more than one key-value pair in the
+alias filter. It's also possible to use multiple filters or use a combination
+of filters and wildcards.
+
+Creating/deleting object instances
+----------------------------------
+
+Given the declarative nature of provisions, we cannot explicitly tell the
+device to create or delete an instance under a given object. Instead, we
+specify the number of instances we want there to be, and based on that GenieACS
+will determine whether or not it needs to create or delete instances. For
+example, the following declaration will ensure we have one and only one
+WANIPConnection object:
+
+.. code:: javascript
+
+  // Example: Ensure we have one and only one WANIPConnection object
+  declare("InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.*", null, {path: 1});
+
+Note the wildcard at the end of the parameter path.
+
+It is also possible to use alias filters as the last path segment which will
+ensure that the declared number of instances is satisfied given the alias
+filter:
+
+.. code:: javascript
+
+  // Ensure that *all* other instances are deleted
+  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[]", null, {path: 0});
+
+  // Add the two entries we care about
+  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[SourceIPAddress:192.168.1.0,SourceNetMask:255.255.255.0]",  {path: now}, {path: 1});
+  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[SourceIPAddress:172.16.12.0,SourceNetMask:255.255.0.0]", {path: now}, {path: 1});
+
+Special GenieACS parameters
+---------------------------
+
+In addition to the parameters exposed in the device's data model through
+TR-069, GenieACS has its own set of special parameters:
+
+``DeviceID``
+~~~~~~~~~~~~
+
+This parameter sub-tree includes the following read-only parameters:
+
+- ``DeviceID.ID``
+- ``DeviceID.SerialNumber``
+- ``DeviceID.ProductClass``
+- ``DeviceID.OUI``
+- ``DeviceID.Manufacturer``
+
+``Tags``
+~~~~~~~~
+
+The ``Tags`` root parameter is used to expose device tags in the data model.
+Tags appear as child parameters that are writable and have boolean value.
+Setting a tag to ``false`` will delete that tag, and setting the value of a
+non-existing tag parameter to ``true`` will create it.
+
+.. code:: javascript
+
+  // Example: Remove "tag1", add "tag2", and read "tag3"
+  declare("Tags.tag1", null, {value: false});
+  declare("Tags.tag2", null, {value: true});
+  let tag3 = declare("Tags.tag3", {value: 1});
+
+``Reboot``
+~~~~~~~~~~
+
+The ``Reboot`` root parameter hold the timestamp of the last reboot command.
+The parameter value is writable and declaring a timestamp value that is larger
+than the current value will trigger a reboot.
+
+.. code:: javascript
+
+  // Example: Reboot the device only if it hasn't been rebooted in the past 300 seconds
+  declare("Reboot", null, {value: Date.now() - (300 * 1000)});
+
+``FactoryReset``
+~~~~~~~~~~~~~~~~
+
+Works like ``Reboot`` parameter but for factory reset.
+
+.. code:: javascript
+
+  // Example: Default the device to factory settings
+  declare("FactoryReset", null, {value: Date.now()});
+
+``Downloads``
+~~~~~~~~~~~~~
+
+The ``Downloads`` sub-tree holds information about the last download
+command(s). A download command is represented as an instance (e.g.
+``Downloads.1``) containing parameters such as ``Download`` (timestamp),
+``LastFileType``, ``LastFileName``. The parameters ``FileType``, ``FileName``,
+``TargetFileName`` and ``Download`` are writable and can be used to trigger a
+new download.
+
+.. code:: javascript
+
+  declare("Downloads.[FileType:1 Firmware Upgrade Image]", {path: 1}, {path: 1});
+  declare("Downloads.[FileType:1 Firmware Upgrade Image].FileName", {value: 1}, {value: "firmware-2017.01.tar"});
+  declare("Downloads.[FileType:1 Firmware Upgrade Image].Download", {value: 1}, {value: Date.now()});
+
+Common file types are:
+
+- ``1 Firmware Upgrade Image``
+- ``2 Web Content``
+- ``3 Vendor Configuration File``
+- ``4 Tone File``
+- ``5 Ringer File``
+
+.. warning::
+
+  Pushing a file to the device is often a service-interrupting operation. It's
+  recommended to only trigger it on certain events such as ``1 BOOT`` or during
+  a predetermined maintenance window).
+
+After the CPE had finished downloading and applying the config file, it will
+send a ``7 TRANSFER COMPLETE`` event. You may use that to trigger a reboot
+after the firmware image or configuration file had been applied.

docs/roles-and-permissions.rst

@@ -0,0 +1,6 @@
+.. _roles-and-permissions:
+
+Roles and Permissions
+=====================
+
+TODO

docs/virtual-parameters.rst

@@ -0,0 +1,105 @@
+.. _virtual-parameters:
+
+Virtual Parameters
+==================
+
+Virtual parameters are user-defined parameters whose values are generated using
+a custom Javascript code. Virtual parameters behave just like regular
+parameters and appear in the data model under ``VirtualParameters.`` path.
+Virtual parameter names cannot contain a period (``.``).
+
+The execution environment for virtual parameters is almost identical to that of
+provisions. See :ref:`provisions` for more details and examples. The only
+differences between the scripts of provisions and virtual parameters are:
+
+- You can't pass custom arguments to virtual parameter scripts. Instead, the
+  variable ``args`` will hold the current vparam timestamps and values as well
+  as the declared timestamps and values. Like this:
+
+.. code:: javascript
+
+  // [<current attr timestamps>, <current attr values>, <declared attr timestamps, declared attr values>]
+  [{path: 1559840000000, value: 1559840000000}, {value: ["cur val", "xsd:string"]}, {path: 1559849387191, value: 1559849387191}, {value: ["new val", "xsd:string"]}]
+
+- Virtual parameter scripts must return an object containing the attributes of
+  this parameter.
+
+.. note::
+
+  Just like a regular parameter, creating a virtual parameter does not
+  automatically add it to the parameter list for a device. It needs to fetched
+  (manually or via a preset) before you can see it in the data model.
+
+Examples
+--------
+
+Unified MAC parameter across different device models
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  // Example: Unified MAC parameter across different device models
+  let m = "00:00:00:00:00:00";
+  let d = declare("Device.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress", {value: Date.now()});
+  let igd = declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANPPPConnection.*.MACAddress", {value: Date.now()});
+
+  if (d.size) {
+    for (let p of d) {
+      if (p.value[0]) {
+        m = p.value[0];
+        break;
+      }
+    }  
+  }
+  else if (igd.size) {
+    for (let p of igd) {
+      if (p.value[0]) {
+        m = p.value[0];
+        break;
+      }
+    }  
+  }
+
+  return {writable: false, value: [m, "xsd:string"]};
+
+Expose an external value as a virtual parameter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  // Example: Expose an external value as a virtual parameter
+  let serial = declare("DeviceID.SerialNumber", {value: 1});
+  if (values.value) {
+    ext("example-ext", "set", serial.value[0], values.value[0]);
+    return {writable: true, value: [values.value[0], "xsd:string"]};
+  }
+  else {
+    let v = ext("example-ext", "get", serial.value[0]);
+    return {writable: true, value: [v, "xsd:string"]};
+  }
+
+Create an editable virtual parameter for WPA passphrase
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  // Example: Create an editable virtual parameter for WPA passphrase
+  let m = "";
+  if (VALUES && VALUES.value) {
+    m = VALUES.value[0];
+    declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", null, {value: m});
+    declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", null, {value: m});
+  }
+  else {
+    let d = declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", {value: Date.now()});
+    let igd = declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", {value: Date.now()});
+
+    if (d.size) {
+      m = d.value[0];
+    }
+    else if (igd.size) {
+      m = igd.value[0];  
+    }
+  }
+
+  return {writable: true, value: [m, "xsd:string"]};

lib/api-functions.coffee

@@ -1,250 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This file incorporates work covered by the following copyright and
-# permission notice:
-#
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-config = require './config'
-db = require './db'
-http = require 'http'
-common = require './common'
-util = require 'util'
-URL = require 'url'
-auth = require './auth'
-
-
-connectionRequest = (deviceId, callback) ->
-  # ensure socket is reused in case of digest authentication
-  agent = new http.Agent({maxSockets : 1})
-
-  statusToError = (statusCode) ->
-    switch statusCode
-      when 200, 204
-        null
-      when 401
-        new Error('Incorrect connection request credentials')
-      when 0
-        new Error('Device is offline')
-      else
-        new Error("Unexpected response code from device: #{statusCode}")
-
-  conReq = (url, authString, callback) ->
-    options = URL.parse(url)
-    options.agent = agent
-
-    if authString
-      options.headers = {'Authorization' : authString}
-
-    request = http.get(options, (res) ->
-      if res.statusCode == 401 and res.headers['www-authenticate']?
-        authHeader = auth.parseAuthHeader(res.headers['www-authenticate'])
-      callback(res.statusCode, authHeader)
-      # don't need body, go ahead and emit events to free up the socket for possible reuse
-      res.resume()
-    )
-
-    request.on('error', (err) ->
-      # error event when request is aborted
-      request.abort()
-      callback(0)
-    )
-
-    request.on('socket', (socket) ->
-      socket.setTimeout(2000)
-      socket.on('timeout', () ->
-        request.abort()
-      )
-    )
-
-  proj = {
-    'Device.ManagementServer.ConnectionRequestURL._value' : 1,
-    'InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value' : 1
-  }
-
-  db.devicesCollection.findOne({_id : deviceId}, proj, (err, device)->
-    if err
-      callback(err)
-      return
-
-    if device.Device? # TR-181 data model
-      connectionRequestUrl = device.Device.ManagementServer.ConnectionRequestURL._value
-    else # TR-098 data model
-      connectionRequestUrl = device.InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value
-
-    # for testing
-    #connectionRequestUrl = connectionRequestUrl.replace(/^(http:\/\/)([0-9\.]+)(\:[0-9]+\/[a-zA-Z0-9]+\/?$)/, '$110.1.1.254$3')
-    conReq(connectionRequestUrl, null, (statusCode, authHeader) ->
-      if statusCode == 401
-        [username, password] = config.auth.connectionRequest(deviceId)
-        if authHeader.method is 'Basic'
-          authString = auth.basic(username, password)
-        else if authHeader.method is 'Digest'
-          uri = URL.parse(connectionRequestUrl)
-          authString = auth.digest(username, password, uri.path, 'GET', null, authHeader)
-
-        conReq(connectionRequestUrl, authString, (statusCode, authHeader) ->
-          callback(statusToError(statusCode))
-        )
-      else
-        callback(statusToError(statusCode))
-    )
-  )
-
-
-watchTask = (taskId, timeout, callback) ->
-  setTimeout( () ->
-    db.tasksCollection.findOne({_id : taskId}, {'_id' : 1}, (err, task) ->
-      if task
-        timeout -= 500
-        if timeout < 0
-          callback('timeout')
-        else
-          watchTask(taskId, timeout, callback)
-      else
-        callback(err)
-    )
-  , 500)
-
-
-expandParam = (param, aliases) ->
-  params = [param]
-  for a,aa of aliases
-    if a == param or common.startsWith(a, "#{param}.")
-      for p in aa
-        params.push(p) if p[p.lastIndexOf('.') + 1] != '_'
-
-  return params
-
-
-sanitizeTask = (task, aliases, callback) ->
-  task.timestamp = new Date(task.timestamp ? Date.now())
-  if task.expiry?
-    if common.typeOf(task.expiry) is common.DATE_TYPE or isNaN(task.expiry)
-      task.expiry = new Date(task.expiry)
-    else
-      task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000)
-
-  switch task.name
-    when 'getParameterValues'
-      projection = {}
-      for p in task.parameterNames
-        for pp in expandParam(p, aliases)
-          projection[pp] = 1
-      db.devicesCollection.findOne({_id : task.device}, projection, (err, device) ->
-        parameterNames = []
-        for k of projection
-          if common.getParamValueFromPath(device, k)?
-            parameterNames.push(k)
-        task.parameterNames = parameterNames
-        callback(task)
-      )
-    when 'setParameterValues'
-      projection = {}
-      values = {}
-      for p in task.parameterValues
-        for pp in expandParam(p[0], aliases)
-          projection[pp] = 1
-          values[pp] = p[1]
-      db.devicesCollection.findOne({_id : task.device}, projection, (err, device) ->
-        parameterValues = []
-        for k of projection
-          param = common.getParamValueFromPath(device, k)
-          if param?
-            parameterValues.push([k, values[k], param._type])
-        task.parameterValues = parameterValues
-        callback(task)
-      )
-    else
-      # TODO implement setParameterValues
-      callback(task)
-
-
-addAliases = (device, aliases) ->
-  for k,v of aliases
-    for p in v
-      pp = p.split('.')
-      obj = device
-      for i in pp
-        if not obj[i]?
-          obj = null
-          break
-        obj = obj[i]
-
-      device[k] = obj if obj?
-
-
-insertTasks = (tasks, aliases, callback) ->
-  if tasks? and common.typeOf(tasks) isnt common.ARRAY_TYPE
-    tasks = [tasks]
-  else if not tasks? or tasks.length == 0
-    return callback(tasks)
-
-  counter = tasks.length
-
-  for task in tasks
-    sanitizeTask(task, aliases, (t) ->
-      if t.uniqueKey?
-        db.tasksCollection.remove({device : t.device, uniqueKey : t.uniqueKey}, (err, removed) ->
-        )
-
-      --counter
-      if counter == 0
-        db.tasksCollection.insert(tasks, (err, _tasks) ->
-          #util.log("#{_task.device}: Added task #{_task.name}(#{_task._id})") for _task in _tasks
-          callback(err, _tasks)
-        )
-    )
-
-
-deleteDevice = (deviceId, callback) ->
-  db.tasksCollection.remove({'device' : deviceId}, (err, removed) ->
-    return callback(err) if err
-    db.devicesCollection.remove({'_id' : deviceId}, (err, removed) ->
-      return callback(err) if err
-      db.redisClient.del("#{deviceId}_presets_hash", "#{deviceId}_inform_hash", (err) ->
-        callback(err)
-      )
-    )
-  )
-
-
-exports.addAliases = addAliases
-exports.sanitizeTask = sanitizeTask
-exports.connectionRequest = connectionRequest
-exports.watchTask = watchTask
-exports.insertTasks = insertTasks
-exports.deleteDevice = deleteDevice

lib/api-functions.ts

@@ -0,0 +1,279 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { parse } from "url";
+import * as db from "./db";
+import * as common from "./common";
+import * as cache from "./cache";
+import {
+  getCurrentSnapshot,
+  getConfig,
+  getConfigExpression
+} from "./local-cache";
+import {
+  httpConnectionRequest,
+  udpConnectionRequest
+} from "./connection-request";
+import { Expression, Task } from "./types";
+
+export async function connectionRequest(deviceId): Promise<void> {
+  const options = {
+    projection: {
+      _deviceId: 1,
+      "Device.ManagementServer.ConnectionRequestURL._value": 1,
+      "Device.ManagementServer.UDPConnectionRequestAddress._value": 1,
+      "Device.ManagementServer.ConnectionRequestUsername._value": 1,
+      "Device.ManagementServer.ConnectionRequestPassword._value": 1,
+      "InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value": 1,
+      "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress._value": 1,
+      "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername._value": 1,
+      "InternetGatewayDevice.ManagementServer.ConnectionRequestPassword._value": 1
+    }
+  };
+
+  const device = await db.devicesCollection.findOne({ _id: deviceId }, options);
+  if (!device) throw new Error("No such device");
+
+  let managementServer,
+    connectionRequestUrl,
+    udpConnectionRequestAddress,
+    username,
+    password;
+  if (device.Device)
+    // TR-181 data model
+    managementServer = device.Device.ManagementServer;
+  // TR-098 data model
+  else managementServer = device.InternetGatewayDevice.ManagementServer;
+
+  if (managementServer.ConnectionRequestURL)
+    connectionRequestUrl = managementServer.ConnectionRequestURL._value;
+  if (managementServer.UDPConnectionRequestAddress) {
+    udpConnectionRequestAddress =
+      managementServer.UDPConnectionRequestAddress._value;
+  }
+  if (managementServer.ConnectionRequestUsername)
+    username = managementServer.ConnectionRequestUsername._value;
+  if (managementServer.ConnectionRequestPassword)
+    password = managementServer.ConnectionRequestPassword._value;
+
+  const context = {
+    id: device["_id"],
+    serialNumber: device["_deviceId"]["SerialNumber"],
+    productClass: device["_deviceId"]["ProductClass"],
+    oui: device["_deviceId"]["OUI"],
+    remoteAddress: connectionRequestUrl
+      ? parse(connectionRequestUrl).host
+      : null,
+    username: username || "",
+    password: password || ""
+  };
+
+  const snapshot = await getCurrentSnapshot();
+  const now = Date.now();
+  const UDP_CONNECTION_REQUEST_PORT = +getConfig(
+    snapshot,
+    "cwmp.udpConnectionRequestAuth",
+    context,
+    now
+  );
+  const CONNECTION_REQUEST_TIMEOUT = +getConfig(
+    snapshot,
+    "cwmp.connectionRequestTimeout",
+    context,
+    now
+  );
+  const CONNECTION_REQUEST_ALLOW_BASIC_AUTH = !!getConfig(
+    snapshot,
+    "cwmp.connectionRequestAllowBasicAuth",
+    context,
+    now
+  );
+  let authExp: Expression = getConfigExpression(
+    snapshot,
+    "cwmp.connectionRequestAuth"
+  );
+
+  if (authExp === undefined) {
+    authExp = [
+      "FUNC",
+      "AUTH",
+      ["PARAM", "username"],
+      ["PARAM", "password"]
+    ] as Expression;
+  }
+
+  const debug = !!getConfig(snapshot, "cwmp.debug", context, now);
+
+  let udpProm;
+  if (udpConnectionRequestAddress) {
+    udpProm = udpConnectionRequest(
+      udpConnectionRequestAddress,
+      authExp,
+      context,
+      UDP_CONNECTION_REQUEST_PORT,
+      debug,
+      deviceId
+    );
+  }
+
+  try {
+    await httpConnectionRequest(
+      connectionRequestUrl,
+      authExp,
+      context,
+      CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
+      CONNECTION_REQUEST_TIMEOUT,
+      debug,
+      deviceId
+    );
+  } catch (err) {
+    if (!udpProm) throw err;
+    await udpProm;
+  }
+}
+
+export async function watchTask(deviceId, taskId, timeout): Promise<string> {
+  await new Promise(resolve => setTimeout(resolve, 500));
+
+  const task = await db.tasksCollection.findOne(
+    { _id: taskId },
+    { projection: { _id: 1 } }
+  );
+  if (!task) return "completed";
+
+  const q = { _id: `${deviceId}:task_${taskId}` };
+  const fault = await db.faultsCollection.findOne(q, {
+    projection: { _id: 1 }
+  });
+  if (fault) return "fault";
+
+  if ((timeout -= 500) <= 0) return "timeout";
+
+  return watchTask(deviceId, taskId, timeout);
+}
+
+function sanitizeTask(task): void {
+  task.timestamp = new Date(task.timestamp || Date.now());
+  if (task.expiry) {
+    if (common.typeOf(task.expiry) === common.DATE_TYPE || isNaN(task.expiry))
+      task.expiry = new Date(task.expiry);
+    else task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000);
+  }
+
+  const validParamValue = (p): boolean => {
+    if (
+      !Array.isArray(p) ||
+      p.length < 2 ||
+      typeof p[0] !== "string" ||
+      !p[0].length ||
+      !["string", "boolean", "number"].includes(typeof p[1]) ||
+      (p[2] != null && typeof p[2] !== "string")
+    )
+      return false;
+    return true;
+  };
+
+  switch (task.name) {
+    case "getParameterValues":
+      if (!Array.isArray(task.parameterNames) || !task.parameterNames.length)
+        throw new Error("Missing 'parameterNames' property");
+      for (const p of task.parameterNames) {
+        if (typeof p !== "string" || !p.length)
+          throw new Error(`Invalid parameter name '${p}'`);
+      }
+      break;
+
+    case "setParameterValues":
+      if (!Array.isArray(task.parameterValues) || !task.parameterValues.length)
+        throw new Error("Missing 'parameterValues' property");
+      for (const p of task.parameterValues) {
+        if (!validParamValue(p))
+          throw new Error(`Invalid parameter value '${p}'`);
+      }
+      break;
+
+    case "refreshObject":
+    case "deleteObject":
+      if (typeof task.objectName !== "string" || !task.objectName.length)
+        throw new Error("Missing 'objectName' property");
+      break;
+
+    case "addObject":
+      if (task.parameterValues != null) {
+        if (!Array.isArray(task.parameterValues))
+          throw new Error("Invalid 'parameterValues' property");
+        for (const p of task.parameterValues) {
+          if (!validParamValue(p))
+            throw new Error(`Invalid parameter value '${p}'`);
+        }
+      }
+      break;
+
+    case "download":
+      if (typeof task.fileType !== "string" || !task.fileType.length)
+        throw new Error("Missing 'fileType' property");
+
+      if (typeof task.fileName !== "string" || !task.fileName.length)
+        throw new Error("Missing 'fileName' property");
+
+      if (
+        task.targetFileName != null &&
+        typeof task.targetFileName !== "string"
+      )
+        throw new Error("Invalid 'targetFileName' property");
+      break;
+  }
+
+  return task;
+}
+
+export async function insertTasks(tasks): Promise<Task[]> {
+  if (tasks && common.typeOf(tasks) !== common.ARRAY_TYPE) tasks = [tasks];
+  else if (!tasks || tasks.length === 0) return tasks || [];
+
+  for (const task of tasks) {
+    sanitizeTask(task);
+    if (task.uniqueKey) {
+      await db.tasksCollection.deleteOne({
+        device: task.device,
+        uniqueKey: task.uniqueKey
+      });
+    }
+  }
+  await db.tasksCollection.insertMany(tasks);
+  return tasks;
+}
+
+export async function deleteDevice(deviceId): Promise<void> {
+  await Promise.all([
+    db.tasksCollection.deleteMany({ device: deviceId }),
+    db.devicesCollection.deleteOne({ _id: deviceId }),
+    db.faultsCollection.deleteMany({
+      _id: {
+        $regex: `^${common.escapeRegExp(deviceId)}\\:`
+      }
+    }),
+    db.operationsCollection.deleteMany({
+      _id: {
+        $regex: `^${common.escapeRegExp(deviceId)}\\:`
+      }
+    }),
+    cache.del(`${deviceId}_tasks_faults_operations`)
+  ]);
+}

lib/auth.coffee

@@ -1,80 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-crypto = require 'crypto'
-
-
-exports.parseAuthHeader = (authHeader) ->
-  res = {}
-  i = authHeader.indexOf(' ')
-  res['method'] = authHeader.slice(0, i)
-
-  options = authHeader.slice(i + 1)
-  regex = /([a-z]+)="([a-zA-Z0-9\/\.@\-\+,= ]+)"/g
-  while (r = regex.exec(options)) != null
-    res[r[1]] = r[2]
-
-  regex = /([a-z]+)=([a-zA-Z0-9\/\.@\-\+]+)/g
-  while (r = regex.exec(options)) != null
-    res[r[1]] = r[2]
-  return res
-
-
-exports.basic = (username, password) ->
-  "Basic #{new Buffer("#{username}:#{password}").toString('base64')}"
-
-
-exports.digest = (username, password, uri, httpMethod, body, authHeader) ->
-  cnonce = '0a4f113b'
-  nc = '00000001'
-
-  if authHeader.qop?
-    if authHeader.qop.indexOf(',') != -1
-      qop = 'auth' # either auth or auth-int. prefer auth
-    else
-      qop = authHeader.qop
-
-  ha1 = crypto.createHash('md5')
-  ha1.update(username).update(':').update(authHeader.realm).update(':').update(password)
-  # TODO support "MD5-sess" algorithm directive
-  ha1 = ha1.digest('hex')
-
-  ha2 = crypto.createHash('md5')
-  ha2.update(httpMethod).update(':').update(uri)
-  if qop == 'auth-int'
-    ha2.update(':').update(body)
-  ha2 = ha2.digest('hex')
-
-  response = crypto.createHash('md5')
-  response.update(ha1).update(':').update(authHeader.nonce)
-
-  if qop?
-    response.update(':').update(nc).update(':').update(cnonce).update(':').update(qop)
-  response.update(':').update(ha2)
-  response = response.digest('hex')
-
-  authString = "Digest username=\"#{username}\""
-  authString += ",realm=\"#{authHeader.realm}\""
-  authString += ",nonce=\"#{authHeader.nonce}\""
-  authString += ",uri=\"#{uri}\""
-  authString += ",algorithm=#{authHeader.algorithm}" if authHeader.algorithm?
-  authString += ",qop=#{qop}"
-  authString += ",nc=#{nc}"
-  authString += ",cnonce=\"#{cnonce}\""
-  authString += ",response=\"#{response}\""
-  authString += ",opaque=\"#{authHeader.opaque}\""
-  return authString

lib/auth.ts

@@ -0,0 +1,202 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { createHash, randomBytes, pbkdf2 } from "crypto";
+
+function parseHeaderFeilds(str: string): {} {
+  const res = {};
+  const parts = str.split(",");
+
+  let part;
+  while ((part = parts.shift()) != null) {
+    const name = part.split("=", 1)[0];
+    if (name.length === part.length)
+      throw new Error("Unable to parse auth header");
+
+    let value = part.slice(name.length + 1);
+    if (!/^\s*"/.test(value)) {
+      value = value.trim();
+    } else {
+      while (!/[^\\]"\s*$/.test(value)) {
+        const p = parts.shift();
+        if (p == null) throw new Error("Unable to parse auth header");
+        value += "," + p;
+      }
+
+      try {
+        value = JSON.parse(value);
+      } catch (error) {
+        throw new Error("Unable to parse auth header");
+      }
+    }
+    res[name.trim()] = value;
+  }
+  return res;
+}
+
+export function parseAuthorizationHeader(authHeader): { method: string } {
+  authHeader = authHeader.trim();
+  const method = authHeader.split(" ", 1)[0];
+  const res = { method: method };
+
+  if (method === "Basic") {
+    // Inspired by https://github.com/jshttp/basic-auth
+    const USER_PASS_REGEX = /^([^:]*):(.*)$/;
+    const creds = USER_PASS_REGEX.exec(
+      Buffer.from(authHeader.slice(method.length + 1), "base64").toString()
+    );
+
+    if (!creds) throw new Error("Unable to parse auth header");
+    res["username"] = creds[1];
+    res["password"] = creds[2];
+  } else if (method === "Digest") {
+    Object.assign(res, parseHeaderFeilds(authHeader.slice(method.length + 1)));
+  }
+
+  return res;
+}
+
+export function parseWwwAuthenticateHeader(authHeader): {} {
+  authHeader = authHeader.trim();
+  const method = authHeader.split(" ", 1)[0];
+  const res = { method: method };
+  Object.assign(res, parseHeaderFeilds(authHeader.slice(method.length + 1)));
+  return res;
+}
+
+export function basic(username, password): string {
+  return "Basic " + Buffer.from(`${username}:${password}`).toString("base64");
+}
+
+export function digest(
+  username: string | Buffer,
+  realm: string | Buffer,
+  password: string | Buffer,
+  nonce: string | Buffer,
+  httpMethod: string | Buffer,
+  uri: string | Buffer,
+  qop?: string | Buffer,
+  body?: string | Buffer,
+  cnonce?: string | Buffer,
+  nc?: string | Buffer
+): string {
+  const ha1 = createHash("md5");
+  ha1
+    .update(username)
+    .update(":")
+    .update(realm)
+    .update(":")
+    .update(password);
+  // TODO support "MD5-sess" algorithm directive
+  const ha1d = ha1.digest("hex");
+
+  const ha2 = createHash("md5");
+  ha2
+    .update(httpMethod)
+    .update(":")
+    .update(uri);
+
+  if (qop === "auth-int") {
+    const bodyHash = createHash("md5")
+      .update(body || "")
+      .digest("hex");
+    ha2.update(":").update(bodyHash);
+  }
+
+  const ha2d = ha2.digest("hex");
+
+  const hash = createHash("md5");
+  hash
+    .update(ha1d)
+    .update(":")
+    .update(nonce);
+  if (qop) {
+    hash
+      .update(":")
+      .update(nc)
+      .update(":")
+      .update(cnonce)
+      .update(":")
+      .update(qop);
+  }
+  hash.update(":").update(ha2d);
+
+  return hash.digest("hex");
+}
+
+export function solveDigest(
+  username: string | Buffer,
+  password: string | Buffer,
+  uri: string | Buffer,
+  httpMethod: string | Buffer,
+  body: string | Buffer,
+  authHeader
+): string {
+  const cnonce = randomBytes(8).toString("hex");
+  const nc = "00000001";
+
+  let qop;
+  if (authHeader.qop) {
+    if (authHeader.qop.indexOf(",") !== -1) qop = "auth";
+    // Either auth or auth-int, prefer auth
+    else qop = authHeader.qop;
+  }
+
+  const hash = digest(
+    username,
+    authHeader.realm,
+    password,
+    authHeader.nonce,
+    httpMethod,
+    uri,
+    qop,
+    body,
+    cnonce,
+    nc
+  );
+
+  let authString = `Digest username="${username}"`;
+  authString += `,realm="${authHeader.realm}"`;
+  authString += `,nonce="${authHeader.nonce}"`;
+  authString += `,uri="${uri}"`;
+  if (authHeader.algorithm) authString += `,algorithm=${authHeader.algorithm}`;
+  if (qop) authString += `,qop=${qop},nc=${nc},cnonce="${cnonce}"`;
+  authString += `,response="${hash}"`;
+  if (authHeader.opaque) authString += `,opaque="${authHeader.opaque}"`;
+
+  return authString;
+}
+
+export function generateSalt(length): Promise<string> {
+  return new Promise((resolve, reject) => {
+    randomBytes(length, (err, rand) => {
+      if (err) return void reject(err);
+      resolve(rand.toString("hex"));
+    });
+  });
+}
+
+export function hashPassword(pass, salt): Promise<string> {
+  return new Promise((resolve, reject) => {
+    pbkdf2(pass, salt, 10000, 128, "sha512", (err, hash) => {
+      if (err) return void reject(err);
+      resolve(hash.toString("hex"));
+    });
+  });
+}

lib/cache.ts

@@ -0,0 +1,140 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { MongoClient, Collection } from "mongodb";
+import * as config from "./config";
+
+const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
+
+let mongoClient: MongoClient;
+let mongoCollection: Collection;
+let mongoTimeOffset = 0;
+
+export async function connect(): Promise<void> {
+  const MONGODB_CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
+  mongoClient = await MongoClient.connect(MONGODB_CONNECTION_URL, {
+    useNewUrlParser: true
+  });
+  const db = mongoClient.db();
+  mongoCollection = db.collection("cache");
+  await mongoCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
+  const now = Date.now();
+  const res = await db.command({ hostInfo: 1 });
+  mongoTimeOffset = res.system.currentTime.getTime() - now;
+}
+
+export async function disconnect(): Promise<void> {
+  if (mongoClient) await mongoClient.close();
+}
+
+export async function get(key): Promise<any> {
+  const expire = new Date(Date.now() - mongoTimeOffset);
+  if (Array.isArray(key)) {
+    const res = await mongoCollection.find({ _id: { $in: key } }).toArray();
+
+    const indices = {};
+    key.forEach((v, i) => {
+      indices[v] = i;
+    });
+
+    const values = [];
+    res.forEach(r => {
+      if (r["expire"] > expire) values[indices[r["_id"]]] = r["value"];
+    });
+
+    return values;
+  } else {
+    const res = await mongoCollection.findOne({ _id: { $in: [key] } });
+    if (res && res["expire"] > expire) return res["value"];
+    return res;
+  }
+}
+
+export async function del(key): Promise<void> {
+  if (Array.isArray(key))
+    await mongoCollection.deleteMany({ _id: { $in: key } });
+  else await mongoCollection.deleteOne({ _id: key });
+}
+
+export async function set(
+  key: string,
+  value: string | number,
+  ttl: number = MAX_CACHE_TTL
+): Promise<void> {
+  const expire = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
+  await mongoCollection.replaceOne(
+    { _id: key },
+    { _id: key, value: value, expire: expire },
+    { upsert: true }
+  );
+}
+
+export async function pop(key): Promise<any> {
+  const res = await mongoCollection.findOneAndDelete({ _id: key });
+
+  if (
+    res &&
+    res["value"] &&
+    +res["value"]["expire"] - (Date.now() - mongoTimeOffset)
+  )
+    return res["value"]["value"];
+
+  return null;
+}
+
+export async function lock(lockName, ttl): Promise<Function> {
+  const token = Math.random()
+    .toString(36)
+    .slice(2);
+
+  async function unlockOrExtend(extendTtl): Promise<void> {
+    if (!extendTtl) {
+      const res = await mongoCollection.deleteOne({
+        _id: lockName,
+        value: token
+      });
+      if (res["result"]["n"] !== 1) throw new Error("Lock expired");
+    } else {
+      const expire = new Date(Date.now() - mongoTimeOffset + extendTtl * 1000);
+      const res = await mongoCollection.updateOne(
+        { _id: lockName, value: token },
+        { expire: expire }
+      );
+      if (res["result"]["n"] !== 1) throw new Error("Lock expired");
+    }
+  }
+
+  const expireTest = new Date(Date.now() - mongoTimeOffset);
+  const expireSet = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
+
+  try {
+    await mongoCollection.updateOne(
+      { _id: lockName, expire: { $lte: expireTest } },
+      { $set: { value: token, expire: expireSet } },
+      { upsert: true }
+    );
+  } catch (err) {
+    if (err && err.code === 11000) {
+      await new Promise(resolve => setTimeout(resolve, 200));
+      return lock(lockName, ttl);
+    }
+  }
+
+  return unlockOrExtend;
+}

lib/cluster.coffee

@@ -1,50 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-config = require './config'
-cluster = require 'cluster'
-util = require 'util'
-
-# Keep current configuration to pass that to workers as environment variables
-ENVIRONMENT_VARIABLES = {}
-for k, v of config.allConfig
-  ENVIRONMENT_VARIABLES["GENIEACS_#{k}"] = v
-
-
-cluster.on('listening', (worker, address) ->
-  util.log("Worker #{worker.process.pid} listening to #{address.address}:#{address.port}")
-)
-
-cluster.on('exit', (worker, code, signal) ->
-  util.log("Worker #{worker.process.pid} died (#{worker.process.exitCode})")
-  cluster.fork(ENVIRONMENT_VARIABLES)
-)
-
-
-start = (service) ->
-  workerProcesses = config.get("#{service.toUpperCase()}_WORKER_PROCESSES")
-
-  cluster.setupMaster({
-      exec : 'lib/server',
-      args : [service]
-  })
-
-  for [0 ... workerProcesses]
-    cluster.fork(ENVIRONMENT_VARIABLES)
-
-
-exports.start = start

lib/cluster.ts

@@ -0,0 +1,95 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import cluster from "cluster";
+import { cpus } from "os";
+import * as logger from "./logger";
+
+let respawnTimestamp = 0;
+let tooManyCrashesTimestamp = 0;
+
+function restartWorker(worker, code, signal): void {
+  const msg = {
+    message: "Worker died",
+    pid: worker.process.pid,
+    exitCode: null,
+    signal: null
+  };
+
+  if (code != null) msg.exitCode = code;
+
+  if (signal != null) msg.signal = signal;
+
+  logger.error(msg);
+
+  const now = Date.now();
+  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  if (respawnTimestamp === now) {
+    tooManyCrashesTimestamp = now;
+    cluster.fork();
+    return;
+  }
+
+  if (now - tooManyCrashesTimestamp > 60000) {
+    process.exitCode = 1;
+    cluster.removeListener("exit", restartWorker);
+    for (const pid in cluster.workers) cluster.workers[pid].kill();
+
+    logger.error({
+      message: "Too many crashes, exiting",
+      pid: process.pid
+    });
+    return;
+  }
+
+  setTimeout(() => {
+    if (process.exitCode) return;
+    cluster.fork();
+  }, respawnTimestamp - now);
+}
+
+export function start(workerCount, servicePort, serviceAddress): void {
+  cluster.on("listening", (worker, address) => {
+    if (
+      (address.addressType === 4 || address.addressType === 6) &&
+      address.address === serviceAddress &&
+      address.port === servicePort
+    ) {
+      logger.info({
+        message: "Worker listening",
+        pid: worker.process.pid,
+        address: address.address,
+        port: address.port
+      });
+    }
+  });
+
+  cluster.on("exit", restartWorker);
+
+  if (!workerCount) workerCount = Math.max(2, cpus().length);
+
+  for (let i = 0; i < workerCount; ++i) cluster.fork();
+}
+
+export function stop(): void {
+  cluster.removeListener("exit", restartWorker);
+  for (const pid in cluster.workers) cluster.workers[pid].kill();
+}
+
+export const worker = cluster.worker;

lib/common.coffee

@@ -1,121 +0,0 @@
-###
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-buffer = require 'buffer'
-querystring = require 'querystring'
-
-UNDEFINED_TYPE = '[object Undefined]'
-NULL_TYPE = '[object Null]'
-BOOLEAN_TYPE = '[object Boolean]'
-NUMBER_TYPE = '[object Number]'
-STRING_TYPE = '[object String]'
-ARRAY_TYPE = '[object Array]'
-OBJECT_TYPE = '[object Object]'
-REGEXP_TYPE = '[object RegExp]'
-DATE_TYPE = '[object Date]'
-
-
-typeOf = (obj) ->
-  Object.prototype.toString.call(obj)
-
-
-exports.endsWith = (str, suffix) ->
-  str.indexOf(suffix, str.length - suffix.length) isnt -1
-
-
-exports.startsWith = (str, prefix) ->
-  str.substring(0, prefix.length) == prefix
-
-
-exports.arrayToHash = (arr) ->
-  hash = {}
-  for i in arr
-    hash[i[0]] = i[1]
-  return hash
-
-
-exports.generateDeviceId = (deviceIdStruct) ->
-  # Percent escaping function. Escapes everything except alphanumerics and underscore
-  esc = (str) ->
-    str.replace(/[^A-Za-z0-9_]/g, (chr) ->
-      buf = new buffer.Buffer(chr, 'utf8')
-      rep = ''
-      rep += "%#{b.toString(16).toUpperCase()}" for b in buf
-      return rep
-    )
-
-  # Guaranteeing globally unique id as defined in TR-069
-  if deviceIdStruct['ProductClass']
-    return "#{esc(deviceIdStruct['OUI'])}-#{esc(deviceIdStruct['ProductClass'])}-#{esc(deviceIdStruct['SerialNumber'])}"
-
-  return "#{esc(deviceIdStruct['OUI'])}-#{esc(deviceIdStruct['SerialNumber'])}"
-
-
-exports.parseDeviceId = (deviceId) ->
-  parts = deviceId.split('-')
-  ret = {oui : querystring.unescape(parts[0])}
-  if parts.length == 3
-    ret.productClass = querystring.unescape(parts[1])
-    ret.serialNumber = querystring.unescape(parts[2])
-  else
-    ret.serialNumber = querystring.unescape(parts[1])
-  return ret
-
-
-exports.extend = (obj, mixin) ->
-  obj[name] = method for name, method of mixin        
-  obj
-
-
-exports.getParamValueFromPath = (obj, path) ->
-  pp = path.split('.')
-  ref = obj
-  try
-    for p in pp
-      ref = ref[p]
-    return ref
-  catch err
-    return undefined
-
-
-exports.matchType = (src, dst) ->
-  switch typeOf(src)
-    when STRING_TYPE
-      String(dst)
-    when NUMBER_TYPE
-      Number(dst)
-    when BOOLEAN_TYPE
-      v = String(dst).trim().toLowerCase()
-      v == 'true' or v == 'on' or v == 'yes' or v == '1'
-    else
-      dst
-
-
-exports.UNDEFINED_TYPE = UNDEFINED_TYPE
-exports.NULL_TYPE = NULL_TYPE
-exports.NUMBER_TYPE = NUMBER_TYPE
-exports.STRING_TYPE = STRING_TYPE
-exports.ARRAY_TYPE = ARRAY_TYPE
-exports.OBJECT_TYPE = OBJECT_TYPE
-exports.REGEXP_TYPE = REGEXP_TYPE
-
-exports.typeOf = typeOf

lib/common.ts

@@ -0,0 +1,59 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+export const UNDEFINED_TYPE = "[object Undefined]";
+export const NULL_TYPE = "[object Null]";
+export const BOOLEAN_TYPE = "[object Boolean]";
+export const NUMBER_TYPE = "[object Number]";
+export const STRING_TYPE = "[object String]";
+export const ARRAY_TYPE = "[object Array]";
+export const OBJECT_TYPE = "[object Object]";
+export const REGEXP_TYPE = "[object RegExp]";
+export const DATE_TYPE = "[object Date]";
+
+export const typeOf = (obj): string => Object.prototype.toString.call(obj);
+
+export function generateDeviceId(deviceIdStruct): string {
+  // Escapes everything except alphanumerics and underscore
+  function esc(str): string {
+    return str.replace(/[^A-Za-z0-9_]/g, chr => {
+      const buf = Buffer.from(chr, "utf8");
+      let rep = "";
+      for (const b of buf) rep += "%" + b.toString(16).toUpperCase();
+      return rep;
+    });
+  }
+
+  // Guaranteeing globally unique id as defined in TR-069
+  if (deviceIdStruct["ProductClass"]) {
+    return (
+      esc(deviceIdStruct["OUI"]) +
+      "-" +
+      esc(deviceIdStruct["ProductClass"]) +
+      "-" +
+      esc(deviceIdStruct["SerialNumber"])
+    );
+  }
+  return esc(deviceIdStruct["OUI"]) + "-" + esc(deviceIdStruct["SerialNumber"]);
+}
+
+// Source: http://stackoverflow.com/a/6969486
+export function escapeRegExp(str): string {
+  return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
+}

lib/common/authorizer.ts

@@ -0,0 +1,123 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { PermissionSet, Expression } from "../types";
+import { evaluate } from "./expression";
+
+export default class Authorizer {
+  private permissionSets: PermissionSet[];
+  private validatorCache: WeakMap<
+    object,
+    (mutationType, mutation, any) => boolean
+  >;
+  private hasAccessCache: Map<string, boolean>;
+
+  public constructor(permissionSets) {
+    this.permissionSets = permissionSets;
+    this.validatorCache = new WeakMap();
+    this.hasAccessCache = new Map();
+  }
+
+  public hasAccess(resourceType, access): boolean {
+    const cacheKey = `${resourceType}-${access}`;
+    if (this.hasAccessCache.has(cacheKey))
+      return this.hasAccessCache.get(cacheKey);
+
+    let has = false;
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (perm[resourceType]) {
+          if (perm[resourceType].access >= access) {
+            has = true;
+            break;
+          }
+        }
+      }
+    }
+
+    this.hasAccessCache.set(cacheKey, has);
+    return has;
+  }
+
+  public getValidator(
+    resourceType,
+    resource
+  ): (mutationType: string, mutation: any, args: any) => boolean {
+    if (this.validatorCache.has(resource))
+      return this.validatorCache.get(resource);
+
+    const validators: Expression[] = [];
+
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (
+          perm[resourceType] &&
+          perm[resourceType].access >= 3 &&
+          perm[resourceType].validate
+        )
+          validators.push(perm[resourceType].validate);
+      }
+    }
+
+    const validator = (
+      mutationType: string,
+      mutation: any,
+      any: any
+    ): boolean => {
+      if (!validators.length) return false;
+
+      const object = {
+        mutationType,
+        mutation,
+        resourceType,
+        object: resource,
+        options: any
+      };
+
+      const valueFunction = (paramName): any => {
+        const entry = paramName.split(".", 1)[0];
+        paramName = paramName.slice(entry.length + 1);
+        let value = null;
+        if (["mutation", "options"].includes(entry)) {
+          value = object[entry];
+          for (const seg of paramName.split(".")) {
+            // typeof null is "object"
+            if (value != null && typeof value !== "object") value = null;
+            else value = value[seg];
+            if (value == null) break;
+          }
+        } else if (object[entry]) {
+          value = object[entry][paramName];
+        }
+
+        return value;
+      };
+
+      const res = evaluate(
+        validators.length > 1 ? ["OR", validators] : validators[0],
+        valueFunction,
+        Date.now()
+      );
+      return !Array.isArray(res) && !!res;
+    };
+
+    this.validatorCache.set(resource, validator);
+    return validator;
+  }
+}

lib/common/expression-cnf.ts

@@ -0,0 +1,285 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { map, parseLikePattern } from "./expression-parser";
+import { Expression } from "../types";
+
+type CNF = number[][];
+
+function or(cnf1: CNF, cnf2: CNF): CNF {
+  if (cnf1.length === 0) return cnf2;
+  else if (cnf2.length === 0) return cnf1;
+
+  if (cnf1.length > cnf2.length) {
+    const c = cnf1;
+    cnf1 = cnf2;
+    cnf2 = c;
+  }
+
+  const res = [];
+  for (let i = 0; i < cnf1.length; ++i)
+    for (let j = i; j < cnf2.length; ++j) res.push(cnf1[i].concat(cnf2[j]));
+
+  return res;
+}
+
+function likePatternIncludes(pat1, pat2, idx1 = 0, idx2 = 0): boolean {
+  while (idx1 < pat1.length && idx2 < pat2.length) {
+    if (pat1[idx1] === "\\%") {
+      for (let i = idx2; i <= pat2.length; ++i)
+        if (likePatternIncludes(pat1, pat2, idx1 + 1, i)) return true;
+      return false;
+    } else if (pat2[idx2] === "\\%") {
+      if (pat1[idx1] !== "\\%") return false;
+    } else if (pat1[idx1] === "\\_") {
+      // Ignore
+    } else if (pat2[idx2] === "\\_") {
+      return false;
+    } else if (pat2[idx2] !== pat1[idx1]) {
+      return false;
+    }
+    ++idx1;
+    ++idx2;
+  }
+
+  if (idx1 === pat1.length && idx2 === pat2.length) return true;
+  return false;
+}
+
+export function booleanCnf(exp: Expression): { vars: number; clauses: CNF } {
+  const ranges = new Map();
+  const likePatterns = new Map();
+  map(exp, e => {
+    if (!Array.isArray(e)) return e;
+    if (e[0] === "LIKE" || e[0] === "NOT LIKE") {
+      if (
+        typeof e[2] === "string" &&
+        (e.length >= 4 || (e[3] != null && !Array.isArray(e[3])))
+      ) {
+        const key = JSON.stringify(e[1]);
+        let s = likePatterns.get(key);
+        if (!s) likePatterns.set(key, (s = new Map()));
+        s.set(`${e[3] || ""}:${e[2]}`, parseLikePattern(e[2], e[3]));
+      }
+    } else if ([">", ">=", "<", "<=", "=", "<>"].includes(e[0])) {
+      if (!Array.isArray(e[2])) {
+        const key = `${JSON.stringify(e[1])}:${typeof e[2]}`;
+        let s = ranges.get(key);
+        if (!s) ranges.set(key, (s = new Set()));
+        s.add(e[2]);
+      } else if (!Array.isArray(e[1])) {
+        const key = `${JSON.stringify(e[2])}:${typeof e[1]}`;
+        let s = ranges.get(key);
+        if (!s) ranges.set(key, (s = new Set()));
+        s.add(e[2]);
+      }
+    }
+    return e;
+  });
+
+  for (const [k, v] of ranges) {
+    if (k.endsWith(":number")) {
+      ranges.set(
+        k,
+        Array.from(v as number[]).sort((a: number, b: number) => a - b)
+      );
+    } else {
+      ranges.set(k, Array.from(v).sort());
+    }
+  }
+
+  const expressions: Map<string, number> = new Map();
+  const mutuallyExclusive: Map<string, Set<number>> = new Map();
+
+  function getVariable(lhs, op, rhs = ""): number {
+    const key = `${lhs}:${op}:${rhs}`;
+    let f = expressions.get(key);
+    if (!f) {
+      expressions.set(key, (f = expressions.size + 1));
+      let m = mutuallyExclusive.get(lhs);
+      if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
+      m.add(f);
+      if (rhs || op !== "null") m.add(getVariable(lhs, "null"));
+    }
+    return f;
+  }
+
+  function getComparisonVariables(
+    lhs,
+    from,
+    fromInclusive,
+    to,
+    toInclusive
+  ): number[] {
+    const type = from != null ? typeof from : typeof to;
+    const range = ranges.get(`${lhs}:${type}`);
+    const vars = [];
+
+    if (from == null) vars.push(`${lhs}:--${range[0]}`);
+
+    for (let i = 0; i < range.length; ++i) {
+      if (
+        i > 0 &&
+        (from == null || range[i] > from) &&
+        (to == null || range[i] <= to)
+      )
+        vars.push(`${lhs}:${range[i - 1]}--${range[i]}`);
+
+      if (
+        (range[i] > from && range[i] < to) ||
+        (range[i] === from && fromInclusive) ||
+        (range[i] === to && toInclusive)
+      )
+        vars.push(`${lhs}:${range[i]}`);
+    }
+
+    if (to == null) vars.push(`${lhs}:${range[range.length - 1]}--`);
+
+    return vars.map(key => {
+      let f = expressions.get(key);
+      if (!f) {
+        expressions.set(key, (f = expressions.size + 1));
+        let m = mutuallyExclusive.get(lhs);
+        if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
+        m.add(f);
+      }
+      return f;
+    });
+  }
+
+  function getLikeVariables(lhs, pat, esc): number[] {
+    const vars = [];
+    const pats = likePatterns.get(lhs);
+    pat = pats.get(`${esc || ""}:${pat}`);
+    for (const p of pats.values()) {
+      if (pat === p || likePatternIncludes(pat, p))
+        vars.push(`${lhs}:like:${JSON.stringify(p)}`);
+    }
+
+    return vars.map(key => {
+      let f = expressions.get(key);
+      if (!f) expressions.set(key, (f = expressions.size + 1));
+      return f;
+    });
+  }
+
+  interface ComparisonOperators {
+    [name: string]: (v: string) => [string, boolean, string, boolean];
+  }
+
+  const comparisonOperators: ComparisonOperators = {
+    "=": v => [v, true, v, true],
+    ">": v => [v, false, null, null],
+    ">=": v => [v, true, null, null],
+    "<": v => [null, null, v, false],
+    "<=": v => [null, null, v, true]
+  };
+
+  function recursive(clause, negate): CNF {
+    let op = clause[0];
+
+    if ((op === "AND" && !negate) || (op === "OR" && negate)) {
+      let res = [];
+      for (let i = 1; i < clause.length; ++i)
+        res = res.concat(recursive(clause[i], negate));
+      return res;
+    } else if ((op === "OR" && !negate) || (op === "AND" && negate)) {
+      let res = [];
+      for (let i = 1; i < clause.length; ++i)
+        res = or(res, recursive(clause[i], negate));
+      return res;
+    } else if (op === "NOT") {
+      return recursive(clause[1], !negate);
+    } else if (op === "<>") {
+      const vars = getComparisonVariables(
+        JSON.stringify(clause[1]),
+        ...comparisonOperators["="](clause[2])
+      );
+      if (!negate) return vars.map(x => [0 - x]);
+      else return [vars];
+    } else if (op === "IS NULL") {
+      const v = getVariable(JSON.stringify(clause[1]), "null");
+      if (negate) return [[0 - v]];
+      else return [[v]];
+    } else if (op === "IS NOT NULL") {
+      const v = getVariable(JSON.stringify(clause[1]), "null");
+      if (!negate) return [[0 - v]];
+      else return [[v]];
+    } else if (op in comparisonOperators) {
+      if (!Array.isArray(clause[2])) {
+        const vars = getComparisonVariables(
+          JSON.stringify(clause[1]),
+          ...comparisonOperators[op](clause[2])
+        );
+        if (negate) return vars.map(x => [0 - x]);
+        else return [vars];
+      } else if (!Array.isArray(clause[1])) {
+        op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
+        const vars = getComparisonVariables(
+          JSON.stringify(clause[2]),
+          ...comparisonOperators[op](clause[1])
+        );
+        if (negate) return vars.map(x => [0 - x]);
+        else return [vars];
+      } else {
+        let lhs = JSON.stringify(clause[1]);
+        let rhs = JSON.stringify(clause[2]);
+        // For consistency
+        if (rhs > lhs) {
+          const t = rhs;
+          rhs = lhs;
+          lhs = t;
+          op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
+        }
+
+        const v = getVariable(lhs, op, rhs);
+        if (negate) return [[0 - v]];
+        else return [[v]];
+      }
+    } else if (
+      (op === "LIKE" || op === "NOT LIKE") &&
+      !Array.isArray(clause[2]) &&
+      clause[2] != null &&
+      (clause.length >= 4 || clause[3] != null)
+    ) {
+      if (op === "NOT LIKE") negate = !negate;
+      const vars = getLikeVariables(
+        JSON.stringify(clause[1]),
+        clause[2],
+        clause[3]
+      );
+      if (negate) return vars.map(x => [0 - x]);
+      else return [vars];
+    } else {
+      const v = getVariable(JSON.stringify(clause), "");
+      if (negate) return [[0 - v]];
+      else return [[v]];
+    }
+  }
+
+  const cnf = recursive(exp, false);
+
+  for (const m of mutuallyExclusive.values()) {
+    const ar: number[] = Array.from(m);
+    for (let i = 0; i < ar.length; ++i)
+      for (let j = i + 1; j < ar.length; ++j) cnf.push([0 - ar[i], 0 - ar[j]]);
+  }
+
+  return { vars: expressions.size, clauses: cnf };
+}

lib/common/expression-parser.ts

@@ -0,0 +1,412 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as parsimmon from "parsimmon";
+import { Expression } from "../types";
+
+// Turn escaped characters into real ones (e.g. "\\n" becomes "\n").
+function interpretEscapes(str): string {
+  const escapes = {
+    b: "\b",
+    f: "\f",
+    n: "\n",
+    r: "\r",
+    t: "\t"
+  };
+  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
+    const type = escape.charAt(0);
+    const hex = escape.slice(1);
+    if (type === "u") return String.fromCharCode(parseInt(hex, 16));
+
+    if (escapes.hasOwnProperty(type)) return escapes[type];
+
+    return type;
+  });
+}
+
+export function map(exp, callback): Expression {
+  if (!Array.isArray(exp)) return callback(exp);
+
+  let clone;
+  for (let i = 1; i < exp.length; ++i) {
+    const sub = map(exp[i], callback);
+    if (sub !== exp[i]) {
+      clone = clone || exp.slice();
+      clone[i] = sub;
+    }
+  }
+
+  return callback(clone || exp);
+}
+
+export async function mapAsync(exp, callback): Promise<Expression> {
+  if (!Array.isArray(exp)) return callback(exp);
+
+  let clone;
+  for (let i = 1; i < exp.length; ++i) {
+    const sub = await mapAsync(exp[i], callback);
+    if (sub !== exp[i]) {
+      clone = clone || exp.slice();
+      clone[i] = sub;
+    }
+  }
+
+  return callback(clone || exp);
+}
+
+function binaryLeft(operatorsParser, nextParser): parsimmon.Parser<{}> {
+  return parsimmon.seqMap(
+    nextParser,
+    parsimmon.seq(operatorsParser, nextParser).many(),
+    (first, rest) =>
+      rest.reduce((acc, ch) => {
+        const [op, another] = ch;
+        if (Array.isArray(acc) && op === acc[0]) return acc.concat([another]);
+        if (Array.isArray(another) && op === another[0])
+          return [op, acc].concat(another.slice(1));
+        return [op, acc, another];
+      }, first)
+  );
+}
+
+const lang = parsimmon.createLanguage({
+  ComparisonOperator: function() {
+    return parsimmon
+      .alt(
+        parsimmon.string(">="),
+        parsimmon.string("<>"),
+        parsimmon.string("<="),
+        parsimmon.string("="),
+        parsimmon.string(">"),
+        parsimmon.string("<")
+      )
+      .skip(parsimmon.optWhitespace);
+  },
+  LikeOperator: function() {
+    return parsimmon
+      .alt(
+        parsimmon
+          .regexp(/like/i)
+          .result("LIKE")
+          .desc("LIKE"),
+        parsimmon
+          .regexp(/not\s+like/i)
+          .result("NOT LIKE")
+          .desc("NOT LIKE")
+      )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace);
+  },
+  IsNullOperator: function() {
+    return parsimmon
+      .alt(
+        parsimmon
+          .regexp(/is\s+null/i)
+          .result("IS NULL")
+          .desc("IS NULL"),
+        parsimmon
+          .regexp(/is\s+not\s+null/i)
+          .result("IS NOT NULL")
+          .desc("IS NOT NULL")
+      )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace);
+  },
+  NotOperator: function() {
+    return parsimmon
+      .regexp(/not/i)
+      .result("NOT")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .desc("NOT");
+  },
+  AndOperator: function() {
+    return parsimmon
+      .regexp(/and/i)
+      .result("AND")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .desc("AND");
+  },
+  OrOperator: function() {
+    return parsimmon
+      .regexp(/or/i)
+      .result("OR")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .desc("OR");
+  },
+  Parameter: function(r) {
+    return parsimmon
+      .alt(
+        parsimmon.regexp(/[a-zA-Z0-9_.*-]+/),
+        r.Expression.wrap(
+          parsimmon.string("{").skip(parsimmon.optWhitespace),
+          parsimmon.string("}")
+        )
+      )
+      .atLeast(1)
+      .map(x => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
+      .skip(parsimmon.optWhitespace)
+      .desc("parameter");
+  },
+  StringValueSql: function() {
+    return parsimmon
+      .regexp(/'([^']*)'/, 1)
+      .atLeast(1)
+      .skip(parsimmon.optWhitespace)
+      .map(s => s.join("'"))
+      .desc("string");
+  },
+  StringValueJs: function() {
+    return parsimmon
+      .regexp(/"((?:\\.|.)*?)"/, 1)
+      .skip(parsimmon.optWhitespace)
+      .map(interpretEscapes)
+      .desc("string");
+  },
+  NumberValue: function() {
+    return parsimmon
+      .regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/)
+      .skip(parsimmon.optWhitespace)
+      .map(Number)
+      .desc("number");
+  },
+  BooleanValue: function() {
+    return parsimmon
+      .alt(
+        parsimmon
+          .regexp(/true/i)
+          .result(true)
+          .desc("TRUE"),
+        parsimmon
+          .regexp(/false/i)
+          .result(false)
+          .desc("FALSE")
+      )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace);
+  },
+  NullValue: function() {
+    return parsimmon
+      .regexp(/null/i)
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .result(null)
+      .desc("NULL");
+  },
+  FuncValue: function(r) {
+    return parsimmon.seqMap(
+      parsimmon
+        .regexp(/([a-zA-Z0-9_]+)/, 1)
+        .skip(parsimmon.optWhitespace)
+        .desc("function"),
+      r.ValueExpression.sepBy(
+        parsimmon.string(",").skip(parsimmon.optWhitespace)
+      ).wrap(
+        parsimmon.string("(").skip(parsimmon.optWhitespace),
+        parsimmon.string(")").skip(parsimmon.optWhitespace)
+      ),
+      (f, args) => ["FUNC", f.toUpperCase()].concat(args)
+    );
+  },
+  Value: function(r) {
+    return parsimmon.alt(
+      r.NullValue,
+      r.BooleanValue,
+      r.NumberValue,
+      r.StringValueSql,
+      r.StringValueJs,
+      r.FuncValue
+    );
+  },
+  ValueExpression: function(r) {
+    return binaryLeft(
+      parsimmon.string("||").skip(parsimmon.optWhitespace),
+      binaryLeft(
+        parsimmon
+          .alt(parsimmon.string("+"), parsimmon.string("-"))
+          .skip(parsimmon.optWhitespace),
+        binaryLeft(
+          parsimmon
+            .alt(parsimmon.string("*"), parsimmon.string("/"))
+            .skip(parsimmon.optWhitespace),
+          parsimmon.alt(
+            r.Value,
+            r.Parameter,
+            r.Expression.wrap(
+              parsimmon.string("(").skip(parsimmon.optWhitespace),
+              parsimmon.string(")").skip(parsimmon.optWhitespace)
+            )
+          )
+        )
+      )
+    );
+  },
+  Comparison: function(r) {
+    return parsimmon.alt(
+      parsimmon.seqMap(r.ValueExpression, r.IsNullOperator, (p, o) => [o, p]),
+      parsimmon.seqMap(
+        r.ValueExpression,
+        r.ComparisonOperator,
+        r.ValueExpression,
+        (p, o, v) => [o, p, v]
+      ),
+      parsimmon.seqMap(
+        r.ValueExpression,
+        r.LikeOperator,
+        r.ValueExpression.skip(
+          parsimmon
+            .regexp(/escape/i)
+            .result("ESCAPE")
+            .skip(parsimmon.whitespace)
+            .desc("ESCAPE")
+        ),
+        r.ValueExpression,
+        (a, b, c, d) => [b, a, c, d]
+      ),
+      parsimmon.seqMap(
+        r.ValueExpression,
+        r.LikeOperator,
+        r.ValueExpression,
+        (a, b, c) => [b, a, c]
+      )
+    );
+  },
+  Expression: function(r) {
+    function unary(operatorsParser, nextParser): parsimmon.Parser<{}> {
+      return parsimmon.seq(operatorsParser, nextParser).or(nextParser);
+    }
+
+    return binaryLeft(
+      r.OrOperator,
+      binaryLeft(
+        r.AndOperator,
+        unary(r.NotOperator, r.Comparison.or(r.ValueExpression))
+      )
+    ).trim(parsimmon.optWhitespace);
+  }
+});
+
+export function parse(str: string): Expression {
+  if (!str) return null;
+  return lang.Expression.tryParse(str);
+}
+
+export function stringify(exp, level = 0): string {
+  if (!Array.isArray(exp)) return JSON.stringify(exp);
+
+  const opLevels = {
+    OR: 10,
+    AND: 11,
+    NOT: 12,
+    "=": 20,
+    "<>": 20,
+    ">": 20,
+    ">=": 20,
+    "<": 20,
+    "<=": 20,
+    LIKE: 20,
+    "NOT LIKE": 20,
+    "IS NULL": 20,
+    "IS NOT NULL": 20,
+    "||": 30,
+    "+": 31,
+    "-": 31,
+    "*": 32,
+    "/": 32
+  };
+
+  const op = exp[0].toUpperCase();
+
+  function wrap(e): string {
+    if (opLevels[op] <= level) return `(${e})`;
+    else return e;
+  }
+
+  if (op === "FUNC") {
+    return wrap(
+      `${exp[1]}(${exp
+        .slice(2)
+        .map(e => stringify(e))
+        .join(", ")})`
+    );
+  } else if (op === "PARAM") {
+    if (typeof exp[1] === "string") {
+      return wrap(exp[1]);
+    } else if (Array.isArray(exp[1]) && exp[1][0] === "||") {
+      return wrap(
+        exp[1]
+          .slice(1)
+          .map(p => {
+            if (typeof p === "string") return p;
+            else return `{${stringify(p)}}`;
+          })
+          .join("")
+      );
+    } else {
+      return wrap(`{${stringify(exp[1])}}`);
+    }
+  } else if (op === "IS NULL" || op === "IS NOT NULL") {
+    return wrap(`${stringify(exp[1], opLevels[op])} ${op}`);
+  } else if (op === "LIKE" || op === "NOT LIKE") {
+    if (exp[3]) {
+      return wrap(
+        `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
+          exp[2],
+          opLevels[op]
+        )} ESCAPE ${stringify(exp[3], opLevels[op])}`
+      );
+    } else {
+      return wrap(
+        `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
+          exp[2],
+          opLevels[op]
+        )}`
+      );
+    }
+  } else if (op in opLevels) {
+    const parts = exp.slice(1).map((e, i) => {
+      return stringify(e, opLevels[exp[0]] + Math.min(i - 1, 0));
+    });
+
+    if (op === "NOT") return wrap(`${op} ${parts[0]}`);
+    else return wrap(parts.join(` ${op} `));
+  } else {
+    throw new Error(`Unrecognized operator ${exp[0]}`);
+  }
+}
+
+export function parseLikePattern(pat, esc): string[] {
+  const chars = pat.split("");
+
+  for (let i = 0; i < chars.length; ++i) {
+    const c = chars[i];
+    if (c === esc) {
+      chars[i] = chars[i + 1] || "";
+      chars[i + 1] = "";
+    } else if (c === "_") {
+      chars[i] = "\\_";
+    } else if (c === "%") {
+      chars[i] = "\\%";
+      while (chars[i + 1] === "%") chars[++i] = "";
+    }
+  }
+  return chars.filter(c => c);
+}

lib/common/expression.ts

@@ -0,0 +1,329 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { map, mapAsync, parseLikePattern } from "./expression-parser";
+import { booleanCnf } from "./expression-cnf";
+import { naiveDpll } from "./sat-solver";
+import { Expression } from "../types";
+
+const isArray = Array.isArray;
+
+const regExpCache = new WeakMap<object, RegExp>();
+
+const REDUCE_SKIP = {};
+function reduce(exp, callback): Expression {
+  let loop = true;
+  while (loop) {
+    loop = false;
+    for (let i = 2; i < exp.length; ++i) {
+      const res = callback(exp[i - 1], exp[i], i - 2);
+      if (res !== REDUCE_SKIP) {
+        loop = true;
+        exp = exp.slice();
+        exp.splice(i - 1, 2, res);
+      }
+    }
+  }
+  if (exp.length === 2) return exp[1];
+  return exp;
+}
+
+export function likePatternToRegExp(pat, esc = "", flags = ""): RegExp {
+  const convChars = {
+    "-": "\\-",
+    "/": "\\/",
+    "\\": "\\/",
+    "^": "\\^",
+    $: "\\$",
+    "*": "\\*",
+    "+": "\\+",
+    "?": "\\?",
+    ".": "\\.",
+    "(": "\\(",
+    ")": "\\)",
+    "|": "\\|",
+    "[": "\\[",
+    "]": "\\]",
+    "{": "\\{",
+    "}": "\\}",
+    "\\%": ".*",
+    "\\_": "."
+  };
+  let chars = parseLikePattern(pat, esc);
+  if (!chars.length) return new RegExp("^$", flags);
+  chars = chars.map(c => convChars[c] || c);
+  chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
+  const l = chars.length - 1;
+  chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
+  return new RegExp(chars.join(""), flags);
+}
+
+function evalExp(e: Expression): Expression {
+  if (e[0] === "AND") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a)) return a ? b : a;
+      if (!isArray(b)) return b ? a : b;
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "OR") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a)) return a ? a : b;
+      if (!isArray(b)) return b ? b : a;
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "NOT") {
+    if (!isArray(e[1])) return !e[1];
+    else if (e[1][0] === "NOT") return e[1][1];
+  } else if (e[0] === "IS NULL") {
+    if (isArray(e[1])) return e;
+    else if (e[1] == null) return true;
+    else return null;
+  } else if (e[0] === "IS NOT NULL") {
+    if (isArray(e[1])) return e;
+    else if (e[1] != null) return true;
+    else return null;
+  } else if (e[0] === "LIKE") {
+    if (isArray(e[1]) || isArray(e[2]) || isArray(e[3])) return e;
+    else if (
+      e[1] == null ||
+      e[2] == null ||
+      ((e as any[]).length >= 4 && e[3] == null)
+    )
+      return null;
+    let r = regExpCache.get(e as any[]);
+    if (!r) {
+      r = likePatternToRegExp(e[2], e[3]);
+      regExpCache.set(e as any[], r);
+    }
+    return r.test(e[1]);
+  } else if (e[0] === "NOT LIKE") {
+    if (isArray(e[1]) || isArray(e[2]) || isArray(e[3])) return e;
+    else if (
+      e[1] == null ||
+      e[2] == null ||
+      ((e as any[]).length >= 4 && e[3] == null)
+    )
+      return null;
+    let r = regExpCache.get(e as any[]);
+    if (!r) {
+      r = likePatternToRegExp(e[2], e[3]);
+      regExpCache.set(e as any[], r);
+    }
+    return !r.test(e[1]);
+  } else if (e[0] === "=") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] === e[2];
+  } else if (e[0] === "<>") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] !== e[2];
+  } else if (e[0] === ">") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] > e[2];
+  } else if (e[0] === ">=") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] >= e[2];
+  } else if (e[0] === "<") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] < e[2];
+  } else if (e[0] === "<=") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] <= e[2];
+  } else if (e[0] === "*") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a) && !isArray(b)) {
+        if (a == null || b == null) return null;
+        return a * b;
+      }
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "/") {
+    return reduce(e, (a, b, i) => {
+      if (!isArray(a) && !isArray(b)) {
+        if (a == null || b == null) return null;
+        return i === 0 ? a / b : a * b;
+      }
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "+") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a) && !isArray(b)) {
+        if (a == null || b == null) return null;
+        return a + b;
+      }
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "-") {
+    return reduce(e, (a, b, i) => {
+      if (!isArray(a) && !isArray(b)) {
+        if (a == null || b == null) return null;
+        return i === 0 ? a - b : a + b;
+      }
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "||") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a) && !isArray(b)) {
+        if (a == null || b == null) return null;
+        return `${a}${b}`;
+      }
+      return REDUCE_SKIP;
+    });
+  }
+  return e;
+}
+
+export function evaluate(
+  exp,
+  obj,
+  now: number,
+  cb?: Function
+): string | number | boolean | null;
+export function evaluate(exp, obj?, now?: number, cb?: Function): Expression;
+export function evaluate(exp, obj?, now?: number, cb?: Function): Expression {
+  return map(exp, e => {
+    if (cb) e = cb(e);
+    if (!isArray(e)) return e;
+
+    if (e[0] === "FUNC") {
+      if (e[1] === "NOW") {
+        if (now) return now;
+      } else if (e[1] === "UPPER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
+      } else if (e[1] === "LOWER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
+      }
+    } else if (e[0] === "PARAM") {
+      if (e[1] == null) return null;
+      if (obj && !isArray(e[1])) {
+        let v;
+        if (typeof obj === "function") v = obj(e[1]);
+        else v = obj[e[1]];
+
+        if (v == null) return null;
+        if (typeof v === "object") v = v.value ? v.value[0] : null;
+        return v;
+      }
+    }
+    return evalExp(e);
+  });
+}
+
+export async function evaluateAsync(
+  exp,
+  obj,
+  now: number,
+  cb?: Function
+): Promise<string | number | boolean | null>;
+export async function evaluateAsync(
+  exp,
+  obj?,
+  now?: number,
+  cb?: Function
+): Promise<Expression>;
+export async function evaluateAsync(
+  exp,
+  obj?,
+  now?: number,
+  cb?: Function
+): Promise<Expression> {
+  return mapAsync(exp, async e => {
+    if (cb) e = await cb(e);
+    if (!isArray(e)) return e;
+
+    if (e[0] === "FUNC") {
+      if (e[1] === "NOW") {
+        if (now) return now;
+      } else if (e[1] === "UPPER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
+      } else if (e[1] === "LOWER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
+      }
+    } else if (e[0] === "PARAM") {
+      if (e[1] == null) return null;
+      if (obj && !isArray(e[1])) {
+        let v = obj[e[1]];
+        if (v == null) return null;
+        if (typeof v === "object") v = v.value ? v.value[0] : null;
+        return v;
+      }
+    }
+    return evalExp(e);
+  });
+}
+
+export function and(exp1, exp2): Expression {
+  if (!isArray(exp1)) return exp1 ? exp2 : exp1;
+  if (!isArray(exp2)) return exp2 ? exp1 : exp2;
+
+  let res: Expression = ["AND"];
+
+  if (exp1[0] === "AND") res = res.concat(exp1.slice(1));
+  else res.push(exp1);
+
+  if (exp2[0] === "AND") res = res.concat(exp2.slice(1));
+  else res.push(exp2);
+
+  return res;
+}
+
+export function or(exp1, exp2): Expression {
+  if (!isArray(exp1)) return exp1 ? exp1 : exp2;
+  if (!isArray(exp2)) return exp2 ? exp2 : exp1;
+
+  let res: Expression = ["OR"];
+
+  if (exp1[0] === "OR") res = res.concat(exp1.slice(1));
+  else res.push(exp1);
+
+  if (exp2[0] === "OR") res = res.concat(exp2.slice(1));
+  else res.push(exp2);
+
+  return res;
+}
+
+export function not(exp): Expression {
+  if (isArray(exp) && exp[0] === "NOT") return exp[1];
+  return ["NOT", exp];
+}
+
+export function subset(exp1, exp2): boolean {
+  const e = evaluate(["NOT", ["OR", ["NOT", exp1], exp2]]);
+  if (!isArray(e)) return !e;
+  const { vars, clauses } = booleanCnf(e);
+  return !naiveDpll(clauses, vars);
+}
+
+export function extractParams(exp): Expression[] {
+  const params = new Set<Expression>();
+  map(exp, e => {
+    if (isArray(e) && e[0] === "PARAM") params.add(e[1]);
+    return e;
+  });
+  return Array.from(params);
+}

lib/common/memoize.ts

@@ -0,0 +1,63 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+let cache1 = new Map();
+let cache2 = new Map();
+const keys = new WeakMap();
+
+function getKey(obj): string {
+  if (obj === null) return "null";
+  else if (obj === undefined) return "undefined";
+
+  const t = typeof obj;
+  if (t === "number" || t === "boolean" || t === "string") return `${t}:${obj}`;
+  if (t !== "function" && t !== "object")
+    throw new Error(`Cannot memoize ${t} arguments`);
+
+  let k = keys.get(obj);
+  if (!k) {
+    const rnd = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER);
+    k = `${t}:${rnd.toString(36)}`;
+    keys.set(obj, k);
+  }
+  return k;
+}
+
+export default function memoize<T extends Function>(func: T): T {
+  const funcKey = getKey(func);
+  return ((...args) => {
+    const key = JSON.stringify(args.map(getKey)) + funcKey;
+
+    if (cache1.has(key)) return cache1.get(key);
+
+    let r;
+    if (cache2.has(key)) r = cache2.get(key);
+    else r = func(...args);
+    cache1.set(key, r);
+    return r;
+  }) as any;
+}
+
+const interval = setInterval(() => {
+  cache2 = cache1;
+  cache1 = new Map();
+}, 120000);
+
+// Don't hold Node.js process
+if (interval.unref) interval.unref();

lib/common/path-set.ts

@@ -0,0 +1,130 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Path from "./path";
+
+export default class PathSet {
+  private lengthIndex: Set<Path>[];
+  private fragmentIndex: Map<string, Set<Path>>[];
+  private stringIndex: Map<string, Path>;
+
+  public constructor() {
+    this.lengthIndex = [];
+    this.fragmentIndex = [];
+    this.stringIndex = new Map();
+  }
+
+  public get depth(): number {
+    return this.lengthIndex.length;
+  }
+
+  public add(path: Path): Path {
+    if (path.alias) throw new Error("PathSet does not support aliased paths");
+    const p = this.get(path);
+
+    if (p) return p;
+
+    this.stringIndex.set(path.toString(), path);
+
+    while (this.lengthIndex.length <= path.length) {
+      this.lengthIndex.push(new Set());
+      // fragmentIndex is one less than lengthIndex
+      if (this.lengthIndex.length > 1) this.fragmentIndex.push(new Map());
+    }
+
+    const lengthIndex = this.lengthIndex[path.length];
+    lengthIndex.add(path);
+
+    for (let i = 0; i < path.length; ++i) {
+      const fragment = path.segments[i] as string;
+      const fragmentIndex = this.fragmentIndex[i];
+
+      let fragmentIndexSet = fragmentIndex.get(fragment);
+      if (!fragmentIndexSet) {
+        fragmentIndexSet = new Set<Path>();
+        fragmentIndex.set(fragment, fragmentIndexSet);
+      }
+
+      fragmentIndexSet.add(path);
+    }
+
+    return path;
+  }
+
+  public get(path: Path): Path {
+    return this.stringIndex.get(path.toString()) || null;
+  }
+
+  public find(
+    path: Path,
+    superset: boolean = false,
+    subset: boolean = false,
+    depth: number = path.length
+  ): Path[] {
+    if (path.alias) throw new Error("PathSet does not support aliased paths");
+
+    const len = path.length;
+
+    if (!superset && depth === len && (!subset || !path.wildcard)) {
+      const p = this.get(path);
+      return p ? [p] : [];
+    }
+
+    const lengthIndex = this.lengthIndex.slice(len, depth + 1);
+    if (!lengthIndex.length) return [];
+
+    let res;
+    for (let i = len - 1; i >= 0; --i) {
+      let fragmentIndexSet2;
+      const fragmentIndex = this.fragmentIndex[i];
+
+      if ((path.wildcard >> i) & 1) {
+        if (subset) continue;
+      } else if (superset) {
+        fragmentIndexSet2 = fragmentIndex.get("*");
+      }
+
+      const fragment = path.segments[i] as string;
+      const fragmentIndexSet1 = fragmentIndex.get(fragment);
+
+      if (!fragmentIndexSet1) {
+        if (!fragmentIndexSet2) return [];
+        if (!res) res = [...fragmentIndexSet2];
+        else res = res.filter(r => fragmentIndexSet2.has(r));
+      } else if (!fragmentIndexSet2) {
+        if (!res) res = [...fragmentIndexSet1];
+        else res = res.filter(r => fragmentIndexSet1.has(r));
+      } else {
+        if (!res) {
+          res = [...fragmentIndexSet1, ...fragmentIndexSet2];
+        } else {
+          res = res.filter(
+            r => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r)
+          );
+        }
+      }
+      if (!res.length) return res;
+    }
+
+    if (!res) res = [].concat(...lengthIndex.map(a => [...a]));
+    else res = res.filter(r => lengthIndex.some(a => a.has(r)));
+
+    return res;
+  }
+}

lib/common/path.ts

@@ -0,0 +1,237 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+type Segments = (string | Alias)[];
+type Alias = [Path, string][];
+
+let cache1 = new Map<string, Path>();
+let cache2 = new Map<string, Path>();
+
+export default class Path {
+  public readonly segments: Segments;
+  public readonly wildcard: number;
+  public readonly alias: number;
+  protected _string: string;
+  protected _stringIndex: number[];
+
+  protected static parseAlias(
+    pattern: string,
+    index: number
+  ): { index: number; alias: Alias } {
+    const alias: Alias = [];
+    while (index < pattern.length && pattern[index] !== "]") {
+      const { index: idx, segments } = Path.parsePath(pattern, index);
+      let j = (index = idx + 1);
+      while (pattern[j] !== "]" && pattern[j] !== ",") {
+        if (pattern[j] === '"' && index === j) {
+          ++j;
+          while (pattern[j] !== '"' || pattern[j - 1] === "\\") {
+            if (++j >= pattern.length)
+              throw new Error("Invalid alias expression");
+          }
+        }
+        if (++j >= pattern.length) throw new Error("Invalid alias expression");
+      }
+
+      let value = pattern.slice(index, j).trim();
+      index = j;
+      if (value[0] === '"') {
+        try {
+          value = JSON.parse(value);
+        } catch (error) {
+          throw new Error("Invalid alias expression");
+        }
+      }
+
+      alias.push([new Path(segments), value]);
+      if (pattern[index] === ",") ++index;
+    }
+
+    // Ensure identical expressions have idential string representation
+    alias.sort((a, b) => {
+      if (a[0].toString() > b[0].toString()) return 1;
+      else if (a[0].toString() < b[0].toString()) return -1;
+      else if (a[1] > b[1]) return 1;
+      else if (a[1] < b[1]) return -1;
+      else return 0;
+    });
+
+    Object.freeze(alias);
+    return { index, alias };
+  }
+
+  protected static parsePath(
+    pattern: string,
+    index: number
+  ): { index: number; segments: Segments } {
+    const segments = [];
+    // Colon separator is needed for parseAlias
+    if (index < pattern.length && pattern[index] !== ":") {
+      for (;;) {
+        if (pattern[index] === "[") {
+          const { index: idx, alias } = Path.parseAlias(pattern, index + 1);
+          index = idx + 1;
+          segments.push(alias);
+        } else {
+          const j = index;
+          while (
+            index < pattern.length &&
+            pattern[index] !== ":" &&
+            pattern[index] !== "."
+          )
+            ++index;
+          const s = pattern.slice(j, index).trim();
+          segments.push(s);
+        }
+
+        if (index >= pattern.length || pattern[index] === ":") break;
+        else if (pattern[index] !== ".")
+          throw new Error("Invalid alias expression");
+        ++index;
+      }
+    }
+
+    Object.freeze(segments);
+    return { index, segments };
+  }
+
+  protected constructor(segments: Segments) {
+    let alias = 0;
+    let wildcard = 0;
+    const arr = segments.map((s, i) => {
+      if (Array.isArray(s)) {
+        alias |= 1 << i;
+        const parts = s.map(
+          al => `${al[0].toString()}:${JSON.stringify(al[1])}`
+        );
+        return `[${parts.join(",")}]`;
+      } else if (s === "*") {
+        wildcard |= 1 << i;
+      }
+      return s;
+    });
+
+    let offset = 0;
+    const stringIndex = arr.map((s, i) => (offset += s.length) + i);
+
+    this.segments = segments;
+    this.wildcard = wildcard;
+    this.alias = alias;
+    this._string = arr.join(".");
+    this._stringIndex = stringIndex;
+  }
+
+  public static parse(str: string): Path {
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        const { segments } = Path.parsePath(str, 0);
+        path = new Path(segments);
+        if (path.toString() !== str) cache1.set(path.toString(), path);
+      }
+      cache1.set(str, path);
+    }
+    return path;
+  }
+
+  public get length(): number {
+    return this.segments.length;
+  }
+
+  public toString(): string {
+    return this._string;
+  }
+
+  public slice(start: number = 0, end: number = this.segments.length): Path {
+    if (start < 0) start = Math.max(0, this.segments.length + start);
+    if (end < 0) end = Math.max(0, this.segments.length + end);
+
+    let str;
+    if (start >= end) {
+      str = "";
+    } else {
+      const i1 = start > 0 ? this._stringIndex[start - 1] + 1 : 0;
+      const i2 =
+        end <= this.segments.length
+          ? this._stringIndex[end - 1]
+          : this._string.length;
+      str = this._string.slice(i1, i2);
+    }
+
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        const segments = this.segments.slice(start, end);
+        Object.freeze(segments);
+        path = new Path(segments);
+      }
+      cache1.set(str, path);
+    }
+
+    return path;
+  }
+
+  public concat(path2: Path): Path {
+    if (!path2._string) return this;
+    else if (!this._string) return path2;
+
+    const str = `${this._string}.${path2._string}`;
+
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        const segments = this.segments.concat(path2.segments);
+        Object.freeze(segments);
+        path = new Path(segments);
+      }
+      cache1.set(str, path);
+    }
+
+    return path;
+  }
+
+  public stripAlias(): Path {
+    if (!this.alias) return this;
+    const segments = this.segments.map(s => (Array.isArray(s) ? "*" : s));
+    const str = segments.join(".");
+
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        Object.freeze(segments);
+        path = new Path(segments);
+      }
+      cache1.set(str, path);
+    }
+
+    return path;
+  }
+}
+
+const interval = setInterval(() => {
+  cache2 = cache1;
+  cache1 = new Map();
+}, 120000);
+
+// Don't hold Node.js process
+if (interval.unref) interval.unref();

lib/common/sat-solver.ts

@@ -0,0 +1,175 @@
+/**
+ * Naive DPLL solver for propositional logic adapted from
+ * https://github.com/tammet/logictools
+ *
+ * Origianl code by Tanel Tammet released under the MIT license.
+ *
+ * Check whether a clause set is satisfiable or contradictory, using a naive
+ * implementation of the dpll (Davis-Putnam-Logemann-Loveland) procedure
+ * combining truth table search with the resolution-like unit propagation.
+ *
+ * The standard pure-literal rule is not implemented in this naive version,
+ * since it is not really crucial for the algorithm.
+ *
+ * The code is very similar to the pure table solver, except we use a
+ * unit-propagating, derived-varlist-returing satisfiableAt.
+ */
+
+/* eslint-disable @typescript-eslint/no-use-before-define */
+
+function satisfiableAt(clauses, varVals, varnr, val): boolean {
+  if (varnr !== 0) varVals[varnr] = val;
+
+  const propRes = unitPropagate(clauses, varVals);
+  if (propRes === 1) {
+    varVals[varnr] = 0;
+    return true;
+  }
+
+  // if result false (-1) unitPropagate restores the varVals state itself
+  if (propRes === -1) {
+    varVals[varnr] = 0;
+    return false;
+  }
+
+  // find next unassigned var to split
+  let nextVar = 0;
+  for (let i = 1; i < varVals.length; i++) {
+    if (varVals[i] === 0) {
+      nextVar = i;
+      break;
+    }
+  }
+
+  if (nextVar === 0) throw new Error("Error in satisfiableAt");
+
+  if (
+    satisfiableAt(clauses, varVals, nextVar, 1) ||
+    satisfiableAt(clauses, varVals, nextVar, -1)
+  )
+    // not necessary to restore split var: search is halted anyway
+    return true;
+
+  // restore split var
+  varVals[nextVar] = 0;
+
+  // restore derived vars
+  for (let i = 0; i < propRes.length; i++) varVals[propRes[i]] = 0;
+
+  return false;
+}
+
+/**
+ * Check whether clauses are satisfiable under the varVals assignment, without
+ * search.
+ *
+ * Iterate through all clauses and check whether
+ *
+ *  - all literals in the clause are assigned false: undo derived units (set
+ *    values in varVals to 0) and return false immediately
+ *
+ *  - a clause contains a literal assigned true: skip further actions on this
+ *    clause
+ *
+ *  - a single literal in the clause is unassigned, all the rest are assigned
+ *    false: derive this literal as a unit clause.
+ *
+ * In case at least one literal is derived during the process above, iterate
+ * the whole process until no more new literals are derived.
+ *
+ * Returns:
+ *
+ *   1: all clauses were assigned true, hence the clause set is satisfiable.
+ *   The whole search will terminate after that.
+ *
+ *   -1: clause set is contradictory. Derived units have been restored.
+ *
+ *   a list of derived variables:  the status of the clause set is
+ *   undetermined. The values of variables on the list will be restored (set to
+ *   0) during backtracking.
+ */
+function unitPropagate(clauses, varVals): number[] | 1 | -1 {
+  // all vars with a value derived in the forthcoming main loop
+  const derivedVars = [];
+
+  // the main loop is run, deriving new unit clauses, until either:
+  // - the clause set is detected to be unsatisfiable
+  // - the clause set is detected to be satisfiable
+  // - there are no more derived unit clauses
+
+  for (;;) {
+    let allClausesFound = true; // set to false if we find an undetermined clause
+    const derivedQueue = []; // derived units (literals) during one iteration
+    for (let i = 0; i < clauses.length; i++) {
+      const clause = clauses[i];
+      let clauseVal = 0;
+      let unassignedCount = 0; // count unassigned vars in clause
+      let unassignedLit = 0; // 0 means none found for this clause
+
+      for (let j = 0; j < clause.length; j++) {
+        const lit = clause[j];
+        let nr, polarity;
+        if (lit < 0) {
+          nr = 0 - lit;
+          polarity = -1;
+        } else {
+          nr = lit;
+          polarity = 1;
+        }
+
+        if (varVals[nr] === polarity) {
+          clauseVal = 1;
+          break;
+        } else if (varVals[nr] === 0) {
+          unassignedCount++;
+          unassignedLit = lit;
+        }
+      }
+
+      if (clauseVal === 1) continue; // clause is subsumed
+
+      // clause is not subsumed by varVals
+      if (unassignedCount === 0) {
+        for (let j = 0; j < derivedVars.length; j++)
+          varVals[derivedVars[j]] = 0; // restore derived
+        return -1;
+      } else if (unassignedCount === 1) {
+        // unassignedLit is a derived literal
+        derivedQueue.push(unassignedLit);
+      }
+
+      if (unassignedCount !== 0) allClausesFound = false;
+    }
+
+    // if all clauses were subsumed or all the variables in a clause had a value
+    // no need to restore varVals: whole search finished
+    if (allClausesFound) return 1;
+
+    // in case there are any derived units present, store to varVals and
+    // iterate the main loop
+    if (derivedQueue.length === 0) break;
+
+    // some units were derived: iterate the main loop
+    for (let j = 0; j < derivedQueue.length; j++) {
+      const lit = derivedQueue[j];
+      let nr, polarity;
+      if (lit < 0) {
+        nr = 0 - lit;
+        polarity = -1;
+      } else {
+        nr = lit;
+        polarity = 1;
+      }
+      varVals[nr] = polarity;
+      derivedVars.push(nr);
+    }
+  }
+
+  return derivedVars;
+}
+
+export function naiveDpll(clauses, maxvarnr): boolean {
+  // variable values are 0 if not set, 1 if positive, -1 if negative
+  const varVals = new Int32Array(maxvarnr + 1);
+  return satisfiableAt(clauses, varVals, 0, 0);
+}

lib/config.coffee

@@ -1,171 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-path = require 'path'
-common = require './common'
-
-options = {
-  CONFIG_DIR : {type : 'string', default : 'config'},
-  MONGODB_CONNECTION_URL : {type : 'string', default : 'mongodb://127.0.0.1/genieacs'},
-  REDIS_PORT : {type : 'int', default : 6379},
-  REDIS_HOST : {type : 'string', default : '127.0.0.1'},
-  REDIS_DB : {type : 'int', default : 0},
-
-  CWMP_WORKER_PROCESSES : {type : 'int', default : 4},
-  CWMP_PORT : {type : 'int', default : 7547},
-  CWMP_INTERFACE : {type : 'string', default : '0.0.0.0'},
-  CWMP_SSL : {type : 'bool', default : false},
-
-  NBI_WORKER_PROCESSES : {type : 'int', default : 2},
-  NBI_PORT : {type : 'int', default : 7557},
-  NBI_INTERFACE : {type : 'string', default : '0.0.0.0'},
-  NBI_SSL : {type : 'bool', default : false},
-
-  FS_WORKER_PROCESSES : {type : 'int', default : 2},
-  FS_PORT : {type : 'int', default : 7567},
-  FS_INTERFACE : {type : 'string', default : '0.0.0.0'},
-  FS_SSL : {type : 'bool', default : false},
-  FS_IP : {type : 'string', default : '192.168.0.1'},
-
-  PRESETS_CACHE_DURATION : {type : 'int', default : 86400},
-  LOG_INFORMS : {type : 'bool', default : true},
-  DEBUG : {type : 'bool', default : false},
-  RETRY_DELAY : {type : 'int', default : 300},
-  IGNORE_XML_NAMESPACES : {type : 'bool', default : false},
-  SESSION_TIMEOUT : {type : 'int', default : 30},
-  GET_PARAMETER_NAMES_DEPTH_THRESHOLD : {type : 'int', default : 0},
-  TASK_PARAMETERS_BATCH_SIZE : {type : 'int', default : 32},
-  COOKIES_PATH : {type : 'string'},
-
-  # Libxml related configuration
-  XML_PARSE_RECOVER : {type : 'bool'},
-  XML_PARSE_NOENT : {type : 'bool'},
-  XML_PARSE_NOBLANKS : {type : 'bool'},
-  XML_PARSE_NSCLEAN : {type : 'bool'},
-  XML_PARSE_NOCDATA : {type : 'bool'},
-  XML_PARSE_IGNORE_ENC : {type : 'bool'},
-
-  # Should probably never be changed
-  PRESETS_TIME_PADDING : {type : 'int', default : 1},
-  DEVICE_ONLINE_THRESHOLD : {type : 'int', default : 4000}
-}
-
-allConfig = {}
-
-
-setConfig = (name, value, commandLineArgument) ->
-  return true if allConfig[name]?
-
-  cast = (val, type) ->
-    switch type
-      when 'int'
-        Number(val)
-      when 'bool'
-        String(val).trim().toLowerCase() in ['true', 'on', 'yes', '1']
-      when 'string'
-        String(val)
-      else
-        null
-
-  _value = null
-
-  for optionName, optionDetails of options
-    n = optionName
-    if commandLineArgument
-      n = n.toLowerCase().replace(/_/g, '-')
-
-    if name == n
-      _value = cast(value, optionDetails.type)
-      n = optionName
-    else if common.startsWith(name, "#{n}-")
-      _value = cast(value, optionDetails.type)
-      n = "#{optionName}-#{name[optionName.length+1..]}"
-
-    if _value?
-      allConfig[n] = _value
-      # Save as environmnet variable to pass on to any child processes
-      process.env[n] = _value
-      return true
-
-  return false
-
-
-# Command line arguments
-exports.argv = []
-argv = process.argv[2..]
-while argv.length
-  arg = argv.shift()
-  if arg[0] == '-'
-    v = argv.shift()
-    exports.argv[arg] = v
-    setConfig(arg[2..], v, true)
-  else
-    exports.argv.push(arg)
-
-
-# Environment variable
-for k, v of process.env
-  continue if k.lastIndexOf('GENIEACS_', 0) != 0
-  k = k[9..] # remove "GENIEACS_" prefix
-  setConfig(k, v)
-
-
-# Find config dir
-if exports.argv['--config-dir']?
-  allConfig.CONFIG_DIR = exports.argv['--config-dir']
-else if process.env['GENIEACS_CONFIG_DIR']?
-  allConfig.CONFIG_DIR = process.env['GENIEACS_CONFIG_DIR']
-else
-  allConfig.CONFIG_DIR = options.CONFIG_DIR.default
-
-
-# Configuration file
-for k, v of require(path.resolve(allConfig.CONFIG_DIR, 'config'))
-  setConfig(k, v)
-
-
-# Defaults
-for k, v of options
-  setConfig(k, v.default) if v.default?
-
-
-get = (option, deviceId) ->
-  if deviceId?
-    name = "#{option}-#{deviceId}"
-    v = allConfig[name]
-    return v if v?
-    i = name.lastIndexOf('-')
-    v = allConfig[name[0...i]]
-    return v if v?
-    i = name.lastIndexOf('-', i-1)
-    v = allConfig[name[0...i]]
-    return v if v? or i == -1
-
-  return allConfig[option]
-
-
-# load parameter configurations
-exports.PARAMETERS = require(path.resolve(allConfig.CONFIG_DIR, 'parameters'))
-
-# load authentication scripts
-exports.auth = require(path.resolve(allConfig.CONFIG_DIR, 'auth'))
-
-exports.CUSTOM_COMMANDS = require(path.resolve(allConfig.CONFIG_DIR, 'custom_commands'))
-
-exports.get = get
-
-exports.allConfig = allConfig

lib/config.ts

@@ -0,0 +1,321 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { resolve } from "path";
+import { readFileSync, existsSync } from "fs";
+
+// Find project root directory
+export let ROOT_DIR = resolve(__dirname, "..");
+while (!existsSync(`${ROOT_DIR}/package.json`)) {
+  const d = resolve(ROOT_DIR, "..");
+  if (d === ROOT_DIR) {
+    ROOT_DIR = process.cwd();
+    break;
+  }
+  ROOT_DIR = d;
+}
+
+// For compatibility with v1.1
+let configDir, cwmpSsl, nbiSsl, fsSsl, uiSsl;
+
+const options = {
+  EXT_DIR: { type: "path", default: resolve(ROOT_DIR, "config/ext") },
+  MONGODB_CONNECTION_URL: {
+    type: "string",
+    default: "mongodb://127.0.0.1/genieacs"
+  },
+
+  CWMP_WORKER_PROCESSES: { type: "int", default: 0 },
+  CWMP_PORT: { type: "int", default: 7547 },
+  CWMP_INTERFACE: { type: "string", default: "0.0.0.0" },
+  CWMP_SSL_CERT: { type: "string", default: "" },
+  CWMP_SSL_KEY: { type: "string", default: "" },
+  CWMP_LOG_FILE: { type: "path", default: "" },
+  CWMP_ACCESS_LOG_FILE: { type: "path", default: "" },
+  CWMP_KEEP_ALIVE_TIMEOUT: { type: "int", default: 0 },
+
+  NBI_WORKER_PROCESSES: { type: "int", default: 0 },
+  NBI_PORT: { type: "int", default: 7557 },
+  NBI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  NBI_SSL_CERT: { type: "string", default: "" },
+  NBI_SSL_KEY: { type: "string", default: "" },
+  NBI_LOG_FILE: { type: "path", default: "" },
+  NBI_ACCESS_LOG_FILE: { type: "path", default: "" },
+
+  FS_WORKER_PROCESSES: { type: "int", default: 0 },
+  FS_PORT: { type: "int", default: 7567 },
+  FS_INTERFACE: { type: "string", default: "0.0.0.0" },
+  FS_SSL_CERT: { type: "string", default: "" },
+  FS_SSL_KEY: { type: "string", default: "" },
+  FS_HOSTNAME: { type: "string", default: "" },
+  FS_LOG_FILE: { type: "path", default: "" },
+  FS_ACCESS_LOG_FILE: { type: "path", default: "" },
+
+  UI_WORKER_PROCESSES: { type: "int", default: 0 },
+  UI_PORT: { type: "int", default: 3000 },
+  UI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  UI_SSL_CERT: { type: "string", default: "" },
+  UI_SSL_KEY: { type: "string", default: "" },
+  UI_LOG_FILE: { type: "path", default: "" },
+  UI_ACCESS_LOG_FILE: { type: "path", default: "" },
+  UI_JWT_SECRET: { type: "string", default: "" },
+
+  UDP_CONNECTION_REQUEST_PORT: { type: "int", default: 0 },
+
+  DOWNLOAD_TIMEOUT: { type: "int", default: 3600 },
+  EXT_TIMEOUT: { type: "int", default: 3000 },
+  MAX_CACHE_TTL: { type: "int", default: 86400 },
+  DEBUG_FILE: { type: "path", default: "" },
+  DEBUG_FORMAT: { type: "string", default: "yaml" },
+  DEBUG: { type: "bool", default: false },
+  RETRY_DELAY: { type: "int", default: 300 },
+  SESSION_TIMEOUT: { type: "int", default: 30 },
+  CONNECTION_REQUEST_TIMEOUT: { type: "int", default: 2000 },
+  GPN_NEXT_LEVEL: { type: "int", default: 0 },
+  GPV_BATCH_SIZE: { type: "int", default: 32 },
+  MAX_DEPTH: { type: "int", default: 16 },
+  COOKIES_PATH: { type: "string" },
+  LOG_FORMAT: { type: "string", default: "simple" },
+  ACCESS_LOG_FORMAT: { type: "string", default: "" },
+  MAX_CONCURRENT_REQUESTS: { type: "int", default: 20 },
+  DATETIME_MILLISECONDS: { type: "bool", default: true },
+  BOOLEAN_LITERAL: { type: "bool", default: true },
+  CONNECTION_REQUEST_ALLOW_BASIC_AUTH: { type: "bool", default: false },
+  MAX_COMMIT_ITERATIONS: { type: "int", default: 32 },
+
+  // XML configuration
+  XML_RECOVER: { type: "bool", default: false },
+  XML_IGNORE_ENC: { type: "bool", default: false },
+  XML_FORMAT: { type: "bool", default: false },
+  XML_NO_DECL: { type: "bool", default: false },
+  XML_NO_EMPTY: { type: "bool", default: false },
+  XML_LIBXMLJS: { type: "bool", default: false },
+
+  // Should probably never be changed
+  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 }
+};
+
+const allConfig: { [name: string]: string | number } = {};
+
+function setConfig(name, value, commandLineArgument = false): boolean {
+  if (allConfig[name] != null) return true;
+
+  // For compatibility with v1.1
+  if (name === "CONFIG_DIR" || name === "config-dir")
+    configDir = configDir || resolve(ROOT_DIR, value);
+
+  if (name === "CWMP_SSL" || name === "cwmp-ssl") {
+    cwmpSsl =
+      cwmpSsl ||
+      String(value)
+        .toLowerCase()
+        .trim();
+  }
+
+  if (name === "NBI_SSL" || name === "nbi-ssl") {
+    nbiSsl =
+      nbiSsl ||
+      String(value)
+        .toLowerCase()
+        .trim();
+  }
+
+  if (name === "FS_SSL" || name === "fs-ssl") {
+    fsSsl =
+      fsSsl ||
+      String(value)
+        .toLowerCase()
+        .trim();
+  }
+
+  if (name === "UI_SSL" || name === "ui-ssl") {
+    uiSsl =
+      uiSsl ||
+      String(value)
+        .toLowerCase()
+        .trim();
+  }
+
+  // For compatibility with v1.0
+  if (name === "PRESETS_CACHE_DURATION" || name === "presets-cache-duration")
+    setConfig("MAX_CACHE_TTL", value);
+
+  if (
+    name === "GET_PARAMETER_NAMES_DEPTH_THRESHOLD" ||
+    name === "get-parameter-names-depth-threshold"
+  )
+    setConfig("GPN_NEXT_LEVEL", value);
+
+  if (
+    name === "TASK_PARAMETERS_BATCH_SIZE" ||
+    name === "task-parameters-batch-size"
+  )
+    setConfig("GPV_BATCH_SIZE", value);
+
+  if (name === "XML_PARSE_IGNORE_ENC" || name === "xml-parse-ignore-enc")
+    setConfig("XML_IGNORE_ENC", value);
+
+  if (name === "XML_PARSE_RECOVER" || name === "xml-parse-recover")
+    setConfig("XML_RECOVER", value);
+
+  if (name === "FS_IP" || name === "fs-ip") setConfig("FS_HOSTNAME", value);
+
+  function cast(val, type): string | number | boolean {
+    switch (type) {
+      case "int":
+        return Number(val);
+      case "bool":
+        return ["true", "1"].includes(
+          String(val)
+            .trim()
+            .toLowerCase()
+        );
+      case "string":
+        return String(val);
+      case "path":
+        if (!val) return "";
+        return resolve(val);
+      default:
+        return null;
+    }
+  }
+
+  let _value = null;
+  for (const [optionName, optionDetails] of Object.entries(options)) {
+    let n = optionName;
+    if (commandLineArgument) n = n.toLowerCase().replace(/_/g, "-");
+
+    if (name === n) {
+      _value = cast(value, optionDetails.type);
+      n = optionName;
+    } else if (name.startsWith(`${n}-`)) {
+      _value = cast(value, optionDetails.type);
+      n = `${optionName}-${name.slice(optionName.length + 1)}`;
+    }
+
+    if (_value != null) {
+      allConfig[n] = _value;
+      // Save as environmnet variable to pass on to any child process
+      process.env[`GENIEACS_${n}`] = _value;
+      return true;
+    }
+  }
+
+  return false;
+}
+
+// Command line arguments
+const argv = process.argv.slice(2);
+while (argv.length) {
+  const arg = argv.shift();
+  if (arg[0] === "-") {
+    const v = argv.shift();
+    setConfig(arg.slice(2), v, true);
+  }
+}
+
+// Environment variable
+for (const [k, v] of Object.entries(process.env))
+  if (k.startsWith("GENIEACS_")) setConfig(k.slice(9), v);
+
+// Configuration file
+const configFilename = configDir
+  ? `${configDir}/config.json`
+  : `${ROOT_DIR}/config/config.json`;
+
+if (existsSync(configFilename)) {
+  const configFile = JSON.parse(readFileSync(configFilename).toString());
+
+  for (const [k, v] of Object.entries(configFile)) {
+    if (!setConfig(k, v))
+      // Pass as environment variable to be accessable by extensions
+      process.env[`GENIEACS_${k}`] = `${v}`;
+  }
+}
+
+if (configDir) setConfig("EXT_DIR", `${configDir}/ext`);
+
+if (["true", "1"].includes(cwmpSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("CWMP_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("CWMP_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (["true", "1"].includes(nbiSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("NBI_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("NBI_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (["true", "1"].includes(fsSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("FS_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("FS_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (["true", "1"].includes(uiSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("UI_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("UI_SSL_KEY", `${d}/cwmp.key`);
+}
+
+// Defaults
+for (const [k, v] of Object.entries(options))
+  if (v["default"] != null) setConfig(k, v["default"]);
+
+export function get(
+  optionName: string,
+  deviceId?: string
+): string | number | boolean {
+  if (!deviceId) return allConfig[optionName];
+
+  optionName = `${optionName}-${deviceId}`;
+  let v = allConfig[optionName];
+  if (v != null) return v;
+
+  let i = optionName.lastIndexOf("-");
+  v = allConfig[optionName.slice(0, i)];
+  if (v != null) return v;
+
+  i = optionName.lastIndexOf("-", i - 1);
+  v = allConfig[optionName.slice(0, i)];
+  if (v != null) return v;
+
+  i = optionName.lastIndexOf("-", i - 1);
+  v = allConfig[optionName.slice(0, i)];
+  if (v != null) return v;
+
+  i = optionName.lastIndexOf("-", i - 1);
+  if (i > 0) {
+    v = allConfig[optionName.slice(0, i)];
+    if (v != null) return v;
+  }
+
+  return null;
+}
+
+export function getDefault(optionName): string | number | boolean {
+  const option = options[optionName];
+  if (!option) return null;
+
+  let val = option["default"];
+  if (val && option.type === "path") val = resolve(val);
+
+  return val;
+}

lib/connection-request.ts

@@ -0,0 +1,244 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as crypto from "crypto";
+import * as dgram from "dgram";
+import { parse } from "url";
+import * as http from "http";
+import { evaluateAsync } from "./common/expression";
+import { Expression } from "./types";
+import * as auth from "./auth";
+import * as extensions from "./extensions";
+import * as debug from "./debug";
+
+async function extractAuth(
+  exp: Expression,
+  context: {},
+  now: number,
+  dflt: any
+): Promise<[string, string, Expression]> {
+  let username, password;
+  const _exp = await evaluateAsync(
+    exp,
+    context,
+    now,
+    async (e: Expression): Promise<Expression> => {
+      if (!username && Array.isArray(e) && e[0] === "FUNC") {
+        if (e[1] === "EXT") {
+          if (typeof e[2] !== "string" || typeof e[3] !== "string") return null;
+
+          for (let i = 4; i < e.length; i++)
+            if (Array.isArray(e[i])) return null;
+
+          const { fault, value } = await extensions.run(e.slice(2));
+          return fault ? null : value;
+        } else if (e[1] === "AUTH") {
+          if (!Array.isArray(e[2]) && !Array.isArray(e[3])) {
+            username = e[2];
+            password = e[3];
+          }
+          return dflt;
+        }
+      }
+      return e;
+    }
+  );
+  return [username, password, _exp];
+}
+
+function httpGet(
+  options: http.RequestOptions,
+  timeout,
+  _debug: boolean,
+  deviceId: string
+): Promise<{ statusCode: number; headers: {} }> {
+  return new Promise((resolve, reject) => {
+    const req = http
+      .get(options, res => {
+        res.resume();
+        resolve({ statusCode: res.statusCode, headers: res.headers });
+        if (_debug) {
+          debug.outgoingHttpRequest(req, deviceId, options, null);
+          debug.incomingHttpResponse(res, deviceId, null);
+        }
+      })
+      .on("error", err => {
+        req.abort();
+        reject(new Error("Device is offline"));
+        if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
+      })
+      .on("socket", socket => {
+        socket.setTimeout(timeout);
+        socket.on("timeout", () => {
+          req.abort();
+        });
+      });
+  });
+}
+
+export async function httpConnectionRequest(
+  address: string,
+  authExp: Expression,
+  context: {},
+  allowBasicAuth: boolean,
+  timeout: number,
+  _debug: boolean,
+  deviceId: string
+): Promise<void> {
+  const now = Date.now();
+  const options: http.RequestOptions = parse(address);
+  if (options.protocol !== "http:")
+    throw new Error("Invalid connection request URL or protocol");
+
+  options.agent = new http.Agent({
+    maxSockets: 1,
+    keepAlive: true
+  });
+
+  let authHeader: {};
+  let username: string;
+  let password: string;
+
+  while (!authHeader || (username != null && password != null)) {
+    let opts = options;
+    if (authHeader) {
+      if (authHeader["method"] === "Basic") {
+        if (!allowBasicAuth)
+          throw new Error("Basic HTTP authentication not allowed");
+
+        opts = Object.assign(
+          {
+            headers: {
+              Authorization: auth.basic(username || "", password || "")
+            }
+          },
+          options
+        );
+      } else if (authHeader["method"] === "Digest") {
+        opts = Object.assign(
+          {
+            headers: {
+              Authorization: auth.solveDigest(
+                username,
+                password,
+                options.path,
+                "GET",
+                null,
+                authHeader
+              )
+            }
+          },
+          options
+        );
+      } else {
+        throw new Error("Unrecognized auth method");
+      }
+    }
+
+    let res = await httpGet(opts, timeout, _debug, deviceId);
+
+    // Workaround for some devices unexpectedly closing the connection
+    if (res.statusCode === 0 && authHeader)
+      res = await httpGet(opts, timeout, _debug, deviceId);
+    if (res.statusCode === 0) throw new Error("Device is offline");
+    if (res.statusCode === 200 || res.statusCode === 204) return;
+
+    if (res.statusCode === 401 && res.headers["www-authenticate"]) {
+      authHeader = auth.parseWwwAuthenticateHeader(
+        res.headers["www-authenticate"]
+      );
+      [username, password, authExp] = await extractAuth(
+        authExp,
+        context,
+        now,
+        false
+      );
+    } else {
+      throw new Error(
+        `Unexpected response code from device: ${res.statusCode}`
+      );
+    }
+  }
+  throw new Error("Incorrect connection request credentials");
+}
+
+export async function udpConnectionRequest(
+  address: string,
+  authExp: Expression,
+  context: {},
+  sourcePort: number = 0,
+  _debug: boolean,
+  deviceId: string
+): Promise<void> {
+  const [host, portStr] = address.split(":", 2);
+  const port = portStr ? parseInt(portStr) : 80;
+  const now = Date.now();
+
+  const client = dgram.createSocket({ type: "udp4", reuseAddr: true });
+  // When a device is NAT'ed, the UDP Connection Request must originate from
+  // the same address and port used by the STUN server, in order to traverse
+  // the firewall. This does require that the Genieacs NBI and STUN server
+  // are allowed to bind to the same address and port. The STUN server needs
+  // to open its UDP port with the SO_REUSEADDR option, allowing the NBI to
+  // also bind to the same port.
+  if (sourcePort) client.bind({ port: sourcePort, exclusive: true });
+
+  let username: string;
+  let password: string;
+
+  [username, password, authExp] = await extractAuth(
+    authExp,
+    context,
+    now,
+    null
+  );
+
+  if (username == null) username = "";
+  if (password == null) password = "";
+  while (username != null && password != null) {
+    const ts = Math.trunc(now / 1000);
+    const id = Math.trunc(Math.random() * 4294967295);
+    const cn = crypto.randomBytes(8).toString("hex");
+    const sig = crypto
+      .createHmac("sha1", password)
+      .update(`${ts}${id}${username}${cn}`)
+      .digest("hex");
+    const uri = `http://${address}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
+    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${address}\r\n\r\n`;
+    const message = Buffer.from(msg);
+
+    for (let i = 0; i < 3; ++i) {
+      await new Promise((resolve, reject) => {
+        client.send(message, 0, message.length, port, host, (err: Error) => {
+          if (err) reject(err);
+          else resolve();
+          if (_debug) debug.outgoingUdpMessage(host, deviceId, port, msg);
+        });
+      });
+    }
+
+    [username, password, authExp] = await extractAuth(
+      authExp,
+      context,
+      now,
+      null
+    );
+  }
+  client.close();
+}

lib/custom-commands.coffee

@@ -1,52 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-config = require './config'
-common = require './common'
-
-
-exports.execute = (deviceId, command, callback) ->
-  args = command.split(/\s+/)
-  f = args.shift()
-  c = args.shift()
-
-  file = require "../config/custom_commands/#{f}"
-  file[c](deviceId, args.join(' '), callback)
-
-
-exports.getFileCommands = (filename) ->
-  commands = []
-  f = require("../config/custom_commands/#{filename}")
-  for k,v of f
-    commands.push(k)
-  return commands
-
-
-exports.getDeviceCustomCommands = (deviceId) ->
-  commands = {}
-  for k,v of config.CUSTOM_COMMANDS
-    if eval(v).test(deviceId)
-      commands[k] = exports.getFileCommands(k)
-  return commands
-
-
-exports.getDeviceCustomCommandNames = (deviceId) ->
-  commands = []
-  for k,v of config.CUSTOM_COMMANDS
-    if eval(v).test(deviceId)
-      commands.push(k)
-  return commands

lib/cwmp.coffee

@@ -1,564 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This file incorporates work covered by the following copyright and
-# permission notice:
-#
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-config = require './config'
-common = require './common'
-util = require 'util'
-soap = require './soap'
-tasks = require './tasks'
-db = require './db'
-presets = require './presets'
-mongodb = require 'mongodb'
-apiFunctions = require './api-functions'
-customCommands = require './custom-commands'
-crypto = require 'crypto'
-zlib = require 'zlib'
-updateDevice = require './update-device'
-
-
-# Used to reject new TR-069 sessions when under load
-holdUntil = Date.now()
-
-
-writeResponse = (currentRequest, res) ->
-  if config.get('DEBUG', currentRequest.session.deviceId)
-    dump = "# RESPONSE #{new Date(Date.now())}\n" + JSON.stringify(res.headers) + "\n#{res.data}\n\n"
-    require('fs').appendFile("./debug/#{currentRequest.session.deviceId}.dump", dump, (err) ->
-      throw err if err
-    )
-
-  # respond using the same content-encoding as the request
-  if currentRequest.httpRequest.headers['content-encoding']? and res.data.length > 0
-    switch currentRequest.httpRequest.headers['content-encoding']
-      when 'gzip'
-        res.headers['Content-Encoding'] = 'gzip'
-        compress = zlib.gzip
-      when 'deflate'
-        res.headers['Content-Encoding'] = 'deflate'
-        compress = zlib.deflate
-
-  if compress?
-    compress(res.data, (err, data) ->
-      res.headers['Content-Length'] = data.length
-      currentRequest.httpResponse.writeHead(res.code, res.headers)
-      currentRequest.httpResponse.end(data)
-    )
-  else
-    res.headers['Content-Length'] = res.data.length
-    currentRequest.httpResponse.writeHead(res.code, res.headers)
-    currentRequest.httpResponse.end(res.data)
-
-
-endSession = (currentRequest) ->
-  db.redisClient.del("session_#{currentRequest.sessionId}", (err, res) ->
-    throw err if err
-    res = soap.response(null)
-    writeResponse(currentRequest, res)
-  )
-
-
-inform = (currentRequest, cwmpRequest) ->
-  # If overloaded, ask CPE to retry in 5 mins
-  if Date.now() < holdUntil
-    res = {code : 503, headers : {'Retry-After' : 300}, data : ''}
-    return writeResponse(currentRequest, res)
-
-  now = new Date(Date.now())
-
-  if config.get('LOG_INFORMS', currentRequest.session.deviceId)
-    util.log("#{currentRequest.session.deviceId}: Inform (#{cwmpRequest.methodRequest.event}); retry count #{cwmpRequest.methodRequest.retryCount}")
-
-  parameterNames = (p[0] for p in cwmpRequest.methodRequest.parameterList)
-
-  informHash = crypto.createHash('md5').update(JSON.stringify(parameterNames)).update(JSON.stringify(config.CUSTOM_COMMANDS)).digest('hex')
-
-  deviceUpdates = {parameterValues : cwmpRequest.methodRequest.parameterList, informEvents : cwmpRequest.methodRequest.event}
-
-  # Delete all pending tasks on bootstrap (e.g. factory reset)
-  if '0 BOOTSTRAP' in cwmpRequest.methodRequest.event
-    lastBootstrap = now
-    db.tasksCollection.remove({device : currentRequest.session.deviceId}, (err, removed) ->
-      throw err if err
-    )
-
-  db.redisClient.get("#{currentRequest.session.deviceId}_inform_hash", (err, oldInformHash) ->
-    throw err if err
-
-    updateAndRespond = () ->
-      updateDevice.commitUpdates(currentRequest.session.deviceId, deviceUpdates, false, (err) ->
-        throw err if err
-        db.redisClient.setex("session_#{currentRequest.sessionId}", currentRequest.session.sessionTimeout, JSON.stringify(currentRequest.session), (err) ->
-          throw err if err
-          res = soap.response({
-            id : cwmpRequest.id,
-            methodResponse : {type : 'InformResponse'},
-            cwmpVersion : cwmpRequest.cwmpVersion
-          })
-
-          if !!cookiesPath = config.get('COOKIES_PATH', currentRequest.session.deviceId)
-            res.headers['Set-Cookie'] = "session=#{currentRequest.sessionId}; Path=#{cookiesPath}"
-          else
-            res.headers['Set-Cookie'] = "session=#{currentRequest.sessionId}"
-
-          writeResponse(currentRequest, res)
-        )
-      )
-
-    if oldInformHash == informHash and not lastBootstrap?
-      return updateAndRespond()
-
-    db.redisClient.setex("#{currentRequest.session.deviceId}_inform_hash", config.get('PRESETS_CACHE_DURATION', currentRequest.session.deviceId), informHash, (err, res) ->
-      throw err if err
-    )
-
-    # populate projection and parameterStructure
-    parameterStructure = {}
-    projection = {}
-    projection._customCommands = 1
-    projection._timestamp = 1
-    projection._lastBootstrap = 1
-    for param in parameterNames
-      ref = parameterStructure
-      path = ''
-      for p in param.split('.')
-        ref[p] ?= {_path : path}
-        ref = ref[p]
-        path += p + '.'
-        projection[path + '_timestamp'] = 1
-
-    db.devicesCollection.findOne({'_id' : currentRequest.session.deviceId}, projection, (err, device) ->
-      throw err if err
-      lastBootstrap ?= device?._lastBootstrap
-      _tasks = []
-
-      # For any parameters that aren't in the DB model, issue a refresh task
-      traverse = (reference, actual) ->
-        for k of reference
-          continue if k[0] == '_'
-          if not actual[k]?._timestamp? or actual[k]._timestamp < lastBootstrap
-            _tasks.push({device: currentRequest.session.deviceId, name : 'refreshObject', objectName : reference[k]._path})
-          else
-            traverse(reference[k], actual[k])
-
-      if device?
-        traverse(parameterStructure, device)
-
-        # Update custom commands if needed
-        deviceCustomCommands = {}
-        deviceCustomCommands[k] = v._timestamp for k,v of device._customCommands
-
-        for cmd in customCommands.getDeviceCustomCommandNames(currentRequest.session.deviceId)
-          if not (deviceCustomCommands[cmd]?._timestamp < lastBootstrap)
-            _tasks.push({device: currentRequest.session.deviceId, name: 'customCommand', command: "#{cmd} init"})
-          delete deviceCustomCommands[cmd]
-
-        for cmd of deviceCustomCommands
-          deviceUpdates.deletedObjects ?= []
-          deviceUpdates.deletedObjects.push("_customCommands.#{cmd}")
-
-        # clear presets hash if parameters are potentially modified
-        if _tasks.length > 0
-          db.redisClient.del("#{currentRequest.session.deviceId}_presets_hash", (err) ->
-            throw err if err
-          )
-
-        apiFunctions.insertTasks(_tasks, {}, () ->
-          updateAndRespond()
-        )
-      else
-        deviceIdDetails = {}
-        for k, v of cwmpRequest.methodRequest.deviceId
-          deviceIdDetails["_#{k}"] = v
-
-        db.devicesCollection.insert({_id : currentRequest.session.deviceId, _registered : now, _deviceId : deviceIdDetails}, (err) ->
-          throw err if err
-          util.log("#{currentRequest.session.deviceId}: New device registered")
-          _tasks.push({device: currentRequest.session.deviceId, name : 'refreshObject', objectName : ''})
-
-          for cmd in customCommands.getDeviceCustomCommandNames(currentRequest.session.deviceId)
-            _tasks.push({device: currentRequest.session.deviceId, name: 'customCommand', command: "#{cmd} init"})
-
-          apiFunctions.insertTasks(_tasks, {}, () ->
-            updateAndRespond()
-          )
-        )
-    )
-  )
-
-
-runTask = (currentRequest, task, methodResponse) ->
-  timeDiff = process.hrtime()
-  tasks.task(task, methodResponse, (err, status, methodRequest, deviceUpdates) ->
-    throw err if err
-
-    timeDiff = process.hrtime(timeDiff)[0] + 1
-    if timeDiff > 3 # in seconds
-      # Server under load. Hold new sessions temporarily.
-      holdUntil = Math.max(Date.now() + timeDiff * 2000, holdUntil)
-
-    save = status & tasks.STATUS_SAVE
-    switch status & ~tasks.STATUS_SAVE
-      when tasks.STATUS_OK
-        f = () ->
-          db.redisClient.setex(String(task._id), currentRequest.session.sessionTimeout, JSON.stringify(task), (err) ->
-            throw err if err
-            res = soap.response({
-              id : task._id,
-              methodRequest : methodRequest,
-              cwmpVersion : currentRequest.session.cwmpVersion
-            })
-            writeResponse(currentRequest, res)
-            updateSessionExpiry(currentRequest)
-          )
-
-        if save
-          updateDevice.commitUpdates(currentRequest.session.deviceId, deviceUpdates, true, (err) ->
-            throw err if err
-            db.tasksCollection.update({_id : mongodb.ObjectID(String(task._id))}, {$set : {session : task.session}}, (err) ->
-              throw err if err
-              f()
-            )
-          )
-        else
-          updateDevice.queueUpdates(currentRequest.session.deviceId, deviceUpdates, (err) ->
-            f()
-          )
-      when tasks.STATUS_COMPLETED
-        util.log("#{currentRequest.session.deviceId}: Completed task #{task.name}(#{task._id})")
-        updateDevice.commitUpdates(currentRequest.session.deviceId, deviceUpdates, true, (err) ->
-          throw err if err
-          db.tasksCollection.remove({'_id' : mongodb.ObjectID(String(task._id))}, (err, removed) ->
-            throw err if err
-            db.redisClient.del(String(task._id), (err, res) ->
-              throw err if err
-              nextTask(currentRequest)
-            )
-          )
-        )
-      when tasks.STATUS_FAULT
-        retryAfter = config.get('RETRY_DELAY', currentRequest.session.deviceId) * Math.pow(2, task.retries ? 0)
-        util.log("#{currentRequest.session.deviceId}: Fault response for task #{task._id}. Retrying after #{retryAfter} seconds.")
-        taskUpdate = {fault : task.fault, timestamp : new Date(Date.now() + retryAfter * 1000)}
-        if save
-          taskUpdate.session = task.session
-
-        db.tasksCollection.update({_id : mongodb.ObjectID(String(task._id))}, {$set : taskUpdate, $inc : {retries : 1}}, (err) ->
-          throw err if err
-          nextTask(currentRequest)
-        )
-      else
-        throw new Error('Unknown task status')
-  )
-
-
-isTaskExpired = (task) ->
-  task.expiry <= new Date()
-
-
-assertPresets = (currentRequest) ->
-  db.redisClient.mget("#{currentRequest.session.deviceId}_presets_hash", 'presets_hash', (err, res) ->
-    throw err if err
-    devicePresetsHash = res[0]
-    presetsHash = res[1]
-    if devicePresetsHash? and devicePresetsHash == presetsHash
-      # no discrepancy, end the session
-      endSession(currentRequest)
-    else
-      PRESETS_CACHE_DURATION = config.get('PRESETS_CACHE_DURATION', currentRequest.session.deviceId)
-      db.getPresetsObjectsAliases((allPresets, allObjects, allAliases) ->
-        if not presetsHash?
-          presetsHash = presets.calculatePresetsHash(allPresets, allObjects)
-          db.redisClient.setex('presets_hash', PRESETS_CACHE_DURATION, presetsHash, (err, res) ->
-            throw err if err
-          )
-
-        presets.getDevicePreset(currentRequest.session.deviceId, allPresets, allObjects, allAliases, (devicePreset) ->
-          presets.processDevicePreset(currentRequest.session.deviceId, devicePreset, (taskList, addTags, deleteTags, expiry) ->
-            db.redisClient.setex("#{currentRequest.session.deviceId}_presets_hash", Math.floor(Math.max(1, expiry - config.get('PRESETS_TIME_PADDING'))), presetsHash, (err, res) ->
-              throw err if err
-            )
-
-            if addTags.length + deleteTags.length + taskList.length
-              util.log("#{currentRequest.session.deviceId}: Presets discrepancy found")
-
-            if addTags.length + deleteTags.length
-              util.log("#{currentRequest.session.deviceId}: Updating tags +(#{addTags}) -(#{deleteTags})")
-
-            if deleteTags.length
-              db.devicesCollection.update({'_id' : currentRequest.session.deviceId}, {'$pull' : {'_tags' : {'$in' : deleteTags}}}, {}, (err, count) ->
-                throw err if err
-              )
-
-            if addTags.length
-              db.devicesCollection.update({'_id' : currentRequest.session.deviceId}, {'$addToSet' : {'_tags' : {'$each' : addTags}}}, {}, (err, count) ->
-                throw err if err
-              )
-
-            if taskList.length
-              t.expiry = expiry for t in taskList
-              apiFunctions.insertTasks(taskList, allAliases, (err, taskList) ->
-                throw err if err
-                task = taskList[0]
-                util.log("#{currentRequest.session.deviceId}: Started task #{task.name}(#{task._id})")
-                runTask(currentRequest, task, {})
-              )
-            else
-              endSession(currentRequest)
-          )
-        )
-      )
-  )
-
-
-nextTask = (currentRequest) ->
-  now = new Date()
-  cur = db.tasksCollection.find({'device' : currentRequest.session.deviceId, timestamp : {$lte : now}}).sort(['timestamp']).limit(1)
-  cur.nextObject((err, task) ->
-    throw err if err
-
-    if not task
-      # no more tasks, check presets discrepancy
-      assertPresets(currentRequest)
-    else if isTaskExpired(task)
-      util.log("#{currentRequest.session.deviceId}: Task is expired #{task.name}(#{task._id})")
-      db.tasksCollection.remove({'_id' : mongodb.ObjectID(String(task._id))}, {safe: true}, (err, removed) ->
-        throw err if err
-        nextTask(currentRequest)
-      )
-    else
-      util.log("#{currentRequest.session.deviceId}: Started task #{task.name}(#{task._id})")
-      updateDevice.clearUpdatesQueue(currentRequest.session.deviceId, (err) ->
-        throw err if err
-        runTask(currentRequest, task, {})
-      )
-  )
-
-
-getSession = (httpRequest, callback) ->
-  # Separation by comma is important as some devices don't comform to standard
-  COOKIE_REGEX = /\s*([a-zA-Z0-9\-_]+?)\s*=\s*"?([a-zA-Z0-9\-_]*?)"?\s*(,|;|$)/g
-  while match = COOKIE_REGEX.exec(httpRequest.headers.cookie)
-    sessionId = match[2] if match[1] == 'session'
-
-  return callback() if not sessionId?
-
-  db.redisClient.get("session_#{sessionId}", (err, res) ->
-    throw err if err
-    return callback(sessionId, JSON.parse(res))
-  )
-
-
-updateSessionExpiry = (currentRequest) ->
-  db.redisClient.expire("session_#{currentRequest.sessionId}", currentRequest.session.sessionTimeout, (err, res) ->
-    throw err if err
-    if not res
-      # Resave session in the very rare case when session cache expires while processing request
-      db.redisClient.setex("session_#{currentRequest.sessionId}", currentRequest.session.sessionTimeout, JSON.stringify(currentRequest.session), (err) ->
-        throw err if err
-      )
-  )
-
-
-listener = (httpRequest, httpResponse) ->
-  if httpRequest.method != 'POST'
-    httpResponse.writeHead 405, {'Allow': 'POST'}
-    httpResponse.end('405 Method Not Allowed')
-    return
-
-  if httpRequest.headers['content-encoding']?
-    switch httpRequest.headers['content-encoding']
-      when 'gzip'
-        stream = httpRequest.pipe(zlib.createGunzip())
-      when 'deflate'
-        stream = httpRequest.pipe(zlib.createInflate())
-      else
-        httpResponse.writeHead(415)
-        httpResponse.end('415 Unsupported Media Type')
-        return
-  else
-    stream = httpRequest
-
-  chunks = []
-  bytes = 0
-
-  stream.on('data', (chunk) ->
-    chunks.push(chunk)
-    bytes += chunk.length
-  )
-
-  httpRequest.getBody = () ->
-    # Write all chunks into a Buffer
-    body = new Buffer(bytes)
-    offset = 0
-    chunks.forEach((chunk) ->
-      chunk.copy(body, offset, 0, chunk.length)
-      offset += chunk.length
-    )
-    return body
-
-  stream.on('end', () ->
-    getSession(httpRequest, (sessionId, session) ->
-      cwmpRequest = soap.request(httpRequest, session?.cwmpVersion)
-      if not session?
-        if cwmpRequest.methodRequest?.type isnt 'Inform'
-          httpResponse.writeHead(400)
-          httpResponse.end('Session is expired')
-          return
-
-        deviceId = common.generateDeviceId(cwmpRequest.methodRequest.deviceId)
-        sessionId = crypto.randomBytes(8).toString('hex')
-
-        session = {
-          deviceId : deviceId,
-          cwmpVersion : cwmpRequest.cwmpVersion,
-          sessionTimeout : cwmpRequest.sessionTimeout ? config.get('SESSION_TIMEOUT', deviceId)
-        }
-
-        httpRequest.connection.setTimeout(session.sessionTimeout * 1000)
-
-      currentRequest = {
-        httpRequest : httpRequest,
-        httpResponse : httpResponse,
-        sessionId : sessionId,
-        session : session
-      }
-
-      if config.get('DEBUG', currentRequest.session.deviceId)
-        dump = "# REQUEST #{new Date(Date.now())}\n" + JSON.stringify(httpRequest.headers) + "\n#{httpRequest.getBody()}\n\n"
-        require('fs').appendFile("./debug/#{currentRequest.session.deviceId}.dump", dump, (err) ->
-          throw err if err
-        )
-
-      if cwmpRequest.methodRequest?
-        if cwmpRequest.methodRequest.type is 'Inform'
-          inform(currentRequest, cwmpRequest)
-        else if cwmpRequest.methodRequest.type is 'GetRPCMethods'
-          util.log("#{currentRequest.session.deviceId}: GetRPCMethods")
-          res = soap.response({
-            id : cwmpRequest.id,
-            methodResponse : {type : 'GetRPCMethodsResponse', methodList : ['Inform', 'GetRPCMethods', 'TransferComplete', 'RequestDownload']},
-            cwmpVersion : currentRequest.session.cwmpVersion
-          })
-          writeResponse(currentRequest, res)
-          updateSessionExpiry(currentRequest)
-        else if cwmpRequest.methodRequest.type is 'TransferComplete'
-          # do nothing
-          util.log("#{currentRequest.session.deviceId}: Transfer complete")
-          res = soap.response({
-            id : cwmpRequest.id,
-            methodResponse : {type : 'TransferCompleteResponse'},
-            cwmpVersion : currentRequest.session.cwmpVersion
-          })
-          writeResponse(currentRequest, res)
-          updateSessionExpiry(currentRequest)
-        else if cwmpRequest.methodRequest.type is 'RequestDownload'
-          requestDownloadResponse = () ->
-            res = soap.response({
-              id : cwmpRequest.id,
-              methodResponse : {type : 'RequestDownloadResponse'},
-              cwmpVersion : currentRequest.session.cwmpVersion
-            })
-            writeResponse(currentRequest, res)
-            updateSessionExpiry(currentRequest)
-          fileType = cwmpRequest.methodRequest.fileType
-          util.log("#{currentRequest.session.deviceId}: RequestDownload (#{fileType})")
-          if fileType isnt '1 Firmware Upgrade Image'
-            # Only supporting firmware upgrade for now
-            return requestDownloadResponse()
-
-          db.getPresetsObjectsAliases((allPresets, allObjects, allAliases) ->
-            presets.getDevicePreset(currentRequest.session.deviceId, allPresets, allObjects, allAliases, (devicePreset) ->
-              presetSoftwareVersion = devicePreset.softwareVersion?.preset
-              currentSoftwareVersion = devicePreset.softwareVersion?.current._value
-              if presetSoftwareVersion? and presetSoftwareVersion != currentSoftwareVersion
-                deviceIdStruct = common.parseDeviceId(currentRequest.session.deviceId)
-                db.filesCollection.findOne({'metadata.fileType' : '1 Firmware Upgrade Image', 'metadata.oui' : deviceIdStruct.oui, 'metadata.productClass' : deviceIdStruct.productClass, 'metadata.version' : presetSoftwareVersion}, {_id : 1}, (err, file) ->
-                  throw err if err
-                  if not file?
-                    util.error("#{currentRequest.session.deviceId}: Firmware image not found (#{presetSoftwareVersion})")
-                    return requestDownloadResponse()
-
-                  task = {
-                    device : currentRequest.session.deviceId,
-                    name : 'download',
-                    file : file['_id']
-                  }
-                  apiFunctions.insertTasks(task, allAliases, (err, tasks) ->
-                    throw err if err
-                    return requestDownloadResponse()
-                  )
-                )
-              else
-                return requestDownloadResponse()
-            )
-          )
-        else
-          throw new Error('ACS method not supported')
-      else if cwmpRequest.methodResponse?
-        taskId = cwmpRequest.id
-
-        db.getTask(taskId, (err, task) ->
-          throw err if err
-          if not task
-            nextTask(currentRequest)
-          else
-            runTask(currentRequest, task, cwmpRequest.methodResponse)
-        )
-      else if cwmpRequest.fault?
-        taskId = cwmpRequest.id
-        if not taskId
-          # Fault not related to a task. End the session.
-          return endSession(currentRequest)
-
-        db.getTask(taskId, (err, task) ->
-          throw err if err
-          if not task
-            nextTask(currentRequest)
-          else
-            runTask(currentRequest, task, cwmpRequest.fault)
-        )
-      else
-        # cpe sent empty response. start sending acs requests
-        nextTask(currentRequest)
-    )
-  )
-
-
-exports.listener = listener

lib/db.coffee

@@ -1,230 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-config = require './config'
-mongodb = require 'mongodb'
-redis = require('redis')
-parameters = require './parameters'
-
-redisClient = null
-tasksCollection = null
-devicesCollection = null
-presetsCollection = null
-objectsCollection = null
-
-
-# Only flush redis DB if aliases or custom commands config has changed
-flushRedis = (callback) ->
-  redisClient.get('config_hash', (err, res) ->
-    return callback(err) if err
-
-    hash = require('crypto')
-      .createHash('md5')
-      .update(JSON.stringify(config.PARAMETERS))
-      .update(JSON.stringify(config.CUSTOM_COMMANDS))
-      .digest('hex')
-
-    if res == hash
-      return callback()
-
-    redisClient.flushdb((err) ->
-      return callback(err) if err
-      redisClient.set('config_hash', hash, (err, res) ->
-        return callback(err)
-      )
-    )
-  )
-
-
-connect = (callback) ->
-  callbackCounter = 6
-  mongodb.MongoClient.connect(config.get('MONGODB_CONNECTION_URL'), {db:{w:1},server:{autoReconnect:true}}, (err, db) ->
-    return callback(err) if err
-    exports.mongoDb = db
-    db.collection('tasks', (err, collection) ->
-      exports.tasksCollection = tasksCollection = collection
-      collection?.ensureIndex({device: 1, timestamp: 1}, (err) ->
-      )
-
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('devices', (err, collection) ->
-      exports.devicesCollection  = devicesCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('presets', (err, collection) ->
-      exports.presetsCollection = presetsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('objects', (err, collection) ->
-      exports.objectsCollection = objectsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('fs.files', (err, collection) ->
-      exports.filesCollection = filesCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    exports.redisClient = redisClient = redis.createClient(config.get('REDIS_PORT'), config.get('REDIS_HOST'))
-    redisClient.select(config.get('REDIS_DB'), (err) ->
-      if err
-        callbackCounter = 0
-        return callback(err)
-
-      flushRedis((err) ->
-        if --callbackCounter == 0 or err
-          callbackCounter = 0
-          return callback(err)
-      )
-    )
-  )
-
-
-disconnect = () ->
-  exports.mongoDb.close()
-  redisClient.quit()
-
-
-getTask = (taskId, callback) ->
-  tid = String(taskId)
-  redisClient.get(tid, (err, res) ->
-    return callback(err) if err
-    if res?
-      callback(err, JSON.parse(res))
-    else
-      tasksCollection.findOne({_id : mongodb.ObjectID(tid)}, (err, task) ->
-        callback(err, task)
-      )
-  )
-
-
-getCached = (name, valueCallback, valueExpiry, callback) ->
-  redisClient.get(name, (err, res) ->
-    return callback(err) if err
-    return callback(null, JSON.parse(res)) if res?
-
-    lockAcquired = () ->
-      valueCallback((err, value) ->
-        return callback(err, value) if err
-        redisClient.setex(name, valueExpiry, JSON.stringify(value), (err) ->
-          return callback(err, value) if err
-          redisClient.del("lock.#{name}", (err) ->
-            callback(err, value)
-          )
-        )
-      )
-
-    redisClient.setnx("lock.#{name}", Date.now() + 30000, (err, res) ->
-      return callback(err) if err
-      if res == 1
-        lockAcquired()
-      else
-        setTimeout(() ->
-          redisClient.get("lock.#{name}", (err, timestamp) ->
-            return callback(err) if err
-            now = Date.now()
-            if not timestamp?
-              getCached(name, valueCallback, valueExpiry, callback)
-            else if timestamp > now
-              setTimeout(() ->
-                getCached(name, valueCallback, valueExpiry, callback)
-              , 1000)
-            else
-              redisClient.getset("lock.#{name}", now + 30000, (err, timestamp) ->
-                return callback(err) if err
-                if timestamp > now
-                  setTimeout(() ->
-                    getCached(name, valueCallback, valueExpiry, callback)
-                  , 1000)
-                else
-                  lockAcquired()
-              )
-          )
-        , 1000)
-    )
-  )
-
-
-getAliases = (callback) ->
-  getCached('aliases', parameters.compileAliases, config.get('PRESETS_CACHE_DURATION'), (err, res) ->
-    throw err if err
-    callback(res)
-  )
-
-
-getPresetsObjectsAliases = (callback) ->
-  redisClient.mget('presets', 'objects', 'aliases', (err, res) ->
-    presets = JSON.parse(res[0])
-    objects = JSON.parse(res[1])
-    aliases = JSON.parse(res[2])
-
-    if presets and objects and aliases
-      return callback(presets, objects, aliases)
-
-    if not presets
-      getCached('presets', (callback) ->
-        presetsCollection.find().toArray((err, res) ->
-          callback(err, res)
-        )
-      , config.get('PRESETS_CACHE_DURATION'), (err, res) ->
-        throw err if err
-        presets = res
-        callback(presets, objects, aliases) if objects and aliases
-      )
-
-    if not objects
-      getCached('objects', (callback) ->
-        objectsCollection.find().toArray((err, res) ->
-          return callback(err) if err
-          objs = {}
-          objs[r._id] = r for r in res
-          callback(null, objs)
-        )
-      , config.get('PRESETS_CACHE_DURATION'), (err, res) ->
-        throw err if err
-        objects = res
-        callback(presets, objects, aliases) if presets and aliases
-      )
-
-    if not aliases
-      getCached('aliases', parameters.compileAliases, config.get('PRESETS_CACHE_DURATION'), (err, res) ->
-        throw err if err
-        aliases = res
-        callback(presets, objects, aliases) if presets and objects
-      )
-  )
-
-
-exports.getTask = getTask
-exports.getPresetsObjectsAliases = getPresetsObjectsAliases
-exports.getAliases = getAliases
-exports.connect = connect
-exports.disconnect = disconnect

lib/db.ts

@@ -0,0 +1,692 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { MongoClient, ObjectID, Collection } from "mongodb";
+import { get } from "./config";
+import { escapeRegExp } from "./common";
+import { parse } from "./common/expression-parser";
+import {
+  DeviceData,
+  Attributes,
+  SessionFault,
+  Task,
+  Operation,
+  Expression
+} from "./types";
+import Path from "./common/path";
+
+export let tasksCollection: Collection,
+  devicesCollection: Collection,
+  presetsCollection: Collection,
+  objectsCollection: Collection,
+  provisionsCollection: Collection,
+  virtualParametersCollection: Collection,
+  faultsCollection: Collection,
+  filesCollection: Collection,
+  operationsCollection: Collection,
+  permissionsCollection: Collection,
+  usersCollection: Collection,
+  configCollection: Collection;
+
+export let client: MongoClient;
+
+export async function connect(): Promise<void> {
+  client = await MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
+    useNewUrlParser: true
+  });
+
+  const db = client.db();
+
+  tasksCollection = db.collection("tasks");
+  await tasksCollection.createIndex({ device: 1, timestamp: 1 });
+
+  devicesCollection = db.collection("devices");
+  presetsCollection = db.collection("presets");
+  objectsCollection = db.collection("objects");
+  filesCollection = db.collection("fs.files");
+  provisionsCollection = db.collection("provisions");
+  virtualParametersCollection = db.collection("virtualParameters");
+  faultsCollection = db.collection("faults");
+  operationsCollection = db.collection("operations");
+  permissionsCollection = db.collection("permissions");
+  usersCollection = db.collection("users");
+  configCollection = db.collection("config");
+}
+
+export async function disconnect(): Promise<void> {
+  if (client) await client.close();
+}
+
+// Optimize projection by removing overlaps
+// This can modify the object
+function optimizeProjection(obj: {}): {} {
+  if (obj[""]) return { "": obj[""] };
+
+  const keys = Object.keys(obj).sort();
+  if (keys.length <= 1) return obj;
+
+  for (let i = 1; i < keys.length; ++i) {
+    const a = keys[i - 1];
+    const b = keys[i];
+    if (b.startsWith(a)) {
+      if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") {
+        delete obj[b];
+        keys.splice(i--, 1);
+      }
+    }
+  }
+  return obj;
+}
+
+export async function fetchDevice(
+  id: string,
+  timestamp: number
+): Promise<[Path, number, Attributes?][]> {
+  const res: [Path, number, Attributes?][] = [
+    [
+      Path.parse("Events"),
+      timestamp,
+      { object: [timestamp, 1], writable: [timestamp, 0] }
+    ],
+    [
+      Path.parse("DeviceID"),
+      timestamp,
+      { object: [timestamp, 1], writable: [timestamp, 0] }
+    ]
+  ];
+
+  const device = await devicesCollection.findOne({ _id: id });
+  if (!device) return null;
+
+  function storeParams(obj, path: string, pathLength: number, ts): void {
+    if (obj["_timestamp"]) obj["_timestamp"] = +obj["_timestamp"];
+
+    const attrs: Attributes = {};
+    let t = obj["_timestamp"] || 1;
+    if (ts > t) t = ts;
+
+    if (obj["_value"] != null) {
+      attrs.value = [obj["_timestamp"] || 1, [obj["_value"], obj["_type"]]];
+      if (obj["_type"] === "xsd:dateTime")
+        attrs.value[1][0] = +attrs.value[1][0];
+
+      obj["_object"] = false;
+    }
+    if (obj["_writable"] != null)
+      attrs.writable = [ts || 1, obj["_writable"] ? 1 : 0];
+
+    if (obj["_object"] != null) attrs.object = [t, obj["_object"] ? 1 : 0];
+
+    res.push([Path.parse(path.slice(0, -1)), t, attrs]);
+
+    for (const [k, v] of Object.entries(obj)) {
+      if (!k.startsWith("_")) {
+        obj["_object"] = true;
+        storeParams(v, path + k + ".", pathLength + 1, obj["_timestamp"]);
+      }
+    }
+
+    if ((obj["_object"] || !pathLength) && obj["_timestamp"])
+      res.push([Path.parse(path + "*"), obj["_timestamp"]]);
+  }
+
+  for (const [k, v] of Object.entries(device)) {
+    switch (k) {
+      case "_lastInform":
+        res.push([
+          Path.parse("Events.Inform"),
+          +v,
+          {
+            object: [+v, 0],
+            writable: [+v, 0],
+            value: [+v, [+v, "xsd:dateTime"]]
+          }
+        ]);
+        break;
+      case "_lastBoot":
+        res.push([
+          Path.parse("Events.1_BOOT"),
+          +v,
+          {
+            object: [+v, 0],
+            writable: [+v, 0],
+            value: [+v, [+v, "xsd:dateTime"]]
+          }
+        ]);
+        break;
+      case "_lastBootstrap":
+        res.push([
+          Path.parse("Events.0_BOOTSTRAP"),
+          +v,
+          {
+            object: [+v, 0],
+            writable: [+v, 0],
+            value: [+v, [+v, "xsd:dateTime"]]
+          }
+        ]);
+        break;
+      case "_registered":
+        // Use current timestamp for registered event attribute timestamps
+        res.push([
+          Path.parse("Events.Registered"),
+          timestamp,
+          {
+            object: [timestamp, 0],
+            writable: [timestamp, 0],
+            value: [timestamp, [+v, "xsd:dateTime"]]
+          }
+        ]);
+        break;
+      case "_id":
+        res.push([
+          Path.parse("DeviceID.ID"),
+          timestamp,
+          {
+            object: [timestamp, 0],
+            writable: [timestamp, 0],
+            value: [timestamp, [v as string, "xsd:string"]]
+          }
+        ]);
+        break;
+      case "_tags":
+        if ((v as string[]).length) {
+          res.push([
+            Path.parse("Tags"),
+            timestamp,
+            { object: [timestamp, 1], writable: [timestamp, 0] }
+          ]);
+        }
+
+        for (let t of v as string[]) {
+          t = t.replace(/[^a-zA-Z0-9-]+/g, "_");
+          res.push([
+            Path.parse("Tags." + t),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 1],
+              value: [timestamp, [true, "xsd:boolean"]]
+            }
+          ]);
+        }
+        break;
+      case "_deviceId":
+        if (v["_Manufacturer"] != null) {
+          res.push([
+            Path.parse("DeviceID.Manufacturer"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_Manufacturer"], "xsd:string"]]
+            }
+          ]);
+        }
+
+        if (v["_OUI"] != null) {
+          res.push([
+            Path.parse("DeviceID.OUI"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_OUI"], "xsd:string"]]
+            }
+          ]);
+        }
+
+        if (v["_ProductClass"] != null) {
+          res.push([
+            Path.parse("DeviceID.ProductClass"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_ProductClass"], "xsd:string"]]
+            }
+          ]);
+        }
+
+        if (v["_SerialNumber"] != null) {
+          res.push([
+            Path.parse("DeviceID.SerialNumber"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_SerialNumber"], "xsd:string"]]
+            }
+          ]);
+        }
+    }
+  }
+
+  storeParams(device, "", 0, 0);
+  return res;
+}
+
+export async function saveDevice(
+  deviceId: string,
+  deviceData: DeviceData,
+  isNew: boolean,
+  sessionTimestamp: number
+): Promise<void> {
+  const update = { $set: {}, $unset: {}, $addToSet: {}, $pull: {} };
+
+  for (const diff of deviceData.timestamps.diff()) {
+    if (diff[0].wildcard !== 1 << (diff[0].length - 1)) continue;
+
+    if (
+      diff[0].segments[0] === "Events" ||
+      diff[0].segments[0] === "DeviceID" ||
+      diff[0].segments[0] === "Tags"
+    )
+      continue;
+
+    const parent = deviceData.paths.get(diff[0].slice(0, -1));
+
+    // Param timestamps may be greater than session timestamp to track revisions
+    if (diff[2] > sessionTimestamp) diff[2] = sessionTimestamp;
+
+    if (diff[2] == null && diff[1] != null) {
+      update["$unset"][
+        parent.length ? parent.toString() + "._timestamp" : "_timestamp"
+      ] = 1;
+    } else {
+      if (parent && (!parent.length || deviceData.attributes.has(parent))) {
+        update["$set"][
+          parent.length ? parent.toString() + "._timestamp" : "_timestamp"
+        ] = new Date(diff[2]);
+      }
+    }
+  }
+
+  for (const diff of deviceData.attributes.diff()) {
+    const path = diff[0];
+    const value1 = (((diff[1] || {}).value || [])[1] || [])[0];
+    const value2 = (((diff[2] || {}).value || [])[1] || [])[0];
+    const valueType1 = (((diff[1] || {}).value || [])[1] || [])[1];
+    const valueType2 = (((diff[2] || {}).value || [])[1] || [])[1];
+    const valueTimestamp1 = ((diff[1] || {}).value || [])[0];
+    const valueTimestamp2 = ((diff[2] || {}).value || [])[0];
+    const object1 = ((diff[1] || {}).object || [])[1];
+    const object2 = ((diff[2] || {}).object || [])[1];
+    const writable2 = ((diff[2] || {}).writable || [])[1];
+    const writable1 = ((diff[1] || {}).writable || [])[1];
+
+    switch (path.segments[0]) {
+      case "Events":
+        if (path.length === 2 && value2 !== value1) {
+          if (!diff[2]) {
+            switch (path.segments[1]) {
+              case "Inform":
+                update["$unset"]["_lastInform"] = 1;
+                break;
+              case "1_BOOT":
+                update["$unset"]["_lastBoot"] = 1;
+                break;
+              case "0_BOOTSTRAP":
+                update["$unset"]["_lastBootstrap"] = 1;
+                break;
+              case "Registered":
+                update["$unset"]["_registered"] = 1;
+            }
+          } else {
+            const t = new Date(diff[2].value[1][0] as number);
+            switch (path.segments[1]) {
+              case "Inform":
+                update["$set"]["_lastInform"] = t;
+                break;
+              case "1_BOOT":
+                update["$set"]["_lastBoot"] = t;
+                break;
+              case "0_BOOTSTRAP":
+                update["$set"]["_lastBootstrap"] = t;
+                break;
+              case "Registered":
+                update["$set"]["_registered"] = t;
+            }
+          }
+        }
+
+        break;
+      case "DeviceID":
+        if (value2 !== value1) {
+          const v = diff[2].value[1][0];
+          switch (path.segments[1]) {
+            case "ID":
+              update["$set"]["_id"] = v;
+              break;
+            case "Manufacturer":
+              update["$set"]["_deviceId._Manufacturer"] = v;
+              break;
+            case "OUI":
+              update["$set"]["_deviceId._OUI"] = v;
+              break;
+            case "ProductClass":
+              update["$set"]["_deviceId._ProductClass"] = v;
+              break;
+            case "SerialNumber":
+              update["$set"]["_deviceId._SerialNumber"] = v;
+          }
+        }
+        break;
+      case "Tags":
+        if (value2 !== value1) {
+          if (value2 != null) {
+            if (!update["$addToSet"]["_tags"])
+              update["$addToSet"]["_tags"] = { $each: [] };
+            update["$addToSet"]["_tags"]["$each"].push(path.segments[1]);
+          } else {
+            if (!update["$pull"]["_tags"]) {
+              update["$pull"]["_tags"] = {
+                $in: []
+              };
+            }
+            update["$pull"]["_tags"]["$in"].push(path.segments[1]);
+          }
+        }
+
+        break;
+      default:
+        if (!diff[2]) {
+          update["$unset"][path.toString()] = 1;
+          continue;
+        }
+
+        for (const attrName of Object.keys(diff[2])) {
+          // Param timestamps may be greater than session timestamp to track revisions
+          if (diff[2][attrName][0] > sessionTimestamp)
+            diff[2][attrName][0] = sessionTimestamp;
+
+          if (diff[2][attrName][1] != null) {
+            switch (attrName) {
+              case "value":
+                if (value2 !== value1) {
+                  if (
+                    valueType2 === "xsd:dateTime" &&
+                    Number.isInteger(value2 as number)
+                  ) {
+                    update["$set"][path.toString() + "._value"] = new Date(
+                      value2 as number
+                    );
+                  } else {
+                    update["$set"][path.toString() + "._value"] = value2;
+                  }
+                }
+
+                if (valueType2 !== valueType1)
+                  update["$set"][path.toString() + "._type"] = valueType2;
+
+                if (valueTimestamp2 !== valueTimestamp1) {
+                  update["$set"][path.toString() + "._timestamp"] = new Date(
+                    valueTimestamp2
+                  );
+                }
+
+                break;
+              case "object":
+                if (!diff[1] || !diff[1].object || object2 !== object1) {
+                  update["$set"][
+                    path.length ? path.toString() + "._object" : "_object"
+                  ] = !!object2;
+                }
+
+                break;
+              case "writable":
+                if (!diff[1] || !diff[1].writable || writable2 !== writable1) {
+                  update["$set"][
+                    path.length ? path.toString() + "._writable" : "_writable"
+                  ] = !!writable2;
+                }
+            }
+          }
+        }
+
+        if (diff[1]) {
+          for (const attrName of Object.keys(diff[1])) {
+            if (
+              diff[1][attrName][1] != null &&
+              (!diff[2] || !diff[2][attrName] || diff[2][attrName][1] == null)
+            ) {
+              const p = path.length ? path.toString() + "." : "";
+              update["$unset"][`${p}_${attrName}`] = 1;
+              if (attrName === "value") {
+                update["$unset"][p + "_type"] = 1;
+                update["$unset"][p + "_timestamp"] = 1;
+              }
+            }
+          }
+        }
+    }
+  }
+
+  update["$unset"] = optimizeProjection(update["$unset"]);
+
+  // Remove overlap possibly caused by parameters changing from objects
+  // to regular parameters or vice versa. Reason being that _timestamp
+  // represents two different things depending on whether the parameter
+  // is an object or not.
+  for (const k of Object.keys(update["$unset"]))
+    if (update["$set"][k] != null) delete update["$unset"][k];
+
+  // Remove empty keys
+  for (const [k, v] of Object.entries(update)) {
+    if (k === "$addToSet") {
+      for (const [kk, vv] of Object.entries(v))
+        if (!vv["$each"].length) delete v[kk];
+    } else if (k === "$pull") {
+      for (const [kk, vv] of Object.entries(v))
+        if (!vv["$in"].length) delete v[kk];
+    }
+    if (!Object.keys(v).length) delete update[k];
+  }
+
+  if (!Object.keys(update).length) return;
+
+  // Mongo doesn't allow $addToSet and $pull at the same time
+  let update2;
+  if (update["$addToSet"] && update["$pull"]) {
+    update2 = { $pull: update["$pull"] };
+    delete update["$pull"];
+  }
+
+  const result = await devicesCollection.updateOne({ _id: deviceId }, update, {
+    upsert: isNew
+  });
+
+  if (result.result.n !== 1)
+    throw new Error(`Device ${deviceId} not found in database`);
+
+  if (update2) {
+    await devicesCollection.updateOne({ _id: deviceId }, update2);
+    return;
+  }
+}
+
+export async function getFaults(
+  deviceId
+): Promise<{ [channel: string]: SessionFault }> {
+  const res = await faultsCollection
+    .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
+    .toArray();
+
+  const faults: { [channel: string]: SessionFault } = {};
+  for (const r of res) {
+    const channel = r._id.slice(deviceId.length + 1);
+    delete r._id;
+    delete r.channel;
+    delete r.device;
+    r.timestamp = +r.timestamp;
+    r.provisions = JSON.parse(r.provisions);
+    faults[channel] = r;
+  }
+
+  return faults;
+}
+
+export async function saveFault(deviceId, channel, fault): Promise<void> {
+  const id = `${deviceId}:${channel}`;
+  fault = Object.assign({}, fault);
+  fault._id = id;
+  fault.device = deviceId;
+  fault.channel = channel;
+  fault.timestamp = new Date(fault.timestamp);
+  fault.provisions = JSON.stringify(fault.provisions);
+  await faultsCollection.replaceOne({ _id: id }, fault, { upsert: true });
+}
+
+export async function deleteFault(deviceId, channel): Promise<void> {
+  await faultsCollection.deleteOne({ _id: `${deviceId}:${channel}` });
+}
+
+export async function getDueTasks(
+  deviceId,
+  timestamp
+): Promise<[Task[], number]> {
+  const cur = tasksCollection.find({ device: deviceId }).sort(["timestamp"]);
+  const tasks = [] as Task[];
+
+  for await (const task of cur) {
+    if (task.timestamp) task.timestamp = +task.timestamp;
+    if (task.expiry) task.expiry = +task.expiry;
+    if (task.timestamp >= timestamp) return [tasks, +task.timestamp];
+    task._id = String(task._id);
+
+    tasks.push(task);
+
+    // For API compatibility
+    if (task.name === "download" && task.file) {
+      let q;
+      if (ObjectID.isValid(task.file))
+        q = { _id: { $in: [task.file, new ObjectID(task.file)] } };
+      else q = { _id: task.file };
+
+      const res = await filesCollection.find(q).toArray();
+
+      if (res[0]) {
+        if (!task.fileType) task.fileType = res[0].metadata.fileType;
+
+        if (!task.fileName)
+          task.fileName = res[0].filename || res[0]._id.toString();
+      }
+    }
+  }
+  return [tasks, null];
+}
+
+export async function clearTasks(deviceId, taskIds): Promise<void> {
+  await tasksCollection.deleteMany({
+    _id: { $in: taskIds.map(id => new ObjectID(id)) }
+  });
+}
+
+export async function getOperations(
+  deviceId
+): Promise<{ [commandKey: string]: Operation }> {
+  const res = await operationsCollection
+    .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
+    .toArray();
+
+  const operations: { [commandKey: string]: Operation } = {};
+  for (const r of res) {
+    const commandKey = r._id.slice(deviceId.length + 1);
+    delete r._id;
+    r.timestamp = +r.timestamp;
+    if (r.args) r.args = JSON.parse(r.args);
+    r.provisions = JSON.parse(r.provisions);
+    r.retries = JSON.parse(r.retries);
+    operations[commandKey] = r;
+  }
+  return operations;
+}
+
+export async function saveOperation(
+  deviceId,
+  commandKey,
+  operation
+): Promise<void> {
+  const id = `${deviceId}:${commandKey}`;
+  operation = Object.assign({}, operation);
+  operation._id = id;
+  operation.timestamp = new Date(operation.timestamp);
+  operation.provisions = JSON.stringify(operation.provisions);
+  operation.retries = JSON.stringify(operation.retries);
+  operation.args = JSON.stringify(operation.args);
+  await operationsCollection.replaceOne({ _id: id }, operation, {
+    upsert: true
+  });
+}
+
+export async function deleteOperation(deviceId, commandKey): Promise<void> {
+  await operationsCollection.deleteOne({ _id: `${deviceId}:${commandKey}` });
+}
+
+export async function getPresets(): Promise<{}[]> {
+  return presetsCollection.find().toArray();
+}
+
+export async function getObjects(): Promise<{}[]> {
+  return objectsCollection.find().toArray();
+}
+
+export async function getProvisions(): Promise<{}[]> {
+  return provisionsCollection.find().toArray();
+}
+
+export async function getVirtualParameters(): Promise<{}[]> {
+  return virtualParametersCollection.find().toArray();
+}
+
+export function getFiles(): Promise<{}[]> {
+  return filesCollection.find().toArray();
+}
+
+export async function getConfig(): Promise<
+  { id: string; value: Expression }[]
+> {
+  const res = await configCollection.find().toArray();
+  return res.map(c => ({
+    id: c["_id"],
+    value: parse(c["value"])
+  }));
+}
+
+interface Permission {
+  role: string;
+  resource: string;
+  access: number;
+  filter: string;
+  validate: string;
+}
+
+export async function getPermissions(): Promise<Permission[]> {
+  return permissionsCollection.find().toArray();
+}
+
+interface User {
+  _id: string;
+  password: string;
+  salt: string;
+  roles: string;
+}
+
+export async function getUsers(): Promise<User[]> {
+  return usersCollection.find().toArray();
+}

lib/debug.ts

@@ -0,0 +1,204 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import {
+  IncomingMessage,
+  ServerResponse,
+  ClientRequest,
+  RequestOptions
+} from "http";
+import { Socket } from "net";
+import { appendFileSync } from "fs";
+import * as yaml from "yaml";
+import * as config from "./config";
+
+const DEBUG_FILE = "" + config.get("DEBUG_FILE");
+const DEBUG_FORMAT = "" + config.get("DEBUG_FORMAT");
+
+const connectionTimestamps = new WeakMap<Socket, Date>();
+
+function getConnectionTimestamp(connection: Socket): Date {
+  let t = connectionTimestamps.get(connection);
+  if (!t) {
+    t = new Date();
+    connectionTimestamps.set(connection, t);
+  }
+  return t;
+}
+
+export function incomingHttpRequest(
+  httpRequest: IncomingMessage,
+  deviceId: string,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpRequest.connection;
+  const msg = {
+    event: "incoming HTTP request",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(con),
+    localPort: con.localPort,
+    method: httpRequest.method,
+    url: httpRequest.url,
+    headers: httpRequest.headers,
+    body: body
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingHttpResponse(
+  httpResponse: ServerResponse,
+  deviceId: string,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpResponse.connection;
+  const msg = {
+    event: "outgoing HTTP response",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(con),
+    statusCode: httpResponse.statusCode,
+    headers: httpResponse.getHeaders(),
+    body: body
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingHttpRequest(
+  httpRequest: ClientRequest,
+  deviceId: string,
+  options: RequestOptions,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpRequest.connection;
+  const msg = {
+    event: "outgoing HTTP request",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(con),
+    remotePort: options.port,
+    method: options.method || "GET",
+    url: options.path,
+    headers: httpRequest.getHeaders(),
+    body: body
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingHttpRequestError(
+  httpRequest: ClientRequest,
+  deviceId: string,
+  options: RequestOptions,
+  err: Error
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing HTTP request",
+    timestamp: now,
+    remoteAddress: options.hostname,
+    deviceId: deviceId,
+    connection: null,
+    remotePort: options.port,
+    method: options.method,
+    url: options.path,
+    headers: httpRequest.getHeaders(),
+    error: err.message
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function incomingHttpResponse(
+  httpResponse: IncomingMessage,
+  deviceId: string,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpResponse.connection;
+  const msg = {
+    event: "incoming HTTP response",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(httpResponse.connection),
+    statusCode: httpResponse.statusCode,
+    headers: httpResponse.headers,
+    body: body
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingUdpMessage(
+  remoteAddress: string,
+  deviceId: string,
+  remotePort: number,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing UDP message",
+    timestamp: now,
+    remoteAddress: remoteAddress,
+    deviceId: deviceId,
+    remotePort: remotePort,
+    body: body
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}

lib/default-provisions.ts

@@ -0,0 +1,174 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Path from "./common/path";
+import * as config from "./config";
+import * as device from "./device";
+import * as scheduling from "./scheduling";
+
+const MAX_DEPTH = +config.get("MAX_DEPTH");
+
+export function refresh(sessionContext, provision, declarations): boolean {
+  const segments = Path.parse(provision[1]).segments.slice();
+  const l = segments.length;
+  segments.length = MAX_DEPTH;
+  segments.fill("*", l);
+  const path = Path.parse(segments.join("."));
+  const every = 1000 * (provision[2] || 1);
+  const offset = scheduling.variance(sessionContext.deviceId, every);
+  const t = scheduling.interval(sessionContext.timestamp, every, offset);
+
+  for (let i = l; i < path.length; ++i) {
+    declarations.push({
+      path: path.slice(0, i),
+      pathGet: t,
+      pathSet: null,
+      attrGet: { object: 1, writable: 1, value: t },
+      attrSet: null,
+      defer: true
+    });
+  }
+
+  return true;
+}
+
+export function value(sessionContext, provision, declarations): boolean {
+  declarations.push({
+    path: Path.parse(provision[1]),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [provision[2]] },
+    defer: true
+  });
+
+  return true;
+}
+
+export function tag(sessionContext, provision, declarations): boolean {
+  declarations.push({
+    path: Path.parse(`Tags.${provision[1]}`),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [provision[2]] },
+    defer: true
+  });
+
+  return true;
+}
+
+export function reboot(sessionContext, provision, declarations): boolean {
+  declarations.push({
+    path: Path.parse("Reboot"),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [sessionContext.timestamp] },
+    defer: true
+  });
+
+  return true;
+}
+
+export function reset(sessionContext, provision, declarations): boolean {
+  declarations.push({
+    path: Path.parse("FactoryReset"),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [sessionContext.timestamp] },
+    defer: true
+  });
+
+  return true;
+}
+
+export function download(sessionContext, provision, declarations): boolean {
+  const alias = [
+    `FileType:${JSON.stringify(provision[1] || "")}`,
+    `FileName:${JSON.stringify(provision[2] || "")}`,
+    `TargetFileName:${JSON.stringify(provision[3] || "")}`
+  ].join(",");
+
+  declarations.push({
+    path: Path.parse(`Downloads.[${alias}]`),
+    pathGet: 1,
+    pathSet: 1,
+    attrGet: null,
+    attrSet: null,
+    defer: true
+  });
+
+  declarations.push({
+    path: Path.parse(`Downloads.[${alias}].Download`),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [sessionContext.timestamp] },
+    defer: true
+  });
+
+  return true;
+}
+
+export function instances(
+  sessionContext,
+  provision,
+  declarations,
+  startRevision,
+  endRevision
+): boolean {
+  let count = Number(provision[2]);
+
+  if (Number.isNaN(count)) return true;
+
+  const path = Path.parse(provision[1]);
+
+  if (provision[2][0] === "+" || provision[2][0] === "-") {
+    declarations.push({
+      path: path,
+      pathGet: 1,
+      pathSet: null,
+      attrGet: null,
+      attrSet: null,
+      defer: true
+    });
+
+    if (endRevision === startRevision) return false;
+
+    const unpacked = device.unpack(
+      sessionContext.deviceData,
+      path,
+      startRevision + 1
+    );
+    count = Math.max(0, unpacked.length + count);
+  }
+
+  declarations.push({
+    path: path,
+    pathGet: 1,
+    pathSet: count,
+    attrGet: null,
+    attrSet: null,
+    defer: true
+  });
+
+  return true;
+}

lib/device.ts

@@ -0,0 +1,405 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Path from "./common/path";
+import { DeviceData, Declaration, Clear } from "./types";
+
+const CHANGE_FLAGS = {
+  object: 2,
+  writable: 4,
+  value: 8
+};
+
+function parseBool(v): boolean {
+  v = "" + v;
+  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
+  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
+    return false;
+  else return null;
+}
+
+export function sanitizeParameterValue(
+  parameterValue: [string | number | boolean, string]
+): [string | number | boolean, string?] {
+  if (parameterValue[0] != null) {
+    switch (parameterValue[1]) {
+      case "xsd:boolean":
+        if (typeof parameterValue[0] !== "boolean") {
+          const b = parseBool(parameterValue[0]);
+          if (b == null) parameterValue[0] = "" + parameterValue[0];
+          else parameterValue[0] = b;
+        }
+        break;
+      case "xsd:string":
+      case "xsd:base64":
+      case "xsd:hexBinary":
+        if (typeof parameterValue[0] !== "string")
+          parameterValue[0] = "" + parameterValue[0];
+
+        break;
+      case "xsd:int":
+      case "xsd:unsignedInt":
+        if (typeof parameterValue[0] !== "number") {
+          const i = parseInt(parameterValue[0] as string);
+          if (isNaN(i)) parameterValue[0] = "" + parameterValue[0];
+          else parameterValue[0] = i;
+        }
+        break;
+      case "xsd:dateTime":
+        if (typeof parameterValue[0] !== "number") {
+          // Don't use parseInt because it reads date string as a number
+          let i = +parameterValue[0];
+          if (isNaN(i)) {
+            i = Date.parse(parameterValue[0] as string);
+            if (isNaN(i)) parameterValue[0] = "" + parameterValue[0];
+            else parameterValue[0] = i;
+          } else {
+            parameterValue[0] = i;
+          }
+        }
+        break;
+      default:
+        parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
+        break;
+    }
+  }
+
+  return parameterValue;
+}
+
+export function getAliasDeclarations(
+  path: Path,
+  timestamp: number,
+  attrGet = null
+): Declaration[] {
+  const stripped = path.stripAlias();
+  let decs: Declaration[] = [
+    {
+      path: stripped,
+      pathGet: timestamp,
+      pathSet: null,
+      attrGet: attrGet,
+      attrSet: null,
+      defer: true
+    }
+  ];
+
+  if (path.alias) {
+    for (const [i, alias] of path.segments.entries()) {
+      if (Array.isArray(alias)) {
+        const parent = stripped.slice(0, i + 1);
+        for (const [p] of alias as [Path, string][]) {
+          decs = decs.concat(
+            getAliasDeclarations(parent.concat(p), timestamp, {
+              value: timestamp
+            })
+          );
+        }
+      }
+    }
+  }
+
+  return decs;
+}
+
+export function unpack(
+  deviceData: DeviceData,
+  path: Path,
+  revision?: number
+): Path[] {
+  let allMatches = [] as Path[];
+  if (!path.alias) {
+    for (const p of deviceData.paths.find(path, false, true))
+      if (deviceData.attributes.has(p, revision)) allMatches.push(p);
+  } else {
+    const wildcardPath = path.stripAlias();
+
+    for (const p of deviceData.paths.find(wildcardPath, false, true))
+      if (deviceData.attributes.has(p, revision)) allMatches.push(p);
+
+    for (let i = path.length - 1; i >= 0; --i) {
+      if (path.alias & (1 << i)) {
+        for (const [param, val] of path.segments[i] as [Path, string][]) {
+          const p = wildcardPath.slice(0, i + 1).concat(param);
+          const unpacked = unpack(deviceData, p, revision);
+          const filtered: Path[] = [];
+          for (const up of unpacked) {
+            const attributes = deviceData.attributes.get(up, revision);
+            if (
+              attributes &&
+              attributes.value &&
+              attributes.value[1] &&
+              sanitizeParameterValue([val, attributes.value[1][1]])[0] ===
+                attributes.value[1][0]
+            ) {
+              for (let m = 0; m < allMatches.length; ++m) {
+                let k;
+                const match = allMatches[m];
+                if (!match) continue;
+                for (k = i; k >= 0; --k)
+                  if (match.segments[k] !== up.segments[k]) break;
+
+                if (k < 0) {
+                  filtered.push(match);
+                  allMatches[m] = null;
+                }
+              }
+            }
+          }
+          allMatches = filtered;
+        }
+      }
+    }
+  }
+
+  allMatches.sort((p1, p2) => {
+    for (let i = 0; i < p1.length; ++i) {
+      const a = p1.segments[i] as string;
+      const b = p2.segments[i] as string;
+      if (a !== b) {
+        // Use numeric sorting for numbers
+        const ia = parseInt(a);
+        const ib = parseInt(b);
+
+        if (ia === +a && ib === +b) return ia - ib;
+        else if (a < b) return -1;
+        else return 1;
+      }
+    }
+    return 0;
+  });
+
+  return allMatches;
+}
+
+export function clear(
+  deviceData,
+  path,
+  timestamp,
+  attributes,
+  changeFlags = 0
+): void {
+  const changeTrackers = {};
+
+  timestamp = timestamp || 0;
+
+  let descendantsTimestamp = timestamp;
+  if (attributes && attributes.object) {
+    if (attributes.object > descendantsTimestamp)
+      descendantsTimestamp = attributes.object;
+    if (!(attributes.object <= attributes.value))
+      attributes.value = attributes.object;
+  }
+
+  for (const p of deviceData.paths.find(
+    path,
+    true,
+    true,
+    descendantsTimestamp ? 99 : path.length
+  )) {
+    const tracker = deviceData.trackers.get(p);
+    for (const k in tracker) changeTrackers[k] |= tracker[k];
+
+    const currentTimestamp = deviceData.timestamps.get(p);
+    if (currentTimestamp === undefined) continue;
+
+    if (
+      timestamp > currentTimestamp ||
+      (descendantsTimestamp > currentTimestamp && p.length > path.length)
+    ) {
+      deviceData.timestamps.delete(p);
+      deviceData.attributes.delete(p);
+      changeFlags |= 1;
+    } else if (attributes && p.length === path.length) {
+      const currentAttributes = deviceData.attributes.get(p);
+      if (currentAttributes) {
+        let newAttrs;
+        for (const attrName in attributes) {
+          if (
+            attrName in currentAttributes &&
+            attributes[attrName] > currentAttributes[attrName][0]
+          ) {
+            changeFlags |= CHANGE_FLAGS[attrName];
+            if (!newAttrs) {
+              newAttrs = Object.assign({}, currentAttributes);
+              deviceData.attributes.set(p, newAttrs);
+            }
+            delete newAttrs[attrName];
+          }
+        }
+      }
+    }
+  }
+
+  // Note: For performance, we're merging all changes together rather than
+  // mark changes based the exact parameters affected.
+  for (const k in changeTrackers)
+    if (changeTrackers[k] & changeFlags) deviceData.changes.add(k);
+}
+
+function compareEquality(a, b): boolean {
+  const t = typeof a;
+  if (
+    a === null ||
+    a === undefined ||
+    t === "number" ||
+    t === "boolean" ||
+    t === "string" ||
+    t === "symbol"
+  )
+    return a === b;
+
+  return JSON.stringify(a) === JSON.stringify(b);
+}
+
+export function set(
+  deviceData: DeviceData,
+  path: Path,
+  timestamp,
+  attributes,
+  toClear?: Clear[]
+): Clear[] {
+  path = deviceData.paths.add(path);
+
+  const currentTimestamp = deviceData.timestamps.get(path);
+
+  let currentAttributes;
+
+  if (path.wildcard) attributes = undefined;
+  else if (currentTimestamp)
+    currentAttributes = deviceData.attributes.get(path);
+
+  let changeFlags = 0;
+
+  if (attributes) {
+    if (
+      attributes.value &&
+      attributes.value[1] &&
+      attributes.value[0] >= (attributes.object ? attributes.object[0] : 0)
+    )
+      attributes.object = [attributes.value[0], 0];
+
+    if (
+      attributes.object &&
+      attributes.object[1] &&
+      attributes.object[0] >= (attributes.value ? attributes.value[0] : 0)
+    )
+      attributes.value = [attributes.object[0]];
+
+    const newAttributes = Object.assign({}, currentAttributes, attributes);
+
+    if (currentAttributes) {
+      for (const attrName in attributes) {
+        timestamp = Math.max(timestamp, attributes[attrName][0]);
+        if (!(attrName in currentAttributes))
+          changeFlags |= CHANGE_FLAGS[attrName];
+        else if (attributes[attrName][0] <= currentAttributes[attrName][0])
+          newAttributes[attrName] = currentAttributes[attrName];
+        else if (
+          !compareEquality(
+            attributes[attrName][1],
+            currentAttributes[attrName][1]
+          )
+        )
+          changeFlags |= CHANGE_FLAGS[attrName];
+      }
+    } else {
+      changeFlags |= 1;
+    }
+
+    deviceData.attributes.set(path, newAttributes);
+
+    if (!(timestamp <= currentTimestamp)) {
+      deviceData.timestamps.set(path, timestamp);
+      if (path.length > 1) {
+        toClear = set(
+          deviceData,
+          path.slice(0, path.length - 1),
+          timestamp,
+          { object: [timestamp, 1] },
+          toClear
+        );
+      }
+    }
+  } else if (!(timestamp <= currentTimestamp)) {
+    deviceData.timestamps.set(path, timestamp);
+
+    if (currentAttributes) {
+      deviceData.attributes.delete(path);
+      changeFlags |= 1;
+    } else if (path.wildcard) {
+      for (const p of deviceData.paths.find(path, false, true, path.length)) {
+        if (timestamp > deviceData.timestamps.get(p)) {
+          toClear = toClear || [];
+          toClear.push([p, timestamp]);
+        }
+      }
+    }
+  }
+
+  if (changeFlags) {
+    if (changeFlags & 1) {
+      toClear = toClear || [];
+      toClear.push([path, timestamp, null, changeFlags]);
+    } else if (changeFlags & CHANGE_FLAGS.object) {
+      toClear = toClear || [];
+      toClear.push([path, 0, { object: attributes.object[0] }, changeFlags]);
+    } else {
+      for (const p of deviceData.paths.find(path, true, false, path.length)) {
+        const tracker = deviceData.trackers.get(p);
+        for (const k in tracker)
+          if (tracker[k] & changeFlags) deviceData.changes.add(k);
+      }
+    }
+  }
+
+  return toClear;
+}
+
+export function track(
+  deviceData: DeviceData,
+  path: Path,
+  marker: string,
+  attributes?
+): void {
+  path = deviceData.paths.add(path);
+  let f = 1;
+
+  if (attributes)
+    for (const attrName of attributes) f |= CHANGE_FLAGS[attrName];
+
+  let cur = deviceData.trackers.get(path);
+  if (!cur) {
+    cur = {};
+    deviceData.trackers.set(path, cur);
+  }
+
+  cur[marker] |= f;
+}
+
+export function clearTrackers(deviceData: DeviceData, tracker): void {
+  if (Array.isArray(tracker)) {
+    for (const v of deviceData.trackers.values())
+      for (const t of tracker) delete v[t];
+    for (const t of tracker) deviceData.changes.delete(t);
+  } else {
+    for (const v of deviceData.trackers.values()) delete v[tracker];
+    deviceData.changes.delete(tracker);
+  }
+}

lib/extensions.ts

@@ -0,0 +1,113 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { spawn, ChildProcess } from "child_process";
+import * as crypto from "crypto";
+import * as config from "./config";
+import { Fault } from "./types";
+
+const TIMEOUT = +config.get("EXT_TIMEOUT");
+
+const processes: { [script: string]: ChildProcess } = {};
+const jobs = new Map();
+
+function messageHandler(message): void {
+  const func = jobs.get(message[0]);
+  if (func) {
+    jobs.delete(message[0]);
+    func({ fault: message[1], value: message[2] });
+  }
+}
+
+export function run(args): Promise<{ fault: Fault; value: any }> {
+  return new Promise(resolve => {
+    const scriptName = args[0];
+
+    const id = crypto.randomBytes(8).toString("hex");
+    jobs.set(id, resolve);
+
+    if (!processes[scriptName]) {
+      const p = spawn("bin/genieacs-ext", [scriptName], {
+        stdio: ["inherit", "inherit", "inherit", "ipc"]
+      });
+      processes[scriptName] = p;
+
+      p.on("error", err => {
+        if (processes[scriptName] === p) {
+          if (jobs.delete(id)) {
+            resolve({
+              fault: { code: err.name, message: err.message },
+              value: null
+            });
+          }
+
+          kill(processes[scriptName]);
+          delete processes[scriptName];
+        }
+      });
+
+      p.on("disconnect", () => {
+        if (processes[scriptName] === p) delete processes[scriptName];
+      });
+
+      p.on("message", messageHandler);
+    }
+
+    setTimeout(() => {
+      if (jobs.delete(id)) {
+        resolve({
+          fault: { code: "timeout", message: "Extension timed out" },
+          value: null
+        });
+      }
+    }, TIMEOUT);
+
+    if (!processes[scriptName].connected) return false;
+
+    return processes[scriptName].send([id, args.slice(1)]);
+  });
+}
+
+function kill(process: ChildProcess): Promise<void> {
+  return new Promise(resolve => {
+    const timeToKill = Date.now() + 5000;
+
+    process.kill();
+
+    const t = setInterval(() => {
+      if (!process.connected) {
+        clearInterval(t);
+        resolve();
+      } else if (Date.now() > timeToKill) {
+        process.kill("SIGKILL");
+        clearInterval(t);
+        resolve();
+      }
+    }, 100);
+  });
+}
+
+export async function killAll(): Promise<void> {
+  await Promise.all(
+    Object.entries(processes).map(([k, p]) => {
+      delete processes[k];
+      return kill(p);
+    })
+  );
+}

lib/fs.coffee

@@ -1,49 +0,0 @@
-###
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-config = require './config'
-url = require 'url'
-mongodb = require 'mongodb'
-querystring = require 'querystring'
-db = require './db'
-
-
-listener = (request, response) ->
-  urlParts = url.parse(request.url, true)
-  if request.method == 'GET'
-    filename = querystring.unescape(urlParts.pathname.substring(1))
-    gs = new mongodb.GridStore(db.mongoDb, filename, 'r', {})
-    gs.open((err, gs) ->
-      if err
-        response.writeHead(404)
-        response.end()
-        return
-      stream = gs.stream(true)
-      response.writeHead(200, {'Content-Type' : 'application/octet-stream', 'Content-Length' : gs.length})
-      stream.pipe(response)
-    )
-  else
-    response.writeHead(405, {'Allow': 'GET'})
-    response.end('405 Method Not Allowed')
-
-
-exports.listener = listener

lib/fs.ts

@@ -0,0 +1,63 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as url from "url";
+import * as querystring from "querystring";
+import { GridFSBucket } from "mongodb";
+import * as db from "./db";
+import * as logger from "./logger";
+
+export function listener(request, response): void {
+  const urlParts = url.parse(request.url, true);
+  if (request.method === "GET") {
+    const filename = querystring.unescape(urlParts.pathname.substring(1));
+
+    const log = {
+      message: "Fetch file",
+      filename: filename,
+      remoteAddress: request.connection.remoteAddress
+    };
+
+    db.filesCollection.findOne({ _id: filename }, (err, file) => {
+      if (err) throw err;
+
+      if (!file) {
+        response.writeHead(404);
+        response.end();
+        log.message += " not found";
+        logger.accessError(log);
+        return;
+      }
+
+      response.writeHead(200, {
+        "Content-Type": file.contentType || "application/octet-stream",
+        "Content-Length": file.length
+      });
+
+      const bucket = new GridFSBucket(db.client.db());
+      const downloadStream = bucket.openDownloadStreamByName(filename);
+      downloadStream.pipe(response);
+
+      logger.accessInfo(log);
+    });
+  } else {
+    response.writeHead(405, { Allow: "GET" });
+    response.end("405 Method Not Allowed");
+  }
+}

lib/gpn-heuristic.ts

@@ -0,0 +1,78 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+const WILDCARD_MULTIPLIER = 2;
+const UNDISCOVERED_DEPTH = 7;
+
+// Simple heuristic to estimate GPN count given a set of patterns to be
+// discovered. Used to decide whether to use nextLevel = false in GPN.
+// gpnPatterns is [pattern, flags]
+// pattern is a path (array)
+// flags is an int where its bits mark the segments in the pattern that
+// need refreshing. Leading 0s indicate that the pattern up to that
+// point has been discovered.
+export function estimateGpnCount(gpnPatterns, depth = 0): number {
+  const children = {};
+  const wildcardChildren = [];
+  let wildcardDiscovered = false;
+  let gpnCount = 0;
+
+  for (const pattern of gpnPatterns) {
+    const path = pattern[0];
+    const flags = pattern[1] >> depth;
+
+    const k = path[depth];
+
+    if (!k) {
+      if (flags & 1) gpnCount = 1;
+      continue;
+    }
+
+    if (flags & 1) {
+      gpnCount = 1;
+
+      if (depth > UNDISCOVERED_DEPTH) continue;
+    } else if (k === "*") {
+      wildcardDiscovered = true;
+    }
+
+    if (k === "*") {
+      wildcardChildren.push(pattern);
+    } else {
+      children[k] = children[k] || [];
+      children[k].push(pattern);
+    }
+  }
+
+  let wildcardGpnCount = 0;
+  if (!wildcardDiscovered && wildcardChildren.length) {
+    wildcardGpnCount +=
+      estimateGpnCount(wildcardChildren, depth + 1) * WILDCARD_MULTIPLIER;
+  }
+
+  for (const k of Object.keys(children)) {
+    const c = estimateGpnCount(children[k].concat(wildcardChildren), depth + 1);
+    wildcardGpnCount -= c;
+    gpnCount += c;
+  }
+
+  gpnCount += Math.max(0, wildcardGpnCount);
+
+  return gpnCount;
+}

lib/init.ts

@@ -0,0 +1,425 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as localCache from "./local-cache";
+import { generateSalt, hashPassword } from "./auth";
+import * as db from "./ui/db";
+import { del } from "./cache";
+
+interface Status {
+  users: boolean;
+  presets: boolean;
+  filters: boolean;
+  device: boolean;
+  index: boolean;
+  overview: boolean;
+}
+
+const BOOTSTRAP_SCRIPT = `
+const now = Date.now();
+
+// Clear cached data model to force a refresh
+clear("Device", now);
+clear("InternetGatewayDevice", now);
+`.trim();
+
+const DEFAULT_SCRIPT = `
+const hourly = Date.now(3600000);
+
+// Refresh basic parameters hourly
+declare("InternetGatewayDevice.DeviceInfo.HardwareVersion", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.DeviceInfo.SoftwareVersion", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID", {path: hourly, value: hourly});
+// Don't refresh password field periodically because CPEs always report blank passowrds for security reasons
+declare("InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase", {path: hourly, value: 1});
+declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress", {path: hourly, value: hourly});
+`.trim();
+
+const INFORM_SCRIPT = `
+// Device ID as user name
+const username = declare("DeviceID.ID", {value: 1}).value[0]
+
+// Password will be fixed a given device because Math.random() is seeded with devcie ID by default.
+const password = "" + Math.random();
+
+const informInterval = 300;
+
+// Refresh values daily
+const daily = Date.now(86400000);
+
+// Unique inform offset per device for better load distribution
+const informTime = daily % 86400000;
+
+declare("InternetGatewayDevice.ManagementServer.ConnectionRequestUsername", {value: daily}, {value: username});
+declare("InternetGatewayDevice.ManagementServer.ConnectionRequestPassword", {value: daily}, {value: password});
+declare("InternetGatewayDevice.ManagementServer.PeriodicInformEnable", {value: daily}, {value: true});
+declare("InternetGatewayDevice.ManagementServer.PeriodicInformInterval", {value: daily}, {value: informInterval});
+declare("InternetGatewayDevice.ManagementServer.PeriodicInformTime", {value: daily}, {value: informTime});
+
+declare("Device.ManagementServer.ConnectionRequestUsername", {value: daily}, {value: username});
+declare("Device.ManagementServer.ConnectionRequestPassword", {value: daily}, {value: password});
+declare("Device.ManagementServer.PeriodicInformEnable", {value: daily}, {value: true});
+declare("Device.ManagementServer.PeriodicInformInterval", {value: daily}, {value: informInterval});
+declare("Device.ManagementServer.PeriodicInformTime", {value: daily}, {value: informTime});
+`.trim();
+
+export async function getStatus(): Promise<Status> {
+  const configSnapshot = await localCache.getCurrentSnapshot();
+  const users = localCache.getUsers(configSnapshot);
+  const presets = localCache.getPresets(configSnapshot);
+  const ui = localCache.getUiConfig(configSnapshot);
+
+  return {
+    users: !Object.keys(users).length,
+    presets: !presets.length,
+    filters: !Object.keys(ui["filters"]).length,
+    device: !Object.keys(ui["device"]).length,
+    index: !Object.keys(ui["index"]).length,
+    overview: !Object.keys(ui["overview"]).length
+  };
+}
+
+export async function seed(options): Promise<void> {
+  const resources = {};
+  const proms = [];
+
+  if (options.users) {
+    resources["permissions"] = [
+      { role: "admin", resource: "devices", access: 3, validate: "true" },
+      { role: "admin", resource: "faults", access: 3, validate: "true" },
+      { role: "admin", resource: "files", access: 3, validate: "true" },
+      { role: "admin", resource: "presets", access: 3, validate: "true" },
+      { role: "admin", resource: "provisions", access: 3, validate: "true" },
+      { role: "admin", resource: "config", access: 3, validate: "true" },
+      { role: "admin", resource: "permissions", access: 3, validate: "true" },
+      { role: "admin", resource: "users", access: 3, validate: "true" },
+      {
+        role: "admin",
+        resource: "virtualParameters",
+        access: 3,
+        validate: "true"
+      }
+    ];
+
+    resources["users"] = [
+      { username: "admin", password: "admin", roles: ["admin"] }
+    ];
+  }
+
+  if (options.filters) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.filters.0.label", value: "'Serial number'" },
+      { _id: "ui.filters.0.parameter", value: "DeviceID.SerialNumber" },
+      { _id: "ui.filters.0.type", value: "'string'" },
+      { _id: "ui.filters.1.label", value: "'Product class'" },
+      { _id: "ui.filters.1.parameter", value: "DeviceID.ProductClass" },
+      { _id: "ui.filters.1.type", value: "'string'" },
+      { _id: "ui.filters.2.label", value: "'Tag'" },
+      { _id: "ui.filters.2.parameter", value: "tag" },
+      { _id: "ui.filters.2.type", value: "'string'" }
+    ]);
+  }
+
+  if (options.device) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.device.0.type", value: "'tags'" },
+      { _id: "ui.device.1.type", value: "'ping'" },
+      { _id: "ui.device.2.type", value: "'parameter-list'" },
+      { _id: "ui.device.2.parameters.0.type", value: "'container'" },
+      { _id: "ui.device.2.parameters.0.element", value: "'span.inform'" },
+      { _id: "ui.device.2.parameters.0.label", value: "'Last inform'" },
+      {
+        _id: "ui.device.2.parameters.0.components.0.type",
+        value: "'parameter'"
+      },
+      { _id: "ui.device.2.parameters.0.components.1.chart", value: "'online'" },
+      {
+        _id: "ui.device.2.parameters.0.components.1.type",
+        value: "'overview-dot'"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.type",
+        value: "'summon-button'"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.0",
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.1",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.2",
+        value:
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.3",
+        value:
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.4",
+        value: "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.5",
+        value:
+          "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.6",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.7",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress"
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.8",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress"
+      },
+      {
+        _id: "ui.device.2.parameters.0.parameter",
+        value: "DATE_STRING(Events.Inform)"
+      },
+      { _id: "ui.device.2.parameters.1.label", value: "'Serial number'" },
+      {
+        _id: "ui.device.2.parameters.1.parameter",
+        value: "DeviceID.SerialNumber"
+      },
+      { _id: "ui.device.2.parameters.2.label", value: "'Product class'" },
+      {
+        _id: "ui.device.2.parameters.2.parameter",
+        value: "DeviceID.ProductClass"
+      },
+      { _id: "ui.device.2.parameters.3.label", value: "'OUI'" },
+      {
+        _id: "ui.device.2.parameters.3.parameter",
+        value: "DeviceID.OUI"
+      },
+      { _id: "ui.device.2.parameters.4.label", value: "'Manufacturer'" },
+      {
+        _id: "ui.device.2.parameters.4.parameter",
+        value: "DeviceID.Manufacturer"
+      },
+      { _id: "ui.device.2.parameters.5.label", value: "'Hardware version'" },
+      {
+        _id: "ui.device.2.parameters.5.parameter",
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion"
+      },
+      { _id: "ui.device.2.parameters.6.label", value: "'Software version'" },
+      {
+        _id: "ui.device.2.parameters.6.parameter",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+      },
+      { _id: "ui.device.2.parameters.7.label", value: "'MAC'" },
+      {
+        _id: "ui.device.2.parameters.7.parameter",
+        value:
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress"
+      },
+      { _id: "ui.device.2.parameters.8.label", value: "'IP'" },
+      {
+        _id: "ui.device.2.parameters.8.parameter",
+        value:
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress"
+      },
+      { _id: "ui.device.2.parameters.9.label", value: "'WLAN SSID'" },
+      {
+        _id: "ui.device.2.parameters.9.parameter",
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID"
+      },
+      { _id: "ui.device.2.parameters.10.label", value: "'WLAN passphrase'" },
+      {
+        _id: "ui.device.2.parameters.10.parameter",
+        value:
+          "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase"
+      },
+      { _id: "ui.device.3.type", value: "'parameter-table'" },
+      { _id: "ui.device.3.label", value: "'WLAN hosts'" },
+      {
+        _id: "ui.device.3.parameter",
+        value: "InternetGatewayDevice.LANDevice.1.Hosts.Host"
+      },
+      { _id: "ui.device.3.childParameters.0.label", value: "'Host name'" },
+      { _id: "ui.device.3.childParameters.0.parameter", value: "HostName" },
+      { _id: "ui.device.3.childParameters.1.label", value: "'IP address'" },
+      { _id: "ui.device.3.childParameters.1.parameter", value: "IPAddress" },
+      { _id: "ui.device.3.childParameters.2.label", value: "'MAC address'" },
+      { _id: "ui.device.3.childParameters.2.parameter", value: "MACAddress" },
+      { _id: "ui.device.4.type", value: "'container'" },
+      { _id: "ui.device.4.element", value: "'div'" },
+      { _id: "ui.device.4.components.0.components.0", value: "'Faults'" },
+      { _id: "ui.device.4.components.0.element", value: "'h3'" },
+      { _id: "ui.device.4.components.0.type", value: "'container'" },
+      { _id: "ui.device.4.components.1.type", value: "'device-faults'" },
+      { _id: "ui.device.5.type", value: "'container'" },
+      { _id: "ui.device.5.element", value: "'div.container-full-width'" },
+      {
+        _id: "ui.device.5.components.0.components.0",
+        value: "'All parameters'"
+      },
+      { _id: "ui.device.5.components.0.element", value: "'h3'" },
+      { _id: "ui.device.5.components.0.type", value: "'container'" },
+      { _id: "ui.device.5.components.1.type", value: "'all-parameters'" },
+      { _id: "ui.device.6.type", value: "'device-actions'" }
+    ]);
+  }
+
+  if (options.index) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.index.0.type", value: "'device-link'" },
+      { _id: "ui.index.0.label", value: "'Serial number'" },
+      { _id: "ui.index.0.parameter", value: "DeviceID.SerialNumber" },
+      { _id: "ui.index.0.components.0.type", value: "'parameter'" },
+      { _id: "ui.index.1.label", value: "'Product class'" },
+      { _id: "ui.index.1.parameter", value: "DeviceID.ProductClass" },
+      { _id: "ui.index.2.label", value: "'Software version'" },
+      {
+        _id: "ui.index.2.parameter",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+      },
+      { _id: "ui.index.3.label", value: "'IP'" },
+      {
+        _id: "ui.index.3.parameter",
+        value:
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress"
+      },
+      { _id: "ui.index.4.label", value: "'SSID'" },
+      {
+        _id: "ui.index.4.parameter",
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID"
+      },
+      { _id: "ui.index.5.type", value: "'container'" },
+      { _id: "ui.index.5.label", value: "'Last inform'" },
+      { _id: "ui.index.5.element", value: "'span.inform'" },
+      { _id: "ui.index.5.parameter", value: "DATE_STRING(Events.Inform)" },
+      { _id: "ui.index.5.components.0.type", value: "'parameter'" },
+      { _id: "ui.index.5.components.1.chart", value: "'online'" },
+      { _id: "ui.index.5.components.1.type", value: "'overview-dot'" },
+      { _id: "ui.index.6.type", value: "'tags'" },
+      { _id: "ui.index.6.label", value: "'Tags'" },
+      { _id: "ui.index.6.parameter", value: "Tags" },
+      { _id: "ui.index.6.unsortable", value: "true" },
+      { _id: "ui.index.6.writable", value: "false" }
+    ]);
+  }
+
+  if (options.overview) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.overview.charts.online.label", value: "'Online status'" },
+      {
+        _id: "ui.overview.charts.online.slices.1_onlineNow.color",
+        value: "'#31a354'"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.1_onlineNow.filter",
+        value: "Events.Inform > NOW() - 5 * 60 * 1000"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.1_onlineNow.label",
+        value: "'Online now'"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.2_past24.color",
+        value: "'#a1d99b'"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.2_past24.filter",
+        value:
+          "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.2_past24.label",
+        value: "'Past 24 hours'"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.3_others.color",
+        value: "'#e5f5e0'"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.3_others.filter",
+        value: "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)"
+      },
+      {
+        _id: "ui.overview.charts.online.slices.3_others.label",
+        value: "'Others'"
+      },
+      { _id: "ui.overview.groups.online.label", value: "''" },
+      { _id: "ui.overview.groups.online.charts.0", value: "'online'" }
+    ]);
+  }
+
+  if (options.presets) {
+    resources["presets"] = [
+      {
+        _id: "bootstrap",
+        weight: 0,
+        channel: "bootstrap",
+        provision: "bootstrap"
+      },
+      { _id: "default", weight: 0, channel: "default", provision: "default" },
+      { _id: "inform", weight: 0, channel: "inform", provision: "inform" }
+    ];
+
+    resources["provisions"] = [
+      { _id: "bootstrap", script: BOOTSTRAP_SCRIPT },
+      { _id: "default", script: DEFAULT_SCRIPT },
+      { _id: "inform", script: INFORM_SCRIPT }
+    ];
+  }
+
+  if (resources["permissions"]) {
+    for (const p of resources["permissions"]) {
+      p["_id"] = `${p["role"]}:${p["resource"]}:${p["access"]}`;
+      proms.push(db.putPermission(p["_id"], p));
+    }
+  }
+
+  if (resources["users"]) {
+    for (const u of resources["users"]) {
+      u["salt"] = await generateSalt(64);
+      u["password"] = await hashPassword(u["password"], u["salt"]);
+      u["roles"] = (u["roles"] || []).join(",");
+      u["_id"] = u["username"];
+      delete u["username"];
+      proms.push(db.putUser(u["_id"], u));
+    }
+  }
+
+  if (resources["provisions"]) {
+    for (const p of resources["provisions"])
+      proms.push(db.putProvision(p["_id"], p));
+  }
+
+  if (resources["presets"])
+    for (const p of resources["presets"]) proms.push(db.putPreset(p["_id"], p));
+
+  if (resources["config"])
+    for (const c of resources["config"]) proms.push(db.putConfig(c["_id"], c));
+
+  await proms;
+  return del("presets_hash");
+}

lib/instance-set.ts

@@ -0,0 +1,131 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+interface Instance {
+  [name: string]: string;
+}
+
+export default class InstanceSet {
+  private set: Set<Instance>;
+
+  public constructor() {
+    this.set = new Set();
+  }
+
+  public add(instance: Instance): void {
+    this.set.add(instance);
+  }
+
+  public delete(instance: Instance): void {
+    this.set.delete(instance);
+  }
+
+  public superset(instance: Instance): Instance[] {
+    const res = [];
+    for (const inst of this.set) {
+      let match = true;
+      for (const k in instance) {
+        if (inst[k] !== instance[k]) {
+          match = false;
+          break;
+        }
+      }
+
+      if (match) res.push(inst);
+    }
+
+    res.sort((a, b) => {
+      const keysA = Object.keys(a);
+      const keysB = Object.keys(b);
+
+      if (keysA.length !== keysB.length) return keysB.length - keysA.length;
+
+      keysA.sort();
+      keysB.sort();
+
+      for (let i = 0; i < keysA.length; ++i) {
+        if (keysA[i] > keysB[i]) return 1;
+        else if (keysA[i] < keysB[i]) return -1;
+        else if (a[keysA[i]] > b[keysB[i]]) return 1;
+        else if (a[keysA[i]] < b[keysB[i]]) return -1;
+      }
+
+      return 0;
+    });
+
+    return res;
+  }
+
+  public subset(instance: Instance): Instance[] {
+    const res = [];
+
+    for (const inst of this.set) {
+      let match = true;
+      for (const k in inst) {
+        if (inst[k] !== instance[k]) {
+          match = false;
+          break;
+        }
+      }
+
+      if (match) res.push(inst);
+    }
+
+    res.sort((a, b) => {
+      const keysA = Object.keys(a);
+      const keysB = Object.keys(b);
+
+      if (keysA.length !== keysB.length) return keysA.length - keysB.length;
+
+      keysA.sort();
+      keysB.sort();
+
+      for (let i = 0; i < keysA.length; ++i) {
+        if (keysA[i] > keysB[i]) return 1;
+        else if (keysA[i] < keysB[i]) return -1;
+        else if (a[keysA[i]] > b[keysB[i]]) return 1;
+        else if (a[keysA[i]] < b[keysB[i]]) return -1;
+      }
+
+      return 0;
+    });
+
+    return res;
+  }
+
+  public [Symbol.iterator](): IterableIterator<Instance> {
+    return this.set.values();
+  }
+
+  public forEach(callback: (instance: Instance) => void): void {
+    this.set.forEach(callback);
+  }
+
+  public values(): IterableIterator<Instance> {
+    return this.set.values();
+  }
+
+  public clear(): void {
+    this.set.clear();
+  }
+
+  public get size(): number {
+    return this.set.size;
+  }
+}

lib/local-cache.ts

@@ -0,0 +1,524 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as vm from "vm";
+import * as crypto from "crypto";
+import * as config from "./config";
+import * as db from "./db";
+import * as cache from "./cache";
+import { mongoQueryToFilter } from "./mongodb-functions";
+import * as logger from "./logger";
+import * as scheduling from "./scheduling";
+import * as expression from "./common/expression";
+import { parse } from "./common/expression-parser";
+import { Preset, Expression } from "./types";
+
+interface Permissions {
+  [role: string]: {
+    [access: number]: {
+      [resource: string]: {
+        access: number;
+        filter: Expression;
+        validate?: Expression;
+      };
+    };
+  };
+}
+
+const REFRESH = 3000;
+const EVICT_TIMEOUT = 60000;
+
+const snapshots = new Map();
+let currentSnapshot = null;
+let nextRefresh = 1;
+
+function computeHash(snapshot): string {
+  // MD5 hash for presets, provisions, virtual parameters for detecting changes
+  const h = crypto.createHash("md5");
+  for (const p of snapshot.presets) {
+    h.update(JSON.stringify(p.name));
+    h.update(JSON.stringify(p.channel));
+    h.update(JSON.stringify(p.schedule));
+    h.update(JSON.stringify(p.events));
+    h.update(JSON.stringify(p.precondition));
+    h.update(JSON.stringify(p.provisions));
+  }
+
+  let keys;
+
+  keys = Object.keys(snapshot.provisions).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(snapshot.provisions[k].md5);
+
+  keys = Object.keys(snapshot.virtualParameters).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(snapshot.virtualParameters[k].md5);
+
+  keys = Object.keys(snapshot.config).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(JSON.stringify(snapshot.config[k]));
+
+  keys = Object.keys(snapshot.files).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(JSON.stringify(snapshot.files[k]));
+
+  keys = Object.keys(snapshot.users).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(JSON.stringify(snapshot.users[k]));
+
+  const roles = Object.keys(snapshot.permissions).sort();
+  h.update(JSON.stringify(roles));
+  for (const r of roles) {
+    const levels = Object.keys(snapshot.permissions[r]).sort();
+    h.update(JSON.stringify(levels));
+    for (const l of levels) {
+      keys = Object.keys(snapshot.permissions[r][l]).sort();
+      h.update(JSON.stringify(keys));
+      for (const k of keys)
+        h.update(JSON.stringify(snapshot.permissions[r][l][k]));
+    }
+  }
+
+  return h.digest("hex");
+}
+
+function flattenObject(src, prefix = "", dst = {}): {} {
+  for (const k of Object.keys(src)) {
+    const v = src[k];
+    if (typeof v === "object" && !Array.isArray(v))
+      flattenObject(v, `${prefix}${k}.`, dst);
+    else dst[`${prefix}${k}`] = v;
+  }
+  return dst;
+}
+
+async function fetchPresets(): Promise<Preset[]> {
+  const res = await db.getPresets();
+  let objects = await db.getObjects();
+
+  objects = objects.map(obj => {
+    // Flatten object
+    obj = flattenObject(obj);
+
+    // If no keys are defined, consider all parameters as keys to keep the
+    // same behavior from v1.0
+    if (!obj["_keys"] || !obj["_keys"].length)
+      obj["_keys"] = Object.keys(obj).filter(k => !k.startsWith("_"));
+
+    return obj;
+  });
+
+  res.sort((a, b) => {
+    if (a["weight"] === b["weight"])
+      return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
+    else return a["weight"] - b["weight"];
+  });
+
+  const presets = [] as Preset[];
+  for (const preset of res) {
+    let schedule: { md5: string; duration: number; schedule: any } = null;
+    if (preset["schedule"]) {
+      const parts = preset["schedule"].trim().split(/\s+/);
+      schedule = {
+        md5: crypto
+          .createHash("md5")
+          .update(preset["schedule"])
+          .digest("hex"),
+        duration: null,
+        schedule: null
+      };
+
+      try {
+        schedule.duration = +parts.shift() * 1000;
+        schedule.schedule = scheduling.parseCron(parts.join(" "));
+      } catch (err) {
+        logger.warn({
+          message: "Invalid preset schedule",
+          preset: preset["_id"],
+          schedule: preset["schedule"]
+        });
+        schedule.schedule = false;
+      }
+    }
+
+    const events = preset["events"] || {};
+    let precondition = true as Expression;
+    if (preset["precondition"]) {
+      try {
+        precondition = parse(preset["precondition"]);
+      } catch (error) {
+        precondition = mongoQueryToFilter(JSON.parse(preset["precondition"]));
+      }
+
+      // Simplify expression
+      precondition = expression.evaluate(precondition);
+    }
+
+    const _provisions = preset["provisions"] || [];
+
+    // Generate provisions from the old configuration format
+    for (const c of preset["configurations"]) {
+      switch (c.type) {
+        case "age":
+          _provisions.push(["refresh", c.name, c.age]);
+          break;
+
+        case "value":
+          _provisions.push(["value", c.name, c.value]);
+          break;
+
+        case "add_tag":
+          _provisions.push(["tag", c.tag, true]);
+          break;
+
+        case "delete_tag":
+          _provisions.push(["tag", c.tag, false]);
+          break;
+
+        case "provision":
+          _provisions.push([c.name].concat(c.args || []));
+          break;
+
+        case "add_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map(k => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 1]);
+
+              for (const k in obj) {
+                if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
+                  _provisions.push(["value", `${p}.${k}`, obj[k]]);
+              }
+            }
+          }
+
+          break;
+
+        case "delete_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map(k => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 0]);
+            }
+          }
+
+          break;
+
+        default:
+          throw new Error(`Unknown configuration type ${c.type}`);
+      }
+    }
+
+    presets.push({
+      name: preset["_id"],
+      channel: (preset["channel"] as string) || "default",
+      schedule: schedule,
+      events: events,
+      precondition: precondition,
+      provisions: _provisions
+    });
+  }
+
+  return presets;
+}
+
+async function fetchProvisions(): Promise<{}> {
+  const res = await db.getProvisions();
+
+  const provisions = {};
+  for (const r of res) {
+    provisions[r["_id"]] = {};
+    provisions[r["_id"]].md5 = crypto
+      .createHash("md5")
+      .update(r["script"])
+      .digest("hex");
+    provisions[r["_id"]].script = new vm.Script(
+      `"use strict";(function(){\n${r["script"]}\n})();`,
+      { filename: r["_id"], lineOffset: -1, timeout: 50 }
+    );
+  }
+
+  return provisions;
+}
+
+async function fetchVirtualParameters(): Promise<{}> {
+  const res = await db.getVirtualParameters();
+
+  const virtualParameters = {};
+  for (const r of res) {
+    virtualParameters[r["_id"]] = {};
+    virtualParameters[r["_id"]].md5 = crypto
+      .createHash("md5")
+      .update(r["script"])
+      .digest("hex");
+    virtualParameters[r["_id"]].script = new vm.Script(
+      `"use strict";(function(){\n${r["script"]}\n})();`,
+      { filename: r["_id"], lineOffset: -1, timeout: 50 }
+    );
+  }
+
+  return virtualParameters;
+}
+
+async function fetchPermissions(): Promise<Permissions> {
+  const perms = await db.getPermissions();
+  const permissions: Permissions = {};
+
+  for (const p of perms) {
+    if (!permissions[p.role]) permissions[p.role] = {};
+    if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {};
+
+    permissions[p.role][p.access][p.resource] = {
+      access: p.access,
+      filter: parse(p.filter || "true")
+    };
+    if (p.validate)
+      permissions[p.role][p.access][p.resource].validate = parse(p.validate);
+  }
+
+  return permissions;
+}
+
+async function fetchFiles(): Promise<{}> {
+  const res = await db.getFiles();
+  const files = {};
+
+  for (const r of res) {
+    const id = r["filename"] || r["_id"].toString();
+    files[id] = {};
+    files[id].length = r["length"];
+    files[id].md5 = r["md5"];
+    files[id].contentType = r["contentType"];
+  }
+
+  return files;
+}
+
+async function fetchUsers(): Promise<{}> {
+  const _users = await db.getUsers();
+  const users = {};
+
+  for (const user of _users) {
+    users[user._id] = {
+      password: user.password,
+      salt: user.salt,
+      roles: user.roles.split(",").map(s => s.trim())
+    };
+  }
+
+  return users;
+}
+
+async function fetchConfig(): Promise<[{}, {}]> {
+  const conf = await db.getConfig();
+
+  conf.sort((a, b) => (a.id > b.id ? 1 : a.id < b.id ? -1 : 0));
+
+  const ui = {
+    filters: {},
+    device: {},
+    index: {},
+    overview: {}
+  };
+  const _config = {};
+
+  for (const c of conf) {
+    // Evaluate expressions to simplify them
+    const val = expression.evaluate(c.value);
+    _config[c.id] = val;
+    if (c.id.startsWith("ui.")) {
+      const keys = c.id.split(".");
+      // remove the first key(ui)
+      keys.shift();
+      let ref = ui;
+      while (keys.length > 1) {
+        const k = keys.shift();
+        if (typeof ref[k] !== "object") ref[k] = {};
+        ref = ref[k];
+      }
+      ref[keys[0]] = val;
+    }
+  }
+
+  return [_config, ui];
+}
+
+async function refresh(): Promise<void> {
+  if (!nextRefresh) {
+    await new Promise(resolve => setTimeout(resolve, 20));
+    await refresh();
+    return;
+  }
+
+  nextRefresh = 0;
+  const now = Date.now();
+
+  const dbHash = await cache.get("presets_hash");
+
+  if (currentSnapshot && dbHash === currentSnapshot) {
+    nextRefresh = now + (REFRESH - (now % REFRESH));
+    return;
+  }
+
+  const unlockOrExtend = await cache.lock("presets_hash_lock", 3);
+
+  const res = await Promise.all([
+    fetchPresets(),
+    fetchProvisions(),
+    fetchVirtualParameters(),
+    fetchFiles(),
+    fetchPermissions(),
+    fetchUsers(),
+    fetchConfig()
+  ]);
+
+  const snapshot = {
+    presets: res[0],
+    provisions: res[1],
+    virtualParameters: res[2],
+    files: res[3],
+    permissions: res[4],
+    users: res[5],
+    config: res[6][0],
+    ui: res[6][1]
+  };
+
+  if (currentSnapshot) {
+    const h = currentSnapshot;
+    const s = snapshots.get(h);
+    setTimeout(() => {
+      if (snapshots.get(h) === s) snapshots.delete(h);
+    }, EVICT_TIMEOUT).unref();
+  }
+
+  currentSnapshot = computeHash(snapshot);
+  snapshots.set(currentSnapshot, snapshot);
+  await cache.set("presets_hash", currentSnapshot, 300);
+  await unlockOrExtend(0);
+
+  nextRefresh = now + (REFRESH - (now % REFRESH));
+}
+
+export async function getCurrentSnapshot(): Promise<string> {
+  if (Date.now() > nextRefresh) await refresh();
+  return currentSnapshot;
+}
+
+export function hasSnapshot(hash): boolean {
+  return snapshots.has(hash);
+}
+
+export function getPresets(snapshotKey): Preset[] {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+  return snapshot.presets;
+}
+
+export function getProvisions(
+  snapshotKey
+): { [name: string]: { md5: string; script: vm.Script } } {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+  return snapshot.provisions;
+}
+
+export function getVirtualParameters(
+  snapshotKey
+): { [name: string]: { md5: string; script: vm.Script } } {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+  return snapshot.virtualParameters;
+}
+
+export function getFiles(
+  snapshotKey
+): { [name: string]: { length: number; md5: string; contentType: string } } {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+  return snapshot.files;
+}
+
+export function getConfig(
+  snapshotKey,
+  key,
+  context?,
+  now?
+): string | number | boolean | null {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  const oldOpts = {
+    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
+    "cwmp.debug": "DEBUG",
+    "cwmp.retryDelay": "RETRY_DELAY",
+    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
+    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
+    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
+    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
+    "cwmp.cookiesPath": "COOKIES_PATH",
+    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
+    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
+    "cwmp.connectionRequestAllowBasicAuth":
+      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
+    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
+    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT"
+  };
+
+  if (!(key in snapshot.config)) {
+    if (key in oldOpts)
+      return config.get(oldOpts[key], context ? context.id : null);
+    else return null;
+  }
+  return expression.evaluate(snapshot.config[key], context, now || Date.now());
+}
+
+export function getConfigExpression(snapshotKey, key): Expression {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  return snapshot.config[key];
+}
+
+export function getUsers(snapshotKey): {} {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  return snapshot.users;
+}
+
+export function getPermissions(snapshotKey): {} {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  return snapshot.permissions;
+}
+
+export function getUiConfig(snapshotKey): {} {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  return snapshot.ui;
+}

lib/logger.ts

@@ -0,0 +1,319 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as fs from "fs";
+import * as os from "os";
+
+import * as config from "./config";
+
+const REOPEN_EVERY = 60000;
+
+const LOG_FORMAT = config.get("LOG_FORMAT");
+const ACCESS_LOG_FORMAT = config.get("ACCESS_LOG_FORMAT") || LOG_FORMAT;
+
+const defaultMeta: { [name: string]: any } = {};
+
+let LOG_SYSTEMD = false;
+let ACCESS_LOG_SYSTEMD = false;
+
+let LOG_FILE, ACCESS_LOG_FILE;
+
+declare global {
+  /* eslint-disable-next-line @typescript-eslint/no-namespace */
+  namespace NodeJS {
+    export interface WritableStream {
+      fd?: number;
+    }
+  }
+}
+
+declare module "fs" {
+  interface WriteStream {
+    fd?: number;
+  }
+}
+
+let logStream = fs.createWriteStream(null, { fd: process.stderr.fd });
+let logStat = fs.fstatSync(logStream.fd);
+let accessLogStream = fs.createWriteStream(null, { fd: process.stdout.fd });
+let accessLogStat = fs.fstatSync(accessLogStream.fd);
+
+// Reopen if original files have been moved (e.g. logrotate)
+function reopen(): void {
+  let counter = 1;
+
+  if (LOG_FILE) {
+    ++counter;
+    fs.stat(LOG_FILE, (err, stat) => {
+      if (err && !err.message.startsWith("ENOENT:")) throw err;
+
+      if (!(stat && stat.dev === logStat.dev && stat.ino === logStat.ino)) {
+        logStream.end();
+        logStream = fs.createWriteStream(null, {
+          fd: fs.openSync(LOG_FILE, "a")
+        });
+        logStat = fs.fstatSync(logStream.fd);
+      }
+
+      if (--counter === 0)
+        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+    });
+  }
+
+  if (ACCESS_LOG_FILE) {
+    ++counter;
+    fs.stat(ACCESS_LOG_FILE, (err, stat) => {
+      if (err && !err.message.startsWith("ENOENT:")) throw err;
+
+      if (
+        !(
+          stat &&
+          stat.dev === accessLogStat.dev &&
+          stat.ino === accessLogStat.ino
+        )
+      ) {
+        accessLogStream.end();
+        accessLogStream = fs.createWriteStream(null, {
+          fd: fs.openSync(ACCESS_LOG_FILE, "a")
+        });
+        accessLogStat = fs.fstatSync(accessLogStream.fd);
+      }
+
+      if (--counter === 0)
+        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+    });
+  }
+
+  if (--counter === 0)
+    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+}
+
+export function init(service, version): void {
+  defaultMeta.hostname = os.hostname();
+  defaultMeta.pid = process.pid;
+  defaultMeta.name = `genieacs-${service}`;
+  defaultMeta.version = version;
+
+  LOG_FILE = config.get(`${service.toUpperCase()}_LOG_FILE`);
+  ACCESS_LOG_FILE = config.get(`${service.toUpperCase()}_ACCESS_LOG_FILE`);
+
+  if (LOG_FILE) {
+    logStream = fs.createWriteStream(null, { fd: fs.openSync(LOG_FILE, "a") });
+    logStat = fs.fstatSync(logStream.fd);
+  }
+
+  if (ACCESS_LOG_FILE) {
+    accessLogStream = fs.createWriteStream(null, {
+      fd: fs.openSync(ACCESS_LOG_FILE, "a")
+    });
+    accessLogStat = fs.fstatSync(accessLogStream.fd);
+  }
+
+  // Determine if logs are going to journald
+  const JOURNAL_STREAM = process.env["JOURNAL_STREAM"];
+
+  if (JOURNAL_STREAM) {
+    const [dev, inode] = JOURNAL_STREAM.split(":").map(parseInt);
+
+    LOG_SYSTEMD = logStat.dev === dev && logStat.ino === inode;
+    ACCESS_LOG_SYSTEMD =
+      accessLogStat.dev === dev && accessLogStat.ino === inode;
+  }
+
+  if (LOG_FILE || ACCESS_LOG_FILE)
+    // Can't use setInterval as we need all workers to cehck at the same time
+    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+}
+
+export function close(): void {
+  accessLogStream.end();
+  logStream.end();
+}
+
+export function flatten(details): {} {
+  if (details.sessionContext) {
+    details.deviceId = details.sessionContext.deviceId;
+    details.remoteAddress =
+      details.sessionContext.httpRequest.connection.remoteAddress;
+    delete details.sessionContext;
+  }
+
+  if (details.exception) {
+    details.exceptionName = details.exception.name;
+    details.exceptionMessage = details.exception.message;
+    details.exceptionStack = details.exception.stack;
+    delete details.exception;
+  }
+
+  if (details.task) {
+    details.taskId = details.task._id;
+    delete details.task;
+  }
+
+  if (details.rpc) {
+    if (details.rpc.acsRequest) {
+      details.acsRequestId = details.rpc.id;
+      details.acsRequestName = details.rpc.acsRequest.name;
+      if (details.rpc.acsRequest.commandKey)
+        details.acsRequestCommandKey = details.rpc.acsRequest.commandKey;
+    } else if (details.rpc.cpeRequest) {
+      details.cpeRequestId = details.rpc.id;
+      if (details.rpc.cpeRequest.name === "Inform") {
+        details.informEvent = details.rpc.cpeRequest.event.join(",");
+        details.informRetryCount = details.rpc.cpeRequest.retryCount;
+      } else {
+        details.cpeRequestName = details.rpc.cpeRequest.name;
+        if (details.rpc.cpeRequest.commandKey)
+          details.cpeRequestCommandKey = details.rpc.cpeRequest.commandKey;
+      }
+    } else if (details.rpc.cpeFault) {
+      details.acsRequestId = details.rpc.id;
+      details.cpeFaultCode = details.rpc.cpeFault.detail.faultCode;
+      details.cpeFaultString = details.rpc.cpeFault.detail.faultString;
+    }
+    delete details.rpc;
+  }
+
+  if (details.fault) {
+    details.faultCode = details.fault.code;
+    details.faultMessage = details.fault.message;
+    delete details.fault;
+  }
+
+  // For genieacs-ui
+  if (details.context) {
+    details.remoteAddress = details.context.request.ip;
+    if (details.context.state.user)
+      details.user = details.context.state.user.username;
+    delete details.context;
+  }
+
+  for (const [k, v] of Object.entries(details))
+    if (v == null) delete details[k];
+
+  return details;
+}
+
+function formatJson(details, systemd): string {
+  if (systemd) {
+    let severity = "";
+    if (details.severity === "info") severity = "<6>";
+    else if (details.severity === "warn") severity = "<4>";
+    else if (details.severity === "error") severity = "<3>";
+
+    return `${severity}${JSON.stringify(flatten(details))}${os.EOL}`;
+  }
+
+  return `${JSON.stringify(flatten(details))}${os.EOL}`;
+}
+
+function formatSimple(details, systemd): string {
+  const skip = {
+    user: true,
+    remoteAddress: true,
+    severity: true,
+    timestamp: true,
+    message: true,
+    deviceId: !!details.sessionContext
+  };
+
+  flatten(details);
+
+  let remote = "";
+  if (details.remoteAddress) {
+    if (details.deviceId && skip["deviceId"])
+      remote = `${details.remoteAddress} ${details.deviceId}: `;
+    else if (details.user)
+      remote = `${details.user}@${details.remoteAddress}: `;
+    else remote = `${details.remoteAddress}: `;
+  }
+
+  const keys = Object.keys(details);
+
+  let meta = "";
+
+  const kv = [];
+  for (const k of keys)
+    if (!skip[k]) kv.push(`${k}=${JSON.stringify(details[k])}`);
+
+  if (kv.length) meta = `; ${kv.join(" ")}`;
+
+  if (systemd) {
+    let severity = "";
+    if (details.severity === "info") severity = "<6>";
+    else if (details.severity === "warn") severity = "<4>";
+    else if (details.severity === "error") severity = "<3>";
+
+    return `${severity}${remote}${details.message}${meta}${os.EOL}`;
+  }
+
+  return `${details.timestamp} [${details.severity.toUpperCase()}] ${remote}${
+    details.message
+  }${meta}${os.EOL}`;
+}
+
+function log(details): void {
+  details.timestamp = new Date().toISOString();
+  if (LOG_FORMAT === "json") {
+    details = Object.assign({}, defaultMeta, details);
+    logStream.write(formatJson(details, LOG_SYSTEMD));
+  } else {
+    logStream.write(formatSimple(details, LOG_SYSTEMD));
+  }
+}
+
+export function info(details): void {
+  details.severity = "info";
+  log(details);
+}
+
+export function warn(details): void {
+  details.severity = "warn";
+  log(details);
+}
+
+export function error(details): void {
+  details.severity = "error";
+  log(details);
+}
+
+export function accessLog(details): void {
+  details.timestamp = new Date().toISOString();
+  if (ACCESS_LOG_FORMAT === "json") {
+    Object.assign(details, defaultMeta);
+    accessLogStream.write(formatJson(details, ACCESS_LOG_SYSTEMD));
+  } else {
+    accessLogStream.write(formatSimple(details, ACCESS_LOG_SYSTEMD));
+  }
+}
+
+export function accessInfo(details): void {
+  details.severity = "info";
+  accessLog(details);
+}
+
+export function accessWarn(details): void {
+  details.severity = "warn";
+  accessLog(details);
+}
+
+export function accessError(details): void {
+  details.severity = "error";
+  accessLog(details);
+}

lib/mongo-query.coffee

@@ -1,141 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This file incorporates work covered by the following copyright and
-# permission notice:
-#
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-common = require './common'
-
-
-test = (obj, query) ->
-  for k,v of query
-    if k.charAt(0) == '$' # this is a logical operator
-      res = testExpressions(obj, v, k)
-    else
-      value = common.getParamValueFromPath(obj, k)
-
-      if common.typeOf(v) isnt common.OBJECT_TYPE
-        # TODO comparing array to regex, array to array, and object to object
-        if common.typeOf(value) is common.ARRAY_TYPE
-          res = value.indexOf(v) != -1
-        else
-          if common.typeOf(v) is common.REGEXP_TYPE
-            res = v.test(value)
-          else
-            res = v == value
-      else
-        for k2,v2 of v
-          switch k2
-            when '$ne'
-              if common.typeOf(value) is common.ARRAY_TYPE
-                res = value.indexOf(v2) == -1
-              else
-                res = value != v2
-            when '$lt'
-              res = value < v2
-            when '$lte'
-              res = value <= v2
-            when '$gt'
-              res = value > v2
-            when '$gte'
-              res = value >= v2
-            when '$regex'
-              res = v2.test(value)
-            when '$in'
-              throw new Error('Operator not supported')        
-            when '$nin'
-              throw new Error('Operator not supported')        
-            when '$all'
-              throw new Error('Operator not supported')        
-            when '$exists'
-              throw new Error('Operator not supported')        
-            else
-              throw new Error('Operator not supported')        
-
-    if not res
-      return false
-
-  return true
-
-
-testExpressions = (obj, expressions, lop) ->
-  for f in expressions
-    res = test(obj, f)
-
-    switch lop
-      when '$and'
-        return false if not res
-      when '$or'
-        return true if res
-      when '$nor'
-        return false if res
-
-  return switch lop
-    when '$and' then true
-    when '$or' then false
-    when '$nor' then true
-    else throw new Error('Unknown logical operator')
-
-
-# generate field projection from a query
-projection = (query, proj) ->
-  for k,v of query
-    if k.charAt(0) == '$' # this is a logical operator
-      for q in v
-        projection(q, proj)
-    else
-      proj[k] = 1
-
-# optimize projection by removing overlaps
-optimizeProjection = (obj) ->
-  keys = Object.keys(obj).sort()
-  return if keys.length <= 1
-  i = 1
-  while i < keys.length
-    a = keys[i-1]
-    b = keys[i]
-    if common.startsWith(b, a)
-      if b.charAt(a.length) == '.' or b.charAt(a.length - 1) == '.'
-        delete obj[b]
-        keys.splice(i--, 1)
-    ++ i
-  return
-
-
-exports.test = test
-exports.projection = projection
-exports.optimizeProjection = optimizeProjection

lib/mongodb-functions.ts

@@ -0,0 +1,629 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { ObjectID } from "mongodb";
+import { map, parse, stringify } from "./common/expression-parser";
+import { likePatternToRegExp, evaluate } from "./common/expression";
+import { Expression, Fault, Task } from "./types";
+
+const isArray = Array.isArray;
+
+export function processDeviceFilter(filter): Expression {
+  return map(filter, exp => {
+    if (!isArray(exp)) return exp;
+
+    if (exp[0] === "PARAM") {
+      const p = exp[1];
+      if (p === "DeviceID.ID") return ["PARAM", "_id"];
+      else if (p === "DeviceID") return ["PARAM", "_deviceId"];
+      else if (p.startsWith("DeviceID."))
+        return ["PARAM", "_deviceId._" + p.slice(9)];
+      else if (p === "Tags") return ["PARAM", "_tags"];
+      else if (p === "tag") return ["PARAM", "_tags"];
+      else if (p === "Events.Inform") return ["PARAM", "_lastInform"];
+      else if (p === "Events.Registered") return ["PARAM", "_registered"];
+      else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"];
+      else if (p === "Events.1_BOOT") return ["PARAM", "_lastBoot"];
+    } else if (
+      isArray(exp[1]) &&
+      exp[1][0] === "PARAM" &&
+      exp[1][1].startsWith("Tags.")
+    ) {
+      const t = exp[1][1].slice(5);
+      if (exp[0] === "IS NULL") return ["<>", ["PARAM", "_tags"], t];
+      else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t];
+      else if (exp[0] === "=" && exp[2] === true)
+        return ["=", ["PARAM", "_tags"], t];
+    } else if (
+      ["=", "<>", ">", ">=", "<", "<=", "LIKE", "NOT LIKE"].includes(exp[0])
+    ) {
+      let e = map(exp, ee => {
+        if (
+          isArray(ee) &&
+          ee[0] === "PARAM" &&
+          typeof ee[1] === "string" &&
+          !ee[1].startsWith("_")
+        )
+          return ["PARAM", `${ee[1]}._value`];
+        return ee;
+      });
+
+      if (typeof e[2] === "number") {
+        const alt = (e as any[]).slice();
+        alt[2] = new Date(e[2]);
+        e = ["OR", e, alt];
+      }
+      return e;
+    }
+    return exp;
+  });
+}
+
+export function processTasksFilter(filter): Expression {
+  return map(filter, exp => {
+    if (!isArray(exp)) return exp;
+    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
+      const e = exp.slice();
+      if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectID(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "timestamp")
+        e[2] = new Date(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
+        e[2] = new Date(e[2]);
+      return e;
+    }
+    return exp;
+  });
+}
+
+export function processFaultsFilter(filter): Expression {
+  return map(filter, exp => {
+    if (!isArray(exp)) return exp;
+    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
+      const e = exp.slice();
+      if (e[1][0] === "PARAM" && e[1][1] === "timestamp") e[2] = new Date(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
+        e[2] = new Date(e[2]);
+      return e;
+    }
+    return exp;
+  });
+}
+
+export function filterToMongoQuery(exp: Expression): {} {
+  const ops = {
+    OR: 0,
+    AND: 0,
+    NOT: 0,
+    "=": 1,
+    "<>": 1,
+    ">": 1,
+    ">=": 1,
+    "<": 1,
+    "<=": 1,
+    LIKE: 1,
+    "NOT LIKE": 1,
+    "IS NULL": 1,
+    "IS NOT NULL": 1
+  };
+
+  function recursive(filter, negate, res = {}): {} {
+    const op = filter[0];
+
+    if (ops[op] === 0) {
+      for (let i = 1; i < filter.length; ++i) {
+        if (!isArray(filter[i]) || ops[filter[i][0]] == null)
+          throw new Error(`Invalid expression in ${op} clause`);
+      }
+
+      if ((!negate && op === "AND") || (negate && op === "OR")) {
+        res["$and"] = res["$and"] || [];
+        for (let i = 1; i < filter.length; ++i)
+          res["$and"].push(recursive(filter[i], negate));
+      } else if ((!negate && op === "OR") || (negate && op === "AND")) {
+        res["$or"] = res["$or"] || [];
+
+        for (let i = 1; i < filter.length; ++i)
+          res["$or"].push(recursive(filter[i], negate));
+      } else if (op === "NOT") {
+        recursive(filter[1], !negate, res);
+      }
+    } else if (ops[op] === 1) {
+      if (isArray(filter[2])) {
+        throw new Error(`Right operand of ${op} clause is not a primitive`);
+      } else if (
+        ["LIKE", "NOT LIKE"].includes(op) &&
+        isArray(filter[1]) &&
+        filter[1][0] === "FUNC" &&
+        ["UPPER", "LOWER"].includes(filter[1][1])
+      ) {
+        if (
+          (filter[1][1] === "UPPER" && filter[2] !== filter[2].toUpperCase()) ||
+          (filter[1][1] === "LOWER" && filter[2] !== filter[2].toLowerCase())
+        ) {
+          throw new Error(
+            `Cannot compare ${
+              filter[1][1]
+            }() against non-${filter[1][1].toLowerCase()}case pattern`
+          );
+        }
+      } else if (!isArray(filter[1]) || filter[1][0] !== "PARAM") {
+        throw new Error(`Left operand of ${op} clause is not a parameter`);
+      }
+
+      if (op === "=") {
+        const param = filter[1][1];
+        const p = (res[param] = res[param] || {});
+        if (negate) p["$ne"] = filter[2];
+        else p["$eq"] = filter[2];
+      } else if (op === "<>") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$ne"] = filter[2];
+        if (param !== "_tags") p["$exists"] = true;
+      } else if (op === ">") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$gt"] = filter[2];
+      } else if (op === ">=") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$gte"] = filter[2];
+      } else if (op === "<") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$lt"] = filter[2];
+      } else if (op === "<=") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$lte"] = filter[2];
+      } else if (op === "IS NULL") {
+        const param = filter[1][1];
+        res[param] = { $exists: negate };
+      } else if (op === "IS NOT NULL") {
+        const param = filter[1][1];
+        res[param] = { $exists: !negate };
+      } else {
+        if (op === "NOT LIKE") negate = !negate;
+        let param;
+        let flags;
+        if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
+          param = filter[1][2][1];
+          flags = "i";
+        } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
+          param = filter[1][2][1];
+          flags = "i";
+        } else {
+          param = filter[1][1];
+          flags = "";
+        }
+        const r = likePatternToRegExp(filter[2], filter[3], flags);
+        if (negate) res[param] = { $not: r };
+        else res[param] = r;
+      }
+    } else {
+      throw new Error(`Unrecognized operator ${op}`);
+    }
+
+    return res;
+  }
+
+  const _exp = evaluate(exp, null, Date.now());
+
+  if (!isArray(_exp)) {
+    if (_exp === true) return {};
+    throw new Error("Primitives are not valid queries");
+  }
+
+  return recursive(_exp, false);
+}
+
+export function processDeviceProjection(projection: {}): {} {
+  if (!projection) return projection;
+  const p = {};
+  for (const [k, v] of Object.entries(projection)) {
+    if (k === "DeviceID.ID") {
+      p["_id"] = 1;
+    } else if (k.startsWith("DeviceID")) {
+      p["_deviceId._SerialNumber"] = v;
+      p["_deviceId._OUI"] = v;
+      p["_deviceId._ProductClass"] = v;
+      p["_deviceId._Manufacturer"] = v;
+    } else if (k.startsWith("Tags")) {
+      p["_tags"] = v;
+    } else if (k.startsWith("Events")) {
+      p["_lastInform"] = v;
+      p["_registered"] = v;
+      p["_lastBoot"] = v;
+      p["_lastBootstrap"] = v;
+    } else {
+      p[k] = v;
+    }
+  }
+
+  return p;
+}
+
+export function processDeviceSort(sort: {}): {} {
+  if (!sort) return sort;
+  const s = {};
+  for (const [k, v] of Object.entries(sort)) {
+    if (k === "DeviceID.ID") s["_id"] = v;
+    else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v;
+    else if (k === "Events.Inform") s["_lastInform"] = v;
+    else if (k === "Events.Registered") s["_registered"] = v;
+    else if (k === "Events.1_BOOT") s["_lastBoot"] = v;
+    else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v;
+    else s[`${k}._value`] = v;
+  }
+
+  return s;
+}
+
+function parseDate(d): number | string {
+  const n = +d;
+  return isNaN(n) ? "" + d : n;
+}
+
+interface FlatDevice {
+  [param: string]: {
+    object: boolean;
+    objectTimestamp: number;
+    writable: boolean;
+    writableTimestamp: number;
+    value: [string | number | boolean, string];
+    valueTimestamp: number;
+  };
+}
+
+export function flattenDevice(device): FlatDevice {
+  function recursive(input, root, output, timestamp): void {
+    for (const [name, tree] of Object.entries(input)) {
+      if (!root) {
+        if (name === "_lastInform") {
+          output["Events.Inform"] = {
+            value: [parseDate(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_registered") {
+          output["Events.Registered"] = {
+            value: [parseDate(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_lastBoot") {
+          output["Events.1_BOOT"] = {
+            value: [parseDate(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_lastBootstrap") {
+          output["Events.0_BOOTSTRAP"] = {
+            value: [parseDate(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_id") {
+          output["DeviceID.ID"] = {
+            value: [tree, "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_deviceId") {
+          output["DeviceID.Manufacturer"] = {
+            value: [tree["_Manufacturer"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+          output["DeviceID.OUI"] = {
+            value: [tree["_OUI"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+          output["DeviceID.ProductClass"] = {
+            value: [tree["_ProductClass"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+          output["DeviceID.SerialNumber"] = {
+            value: [tree["_SerialNumber"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_tags") {
+          output["Tags"] = {
+            writable: true,
+            writableTimestamp: timestamp,
+            object: true,
+            objectTimestamp: timestamp
+          };
+
+          for (const t of tree as string[]) {
+            output[`Tags.${t}`] = {
+              value: [true, "xsd:boolean"],
+              valueTimestamp: timestamp,
+              writable: false,
+              writableTimestamp: timestamp,
+              object: false,
+              objectTimestamp: timestamp
+            };
+          }
+        }
+      }
+
+      if (name.startsWith("_")) continue;
+
+      let childrenTimestamp = timestamp;
+
+      if (!root) childrenTimestamp = +(input["_timestamp"] || 1);
+      else if (+input["_timestamp"] > timestamp)
+        childrenTimestamp = +input["_timestamp"];
+
+      const attrs = {};
+      if (tree["_value"] != null) {
+        attrs["value"] = [
+          tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
+          tree["_type"]
+        ];
+        attrs["valueTimestamp"] = +(tree["_timestamp"] || childrenTimestamp);
+        attrs["object"] = false;
+        attrs["objectTimestamp"] = childrenTimestamp;
+      } else if (tree["_object"] != null) {
+        attrs["object"] = tree["_object"];
+        attrs["objectTimestamp"] = childrenTimestamp;
+      }
+
+      if (tree["_writable"] != null) {
+        attrs["writable"] = tree["_writable"];
+        attrs["writableTimestamp"] = childrenTimestamp;
+      }
+
+      const r = root ? `${root}.${name}` : name;
+      output[r] = attrs;
+
+      if (attrs["object"] || tree["object"] == null)
+        recursive(tree, r, output, childrenTimestamp);
+    }
+  }
+
+  const newDevice = {};
+  const timestamp = new Date(device["_lastInform"] || 1).getTime();
+  recursive(device, "", newDevice, timestamp);
+  return newDevice;
+}
+
+export function flattenFault(fault): Fault {
+  const f = Object.assign({}, fault);
+  if (f.timestamp) f.timestamp = +f.timestamp;
+  if (f.expiry) f.expiry = +f.expiry;
+  return f;
+}
+
+export function flattenTask(task): Task {
+  const t = Object.assign({}, task);
+  t._id = "" + t._id;
+  if (t.timestamp) t.timestamp = +t.timestamp;
+  if (t.expiry) t.expiry = +t.expiry;
+  return t;
+}
+
+export function mongoQueryToFilter(query): Expression {
+  function recursive(_query): Expression {
+    const expressions: Expression[] = [];
+    for (const [k, v] of Object.entries(_query)) {
+      if (k[0] === "$") {
+        if (k === "$and") {
+          const and: Expression = ["AND"];
+          for (const vv of Object.values(v)) and.push(recursive(vv));
+          expressions.push(and);
+        } else if (k === "$or") {
+          const or: Expression = ["OR"];
+          for (const vv of Object.values(v)) or.push(recursive(vv));
+          expressions.push(or);
+        } else {
+          throw new Error(`Operator ${k} not supported`);
+        }
+      } else if (k === "_tags") {
+        if (typeof v === "object") {
+          if (isArray(v)) throw new Error(`Invalid type`);
+
+          if (v.hasOwnProperty("$ne"))
+            expressions.push(["IS NULL", ["PARAM", `Tags.${v["$ne"]}`]]);
+          else if (v.hasOwnProperty("$eq"))
+            expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v["$eq"]}`]]);
+          else throw new Error(`Invalid tag query`);
+        } else {
+          expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v}`]]);
+        }
+      } else if (k.startsWith("Tags.")) {
+        let exists: boolean;
+        if (typeof v === "boolean") exists = v;
+        else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"];
+        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
+        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
+        else throw new Error(`Invalid tag query`);
+
+        expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
+      } else if (typeof v === "object") {
+        if (isArray(v)) throw new Error(`Invalid type`);
+
+        const exps: Expression[] = [];
+        for (const [kk, vv] of Object.entries(v)) {
+          let op;
+          switch (kk) {
+            case "$eq":
+              op = "=";
+              break;
+            case "$ne":
+              op = "<>";
+              break;
+            case "$lt":
+              op = "<";
+              break;
+            case "$lte":
+              op = "<=";
+              break;
+            case "$gt":
+              op = ">";
+              break;
+            case "$gte":
+              op = ">=";
+              break;
+            default:
+              throw new Error(`Operator ${kk} not supported`);
+          }
+          exps.push([op, ["PARAM", k], vv]);
+        }
+        if (exps.length === 1) return exps[0];
+        const and: Expression = ["AND"];
+        expressions.push(and.concat(exps));
+      } else {
+        expressions.push(["=", ["PARAM", k], v]);
+      }
+    }
+    if (expressions.length === 1) return expressions[0];
+    const and: Expression = ["AND"];
+    return and.concat(expressions);
+  }
+
+  // empty filter
+  if (!Object.keys(query).length) return true;
+
+  return recursive(query);
+}
+
+export function flattenPreset(preset): {} {
+  const p = Object.assign({}, preset);
+  if (p.precondition) {
+    try {
+      // Try parse to check expression validity
+      parse(p.precondition);
+    } catch (error) {
+      p.precondition = mongoQueryToFilter(JSON.parse(p.precondition));
+      p.precondition = p.precondition.length ? stringify(p.precondition) : "";
+    }
+  }
+
+  if (p.events) {
+    const e = [];
+    for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`);
+    p.events = e.join(", ");
+  }
+
+  const provision = p.configurations[0];
+  if (
+    p.configurations.length === 1 &&
+    provision.type === "provision" &&
+    provision.name &&
+    provision.name.length
+  ) {
+    p.provision = provision.name;
+    p.provisionArgs = provision.args
+      ? JSON.stringify(provision.args).slice(1, -1)
+      : "";
+  }
+
+  delete p.configurations;
+  return p;
+}
+
+export function flattenFile(file): {} {
+  const f = {};
+  f["_id"] = file["_id"];
+  if (file.metadata) {
+    f["metadata.fileType"] = file.metadata.fileType || "";
+    f["metadata.oui"] = file.metadata.oui || "";
+    f["metadata.productClass"] = file.metadata.productClass || "";
+    f["metadata.version"] = file.metadata.version || "";
+  }
+  return f;
+}
+
+export function preProcessPreset(data): {} {
+  const preset = Object.assign({}, data);
+
+  if (!preset.precondition) preset.precondition = "";
+  // Try parse to check expression validity
+  parse(preset.precondition);
+
+  preset.weight = parseInt(preset.weight) || 0;
+
+  const events = {};
+  if (preset.events) {
+    for (let e of preset.events.split(",")) {
+      let v = true;
+      e = e.trim();
+      if (e.startsWith("-")) {
+        v = false;
+        e = e.slice(1).trim();
+      }
+      if (e) events[e] = v;
+    }
+  }
+
+  preset.events = events;
+
+  if (!preset.provision) throw new Error("Invalid preset provision");
+
+  const configuration = {
+    type: "provision",
+    name: preset.provision,
+    args: null
+  };
+
+  if (preset.provisionArgs)
+    configuration.args = JSON.parse(`[${preset.provisionArgs}]`);
+
+  delete preset.provision;
+  delete preset.provisionArgs;
+  preset.configurations = [configuration];
+  return preset;
+}

lib/nbi.coffee

@@ -1,398 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This file incorporates work covered by the following copyright and
-# permission notice:
-#
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-config = require './config'
-url = require 'url'
-common = require './common'
-db = require './db'
-mongodb = require 'mongodb'
-querystring = require 'querystring'
-query = require './query'
-apiFunctions = require './api-functions'
-presets = require './presets'
-
-# regular expression objects
-DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/tasks\/?$/
-TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-\_\%]+)(\/[a-zA-Z_]*)?$/
-TAGS_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/tags\/([a-zA-Z0-9\-\_\%]+)\/?$/
-PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-\_\%]+)\/?$/
-OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-\_\%]+)\/?$/
-FILES_REGEX = /^\/files\/([a-zA-Z0-9\%\!\*\'\(\)\;\:\@\&\=\+\$\,\?\#\[\]\-\_\.\~]+)\/?$/
-PING_REGEX = /^\/ping\/([a-zA-Z0-9\-\_\.]+)\/?$/
-QUERY_REGEX = /^\/([a-zA-Z0-9_]+s)\/?$/
-DEVICE_PRESET_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/preset\/?$/
-DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/?$/
-
-
-listener = (request, response) ->
-  chunks = []
-  bytes = 0
-
-  request.addListener 'data', (chunk) ->
-    chunks.push(chunk)
-    bytes += chunk.length
-
-  request.getBody = () ->
-    # Write all chunks into a Buffer
-    body = new Buffer(bytes)
-    offset = 0
-    chunks.forEach((chunk) ->
-      chunk.copy(body, offset, 0, chunk.length)
-      offset += chunk.length
-    )
-    return body
-
-  request.addListener 'end', () ->
-    body = request.getBody()
-    urlParts = url.parse(request.url, true)
-    if PRESETS_REGEX.test(urlParts.pathname)
-      presetName = querystring.unescape(PRESETS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        preset = JSON.parse(body)
-        preset._id = presetName
-
-        db.presetsCollection.save(preset, (err) ->
-          db.redisClient.del('presets', 'presets_hash', (err) ->
-            throw err if err
-          )
-          if err
-            response.writeHead(500)
-            response.end(err)
-            return
-          response.writeHead(200)
-          response.end()
-        )
-      else if request.method == 'DELETE'
-        db.presetsCollection.remove({'_id' : presetName}, (err, removedCount) ->
-          db.redisClient.del('presets', 'presets_hash', (err) ->
-            throw err if err
-          )
-          if err
-            response.writeHead(500)
-            response.end(err)
-            return
-          response.writeHead(200)
-          response.end()
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if OBJECTS_REGEX.test(urlParts.pathname)
-      objectName = querystring.unescape(OBJECTS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        object = JSON.parse(body)
-        object._id = objectName
-
-        db.objectsCollection.save(object, (err) ->
-          db.redisClient.del('objects', 'presets_hash', (err) ->
-            throw err if err
-          )
-          if err
-            response.writeHead(500)
-            response.end(err)
-            return
-          response.writeHead(200)
-          response.end()
-        )
-      else if request.method == 'DELETE'
-        db.objectsCollection.remove({'_id' : objectName}, (err, removedCount) ->
-          db.redisClient.del('objects', 'presets_hash', (err) ->
-            throw err if err
-          )
-          if err
-            response.writeHead(500)
-            response.end(err)
-            return
-          response.writeHead(200)
-          response.end()
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if TAGS_REGEX.test(urlParts.pathname)
-      r = TAGS_REGEX.exec(urlParts.pathname)
-      deviceId = querystring.unescape(r[1])
-      tag = querystring.unescape(r[2])
-      if request.method == 'POST'
-        db.devicesCollection.update({'_id' : deviceId}, {'$addToSet' : {'_tags' : tag}}, {safe: true}, (err) ->
-          db.redisClient.del("#{deviceId}_presets_hash", (err) ->
-            throw err if err
-          )
-          if err
-            response.writeHead(500)
-            response.end(err)
-            return
-          response.writeHead(200)
-          response.end()
-        )
-      else if request.method == 'DELETE'
-        db.devicesCollection.update({'_id' : deviceId}, {'$pull' : {'_tags' : tag}}, {safe: true}, (err) ->
-          db.redisClient.del("#{deviceId}_presets_hash", (err) ->
-            throw err if err
-          )
-          if err
-            response.writeHead(500)
-            response.end(err)
-            return
-          response.writeHead(200)
-          response.end()
-        )
-      else
-        response.writeHead 405, {'Allow': 'POST, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if DEVICE_TASKS_REGEX.test(urlParts.pathname)
-      if request.method == 'POST'
-        deviceId = querystring.unescape(DEVICE_TASKS_REGEX.exec(urlParts.pathname)[1])
-        if body.length
-          task = JSON.parse(body)
-          task.device = deviceId
-          db.getAliases((aliases) ->
-            apiFunctions.insertTasks(task, aliases, (err) ->
-              db.redisClient.del("#{deviceId}_presets_hash", (err) ->
-                throw err if err
-              )
-              if err
-                response.writeHead(500)
-                response.end(err)
-                return
-
-              if urlParts.query.connection_request?
-                apiFunctions.connectionRequest(deviceId, (err) ->
-                  if err
-                    response.writeHead(202)
-                    response.end()
-                  else
-                    apiFunctions.watchTask(task._id, config.get('DEVICE_ONLINE_THRESHOLD', deviceId), (err) ->
-                      if err
-                        response.writeHead(202)
-                        response.end()
-                        return
-                      response.writeHead(200)
-                      response.end()
-                    )
-                )
-              else
-                response.writeHead(202)
-                response.end()
-            )
-          )
-        else if urlParts.query.connection_request?
-          # no task, send connection request only
-          apiFunctions.connectionRequest(deviceId, (err) ->
-            if err
-              response.writeHead 504
-              response.end()
-              return
-            response.writeHead 200
-            response.end()
-          )
-        else
-          response.writeHead(400)
-          response.end()
-      else
-        response.writeHead 405, {'Allow': 'POST'}
-        response.end('405 Method Not Allowed')
-    else if DEVICE_PRESET_REGEX.test(urlParts.pathname)
-      deviceId = querystring.unescape(DEVICE_PRESET_REGEX.exec(urlParts.pathname)[1])
-      if request.method is 'GET'
-        db.getPresetsObjectsAliases((allPresets, allObjects, allAliases) ->
-          presets.getDevicePreset(deviceId, allPresets, allObjects, allAliases, (devicePreset) ->
-            response.writeHead(200, {'Content-Type' : 'application/json'})
-            response.end(JSON.stringify(devicePreset))
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'GET'}
-        response.end('405 Method Not Allowed')
-    else if TASKS_REGEX.test(urlParts.pathname)
-      r = TASKS_REGEX.exec(urlParts.pathname)
-      taskId = mongodb.ObjectID(querystring.unescape(r[1]))
-      action = try querystring.unescape(r[2])
-      if not action? or action is '/'
-        if request.method == 'DELETE'
-          db.tasksCollection.remove({'_id' : taskId}, (err, removedCount) ->
-            if err
-              response.writeHead(500)
-              response.end(err)
-              return
-            response.writeHead(200)
-            response.end()
-          )
-        else
-          response.writeHead 405, {'Allow': 'PUT DELETE'}
-          response.end('405 Method Not Allowed')
-      else if action is '/retry'
-        if request.method == 'POST'
-          db.tasksCollection.update({_id : taskId}, {$unset : {fault : 1}, $set : {timestamp : new Date()}}, (err, count) ->
-            # TODO need to invalidate presets hash for the device
-            response.writeHead(200)
-            response.end()
-          )
-        else
-          response.writeHead 405, {'Allow': 'POST'}
-          response.end('405 Method Not Allowed')
-      else
-        response.writeHead(404)
-        response.end()
-    else if FILES_REGEX.test(urlParts.pathname)
-      filename = querystring.unescape(FILES_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        metadata = {
-          fileType : request.headers.filetype,
-          oui : request.headers.oui,
-          productClass : request.headers.productclass,
-          version : request.headers.version,
-        }
-
-        gs = new mongodb.GridStore(db.mongoDb, filename, 'w', {metadata : metadata})
-        gs.open((err, gs) ->
-          gs.write(body, (err, res) ->
-            throw err if err
-            gs.close((err) ->
-              throw err if err
-              response.writeHead(201)
-              response.end()
-            )
-          )
-        )
-      else if request.method == 'DELETE'
-        mongodb.GridStore.unlink(db.mongoDb, filename, (err) ->
-          response.writeHead(200)
-          response.end()
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if PING_REGEX.test(urlParts.pathname)
-      host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1])
-      require('child_process').exec("ping -w 1 -i 0.2 -c 3 #{host}", (err, stdout, stderr) ->
-        if err?
-          response.writeHead(404, {'Cache-Control' : 'no-cache'})
-          response.end()
-          return
-        response.writeHead(200, {'Content-Type' : 'text/plain', 'Cache-Control' : 'no-cache'})
-        response.end(stdout)
-      )
-    else if DELETE_DEVICE_REGEX.test(urlParts.pathname)
-      if request.method isnt 'DELETE'
-        response.writeHead(405, {'Allow' : 'DELETE'})
-        response.end('405 Method Not Allowed')
-        return
-
-      deviceId = querystring.unescape(DELETE_DEVICE_REGEX.exec(urlParts.pathname)[1])
-      apiFunctions.deleteDevice(deviceId, (err) ->
-        if err
-          response.writeHead(500)
-          response.end(err.message)
-          return
-        response.writeHead(200)
-        response.end()
-      )
-    else if QUERY_REGEX.test(urlParts.pathname)
-      collectionName = QUERY_REGEX.exec(urlParts.pathname)[1]
-      if request.method not in ['GET', 'HEAD']
-        response.writeHead 405, {'Allow' : 'GET, HEAD'}
-        response.end('405 Method Not Allowed')
-        return
-      collection = db["#{collectionName}Collection"]
-      if not collection?
-        response.writeHead 404
-        response.end('404 Not Found')
-        return
-
-      func = (aliases) ->
-        if urlParts.query.query?
-          q = JSON.parse(urlParts.query.query)
-        else
-          q = {}
-        q = query.expand(q, aliases) if collectionName is 'devices'
-
-        if urlParts.query.projection?
-          projection = {}
-          for p in urlParts.query.projection.split(',')
-            p = p.trim()
-            projection[p] = 1
-            if collectionName is 'devices'
-              for k,v of aliases
-                if k == p or common.startsWith(k, "#{p}.")
-                  projection[a] = 1 for a in v
-
-        cur = collection.find(q, projection, {batchSize : 50})
-        if urlParts.query.sort?
-          s = JSON.parse(urlParts.query.sort)
-          sort = {}
-          for k,v of s
-            if aliases[k]?
-              sort[a] = v for a in aliases[k]
-            else
-              sort[k] = v
-          cur.sort(sort)
-
-        cur.skip(parseInt(urlParts.query.skip)) if urlParts.query.skip?
-        cur.limit(parseInt(urlParts.query.limit)) if urlParts.query.limit?
-        cur.count((err, total) ->
-          response.writeHead(200, {'Content-Type' : 'application/json', 'total' : total})
-          if request.method is 'HEAD'
-            response.end()
-            return
-          response.write("[\n")
-          i = 0
-          cur.each((err, item) ->
-            if item is null
-              response.end("\n]")
-            else
-              response.write(",\n") if i++
-              apiFunctions.addAliases(item, aliases) if collectionName is 'devices'
-              response.write(JSON.stringify(item))
-          )
-        )
-
-      if collectionName is 'devices'
-        db.getAliases((aliases) ->
-          func(aliases)
-        )
-      else
-        func({})
-    else
-      response.writeHead 404
-      response.end('404 Not Found')
-
-
-exports.listener = listener

lib/nbi.ts

@@ -0,0 +1,741 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as url from "url";
+import * as mongodb from "mongodb";
+import * as querystring from "querystring";
+import * as vm from "vm";
+import { exec } from "child_process";
+import * as config from "./config";
+import * as db from "./db";
+import * as query from "./query";
+import * as apiFunctions from "./api-functions";
+import * as cache from "./cache";
+import { version as VERSION } from "../package.json";
+
+const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
+const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
+const TAGS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tags\/([a-zA-Z0-9\-_%]+)\/?$/;
+const PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-_%]+)\/?$/;
+const OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-_%]+)\/?$/;
+const FILES_REGEX = /^\/files\/([a-zA-Z0-9%!*'();:@&=+$,?#[\]\-_.~]+)\/?$/;
+const PING_REGEX = /^\/ping\/([a-zA-Z0-9\-_.:]+)\/?$/;
+const QUERY_REGEX = /^\/([a-zA-Z0-9_]+)\/?$/;
+const DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/?$/;
+const PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-_%]+)\/?$/;
+const VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
+const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
+
+function throwError(err, httpResponse?): never {
+  if (httpResponse) {
+    httpResponse.writeHead(500, { Connection: "close" });
+    httpResponse.end(`${err.name}: ${err.message}`);
+  }
+  throw err;
+}
+
+export function listener(request, response): void {
+  const chunks = [];
+  let bytes = 0;
+  response.setHeader("GenieACS-Version", VERSION);
+
+  request.addListener("data", (chunk): void => {
+    chunks.push(chunk);
+    bytes += chunk.length;
+  });
+
+  function getBody(): Buffer {
+    // Write all chunks into a Buffer
+    const body = Buffer.allocUnsafe(bytes);
+    let offset = 0;
+    for (const chunk of chunks) {
+      chunk.copy(body, offset, 0, chunk.length);
+      offset += chunk.length;
+    }
+    return body;
+  }
+
+  request.addListener("end", () => {
+    const body = getBody();
+    const urlParts = url.parse(request.url, true);
+    if (PRESETS_REGEX.test(urlParts.pathname)) {
+      const presetName = querystring.unescape(
+        PRESETS_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const preset = JSON.parse(body.toString());
+        preset._id = presetName;
+        db.presetsCollection.replaceOne(
+          { _id: presetName },
+          preset,
+          { upsert: true },
+          err => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch(err => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else if (request.method === "DELETE") {
+        db.presetsCollection.deleteOne({ _id: presetName }, err => {
+          if (err) return void throwError(err, response);
+
+          cache
+            .del("presets_hash")
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch(err => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        });
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (OBJECTS_REGEX.test(urlParts.pathname)) {
+      const objectName = querystring.unescape(
+        OBJECTS_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const object = JSON.parse(body.toString());
+        object._id = objectName;
+        db.objectsCollection.replaceOne(
+          { _id: objectName },
+          object,
+          { upsert: true },
+          err => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch(err => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else if (request.method === "DELETE") {
+        db.objectsCollection.deleteOne({ _id: objectName }, err => {
+          if (err) return void throwError(err, response);
+
+          cache
+            .del("presets_hash")
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch(err => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        });
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (PROVISIONS_REGEX.test(urlParts.pathname)) {
+      const provisionName = querystring.unescape(
+        PROVISIONS_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const object = {
+          _id: provisionName,
+          script: body.toString()
+        };
+
+        try {
+          new vm.Script(`"use strict";(function(){\n${object.script}\n})();`);
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+
+        db.provisionsCollection.replaceOne(
+          { _id: provisionName },
+          object,
+          { upsert: true },
+          err => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch(err => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else if (request.method === "DELETE") {
+        db.provisionsCollection.deleteOne({ _id: provisionName }, err => {
+          if (err) return void throwError(err, response);
+
+          cache
+            .del("presets_hash")
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch(err => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        });
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (VIRTUAL_PARAMETERS_REGEX.test(urlParts.pathname)) {
+      const virtualParameterName = querystring.unescape(
+        VIRTUAL_PARAMETERS_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const object = {
+          _id: virtualParameterName,
+          script: body.toString()
+        };
+
+        try {
+          new vm.Script(`"use strict";(function(){\n${object.script}\n})();`);
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+
+        db.virtualParametersCollection.replaceOne(
+          { _id: virtualParameterName },
+          object,
+          { upsert: true },
+          err => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch(err => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else if (request.method === "DELETE") {
+        db.virtualParametersCollection.deleteOne(
+          { _id: virtualParameterName },
+          err => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch(err => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (TAGS_REGEX.test(urlParts.pathname)) {
+      const r = TAGS_REGEX.exec(urlParts.pathname);
+      const deviceId = querystring.unescape(r[1]);
+      const tag = querystring.unescape(r[2]);
+      if (request.method === "POST") {
+        db.devicesCollection.updateOne(
+          { _id: deviceId },
+          { $addToSet: { _tags: tag } },
+          err => {
+            if (err) return void throwError(err, response);
+            response.writeHead(200);
+            response.end();
+          }
+        );
+      } else if (request.method === "DELETE") {
+        db.devicesCollection.updateOne(
+          { _id: deviceId },
+          { $pull: { _tags: tag } },
+          err => {
+            if (err) return void throwError(err, response);
+
+            response.writeHead(200);
+            response.end();
+          }
+        );
+      } else {
+        response.writeHead(405, { Allow: "POST, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (FAULTS_REGEX.test(urlParts.pathname)) {
+      if (request.method === "DELETE") {
+        const faultId = querystring.unescape(
+          FAULTS_REGEX.exec(urlParts.pathname)[1]
+        );
+        const deviceId = faultId.split(":", 1)[0];
+        const channel = faultId.slice(deviceId.length + 1);
+        db.faultsCollection.deleteOne({ _id: faultId }, err => {
+          if (err) return void throwError(err, response);
+
+          if (channel.startsWith("task_")) {
+            const objId = new mongodb.ObjectID(channel.slice(5));
+            return void db.tasksCollection.deleteOne({ _id: objId }, err => {
+              if (err) return void throwError(err, response);
+
+              cache
+                .del(`${deviceId}_tasks_faults_operations`)
+                .then(() => {
+                  response.writeHead(200);
+                  response.end();
+                })
+                .catch(err => {
+                  setTimeout(() => {
+                    throwError(err, response);
+                  });
+                });
+            });
+          }
+
+          cache
+            .del(`${deviceId}_tasks_faults_operations`)
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch(err => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        });
+      } else {
+        response.writeHead(405, { Allow: "DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (DEVICE_TASKS_REGEX.test(urlParts.pathname)) {
+      if (request.method === "POST") {
+        const deviceId = querystring.unescape(
+          DEVICE_TASKS_REGEX.exec(urlParts.pathname)[1]
+        );
+        if (body.length) {
+          const task = JSON.parse(body.toString());
+          task.device = deviceId;
+          apiFunctions
+            .insertTasks(task)
+            .then(() => {
+              cache
+                .del(`${deviceId}_tasks_faults_operations`)
+                .then(() => {
+                  if (urlParts.query.connection_request != null) {
+                    apiFunctions
+                      .connectionRequest(deviceId)
+                      .then(() => {
+                        const taskTimeout =
+                          (urlParts.query.timeout &&
+                            parseInt(urlParts.query.timeout as string)) ||
+                          config.get("DEVICE_ONLINE_THRESHOLD", deviceId);
+
+                        apiFunctions
+                          .watchTask(deviceId, task._id, taskTimeout)
+                          .then(status => {
+                            if (status === "timeout") {
+                              response.writeHead(
+                                202,
+                                "Task queued but not processed",
+                                {
+                                  "Content-Type": "application/json"
+                                }
+                              );
+                              response.end(JSON.stringify(task));
+                            } else if (status === "fault") {
+                              db.tasksCollection.findOne(
+                                { _id: task._id },
+                                (err, task2) => {
+                                  if (err)
+                                    return void throwError(err, response);
+
+                                  response.writeHead(202, "Task faulted", {
+                                    "Content-Type": "application/json"
+                                  });
+                                  response.end(JSON.stringify(task2));
+                                }
+                              );
+                            } else {
+                              response.writeHead(200, {
+                                "Content-Type": "application/json"
+                              });
+                              response.end(JSON.stringify(task));
+                            }
+                          })
+                          .catch(err => {
+                            setTimeout(() => {
+                              throwError(err, response);
+                            });
+                          });
+                      })
+                      .catch(err => {
+                        response.writeHead(202, err.message, {
+                          "Content-Type": "application/json"
+                        });
+                        response.end(JSON.stringify(task));
+                      });
+                  } else {
+                    response.writeHead(202, {
+                      "Content-Type": "application/json"
+                    });
+                    response.end(JSON.stringify(task));
+                  }
+                })
+                .catch(err => {
+                  setTimeout(() => {
+                    throwError(err, response);
+                  });
+                });
+            })
+            .catch(err => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        } else if (urlParts.query.connection_request != null) {
+          // No task, send connection request only
+          apiFunctions
+            .connectionRequest(deviceId)
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch(err => {
+              response.writeHead(504);
+              response.end(`${err.name}: ${err.message}`);
+            });
+        } else {
+          response.writeHead(400);
+          response.end();
+        }
+      } else {
+        response.writeHead(405, { Allow: "POST" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (TASKS_REGEX.test(urlParts.pathname)) {
+      const r = TASKS_REGEX.exec(urlParts.pathname);
+      const taskId = querystring.unescape(r[1]);
+      const action = r[2];
+      if (!action || action === "/") {
+        if (request.method === "DELETE") {
+          db.tasksCollection.findOne(
+            { _id: new mongodb.ObjectID(taskId) },
+            { projection: { device: 1 } },
+            (err, task) => {
+              if (err) return void throwError(err, response);
+
+              if (!task) {
+                response.writeHead(404);
+                response.end("Task not found");
+                return;
+              }
+
+              const deviceId = task.device;
+              db.tasksCollection.deleteOne(
+                { _id: new mongodb.ObjectID(taskId) },
+                err => {
+                  if (err) return void throwError(err, response);
+
+                  db.faultsCollection.deleteOne(
+                    { _id: `${deviceId}:task_${taskId}` },
+                    err => {
+                      if (err) return void throwError(err, response);
+
+                      cache
+                        .del(`${deviceId}_tasks_faults_operations`)
+                        .then(() => {
+                          response.writeHead(200);
+                          response.end();
+                        })
+                        .catch(err => {
+                          setTimeout(() => {
+                            throwError(err, response);
+                          });
+                        });
+                    }
+                  );
+                }
+              );
+            }
+          );
+        } else {
+          response.writeHead(405, { Allow: "PUT DELETE" });
+          response.end("405 Method Not Allowed");
+        }
+      } else if (action === "/retry") {
+        if (request.method === "POST") {
+          db.tasksCollection.findOne(
+            { _id: new mongodb.ObjectID(taskId) },
+            { projection: { device: 1 } },
+            (err, task) => {
+              if (err) return void throwError(err, response);
+
+              const deviceId = task.device;
+              db.faultsCollection.deleteOne(
+                { _id: `${deviceId}:task_${taskId}` },
+                err => {
+                  if (err) return void throwError(err, response);
+
+                  cache
+                    .del(`${deviceId}_tasks_faults_operations`)
+                    .then(() => {
+                      response.writeHead(200);
+                      response.end();
+                    })
+                    .catch(err => {
+                      setTimeout(() => {
+                        throwError(err, response);
+                      });
+                    });
+                }
+              );
+            }
+          );
+        } else {
+          response.writeHead(405, { Allow: "POST" });
+          response.end("405 Method Not Allowed");
+        }
+      } else {
+        response.writeHead(404);
+        response.end();
+      }
+    } else if (FILES_REGEX.test(urlParts.pathname)) {
+      const filename = querystring.unescape(
+        FILES_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const metadata = {
+          fileType: request.headers.filetype,
+          oui: request.headers.oui,
+          productClass: request.headers.productclass,
+          version: request.headers.version
+        };
+        const bucket = new mongodb.GridFSBucket(db.client.db());
+        const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
+          metadata: metadata
+        });
+
+        uploadStream.on("error", err => {
+          throwError(err, response);
+        });
+
+        uploadStream.end(body, () => {
+          response.writeHead(201);
+          response.end();
+        });
+      } else if (request.method === "DELETE") {
+        const bucket = new mongodb.GridFSBucket(db.client.db());
+        bucket.delete((filename as unknown) as mongodb.ObjectId, err => {
+          if (err) return void throwError(err, response);
+
+          response.writeHead(200);
+          response.end();
+        });
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (PING_REGEX.test(urlParts.pathname)) {
+      const host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1]);
+      exec(`ping -w 1 -i 0.2 -c 3 ${host}`, (err, stdout) => {
+        if (err) {
+          response.writeHead(404, { "Cache-Control": "no-cache" });
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+
+        response.writeHead(200, {
+          "Content-Type": "text/plain",
+          "Cache-Control": "no-cache"
+        });
+        response.end(stdout);
+      });
+    } else if (DELETE_DEVICE_REGEX.test(urlParts.pathname)) {
+      if (request.method !== "DELETE") {
+        response.writeHead(405, { Allow: "DELETE" });
+        response.end("405 Method Not Allowed");
+        return;
+      }
+
+      const deviceId = querystring.unescape(
+        DELETE_DEVICE_REGEX.exec(urlParts.pathname)[1]
+      );
+      apiFunctions
+        .deleteDevice(deviceId)
+        .then(() => {
+          response.writeHead(200);
+          response.end();
+        })
+        .catch(err => {
+          setTimeout(() => {
+            throwError(err, response);
+          });
+        });
+    } else if (QUERY_REGEX.test(urlParts.pathname)) {
+      let collectionName = QUERY_REGEX.exec(urlParts.pathname)[1];
+
+      // Convert to camel case
+      let i = collectionName.indexOf("_");
+      while (i++ >= 0) {
+        const up =
+          i < collectionName.length ? collectionName[i].toUpperCase() : "";
+        collectionName =
+          collectionName.slice(0, i - 1) + up + collectionName.slice(i + 1);
+        i = collectionName.indexOf("_", i);
+      }
+
+      if (request.method !== "GET" && request.method !== "HEAD") {
+        response.writeHead(405, { Allow: "GET, HEAD" });
+        response.end("405 Method Not Allowed");
+        return;
+      }
+
+      const collection = db[`${collectionName}Collection`];
+      if (!collection) {
+        response.writeHead(404);
+        response.end("404 Not Found");
+        return;
+      }
+
+      let q = {};
+      if (urlParts.query.query) {
+        try {
+          q = JSON.parse(urlParts.query.query as string);
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+      }
+
+      switch (collectionName) {
+        case "devices":
+          q = query.expand(q);
+          break;
+        case "tasks":
+          q = query.sanitizeQueryTypes(q, {
+            _id: v => new mongodb.ObjectID(v),
+            timestamp: v => new Date(v),
+            retries: Number
+          });
+          break;
+        case "faults":
+          q = query.sanitizeQueryTypes(q, {
+            timestamp: v => new Date(v),
+            retries: Number
+          });
+      }
+
+      let projection = null;
+      if (urlParts.query.projection) {
+        projection = {};
+        for (const p of (urlParts.query.projection as string).split(","))
+          projection[p.trim()] = 1;
+      }
+
+      const cur = collection.find(q, { projection: projection });
+
+      if (urlParts.query.sort) {
+        const s = JSON.parse(urlParts.query.sort as string);
+        const sort = {};
+        for (const [k, v] of Object.entries(s)) {
+          if (k[k.lastIndexOf(".") + 1] !== "_" && collectionName === "devices")
+            sort[`${k}._value`] = v;
+          else sort[k] = v;
+        }
+
+        cur.sort(sort);
+      }
+
+      if (urlParts.query.skip)
+        cur.skip(parseInt(urlParts.query.skip as string));
+
+      if (urlParts.query.limit)
+        cur.limit(parseInt(urlParts.query.limit as string));
+
+      cur.count(false, (err, total) => {
+        if (err) return void throwError(err);
+
+        response.writeHead(200, {
+          "Content-Type": "application/json",
+          total: total
+        });
+
+        if (request.method === "HEAD") {
+          response.end();
+          return;
+        }
+
+        response.write("[\n");
+        i = 0;
+        cur.forEach(
+          item => {
+            if (i++) response.write(",\n");
+            response.write(JSON.stringify(item));
+          },
+          err => {
+            if (err) return void throwError(err);
+            response.end("\n]");
+          }
+        );
+      });
+    } else {
+      response.writeHead(404);
+      response.end("404 Not Found");
+    }
+  });
+}

lib/normalize.coffee

@@ -1,132 +0,0 @@
-###
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-# Credit to Netanel Rubin from Check Point Software Technologies for discovering
-# vulnerability caused by use of eval to generate regex from user input.
-
-common = require './common'
-parameters = require './parameters'
-
-
-stringToRegexp = (input) ->
-  if (input.indexOf('*') == -1)
-    return false
-
-  output = input.replace(/[\[\]\\\^\$\.\|\?\+\(\)]/, "\\$&")
-  if output[0] == '*'
-    output = output.replace(/^\*+/g, '')
-  else
-    output = '^' + output
-
-  if output[output.length - 1] == '*'
-    output = output.replace(/\*+$/g, '')
-  else
-    output = output + '$'
-
-  output = output.replace(/[\*]/, '.*')
-
-  return new RegExp(output)
-
-
-normalizers = {}
-
-normalizers.default = (input, normType) ->
-  if normType is 'query' and common.typeOf(input) is common.STRING_TYPE
-    vals = []
-    str = normalizers.string(input, normType)
-    if common.typeOf(str) == common.ARRAY_TYPE
-      vals = vals.concat(str)
-    else
-      vals.push(str)
-
-    f = parseFloat(input)
-    if not isNaN(f)
-      vals.push(f)
-
-    d = new Date(input)
-    if input.length >= 8 and d.getFullYear() > 1983
-      vals.push(d)
-
-    r = stringToRegexp(input)
-    if r isnt false
-      vals.push({'$regex' : r})
-
-    return vals
-  return input
-
-
-FLOAT_REGEX = /^([\-\+]?[0-9\.]+)/
-normalizers.float = (input, normType) ->
-  res = FLOAT_REGEX.exec(input)
-  if res != null
-    parseFloat(res[1])
-  else
-    null
-
-
-normalizers.date = (input, normType) ->
-  return new Date(input)
-
-
-normalizers.string = (input, normType) ->
-  if normType is 'query'
-    if (m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input))
-      return [{'$regex' : new RegExp(m[1], m[2])}, input]
-  return input
-
-
-colonizeMac = (input) ->
-  if (i = input.indexOf(':')) != -1
-    i = if (i % 2) == 0 then 0 else 1
-    while i < input.length
-      if input[i-1] != ':' and input[i-2] != ':'
-        if input[i] != ':' and i != 0
-          input = input.substring(0, i) + ':' + input.substring(i)
-          ++i
-        ++i
-      ++ i
-    return input
-  else
-    return input.replace(/[0-9A-F]{1}(?!$)/g, '$&:?')
-
-
-normalizers.mac = (input, normType) ->
-  if normType isnt 'query'
-    return input
-
-  input = input.trim().toUpperCase().replace(/[^0-9A-F\*\-\:]/g, '').replace(/[\-\:]/g, ':')
-
-  if input.indexOf('*') != -1
-    input = (colonizeMac(a) for a in input.split('*')).join('*')
-    return stringToRegexp(input, false)
-
-  input = colonizeMac(input)
-
-  if input.length == 17
-    return input
-
-  return {'$regex' : new RegExp(input)}
-
-
-exports.normalize = (path, value, normType) ->
-  type = parameters.getType(path) ? 'default'
-  normalizers[type](value, normType)

lib/parameters.coffee

@@ -1,212 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-config = require './config'
-db = require './db'
-
-
-PATH_REGEX = /\w+|(\/.+?\/\w*)/g
-
-# Javascript represents numbers as 64-bit floating point numbers
-# Integers up to 2^53 can be encoded precisely
-MAX_PARAMETERS = 52
-
-
-# get the index of the least significant bit
-getLsb = (num) ->
-  i = -1
-  while num > 0
-    ++ i
-    break if num & 1
-    num >>= 1
-  return i
-
-
-createParameterAttributeFinder = (attributeName) ->
-  # compile attributes into data types optimized for quick lookup
-  attributes = []
-  stringSegments = {}
-  regexSegments = {}
-  paramRegex = {}
-
-  for k, v of config.PARAMETERS
-    continue if not v[attributeName]?
-    attributeIndex = attributes.push(v[attributeName]) - 1
-    throw new Error("There cannot be more than #{MAX_PARAMETERS} configured parameters") if attributeIndex >= MAX_PARAMETERS
-    segmentIndex = 0
-    parts = k.match(PATH_REGEX)
-    for i in [0 ... parts.length]
-      part = parts[i]
-      positionHash = i + parts.length * MAX_PARAMETERS
-
-      if part[0] == '/' # this is a regex part
-        j = part.lastIndexOf('/')
-        regExp = new RegExp(part[1...j], part[j+1..])
-        regexSegments[positionHash] ?= 0
-        regexSegments[positionHash] |= Math.pow(2, attributeIndex)
-        paramRegex[attributeIndex] ?= {}
-        paramRegex[attributeIndex][i] = regExp
-      else
-        stringSegments[positionHash] ?= {}
-        stringSegments[positionHash][part] ?= 0
-        stringSegments[positionHash][part] |= Math.pow(2, attributeIndex)
-
-  # Finder function
-  return (param) ->
-    parts = param.split('.')
-    positionHash = parts.length * MAX_PARAMETERS
-
-    stringIndices = 0 | stringSegments[positionHash]?[parts[0]]
-    allIndices = stringIndices | regexSegments[positionHash]
-
-    for i in [1...parts.length]
-      positionHash = i + parts.length * MAX_PARAMETERS
-      strIdx = stringSegments[positionHash]?[parts[i]]
-      rgxIdx = regexSegments[positionHash]
-      stringIndices &= strIdx
-      allIndices &= (strIdx | rgxIdx)
-
-    if stringIndices > 0
-      return attributes[getLsb(stringIndices)]
-
-    attributeIndex = 0
-    while allIndices > 0
-      if allIndices & 1
-        match = true
-        for i, r of paramRegex[attributeIndex]
-          if not r.test(parts[i])
-            match = false
-            break
-
-        if match
-          return attributes[attributeIndex]
-
-      ++ attributeIndex
-      allIndices >>= 1
-
-    return null
-
-
-splitIds = (batches, callback) ->
-  db.devicesCollection.count((err, count) ->
-    throw err if err
-    return callback(null) if count == 0
-    ids = [{}]
-    return callback(ids) if count <= 1000 or batches == 1
-    batchSize = Math.floor(count / batches)
-    for i in [1 ... batches]
-      ids[i] = {}
-      do (i) ->
-        cursor = db.devicesCollection.find({}, {_id:1}).sort({'_id':1}).skip(i * batchSize).limit(1)
-        cursor.nextObject((err, obj) ->
-          ids[i]['$gte'] = obj._id
-          ids[i-1]['$lt'] = obj._id
-          if i == batches - 1
-            callback(ids)
-        )
-  )
-
-
-compileAliases = (callback) ->
-  map = () ->
-    recurse = (partIndex, ref) ->
-      part = parts[partIndex]
-      res = []
-      if part.test?
-        for p of ref
-          if part.test(p)
-            if partIndex == parts.length - 1
-              res.push(p)
-            else
-              for r in recurse(partIndex + 1, ref[p])
-                res.push("#{p}.#{r}")
-      else
-        if ref[part]?
-          if partIndex == parts.length - 1
-            return [part]
-          else
-            for r in recurse(partIndex + 1, ref[part])
-              res.push("#{part}.#{r}")
-
-      return res
-
-    for key, parts of keys
-      for alias in recurse(0, this)
-        emit(alias, key)
-    return
-
-  reduce = (k, v) ->
-    return v[0]
-
-  aliases = {}
-  keys = {}
-  for k, v of config.PARAMETERS
-    alias = v.alias
-    continue if not alias?
-    isStatic = true
-    ar = []
-    for p in k.match(/\w+|(\/.+?\/\w*)/g)
-      if p[0] == '/' # this is a regex
-        isStatic = false
-        i = p.lastIndexOf('/')
-        r = new RegExp(p[1...i], p[i+1..])
-        ar.push(r)
-      else
-        ar.push(p)
-
-    if isStatic
-      aliases[alias] ?= []
-      aliases[alias].push(k)
-    else
-      keys[k] = ar
-
-  counter = 0
-  splitIds(4, (batches) ->
-    if not batches?
-      return callback(null, aliases)
-
-    for batch in batches
-      options = {
-        out : {inline : 1}
-        jsMode : true
-        sort : {$natural : 1}
-        scope : {keys : keys, flags : {}}
-        query : {_id : batch} if batches.length > 1
-      }
-
-      db.devicesCollection.mapReduce(map, reduce, options, (err, out) ->
-        if err
-          callback?(err)
-          return callback = null
-
-        for o in out
-          alias = config.PARAMETERS[o.value].alias
-          aliases[alias] ?= []
-          id = String(o._id)
-          if id not in aliases[alias]
-            aliases[alias].push(id)
-
-        if ++counter == batches.length
-          callback?(null, aliases)
-      )
-    return
-  )
-
-
-exports.getType = createParameterAttributeFinder('type')
-exports.getAlias = createParameterAttributeFinder('alias')
-exports.compileAliases = compileAliases

lib/presets.coffee

@@ -1,331 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This file incorporates work covered by the following copyright and
-# permission notice:
-#
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-config = require './config'
-common = require './common'
-db = require './db'
-mongoQuery = require './mongo-query'
-query = require './query'
-log = require('util').log
-customCommands = require './custom-commands'
-
-
-exports.calculatePresetsHash = (presets, objects) ->
-  crypto = require('crypto')
-  hash = crypto.createHash('md5').update(JSON.stringify(presets) + JSON.stringify(objects)).digest('hex')
-  return hash
-
-
-matchObject = (object, param) ->
-  return false if not object._keys? or object._keys.length == 0
-  for k in object._keys
-    return false if not param[k]?
-    v = common.matchType(param[k]._value, object[k])
-    if param[k]._value != v
-      return false
-  return true
-
-
-exports.getDevicePreset = (deviceId, presets, objects, aliases, callback) ->
-  deviceCustomCommands = customCommands.getDeviceCustomCommands(deviceId)
-  # only fetch relevant params
-  projection = {_id : 1}
-  for p in presets
-    if common.typeOf(p.precondition) is common.STRING_TYPE
-      p.precondition = query.expand(JSON.parse(p.precondition), aliases)
-    else
-      # Accept an object for backward compatiblity
-      p.precondition = query.expand(p.precondition ? {}, aliases)
-    mongoQuery.projection(p.precondition, projection)
-
-    for c in p.configurations
-      switch c.type
-        when 'value', 'age'
-          projection[c.name] = 1
-        when 'custom_command_value', 'custom_command_age'
-          commandfile = c.command.split(' ', 1)
-          if commandfile of deviceCustomCommands
-            projection["_customCommands.#{c.command.split(' ', 1)}"] = 1
-        when 'software_version'
-          projection['Device.DeviceInfo.SoftwareVersion'] = 1
-          projection['InternetGatewayDevice.DeviceInfo.SoftwareVersion'] = 1
-        when 'add_tag', 'delete_tag'
-          projection['_tags'] = 1
-        when 'add_object', 'delete_object'
-          projection[c.name] = 1
-        else
-          throw new Error('Unknown configuration type')
-
-  mongoQuery.optimizeProjection(projection)
-
-  db.devicesCollection.findOne({'_id' : deviceId}, projection, (err, device) ->
-    throw err if err
-    devicePresets = []
-    for p in presets
-      if mongoQuery.test(device, p.precondition)
-        devicePresets.push(p)
-
-    configurations = accumulateConfigurations(devicePresets, objects)
-
-    getParamDetails = (device, path) ->
-      p = common.getParamValueFromPath(device, path)
-      return undefined if not p?
-      details = {}
-      for k,v of p
-        details[k] = v if k[0] == '_'
-      return details
-
-    devicePreset = {}
-    for c in configurations
-      switch c.type
-        when 'value'
-          devicePreset.parameters ?= {}
-          devicePreset.parameters[c.name] ?= {}
-          devicePreset.parameters[c.name].preset ?= {}
-          devicePreset.parameters[c.name].preset.value = c.value
-          devicePreset.parameters[c.name].current = getParamDetails(device, c.name)
-
-        when 'age'
-          devicePreset.parameters ?= {}
-          devicePreset.parameters[c.name] ?= {}
-          devicePreset.parameters[c.name].preset ?= {}
-          devicePreset.parameters[c.name].preset.expiry = parseInt(c.age)
-          devicePreset.parameters[c.name].current = getParamDetails(device, c.name)
-
-        when 'add_tag'
-          devicePreset.tags ?= {}
-          devicePreset.tags[c.tag] = 2 | c.tag in (device._tags ? [])
-
-        when 'delete_tag'
-          devicePreset.tags ?= {}
-          devicePreset.tags[c.tag] = 0 | c.tag in (device._tags ? [])
-
-        when 'add_object'
-          devicePreset.objects ?= {}
-          devicePreset.objects[c.name] ?= {}
-          devicePreset.objects[c.name][c.object] ?= {}
-          devicePreset.objects[c.name][c.object].preset = objects[c.object]
-          continue if not param = common.getParamValueFromPath(device, c.name)
-          devicePreset.objects[c.name][c.object].current ?= {}
-          for k,p of param
-            continue if k[0] == '_'
-            if p._name?
-              if p._name == c.object
-                devicePreset.objects[c.name][c.object].current[k] = p
-            else if matchObject(objects[c.object], p)
-              u = {}
-              u["#{c.name}.#{k}._name"] = c.object
-              db.devicesCollection.update({'_id' : deviceId}, {'$set' : u}, {}, (err, count) ->
-                throw err if err
-              )
-              devicePreset.objects[c.name][c.object].current[k] = p
-
-        when 'delete_object'
-          devicePreset.objects ?= {}
-          devicePreset.objects[c.name] ?= {}
-          devicePreset.objects[c.name][c.object] ?= {}
-          continue if not param = common.getParamValueFromPath(device, c.name)
-          devicePreset.objects[c.name][c.object].current ?= {}
-          for k,p of param
-            continue if k[0] == '_'
-            if p._name?
-              if p._name == c.object
-                devicePreset.objects[c.name][c.object].current[k] = p
-            else if matchObject(objects[c.object], p)
-              u = {}
-              u["#{c.name}.#{k}._name"] = c.object
-              db.devicesCollection.update({'_id' : deviceId}, {'$set' : u}, {}, (err, count) ->
-                throw err if err
-              )
-              devicePreset.objects[c.name][c.object].current[k] = p
-
-        when 'custom_command_value'
-          [filename, commandName] = c.command.split(' ', 2)
-          devicePreset.customCommands ?= {}
-          devicePreset.customCommands[filename] ?= {}
-          devicePreset.customCommands[filename].preset ?= {}
-          devicePreset.customCommands[filename].preset.value = c.value
-          devicePreset.customCommands[filename].preset.valueCommand = c.command
-          continue if not deviceCustomCommands[filename]? or commandName not in deviceCustomCommands[filename]
-          if cmd = common.getParamValueFromPath(device, "_customCommands.#{filename}")
-            devicePreset.customCommands[filename].current = cmd
-          else
-            devicePreset.customCommands[filename].current ?= {}
-
-        when 'custom_command_age'
-          [filename, commandName] = c.command.split(' ', 2)
-          devicePreset.customCommands ?= {}
-          devicePreset.customCommands[filename] ?= {}
-          devicePreset.customCommands[filename].preset ?= {}
-          devicePreset.customCommands[filename].preset.expiry = parseInt(c.age)
-          devicePreset.customCommands[filename].preset.expiryCommand = c.command
-          continue if not deviceCustomCommands[filename]? or commandName not in deviceCustomCommands[filename]
-          if cmd = common.getParamValueFromPath(device, "_customCommands.#{filename}")
-            devicePreset.customCommands[filename].current = cmd
-          else
-            devicePreset.customCommands[filename].current ?= {}
-
-        when 'software_version'
-          devicePreset.softwareVersion ?= {}
-          devicePreset.softwareVersion.preset = c.software_version
-          if device?.Device? # TR-181 data model
-            devicePreset.softwareVersion.current = common.getParamValueFromPath(device, 'Device.DeviceInfo.SoftwareVersion')
-          else # TR-098 data model
-            devicePreset.softwareVersion.current = common.getParamValueFromPath(device, 'InternetGatewayDevice.DeviceInfo.SoftwareVersion')
-
-        else
-          throw new Error('Unknown configuration type')
-
-    callback(devicePreset)
-  )
-
-
-exports.processDevicePreset = (deviceId, devicePreset, callback) ->
-  PRESETS_TIME_PADDING = config.get('PRESETS_TIME_PADDING')
-  now = new Date()
-  expiry = config.get('PRESETS_CACHE_DURATION', deviceId)
-  getParameterValues = []
-  setParameterValues = []
-  addTags = []
-  deleteTags = []
-  taskList = []
-
-  # Parameters
-  for parameterPath, parameterDetails of devicePreset.parameters ? {}
-    continue if not parameterDetails.current?
-    expiry = Math.min(expiry, parameterDetails.preset.expiry) if parameterDetails.preset.expiry?
-    presetValue = parameterDetails.preset.value
-    currentValue = parameterDetails.current._value
-    if presetValue? and currentValue != (presetValue = common.matchType(currentValue, presetValue))
-      setParameterValues.push([parameterPath, presetValue, parameterDetails.current._type])
-    else if parameterDetails.preset.expiry?
-      diff = parameterDetails.preset.expiry - (now - (parameterDetails.current._timestamp ? 0)) / 1000
-      if diff <= PRESETS_TIME_PADDING
-        if parameterDetails.current._value?
-          getParameterValues.push(parameterPath)
-        else
-          taskList.push({device : deviceId, name : 'refreshObject', objectName : parameterPath})
-      else
-        expiry = Math.min(expiry, diff)
-
-  # Objects
-  for parameterPath, object of devicePreset.objects ? {}
-    for objectName, objectDetails of object
-      continue if not objectDetails.current?
-      if objectDetails.preset?
-        if Object.keys(objectDetails.current).length > 0
-          for i, obj of objectDetails.current
-            for paramName, paramValue of objectDetails.preset
-              continue if paramName[0] == '_'
-              if obj[paramName]
-                currentValue = obj[paramName]._value
-                presetValue = common.matchType(currentValue, objectDetails.preset[paramName])
-                if currentValue != presetValue
-                  setParameterValues.push(["#{parameterPath}.#{i}.#{paramName}", presetValue, obj[paramName]._type])
-        else
-          vals = []
-          for k,v of objectDetails.preset
-            vals.push([k, v]) if k[0] != '_'
-          taskList.push({device : deviceId, name : 'addObject', objectName : parameterPath, parameterValues : vals, instanceName : objectName})
-      else if Object.keys(objectDetails.current).length > 0
-        for i, obj of objectDetails.current
-          taskList.push({device : deviceId, name : 'deleteObject', objectName : "#{parameterPath}.#{i}"})
-
-  # Tags
-  for tag, v of devicePreset.tags ? {}
-    if v == 1
-      deleteTags.push(tag)
-    else if v == 2
-      addTags.push(tag)
-
-  # Custom commands
-  for commandName, commandDetails of devicePreset.customCommands ? {}
-    continue if not commandDetails.current?
-    expiry = Math.min(expiry, commandDetails.preset.expiry) if commandDetails.preset.expiry?
-    currentValue = commandDetails.current._value
-    presetValue = common.matchType(currentValue, commandDetails.preset.value)
-    if commandDetails.preset.value? and currentValue != presetValue
-      taskList.push({device : deviceId, name : 'customCommand', command : commandDetails.preset.valueCommand})
-    else if commandDetails.preset.expiry?
-      diff = commandDetails.preset.expiry - (now - (commandDetails.current._timestamp ? 0)) / 1000
-      if diff <= PRESETS_TIME_PADDING
-        taskList.push({device : deviceId, name : 'customCommand', command : commandDetails.preset.expiryCommand})
-      else
-        expiry = Math.min(expiry, diff)
-
-  if setParameterValues.length
-    taskList.push {device : deviceId, name : 'setParameterValues', parameterValues: setParameterValues, timestamp : now}
-
-  if getParameterValues.length
-    taskList.push {device : deviceId, name : 'getParameterValues', parameterNames: getParameterValues, timestamp : now}
-
-  callback(taskList, addTags, deleteTags, expiry)
-
-
-getObjectHash = (object) ->
-  return object._id if not object._keys? or object._keys.length == 0
-  hash = ''
-  for k in object._keys
-    hash += "#{k}=#{object[k]}"
-  return hash
-
-
-accumulateConfigurations = (presets, objects) ->
-  maxWeights = {}
-  configurations = {}
-  for p in presets
-    for c in p.configurations
-      configurationHash = switch c.type
-        when 'add_tag', 'delete_tag'
-          "tag_#{c.tag}"
-        when 'add_object', 'delete_object'
-          objectHash = getObjectHash(objects[c.object])
-          "object_#{c.name}_#{objectHash}"
-        when 'software_version'
-          'software_version'
-        else
-          "#{c.type}_#{c.name}"
-
-      if not maxWeights[configurationHash]? or p.weight > maxWeights[configurationHash]
-        configurations[configurationHash] = c
-        maxWeights[configurationHash] = p.weight
-
-  configurationsList = (configurations[c] for c of configurations)
-  return configurationsList

lib/query.coffee

@@ -1,107 +0,0 @@
-###
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-common = require './common'
-normalize = require './normalize'
-
-
-expandValue = (param, value) ->
-  if common.typeOf(value) is common.ARRAY_TYPE
-    a = []
-    for j in value
-      a = a.concat(expandValue(param, j))
-    return [a]
-  else if common.typeOf(value) isnt common.OBJECT_TYPE
-    n = normalize.normalize(param, value, 'query')
-    if common.typeOf(n) isnt common.ARRAY_TYPE
-      return [n]
-    else
-      return n
-
-  objs = []
-  indices = []
-  keys = []
-  values = []
-  for k,v of value
-    keys.push(k)
-    values.push(expandValue(param, v))
-    indices.push(0)
-
-  i = 0
-  while i < indices.length
-    obj = {}
-    for i in [0...keys.length]
-      obj[keys[i]] = values[i][indices[i]]
-    objs.push(obj)
-
-    for i in [0...indices.length]
-      indices[i] += 1
-      if indices[i] < values[i].length
-        break
-      indices[i] = 0
-  return objs
-
-
-permute = (param, val, aliases) ->
-  keys = []
-  if aliases[param]?
-    for p in aliases[param]
-      keys.push(p)
-  else
-    keys.push(param)
-
-  conditions = []
-  for k in keys
-    values = expandValue(k, val)
-    if k[k.lastIndexOf('.') + 1] != '_'
-      k += '._value'
-  
-    for v in values
-      obj = {}
-      obj[k] = v
-      conditions.push(obj)
-
-  return conditions
-
-
-expand = (query, aliases) ->
-  new_query = {}
-  for k,v of query
-    if k[0] == '$' # operator
-      expressions = []
-      for e in v
-        expressions.push(expand(e, aliases))
-      new_query[k] = expressions
-    else
-      conditions = permute(k, v, aliases)
-      if conditions.length > 1
-        if new_query['$and']?
-          new_query['$and'].push({'$or' : conditions})
-        else
-          new_query['$and'] = [{'$or' : conditions}]
-      else
-        common.extend(new_query, conditions[0])
-
-  return new_query
-
-
-exports.expand = expand

lib/query.ts

@@ -0,0 +1,255 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as common from "./common";
+
+function stringToRegexp(input, flags?): RegExp | false {
+  if (input.indexOf("*") === -1) return false;
+
+  let output = input.replace(/[[\]\\^$.|?+()]/, "\\$&");
+  if (output[0] === "*") output = output.replace(/^\*+/g, "");
+  else output = "^" + output;
+
+  if (output[output.length - 1] === "*") output = output.replace(/\*+$/g, "");
+  else output = output + "$";
+
+  output = output.replace(/[*]/, ".*");
+  return new RegExp(output, flags);
+}
+
+function normalize(input): any {
+  if (common.typeOf(input) === common.STRING_TYPE) {
+    const vals = [input];
+    const m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input);
+    if (m) vals.push({ $regex: new RegExp(m[1], m[2]) });
+
+    if (+input === parseFloat(input)) vals.push(+input);
+
+    const d = new Date(input);
+    if (input.length >= 8 && d.getFullYear() > 1983) vals.push(d);
+
+    const r = stringToRegexp(input);
+    if (r !== false) vals.push({ $regex: r });
+
+    return vals;
+  }
+  return input;
+}
+
+export function expandValue(value): any[] {
+  if (common.typeOf(value) === common.ARRAY_TYPE) {
+    let a = [];
+    for (const j of value) a = a.concat(expandValue(j));
+    return [a];
+  } else if (common.typeOf(value) !== common.OBJECT_TYPE) {
+    const n = normalize(value);
+    if (common.typeOf(n) !== common.ARRAY_TYPE) return [n];
+    else return n;
+  }
+
+  const objs = [];
+  const indices = [];
+  const keys = [];
+  const values = [];
+  for (const [k, v] of Object.entries(value)) {
+    keys.push(k);
+    values.push(expandValue(v));
+    indices.push(0);
+  }
+
+  let i = 0;
+  while (i < indices.length) {
+    const obj = {};
+    for (let j = 0; j < keys.length; ++j) obj[keys[j]] = values[j][indices[j]];
+    objs.push(obj);
+
+    for (i = 0; i < indices.length; ++i) {
+      indices[i] += 1;
+      if (indices[i] < values[i].length) break;
+      indices[i] = 0;
+    }
+  }
+  return objs;
+}
+
+function permute(param, val): any[] {
+  const conditions = [];
+  const values = expandValue(val);
+
+  if (param[param.lastIndexOf(".") + 1] !== "_") param += "._value";
+
+  for (const v of values) {
+    const obj = {};
+    obj[param] = v;
+    conditions.push(obj);
+  }
+
+  return conditions;
+}
+
+export function expand(query): {} {
+  const newQuery = {};
+  for (const [k, v] of Object.entries(query)) {
+    if (k[0] === "$") {
+      // Operator
+      newQuery[k] = (v as any[]).map(e => expand(e));
+    } else {
+      const conditions = permute(k, v);
+      if (conditions.length > 1) {
+        newQuery["$and"] = newQuery["$and"] || [];
+        if (v && (v["$ne"] != null || v["$not"] != null)) {
+          if (Object.keys(v).length > 1)
+            throw new Error("Cannot mix $ne or $not with other operators");
+          for (const c of conditions) newQuery["$an"].push(c);
+        } else {
+          newQuery["$and"].push({ $or: conditions });
+        }
+      } else {
+        Object.assign(newQuery, conditions[0]);
+      }
+    }
+  }
+
+  return newQuery;
+}
+
+function testExpressions(params, expressions, lop): boolean {
+  for (const f of expressions) {
+    const res = test(params, f);
+    switch (lop) {
+      case "$and":
+        if (!res) return false;
+        break;
+      case "$or":
+        if (res) return true;
+        break;
+      case "$nor":
+        if (res) return false;
+    }
+  }
+
+  switch (lop) {
+    case "$and":
+      return true;
+    case "$or":
+      return false;
+    case "$nor":
+      return true;
+    default:
+      throw new Error("Unknown logical operator");
+  }
+}
+
+export function test(params, query): boolean {
+  let res;
+  for (const [k, v] of Object.entries(query)) {
+    if (k.charAt(0) === "$") {
+      // Logical operator
+      res = testExpressions(params, v, k);
+    } else {
+      const value = params[k];
+      if (common.typeOf(v) !== common.OBJECT_TYPE) {
+        if (common.typeOf(value) === common.ARRAY_TYPE) {
+          // TODO comparing array to regex, array to array, and object to object
+          res = value.indexOf(v) !== -1;
+        } else {
+          if (common.typeOf(v) === common.REGEXP_TYPE)
+            res = (v as RegExp).test(value);
+          else res = v === value;
+        }
+      } else {
+        for (const [k2, v2] of Object.entries(v)) {
+          switch (k2) {
+            case "$ne":
+              if (common.typeOf(value) === common.ARRAY_TYPE)
+                res = value.indexOf(v2) === -1;
+              else res = value !== v2;
+              break;
+            case "$lt":
+              res = value < v2;
+              break;
+            case "$lte":
+              res = value <= v2;
+              break;
+            case "$gt":
+              res = value > v2;
+              break;
+            case "$gte":
+              res = value >= v2;
+              break;
+            case "$regex":
+              res = v2.test(value);
+              break;
+            case "$in":
+              throw new Error("Operator not supported");
+            case "$nin":
+              throw new Error("Operator not supported");
+            case "$all":
+              throw new Error("Operator not supported");
+            case "$exists":
+              throw new Error("Operator not supported");
+            default:
+              throw new Error("Operator not supported");
+          }
+        }
+      }
+    }
+
+    if (!res) return false;
+  }
+  return true;
+}
+
+export function sanitizeQueryTypes(query, types): {} {
+  for (const [k, v] of Object.entries(query)) {
+    if (k[0] === "$") {
+      // Logical operator
+      for (const vv of v as any[]) sanitizeQueryTypes(vv, types);
+    } else if (k in types) {
+      if (common.typeOf(v) === common.OBJECT_TYPE) {
+        for (const [kk, vv] of Object.entries(v)) {
+          switch (kk) {
+            case "$in":
+            case "$nin":
+              for (let i = 0; i < vv.length; ++i) vv[i] = types[k](vv[i]);
+              break;
+            case "$eq":
+            case "$gt":
+            case "$gte":
+            case "$lt":
+            case "$lte":
+            case "$ne":
+              v[kk] = types[k](vv);
+              break;
+            case "$exists":
+            case "$type":
+              // Ignore
+              break;
+            default:
+              throw new Error("Operator not supported");
+          }
+        }
+      } else {
+        query[k] = types[k](query[k]);
+      }
+    }
+  }
+
+  return query;
+}

lib/sandbox.ts

@@ -0,0 +1,438 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as vm from "vm";
+import seedrandom from "seedrandom";
+import * as device from "./device";
+import * as extensions from "./extensions";
+import * as logger from "./logger";
+import * as scheduling from "./scheduling";
+import Path from "./common/path";
+import { Fault, SessionContext, ScriptResult } from "./types";
+
+// Used for throwing to exit user script and commit
+const COMMIT = Symbol();
+
+// Used to execute extensions and restart
+const EXT = Symbol();
+
+const UNDEFINED = undefined;
+
+const context = vm.createContext();
+
+let state;
+
+const runningExtensions = new WeakMap<
+  SessionContext,
+  Map<string, Promise<Fault>>
+>();
+function runExtension(sessionContext, key, extCall): Promise<Fault> {
+  let re = runningExtensions.get(sessionContext);
+  if (!re) {
+    re = new Map<string, Promise<Fault>>();
+    runningExtensions.set(sessionContext, re);
+  }
+
+  let prom = re.get(key);
+  if (!prom) {
+    re.set(
+      key,
+      (prom = new Promise((resolve, reject) => {
+        extensions
+          .run(extCall)
+          .then(({ fault, value }) => {
+            re.delete(key);
+            if (!fault) sessionContext.extensionsCache[key] = value;
+            resolve(fault);
+          })
+          .catch(reject);
+      }))
+    );
+  }
+
+  return prom;
+}
+
+class SandboxDate {
+  public constructor(
+    ...argumentList: [
+      number?,
+      number?,
+      number?,
+      number?,
+      number?,
+      number?,
+      number?
+    ]
+  ) {
+    if (argumentList.length) return new Date(...argumentList);
+
+    return new Date(state.sessionContext.timestamp);
+  }
+
+  public static now(intervalOrCron, variance): number {
+    let t = state.sessionContext.timestamp;
+
+    if (typeof intervalOrCron === "number") {
+      if (variance == null) variance = intervalOrCron;
+
+      let offset = 0;
+      if (variance)
+        offset = scheduling.variance(state.sessionContext.deviceId, variance);
+
+      t = scheduling.interval(t, intervalOrCron, offset);
+    } else if (typeof intervalOrCron === "string") {
+      let offset = 0;
+      if (variance)
+        offset = scheduling.variance(state.sessionContext.deviceId, variance);
+      const cron = scheduling.parseCron(intervalOrCron);
+      t = scheduling.cron(t, cron, offset)[0];
+    } else if (intervalOrCron) {
+      throw new Error("Invalid Date.now() argument");
+    }
+
+    return t;
+  }
+}
+
+function random(): number {
+  if (!state.rng) state.rng = seedrandom(state.sessionContext.deviceId);
+
+  return state.rng();
+}
+
+random.seed = function(s) {
+  state.rng = seedrandom(s);
+};
+
+class ParameterWrapper {
+  public constructor(path: Path, attributes, unpacked?, unpackedRevision?) {
+    for (const attrName of attributes) {
+      Object.defineProperty(this, attrName, {
+        get: function() {
+          if (state.uncommitted) commit();
+
+          if (state.revision !== unpackedRevision) {
+            unpackedRevision = state.revision;
+            unpacked = device.unpack(
+              state.sessionContext.deviceData,
+              path,
+              state.revision
+            );
+          }
+
+          if (!unpacked.length) return UNDEFINED;
+
+          const attr = state.sessionContext.deviceData.attributes.get(
+            unpacked[0],
+            state.revision
+          )[attrName];
+
+          if (!attr) return UNDEFINED;
+
+          return attr[1];
+        }
+      });
+    }
+
+    Object.defineProperty(this, "path", {
+      get: function() {
+        if (state.uncommitted) commit();
+
+        if (state.revision !== unpackedRevision) {
+          unpackedRevision = state.revision;
+          unpacked = device.unpack(
+            state.sessionContext.deviceData,
+            path,
+            state.revision
+          );
+        }
+
+        if (!unpacked.length) return UNDEFINED;
+
+        return unpacked[0].toString();
+      }
+    });
+
+    Object.defineProperty(this, "size", {
+      get: function() {
+        if (state.uncommitted) commit();
+
+        if (state.revision !== unpackedRevision) {
+          unpackedRevision = state.revision;
+          unpacked = device.unpack(
+            state.sessionContext.deviceData,
+            path,
+            state.revision
+          );
+        }
+
+        if (!unpacked.length) return UNDEFINED;
+
+        return unpacked.length;
+      }
+    });
+
+    this[Symbol.iterator] = function*() {
+      if (state.uncommitted) commit();
+
+      if (state.revision !== unpackedRevision) {
+        unpackedRevision = state.revision;
+        unpacked = device.unpack(
+          state.sessionContext.deviceData,
+          path,
+          state.revision
+        );
+      }
+
+      for (const p of unpacked)
+        yield new ParameterWrapper(p, attributes, [p], state.revision);
+    };
+  }
+}
+
+function declare(
+  path: string,
+  timestamps: { [attr: string]: number },
+  values: { [attr: string]: any }
+): ParameterWrapper {
+  state.uncommitted = true;
+  if (!timestamps) timestamps = {};
+
+  if (!values) values = {};
+
+  const parsedPath = Path.parse(path);
+
+  const declaration = {
+    path: parsedPath,
+    pathGet: 1,
+    pathSet: null,
+    attrGet: null,
+    attrSet: null,
+    defer: true
+  };
+
+  const attrs = new Set();
+
+  for (const [attrName, attrValue] of Object.entries(values)) {
+    if (attrName === "path") {
+      declaration.pathSet = attrValue;
+    } else {
+      attrs.add(attrName);
+      if (!declaration.attrGet) declaration.attrGet = {};
+      if (!declaration.attrSet) declaration.attrSet = {};
+      declaration.attrGet[attrName] = 1;
+      if (attrName === "value" && !Array.isArray(values.value))
+        declaration.attrSet.value = [values.value];
+      else declaration.attrSet[attrName] = values[attrName];
+    }
+  }
+
+  for (const [attrName, attrTimestamp] of Object.entries(timestamps)) {
+    if (attrName === "path") {
+      declaration.pathGet = attrTimestamp;
+    } else {
+      attrs.add(attrName);
+      if (!declaration.attrGet) declaration.attrGet = {};
+      declaration.attrGet[attrName] = attrTimestamp;
+    }
+  }
+
+  state.declarations.push(declaration);
+
+  return new ParameterWrapper(parsedPath, attrs);
+}
+
+function clear(path: string, timestamp: number, attributes?): void {
+  state.uncommitted = true;
+
+  if (state.revision === state.maxRevision)
+    state.clear.push([Path.parse(path), timestamp, attributes]);
+}
+
+function commit(): void {
+  ++state.revision;
+  state.uncommitted = false;
+
+  if (state.revision === state.maxRevision + 1) {
+    for (const d of state.declarations) d.defer = false;
+    throw COMMIT;
+  } else if (state.revision > state.maxRevision + 1) {
+    throw new Error(
+      "Declare function should not be called from within a try/catch block"
+    );
+  }
+}
+
+function ext(): any {
+  ++state.extCounter;
+  const extCall = Array.from(arguments).map(String);
+  const key = `${state.revision}: ${JSON.stringify(extCall)}`;
+
+  if (key in state.sessionContext.extensionsCache)
+    return state.sessionContext.extensionsCache[key];
+
+  state.extensions[key] = extCall;
+  throw EXT;
+}
+
+function log(msg: string, meta: {}): void {
+  if (state.revision === state.maxRevision && state.extCounter >= 0) {
+    const details = Object.assign({}, meta, {
+      sessionContext: state.sessionContext,
+      message: `Script: ${msg}`
+    });
+
+    delete details["hostname"];
+    delete details["pid"];
+    delete details["name"];
+    delete details["version"];
+    delete details["deviceId"];
+    delete details["remoteAddress"];
+
+    logger.accessInfo(details);
+  }
+}
+
+Object.defineProperty(context, "Date", { value: SandboxDate });
+Object.defineProperty(context, "declare", { value: declare });
+Object.defineProperty(context, "clear", { value: clear });
+Object.defineProperty(context, "commit", { value: commit });
+Object.defineProperty(context, "ext", { value: ext });
+Object.defineProperty(context, "log", { value: log });
+
+// Monkey-patch Math.random() to make it deterministic
+context.random = random;
+vm.runInContext("Math.random = random;", context);
+delete context.random;
+
+function errorToFault(err: Error): Fault {
+  if (!err) return null;
+
+  if (!err.name) return { code: "script", message: `${err}` };
+
+  const fault: Fault = {
+    code: `script.${err.name}`,
+    message: err.message,
+    detail: {
+      name: err.name,
+      message: err.message
+    }
+  };
+
+  if (err.stack) {
+    fault.detail["stack"] = err.stack;
+    // Trim the stack trace at the self-executing anonymous wrapper function
+    const stackTrimIndex = fault.detail["stack"].match(
+      /\s+at\s[^\s]+\s+at\s[^\s]+\s\(vm\.js.+\)/
+    );
+    if (stackTrimIndex) {
+      fault.detail["stack"] = fault.detail["stack"].slice(
+        0,
+        stackTrimIndex.index
+      );
+    }
+  }
+
+  return fault;
+}
+
+export async function run(
+  script: vm.Script,
+  globals: {},
+  sessionContext: SessionContext,
+  startRevision: number,
+  maxRevision: number,
+  extCounter = 0
+): Promise<ScriptResult> {
+  state = {
+    sessionContext: sessionContext,
+    revision: startRevision,
+    maxRevision: maxRevision,
+    uncommitted: false,
+    declarations: [],
+    extensions: {},
+    clear: [],
+    rng: null,
+    extCounter: extCounter
+  };
+
+  for (const n of Object.keys(context)) delete context[n];
+
+  Object.assign(context, globals);
+
+  let ret, status;
+
+  try {
+    ret = script.runInContext(context, { displayErrors: false });
+    status = 0;
+  } catch (err) {
+    if (err === COMMIT) {
+      status = 1;
+    } else if (err === EXT) {
+      status = 2;
+    } else {
+      return {
+        fault: errorToFault(err),
+        clear: null,
+        declare: null,
+        done: false,
+        returnValue: null
+      };
+    }
+  }
+
+  const _state = state;
+  let fault;
+
+  await Promise.all(
+    Object.entries(_state.extensions).map(async ([k, v]) => {
+      fault = (await runExtension(_state.sessionContext, k, v)) || fault;
+    })
+  );
+
+  if (fault) {
+    return {
+      fault: fault,
+      clear: null,
+      declare: null,
+      done: false,
+      returnValue: null
+    };
+  }
+
+  if (status === 2) {
+    return run(
+      script,
+      globals,
+      sessionContext,
+      startRevision,
+      maxRevision,
+      extCounter - _state.extCounter
+    );
+  }
+
+  return {
+    fault: null,
+    clear: _state.clear,
+    declare: _state.declarations,
+    done: status === 0,
+    returnValue: ret
+  };
+}

lib/scheduling.ts

@@ -0,0 +1,62 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as crypto from "crypto";
+import * as later from "later";
+
+function md532(str): number {
+  const digest = crypto
+    .createHash("md5")
+    .update(str)
+    .digest();
+  return (
+    digest.readUInt32LE(0) ^
+    digest.readUInt32LE(4) ^
+    digest.readUInt32LE(8) ^
+    digest.readUInt32LE(12)
+  );
+}
+
+export function variance(deviceId, vrnc): number {
+  return (md532(deviceId) >>> 0) % vrnc;
+}
+
+export function interval(timestamp, intrvl, offset = 0): number {
+  return Math.trunc((timestamp + offset) / intrvl) * intrvl - offset;
+}
+
+export function parseCron(cronExp): any {
+  const parts = cronExp.trim().split(/\s+/);
+  if (parts.length === 5) parts.unshift("*");
+
+  return later.schedule(later.parse.cron(parts.join(" "), true));
+}
+
+export function cron(timestamp, schedule, offset = 0): number[] {
+  // TODO later.js doesn't throw erorr if expression is invalid!
+  const ret = [0, 0];
+
+  const prev = schedule.prev(1, new Date(timestamp + offset));
+  if (prev) ret[0] = prev.setMilliseconds(0) - offset;
+
+  const next = schedule.next(1, new Date(timestamp + offset + 1000));
+  if (next) ret[1] = next.setMilliseconds(0) - offset;
+
+  return ret;
+}

lib/server.coffee

@@ -1,83 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-domain = require 'domain'
-util = require 'util'
-cluster = require 'cluster'
-config = require './config'
-db = require './db'
-
-service = config.argv[0]
-if not service?
-  throw new Error('Missing argument cwmp, fs, or nbi')
-
-networkInterface = config.get("#{service.toUpperCase()}_INTERFACE")
-port = config.get("#{service.toUpperCase()}_PORT")
-useHttps = config.get("#{service.toUpperCase()}_SSL")
-
-serviceListener = require("./#{service}").listener
-
-server = null
-
-listener = (httpRequest, httpResponse) ->
-  d = domain.create()
-
-  d.on('error', (err) ->
-    util.error("#{new Date().toISOString()} - #{err.stack}\n")
-    try
-      killTimer = setTimeout(() ->
-        process.exit(1)
-      , 30000)
-
-      killTimer.unref()
-
-      cluster.worker?.disconnect()
-      server.close(() ->
-        db.disconnect()
-      )
-
-      httpResponse.writeHead(500, {'Connection' : 'close'})
-      httpResponse.end()
-    catch err2
-      util.error("#{new Date().toISOString()} - #{err2.stack}\n")
-  )
-
-  d.add(httpRequest)
-  d.add(httpResponse)
-
-  d.run(() ->
-    serviceListener(httpRequest, httpResponse)
-  )
-
-
-if useHttps
-  path = require 'path'
-  fs = require 'fs'
-  httpsKey = path.resolve(config.get('CONFIG_DIR'), "#{service}.key")
-  httpsCert = path.resolve(config.get('CONFIG_DIR'), "#{service}.crt")
-  options = {
-    key: fs.readFileSync(httpsKey),
-    cert: fs.readFileSync(httpsCert)
-  }
-  server = require('https').createServer(options, listener)
-else
-  server = require('http').createServer(listener)
-
-db.connect((err) ->
-  throw err if err
-  server.listen(port, networkInterface)
-)

lib/server.ts

@@ -0,0 +1,91 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as fs from "fs";
+import * as http from "http";
+import * as https from "https";
+import * as path from "path";
+import { ROOT_DIR } from "./config";
+
+let server: http.Server | https.Server;
+let listener: (...args) => void;
+
+function closeServer(timeout, callback): void {
+  if (!server) return void callback();
+
+  setTimeout(() => {
+    if (!callback) return;
+
+    // Ignore HTTP requests from connection that may still be open
+    server.removeListener("request", listener);
+    server.setTimeout(1);
+
+    const cb = callback;
+    callback = null;
+    setTimeout(cb, 1000);
+  }, timeout).unref();
+
+  server.close(() => {
+    if (!callback) return;
+
+    const cb = callback;
+    callback = null;
+    cb();
+  });
+}
+
+export function start(
+  port,
+  networkInterface,
+  ssl,
+  _listener,
+  onConnection?,
+  keepAliveTimeout: number = 0
+): void {
+  listener = _listener;
+
+  if (ssl && ssl.key && ssl.cert) {
+    const options = {
+      key: ssl.key
+        .split(":")
+        .map(f => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
+      cert: ssl.cert
+        .split(":")
+        .map(f => fs.readFileSync(path.resolve(ROOT_DIR, f.trim())))
+    };
+
+    server = https.createServer(options, listener);
+    if (onConnection != null) server.on("secureConnection", onConnection);
+  } else {
+    server = http.createServer(listener);
+    if (onConnection != null) server.on("connection", onConnection);
+  }
+
+  if (keepAliveTimeout) server.keepAliveTimeout = keepAliveTimeout;
+  server.listen(port, networkInterface);
+}
+
+export function stop(): Promise<void> {
+  return new Promise((resolve, reject) => {
+    setTimeout(() => {
+      reject(new Error("Could not close server in a timely manner"));
+    }, 30000).unref();
+    closeServer(20000, resolve);
+  });
+}

lib/soap-libxmljs.ts

@@ -0,0 +1,745 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { parseXml, Document, Element, ParserOptions } from "libxmljs";
+import * as config from "./config";
+import { version as VERSION } from "../package.json";
+import {
+  CpeGetResponse,
+  CpeSetResponse,
+  InformRequest,
+  AcsResponse,
+  CpeRequest,
+  FaultStruct,
+  SpvFault,
+  CpeFault,
+  SoapMessage,
+  TransferCompleteRequest
+} from "./types";
+
+const SERVER_NAME = `GenieACS/${VERSION}`;
+
+const NAMESPACES = {
+  "1.0": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-0"
+  },
+  "1.1": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-1"
+  },
+  "1.2": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-2"
+  },
+  "1.3": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-2"
+  },
+  "1.4": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-3"
+  }
+};
+
+const LIBXMLJS_PARSE_OPTIONS: ParserOptions = {
+  nocdata: true,
+  recover: !!config.get("XML_RECOVER"),
+  /* eslint-disable-next-line @typescript-eslint/camelcase */
+  ignore_enc: !!config.get("XML_IGNORE_ENC")
+};
+
+interface WriterOptions {
+  declaration?: boolean;
+  format?: boolean;
+  selfCloseEmpty?: boolean;
+  whitespace?: boolean;
+  type?: "XML" | "HTML" | "XHTML";
+}
+
+declare module "libxmljs" {
+  interface Document {
+    toString(formatting: boolean | WriterOptions): string;
+  }
+}
+
+const LIBXMLJS_SAVE_OPTIONS: WriterOptions = {
+  format: !!config.get("XML_FORMAT"),
+  declaration: !config.get("XML_NO_DECL"),
+  selfCloseEmpty: !config.get("XML_NO_EMPTY")
+};
+
+let warnings;
+
+function parseBool(v): boolean {
+  v = "" + v;
+  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
+  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
+    return false;
+  else return null;
+}
+
+function event(xml): string[] {
+  return xml
+    .childNodes()
+    .filter(n => n.name() === "EventStruct")
+    .map(c =>
+      c
+        .childNodes()
+        .find(n => n.name() === "EventCode")
+        .text()
+        .trim()
+    );
+}
+
+function parameterInfoList(xml): [string, boolean][] {
+  return xml
+    .childNodes()
+    .filter(e => e.name() === "ParameterInfoStruct")
+    .map(e => {
+      let param, value;
+      for (const c of e.childNodes()) {
+        switch (c.name()) {
+          case "Name":
+            param = c.text();
+            break;
+          case "Writable":
+            value = c.text();
+            break;
+        }
+      }
+
+      let parsed = parseBool(value);
+
+      if (parsed === null) {
+        warnings.push({
+          message: "Invalid writable attribute",
+          parameter: param
+        });
+        parsed = false;
+      }
+
+      return [param, parsed];
+    });
+}
+
+function parameterValueList(
+  xml
+): [string, string | number | boolean, string][] {
+  return xml
+    .childNodes()
+    .filter(e => e.name() === "ParameterValueStruct")
+    .map(e => {
+      let valueElement, param;
+      for (const c of e.childNodes()) {
+        switch (c.name()) {
+          case "Name":
+            param = c.text();
+            break;
+          case "Value":
+            valueElement = c;
+            break;
+        }
+      }
+
+      const valueType = valueElement.attr("type").value();
+      const value = valueElement.text();
+      let parsed = value;
+      if (valueType === "xsd:boolean") {
+        parsed = parseBool(value);
+        if (parsed === null) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      } else if (valueType === "xsd:int" || valueType === "xsd:unsignedInt") {
+        parsed = parseInt(value);
+        if (isNaN(parsed)) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      } else if (valueType === "xsd:dateTime") {
+        parsed = Date.parse(value);
+        if (isNaN(parsed)) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      }
+
+      return [param, parsed, valueType];
+    });
+}
+
+function GetParameterNames(xml, methodRequest): void {
+  const el = xml.node("cwmp:GetParameterNames");
+  el.node("ParameterPath").text(methodRequest.parameterPath);
+  el.node("NextLevel").text(+methodRequest.nextLevel);
+}
+
+function GetParameterNamesResponse(xml): CpeGetResponse {
+  return {
+    name: "GetParameterNamesResponse",
+    parameterList: parameterInfoList(
+      xml.childNodes().find(n => n.name() === "ParameterList")
+    )
+  };
+}
+
+function GetParameterValues(xml, methodRequest): void {
+  const el = xml.node("cwmp:GetParameterValues").node("ParameterNames");
+  el.attr({
+    "soap-enc:arrayType": `xsd:string[${methodRequest.parameterNames.length}]`
+  });
+
+  for (const p of methodRequest.parameterNames) el.node("string").text(p);
+}
+
+function GetParameterValuesResponse(xml): CpeGetResponse {
+  return {
+    name: "GetParameterValuesResponse",
+    parameterList: parameterValueList(
+      xml.childNodes().find(n => n.name() === "ParameterList")
+    )
+  };
+}
+
+function SetParameterValues(xml, methodRequest): void {
+  const el = xml.node("cwmp:SetParameterValues");
+  const paramList = el.node("ParameterList");
+  paramList.attr({
+    "soap-enc:arrayType": `cwmp:ParameterValueStruct[${methodRequest.parameterList.length}]`
+  });
+
+  for (const p of methodRequest.parameterList) {
+    const pvs = paramList.node("ParameterValueStruct");
+    pvs.node("Name").text(p[0]);
+    let val = p[1];
+    if (p[2] === "xsd:dateTime" && typeof val === "number") {
+      val = new Date(val).toISOString();
+      if (methodRequest.DATETIME_MILLISECONDS === false)
+        val = val.replace(".000", "");
+    }
+    if (p[2] === "xsd:boolean" && typeof val === "boolean")
+      if (methodRequest.BOOLEAN_LITERAL === false) val = +val;
+
+    pvs
+      .node("Value")
+      .attr({ "xsi:type": p[2] })
+      .text(val);
+  }
+
+  el.node("ParameterKey").text(methodRequest.parameterKey || "");
+}
+
+function SetParameterValuesResponse(xml): CpeSetResponse {
+  return {
+    name: "SetParameterValuesResponse",
+    status: parseInt(
+      xml
+        .childNodes()
+        .find(n => n.name() === "Status")
+        .text()
+    )
+  };
+}
+
+function AddObject(xml, methodRequest): void {
+  const el = xml.node("cwmp:AddObject");
+  el.node("ObjectName").text(methodRequest.objectName);
+  el.node("ParameterKey").text(methodRequest.parameterKey || "");
+}
+
+function AddObjectResponse(xml): CpeSetResponse {
+  let instanceNumber, status;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "InstanceNumber":
+        instanceNumber = parseInt(c.text());
+        break;
+      case "Status":
+        status = parseInt(c.text());
+        break;
+    }
+  }
+
+  return {
+    name: "AddObjectResponse",
+    instanceNumber: instanceNumber,
+    status: status
+  };
+}
+
+function DeleteObject(xml, methodRequest): void {
+  const el = xml.node("cwmp:DeleteObject");
+  el.node("ObjectName").text(methodRequest.objectName);
+  el.node("ParameterKey").text(methodRequest.parameterKey || "");
+}
+
+function DeleteObjectResponse(xml): CpeSetResponse {
+  return {
+    name: "DeleteObjectResponse",
+    status: parseInt(
+      xml
+        .childNodes()
+        .find(n => n.name() === "Status")
+        .text()
+    )
+  };
+}
+
+function Reboot(xml, methodRequest): void {
+  const el = xml.node("cwmp:Reboot");
+  el.node("CommandKey").text(methodRequest.commandKey || "");
+}
+
+function RebootResponse(): CpeSetResponse {
+  return {
+    name: "RebootResponse"
+  };
+}
+
+function FactoryReset(xml): void {
+  xml.node("cwmp:FactoryReset");
+}
+
+function FactoryResetResponse(): CpeSetResponse {
+  return {
+    name: "FactoryResetResponse"
+  };
+}
+
+function Download(xml, methodRequest): void {
+  const el = xml.node("cwmp:Download");
+  el.node("CommandKey").text(methodRequest.commandKey || "");
+  el.node("FileType").text(methodRequest.fileType);
+  el.node("URL").text(methodRequest.url);
+  el.node("Username").text(methodRequest.username || "");
+  el.node("Password").text(methodRequest.password || "");
+  el.node("FileSize").text(methodRequest.fileSize || "0");
+  el.node("TargetFileName").text(methodRequest.targetFileName || "");
+  el.node("DelaySeconds").text(methodRequest.delaySeconds || "0");
+  el.node("SuccessURL").text(methodRequest.successUrl || "");
+  el.node("FailureURL").text(methodRequest.failureUrl || "");
+}
+
+function DownloadResponse(xml): CpeSetResponse {
+  let status, startTime, completeTime;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "Status":
+        status = parseInt(c.text());
+        break;
+      case "StartTime":
+        startTime = Date.parse(c.text());
+        break;
+      case "CompleteTime":
+        completeTime = Date.parse(c.text());
+        break;
+    }
+  }
+
+  return {
+    name: "DownloadResponse",
+    status: status,
+    startTime: startTime,
+    completeTime: completeTime
+  };
+}
+
+function Inform(xml): InformRequest {
+  let retryCount, evnt, parameterList;
+  const deviceId = {
+    Manufacturer: null,
+    OUI: null,
+    ProductClass: null,
+    SerialNumber: null
+  };
+
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "ParameterList":
+        parameterList = parameterValueList(c);
+        break;
+      case "DeviceId":
+        for (const cc of c.childNodes()) {
+          const n = cc.name();
+          if (n in deviceId) deviceId[n] = cc.text();
+        }
+        break;
+      case "Event":
+        evnt = event(c);
+        break;
+      case "RetryCount":
+        retryCount = parseInt(c.text());
+        break;
+    }
+  }
+
+  return {
+    name: "Inform",
+    parameterList: parameterList,
+    deviceId: deviceId,
+    event: evnt,
+    retryCount: retryCount
+  };
+}
+
+function InformResponse(xml): void {
+  xml
+    .node("cwmp:InformResponse")
+    .node("MaxEnvelopes")
+    .text(1);
+}
+
+function GetRPCMethods(): AcsResponse {
+  return { name: "GetRPCMethods" };
+}
+
+function GetRPCMethodsResponse(xml, methodResponse): void {
+  const el = xml.node("cwmp:GetRPCMethodsResponse").node("MethodList");
+  el.attr({
+    "soap-enc:arrayType": `xsd:string[${methodResponse.methodList.length}]`
+  });
+
+  for (const m of methodResponse.methodList) el.node("string").text(m);
+}
+
+function TransferComplete(xml): TransferCompleteRequest {
+  let commandKey, _faultStruct, startTime, completeTime;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "CommandKey":
+        commandKey = c.text();
+        break;
+      case "FaultStruct":
+        _faultStruct = faultStruct(c);
+        break;
+      case "StartTime":
+        startTime = Date.parse(c.text());
+        break;
+      case "CompleteTime":
+        completeTime = Date.parse(c.text());
+        break;
+    }
+  }
+
+  return {
+    name: "TransferComplete",
+    commandKey: commandKey,
+    faultStruct: _faultStruct,
+    startTime: startTime,
+    completeTime: completeTime
+  };
+}
+
+function TransferCompleteResponse(xml): void {
+  xml.node("cwmp:TransferCompleteResponse").text("");
+}
+
+function RequestDownload(xml): CpeRequest {
+  return {
+    name: "RequestDownload",
+    fileType: xml
+      .childNodes()
+      .find(n => n.name() === "FileType")
+      .text()
+  };
+}
+
+function RequestDownloadResponse(xml): void {
+  xml.node("cwmp:RequestDownloadResponse").text("");
+}
+
+function faultStruct(xml): FaultStruct {
+  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "FaultCode":
+        faultCode = c.text();
+        break;
+      case "FaultString":
+        faultString = c.text();
+        break;
+      case "SetParameterValuesFault":
+        setParameterValuesFault = setParameterValuesFault || [];
+        for (const cc of c.childNodes()) {
+          switch (cc.name()) {
+            case "ParameterName":
+              pn = cc.text();
+              break;
+            case "FaultCode":
+              fc = cc.text();
+              break;
+            case "FaultString":
+              fs = cc.text();
+              break;
+          }
+        }
+        setParameterValuesFault.push({
+          parameterName: pn,
+          faultCode: fc,
+          faultString: fs
+        });
+    }
+  }
+
+  return { faultCode, faultString, setParameterValuesFault };
+}
+
+function fault(xml): CpeFault {
+  let faultCode, faultString, detail;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "faultcode":
+        faultCode = c.text();
+        break;
+      case "faultstring":
+        faultString = c.text();
+        break;
+      case "detail":
+        detail = faultStruct(c.childNodes().find(n => n.name() === "Fault"));
+        break;
+    }
+  }
+
+  return { faultCode, faultString, detail };
+}
+
+export function request(data, cwmpVersion, warn): SoapMessage {
+  warnings = warn;
+
+  const rpc = {
+    id: null,
+    cwmpVersion: cwmpVersion,
+    sessionTimeout: null,
+    cpeRequest: null,
+    cpeFault: null,
+    cpeResponse: null
+  };
+
+  if (!data.length) return rpc;
+
+  const xml = parseXml(data, LIBXMLJS_PARSE_OPTIONS);
+
+  let headerElement, bodyElement;
+
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "Header":
+        headerElement = c;
+        break;
+      case "Body":
+        bodyElement = c;
+        break;
+    }
+  }
+
+  if (headerElement) {
+    for (const c of headerElement.childNodes()) {
+      switch (c.name()) {
+        case "ID":
+          rpc.id = c.text();
+          break;
+        case "sessionTimeout":
+          rpc.sessionTimeout = parseInt(c.text());
+          break;
+      }
+    }
+  }
+
+  for (const methodElement of bodyElement.childNodes()) {
+    const methodName = methodElement.name();
+    if (methodName === "text") continue;
+
+    if (!rpc.cwmpVersion && methodName !== "Fault") {
+      switch (methodElement.namespace().href()) {
+        case "urn:dslforum-org:cwmp-1-0":
+          rpc.cwmpVersion = "1.0";
+          break;
+        case "urn:dslforum-org:cwmp-1-1":
+          rpc.cwmpVersion = "1.1";
+          break;
+        case "urn:dslforum-org:cwmp-1-2":
+          if (rpc.sessionTimeout) rpc.cwmpVersion = "1.3";
+          else rpc.cwmpVersion = "1.2";
+
+          break;
+        case "urn:dslforum-org:cwmp-1-3":
+          rpc.cwmpVersion = "1.4";
+          break;
+        default:
+          throw new Error("Unrecognized CWMP version");
+      }
+    }
+
+    switch (methodName) {
+      case "Inform":
+        rpc.cpeRequest = Inform(methodElement);
+        break;
+      case "GetRPCMethods":
+        rpc.cpeRequest = GetRPCMethods();
+        break;
+      case "TransferComplete":
+        rpc.cpeRequest = TransferComplete(methodElement);
+        break;
+      case "RequestDownload":
+        rpc.cpeRequest = RequestDownload(methodElement);
+        break;
+      case "GetParameterNamesResponse":
+        rpc.cpeResponse = GetParameterNamesResponse(methodElement);
+        break;
+      case "GetParameterValuesResponse":
+        rpc.cpeResponse = GetParameterValuesResponse(methodElement);
+        break;
+      case "SetParameterValuesResponse":
+        rpc.cpeResponse = SetParameterValuesResponse(methodElement);
+        break;
+      case "AddObjectResponse":
+        rpc.cpeResponse = AddObjectResponse(methodElement);
+        break;
+      case "DeleteObjectResponse":
+        rpc.cpeResponse = DeleteObjectResponse(methodElement);
+        break;
+      case "RebootResponse":
+        rpc.cpeResponse = RebootResponse();
+        break;
+      case "FactoryResetResponse":
+        rpc.cpeResponse = FactoryResetResponse();
+        break;
+      case "DownloadResponse":
+        rpc.cpeResponse = DownloadResponse(methodElement);
+        break;
+      case "Fault":
+        rpc.cpeFault = fault(methodElement);
+        break;
+      default:
+        throw new Error(`8000 Method not supported ${methodName}`);
+    }
+  }
+
+  return rpc;
+}
+
+function createSoapEnv(cwmpVersion: string): Element {
+  const xml = new Document();
+  const env = xml.node("soap-env:Envelope");
+
+  for (const prefix of Object.keys(NAMESPACES[cwmpVersion]))
+    env.defineNamespace(prefix, NAMESPACES[cwmpVersion][prefix]);
+
+  return env;
+}
+
+export function response(rpc): { code: number; headers: {}; data: string } {
+  const headers = {
+    Server: SERVER_NAME,
+    SOAPServer: SERVER_NAME
+  };
+
+  if (!rpc) return { code: 204, headers: headers, data: "" };
+
+  const env = createSoapEnv(rpc.cwmpVersion);
+  const header = env.node("soap-env:Header");
+  header
+    .node("cwmp:ID")
+    .attr({
+      "soap-env:mustUnderstand": "1"
+    })
+    .text(rpc.id);
+
+  const body = env.node("soap-env:Body");
+
+  if (rpc.acsResponse) {
+    switch (rpc.acsResponse.name) {
+      case "InformResponse":
+        InformResponse(body);
+        break;
+      case "GetRPCMethodsResponse":
+        GetRPCMethodsResponse(body, rpc.acsResponse);
+        break;
+      case "TransferCompleteResponse":
+        TransferCompleteResponse(body);
+        break;
+      case "RequestDownloadResponse":
+        RequestDownloadResponse(body);
+        break;
+      default:
+        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
+    }
+  } else if (rpc.acsRequest) {
+    switch (rpc.acsRequest.name) {
+      case "GetParameterNames":
+        GetParameterNames(body, rpc.acsRequest);
+        break;
+      case "GetParameterValues":
+        GetParameterValues(body, rpc.acsRequest);
+        break;
+      case "SetParameterValues":
+        SetParameterValues(body, rpc.acsRequest);
+        break;
+      case "AddObject":
+        AddObject(body, rpc.acsRequest);
+        break;
+      case "DeleteObject":
+        DeleteObject(body, rpc.acsRequest);
+        break;
+      case "Reboot":
+        Reboot(body, rpc.acsRequest);
+        break;
+      case "FactoryReset":
+        FactoryReset(body);
+        break;
+      case "Download":
+        Download(body, rpc.acsRequest);
+        break;
+      default:
+        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
+    }
+  }
+
+  headers["Content-Type"] = 'text/xml; charset="utf-8"';
+  return {
+    code: 200,
+    headers: headers,
+    data: env.doc().toString(LIBXMLJS_SAVE_OPTIONS)
+  };
+}

lib/soap.coffee

@@ -1,433 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This file incorporates work covered by the following copyright and
-# permission notice:
-#
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-config = require './config'
-libxmljs = require 'libxmljs'
-
-SERVER_NAME = "GenieACS/#{require('../package.json').version}"
-
-NAMESPACES = {
-  '1.0' : {
-    'soap-enc' : 'http://schemas.xmlsoap.org/soap/encoding/',
-    'soap-env' : 'http://schemas.xmlsoap.org/soap/envelope/',
-    'xsd' : 'http://www.w3.org/2001/XMLSchema',
-    'xsi' : 'http://www.w3.org/2001/XMLSchema-instance',
-    'cwmp' : 'urn:dslforum-org:cwmp-1-0'
-  },
-  '1.1' : {
-    'soap-enc' : 'http://schemas.xmlsoap.org/soap/encoding/',
-    'soap-env' : 'http://schemas.xmlsoap.org/soap/envelope/',
-    'xsd' : 'http://www.w3.org/2001/XMLSchema',
-    'xsi' : 'http://www.w3.org/2001/XMLSchema-instance',
-    'cwmp' : 'urn:dslforum-org:cwmp-1-1'
-  },
-  '1.2' : {
-    'soap-enc' : 'http://schemas.xmlsoap.org/soap/encoding/',
-    'soap-env' : 'http://schemas.xmlsoap.org/soap/envelope/',
-    'xsd' : 'http://www.w3.org/2001/XMLSchema',
-    'xsi' : 'http://www.w3.org/2001/XMLSchema-instance',
-    'cwmp' : 'urn:dslforum-org:cwmp-1-2'
-  },
-  '1.3' : {
-    'soap-enc' : 'http://schemas.xmlsoap.org/soap/encoding/',
-    'soap-env' : 'http://schemas.xmlsoap.org/soap/envelope/',
-    'xsd' : 'http://www.w3.org/2001/XMLSchema',
-    'xsi' : 'http://www.w3.org/2001/XMLSchema-instance',
-    'cwmp' : 'urn:dslforum-org:cwmp-1-2'
-  },
-  '1.4' : {
-    'soap-enc' : 'http://schemas.xmlsoap.org/soap/encoding/',
-    'soap-env' : 'http://schemas.xmlsoap.org/soap/envelope/',
-    'xsd' : 'http://www.w3.org/2001/XMLSchema',
-    'xsi' : 'http://www.w3.org/2001/XMLSchema-instance',
-    'cwmp' : 'urn:dslforum-org:cwmp-1-3'
-  }
-}
-
-# Generate Libxmljs options
-LIBXMLJS_OPTIONS = {}
-for k, v of config.allConfig
-  if k.lastIndexOf('XML_PARSE_', 0) == 0
-    LIBXMLJS_OPTIONS[k] = v
-
-
-# Workaround for devices that don't use correct namespace declarations or prefixes
-if config.get('IGNORE_XML_NAMESPACES')
-  libxmljs.Element.prototype.__find = libxmljs.Element.prototype.find
-  libxmljs.Element.prototype.find = (xpath, namespaces) ->
-    # Modify xpath queries to work regardless of element's namespace
-    p = xpath.replace(/([^\/:]+:)?([^\/]+)(\/|$)/g, '$2$3') \
-      .replace(/([a-zA-Z0-9_-]+)([^\/]*)(\/|$)/g, "*[local-name()='$1']$2$3")
-    libxmljs.Element.prototype.__find.call(this, p, namespaces)
-
-
-event = (xml) ->
-  e.text().trim() for e in xml.find('EventStruct/EventCode')
-
-
-parameterInfoList = (xml) ->
-  infoList = []
-  for e in xml.find('ParameterInfoStruct')
-    name = e.get('Name').text().trim()
-    writable = Boolean(JSON.parse(e.get('Writable').text()))
-    infoList.push([name, writable])
-  return infoList
-
-
-parameterValueList = (xml) ->
-  valueList = []
-  for e in xml.find('ParameterValueStruct')
-    valueType = e.get('Value').attr('type').value().trim()
-    name = e.get('Name').text().trim()
-    value = e.get('Value').text().trim()
-    try
-      switch valueType
-        when 'xsd:boolean'
-          value = Boolean(JSON.parse(value))
-        when 'xsd:unsignedInt'
-          value = JSON.parse(value)
-        when 'xsd:int'
-          value = JSON.parse(value)
-    catch err
-      value = null
-    valueList.push([name, value, valueType])
-  return valueList
-
-
-cpeGetParameterNames = (xml, methodRequest) ->
-  el = xml.node('cwmp:GetParameterNames')
-  el.node('ParameterPath').text(methodRequest.parameterPath)
-  el.node('NextLevel').text(+methodRequest.nextLevel)
-
-
-cpeGetParameterNamesResponse = (xml) ->
-  {parameterList : parameterInfoList(xml.get('ParameterList'))}
-
-
-cpeGetParameterValues = (xml, methodRequest) ->
-  el = xml.node('cwmp:GetParameterValues').node('ParameterNames')
-  el.attr({'soap-enc:arrayType' : "xsd:string[#{methodRequest.parameterNames.length}]"})
-  for p in methodRequest.parameterNames
-    el.node('string').text(p)
-
-
-cpeGetParameterValuesResponse = (xml) ->
-  {parameterList : parameterValueList(xml.get('ParameterList'))}
-
-
-cpeSetParameterValues = (xml, methodRequest) ->
-  el = xml.node('cwmp:SetParameterValues')
-  paramList = el.node('ParameterList')
-  paramList.attr({'soap-enc:arrayType' : "cwmp:ParameterValueStruct[#{methodRequest.parameterList.length}]"})
-  for i in methodRequest.parameterList
-    pvs = paramList.node('ParameterValueStruct')
-    pvs.node('Name').text(i[0])
-    v = pvs.node('Value')
-    v.text(i[1])
-    v.attr({'xsi:type' : i[2]}) if i[2]?
-  el.node('ParameterKey').text(methodRequest.parameterKey ? '')
-
-
-cpeSetParameterValuesResponse = (xml) ->
-  {status : JSON.parse(xml.get('Status').text())}
-
-
-cpeAddObject = (xml, methodRequest) ->
-  el = xml.node('cwmp:AddObject')
-  el.node('ObjectName').text(methodRequest.objectName)
-  el.node('ParameterKey').text(methodRequest.parameterKey ? '')
-
-
-cpeAddObjectResponse = (xml) ->
-  {
-    instanceNumber : parseInt(xml.get('InstanceNumber').text()),
-    status : parseInt(xml.get('Status').text())
-  }
-
-
-cpeDeleteObject = (xml, methodRequest) ->
-  el = xml.node('cwmp:DeleteObject')
-  el.node('ObjectName').text(methodRequest.objectName)
-  el.node('ParameterKey').text(methodRequest.parameterKey ? '')
-
-
-cpeDeleteObjectResponse = (xml) ->
-  {status : parseInt(xml.get('Status').text())}
-
-
-cpeReboot = (xml, methodRequest) ->
-  el = xml.node('cwmp:Reboot')
-  el.node('CommandKey').text(methodRequest.commandKey ? '')
-
-
-cpeRebootResponse = (xml) ->
-  {}
-
-
-cpeFactoryReset = (xml, methodRequest) ->
-  el = xml.node('cwmp:FactoryReset')
-
-
-cpeFactoryResetResponse = (xml, methodRequest) ->
-  {}
-
-
-cpeDownload = (xml, methodRequest) ->
-  el = xml.node('cwmp:Download')
-  el.node('CommandKey').text(methodRequest.commandKey ? '')
-  el.node('FileType').text(methodRequest.fileType)
-  el.node('URL').text(methodRequest.url)
-  el.node('Username').text(methodRequest.username ? '')
-  el.node('Password').text(methodRequest.password ? '')
-  el.node('FileSize').text(methodRequest.fileSize ? '0')
-  el.node('TargetFileName').text(methodRequest.TargetFileName ? '')
-  el.node('DelaySeconds').text(methodRequest.delaySeconds ? '0')
-  el.node('SuccessURL').text(methodRequest.successUrl ? '')
-  el.node('FailureURL').text(methodRequest.failureUrl ? '')
-
-
-cpeDownloadResponse = (xml) ->
-  {status : JSON.parse(xml.get('Status').text())}
-
-
-acsInform = (xml) ->
-  {
-    parameterList : parameterValueList(xml.get('ParameterList')),
-    deviceId : traverseXml(xml.get('DeviceId')),
-    event : event(xml.get('Event')),
-    retryCount : parseInt(xml.get('RetryCount').text())
-  }
-
-
-acsInformResponse = (xml, methodResponse) ->
-  xml.node('cwmp:InformResponse').node('MaxEnvelopes').text(1)
-
-
-acsGetRPCMethods = (xml) ->
-  {}
-
-
-acsGetRPCMethodsResponse = (xml, methodResponse) ->
-  el = xml.node('cwmp:GetRPCMethodsResponse').node('MethodList')
-  el.attr({'soap-enc:arrayType' : "xsd:string[#{methodResponse.methodList.length}]"})
-  for m in methodResponse.methodList
-    el.node('string').text(m)
-
-
-acsTransferComplete = (xml) ->
-  {
-    commandKey : xml.get('CommandKey').text(),
-    faultStruct : traverseXml(xml.get('FaultStruct')),
-    startTime : xml.get('StartTime').text(), # TODO convert to datetime
-    completeTime : xml.get('CompleteTime').text() # TODO convert to datetime
-  }
-
-
-acsTransferCompleteResponse = (xml, methodResponse) ->
-  xml.node('cwmp:TransferCompleteResponse').text('')
-
-
-acsRequestDownload = (xml) ->
-  # TODO FileTypeArg
-  {
-    fileType : xml.get('FileType').text()
-  }
-
-
-acsRequestDownloadResponse = (xml, methodResponse) ->
-  xml.node('cwmp:RequestDownloadResponse').text('')
-
-
-traverseXml = (xml) ->
-  obj = {}
-  for n in xml.childNodes()
-    if n.type() == 'element'
-      obj[n.name()] = traverseXml(n)
-
-  if Object.keys(obj).length == 0
-    xml.text()
-  else
-    obj
-
-
-fault = (xml) ->
-  return traverseXml(xml)
-
-
-exports.request = (httpRequest, cwmpVersion) ->
-  cwmpRequest = {cwmpVersion : cwmpVersion}
-
-  data = httpRequest.getBody()
-
-  if data.length > 0
-    xml = libxmljs.parseXml(data, LIBXMLJS_OPTIONS)
-
-    if not cwmpRequest.cwmpVersion?
-      # cwmpVersion not passed, thus it's an inform request
-      methodElement = xml.get('/soap-env:Envelope/soap-env:Body/*', NAMESPACES['1.0'])
-      switch methodElement.namespace().href()
-        when 'urn:dslforum-org:cwmp-1-0'
-          cwmpRequest.cwmpVersion = '1.0'
-        when 'urn:dslforum-org:cwmp-1-1'
-          cwmpRequest.cwmpVersion = '1.1'
-        when 'urn:dslforum-org:cwmp-1-2'
-          cwmpRequest.sessionTimeout = try xml.get('/soap-env:Envelope/soap-env:Header/cwmp:sessionTimeout', NAMESPACES['1.2']).text() catch then null
-          if cwmpRequest.sessionTimeout?
-            cwmpRequest.cwmpVersion = '1.3'
-          else
-            cwmpRequest.cwmpVersion = '1.2'
-        when 'urn:dslforum-org:cwmp-1-3'
-          cwmpRequest.cwmpVersion = '1.4'
-        else
-          throw new Error('Unrecognized CWMP version')
-
-      cwmpRequest.sessionTimeout ?= try xml.get('/soap-env:Envelope/soap-env:Header/cwmp:sessionTimeout', NAMESPACES[cwmpRequest.cwmpVersion]).text() catch then null
-    else
-      methodElement = xml.get('/soap-env:Envelope/soap-env:Body/cwmp:*', NAMESPACES[cwmpRequest.cwmpVersion])
-
-    cwmpRequest.id = try xml.get('/soap-env:Envelope/soap-env:Header/cwmp:ID', NAMESPACES[cwmpRequest.cwmpVersion]).text() catch then null
-
-    if methodElement? and not (config.get('IGNORE_XML_NAMESPACES') and methodElement.name() is 'Fault')
-      switch methodElement.name()
-        when 'Inform'
-          cwmpRequest.methodRequest = acsInform(methodElement)
-          cwmpRequest.methodRequest.type = 'Inform'
-        when 'GetRPCMethods'
-          cwmpRequest.methodRequest = acsGetRPCMethods(methodElement)
-          cwmpRequest.methodRequest.type = 'GetRPCMethods'
-        when 'TransferComplete'
-          cwmpRequest.methodRequest = acsTransferComplete(methodElement)
-          cwmpRequest.methodRequest.type = 'TransferComplete'
-        when 'RequestDownload'
-          cwmpRequest.methodRequest = acsRequestDownload(methodElement)
-          cwmpRequest.methodRequest.type = 'RequestDownload'
-        when 'GetParameterNamesResponse'
-          cwmpRequest.methodResponse = cpeGetParameterNamesResponse(methodElement)
-          cwmpRequest.methodResponse.type = 'GetParameterNamesResponse'
-        when 'GetParameterValuesResponse'
-          cwmpRequest.methodResponse = cpeGetParameterValuesResponse(methodElement)
-          cwmpRequest.methodResponse.type = 'GetParameterValuesResponse'
-        when 'SetParameterValuesResponse'
-          cwmpRequest.methodResponse = cpeSetParameterValuesResponse(methodElement)
-          cwmpRequest.methodResponse.type = 'SetParameterValuesResponse'
-        when 'AddObjectResponse'
-          cwmpRequest.methodResponse = cpeAddObjectResponse(methodElement)
-          cwmpRequest.methodResponse.type = 'AddObjectResponse'
-        when 'DeleteObjectResponse'
-          cwmpRequest.methodResponse = cpeDeleteObjectResponse(methodElement)
-          cwmpRequest.methodResponse.type = 'DeleteObjectResponse'
-        when 'RebootResponse'
-          cwmpRequest.methodResponse = cpeRebootResponse(methodElement)
-          cwmpRequest.methodResponse.type = 'RebootResponse'
-        when 'FactoryResetResponse'
-          cwmpRequest.methodResponse = cpeFactoryResetResponse(methodElement)
-          cwmpRequest.methodResponse.type = 'FactoryResetResponse'
-        when 'DownloadResponse'
-          cwmpRequest.methodResponse = cpeDownloadResponse(methodElement)
-          cwmpRequest.methodResponse.type = 'DownloadResponse'
-        else
-          throw Error('8000 Method not supported ' + methodElement.name())
-    else
-      faultElement = xml.get('/soap-env:Envelope/soap-env:Body/soap-env:Fault', NAMESPACES[cwmpRequest.cwmpVersion])
-      cwmpRequest.fault = fault(faultElement)
-    
-  return cwmpRequest
-
-
-createSoapEnv = (cwmpVersion) ->
-  xml = libxmljs.Document()
-  env = xml.node('soap-env:Envelope')
-  for prefix, url of NAMESPACES[cwmpVersion]
-    env.defineNamespace prefix, url
-  return env
-
-
-exports.response = (cwmpResponse) ->
-  headers = {
-    'Server' : SERVER_NAME,
-    'SOAPServer' : SERVER_NAME,
-  }
-
-  if cwmpResponse?.methodResponse?
-    env = createSoapEnv(cwmpResponse.cwmpVersion)
-    header = env.node('soap-env:Header')
-    header.node('cwmp:ID').attr({'soap-env:mustUnderstand' : 1}).text(cwmpResponse.id)
-    body = env.node('soap-env:Body')
-    switch cwmpResponse.methodResponse.type
-      when 'InformResponse'
-        acsInformResponse(body, cwmpResponse.methodResponse)
-      when 'GetRPCMethodsResponse'
-        acsGetRPCMethodsResponse(body, cwmpResponse.methodResponse)
-      when 'TransferCompleteResponse'
-        acsTransferCompleteResponse(body, cwmpResponse.methodResponse)
-      when 'RequestDownloadResponse'
-        acsRequestDownloadResponse(body, cwmpResponse.methodResponse)
-      else
-        throw Error("Unknown method response type #{cwmpResponse.methodResponse.type}")
-  else if cwmpResponse?.methodRequest?
-    env = createSoapEnv(cwmpResponse.cwmpVersion)
-    header = env.node('soap-env:Header')
-    header.node('cwmp:ID').attr({'soap-env:mustUnderstand' : 1}).text(cwmpResponse.id)
-    body = env.node('soap-env:Body')
-    switch cwmpResponse.methodRequest.type
-      when 'GetParameterNames'
-        cpeGetParameterNames(body, cwmpResponse.methodRequest)
-      when 'GetParameterValues'
-        cpeGetParameterValues(body, cwmpResponse.methodRequest)
-      when 'SetParameterValues'
-        cpeSetParameterValues(body, cwmpResponse.methodRequest)
-      when 'AddObject'
-        cpeAddObject(body, cwmpResponse.methodRequest)
-      when 'DeleteObject'
-        cpeDeleteObject(body, cwmpResponse.methodRequest)
-      when 'Reboot'
-        cpeReboot(body, cwmpResponse.methodRequest)
-      when 'FactoryReset'
-        cpeFactoryReset(body, cwmpResponse.methodRequest)
-      when 'Download'
-        cpeDownload(body, cwmpResponse.methodRequest)
-      else
-        throw Error("Unknown method request #{cwmpResponse.methodRequest.type}")
-  
-  if env?
-    headers['Content-Type'] = 'text/xml; charset="utf-8"'
-    return {code: 200, headers: headers, data: new Buffer(env.doc().toString(false))}
-  else
-    return {code: 204, headers: headers, data: new Buffer(0)}

lib/soap.ts

@@ -0,0 +1,726 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import {
+  parseXml,
+  Element,
+  parseAttrs,
+  encodeEntities,
+  decodeEntities
+} from "./xml-parser";
+import memoize from "./common/memoize";
+import { version as VERSION } from "../package.json";
+import {
+  CpeGetResponse,
+  CpeSetResponse,
+  InformRequest,
+  AcsResponse,
+  CpeRequest,
+  FaultStruct,
+  SpvFault,
+  CpeFault,
+  SoapMessage,
+  TransferCompleteRequest
+} from "./types";
+
+const SERVER_NAME = `GenieACS/${VERSION}`;
+
+const NAMESPACES = {
+  "1.0": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-0"
+  },
+  "1.1": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-1"
+  },
+  "1.2": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-2"
+  },
+  "1.3": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-2"
+  },
+  "1.4": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-3"
+  }
+};
+
+let warnings;
+
+const memoizedParseAttrs = memoize(parseAttrs);
+
+function parseBool(v): boolean {
+  v = "" + v;
+  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
+  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
+    return false;
+  else return null;
+}
+
+function event(xml: Element): string[] {
+  return xml.children
+    .filter(n => n.localName === "EventStruct")
+    .map(c => c.children.find(n => n.localName === "EventCode").text.trim());
+}
+
+function parameterInfoList(xml: Element): [string, boolean][] {
+  return xml.children
+    .filter(e => e.localName === "ParameterInfoStruct")
+    .map<[string, boolean]>(e => {
+      let param: string, value: string;
+      for (const c of e.children) {
+        switch (c.localName) {
+          case "Name":
+            param = c.text;
+            break;
+          case "Writable":
+            value = c.text;
+            break;
+        }
+      }
+
+      let parsed: boolean = parseBool(value);
+
+      if (parsed === null) {
+        warnings.push({
+          message: "Invalid writable attribute",
+          parameter: param
+        });
+        parsed = false;
+      }
+
+      return [param, parsed];
+    });
+}
+
+const getValueType = memoize((str: string) =>
+  parseAttrs(str)
+    .find(s => s.localName === "type")
+    .value.trim()
+);
+
+function parameterValueList(
+  xml: Element
+): [string, string | number | boolean, string][] {
+  return xml.children
+    .filter(e => e.localName === "ParameterValueStruct")
+    .map<[string, string | number | boolean, string]>(e => {
+      let valueElement: Element, param: string;
+      for (const c of e.children) {
+        switch (c.localName) {
+          case "Name":
+            param = c.text;
+            break;
+          case "Value":
+            valueElement = c;
+            break;
+        }
+      }
+
+      const valueType = getValueType(valueElement.attrs);
+
+      const value = decodeEntities(valueElement.text);
+      let parsed: string | number | boolean = value;
+      if (valueType === "xsd:boolean") {
+        parsed = parseBool(value);
+        if (parsed === null) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      } else if (valueType === "xsd:int" || valueType === "xsd:unsignedInt") {
+        parsed = parseInt(value);
+        if (isNaN(parsed)) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      } else if (valueType === "xsd:dateTime") {
+        parsed = Date.parse(value);
+        if (isNaN(parsed)) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      }
+
+      return [param, parsed, valueType];
+    });
+}
+
+function GetParameterNames(methodRequest): string {
+  return `<cwmp:GetParameterNames><ParameterPath>${
+    methodRequest.parameterPath
+  }</ParameterPath><NextLevel>${+methodRequest.nextLevel}</NextLevel></cwmp:GetParameterNames>`;
+}
+
+function GetParameterNamesResponse(xml): CpeGetResponse {
+  return {
+    name: "GetParameterNamesResponse",
+    parameterList: parameterInfoList(
+      xml.children.find(n => n.localName === "ParameterList")
+    )
+  };
+}
+
+function GetParameterValues(methodRequest): string {
+  return `<cwmp:GetParameterValues><ParameterNames soap-enc:arrayType="xsd:string[${
+    methodRequest.parameterNames.length
+  }]">${methodRequest.parameterNames
+    .map(p => `<string>${p}</string>`)
+    .join("")}</ParameterNames></cwmp:GetParameterValues>`;
+}
+
+function GetParameterValuesResponse(xml: Element): CpeGetResponse {
+  return {
+    name: "GetParameterValuesResponse",
+    parameterList: parameterValueList(
+      xml.children.find(n => n.localName === "ParameterList")
+    )
+  };
+}
+
+function SetParameterValues(methodRequest): string {
+  const params = methodRequest.parameterList.map(p => {
+    let val = p[1];
+    if (p[2] === "xsd:dateTime" && typeof val === "number") {
+      val = new Date(val).toISOString();
+      if (methodRequest.DATETIME_MILLISECONDS === false)
+        val = val.replace(".000", "");
+    }
+    if (p[2] === "xsd:boolean" && typeof val === "boolean")
+      if (methodRequest.BOOLEAN_LITERAL === false) val = +val;
+    return `<ParameterValueStruct><Name>${p[0]}</Name><Value xsi:type="${
+      p[2]
+    }">${encodeEntities("" + val)}</Value></ParameterValueStruct>`;
+  });
+
+  return `<cwmp:SetParameterValues><ParameterList soap-enc:arrayType="cwmp:ParameterValueStruct[${
+    methodRequest.parameterList.length
+  }]">${params.join(
+    ""
+  )}</ParameterList><ParameterKey>${methodRequest.parameterKey ||
+    ""}</ParameterKey></cwmp:SetParameterValues>`;
+}
+
+function SetParameterValuesResponse(xml: Element): CpeSetResponse {
+  return {
+    name: "SetParameterValuesResponse",
+    status: parseInt(xml.children.find(n => n.localName === "Status").text)
+  };
+}
+
+function AddObject(methodRequest): string {
+  return `<cwmp:AddObject><ObjectName>${
+    methodRequest.objectName
+  }</ObjectName><ParameterKey>${methodRequest.parameterKey ||
+    ""}</ParameterKey></cwmp:AddObject>`;
+}
+
+function AddObjectResponse(xml: Element): CpeSetResponse {
+  let instanceNumber, status;
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "InstanceNumber":
+        instanceNumber = parseInt(c.text);
+        break;
+      case "Status":
+        status = parseInt(c.text);
+        break;
+    }
+  }
+
+  return {
+    name: "AddObjectResponse",
+    instanceNumber: instanceNumber,
+    status: status
+  };
+}
+
+function DeleteObject(methodRequest): string {
+  return `<cwmp:DeleteObject><ObjectName>${
+    methodRequest.objectName
+  }</ObjectName><ParameterKey>${methodRequest.parameterKey ||
+    ""}</ParameterKey></cwmp:DeleteObject>`;
+}
+
+function DeleteObjectResponse(xml: Element): CpeSetResponse {
+  return {
+    name: "DeleteObjectResponse",
+    status: parseInt(xml.children.find(n => n.localName === "Status").text)
+  };
+}
+
+function Reboot(methodRequest): string {
+  return `<cwmp:Reboot><CommandKey>${methodRequest.commandKey ||
+    ""}</CommandKey></cwmp:Reboot>`;
+}
+
+function RebootResponse(): CpeSetResponse {
+  return {
+    name: "RebootResponse"
+  };
+}
+
+function FactoryReset(): string {
+  return "<cwmp:FactoryReset></cwmp:FactoryReset>";
+}
+
+function FactoryResetResponse(): CpeSetResponse {
+  return {
+    name: "FactoryResetResponse"
+  };
+}
+
+function Download(methodRequest): string {
+  return `<cwmp:Download><CommandKey>${methodRequest.commandKey ||
+    ""}</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
+    methodRequest.url
+  }</URL><Username>${encodeEntities(
+    methodRequest.username || ""
+  )}</Username><Password>${encodeEntities(
+    methodRequest.password || ""
+  )}</Password><FileSize>${methodRequest.fileSize ||
+    "0"}</FileSize><TargetFileName>${encodeEntities(
+    methodRequest.targetFileName || ""
+  )}</TargetFileName><DelaySeconds>${methodRequest.delaySeconds ||
+    "0"}</DelaySeconds><SuccessURL>${encodeEntities(
+    methodRequest.successUrl || ""
+  )}</SuccessURL><FailureURL>${encodeEntities(
+    methodRequest.failureUrl || ""
+  )}</FailureURL></cwmp:Download>`;
+}
+
+function DownloadResponse(xml: Element): CpeSetResponse {
+  let status, startTime, completeTime;
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "Status":
+        status = parseInt(c.text);
+        break;
+      case "StartTime":
+        startTime = Date.parse(c.text);
+        break;
+      case "CompleteTime":
+        completeTime = Date.parse(c.text);
+        break;
+    }
+  }
+
+  return {
+    name: "DownloadResponse",
+    status: status,
+    startTime: startTime,
+    completeTime: completeTime
+  };
+}
+
+function Inform(xml: Element): InformRequest {
+  let retryCount, evnt, parameterList;
+  const deviceId = {
+    Manufacturer: null,
+    OUI: null,
+    ProductClass: null,
+    SerialNumber: null
+  };
+
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "ParameterList":
+        parameterList = parameterValueList(c);
+        break;
+      case "DeviceId":
+        for (const cc of c.children) {
+          const n = cc.localName;
+          if (n in deviceId) deviceId[n] = decodeEntities(cc.text);
+        }
+        break;
+      case "Event":
+        evnt = event(c);
+        break;
+      case "RetryCount":
+        retryCount = parseInt(c.text);
+        break;
+    }
+  }
+
+  return {
+    name: "Inform",
+    parameterList: parameterList,
+    deviceId: deviceId,
+    event: evnt,
+    retryCount: retryCount
+  };
+}
+
+function InformResponse(): string {
+  return "<cwmp:InformResponse><MaxEnvelopes>1</MaxEnvelopes></cwmp:InformResponse>";
+}
+
+function GetRPCMethods(): AcsResponse {
+  return { name: "GetRPCMethods" };
+}
+
+function GetRPCMethodsResponse(methodResponse): string {
+  return `<cwmp:GetRPCMethodsResponse><MethodList soap-enc:arrayType="xsd:string[${
+    methodResponse.methodList.length
+  }]">${methodResponse.methodList
+    .map(m => `<string>${m}</string>`)
+    .join("")}</MethodList></cwmp:GetRPCMethodsResponse>`;
+}
+
+function TransferComplete(xml: Element): TransferCompleteRequest {
+  let commandKey, _faultStruct, startTime, completeTime;
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "CommandKey":
+        commandKey = c.text;
+        break;
+      case "FaultStruct":
+        _faultStruct = faultStruct(c);
+        break;
+      case "StartTime":
+        startTime = Date.parse(c.text);
+        break;
+      case "CompleteTime":
+        completeTime = Date.parse(c.text);
+        break;
+    }
+  }
+
+  return {
+    name: "TransferComplete",
+    commandKey: commandKey,
+    faultStruct: _faultStruct,
+    startTime: startTime,
+    completeTime: completeTime
+  };
+}
+
+function TransferCompleteResponse(): string {
+  return "<cwmp:TransferCompleteResponse></cwmp:TransferCompleteResponse>";
+}
+
+function RequestDownload(xml: Element): CpeRequest {
+  return {
+    name: "RequestDownload",
+    fileType: xml.children.find(n => n.localName === "FileType").text
+  };
+}
+
+function RequestDownloadResponse(): string {
+  return "<cwmp:RequestDownloadResponse></cwmp:RequestDownloadResponse>";
+}
+
+function faultStruct(xml: Element): FaultStruct {
+  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "FaultCode":
+        faultCode = c.text;
+        break;
+      case "FaultString":
+        faultString = decodeEntities(c.text);
+        break;
+      case "SetParameterValuesFault":
+        setParameterValuesFault = setParameterValuesFault || [];
+        for (const cc of c.children) {
+          switch (cc.localName) {
+            case "ParameterName":
+              pn = cc.text;
+              break;
+            case "FaultCode":
+              fc = cc.text;
+              break;
+            case "FaultString":
+              fs = decodeEntities(cc.text);
+              break;
+          }
+        }
+        setParameterValuesFault.push({
+          parameterName: pn,
+          faultCode: fc,
+          faultString: fs
+        });
+    }
+  }
+
+  return { faultCode, faultString, setParameterValuesFault };
+}
+
+function fault(xml: Element): CpeFault {
+  let faultCode, faultString, detail;
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "faultcode":
+        faultCode = c.text;
+        break;
+      case "faultstring":
+        faultString = decodeEntities(c.text);
+        break;
+      case "detail":
+        detail = faultStruct(c.children.find(n => n.localName === "Fault"));
+        break;
+    }
+  }
+
+  return { faultCode, faultString, detail };
+}
+
+export function request(body: string, cwmpVersion, warn): SoapMessage {
+  warnings = warn;
+
+  const rpc = {
+    id: null,
+    cwmpVersion: cwmpVersion,
+    sessionTimeout: null,
+    cpeRequest: null,
+    cpeFault: null,
+    cpeResponse: null
+  };
+
+  if (!body.length) return rpc;
+
+  const xml = parseXml(body);
+
+  if (!xml.children.length) return rpc;
+
+  const envelope = xml.children[0];
+
+  let headerElement: Element, bodyElement: Element;
+
+  for (const c of envelope.children) {
+    switch (c.localName) {
+      case "Header":
+        headerElement = c;
+        break;
+      case "Body":
+        bodyElement = c;
+        break;
+    }
+  }
+
+  if (headerElement) {
+    for (const c of headerElement.children) {
+      switch (c.localName) {
+        case "ID":
+          rpc.id = decodeEntities(c.text);
+          break;
+        case "sessionTimeout":
+          rpc.sessionTimeout = parseInt(c.text);
+          break;
+      }
+    }
+  }
+
+  const methodElement = bodyElement.children[0];
+
+  if (!rpc.cwmpVersion && methodElement.localName !== "Fault") {
+    let namespace, namespaceHref;
+    for (const e of [methodElement, bodyElement, envelope]) {
+      namespace = namespace || e.namespace;
+      if (e.attrs) {
+        const attrs = memoizedParseAttrs(e.attrs);
+        const attr = namespace
+          ? attrs.find(
+              s => s.namespace === "xmlns" && s.localName === namespace
+            )
+          : attrs.find(s => s.name === "xmlns");
+
+        if (attr) namespaceHref = attr.value;
+      }
+    }
+
+    switch (namespaceHref) {
+      case "urn:dslforum-org:cwmp-1-0":
+        rpc.cwmpVersion = "1.0";
+        break;
+      case "urn:dslforum-org:cwmp-1-1":
+        rpc.cwmpVersion = "1.1";
+        break;
+      case "urn:dslforum-org:cwmp-1-2":
+        if (rpc.sessionTimeout) rpc.cwmpVersion = "1.3";
+        else rpc.cwmpVersion = "1.2";
+
+        break;
+      case "urn:dslforum-org:cwmp-1-3":
+        rpc.cwmpVersion = "1.4";
+        break;
+      default:
+        throw new Error("Unrecognized CWMP version");
+    }
+  }
+
+  switch (methodElement.localName) {
+    case "Inform":
+      rpc.cpeRequest = Inform(methodElement);
+      break;
+    case "GetRPCMethods":
+      rpc.cpeRequest = GetRPCMethods();
+      break;
+    case "TransferComplete":
+      rpc.cpeRequest = TransferComplete(methodElement);
+      break;
+    case "RequestDownload":
+      rpc.cpeRequest = RequestDownload(methodElement);
+      break;
+    case "GetParameterNamesResponse":
+      rpc.cpeResponse = GetParameterNamesResponse(methodElement);
+      break;
+    case "GetParameterValuesResponse":
+      rpc.cpeResponse = GetParameterValuesResponse(methodElement);
+      break;
+    case "SetParameterValuesResponse":
+      rpc.cpeResponse = SetParameterValuesResponse(methodElement);
+      break;
+    case "AddObjectResponse":
+      rpc.cpeResponse = AddObjectResponse(methodElement);
+      break;
+    case "DeleteObjectResponse":
+      rpc.cpeResponse = DeleteObjectResponse(methodElement);
+      break;
+    case "RebootResponse":
+      rpc.cpeResponse = RebootResponse();
+      break;
+    case "FactoryResetResponse":
+      rpc.cpeResponse = FactoryResetResponse();
+      break;
+    case "DownloadResponse":
+      rpc.cpeResponse = DownloadResponse(methodElement);
+      break;
+    case "Fault":
+      rpc.cpeFault = fault(methodElement);
+      break;
+    default:
+      throw new Error(`8000 Method not supported ${methodElement.localName}`);
+  }
+
+  return rpc;
+}
+
+const namespacesAttrs = {
+  "1.0": Object.entries(NAMESPACES["1.0"])
+    .map(([k, v]) => `xmlns:${k}="${v}"`)
+    .join(" "),
+  "1.1": Object.entries(NAMESPACES["1.1"])
+    .map(([k, v]) => `xmlns:${k}="${v}"`)
+    .join(" "),
+  "1.2": Object.entries(NAMESPACES["1.2"])
+    .map(([k, v]) => `xmlns:${k}="${v}"`)
+    .join(" "),
+  "1.3": Object.entries(NAMESPACES["1.3"])
+    .map(([k, v]) => `xmlns:${k}="${v}"`)
+    .join(" "),
+  "1.4": Object.entries(NAMESPACES["1.4"])
+    .map(([k, v]) => `xmlns:${k}="${v}"`)
+    .join(" ")
+};
+
+export function response(rpc): { code: number; headers: {}; data: string } {
+  const headers = {
+    Server: SERVER_NAME,
+    SOAPServer: SERVER_NAME
+  };
+
+  if (!rpc) return { code: 204, headers: headers, data: "" };
+
+  let body;
+  if (rpc.acsResponse) {
+    switch (rpc.acsResponse.name) {
+      case "InformResponse":
+        body = InformResponse();
+        break;
+      case "GetRPCMethodsResponse":
+        body = GetRPCMethodsResponse(rpc.acsResponse);
+        break;
+      case "TransferCompleteResponse":
+        body = TransferCompleteResponse();
+        break;
+      case "RequestDownloadResponse":
+        body = RequestDownloadResponse();
+        break;
+      default:
+        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
+    }
+  } else if (rpc.acsRequest) {
+    switch (rpc.acsRequest.name) {
+      case "GetParameterNames":
+        body = GetParameterNames(rpc.acsRequest);
+        break;
+      case "GetParameterValues":
+        body = GetParameterValues(rpc.acsRequest);
+        break;
+      case "SetParameterValues":
+        body = SetParameterValues(rpc.acsRequest);
+        break;
+      case "AddObject":
+        body = AddObject(rpc.acsRequest);
+        break;
+      case "DeleteObject":
+        body = DeleteObject(rpc.acsRequest);
+        break;
+      case "Reboot":
+        body = Reboot(rpc.acsRequest);
+        break;
+      case "FactoryReset":
+        body = FactoryReset();
+        break;
+      case "Download":
+        body = Download(rpc.acsRequest);
+        break;
+      default:
+        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
+    }
+  }
+
+  headers["Content-Type"] = 'text/xml; charset="utf-8"';
+  return {
+    code: 200,
+    headers: headers,
+    data: `<?xml version="1.0" encoding="UTF-8"?>\n<soap-env:Envelope ${
+      namespacesAttrs[rpc.cwmpVersion]
+    }><soap-env:Header><cwmp:ID soap-env:mustUnderstand="1">${
+      rpc.id
+    }</cwmp:ID></soap-env:Header><soap-env:Body>${body}</soap-env:Body></soap-env:Envelope>`
+  };
+}

lib/tasks.coffee

@@ -1,360 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This file incorporates work covered by the following copyright and
-# permission notice:
-#
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-config = require './config'
-common = require './common'
-db = require './db'
-mongodb = require 'mongodb'
-customCommands = require './custom-commands'
-
-exports.STATUS_OK = STATUS_OK = 1
-exports.STATUS_FAULT = STATUS_FAULT = 2
-exports.STATUS_COMPLETED = STATUS_COMPLETED = 4
-exports.STATUS_SAVE = STATUS_SAVE = 128 # not mutually exclusive with the rest
-
-
-this.refreshObject = (task, methodResponse, callback) ->
-  task.session = {} if not task.session?
-
-  if not task.session.subtask?
-    path = task.objectName
-    path += '.' if path != '' and not common.endsWith(path, '.')
-    task.session.subtask = {device : task.device, name : 'getParameterNames', parameterPath : path}
-
-  if methodResponse.faultcode?
-    task.fault = methodResponse
-    callback(null, STATUS_FAULT)
-    return
-
-  if task.session.subtask.name is 'getParameterNames'
-    this.getParameterNames(task.session.subtask, methodResponse, (err, status, methodRequest, deviceUpdates) =>
-      task.session.parameterNames ?= []
-      if deviceUpdates?.parameterNames?
-        for p in deviceUpdates.parameterNames
-          if not common.endsWith(p[0], '.')
-            task.session.parameterNames.push(p[0])
-
-      if status & STATUS_COMPLETED
-        task.session.subtask = {device : task.device, name : 'getParameterValues', parameterNames : task.session.parameterNames}
-        delete task.session.parameterNames
-
-        this.getParameterValues(task.session.subtask, {}, (err, status, methodRequest) ->
-          # ignore deviceUpdates returned by firt call to getParameterValues
-          callback(err, status, methodRequest, deviceUpdates)
-        )
-      else if status & STATUS_OK
-        callback(err, STATUS_OK, methodRequest, deviceUpdates)
-      else
-        throw new Error('Unexpected subtask status')
-    )
-  else if task.session.subtask.name is 'getParameterValues'
-    if methodResponse.faultcode?
-      task.fault = methodResponse
-      return callback(null, STATUS_FAULT)
-
-    this.getParameterValues(task.session.subtask, methodResponse, (err, status, methodRequest, deviceUpdates) ->
-      callback(err, status, methodRequest, deviceUpdates)
-    )
-  else
-    throw Error('Unexpected subtask name')
-
-
-this.getParameterNames = (task, methodResponse, callback) ->
-  if methodResponse.faultcode?
-    task.fault = methodResponse
-    callback(null, STATUS_FAULT)
-    return
-
-  GET_PARAMETER_NAMES_DEPTH_THRESHOLD = config.get('GET_PARAMETER_NAMES_DEPTH_THRESHOLD', task.device)
-
-  getParameterDepth = (param) ->
-    return 0 if !param
-    return (param[...-1] + '.').split('.').length - 1
-
-  if methodResponse.type is 'GetParameterNamesResponse'
-    path = task.session.queue.pop()
-
-    # If parameter depth higher than the threshold, nextLevel was set to false
-    if not task.nextLevel? and getParameterDepth(path) < GET_PARAMETER_NAMES_DEPTH_THRESHOLD
-      for p in methodResponse.parameterList
-        task.session.queue.push(p[0]) if p[0][-1..] == '.'
-
-    deviceUpdates = {parameterNames : methodResponse.parameterList}
-
-    # Make sure that for each parameter, all its parents are explicitly included
-    found = {}
-    found[path] = 0 if !!path
-    for p in deviceUpdates.parameterNames
-      param = p[0]
-      i = param.length
-      while (i = param.lastIndexOf('.', i-1)) > path.length
-        pp = param.slice(0, i + 1)
-        break if found[pp]?
-        found[pp] = 0
-      found[p[0]] = 1
-
-    for k, v of found
-      deviceUpdates.parameterNames.push([k]) if v == 0
-  else
-    task.session = {queue : [task.parameterPath]}
-
-  if task.session.queue.length > 0
-    path = task.session.queue[-1..][0]
-    methodRequest = {
-      type : 'GetParameterNames',
-      parameterPath : path,
-      nextLevel : task.nextLevel ? getParameterDepth(path) < GET_PARAMETER_NAMES_DEPTH_THRESHOLD
-    }
-    return callback(null, STATUS_OK, methodRequest, deviceUpdates)
-  else
-    return callback(null, STATUS_COMPLETED, null, deviceUpdates)
-
-
-this.getParameterValues = (task, methodResponse, callback) ->
-  task.session = {} if not task.session?
-
-  if methodResponse.faultcode?
-    task.fault = methodResponse
-    callback(null, STATUS_FAULT)
-    return
-
-  if not task.session.currentIndex?
-    task.session.currentIndex = 0
-  else if methodResponse.parameterList?
-    task.session.currentIndex = task.session.nextIndex
-
-  task.session.nextIndex = Math.min(task.session.currentIndex + config.get('TASK_PARAMETERS_BATCH_SIZE', task.device), task.parameterNames.length)
-  names = task.parameterNames.slice(task.session.currentIndex, task.session.nextIndex)
-
-  if methodResponse.type is 'GetParameterValuesResponse'
-    deviceUpdates = {parameterValues : methodResponse.parameterList}
-
-  if names.length == 0
-    callback(null, STATUS_COMPLETED, null, deviceUpdates)
-  else
-    methodRequest = {
-      type : 'GetParameterValues',
-      parameterNames : names
-    }
-    callback(null, STATUS_OK, methodRequest, deviceUpdates)
-
-
-this.setParameterValues = (task, methodResponse, callback) ->
-  task.session = {} if not task.session?
-
-  if methodResponse.faultcode?
-    task.fault = methodResponse
-    callback(null, STATUS_FAULT)
-    return
-
-  if not task.session.currentIndex?
-    task.session.currentIndex = 0
-  else if methodResponse.type is 'SetParameterValuesResponse'
-    prevValues = task.parameterValues.slice(task.session.currentIndex, task.session.nextIndex)
-    task.session.currentIndex = task.session.nextIndex
-
-  task.session.nextIndex = Math.min(task.session.currentIndex + config.get('TASK_PARAMETERS_BATCH_SIZE', task.device), task.parameterValues.length)
-  values = task.parameterValues.slice(task.session.currentIndex, task.session.nextIndex)
-
-  if prevValues?
-    deviceUpdates = {parameterValues : prevValues}
-
-  if values.length == 0
-    callback(null, STATUS_COMPLETED, null, deviceUpdates)
-  else
-    callback(null, STATUS_OK, {type : 'SetParameterValues', parameterList : values}, deviceUpdates)
-
-
-this.addObject = (task, methodResponse, callback) ->
-  task.session = {} if not task.session?
-
-  if methodResponse.faultcode?
-    task.fault = methodResponse
-    callback(null, STATUS_FAULT)
-    return
-
-  allDeviceUpdates = {}
-
-  if not task.session.instanceNumber?
-    if methodResponse.type is 'AddObjectResponse'
-      status_save = STATUS_SAVE
-      task.session.instanceNumber = methodResponse.instanceNumber
-      # TODO Don't specify nextLevel explicity. Also consider using refreshObject instead.
-      task.session.subtask = {device : task.device, name : 'getParameterNames', parameterPath : "#{task.objectName}.#{task.session.instanceNumber}.", nextLevel : false}
-      task.session.appliedParameterValues = []
-      task.session.parameterNames = []
-      allDeviceUpdates.instanceName = [["#{task.objectName}.#{task.session.instanceNumber}", task.instanceName]] if task.instanceName?
-    else
-      callback(null, STATUS_OK, {type : 'AddObject', objectName : "#{task.objectName}."})
-      return
-
-  subtask = () =>
-    switch task.session.subtask.name
-      when 'getParameterNames'
-        this.getParameterNames(task.session.subtask, methodResponse, (err, status, methodRequest, deviceUpdates) =>
-          common.extend(allDeviceUpdates, deviceUpdates)
-          if deviceUpdates and deviceUpdates.parameterNames
-            for p in deviceUpdates.parameterNames
-              task.session.parameterNames.push(p[0]) if not common.endsWith(p[0], '.')
-
-          if status & STATUS_COMPLETED
-            task.session.subtask = {device : task.device, name : 'getParameterValues', parameterNames : task.session.parameterNames}
-            subtask()
-          else if status & STATUS_OK
-            # Use STATUS_SAVE to avoid adding duplicate object in case of error
-            callback(err, STATUS_OK | status_save, methodRequest, allDeviceUpdates)
-          else
-            throw Error('Unexpected subtask status')
-        )
-      when 'getParameterValues'
-        this.getParameterValues(task.session.subtask, methodResponse, (err, status, methodRequest, deviceUpdates) =>
-          common.extend(allDeviceUpdates, deviceUpdates)
-          # if values are given, compare with default values
-          if task.parameterValues?
-            if deviceUpdates and deviceUpdates.parameterValues?
-              for p1 in deviceUpdates.parameterValues
-                for p2 in task.parameterValues
-                  if common.endsWith(p1[0], ".#{p2[0]}")
-                    t = if p2[2] then p2[2] else p1[2]
-                    v = common.matchType(p1[1], p2[1])
-                    # TODO only include if writable
-                    task.session.appliedParameterValues.push([p1[0], v, t])
-
-          if methodResponse.faultcode?
-            task.fault = methodResponse
-            return callback(null, STATUS_FAULT)
-
-          if status & STATUS_COMPLETED and task.session.appliedParameterValues.length > 0
-            task.session.subtask = {device : task.device, name : 'setParameterValues', parameterValues : task.session.appliedParameterValues}
-            subtask()
-          else
-            callback(err, status, methodRequest, allDeviceUpdates)
-        )
-      when 'setParameterValues'
-        this.setParameterValues(task.session.subtask, methodResponse, (err, status, methodRequest, deviceUpdates) =>
-          common.extend(allDeviceUpdates, deviceUpdates)
-          callback(err, status, methodRequest, allDeviceUpdates)
-        )
-  subtask()
-
-
-this.deleteObject = (task, methodResponse, callback) ->
-  if methodResponse.faultcode?
-    task.fault = methodResponse
-    callback(null, STATUS_FAULT)
-    return
-
-  if methodResponse.type is 'DeleteObjectResponse'
-    callback(null, STATUS_COMPLETED, null, {deletedObjects : [task.objectName]})
-  else
-    methodRequest = {
-      type : 'DeleteObject',
-      objectName : "#{task.objectName}."
-    }
-    callback(null, STATUS_OK, methodRequest)
-
-
-this.reboot = (task, methodResponse, callback) ->
-  if methodResponse.faultcode?
-    task.fault = methodResponse
-    callback(null, STATUS_FAULT)
-    return
-  
-  if methodResponse.type isnt 'RebootResponse'
-    callback(null, STATUS_OK, {type : 'Reboot'})
-  else
-    callback(null, STATUS_COMPLETED)
-
-
-this.factoryReset = (task, methodResponse, callback) ->
-  if methodResponse.faultcode?
-    task.fault = methodResponse
-    callback(null, STATUS_FAULT)
-    return
-
-  if methodResponse.type isnt 'FactoryResetResponse'
-    callback(null, STATUS_OK, {type : 'FactoryReset'})
-  else
-    callback(null, STATUS_COMPLETED)
-
-
-this.download = (task, methodResponse, callback) ->
-  if methodResponse.faultcode?
-    task.fault = methodResponse
-    callback(null, STATUS_FAULT)
-    return
-
-  if methodResponse.type isnt 'DownloadResponse'
-    db.filesCollection.findOne({_id : mongodb.ObjectID(String(task.file))}, (err, file) ->
-      if not file?
-        callback('File not found')
-        return
-      else if err?
-        callback(err)
-        return
-
-      methodRequest = {
-        type : 'Download',
-        fileType : file.metadata.fileType,
-        fileSize : file.length,
-        url : "http://#{config.get('FS_IP')}:#{config.get('FS_PORT')}/#{encodeURIComponent(file.filename)}",
-        successUrl : task.successUrl,
-        failureUrl : task.failureUrl
-      }
-      callback(null, STATUS_OK, methodRequest)
-    )
-  else
-    callback(null, STATUS_COMPLETED)
-
-
-this.customCommand = (task, methodResponse, callback) ->
-  # TODO implement timeout
-  customCommands.execute(task.device, task.command, (err, value) ->
-    if err?
-      task.fault = err
-      callback(null, STATUS_FAULT)
-    else
-      commandName = task.command.split(' ', 2)[0]
-      callback(null, STATUS_COMPLETED, null, {customCommands : [[commandName, value]]})
-  )
-
-
-exports.task = (task, methodResponse, callback) ->
-  this[task.name](task, methodResponse, callback)

lib/types.ts

@@ -0,0 +1,351 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Path from "./common/path";
+import PathSet from "./common/path-set";
+import VersionedMap from "./versioned-map";
+import InstanceSet from "./instance-set";
+import { IncomingMessage, ServerResponse } from "http";
+
+export type Expression = string | number | boolean | null | any[];
+
+export interface Fault {
+  code: string;
+  message: string;
+  detail?:
+    | FaultStruct
+    | {
+        name: string;
+        message: string;
+        stack?: string;
+      };
+  timestamp?: number;
+}
+
+export interface SessionFault extends Fault {
+  timestamp: number;
+  provisions: string[][];
+  retryNow?: boolean;
+  precondition?: boolean;
+  retries?: number;
+  expiry?: number;
+}
+
+export interface Attributes {
+  object?: [number, 1 | 0];
+  writable?: [number, 1 | 0];
+  value?: [number, [string | number | boolean, string]];
+}
+
+export interface AttributeTimestamps {
+  object?: number;
+  writable?: number;
+  value?: number;
+}
+
+export interface AttributeValues {
+  object?: boolean;
+  writable?: boolean;
+  value?: [string | number | boolean, string?];
+}
+
+export interface DeviceData {
+  paths: PathSet;
+  timestamps: VersionedMap<Path, number>;
+  attributes: VersionedMap<Path, Attributes>;
+  trackers: Map<Path, { [name: string]: number }>;
+  changes: Set<string>;
+}
+
+export type VirtualParameterDeclaration = [
+  Path,
+  { path?: number; object?: number; writable?: number; value?: number }?,
+  {
+    path?: [number, number];
+    object?: boolean;
+    writable?: boolean;
+    value?: [string | number | boolean, string?];
+  }?
+];
+
+export interface SyncState {
+  refreshAttributes: {
+    exist: Set<Path>;
+    object: Set<Path>;
+    writable: Set<Path>;
+    value: Set<Path>;
+  };
+  spv: Map<Path, [string | number | boolean, string]>;
+  gpn: Set<Path>;
+  gpnPatterns: Map<Path, number>;
+  tags: Map<Path, boolean>;
+  virtualParameterDeclarations: VirtualParameterDeclaration[][];
+  instancesToDelete: Map<Path, Set<Path>>;
+  instancesToCreate: Map<Path, InstanceSet>;
+  downloadsToDelete: Set<Path>;
+  downloadsToCreate: InstanceSet;
+  downloadsValues: Map<Path, string | number>;
+  downloadsDownload: Map<Path, number>;
+  reboot: number;
+  factoryReset: number;
+}
+
+export interface SessionContext {
+  sessionId?: string;
+  timestamp: number;
+  deviceId: string;
+  deviceData: DeviceData;
+  cwmpVersion: string;
+  timeout: number;
+  provisions: any[];
+  channels: { [channel: string]: number };
+  virtualParameters: [
+    string,
+    AttributeTimestamps,
+    AttributeValues,
+    AttributeTimestamps,
+    AttributeValues
+  ][][];
+  revisions: number[];
+  rpcCount: number;
+  iteration: number;
+  cycle: number;
+  extensionsCache: any;
+  declarations: Declaration[][];
+  faults?: { [channel: string]: SessionFault };
+  retries?: { [channel: string]: number };
+  cacheSnapshot?: string;
+  configContext?: { [name: string]: any };
+  httpResponse?: ServerResponse;
+  httpRequest?: IncomingMessage;
+  faultsTouched?: { [channel: string]: boolean };
+  presetCycles?: number;
+  new?: boolean;
+  debug?: boolean;
+  state: number;
+  tasks?: Task[];
+  operations?: { [commandKey: string]: Operation };
+  cacheUntil?: number;
+  syncState?: SyncState;
+  lastActivity?: number;
+  rpcRequest?: AcsRequest;
+  operationsTouched?: { [commandKey: string]: 1 | 0 };
+  provisionsRet?: any[];
+  doneTasks?: string[];
+}
+
+export interface Task {
+  _id: string;
+  name: string;
+  parameterNames?: string[];
+  parameterValues?: [string, string | number | boolean, string?][];
+  objectName?: string;
+  fileType?: string;
+  fileName?: string;
+  targetFileName?: string;
+  expiry?: number;
+}
+
+export interface Operation {
+  name: string;
+  timestamp: number;
+  provisions: string[][];
+  channels: { [channel: string]: number };
+  retries: { [channel: string]: number };
+  args: {
+    instance: string;
+    fileType: string;
+    fileName: string;
+    targetFileName: string;
+  };
+}
+
+export interface AcsRequest {
+  name: string;
+  next?: string;
+}
+
+export interface GetAcsRequest extends AcsRequest {
+  name: "GetParameterNames" | "GetParameterValues";
+  objectName?: string;
+  parameterNames?: string[];
+  parameterPath?: string;
+  nextLevel?: boolean;
+  instanceValues?: { [name: string]: string };
+}
+
+export interface SetAcsRequest extends AcsRequest {
+  name:
+    | "SetParameterValues"
+    | "AddObject"
+    | "DeleteObject"
+    | "FactoryReset"
+    | "Reboot"
+    | "Download";
+  parameterList?: [string, string | number | boolean, string][];
+  instanceValues?: { [param: string]: string | number | boolean };
+  objectName?: string;
+  DATETIME_MILLISECONDS?: boolean;
+  BOOLEAN_LITERAL?: boolean;
+  commandKey?: string;
+  instance?: string;
+  fileType?: string;
+  fileSize?: number;
+  url?: string;
+  fileName?: string;
+  targetFileName?: string;
+}
+
+export interface CpeResponse {
+  name: string;
+}
+
+export interface SpvFault {
+  parameterName: string;
+  faultCode: string;
+  faultString: string;
+}
+
+export interface FaultStruct {
+  faultCode: string;
+  faultString: string;
+  setParameterValuesFault?: SpvFault[];
+}
+
+export interface CpeFault {
+  faultCode: string;
+  faultString: string;
+  detail?: FaultStruct;
+}
+
+export interface CpeGetResponse extends CpeResponse {
+  name: "GetParameterNamesResponse" | "GetParameterValuesResponse";
+  parameterList?:
+    | [string, boolean][]
+    | [string, string | number | boolean, string][];
+}
+
+export interface CpeSetResponse extends CpeResponse {
+  name:
+    | "SetParameterValuesResponse"
+    | "AddObjectResponse"
+    | "DeleteObjectResponse"
+    | "RebootResponse"
+    | "FactoryResetResponse"
+    | "DownloadResponse";
+  status?: number;
+  instanceNumber?: string;
+  startTime?: number;
+  completeTime?: number;
+}
+
+export interface CpeRequest {
+  name: string;
+  fileType?: string;
+}
+
+export interface InformRequest extends CpeRequest {
+  name: "Inform";
+  deviceId: {
+    Manufacturer: string;
+    OUI: string;
+    ProductClass?: string;
+    SerialNumber: string;
+  };
+  event: string[];
+  retryCount: number;
+  parameterList: [string, string | number | boolean, string][];
+}
+
+export interface TransferCompleteRequest extends CpeRequest {
+  name: "TransferComplete";
+  commandKey?: string;
+  faultStruct?: FaultStruct;
+  startTime?: number;
+  completeTime?: number;
+}
+
+export interface AcsResponse {
+  name: string;
+  commandKey?: string;
+  faultStruct?: FaultStruct;
+}
+
+export interface QueryOptions {
+  projection?: any;
+  skip?: number;
+  limit?: number;
+  sort?: {
+    [param: string]: number;
+  };
+}
+
+export interface Declaration {
+  path: Path;
+  pathGet: number;
+  pathSet?: [number, number];
+  attrGet?: { object?: number; writable?: number; value?: number };
+  attrSet?: {
+    object?: boolean;
+    writable?: boolean;
+    value?: [string | number | boolean, string?];
+  };
+  defer: boolean;
+}
+
+export type Clear = [
+  Path,
+  number,
+  { object?: number; writable?: number; value?: number }?,
+  number?
+];
+
+export interface Preset {
+  name: string;
+  channel: string;
+  schedule?: { md5: string; duration: number; schedule: any };
+  events?: { [event: string]: boolean };
+  precondition?: {};
+  provisions: string[][];
+}
+
+export type PermissionSet = {
+  [resource: string]: {
+    access: number;
+    validate?: Expression;
+    filter: Expression;
+  };
+}[];
+
+export interface SoapMessage {
+  id: string;
+  cwmpVersion: string;
+  sessionTimeout: number;
+  cpeRequest?: CpeRequest;
+  cpeFault?: CpeFault;
+  cpeResponse?: CpeResponse;
+}
+
+export interface ScriptResult {
+  fault: Fault;
+  clear: Clear[];
+  declare: Declaration[];
+  done: boolean;
+  returnValue: any;
+}

lib/ui.ts

@@ -0,0 +1,238 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { Z_SYNC_FLUSH } from "zlib";
+import Koa from "koa";
+import Router from "koa-router";
+import * as jwt from "jsonwebtoken";
+import koaStatic from "koa-static";
+import koaCompress from "koa-compress";
+import koaBodyParser from "koa-bodyparser";
+import koaJwt from "koa-jwt";
+import * as config from "./config";
+import api from "./ui/api";
+import Authorizer from "./common/authorizer";
+import * as logger from "./logger";
+import * as localCache from "./local-cache";
+import { PermissionSet } from "./types";
+import { authSimple } from "./ui/api-functions";
+import * as init from "./init";
+import { version as VERSION } from "../package.json";
+
+declare module "koa" {
+  interface Request {
+    body: any;
+  }
+}
+
+const koa = new Koa();
+const router = new Router();
+
+const JWT_SECRET = "" + config.get("UI_JWT_SECRET");
+const JWT_COOKIE = "genieacs-ui-jwt";
+
+function getPermissionSets(ctx): PermissionSet[] {
+  const allPermissions = localCache.getPermissions(ctx.state.configSnapshot);
+  const users = localCache.getUsers(ctx.state.configSnapshot);
+  const user = users[ctx.state.user.username];
+  const permissionSets = user.roles.map(role =>
+    Object.values(allPermissions[role] || {})
+  );
+  return permissionSets;
+}
+
+koa.on("error", async err => {
+  throw err;
+});
+
+koa.use(async (ctx, next) => {
+  const configSnapshot = await localCache.getCurrentSnapshot();
+  ctx.state.configSnapshot = configSnapshot;
+  ctx.set("X-Config-Snapshot", configSnapshot);
+  ctx.set("GenieACS-Version", VERSION);
+  return next();
+});
+
+koa.use(
+  koaJwt({
+    secret: JWT_SECRET,
+    passthrough: true,
+    cookie: JWT_COOKIE
+  })
+);
+
+koa.use(async (ctx, next) => {
+  if (ctx.state.user && ctx.state.user.username)
+    ctx.state.authorizer = new Authorizer(getPermissionSets(ctx));
+  else ctx.state.authorizer = new Authorizer([]);
+
+  return next();
+});
+
+router.post("/login", async ctx => {
+  if (!JWT_SECRET) {
+    ctx.status = 500;
+    ctx.body = "UI_JWT_SECRET is not set";
+    logger.error({ message: "UI_JWT_SECRET is not set" });
+    return;
+  }
+
+  const username = ctx.request.body.username;
+  const password = ctx.request.body.password;
+
+  const log = {
+    message: "Log in",
+    context: ctx,
+    username: username,
+    method: null
+  };
+
+  function success(method): void {
+    log.method = method;
+    const token = jwt.sign({ username }, JWT_SECRET);
+    ctx.cookies.set(JWT_COOKIE, token, { sameSite: "lax" });
+    ctx.body = JSON.stringify(token);
+    logger.accessInfo(log);
+  }
+
+  function failure(): void {
+    ctx.status = 400;
+    ctx.body = "Incorrect username or password";
+    log.message += " failed";
+    logger.accessWarn(log);
+  }
+
+  const roles = await authSimple(ctx.state.configSnapshot, username, password);
+
+  if (roles) return void success("simple");
+
+  failure();
+});
+
+router.post("/logout", async ctx => {
+  ctx.cookies.set(JWT_COOKIE); // Delete cookie
+  ctx.body = "";
+
+  logger.accessInfo({
+    message: "Log out",
+    context: ctx
+  });
+});
+
+koa.use(async (ctx, next) => {
+  if (ctx.request.type === "application/octet-stream")
+    ctx.disableBodyParser = true;
+
+  return next();
+});
+
+koa.use(koaBodyParser());
+router.use("/api", api.routes(), api.allowedMethods());
+
+router.get("/status", ctx => {
+  ctx.body = "OK";
+});
+
+router.get("/init", async ctx => {
+  const status = await init.getStatus();
+  if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
+    if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
+    if (!ctx.state.authorizer.hasAccess("permissions", 3))
+      status["users"] = false;
+    if (!ctx.state.authorizer.hasAccess("config", 3)) {
+      status["filters"] = false;
+      status["device"] = false;
+      status["index"] = false;
+      status["overview"] = false;
+    }
+    if (!ctx.state.authorizer.hasAccess("presets", 3))
+      status["presets"] = false;
+    if (!ctx.state.authorizer.hasAccess("provisions", 3))
+      status["presets"] = false;
+  }
+
+  ctx.body = status;
+});
+
+router.post("/init", async ctx => {
+  const status = ctx.request.body;
+  if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
+    if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
+    if (!ctx.state.authorizer.hasAccess("permissions", 3))
+      status["users"] = false;
+    if (!ctx.state.authorizer.hasAccess("config", 3)) {
+      status["filters"] = false;
+      status["device"] = false;
+      status["index"] = false;
+      status["overview"] = false;
+    }
+    if (!ctx.state.authorizer.hasAccess("presets", 3))
+      status["presets"] = false;
+    if (!ctx.state.authorizer.hasAccess("provisions", 3))
+      status["presets"] = false;
+  }
+  await init.seed(status);
+  ctx.body = "";
+});
+
+router.get("/", async ctx => {
+  let permissionSets = [];
+  if (ctx.state.user && ctx.state.user.username)
+    permissionSets = getPermissionSets(ctx);
+
+  let wizard = "";
+  if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)
+    wizard = '<script>window.location.hash = "#!/wizard";</script>';
+
+  ctx.body = `
+  <html>
+    <head>
+      <title>GenieACS</title>
+      <link rel="shortcut icon" type="image/png" href="favicon.png" />
+      <link rel="stylesheet" href="app.css">
+    </head>
+    <body>
+    <noscript>GenieACS UI requires JavaScript to work. Please enable JavaScript in your browser.</noscript>
+      <script>
+        window.clientConfig = ${JSON.stringify({
+          ui: localCache.getUiConfig(ctx.state.configSnapshot)
+        })};
+        window.configSnapshot = ${JSON.stringify(ctx.state.configSnapshot)};
+        window.genieacsVersion = ${JSON.stringify(VERSION)};
+        window.username = ${JSON.stringify(
+          ctx.state.user ? ctx.state.user.username : ""
+        )};
+        window.permissionSets = ${JSON.stringify(permissionSets)};
+      </script>
+      <script src="app.js"></script>${wizard} 
+    </body>
+  </html>
+  `;
+});
+
+koa.use(
+  koaCompress({
+    flush: Z_SYNC_FLUSH
+  })
+);
+
+koa.use(router.routes());
+koa.use(koaStatic(config.ROOT_DIR + "/public"));
+
+export const listener = koa.callback();

lib/ui/api-functions.ts

@@ -0,0 +1,207 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { exec } from "child_process";
+import { ObjectID } from "mongodb";
+import * as db from "./db";
+import { del } from "../cache";
+import { getUsers } from "../local-cache";
+import { hashPassword } from "../auth";
+import {
+  insertTasks,
+  watchTask,
+  connectionRequest,
+  deleteDevice
+} from "../api-functions";
+
+async function deleteFault(id): Promise<void> {
+  const deviceId = id.split(":", 1)[0];
+  const channel = id.slice(deviceId.length + 1);
+
+  await Promise.all([
+    db.deleteFault(id),
+    channel.startsWith("task_")
+      ? db.deleteTask(new ObjectID(channel.slice(5)))
+      : null
+  ]);
+
+  await del(`${deviceId}_tasks_faults_operations`);
+}
+
+export async function deleteResource(resource, id): Promise<void> {
+  switch (resource) {
+    case "devices":
+      await deleteDevice(id);
+      break;
+
+    case "files":
+      await db.deleteFile(id);
+      break;
+
+    case "faults":
+      await deleteFault(id);
+      break;
+
+    case "provisions":
+      await db.deleteProvision(id);
+      break;
+
+    case "presets":
+      await db.deletePreset(id);
+      break;
+
+    case "virtualParameters":
+      await db.deleteVirtualParameter(id);
+      break;
+
+    case "config":
+      await db.deleteConfig(id);
+      break;
+
+    case "permissions":
+      await db.deletePermission(id);
+      break;
+
+    case "users":
+      await db.deleteUser(id);
+      break;
+  }
+
+  await del("presets_hash");
+}
+
+export async function postTasks(
+  deviceId,
+  tasks,
+  timeout
+): Promise<{ connectionRequest: string; tasks: any[] }> {
+  for (const task of tasks) {
+    delete task._id;
+    task.device = deviceId;
+  }
+
+  tasks = await insertTasks(tasks);
+  const statuses = tasks.map(t => {
+    return { _id: t._id, status: "pending" };
+  });
+
+  await del(`${deviceId}_tasks_faults_operations`);
+
+  try {
+    await connectionRequest(deviceId);
+  } catch (err) {
+    return {
+      connectionRequest: err.message,
+      tasks: statuses
+    };
+  }
+
+  const sample = tasks[tasks.length - 1];
+
+  // Waiting for session to finish or timeout
+  await watchTask(deviceId, sample._id, timeout);
+
+  const promises = [];
+  for (const s of statuses) {
+    promises.push(db.query("tasks", ["=", ["PARAM", "_id"], s._id]));
+    promises.push(
+      db.query("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
+    );
+  }
+
+  const res = await Promise.all(promises);
+  for (const [i, r] of statuses.entries()) {
+    if (res[i * 2].length === 0) {
+      r.status = "done";
+    } else if (res[i * 2 + 1].length === 1) {
+      r.status = "fault";
+      r.fault = res[i * 2 + 1][0];
+    }
+    db.deleteTask(r._id);
+  }
+
+  return { connectionRequest: "OK", tasks: statuses };
+}
+
+interface PingResponse {
+  packetsTransmitted: number;
+  packetsReceived: number;
+  packetLoss: string;
+  min: number;
+  avg: number;
+  max: number;
+  mdev: number;
+}
+
+export function ping(host): Promise<PingResponse> {
+  return new Promise((resolve, reject) => {
+    exec(`ping -w 1 -i 0.2 -c 3 ${host}`, (err, stdout) => {
+      if (err) return void reject(err);
+
+      const m = stdout.match(
+        /(\d) packets transmitted, (\d) received, ([\d.%]+) packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/
+      );
+      if (!m) return void reject(new Error("Could not parse ping response"));
+
+      resolve({
+        packetsTransmitted: +m[1],
+        packetsReceived: +m[2],
+        packetLoss: m[3],
+        min: +m[4],
+        avg: +m[5],
+        max: +m[6],
+        mdev: +m[7]
+      });
+    });
+  });
+}
+
+export async function putResource(resource, id, data): Promise<void> {
+  if (resource === "presets") {
+    await db.putPreset(id, data);
+  } else if (resource === "provisions") {
+    await db.putProvision(id, data);
+  } else if (resource === "virtualParameters") {
+    await db.putVirtualParameter(id, data);
+  } else if (resource === "config") {
+    await db.putConfig(id, data);
+  } else if (resource === "permissions") {
+    await db.putPermission(id, data);
+  } else if (resource === "users") {
+    delete data.password;
+    delete data.salt;
+    await db.putUser(id, data);
+  }
+
+  await del("presets_hash");
+}
+
+export function authSimple(snapshot, username, password): Promise<string[]> {
+  return new Promise((resolve, reject) => {
+    const users = getUsers(snapshot);
+    const user = users[username];
+    if (!user || !user.password) return void resolve(null);
+    hashPassword(password, user.salt)
+      .then(hash => {
+        if (hash === user.password) resolve(user.roles);
+        else resolve(null);
+      })
+      .catch(reject);
+  });
+}

lib/ui/api.ts

@@ -0,0 +1,580 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as stream from "stream";
+import Router from "koa-router";
+import * as db from "./db";
+import * as apiFunctions from "./api-functions";
+import { evaluate, and, extractParams } from "../common/expression";
+import { parse } from "../common/expression-parser";
+import * as logger from "../logger";
+import { getConfig } from "../local-cache";
+import { QueryOptions, Expression } from "../types";
+import { generateSalt, hashPassword } from "../auth";
+import { del } from "../cache";
+
+const router = new Router();
+export default router;
+
+function logUnauthorizedWarning(log): void {
+  log.message += " not authorized";
+  logger.accessWarn(log);
+}
+
+const RESOURCE_DELETE = 1 << 0;
+const RESOURCE_PUT = 1 << 1;
+
+const RESOURCE_IDS = {
+  devices: "DeviceID.ID",
+  presets: "_id",
+  provisions: "_id",
+  files: "_id",
+  virtualParameters: "_id",
+  config: "_id",
+  permissions: "_id",
+  users: "_id",
+  faults: "_id",
+  tasks: "_id"
+};
+
+const resources = {
+  devices: 0 | RESOURCE_DELETE,
+  presets: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  provisions: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  files: 0 | RESOURCE_DELETE,
+  virtualParameters: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  config: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  permissions: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  users: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  faults: 0 | RESOURCE_DELETE,
+  tasks: 0
+};
+
+router.get(`/devices/:id.csv`, async ctx => {
+  const log = {
+    message: "Query device (CSV)",
+    context: ctx,
+    id: ctx.params.id
+  };
+
+  const filter = ["=", ["PARAM", RESOURCE_IDS.devices], ctx.params.id];
+
+  if (!ctx.state.authorizer.hasAccess("devices", 2)) {
+    logUnauthorizedWarning(log);
+    return void (ctx.status = 404);
+  }
+
+  const res = await db.query("devices", filter);
+  if (!res[0]) return void (ctx.status = 404);
+
+  ctx.type = "text/csv";
+  ctx.attachment(
+    `device-${ctx.params.id}-${new Date()
+      .toISOString()
+      .replace(/[:.]/g, "")}.csv`
+  );
+
+  ctx.body = new stream.PassThrough();
+  ctx.body.write(
+    "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp\n"
+  );
+
+  for (const k of Object.keys(res[0]).sort()) {
+    const p = res[0][k];
+    const row = [
+      k,
+      p.object,
+      p.objectTimestamp,
+      p.writable,
+      p.writableTimestamp,
+      p.value != null ? `"${p.value[0].toString().replace(/"/g, '""')}"` : "",
+      p.value != null ? p.value[1] : "",
+      p.valueTimestamp
+    ];
+    ctx.body.write(row.map(r => (r != null ? r : "")).join(",") + "\n");
+  }
+  ctx.body.end();
+  logger.accessInfo(log);
+});
+
+for (const [resource, flags] of Object.entries(resources)) {
+  router.head(`/${resource}`, async (ctx, next) => {
+    let filter: Expression = true;
+    if (ctx.request.query.filter) filter = parse(ctx.request.query.filter);
+
+    const log = {
+      message: `Count ${resource}`,
+      context: ctx,
+      filter: ctx.request.query.filter,
+      count: null
+    };
+
+    if (!ctx.state.authorizer.hasAccess(resource, 1)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    // Exclude temporary tasks and faults
+    if (resource === "tasks" || resource === "faults") {
+      filter = and(filter, [
+        "NOT",
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+      ]);
+    }
+
+    const count = await db.count(resource, filter);
+
+    ctx.set("X-Total-Count", `${count}`);
+    ctx.body = "";
+
+    log.count = count;
+    logger.accessInfo(log);
+  });
+
+  router.get(`/${resource}`, async (ctx, next) => {
+    const options: QueryOptions = {};
+    let filter: Expression = true;
+    if (ctx.request.query.filter) filter = parse(ctx.request.query.filter);
+    if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
+    if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
+    if (ctx.request.query.sort)
+      options.sort = JSON.parse(ctx.request.query.sort);
+    if (ctx.request.query.projection) {
+      options.projection = ctx.request.query.projection
+        .split(",")
+        .reduce((obj, k) => Object.assign(obj, { [k]: 1 }), {});
+    }
+
+    const log = {
+      message: `Query ${resource}`,
+      context: ctx,
+      filter: ctx.request.query.filter,
+      limit: options.limit,
+      skip: options.skip,
+      sort: options.sort,
+      projection: options.projection
+    };
+
+    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    // Exclude temporary tasks and faults
+    if (resource === "tasks" || resource === "faults") {
+      filter = and(filter, [
+        "NOT",
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+      ]);
+    }
+
+    ctx.body = new stream.PassThrough();
+    ctx.type = "application/json";
+
+    let c = 0;
+    ctx.body.write("[\n");
+    await db.query(resource, filter, options, obj => {
+      ctx.body.write((c++ ? "," : "") + JSON.stringify(obj) + "\n");
+    });
+    ctx.body.end("]");
+
+    logger.accessInfo(log);
+  });
+
+  // CSV download
+  router.get(`/${resource}.csv`, async (ctx, next) => {
+    const options: QueryOptions = { projection: {} };
+    let filter: Expression = true;
+    if (ctx.request.query.filter) filter = parse(ctx.request.query.filter);
+    if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
+    if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
+    if (ctx.request.query.sort)
+      options.sort = JSON.parse(ctx.request.query.sort);
+
+    const log = {
+      message: `Query ${resource} (CSV)`,
+      context: ctx,
+      filter: ctx.request.query.filter,
+      limit: options.limit,
+      skip: options.skip,
+      sort: options.sort
+    };
+
+    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    const columns = JSON.parse(ctx.request.query.columns);
+    const now = Date.now();
+
+    for (const [k, v] of Object.entries(columns)) {
+      const e = evaluate(parse(v as string), null, now);
+      columns[k] = e;
+      for (const p of extractParams(e))
+        if (typeof p === "string") options.projection[p] = 1;
+    }
+
+    // Exclude temporary tasks and faults
+    if (resource === "tasks" || resource === "faults") {
+      filter = and(filter, [
+        "NOT",
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+      ]);
+    }
+
+    ctx.body = new stream.PassThrough();
+    ctx.type = "text/csv";
+    ctx.attachment(
+      `${resource}-${new Date(now).toISOString().replace(/[:.]/g, "")}.csv`
+    );
+
+    ctx.body.write(
+      Object.keys(columns).map(k => `"${k.replace(/"/, '""')}"`) + "\n"
+    );
+    await db.query(resource, filter, options, obj => {
+      const arr = Object.values(columns).map(exp => {
+        const v = evaluate(exp, obj, null, e => {
+          if (Array.isArray(e)) {
+            if (e[0] === "PARAM") {
+              if (resource === "devices") {
+                if (e[1] === "Tags") {
+                  const tags = [];
+                  for (const p in obj)
+                    if (p.startsWith("Tags.")) tags.push(p.slice(5));
+
+                  return tags.join(", ");
+                }
+              }
+            } else if (e[0] === "FUNC") {
+              if (e[1] === "DATE_STRING") {
+                if (e[2] && !Array.isArray(e[2]))
+                  return new Date(e[2]).toISOString();
+              }
+            }
+          }
+
+          return e;
+        });
+
+        if (Array.isArray(v) || v == null) return "";
+        if (typeof v === "string") return `"${v.replace(/"/g, '""')}"`;
+        return v;
+      });
+      ctx.body.write(arr.join(",") + "\n");
+    });
+    ctx.body.end();
+
+    logger.accessInfo(log);
+  });
+
+  router.head(`/${resource}/:id`, async (ctx, next) => {
+    const log = {
+      message: `Count ${resource}`,
+      context: ctx,
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
+    };
+
+    const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
+    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    const res = await db.query(resource, filter);
+
+    if (!res.length) return void next();
+
+    logger.accessInfo(log);
+    ctx.body = "";
+  });
+
+  router.get(`/${resource}/:id`, async (ctx, next) => {
+    const log = {
+      message: `Query ${resource}`,
+      context: ctx,
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
+    };
+
+    const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
+    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    const res = await db.query(resource, filter);
+
+    if (!res.length) return void next();
+
+    logger.accessInfo(log);
+    ctx.body = res[0];
+  });
+
+  // TODO add PUT, PATCH routes
+  if (flags & RESOURCE_DELETE) {
+    router.delete(`/${resource}/:id`, async (ctx, next) => {
+      const log = {
+        message: `Delete ${resource}`,
+        context: ctx,
+        id: ctx.params.id
+      };
+
+      const authorizer = ctx.state.authorizer;
+      const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
+      if (!authorizer.hasAccess(resource, 3)) {
+        logUnauthorizedWarning(log);
+        return void next();
+      }
+      const res = await db.query(resource, filter);
+      if (!res.length) return void next();
+
+      const validate = authorizer.getValidator(resource, res[0]);
+      if (!validate("delete")) {
+        logUnauthorizedWarning(log);
+        return void (ctx.status = 403);
+      }
+
+      await apiFunctions.deleteResource(resource, ctx.params.id);
+
+      logger.accessInfo(log);
+
+      ctx.body = "";
+    });
+  }
+
+  if (flags & RESOURCE_PUT) {
+    router.put(`/${resource}/:id`, async (ctx, next) => {
+      const id = ctx.params.id;
+
+      const log = {
+        message: `Put ${resource}`,
+        context: ctx,
+        id: id
+      };
+
+      const authorizer = ctx.state.authorizer;
+      if (!authorizer.hasAccess(resource, 3)) {
+        logUnauthorizedWarning(log);
+        return void next();
+      }
+
+      const obj = ctx.request.body;
+
+      const validate = authorizer.getValidator(resource, obj);
+      if (!validate("put")) {
+        logUnauthorizedWarning(log);
+        return void (ctx.status = 403);
+      }
+
+      try {
+        await apiFunctions.putResource(resource, id, obj);
+      } catch (err) {
+        log.message += " failed";
+        logger.accessWarn(log);
+        ctx.body = `${err.name}: ${err.message}`;
+        return void (ctx.status = 400);
+      }
+
+      logger.accessInfo(log);
+
+      ctx.body = "";
+    });
+  }
+}
+
+router.put("/files/:id", async (ctx, next) => {
+  const resource = "files";
+  const id = ctx.params.id;
+
+  const log = {
+    message: `Upload ${resource}`,
+    context: ctx,
+    id: id,
+    metadata: null
+  };
+
+  const authorizer = ctx.state.authorizer;
+  if (!authorizer.hasAccess(resource, 3)) {
+    logUnauthorizedWarning(log);
+    return void next();
+  }
+
+  const metadata = {
+    fileType: ctx.request.headers["metadata.filetype"] || "",
+    oui: ctx.request.headers["metadata.oui"] || "",
+    productClass: ctx.request.headers["metadata.productclass"] || "",
+    version: ctx.request.headers["metadata.version"] || ""
+  };
+
+  const validate = authorizer.getValidator(resource, metadata);
+  if (!validate("put")) {
+    logUnauthorizedWarning(log);
+    return void (ctx.status = 403);
+  }
+
+  await db.putFile(id, metadata, ctx.req);
+  log.metadata = metadata;
+  logger.accessInfo(log);
+
+  ctx.body = "";
+});
+
+router.post("/devices/:id/tasks", async (ctx, next) => {
+  const log = {
+    message: "Commit tasks",
+    context: ctx,
+    deviceId: ctx.params.id,
+    tasks: null
+  };
+
+  const authorizer = ctx.state.authorizer;
+  const filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
+  if (!authorizer.hasAccess("devices", 3)) {
+    logUnauthorizedWarning(log);
+    return void next();
+  }
+  const devices = await db.query("devices", filter);
+  if (!devices.length) return void next();
+
+  const validate = authorizer.getValidator("devices", devices[0]);
+  for (const t of ctx.request.body) {
+    if (!validate("task", t)) {
+      logUnauthorizedWarning(log);
+      return void (ctx.status = 403);
+    }
+  }
+
+  // TODO Pass context and timestamp
+  const onlineThreshold = getConfig(
+    ctx.state.configSnapshot,
+    "cwmp.deviceOnlineThreshold"
+  );
+
+  const res = await apiFunctions.postTasks(
+    ctx.params.id,
+    ctx.request.body,
+    onlineThreshold
+  );
+
+  log.tasks = res.tasks.map(t => t._id).join(",");
+
+  logger.accessInfo(log);
+
+  ctx.set("Connection-Request", res.connectionRequest);
+  ctx.body = res.tasks;
+});
+
+router.post("/devices/:id/tags", async (ctx, next) => {
+  const log = {
+    message: "Update tags",
+    context: ctx,
+    deviceId: ctx.params.id,
+    tags: ctx.request.body
+  };
+
+  const authorizer = ctx.state.authorizer;
+  const filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
+  if (!authorizer.hasAccess("devices", 3)) {
+    logUnauthorizedWarning(log);
+    return void next();
+  }
+  const res = await db.query("devices", filter);
+  if (!res.length) return void next();
+
+  const validate = authorizer.getValidator("devices", res[0]);
+  if (!validate("tags", ctx.request.body)) {
+    logUnauthorizedWarning(log);
+    return void (ctx.status = 403);
+  }
+
+  try {
+    await db.updateDeviceTags(ctx.params.id, ctx.request.body);
+  } catch (error) {
+    log.message += " failed";
+    logger.accessWarn(log);
+    ctx.body = error.message;
+    return void (ctx.status = 400);
+  }
+
+  logger.accessInfo(log);
+
+  ctx.body = "";
+});
+
+router.get("/ping/:host", async ctx => {
+  try {
+    ctx.body = await apiFunctions.ping(ctx.params.host);
+  } catch (err) {
+    ctx.status = 404;
+    ctx.response.headers["Cache-Control"] = "no-cache";
+    ctx.body = `${err.name}: ${err.message}`;
+  }
+});
+
+router.put("/users/:id/password", async (ctx, next) => {
+  const username = ctx.params.id;
+  const log = {
+    message: "Change password",
+    context: ctx,
+    username: username
+  };
+
+  const authorizer = ctx.state.authorizer;
+
+  if (!ctx.state.user) {
+    // User not logged in
+    if (
+      !(await apiFunctions.authSimple(
+        ctx.state.configSnapshot,
+        username,
+        ctx.request.body.authPassword
+      ))
+    ) {
+      logUnauthorizedWarning(log);
+      ctx.status = 401;
+      ctx.body = "Authentication failed, check your username and password";
+      return;
+    }
+  } else if (!authorizer.hasAccess("users", 3)) {
+    logUnauthorizedWarning(log);
+    return void next();
+  }
+
+  const filter = ["=", ["PARAM", RESOURCE_IDS.users], username];
+  const res = await db.query("users", filter);
+  if (!res.length) return void next();
+
+  const newPassword = ctx.request.body.newPassword;
+  if (ctx.state.user) {
+    const validate = authorizer.getValidator("users", res[0]);
+    if (!validate("password", { password: newPassword })) {
+      logUnauthorizedWarning(log);
+      return void (ctx.status = 403);
+    }
+  }
+
+  const salt = await generateSalt(64);
+  const password = await hashPassword(newPassword, salt);
+  await db.putUser(username, { password, salt });
+
+  await del("presets_hash");
+
+  logger.accessInfo(log);
+  ctx.body = "";
+});

lib/ui/db.ts

@@ -0,0 +1,414 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { MongoClient, GridFSBucket } from "mongodb";
+import { Script } from "vm";
+import * as config from "../config";
+import * as mongodbFunctions from "../mongodb-functions";
+import * as expression from "../common/expression";
+import { QueryOptions } from "../types";
+
+const CACHE_TTL = 300000;
+
+let _clientPromise = null;
+
+const RESOURCE_COLLECTION = {
+  files: "fs.files"
+};
+
+function ensureIndexes(client): void {
+  client
+    .db()
+    .collection("cache")
+    .createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
+}
+
+function getClient(): Promise<MongoClient> {
+  if (!_clientPromise) {
+    _clientPromise = new Promise((resolve, reject) => {
+      const CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
+      MongoClient.connect(
+        CONNECTION_URL,
+        { useNewUrlParser: true },
+        (err, client) => {
+          if (err) return void reject(err);
+          ensureIndexes(client);
+          resolve(client);
+        }
+      );
+    });
+  }
+
+  return _clientPromise;
+}
+
+export function cache<T>(key, valueGetter: () => Promise<T>, ttl): Promise<T> {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const collection = client.db().collection("cache");
+        collection.findOne({ _id: key }, (err, doc) => {
+          if (err) return void reject(err);
+          if (doc != null) return void resolve(JSON.parse(doc.value) as T);
+          valueGetter()
+            .then(res => {
+              const expire = Date.now() + (ttl || CACHE_TTL);
+              const cacheDoc = {
+                _id: key,
+                value: JSON.stringify(res),
+                expire: new Date(expire)
+              };
+              collection.updateOne(
+                { _id: key },
+                { $set: cacheDoc },
+                { upsert: true },
+                err => {
+                  if (err) reject(err);
+                  else resolve(res);
+                }
+              );
+            })
+            .catch(reject);
+        });
+      })
+      .catch(reject);
+  });
+}
+
+export function query(
+  resource,
+  filter,
+  options?: QueryOptions,
+  callback?: (doc: any) => void
+): Promise<any[]> {
+  options = options || {};
+  let q;
+  filter = expression.evaluate(filter, null, Date.now());
+
+  if (Array.isArray(filter)) {
+    if (resource === "devices") {
+      filter = mongodbFunctions.processDeviceFilter(filter);
+    } else if (resource === "tasks") {
+      filter = mongodbFunctions.processTasksFilter(filter);
+    } else if (resource === "faults") {
+      filter = mongodbFunctions.processFaultsFilter(filter);
+    } else if (resource === "users") {
+      // Protect against brute force, and dictionary attacks
+      const params = expression.extractParams(filter);
+      if (params.includes("password") || params.includes("salt"))
+        return Promise.reject(new Error("Invalid users filter"));
+    }
+
+    q = mongodbFunctions.filterToMongoQuery(filter);
+  } else if (filter != null && !filter) {
+    return Promise.resolve([]);
+  }
+
+  return new Promise((resolve, reject) => {
+    getClient().then(client => {
+      const collection = client
+        .db()
+        .collection(RESOURCE_COLLECTION[resource] || resource);
+      const cursor = collection.find(q);
+      if (options.projection) {
+        let projection = options.projection;
+        if (resource === "devices") {
+          projection = mongodbFunctions.processDeviceProjection(
+            options.projection
+          );
+        }
+
+        if (resource === "presets") projection.configurations = 1;
+        cursor.project(projection);
+      }
+
+      if (resource === "users") cursor.project({ password: 0, salt: 0 });
+
+      if (options.skip) cursor.skip(options.skip);
+      if (options.limit) cursor.limit(options.limit);
+
+      if (options.sort) {
+        let s = Object.entries(options.sort)
+          .sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
+          .reduce(
+            (obj, [k, v]) =>
+              Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
+            {}
+          );
+
+        if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
+        cursor.sort(s);
+      }
+
+      if (!callback) {
+        cursor.toArray((err, docs) => {
+          if (err) return reject(err);
+          if (resource === "devices")
+            docs = docs.map(d => mongodbFunctions.flattenDevice(d));
+          else if (resource === "faults")
+            docs = docs.map(d => mongodbFunctions.flattenFault(d));
+          else if (resource === "tasks")
+            docs = docs.map(d => mongodbFunctions.flattenTask(d));
+          else if (resource === "presets")
+            docs = docs.map(d => mongodbFunctions.flattenPreset(d));
+          else if (resource === "files")
+            docs = docs.map(d => mongodbFunctions.flattenFile(d));
+          return resolve(docs);
+        });
+      } else {
+        cursor.forEach(
+          doc => {
+            if (resource === "devices")
+              doc = mongodbFunctions.flattenDevice(doc);
+            else if (resource === "faults")
+              doc = mongodbFunctions.flattenFault(doc);
+            else if (resource === "tasks")
+              doc = mongodbFunctions.flattenTask(doc);
+            else if (resource === "presets")
+              doc = mongodbFunctions.flattenPreset(doc);
+            else if (resource === "files")
+              doc = mongodbFunctions.flattenFile(doc);
+            callback(doc);
+          },
+          err => {
+            if (err) reject(err);
+            else resolve();
+          }
+        );
+      }
+    });
+  });
+}
+
+export function count(resource, filter): Promise<number> {
+  let q;
+  filter = expression.evaluate(filter, null, Date.now());
+
+  if (Array.isArray(filter)) {
+    if (resource === "devices")
+      filter = mongodbFunctions.processDeviceFilter(filter);
+    else if (resource === "tasks")
+      filter = mongodbFunctions.processTasksFilter(filter);
+    else if (resource === "faults")
+      filter = mongodbFunctions.processFaultsFilter(filter);
+    q = mongodbFunctions.filterToMongoQuery(filter);
+  } else if (filter != null && !filter) {
+    return Promise.resolve(0);
+  }
+
+  return new Promise((resolve, reject) => {
+    getClient().then(client => {
+      const collection = client
+        .db()
+        .collection(RESOURCE_COLLECTION[resource] || resource);
+      collection.find(q).count((err, c) => {
+        if (err) reject(err);
+        else resolve(c);
+      });
+    });
+  });
+}
+
+export function updateDeviceTags(deviceId, tags): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const add = [];
+    const pull = [];
+
+    const regex = /^[0-9a-zA-Z_]+$/;
+    for (let [tag, onOff] of Object.entries(tags)) {
+      tag = tag.trim();
+      if (onOff) {
+        if (!tag.match(regex))
+          return void reject(new Error(`Invalid tag '${tag}'`));
+        add.push(tag);
+      } else {
+        pull.push(tag);
+      }
+    }
+    getClient()
+      .then(client => {
+        const collection = client.db().collection("devices");
+
+        const object = {};
+        if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
+        if (pull && pull.length) object["$pullAll"] = { _tags: pull };
+        collection.updateOne({ _id: deviceId }, object, err => {
+          if (err) return void reject(err);
+          resolve();
+        });
+      })
+      .catch(reject);
+  });
+}
+
+function putResource(resource, id, object): Promise<void> {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const collection = client
+          .db()
+          .collection(RESOURCE_COLLECTION[resource] || resource);
+        collection.replaceOne({ _id: id }, object, { upsert: true }, err => {
+          if (err) return void reject(err);
+          resolve();
+        });
+      })
+      .catch(reject);
+  });
+}
+
+function deleteResource(resource, id): Promise<void> {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const collection = client
+          .db()
+          .collection(RESOURCE_COLLECTION[resource] || resource);
+        collection.deleteOne({ _id: id }, err => {
+          if (err) return void reject(err);
+          resolve();
+        });
+      })
+      .catch(reject);
+  });
+}
+
+export function putPreset(id, object): Promise<void> {
+  object = mongodbFunctions.preProcessPreset(object);
+  return putResource("presets", id, object);
+}
+
+export function deletePreset(id): Promise<void> {
+  return deleteResource("presets", id);
+}
+
+export function putProvision(id, object): Promise<void> {
+  if (!object.script) object.script = "";
+  try {
+    new Script(`"use strict";(function(){\n${object.script}\n})();`);
+  } catch (error) {
+    return Promise.reject(error);
+  }
+  return putResource("provisions", id, object);
+}
+
+export function deleteProvision(id): Promise<void> {
+  return deleteResource("provisions", id);
+}
+
+export function putVirtualParameter(id, object): Promise<void> {
+  if (!object.script) object.script = "";
+  try {
+    new Script(`"use strict";(function(){\n${object.script}\n})();`);
+  } catch (error) {
+    return Promise.reject(error);
+  }
+  return putResource("virtualParameters", id, object);
+}
+
+export function deleteVirtualParameter(id): Promise<void> {
+  return deleteResource("virtualParameters", id);
+}
+
+export function putConfig(id, object): Promise<void> {
+  return putResource("config", id, object);
+}
+
+export function deleteConfig(id): Promise<void> {
+  return deleteResource("config", id);
+}
+
+export function putPermission(id, object): Promise<void> {
+  return putResource("permissions", id, object);
+}
+
+export function deletePermission(id): Promise<void> {
+  return deleteResource("permissions", id);
+}
+
+export function putUser(id, object): Promise<void> {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const collection = client.db().collection("users");
+        // update instead of replace to keep the password if not set by user
+        collection.updateOne(
+          { _id: id },
+          { $set: object },
+          { upsert: true },
+          err => {
+            if (err) return void reject(err);
+            resolve();
+          }
+        );
+      })
+      .catch(reject);
+  });
+}
+
+export function deleteUser(id): Promise<void> {
+  return deleteResource("users", id);
+}
+
+export function putFile(filename, metadata, contentStream): Promise<void> {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const bucket = new GridFSBucket(client.db());
+        const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
+          metadata: metadata
+        });
+        uploadStream.on("error", reject);
+        contentStream.on("error", reject);
+        uploadStream.on("finish", resolve);
+        contentStream.pipe(uploadStream);
+      })
+      .catch(reject);
+  });
+}
+
+export function deleteFile(filename): Promise<void> {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const bucket = new GridFSBucket(client.db());
+        bucket.delete(filename, err => {
+          if (err) return void reject(err);
+          resolve();
+        });
+      })
+      .catch(reject);
+  });
+}
+
+export function deleteFault(id): Promise<void> {
+  return deleteResource("faults", id);
+}
+
+export function deleteTask(id): Promise<void> {
+  return deleteResource("tasks", id);
+}
+
+export function disconnect(): void {
+  if (_clientPromise) {
+    _clientPromise.then(client => {
+      client.close();
+    });
+  }
+}

lib/update-device.coffee

@@ -1,221 +0,0 @@
-###
-# Copyright 2013, 2014  Zaid Abdulla
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-common = require './common'
-db = require './db'
-normalize = require('./normalize').normalize
-mongoQuery = require './mongo-query'
-parameters = require './parameters'
-
-OBJECT_REGEX = /\.$/
-INSTANCE_REGEX = /\.[\d]+\.$/
-
-
-queueUpdates = (deviceId, deviceUpdates, callback) ->
-  return callback() if not deviceUpdates? or Object.keys(deviceUpdates).length == 0
-  # TODO consider setting expiry
-  db.redisClient.rpush("#{deviceId}_updates", JSON.stringify(deviceUpdates), (err, res) ->
-    return callback(err)
-  )
-
-
-clearUpdatesQueue = (deviceId, callback) ->
-  db.redisClient.del("#{deviceId}_updates", (err, res) ->
-    return callback(err)
-  )
-
-
-commitUpdates = (deviceId, deviceUpdates, commitQueue, callback) ->
-  now = new Date(Date.now())
-  update = {'$set' : {}, '$unset' : {}}
-
-  # Used to find parameters that no longer exist in device
-  presentPaths = null
-  projection = null
-
-  # Keep track of alias changes to clear cache if needed
-  newAliases = null
-
-  processBatch = (updatesBatch, cb) ->
-    if updatesBatch.informEvents?
-      update['$set']['_lastInform'] = now
-      update['$set']['_lastBoot'] if '1 BOOT' in updatesBatch.informEvents
-      update['$set']['_lastBootstrap'] if '0 BOOTSTRAP' in updatesBatch.informEvents
-
-    if updatesBatch.parameterValues?
-      for p in updatesBatch.parameterValues
-        origValue = p[1]
-        v = normalize(p[0], origValue)
-
-        path = if common.endsWith(p[0], '.') then p[0] else "#{p[0]}."
-        if v == origValue
-          update['$unset']["#{path}_orig"] = 1
-        else
-          update['$set']["#{path}_orig"] = origValue
-        update['$set']["#{path}_value"] = v
-        update['$set']["#{path}_timestamp"] = now
-        if p[2]?
-          update['$set']["#{path}_type"] = p[2]
-
-    if updatesBatch.deletedObjects?
-      for p in updatesBatch.deletedObjects
-        update['$unset'][p] = 1
-
-    if updatesBatch.instanceName?
-      for i in updatesBatch.instanceName
-        update['$set']["#{i[0]}._name"] = i[1]
-
-    if updatesBatch.parameterNames?
-      presentPaths ?= {}
-      projection ?= {}
-
-      for p in updatesBatch.parameterNames
-        if OBJECT_REGEX.test(p[0])
-          path = p[0].slice(0, -1)
-          if INSTANCE_REGEX.test(p[0])
-            update['$set']["#{path}._instance"] = true
-          else
-            update['$set']["#{path}._object"] = true
-        else
-          path = p[0]
-        update['$set']["#{path}._writable"] = p[1] if p[1]?
-        update['$set']["#{path}._timestamp"] = now
-
-        # Add parameter (sans any dot suffix) if has an alias
-        if (a = parameters.getAlias(path))?
-          newAliases ?= {}
-          newAliases[path] = a
-
-        projection[path] = 1
-
-        # Store this param path and all its parents for later look up
-        presentPaths[path] = true
-        i = p[0].indexOf('.')
-        while i != -1
-          presentPaths[p[0][0...i]] = true
-          i = p[0].indexOf('.', i + 1)
-
-    if updatesBatch.customCommands?
-      for p in updatesBatch.customCommands
-        commandName = p[0]
-        update['$set']["_customCommands.#{commandName}._value"] = p[1]
-        update['$set']["_customCommands.#{commandName}._timestamp"] = now
-
-  processBatches = (cb) ->
-    if not commitQueue
-      processBatch(deviceUpdates) if deviceUpdates?
-      return cb()
-
-    db.redisClient.lpop("#{deviceId}_updates", (err, res) ->
-      callback(err) if err
-
-      if not res?
-        processBatch(deviceUpdates) if deviceUpdates?
-        return cb()
-
-      processBatch(JSON.parse(res))
-      return processBatches(cb)
-    )
-
-  commit = () ->
-    for k of update
-      delete update[k] if Object.keys(update[k]).length == 0
-
-    if Object.keys(update).length
-      db.devicesCollection.update({'_id' : deviceId}, update, {}, (err, count) ->
-        return callback(err) if err
-
-        # Clear aliases cache if there's a new aliased parameter that has not been included in aliases cache
-        if newAliases?
-          newAliasFound = false
-          db.getAliases((aliases) ->
-            for p, a of newAliases
-              if not aliases[a]? or p not in aliases[a]
-                newAliasFound = true
-                break
-
-            return callback() if not newAliasFound
-            db.redisClient.del('aliases', (err, res) ->
-              return callback(err)
-            )
-          )
-        else
-          return callback()
-      )
-    else
-      return callback()
-
-  processBatches(() ->
-    if projection?
-      mongoQuery.optimizeProjection(projection)
-
-      # Fetch device from DB to discover and delete parameters that no longer exist
-      # While we're fetching the object, we might as well use the information
-      # to optimize our update query
-      db.devicesCollection.findOne({'_id' : deviceId}, projection, (err, device) ->
-        return callback(err) if err
-
-        recursive = (obj, prefix) ->
-          for k, v of obj
-            p = prefix + k
-
-            # Mark unset operations we want to keep
-            if update['$unset'][p]?
-              update['$unset'][p] = 2
-              continue
-
-            if k[0] == '_'
-              # Remove set operations for unmodified values for optimization
-              if update['$set'][p] == v
-                delete update['$set'][p]
-              continue
-
-            if presentPaths[p]
-              recursive(v, "#{p}.")
-            else
-              update['$unset'][p] = 2
-
-        # Scan the DB object and remove parameters that no longer exists in device
-        for path of projection
-          root = device
-          rootPath = ''
-          for p in path.split('.')
-            rootPath += "#{p}."
-            root = root[p]
-            break if not root?
-
-          if root
-            recursive(root, rootPath)
-          else
-            if not presentPaths[rootPath.slice(0, -1)]?
-              update['$unset'][rootPath.slice(0, -1)] = 2
-
-          # Remove unnecessary unset updates
-          for k, v of update['$unset']
-            if common.startsWith(k, path + '.') and v != 2
-              delete update['$unset'][k]
-
-        commit()
-      )
-    else
-      commit()
-  )
-
-
-exports.queueUpdates = queueUpdates
-exports.clearUpdatesQueue = clearUpdatesQueue
-exports.commitUpdates = commitUpdates

lib/versioned-map.ts

@@ -0,0 +1,200 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+const NONEXISTENT = Symbol();
+const UNDEFINED = undefined;
+
+interface Revisions<V> {
+  [rev: number]: V;
+  delete?: number;
+}
+
+export default class VersionedMap<K, V> {
+  private _sizeDiff: number[];
+  private _revision: number;
+  private map: Map<K, (V | symbol)[]>;
+  public dirty: number;
+
+  public constructor() {
+    this._sizeDiff = [0];
+    this._revision = 0;
+    this.map = new Map();
+    this.dirty = 0;
+  }
+
+  public get size(): number {
+    return this.map.size + this._sizeDiff[this.revision];
+  }
+
+  public get revision(): number {
+    return this._revision;
+  }
+
+  public set revision(rev) {
+    for (let i = this._sizeDiff.length; i <= rev; ++i)
+      this._sizeDiff[i] = this._sizeDiff[i - 1];
+
+    this._revision = rev;
+  }
+
+  public get(key: K, rev = this._revision): V {
+    const revisions = this.map.get(key);
+    if (!revisions) return UNDEFINED;
+
+    const v = revisions[Math.min(revisions.length - 1, rev)];
+    if (v === NONEXISTENT) return UNDEFINED;
+    return v as V;
+  }
+
+  public has(key: K, rev = this._revision): boolean {
+    const revisions = this.map.get(key);
+    if (!revisions) return false;
+
+    const v = revisions[Math.min(revisions.length - 1, rev)];
+    if (v === NONEXISTENT) return false;
+    return true;
+  }
+
+  public set(key: K, value: V, rev = this._revision): this {
+    let revisions = this.map.get(key);
+    if (!revisions) {
+      this.dirty |= 1 << rev;
+      for (let i = 0; i < rev; ++i) this._sizeDiff[i] -= 1;
+
+      revisions = [];
+      for (let i = 0; i < rev; ++i) revisions[i] = NONEXISTENT;
+      revisions[rev] = value;
+      this.map.set(key, revisions);
+      return this;
+    }
+
+    // Can't modify old revisions
+    if (rev < revisions.length - 1) return null;
+
+    const old = revisions[revisions.length - 1];
+
+    this.dirty |= 1 << rev;
+    if (old === NONEXISTENT) ++this._sizeDiff[rev];
+
+    for (let i = revisions.length; i < rev; ++i) revisions[i] = old;
+    revisions[rev] = value;
+
+    return this;
+  }
+
+  public delete(key: K, rev = this._revision): boolean {
+    const revisions = this.map.get(key);
+    if (!revisions) return false;
+
+    // Can't modify old revisions
+    if (rev < revisions.length - 1) return null;
+
+    const old = revisions[revisions.length - 1];
+    if (old === NONEXISTENT) return false;
+
+    this.dirty |= 1 << rev;
+    --this._sizeDiff[rev];
+
+    for (let i = revisions.length; i < rev; ++i) revisions[i] = old;
+    revisions[rev] = NONEXISTENT;
+
+    return true;
+  }
+
+  public getRevisions(key: K): Revisions<V> {
+    const revisions = this.map.get(key);
+    if (!revisions) return null;
+
+    const res: Revisions<V> = {};
+
+    let prev: V | symbol = NONEXISTENT;
+    for (const [i, v] of revisions.entries()) {
+      if (v === prev) continue;
+      if (v === NONEXISTENT) res.delete |= 1 << i;
+      else res[i] = v as V;
+      prev = v;
+    }
+
+    return res;
+  }
+
+  public setRevisions(key: K, revisionsObj: Revisions<V>): void {
+    const del = revisionsObj.delete || 0;
+    const mutations = Object.keys(revisionsObj).reduce(
+      (acc, cur) => (cur === "delete" ? acc : acc | (1 << +cur)),
+      del
+    );
+
+    const revisions = [];
+
+    let prev: V | symbol = NONEXISTENT;
+    for (let i = 0; mutations >> i; ++i) {
+      let v = prev;
+      if (del & (1 << i)) v = NONEXISTENT;
+      else if (i in revisionsObj) v = revisionsObj[i];
+      if (v !== prev) this.dirty |= 1 << i;
+      revisions[i] = v;
+      prev = v;
+    }
+
+    this.map.set(key, revisions);
+  }
+
+  public getDiff(key: K): [V, V] {
+    const revisions = this.map.get(key);
+    if (!revisions) return [UNDEFINED, UNDEFINED];
+    let first = revisions[0];
+    if (first === NONEXISTENT) first = UNDEFINED;
+    let last = revisions[revisions.length - 1];
+    if (last === NONEXISTENT) last = UNDEFINED;
+    return [first as V, last as V];
+  }
+
+  public *diff(): IterableIterator<[K, V, V]> {
+    for (const [key, revisions] of this.map) {
+      let first = revisions[0];
+      let last = revisions[revisions.length - 1];
+      if (first === NONEXISTENT && last === NONEXISTENT) continue;
+      if (first === NONEXISTENT) first = UNDEFINED;
+      if (last === NONEXISTENT) last = UNDEFINED;
+      yield [key, first as V, last as V];
+    }
+  }
+
+  public collapse(revision: number): void {
+    if (this._sizeDiff.length <= revision) return;
+
+    this._sizeDiff[revision] = this._sizeDiff[this._sizeDiff.length - 1];
+    this._sizeDiff.splice(revision + 1, this._sizeDiff.length);
+
+    const d = this.dirty >> revision;
+    this.dirty = this.dirty ^ (d << revision);
+    this.dirty |= +!!d << revision;
+
+    for (const [k, v] of this.map) {
+      const l = v.length - 1;
+      if (l <= revision) continue;
+      const last = v[l];
+      v.splice(revision, l - revision);
+
+      if (last === NONEXISTENT && !v.some(vv => vv !== NONEXISTENT))
+        this.map.delete(k);
+    }
+  }
+}

lib/xml-parser.ts

@@ -0,0 +1,357 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+const CHAR_SINGLE_QUOTE = 39;
+const CHAR_DOUBLE_QUOTE = 34;
+const CHAR_LESS_THAN = 60;
+const CHAR_GREATER_THAN = 62;
+const CHAR_COLON = 58;
+const CHAR_SPACE = 32;
+const CHAR_TAB = 9;
+const CHAR_CR = 13;
+const CHAR_LF = 10;
+const CHAR_SLASH = 47;
+const CHAR_EXMARK = 33;
+const CHAR_QMARK = 63;
+const CHAR_EQUAL = 61;
+
+const STATE_LESS_THAN = 1;
+const STATE_SINGLE_QUOTE = 2;
+const STATE_DOUBLE_QUOTE = 3;
+
+export interface Attribute {
+  name: string;
+  namespace: string;
+  localName: string;
+  value: string;
+}
+
+export interface Element {
+  name: string;
+  namespace: string;
+  localName: string;
+  attrs: string;
+  text: string;
+  bodyIndex: number;
+  children: Element[];
+}
+
+export function parseXmlDeclaration(buffer: Buffer): Attribute[] {
+  for (const enc of ["utf16le", "utf8", "latin1", "ascii"]) {
+    let str = buffer.toString(enc, 0, 150);
+    if (str.startsWith("<?xml")) {
+      str = str.split("\n")[0].trim();
+      try {
+        return parseAttrs(str.slice(5, -2));
+      } catch (err) {
+        // Ignore
+      }
+    }
+  }
+  return null;
+}
+
+export function parseAttrs(string: string): Attribute[] {
+  const attrs: Attribute[] = [];
+  const len = string.length;
+
+  let state = 0;
+  let name = "";
+  let namespace = "";
+  let localName = "";
+  let idx = 0;
+  let colonIdx = 0;
+  for (let i = 0; i < len; ++i) {
+    const c = string.charCodeAt(i);
+    switch (c) {
+      case CHAR_SINGLE_QUOTE:
+      case CHAR_DOUBLE_QUOTE:
+        if (state === c) {
+          state = 0;
+          if (name) {
+            const value = string.slice(idx + 1, i);
+            const e = {
+              name: name,
+              namespace: namespace,
+              localName: localName,
+              value: value
+            };
+            attrs.push(e);
+            name = "";
+            idx = i + 1;
+          }
+        } else {
+          state = c;
+          idx = i;
+        }
+        continue;
+
+      case CHAR_COLON:
+        if (idx >= colonIdx) colonIdx = i;
+        continue;
+
+      case CHAR_EQUAL:
+        if (name) throw new Error(`Unexpected character at ${i}`);
+        name = string.slice(idx, i).trim();
+        // TODO validate name
+        if (colonIdx > idx) {
+          namespace = string.slice(idx, colonIdx).trim();
+          localName = string.slice(colonIdx + 1, i).trim();
+        } else {
+          namespace = "";
+          localName = name;
+        }
+    }
+  }
+
+  if (name) throw new Error(`Attribute must have value at ${idx}`);
+
+  const tail = string.slice(idx);
+  if (tail.trim()) throw new Error(`Unexpected string at ${len - tail.length}`);
+
+  return attrs;
+}
+
+export function decodeEntities(string): string {
+  return string.replace(/&[0-9a-z#]+;/gi, match => {
+    switch (match) {
+      case "&quot;":
+        return '"';
+
+      case "&amp;":
+        return "&";
+
+      case "&apos;":
+        return "'";
+
+      case "&lt;":
+        return "<";
+
+      case "&gt;":
+        return ">";
+
+      default:
+        if (match.startsWith("&#x")) {
+          const str = match.slice(3, -1).toLowerCase();
+          const n = parseInt(str, 16);
+          if (str.endsWith(n.toString(16))) return String.fromCharCode(n);
+        } else if (match.startsWith("&#")) {
+          const str = match.slice(2, -1);
+          const n = parseInt(str);
+          if (str.endsWith(n.toString())) return String.fromCharCode(n);
+        }
+    }
+    return match;
+  });
+}
+
+export function encodeEntities(string): string {
+  const entities = {
+    "&": "&amp;",
+    '"': "&quot;",
+    "'": "&apos;",
+    "<": "&lt;",
+    ">": "&gt;"
+  };
+  return string.replace(/[&"'<>]/g, m => entities[m]);
+}
+
+export function parseXml(string: string): Element {
+  const len = string.length;
+  let state1 = 0;
+  let state1Index = 0;
+  let state2 = 0;
+  let state2Index = 0;
+
+  const root: Element = {
+    name: "root",
+    namespace: "",
+    localName: "root",
+    attrs: "",
+    text: "",
+    bodyIndex: 0,
+    children: []
+  };
+
+  const stack: Element[] = [root];
+
+  for (let i = 0; i < len; ++i) {
+    switch (string.charCodeAt(i)) {
+      case CHAR_SINGLE_QUOTE:
+        switch (state1 & 0xff) {
+          case STATE_SINGLE_QUOTE:
+            state1 = state2;
+            state1Index = state2Index;
+            state2 = 0;
+            continue;
+
+          case STATE_LESS_THAN:
+            state2 = state1;
+            state2Index = state1Index;
+            state1 = STATE_SINGLE_QUOTE;
+            state1Index = i;
+            continue;
+        }
+        continue;
+
+      case CHAR_DOUBLE_QUOTE:
+        switch (state1 & 0xff) {
+          case STATE_DOUBLE_QUOTE:
+            state1 = state2;
+            state1Index = state2Index;
+            state2 = 0;
+            continue;
+
+          case STATE_LESS_THAN:
+            state2 = state1;
+            state2Index = state1Index;
+            state1 = STATE_DOUBLE_QUOTE;
+            state1Index = i;
+            continue;
+        }
+        continue;
+
+      case CHAR_LESS_THAN:
+        if ((state1 & 0xff) === 0) {
+          state2 = state1;
+          state2Index = state1Index;
+          state1 = STATE_LESS_THAN;
+          state1Index = i;
+        }
+        continue;
+
+      case CHAR_COLON:
+        if ((state1 & 0xff) === STATE_LESS_THAN) {
+          const colonIndex = (state1 >> 8) & 0xff;
+          if (colonIndex === 0) state1 ^= ((i - state1Index) & 0xff) << 8;
+        }
+        continue;
+
+      case CHAR_SPACE:
+      case CHAR_TAB:
+      case CHAR_CR:
+      case CHAR_LF:
+        if ((state1 & 0xff) === STATE_LESS_THAN) {
+          const wsIndex = (state1 >> 16) & 0xff;
+          if (wsIndex === 0) state1 ^= ((i - state1Index) & 0xff) << 16;
+        }
+        continue;
+
+      case CHAR_GREATER_THAN:
+        if ((state1 & 0xff) === STATE_LESS_THAN) {
+          const secondChar = string.charCodeAt(state1Index + 1);
+          const wsIndex: number = (state1 >> 16) & 0xff;
+          let name: string,
+            colonIndex: number,
+            e: Element,
+            parent: Element,
+            selfClosing: number,
+            localName: string,
+            namespace: string;
+
+          switch (secondChar) {
+            case CHAR_SLASH:
+              e = stack.pop();
+              name =
+                wsIndex === 0
+                  ? string.slice(state1Index + 2, i)
+                  : string.slice(state1Index + 2, state1Index + wsIndex);
+              if (e.name !== name)
+                throw new Error(`Unmatched closing tag at ${i}`);
+              if (!e.children.length)
+                e.text = string.slice(e.bodyIndex, state1Index);
+              state1 = state2;
+              state1Index = state2Index;
+              state2 = 0;
+              continue;
+
+            case CHAR_EXMARK:
+              if (string.startsWith("![CDATA[", state1Index + 1)) {
+                if (string.endsWith("]]", i))
+                  throw new Error(`CDATA nodes are not supported at ${i}`);
+              } else if (string.startsWith("!--", state1Index + 1)) {
+                // Comment node, ignore
+                if (string.endsWith("--", i)) {
+                  state1 = state2;
+                  state1Index = state2Index;
+                  state2 = 0;
+                }
+              }
+              continue;
+
+            case CHAR_QMARK:
+              if (string.charCodeAt(i - 1) === CHAR_QMARK) {
+                // XML declaration node, ignore
+                state1 = state2;
+                state1Index = state2Index;
+                state2 = 0;
+              }
+              continue;
+
+            default:
+              selfClosing = +(string.charCodeAt(i - 1) === CHAR_SLASH);
+              parent = stack[stack.length - 1];
+              colonIndex = (state1 >> 8) & 0xff;
+
+              name =
+                wsIndex === 0
+                  ? string.slice(state1Index + 1, i - selfClosing)
+                  : string.slice(state1Index + 1, state1Index + wsIndex);
+              if (colonIndex && (!wsIndex || colonIndex < wsIndex)) {
+                localName = name.slice(colonIndex);
+                namespace = name.slice(0, colonIndex - 1);
+              } else {
+                localName = name;
+                namespace = "";
+              }
+
+              e = {
+                name: name,
+                namespace: namespace,
+                localName: localName,
+                attrs: wsIndex
+                  ? string.slice(state1Index + wsIndex + 1, i - selfClosing)
+                  : "",
+                text: "",
+                bodyIndex: i + 1,
+                children: []
+              };
+              parent.children.push(e);
+              if (!selfClosing) stack.push(e);
+
+              state1 = state2;
+              state1Index = state2Index;
+              state2 = 0;
+              continue;
+          }
+        }
+        continue;
+    }
+  }
+
+  if (state1) throw new Error(`Unclosed token at ${state1Index}`);
+
+  if (stack.length > 1) {
+    const e = stack[stack.length - 1];
+    throw new Error(`Unclosed XML element at ${e.bodyIndex}`);
+  }
+
+  if (!root.children.length) root.text = string;
+  return root;
+}

package.json

@@ -1,41 +1,87 @@
 {
   "name": "genieacs",
-  "version": "1.0.0",
-  "description": "A blazing fast TR-069 auto configuration server (ACS) built with Node.js, Redis, and MongoDB",
+  "version": "1.2.0-beta.0",
+  "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
-    "url": "https://github.com/zaidka/genieacs.git"
+    "url": "https://github.com/genieacs/genieacs.git"
   },
+  "homepage": "https://genieacs.com",
   "keywords": [
     "TR-069",
     "CWMP",
     "ACS"
   ],
-  "author": "Zaid Abdulla",
-  "license": "AGPL3",
-  "readmeFilename": "README.md",
+  "author": "Zaid Abdulla <zaid@genieacs.com>",
+  "license": "AGPL-3.0",
   "preferGlobal": true,
   "directories": {
-    "test": "test",
-    "bin": "bin"
+    "bin": "bin",
+    "tools": "tools"
   },
   "dependencies": {
-    "libxmljs": "~0.11.0",
-    "redis": "~0.12.1",
-    "hiredis": "~0.1.17",
-    "mongodb": "~1.4.8"
+    "jsonwebtoken": "^8.5.1",
+    "koa": "^2.7.0",
+    "koa-bodyparser": "^4.2.1",
+    "koa-compress": "^3.0.0",
+    "koa-jwt": "^3.6.0",
+    "koa-router": "^7.4.0",
+    "koa-static": "^5.0.0",
+    "later": "^1.2.0",
+    "mongodb": "^3.2.6",
+    "parsimmon": "^1.13.0",
+    "seedrandom": "^3.0.1",
+    "yaml": "^1.6.0"
+  },
+  "optionalDependencies": {
+    "libxmljs": "^0.19.5"
   },
   "devDependencies": {
-    "coffee-script": "~1.7.1"
+    "@types/jsonwebtoken": "^8.3.2",
+    "@types/koa": "^2.0.49",
+    "@types/koa-compress": "^2.0.9",
+    "@types/koa-router": "^7.0.42",
+    "@types/koa-static": "^4.0.1",
+    "@types/libxmljs": "^0.18.3",
+    "@types/mithril": "^2.0.0",
+    "@types/mongodb": "^3.1.31",
+    "@types/parsimmon": "^1.10.0",
+    "@types/seedrandom": "^2.4.28",
+    "@types/svgo": "^1.2.1",
+    "@typescript-eslint/eslint-plugin": "^1.13.0",
+    "@typescript-eslint/parser": "^1.13.0",
+    "ava": "^2.2.0",
+    "codemirror": "^5.48.2",
+    "cssnano": "^4.1.10",
+    "eslint": "^6.1.0",
+    "eslint-config-prettier": "^6.0.0",
+    "esm": "^3.2.25",
+    "mithril": "^2.0.3",
+    "postcss": "^7.0.17",
+    "postcss-import": "^12.0.1",
+    "postcss-preset-env": "^6.7.0",
+    "prettier": "^1.18.2",
+    "rollup": "^1.17.0",
+    "rollup-plugin-json": "^4.0.0",
+    "rollup-plugin-replace": "^2.2.0",
+    "rollup-plugin-terser": "^5.1.1",
+    "rollup-plugin-typescript": "^1.0.1",
+    "source-map-loader": "^0.2.4",
+    "svgo": "^1.3.0",
+    "ts-node": "^8.3.0",
+    "typescript": "^3.5.3",
+    "webpack": "^4.38.0"
   },
   "engines": {
-    "node": ">=0.10.0 <0.12"
+    "node": ">=8.14.0"
   },
   "scripts": {
-    "prepublish": "npm run compile",
-    "compile": "coffee -cb lib/*.coffee test/*.coffee tools/*.coffee",
-    "configure": "cp ./config/config-sample.json ./config/config.json; cp ./config/parameters-sample.json ./config/parameters.json; cp ./config/custom_commands-sample.json ./config/custom_commands.json; cp ./config/auth-sample.js ./config/auth.js",
-    "install": "npm run configure",
-    "test": "node test"
+    "test": "ava --verbose --color",
+    "lint-ln": "find bin -type f -exec sh -c 'ln -fs \"$(sh -c pwd)/$1\" \"bin/.${1#bin/}.ts\"' _ {} \\;; find tools -type f -exec sh -c 'ln -fs \"$(sh -c pwd)/$1\" \"tools/.${1#tools/}.ts\"' _ {} \\;",
+    "lint-eslint": "eslint --fix \"bin/**/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/**/*.ts\"",
+    "lint-prettier": "prettier --parser typescript --write \"bin/**/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/**/*.ts\"",
+    "lint-tsc": "tsc --noEmit",
+    "lint": "npm run lint-ln && npm run lint-eslint && npm run lint-prettier && npm run lint-tsc",
+    "build": "node -r esm -r ts-node/register/transpile-only build/build"
   }
 }

public/logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="310" height="64" viewBox="0 0 82.020832 16.933334"><defs><linearGradient id="b"><stop offset="0" stop-color="#737373"/><stop offset="1" stop-color="#4b4b4b"/></linearGradient><linearGradient id="a"><stop offset="0" stop-color="#b72f5f"/><stop offset="1" stop-color="#872346"/></linearGradient><linearGradient xlink:href="#a" id="c" gradientUnits="userSpaceOnUse" gradientTransform="translate(173.233 -461.003) scale(.21343)" x1="16.045" y1="132.803" x2="16.045" y2="-81.164"/><linearGradient xlink:href="#a" id="d" gradientUnits="userSpaceOnUse" x1="46.459" y1="281.163" x2="46.459" y2="296.66" gradientTransform="translate(1.09 16.9) scale(.94143)"/><linearGradient xlink:href="#b" id="e" gradientUnits="userSpaceOnUse" gradientTransform="translate(11.105 -994.648) scale(3.55816)" x1="46.459" y1="281.163" x2="46.459" y2="296.66"/></defs><path d="M173.233-461.003l1.953-3.714c-.868-1.509-2.042-2.814-3.522-3.917-2.419-1.43-4.386-1.199-5.901.694 2.931 1.323 5.421 3.635 7.47 6.937m-7.577 11.151a6.095 6.095 0 0 0 .673-3.49 23.76 23.76 0 0 0-1.473-6.285 20.998 20.998 0 0 1-1.206-4.855 6.766 6.766 0 0 1 .48-3.426c.463-1.117 1.298-2.074 2.508-2.87-1.302-.392-2.44-.499-3.415-.32-1.707.326-2.82 1.354-3.34 3.083a10.152 10.152 0 0 0-.267 5.08 33.38 33.38 0 0 0 1.238 4.674 32.993 32.993 0 0 1 1.227 4.856c.107.626.118 1.277.032 1.953a4.72 4.72 0 0 1-.779 2.006 4.498 4.498 0 0 1-1.707 1.44c.697.129 1.387.235 2.07.32a4.124 4.124 0 0 0 2.07-.351c.84-.37 1.47-.975 1.89-1.815m22.463 5.123l.63-.62-5.25-4.77 1.579-1.579 4.77 5.25.619-.629-5.24-4.77 1.483-1.483 4.77 5.24.63-.62-5.25-4.77 1.58-1.58 5.015 5.016 1.067-1.045-4.503-4.504h.01l-5.87-5.859-3.563-1.462-5.102 3.831-.02.022-.022-.022.01-.02 3.394-4.526-1.131-.458-4.973 4.973 1.26 3.073 4.652-3.49.01-.01.033.02-.01.012-3.95 5.26.64 1.57 4.814 4.802h-.01l5.559 5.549 1.057-1.046-5.027-5.026 1.569-1.569 4.77 5.24m4.14-28.173c4.405 4.403 6.607 9.722 6.607 15.954 0 6.218-2.202 11.536-6.606 15.954-4.418 4.404-9.736 6.606-15.954 6.606-6.233 0-11.55-2.202-15.955-6.606-2.703-2.71-4.574-5.756-5.613-9.135l.448.245c2.241 1.181 4.226.577 5.955-1.814-1.565-.32-2.82-.658-3.767-1.014a5.467 5.467 0 0 1-1.43-.821c-.96-.762-1.565-1.73-1.814-2.903a11.315 11.315 0 0 1 .032-5.005c.81-4.34 2.874-8.16 6.19-11.461 4.403-4.411 9.721-6.617 15.954-6.617 6.225 0 11.543 2.206 15.954 6.617" fill="url(#c)" transform="matrix(.35176 0 0 -.35176 -53.595 -152.27)"/><path style="line-height:6.61458302px;-inkscape-font-specification:Norwegian" d="M47.104 281.834c-.456 0-.816.388-.816.849 0 .46.355.816.816.816a.82.82 0 0 0 .833-.816.85.85 0 0 0-.833-.849zm-21.502 1.404c-.205 0-.382.046-.5.13-.118.085-.187.201-.187.4v.981l-.151-.031a9.89 9.89 0 0 0-1.872-.173c-1.288 0-2.148.066-2.657.374a1.264 1.264 0 0 0-.548.676c-.119.313-.177.73-.177 1.28v6.123c0 .982.25 1.508.787 1.824.539.315 1.407.41 2.628.41 1.308 0 2.178-.13 2.701-.484.524-.354.745-.932.745-1.942 0-.821-.147-1.37-.543-1.722-.395-.352-1.075-.528-2.186-.528h-2.787v-1.234l.148.03c.478.092 1.094.11 1.89.11 1.292 0 2.148-.112 2.658-.453.51-.34.724-.915.724-1.958v-3.284c0-.198-.07-.314-.186-.399a.843.843 0 0 0-.487-.13zm6.112 1.29c-1.226 0-2.038.137-2.526.495-.488.357-.697.945-.697 1.947v2.84c0 .935.23 1.54.743 1.932.512.39 1.337.571 2.544.571 1.392 0 2.17-.058 2.567-.196.199-.069.298-.15.355-.246.058-.097.078-.227.078-.405 0-.208-.04-.378-.117-.485-.076-.108-.186-.171-.396-.171-.794 0-1.057.19-2.408.19-.593 0-1.06-.078-1.395-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.717.717 0 0 0 .539-.213c.13-.138.213-.342.213-.62v-1.212c0-1.037-.179-1.623-.654-1.97-.476-.348-1.3-.472-2.617-.472zm8.83 0c-1.303 0-2.117.086-2.59.49-.472.403-.665 1.168-.665 2.623v4.002c0 .215.074.338.197.426.123.088.31.134.524.134.223 0 .412-.046.537-.134.125-.088.198-.211.198-.426v-3.81c0-.424.006-.754.04-1.019.036-.265.102-.47.236-.623.135-.152.328-.236.574-.28.246-.044.552-.053.949-.053.4 0 .712.01.963.053.25.044.447.126.587.278.139.151.211.358.25.623.038.265.046.597.046 1.02v3.81c0 .217.07.34.19.427.122.088.308.134.53.134.215 0 .401-.046.525-.134.123-.088.196-.211.196-.426v-4.002c0-1.455-.201-2.219-.682-2.623-.48-.404-1.303-.49-2.605-.49zm12.996 0c-1.227 0-2.038.137-2.527.495-.488.357-.696.945-.696 1.947v2.84c0 .935.23 1.54.742 1.932.513.39 1.338.571 2.545.571 1.392 0 2.17-.058 2.567-.196.2-.069.298-.15.355-.246.058-.097.079-.227.079-.405 0-.208-.04-.378-.117-.485-.077-.108-.188-.171-.398-.171-.793 0-1.056.19-2.406.19-.594 0-1.062-.078-1.396-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.716.716 0 0 0 .539-.213c.13-.138.213-.342.213-.62v-1.212c0-1.037-.18-1.623-.655-1.97-.475-.348-1.3-.472-2.616-.472zm-6.436.13c-.212 0-.398.047-.522.14-.124.09-.198.22-.198.435v6.41c0 .215.073.338.196.426.123.088.31.134.524.134.223 0 .414-.046.54-.134.124-.088.198-.211.198-.426v-6.41c0-.215-.075-.342-.201-.434-.126-.092-.316-.142-.537-.142zm-24.212 1.135c.662 0 1.4.065 1.918.146l.105.016v.92c0 .246-.01.453-.058.632a.757.757 0 0 1-Line truncated