Include notification and accessList in CSV

I omitted the "/" on the initial commit for the SPAWN error on extensions

bin/genieacs-cwmp

@@ -35,10 +35,13 @@ const SERVICE_ADDRESS = config.get("CWMP_INTERFACE");
 const SERVICE_PORT = config.get("CWMP_PORT");
 
 function exitWorkerGracefully(): void {
-  db.disconnect();
-  cache.disconnect();
-  extensions.killAll();
-  cluster.worker.disconnect();
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    cache.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect()
+  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
@@ -81,8 +84,6 @@ if (!cluster.worker) {
     cert: config.get("CWMP_SSL_CERT")
   };
 
-  const keepAliveTimeout = config.get("CWMP_KEEP_ALIVE_TIMEOUT") as number;
-
   process.on("uncaughtException", err => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
@@ -96,21 +97,7 @@ if (!cluster.worker) {
       .catch(exitWorkerUngracefully);
   });
 
-  process.on("SIGINT", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  Promise.all([db.connect(), cache.connect()])
+  const initPromise = Promise.all([db.connect(), cache.connect()])
     .then(() => {
       server.start(
         SERVICE_PORT,
@@ -118,7 +105,7 @@ if (!cluster.worker) {
         ssl,
         cwmp.listener,
         cwmp.onConnection,
-        keepAliveTimeout
+        0
       );
     })
     .catch(err => {
@@ -126,4 +113,22 @@ if (!cluster.worker) {
         throw err;
       });
     });
+
+  process.on("SIGINT", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
 }

bin/genieacs-fs

@@ -34,9 +34,12 @@ const SERVICE_ADDRESS = config.get("FS_INTERFACE");
 const SERVICE_PORT = config.get("FS_PORT");
 
 function exitWorkerGracefully(): void {
-  db.disconnect();
-  cache.disconnect();
-  cluster.worker.disconnect();
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    cache.disconnect(),
+    cluster.worker.disconnect()
+  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
@@ -90,21 +93,7 @@ if (!cluster.worker) {
       .catch(exitWorkerUngracefully);
   });
 
-  process.on("SIGINT", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  Promise.all([db.connect(), cache.connect()])
+  const initPromise = Promise.all([db.connect(), cache.connect()])
     .then(() => {
       server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
     })
@@ -113,4 +102,22 @@ if (!cluster.worker) {
         throw err;
       });
     });
+
+  process.on("SIGINT", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
 }

bin/genieacs-nbi

@@ -35,10 +35,13 @@ const SERVICE_ADDRESS = config.get("NBI_INTERFACE");
 const SERVICE_PORT = config.get("NBI_PORT");
 
 function exitWorkerGracefully(): void {
-  db.disconnect();
-  cache.disconnect();
-  extensions.killAll();
-  cluster.worker.disconnect();
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    cache.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect()
+  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
@@ -94,21 +97,7 @@ if (!cluster.worker) {
       .catch(exitWorkerUngracefully);
   });
 
-  process.on("SIGINT", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  Promise.all([db.connect(), cache.connect()])
+  const initPromise = Promise.all([db.connect(), cache.connect()])
     .then(() => {
       server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
     })
@@ -117,4 +106,22 @@ if (!cluster.worker) {
         throw err;
       });
     });
+
+  process.on("SIGINT", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
 }

bin/genieacs-ui

@@ -36,11 +36,14 @@ const SERVICE_ADDRESS = config.get("UI_INTERFACE");
 const SERVICE_PORT = config.get("UI_PORT");
 
 function exitWorkerGracefully(): void {
-  db.disconnect();
-  db2.disconnect();
-  cache.disconnect();
-  extensions.killAll();
-  cluster.worker.disconnect();
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    db2.disconnect(),
+    cache.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect()
+  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
@@ -99,28 +102,12 @@ if (!cluster.worker) {
       .catch(exitWorkerUngracefully);
   });
 
-  process.on("SIGINT", () => {
-    stopping = true;
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    stopping = true;
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
   const _listener = (req, res): void => {
     if (stopping) res.setHeader("Connection", "close");
     listener(req, res);
   };
 
-  Promise.all([db2.connect(), cache.connect()])
+  const initPromise = Promise.all([db2.connect(), cache.connect()])
     .then(() => {
       server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
     })
@@ -129,4 +116,24 @@ if (!cluster.worker) {
         throw err;
       });
     });
+
+  process.on("SIGINT", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
 }

build/build.ts

@@ -64,6 +64,7 @@ const externals = [
   "child_process",
   "dgram",
   "url",
+  "iconv-lite",
   "koa",
   "koa-router",
   "koa-compress",

docs/administration-faq.rst

@@ -0,0 +1,64 @@
+.. _administration-faq:
+
+Administration FAQ
+==================
+
+.. _administration-faq-duplicate-log-entries:
+
+Duplicate log entries when using :func:`log()` function
+-------------------------------------------------------
+
+Because GenieACS uses a full fledged scripting language for device
+configuration, the only way to guarantee that it has satisfied the 'desired
+state' is by repeatedly executing the script until there's no more
+discrepancies with the current device state. Though it may seem like this will
+cause duplicate requests going to the device, this isn't actually the case
+because device configuration are stated declaratively and that the scripts
+themselves are pure functions in the context of a session (e.g. Date.now()
+always returns the same value within the session).
+
+To illustrate with an example, consider the following script:
+
+.. code:: javascript
+
+  log("Executing script");
+  declare("Device.param", null, {value: 1});
+  commit();
+  declare("Device.param", null, {value: 2});
+
+This will set the value of the 'Device.param' to 1, then to 2. Then as the
+script is run again the value is set back to 1 and so on. A stable state will
+never be reached so GenieACS will execute the script a few times until it gives
+up and throws a fault. This is an edge case that should be avoided. A more
+typical case is where the script is run once or twice. Essentially if an
+execution doesn't result in any request to the CPE or a change in the data
+model then a stable state is deemed to have been reached.
+
+Configurations not pushed to device after factory reset
+---------------------------------------------------------
+
+After a device is reset to its factory default state, the cached data model in
+GenieACS's database needs to be invalidated to force rediscovery. Ensure the
+following lines are called on ``0 BOOTSTRAP`` event:
+
+.. code:: javascript
+
+  const now = Date.now();
+
+  // Clear cached data model to force a refresh
+  clear("Device", now);
+  clear("InternetGatewayDevice", now);
+
+
+Most device parameters are missing
+----------------------------------
+
+For performance reasons (server, client, and network), GenieACS by default only
+fetches parts of the data model that are necessary to satisfy the declarations
+in your provision scripts. Create declarations for any parameters you need
+fetched by default.
+
+If you're unsure and want to explore the available parameters exposed by the
+device, refresh the root parameter (e.g. ``InternetGatewayDevice``) from
+GenieACS's UI. You typically only need to do that one time for a given CPE
+model.

docs/api-reference.rst

@@ -415,6 +415,15 @@ parameterValues key. For example:
   -X POST \
   --data '{"name":"addObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection"}'
 
+``deleteObject``
+~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name":"deleteObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection.1"}'
+
 ``reboot``
 ~~~~~~~~~~
 

docs/environment-variables.rst

@@ -3,4 +3,214 @@
 Environment Variables
 =====================
 
-TODO
+Configuring GenieACS services can be done through the following environment
+variables:
+
+.. attention::
+
+  All GenieACS environment variables must be prefixed with ``GENIEACS_``.
+
+MONGODB_CONNECTION_URL
+  MongoDB connection string.
+
+  Default: ``mongodb://127.0.0.1/genieacs``
+
+EXT_DIR
+  The directory from which to look up extension scripts.
+
+  Default: ``<installation dir>/config/ext``
+
+EXT_TIMEOUT
+  Timeout (in milliseconds) to allow for calls to extensions to return a
+  response.
+
+  Default: ``3000``
+
+DEBUG_FILE
+  File to dump CPE debug log.
+
+  Default: unset
+
+DEBUG_FORMAT
+  Debug log format. Valid values are 'yaml' and 'json'.
+
+  Default: ``yaml``
+
+LOG_FORMAT
+  The format used for the log entries in ``CWMP_LOG_FILE``, ``NBI_LOG_FILE``,
+  ``FS_LOG_FILE``, and ``UI_LOG_FILE``. Possible values are ``simple`` and
+  ``json``.
+
+  Default: ``simple``
+
+ACCESS_LOG_FORMAT
+  The format used for the log entries in ``CWMP_ACCESS_LOG_FILE``,
+  ``NBI_ACCESS_LOG_FILE``, ``FS_ACCESS_LOG_FILE``, and ``UI_ACCESS_LOG_FILE``.
+  Possible values are ``simple`` and ``json``.
+
+  Default: ``simple``
+
+CWMP_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-cwmp. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+CWMP_PORT
+  The TCP port that genieacs-cwmp listens on.
+
+  Default: ``7547``
+
+CWMP_INTERFACE
+  The network interface that genieacs-cwmp binds to.
+
+  Default: ``0.0.0.0``
+
+CWMP_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+CWMP_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+CWMP_LOG_FILE
+  File to log process related events for genieacs-cwmp. If omitted, logs will
+  go to stderr.
+
+  Default: unset
+
+CWMP_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-cwmp. If omitted, logs will go to
+  stdout.
+
+  Default: unest
+
+NBI_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-nbi. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+NBI_PORT
+  The TCP port that genieacs-nbi listens on.
+
+  Default: ``7557``
+
+NBI_INTERFACE
+  The network interface that genieacs-nbi binds to.
+
+  Default: ``0.0.0.0``
+
+NBI_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unest
+
+NBI_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+NBI_LOG_FILE
+  File to log process related events for genieacs-nbi. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+NBI_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-nbi. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+FS_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-fs. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+FS_PORT
+  The TCP port that genieacs-fs listens on.
+
+  Default: ``7567``
+
+FS_INTERFACE
+  The network interface that genieacs-fs binds to.
+
+  Default: ``0.0.0.0``
+
+FS_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+FS_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+FS_LOG_FILE
+  File to log process related events for genieacs-fs. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+FS_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-fs. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+FS_HOSTNAME
+  The hostname to use when generating the file URL for TR-069 Download
+  requests. Set this if the hostname where genieacs-fs is listening differs
+  from the hostname in the ACS URL.
+
+  Default: the hostname from the ACS URL
+
+UI_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-ui. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+UI_PORT
+  The TCP port that genieacs-ui listens on.
+
+  Default: ``3000``
+
+UI_INTERFACE
+  The network interface that genieacs-ui binds to.
+
+  Default: ``0.0.0.0``
+
+UI_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+UI_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+UI_LOG_FILE
+  File to log process related events for genieacs-ui. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+UI_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-ui. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+UI_JWT_SECRET
+  The key used for signing JWT tokens that are stored in browser cookies. The
+  string can be up to 64 characters in length.
+
+  Default: unset

docs/index.rst

@@ -30,6 +30,7 @@ Welcome to GenieACS's documentation!
 
   provisions
   virtual-parameters
+  administration-faq
 
 .. raw:: latex
 

docs/installation-guide.rst

@@ -34,7 +34,7 @@ Install GenieACS
 
   .. parsed-literal::
 
-    sudo npm install -g genieacs@\ |release|
+    sudo npm install -g --unsafe-perm genieacs@\ |release|
 
 .. topic:: Installing from source
 
@@ -68,11 +68,12 @@ Configure systemd
 
   .. code:: bash
 
-    GENIEACS_CWMP_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-cmwp-access.log
+    GENIEACS_CWMP_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-cwmp-access.log
     GENIEACS_NBI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-nbi-access.log
     GENIEACS_FS_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-fs-access.log
     GENIEACS_UI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-ui-access.log
     GENIEACS_DEBUG_FILE=/var/log/genieacs/genieacs-debug.yaml
+    GENIEACS_EXT_DIR=/opt/genieacs/ext
     GENIEACS_UI_JWT_SECRET=secret
   
   Set file ownership and permissions:
@@ -100,11 +101,16 @@ Configure systemd
   :file:`/var/log/genieacs/` while process logs go to *journald*. Use
   ``journalctl`` command to view process logs.
 
+  .. attention::
+
+    If the command :command:`systemctl edit --force --full` fails, you can
+    create the unit file manually.
+
   1. Run the following command to create ``genieacs-cwmp`` service:
   
     .. code:: bash
 
-      sudo systemctl edit --force genieacs-cwmp
+      sudo systemctl edit --force --full genieacs-cwmp
     
     Then paste the following in the editor and save:
 
@@ -117,8 +123,7 @@ Configure systemd
       [Service]
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=genieacs-cwmp
-      KillMode=process
+      ExecStart=/usr/bin/genieacs-cwmp
 
       [Install]
       WantedBy=default.target
@@ -127,7 +132,7 @@ Configure systemd
   
     .. code:: bash
 
-      sudo systemctl edit --force genieacs-nbi
+      sudo systemctl edit --force --full genieacs-nbi
     
     Then paste the following in the editor and save:
 
@@ -140,8 +145,7 @@ Configure systemd
       [Service]
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=genieacs-nbi
-      KillMode=process
+      ExecStart=/usr/bin/genieacs-nbi
 
       [Install]
       WantedBy=default.target
@@ -150,7 +154,7 @@ Configure systemd
   
     .. code:: bash
 
-      sudo systemctl edit --force genieacs-fs
+      sudo systemctl edit --force --full genieacs-fs
     
     Then paste the following in the editor and save:
 
@@ -163,8 +167,7 @@ Configure systemd
       [Service]
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=genieacs-fs
-      KillMode=process
+      ExecStart=/usr/bin/genieacs-fs
 
       [Install]
       WantedBy=default.target
@@ -173,7 +176,7 @@ Configure systemd
   
     .. code:: bash
 
-      sudo systemctl edit --force genieacs-ui
+      sudo systemctl edit --force --full genieacs-ui
     
     Then paste the following in the editor and save:
 
@@ -186,19 +189,18 @@ Configure systemd
       [Service]
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=genieacs-ui
-      KillMode=process
+      ExecStart=/usr/bin/genieacs-ui
 
       [Install]
       WantedBy=default.target
 
 .. topic:: Configure log file rotation using logrotate
 
-  Save the following under :file:`/etc/logrotate.d/genieacs`
+  Save the following as :file:`/etc/logrotate.d/genieacs`
 
   .. code::
   
-    /var/log/genieacs/*.{log,yaml} {
+    /var/log/genieacs/*.log /var/log/genieacs/*.yaml {
         daily
         rotate 30
         compress

docs/provisions.rst

@@ -128,6 +128,14 @@ script filename while second argument is the function name within that script.
 Any remaining arguments will be passed to that function. See :ref:`extensions`
 for more details.
 
+``log(message)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Prints out a string in genieacs-cwmp's access log. It's meant to be used for
+debugging. Note that you may see multiple log entries as the script can be
+executed multiple times in a session. See :ref:`this FAQ
+<administration-faq-duplicate-log-entries>`.
+
 .. _path-format:
 
 Path format

lib/api-functions.ts

@@ -88,7 +88,7 @@ export async function connectionRequest(deviceId): Promise<void> {
   const now = Date.now();
   const UDP_CONNECTION_REQUEST_PORT = +getConfig(
     snapshot,
-    "cwmp.udpConnectionRequestAuth",
+    "cwmp.udpConnectionRequestPort",
     context,
     now
   );
@@ -226,11 +226,14 @@ function sanitizeTask(task): void {
       break;
 
     case "download":
+      // genieacs-gui sends file ID instead of fileName and fileType
+      if (!task.file) {
         if (typeof task.fileType !== "string" || !task.fileType.length)
           throw new Error("Missing 'fileType' property");
 
         if (typeof task.fileName !== "string" || !task.fileName.length)
           throw new Error("Missing 'fileName' property");
+      }
 
       if (
         task.targetFileName != null &&
@@ -238,6 +241,18 @@ function sanitizeTask(task): void {
       )
         throw new Error("Invalid 'targetFileName' property");
       break;
+
+    case "provisions":
+      if (
+        !Array.isArray(task.provisions) ||
+        !task.provisions.every(arr =>
+          arr.every(
+            s => s == null || ["boolean", "number", "string"].includes(typeof s)
+          )
+        )
+      )
+        throw new Error("Invalid 'provisions' property");
+      break;
   }
 
   return task;

lib/cache.ts

@@ -22,16 +22,16 @@ import * as config from "./config";
 
 const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
 
-let mongoClient: MongoClient;
+let clientPromise: Promise<MongoClient>;
 let mongoCollection: Collection;
 let mongoTimeOffset = 0;
 
 export async function connect(): Promise<void> {
   const MONGODB_CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
-  mongoClient = await MongoClient.connect(MONGODB_CONNECTION_URL, {
+  clientPromise = MongoClient.connect(MONGODB_CONNECTION_URL, {
     useNewUrlParser: true
   });
-  const db = mongoClient.db();
+  const db = (await clientPromise).db();
   mongoCollection = db.collection("cache");
   await mongoCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
   const now = Date.now();
@@ -40,7 +40,7 @@ export async function connect(): Promise<void> {
 }
 
 export async function disconnect(): Promise<void> {
-  if (mongoClient) await mongoClient.close();
+  if (clientPromise) await (await clientPromise).close();
 }
 
 export async function get(key): Promise<any> {
@@ -62,7 +62,7 @@ export async function get(key): Promise<any> {
   } else {
     const res = await mongoCollection.findOne({ _id: { $in: [key] } });
     if (res && res["expire"] > expire) return res["value"];
-    return res;
+    return null;
   }
 }
 

lib/cluster.ts

@@ -22,7 +22,19 @@ import { cpus } from "os";
 import * as logger from "./logger";
 
 let respawnTimestamp = 0;
-let tooManyCrashesTimestamp = 0;
+let crashes: number[] = [];
+
+function fork(): cluster.Worker {
+  const w = cluster.fork();
+  w.on("error", (err: NodeJS.ErrnoException) => {
+    // Avoid exception when attempting to kill the process just as it's exiting
+    if (err.code !== "EPIPE") throw err;
+    setTimeout(() => {
+      if (!w.isDead()) throw err;
+    }, 50);
+  });
+  return w;
+}
 
 function restartWorker(worker, code, signal): void {
   const msg = {
@@ -39,14 +51,21 @@ function restartWorker(worker, code, signal): void {
   logger.error(msg);
 
   const now = Date.now();
-  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
-  if (respawnTimestamp === now) {
-    tooManyCrashesTimestamp = now;
-    cluster.fork();
-    return;
-  }
+  crashes.push(now);
 
-  if (now - tooManyCrashesTimestamp > 60000) {
+  let min1 = 0,
+    min2 = 0,
+    min3 = 0;
+
+  crashes = crashes.filter(n => {
+    if (n > now - 60000) ++min1;
+    else if (n > now - 120000) ++min2;
+    else if (n > now - 180000) ++min3;
+    else return false;
+    return true;
+  });
+
+  if (min1 > 5 && min2 > 5 && min3 > 5) {
     process.exitCode = 1;
     cluster.removeListener("exit", restartWorker);
     for (const pid in cluster.workers) cluster.workers[pid].kill();
@@ -58,9 +77,15 @@ function restartWorker(worker, code, signal): void {
     return;
   }
 
+  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  if (respawnTimestamp === now) {
+    fork();
+    return;
+  }
+
   setTimeout(() => {
     if (process.exitCode) return;
-    cluster.fork();
+    fork();
   }, respawnTimestamp - now);
 }
 
@@ -84,7 +109,7 @@ export function start(workerCount, servicePort, serviceAddress): void {
 
   if (!workerCount) workerCount = Math.max(2, cpus().length);
 
-  for (let i = 0; i < workerCount; ++i) cluster.fork();
+  for (let i = 0; i < workerCount; ++i) fork();
 }
 
 export function stop(): void {

lib/common/authorizer.ts

@@ -18,7 +18,7 @@
  */
 
 import { PermissionSet, Expression } from "../types";
-import { evaluate } from "./expression";
+import { evaluate, or } from "./expression";
 
 export default class Authorizer {
   private permissionSets: PermissionSet[];
@@ -27,14 +27,16 @@ export default class Authorizer {
     (mutationType, mutation, any) => boolean
   >;
   private hasAccessCache: Map<string, boolean>;
+  private getFilterCache: Map<string, Expression>;
 
   public constructor(permissionSets) {
     this.permissionSets = permissionSets;
     this.validatorCache = new WeakMap();
     this.hasAccessCache = new Map();
+    this.getFilterCache = new Map();
   }
 
-  public hasAccess(resourceType, access): boolean {
+  public hasAccess(resourceType: string, access: number): boolean {
     const cacheKey = `${resourceType}-${access}`;
     if (this.hasAccessCache.has(cacheKey))
       return this.hasAccessCache.get(cacheKey);
@@ -55,10 +57,29 @@ export default class Authorizer {
     return has;
   }
 
+  public getFilter(resourceType: string, access: number): Expression {
+    const cacheKey = `${resourceType}-${access}`;
+    if (this.getFilterCache.has(cacheKey))
+      return this.getFilterCache.get(cacheKey);
+
+    let filter: Expression = null;
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (perm[resourceType]) {
+          if (perm[resourceType].access >= access)
+            filter = or(filter, perm[resourceType].filter);
+        }
+      }
+    }
+
+    this.getFilterCache.set(cacheKey, filter);
+    return filter;
+  }
+
   public getValidator(
     resourceType,
     resource
-  ): (mutationType: string, mutation: any, args: any) => boolean {
+  ): (mutationType: string, mutation?: any, args?: any) => boolean {
     if (this.validatorCache.has(resource))
       return this.validatorCache.get(resource);
 
@@ -120,4 +141,8 @@ export default class Authorizer {
     this.validatorCache.set(resource, validator);
     return validator;
   }
+
+  public getPermissionSets(): PermissionSet[] {
+    return this.permissionSets;
+  }
 }

lib/common/expression.ts

@@ -277,7 +277,7 @@ export async function evaluateAsync(
   });
 }
 
-export function and(exp1, exp2): Expression {
+export function and(exp1: Expression, exp2: Expression): Expression {
   if (!isArray(exp1)) return exp1 ? exp2 : exp1;
   if (!isArray(exp2)) return exp2 ? exp1 : exp2;
 
@@ -292,7 +292,7 @@ export function and(exp1, exp2): Expression {
   return res;
 }
 
-export function or(exp1, exp2): Expression {
+export function or(exp1: Expression, exp2: Expression): Expression {
   if (!isArray(exp1)) return exp1 ? exp1 : exp2;
   if (!isArray(exp2)) return exp2 ? exp2 : exp1;
 

lib/config.ts

@@ -48,7 +48,6 @@ const options = {
   CWMP_SSL_KEY: { type: "string", default: "" },
   CWMP_LOG_FILE: { type: "path", default: "" },
   CWMP_ACCESS_LOG_FILE: { type: "path", default: "" },
-  CWMP_KEEP_ALIVE_TIMEOUT: { type: "int", default: 0 },
 
   NBI_WORKER_PROCESSES: { type: "int", default: 0 },
   NBI_PORT: { type: "int", default: 7557 },

lib/cwmp.ts

@@ -50,6 +50,7 @@ import { IncomingMessage, ServerResponse } from "http";
 import { Readable } from "stream";
 import { promisify } from "util";
 import { TLSSocket } from "tls";
+import { decode, encodingExists } from "iconv-lite";
 import { dependencies as DEPENDENCIES } from "../package.json";
 import { parseXmlDeclaration } from "./xml-parser";
 import * as debug from "./debug";
@@ -189,14 +190,15 @@ async function writeResponse(
   }
 
   const httpResponse = sessionContext.httpResponse;
+  const connection = httpResponse.connection;
+
   httpResponse.setHeader("Content-Length", Buffer.byteLength(data));
   httpResponse.writeHead(res.code, res.headers);
-  httpResponse.end(data);
-
   if (sessionContext.debug)
     debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, res.data);
+  httpResponse.end(data);
 
-  if (sessionContext.httpRequest.connection.destroyed) {
+  if (connection.destroyed) {
     logger.accessError({
       sessionContext: sessionContext,
       message: "Connection dropped"
@@ -211,7 +213,7 @@ async function writeResponse(
     }
   } else {
     sessionContext.lastActivity = Date.now();
-    currentSessions.set(sessionContext.httpRequest.connection, sessionContext);
+    currentSessions.set(connection, sessionContext);
   }
 }
 
@@ -668,7 +670,7 @@ async function nextRpc(sessionContext: SessionContext): Promise<void> {
       break;
     case "download":
       session.addProvisions(sessionContext, `task_${task._id}`, [
-        ["download", task.fileType, task.fileName, task.targetFileName]
+        ["download", task.fileType, task.fileName, task.targetFileName || ""]
       ]);
       break;
     case "addObject":
@@ -684,6 +686,13 @@ async function nextRpc(sessionContext: SessionContext): Promise<void> {
         ["instances", task.objectName, 0]
       ]);
       break;
+    case "provisions":
+      session.addProvisions(
+        sessionContext,
+        `task_${task._id}`,
+        task.provisions
+      );
+      break;
     default:
       throw new Error("Task name not recognized");
   }
@@ -772,7 +781,7 @@ async function sendAcsRequest(
     downloadRequest.fileSize = 0;
     if (!downloadRequest.url) {
       const FS_PORT = config.get("FS_PORT");
-      const FS_SSL = config.get("FS_SSL");
+      const FS_SSL = config.get("FS_SSL_CERT");
       let hostname = config.get("FS_HOSTNAME");
       if (!hostname) {
         if (sessionContext.httpRequest.headers["host"])
@@ -822,11 +831,15 @@ async function getSession(connection, sessionId): Promise<SessionContext> {
   return session.deserialize(sessionContextString);
 }
 
+// Only needed to prevent tree shaking from removing the remoteAddress
+// workaround in onConnection function.
+const remoteAddressWorkaround = new WeakMap<Socket, string>();
+
 // When socket closes, store active sessions in cache
-export function onConnection(socket): void {
+export function onConnection(socket: Socket): void {
   // The property remoteAddress may be undefined after the connection is
   // closed, unless we read it at least once (caching?)
-  socket.remoteAddress;
+  remoteAddressWorkaround.set(socket, socket.remoteAddress);
 
   socket.on("close", async () => {
     const sessionContext = currentSessions.get(socket);
@@ -952,9 +965,9 @@ async function reportBadState(sessionContext: SessionContext): Promise<void> {
   const body = "Bad session state";
   httpResponse.setHeader("Content-Length", Buffer.byteLength(body));
   httpResponse.writeHead(400, { Connection: "close" });
-  httpResponse.end(body);
   if (sessionContext.debug)
     debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, body);
+  httpResponse.end(body);
 }
 
 async function responseUnauthorized(
@@ -989,9 +1002,9 @@ async function responseUnauthorized(
   const body = "Unauthorized";
   httpResponse.setHeader("Content-Length", Buffer.byteLength(body));
   httpResponse.writeHead(401, resHeaders);
-  httpResponse.end(body);
   if (sessionContext.debug)
     debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, body);
+  httpResponse.end(body);
 }
 
 async function processRequest(
@@ -1105,6 +1118,15 @@ export function listener(
     });
 }
 
+function decodeString(buffer: Buffer, charset: string): string {
+  try {
+    return buffer.toString(charset);
+  } catch (err) {
+    if (encodingExists(charset)) return decode(buffer, charset);
+  }
+  return null;
+}
+
 async function listenerAsync(
   httpRequest: IncomingMessage,
   httpResponse: ServerResponse
@@ -1202,7 +1224,6 @@ async function listenerAsync(
       const _body = "Invalid session";
       httpResponse.setHeader("Content-Length", Buffer.byteLength(_body));
       httpResponse.writeHead(400, { Connection: "close" });
-      httpResponse.end(_body);
       if (sessionContext.debug) {
         debug.outgoingHttpResponse(
           httpResponse,
@@ -1210,7 +1231,7 @@ async function listenerAsync(
           _body
         );
       }
-
+      httpResponse.end(_body);
       return;
     }
   } else if (stats.concurrentRequests > MAX_CONCURRENT_REQUESTS) {
@@ -1233,10 +1254,36 @@ async function listenerAsync(
   if (!charset) {
     const parse = parseXmlDeclaration(body);
     const e = parse ? parse.find(s => s.localName === "encoding") : null;
-    charset = e ? e.value : "utf8";
+    charset = e ? e.value.toLowerCase() : "utf8";
   }
 
-  const bodyStr = body.toString(charset);
+  const bodyStr = decodeString(body, charset);
+
+  if (bodyStr == null) {
+    const msg = `Unknown encoding '${charset}'`;
+    logger.accessError({
+      message: "XML parse error",
+      parseError: msg,
+      sessionContext: sessionContext || {
+        httpRequest: httpRequest,
+        httpResponse: httpResponse
+      }
+    });
+    httpResponse.setHeader("Content-Length", Buffer.byteLength(msg));
+    httpResponse.writeHead(400, { Connection: "close" });
+    if (sessionContext) {
+      if (sessionContext.debug)
+        debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, msg);
+    } else {
+      const cacheSnapshot = await localCache.getCurrentSnapshot();
+      const d = !!localCache.getConfig(cacheSnapshot, "cwmp.debug", {
+        remoteAddress: httpResponse.connection.remoteAddress
+      });
+      if (d) debug.outgoingHttpResponse(httpResponse, null, msg);
+    }
+    httpResponse.end(msg);
+    return;
+  }
 
   async function parsedRpc(_sessionContext, rpc, parseWarnings): Promise<void> {
     for (const w of parseWarnings) {
@@ -1270,7 +1317,6 @@ async function listenerAsync(
     });
     httpResponse.setHeader("Content-Length", Buffer.byteLength(error.message));
     httpResponse.writeHead(400, { Connection: "close" });
-    httpResponse.end(error.message);
     if (sessionContext) {
       if (sessionContext.debug) {
         debug.outgoingHttpResponse(
@@ -1284,14 +1330,9 @@ async function listenerAsync(
       const d = !!localCache.getConfig(cacheSnapshot, "cwmp.debug", {
         remoteAddress: httpResponse.connection.remoteAddress
       });
-      if (d) {
-        debug.outgoingHttpResponse(
-          httpResponse,
-          sessionContext.deviceId,
-          error.message
-        );
-      }
+      if (d) debug.outgoingHttpResponse(httpResponse, null, error.message);
     }
+    httpResponse.end(error.message);
     return;
   }
 
@@ -1318,14 +1359,13 @@ async function listenerAsync(
     const _body = "Invalid session";
     httpResponse.setHeader("Content-Length", Buffer.byteLength(_body));
     httpResponse.writeHead(400, { Connection: "close" });
-    httpResponse.end(_body);
     const cacheSnapshot = await localCache.getCurrentSnapshot();
     const d = !!localCache.getConfig(cacheSnapshot, "cwmp.debug", {
       remoteAddress: httpResponse.connection.remoteAddress
     });
     if (d)
       debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, _body);
-
+    httpResponse.end(_body);
     return;
   }
 

lib/db.ts

@@ -44,13 +44,21 @@ export let tasksCollection: Collection,
   usersCollection: Collection,
   configCollection: Collection;
 
+let clientPromise: Promise<MongoClient>;
 export let client: MongoClient;
 
+function compareAccessLists(list1: string[], list2: string[]): boolean {
+  if (list1.length !== list2.length) return false;
+  for (const [i, v] of list1.entries()) if (v !== list2[i]) return false;
+  return true;
+}
+
 export async function connect(): Promise<void> {
-  client = await MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
+  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
     useNewUrlParser: true
   });
 
+  client = await clientPromise;
   const db = client.db();
 
   tasksCollection = db.collection("tasks");
@@ -70,7 +78,7 @@ export async function connect(): Promise<void> {
 }
 
 export async function disconnect(): Promise<void> {
-  if (client) await client.close();
+  if (clientPromise) await (await clientPromise).close();
 }
 
 // Optimize projection by removing overlaps
@@ -116,6 +124,8 @@ export async function fetchDevice(
 
   function storeParams(obj, path: string, pathLength: number, ts): void {
     if (obj["_timestamp"]) obj["_timestamp"] = +obj["_timestamp"];
+    if (obj["_attributesTimestamp"])
+      obj["_attributesTimestamp"] = +obj["_attributesTimestamp"];
 
     const attrs: Attributes = {};
     let t = obj["_timestamp"] || 1;
@@ -133,6 +143,16 @@ export async function fetchDevice(
 
     if (obj["_object"] != null) attrs.object = [t, obj["_object"] ? 1 : 0];
 
+    if (obj["_notification"] != null) {
+      attrs.notification = [
+        obj["_attributesTimestamp"] || 1,
+        obj["_notification"]
+      ];
+    }
+
+    if (obj["_accessList"] != null)
+      attrs.accessList = [obj["_attributesTimestamp"] || 1, obj["_accessList"]];
+
     res.push([Path.parse(path.slice(0, -1)), t, attrs]);
 
     for (const [k, v] of Object.entries(obj)) {
@@ -329,6 +349,12 @@ export async function saveDevice(
     const object2 = ((diff[2] || {}).object || [])[1];
     const writable2 = ((diff[2] || {}).writable || [])[1];
     const writable1 = ((diff[1] || {}).writable || [])[1];
+    const attributesTimestamp1 = ((diff[1] || {}).notification || [])[0];
+    const attributesTimestamp2 = ((diff[2] || {}).notification || [])[0];
+    const notification1 = ((diff[1] || {}).notification || [])[1];
+    const notification2 = ((diff[2] || {}).notification || [])[1];
+    const accessList1 = ((diff[1] || {}).accessList || [])[1];
+    const accessList2 = ((diff[2] || {}).accessList || [])[1];
 
     switch (path.segments[0]) {
       case "Events":
@@ -455,6 +481,46 @@ export async function saveDevice(
                     path.length ? path.toString() + "._writable" : "_writable"
                   ] = !!writable2;
                 }
+
+                break;
+              case "notification":
+                if (
+                  !diff[1] ||
+                  !diff[1].notification ||
+                  notification2 !== notification1
+                ) {
+                  update["$set"][
+                    path.length
+                      ? path.toString() + "._notification"
+                      : "_notification"
+                  ] = notification2;
+                }
+
+                if (attributesTimestamp2 !== attributesTimestamp1) {
+                  update["$set"][
+                    path.toString() + "._attributesTimestamp"
+                  ] = new Date(attributesTimestamp2);
+                }
+
+                break;
+              case "accessList":
+                if (
+                  !diff[1] ||
+                  !diff[1].accessList ||
+                  !compareAccessLists(accessList2, accessList1)
+                ) {
+                  update["$set"][
+                    path.length
+                      ? path.toString() + "._accessList"
+                      : "_accessList"
+                  ] = accessList2;
+                }
+
+                if (attributesTimestamp2 !== attributesTimestamp1) {
+                  update["$set"][
+                    path.toString() + "._attributesTimestamp"
+                  ] = new Date(attributesTimestamp2);
+                }
             }
           }
         }
@@ -470,6 +536,12 @@ export async function saveDevice(
               if (attrName === "value") {
                 update["$unset"][p + "_type"] = 1;
                 update["$unset"][p + "_timestamp"] = 1;
+              } else if (attrName === "notification") {
+                if (accessList2 == null)
+                  update["$unset"][`${p}_attributesTimestamp`] = 1;
+              } else if (attrName === "accessList") {
+                if (notification2 == null)
+                  update["$unset"][`${p}_attributesTimestamp`] = 1;
               }
             }
           }

lib/default-provisions.ts

@@ -25,21 +25,49 @@ import * as scheduling from "./scheduling";
 const MAX_DEPTH = +config.get("MAX_DEPTH");
 
 export function refresh(sessionContext, provision, declarations): boolean {
-  const segments = Path.parse(provision[1]).segments.slice();
-  const l = segments.length;
-  segments.length = MAX_DEPTH;
-  segments.fill("*", l);
-  const path = Path.parse(segments.join("."));
+  if (
+    (provision.length !== 2 || typeof provision[1] !== "string") &&
+    (provision.length !== 3 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "number") &&
+    (provision.length < 4 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "number" ||
+      typeof provision[3] !== "boolean")
+  )
+    throw new Error("Invalid arguments");
+
   const every = 1000 * (provision[2] || 1);
   const offset = scheduling.variance(sessionContext.deviceId, every);
   const t = scheduling.interval(sessionContext.timestamp, every, offset);
 
-  for (let i = l; i < path.length; ++i) {
+  let attrGet;
+  let refreshChildren;
+  if (provision[3] == null) {
+    refreshChildren = true;
+    attrGet = { object: 1, writable: 1, value: t };
+  } else {
+    attrGet = {};
+    refreshChildren = !!provision[3];
+    for (const a of provision.slice(4)) attrGet[a] = t;
+  }
+
+  let path = Path.parse(provision[1]);
+  let l = path.length;
+  if (refreshChildren) {
+    const segments = path.segments.slice();
+    l = segments.length;
+    segments.length = MAX_DEPTH;
+    segments.fill("*", l);
+    path = Path.parse(segments.join("."));
+  }
+
+  for (let i = l; i <= path.length; ++i) {
     declarations.push({
       path: path.slice(0, i),
       pathGet: t,
       pathSet: null,
-      attrGet: { object: 1, writable: 1, value: t },
+      attrGet: attrGet,
       attrSet: null,
       defer: true
     });
@@ -49,12 +77,38 @@ export function refresh(sessionContext, provision, declarations): boolean {
 }
 
 export function value(sessionContext, provision, declarations): boolean {
+  if (
+    provision.length < 3 ||
+    provision.length > 4 ||
+    typeof provision[1] !== "string"
+  )
+    throw new Error("Invalid arguments");
+
+  let attr: string, val: any;
+
+  if (provision.length === 3) {
+    attr = "value";
+    val = provision[2];
+  } else {
+    attr = provision[2] || "";
+    val = provision[3];
+  }
+
+  if (attr === "accessList") {
+    val = (val || "")
+      .split(",")
+      .map(s => s.trim())
+      .filter(s => !!s);
+  } else if (attr === "value") {
+    val = [val];
+  }
+
   declarations.push({
     path: Path.parse(provision[1]),
     pathGet: 1,
     pathSet: null,
-    attrGet: { value: 1 },
-    attrSet: { value: [provision[2]] },
+    attrGet: { [attr]: 1 },
+    attrSet: { [attr]: val },
     defer: true
   });
 
@@ -62,6 +116,13 @@ export function value(sessionContext, provision, declarations): boolean {
 }
 
 export function tag(sessionContext, provision, declarations): boolean {
+  if (
+    provision.length !== 3 ||
+    typeof provision[1] !== "string" ||
+    typeof provision[2] !== "boolean"
+  )
+    throw new Error("Invalid arguments");
+
   declarations.push({
     path: Path.parse(`Tags.${provision[1]}`),
     pathGet: 1,
@@ -75,6 +136,8 @@ export function tag(sessionContext, provision, declarations): boolean {
 }
 
 export function reboot(sessionContext, provision, declarations): boolean {
+  if (provision.length !== 1) throw new Error("Invalid arguments");
+
   declarations.push({
     path: Path.parse("Reboot"),
     pathGet: 1,
@@ -88,6 +151,8 @@ export function reboot(sessionContext, provision, declarations): boolean {
 }
 
 export function reset(sessionContext, provision, declarations): boolean {
+  if (provision.length !== 1) throw new Error("Invalid arguments");
+
   declarations.push({
     path: Path.parse("FactoryReset"),
     pathGet: 1,
@@ -101,6 +166,17 @@ export function reset(sessionContext, provision, declarations): boolean {
 }
 
 export function download(sessionContext, provision, declarations): boolean {
+  if (
+    (provision.length !== 3 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "string") &&
+    (provision.length !== 4 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "string" ||
+      typeof provision[3] !== "string")
+  )
+    throw new Error("Invalid arguments");
+
   const alias = [
     `FileType:${JSON.stringify(provision[1] || "")}`,
     `FileName:${JSON.stringify(provision[2] || "")}`,
@@ -135,9 +211,12 @@ export function instances(
   startRevision,
   endRevision
 ): boolean {
+  if (provision.length !== 3 || typeof provision[1] !== "string")
+    throw new Error("Invalid arguments");
+
   let count = Number(provision[2]);
 
-  if (Number.isNaN(count)) return true;
+  if (Number.isNaN(count)) throw new Error("Invalid arguments");
 
   const path = Path.parse(provision[1]);
 

lib/device.ts

@@ -23,7 +23,9 @@ import { DeviceData, Declaration, Clear } from "./types";
 const CHANGE_FLAGS = {
   object: 2,
   writable: 4,
-  value: 8
+  value: 8,
+  notification: 16,
+  accessList: 32
 };
 
 function parseBool(v): boolean {

lib/extensions.ts

@@ -21,6 +21,7 @@ import { spawn, ChildProcess } from "child_process";
 import * as crypto from "crypto";
 import * as config from "./config";
 import { Fault } from "./types";
+import { ROOT_DIR } from "./config";
 
 const TIMEOUT = +config.get("EXT_TIMEOUT");
 
@@ -43,7 +44,7 @@ export function run(args): Promise<{ fault: Fault; value: any }> {
     jobs.set(id, resolve);
 
     if (!processes[scriptName]) {
-      const p = spawn("bin/genieacs-ext", [scriptName], {
+      const p = spawn(ROOT_DIR + "/bin/genieacs-ext", [scriptName], {
         stdio: ["inherit", "inherit", "inherit", "ipc"]
       });
       processes[scriptName] = p;

lib/gpn-heuristic.ts

@@ -1,3 +1,5 @@
+import Path from "./common/path";
+
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -27,9 +29,12 @@ const UNDISCOVERED_DEPTH = 7;
 // flags is an int where its bits mark the segments in the pattern that
 // need refreshing. Leading 0s indicate that the pattern up to that
 // point has been discovered.
-export function estimateGpnCount(gpnPatterns, depth = 0): number {
-  const children = {};
-  const wildcardChildren = [];
+export function estimateGpnCount(
+  gpnPatterns: [Path, number][],
+  depth: number = 0
+): number {
+  const children: { [segment: string]: [Path, number][] } = {};
+  const wildcardChildren: [Path, number][] = [];
   let wildcardDiscovered = false;
   let gpnCount = 0;
 
@@ -37,7 +42,7 @@ export function estimateGpnCount(gpnPatterns, depth = 0): number {
     const path = pattern[0];
     const flags = pattern[1] >> depth;
 
-    const k = path[depth];
+    const k = path.segments[depth] as string;
 
     if (!k) {
       if (flags & 1) gpnCount = 1;

lib/init.ts

@@ -60,7 +60,7 @@ const INFORM_SCRIPT = `
 const username = declare("DeviceID.ID", {value: 1}).value[0]
 
 // Password will be fixed a given device because Math.random() is seeded with devcie ID by default.
-const password = "" + Math.random();
+const password = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER).toString(36);
 
 const informInterval = 300;
 
@@ -135,8 +135,7 @@ export async function seed(options): Promise<void> {
       { _id: "ui.filters.1.parameter", value: "DeviceID.ProductClass" },
       { _id: "ui.filters.1.type", value: "'string'" },
       { _id: "ui.filters.2.label", value: "'Tag'" },
-      { _id: "ui.filters.2.parameter", value: "tag" },
-      { _id: "ui.filters.2.type", value: "'string'" }
+      { _id: "ui.filters.2.type", value: "'tag'" }
     ]);
   }
 
@@ -378,6 +377,7 @@ export async function seed(options): Promise<void> {
         _id: "bootstrap",
         weight: 0,
         channel: "bootstrap",
+        events: "0 BOOTSTRAP",
         provision: "bootstrap"
       },
       { _id: "default", weight: 0, channel: "default", provision: "default" },

lib/mongodb-functions.ts

@@ -34,8 +34,6 @@ export function processDeviceFilter(filter): Expression {
       else if (p === "DeviceID") return ["PARAM", "_deviceId"];
       else if (p.startsWith("DeviceID."))
         return ["PARAM", "_deviceId._" + p.slice(9)];
-      else if (p === "Tags") return ["PARAM", "_tags"];
-      else if (p === "tag") return ["PARAM", "_tags"];
       else if (p === "Events.Inform") return ["PARAM", "_lastInform"];
       else if (p === "Events.Registered") return ["PARAM", "_registered"];
       else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"];
@@ -50,6 +48,8 @@ export function processDeviceFilter(filter): Expression {
       else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t];
       else if (exp[0] === "=" && exp[2] === true)
         return ["=", ["PARAM", "_tags"], t];
+      else if (exp[0] === "<>" && exp[2] !== true)
+        return ["=", ["PARAM", "_tags"], t];
     } else if (
       ["=", "<>", ">", ">=", "<", "<=", "LIKE", "NOT LIKE"].includes(exp[0])
     ) {
@@ -285,19 +285,30 @@ function parseDate(d): number | string {
   return isNaN(n) ? "" + d : n;
 }
 
+interface FlatAttributes {
+  object?: boolean;
+  objectTimestamp?: number;
+  writable?: boolean;
+  writableTimestamp?: number;
+  value?: [string | number | boolean, string];
+  valueTimestamp?: number;
+  notification?: number;
+  notificationTimestamp?: number;
+  accessList?: string[];
+  accessListTimestamp?: number;
+}
+
 interface FlatDevice {
-  [param: string]: {
-    object: boolean;
-    objectTimestamp: number;
-    writable: boolean;
-    writableTimestamp: number;
-    value: [string | number | boolean, string];
-    valueTimestamp: number;
-  };
+  [param: string]: FlatAttributes;
 }
 
 export function flattenDevice(device): FlatDevice {
-  function recursive(input, root, output, timestamp): void {
+  function recursive(
+    input,
+    root: string,
+    output: FlatDevice,
+    timestamp: number
+  ): void {
     for (const [name, tree] of Object.entries(input)) {
       if (!root) {
         if (name === "_lastInform") {
@@ -338,7 +349,7 @@ export function flattenDevice(device): FlatDevice {
           };
         } else if (name === "_id") {
           output["DeviceID.ID"] = {
-            value: [tree, "xsd:string"],
+            value: [tree as string, "xsd:string"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
@@ -380,7 +391,7 @@ export function flattenDevice(device): FlatDevice {
           };
         } else if (name === "_tags") {
           output["Tags"] = {
-            writable: true,
+            writable: false,
             writableTimestamp: timestamp,
             object: true,
             objectTimestamp: timestamp
@@ -390,7 +401,7 @@ export function flattenDevice(device): FlatDevice {
             output[`Tags.${t}`] = {
               value: [true, "xsd:boolean"],
               valueTimestamp: timestamp,
-              writable: false,
+              writable: true,
               writableTimestamp: timestamp,
               object: false,
               objectTimestamp: timestamp
@@ -407,34 +418,44 @@ export function flattenDevice(device): FlatDevice {
       else if (+input["_timestamp"] > timestamp)
         childrenTimestamp = +input["_timestamp"];
 
-      const attrs = {};
+      const attrs: FlatAttributes = {};
       if (tree["_value"] != null) {
-        attrs["value"] = [
+        attrs.value = [
           tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
           tree["_type"]
         ];
-        attrs["valueTimestamp"] = +(tree["_timestamp"] || childrenTimestamp);
-        attrs["object"] = false;
-        attrs["objectTimestamp"] = childrenTimestamp;
+        attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
+        attrs.object = false;
+        attrs.objectTimestamp = childrenTimestamp;
       } else if (tree["_object"] != null) {
-        attrs["object"] = tree["_object"];
-        attrs["objectTimestamp"] = childrenTimestamp;
+        attrs.object = tree["_object"];
+        attrs.objectTimestamp = childrenTimestamp;
       }
 
       if (tree["_writable"] != null) {
-        attrs["writable"] = tree["_writable"];
-        attrs["writableTimestamp"] = childrenTimestamp;
+        attrs.writable = tree["_writable"];
+        attrs.writableTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_notification"] != null) {
+        attrs.notification = tree["_notification"];
+        attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      if (tree["_accessList"] != null) {
+        attrs.accessList = tree["_accessList"];
+        attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1;
       }
 
       const r = root ? `${root}.${name}` : name;
       output[r] = attrs;
 
-      if (attrs["object"] || tree["object"] == null)
+      if (attrs.object || tree["object"] == null)
         recursive(tree, r, output, childrenTimestamp);
     }
   }
 
-  const newDevice = {};
+  const newDevice: FlatDevice = {};
   const timestamp = new Date(device["_lastInform"] || 1).getTime();
   recursive(device, "", newDevice, timestamp);
   return newDevice;
@@ -522,9 +543,12 @@ export function mongoQueryToFilter(query): Expression {
           }
           exps.push([op, ["PARAM", k], vv]);
         }
-        if (exps.length === 1) return exps[0];
+        if (exps.length === 1) {
+          expressions.push(exps[0]);
+        } else {
           const and: Expression = ["AND"];
           expressions.push(and.concat(exps));
+        }
       } else {
         expressions.push(["=", ["PARAM", k], v]);
       }

lib/nbi.ts

@@ -21,13 +21,13 @@ import * as url from "url";
 import * as mongodb from "mongodb";
 import * as querystring from "querystring";
 import * as vm from "vm";
-import { exec } from "child_process";
 import * as config from "./config";
 import * as db from "./db";
 import * as query from "./query";
 import * as apiFunctions from "./api-functions";
 import * as cache from "./cache";
 import { version as VERSION } from "../package.json";
+import { ping } from "./ping";
 
 const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
 const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
@@ -591,8 +591,9 @@ export function listener(request, response): void {
       }
     } else if (PING_REGEX.test(urlParts.pathname)) {
       const host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1]);
-      exec(`ping -w 1 -i 0.2 -c 3 ${host}`, (err, stdout) => {
+      ping(host, (err, res, stdout) => {
         if (err) {
+          if (!res) throwError(err, response);
           response.writeHead(404, { "Cache-Control": "no-cache" });
           response.end(`${err.name}: ${err.message}`);
           return;

lib/ping.ts

@@ -0,0 +1,89 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { platform } from "os";
+import { exec } from "child_process";
+
+interface Ping {
+  packetsTransmitted: number;
+  packetsReceived: number;
+  packetLoss: number;
+  min: number;
+  avg: number;
+  max: number;
+  mdev: number;
+}
+
+export function ping(
+  host: string,
+  callback: (err, res?, stdout?) => void
+): void {
+  let cmd: string, parseRegExp1: RegExp, parseRegExp2: RegExp;
+  switch (platform()) {
+    case "linux":
+      cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
+      parseRegExp1 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
+      parseRegExp2 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss/;
+      break;
+
+    case "freebsd":
+      // Send a single packet because on FreeBSD only superuser can send
+      // packets that are only 200 ms apart.
+      cmd = `ping -t 1 -c 3 ${host}`;
+      parseRegExp1 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
+      parseRegExp2 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
+      break;
+
+    default:
+      return callback(new Error("Platform not supported"));
+  }
+
+  exec(cmd, (err, stdout) => {
+    let parsed: Ping;
+    if (stdout) {
+      const m1 = stdout.match(parseRegExp1);
+      if (m1) {
+        parsed = {
+          packetsTransmitted: +m1[1],
+          packetsReceived: +m1[2],
+          packetLoss: +m1[3],
+          min: +m1[4],
+          avg: +m1[5],
+          max: +m1[6],
+          mdev: +m1[7]
+        };
+      } else {
+        const m2 = stdout.match(parseRegExp2);
+        if (m2) {
+          parsed = {
+            packetsTransmitted: +m2[1],
+            packetsReceived: +m2[2],
+            packetLoss: +m2[3],
+            min: null,
+            avg: null,
+            max: null,
+            mdev: null
+          };
+        }
+      }
+    }
+
+    callback(err, parsed, stdout);
+  });
+}

lib/server.ts

@@ -46,7 +46,8 @@ function closeServer(timeout, callback): void {
 
     const cb = callback;
     callback = null;
-    cb();
+    // Allow some time for connection close events to fire
+    setTimeout(cb, 50);
   });
 }
 
@@ -56,7 +57,7 @@ export function start(
   ssl,
   _listener,
   onConnection?,
-  keepAliveTimeout: number = 0
+  keepAliveTimeout: number = -1
 ): void {
   listener = _listener;
 
@@ -77,7 +78,7 @@ export function start(
     if (onConnection != null) server.on("connection", onConnection);
   }
 
-  if (keepAliveTimeout) server.keepAliveTimeout = keepAliveTimeout;
+  if (keepAliveTimeout >= 0) server.keepAliveTimeout = keepAliveTimeout;
   server.listen(port, networkInterface);
 }
 

lib/session.ts

@@ -537,15 +537,29 @@ async function runProvisions(
       if (!allProvisions[provision[0]]) {
         if (defaultProvisions[provision[0]]) {
           const dec = [];
-          const done = defaultProvisions[provision[0]](
+          let done = true;
+          let fault = null;
+          try {
+            done = defaultProvisions[provision[0]](
               sessionContext,
               provision,
               dec,
               startRevision,
               endRevision
             );
+          } catch (err) {
+            fault = {
+              code: `script.${err.name}`,
+              message: err.message,
+              detail: {
+                name: err.name,
+                message: err.message,
+                stack: `${err.name}: ${err.message}\n    at ${provision[0]}`
+              }
+            };
+          }
           return {
-            fault: null,
+            fault: fault,
             clear: null,
             declare: dec,
             done: done,
@@ -715,9 +729,12 @@ function runDeclarations(
         exist: new Set(),
         object: new Set(),
         writable: new Set(),
-        value: new Set()
+        value: new Set(),
+        notification: new Set(),
+        accessList: new Set()
       },
       spv: new Map(),
+      spa: new Map(),
       gpn: new Set<Path>(),
       gpnPatterns: new Map(),
       tags: new Map(),
@@ -771,6 +788,9 @@ function runDeclarations(
     let path = declaration.path;
     let unpacked: Path[];
 
+    // Can't run declarations on root
+    if (!path.length) continue;
+
     if (
       (path.alias | path.wildcard) & 1 ||
       path.segments[0] === "VirtualParameters"
@@ -1446,6 +1466,8 @@ function generateGetRpcRequest(sessionContext: SessionContext): GetAcsRequest {
         syncState.refreshAttributes.value.has(p) ||
         syncState.refreshAttributes.object.has(p) ||
         syncState.refreshAttributes.writable.has(p) ||
+        syncState.refreshAttributes.notification.has(p) ||
+        syncState.refreshAttributes.accessList.has(p) ||
         syncState.gpn.has(p)
       ) {
         found = true;
@@ -1474,7 +1496,9 @@ function generateGetRpcRequest(sessionContext: SessionContext): GetAcsRequest {
         syncState.refreshAttributes.value.has(p) ||
         (p.length > path.length &&
           (syncState.refreshAttributes.object.has(p) ||
-            syncState.refreshAttributes.writable.has(p)))
+            syncState.refreshAttributes.writable.has(p) ||
+            syncState.refreshAttributes.notification.has(p) ||
+            syncState.refreshAttributes.accessList.has(p)))
       ) {
         found = true;
         break;
@@ -1522,7 +1546,7 @@ function generateGetRpcRequest(sessionContext: SessionContext): GetAcsRequest {
       let nextLevel;
       let est = 0;
       if (path.length >= GPN_NEXT_LEVEL) {
-        const patterns = [[path, 0]];
+        const patterns: [Path, number][] = [[path, 0]];
         for (const p of sessionContext.deviceData.paths.find(
           path,
           true,
@@ -1582,9 +1606,57 @@ function generateGetRpcRequest(sessionContext: SessionContext): GetAcsRequest {
       };
     }
   }
+
+  if (
+    syncState.refreshAttributes.notification.size ||
+    syncState.refreshAttributes.accessList.size
+  ) {
+    const GPV_BATCH_SIZE = localCache.getConfig(
+      sessionContext.cacheSnapshot,
+      "cwmp.gpvBatchSize",
+      sessionContext.configContext
+    );
+
+    const parameterNames: string[] = [];
+    for (const path of syncState.refreshAttributes.notification) {
+      syncState.refreshAttributes.notification.delete(path);
+      syncState.refreshAttributes.accessList.delete(path);
+      // Need to check in case param is deleted
+      const attrs = sessionContext.deviceData.attributes.get(path);
+      if (attrs) {
+        parameterNames.push(path.toString());
+        if (parameterNames.length >= GPV_BATCH_SIZE) break;
+      }
+    }
+
+    if (parameterNames.length < GPV_BATCH_SIZE) {
+      for (const path of syncState.refreshAttributes.accessList) {
+        syncState.refreshAttributes.accessList.delete(path);
+        const attrs = sessionContext.deviceData.attributes.get(path);
+        if (attrs) {
+          parameterNames.push(path.toString());
+          if (parameterNames.length >= GPV_BATCH_SIZE) break;
+        }
+      }
+    }
+
+    if (parameterNames.length) {
+      return {
+        name: "GetParameterAttributes",
+        parameterNames: parameterNames
+      };
+    }
+  }
+
   return null;
 }
 
+function compareAccessLists(list1: string[], list2: string[]): boolean {
+  if (list1.length !== list2.length) return false;
+  for (const [i, v] of list1.entries()) if (v !== list2[i]) return false;
+  return true;
+}
+
 function generateSetRpcRequest(sessionContext: SessionContext): SetAcsRequest {
   const syncState = sessionContext.syncState;
   if (!syncState) return null;
@@ -1637,7 +1709,7 @@ function generateSetRpcRequest(sessionContext: SessionContext): SetAcsRequest {
     sessionContext.configContext
   );
 
-  const parameterValues: [Path, string | number | boolean, string][] = [];
+  const parameterValues: [string, string | number | boolean, string][] = [];
   for (const [k, v] of syncState.spv) {
     syncState.spv.delete(k);
     const attrs = sessionContext.deviceData.attributes.get(k);
@@ -1656,7 +1728,7 @@ function generateSetRpcRequest(sessionContext: SessionContext): SetAcsRequest {
         val[0] -= val[0] % 1000;
 
       if (val[0] !== curVal[0] || val[1] !== curVal[1])
-        parameterValues.push([k, val[0], val[1]]);
+        parameterValues.push([k.toString(), val[0], val[1]]);
 
       if (parameterValues.length >= GPV_BATCH_SIZE) break;
     }
@@ -1665,19 +1737,44 @@ function generateSetRpcRequest(sessionContext: SessionContext): SetAcsRequest {
   if (parameterValues.length) {
     return {
       name: "SetParameterValues",
-      parameterList: parameterValues.map(
-        p =>
-          [p[0].toString(), p[1], p[2]] as [
-            string,
-            string | number | boolean,
-            string
-          ]
-      ),
+      parameterList: parameterValues,
       DATETIME_MILLISECONDS: DATETIME_MILLISECONDS,
       BOOLEAN_LITERAL: BOOLEAN_LITERAL
     };
   }
 
+  // Set attributes
+  const parameterAttributes: [string, number, string[]][] = [];
+  for (const [k, v] of syncState.spa) {
+    syncState.spa.delete(k);
+    const attrs = sessionContext.deviceData.attributes.get(k);
+
+    if (
+      v.notification != null &&
+      (!attrs.notification || v.notification === attrs.notification[1])
+    )
+      v.notification = null;
+
+    if (
+      v.accessList != null &&
+      (!attrs.accessList ||
+        compareAccessLists(v.accessList, attrs.accessList[1]))
+    )
+      v.accessList = null;
+
+    if (v.notification != null || v.accessList != null)
+      parameterAttributes.push([k.toString(), v.notification, v.accessList]);
+
+    if (parameterAttributes.length >= GPV_BATCH_SIZE) break;
+  }
+
+  if (parameterAttributes.length) {
+    return {
+      name: "SetParameterAttributes",
+      parameterList: parameterAttributes
+    };
+  }
+
   // Downloads
   for (const [p, t] of syncState.downloadsDownload) {
     const attrs = deviceData.attributes.get(p);
@@ -2092,8 +2189,34 @@ function processDeclarations(
               }
             }
           }
-          if (declareAttributeValues && declareAttributeValues.value != null)
+          if (declareAttributeValues) {
+            if (declareAttributeValues.value != null)
               syncState.spv.set(currentPath, declareAttributeValues.value);
+
+            if (declareAttributeValues.notification != null) {
+              const spa = syncState.spa.get(currentPath);
+              if (spa) {
+                spa.notification = declareAttributeValues.notification;
+              } else {
+                syncState.spa.set(currentPath, {
+                  notification: declareAttributeValues.notification,
+                  accessList: null
+                });
+              }
+            }
+
+            if (declareAttributeValues.accessList != null) {
+              const spa = syncState.spa.get(currentPath);
+              if (spa) {
+                spa.accessList = declareAttributeValues.accessList;
+              } else {
+                syncState.spa.set(currentPath, {
+                  notification: null,
+                  accessList: declareAttributeValues.accessList
+                });
+              }
+            }
+          }
         }
     }
 
@@ -2226,7 +2349,7 @@ export async function rpcResponse(
       if (p[1] !== rpcReq.instanceValues[p[0]]) {
         const v = device.sanitizeParameterValue([
           rpcReq.instanceValues[p[0]],
-          p[2]
+          p[2] as string
         ]);
         parameterList.push([p[0], v[0], v[1]]);
       }
@@ -2271,6 +2394,21 @@ export async function rpcResponse(
         );
       }
 
+      break;
+    case "GetParameterAttributesResponse":
+      if (rpcReq.name !== "GetParameterAttributes")
+        throw new Error("Response name does not match request name");
+
+      for (const p of rpcRes.parameterList) {
+        toClear = device.set(
+          sessionContext.deviceData,
+          Path.parse(p[0]),
+          timestamp,
+          { notification: [timestamp, p[1]], accessList: [timestamp, p[2]] },
+          toClear
+        );
+      }
+
       break;
     case "GetParameterNamesResponse":
       if (rpcReq.name !== "GetParameterNames")
@@ -2387,6 +2525,38 @@ export async function rpcResponse(
         );
       }
 
+      break;
+    case "SetParameterAttributesResponse":
+      if (rpcReq.name !== "SetParameterAttributes")
+        throw new Error("Response name does not match request name");
+
+      for (const p of rpcReq.parameterList) {
+        let attrs;
+
+        if (p[1] != null && p[2] != null) {
+          attrs = {
+            notification: [timestamp + 1, p[1]],
+            accessList: [timestamp + 1, p[2]]
+          };
+        } else if (p[1] != null) {
+          attrs = {
+            notification: [timestamp + 1, p[1]]
+          };
+        } else if (p[2] != null) {
+          attrs = {
+            accessList: [timestamp + 1, p[2]]
+          };
+        }
+
+        toClear = device.set(
+          sessionContext.deviceData,
+          Path.parse(p[0]),
+          timestamp + 1,
+          attrs,
+          toClear
+        );
+      }
+
       break;
     case "AddObjectResponse":
       toClear = device.set(
@@ -2531,7 +2701,7 @@ export async function rpcFault(
 
   // Recover from invalid parameter name faults
   if (faultResponse.detail.faultCode === "9005") {
-    const timestamp = sessionContext.timestamp + sessionContext.iteration;
+    const timestamp = sessionContext.timestamp + sessionContext.iteration + 1;
     const revision =
       (sessionContext.revisions[sessionContext.revisions.length - 1] || 0) + 1;
     sessionContext.deviceData.timestamps.revision = revision;
@@ -2549,13 +2719,23 @@ export async function rpcFault(
         p => [Path.parse(p.replace(/\.$/, "")), timestamp] as Clear
       );
     } else if (rpcReq.name === "SetParameterValues") {
-      toClear = rpcReq.parameterList.map(
-        p => [Path.parse(p[0].replace(/\.$/, "")), timestamp] as Clear
-      );
+      toClear = (rpcReq.parameterList as [
+        string,
+        string | number | boolean,
+        string
+      ][]).map(p => [Path.parse(p[0].replace(/\.$/, "")), timestamp] as Clear);
     } else if (rpcReq.name === "AddObject") {
       toClear = [[Path.parse(rpcReq.objectName.replace(/\.$/, "")), timestamp]];
     } else if (rpcReq.name === "DeleteObject") {
       toClear = [[Path.parse(rpcReq.objectName.replace(/\.$/, "")), timestamp]];
+    } else if (rpcReq.name === "GetParameterAttributes") {
+      toClear = rpcReq.parameterNames.map(
+        p => [Path.parse(p.replace(/\.$/, "")), timestamp] as Clear
+      );
+    } else if (rpcReq.name === "SetParameterAttributes") {
+      toClear = (rpcReq.parameterList as [string, number, string[]][]).map(
+        p => [Path.parse(p[0].replace(/\.$/, "")), timestamp] as Clear
+      );
     }
 
     if (toClear) {
@@ -2621,8 +2801,13 @@ export async function serialize(
     deviceData.push(e);
   }
 
+  const declarations = sessionContext.declarations.map(decs => {
+    return decs.map(d => Object.assign({}, d, { path: d.path.toString() }));
+  });
+
   const jsonSessionContext = Object.assign({}, sessionContext, {
     deviceData: deviceData,
+    declarations: declarations,
     syncState: null,
     toLoad: null,
     httpRequest: null,

lib/soap.ts

@@ -188,6 +188,45 @@ function parameterValueList(
     });
 }
 
+function parameterAttributeList(xml: Element): [string, number, string[]][] {
+  return xml.children
+    .filter(e => e.localName === "ParameterAttributeStruct")
+    .map<[string, number, string[]]>(e => {
+      let notificationElement: Element,
+        accessListElement: Element,
+        param: string;
+      for (const c of e.children) {
+        switch (c.localName) {
+          case "Name":
+            param = c.text;
+            break;
+          case "Notification":
+            notificationElement = c;
+            break;
+          case "AccessList":
+            accessListElement = c;
+            break;
+        }
+      }
+
+      const notification = decodeEntities(notificationElement.text);
+      const notif = +notification;
+
+      if (notif !== parseInt(notification)) {
+        warnings.push({
+          message: "Invalid notification attribute",
+          parameter: param
+        });
+      }
+
+      const accessList = accessListElement.children
+        .filter(c => c.localName === "string")
+        .map(c => decodeEntities(c.text));
+
+      return [param, notif, accessList];
+    });
+}
+
 function GetParameterNames(methodRequest): string {
   return `<cwmp:GetParameterNames><ParameterPath>${
     methodRequest.parameterPath
@@ -220,6 +259,23 @@ function GetParameterValuesResponse(xml: Element): CpeGetResponse {
   };
 }
 
+function GetParameterAttributes(methodRequest): string {
+  return `<cwmp:GetParameterAttributes><ParameterNames soap-enc:arrayType="xsd:string[${
+    methodRequest.parameterNames.length
+  }]">${methodRequest.parameterNames
+    .map(p => `<string>${p}</string>`)
+    .join("")}</ParameterNames></cwmp:GetParameterAttributes>`;
+}
+
+function GetParameterAttributesResponse(xml: Element): CpeGetResponse {
+  return {
+    name: "GetParameterAttributesResponse",
+    parameterList: parameterAttributeList(
+      xml.children.find(n => n.localName === "ParameterList")
+    )
+  };
+}
+
 function SetParameterValues(methodRequest): string {
   const params = methodRequest.parameterList.map(p => {
     let val = p[1];
@@ -250,6 +306,36 @@ function SetParameterValuesResponse(xml: Element): CpeSetResponse {
   };
 }
 
+function SetParameterAttributes(methodRequest): string {
+  const params = methodRequest.parameterList.map(p => {
+    return `<SetParameterAttributesStruct><Name>${
+      p[0]
+    }</Name><NotificationChange>${
+      p[1] == null ? "false" : "true"
+    }</NotificationChange><Notification>${
+      p[1] == null ? "" : p[1]
+    }</Notification><AccessListChange>${
+      p[2] == null ? "false" : "true"
+    }</AccessListChange><AccessList soap-enc:arrayType="xsd:string[${
+      (p[2] || []).length
+    }]">${
+      p[2] == null
+        ? ""
+        : p[2].map(s => `<string>${encodeEntities(s)}</string>`).join("")
+    }</AccessList></SetParameterAttributesStruct>`;
+  });
+
+  return `<cwmp:SetParameterAttributes><ParameterList soap-enc:arrayType="cwmp:SetParameterAttributesStruct[${
+    methodRequest.parameterList.length
+  }]">${params.join("")}</ParameterList></cwmp:SetParameterAttributes>`;
+}
+
+function SetParameterAttributesResponse(): CpeSetResponse {
+  return {
+    name: "SetParameterAttributesResponse"
+  };
+}
+
 function AddObject(methodRequest): string {
   return `<cwmp:AddObject><ObjectName>${
     methodRequest.objectName
@@ -610,9 +696,15 @@ export function request(body: string, cwmpVersion, warn): SoapMessage {
     case "GetParameterValuesResponse":
       rpc.cpeResponse = GetParameterValuesResponse(methodElement);
       break;
+    case "GetParameterAttributesResponse":
+      rpc.cpeResponse = GetParameterAttributesResponse(methodElement);
+      break;
     case "SetParameterValuesResponse":
       rpc.cpeResponse = SetParameterValuesResponse(methodElement);
       break;
+    case "SetParameterAttributesResponse":
+      rpc.cpeResponse = SetParameterAttributesResponse();
+      break;
     case "AddObjectResponse":
       rpc.cpeResponse = AddObjectResponse(methodElement);
       break;
@@ -690,9 +782,15 @@ export function response(rpc): { code: number; headers: {}; data: string } {
       case "GetParameterValues":
         body = GetParameterValues(rpc.acsRequest);
         break;
+      case "GetParameterAttributes":
+        body = GetParameterAttributes(rpc.acsRequest);
+        break;
       case "SetParameterValues":
         body = SetParameterValues(rpc.acsRequest);
         break;
+      case "SetParameterAttributes":
+        body = SetParameterAttributes(rpc.acsRequest);
+        break;
       case "AddObject":
         body = AddObject(rpc.acsRequest);
         break;

lib/types.ts

@@ -51,18 +51,24 @@ export interface Attributes {
   object?: [number, 1 | 0];
   writable?: [number, 1 | 0];
   value?: [number, [string | number | boolean, string]];
+  notification?: [number, number];
+  accessList?: [number, string[]];
 }
 
 export interface AttributeTimestamps {
   object?: number;
   writable?: number;
   value?: number;
+  notification?: number;
+  accessList?: number;
 }
 
 export interface AttributeValues {
   object?: boolean;
   writable?: boolean;
   value?: [string | number | boolean, string?];
+  notification?: number;
+  accessList?: string[];
 }
 
 export interface DeviceData {
@@ -75,12 +81,21 @@ export interface DeviceData {
 
 export type VirtualParameterDeclaration = [
   Path,
-  { path?: number; object?: number; writable?: number; value?: number }?,
+  {
+    path?: number;
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  }?,
   {
     path?: [number, number];
     object?: boolean;
     writable?: boolean;
     value?: [string | number | boolean, string?];
+    notification?: number;
+    accessList?: string[];
   }?
 ];
 
@@ -90,8 +105,11 @@ export interface SyncState {
     object: Set<Path>;
     writable: Set<Path>;
     value: Set<Path>;
+    notification: Set<Path>;
+    accessList: Set<Path>;
   };
   spv: Map<Path, [string | number | boolean, string]>;
+  spa: Map<Path, { notification: number; accessList: string[] }>;
   gpn: Set<Path>;
   gpnPatterns: Map<Path, number>;
   tags: Map<Path, boolean>;
@@ -160,6 +178,7 @@ export interface Task {
   fileName?: string;
   targetFileName?: string;
   expiry?: number;
+  provisions?: (string | number | boolean)[][];
 }
 
 export interface Operation {
@@ -182,7 +201,7 @@ export interface AcsRequest {
 }
 
 export interface GetAcsRequest extends AcsRequest {
-  name: "GetParameterNames" | "GetParameterValues";
+  name: "GetParameterNames" | "GetParameterValues" | "GetParameterAttributes";
   objectName?: string;
   parameterNames?: string[];
   parameterPath?: string;
@@ -193,12 +212,15 @@ export interface GetAcsRequest extends AcsRequest {
 export interface SetAcsRequest extends AcsRequest {
   name:
     | "SetParameterValues"
+    | "SetParameterAttributes"
     | "AddObject"
     | "DeleteObject"
     | "FactoryReset"
     | "Reboot"
     | "Download";
-  parameterList?: [string, string | number | boolean, string][];
+  parameterList?:
+    | [string, string | number | boolean, string][]
+    | [string, number, string[]][];
   instanceValues?: { [param: string]: string | number | boolean };
   objectName?: string;
   DATETIME_MILLISECONDS?: boolean;
@@ -235,15 +257,20 @@ export interface CpeFault {
 }
 
 export interface CpeGetResponse extends CpeResponse {
-  name: "GetParameterNamesResponse" | "GetParameterValuesResponse";
+  name:
+    | "GetParameterNamesResponse"
+    | "GetParameterValuesResponse"
+    | "GetParameterAttributesResponse";
   parameterList?:
     | [string, boolean][]
-    | [string, string | number | boolean, string][];
+    | [string, string | number | boolean, string][]
+    | [string, number, string[]][];
 }
 
 export interface CpeSetResponse extends CpeResponse {
   name:
     | "SetParameterValuesResponse"
+    | "SetParameterAttributesResponse"
     | "AddObjectResponse"
     | "DeleteObjectResponse"
     | "RebootResponse"
@@ -300,11 +327,19 @@ export interface Declaration {
   path: Path;
   pathGet: number;
   pathSet?: [number, number];
-  attrGet?: { object?: number; writable?: number; value?: number };
+  attrGet?: {
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  };
   attrSet?: {
     object?: boolean;
     writable?: boolean;
     value?: [string | number | boolean, string?];
+    notification?: number;
+    accessList?: string[];
   };
   defer: boolean;
 }
@@ -312,7 +347,13 @@ export interface Declaration {
 export type Clear = [
   Path,
   number,
-  { object?: number; writable?: number; value?: number }?,
+  {
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  }?,
   number?
 ];
 

lib/ui.ts

@@ -31,7 +31,7 @@ import Authorizer from "./common/authorizer";
 import * as logger from "./logger";
 import * as localCache from "./local-cache";
 import { PermissionSet } from "./types";
-import { authSimple } from "./ui/api-functions";
+import { authLocal } from "./ui/api-functions";
 import * as init from "./init";
 import { version as VERSION } from "../package.json";
 
@@ -47,16 +47,6 @@ const router = new Router();
 const JWT_SECRET = "" + config.get("UI_JWT_SECRET");
 const JWT_COOKIE = "genieacs-ui-jwt";
 
-function getPermissionSets(ctx): PermissionSet[] {
-  const allPermissions = localCache.getPermissions(ctx.state.configSnapshot);
-  const users = localCache.getUsers(ctx.state.configSnapshot);
-  const user = users[ctx.state.user.username];
-  const permissionSets = user.roles.map(role =>
-    Object.values(allPermissions[role] || {})
-  );
-  return permissionSets;
-}
-
 koa.on("error", async err => {
   throw err;
 });
@@ -73,14 +63,39 @@ koa.use(
   koaJwt({
     secret: JWT_SECRET,
     passthrough: true,
-    cookie: JWT_COOKIE
+    cookie: JWT_COOKIE,
+    isRevoked: async (ctx, token) => {
+      if (token["authMethod"] === "local") {
+        return !localCache.getUsers(ctx.state.configSnapshot)[
+          token["username"]
+        ];
+      }
+
+      return true;
+    }
   })
 );
 
 koa.use(async (ctx, next) => {
-  if (ctx.state.user && ctx.state.user.username)
-    ctx.state.authorizer = new Authorizer(getPermissionSets(ctx));
-  else ctx.state.authorizer = new Authorizer([]);
+  const permissionSets: PermissionSet[] = [];
+
+  if (ctx.state.user && ctx.state.user.username) {
+    const allPermissions = localCache.getPermissions(ctx.state.configSnapshot);
+
+    let user;
+    if (ctx.state.user.authMethod === "local") {
+      user = localCache.getUsers(ctx.state.configSnapshot)[
+        ctx.state.user.username
+      ];
+    } else {
+      throw new Error("Invalid auth method");
+    }
+
+    for (const role of (user.roles || []).sort())
+      permissionSets.push(Object.values(allPermissions[role] || {}));
+  }
+
+  ctx.state.authorizer = new Authorizer(permissionSets);
 
   return next();
 });
@@ -103,9 +118,9 @@ router.post("/login", async ctx => {
     method: null
   };
 
-  function success(method): void {
-    log.method = method;
-    const token = jwt.sign({ username }, JWT_SECRET);
+  function success(authMethod): void {
+    log.method = authMethod;
+    const token = jwt.sign({ username, authMethod }, JWT_SECRET);
     ctx.cookies.set(JWT_COOKIE, token, { sameSite: "lax" });
     ctx.body = JSON.stringify(token);
     logger.accessInfo(log);
@@ -118,9 +133,8 @@ router.post("/login", async ctx => {
     logger.accessWarn(log);
   }
 
-  const roles = await authSimple(ctx.state.configSnapshot, username, password);
-
-  if (roles) return void success("simple");
+  if (await authLocal(ctx.state.configSnapshot, username, password))
+    return void success("local");
 
   failure();
 });
@@ -192,9 +206,7 @@ router.post("/init", async ctx => {
 });
 
 router.get("/", async ctx => {
-  let permissionSets = [];
-  if (ctx.state.user && ctx.state.user.username)
-    permissionSets = getPermissionSets(ctx);
+  const permissionSets = ctx.state.authorizer.getPermissionSets();
 
   let wizard = "";
   if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)

lib/ui/api-functions.ts

@@ -17,7 +17,6 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { exec } from "child_process";
 import { ObjectID } from "mongodb";
 import * as db from "./db";
 import { del } from "../cache";
@@ -149,29 +148,6 @@ interface PingResponse {
   mdev: number;
 }
 
-export function ping(host): Promise<PingResponse> {
-  return new Promise((resolve, reject) => {
-    exec(`ping -w 1 -i 0.2 -c 3 ${host}`, (err, stdout) => {
-      if (err) return void reject(err);
-
-      const m = stdout.match(
-        /(\d) packets transmitted, (\d) received, ([\d.%]+) packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/
-      );
-      if (!m) return void reject(new Error("Could not parse ping response"));
-
-      resolve({
-        packetsTransmitted: +m[1],
-        packetsReceived: +m[2],
-        packetLoss: m[3],
-        min: +m[4],
-        avg: +m[5],
-        max: +m[6],
-        mdev: +m[7]
-      });
-    });
-  });
-}
-
 export async function putResource(resource, id, data): Promise<void> {
   if (resource === "presets") {
     await db.putPreset(id, data);
@@ -192,15 +168,15 @@ export async function putResource(resource, id, data): Promise<void> {
   await del("presets_hash");
 }
 
-export function authSimple(snapshot, username, password): Promise<string[]> {
+export function authLocal(snapshot, username, password): Promise<boolean> {
   return new Promise((resolve, reject) => {
     const users = getUsers(snapshot);
     const user = users[username];
     if (!user || !user.password) return void resolve(null);
     hashPassword(password, user.salt)
       .then(hash => {
-        if (hash === user.password) resolve(user.roles);
-        else resolve(null);
+        if (hash === user.password) resolve(true);
+        else resolve(false);
       })
       .catch(reject);
   });

lib/ui/api.ts

@@ -28,6 +28,8 @@ import { getConfig } from "../local-cache";
 import { QueryOptions, Expression } from "../types";
 import { generateSalt, hashPassword } from "../auth";
 import { del } from "../cache";
+import Authorizer from "../common/authorizer";
+import { ping } from "../ping";
 
 const router = new Router();
 export default router;
@@ -67,15 +69,20 @@ const resources = {
 };
 
 router.get(`/devices/:id.csv`, async ctx => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Query device (CSV)",
     context: ctx,
     id: ctx.params.id
   };
 
-  const filter = ["=", ["PARAM", RESOURCE_IDS.devices], ctx.params.id];
+  const filter = and(authorizer.getFilter("devices", 2), [
+    "=",
+    ["PARAM", RESOURCE_IDS.devices],
+    ctx.params.id
+  ]);
 
-  if (!ctx.state.authorizer.hasAccess("devices", 2)) {
+  if (!authorizer.hasAccess("devices", 2)) {
     logUnauthorizedWarning(log);
     return void (ctx.status = 404);
   }
@@ -92,7 +99,7 @@ router.get(`/devices/:id.csv`, async ctx => {
 
   ctx.body = new stream.PassThrough();
   ctx.body.write(
-    "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp\n"
+    "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp,Notification,Notification timestamp,Access list,Access list timestamp\n"
   );
 
   for (const k of Object.keys(res[0]).sort()) {
@@ -105,7 +112,11 @@ router.get(`/devices/:id.csv`, async ctx => {
       p.writableTimestamp,
       p.value != null ? `"${p.value[0].toString().replace(/"/g, '""')}"` : "",
       p.value != null ? p.value[1] : "",
-      p.valueTimestamp
+      p.valueTimestamp,
+      p.notification,
+      p.notificationTimestamp,
+      p.accessList ? p.accessList.join(", ") : "",
+      p.accessListTimestamp
     ];
     ctx.body.write(row.map(r => (r != null ? r : "")).join(",") + "\n");
   }
@@ -115,8 +126,10 @@ router.get(`/devices/:id.csv`, async ctx => {
 
 for (const [resource, flags] of Object.entries(resources)) {
   router.head(`/${resource}`, async (ctx, next) => {
-    let filter: Expression = true;
-    if (ctx.request.query.filter) filter = parse(ctx.request.query.filter);
+    const authorizer: Authorizer = ctx.state.authorizer;
+    let filter: Expression = authorizer.getFilter(resource, 1);
+    if (ctx.request.query.filter)
+      filter = and(filter, parse(ctx.request.query.filter));
 
     const log = {
       message: `Count ${resource}`,
@@ -125,7 +138,7 @@ for (const [resource, flags] of Object.entries(resources)) {
       count: null
     };
 
-    if (!ctx.state.authorizer.hasAccess(resource, 1)) {
+    if (!authorizer.hasAccess(resource, 1)) {
       logUnauthorizedWarning(log);
       return void next();
     }
@@ -148,9 +161,11 @@ for (const [resource, flags] of Object.entries(resources)) {
   });
 
   router.get(`/${resource}`, async (ctx, next) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = {};
-    let filter: Expression = true;
-    if (ctx.request.query.filter) filter = parse(ctx.request.query.filter);
+    let filter: Expression = authorizer.getFilter(resource, 2);
+    if (ctx.request.query.filter)
+      filter = and(filter, parse(ctx.request.query.filter));
     if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
     if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
     if (ctx.request.query.sort)
@@ -171,7 +186,7 @@ for (const [resource, flags] of Object.entries(resources)) {
       projection: options.projection
     };
 
-    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+    if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
       return void next();
     }
@@ -199,9 +214,11 @@ for (const [resource, flags] of Object.entries(resources)) {
 
   // CSV download
   router.get(`/${resource}.csv`, async (ctx, next) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = { projection: {} };
-    let filter: Expression = true;
-    if (ctx.request.query.filter) filter = parse(ctx.request.query.filter);
+    let filter: Expression = authorizer.getFilter(resource, 2);
+    if (ctx.request.query.filter)
+      filter = and(filter, parse(ctx.request.query.filter));
     if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
     if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
     if (ctx.request.query.sort)
@@ -216,7 +233,7 @@ for (const [resource, flags] of Object.entries(resources)) {
       sort: options.sort
     };
 
-    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+    if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
       return void next();
     }
@@ -285,14 +302,19 @@ for (const [resource, flags] of Object.entries(resources)) {
   });
 
   router.head(`/${resource}/:id`, async (ctx, next) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Count ${resource}`,
       context: ctx,
       filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
     };
 
-    const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
-    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+    const filter = and(authorizer.getFilter(resource, 2), [
+      "=",
+      ["PARAM", RESOURCE_IDS[resource]],
+      ctx.params.id
+    ]);
+    if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
       return void next();
     }
@@ -306,14 +328,19 @@ for (const [resource, flags] of Object.entries(resources)) {
   });
 
   router.get(`/${resource}/:id`, async (ctx, next) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Query ${resource}`,
       context: ctx,
       filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
     };
 
-    const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
-    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+    const filter = and(authorizer.getFilter(resource, 2), [
+      "=",
+      ["PARAM", RESOURCE_IDS[resource]],
+      ctx.params.id
+    ]);
+    if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
       return void next();
     }
@@ -326,17 +353,20 @@ for (const [resource, flags] of Object.entries(resources)) {
     ctx.body = res[0];
   });
 
-  // TODO add PUT, PATCH routes
   if (flags & RESOURCE_DELETE) {
     router.delete(`/${resource}/:id`, async (ctx, next) => {
+      const authorizer: Authorizer = ctx.state.authorizer;
       const log = {
         message: `Delete ${resource}`,
         context: ctx,
         id: ctx.params.id
       };
 
-      const authorizer = ctx.state.authorizer;
-      const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
+      const filter = and(authorizer.getFilter(resource, 3), [
+        "=",
+        ["PARAM", RESOURCE_IDS[resource]],
+        ctx.params.id
+      ]);
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
         return void next();
@@ -360,6 +390,7 @@ for (const [resource, flags] of Object.entries(resources)) {
 
   if (flags & RESOURCE_PUT) {
     router.put(`/${resource}/:id`, async (ctx, next) => {
+      const authorizer: Authorizer = ctx.state.authorizer;
       const id = ctx.params.id;
 
       const log = {
@@ -368,7 +399,6 @@ for (const [resource, flags] of Object.entries(resources)) {
         id: id
       };
 
-      const authorizer = ctx.state.authorizer;
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
         return void next();
@@ -399,6 +429,7 @@ for (const [resource, flags] of Object.entries(resources)) {
 }
 
 router.put("/files/:id", async (ctx, next) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const resource = "files";
   const id = ctx.params.id;
 
@@ -409,7 +440,6 @@ router.put("/files/:id", async (ctx, next) => {
     metadata: null
   };
 
-  const authorizer = ctx.state.authorizer;
   if (!authorizer.hasAccess(resource, 3)) {
     logUnauthorizedWarning(log);
     return void next();
@@ -436,6 +466,7 @@ router.put("/files/:id", async (ctx, next) => {
 });
 
 router.post("/devices/:id/tasks", async (ctx, next) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Commit tasks",
     context: ctx,
@@ -443,8 +474,11 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
     tasks: null
   };
 
-  const authorizer = ctx.state.authorizer;
-  const filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
+  const filter = and(authorizer.getFilter("devices", 3), [
+    "=",
+    ["PARAM", "DeviceID.ID"],
+    ctx.params.id
+  ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
     return void next();
@@ -481,6 +515,7 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
 });
 
 router.post("/devices/:id/tags", async (ctx, next) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Update tags",
     context: ctx,
@@ -488,8 +523,11 @@ router.post("/devices/:id/tags", async (ctx, next) => {
     tags: ctx.request.body
   };
 
-  const authorizer = ctx.state.authorizer;
-  const filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
+  const filter = and(authorizer.getFilter("devices", 3), [
+    "=",
+    ["PARAM", "DeviceID.ID"],
+    ctx.params.id
+  ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
     return void next();
@@ -518,16 +556,21 @@ router.post("/devices/:id/tags", async (ctx, next) => {
 });
 
 router.get("/ping/:host", async ctx => {
-  try {
-    ctx.body = await apiFunctions.ping(ctx.params.host);
-  } catch (err) {
-    ctx.status = 404;
-    ctx.response.headers["Cache-Control"] = "no-cache";
+  return new Promise(resolve => {
+    ping(ctx.params.host, (err, parsed) => {
+      if (parsed) {
+        ctx.body = parsed;
+      } else {
+        ctx.status = 500;
         ctx.body = `${err.name}: ${err.message}`;
       }
+      resolve();
+    });
+  });
 });
 
 router.put("/users/:id/password", async (ctx, next) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const username = ctx.params.id;
   const log = {
     message: "Change password",
@@ -535,12 +578,10 @@ router.put("/users/:id/password", async (ctx, next) => {
     username: username
   };
 
-  const authorizer = ctx.state.authorizer;
-
   if (!ctx.state.user) {
     // User not logged in
     if (
-      !(await apiFunctions.authSimple(
+      !(await apiFunctions.authLocal(
         ctx.state.configSnapshot,
         username,
         ctx.request.body.authPassword
@@ -556,7 +597,11 @@ router.put("/users/:id/password", async (ctx, next) => {
     return void next();
   }
 
-  const filter = ["=", ["PARAM", RESOURCE_IDS.users], username];
+  const filter = and(authorizer.getFilter("users", 3), [
+    "=",
+    ["PARAM", RESOURCE_IDS.users],
+    username
+  ]);
   const res = await db.query("users", filter);
   if (!res.length) return void next();
 

lib/ui/db.ts

@@ -26,7 +26,7 @@ import { QueryOptions } from "../types";
 
 const CACHE_TTL = 300000;
 
-let _clientPromise = null;
+let clientPromise: Promise<MongoClient>;
 
 const RESOURCE_COLLECTION = {
   files: "fs.files"
@@ -40,8 +40,8 @@ function ensureIndexes(client): void {
 }
 
 function getClient(): Promise<MongoClient> {
-  if (!_clientPromise) {
-    _clientPromise = new Promise((resolve, reject) => {
+  if (!clientPromise) {
+    clientPromise = new Promise((resolve, reject) => {
       const CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
       MongoClient.connect(
         CONNECTION_URL,
@@ -55,7 +55,7 @@ function getClient(): Promise<MongoClient> {
     });
   }
 
-  return _clientPromise;
+  return clientPromise;
 }
 
 export function cache<T>(key, valueGetter: () => Promise<T>, ttl): Promise<T> {
@@ -405,10 +405,6 @@ export function deleteTask(id): Promise<void> {
   return deleteResource("tasks", id);
 }
 
-export function disconnect(): void {
-  if (_clientPromise) {
-    _clientPromise.then(client => {
-      client.close();
-    });
-  }
+export async function disconnect(): Promise<void> {
+  if (clientPromise) await (await clientPromise).close();
 }

npm-shrinkwrap.json

@@ -3670,6 +3670,17 @@
         "chardet": "^0.7.0",
         "iconv-lite": "^0.4.24",
         "tmp": "^0.0.33"
+      },
+      "dependencies": {
+        "iconv-lite": {
+          "version": "0.4.24",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+          "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+          "dev": true,
+          "requires": {
+            "safer-buffer": ">= 2.1.2 < 3"
+          }
+        }
       }
     },
     "extglob": {
@@ -4355,9 +4366,9 @@
       "dev": true
     },
     "iconv-lite": {
-      "version": "0.4.24",
-      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
-      "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.0.tgz",
+      "integrity": "sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==",
       "requires": {
         "safer-buffer": ">= 2.1.2 < 3"
       }
@@ -5937,6 +5948,15 @@
           "requires": {
             "ms": "^2.1.1"
           }
+        },
+        "iconv-lite": {
+          "version": "0.4.24",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+          "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+          "optional": true,
+          "requires": {
+            "safer-buffer": ">= 2.1.2 < 3"
+          }
         }
       }
     },
@@ -7664,6 +7684,16 @@
         "http-errors": "1.7.3",
         "iconv-lite": "0.4.24",
         "unpipe": "1.0.0"
+      },
+      "dependencies": {
+        "iconv-lite": {
+          "version": "0.4.24",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+          "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+          "requires": {
+            "safer-buffer": ">= 2.1.2 < 3"
+          }
+        }
       }
     },
     "rc": {

package.json

@@ -20,6 +20,7 @@
     "tools": "tools"
   },
   "dependencies": {
+    "iconv-lite": "^0.5.0",
     "jsonwebtoken": "^8.5.1",
     "koa": "^2.7.0",
     "koa-bodyparser": "^4.2.1",

test/mongodb-functions.ts

@@ -32,6 +32,10 @@ ava("mongoQueryToFilter", t => {
     [
       { test: { $gte: "test1", $ne: "test2" } },
       'test >= "test1" AND test <> "test2"'
+    ],
+    [
+      { test: "test", test2: { $ne: "test2" } },
+      'test = "test" AND test2 <> "test2"'
     ]
   ];
 

ui/components/all-parameters.ts

@@ -140,7 +140,9 @@ const component: ClosureComponent = (): Component => {
         m(
           "a.download-csv",
           {
-            href: `/api/devices/${device["DeviceID.ID"].value[0]}.csv`,
+            href: `/api/devices/${encodeURIComponent(
+              device["DeviceID.ID"].value[0]
+            )}.csv`,
             download: "",
             style: "float: right;"
           },

ui/components/parameter-table.ts

@@ -83,7 +83,8 @@ const component: ClosureComponent = (): Component => {
               p.type || "parameter",
               Object.assign({}, p, {
                 device: device,
-                parameter: param
+                parameter: param,
+                label: null
               })
             )
           );

ui/components/ping.ts

@@ -17,58 +17,62 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { ClosureComponent, Component } from "mithril";
+import { ClosureComponent, Component, VnodeDOM } from "mithril";
 import { m } from "../components";
 import * as store from "../store";
 
 const REFRESH_INTERVAL = 3000;
 
-const component: ClosureComponent = (): Component => {
+const component: ClosureComponent = (vn): Component => {
+  let interval: ReturnType<typeof setInterval>;
+  let host: string;
+
+  const refresh = async (): Promise<void> => {
+    let status = "";
+    if (host) {
+      try {
+        const res = await store.ping(host);
+        if (res["avg"] != null) status = `${Math.trunc(res["avg"])} ms`;
+        else status = "Unreachable";
+      } catch (err) {
+        setTimeout(() => {
+          throw err;
+        }, 0);
+      }
+    }
+
+    const dom = (vn as VnodeDOM).dom;
+
+    if (dom) dom.innerHTML = `Pinging ${host}: ${status}`;
+  };
+
   return {
-    oninit: vnode => {
-      vnode.state["timestamp"] = 0;
-      vnode.state["ping"] = null;
-      vnode.state["host"] = null;
-      vnode.state["timeout"] = null;
+    onremove: () => {
+      clearInterval(interval);
     },
     view: vnode => {
-      const refresh = (): void => {
       const device = vnode.attrs["device"];
       let param =
         device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"];
       if (!param)
         param = device["Device.ManagementServer.ConnectionRequestURL"];
-        if (!param || !param.value) return;
 
+      let h;
+      if (param && param.value) {
         const url = new URL(param.value[0]);
-        vnode.state["host"] = url.hostname;
-        vnode.state["timestamp"] = Date.now();
-        store
-          .ping(vnode.state["host"])
-          .then(res => {
-            vnode.state["ping"] = res;
-          })
-          .catch(() => {
-            // Do nothing
-          });
-      };
-
-      const t = vnode.state["timestamp"] + REFRESH_INTERVAL - Date.now();
-      clearTimeout(vnode.state["timeout"]);
-      if (t <= 0) refresh();
-      else vnode.state["timeout"] = setTimeout(refresh, t);
-
-      if (vnode.state["host"]) {
-        let status = "";
-        if (vnode.state["ping"]) {
-          if (vnode.state["ping"].avg != null)
-            status = `${Math.trunc(vnode.state["ping"].avg)} ms`;
-          else status = "Unreachable";
+        h = url.hostname;
       }
 
-        return m("div", `Pinging ${vnode.state["host"]}: ${status}`);
+      if (host !== h) {
+        host = h;
+        clearInterval(interval);
+        if (host) {
+          refresh();
+          interval = setInterval(refresh, REFRESH_INTERVAL);
         }
-      return null;
+      }
+
+      return m("div", `Pinging ${host}:`);
     }
   };
 };

ui/smart-query.ts

@@ -132,6 +132,11 @@ function queryMac(param, value): Expression {
   ];
 }
 
+function queryTag(tag: string): Expression {
+  const t = tag.replace(/[^a-zA-Z0-9-]+/g, "_");
+  return ["IS NOT NULL", ["PARAM", `Tags.${t}`]];
+}
+
 export function getTip(resource, label): string {
   let tip;
   if (resources[resource] && resources[resource][label]) {
@@ -156,6 +161,9 @@ export function getTip(resource, label): string {
         case "mac":
           tips.push("partial case insensitive MAC address");
           break;
+        case "tag":
+          tips.push("case sensitive string");
+          break;
       }
     }
 
@@ -190,6 +198,11 @@ export function unpack(resource, label, value): Expression {
     if (q) res.push(q);
   }
 
+  if (type.includes("tag")) {
+    const q = queryTag(value);
+    if (q) res.push(q);
+  }
+
   if (res.length <= 1) return null;
   else if (res.length === 2) return res[1];
   else return res;

ui/store.ts

@@ -18,7 +18,7 @@
  */
 
 import m from "mithril";
-import { map, stringify } from "../lib/common/expression-parser";
+import { stringify } from "../lib/common/expression-parser";
 import { or, and, not, evaluate, subset } from "../lib/common/expression";
 import memoize from "../lib/common/memoize";
 import { QueryOptions, Expression } from "../lib/types";
@@ -299,22 +299,6 @@ function compareFunction(sort: {
 }
 
 function findMatches(resourceType, filter, sort, limit): any[] {
-  // Handle "tag =" and "tag <>" special cases
-  if (resourceType === "devices") {
-    filter = map(filter, e => {
-      if (
-        Array.isArray(e) &&
-        Array.isArray(e[1]) &&
-        e[1][0] === "PARAM" &&
-        e[1][1] === "tag"
-      ) {
-        if (e[0] === "=") return ["IS NOT NULL", ["PARAM", `Tags.${e[2]}`]];
-        else if (e[0] === "<>") return ["IS NULL", ["PARAM", `Tags.${e[2]}`]];
-      }
-      return e;
-    });
-  }
-
   let value = [];
   for (const obj of resources[resourceType].objects.values())
     if (evaluate(filter, obj, fulfillTimestamp)) value.push(obj);
@@ -709,6 +693,7 @@ export function logOut(): Promise<void> {
 
 export function ping(host): Promise<{}> {
   return xhrRequest({
-    url: `/api/ping/${encodeURIComponent(host)}`
+    url: `/api/ping/${encodeURIComponent(host)}`,
+    background: true
   });
 }