1.2.9

Fixes occassional lock expired errors after updating presets, etc.

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Change Log
 
+## 1.2.9 (2022-08-22)
+
+- New config option `cwmp.skipRootGpn` to enable a workaround for some
+  problematic CPEs that reject GPN requests on data model root.
+- Stream query results and CSV downloads as data becomes available instead of
+  buffering the entire response.
+- Log HTTP/HTTPS client errors in debug log.
+- Fix occasional lock expired errors after updating presets, etc.
+- Fix bug where queries containing `<>` operator may return incorrect results.
+- Fix a performance hit caused by DB calls containing the entire CPE data model
+  rather then just the updated parameters.
+- Fix bug where tags containing special characters are saved in their encoded
+  form when set via a Provision script.
+- Fix issue causing invalid `xsd:dateTime` values to be saved in DB as `NaN`
+  rather than maintain the original string value.
+- Fix bug using `$regex` operator with a numeric or a datetime string.
+- Fix ping not working on certain platforms.
+- Ping requests are now authenticated.
+- Fix error when the `FORWARDED_HEADER` environment variable contains IPv4 CIDR
+  while the listening interface is IPv6 and vice versa.
+- Fix false warning "unexpected parameter in response" showing in
+  `genieacs-cwmp` logs.
+- Other minor fixes and stability improvements.
+
 ## 1.2.8 (2021-10-27)
 
 - Fix a remote code execution security vulnerability in genieacs-ui.

bin/genieacs-cwmp.ts

@@ -83,6 +83,7 @@ if (!cluster.worker) {
     host: SERVICE_ADDRESS,
     ssl: key && cert ? { key, cert } : null,
     onConnection: cwmp.onConnection,
+    onClientError: cwmp.onClientError,
     timeout: 30000,
     keepAliveTimeout: 0,
   };

build/build.ts

@@ -28,6 +28,7 @@ import commonjs from "@rollup/plugin-commonjs";
 import postcss from "postcss";
 import postcssImport from "postcss-import";
 import postcssPresetEnv from "postcss-preset-env";
+import postcssColorMod from "postcss-color-mod-function";
 import cssnano from "cssnano";
 import { optimize } from "svgo";
 import * as xmlParser from "../lib/xml-parser";
@@ -185,10 +186,10 @@ async function generateCss(): Promise<void> {
       stage: 3,
       features: {
         "nesting-rules": true,
-        "color-mod-function": true,
       },
     }),
-    cssnano,
+    postcssColorMod,
+    ...(MODE === "production" ? [cssnano] : []),
   ]).process(cssIn, { from: cssInPath, to: cssOutPath });
   fs.writeFileSync(cssOutPath, cssOut.css);
 }

build/spellcheck-dict.pws

@@ -170,3 +170,7 @@ TransferComplete
 nginx
 ENCODEURICOMPONENT
 pageSize
+skipRootGpn
+GPN
+NaN
+CIDR

lib/cache.ts

@@ -20,11 +20,12 @@
 import { Collection } from "mongodb";
 import { onConnect } from "./db";
 import * as config from "./config";
+import { Cache } from "./mongodb-types";
 
 const CLOCK_SKEW_TOLERANCE = 30000;
 const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
 
-let cacheCollection: Collection;
+let cacheCollection: Collection<Cache>;
 
 onConnect(async (db) => {
   cacheCollection = db.collection("cache");
@@ -33,8 +34,7 @@ onConnect(async (db) => {
 
 export async function get(key: string): Promise<string> {
   const res = await cacheCollection.findOne({ _id: key });
-  if (res) return res["value"];
+  return res?.value;
-  return null;
 }
 
 export async function del(key: string): Promise<void> {
@@ -52,15 +52,14 @@ export async function set(
   );
   await cacheCollection.replaceOne(
     { _id: key },
-    { _id: key, value, expire, timestamp },
+    { value, expire, timestamp },
     { upsert: true }
   );
 }
 
-export async function pop(key: string): Promise<any> {
+export async function pop(key: string): Promise<string> {
   const res = await cacheCollection.findOneAndDelete({ _id: key });
-  if (res?.["value"]) return res["value"]["value"];
+  return res.value?.value;
-  return null;
 }
 
 export async function acquireLock(
@@ -72,35 +71,23 @@ export async function acquireLock(
   try {
     const now = Date.now();
     const r = await cacheCollection.findOneAndUpdate(
-      { _id: lockName, token },
+      { _id: lockName, value: token },
       {
         $set: {
-          value: token,
           expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
         },
-        $currentDate: { timestamp: true as unknown as Date },
+        $currentDate: { timestamp: true },
       },
       { upsert: true, returnDocument: "after" }
     );
-    const v = r.value;
+    if (Math.abs(r.value.timestamp.getTime() - now) > CLOCK_SKEW_TOLERANCE)
-    if (Math.abs(v["timestamp"].getTime() - now) > CLOCK_SKEW_TOLERANCE)
       throw new Error("Database clock skew too great");
   } catch (err) {
-    if (err.code === 11000) {
+    if (err.code !== 11000) throw err;
-      if (timeout > 0) {
+    if (!(timeout > 0)) return null;
-        return new Promise((resolve, reject) => {
+    const w = 50 + Math.random() * 50;
-          const w = 50 + Math.random() * 50;
+    await new Promise((resolve) => setTimeout(resolve, w));
-          setTimeout(() => {
+    return acquireLock(lockName, ttl, timeout - w, token);
-            acquireLock(lockName, ttl, timeout - w, token).then(
-              resolve,
-              reject
-            );
-          }, w);
-        });
-      }
-      throw new Error("Failed to acquire lock");
-    }
-    throw err;
   }
 
   return token;

lib/common/boolean-expression.ts

@@ -1108,13 +1108,7 @@ function getCanRaiseCallback(
   return (idx: number, set: Set<number>): boolean => {
     const i = idx >> 2;
     const vars = context.getIsNullRelations(i);
-    if (!context.isIsNull(i)) {
+    if (!context.isIsNull(i)) return !(idx & 1) || !set.has(idx ^ 3);
-      if (!(idx & 1)) return true;
-      for (const n of vars) if (set.has((n << 2) ^ 3)) return true;
-      if (set.has(idx ^ 3)) return false;
-      for (const n of vars) if (!set.has((n << 2) ^ 2)) return false;
-      return true;
-    }
 
     for (const k of vars) {
       if (set.has((k << 2) ^ 1)) continue;

lib/common/debounce.ts

@@ -0,0 +1,16 @@
+export default function debounce<T>(
+  func: (args: T[]) => void,
+  timeout: number
+): (arg: T) => void {
+  let timer: ReturnType<typeof setTimeout>;
+  let args: T[] = [];
+  return (arg: T) => {
+    args.push(arg);
+    clearTimeout(timer);
+    timer = setTimeout(() => {
+      const argscopy = args;
+      args = [];
+      func(argscopy);
+    }, timeout);
+  };
+}

lib/cwmp.ts

@@ -54,6 +54,7 @@ import { decode, encodingExists } from "iconv-lite";
 import { parseXmlDeclaration } from "./xml-parser";
 import * as debug from "./debug";
 import { getRequestOrigin } from "./forwarded";
+import { getSocketEndpoints } from "./server";
 
 const gzipPromisified = promisify(zlib.gzip);
 const deflatePromisified = promisify(zlib.deflate);
@@ -212,12 +213,13 @@ async function writeResponse(
     currentSessions.set(connection, sessionContext);
     if (now >= sessionContext.extendLock) {
       sessionContext.extendLock = now + 10000;
-      await cache.acquireLock(
+      const lockToken = await cache.acquireLock(
         `cwmp_session_${sessionContext.deviceId}`,
         sessionContext.timeout * 1000 + 15000,
         0,
         sessionContext.sessionId
       );
+      if (!lockToken) throw new Error("Failed to extend lock");
     }
   }
 }
@@ -534,7 +536,7 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
               session.configContextCallback(sessionContext, e)
             )
           ),
-      ]);
+      ]) as [string, ...Expression[]][];
       if (blackList[p.channel] === 2) {
         appendProvisionsToFaults[p.channel] = (
           appendProvisionsToFaults[p.channel] || []
@@ -883,16 +885,8 @@ async function sendAcsRequest(
   return writeResponse(sessionContext, res);
 }
 
-// Only needed to prevent tree shaking from removing the remoteAddress
-// workaround in onConnection function.
-const remoteAddressWorkaround = new WeakMap<Socket, string>();
-
 // When socket closes, store active sessions in cache
 export function onConnection(socket: Socket): void {
-  // The property remoteAddress may be undefined after the connection is
-  // closed, unless we read it at least once (caching?)
-  remoteAddressWorkaround.set(socket, socket.remoteAddress);
-
   socket.on("close", async () => {
     const sessionContext = currentSessions.get(socket);
     if (!sessionContext) return;
@@ -947,6 +941,32 @@ export function onConnection(socket: Socket): void {
   });
 }
 
+export function onClientError(err: Error, socket: Socket): void {
+  const remoteAddress = getSocketEndpoints(socket).remoteAddress;
+  localCache
+    .getCurrentSnapshot()
+    .then((cacheSnapshot) => {
+      const debugEnabled = !!localCache.getConfig(
+        cacheSnapshot,
+        "cwmp.debug",
+        {
+          remoteAddress: remoteAddress,
+        },
+        Date.now(),
+        (e) => {
+          if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
+            return remoteAddress;
+          return e;
+        }
+      );
+
+      if (debugEnabled) debug.clientError(remoteAddress, err);
+    })
+    .catch((err) => {
+      throw err;
+    });
+}
+
 setInterval(() => {
   if (stats.droppedRequests) {
     logger.warn({
@@ -1130,15 +1150,15 @@ async function processRequest(
       }
     }
 
-    try {
+    sessionContext.extendLock = sessionContext.timestamp + 10000;
-      sessionContext.extendLock = sessionContext.timestamp + 10000;
+    const lockToken = await cache.acquireLock(
-      await cache.acquireLock(
+      `cwmp_session_${sessionContext.deviceId}`,
-        `cwmp_session_${sessionContext.deviceId}`,
+      sessionContext.timeout * 1000 + 15000,
-        sessionContext.timeout * 1000 + 15000,
+      0,
-        0,
+      sessionContext.sessionId
-        sessionContext.sessionId
+    );
-      );
+
-    } catch (err) {
+    if (!lockToken) {
       logger.accessError({
         message: "CPE already in session",
         sessionContext: sessionContext,

lib/db.ts

@@ -30,19 +30,20 @@ import {
   Expression,
 } from "./types";
 import Path from "./common/path";
+import * as MongoTypes from "./mongodb-types";
 
-export let tasksCollection: Collection,
+export let tasksCollection: Collection<MongoTypes.Task>,
   devicesCollection: Collection,
   presetsCollection: Collection,
   objectsCollection: Collection,
   provisionsCollection: Collection,
   virtualParametersCollection: Collection,
-  faultsCollection: Collection,
+  faultsCollection: Collection<MongoTypes.Fault>,
   filesCollection: Collection,
-  operationsCollection: Collection,
+  operationsCollection: Collection<MongoTypes.Operation>,
   permissionsCollection: Collection,
   usersCollection: Collection,
-  configCollection: Collection;
+  configCollection: Collection<MongoTypes.Config>;
 
 let clientPromise: Promise<MongoClient>;
 
@@ -91,7 +92,9 @@ export async function disconnect(): Promise<void> {
 
 // Optimize projection by removing overlaps
 // This can modify the object
-function optimizeProjection(obj: { [path: string]: 1 }): { [path: string]: 1 } {
+export function optimizeProjection(obj: { [path: string]: 1 }): {
+  [path: string]: 1;
+} {
   if (obj[""]) return { "": obj[""] };
 
   const keys = Object.keys(obj).sort();
@@ -146,7 +149,7 @@ export async function fetchDevice(
 
     if (obj["_value"] != null) {
       attrs.value = [obj["_timestamp"] || 1, [obj["_value"], obj["_type"]]];
-      if (obj["_type"] === "xsd:dateTime")
+      if (obj["_type"] === "xsd:dateTime" && obj["_value"] instanceof Date)
         attrs.value[1][0] = +attrs.value[1][0];
 
       obj["_object"] = false;
@@ -446,14 +449,18 @@ export async function saveDevice(
           if (value2 != null) {
             if (!update["$addToSet"]["_tags"])
               update["$addToSet"]["_tags"] = { $each: [] };
-            update["$addToSet"]["_tags"]["$each"].push(path.segments[1]);
+            update["$addToSet"]["_tags"]["$each"].push(
+              decodeTag(path.segments[1] as string)
+            );
           } else {
             if (!update["$pull"]["_tags"]) {
               update["$pull"]["_tags"] = {
                 $in: [],
               };
             }
-            update["$pull"]["_tags"]["$in"].push(path.segments[1]);
+            update["$pull"]["_tags"]["$in"].push(
+              decodeTag(path.segments[1] as string)
+            );
           }
         }
 
@@ -637,12 +644,16 @@ export async function getFaults(
   const faults: { [channel: string]: SessionFault } = {};
   for (const r of res) {
     const channel = r._id.slice(deviceId.length + 1);
-    delete r._id;
+    const fault: SessionFault = {
-    delete r.channel;
+      code: r.code,
-    delete r.device;
+      message: r.message,
-    r.timestamp = +r.timestamp;
+      ...(r.detail && { detail: r.detail }),
-    r.provisions = JSON.parse(r.provisions);
+      timestamp: +r.timestamp,
-    faults[channel] = r as SessionFault;
+      provisions: JSON.parse(r.provisions),
+      retries: r.retries,
+      ...(r.expiry && { expiry: +r.expiry }),
+    };
+    faults[channel] = fault;
   }
 
   return faults;
@@ -654,12 +665,18 @@ export async function saveFault(
   fault: SessionFault
 ): Promise<void> {
   const id = `${deviceId}:${channel}`;
-  const f = Object.assign({}, fault) as Record<string, any>;
+  const f: MongoTypes.Fault = {
-  f["_id"] = id;
+    _id: id,
-  f["device"] = deviceId;
+    device: deviceId,
-  f["channel"] = channel;
+    channel: channel,
-  f["timestamp"] = new Date(fault.timestamp);
+    timestamp: new Date(fault.timestamp),
-  f["provisions"] = JSON.stringify(fault.provisions);
+    code: fault.code,
+    message: fault.message,
+    ...(fault.detail && { detail: fault.detail }),
+    retries: fault.retries,
+    ...(fault.expiry && { expiry: new Date(fault.expiry) }),
+    provisions: JSON.stringify(fault.provisions),
+  };
   await faultsCollection.replaceOne({ _id: id }, f, { upsert: true });
 }
 
@@ -677,20 +694,47 @@ export async function getDueTasks(
   const cur = tasksCollection.find({ device: deviceId }).sort({ timestamp: 1 });
   const tasks = [] as Task[];
 
-  for await (const task of cur) {
+  for await (const t of cur) {
-    if (task.timestamp) task.timestamp = +task.timestamp;
+    if (+t.timestamp >= timestamp) return [tasks, +t.timestamp];
-    if (task.expiry) task.expiry = +task.expiry;
+    const task: Task = {
-    if (task.timestamp >= timestamp) return [tasks, +task.timestamp];
+      _id: t._id.toString(),
-    task._id = String(task._id);
+      name: t.name,
+      ...(t.timestamp && { timestamp: +t.timestamp }),
+      ...(t.expiry && { expiry: +t.expiry }),
+      ...(t.name === "getParameterValues" && {
+        parameterNames: t.parameterNames,
+      }),
+      ...(t.name === "setParameterValues" && {
+        parameterValues: t.parameterValues,
+      }),
+      ...(t.name === "refreshObject" && {
+        objectName: t.objectName,
+      }),
+      ...(t.name === "download" && {
+        fileType: t.fileType,
+        fileName: t.fileName,
+        targetFileName: t.targetFileName,
+      }),
+      ...(t.name === "addObject" && {
+        objectName: t.objectName,
+        parameterValues: t.parameterValues,
+      }),
+      ...(t.name === "deleteObject" && {
+        objectName: t.objectName,
+      }),
+      ...(t.name === "provisions" && {
+        provisions: t.provisions,
+      }),
+    };
 
-    tasks.push(task as Task);
+    tasks.push(task);
 
     // For API compatibility
-    if (task.name === "download" && task.file) {
+    if (task.name === "download" && t["file"]) {
       let q;
-      if (ObjectId.isValid(task.file))
+      if (ObjectId.isValid(t["file"]))
-        q = { _id: { $in: [task.file, new ObjectId(task.file)] } };
+        q = { _id: { $in: [t["file"], new ObjectId(t["file"])] } };
-      else q = { _id: task.file };
+      else q = { _id: t["file"] };
 
       const res = await filesCollection.find(q).toArray();
 
@@ -724,17 +768,22 @@ export async function getOperations(
   const operations: { [commandKey: string]: Operation } = {};
   for (const r of res) {
     const commandKey = r._id.slice(deviceId.length + 1);
-    delete r._id;
     // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
     if (typeof r.provisions !== "string") {
-      operations[commandKey] = r as Operation;
+      delete r._id;
+      operations[commandKey] = r as unknown as Operation;
       continue;
     }
-    r.timestamp = +r.timestamp;
+    const operation: Operation = {
-    if (r.args) r.args = JSON.parse(r.args);
+      name: r.name,
-    r.provisions = JSON.parse(r.provisions);
+      timestamp: +r.timestamp,
-    r.retries = JSON.parse(r.retries);
+      channels:
-    operations[commandKey] = r as Operation;
+        typeof r.channels === "string" ? JSON.parse(r.channels) : r.channels,
+      retries: JSON.parse(r.retries),
+      provisions: JSON.parse(r.provisions),
+      ...(r.args && { args: JSON.parse(r.args) }),
+    };
+    operations[commandKey] = operation;
   }
   return operations;
 }
@@ -745,12 +794,15 @@ export async function saveOperation(
   operation: Operation
 ): Promise<void> {
   const id = `${deviceId}:${commandKey}`;
-  const o = Object.assign({}, operation) as Record<string, any>;
+  const o: MongoTypes.Operation = {
-  o["_id"] = id;
+    _id: id,
-  o["timestamp"] = new Date(operation.timestamp);
+    name: operation.name,
-  o["provisions"] = JSON.stringify(operation.provisions);
+    timestamp: new Date(operation.timestamp),
-  o["retries"] = JSON.stringify(operation.retries);
+    channels: JSON.stringify(operation.channels),
-  o["args"] = JSON.stringify(operation.args);
+    provisions: JSON.stringify(operation.provisions),
+    retries: JSON.stringify(operation.retries),
+    args: JSON.stringify(operation.args),
+  };
   await operationsCollection.replaceOne({ _id: id }, o, {
     upsert: true,
   });

lib/debug.ts

@@ -22,6 +22,7 @@ import { Socket } from "net";
 import { appendFileSync } from "fs";
 import { stringify } from "./common/yaml";
 import * as config from "./config";
+import { getSocketEndpoints } from "./server";
 
 const DEBUG_FILE = "" + config.get("DEBUG_FILE");
 const DEBUG_FORMAT = "" + config.get("DEBUG_FORMAT");
@@ -45,13 +46,14 @@ export function incomingHttpRequest(
   if (!DEBUG_FILE) return;
   const now = new Date();
   const con = httpRequest.socket;
+  const socketEndpoints = getSocketEndpoints(con);
   const msg = {
     event: "incoming HTTP request",
     timestamp: now,
-    remoteAddress: con.remoteAddress,
+    remoteAddress: socketEndpoints.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
-    localPort: con.localPort,
+    localPort: socketEndpoints.localPort,
     method: httpRequest.method,
     url: httpRequest.url,
     headers: httpRequest.headers,
@@ -73,10 +75,11 @@ export function outgoingHttpResponse(
   if (!DEBUG_FILE) return;
   const now = new Date();
   const con = httpResponse.socket;
+  const socketEndpoints = getSocketEndpoints(con);
   const msg = {
     event: "outgoing HTTP response",
     timestamp: now,
-    remoteAddress: con.remoteAddress,
+    remoteAddress: socketEndpoints.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
     statusCode: httpResponse.statusCode,
@@ -199,3 +202,20 @@ export function outgoingUdpMessage(
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
 }
+
+export function clientError(remoteAddress: string, err: Error): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "client error",
+    timestamp: now,
+    remoteAddress: remoteAddress,
+    error: err.message,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}

lib/forwarded.ts

@@ -21,6 +21,7 @@ import { IncomingMessage } from "http";
 import { TLSSocket } from "tls";
 import { parseCIDR, parse, IPv6, IPv4 } from "ipaddr.js";
 import * as config from "./config";
+import { getSocketEndpoints } from "./server";
 
 interface RequestOrigin {
   localAddress: string;
@@ -98,19 +99,22 @@ export function getRequestOrigin(request: IncomingMessage): RequestOrigin {
   let origin = cache.get(request);
   if (!origin) {
     const soc = request.socket;
+    const socketEndpoints = getSocketEndpoints(soc);
     origin = {
-      localAddress: soc.localAddress,
+      localAddress: socketEndpoints.localAddress,
-      localPort: soc.localPort,
+      localPort: socketEndpoints.localPort,
-      remoteAddress: soc.remoteAddress,
+      remoteAddress: socketEndpoints.remoteAddress,
-      remotePort: soc.remotePort,
+      remotePort: socketEndpoints.remotePort,
       host: request.headers["host"],
       encrypted: !!(request.socket as TLSSocket).encrypted,
     };
 
     const header = request.headers["forwarded"];
     if (header) {
-      const ip = parse(soc.remoteAddress) as IPv4;
+      const ip = parse(socketEndpoints.remoteAddress);
-      if (cidrs.some((cidr) => ip.match(cidr as [IPv4, number]))) {
+      if (
+        cidrs.some((cidr) => ip.kind() === cidr[0].kind() && ip.match(cidr))
+      ) {
         const parsed = parseForwardedHeader(header);
 
         if (parsed["proto"] === "https") {

lib/local-cache.ts

@@ -380,13 +380,15 @@ async function fetchConfig(): Promise<[Config, UiConfig]> {
 async function refresh(): Promise<void> {
   if (!nextRefresh) {
     await new Promise((resolve) => setTimeout(resolve, 20));
-    await refresh();
+    return refresh();
-    return;
   }
 
-  nextRefresh = 0;
   const now = Date.now();
 
+  if (now < nextRefresh) return;
+
+  nextRefresh = 0;
+
   const dbHash = await cache.get("presets_hash");
 
   if (currentSnapshot && dbHash === currentSnapshot) {
@@ -394,7 +396,7 @@ async function refresh(): Promise<void> {
     return;
   }
 
-  const lockToken = await cache.acquireLock("presets_hash_lock", 5000, 5000);
+  const lockToken = await cache.acquireLock("presets_hash_lock", 5000);
 
   const res = await Promise.all([
     fetchPresets(),
@@ -427,8 +429,12 @@ async function refresh(): Promise<void> {
 
   currentSnapshot = computeHash(snapshot);
   snapshots.set(currentSnapshot, snapshot);
-  await cache.set("presets_hash", currentSnapshot, 300);
+
-  await cache.releaseLock("presets_hash_lock", lockToken);
+  if (lockToken) {
+    if (currentSnapshot !== dbHash)
+      await cache.set("presets_hash", currentSnapshot, 300);
+    await cache.releaseLock("presets_hash_lock", lockToken);
+  }
 
   nextRefresh = now + (REFRESH - (now % REFRESH));
 }

lib/mongodb-functions.ts

@@ -77,7 +77,8 @@ export function processDeviceFilter(filter: Expression): Expression {
       if (typeof e[2] === "number") {
         const alt = (e as any[]).slice();
         alt[2] = new Date(e[2]);
-        e = ["OR", e, alt];
+        if (exp[0] === "<>" || exp[0] === "NOT LIKE") e = ["AND", e, alt];
+        else e = ["OR", e, alt];
       }
       return e;
     }

lib/mongodb-types.ts

@@ -0,0 +1,107 @@
+import { ObjectId } from "mongodb";
+import { Expression, FaultStruct } from "./types";
+
+export interface Fault {
+  _id: string;
+  device: string;
+  channel: string;
+  timestamp: Date;
+  provisions: string;
+  retries: number;
+  code: string;
+  message: string;
+  detail?:
+    | FaultStruct
+    | {
+        name: string;
+        message: string;
+        stack?: string;
+      };
+  expiry?: Date;
+}
+
+interface TaskBase {
+  _id: ObjectId;
+  timestamp?: Date;
+  expiry?: Date;
+  name: string;
+}
+
+interface TaskGetParameterValues extends TaskBase {
+  name: "getParameterValues";
+  parameterNames: string[];
+}
+
+interface TaskSetParameterValues extends TaskBase {
+  name: "setParameterValues";
+  parameterValues: [string, string | number | boolean, string?][];
+}
+
+interface TaskRefreshObject extends TaskBase {
+  name: "refreshObject";
+  objectName: string;
+}
+
+interface TaskReboot extends TaskBase {
+  name: "reboot";
+}
+
+interface TaskFactoryReset extends TaskBase {
+  name: "factoryReset";
+}
+
+interface TaskDownload extends TaskBase {
+  name: "download";
+  fileType: string;
+  fileName: string;
+  targetFileName?: string;
+}
+
+interface TaskAddObject extends TaskBase {
+  name: "addObject";
+  objectName: string;
+  parameterValues: [string, string | number | boolean, string?][];
+}
+
+interface TaskDeleteObject extends TaskBase {
+  name: "deleteObject";
+  objectName: string;
+}
+
+interface TaskProvisions extends TaskBase {
+  name: "provisions";
+  provisions?: [string, ...Expression[]][];
+}
+
+export type Task =
+  | TaskGetParameterValues
+  | TaskSetParameterValues
+  | TaskRefreshObject
+  | TaskReboot
+  | TaskFactoryReset
+  | TaskDownload
+  | TaskAddObject
+  | TaskDeleteObject
+  | TaskProvisions;
+
+export interface Operation {
+  _id: string;
+  name: string;
+  timestamp: Date;
+  channels: string;
+  retries: string;
+  provisions: string;
+  args: string;
+}
+
+export interface Config {
+  _id: string;
+  value: string;
+}
+
+export interface Cache {
+  _id: string;
+  value: string;
+  timestamp: Date;
+  expire: Date;
+}

lib/nbi.ts

@@ -21,7 +21,7 @@ import * as url from "url";
 import { Collection, GridFSBucket, ObjectId } from "mongodb";
 import * as vm from "vm";
 import * as config from "./config";
-import { onConnect } from "./db";
+import { onConnect, optimizeProjection } from "./db";
 import * as query from "./query";
 import * as apiFunctions from "./api-functions";
 import { IncomingMessage, ServerResponse } from "http";
@@ -30,6 +30,7 @@ import { version as VERSION } from "../package.json";
 import { ping } from "./ping";
 import * as logger from "./logger";
 import { flattenDevice } from "./mongodb-functions";
+import { getSocketEndpoints } from "./server";
 
 const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
 const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
@@ -101,10 +102,11 @@ export function listener(
   request.addListener("end", () => {
     const body = getBody();
     const urlParts = url.parse(request.url, true);
+    const socketEndpoints = getSocketEndpoints(request.socket);
 
     logger.accessInfo(
       Object.assign({}, urlParts.query, {
-        remoteAddress: request.socket.remoteAddress,
+        remoteAddress: socketEndpoints.remoteAddress,
         message: `${request.method} ${urlParts.pathname}`,
       })
     );
@@ -819,6 +821,7 @@ export function listener(
         projection = {};
         for (const p of (urlParts.query.projection as string).split(","))
           projection[p.trim()] = 1;
+        projection = optimizeProjection(projection);
       }
 
       const cur = collection.find(q, { projection: projection });
@@ -842,7 +845,7 @@ export function listener(
         cur.sort(sort);
       }
 
-      cur.count((err, total) => {
+      collection.countDocuments(q, (err, total) => {
         if (err) return void throwError(err);
 
         response.writeHead(200, {

lib/ping.ts

@@ -40,68 +40,78 @@ function isValidHost(host: string): boolean {
   return /^[a-zA-Z0-9\-.:[\]-]+$/.test(domainToASCII(host));
 }
 
-export function ping(
+export function parsePing(osPlatform: string, stdout: string): PingResult {
-  host: string,
+  let parseRegExp1: RegExp, parseRegExp2: RegExp, parsed: PingResult;
-  callback: (err: Error, res?: PingResult, stdout?: string) => void
+  switch (osPlatform) {
-): void {
-  let cmd: string, parseRegExp1: RegExp, parseRegExp2: RegExp;
-  // Validate input to prevent possible remote code execution
-  // Credit to Alex Hordijk for reporting this vulnerability
-  if (!isValidHost(host)) return callback(new Error("Invalid host"));
-  host = host.replace("[", "").replace("]", "");
-  switch (platform()) {
     case "linux":
-      cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
       parseRegExp1 =
-        /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
+        /(\d+) packets transmitted, (\d+) .*received, ([\d.]+)% .*loss[^]*= ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
       parseRegExp2 =
-        /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss/;
+        /(\d+) packets transmitted, (\d+) .*received, ([\d.]+)% .*loss/;
       break;
 
     case "freebsd":
-      // Send a single packet because on FreeBSD only superuser can send
-      // packets that are only 200 ms apart.
-      cmd = `ping -t 1 -c 3 ${host}`;
       parseRegExp1 =
         /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
       parseRegExp2 =
         /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
       break;
+  }
 
+  const m1 = stdout.match(parseRegExp1);
+  if (m1) {
+    parsed = {
+      packetsTransmitted: +m1[1],
+      packetsReceived: +m1[2],
+      packetLoss: +m1[3],
+      min: +m1[4],
+      avg: +m1[5],
+      max: +m1[6],
+      mdev: +m1[7],
+    };
+  } else {
+    const m2 = stdout.match(parseRegExp2);
+    if (m2) {
+      parsed = {
+        packetsTransmitted: +m2[1],
+        packetsReceived: +m2[2],
+        packetLoss: +m2[3],
+        min: null,
+        avg: null,
+        max: null,
+        mdev: null,
+      };
+    }
+  }
+  return parsed;
+}
+
+export function ping(
+  host: string,
+  callback: (err: Error, res?: PingResult, stdout?: string) => void
+): void {
+  // Validate input to prevent possible remote code execution
+  // Credit to Alex Hordijk for reporting this vulnerability
+  if (!isValidHost(host)) return callback(new Error("Invalid host"));
+  host = host.replace("[", "").replace("]", "");
+  let cmd: string;
+
+  switch (platform()) {
+    case "linux":
+      cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
+      break;
+    case "freebsd":
+      // Send a single packet because on FreeBSD only superuser can send
+      // packets that are only 200 ms apart.
+      cmd = `ping -t 1 -c 3 ${host}`;
+      break;
     default:
       return callback(new Error("Platform not supported"));
   }
 
   exec(cmd, (err, stdout) => {
-    let parsed: PingResult;
+    if (err) return callback(err);
-    if (stdout) {
+    const parsed: PingResult = parsePing(platform(), stdout);
-      const m1 = stdout.match(parseRegExp1);
+    return callback(err, parsed, stdout);
-      if (m1) {
-        parsed = {
-          packetsTransmitted: +m1[1],
-          packetsReceived: +m1[2],
-          packetLoss: +m1[3],
-          min: +m1[4],
-          avg: +m1[5],
-          max: +m1[6],
-          mdev: +m1[7],
-        };
-      } else {
-        const m2 = stdout.match(parseRegExp2);
-        if (m2) {
-          parsed = {
-            packetsTransmitted: +m2[1],
-            packetsReceived: +m2[2],
-            packetLoss: +m2[3],
-            min: null,
-            avg: null,
-            max: null,
-            mdev: null,
-          };
-        }
-      }
-    }
-
-    callback(err, parsed, stdout);
   });
 }

lib/query.ts

@@ -54,6 +54,17 @@ function normalize(input): any {
   return input;
 }
 
+const EXPAND_OPS = new Set([
+  "$eq",
+  "$gt",
+  "$gte",
+  "$in",
+  "$lt",
+  "$lte",
+  "$ne",
+  "$nin",
+]);
+
 function expandValue(value: unknown): unknown[] {
   if (Array.isArray(value)) {
     let a = [];
@@ -71,7 +82,8 @@ function expandValue(value: unknown): unknown[] {
   const values = [];
   for (const [k, v] of Object.entries(value)) {
     keys.push(k);
-    values.push(expandValue(v));
+    if (EXPAND_OPS.has(k)) values.push(expandValue(v));
+    else values.push([v]);
     indices.push(0);
   }
 

lib/sandbox.ts

@@ -255,6 +255,7 @@ function declare(
   }
 
   for (const [attrName, attrTimestamp] of Object.entries(timestamps)) {
+    if (!(attrTimestamp >= 1)) continue;
     if (attrName === "path") {
       declaration.pathGet = attrTimestamp;
     } else {

lib/server.ts

@@ -59,8 +59,20 @@ interface ServerOptions {
   timeout?: number;
   keepAliveTimeout?: number;
   onConnection?: (socket: Socket) => void;
+  onClientError?: (err: Error, socket: Socket) => void;
 }
 
+interface SocketEndpoint {
+  localAddress: string;
+  localPort: number;
+  remoteAddress: string;
+  remotePort: number;
+  remoteFamily: "IPv4" | "IPv6";
+}
+
+// Save this info as they're not accessible after a socket has been closed
+const socketEndpoints: WeakMap<Socket, SocketEndpoint> = new WeakMap();
+
 export function start(
   options: ServerOptions,
   _listener: http.RequestListener
@@ -85,6 +97,27 @@ export function start(
     if (options.onConnection) server.on("connection", options.onConnection);
   }
 
+  server.on("connection", (socket: Socket) => {
+    socketEndpoints.set(socket, {
+      localAddress: socket.localAddress,
+      localPort: socket.localPort,
+      remoteAddress: socket.remoteAddress,
+      remotePort: socket.remotePort,
+      remoteFamily: socket.remoteFamily as "IPv4" | "IPv6",
+    });
+  });
+
+  if (options.onClientError) {
+    server.on("clientError", (err, socket: Socket) => {
+      if (err["code"] !== "ECONNRESET" && socket.writable)
+        socket.end("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n");
+
+      // As per Node docs: This event is guaranteed to be passed an instance
+      // of the <net.Socket> class
+      options.onClientError(err, socket as Socket);
+    });
+  }
+
   server.timeout = options.timeout || 0;
   if (options.keepAliveTimeout != null)
     server.keepAliveTimeout = options.keepAliveTimeout;
@@ -99,3 +132,8 @@ export function stop(): Promise<void> {
     closeServer(20000, resolve);
   });
 }
+
+export function getSocketEndpoints(socket: Socket): SocketEndpoint {
+  // TLSSocket keeps a reference to the raw TCP socket in _parent
+  return socketEndpoints.get(socket["_parent"] ?? socket);
+}

lib/session.ts

@@ -504,7 +504,7 @@ export async function timeoutOperations(
 export function addProvisions(
   sessionContext: SessionContext,
   channel: string,
-  provisions: (string | number | boolean)[][]
+  provisions: [string, ...Expression[]][]
 ): void {
   // Multiply by two because every iteration is two
   // phases: read and update
@@ -1624,6 +1624,20 @@ function generateGetRpcRequest(
       (a, b) => b.length - a.length
     );
     let path = paths.pop();
+
+    // Skip root GPN workaround
+    if (path && !path.length) {
+      const SKIP_ROOT_GPN = !!localCache.getConfig(
+        sessionContext.cacheSnapshot,
+        "cwmp.skipRootGpn",
+        {},
+        sessionContext.timestamp,
+        (e) => configContextCallback(sessionContext, e)
+      );
+
+      if (SKIP_ROOT_GPN) path = paths.pop();
+    }
+
     while (
       path &&
       path.length &&
@@ -2678,7 +2692,10 @@ export async function rpcResponse(
     const wildcardParams: Path[] = [root.concat(wildcardPath)];
 
     for (const [path, object, writable] of rpcRes.parameterList) {
-      if (!path.toString().startsWith(rpcReq.parameterPath)) {
+      if (
+        !path.toString().startsWith(rpcReq.parameterPath) &&
+        !(`${path.toString()}.` === rpcReq.parameterPath && !rpcReq.nextLevel)
+      ) {
         logger.accessWarn({
           sessionContext: sessionContext,
           message: "Unexpected parameter in response",

lib/types.ts

@@ -180,7 +180,7 @@ export interface Task {
   fileName?: string;
   targetFileName?: string;
   expiry?: number;
-  provisions?: (string | number | boolean)[][];
+  provisions?: [string, ...Expression[]][];
 }
 
 export interface Operation {

lib/ui.ts

@@ -228,7 +228,7 @@ router.get("/", async (ctx) => {
   if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)
     wizard = '<script>window.location.hash = "#!/wizard";</script>';
 
-  ctx.body = `
+  ctx.body = `<!DOCTYPE html>
   <html>
     <head>
       <title>GenieACS</title>
@@ -263,6 +263,7 @@ koa.use(
       flush: constants.Z_SYNC_FLUSH,
     },
     br: {
+      flush: constants.BROTLI_OPERATION_FLUSH,
       params: {
         [constants.BROTLI_PARAM_QUALITY]: 5,
       },

lib/ui/api.ts

@@ -125,7 +125,7 @@ router.get(`/devices/:id.csv`, async (ctx) => {
       new Date(p.objectTimestamp).toJSON(),
       p.writable,
       new Date(p.writableTimestamp).toJSON(),
-      `"${value.toString().replace(/"/g, '""')}"`,
+      `"${String(value).replace(/"/g, '""')}"`,
       type,
       new Date(p.valueTimestamp).toJSON(),
       p.notification,
@@ -221,12 +221,16 @@ for (const [resource, flags] of Object.entries(resources)) {
 
     let c = 0;
     ctx.body.write("[\n");
-    await db.query(resource, filter, options, (obj) => {
+    db.query(resource, filter, options, (obj) => {
       ctx.body.write((c++ ? "," : "") + JSON.stringify(obj) + "\n");
-    });
+    })
-    ctx.body.end("]");
+      .then(() => {
-
+        ctx.body.end("]");
-    logger.accessInfo(log);
+        logger.accessInfo(log);
+      })
+      .catch((err) => {
+        ctx.body.emit("error", err);
+      });
   });
 
   // CSV download
@@ -285,7 +289,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     ctx.body.write(
       Object.keys(columns).map((k) => `"${k.replace(/"/, '""')}"`) + "\n"
     );
-    await db.query(resource, filter, options, (obj) => {
+    db.query(resource, filter, options, (obj) => {
       const arr = Object.values(columns).map((exp) => {
         const v = evaluate(exp, obj, null, (e) => {
           if (Array.isArray(e)) {
@@ -327,10 +331,14 @@ for (const [resource, flags] of Object.entries(resources)) {
         return v;
       });
       ctx.body.write(arr.join(",") + "\n");
-    });
+    })
-    ctx.body.end();
+      .then(() => {
-
+        ctx.body.end();
-    logger.accessInfo(log);
+        logger.accessInfo(log);
+      })
+      .catch((err) => {
+        ctx.body.emit("error", err);
+      });
   });
 
   router.head(`/${resource}/:id`, async (ctx) => {
@@ -652,6 +660,7 @@ router.post("/devices/:id/tags", async (ctx) => {
 });
 
 router.get("/ping/:host", async (ctx) => {
+  if (!ctx.state.user) return void (ctx.status = 401);
   return new Promise<void>((resolve) => {
     ping(ctx.params.host, (err, parsed) => {
       if (parsed) {

lib/ui/db.ts

@@ -19,7 +19,7 @@
 
 import { Db, GridFSBucket, ObjectId } from "mongodb";
 import { Script } from "vm";
-import { onConnect } from "../db";
+import { onConnect, optimizeProjection } from "../db";
 import * as mongodbFunctions from "../mongodb-functions";
 import * as expression from "../common/expression";
 import { QueryOptions, Expression } from "../types";
@@ -89,6 +89,7 @@ export function query(
       }
 
       if (resource === "presets") projection.configurations = 1;
+      projection = optimizeProjection(projection);
       cursor.project(projection);
     }
 
@@ -114,30 +115,31 @@ export function query(
       cursor.toArray((err, docs) => {
         if (err) return reject(err);
         if (resource === "devices")
-          docs = docs.map((d) => mongodbFunctions.flattenDevice(d));
+          return resolve(docs.map((d) => mongodbFunctions.flattenDevice(d)));
         else if (resource === "faults")
-          docs = docs.map((d) => mongodbFunctions.flattenFault(d));
+          return resolve(docs.map((d) => mongodbFunctions.flattenFault(d)));
         else if (resource === "tasks")
-          docs = docs.map((d) => mongodbFunctions.flattenTask(d));
+          return resolve(docs.map((d) => mongodbFunctions.flattenTask(d)));
         else if (resource === "presets")
-          docs = docs.map((d) => mongodbFunctions.flattenPreset(d));
+          return resolve(docs.map((d) => mongodbFunctions.flattenPreset(d)));
         else if (resource === "files")
-          docs = docs.map((d) => mongodbFunctions.flattenFile(d));
+          return resolve(docs.map((d) => mongodbFunctions.flattenFile(d)));
         return resolve(docs);
       });
     } else {
       cursor.forEach(
         (doc) => {
-          if (resource === "devices") doc = mongodbFunctions.flattenDevice(doc);
+          if (resource === "devices")
+            callback(mongodbFunctions.flattenDevice(doc));
           else if (resource === "faults")
-            doc = mongodbFunctions.flattenFault(doc);
+            callback(mongodbFunctions.flattenFault(doc));
           else if (resource === "tasks")
-            doc = mongodbFunctions.flattenTask(doc);
+            callback(mongodbFunctions.flattenTask(doc));
           else if (resource === "presets")
-            doc = mongodbFunctions.flattenPreset(doc);
+            callback(mongodbFunctions.flattenPreset(doc));
           else if (resource === "files")
-            doc = mongodbFunctions.flattenFile(doc);
+            callback(mongodbFunctions.flattenFile(doc));
-          callback(doc);
+          else callback(doc);
         },
         (err) => {
           if (err) reject(err);
@@ -149,7 +151,8 @@ export function query(
 }
 
 export function count(resource: string, filter: Expression): Promise<number> {
-  let q;
+  const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
+  let q: Parameters<typeof collection.countDocuments>[0];
   filter = expression.evaluate(filter, null, Date.now());
   filter = minimize(filter, true);
 
@@ -165,13 +168,7 @@ export function count(resource: string, filter: Expression): Promise<number> {
     return Promise.resolve(0);
   }
 
-  return new Promise((resolve, reject) => {
+  return collection.countDocuments(q);
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-    collection.find(q).count((err, c) => {
-      if (err) reject(err);
-      else resolve(c);
-    });
-  });
 }
 
 export async function updateDeviceTags(

lib/versioned-map.ts

@@ -168,6 +168,7 @@ export default class VersionedMap<K, V> {
 
   public *diff(): IterableIterator<[K, V, V]> {
     for (const [key, revisions] of this.map) {
+      if (revisions.length === 1) continue;
       let first = revisions[0];
       let last = revisions[revisions.length - 1];
       if (first === NONEXISTENT && last === NONEXISTENT) continue;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.8",
+  "version": "1.2.9",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -28,46 +28,47 @@
     "koa-bodyparser": "^4.3.0",
     "koa-compress": "^5.1.0",
     "koa-jwt": "^4.0.3",
-    "koa-router": "^10.1.1",
+    "koa-router": "^12.0.0",
     "koa-static": "^5.0.0",
-    "mongodb": "^4.1.3",
+    "mongodb": "^4.9.0",
-    "parsimmon": "^1.18.0",
+    "parsimmon": "^1.18.1",
     "seedrandom": "^3.0.5"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^21.0.1",
+    "@rollup/plugin-commonjs": "^22.0.2",
     "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^13.0.6",
+    "@rollup/plugin-node-resolve": "^13.3.0",
-    "@rollup/plugin-typescript": "^8.3.0",
+    "@rollup/plugin-typescript": "^8.3.4",
     "@types/codemirror": "^5.60.5",
-    "@types/jsonwebtoken": "^8.5.5",
+    "@types/jsonwebtoken": "^8.5.8",
-    "@types/koa": "^2.13.4",
+    "@types/koa": "^2.13.5",
     "@types/koa-compress": "^4.0.3",
     "@types/koa-router": "^7.4.4",
     "@types/koa-static": "^4.0.2",
-    "@types/mithril": "^2.0.8",
+    "@types/mithril": "^2.0.11",
-    "@types/node": "^16.11.6",
+    "@types/node": "^18.7.9",
     "@types/parsimmon": "^1.10.6",
-    "@types/seedrandom": "^3.0.1",
+    "@types/seedrandom": "^3.0.2",
-    "@typescript-eslint/eslint-plugin": "^5.2.0",
+    "@typescript-eslint/eslint-plugin": "^5.33.1",
-    "@typescript-eslint/parser": "^5.2.0",
+    "@typescript-eslint/parser": "^5.33.1",
     "ava": "^3.15.0",
-    "codemirror": "^5.63.3",
+    "codemirror": "^5.65.8",
-    "cssnano": "^5.0.8",
+    "cssnano": "^5.1.13",
-    "eslint": "^8.1.0",
+    "eslint": "^8.22.0",
-    "eslint-config-prettier": "^8.3.0",
+    "eslint-config-prettier": "^8.5.0",
     "esm": "^3.2.25",
-    "mithril": "^2.0.4",
+    "mithril": "^2.2.2",
-    "postcss": "^8.3.11",
+    "postcss": "^8.4.16",
-    "postcss-import": "^14.0.2",
+    "postcss-color-mod-function": "^3.0.3",
-    "postcss-preset-env": "^6.7.0",
+    "postcss-import": "^14.1.0",
-    "prettier": "^2.4.1",
+    "postcss-preset-env": "^7.8.0",
-    "rollup": "^2.58.3",
+    "prettier": "^2.7.1",
+    "rollup": "^2.78.1",
     "rollup-plugin-terser": "^7.0.2",
-    "sql.js": "^1.6.2",
+    "sql.js": "^1.7.0",
-    "svgo": "^2.7.0",
+    "svgo": "^2.8.0",
-    "ts-node": "^10.4.0",
+    "ts-node": "^10.9.1",
-    "typescript": "^4.4.4",
+    "typescript": "^4.7.4",
     "yaml": "^1.10.2"
   },
   "engines": {

test/boolean-expression.ts

@@ -1,7 +1,7 @@
 import ava, { TestInterface } from "ava";
 import { minimize, unionDiff } from "../lib/common/boolean-expression";
 import { parse, stringify } from "../lib/common/expression-parser";
-import initSqlJs from "sql.js";
+import initSqlJs from "sql.js/dist/sql-asm.js";
 
 const test = ava as TestInterface<{
   query: (filter: string) => Promise<Set<number>>;

test/mongodb-functions.ts

@@ -80,6 +80,24 @@ ava("filterToMongoQuery", async (t) => {
     ["Tags.tag1 = 123", { "Tags.tag1": 123 }],
     ["Param1 = 'value1'", { "Param1._value": "value1" }],
     ["Param1 <> 'value1'", { "Param1._value": { $nin: ["value1", null] } }],
+    [
+      "Param1 <> 1657844103524",
+      {
+        $and: [
+          { "Param1._value": { $nin: [1657844103524, null] } },
+          { "Param1._value": { $nin: [new Date(1657844103524), null] } },
+        ],
+      },
+    ],
+    [
+      "Param1 = 1657844103524",
+      {
+        $or: [
+          { "Param1._value": 1657844103524 },
+          { "Param1._value": new Date(1657844103524) },
+        ],
+      },
+    ],
     ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }],
     ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }],
     ["Param1 LIKE 'value'", { "Param1._value": /^value$/ }],

test/ping.ts

@@ -0,0 +1,36 @@
+import ava from "ava";
+import { parsePing } from "../lib/ping";
+
+ava("linux Case-1", (t) => {
+  const platform = "linux";
+  const stdout =
+    "PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data\n64 bytes from 8.8.8.8: icmp_seq=1 ttl=118 time=6.66 ms\n64 bytes from 8.8.8.8: icmp_seq=2 ttl=118 time=6.50 ms\n64 bytes from 8.8.8.8: icmp_seq=3 ttl=118 time=6.38 ms\n\n--- 8.8.8.8 ping statistics ---\n3 packets transmitted, 3 received, 0% packet loss, time 402ms\nrtt min/avg/max/mdev = 6.381/6.511/6.656/0.112 ms";
+  const parsedResult = {
+    packetsTransmitted: 3,
+    packetsReceived: 3,
+    packetLoss: 0,
+    min: 6.381,
+    avg: 6.511,
+    max: 6.656,
+    mdev: 0.112,
+  };
+  const parsed = parsePing(platform, stdout);
+  t.deepEqual(parsedResult, parsed);
+});
+
+ava("linux Case-2", (t) => {
+  const platform = "linux";
+  const stdout =
+    "PING 10.251.9.108 (10.251.9.108): 56 data bytes\n64 bytes from 10.251.9.108: icmp_seq=0 ttl=57 time=36.758 ms\n64 bytes from 10.251.9.108: icmp_seq=1 ttl=57 time=69.094 ms\n64 bytes from 10.251.9.108: icmp_seq=2 ttl=57 time=28.868 ms\n--- 10.251.9.108 ping statistics ---3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max/stddev = 28.868/44.907/69.094/17.404 ms";
+  const parsedResult = {
+    packetsTransmitted: 3,
+    packetsReceived: 3,
+    packetLoss: 0,
+    min: 28.868,
+    avg: 44.907,
+    max: 69.094,
+    mdev: 17.404,
+  };
+  const parsed = parsePing(platform, stdout);
+  t.deepEqual(parsedResult, parsed);
+});

ui/autocomplete-compnent.ts

@@ -99,9 +99,9 @@ export default class Autocomplete {
       if (this.visible) this.hide();
       return;
     }
-    this.container.style.left = `${domRect.left + window.pageXOffset}px`;
+    this.container.style.left = `${domRect.left + window.scrollX}px`;
     this.container.style.width = `${domRect.width}px`;
-    this.container.style.top = `${domRect.bottom + window.pageYOffset}px`;
+    this.container.style.top = `${domRect.bottom + window.scrollY}px`;
   }
 
   private hide(): void {

ui/components/all-parameters.ts

@@ -24,6 +24,7 @@ import { parse } from "../../lib/common/expression-parser";
 import memoize from "../../lib/common/memoize";
 import { getIcon } from "../icons";
 import { evaluateExpression } from "../store";
+import debounce from "../../lib/common/debounce";
 
 const memoizedParse = memoize(parse);
 
@@ -52,6 +53,12 @@ function orderKeysByDepth(device: Record<string, unknown>): string[][] {
 }
 
 const component: ClosureComponent = (): Component => {
+  let queryString: string;
+  const formQueryString = debounce((args: string[]) => {
+    queryString = args[args.length - 1];
+    m.redraw();
+  }, 500);
+
   return {
     view: (vnode) => {
       const device = vnode.attrs["device"];
@@ -62,19 +69,15 @@ const component: ClosureComponent = (): Component => {
         type: "text",
         placeholder: "Search parameters",
         oninput: (e) => {
-          vnode.state["searchString"] = e.target.value;
+          formQueryString(e.target.value);
           e.redraw = false;
-          clearTimeout(vnode.state["timeout"]);
-          vnode.state["timeout"] = setTimeout(m.redraw, 500);
         },
       });
 
       const instanceRegex = /\.[0-9]+$/;
       let re;
-      if (vnode.state["searchString"]) {
+      if (queryString) {
-        const keywords = vnode.state["searchString"]
+        const keywords = queryString.split(" ").filter((s) => s);
-          .split(" ")
-          .filter((s) => s);
         if (keywords.length)
           re = new RegExp(keywords.map((s) => escapeRegExp(s)).join(".*"), "i");
       }

ui/css/app.css

@@ -87,7 +87,7 @@ i {
 }
 
 body {
-  height: 100%;
+  height: 100vh;
   display: flex;
   flex-direction: column;
 }
@@ -169,6 +169,7 @@ body {
 
         & input {
           width: 100%;
+          box-sizing: border-box;
           padding: 0.3em;
           margin-top: 0.3em;
           margin-bottom: 0.3em;
@@ -291,6 +292,7 @@ body {
     & > img {
       margin: 10px;
       height: 56px;
+      vertical-align: top;
     }
 
     & > .version {
@@ -581,6 +583,7 @@ span.na {
 .all-parameters {
   & > input {
     font-family: monospace;
+    font-size: 14px;
     margin-bottom: 2px;
     padding: 4px;
     width: 700px;
@@ -588,6 +591,10 @@ span.na {
 
   & > .parameter-list {
     font-family: monospace;
+    font-size: 14px;
+    & button {
+      font-size: 16px;
+    }
     overflow-y: scroll;
     height: 200px;
 
@@ -641,7 +648,7 @@ span.na {
 
 span.tag {
   display: inline-block;
-  background-image: url('data:image/svg+xml;utf-8,<svg xmlns="http://www.w3.org/2000/svg" width="190" height="16"><path d="M6.59 15.42l-6-6a2 2 0 0 1 0-2.83l6-6A2 2 0 0 1 8 0h180a2 2 0 0 1 2 2v12a2 2 0 0 1-2 2H8a2 2 0 0 1-1.41-.58zM4.35 6.59A2 2 0 1 0 7.17 9.4 2 2 0 0 0 4.35 6.6z" fill="#e8d5b2"></path></svg>');
+  background-image: url('data:image/svg+xml;utf-8,<svg xmlns="http://www.w3.org/2000/svg" width="190" height="16"><path d="M6.59 15.42l-6-6a2 2 0 0 1 0-2.83l6-6A2 2 0 0 1 8 0h180a2 2 0 0 1 2 2v12a2 2 0 0 1-2 2H8a2 2 0 0 1-1.41-.58zM4.35 6.59A2 2 0 1 0 7.17 9.4 2 2 0 0 0 4.35 6.6z" fill="%23e8d5b2"></path></svg>');
   background-size: cover;
   background-repeat: no-repeat;
   border-radius: 0.2em;
@@ -725,6 +732,7 @@ span.long-text-overflowed {
 
 textarea.long-text {
   font-family: monospace;
+  font-size: 14px;
   min-width: 10vw;
   min-height: 10vh;
 }
@@ -746,8 +754,7 @@ span.parameter-value {
 }
 
 .icon {
-  display: inline-block;
+  display: block;
-  vertical-align: middle;
   width: 1em;
   height: 1em;
   fill: none;
@@ -797,3 +804,9 @@ span.parameter-value {
     background-color: #b12d5c;
   }
 }
+
+/* To keep consistent style after changing from Quirks to Standards mode */
+td,
+th {
+  line-height: normal;
+}

ui/devices-page.ts

@@ -268,18 +268,14 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortedAttrs): void {
-        let _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortedAttrs)) {
+        for (const index of sortedAttrs) {
-          const param = memoizedGetSortable(attributes[index].parameter);
+          const param = memoizedGetSortable(
-          if (param) {
+            attributes[Math.abs(index) - 1].parameter
-            // Changing the priority of columns
+          );
-            delete _sort[param];
+          _sort[param] = Math.sign(index);
-            if (direction) _sort = Object.assign({ [param]: direction }, _sort);
-          }
         }
-
+        const ops = { sort: JSON.stringify(_sort) };
-        const ops = {};
-        if (Object.keys(_sort).length) ops["sort"] = JSON.stringify(_sort);
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/devices", ops);
       }

ui/drawer-component.ts

@@ -388,7 +388,7 @@ const component: ClosureComponent = (): Component => {
       function repositionNotifications(): void {
         let top = 10;
         for (const c of notificationElements as VnodeDOM[]) {
-          (c.dom as HTMLDivElement).style.top = `${top}`;
+          (c.dom as HTMLDivElement).style.top = `${top}px`;
           top += (c.dom as HTMLDivElement).offsetHeight + 10;
         }
       }
@@ -408,7 +408,7 @@ const component: ClosureComponent = (): Component => {
           for (const c of drawerElement.children)
             height = Math.max(height, c.dom.offsetTop + c.dom.offsetHeight);
         }
-        drawerElement.dom.style.height = height;
+        drawerElement.dom.style.height = height + "px";
       }
 
       if (stagingElements.length + queueElements.length) {

ui/faults-page.ts

@@ -107,16 +107,10 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortAttrs): void {
-        let _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
+        const ops = { sort: JSON.stringify(_sort) };
-          if (direction)
-            _sort = Object.assign({ [attributes[index].id]: direction }, _sort);
-        }
-
-        const ops = {};
-        if (Object.keys(_sort).length) ops["sort"] = JSON.stringify(_sort);
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/faults", ops);
       }

ui/files-page.ts

@@ -151,16 +151,10 @@ export const component: ClosureComponent = (): Component => {
         sortAttributes[i] = sort[attributes[i].id] || 0;
 
       function onSortChange(sortAttrs): void {
-        let _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
+        const ops = { sort: JSON.stringify(_sort) };
-          if (direction)
-            _sort = Object.assign({ [attributes[index].id]: direction }, _sort);
-        }
-
-        const ops = {};
-        if (Object.keys(_sort).length) ops["sort"] = JSON.stringify(_sort);
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/files", ops);
       }

ui/index-table-component.ts

@@ -20,6 +20,7 @@
 import { ClosureComponent, Component, Children } from "mithril";
 import { m } from "./components";
 import { getIcon } from "./icons";
+import debounce from "../lib/common/debounce";
 
 interface Attribute {
   id?: string;
@@ -36,7 +37,7 @@ function renderTable(
   showMoreCallback: () => void,
   selected: Set<string>,
   sortAttributes: Record<string, any>,
-  onSortChange: (obj: Record<string, number>) => void,
+  onSort: (i: number) => void,
   downloadUrl?: string,
   valueCallback?: (attr: Attribute, record: Record<string, any>) => Children,
   actionsCallback?: Children | ((sel: Set<string>) => Children),
@@ -86,9 +87,9 @@ function renderTable(
     const sortable = m(
       "button",
       {
-        onclick: () => {
+        onclick: (e) => {
-          const direction = ((sortAttributes[i] + 2) % 3) - 1;
+          e.redraw = false;
-          return onSortChange({ [i]: direction });
+          onSort(i);
         },
       },
       symbol
@@ -213,7 +214,10 @@ function renderTable(
 
 const component: ClosureComponent = (): Component => {
   let selected = new Set<string>();
-
+  let sortingfunction: (events: number[]) => void;
+  const onSort = debounce((events: number[]) => {
+    sortingfunction(events);
+  }, 500);
   return {
     view: (vnode) => {
       const attributes = vnode.attrs["attributes"];
@@ -232,6 +236,20 @@ const component: ClosureComponent = (): Component => {
         const id = record["_id"] || record["DeviceID.ID"].value[0];
         if (selected.has(id)) _selected.add(id);
       }
+
+      sortingfunction = (events) => {
+        const sortArray = new Set(
+          Object.keys(sortAttributes)
+            .map((x) => (parseInt(x) + 1) * sortAttributes[x])
+            .filter((x) => x)
+        );
+        for (const num of events) {
+          if (sortArray.delete(num + 1)) sortArray.add(-(num + 1));
+          else if (!sortArray.delete((num + 1) * -1)) sortArray.add(num + 1);
+        }
+        onSortChange(Array.from(sortArray).reverse());
+      };
+
       selected = _selected;
 
       return renderTable(
@@ -241,7 +259,7 @@ const component: ClosureComponent = (): Component => {
         showMoreCallback,
         selected,
         sortAttributes,
-        onSortChange,
+        onSort,
         downloadUrl,
         valueCallback,
         actionsCallback,

ui/permissions-page.ts

@@ -205,16 +205,10 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortAttrs): void {
-        let _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
+        const ops = { sort: JSON.stringify(_sort) };
-          if (direction)
-            _sort = Object.assign({ [attributes[index].id]: direction }, _sort);
-        }
-
-        const ops = {};
-        if (Object.keys(_sort).length) ops["sort"] = JSON.stringify(_sort);
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/permissions", ops);
       }

ui/presets-page.ts

@@ -190,16 +190,10 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortAttrs): void {
-        let _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
+        const ops = { sort: JSON.stringify(_sort) };
-          if (direction)
-            _sort = Object.assign({ [attributes[index].id]: direction }, _sort);
-        }
-
-        const ops = {};
-        if (Object.keys(_sort).length) ops["sort"] = JSON.stringify(_sort);
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/presets", ops);
       }

ui/provisions-page.ts

@@ -173,16 +173,10 @@ export const component: ClosureComponent = (): Component => {
         sortAttributes[i] = sort[attributes[i].id] || 0;
 
       function onSortChange(sortAttrs): void {
-        let _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
+        const ops = { sort: JSON.stringify(_sort) };
-          if (direction)
-            _sort = Object.assign({ [attributes[index].id]: direction }, _sort);
-        }
-
-        const ops = {};
-        if (Object.keys(_sort).length) ops["sort"] = JSON.stringify(_sort);
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/provisions", ops);
       }

ui/users-page.ts

@@ -186,16 +186,10 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortAttrs): void {
-        let _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
+        const ops = { sort: JSON.stringify(_sort) };
-          if (direction)
-            _sort = Object.assign({ [attributes[index].id]: direction }, _sort);
-        }
-
-        const ops = {};
-        if (Object.keys(_sort).length) ops["sort"] = JSON.stringify(_sort);
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/users", ops);
       }

ui/virtual-parameters-page.ts

@@ -173,16 +173,10 @@ export const component: ClosureComponent = (): Component => {
         sortAttributes[i] = sort[attributes[i].id] || 0;
 
       function onSortChange(sortAttrs): void {
-        let _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
+        const ops = { sort: JSON.stringify(_sort) };
-          if (direction)
-            _sort = Object.assign({ [attributes[index].id]: direction }, _sort);
-        }
-
-        const ops = {};
-        if (Object.keys(_sort).length) ops["sort"] = JSON.stringify(_sort);
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/virtualParameters", ops);
       }