Complete implementation of ChangeDUState

Lookbehind in regex is not supported in Safari browsers. This implementation
should be more efficient so why not.

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Change Log
 
+## 1.2.5 (2021-03-12)
+
+- Support specifying custom types when uploading files.
+- Fix JS compatibility issue with Safari browser.
+
 ## 1.2.4 (2021-02-24)
 
 - The data model state of a CPE is no longer forgotten after unsuccessful

README.md

@@ -17,7 +17,7 @@ supported versions are:
 
 Install GenieACS from NPM:
 
-    sudo npm install -g --unsafe-perm genieacs
+    sudo npm install -g genieacs
 
 To build from source instead, clone this repo or download the source archive
 then _cd_ into the source directory then run:

docs/installation-guide.rst

@@ -34,7 +34,7 @@ Install GenieACS
 
   .. parsed-literal::
 
-    sudo npm install -g --unsafe-perm genieacs@\ |release|
+    sudo npm install -g genieacs@\ |release|
 
 .. topic:: Installing from source
 

lib/api-functions.ts

@@ -145,15 +145,24 @@ export async function connectionRequest(
 
   const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);
 
-  let udpProm;
+  let udpProm = Promise.resolve(false);
   if (udpConnectionRequestAddress) {
-    udpProm = udpConnectionRequest(
+    try {
-      udpConnectionRequestAddress,
+      const u = new URL("udp://" + udpConnectionRequestAddress);
-      authExp,
+      udpProm = udpConnectionRequest(
-      UDP_CONNECTION_REQUEST_PORT,
+        u.hostname,
-      debug,
+        parseInt(u.port || "80"),
-      deviceId
+        authExp,
-    );
+        UDP_CONNECTION_REQUEST_PORT,
+        debug,
+        deviceId
+      ).then(
+        () => true,
+        () => false
+      );
+    } catch (err) {
+      // Ignore invalid address
+    }
   }
 
   const status = await httpConnectionRequest(
@@ -165,10 +174,7 @@ export async function connectionRequest(
     deviceId
   );
 
-  if (udpProm) {
+  if (await udpProm) return "";
-    await udpProm;
-    return "";
-  }
 
   return status;
 }

lib/common.ts

@@ -51,7 +51,7 @@ export function escapeRegExp(str: string): string {
 export function encodeTag(tag: string): string {
   return encodeURIComponent(tag)
     .replace(
-      /[!~*'()]/g,
+      /[!~*'().]/g,
       (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
     )
     .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")

lib/common/path.ts

@@ -23,6 +23,31 @@ type Alias = [Path, string][];
 let cache1 = new Map<string, Path>();
 let cache2 = new Map<string, Path>();
 
+const CHAR_SPACE = 32;
+const CHAR_BACKSLASH = 92;
+const CHAR_COLON = 58;
+const CHAR_QUOTE = 34;
+const CHAR_OPEN_BRACKET = 91;
+const CHAR_CLOSE_BRACKET = 93;
+const CHAR_ASTERISK = 42;
+const CHAR_COMMA = 44;
+const CHAR_DOT = 46;
+
+function charCodeAt(str: string, index: number): number {
+  if (index >= str.length) return 0;
+  return str.charCodeAt(index);
+}
+
+function legalChar(c: number): boolean {
+  return (
+    (c >= 97 && c <= 122) ||
+    (c >= 65 && c <= 90) ||
+    (c >= 48 && c <= 57) ||
+    c === 95 ||
+    c === 45
+  );
+}
+
 export default class Path {
   public readonly segments: Segments;
   public readonly wildcard: number;
@@ -30,37 +55,70 @@ export default class Path {
   protected _string: string;
   protected _stringIndex: number[];
 
+  protected static parseAliasValue(
+    pattern: string,
+    index: number
+  ): { index: number; value: string } {
+    let i = index;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    if (charCodeAt(pattern, i) === CHAR_QUOTE) {
+      for (let j = i + 1; j < pattern.length; ++j) {
+        if (
+          pattern.charCodeAt(j) === CHAR_QUOTE &&
+          pattern.charCodeAt(j - 1) !== CHAR_BACKSLASH
+        ) {
+          try {
+            ++j;
+            const v = JSON.parse(pattern.slice(i, j));
+            return { index: j, value: v };
+          } catch {
+            return { index, value: null };
+          }
+        }
+      }
+      return { index, value: null };
+    }
+
+    for (; i < pattern.length; ++i) {
+      const c = pattern.charCodeAt(i);
+      if (c === CHAR_CLOSE_BRACKET || c === CHAR_COMMA) break;
+    }
+
+    return { index: i, value: pattern.slice(index, i).trim() };
+  }
+
+  protected static parseAliasPath(
+    pattern: string,
+    index: number
+  ): { index: number; path: Path } {
+    let i = index;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    const { index: idx, segments } = Path.parsePath(pattern, i);
+    if (idx === i) return { index, path: null };
+    i = idx;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    if (charCodeAt(pattern, i) !== CHAR_COLON) return { index, path: null };
+    return { index: i + 1, path: new Path(segments) };
+  }
+
   protected static parseAlias(
     pattern: string,
     index: number
   ): { index: number; alias: Alias } {
     const alias: Alias = [];
-    while (index < pattern.length && pattern[index] !== "]") {
+    let i = index;
-      const { index: idx, segments } = Path.parsePath(pattern, index);
+    for (;;) {
-      let j = (index = idx + 1);
+      let path: Path;
-      while (pattern[j] !== "]" && pattern[j] !== ",") {
+      let value: string;
-        if (pattern[j] === '"' && index === j) {
+      ({ index: i, path } = Path.parseAliasPath(pattern, i));
-          ++j;
+      if (!path) break;
-          while (pattern[j] !== '"' || pattern[j - 1] === "\\") {
+      ({ index: i, value } = Path.parseAliasValue(pattern, i));
-            if (++j >= pattern.length)
+      if (value == null) break;
-              throw new Error("Invalid alias expression");
+      alias.push([path, value]);
-          }
+      index = i;
-        }
-        if (++j >= pattern.length) throw new Error("Invalid alias expression");
-      }
 
-      let value = pattern.slice(index, j).trim();
+      if (charCodeAt(pattern, index) !== CHAR_COMMA) break;
-      index = j;
+      ++i;
-      if (value[0] === '"') {
-        try {
-          value = JSON.parse(value);
-        } catch (error) {
-          throw new Error("Invalid alias expression");
-        }
-      }
-
-      alias.push([new Path(segments), value]);
-      if (pattern[index] === ",") ++index;
     }
 
     // Ensure identical expressions have idential string representation
@@ -81,30 +139,26 @@ export default class Path {
     index: number
   ): { index: number; segments: Segments } {
     const segments = [];
-    // Colon separator is needed for parseAlias
-    if (index < pattern.length && pattern[index] !== ":") {
-      for (;;) {
-        if (pattern[index] === "[") {
-          const { index: idx, alias } = Path.parseAlias(pattern, index + 1);
-          index = idx + 1;
-          segments.push(alias);
-        } else {
-          const j = index;
-          while (
-            index < pattern.length &&
-            pattern[index] !== ":" &&
-            pattern[index] !== "."
-          )
-            ++index;
-          const s = pattern.slice(j, index).trim();
-          if (!s) throw new Error("Invalid parameter path");
-          segments.push(s);
-        }
 
-        if (index >= pattern.length || pattern[index] === ":") break;
+    let segStart = index;
-        else if (pattern[index] !== ".")
+    let wildcard = -1;
-          throw new Error("Invalid alias expression");
+    for (let i = index; i <= pattern.length; ++i) {
-        ++index;
+      const c = charCodeAt(pattern, i);
+      if (c === CHAR_OPEN_BRACKET && i === segStart) {
+        const { index: idx, alias } = Path.parseAlias(pattern, i + 1);
+        if (charCodeAt(pattern, idx) !== CHAR_CLOSE_BRACKET) break;
+        segments.push(alias);
+        index = i = idx + 1;
+        segStart = i + 1;
+      } else if (c === CHAR_ASTERISK && i === segStart) {
+        wildcard = i + 1;
+      } else if (!legalChar(c) || i === wildcard) {
+        if (i === segStart) break;
+        const s = pattern.slice(segStart, i);
+        segments.push(s);
+        index = i;
+        if (c !== CHAR_DOT) break;
+        segStart = i + 1;
       }
     }
 
@@ -143,7 +197,8 @@ export default class Path {
     if (!path) {
       path = cache2.get(str);
       if (!path) {
-        const { segments } = Path.parsePath(str, 0);
+        const { index, segments } = Path.parsePath(str, 0);
+        if (index < str.length) throw new Error("Invalid parameter path");
         path = new Path(segments);
         if (path.toString() !== str) cache1.set(path.toString(), path);
       }

lib/common/yaml.ts

@@ -29,21 +29,36 @@ function stringifyKey(str: string): string {
 function foldString(str: string): string[] {
   if (str.length <= LINE_WIDTH) return [str];
   if (str.startsWith(" ")) return [str];
-  const words = str.split(/(?<=[^ ]) (?=[^ ])/);
   const lines: string[] = [];
 
   let idx = 0;
-  let len = 0;
+  let cand = 0;
-  for (const [i, word] of words.entries()) {
+  for (let i = 1; i < str.length - 1; ++i) {
-    len += word.length + 1;
+    if (str[i] !== " ") continue;
-    if (len >= LINE_WIDTH && len !== word.length) {
+
-      lines.push(words.slice(idx, i).join(" "));
+    if (str[i + 1] === " ") {
-      idx = i;
+      i += 2;
-      len = 0;
+      while (str[i] === " ") ++i;
+      continue;
     }
+
+    if (i <= idx + LINE_WIDTH) {
+      cand = i;
+      continue;
+    }
+
+    const c = cand > idx ? cand : i;
+    lines.push(str.slice(idx, c));
+    idx = c + 1;
+    cand = i;
   }
 
-  if (idx < words.length) lines.push(words.slice(idx).join(" "));
+  if (cand > idx && str.length > idx + LINE_WIDTH) {
+    lines.push(str.slice(idx, cand));
+    idx = cand + 1;
+  }
+
+  lines.push(str.slice(idx));
 
   return lines;
 }

lib/connection-request.ts

@@ -169,9 +169,13 @@ export async function httpConnectionRequest(
     if (res.statusCode === 503) return "Device is offline";
 
     if (res.statusCode === 401 && res.headers["www-authenticate"]) {
-      authHeader = auth.parseWwwAuthenticateHeader(
+      try {
-        res.headers["www-authenticate"]
+        authHeader = auth.parseWwwAuthenticateHeader(
-      );
+          res.headers["www-authenticate"]
+        );
+      } catch (err) {
+        return "Connection request error: Error parsing www-authenticate header";
+      }
       [username, password, authExp] = await extractAuth(authExp, false);
     } else {
       return `Connection request error: Unexpected status code ${res.statusCode}`;
@@ -182,14 +186,13 @@ export async function httpConnectionRequest(
 }
 
 export async function udpConnectionRequest(
-  address: string,
+  host: string,
+  port: number,
   authExp: Expression,
   sourcePort = 0,
   _debug: boolean,
   deviceId: string
 ): Promise<void> {
-  const [host, portStr] = address.split(":", 2);
-  const port = portStr ? parseInt(portStr) : 80;
   const now = Date.now();
 
   const client = dgram.createSocket({ type: "udp4", reuseAddr: true });
@@ -216,8 +219,8 @@ export async function udpConnectionRequest(
       .createHmac("sha1", password)
       .update(`${ts}${id}${username}${cn}`)
       .digest("hex");
-    const uri = `http://${address}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
+    const uri = `http://${host}:${port}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
-    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${address}\r\n\r\n`;
+    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${host}:${port}\r\n\r\n`;
     const message = Buffer.from(msg);
 
     for (let i = 0; i < 3; ++i) {

lib/cwmp.ts

@@ -84,9 +84,13 @@ async function authenticate(
   let authentication;
 
   if (sessionContext.httpRequest.headers["authorization"]) {
-    authentication = auth.parseAuthorizationHeader(
+    try {
-      sessionContext.httpRequest.headers["authorization"]
+      authentication = auth.parseAuthorizationHeader(
-    );
+        sessionContext.httpRequest.headers["authorization"]
+      );
+    } catch (err) {
+      return false;
+    }
   }
 
   if (authentication && authentication.method === "Digest") {
@@ -208,7 +212,7 @@ async function writeResponse(
       sessionContext.extendLock = now + 10000;
       await cache.acquireLock(
         `cwmp_session_${sessionContext.deviceId}`,
-        sessionContext.timeout + 15000,
+        sessionContext.timeout * 1000 + 15000,
         0,
         sessionContext.sessionId
       );
@@ -357,6 +361,39 @@ async function transferComplete(sessionContext, rpc): Promise<void> {
   return writeResponse(sessionContext, res);
 }
 
+async function duStateChangeComplete(sessionContext, rpc): Promise<void> {
+  const { acsResponse, operation, fault } = await session.duStateChangeComplete(
+    sessionContext,
+    rpc.cpeRequest
+  );
+
+  if (!operation) {
+    logger.accessWarn({
+      sessionContext: sessionContext,
+      message: "Unrecognized command key",
+      rpc: rpc,
+    });
+  }
+
+  if (fault) {
+    Object.assign(sessionContext.retries, operation.retries);
+    recordFault(
+      sessionContext,
+      fault,
+      operation.provisions,
+      operation.channels
+    );
+  }
+
+  const res = soap.response({
+    id: rpc.id,
+    acsResponse: acsResponse,
+    cwmpVersion: sessionContext.cwmpVersion,
+  });
+
+  return writeResponse(sessionContext, res);
+}
+
 // Append provisions and remove duplicates
 function appendProvisions(original, toAppend): boolean {
   let modified = false;
@@ -891,7 +928,7 @@ async function getSession(
   const sessionContextString = await cache.pop(`session_${sessionId}`);
   if (!sessionContextString) return null;
   sessionContext = await session.deserialize(sessionContextString);
-  connection.setTimeout(sessionContext.timeout);
+  connection.setTimeout(sessionContext.timeout * 1000);
   return sessionContext;
 }
 
@@ -1146,7 +1183,7 @@ async function processRequest(
       sessionContext.extendLock = sessionContext.timestamp + 10000;
       await cache.acquireLock(
         `cwmp_session_${sessionContext.deviceId}`,
-        sessionContext.timeout + 15000,
+        sessionContext.timeout * 1000 + 15000,
         0,
         sessionContext.sessionId
       );
@@ -1235,6 +1272,16 @@ async function processRequest(
         cwmpVersion: sessionContext.cwmpVersion,
       });
       return writeResponse(sessionContext, res);
+    } else if (rpc.cpeRequest.name === "DUStateChangeComplete") {
+      if (sessionContext.state !== 1) return reportBadState(sessionContext);
+
+      logger.accessInfo({
+        sessionContext: sessionContext,
+        message: "CPE request",
+        rpc: rpc,
+      });
+
+      return duStateChangeComplete(sessionContext, rpc);
     } else {
       if (sessionContext.state !== 1 || rpc.cpeRequest.name === "Inform")
         return void reportBadState(sessionContext);

lib/db.ts

@@ -19,7 +19,7 @@
 
 import { MongoClient, ObjectID, Collection, Db } from "mongodb";
 import { get } from "./config";
-import { encodeTag, escapeRegExp } from "./common";
+import { decodeTag, encodeTag, escapeRegExp } from "./common";
 import { parse } from "./common/expression-parser";
 import {
   DeviceData,
@@ -46,6 +46,8 @@ export let tasksCollection: Collection,
 
 let clientPromise: Promise<MongoClient>;
 
+const INVALID_PATH_SUFFIX = "__invalid";
+
 function compareAccessLists(list1: string[], list2: string[]): boolean {
   if (list1.length !== list2.length) return false;
   for (const [i, v] of list1.entries()) if (v !== list2[i]) return false;
@@ -167,7 +169,19 @@ export async function fetchDevice(
     if (obj["_accessList"] != null)
       attrs.accessList = [obj["_attributesTimestamp"] || 1, obj["_accessList"]];
 
-    res.push([Path.parse(path), t, attrs]);
+    try {
+      res.push([Path.parse(path), t, attrs]);
+    } catch (err) {
+      // The path parser is now more strict so we might be in a situation where
+      // the database contains invalid paths from before this change So here we
+      // encode the invalid characters.
+      const splits = path.split(".");
+      splits[splits.length - 1] =
+        encodeTag(splits[splits.length - 1]) + INVALID_PATH_SUFFIX;
+      path = splits.join(".");
+      res.push([Path.parse(path), t, attrs]);
+      return;
+    }
 
     for (const [k, v] of Object.entries(obj)) {
       if (!k.startsWith("_")) {
@@ -449,7 +463,16 @@ export async function saveDevice(
         break;
       default:
         if (!diff[2]) {
-          update["$unset"][path.toString()] = 1;
+          let pathStr = path.toString();
+          // Paths with that suffix are encoded and need to be decoded
+          if (pathStr.endsWith(INVALID_PATH_SUFFIX)) {
+            const splits = pathStr.split(".");
+            splits[splits.length - 1] = decodeTag(
+              splits[splits.length - 1].slice(0, 0 - INVALID_PATH_SUFFIX.length)
+            );
+            pathStr = splits.join(".");
+          }
+          update["$unset"][pathStr] = 1;
           continue;
         }
 

lib/extensions.ts

@@ -30,14 +30,6 @@ const TIMEOUT = +config.get("EXT_TIMEOUT");
 const processes: { [script: string]: ChildProcess } = {};
 const jobs = new Map();
 
-function messageHandler(message): void {
-  const func = jobs.get(message[0]);
-  if (func) {
-    jobs.delete(message[0]);
-    func({ fault: message[1], value: message[2] });
-  }
-}
-
 export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
   return new Promise((resolve) => {
     const scriptName = args[0];
@@ -70,7 +62,16 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
         if (processes[scriptName] === p) delete processes[scriptName];
       });
 
-      p.on("message", messageHandler);
+      p.on("message", (message) => {
+        const func = jobs.get(message[0]);
+        if (func) {
+          jobs.delete(message[0]);
+          // Wait for any disconnect even to fire
+          setTimeout(() => {
+            func({ fault: message[1], value: message[2] });
+          });
+        }
+      });
 
       const rlstdout = readline.createInterface(p.stdout);
       rlstdout.on("line", (line) => {

lib/session.ts

@@ -57,8 +57,13 @@ import {
   FactoryReset,
   AddObjectResponse,
   GetParameterValuesResponse,
+  ChangeDUStateOperation,
+  DownloadOperation,
+  DUStateChangeCompleteRequest,
+  ChangeDUState,
 } from "./types";
 import { getRequestOrigin } from "./forwarded";
+import * as logger from "./logger";
 
 const VALID_PARAM_TYPES = new Set([
   "xsd:int",
@@ -283,6 +288,8 @@ export async function transferComplete(
   sessionContext.deviceData.attributes.revision = revision;
   const commandKey = rpcReq.commandKey;
   const operation = sessionContext.operations[commandKey];
+  if (operation.name !== "Download")
+    throw new Error("Unexpected operation name");
 
   if (!operation) {
     return {
@@ -432,6 +439,142 @@ function revertDownloadParameters(
   }
 }
 
+export async function duStateChangeComplete(
+  sessionContext: SessionContext,
+  rpcReq: DUStateChangeCompleteRequest
+): Promise<{
+  acsResponse: AcsResponse;
+  operation: ChangeDUStateOperation;
+  fault: Fault;
+}> {
+  const revision =
+    (sessionContext.revisions[sessionContext.revisions.length - 1] || 0) + 1;
+  sessionContext.deviceData.timestamps.revision = revision;
+  sessionContext.deviceData.attributes.revision = revision;
+  const commandKey = rpcReq.commandKey;
+  const operation = sessionContext.operations[commandKey];
+
+  if (!operation) {
+    return {
+      acsResponse: { name: "DUStateChangeCompleteResponse" },
+      operation: null,
+      fault: null,
+    };
+  }
+
+  if (operation.name !== "ChangeDUState")
+    throw new Error("Unexpected operation name");
+
+  if (operation.args.operations.length !== rpcReq.results.length) {
+    return {
+      acsResponse: { name: "DUStateChangeCompleteResponse" },
+      operation: operation,
+      fault: {
+        code: "invalid_response",
+        message: "Unexpected number of results in DUStateChangeComplete",
+        timestamp: operation.timestamp,
+      },
+    };
+  }
+
+  delete sessionContext.operations[commandKey];
+  if (!sessionContext.operationsTouched) sessionContext.operationsTouched = {};
+  sessionContext.operationsTouched[commandKey] = 1;
+
+  let toClear = null;
+  const timestamp = sessionContext.timestamp + sessionContext.iteration + 1;
+
+  let fault: Fault;
+  for (const [i, res] of rpcReq.results.entries()) {
+    const op = operation.args.operations[i];
+    if (res.fault && res.fault.faultCode !== "0") {
+      revertChangeDuStateParameters(sessionContext, op.instance);
+      fault = {
+        code: `cwmp.${res.fault.faultCode}`,
+        message: res.fault.faultString,
+        timestamp: operation.timestamp,
+        detail: res.fault,
+      };
+      continue;
+    }
+
+    const toSet: Record<string, [string | number | boolean, string]> = {
+      LastChangeDUState: [operation.timestamp, "xsd:dateTime"],
+      LastOperationType: [op.operationType, "xsd:string"],
+      LastURL: [op.url, "xsd:string"],
+      LastUUID: [op.uuid, "xsd:string"],
+      LastExecutionEnvRef: [op.executionEnvRef, "xsd:string"],
+      StartTime: [+res.startTime, "xsd:dateTime"],
+      CompleteTime: [+res.completeTime, "xsd:dateTime"],
+      CurrentState: [res.currentState, "xsd:string"],
+      DeploymentUnitRef: [res.deploymentUnitRef, "xsd:string"],
+      Version: [res.version, "xsd:string"],
+      Resolved: [res.resolved, "xsd:string"],
+      ExecutionUnitRefList: [res.executionUnitRefList, "xsd:string"],
+    };
+
+    for (const [k, v] of Object.entries(toSet)) {
+      const p = sessionContext.deviceData.paths.add(
+        Path.parse(`DUStates.${op.instance}.${k}`)
+      );
+      toClear = device.set(
+        sessionContext.deviceData,
+        p,
+        timestamp,
+        { value: [timestamp, v] },
+        toClear
+      );
+    }
+  }
+
+  if (toClear) {
+    for (const c of toClear)
+      device.clear(sessionContext.deviceData, c[0], c[1], c[2], c[3]);
+  }
+
+  return {
+    acsResponse: { name: "DUStateChangeCompleteResponse" },
+    operation: operation,
+    fault: fault,
+  };
+}
+
+function revertChangeDuStateParameters(
+  sessionContext: SessionContext,
+  instance
+): void {
+  const timestamp = sessionContext.timestamp + sessionContext.iteration + 1;
+
+  let p;
+
+  p = sessionContext.deviceData.paths.add(
+    Path.parse(`DUStates.${instance}.LastChangeDUState`)
+  );
+
+  const lastChangeDuState = sessionContext.deviceData.attributes.get(p);
+
+  p = sessionContext.deviceData.paths.add(
+    Path.parse(`DUStates.${instance}.ChangeDUState`)
+  );
+
+  const toClear = device.set(sessionContext.deviceData, p, timestamp, {
+    value: [
+      timestamp,
+      [
+        lastChangeDuState && lastChangeDuState.value[1]
+          ? lastChangeDuState.value[1][0]
+          : 0,
+        "xsd:dateTime",
+      ],
+    ],
+  });
+
+  if (toClear) {
+    for (const c of toClear)
+      device.clear(sessionContext.deviceData, c[0], c[1], c[2], c[3]);
+  }
+}
+
 export async function timeoutOperations(
   sessionContext: SessionContext
 ): Promise<{ faults: Fault[]; operations: Operation[] }> {
@@ -793,6 +936,10 @@ function runDeclarations(
       downloadsToCreate: new InstanceSet(),
       downloadsValues: new Map(),
       downloadsDownload: new Map(),
+      duStatesToDelete: new Set(),
+      duStatesToCreate: new InstanceSet(),
+      duStatesValues: new Map(),
+      duStatesChangeDUState: new Map(),
       reboot: 0,
       factoryReset: 0,
     };
@@ -1438,6 +1585,147 @@ export async function rpcRequest(
         }
       }
 
+      // DUState
+      index = null;
+      for (const instance of sessionContext.syncState.duStatesToCreate) {
+        if (index == null) {
+          index = 0;
+          for (const p of sessionContext.deviceData.paths.find(
+            Path.parse("DUStates.*"),
+            false,
+            true
+          )) {
+            if (
+              +p.segments[1] > index &&
+              sessionContext.deviceData.attributes.has(p)
+            )
+              index = +p.segments[1];
+          }
+        }
+
+        ++index;
+
+        toClear = device.set(
+          sessionContext.deviceData,
+          Path.parse("DUStates"),
+          timestamp,
+          { object: [timestamp, 1], writable: [timestamp, 1] },
+          toClear
+        );
+
+        toClear = device.set(
+          sessionContext.deviceData,
+          Path.parse(`DUStates.${index}`),
+          timestamp,
+          { object: [timestamp, 1], writable: [timestamp, 1] },
+          toClear
+        );
+
+        const params = {
+          URL: {
+            writable: 1,
+            value: [instance.URL || "", "xsd:string"],
+          },
+          UUID: {
+            writable: 1,
+            value: [instance.UUID || "", "xsd:string"],
+          },
+          ExecutionEnvRef: {
+            writable: 1,
+            value: [instance.ExecutionEnvRef || "", "xsd:string"],
+          },
+          Username: {
+            writable: 1,
+            value: [instance.Username || "", "xsd:string"],
+          },
+          Password: {
+            writable: 1,
+            value: [instance.Password || "", "xsd:string"],
+          },
+          ChangeDUState: {
+            writable: 1,
+            value: [instance.ChangeDUState || "", "xsd:dateTime"],
+          },
+          OperationType: {
+            writable: 1,
+            value: [instance.OperationType || "", "xsd:string"],
+          },
+          LastURL: { writable: 0, value: ["", "xsd:string"] },
+          LastUUID: { writable: 0, value: ["", "xsd:string"] },
+          LastExecutionEnvRef: { writable: 0, value: ["", "xsd:string"] },
+          LastChangeDUState: { writable: 0, value: [0, "xsd:dateTime"] },
+          LastOperationType: { writable: 0, value: [0, "xsd:string"] },
+          Version: { writable: 0, value: [0, "xsd:string"] },
+          CurrentState: { writable: 0, value: [0, "xsd:string"] },
+          ExecutionUnitRefList: { writable: 0, value: [0, "xsd:string"] },
+          DeploymentUnitRef: { writable: 0, value: [0, "xsd:string"] },
+          Resolved: { writable: 0, value: [0, "xsd:boolean"] },
+          StartTime: { writable: 0, value: [0, "xsd:dateTime"] },
+          CompleteTime: { writable: 0, value: [0, "xsd:dateTime"] },
+        };
+
+        for (const [k, v] of Object.entries(params)) {
+          toClear = device.set(
+            sessionContext.deviceData,
+            Path.parse(`DUStates.${index}.${k}`),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, v.writable as 0 | 1],
+              value: [
+                timestamp,
+                v.value as [string | number | boolean, string],
+              ],
+            },
+            toClear
+          );
+        }
+
+        toClear = device.set(
+          sessionContext.deviceData,
+          Path.parse(`DUStates.${index}.*`),
+          timestamp,
+          null,
+          toClear
+        );
+      }
+
+      sessionContext.syncState.duStatesToCreate.clear();
+
+      for (const instance of sessionContext.syncState.duStatesToDelete) {
+        toClear = device.set(
+          sessionContext.deviceData,
+          instance,
+          timestamp,
+          null,
+          toClear
+        );
+        for (const p of sessionContext.syncState.duStatesValues.keys()) {
+          if (p.segments[1] === instance.segments[1])
+            sessionContext.syncState.duStatesValues.delete(p);
+        }
+      }
+
+      sessionContext.syncState.duStatesToDelete.clear();
+
+      for (const [p, v] of sessionContext.syncState.duStatesValues) {
+        const attrs = sessionContext.deviceData.attributes.get(p);
+        if (attrs) {
+          if (attrs.writable && attrs.writable[1] && attrs.value) {
+            const val = device.sanitizeParameterValue([v, attrs.value[1][1]]);
+            if (val[0] !== attrs.value[1][0]) {
+              toClear = device.set(
+                sessionContext.deviceData,
+                p,
+                timestamp,
+                { value: [timestamp, val] },
+                toClear
+              );
+            }
+          }
+        }
+      }
+
       if (toClear || sessionContext.deviceData.changes.has("prerequisite")) {
         if (toClear) {
           for (const c of toClear)
@@ -1740,6 +2028,7 @@ function generateSetRpcRequest(
   | FactoryReset
   | Reboot
   | Download
+  | ChangeDUState
 ) & { next?: string } {
   const syncState = sessionContext.syncState;
   if (!syncState) return null;
@@ -1918,6 +2207,76 @@ function generateSetRpcRequest(
     }
   }
 
+  // ChangeDUState
+  const duStateOps: ChangeDUState["operations"] = [];
+  for (const [p, t] of syncState.duStatesChangeDUState) {
+    const attrs = deviceData.attributes.get(p);
+    if (attrs && attrs.value && t <= attrs.value[1][0]) continue;
+    const urlAttrs = deviceData.attributes.get(
+      deviceData.paths.get(p.slice(0, -1).concat(Path.parse("URL")))
+    );
+    const operationTypeAttrs = deviceData.attributes.get(
+      deviceData.paths.get(p.slice(0, -1).concat(Path.parse("OperationType")))
+    );
+    const uuidAttrs = deviceData.attributes.get(
+      deviceData.paths.get(p.slice(0, -1).concat(Path.parse("UUID")))
+    );
+    const executionEnvRefAttrs = deviceData.attributes.get(
+      deviceData.paths.get(p.slice(0, -1).concat(Path.parse("ExecutionEnvRef")))
+    );
+    const versionAttrs = deviceData.attributes.get(
+      deviceData.paths.get(p.slice(0, -1).concat(Path.parse("Version")))
+    );
+    const usernameAttrs = deviceData.attributes.get(
+      deviceData.paths.get(p.slice(0, -1).concat(Path.parse("Username")))
+    );
+    const passwordAttrs = deviceData.attributes.get(
+      deviceData.paths.get(p.slice(0, -1).concat(Path.parse("Password")))
+    );
+
+    const operationType = operationTypeAttrs
+      ? (operationTypeAttrs.value[1][0] as string)
+      : null;
+    const url = urlAttrs ? (urlAttrs.value[1][0] as string) : null;
+    const uuid = uuidAttrs ? (uuidAttrs.value[1][0] as string) : null;
+    const executionEnvRef = executionEnvRefAttrs
+      ? (executionEnvRefAttrs.value[1][0] as string)
+      : null;
+    const version = versionAttrs ? (versionAttrs.value[1][0] as string) : null;
+    const username = usernameAttrs
+      ? (usernameAttrs.value[1][0] as string)
+      : null;
+    const password = passwordAttrs
+      ? (passwordAttrs.value[1][0] as string)
+      : null;
+
+    if (
+      operationType !== "install" &&
+      operationType !== "update" &&
+      operationType !== "uninstall"
+    )
+      continue;
+
+    duStateOps.push({
+      operationType: operationType,
+      instance: p.segments[1] as string,
+      url: url,
+      uuid: uuid,
+      executionEnvRef: executionEnvRef,
+      version: version,
+      username: username,
+      password: password,
+    });
+  }
+
+  if (duStateOps.length) {
+    return {
+      name: "ChangeDUState",
+      commandKey: generateRpcId(sessionContext),
+      operations: duStateOps,
+    };
+  }
+
   // Reboot
   if (syncState.reboot) {
     const p = sessionContext.deviceData.paths.get(Path.parse("Reboot"));
@@ -2199,6 +2558,26 @@ function processDeclarations(
           }
         }
         break;
+      case "DUStates":
+        if (
+          currentPath.length === 3 &&
+          currentPath.wildcard === 0 &&
+          declareAttributeValues &&
+          declareAttributeValues.value
+        ) {
+          if (currentPath.segments[2] === "ChangeDUState") {
+            syncState.duStatesChangeDUState.set(
+              currentPath,
+              declareAttributeValues.value[0]
+            );
+          } else {
+            syncState.duStatesValues.set(
+              currentPath,
+              declareAttributeValues.value[0]
+            );
+          }
+        }
+        break;
       case "VirtualParameters":
         if (currentPath.length <= 2) {
           let d;
@@ -2290,7 +2669,7 @@ function processDeclarations(
                     syncState.refreshAttributes.object.add(currentPath);
                   else if (currentAttributes.object[1] === 0)
                     syncState.refreshAttributes.value.add(currentPath);
-                } else {
+                } else if (attrName in syncState.refreshAttributes) {
                   syncState.refreshAttributes[attrName].add(currentPath);
                 }
               }
@@ -2369,6 +2748,10 @@ function processInstances(
     if (parent.length !== 1) return;
     instancesToDelete = sessionContext.syncState.downloadsToDelete;
     instancesToCreate = sessionContext.syncState.downloadsToCreate;
+  } else if (parent.segments[0] === "DUStates") {
+    if (parent.length !== 1) return;
+    instancesToDelete = sessionContext.syncState.duStatesToDelete;
+    instancesToCreate = sessionContext.syncState.duStatesToCreate;
   } else {
     instancesToDelete = sessionContext.syncState.instancesToDelete.get(parent);
     if (instancesToDelete == null) {
@@ -2420,8 +2803,15 @@ export async function rpcResponse(
   id: string,
   _rpcRes: CpeResponse
 ): Promise<Fault> {
+  function invalidResponse(message: string): Fault {
+    return {
+      code: "invalid_response",
+      message: message,
+    };
+  }
+
   if (id !== generateRpcId(sessionContext))
-    throw new Error("Request ID not recognized");
+    return invalidResponse("Request ID not recognized");
 
   ++sessionContext.rpcCount;
 
@@ -2512,47 +2902,99 @@ export async function rpcResponse(
 
   if (rpcRes.name === "GetParameterValuesResponse") {
     if (rpcReq.name !== "GetParameterValues")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
-    if (rpcRes.parameterList.length !== rpcReq.parameterNames.length) {
+    const requested = new Set(rpcReq.parameterNames);
-      return {
-        code: `invalid_response`,
-        message:
-          "The list of parameters returned in the GetParameterValuesResponse message does not match the requested parameters",
-      };
-    }
 
-    for (const p of rpcRes.parameterList) {
+    for (const [path, value, type] of rpcRes.parameterList) {
+      if (!requested.delete(path.toString())) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Unexpected parameter in response",
+          parameter: path.toString(),
+        });
+        continue;
+      }
       toClear = device.set(
         sessionContext.deviceData,
-        p[0],
+        path,
         timestamp,
         {
           object: [timestamp, 0],
-          value: [timestamp, p.slice(1) as [string | number | boolean, string]],
+          value: [timestamp, [value, type]],
         },
         toClear
       );
     }
+
+    if (requested.size) {
+      for (const p of requested) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Missing parameter in response",
+          parameter: p,
+        });
+        toClear = device.set(
+          sessionContext.deviceData,
+          Path.parse(p),
+          timestamp,
+          {
+            object: [timestamp, 0],
+            value: [timestamp, ["", "xsd:string"]],
+          },
+          toClear
+        );
+      }
+    }
   } else if (rpcRes.name === "GetParameterAttributesResponse") {
     if (rpcReq.name !== "GetParameterAttributes")
       throw new Error("Response name does not match request name");
 
-    for (const p of rpcRes.parameterList) {
+    const requested = new Set(rpcReq.parameterNames);
+
+    for (const [path, notification, accessList] of rpcRes.parameterList) {
+      if (!requested.delete(path.toString())) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Unexpected parameter in response",
+          parameter: path.toString(),
+        });
+        continue;
+      }
       toClear = device.set(
         sessionContext.deviceData,
-        p[0],
+        path,
         timestamp,
         {
-          notification: [timestamp, p[1] as number],
+          notification: [timestamp, notification],
-          accessList: [timestamp, p[2] as string[]],
+          accessList: [timestamp, accessList],
         },
         toClear
       );
     }
+
+    if (requested.size) {
+      for (const p of requested) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Missing parameter in response",
+          parameter: p,
+        });
+        toClear = device.set(
+          sessionContext.deviceData,
+          Path.parse(p),
+          timestamp,
+          {
+            notification: [timestamp, 0],
+            accessList: [timestamp, []],
+          },
+          toClear
+        );
+      }
+    }
   } else if (rpcRes.name === "GetParameterNamesResponse") {
     if (rpcReq.name !== "GetParameterNames")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     let root: Path;
     if (rpcReq.parameterPath.endsWith("."))
@@ -2598,6 +3040,7 @@ export async function rpcResponse(
         "FactoryReset",
         "VirtualParameters",
         "Downloads",
+        "DUStates",
       ]) {
         const p = sessionContext.deviceData.paths.get(Path.parse(n));
         if (p && sessionContext.deviceData.attributes.has(p))
@@ -2609,6 +3052,14 @@ export async function rpcResponse(
     const wildcardParams: Path[] = [root.concat(wildcardPath)];
 
     for (const [path, object, writable] of rpcRes.parameterList) {
+      if (!path.toString().startsWith(rpcReq.parameterPath)) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Unexpected parameter in response",
+          parameter: path.toString(),
+        });
+        continue;
+      }
       if (object && !rpcReq.nextLevel)
         wildcardParams.push(path.concat(wildcardPath));
 
@@ -2635,7 +3086,7 @@ export async function rpcResponse(
     }
   } else if (rpcRes.name === "SetParameterValuesResponse") {
     if (rpcReq.name !== "SetParameterValues")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     for (const p of rpcReq.parameterList) {
       toClear = device.set(
@@ -2654,7 +3105,7 @@ export async function rpcResponse(
     }
   } else if (rpcRes.name === "SetParameterAttributesResponse") {
     if (rpcReq.name !== "SetParameterAttributes")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     for (const p of rpcReq.parameterList) {
       let attrs;
@@ -2684,7 +3135,7 @@ export async function rpcResponse(
     }
   } else if (rpcRes.name === "AddObjectResponse") {
     if (rpcReq.name !== "AddObject")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2695,7 +3146,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "DeleteObjectResponse") {
     if (rpcReq.name !== "DeleteObject")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2706,7 +3157,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "RebootResponse") {
     if (rpcReq.name !== "Reboot")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2717,7 +3168,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "FactoryResetResponse") {
     if (rpcReq.name !== "FactoryReset")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2728,7 +3179,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "DownloadResponse") {
     if (rpcReq.name !== "Download")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2789,7 +3240,7 @@ export async function rpcResponse(
         toClear
       );
     } else {
-      const operation = {
+      const operation: DownloadOperation = {
         name: "Download",
         timestamp: sessionContext.timestamp,
         provisions: sessionContext.provisions,
@@ -2813,8 +3264,42 @@ export async function rpcResponse(
         sessionContext.operationsTouched = {};
       sessionContext.operationsTouched[rpcReq.commandKey] = 1;
     }
+  } else if (rpcRes.name === "ChangeDUStateResponse") {
+    if (rpcReq.name !== "ChangeDUState")
+      return invalidResponse("Response name does not match request name");
+
+    for (const op of rpcReq.operations) {
+      toClear = device.set(
+        sessionContext.deviceData,
+        Path.parse(`DUStates.${op.instance}.ChangeDUState`),
+        timestamp + 1,
+        { value: [timestamp + 1, [sessionContext.timestamp, "xsd:dateTime"]] },
+        toClear
+      );
+    }
+
+    const operation = {
+      name: "ChangeDUState",
+      timestamp: sessionContext.timestamp,
+      provisions: sessionContext.provisions,
+      channels: sessionContext.channels,
+      retries: {},
+      args: {
+        operations: rpcReq.operations,
+      },
+    };
+
+    for (const channel of Object.keys(sessionContext.channels)) {
+      if (sessionContext.retries[channel] != null)
+        operation.retries[channel] = sessionContext.retries[channel];
+    }
+
+    sessionContext.operations[rpcReq.commandKey] = operation as Operation;
+    if (!sessionContext.operationsTouched)
+      sessionContext.operationsTouched = {};
+    sessionContext.operationsTouched[rpcReq.commandKey] = 1;
   } else {
-    throw new Error("Response name not recognized");
+    return invalidResponse("Response name not recognized");
   }
 
   if (toClear) {

lib/soap.ts

@@ -48,6 +48,9 @@ import {
   GetRPCMethodsRequest,
   RequestDownloadRequest,
   AcsResponse,
+  ChangeDUState,
+  ChangeDUStateResponse,
+  DUStateChangeCompleteRequest,
 } from "./types";
 import Path from "./common/path";
 
@@ -109,8 +112,8 @@ function event(xml: Element): string[] {
 
 function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterInfoStruct")
     .map<[Path, boolean, boolean]>((e) => {
+      if (e.localName !== "ParameterInfoStruct") return null;
       let param: string, value: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -134,10 +137,20 @@ function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
         parsed = false;
       }
 
-      if (param && !param.endsWith("."))
+      try {
-        return [Path.parse(param), false, parsed];
+        if (param && !param.endsWith("."))
-      else return [Path.parse(param.slice(0, -1)), true, parsed];
+          return [Path.parse(param), false, parsed];
-    });
+        else return [Path.parse(param.slice(0, -1)), true, parsed];
+      } catch (err) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
+        });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
 const getValueType = memoize((str: string) => {
@@ -151,8 +164,8 @@ function parameterValueList(
   xml: Element
 ): [Path, string | number | boolean, string][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterValueStruct")
     .map<[Path, string | number | boolean, string]>((e) => {
+      if (e.localName !== "ParameterValueStruct") return null;
       let valueElement: Element, param: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -208,15 +221,24 @@ function parameterValueList(
           parsed = value;
         }
       }
-
+      try {
-      return [Path.parse(param), parsed, valueType];
+        return [Path.parse(param), parsed, valueType];
-    });
+      } catch (err) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
+        });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
 function parameterAttributeList(xml: Element): [Path, number, string[]][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterAttributeStruct")
     .map<[Path, number, string[]]>((e) => {
+      if (e.localName !== "ParameterAttributeStruct") return null;
       let notificationElement: Element,
         accessListElement: Element,
         param: string;
@@ -248,8 +270,18 @@ function parameterAttributeList(xml: Element): [Path, number, string[]][] {
         .filter((c) => c.localName === "string")
         .map((c) => decodeEntities(c.text));
 
-      return [Path.parse(param), notification, accessList];
+      try {
-    });
+        return [Path.parse(param), notification, accessList];
+      } catch (err) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
+        });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
 function GetParameterNames(methodRequest): string {
@@ -544,6 +576,29 @@ function DownloadResponse(xml: Element): DownloadResponse {
   };
 }
 
+function ChangeDUState(methodRequest: ChangeDUState): string {
+  const operations = methodRequest.operations.map((op) => {
+    if (op.operationType === "install")
+      return `<Operations xsi:type="cwmp:InstallOpStruct"><UUID>${op.uuid}</UUID><URL>${op.url}</URL><ExecutionEnvRef>${op.executionEnvRef}</ExecutionEnvRef><Username>${op.username}</Username><Password>${op.password}</Password></Operations>`;
+    else if (op.operationType === "update")
+      return `<Operations xsi:type="cwmp:UpdateOpStruct"><UUID>${op.uuid}</UUID><URL>${op.url}</URL><Version>${op.version}</Version><Username>${op.username}</Username><Password>${op.password}</Password></Operations>`;
+    else if (op.operationType === "uninstall")
+      return `<Operations xsi:type="cwmp:UninstallOpStruct"><UUID>${op.uuid}</UUID><Version>${op.version}</Version><ExecutionEnvRef>${op.executionEnvRef}</ExecutionEnvRef></Operations>`;
+
+    return "";
+  });
+
+  return `<cwmp:ChangeDUState><CommandKey>${
+    methodRequest.commandKey || ""
+  }</CommandKey>${operations.join("")}</cwmp:ChangeDUState>`;
+}
+
+function ChangeDUStateResponse(): ChangeDUStateResponse {
+  return {
+    name: "ChangeDUStateResponse",
+  };
+}
+
 function Inform(xml: Element): InformRequest {
   let retryCount: number, evnt: string[];
   let parameterList: [Path, string | number | boolean, string][];
@@ -690,6 +745,97 @@ function TransferCompleteResponse(): string {
   return "<cwmp:TransferCompleteResponse></cwmp:TransferCompleteResponse>";
 }
 
+function DUStateChangeComplete(xml: Element): DUStateChangeCompleteRequest {
+  let commandKey: string,
+    results: DUStateChangeCompleteRequest["results"],
+    res: DUStateChangeCompleteRequest["results"][0];
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "CommandKey":
+        commandKey = c.text;
+        break;
+      case "Results":
+        if (!results) results = [];
+        res = {
+          uuid: null,
+          deploymentUnitRef: null,
+          version: null,
+          currentState: null,
+          resolved: true,
+          executionUnitRefList: null,
+          startTime: null,
+          completeTime: null,
+          fault: null,
+        };
+        for (const cc of c.children) {
+          switch (cc.localName) {
+            case "UUID":
+              res.uuid = cc.text;
+              break;
+            case "DeploymentUnitRef":
+              res.deploymentUnitRef = cc.text;
+              break;
+            case "Version":
+              res.version = cc.text;
+              break;
+            case "CurrentState":
+              res.currentState = cc.text;
+              break;
+            case "Resolved":
+              res.resolved = parseBool(cc.text);
+              break;
+            case "ExecutionUnitRefList":
+              res.executionUnitRefList = cc.text;
+              break;
+            case "StartTime":
+              if (c.text) res.startTime = Date.parse(c.text);
+              break;
+            case "CompleteTime":
+              if (c.text) res.completeTime = Date.parse(c.text);
+              break;
+            case "Fault":
+              res.fault = {
+                faultCode: cc.children.find((n) => n.localName === "FaultCode")
+                  .text,
+                faultString: cc.children.find(
+                  (n) => n.localName === "FaultString"
+                ).text,
+              };
+              break;
+          }
+        }
+        results.push(res);
+        break;
+    }
+  }
+
+  if (commandKey == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CommandKey",
+    });
+    commandKey = "";
+  }
+
+  if (!results) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Results",
+    });
+    results = [];
+  }
+
+  return {
+    name: "DUStateChangeComplete",
+    commandKey: commandKey,
+    results: results,
+  };
+}
+
+function DUStateChangeCompleteResponse(): string {
+  return "<cwmp:DUStateChangeCompleteResponse></cwmp:DUStateChangeCompleteResponse>";
+}
+
 function RequestDownload(xml: Element): RequestDownloadRequest {
   return {
     name: "RequestDownload",
@@ -894,6 +1040,9 @@ export function request(
     case "TransferComplete":
       rpc.cpeRequest = TransferComplete(methodElement);
       break;
+    case "DUStateChangeComplete":
+      rpc.cpeRequest = DUStateChangeComplete(methodElement);
+      break;
     case "RequestDownload":
       rpc.cpeRequest = RequestDownload(methodElement);
       break;
@@ -927,6 +1076,9 @@ export function request(
     case "DownloadResponse":
       rpc.cpeResponse = DownloadResponse(methodElement);
       break;
+    case "ChangeDUStateResponse":
+      rpc.cpeResponse = ChangeDUStateResponse();
+      break;
     case "Fault":
       rpc.cpeFault = fault(methodElement);
       break;
@@ -980,6 +1132,9 @@ export function response(rpc: {
       case "TransferCompleteResponse":
         body = TransferCompleteResponse();
         break;
+      case "DUStateChangeCompleteResponse":
+        body = DUStateChangeCompleteResponse();
+        break;
       case "RequestDownloadResponse":
         body = RequestDownloadResponse();
         break;
@@ -1022,6 +1177,9 @@ export function response(rpc: {
       case "Download":
         body = Download(rpc.acsRequest);
         break;
+      case "ChangeDUState":
+        body = ChangeDUState(rpc.acsRequest);
+        break;
       default:
         throw new Error(
           `Unknown method request ${(rpc.acsRequest as AcsRequest).name}`

lib/types.ts

@@ -121,6 +121,10 @@ export interface SyncState {
   downloadsToCreate: InstanceSet;
   downloadsValues: Map<Path, string | number>;
   downloadsDownload: Map<Path, number>;
+  duStatesToDelete: Set<Path>;
+  duStatesToCreate: InstanceSet;
+  duStatesValues: Map<Path, string | number>;
+  duStatesChangeDUState: Map<Path, number>;
   reboot: number;
   factoryReset: number;
 }
@@ -183,12 +187,17 @@ export interface Task {
   provisions?: (string | number | boolean)[][];
 }
 
-export interface Operation {
+interface OperationBase {
   name: string;
   timestamp: number;
   provisions: string[][];
   channels: { [channel: string]: number };
   retries: { [channel: string]: number };
+  args: any;
+}
+
+export interface DownloadOperation extends OperationBase {
+  name: "Download";
   args: {
     instance: string;
     fileType: string;
@@ -197,6 +206,15 @@ export interface Operation {
   };
 }
 
+export interface ChangeDUStateOperation extends OperationBase {
+  name: "ChangeDUState";
+  args: {
+    operations: ChangeDUState["operations"];
+  };
+}
+
+export type Operation = DownloadOperation | ChangeDUStateOperation;
+
 export type AcsRequest =
   | GetParameterNames
   | GetParameterValues
@@ -207,7 +225,8 @@ export type AcsRequest =
   | DeleteObject
   | FactoryReset
   | Reboot
-  | Download;
+  | Download
+  | ChangeDUState;
 
 export interface GetParameterNames {
   name: "GetParameterNames";
@@ -277,6 +296,21 @@ export interface Download {
   failureUrl?: string;
 }
 
+export interface ChangeDUState {
+  name: "ChangeDUState";
+  commandKey: string;
+  operations: {
+    operationType: "install" | "update" | "uninstall";
+    instance: string;
+    url: string;
+    uuid: string;
+    version: string;
+    executionEnvRef: string;
+    username: string;
+    password: string;
+  }[];
+}
+
 export interface SpvFault {
   parameterName: string;
   faultCode: string;
@@ -305,7 +339,8 @@ export type CpeResponse =
   | DeleteObjectResponse
   | RebootResponse
   | FactoryResetResponse
-  | DownloadResponse;
+  | DownloadResponse
+  | ChangeDUStateResponse;
 
 export interface GetParameterNamesResponse {
   name: "GetParameterNamesResponse";
@@ -342,6 +377,10 @@ export interface DeleteObjectResponse {
   status: number;
 }
 
+export interface ChangeDUStateResponse {
+  name: "ChangeDUStateResponse";
+}
+
 export interface RebootResponse {
   name: "RebootResponse";
 }
@@ -361,7 +400,8 @@ export type CpeRequest =
   | InformRequest
   | TransferCompleteRequest
   | GetRPCMethodsRequest
-  | RequestDownloadRequest;
+  | RequestDownloadRequest
+  | DUStateChangeCompleteRequest;
 
 export interface InformRequest {
   name: "Inform";
@@ -384,6 +424,22 @@ export interface TransferCompleteRequest {
   completeTime?: number;
 }
 
+export interface DUStateChangeCompleteRequest {
+  name: "DUStateChangeComplete";
+  commandKey: string;
+  results: {
+    uuid: string;
+    deploymentUnitRef: string;
+    version: string;
+    currentState: string;
+    resolved: boolean;
+    executionUnitRefList: string;
+    fault?: { faultCode: string; faultString: string };
+    startTime?: number;
+    completeTime?: number;
+  }[];
+}
+
 export interface GetRPCMethodsRequest {
   name: "GetRPCMethods";
 }
@@ -397,7 +453,8 @@ export type AcsResponse =
   | InformResponse
   | GetRPCMethodsResponse
   | TransferCompleteResponse
-  | RequestDownloadResponse;
+  | RequestDownloadResponse
+  | DUStateChangeCompleteResponse;
 
 export interface InformResponse {
   name: "InformResponse";
@@ -415,6 +472,9 @@ export interface TransferCompleteResponse {
 export interface RequestDownloadResponse {
   name: "RequestDownloadResponse";
 }
+export interface DUStateChangeCompleteResponse {
+  name: "DUStateChangeCompleteResponse";
+}
 
 export interface QueryOptions {
   projection?: any;

lib/ui.ts

@@ -59,8 +59,10 @@ const getAuthorizer = memoize(
   }
 );
 
-koa.on("error", async (err) => {
+koa.on("error", async (err, ctx) => {
   setTimeout(() => {
+    // Ignored errors resulting from aborted requests
+    if (ctx && ctx.req.aborted) return;
     throw err;
   });
 });

lib/ui/api.ts

@@ -659,7 +659,7 @@ router.get("/ping/:host", async (ctx) => {
         ctx.body = parsed;
       } else {
         ctx.status = 500;
-        ctx.body = `${err.name}: ${err.message}`;
+        ctx.body = err ? `${err.name}: ${err.message}` : "Unknown error";
       }
       resolve();
     });

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.4",
+  "version": "1.2.5",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -41,33 +41,33 @@
     "@rollup/plugin-typescript": "^8.2.0",
     "@types/codemirror": "0.0.108",
     "@types/jsonwebtoken": "^8.5.0",
-    "@types/koa": "^2.13.0",
+    "@types/koa": "^2.13.1",
     "@types/koa-compress": "^4.0.1",
     "@types/koa-router": "^7.4.1",
     "@types/koa-static": "^4.0.1",
     "@types/mithril": "^2.0.6",
-    "@types/mongodb": "^3.6.8",
+    "@types/mongodb": "^3.6.9",
-    "@types/node": "^14.14.31",
+    "@types/node": "^14.14.34",
     "@types/parsimmon": "^1.10.6",
-    "@types/seedrandom": "^2.4.28",
+    "@types/seedrandom": "^3.0.0",
-    "@typescript-eslint/eslint-plugin": "^4.15.2",
+    "@typescript-eslint/eslint-plugin": "^4.17.0",
-    "@typescript-eslint/parser": "^4.15.2",
+    "@typescript-eslint/parser": "^4.17.0",
     "ava": "^3.15.0",
     "codemirror": "^5.59.4",
     "cssnano": "^4.1.10",
-    "eslint": "^7.20.0",
+    "eslint": "^7.22.0",
     "eslint-config-prettier": "^8.1.0",
     "esm": "^3.2.25",
     "mithril": "^2.0.4",
-    "postcss": "^8.2.6",
+    "postcss": "^8.2.8",
     "postcss-import": "^14.0.0",
     "postcss-preset-env": "^6.7.0",
     "prettier": "^2.2.1",
-    "rollup": "^2.39.1",
+    "rollup": "^2.41.2",
     "rollup-plugin-terser": "^7.0.2",
-    "svgo": "^2.1.0",
+    "svgo": "^2.2.2",
     "ts-node": "^9.1.1",
-    "typescript": "^4.1.5",
+    "typescript": "^4.2.3",
     "yaml": "^1.10.0"
   },
   "engines": {

test/path.ts

@@ -1,6 +1,25 @@
 import ava from "ava";
 import Path from "../lib/common/path";
 
+ava("parse", (t) => {
+  t.throws(() => Path.parse("."));
+  t.throws(() => Path.parse("a "));
+  t.throws(() => Path.parse(".a"));
+  t.throws(() => Path.parse("a."));
+  t.throws(() => Path.parse("a..b"));
+  t.throws(() => Path.parse("b*"));
+  t.throws(() => Path.parse("*b"));
+  t.throws(() => Path.parse("a.b c.d"));
+  t.throws(() => Path.parse("a["));
+  t.throws(() => Path.parse("a[b"));
+  t.throws(() => Path.parse("a[b:"));
+  t.throws(() => Path.parse('a[b:"waef]'));
+  t.notThrows(() => Path.parse("[]"));
+  t.notThrows(() => Path.parse("[a:]"));
+  t.notThrows(() => Path.parse("*"));
+  t.notThrows(() => Path.parse(""));
+});
+
 ava("toString", (t) => {
   const path1 = Path.parse('abc.[ abc : 123 , 123 : abc , 123: " abc "].123');
   const path2 = Path.parse('abc.[123:  " abc ",abc:123,123:abc].123');

ui/components/parameter.ts

@@ -78,7 +78,7 @@ const component: ClosureComponent = (): Component => {
       const { value, timestamp, parameter } = evaluateParam(
         vnode.attrs["parameter"],
         vnode.attrs["device"],
-        store.getTimestamp()
+        store.getTimestamp() + store.getClockSkew()
       );
 
       if (value == null) return null;
@@ -117,7 +117,7 @@ const component: ClosureComponent = (): Component => {
             e.redraw = false;
             // Don't update any child element
             if (e.target === (el as VnodeDOM).dom) {
-              const now = Date.now();
+              const now = Date.now() + store.getClockSkew();
               const localeString = new Date(timestamp).toLocaleString();
               e.target.title = `${localeString} (${timeAgo(now - timestamp)})`;
             }

ui/css/app.css

@@ -342,6 +342,13 @@ body {
   flex-grow: 1;
   margin: 10px;
   min-height: 250px;
+
+  /* Workaround for Chrome ignoring bottom margin when flex container overflows */
+  &::after {
+    content: "";
+    display: block;
+    padding-bottom: 1px;
+  }
 }
 
 table.table {
@@ -780,3 +787,13 @@ span.parameter-value {
     background-size: 100% 100%;
   }
 }
+
+.progress {
+  background-color: #eee;
+  overflow: hidden;
+  height: 3px;
+  & > .progress-bar {
+    height: 100%;
+    background-color: #b12d5c;
+  }
+}

ui/datalist.ts

@@ -0,0 +1,62 @@
+/**
+ * Copyright 2013-2021  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import m, { ClosureComponent, Component, Vnode } from "mithril";
+
+const elements: Map<string, Vnode> = new Map();
+
+// Source: https://werxltd.com/wp/2010/05/13/javascript-implementation-of-javas-string-hashcode-method/
+function hash(str: string): number {
+  let res = 0;
+  for (let i = 0; i < str.length; ++i) {
+    const c = str.charCodeAt(i);
+    res = (res << 5) - res + c;
+    res |= 0;
+  }
+  return res;
+}
+
+export function getDatalistId(options: string[]): string {
+  const id = "datalist" + options.reduce((acc, cur) => acc ^ hash(cur), 0);
+  if (!elements.has(id)) {
+    const n = m(
+      "datalist",
+      { id },
+      options.map((o) => m("option", { value: o }))
+    );
+    elements.set(id, n);
+  }
+  return id;
+}
+
+const component: ClosureComponent = (): Component => {
+  return {
+    view: () => {
+      return [...elements.values()];
+    },
+    onupdate: () => {
+      for (const id of elements.keys()) {
+        const used = document.querySelector(`[list='${id}']`);
+        if (!used) elements.delete(id);
+      }
+    },
+  };
+};
+
+export default component;

ui/drawer-component.ts

@@ -121,12 +121,15 @@ function renderStagingDownload(task: StageTask): Children {
   if (productClass) productClass = decodeURIComponent(productClass);
 
   const typesList = [
-    "",
+    ...new Set([
-    "1 Firmware Upgrade Image",
+      "",
-    "2 Web Content",
+      "1 Firmware Upgrade Image",
-    "3 Vendor Configuration File",
+      "2 Web Content",
-    "4 Tone File",
+      "3 Vendor Configuration File",
-    "5 Ringer File",
+      "4 Tone File",
+      "5 Ringer File",
+      ...files.value.map((f) => f["metadata.fileType"]).filter((f) => f),
+    ]),
   ].map((t) =>
     m(
       "option",

ui/files-page.ts

@@ -45,7 +45,6 @@ const attributes: {
   {
     id: "metadata.fileType",
     label: "Type",
-    type: "combo",
     options: [
       "1 Firmware Upgrade Image",
       "2 Web Content",
@@ -74,58 +73,32 @@ const unpackSmartQuery = memoize((query) => {
   });
 });
 
-interface ValidationErrors {
+function upload(
-  [prop: string]: string;
+  file: File,
-}
+  headers: Record<string, string>,
-
+  abortSignal?: AbortSignal,
-function putActionHandler(action, _object): Promise<ValidationErrors> {
+  progressListener?: (e: ProgressEvent) => void
-  return new Promise((resolve, reject) => {
+): Promise<void> {
-    const object = Object.assign({}, _object);
+  headers = Object.assign(
-    if (action === "save") {
+    {
-      const file = object["file"] ? object["file"][0] : null;
+      "Content-Type": "application/octet-stream",
-      delete object["file"];
+    },
-
+    headers
-      if (!file) return void resolve({ file: "File not selected" });
+  );
-
+  return store.xhrRequest({
-      const id = file.name;
+    method: "PUT",
-
+    headers: headers,
-      store
+    url: `api/files/${encodeURIComponent(file.name)}`,
-        .resourceExists("files", id)
+    serialize: (body) => body, // Identity function to prevent JSON.parse on blob data
-        .then((exists) => {
+    body: file,
-          if (exists) {
+    config: (xhr) => {
-            store.setTimestamp(Date.now());
+      if (progressListener)
-            return void resolve({ file: "File already exists" });
+        xhr.upload.addEventListener("progress", progressListener);
-          }
+      if (abortSignal) {
-          const headers = Object.assign(
+        if (abortSignal.aborted) xhr.abort();
-            {
+        abortSignal.addEventListener("abort", () => xhr.abort());
-              "Content-Type": "application/octet-stream",
+      }
-              Accept: "application/octet-stream",
+    },
-            },
-            object
-          );
-
-          store
-            .xhrRequest({
-              method: "PUT",
-              headers: headers,
-              url: `api/files/${encodeURIComponent(id)}`,
-              serialize: (body) => body, // Identity function to prevent JSON.parse on blob data
-              body: file,
-            })
-            .then(() => {
-              notifications.push(
-                "success",
-                `File ${exists ? "updated" : "created"}`
-              );
-              store.setTimestamp(Date.now());
-              resolve(null);
-            })
-            .catch(reject);
-        })
-        .catch(reject);
-    } else {
-      reject(new Error("Undefined action"));
-    }
   });
 }
 
@@ -225,42 +198,77 @@ export const component: ClosureComponent = (): Component => {
                 title: "Create new file",
                 onclick: () => {
                   let cb: () => Children = null;
+                  const abortController = new AbortController();
+                  let progress = -1;
                   const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
-                        actionHandler: (action, object) => {
+                        actionHandler: async (action, _object) => {
-                          return new Promise<void>((resolve) => {
+                          if (action !== "save")
-                            putActionHandler(action, object)
+                            throw new Error("Undefined action");
-                              .then((errors) => {
+                          const object = Object.assign({}, _object);
-                                const errorList = errors
+                          const file = object["file"]
-                                  ? Object.values(errors)
+                            ? object["file"][0]
-                                  : [];
+                            : null;
-                                if (errorList.length) {
+                          delete object["file"];
-                                  for (const err of errorList)
+                          if (!file) {
-                                    notifications.push("error", err);
+                            notifications.push("error", "File not selected");
-                                } else {
+                            return;
-                                  overlay.close(cb);
+                          }
-                                }
+
-                                resolve();
+                          if (await store.resourceExists("files", file.name)) {
-                              })
+                            store.setTimestamp(Date.now());
-                              .catch((err) => {
+                            notifications.push("error", "File already exists");
-                                notifications.push("error", err.message);
+                            return;
-                                resolve();
+                          }
-                              });
+
-                          });
+                          const progressListener = (e: ProgressEvent): void => {
+                            progress = e.loaded / e.total;
+                            m.redraw();
+                          };
+
+                          progress = 0;
+                          try {
+                            await upload(
+                              file,
+                              object,
+                              abortController.signal,
+                              progressListener
+                            );
+                            store.setTimestamp(Date.now());
+                            notifications.push("success", "File created");
+                            overlay.close(cb);
+                          } catch (err) {
+                            notifications.push("error", err.message);
+                          }
+                          progress = -1;
                         },
                       },
                       formData
                     )
                   );
-                  cb = () => comp;
+                  cb = () => {
-                  overlay.open(
+                    if (progress < 0) return [null, comp];
-                    cb,
+                    return [
-                    () =>
+                      m(
-                      !comp.state["current"]["modified"] ||
+                        "div.progress",
-                      confirm("You have unsaved changes. Close anyway?")
+                        m("div.progress-bar", {
-                  );
+                          style: `width: ${Math.trunc(progress * 100)}%`,
+                        })
+                      ),
+                      comp,
+                    ];
+                  };
+                  overlay.open(cb, () => {
+                    if (
+                      comp.state["current"]["modified"] &&
+                      !confirm("You have unsaved changes. Close anyway?")
+                    )
+                      return false;
+                    abortController.abort();
+                    return true;
+                  });
                 },
               },
               "New"

ui/layout.ts

@@ -24,6 +24,7 @@ import userMenu from "./user-menu";
 import adminMenu from "./admin-menu";
 import * as overlay from "./overlay";
 import { version as VERSION } from "../package.json";
+import datalist from "./datalist";
 
 const adminPages = [
   "presets",
@@ -69,6 +70,7 @@ const component: ClosureComponent = (): Component => {
           ])
         ),
         overlay.render(),
+        m(datalist),
       ];
     },
   };

ui/put-form-component.ts

@@ -20,6 +20,7 @@
 import { VnodeDOM, ClosureComponent, Component, Children } from "mithril";
 import { m } from "./components";
 import codeEditorComponent from "./code-editor-component";
+import { getDatalistId } from "./datalist";
 
 const singular = {
   presets: "preset",
@@ -151,9 +152,14 @@ function createField(current, attr, focus): Children {
     });
   }
 
+  let datalist: string = null;
+  if (attr.options) datalist = getDatalistId(attr.options);
+
   return m("input", {
     type: attr.type === "password" ? "password" : "text",
     name: attr.id,
+    list: datalist,
+    autocomplete: datalist ? "off" : null,
     disabled: attr.id === "_id" && !current.isNew,
     value: current.object[attr.id],
     oncreate: focus

ui/store.ts

@@ -34,6 +34,8 @@ const memoizedEvaluate = memoize(evaluate);
 let fulfillTimestamp = 0;
 let connectionNotification, configNotification, versionNotification;
 
+let clockSkew = 0;
+
 const queries = {
   filter: new WeakMap(),
   bookmark: new WeakMap(),
@@ -93,11 +95,13 @@ class QueryResponse {
 }
 
 function checkConnection(): void {
+  const now1 = Date.now();
   m.request({
     url: "status",
     method: "GET",
     background: true,
     extract: (xhr) => {
+      const now2 = Date.now();
       if (xhr.status !== 200) {
         if (!connectionNotification) {
           connectionNotification = notifications.push(
@@ -112,6 +116,21 @@ function checkConnection(): void {
           connectionNotification = null;
         }
 
+        try {
+          const nowAvg = Math.trunc((now1 + now2) / 2);
+          const skew = Date.parse(xhr.getResponseHeader("Date")) - nowAvg;
+          if (Math.abs(skew - clockSkew) > 5000 && now2 - now1 < 1000) {
+            clockSkew = skew;
+            console.warn(
+              `System and server clocks are out of sync. Adding ${clockSkew}ms offset to any client-side time relative calculations.`
+            );
+            setTimestamp(now2);
+            m.redraw();
+          }
+        } catch (err) {
+          // Ignore in case of missing or invalid Date header
+        }
+
         const configChanged =
           xhr.getResponseHeader("x-config-snapshot") !== configSnapshot;
         const versionChanged =
@@ -209,7 +228,7 @@ export async function xhrRequest(
 
 export function unpackExpression(exp: Expression): Expression {
   if (!Array.isArray(exp)) return exp;
-  const e = memoizedEvaluate(exp, null, fulfillTimestamp);
+  const e = memoizedEvaluate(exp, null, fulfillTimestamp + clockSkew);
   return e;
 }
 
@@ -295,7 +314,7 @@ function compareFunction(sort: {
 function findMatches(resourceType, filter, sort, limit): any[] {
   let value = [];
   for (const obj of resources[resourceType].objects.values())
-    if (evaluate(filter, obj, fulfillTimestamp)) value.push(obj);
+    if (evaluate(filter, obj, fulfillTimestamp + clockSkew)) value.push(obj);
 
   value = value.sort(compareFunction(sort));
   if (limit) value = value.slice(0, limit);
@@ -461,7 +480,7 @@ export function fulfill(accessTimestamp: number): void {
 
         for (const queryResponse of toFetch) {
           let filter = queries.filter.get(queryResponse);
-          filter = memoizedEvaluate(filter, null, fulfillTimestamp);
+          filter = memoizedEvaluate(filter, null, fulfillTimestamp + clockSkew);
           const bookmark = queries.bookmark.get(queryResponse);
           const sort = queries.sort.get(queryResponse);
           if (bookmark) filter = limitFilter(filter, sort, bookmark);
@@ -476,7 +495,11 @@ export function fulfill(accessTimestamp: number): void {
         if (!diff) {
           for (const queryResponse of toFetch) {
             let filter = queries.filter.get(queryResponse);
-            filter = memoizedEvaluate(filter, null, fulfillTimestamp);
+            filter = memoizedEvaluate(
+              filter,
+              null,
+              fulfillTimestamp + clockSkew
+            );
             const limit = queries.limit.get(queryResponse);
             const bookmark = queries.bookmark.get(queryResponse);
             const sort = queries.sort.get(queryResponse);
@@ -521,7 +544,9 @@ export function fulfill(accessTimestamp: number): void {
 
             for (const d of deleted) {
               const obj = resources[resourceType].objects.get(d);
-              if (evaluate(combinedFilterDiff, obj, fulfillTimestamp))
+              if (
+                evaluate(combinedFilterDiff, obj, fulfillTimestamp + clockSkew)
+              )
                 resources[resourceType].objects.delete(d);
             }
 
@@ -563,6 +588,10 @@ export function setTimestamp(t: number): void {
   }
 }
 
+export function getClockSkew(): number {
+  return clockSkew;
+}
+
 export function postTasks(
   deviceId: string,
   tasks: QueueTask[]
@@ -664,7 +693,7 @@ export function evaluateExpression(
   obj: Record<string, unknown>
 ): Expression {
   if (!Array.isArray(exp)) return exp;
-  return memoizedEvaluate(exp, obj, fulfillTimestamp);
+  return memoizedEvaluate(exp, obj, fulfillTimestamp + clockSkew);
 }
 
 export function changePassword(