Utilize locks to prevent certain edge case issues

lib/api-functions.ts

@@ -20,9 +20,8 @@
 import { ObjectId } from "mongodb";
 import { collections } from "./db/db";
 import {
-  count,
   deleteConfig,
-  deleteFault,
+  deleteFault as dbDeleteFault,
   deleteFile,
   deletePermission,
   deletePreset,
@@ -39,7 +38,7 @@ import {
 } from "./ui/db";
 import * as common from "./util";
 import * as cache from "./cache";
-import { lockExists } from "./lock";
+import { acquireLock, getToken, releaseLock } from "./lock";
 import {
   getRevision,
   getConfig,
@@ -54,6 +53,7 @@ import { Expression, Task } from "./types";
 import { evaluate } from "./common/expression/util";
 import { hashPassword } from "./auth";
 import { flattenDevice } from "./ui/db";
+import { ResourceLockedError } from "./common/errors";
 
 export async function connectionRequest(
   deviceId: string,
@@ -212,7 +212,8 @@ export async function awaitSessionStart(
   );
   const li = (device["_lastInform"] as Date).getTime();
   if (li > lastInform) return true;
-  if (await lockExists(`cwmp_session_${deviceId}`)) return true;
+  const token = await getToken(`cwmp_session_${deviceId}`);
+  if (token?.startsWith("cwmp_session_")) return true;
   if (timeout < 500) return false;
   await new Promise((resolve) => setTimeout(resolve, 500));
   timeout -= Date.now() - now;
@@ -224,7 +225,8 @@ export async function awaitSessionEnd(
   timeout: number
 ): Promise<boolean> {
   const now = Date.now();
-  if (!(await lockExists(`cwmp_session_${deviceId}`))) return true;
+  const token = await getToken(`cwmp_session_${deviceId}`);
+  if (!token?.startsWith("cwmp_session_")) return true;
   if (timeout < 500) return false;
   await new Promise((resolve) => setTimeout(resolve, 500));
   timeout -= Date.now() - now;
@@ -354,20 +356,41 @@ export async function insertTasks(tasks: any[]): Promise<Task[]> {
 }
 
 export async function deleteDevice(deviceId: string): Promise<void> {
-  await Promise.all([
-    collections.tasks.deleteMany({ device: deviceId }),
-    collections.devices.deleteOne({ _id: deviceId }),
-    collections.faults.deleteMany({
-      _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
-      },
-    }),
-    collections.operations.deleteMany({
-      _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
-      },
-    }),
-  ]);
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+  if (!token) throw new ResourceLockedError("Device is in session");
+  try {
+    await Promise.all([
+      collections.tasks.deleteMany({ device: deviceId }),
+      collections.devices.deleteOne({ _id: deviceId }),
+      collections.faults.deleteMany({
+        _id: {
+          $regex: `^${common.escapeRegExp(deviceId)}\\:`,
+        },
+      }),
+      collections.operations.deleteMany({
+        _id: {
+          $regex: `^${common.escapeRegExp(deviceId)}\\:`,
+        },
+      }),
+    ]);
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
+  }
+}
+
+export async function deleteFault(id: string): Promise<void> {
+  const deviceId = id.split(":", 1)[0];
+  const channel = id.slice(deviceId.length + 1);
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+  if (!token) throw new ResourceLockedError("Device is in session");
+  try {
+    const proms = [dbDeleteFault(id)];
+    if (channel.startsWith("task_"))
+      proms.push(deleteTask(new ObjectId(channel.slice(5))));
+    await Promise.all(proms);
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
+  }
 }
 
 export async function deleteResource(
@@ -380,12 +403,7 @@ export async function deleteResource(
     await deleteFile(id);
     await cache.del("cwmp-local-cache-hash");
   } else if (resource === "faults") {
-    const deviceId = id.split(":", 1)[0];
-    const channel = id.slice(deviceId.length + 1);
-    const proms = [deleteFault(id)];
-    if (channel.startsWith("task_"))
-      proms.push(deleteTask(new ObjectId(channel.slice(5))));
-    await Promise.all(proms);
+    await deleteFault(id);
   } else if (resource === "provisions") {
     await deleteProvision(id);
     await cache.del("cwmp-local-cache-hash");
@@ -412,82 +430,6 @@ export async function deleteResource(
   }
 }
 
-export async function postTasks(
-  deviceId: string,
-  tasks: Task[],
-  timeout: number,
-  device: Record<string, { value?: [boolean | number | string, string] }>
-): Promise<{ connectionRequest: string; tasks: any[] }> {
-  for (const task of tasks) {
-    delete task._id;
-    task["device"] = deviceId;
-  }
-
-  tasks = await insertTasks(tasks);
-  const statuses = tasks.map((t) => {
-    return { _id: t._id, status: "pending" };
-  });
-  const lastInform = Date.now();
-  const notInSession = await awaitSessionEnd(deviceId, 30000);
-  if (!notInSession) {
-    await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
-
-  const status = await connectionRequest(deviceId, device);
-
-  if (status) {
-    await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: status,
-      tasks: statuses,
-    };
-  }
-
-  const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
-  if (!sessionStarted) {
-    await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: "No contact from CPE",
-      tasks: statuses,
-    };
-  }
-
-  const sessionEnded = await awaitSessionEnd(deviceId, 120000);
-  if (!sessionEnded) {
-    await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
-
-  const promises: Promise<number>[] = [];
-  for (const s of statuses) {
-    promises.push(count("tasks", ["=", ["PARAM", "_id"], s._id]));
-    promises.push(
-      count("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
-    );
-  }
-
-  const res = await Promise.all(promises);
-  for (const [i, r] of statuses.entries()) {
-    if (!res[i * 2]) {
-      r.status = "done";
-    } else if (res[i * 2 + 1]) {
-      r.status = "fault";
-      r["fault"] = res[i * 2 + 1][0];
-    }
-  }
-
-  await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
-
-  return { connectionRequest: "OK", tasks: statuses };
-}
-
 // TODO Implement validation
 export async function putResource(
   resource: string,

lib/common/errors.ts

@@ -0,0 +1,6 @@
+export class ResourceLockedError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ResourceLockedError";
+  }
+}

lib/cwmp.ts

@@ -231,7 +231,7 @@ async function writeResponse(
         `cwmp_session_${sessionContext.deviceId}`,
         sessionContext.timeout * 1000 + 15000,
         0,
-        sessionContext.sessionId
+        `cwmp_session_${sessionContext.sessionId}`
       );
       if (!lockToken) throw new Error("Failed to extend lock");
     }
@@ -826,7 +826,7 @@ async function endSession(sessionContext: SessionContext): Promise<void> {
   await Promise.all(promises);
   await lock.releaseLock(
     `cwmp_session_${sessionContext.deviceId}`,
-    sessionContext.sessionId
+    `cwmp_session_${sessionContext.sessionId}`
   );
   if (sessionContext.new) {
     logger.accessInfo({
@@ -1080,7 +1080,7 @@ async function processRequest(
       `cwmp_session_${sessionContext.deviceId}`,
       sessionContext.timeout * 1000 + 15000,
       0,
-      sessionContext.sessionId
+      `cwmp_session_${sessionContext.sessionId}`
     );
 
     if (!lockToken) {

lib/lock.ts

@@ -44,7 +44,7 @@ export async function releaseLock(
   if (res.deletedCount !== 1) throw new Error("Lock expired");
 }
 
-export async function lockExists(lockName: string): Promise<boolean> {
+export async function getToken(lockName: string): Promise<string> {
   const res = await collections.locks.findOne({ _id: lockName });
-  return res != null;
+  return res?.value;
 }

lib/nbi.ts

@@ -19,7 +19,7 @@
 
 import { Collection, ObjectId } from "mongodb";
 import * as vm from "vm";
-import * as config from "./config";
+import { getRevision, getConfig } from "./ui/local-cache";
 import { filesBucket, collections } from "./db/db";
 import { optimizeProjection } from "./db/util";
 import * as query from "./query";
@@ -31,6 +31,8 @@ import { ping } from "./ping";
 import * as logger from "./logger";
 import { flattenDevice } from "./ui/db";
 import { getRequestOrigin } from "./forwarded";
+import { acquireLock, releaseLock } from "./lock";
+import { ResourceLockedError } from "./common/errors";
 
 const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
 const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
@@ -255,15 +257,15 @@ async function handler(
   } else if (FAULTS_REGEX.test(url.pathname)) {
     if (request.method === "DELETE") {
       const faultId = decodeURIComponent(FAULTS_REGEX.exec(url.pathname)[1]);
-      const deviceId = faultId.split(":", 1)[0];
-      const channel = faultId.slice(deviceId.length + 1);
-      await collections.faults.deleteOne({ _id: faultId });
-      if (channel.startsWith("task_")) {
-        const objId = new ObjectId(channel.slice(5));
-        await collections.tasks.deleteOne({ _id: objId });
-        response.writeHead(200);
-        response.end();
-        return;
+      try {
+        await apiFunctions.deleteFault(faultId);
+      } catch (err) {
+        if (err instanceof ResourceLockedError) {
+          response.writeHead(503);
+          response.end("Device is in session");
+          return;
+        }
+        throw err;
       }
 
       response.writeHead(200);
@@ -277,153 +279,159 @@ async function handler(
       const deviceId = decodeURIComponent(
         DEVICE_TASKS_REGEX.exec(url.pathname)[1]
       );
+
+      const conReq = url.searchParams.has("connection_request");
+      let task;
       if (body.length) {
-        let task;
         try {
           task = JSON.parse(body.toString());
+          task.device = deviceId;
         } catch (err) {
           response.writeHead(400);
           response.end(`${err.name}: ${err.message}`);
           return;
         }
-        task.device = deviceId;
-        const dev = await collections.devices.findOne({
-          _id: deviceId,
-        });
+      }
+
+      if (!task && !conReq) {
+        response.writeHead(400);
+        response.end();
+        return;
+      }
+
+      if (!task || !conReq) {
+        const dev = await collections.devices.findOne({ _id: deviceId });
+        if (!dev) {
+          response.writeHead(404);
+          response.end("No such device");
+          return;
+        }
+
+        if (task) {
+          await apiFunctions.insertTasks(task);
+          response.writeHead(202, { "Content-Type": "application/json" });
+          response.end(JSON.stringify(task));
+        } else {
+          const status = await apiFunctions.connectionRequest(
+            deviceId,
+            flattenDevice(dev)
+          );
+          if (status) {
+            response.writeHead(504, status);
+            response.end(status);
+          } else {
+            response.writeHead(200);
+            response.end();
+          }
+        }
+        return;
+      }
+
+      const socketTimeout: number = request.socket.timeout;
+
+      // Extend socket timeout while waiting for session
+      if (socketTimeout) request.socket.setTimeout(300000);
+
+      const token = await acquireLock(`cwmp_session_${deviceId}`, 5000, 30000);
+      if (!token) {
+        // Restore socket timeout
+        if (socketTimeout) request.socket.setTimeout(socketTimeout);
+        const dev = await collections.devices.findOne({ _id: deviceId });
         if (!dev) {
           response.writeHead(404);
           response.end("No such device");
           return;
         }
 
-        const device = flattenDevice(dev);
         await apiFunctions.insertTasks(task);
+        response.writeHead(202, "Task queued but not processed", {
+          "Content-Type": "application/json",
+        });
+        response.end(JSON.stringify(task));
+        return;
+      }
 
-        const lastInform = Date.now();
+      let dev;
 
-        const socketTimeout: number = request.socket["timeout"];
-
-        // Disable socket timeout while waiting for session
-        if (socketTimeout) request.socket.setTimeout(0);
-
-        const notInSession = await apiFunctions.awaitSessionEnd(
-          deviceId,
-          30000
-        );
-        if (url.searchParams.has("connection_request")) {
-          if (socketTimeout) request.socket.setTimeout(socketTimeout);
-          response.writeHead(202, {
-            "Content-Type": "application/json",
-          });
-          response.end(JSON.stringify(task));
+      try {
+        dev = await collections.devices.findOne({ _id: deviceId });
+        if (!dev) {
+          response.writeHead(404);
+          response.end("No such device");
           return;
         }
+        await apiFunctions.insertTasks(task);
+      } finally {
+        await releaseLock(`cwmp_session_${deviceId}`, token);
+      }
 
-        if (!notInSession) {
-          if (socketTimeout) request.socket.setTimeout(socketTimeout);
-          response.writeHead(202, "Task queued but not processed", {
-            "Content-Type": "application/json",
-          });
-          response.end(JSON.stringify(task));
-          return;
-        }
+      const lastInform = (dev["_lastInform"] as Date).getTime();
+      const device = flattenDevice(dev);
 
-        const status = await apiFunctions.connectionRequest(deviceId, device);
-
-        if (status) {
-          if (socketTimeout) request.socket.setTimeout(socketTimeout);
-          response.writeHead(202, status, {
-            "Content-Type": "application/json",
-          });
-          response.end(JSON.stringify(task));
-          return;
-        }
-
-        const onlineThreshold =
-          (url.searchParams.has("timeout") &&
-            parseInt(url.searchParams.get("timeout"))) ||
-          (config.get("DEVICE_ONLINE_THRESHOLD", deviceId) as number);
+      let onlineThreshold: number;
+      if (url.searchParams.has("timeout")) {
+        onlineThreshold = parseInt(url.searchParams.get("timeout"));
+      } else {
+        const revision = await getRevision();
+        onlineThreshold = getConfig(
+          revision,
+          "cwmp.deviceOnlineThreshold",
+          {},
+          Date.now(),
+          (exp) => {
+            if (!Array.isArray(exp)) return exp;
+            if (exp[0] === "PARAM") {
+              const p = device[exp[1]];
+              if (p?.value) return p.value[0];
+            } else if (exp[0] === "FUNC") {
+              if (exp[1] === "REMOTE_ADDRESS") {
+                for (const root of ["InternetGatewayDevice", "Device"]) {
+                  const p =
+                    device[`${root}.ManagementServer.ConnectionRequestURL`];
+                  if (p?.value) return new URL(p.value[0] as string).hostname;
+                }
+                return null;
+              }
+            }
+            return exp;
+          }
+        ) as number;
+      }
 
+      let status = await apiFunctions.connectionRequest(deviceId, device);
+      if (!status) {
         const sessionStarted = await apiFunctions.awaitSessionStart(
           deviceId,
           lastInform,
           onlineThreshold
         );
         if (!sessionStarted) {
-          if (socketTimeout) request.socket.setTimeout(socketTimeout);
-          response.writeHead(202, "Task queued but not processed", {
-            "Content-Type": "application/json",
-          });
-          response.end(JSON.stringify(task));
-          return;
-        }
-
-        const sessionEnded = await apiFunctions.awaitSessionEnd(
-          deviceId,
-          120000
-        );
-        if (!sessionEnded) {
-          if (socketTimeout) request.socket.setTimeout(socketTimeout);
-          response.writeHead(202, "Task queued but not processed", {
-            "Content-Type": "application/json",
-          });
-          response.end(JSON.stringify(task));
-          return;
-        }
-
-        const prom1 = collections.tasks.findOne(
-          { _id: task._id },
-          { projection: { _id: 1 } }
-        );
-        const prom2 = collections.faults.findOne(
-          { _id: `${deviceId}:task_${task._id}` },
-          {
-            projection: { _id: 1 },
-          }
-        );
-
-        const [t, f] = await Promise.all([prom1, prom2]);
-
-        // Restore socket timeout
-        if (socketTimeout) request.socket.setTimeout(socketTimeout);
-
-        if (f) {
-          response.writeHead(202, "Task faulted", {
-            "Content-Type": "application/json",
-          });
-          response.end(JSON.stringify(t || task));
-        } else if (t) {
-          response.writeHead(202, "Task queued but not processed", {
-            "Content-Type": "application/json",
-          });
-          response.end(JSON.stringify(t));
+          status = "Task queued but not processed";
         } else {
-          response.writeHead(200, {
-            "Content-Type": "application/json",
-          });
-          response.end(JSON.stringify(task));
+          const sessionEnded = await apiFunctions.awaitSessionEnd(
+            deviceId,
+            120000
+          );
+          if (!sessionEnded) {
+            status = "Task queued but not processed";
+          } else {
+            const f = await collections.faults.count({
+              _id: `${deviceId}:task_${task._id}`,
+            });
+            if (f) status = "Task faulted";
+          }
         }
-      } else if (url.searchParams.has("connection_request")) {
-        // No task, send connection request only
-        const dev = await collections.devices.findOne({
-          _id: deviceId,
-        });
-        if (!dev) {
-          response.writeHead(404);
-          response.end("No such device");
-          return;
-        }
-        const status = await apiFunctions.connectionRequest(deviceId);
-        if (status) {
-          response.writeHead(504, status);
-          response.end(status);
-          return;
-        }
-        response.writeHead(200);
-        response.end();
+      }
+
+      // Restore socket timeout
+      if (socketTimeout) request.socket.setTimeout(socketTimeout);
+
+      if (status) {
+        response.writeHead(202, status, { "Content-Type": "application/json" });
+        response.end(JSON.stringify(task));
       } else {
-        response.writeHead(400);
-        response.end();
+        response.writeHead(200, { "Content-Type": "application/json" });
+        response.end(JSON.stringify(task));
       }
     } else {
       response.writeHead(405, { Allow: "POST" });
@@ -447,11 +455,21 @@ async function handler(
         }
 
         const deviceId = task.device;
-        await collections.tasks.deleteOne({ _id: new ObjectId(taskId) });
+        const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+        if (!token) {
+          response.writeHead(503);
+          response.end("Device is in session");
+          return;
+        }
 
-        await collections.faults.deleteOne({
-          _id: `${deviceId}:task_${taskId}`,
-        });
+        try {
+          await Promise.all([
+            collections.tasks.deleteOne({ _id: new ObjectId(taskId) }),
+            collections.faults.deleteOne({ _id: `${deviceId}:task_${taskId}` }),
+          ]);
+        } finally {
+          await releaseLock(`cwmp_session_${deviceId}`, token);
+        }
 
         response.writeHead(200);
         response.end();
@@ -467,9 +485,19 @@ async function handler(
         );
 
         const deviceId = task.device;
-        await collections.faults.deleteOne({
-          _id: `${deviceId}:task_${taskId}`,
-        });
+        const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+        if (!token) {
+          response.writeHead(503);
+          response.end("Device is in session");
+          return;
+        }
+        try {
+          await collections.faults.deleteOne({
+            _id: `${deviceId}:task_${taskId}`,
+          });
+        } finally {
+          await releaseLock(`cwmp_session_${deviceId}`, token);
+        }
 
         response.writeHead(200);
         response.end();
@@ -563,7 +591,18 @@ async function handler(
     const deviceId = decodeURIComponent(
       DELETE_DEVICE_REGEX.exec(url.pathname)[1]
     );
-    await apiFunctions.deleteDevice(deviceId);
+
+    try {
+      await apiFunctions.deleteDevice(deviceId);
+    } catch (err) {
+      if (err instanceof ResourceLockedError) {
+        response.writeHead(503);
+        response.end("Device is in session");
+        return;
+      }
+      throw err;
+    }
+
     response.writeHead(200);
     response.end();
   } else if (QUERY_REGEX.test(url.pathname)) {

lib/ui/api.ts

@@ -19,19 +19,23 @@
 
 import { PassThrough } from "stream";
 import Router from "koa-router";
+import { ObjectId } from "mongodb";
 import * as db from "./db";
 import * as apiFunctions from "../api-functions";
 import { evaluate, and, extractParams } from "../common/expression/util";
 import { parse } from "../common/expression/parser";
 import * as logger from "../logger";
 import { getConfig } from "../ui/local-cache";
-import { Expression } from "../types";
+import { Expression, Task } from "../types";
 import { generateSalt, hashPassword } from "../auth";
 import { del } from "../cache";
 import Authorizer from "../common/authorizer";
 import { ping } from "../ping";
 import { decodeTag } from "../util";
 import { stringify as yamlStringify } from "../common/yaml";
+import { ResourceLockedError } from "../common/errors";
+import { acquireLock, releaseLock } from "../lock";
+import { collections } from "../db/db";
 
 const router = new Router();
 export default router;
@@ -410,7 +414,18 @@ for (const [resource, flags] of Object.entries(resources)) {
         return void (ctx.status = 403);
       }
 
-      await apiFunctions.deleteResource(resource, ctx.params.id);
+      try {
+        await apiFunctions.deleteResource(resource, ctx.params.id);
+      } catch (err) {
+        if (err instanceof ResourceLockedError) {
+          log.message += " failed";
+          logger.accessWarn(log);
+          ctx.status = 503;
+          ctx.body = err.message;
+          return;
+        }
+        throw err;
+      }
 
       logger.accessInfo(log);
 
@@ -533,32 +548,69 @@ router.put("/files/:id", async (ctx) => {
 });
 
 router.post("/devices/:id/tasks", async (ctx) => {
+  const deviceId = ctx.params.id;
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Commit tasks",
     context: ctx,
-    deviceId: ctx.params.id,
+    deviceId: deviceId,
     tasks: null,
   };
 
-  const filter = and(authorizer.getFilter("devices", 3), [
-    "=",
-    ["PARAM", "DeviceID.ID"],
-    ctx.params.id,
-  ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
     return void (ctx.status = 403);
   }
-  const { value: device } = await db.query("devices", filter).next();
-  if (!device) return void (ctx.status = 404);
 
-  const validate = authorizer.getValidator("devices", device);
-  for (const t of ctx.request.body) {
-    if (!validate("task", t)) {
-      logUnauthorizedWarning(log);
-      return void (ctx.status = 403);
+  const socketTimeout: number = ctx.socket.timeout;
+
+  // Extend socket timeout while waiting for session
+  if (socketTimeout) ctx.socket.setTimeout(300000);
+
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000, 30000);
+  if (!token) {
+    log.message += " failed";
+    logger.accessWarn(log);
+    ctx.body = "Device is in session";
+    ctx.status = 503;
+    // Restore socket timeout
+    if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
+    return;
+  }
+
+  let device;
+
+  let statuses: { _id: string; status: string }[];
+
+  try {
+    const filter = and(authorizer.getFilter("devices", 3), [
+      "=",
+      ["PARAM", "DeviceID.ID"],
+      deviceId,
+    ]);
+    device = (await db.query("devices", filter).next()).value;
+    if (!device) return void (ctx.status = 404);
+
+    const validate = authorizer.getValidator("devices", device);
+    for (const t of ctx.request.body) {
+      if (!validate("task", t)) {
+        logUnauthorizedWarning(log);
+        return void (ctx.status = 403);
+      }
     }
+
+    let tasks = ctx.request.body as Task[];
+
+    for (const task of tasks) {
+      delete task._id;
+      task["device"] = deviceId;
+    }
+
+    tasks = await apiFunctions.insertTasks(tasks);
+
+    statuses = tasks.map((t) => ({ _id: t._id, status: "pending" }));
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
   }
 
   const onlineThreshold = getConfig(
@@ -584,27 +636,43 @@ router.post("/devices/:id/tasks", async (ctx) => {
     }
   ) as number;
 
-  const socketTimeout: number = ctx.socket["timeout"];
+  const lastInform = device["Events.Inform"].value[0] as number;
 
-  // Disable socket timeout while waiting for postTasks()
-  if (socketTimeout) ctx.socket.setTimeout(0);
+  let status = await apiFunctions.connectionRequest(deviceId, device);
+  if (!status) {
+    const sessionStarted = await apiFunctions.awaitSessionStart(
+      deviceId,
+      lastInform,
+      onlineThreshold
+    );
+    if (!sessionStarted) {
+      status = "No contact from CPE";
+    } else {
+      const sessionEnded = await apiFunctions.awaitSessionEnd(deviceId, 120000);
+      if (!sessionEnded) status = "Session took too long to complete";
+    }
+  }
 
-  const res = await apiFunctions.postTasks(
-    ctx.params.id,
-    ctx.request.body,
-    onlineThreshold,
-    device
-  );
+  if (!status) {
+    const promises = statuses.map((t) =>
+      collections.faults.count({ _id: `${deviceId}:task_${t._id}` })
+    );
+
+    const res = await Promise.all(promises);
+    for (const [i, r] of statuses.entries())
+      r.status = res[i] ? "fault" : "done";
+  }
+
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
 
   // Restore socket timeout
   if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
 
-  log.tasks = res.tasks.map((t) => t._id).join(",");
-
+  log.tasks = statuses.map((t) => t._id).join(",");
   logger.accessInfo(log);
 
-  ctx.set("Connection-Request", res.connectionRequest);
-  ctx.body = res.tasks;
+  ctx.set("Connection-Request", status || "OK");
+  ctx.body = statuses;
 });
 
 router.post("/devices/:id/tags", async (ctx) => {