rodrigo/genieacs
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Utilize locks to prevent certain edge case issues

Browse Source
This commit is contained in:
Zaid Humaimidi committed 2023-12-06 11:10:23 -08:00
1 parent f4596db9f7
commit 342210faf1
7 files changed
+323 -269
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lib/api-functions.ts
+43 -101
View File
@@ -20,9 +20,8 @@
import { ObjectId } from "mongodb";
import { collections } from "./db/db";
import {
count,
deleteConfig,
deleteFault,
deleteFault as dbDeleteFault,
deleteFile,
deletePermission,
deletePreset,
@@ -39,7 +38,7 @@ import {
} from "./ui/db";
import * as common from "./util";
import * as cache from "./cache";
import { lockExists } from "./lock";
import { acquireLock, getToken, releaseLock } from "./lock";
import {
getRevision,
getConfig,
@@ -54,6 +53,7 @@ import { Expression, Task } from "./types";
import { evaluate } from "./common/expression/util";
import { hashPassword } from "./auth";
import { flattenDevice } from "./ui/db";
import { ResourceLockedError } from "./common/errors";
export async function connectionRequest(
deviceId: string,
@@ -212,7 +212,8 @@ export async function awaitSessionStart(
);
const li = (device["_lastInform"] as Date).getTime();
if (li > lastInform) return true;
if (await lockExists(`cwmp_session_${deviceId}`)) return true;
const token = await getToken(`cwmp_session_${deviceId}`);
if (token?.startsWith("cwmp_session_")) return true;
if (timeout < 500) return false;
await new Promise((resolve) => setTimeout(resolve, 500));
timeout -= Date.now() - now;
@@ -224,7 +225,8 @@ export async function awaitSessionEnd(
timeout: number
): Promise<boolean> {
const now = Date.now();
if (!(await lockExists(`cwmp_session_${deviceId}`))) return true;
const token = await getToken(`cwmp_session_${deviceId}`);
if (!token?.startsWith("cwmp_session_")) return true;
if (timeout < 500) return false;
await new Promise((resolve) => setTimeout(resolve, 500));
timeout -= Date.now() - now;
@@ -354,20 +356,41 @@ export async function insertTasks(tasks: any[]): Promise<Task[]> {
}
export async function deleteDevice(deviceId: string): Promise<void> {
await Promise.all([
collections.tasks.deleteMany({ device: deviceId }),
collections.devices.deleteOne({ _id: deviceId }),
collections.faults.deleteMany({
_id: {
$regex: `^${common.escapeRegExp(deviceId)}\\:`,
},
}),
collections.operations.deleteMany({
_id: {
$regex: `^${common.escapeRegExp(deviceId)}\\:`,
},
}),
]);
const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
if (!token) throw new ResourceLockedError("Device is in session");
try {
await Promise.all([
collections.tasks.deleteMany({ device: deviceId }),
collections.devices.deleteOne({ _id: deviceId }),
collections.faults.deleteMany({
_id: {
$regex: `^${common.escapeRegExp(deviceId)}\\:`,
},
}),
collections.operations.deleteMany({
_id: {
$regex: `^${common.escapeRegExp(deviceId)}\\:`,
},
}),
]);
} finally {
await releaseLock(`cwmp_session_${deviceId}`, token);
}
}
export async function deleteFault(id: string): Promise<void> {
const deviceId = id.split(":", 1)[0];
const channel = id.slice(deviceId.length + 1);
const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
if (!token) throw new ResourceLockedError("Device is in session");
try {
const proms = [dbDeleteFault(id)];
if (channel.startsWith("task_"))
proms.push(deleteTask(new ObjectId(channel.slice(5))));
await Promise.all(proms);
} finally {
await releaseLock(`cwmp_session_${deviceId}`, token);
}
}
export async function deleteResource(
@@ -380,12 +403,7 @@ export async function deleteResource(
await deleteFile(id);
await cache.del("cwmp-local-cache-hash");
} else if (resource === "faults") {
const deviceId = id.split(":", 1)[0];
const channel = id.slice(deviceId.length + 1);
const proms = [deleteFault(id)];
if (channel.startsWith("task_"))
proms.push(deleteTask(new ObjectId(channel.slice(5))));
await Promise.all(proms);
await deleteFault(id);
} else if (resource === "provisions") {
await deleteProvision(id);
await cache.del("cwmp-local-cache-hash");
@@ -412,82 +430,6 @@ export async function deleteResource(
}
}
export async function postTasks(
deviceId: string,
tasks: Task[],
timeout: number,
device: Record<string, { value?: [boolean | number | string, string] }>
): Promise<{ connectionRequest: string; tasks: any[] }> {
for (const task of tasks) {
delete task._id;
task["device"] = deviceId;
}
tasks = await insertTasks(tasks);
const statuses = tasks.map((t) => {
return { _id: t._id, status: "pending" };
});
const lastInform = Date.now();
const notInSession = await awaitSessionEnd(deviceId, 30000);
if (!notInSession) {
await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
return {
connectionRequest: "Session took too long to complete",
tasks: statuses,
};
}
const status = await connectionRequest(deviceId, device);
if (status) {
await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
return {
connectionRequest: status,
tasks: statuses,
};
}
const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
if (!sessionStarted) {
await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
return {
connectionRequest: "No contact from CPE",
tasks: statuses,
};
}
const sessionEnded = await awaitSessionEnd(deviceId, 120000);
if (!sessionEnded) {
await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
return {
connectionRequest: "Session took too long to complete",
tasks: statuses,
};
}
const promises: Promise<number>[] = [];
for (const s of statuses) {
promises.push(count("tasks", ["=", ["PARAM", "_id"], s._id]));
promises.push(
count("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
);
}
const res = await Promise.all(promises);
for (const [i, r] of statuses.entries()) {
if (!res[i * 2]) {
r.status = "done";
} else if (res[i * 2 + 1]) {
r.status = "fault";
r["fault"] = res[i * 2 + 1][0];
}
}
await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
return { connectionRequest: "OK", tasks: statuses };
}
// TODO Implement validation
export async function putResource(
resource: string,
lib/common/errors.ts
+6
View File
@@ -0,0 +1,6 @@
export class ResourceLockedError extends Error {
constructor(message: string) {
super(message);
this.name = "ResourceLockedError";
}
}
lib/cwmp.ts
+3 -3
View File
@@ -231,7 +231,7 @@ async function writeResponse(
`cwmp_session_${sessionContext.deviceId}`,
sessionContext.timeout * 1000 + 15000,
0,
sessionContext.sessionId
`cwmp_session_${sessionContext.sessionId}`
);
if (!lockToken) throw new Error("Failed to extend lock");
}
@@ -826,7 +826,7 @@ async function endSession(sessionContext: SessionContext): Promise<void> {
await Promise.all(promises);
await lock.releaseLock(
`cwmp_session_${sessionContext.deviceId}`,
sessionContext.sessionId
`cwmp_session_${sessionContext.sessionId}`
);
if (sessionContext.new) {
logger.accessInfo({
@@ -1080,7 +1080,7 @@ async function processRequest(
`cwmp_session_${sessionContext.deviceId}`,
sessionContext.timeout * 1000 + 15000,
0,
sessionContext.sessionId
`cwmp_session_${sessionContext.sessionId}`
);
if (!lockToken) {
lib/lock.ts
+2 -2
View File
@@ -44,7 +44,7 @@ export async function releaseLock(
if (res.deletedCount !== 1) throw new Error("Lock expired");
}
export async function lockExists(lockName: string): Promise<boolean> {
export async function getToken(lockName: string): Promise<string> {
const res = await collections.locks.findOne({ _id: lockName });
return res != null;
return res?.value;
}
lib/nbi.ts
+173 -134
View File
@@ -19,7 +19,7 @@
import { Collection, ObjectId } from "mongodb";
import * as vm from "vm";
import * as config from "./config";
import { getRevision, getConfig } from "./ui/local-cache";
import { filesBucket, collections } from "./db/db";
import { optimizeProjection } from "./db/util";
import * as query from "./query";
@@ -31,6 +31,8 @@ import { ping } from "./ping";
import * as logger from "./logger";
import { flattenDevice } from "./ui/db";
import { getRequestOrigin } from "./forwarded";
import { acquireLock, releaseLock } from "./lock";
import { ResourceLockedError } from "./common/errors";
const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
@@ -255,15 +257,15 @@ async function handler(
} else if (FAULTS_REGEX.test(url.pathname)) {
if (request.method === "DELETE") {
const faultId = decodeURIComponent(FAULTS_REGEX.exec(url.pathname)[1]);
const deviceId = faultId.split(":", 1)[0];
const channel = faultId.slice(deviceId.length + 1);
await collections.faults.deleteOne({ _id: faultId });
if (channel.startsWith("task_")) {
const objId = new ObjectId(channel.slice(5));
await collections.tasks.deleteOne({ _id: objId });
response.writeHead(200);
response.end();
return;
try {
await apiFunctions.deleteFault(faultId);
} catch (err) {
if (err instanceof ResourceLockedError) {
response.writeHead(503);
response.end("Device is in session");
return;
}
throw err;
}
response.writeHead(200);
@@ -277,153 +279,159 @@ async function handler(
const deviceId = decodeURIComponent(
DEVICE_TASKS_REGEX.exec(url.pathname)[1]
);
const conReq = url.searchParams.has("connection_request");
let task;
if (body.length) {
let task;
try {
task = JSON.parse(body.toString());
task.device = deviceId;
} catch (err) {
response.writeHead(400);
response.end(`${err.name}: ${err.message}`);
return;
}
task.device = deviceId;
const dev = await collections.devices.findOne({
_id: deviceId,
});
}
if (!task && !conReq) {
response.writeHead(400);
response.end();
return;
}
if (!task || !conReq) {
const dev = await collections.devices.findOne({ _id: deviceId });
if (!dev) {
response.writeHead(404);
response.end("No such device");
return;
}
if (task) {
await apiFunctions.insertTasks(task);
response.writeHead(202, { "Content-Type": "application/json" });
response.end(JSON.stringify(task));
} else {
const status = await apiFunctions.connectionRequest(
deviceId,
flattenDevice(dev)
);
if (status) {
response.writeHead(504, status);
response.end(status);
} else {
response.writeHead(200);
response.end();
}
}
return;
}
const socketTimeout: number = request.socket.timeout;
// Extend socket timeout while waiting for session
if (socketTimeout) request.socket.setTimeout(300000);
const token = await acquireLock(`cwmp_session_${deviceId}`, 5000, 30000);
if (!token) {
// Restore socket timeout
if (socketTimeout) request.socket.setTimeout(socketTimeout);
const dev = await collections.devices.findOne({ _id: deviceId });
if (!dev) {
response.writeHead(404);
response.end("No such device");
return;
}
const device = flattenDevice(dev);
await apiFunctions.insertTasks(task);
response.writeHead(202, "Task queued but not processed", {
"Content-Type": "application/json",
});
response.end(JSON.stringify(task));
return;
}
const lastInform = Date.now();
let dev;
const socketTimeout: number = request.socket["timeout"];
// Disable socket timeout while waiting for session
if (socketTimeout) request.socket.setTimeout(0);
const notInSession = await apiFunctions.awaitSessionEnd(
deviceId,
30000
);
if (url.searchParams.has("connection_request")) {
if (socketTimeout) request.socket.setTimeout(socketTimeout);
response.writeHead(202, {
"Content-Type": "application/json",
});
response.end(JSON.stringify(task));
try {
dev = await collections.devices.findOne({ _id: deviceId });
if (!dev) {
response.writeHead(404);
response.end("No such device");
return;
}
await apiFunctions.insertTasks(task);
} finally {
await releaseLock(`cwmp_session_${deviceId}`, token);
}
if (!notInSession) {
if (socketTimeout) request.socket.setTimeout(socketTimeout);
response.writeHead(202, "Task queued but not processed", {
"Content-Type": "application/json",
});
response.end(JSON.stringify(task));
return;
}
const lastInform = (dev["_lastInform"] as Date).getTime();
const device = flattenDevice(dev);
const status = await apiFunctions.connectionRequest(deviceId, device);
if (status) {
if (socketTimeout) request.socket.setTimeout(socketTimeout);
response.writeHead(202, status, {
"Content-Type": "application/json",
});
response.end(JSON.stringify(task));
return;
}
const onlineThreshold =
(url.searchParams.has("timeout") &&
parseInt(url.searchParams.get("timeout"))) ||
(config.get("DEVICE_ONLINE_THRESHOLD", deviceId) as number);
let onlineThreshold: number;
if (url.searchParams.has("timeout")) {
onlineThreshold = parseInt(url.searchParams.get("timeout"));
} else {
const revision = await getRevision();
onlineThreshold = getConfig(
revision,
"cwmp.deviceOnlineThreshold",
{},
Date.now(),
(exp) => {
if (!Array.isArray(exp)) return exp;
if (exp[0] === "PARAM") {
const p = device[exp[1]];
if (p?.value) return p.value[0];
} else if (exp[0] === "FUNC") {
if (exp[1] === "REMOTE_ADDRESS") {
for (const root of ["InternetGatewayDevice", "Device"]) {
const p =
device[`${root}.ManagementServer.ConnectionRequestURL`];
if (p?.value) return new URL(p.value[0] as string).hostname;
}
return null;
}
}
return exp;
}
) as number;
}
let status = await apiFunctions.connectionRequest(deviceId, device);
if (!status) {
const sessionStarted = await apiFunctions.awaitSessionStart(
deviceId,
lastInform,
onlineThreshold
);
if (!sessionStarted) {
if (socketTimeout) request.socket.setTimeout(socketTimeout);
response.writeHead(202, "Task queued but not processed", {
"Content-Type": "application/json",
});
response.end(JSON.stringify(task));
return;
}
const sessionEnded = await apiFunctions.awaitSessionEnd(
deviceId,
120000
);
if (!sessionEnded) {
if (socketTimeout) request.socket.setTimeout(socketTimeout);
response.writeHead(202, "Task queued but not processed", {
"Content-Type": "application/json",
});
response.end(JSON.stringify(task));
return;
}
const prom1 = collections.tasks.findOne(
{ _id: task._id },
{ projection: { _id: 1 } }
);
const prom2 = collections.faults.findOne(
{ _id: `${deviceId}:task_${task._id}` },
{
projection: { _id: 1 },
}
);
const [t, f] = await Promise.all([prom1, prom2]);
// Restore socket timeout
if (socketTimeout) request.socket.setTimeout(socketTimeout);
if (f) {
response.writeHead(202, "Task faulted", {
"Content-Type": "application/json",
});
response.end(JSON.stringify(t || task));
} else if (t) {
response.writeHead(202, "Task queued but not processed", {
"Content-Type": "application/json",
});
response.end(JSON.stringify(t));
status = "Task queued but not processed";
} else {
response.writeHead(200, {
"Content-Type": "application/json",
});
response.end(JSON.stringify(task));
const sessionEnded = await apiFunctions.awaitSessionEnd(
deviceId,
120000
);
if (!sessionEnded) {
status = "Task queued but not processed";
} else {
const f = await collections.faults.count({
_id: `${deviceId}:task_${task._id}`,
});
if (f) status = "Task faulted";
}
}
} else if (url.searchParams.has("connection_request")) {
// No task, send connection request only
const dev = await collections.devices.findOne({
_id: deviceId,
});
if (!dev) {
response.writeHead(404);
response.end("No such device");
return;
}
const status = await apiFunctions.connectionRequest(deviceId);
if (status) {
response.writeHead(504, status);
response.end(status);
return;
}
response.writeHead(200);
response.end();
}
// Restore socket timeout
if (socketTimeout) request.socket.setTimeout(socketTimeout);
if (status) {
response.writeHead(202, status, { "Content-Type": "application/json" });
response.end(JSON.stringify(task));
} else {
response.writeHead(400);
response.end();
response.writeHead(200, { "Content-Type": "application/json" });
response.end(JSON.stringify(task));
}
} else {
response.writeHead(405, { Allow: "POST" });
@@ -447,11 +455,21 @@ async function handler(
}
const deviceId = task.device;
await collections.tasks.deleteOne({ _id: new ObjectId(taskId) });
const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
if (!token) {
response.writeHead(503);
response.end("Device is in session");
return;
}
await collections.faults.deleteOne({
_id: `${deviceId}:task_${taskId}`,
});
try {
await Promise.all([
collections.tasks.deleteOne({ _id: new ObjectId(taskId) }),
collections.faults.deleteOne({ _id: `${deviceId}:task_${taskId}` }),
]);
} finally {
await releaseLock(`cwmp_session_${deviceId}`, token);
}
response.writeHead(200);
response.end();
@@ -467,9 +485,19 @@ async function handler(
);
const deviceId = task.device;
await collections.faults.deleteOne({
_id: `${deviceId}:task_${taskId}`,
});
const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
if (!token) {
response.writeHead(503);
response.end("Device is in session");
return;
}
try {
await collections.faults.deleteOne({
_id: `${deviceId}:task_${taskId}`,
});
} finally {
await releaseLock(`cwmp_session_${deviceId}`, token);
}
response.writeHead(200);
response.end();
@@ -563,7 +591,18 @@ async function handler(
const deviceId = decodeURIComponent(
DELETE_DEVICE_REGEX.exec(url.pathname)[1]
);
await apiFunctions.deleteDevice(deviceId);
try {
await apiFunctions.deleteDevice(deviceId);
} catch (err) {
if (err instanceof ResourceLockedError) {
response.writeHead(503);
response.end("Device is in session");
return;
}
throw err;
}
response.writeHead(200);
response.end();
} else if (QUERY_REGEX.test(url.pathname)) {
lib/ui/api.ts
+96 -28
View File
@@ -19,19 +19,23 @@
import { PassThrough } from "stream";
import Router from "koa-router";
import { ObjectId } from "mongodb";
import * as db from "./db";
import * as apiFunctions from "../api-functions";
import { evaluate, and, extractParams } from "../common/expression/util";
import { parse } from "../common/expression/parser";
import * as logger from "../logger";
import { getConfig } from "../ui/local-cache";
import { Expression } from "../types";
import { Expression, Task } from "../types";
import { generateSalt, hashPassword } from "../auth";
import { del } from "../cache";
import Authorizer from "../common/authorizer";
import { ping } from "../ping";
import { decodeTag } from "../util";
import { stringify as yamlStringify } from "../common/yaml";
import { ResourceLockedError } from "../common/errors";
import { acquireLock, releaseLock } from "../lock";
import { collections } from "../db/db";
const router = new Router();
export default router;
@@ -410,7 +414,18 @@ for (const [resource, flags] of Object.entries(resources)) {
return void (ctx.status = 403);
}
await apiFunctions.deleteResource(resource, ctx.params.id);
try {
await apiFunctions.deleteResource(resource, ctx.params.id);
} catch (err) {
if (err instanceof ResourceLockedError) {
log.message += " failed";
logger.accessWarn(log);
ctx.status = 503;
ctx.body = err.message;
return;
}
throw err;
}
logger.accessInfo(log);
@@ -533,32 +548,69 @@ router.put("/files/:id", async (ctx) => {
});
router.post("/devices/:id/tasks", async (ctx) => {
const deviceId = ctx.params.id;
const authorizer: Authorizer = ctx.state.authorizer;
const log = {
message: "Commit tasks",
context: ctx,
deviceId: ctx.params.id,
deviceId: deviceId,
tasks: null,
};
const filter = and(authorizer.getFilter("devices", 3), [
"=",
["PARAM", "DeviceID.ID"],
ctx.params.id,
]);
if (!authorizer.hasAccess("devices", 3)) {
logUnauthorizedWarning(log);
return void (ctx.status = 403);
}
const { value: device } = await db.query("devices", filter).next();
if (!device) return void (ctx.status = 404);
const validate = authorizer.getValidator("devices", device);
for (const t of ctx.request.body) {
if (!validate("task", t)) {
logUnauthorizedWarning(log);
return void (ctx.status = 403);
const socketTimeout: number = ctx.socket.timeout;
// Extend socket timeout while waiting for session
if (socketTimeout) ctx.socket.setTimeout(300000);
const token = await acquireLock(`cwmp_session_${deviceId}`, 5000, 30000);
if (!token) {
log.message += " failed";
logger.accessWarn(log);
ctx.body = "Device is in session";
ctx.status = 503;
// Restore socket timeout
if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
return;
}
let device;
let statuses: { _id: string; status: string }[];
try {
const filter = and(authorizer.getFilter("devices", 3), [
"=",
["PARAM", "DeviceID.ID"],
deviceId,
]);
device = (await db.query("devices", filter).next()).value;
if (!device) return void (ctx.status = 404);
const validate = authorizer.getValidator("devices", device);
for (const t of ctx.request.body) {
if (!validate("task", t)) {
logUnauthorizedWarning(log);
return void (ctx.status = 403);
}
}
let tasks = ctx.request.body as Task[];
for (const task of tasks) {
delete task._id;
task["device"] = deviceId;
}
tasks = await apiFunctions.insertTasks(tasks);
statuses = tasks.map((t) => ({ _id: t._id, status: "pending" }));
} finally {
await releaseLock(`cwmp_session_${deviceId}`, token);
}
const onlineThreshold = getConfig(
@@ -584,27 +636,43 @@ router.post("/devices/:id/tasks", async (ctx) => {
}
) as number;
const socketTimeout: number = ctx.socket["timeout"];
const lastInform = device["Events.Inform"].value[0] as number;
// Disable socket timeout while waiting for postTasks()
if (socketTimeout) ctx.socket.setTimeout(0);
let status = await apiFunctions.connectionRequest(deviceId, device);
if (!status) {
const sessionStarted = await apiFunctions.awaitSessionStart(
deviceId,
lastInform,
onlineThreshold
);
if (!sessionStarted) {
status = "No contact from CPE";
} else {
const sessionEnded = await apiFunctions.awaitSessionEnd(deviceId, 120000);
if (!sessionEnded) status = "Session took too long to complete";
}
}
const res = await apiFunctions.postTasks(
ctx.params.id,
ctx.request.body,
onlineThreshold,
device
);
if (!status) {
const promises = statuses.map((t) =>
collections.faults.count({ _id: `${deviceId}:task_${t._id}` })
);
const res = await Promise.all(promises);
for (const [i, r] of statuses.entries())
r.status = res[i] ? "fault" : "done";
}
await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
// Restore socket timeout
if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
log.tasks = res.tasks.map((t) => t._id).join(",");
log.tasks = statuses.map((t) => t._id).join(",");
logger.accessInfo(log);
ctx.set("Connection-Request", res.connectionRequest);
ctx.body = res.tasks;
ctx.set("Connection-Request", status || "OK");
ctx.body = statuses;
});
router.post("/devices/:id/tags", async (ctx) => {
ui/store.ts
-1
View File
@@ -618,7 +618,6 @@ export function postTasks(
for (const [i, t] of st.entries()) {
tasks[i]._id = t._id;
tasks[i].status = t.status;
tasks[i]["fault"] = t.fault;
}
return connectionRequestStatus;
},