Refactor db and local-cache code

Separated cwmp and ui code into distinct files to enhance clarity and
maintainability with varied internal data representations.

lib/ui/api-functions.ts

@@ -1,211 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { ObjectId } from "mongodb";
-import * as db from "./db";
-import { del } from "../cache";
-import { getUsers } from "../local-cache";
-import { hashPassword } from "../auth";
-import {
-  insertTasks,
-  awaitSessionEnd,
-  awaitSessionStart,
-  connectionRequest,
-  deleteDevice,
-} from "../api-functions";
-import { Task } from "../types";
-
-async function deleteFault(id): Promise<void> {
-  const deviceId = id.split(":", 1)[0];
-  const channel = id.slice(deviceId.length + 1);
-
-  await Promise.all([
-    db.deleteFault(id),
-    channel.startsWith("task_")
-      ? db.deleteTask(new ObjectId(channel.slice(5)))
-      : null,
-  ]);
-
-  await del(`${deviceId}_tasks_faults_operations`);
-}
-
-export async function deleteResource(
-  resource: string,
-  id: string
-): Promise<void> {
-  switch (resource) {
-    case "devices":
-      await deleteDevice(id);
-      break;
-
-    case "files":
-      await db.deleteFile(id);
-      break;
-
-    case "faults":
-      await deleteFault(id);
-      break;
-
-    case "provisions":
-      await db.deleteProvision(id);
-      break;
-
-    case "presets":
-      await db.deletePreset(id);
-      break;
-
-    case "virtualParameters":
-      await db.deleteVirtualParameter(id);
-      break;
-
-    case "config":
-      await db.deleteConfig(id);
-      break;
-
-    case "permissions":
-      await db.deletePermission(id);
-      break;
-
-    case "users":
-      await db.deleteUser(id);
-      break;
-  }
-
-  await del("presets_hash");
-}
-
-export async function postTasks(
-  deviceId: string,
-  tasks: Task[],
-  timeout: number,
-  device: Record<string, { value?: [boolean | number | string, string] }>
-): Promise<{ connectionRequest: string; tasks: any[] }> {
-  for (const task of tasks) {
-    delete task._id;
-    task["device"] = deviceId;
-  }
-
-  tasks = await insertTasks(tasks);
-  const statuses = tasks.map((t) => {
-    return { _id: t._id, status: "pending" };
-  });
-  const lastInform = Date.now();
-  const notInSession = await awaitSessionEnd(deviceId, 30000);
-  if (!notInSession) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
-
-  await del(`${deviceId}_tasks_faults_operations`);
-
-  const status = await connectionRequest(deviceId, device);
-
-  if (status) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: status,
-      tasks: statuses,
-    };
-  }
-
-  const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
-  if (!sessionStarted) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: "No contact from CPE",
-      tasks: statuses,
-    };
-  }
-
-  const sessionEnded = await awaitSessionEnd(deviceId, 120000);
-  if (!sessionEnded) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
-
-  const promises: Promise<number>[] = [];
-  for (const s of statuses) {
-    promises.push(db.count("tasks", ["=", ["PARAM", "_id"], s._id]));
-    promises.push(
-      db.count("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
-    );
-  }
-
-  const res = await Promise.all(promises);
-  for (const [i, r] of statuses.entries()) {
-    if (!res[i * 2]) {
-      r.status = "done";
-    } else if (res[i * 2 + 1]) {
-      r.status = "fault";
-      r["fault"] = res[i * 2 + 1][0];
-    }
-  }
-
-  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-
-  return { connectionRequest: "OK", tasks: statuses };
-}
-
-export async function putResource(
-  resource: string,
-  id: string,
-  data: Record<string, unknown>
-): Promise<void> {
-  if (resource === "presets") {
-    await db.putPreset(id, data);
-  } else if (resource === "provisions") {
-    await db.putProvision(id, data);
-  } else if (resource === "virtualParameters") {
-    await db.putVirtualParameter(id, data);
-  } else if (resource === "config") {
-    await db.putConfig(id, data);
-  } else if (resource === "permissions") {
-    await db.putPermission(id, data);
-  } else if (resource === "users") {
-    delete data["password"];
-    delete data["salt"];
-    await db.putUser(id, data);
-  }
-
-  await del("presets_hash");
-}
-
-export function authLocal(
-  snapshot: string,
-  username: string,
-  password: string
-): Promise<boolean> {
-  return new Promise((resolve, reject) => {
-    const users = getUsers(snapshot);
-    const user = users[username];
-    if (!user?.password) return void resolve(null);
-    hashPassword(password, user.salt)
-      .then((hash) => {
-        if (hash === user.password) resolve(true);
-        else resolve(false);
-      })
-      .catch(reject);
-  });
-}

lib/ui/api.ts

@@ -20,17 +20,17 @@
 import { PassThrough } from "stream";
 import Router from "koa-router";
 import * as db from "./db";
-import * as apiFunctions from "./api-functions";
+import * as apiFunctions from "../api-functions";
 import { evaluate, and, extractParams } from "../common/expression/util";
 import { parse } from "../common/expression/parser";
 import * as logger from "../logger";
-import { getConfig } from "../local-cache";
-import { QueryOptions, Expression } from "../types";
+import { getConfig } from "../ui/local-cache";
+import { Expression } from "../types";
 import { generateSalt, hashPassword } from "../auth";
 import { del } from "../cache";
 import Authorizer from "../common/authorizer";
 import { ping } from "../ping";
-import { decodeTag } from "../common";
+import { decodeTag } from "../util";
 import { stringify as yamlStringify } from "../common/yaml";
 
 const router = new Router();
@@ -178,7 +178,7 @@ for (const [resource, flags] of Object.entries(resources)) {
 
   router.get(`/${resource}`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
-    const options: QueryOptions = {};
+    const options: Parameters<typeof db.query>[2] = {};
     let filter: Expression = authorizer.getFilter(resource, 2);
     if (ctx.request.query.filter)
       filter = and(filter, parse(singleParam(ctx.request.query.filter)));
@@ -231,7 +231,7 @@ for (const [resource, flags] of Object.entries(resources)) {
   // CSV download
   router.get(`/${resource}.csv`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
-    const options: QueryOptions = { projection: {} };
+    const options: Parameters<typeof db.query>[2] = { projection: {} };
     let filter: Expression = authorizer.getFilter(resource, 2);
     if (ctx.request.query.filter)
       filter = and(filter, parse(singleParam(ctx.request.query.filter)));
@@ -712,7 +712,7 @@ router.put("/users/:id/password", async (ctx) => {
   const password = await hashPassword(newPassword, salt);
   await db.putUser(username, { password, salt });
 
-  await del("presets_hash");
+  await del("ui-local-cache-hash");
 
   logger.accessInfo(log);
   ctx.body = "";

lib/ui/db.ts

@@ -17,19 +17,362 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Collection, GridFSBucket, ObjectId } from "mongodb";
-import { Script } from "vm";
-import { collections, onConnect, optimizeProjection } from "../db";
-import * as mongodbFunctions from "../mongodb-functions";
+import { encodeTag } from "../util";
 import { evaluate } from "../common/expression/util";
-import { QueryOptions, Expression } from "../types";
+import { Expression, Fault, Task } from "../types";
+import { collections, filesBucket } from "../db/db";
+import { convertOldPrecondition, optimizeProjection } from "../db/util";
+import * as MongoTypes from "../db/types";
+import { parse, parseList, stringify } from "../common/expression/parser";
+import { toMongoQuery } from "../db/synth";
+import { Collection, ObjectId, WithoutId } from "mongodb";
+import { Script } from "vm";
 import { Readable } from "stream";
 
-let filesBucket: GridFSBucket;
+function processDeviceProjection(
+  projection: Record<string, 1>
+): Record<string, 1> {
+  if (!projection) return projection;
+  const p = {};
+  for (const [k, v] of Object.entries(projection)) {
+    if (k === "DeviceID.ID") {
+      p["_id"] = 1;
+    } else if (k.startsWith("DeviceID")) {
+      p["_deviceId._SerialNumber"] = v;
+      p["_deviceId._OUI"] = v;
+      p["_deviceId._ProductClass"] = v;
+      p["_deviceId._Manufacturer"] = v;
+    } else if (k.startsWith("Tags")) {
+      p["_tags"] = v;
+    } else if (k.startsWith("Events")) {
+      p["_lastInform"] = v;
+      p["_registered"] = v;
+      p["_lastBoot"] = v;
+      p["_lastBootstrap"] = v;
+    } else {
+      p[k] = v;
+    }
+  }
 
-onConnect(async (db) => {
-  filesBucket = new GridFSBucket(db);
-});
+  return p;
+}
+
+function processDeviceSort(
+  sort: Record<string, number>
+): Record<string, number> {
+  if (!sort) return sort;
+  const s = {};
+  for (const [k, v] of Object.entries(sort)) {
+    if (k === "DeviceID.ID") s["_id"] = v;
+    else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v;
+    else if (k === "Events.Inform") s["_lastInform"] = v;
+    else if (k === "Events.Registered") s["_registered"] = v;
+    else if (k === "Events.1_BOOT") s["_lastBoot"] = v;
+    else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v;
+    else s[`${k}._value`] = v;
+  }
+
+  return s;
+}
+
+function parseDate(d: Date): number | string {
+  const n = +d;
+  return isNaN(n) ? "" + d : n;
+}
+
+interface FlatAttributes {
+  object?: boolean;
+  objectTimestamp?: number;
+  writable?: boolean;
+  writableTimestamp?: number;
+  value?: [string | number | boolean, string];
+  valueTimestamp?: number;
+  notification?: number;
+  notificationTimestamp?: number;
+  accessList?: string[];
+  accessListTimestamp?: number;
+}
+
+interface FlatDevice {
+  [param: string]: FlatAttributes;
+}
+
+export function flattenDevice(device: Record<string, unknown>): FlatDevice {
+  function recursive(
+    input,
+    root: string,
+    output: FlatDevice,
+    timestamp: number
+  ): void {
+    for (const [name, tree] of Object.entries(input)) {
+      if (!root) {
+        if (name === "_lastInform") {
+          output["Events.Inform"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_registered") {
+          output["Events.Registered"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_lastBoot") {
+          output["Events.1_BOOT"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_lastBootstrap") {
+          output["Events.0_BOOTSTRAP"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_id") {
+          output["DeviceID.ID"] = {
+            value: [tree as string, "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_deviceId") {
+          output["DeviceID.Manufacturer"] = {
+            value: [tree["_Manufacturer"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.OUI"] = {
+            value: [tree["_OUI"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.ProductClass"] = {
+            value: [tree["_ProductClass"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.SerialNumber"] = {
+            value: [tree["_SerialNumber"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_tags") {
+          output["Tags"] = {
+            writable: false,
+            writableTimestamp: timestamp,
+            object: true,
+            objectTimestamp: timestamp,
+          };
+
+          for (const t of tree as string[]) {
+            output[`Tags.${encodeTag(t)}`] = {
+              value: [true, "xsd:boolean"],
+              valueTimestamp: timestamp,
+              writable: true,
+              writableTimestamp: timestamp,
+              object: false,
+              objectTimestamp: timestamp,
+            };
+          }
+        }
+      }
+
+      if (name.startsWith("_")) continue;
+
+      let childrenTimestamp = timestamp;
+
+      if (!root) childrenTimestamp = +(input["_timestamp"] || 1);
+      else if (+input["_timestamp"] > timestamp)
+        childrenTimestamp = +input["_timestamp"];
+
+      const attrs: FlatAttributes = {};
+      if (tree["_value"] != null) {
+        attrs.value = [
+          tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
+          tree["_type"],
+        ];
+        attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
+        attrs.object = false;
+        attrs.objectTimestamp = childrenTimestamp;
+      } else if (tree["_object"] != null) {
+        attrs.object = tree["_object"];
+        attrs.objectTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_writable"] != null) {
+        attrs.writable = tree["_writable"];
+        attrs.writableTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_notification"] != null) {
+        attrs.notification = tree["_notification"];
+        attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      if (tree["_accessList"] != null) {
+        attrs.accessList = tree["_accessList"];
+        attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      const r = root ? `${root}.${name}` : name;
+      output[r] = attrs;
+
+      if (attrs.object || tree["object"] == null)
+        recursive(tree, r, output, childrenTimestamp);
+    }
+  }
+
+  const newDevice: FlatDevice = {};
+  const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime();
+  recursive(device, "", newDevice, timestamp);
+  return newDevice;
+}
+
+function flattenFault(fault: unknown): Fault {
+  const f = Object.assign({}, fault) as Fault;
+  if (f.timestamp) f.timestamp = +f.timestamp;
+  if (f["expiry"]) f["expiry"] = +f["expiry"];
+  return f as Fault;
+}
+
+function flattenTask(task: unknown): Task {
+  const t = Object.assign({}, task) as Task;
+  t._id = "" + t._id;
+  if (t["timestamp"]) t["timestamp"] = +t["timestamp"];
+  if (t.expiry) t.expiry = +t.expiry;
+  return t;
+}
+
+function flattenPreset(
+  preset: Record<string, unknown>
+): Record<string, unknown> {
+  const p = Object.assign({}, preset);
+  if (p.precondition) {
+    try {
+      // Try parse to check expression validity
+      parse(p.precondition as string);
+    } catch (error) {
+      p.precondition = convertOldPrecondition(
+        JSON.parse(p.precondition as string)
+      );
+      p.precondition = (p.precondition as string).length
+        ? stringify(p.precondition as Expression)
+        : "";
+    }
+  }
+
+  if (p.events) {
+    const e = [];
+    for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`);
+    p.events = e.join(", ");
+  }
+
+  const provision = p.configurations[0];
+  if (
+    (p.configurations as any[]).length === 1 &&
+    provision.type === "provision" &&
+    provision.name &&
+    provision.name.length
+  ) {
+    p.provision = provision.name;
+    p.provisionArgs = provision.args
+      ? provision.args.map((a) => stringify(a)).join(", ")
+      : "";
+  }
+
+  delete p.configurations;
+  return p;
+}
+
+function flattenFile(file: Record<string, unknown>): Record<string, unknown> {
+  const f = {};
+  f["_id"] = file["_id"];
+  if (file.metadata) {
+    f["metadata.fileType"] = file["metadata"]["fileType"] || "";
+    f["metadata.oui"] = file["metadata"]["oui"] || "";
+    f["metadata.productClass"] = file["metadata"]["productClass"] || "";
+    f["metadata.version"] = file["metadata"]["version"] || "";
+  }
+  return f;
+}
+
+function preProcessPreset(data: Record<string, unknown>): MongoTypes.Preset {
+  const preset = Object.assign({}, data);
+
+  if (!preset.precondition) preset.precondition = "";
+  // Try parse to check expression validity
+  parse(preset.precondition as string);
+
+  preset.weight = parseInt(preset.weight as string) || 0;
+
+  const events = {};
+  if (preset.events) {
+    for (let e of (preset.events as string).split(",")) {
+      let v = true;
+      e = e.trim();
+      if (e.startsWith("-")) {
+        v = false;
+        e = e.slice(1).trim();
+      }
+      if (e) events[e] = v;
+    }
+  }
+
+  preset.events = events;
+
+  if (!preset.provision) throw new Error("Invalid preset provision");
+
+  const configuration = {
+    type: "provision",
+    name: preset.provision,
+    args: null,
+  };
+
+  if (preset.provisionArgs)
+    configuration.args = parseList(preset.provisionArgs as string);
+
+  delete preset.provision;
+  delete preset.provisionArgs;
+  preset.configurations = [configuration];
+  return preset as unknown as MongoTypes.Preset;
+}
+
+interface QueryOptions {
+  projection?: any;
+  skip?: number;
+  limit?: number;
+  sort?: {
+    [param: string]: number;
+  };
+}
 
 export async function* query(
   resource: string,
@@ -38,7 +381,7 @@ export async function* query(
 ): AsyncGenerator<any, void, undefined> {
   options = options || {};
   filter = evaluate(filter, null, Date.now());
-  const q = mongodbFunctions.toMongoQuery(filter, resource);
+  const q = toMongoQuery(filter, resource);
   if (!q) return;
 
   const collection = collections[resource] as Collection<any>;
@@ -46,7 +389,7 @@ export async function* query(
   if (options.projection) {
     let projection = options.projection;
     if (resource === "devices")
-      projection = mongodbFunctions.processDeviceProjection(options.projection);
+      projection = processDeviceProjection(options.projection);
 
     if (resource === "presets") projection.configurations = 1;
     projection = optimizeProjection(projection);
@@ -67,25 +410,25 @@ export async function* query(
         {}
       );
 
-    if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
+    if (resource === "devices") s = processDeviceSort(s);
     cursor.sort(s);
   }
 
   for await (let doc of cursor) {
-    if (resource === "devices") doc = mongodbFunctions.flattenDevice(doc);
-    else if (resource === "faults") doc = mongodbFunctions.flattenFault(doc);
-    else if (resource === "tasks") doc = mongodbFunctions.flattenTask(doc);
-    else if (resource === "presets") doc = mongodbFunctions.flattenPreset(doc);
-    else if (resource === "files") doc = mongodbFunctions.flattenFile(doc);
+    if (resource === "devices") doc = flattenDevice(doc);
+    else if (resource === "faults") doc = flattenFault(doc);
+    else if (resource === "tasks") doc = flattenTask(doc);
+    else if (resource === "presets") doc = flattenPreset(doc);
+    else if (resource === "files") doc = flattenFile(doc);
 
     yield doc;
   }
 }
 
 export function count(resource: string, filter: Expression): Promise<number> {
-  const collection = collections[resource] as Collection<unknown>;
+  const collection = collections[resource] as Collection<any>;
   filter = evaluate(filter, null, Date.now());
-  const q = mongodbFunctions.toMongoQuery(filter, resource);
+  const q = toMongoQuery(filter, resource);
   if (!q) return Promise.resolve(0);
   return collection.countDocuments(q);
 }
@@ -110,34 +453,21 @@ export async function updateDeviceTags(
   await collections.devices.updateOne({ _id: deviceId }, object);
 }
 
-async function putResource(resource, id, object): Promise<void> {
-  const collection = collections[resource] as Collection<unknown>;
-  await collection.replaceOne({ _id: id }, object, { upsert: true });
-}
-
-async function deleteResource(
-  resource: string,
-  id: string | ObjectId
-): Promise<void> {
-  const collection = collections[resource] as Collection<unknown>;
-  await collection.deleteOne({ _id: id });
-}
-
-export function putPreset(
+export async function putPreset(
   id: string,
   object: Record<string, unknown>
 ): Promise<void> {
-  object = mongodbFunctions.preProcessPreset(object);
-  return putResource("presets", id, object);
+  const p = preProcessPreset(object);
+  await collections.presets.replaceOne({ _id: id }, p, { upsert: true });
 }
 
-export function deletePreset(id: string): Promise<void> {
-  return deleteResource("presets", id);
+export async function deletePreset(id: string): Promise<void> {
+  await collections.presets.deleteOne({ _id: id });
 }
 
-export function putProvision(
+export async function putProvision(
   id: string,
-  object: Record<string, unknown>
+  object: { script: string }
 ): Promise<void> {
   if (!object.script) object.script = "";
   try {
@@ -153,16 +483,18 @@ export function putProvision(
     }
     return Promise.reject(err);
   }
-  return putResource("provisions", id, object);
+  await collections.provisions.replaceOne({ _id: id }, object, {
+    upsert: true,
+  });
 }
 
-export function deleteProvision(id: string): Promise<void> {
-  return deleteResource("provisions", id);
+export async function deleteProvision(id: string): Promise<void> {
+  await collections.provisions.deleteOne({ _id: id });
 }
 
-export function putVirtualParameter(
+export async function putVirtualParameter(
   id: string,
-  object: Record<string, unknown>
+  object: { script: string }
 ): Promise<void> {
   if (!object.script) object.script = "";
   try {
@@ -178,38 +510,42 @@ export function putVirtualParameter(
     }
     return Promise.reject(err);
   }
-  return putResource("virtualParameters", id, object);
+  await collections.virtualParameters.replaceOne({ _id: id }, object, {
+    upsert: true,
+  });
 }
 
-export function deleteVirtualParameter(id: string): Promise<void> {
-  return deleteResource("virtualParameters", id);
+export async function deleteVirtualParameter(id: string): Promise<void> {
+  await collections.virtualParameters.deleteOne({ _id: id });
 }
 
-export function putConfig(
+export async function putConfig(
   id: string,
-  object: Record<string, unknown>
+  object: WithoutId<MongoTypes.Config>
 ): Promise<void> {
-  return putResource("config", id, object);
+  await collections.config.replaceOne({ _id: id }, object, { upsert: true });
 }
 
-export function deleteConfig(id: string): Promise<void> {
-  return deleteResource("config", id);
+export async function deleteConfig(id: string): Promise<void> {
+  await collections.config.deleteOne({ _id: id });
 }
 
-export function putPermission(
+export async function putPermission(
   id: string,
-  object: Record<string, unknown>
+  object: WithoutId<MongoTypes.Permission>
 ): Promise<void> {
-  return putResource("permissions", id, object);
+  await collections.permissions.replaceOne({ _id: id }, object, {
+    upsert: true,
+  });
 }
 
-export function deletePermission(id: string): Promise<void> {
-  return deleteResource("permissions", id);
+export async function deletePermission(id: string): Promise<void> {
+  await collections.permissions.deleteOne({ _id: id });
 }
 
 export async function putUser(
   id: string,
-  object: Record<string, unknown>
+  object: Partial<WithoutId<MongoTypes.User>>
 ): Promise<void> {
   // update instead of replace to keep the password if not set by user
   await collections.users.updateOne(
@@ -219,8 +555,8 @@ export async function putUser(
   );
 }
 
-export function deleteUser(id: string): Promise<void> {
-  return deleteResource("users", id);
+export async function deleteUser(id: string): Promise<void> {
+  await collections.users.deleteOne({ _id: id });
 }
 
 export function downloadFile(filename: string): Readable {
@@ -267,10 +603,10 @@ export async function deleteFile(filename: string): Promise<void> {
   await filesBucket.delete(filename as any);
 }
 
-export function deleteFault(id: string): Promise<void> {
-  return deleteResource("faults", id);
+export async function deleteFault(id: string): Promise<void> {
+  await collections.faults.deleteOne({ _id: id });
 }
 
-export function deleteTask(id: ObjectId): Promise<void> {
-  return deleteResource("tasks", id);
+export async function deleteTask(id: ObjectId): Promise<void> {
+  await collections.tasks.deleteOne({ _id: id });
 }

lib/ui/local-cache.ts

@@ -0,0 +1,187 @@
+import * as crypto from "crypto";
+import * as config from "../config";
+import { collections } from "../db/db";
+import * as expression from "../common/expression/util";
+import { parse } from "../common/expression/parser";
+import { Expression, Users, Permissions, Config, UiConfig } from "../types";
+import { LocalCache } from "../local-cache";
+
+interface Snapshot {
+  permissions: Permissions;
+  users: Users;
+  config: Config;
+  ui: UiConfig;
+}
+
+async function fetchPermissions(): Promise<[string, Permissions]> {
+  const perms = await collections.permissions.find().toArray();
+  perms.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto
+    .createHash("md5")
+    .update(JSON.stringify(perms))
+    .digest("hex");
+  const permissions: Permissions = {};
+
+  for (const p of perms) {
+    if (!permissions[p.role]) permissions[p.role] = {};
+    if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {};
+
+    permissions[p.role][p.access][p.resource] = {
+      access: p.access,
+      filter: parse(p.filter || "true"),
+    };
+    if (p.validate)
+      permissions[p.role][p.access][p.resource].validate = parse(p.validate);
+  }
+
+  return [h, permissions];
+}
+
+async function fetchUsers(): Promise<[string, Users]> {
+  const _users = await collections.users.find().toArray();
+  _users.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto
+    .createHash("md5")
+    .update(JSON.stringify(_users))
+    .digest("hex");
+  const users = {};
+
+  for (const user of _users) {
+    users[user._id] = {
+      password: user.password,
+      salt: user.salt,
+      roles: user.roles.split(",").map((s) => s.trim()),
+    };
+  }
+
+  return [h, users];
+}
+
+async function fetchConfig(): Promise<[string, Config, UiConfig]> {
+  const conf = await collections.config.find().toArray();
+  conf.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(conf)).digest("hex");
+
+  const ui = {
+    filters: {},
+    device: {},
+    index: {},
+    overview: {},
+    pageSize: null,
+  };
+
+  const _config = {};
+
+  for (const c of conf) {
+    // Evaluate expressions to simplify them
+    const val = expression.evaluate(parse(c.value));
+    _config[c._id] = val;
+    if (c._id.startsWith("ui.")) {
+      const keys = c._id.split(".");
+      if (!(keys[1] in ui)) continue;
+      // remove the first key(ui)
+      keys.shift();
+      let ref = ui;
+      while (keys.length > 1) {
+        const k = keys.shift();
+        if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {};
+        ref = ref[k];
+      }
+      ref[keys[0]] = val;
+    }
+  }
+
+  return [h, _config, ui];
+}
+
+const localCache = new LocalCache<Snapshot>("ui-local-cache-hash", refresh);
+
+async function refresh(): Promise<[string, Snapshot]> {
+  const res = await Promise.all([
+    fetchPermissions(),
+    fetchUsers(),
+    fetchConfig(),
+  ]);
+
+  const h = crypto.createHash("md5");
+  for (const r of res) h.update(r[0]);
+
+  const snapshot = {
+    permissions: res[0][1],
+    users: res[1][1],
+    config: res[2][1],
+    ui: res[2][2],
+  };
+
+  return [h.digest("hex"), snapshot];
+}
+
+export async function getRevision(): Promise<string> {
+  return await localCache.getRevision();
+}
+
+export function getConfig(
+  revision: string,
+  key: string,
+  context: Record<string, unknown>,
+  now: number,
+  cb?: (e: Expression) => Expression
+): string | number | boolean | null {
+  const snapshot = localCache.get(revision);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  const oldOpts = {
+    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
+    "cwmp.debug": "DEBUG",
+    "cwmp.retryDelay": "RETRY_DELAY",
+    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
+    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
+    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
+    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
+    "cwmp.cookiesPath": "COOKIES_PATH",
+    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
+    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
+    "cwmp.connectionRequestAllowBasicAuth":
+      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
+    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
+    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
+  };
+
+  if (!(key in snapshot.config)) {
+    if (key in oldOpts) {
+      let id;
+      if (context?.["id"]) {
+        id = context["id"];
+      } else if (cb) {
+        id = cb(["PARAM", "DeviceID.ID"]);
+        if (Array.isArray(id)) id = null;
+      }
+      return config.get(oldOpts[key], id);
+    }
+    return null;
+  }
+
+  const v = expression.evaluate(snapshot.config[key], context, now, cb);
+  return Array.isArray(v) ? null : v;
+}
+
+export function getConfigExpression(revision: string, key: string): Expression {
+  const snapshot = localCache.get(revision);
+  return snapshot.config[key];
+}
+
+export function getUsers(revision: string): Users {
+  const snapshot = localCache.get(revision);
+  return snapshot.users;
+}
+
+export function getPermissions(revision: string): Permissions {
+  const snapshot = localCache.get(revision);
+  return snapshot.permissions;
+}
+
+export function getUiConfig(revision: string): UiConfig {
+  const snapshot = localCache.get(revision);
+  return snapshot.ui;
+}