Refactor db and local-cache code

Separated cwmp and ui code into distinct files to enhance clarity and
maintainability with varied internal data representations.

.eslintrc.json

@@ -21,7 +21,6 @@
     "no-buffer-constructor": "error",
     "prefer-const": ["error", { "destructuring": "all" }],
     "eqeqeq": ["error", "always", { "null": "ignore" }],
-    "no-return-await": "error",
     "@typescript-eslint/no-explicit-any": 0,
     "@typescript-eslint/no-use-before-define": [
       "error",

bin/genieacs-cwmp.ts

@@ -22,7 +22,7 @@ import * as logger from "../lib/logger";
 import * as cluster from "../lib/cluster";
 import * as server from "../lib/server";
 import * as cwmp from "../lib/cwmp";
-import * as db from "../lib/db";
+import * as db from "../lib/db/db";
 import * as extensions from "../lib/extensions";
 import { version as VERSION } from "../package.json";
 

bin/genieacs-fs.ts

@@ -22,7 +22,7 @@ import * as logger from "../lib/logger";
 import * as cluster from "../lib/cluster";
 import * as server from "../lib/server";
 import { listener } from "../lib/fs";
-import * as db from "../lib/db";
+import * as db from "../lib/db/db";
 import { version as VERSION } from "../package.json";
 
 logger.init("fs", VERSION);

bin/genieacs-nbi.ts

@@ -22,7 +22,7 @@ import * as logger from "../lib/logger";
 import * as cluster from "../lib/cluster";
 import * as server from "../lib/server";
 import { listener } from "../lib/nbi";
-import * as db from "../lib/db";
+import * as db from "../lib/db/db";
 import * as extensions from "../lib/extensions";
 import { version as VERSION } from "../package.json";
 

bin/genieacs-ui.ts

@@ -23,7 +23,7 @@ import * as cluster from "../lib/cluster";
 import * as server from "../lib/server";
 import { listener } from "../lib/ui";
 import * as extensions from "../lib/extensions";
-import * as db from "../lib/db";
+import * as db from "../lib/db/db";
 import { version as VERSION } from "../package.json";
 
 logger.init("ui", VERSION);

lib/api-functions.ts

@@ -17,21 +17,42 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { collections } from "./db";
+import { ObjectId } from "mongodb";
-import * as common from "./common";
+import { collections } from "./db/db";
+import {
+  count,
+  deleteConfig,
+  deleteFault,
+  deleteFile,
+  deletePermission,
+  deletePreset,
+  deleteProvision,
+  deleteTask,
+  deleteUser,
+  deleteVirtualParameter,
+  putConfig,
+  putPermission,
+  putPreset,
+  putProvision,
+  putUser,
+  putVirtualParameter,
+} from "./ui/db";
+import * as common from "./util";
 import * as cache from "./cache";
 import {
-  getCurrentSnapshot,
+  getRevision,
   getConfig,
   getConfigExpression,
-} from "./local-cache";
+  getUsers,
+} from "./ui/local-cache";
 import {
   httpConnectionRequest,
   udpConnectionRequest,
 } from "./connection-request";
 import { Expression, Task } from "./types";
-import { flattenDevice } from "./mongodb-functions";
 import { evaluate } from "./common/expression/util";
+import { hashPassword } from "./auth";
+import { flattenDevice } from "./ui/db";
 
 export async function connectionRequest(
   deviceId: string,
@@ -103,7 +124,7 @@ export async function connectionRequest(
     return exp;
   };
 
-  const snapshot = await getCurrentSnapshot();
+  const snapshot = await getRevision();
   const now = Date.now();
   const UDP_CONNECTION_REQUEST_PORT = +getConfig(
     snapshot,
@@ -350,3 +371,176 @@ export async function deleteDevice(deviceId: string): Promise<void> {
     cache.del(`${deviceId}_tasks_faults_operations`),
   ]);
 }
+
+export async function deleteResource(
+  resource: string,
+  id: string
+): Promise<void> {
+  if (resource === "devices") {
+    await deleteDevice(id);
+  } else if (resource === "files") {
+    await deleteFile(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "faults") {
+    const deviceId = id.split(":", 1)[0];
+    const channel = id.slice(deviceId.length + 1);
+    const proms = [deleteFault(id)];
+    if (channel.startsWith("task_"))
+      proms.push(deleteTask(new ObjectId(channel.slice(5))));
+    await Promise.all(proms);
+    await cache.del(`${deviceId}_tasks_faults_operations`);
+  } else if (resource === "provisions") {
+    await deleteProvision(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "presets") {
+    await deletePreset(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "virtualParameters") {
+    await deleteVirtualParameter(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "config") {
+    await deleteConfig(id);
+    await Promise.all([
+      cache.del("ui-local-cache-hash"),
+      cache.del("cwmp-local-cache-hash"),
+    ]);
+  } else if (resource === "permissions") {
+    await deletePermission(id);
+    await cache.del("ui-local-cache-hash");
+  } else if (resource === "users") {
+    await deleteUser(id);
+    await cache.del("ui-local-cache-hash");
+  } else {
+    throw new Error(`Unknown resource ${resource}`);
+  }
+}
+
+export async function postTasks(
+  deviceId: string,
+  tasks: Task[],
+  timeout: number,
+  device: Record<string, { value?: [boolean | number | string, string] }>
+): Promise<{ connectionRequest: string; tasks: any[] }> {
+  for (const task of tasks) {
+    delete task._id;
+    task["device"] = deviceId;
+  }
+
+  tasks = await insertTasks(tasks);
+  const statuses = tasks.map((t) => {
+    return { _id: t._id, status: "pending" };
+  });
+  const lastInform = Date.now();
+  const notInSession = await awaitSessionEnd(deviceId, 30000);
+  if (!notInSession) {
+    await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
+    return {
+      connectionRequest: "Session took too long to complete",
+      tasks: statuses,
+    };
+  }
+
+  await cache.del(`${deviceId}_tasks_faults_operations`);
+
+  const status = await connectionRequest(deviceId, device);
+
+  if (status) {
+    await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
+    return {
+      connectionRequest: status,
+      tasks: statuses,
+    };
+  }
+
+  const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
+  if (!sessionStarted) {
+    await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
+    return {
+      connectionRequest: "No contact from CPE",
+      tasks: statuses,
+    };
+  }
+
+  const sessionEnded = await awaitSessionEnd(deviceId, 120000);
+  if (!sessionEnded) {
+    await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
+    return {
+      connectionRequest: "Session took too long to complete",
+      tasks: statuses,
+    };
+  }
+
+  const promises: Promise<number>[] = [];
+  for (const s of statuses) {
+    promises.push(count("tasks", ["=", ["PARAM", "_id"], s._id]));
+    promises.push(
+      count("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
+    );
+  }
+
+  const res = await Promise.all(promises);
+  for (const [i, r] of statuses.entries()) {
+    if (!res[i * 2]) {
+      r.status = "done";
+    } else if (res[i * 2 + 1]) {
+      r.status = "fault";
+      r["fault"] = res[i * 2 + 1][0];
+    }
+  }
+
+  await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id))));
+
+  return { connectionRequest: "OK", tasks: statuses };
+}
+
+// TODO Implement validation
+export async function putResource(
+  resource: string,
+  id: string,
+  data: any
+): Promise<void> {
+  if (resource === "presets") {
+    await putPreset(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "provisions") {
+    await putProvision(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "virtualParameters") {
+    await putVirtualParameter(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "config") {
+    await putConfig(id, data);
+    await Promise.all([
+      cache.del("ui-local-cache-hash"),
+      cache.del("cwmp-local-cache-hash"),
+    ]);
+  } else if (resource === "permissions") {
+    await putPermission(id, data);
+    await cache.del("ui-local-cache-hash");
+  } else if (resource === "users") {
+    delete data["password"];
+    delete data["salt"];
+    await putUser(id, data);
+    await cache.del("ui-local-cache-hash");
+  } else {
+    throw new Error(`Unknown resource ${resource}`);
+  }
+}
+
+export function authLocal(
+  snapshot: string,
+  username: string,
+  password: string
+): Promise<boolean> {
+  return new Promise((resolve, reject) => {
+    const users = getUsers(snapshot);
+    const user = users[username];
+    if (!user?.password) return void resolve(null);
+    hashPassword(password, user.salt)
+      .then((hash) => {
+        if (hash === user.password) resolve(true);
+        else resolve(false);
+      })
+      .catch(reject);
+  });
+}

lib/cache.ts

@@ -17,28 +17,19 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Collection } from "mongodb";
+import { collections } from "./db/db";
-import { onConnect } from "./db";
 import * as config from "./config";
-import { Cache } from "./mongodb-types";
 
 const CLOCK_SKEW_TOLERANCE = 30000;
 const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
 
-let cacheCollection: Collection<Cache>;
-
-onConnect(async (db) => {
-  cacheCollection = db.collection("cache");
-  await cacheCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
-});
-
 export async function get(key: string): Promise<string> {
-  const res = await cacheCollection.findOne({ _id: key });
+  const res = await collections.cache.findOne({ _id: key });
   return res?.value;
 }
 
 export async function del(key: string): Promise<void> {
-  await cacheCollection.deleteOne({ _id: key });
+  await collections.cache.deleteOne({ _id: key });
 }
 
 export async function set(
@@ -50,7 +41,7 @@ export async function set(
   const expire = new Date(
     timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000
   );
-  await cacheCollection.replaceOne(
+  await collections.cache.replaceOne(
     { _id: key },
     { value, expire, timestamp },
     { upsert: true }
@@ -58,7 +49,7 @@ export async function set(
 }
 
 export async function pop(key: string): Promise<string> {
-  const res = await cacheCollection.findOneAndDelete({ _id: key });
+  const res = await collections.cache.findOneAndDelete({ _id: key });
   return res.value?.value;
 }
 
@@ -70,7 +61,7 @@ export async function acquireLock(
 ): Promise<string> {
   try {
     const now = Date.now();
-    const r = await cacheCollection.findOneAndUpdate(
+    const r = await collections.cache.findOneAndUpdate(
       { _id: lockName, value: token },
       {
         $set: {
@@ -97,7 +88,7 @@ export async function releaseLock(
   lockName: string,
   token: string
 ): Promise<void> {
-  const res = await cacheCollection.deleteOne({
+  const res = await collections.cache.deleteOne({
     _id: lockName,
     value: token,
   });

lib/cwmp.ts

@@ -22,7 +22,7 @@ import * as crypto from "crypto";
 import { Socket } from "net";
 import * as auth from "./auth";
 import * as config from "./config";
-import * as common from "./common";
+import { generateDeviceId, once, setTimeoutPromise } from "./util";
 import * as soap from "./soap";
 import * as session from "./session";
 import {
@@ -31,8 +31,19 @@ import {
   extractParams,
 } from "./common/expression/util";
 import * as cache from "./cache";
-import * as localCache from "./local-cache";
+import * as localCache from "./cwmp/local-cache";
-import * as db from "./db";
+import {
+  clearTasks,
+  deleteFault,
+  deleteOperation,
+  fetchDevice,
+  getDueTasks,
+  getFaults,
+  getOperations,
+  saveDevice,
+  saveFault,
+  saveOperation,
+} from "./cwmp/db";
 import * as logger from "./logger";
 import * as scheduling from "./scheduling";
 import Path from "./common/path";
@@ -59,7 +70,6 @@ import { parseXmlDeclaration } from "./xml-parser";
 import * as debug from "./debug";
 import { getRequestOrigin } from "./forwarded";
 import { getSocketEndpoints } from "./server";
-import { EventEmitter } from "events";
 
 const gzipPromisified = promisify(zlib.gzip);
 const deflatePromisified = promisify(zlib.deflate);
@@ -773,7 +783,7 @@ async function endSession(sessionContext: SessionContext): Promise<void> {
   const promises = [];
 
   promises.push(
-    db.saveDevice(
+    saveDevice(
       sessionContext.deviceId,
       sessionContext.deviceData,
       sessionContext.new,
@@ -786,14 +796,14 @@ async function endSession(sessionContext: SessionContext): Promise<void> {
       saveCache = true;
       if (sessionContext.operations[k]) {
         promises.push(
-          db.saveOperation(
+          saveOperation(
             sessionContext.deviceId,
             k,
             sessionContext.operations[k]
           )
         );
       } else {
-        promises.push(db.deleteOperation(sessionContext.deviceId, k));
+        promises.push(deleteOperation(sessionContext.deviceId, k));
       }
     }
   }
@@ -801,7 +811,7 @@ async function endSession(sessionContext: SessionContext): Promise<void> {
   if (sessionContext.doneTasks?.length) {
     saveCache = true;
     promises.push(
-      db.clearTasks(sessionContext.deviceId, sessionContext.doneTasks)
+      clearTasks(sessionContext.deviceId, sessionContext.doneTasks)
     );
   }
 
@@ -811,10 +821,10 @@ async function endSession(sessionContext: SessionContext): Promise<void> {
       if (sessionContext.faults[k]) {
         sessionContext.faults[k].retries = sessionContext.retries[k];
         promises.push(
-          db.saveFault(sessionContext.deviceId, k, sessionContext.faults[k])
+          saveFault(sessionContext.deviceId, k, sessionContext.faults[k])
         );
       } else {
-        promises.push(db.deleteFault(sessionContext.deviceId, k));
+        promises.push(deleteFault(sessionContext.deviceId, k));
       }
     }
   }
@@ -890,30 +900,6 @@ async function sendAcsRequest(
   return writeResponse(sessionContext, res);
 }
 
-function once(
-  emitter: EventEmitter,
-  event: string,
-  timeout: number
-): Promise<unknown[]> {
-  return new Promise((resolve, reject) => {
-    const timer = setTimeout(() => {
-      reject(new Error(`Event ${event} timed out after ${timeout} ms`));
-    }, timeout);
-
-    emitter.once(event, (...args: unknown[]) => {
-      clearTimeout(timer);
-      resolve(args);
-    });
-  });
-}
-
-function setTimeoutPromise(delay: number, ref = true): Promise<void> {
-  return new Promise((resolve) => {
-    const timerId = setTimeout(resolve, delay);
-    if (!ref) timerId.unref();
-  });
-}
-
 // When socket closes, store active sessions in cache
 export async function onConnection(socket: Socket): Promise<void> {
   await once(socket, "close", 300000);
@@ -966,7 +952,7 @@ export async function onConnection(socket: Socket): Promise<void> {
 
 export async function onClientError(err: Error, socket: Socket): Promise<void> {
   const remoteAddress = getSocketEndpoints(socket).remoteAddress;
-  const cacheSnapshot = await localCache.getCurrentSnapshot();
+  const cacheSnapshot = await localCache.getRevision();
   const debugEnabled = !!localCache.getConfig(
     cacheSnapshot,
     "cwmp.debug",
@@ -1021,9 +1007,9 @@ async function getDueTasksAndFaultsAndOperations(
   }
 
   const res2 = await Promise.all([
-    db.getDueTasks(deviceId, timestamp),
+    getDueTasks(deviceId, timestamp),
-    db.getFaults(deviceId),
+    getFaults(deviceId),
-    db.getOperations(deviceId),
+    getOperations(deviceId),
   ]);
   return {
     tasks: res2[0][0],
@@ -1383,7 +1369,7 @@ async function clientError(
     debugEnabled = sessionContext.debug;
     deviceId = sessionContext.deviceId;
   } else {
-    const cacheSnapshot = await localCache.getCurrentSnapshot();
+    const cacheSnapshot = await localCache.getRevision();
     debugEnabled = !!localCache.getConfig(
       cacheSnapshot,
       "cwmp.debug",
@@ -1637,9 +1623,9 @@ async function listenerAsync(
   }
 
   stats.initiatedSessions += 1;
-  const deviceId = common.generateDeviceId(rpc.cpeRequest.deviceId);
+  const deviceId = generateDeviceId(rpc.cpeRequest.deviceId);
 
-  const cacheSnapshot = await localCache.getCurrentSnapshot();
+  const cacheSnapshot = await localCache.getRevision();
 
   const _sessionContext = session.init(
     deviceId,
@@ -1679,7 +1665,7 @@ async function listenerAsync(
     }
   }
 
-  const parameters = await db.fetchDevice(
+  const parameters = await fetchDevice(
     _sessionContext.deviceId,
     _sessionContext.timestamp
   );

lib/cwmp/db.ts

@@ -1,53 +1,16 @@
-/**
+import { decodeTag, encodeTag, escapeRegExp } from "../util";
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { MongoClient, ObjectId, Collection, Db } from "mongodb";
-import { get } from "./config";
-import { decodeTag, encodeTag, escapeRegExp } from "./common";
-import { parse } from "./common/expression/parser";
 import {
   DeviceData,
   Attributes,
   SessionFault,
   Task,
   Operation,
-  Expression,
+} from "../types";
-} from "./types";
+import Path from "../common/path";
-import Path from "./common/path";
+import { collections } from "../db/db";
-import * as MongoTypes from "./mongodb-types";
+import { optimizeProjection } from "../db/util";
-
+import * as MongoTypes from "../db/types";
-export const collections = {
+import { ObjectId } from "mongodb";
-  devices: null as Collection<MongoTypes.Device>,
-  presets: null as Collection<MongoTypes.Preset>,
-  objects: null as Collection<MongoTypes.Object>,
-  provisions: null as Collection<MongoTypes.Provision>,
-  virtualParameters: null as Collection<MongoTypes.VirtualParameter>,
-  faults: null as Collection<MongoTypes.Fault>,
-  tasks: null as Collection<MongoTypes.Task>,
-  files: null as Collection<MongoTypes.File>,
-  operations: null as Collection<MongoTypes.Operation>,
-  permissions: null as Collection<MongoTypes.Permission>,
-  users: null as Collection<MongoTypes.User>,
-  config: null as Collection<MongoTypes.Config>,
-};
-
-let clientPromise: Promise<MongoClient>;
 
 const INVALID_PATH_SUFFIX = "__invalid";
 
@@ -57,64 +20,6 @@ function compareAccessLists(list1: string[], list2: string[]): boolean {
   return true;
 }
 
-const onConnectCallbacks: ((db: Db) => Promise<void>)[] = [];
-
-export function onConnect(callback: (db: Db) => Promise<void>): void {
-  onConnectCallbacks.push(callback);
-}
-
-export async function connect(): Promise<void> {
-  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"));
-
-  const client = await clientPromise;
-  const db = client.db();
-  await Promise.all(onConnectCallbacks.map((c) => c(db)));
-}
-
-onConnect(async (db) => {
-  collections.tasks = db.collection("tasks");
-  await collections.tasks.createIndex({ device: 1, timestamp: 1 });
-
-  collections.devices = db.collection("devices");
-  collections.presets = db.collection("presets");
-  collections.objects = db.collection("objects");
-  collections.files = db.collection("fs.files");
-  collections.provisions = db.collection("provisions");
-  collections.virtualParameters = db.collection("virtualParameters");
-  collections.faults = db.collection("faults");
-  collections.operations = db.collection("operations");
-  collections.permissions = db.collection("permissions");
-  collections.users = db.collection("users");
-  collections.config = db.collection("config");
-});
-
-export async function disconnect(): Promise<void> {
-  if (clientPromise != null) await (await clientPromise).close();
-}
-
-// Optimize projection by removing overlaps
-// This can modify the object
-export function optimizeProjection(obj: { [path: string]: 1 }): {
-  [path: string]: 1;
-} {
-  if (obj[""]) return { "": obj[""] };
-
-  const keys = Object.keys(obj).sort();
-  if (keys.length <= 1) return obj;
-
-  for (let i = 1; i < keys.length; ++i) {
-    const a = keys[i - 1];
-    const b = keys[i];
-    if (b.startsWith(a)) {
-      if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") {
-        delete obj[b];
-        keys.splice(i--, 1);
-      }
-    }
-  }
-  return obj;
-}
-
 export async function fetchDevice(
   id: string,
   timestamp: number
@@ -822,56 +727,3 @@ export async function deleteOperation(
 ): Promise<void> {
   await collections.operations.deleteOne({ _id: `${deviceId}:${commandKey}` });
 }
-
-export async function getPresets(): Promise<Record<string, any>[]> {
-  return collections.presets.find().toArray();
-}
-
-export async function getObjects(): Promise<Record<string, any>[]> {
-  return collections.objects.find().toArray();
-}
-
-export async function getProvisions(): Promise<Record<string, any>[]> {
-  return collections.provisions.find().toArray();
-}
-
-export async function getVirtualParameters(): Promise<Record<string, any>[]> {
-  return collections.virtualParameters.find().toArray();
-}
-
-export function getFiles(): Promise<Record<string, any>[]> {
-  return collections.files.find().toArray();
-}
-
-export async function getConfig(): Promise<
-  { id: string; value: Expression }[]
-> {
-  const res = await collections.config.find().toArray();
-  return res.map((c) => ({
-    id: c["_id"],
-    value: parse(c["value"]),
-  }));
-}
-
-interface Permission {
-  role: string;
-  resource: string;
-  access: number;
-  filter: string;
-  validate: string;
-}
-
-export async function getPermissions(): Promise<Permission[]> {
-  return collections.permissions.find().toArray() as unknown as Permission[];
-}
-
-interface User {
-  _id: string;
-  password: string;
-  salt: string;
-  roles: string;
-}
-
-export async function getUsers(): Promise<User[]> {
-  return collections.users.find().toArray() as unknown as User[];
-}

lib/cwmp/local-cache.ts

@@ -0,0 +1,359 @@
+import * as vm from "vm";
+import * as crypto from "crypto";
+import * as config from "../config";
+import { collections } from "../db/db";
+import { convertOldPrecondition } from "../db/util";
+import * as logger from "../logger";
+import * as scheduling from "../scheduling";
+import * as expression from "../common/expression/util";
+import { parse } from "../common/expression/parser";
+import {
+  Preset,
+  Expression,
+  Provisions,
+  VirtualParameters,
+  Files,
+  Config,
+} from "../types";
+import { LocalCache } from "../local-cache";
+
+interface Snapshot {
+  presets: Preset[];
+  provisions: Provisions;
+  virtualParameters: VirtualParameters;
+  files: Files;
+  config: Config;
+}
+
+function flattenObject<T extends Record<keyof T, unknown>>(
+  src: T,
+  prefix = "",
+  dst = {} as T
+): T {
+  const t: Record<string, string> = {};
+  t["hi"] = "hi";
+  for (const k of Object.keys(src)) {
+    const v = src[k];
+    if (typeof v === "object" && !Array.isArray(v))
+      flattenObject(v as T, `${prefix}${k}.`, dst);
+    else dst[`${prefix}${k}`] = v;
+  }
+  return dst;
+}
+
+async function fetchPresets(): Promise<[string, Preset[]]> {
+  const res = await collections.presets.find().toArray();
+  let objects = await collections.objects.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  objects.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto
+    .createHash("md5")
+    .update(JSON.stringify(res))
+    .update(JSON.stringify(objects))
+    .digest("hex");
+
+  objects = objects.map((obj) => {
+    // Flatten object
+    obj = flattenObject(obj);
+
+    // If no keys are defined, consider all parameters as keys to keep the
+    // same behavior from v1.0
+    if (!obj["_keys"]?.length)
+      obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
+
+    return obj;
+  });
+
+  res.sort((a, b) => {
+    if (a["weight"] === b["weight"])
+      return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
+    else return a["weight"] - b["weight"];
+  });
+
+  const presets = [] as Preset[];
+  for (const preset of res) {
+    let schedule: { md5: string; duration: number; schedule: any } = null;
+    if (preset["schedule"]) {
+      const parts = preset["schedule"].trim().split(/\s+/);
+      schedule = {
+        md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
+        duration: null,
+        schedule: null,
+      };
+
+      try {
+        schedule.duration = +parts.shift() * 1000;
+        schedule.schedule = scheduling.parseCron(parts.join(" "));
+      } catch (err) {
+        logger.warn({
+          message: "Invalid preset schedule",
+          preset: preset["_id"],
+          schedule: preset["schedule"],
+        });
+        schedule.schedule = false;
+      }
+    }
+
+    const events = preset["events"] || {};
+    let precondition = true as Expression;
+    if (preset["precondition"]) {
+      try {
+        precondition = parse(preset["precondition"]);
+      } catch (error) {
+        precondition = convertOldPrecondition(
+          JSON.parse(preset["precondition"])
+        );
+      }
+
+      // Simplify expression
+      precondition = expression.evaluate(precondition);
+    }
+
+    const _provisions: Preset["provisions"] = [];
+
+    // Generate provisions from the old configuration format
+    for (const c of preset.configurations) {
+      switch (c.type) {
+        case "age":
+          _provisions.push(["refresh", c.name, +c.age]);
+          break;
+
+        case "value":
+          _provisions.push(["value", c.name, c.value]);
+          break;
+
+        case "add_tag":
+          _provisions.push(["tag", c.tag, true]);
+          break;
+
+        case "delete_tag":
+          _provisions.push(["tag", c.tag, false]);
+          break;
+
+        case "provision":
+          _provisions.push([c.name, ...(c.args || [])]);
+          break;
+
+        case "add_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 1]);
+
+              for (const k in obj) {
+                if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
+                  _provisions.push(["value", `${p}.${k}`, obj[k]]);
+              }
+            }
+          }
+
+          break;
+
+        case "delete_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 0]);
+            }
+          }
+
+          break;
+
+        default: {
+          const exhaustiveCheck: never = c;
+          throw new Error(
+            `Unknown configuration type ${exhaustiveCheck["type"]}`
+          );
+        }
+      }
+    }
+
+    presets.push({
+      name: preset["_id"],
+      channel: (preset["channel"] as string) || "default",
+      schedule: schedule,
+      events: events,
+      precondition: precondition,
+      provisions: _provisions,
+    });
+  }
+
+  return [h, presets];
+}
+
+async function fetchProvisions(): Promise<[string, Provisions]> {
+  const res = await collections.provisions.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+
+  const provisions = {};
+  for (const r of res) {
+    provisions[r._id] = {};
+    provisions[r._id].md5 = crypto
+      .createHash("md5")
+      .update(r.script)
+      .digest("hex");
+    provisions[r._id].script = new vm.Script(
+      `"use strict";(function(){\n${r.script}\n})();`,
+      { filename: r._id, lineOffset: -1, timeout: 50 }
+    );
+  }
+
+  return [h, provisions];
+}
+
+async function fetchVirtualParameters(): Promise<[string, VirtualParameters]> {
+  const res = await collections.virtualParameters.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+
+  const virtualParameters = {};
+  for (const r of res) {
+    virtualParameters[r._id] = {};
+    virtualParameters[r._id].md5 = crypto
+      .createHash("md5")
+      .update(r.script)
+      .digest("hex");
+    virtualParameters[r._id].script = new vm.Script(
+      `"use strict";(function(){\n${r.script}\n})();`,
+      { filename: r._id, lineOffset: -1, timeout: 50 }
+    );
+  }
+
+  return [h, virtualParameters];
+}
+
+async function fetchFiles(): Promise<[string, Files]> {
+  const res = await collections.files.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+  const files = {};
+
+  for (const r of res) {
+    const id = r.filename || r._id.toString();
+    files[id] = {};
+    files[id].length = r.length;
+  }
+
+  return [h, files];
+}
+
+async function fetchConfig(): Promise<[string, Config]> {
+  const conf = await collections.config.find().toArray();
+  conf.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(conf)).digest("hex");
+
+  const _config = {};
+
+  for (const c of conf) {
+    // Evaluate expressions to simplify them
+    _config[c._id] = expression.evaluate(parse(c.value));
+  }
+
+  return [h, _config];
+}
+
+const localCache = new LocalCache<Snapshot>("cwmp-local-cache-hash", refresh);
+
+async function refresh(): Promise<[string, Snapshot]> {
+  const res = await Promise.all([
+    fetchPresets(),
+    fetchProvisions(),
+    fetchVirtualParameters(),
+    fetchFiles(),
+    fetchConfig(),
+  ]);
+
+  const h = crypto.createHash("md5");
+  for (const r of res) h.update(r[0]);
+
+  const snapshot = {
+    presets: res[0][1],
+    provisions: res[1][1],
+    virtualParameters: res[2][1],
+    files: res[3][1],
+    config: res[4][1],
+  };
+
+  return [h.digest("hex"), snapshot];
+}
+
+export async function getRevision(): Promise<string> {
+  return await localCache.getRevision();
+}
+
+export function getPresets(revision: string): Preset[] {
+  return localCache.get(revision).presets;
+}
+
+export function getProvisions(revision: string): Provisions {
+  return localCache.get(revision).provisions;
+}
+
+export function getVirtualParameters(revision: string): VirtualParameters {
+  return localCache.get(revision).virtualParameters;
+}
+
+export function getFiles(revision: string): Files {
+  return localCache.get(revision).files;
+}
+
+export function getConfig(
+  snapshotKey: string,
+  key: string,
+  context: Record<string, unknown>,
+  now: number,
+  cb?: (e: Expression) => Expression
+): string | number | boolean | null {
+  const snapshot = localCache.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  const oldOpts = {
+    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
+    "cwmp.debug": "DEBUG",
+    "cwmp.retryDelay": "RETRY_DELAY",
+    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
+    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
+    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
+    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
+    "cwmp.cookiesPath": "COOKIES_PATH",
+    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
+    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
+    "cwmp.connectionRequestAllowBasicAuth":
+      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
+    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
+    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
+  };
+
+  if (!(key in snapshot.config)) {
+    if (key in oldOpts) {
+      let id;
+      if (context?.["id"]) {
+        id = context["id"];
+      } else if (cb) {
+        id = cb(["PARAM", "DeviceID.ID"]);
+        if (Array.isArray(id)) id = null;
+      }
+      return config.get(oldOpts[key], id);
+    }
+    return null;
+  }
+
+  const v = expression.evaluate(snapshot.config[key], context, now, cb);
+  return Array.isArray(v) ? null : v;
+}
+
+export function getConfigExpression(
+  snapshotKey: string,
+  key: string
+): Expression {
+  const snapshot = localCache.get(snapshotKey);
+  return snapshot.config[key];
+}

lib/db/db.ts

@@ -0,0 +1,73 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { MongoClient, Collection, GridFSBucket } from "mongodb";
+import { get } from "../config";
+import * as MongoTypes from "./types";
+
+export let filesBucket: GridFSBucket;
+
+export const collections = {
+  devices: null as Collection<MongoTypes.Device>,
+  presets: null as Collection<MongoTypes.Preset>,
+  objects: null as Collection<MongoTypes.Object>,
+  provisions: null as Collection<MongoTypes.Provision>,
+  virtualParameters: null as Collection<MongoTypes.VirtualParameter>,
+  faults: null as Collection<MongoTypes.Fault>,
+  tasks: null as Collection<MongoTypes.Task>,
+  files: null as Collection<MongoTypes.File>,
+  operations: null as Collection<MongoTypes.Operation>,
+  permissions: null as Collection<MongoTypes.Permission>,
+  users: null as Collection<MongoTypes.User>,
+  config: null as Collection<MongoTypes.Config>,
+  cache: null as Collection<MongoTypes.Cache>,
+};
+
+let clientPromise: Promise<MongoClient>;
+
+export async function connect(): Promise<void> {
+  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"));
+
+  const client = await clientPromise;
+  const db = client.db();
+
+  collections.tasks = db.collection("tasks");
+  collections.devices = db.collection("devices");
+  collections.presets = db.collection("presets");
+  collections.objects = db.collection("objects");
+  collections.files = db.collection("fs.files");
+  collections.provisions = db.collection("provisions");
+  collections.virtualParameters = db.collection("virtualParameters");
+  collections.faults = db.collection("faults");
+  collections.operations = db.collection("operations");
+  collections.permissions = db.collection("permissions");
+  collections.users = db.collection("users");
+  collections.config = db.collection("config");
+  collections.cache = db.collection("cache");
+  filesBucket = new GridFSBucket(db);
+
+  await Promise.all([
+    collections.tasks.createIndex({ device: 1, timestamp: 1 }),
+    collections.cache.createIndex({ expire: 1 }, { expireAfterSeconds: 0 }),
+  ]);
+}
+
+export async function disconnect(): Promise<void> {
+  if (clientPromise != null) await (await clientPromise).close();
+}

lib/db/synth.ts

@@ -1,461 +1,16 @@
 import { Filter } from "mongodb";
 import { EJSON } from "bson";
-import {
+import { parseLikePattern } from "../common/expression/parser";
-  parse,
+import { Expression } from "../types";
-  stringify,
+import { decodeTag } from "../util";
-  parseList,
-  parseLikePattern,
-} from "./common/expression/parser";
-import { Expression, Fault, Task } from "./types";
-import { decodeTag, encodeTag } from "./common";
 import {
   SynthContextBase,
   likeDisjoint,
   likeImplies,
-} from "./common/expression/synth";
+  Clause,
-import normalize from "./common/expression/normalize";
+} from "../common/expression/synth";
-import { Clause } from "./common/expression/synth";
+import normalize from "../common/expression/normalize";
-import * as BI from "./common/expression/bigint";
+import * as BI from "../common/expression/bigint";
-
-export function processDeviceProjection(
-  projection: Record<string, 1>
-): Record<string, 1> {
-  if (!projection) return projection;
-  const p = {};
-  for (const [k, v] of Object.entries(projection)) {
-    if (k === "DeviceID.ID") {
-      p["_id"] = 1;
-    } else if (k.startsWith("DeviceID")) {
-      p["_deviceId._SerialNumber"] = v;
-      p["_deviceId._OUI"] = v;
-      p["_deviceId._ProductClass"] = v;
-      p["_deviceId._Manufacturer"] = v;
-    } else if (k.startsWith("Tags")) {
-      p["_tags"] = v;
-    } else if (k.startsWith("Events")) {
-      p["_lastInform"] = v;
-      p["_registered"] = v;
-      p["_lastBoot"] = v;
-      p["_lastBootstrap"] = v;
-    } else {
-      p[k] = v;
-    }
-  }
-
-  return p;
-}
-
-export function processDeviceSort(
-  sort: Record<string, number>
-): Record<string, number> {
-  if (!sort) return sort;
-  const s = {};
-  for (const [k, v] of Object.entries(sort)) {
-    if (k === "DeviceID.ID") s["_id"] = v;
-    else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v;
-    else if (k === "Events.Inform") s["_lastInform"] = v;
-    else if (k === "Events.Registered") s["_registered"] = v;
-    else if (k === "Events.1_BOOT") s["_lastBoot"] = v;
-    else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v;
-    else s[`${k}._value`] = v;
-  }
-
-  return s;
-}
-
-function parseDate(d: Date): number | string {
-  const n = +d;
-  return isNaN(n) ? "" + d : n;
-}
-
-interface FlatAttributes {
-  object?: boolean;
-  objectTimestamp?: number;
-  writable?: boolean;
-  writableTimestamp?: number;
-  value?: [string | number | boolean, string];
-  valueTimestamp?: number;
-  notification?: number;
-  notificationTimestamp?: number;
-  accessList?: string[];
-  accessListTimestamp?: number;
-}
-
-interface FlatDevice {
-  [param: string]: FlatAttributes;
-}
-
-export function flattenDevice(device: Record<string, unknown>): FlatDevice {
-  function recursive(
-    input,
-    root: string,
-    output: FlatDevice,
-    timestamp: number
-  ): void {
-    for (const [name, tree] of Object.entries(input)) {
-      if (!root) {
-        if (name === "_lastInform") {
-          output["Events.Inform"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_registered") {
-          output["Events.Registered"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_lastBoot") {
-          output["Events.1_BOOT"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_lastBootstrap") {
-          output["Events.0_BOOTSTRAP"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_id") {
-          output["DeviceID.ID"] = {
-            value: [tree as string, "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_deviceId") {
-          output["DeviceID.Manufacturer"] = {
-            value: [tree["_Manufacturer"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-          output["DeviceID.OUI"] = {
-            value: [tree["_OUI"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-          output["DeviceID.ProductClass"] = {
-            value: [tree["_ProductClass"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-          output["DeviceID.SerialNumber"] = {
-            value: [tree["_SerialNumber"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_tags") {
-          output["Tags"] = {
-            writable: false,
-            writableTimestamp: timestamp,
-            object: true,
-            objectTimestamp: timestamp,
-          };
-
-          for (const t of tree as string[]) {
-            output[`Tags.${encodeTag(t)}`] = {
-              value: [true, "xsd:boolean"],
-              valueTimestamp: timestamp,
-              writable: true,
-              writableTimestamp: timestamp,
-              object: false,
-              objectTimestamp: timestamp,
-            };
-          }
-        }
-      }
-
-      if (name.startsWith("_")) continue;
-
-      let childrenTimestamp = timestamp;
-
-      if (!root) childrenTimestamp = +(input["_timestamp"] || 1);
-      else if (+input["_timestamp"] > timestamp)
-        childrenTimestamp = +input["_timestamp"];
-
-      const attrs: FlatAttributes = {};
-      if (tree["_value"] != null) {
-        attrs.value = [
-          tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
-          tree["_type"],
-        ];
-        attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
-        attrs.object = false;
-        attrs.objectTimestamp = childrenTimestamp;
-      } else if (tree["_object"] != null) {
-        attrs.object = tree["_object"];
-        attrs.objectTimestamp = childrenTimestamp;
-      }
-
-      if (tree["_writable"] != null) {
-        attrs.writable = tree["_writable"];
-        attrs.writableTimestamp = childrenTimestamp;
-      }
-
-      if (tree["_notification"] != null) {
-        attrs.notification = tree["_notification"];
-        attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1;
-      }
-
-      if (tree["_accessList"] != null) {
-        attrs.accessList = tree["_accessList"];
-        attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1;
-      }
-
-      const r = root ? `${root}.${name}` : name;
-      output[r] = attrs;
-
-      if (attrs.object || tree["object"] == null)
-        recursive(tree, r, output, childrenTimestamp);
-    }
-  }
-
-  const newDevice: FlatDevice = {};
-  const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime();
-  recursive(device, "", newDevice, timestamp);
-  return newDevice;
-}
-
-export function flattenFault(fault: unknown): Fault {
-  const f = Object.assign({}, fault) as Fault;
-  if (f.timestamp) f.timestamp = +f.timestamp;
-  if (f["expiry"]) f["expiry"] = +f["expiry"];
-  return f as Fault;
-}
-
-export function flattenTask(task: unknown): Task {
-  const t = Object.assign({}, task) as Task;
-  t._id = "" + t._id;
-  if (t["timestamp"]) t["timestamp"] = +t["timestamp"];
-  if (t.expiry) t.expiry = +t.expiry;
-  return t;
-}
-
-export function convertOldPrecondition(
-  query: Record<string, unknown>
-): Expression {
-  function recursive(_query): Expression {
-    const expressions: Expression[] = [];
-    for (const [k, v] of Object.entries(_query)) {
-      if (k[0] === "$") {
-        if (k === "$and") {
-          const and: Expression = ["AND"];
-          for (const vv of Object.values(v)) and.push(recursive(vv));
-          expressions.push(and);
-        } else if (k === "$or") {
-          const or: Expression = ["OR"];
-          for (const vv of Object.values(v)) or.push(recursive(vv));
-          expressions.push(or);
-        } else {
-          throw new Error(`Operator ${k} not supported`);
-        }
-      } else if (k === "_tags") {
-        if (typeof v === "object") {
-          if (Array.isArray(v)) throw new Error(`Invalid type`);
-          const conjs: Expression[] = [];
-          for (const [op, val] of Object.entries(v)) {
-            if (op === "$ne") {
-              if (typeof v["$ne"] !== "string")
-                throw new Error("Only string values are allowed for _tags");
-              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
-            } else if (op === "$eq") {
-              if (typeof v["$eq"] !== "string")
-                throw new Error("Only string values are allowed for _tags");
-              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
-            } else {
-              throw new Error(`Invalid tag query`);
-            }
-          }
-          if (conjs.length === 1) expressions.push(conjs[0]);
-          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
-        } else {
-          expressions.push([
-            "IS NOT NULL",
-            ["PARAM", `Tags.${encodeTag(v as string)}`],
-          ]);
-        }
-      } else if (k.startsWith("Tags.")) {
-        let exists: boolean;
-        if (typeof v === "boolean") exists = v;
-        else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"];
-        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
-        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
-        else throw new Error(`Invalid tag query`);
-
-        expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
-      } else if (typeof v === "object") {
-        if (Array.isArray(v)) throw new Error(`Invalid type`);
-
-        const exps: Expression[] = [];
-        for (const [kk, vv] of Object.entries(v)) {
-          if (kk === "$eq") {
-            exps.push(["=", ["PARAM", k], vv]);
-          } else if (kk === "$ne") {
-            exps.push([
-              "OR",
-              ["<>", ["PARAM", k], vv],
-              ["IS NULL", ["PARAM", k]],
-            ]);
-          } else if (kk === "$lt") {
-            exps.push(["<", ["PARAM", k], vv]);
-          } else if (kk === "$lte") {
-            exps.push(["<=", ["PARAM", k], vv]);
-          } else if (kk === "$gt") {
-            exps.push([">", ["PARAM", k], vv]);
-          } else if (kk === "$gte") {
-            exps.push([">=", ["PARAM", k], vv]);
-          } else {
-            throw new Error(`Operator ${kk} not supported`);
-          }
-          if (!["string", "number", "boolean"].includes(typeof vv))
-            throw new Error(`Invalid value for ${kk} operator`);
-        }
-        if (exps.length === 1) {
-          expressions.push(exps[0]);
-        } else if (exps.length > 1) {
-          const and: Expression = ["AND"];
-          expressions.push(and.concat(exps));
-        }
-      } else {
-        expressions.push(["=", ["PARAM", k], v]);
-      }
-    }
-    if (expressions.length === 1) return expressions[0];
-    if (expressions.length === 0) return true;
-    return ["AND", ...expressions];
-  }
-
-  // empty filter
-  if (!Object.keys(query).length) return true;
-
-  return recursive(query);
-}
-
-export function flattenPreset(
-  preset: Record<string, unknown>
-): Record<string, unknown> {
-  const p = Object.assign({}, preset);
-  if (p.precondition) {
-    try {
-      // Try parse to check expression validity
-      parse(p.precondition as string);
-    } catch (error) {
-      p.precondition = convertOldPrecondition(
-        JSON.parse(p.precondition as string)
-      );
-      p.precondition = (p.precondition as string).length
-        ? stringify(p.precondition as Expression)
-        : "";
-    }
-  }
-
-  if (p.events) {
-    const e = [];
-    for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`);
-    p.events = e.join(", ");
-  }
-
-  const provision = p.configurations[0];
-  if (
-    (p.configurations as any[]).length === 1 &&
-    provision.type === "provision" &&
-    provision.name &&
-    provision.name.length
-  ) {
-    p.provision = provision.name;
-    p.provisionArgs = provision.args
-      ? provision.args.map((a) => stringify(a)).join(", ")
-      : "";
-  }
-
-  delete p.configurations;
-  return p;
-}
-
-export function flattenFile(
-  file: Record<string, unknown>
-): Record<string, unknown> {
-  const f = {};
-  f["_id"] = file["_id"];
-  if (file.metadata) {
-    f["metadata.fileType"] = file["metadata"]["fileType"] || "";
-    f["metadata.oui"] = file["metadata"]["oui"] || "";
-    f["metadata.productClass"] = file["metadata"]["productClass"] || "";
-    f["metadata.version"] = file["metadata"]["version"] || "";
-  }
-  return f;
-}
-
-export function preProcessPreset(
-  data: Record<string, unknown>
-): Record<string, unknown> {
-  const preset = Object.assign({}, data);
-
-  if (!preset.precondition) preset.precondition = "";
-  // Try parse to check expression validity
-  parse(preset.precondition as string);
-
-  preset.weight = parseInt(preset.weight as string) || 0;
-
-  const events = {};
-  if (preset.events) {
-    for (let e of (preset.events as string).split(",")) {
-      let v = true;
-      e = e.trim();
-      if (e.startsWith("-")) {
-        v = false;
-        e = e.slice(1).trim();
-      }
-      if (e) events[e] = v;
-    }
-  }
-
-  preset.events = events;
-
-  if (!preset.provision) throw new Error("Invalid preset provision");
-
-  const configuration = {
-    type: "provision",
-    name: preset.provision,
-    args: null,
-  };
-
-  if (preset.provisionArgs)
-    configuration.args = parseList(preset.provisionArgs as string);
-
-  delete preset.provision;
-  delete preset.provisionArgs;
-  preset.configurations = [configuration];
-  return preset;
-}
 
 type Minterm = number[];
 

lib/db/types.ts

@@ -1,5 +1,5 @@
 import { ObjectId } from "mongodb";
-import { Expression, FaultStruct } from "./types";
+import { Expression, FaultStruct } from "../types";
 
 export interface Fault {
   _id: string;
@@ -115,13 +115,21 @@ export interface Device {
   _timestamp?: Date;
 }
 
+type Configuration =
+  | { type: "age"; name: string; age: number }
+  | { type: "value"; name: string; value: boolean | number | string }
+  | { type: "add_tag"; tag: string }
+  | { type: "delete_tag"; tag: string }
+  | { type: "add_object"; name: string; object: string }
+  | { type: "delete_object"; name: string; object: string }
+  | { type: "provision"; name: string; args?: Expression[] };
+
 export interface Preset {
   _id: string;
   weight: number;
-  channels: string;
+  channel: string;
-  provisions: string;
   events: Record<string, boolean>;
-  configurations: { type: string; name: string; args: Expression[] }[];
+  configurations: Configuration[];
 }
 
 export interface Object {
@@ -155,7 +163,8 @@ export interface Permission {
   role: string;
   resource: string;
   access: 1 | 2 | 3;
-  validate: string;
+  filter?: string;
+  validate?: string;
 }
 
 export interface User {

lib/db/util.ts

@@ -0,0 +1,123 @@
+import { Expression } from "../types";
+import { encodeTag } from "../util";
+
+// Optimize projection by removing overlaps
+// This can modify the object
+export function optimizeProjection(obj: { [path: string]: 1 }): {
+  [path: string]: 1;
+} {
+  if (obj[""]) return { "": obj[""] };
+
+  const keys = Object.keys(obj).sort();
+  if (keys.length <= 1) return obj;
+
+  for (let i = 1; i < keys.length; ++i) {
+    const a = keys[i - 1];
+    const b = keys[i];
+    if (b.startsWith(a)) {
+      if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") {
+        delete obj[b];
+        keys.splice(i--, 1);
+      }
+    }
+  }
+  return obj;
+}
+
+export function convertOldPrecondition(q: Record<string, unknown>): Expression {
+  function recursive(_query): Expression {
+    const expressions: Expression[] = [];
+    for (const [k, v] of Object.entries(_query)) {
+      if (k[0] === "$") {
+        if (k === "$and") {
+          const and: Expression = ["AND"];
+          for (const vv of Object.values(v)) and.push(recursive(vv));
+          expressions.push(and);
+        } else if (k === "$or") {
+          const or: Expression = ["OR"];
+          for (const vv of Object.values(v)) or.push(recursive(vv));
+          expressions.push(or);
+        } else {
+          throw new Error(`Operator ${k} not supported`);
+        }
+      } else if (k === "_tags") {
+        if (typeof v === "object") {
+          if (Array.isArray(v)) throw new Error(`Invalid type`);
+          const conjs: Expression[] = [];
+          for (const [op, val] of Object.entries(v)) {
+            if (op === "$ne") {
+              if (typeof v["$ne"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else if (op === "$eq") {
+              if (typeof v["$eq"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else {
+              throw new Error(`Invalid tag query`);
+            }
+          }
+          if (conjs.length === 1) expressions.push(conjs[0]);
+          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
+        } else {
+          expressions.push([
+            "IS NOT NULL",
+            ["PARAM", `Tags.${encodeTag(v as string)}`],
+          ]);
+        }
+      } else if (k.startsWith("Tags.")) {
+        let exists: boolean;
+        if (typeof v === "boolean") exists = v;
+        else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"];
+        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
+        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
+        else throw new Error(`Invalid tag query`);
+
+        expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
+      } else if (typeof v === "object") {
+        if (Array.isArray(v)) throw new Error(`Invalid type`);
+
+        const exps: Expression[] = [];
+        for (const [kk, vv] of Object.entries(v)) {
+          if (kk === "$eq") {
+            exps.push(["=", ["PARAM", k], vv]);
+          } else if (kk === "$ne") {
+            exps.push([
+              "OR",
+              ["<>", ["PARAM", k], vv],
+              ["IS NULL", ["PARAM", k]],
+            ]);
+          } else if (kk === "$lt") {
+            exps.push(["<", ["PARAM", k], vv]);
+          } else if (kk === "$lte") {
+            exps.push(["<=", ["PARAM", k], vv]);
+          } else if (kk === "$gt") {
+            exps.push([">", ["PARAM", k], vv]);
+          } else if (kk === "$gte") {
+            exps.push([">=", ["PARAM", k], vv]);
+          } else {
+            throw new Error(`Operator ${kk} not supported`);
+          }
+          if (!["string", "number", "boolean"].includes(typeof vv))
+            throw new Error(`Invalid value for ${kk} operator`);
+        }
+        if (exps.length === 1) {
+          expressions.push(exps[0]);
+        } else if (exps.length > 1) {
+          const and: Expression = ["AND"];
+          expressions.push(and.concat(exps));
+        }
+      } else {
+        expressions.push(["=", ["PARAM", k], v]);
+      }
+    }
+    if (expressions.length === 1) return expressions[0];
+    if (expressions.length === 0) return true;
+    return ["AND", ...expressions];
+  }
+
+  // empty filter
+  if (!Object.keys(q).length) return true;
+
+  return recursive(q);
+}

lib/fs.ts

@@ -20,18 +20,11 @@
 import * as url from "url";
 import { IncomingMessage, ServerResponse } from "http";
 import { PassThrough, pipeline, Readable } from "stream";
-import { GridFSBucket } from "mongodb";
+import { filesBucket, collections } from "./db/db";
-import { onConnect, collections } from "./db";
 import * as logger from "./logger";
 import { getRequestOrigin } from "./forwarded";
 import memoize from "./common/memoize";
 
-let filesBucket: GridFSBucket;
-
-onConnect(async (db) => {
-  filesBucket = new GridFSBucket(db);
-});
-
 const getFile = memoize(
   async (
     uploadDate: number,

lib/init.ts

@@ -17,9 +17,16 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import * as localCache from "./local-cache";
+import { getRevision, getUiConfig, getUsers } from "./ui/local-cache";
 import { generateSalt, hashPassword } from "./auth";
-import * as db from "./ui/db";
+import { collections } from "./db/db";
+import {
+  putConfig,
+  putPermission,
+  putPreset,
+  putProvision,
+  putUser,
+} from "./ui/db";
 import { del } from "./cache";
 
 interface Status {
@@ -84,14 +91,16 @@ declare("Device.ManagementServer.PeriodicInformTime", {value: daily}, {value: in
 `.trim();
 
 export async function getStatus(): Promise<Status> {
-  const configSnapshot = await localCache.getCurrentSnapshot();
+  const [configSnapshot, presetCount] = await Promise.all([
-  const users = localCache.getUsers(configSnapshot);
+    getRevision(),
-  const presets = localCache.getPresets(configSnapshot);
+    collections.presets.countDocuments(),
-  const ui = localCache.getUiConfig(configSnapshot);
+  ]);
+  const users = getUsers(configSnapshot);
+  const ui = getUiConfig(configSnapshot);
 
   return {
     users: !Object.keys(users).length,
-    presets: !presets.length,
+    presets: !presetCount,
     filters: !Object.keys(ui["filters"]).length,
     device: !Object.keys(ui["device"]).length,
     index: !Object.keys(ui["index"]).length,
@@ -395,7 +404,7 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
   if (resources["permissions"]) {
     for (const p of resources["permissions"]) {
       p["_id"] = `${p["role"]}:${p["resource"]}:${p["access"]}`;
-      proms.push(db.putPermission(p["_id"], p));
+      proms.push(putPermission(p["_id"], p));
     }
   }
 
@@ -406,21 +415,21 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
       u["roles"] = (u["roles"] || []).join(",");
       u["_id"] = u["username"];
       delete u["username"];
-      proms.push(db.putUser(u["_id"], u));
+      proms.push(putUser(u["_id"], u));
     }
   }
 
   if (resources["provisions"]) {
     for (const p of resources["provisions"])
-      proms.push(db.putProvision(p["_id"], p));
+      proms.push(putProvision(p["_id"], p));
   }
 
   if (resources["presets"])
-    for (const p of resources["presets"]) proms.push(db.putPreset(p["_id"], p));
+    for (const p of resources["presets"]) proms.push(putPreset(p["_id"], p));
 
   if (resources["config"])
-    for (const c of resources["config"]) proms.push(db.putConfig(c["_id"], c));
+    for (const c of resources["config"]) proms.push(putConfig(c["_id"], c));
 
   await proms;
-  return del("presets_hash");
+  await Promise.all([del("ui-local-cache-hash"), del("cwmp-local-cache-hash")]);
 }

lib/local-cache.ts

@@ -17,534 +17,75 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import * as vm from "vm";
+import { setTimeoutPromise } from "./util";
-import * as crypto from "crypto";
+import { acquireLock, get, releaseLock, set } from "./cache";
-import * as config from "./config";
-import * as db from "./db";
-import * as cache from "./cache";
-import { convertOldPrecondition } from "./mongodb-functions";
-import * as logger from "./logger";
-import * as scheduling from "./scheduling";
-import * as expression from "./common/expression/util";
-import { parse } from "./common/expression/parser";
-import {
-  Preset,
-  Expression,
-  Provisions,
-  VirtualParameters,
-  Files,
-  Users,
-  Permissions,
-  Config,
-  UiConfig,
-} from "./types";
-
-interface Snapshot {
-  presets: Preset[];
-  provisions: Provisions;
-  virtualParameters: VirtualParameters;
-  files: Files;
-  permissions: Permissions;
-  users: Users;
-  config: Config;
-  ui: UiConfig;
-}
 
 const REFRESH = 5000;
 const EVICT_TIMEOUT = 120000;
 
-const snapshots = new Map<string, Snapshot>();
+export class LocalCache<T> {
-let currentSnapshot: string = null;
+  private nextRefresh = 1;
-let nextRefresh = 1;
+  private currentRevision: string = null;
+  private snapshots: Map<string, T> = new Map();
 
-function computeHash(snapshot): string {
+  constructor(
-  // MD5 hash for presets, provisions, virtual parameters for detecting changes
+    private cacheKey: string,
-  const h = crypto.createHash("md5");
+    private callback: () => Promise<[string, T]>
-  for (const p of snapshot.presets) {
+  ) {}
-    h.update(JSON.stringify(p.name));
+
-    h.update(JSON.stringify(p.channel));
+  async getRevision(): Promise<string> {
-    h.update(JSON.stringify(p.schedule));
+    if (Date.now() > this.nextRefresh) await this.refresh();
-    h.update(JSON.stringify(p.events));
+    return this.currentRevision;
-    h.update(JSON.stringify(p.precondition));
-    h.update(JSON.stringify(p.provisions));
   }
 
-  let keys;
+  hasRevision(revision: string): boolean {
-
+    return this.snapshots.has(revision);
-  keys = Object.keys(snapshot.provisions).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(snapshot.provisions[k].md5);
-
-  keys = Object.keys(snapshot.virtualParameters).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(snapshot.virtualParameters[k].md5);
-
-  keys = Object.keys(snapshot.config).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.config[k]));
-
-  keys = Object.keys(snapshot.files).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.files[k]));
-
-  keys = Object.keys(snapshot.users).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.users[k]));
-
-  const roles = Object.keys(snapshot.permissions).sort();
-  h.update(JSON.stringify(roles));
-  for (const r of roles) {
-    const levels = Object.keys(snapshot.permissions[r]).sort();
-    h.update(JSON.stringify(levels));
-    for (const l of levels) {
-      keys = Object.keys(snapshot.permissions[r][l]).sort();
-      h.update(JSON.stringify(keys));
-      for (const k of keys)
-        h.update(JSON.stringify(snapshot.permissions[r][l][k]));
-    }
   }
 
-  return h.digest("hex");
+  get(revision: string): T {
+    const snapshot = this.snapshots.get(revision);
+    if (!snapshot) throw new Error("Cache snapshot does not exist");
+    return snapshot;
   }
 
-function flattenObject(
+  async refresh(): Promise<void> {
-  src: Record<string, unknown>,
+    if (!this.nextRefresh) {
-  prefix = "",
+      await setTimeoutPromise(20);
-  dst = {}
+      await this.refresh();
-): Record<string, unknown> {
-  for (const k of Object.keys(src)) {
-    const v = src[k];
-    if (typeof v === "object" && !Array.isArray(v))
-      flattenObject(v as Record<string, unknown>, `${prefix}${k}.`, dst);
-    else dst[`${prefix}${k}`] = v;
-  }
-  return dst;
-}
-
-async function fetchPresets(): Promise<Preset[]> {
-  const res = await db.getPresets();
-  let objects = await db.getObjects();
-
-  objects = objects.map((obj) => {
-    // Flatten object
-    obj = flattenObject(obj);
-
-    // If no keys are defined, consider all parameters as keys to keep the
-    // same behavior from v1.0
-    if (!obj["_keys"]?.length)
-      obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
-
-    return obj;
-  });
-
-  res.sort((a, b) => {
-    if (a["weight"] === b["weight"])
-      return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
-    else return a["weight"] - b["weight"];
-  });
-
-  const presets = [] as Preset[];
-  for (const preset of res) {
-    let schedule: { md5: string; duration: number; schedule: any } = null;
-    if (preset["schedule"]) {
-      const parts = preset["schedule"].trim().split(/\s+/);
-      schedule = {
-        md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
-        duration: null,
-        schedule: null,
-      };
-
-      try {
-        schedule.duration = +parts.shift() * 1000;
-        schedule.schedule = scheduling.parseCron(parts.join(" "));
-      } catch (err) {
-        logger.warn({
-          message: "Invalid preset schedule",
-          preset: preset["_id"],
-          schedule: preset["schedule"],
-        });
-        schedule.schedule = false;
-      }
-    }
-
-    const events = preset["events"] || {};
-    let precondition = true as Expression;
-    if (preset["precondition"]) {
-      try {
-        precondition = parse(preset["precondition"]);
-      } catch (error) {
-        precondition = convertOldPrecondition(
-          JSON.parse(preset["precondition"])
-        );
-      }
-
-      // Simplify expression
-      precondition = expression.evaluate(precondition);
-    }
-
-    const _provisions = preset["provisions"] || [];
-
-    // Generate provisions from the old configuration format
-    for (const c of preset["configurations"]) {
-      switch (c.type) {
-        case "age":
-          _provisions.push(["refresh", c.name, +c.age]);
-          break;
-
-        case "value":
-          _provisions.push(["value", c.name, c.value]);
-          break;
-
-        case "add_tag":
-          _provisions.push(["tag", c.tag, true]);
-          break;
-
-        case "delete_tag":
-          _provisions.push(["tag", c.tag, false]);
-          break;
-
-        case "provision":
-          _provisions.push([c.name].concat(c.args || []));
-          break;
-
-        case "add_object":
-          for (const obj of objects) {
-            if (obj["_id"] === c.object) {
-              const alias = obj["_keys"]
-                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
-                .join(",");
-              const p = `${c.name}.[${alias}]`;
-              _provisions.push(["instances", p, 1]);
-
-              for (const k in obj) {
-                if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
-                  _provisions.push(["value", `${p}.${k}`, obj[k]]);
-              }
-            }
-          }
-
-          break;
-
-        case "delete_object":
-          for (const obj of objects) {
-            if (obj["_id"] === c.object) {
-              const alias = obj["_keys"]
-                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
-                .join(",");
-              const p = `${c.name}.[${alias}]`;
-              _provisions.push(["instances", p, 0]);
-            }
-          }
-
-          break;
-
-        default:
-          throw new Error(`Unknown configuration type ${c.type}`);
-      }
-    }
-
-    presets.push({
-      name: preset["_id"],
-      channel: (preset["channel"] as string) || "default",
-      schedule: schedule,
-      events: events,
-      precondition: precondition,
-      provisions: _provisions,
-    });
-  }
-
-  return presets;
-}
-
-async function fetchProvisions(): Promise<Provisions> {
-  const res = await db.getProvisions();
-
-  const provisions = {};
-  for (const r of res) {
-    provisions[r["_id"]] = {};
-    provisions[r["_id"]].md5 = crypto
-      .createHash("md5")
-      .update(r["script"])
-      .digest("hex");
-    provisions[r["_id"]].script = new vm.Script(
-      `"use strict";(function(){\n${r["script"]}\n})();`,
-      { filename: r["_id"], lineOffset: -1, timeout: 50 }
-    );
-  }
-
-  return provisions;
-}
-
-async function fetchVirtualParameters(): Promise<VirtualParameters> {
-  const res = await db.getVirtualParameters();
-
-  const virtualParameters = {};
-  for (const r of res) {
-    virtualParameters[r["_id"]] = {};
-    virtualParameters[r["_id"]].md5 = crypto
-      .createHash("md5")
-      .update(r["script"])
-      .digest("hex");
-    virtualParameters[r["_id"]].script = new vm.Script(
-      `"use strict";(function(){\n${r["script"]}\n})();`,
-      { filename: r["_id"], lineOffset: -1, timeout: 50 }
-    );
-  }
-
-  return virtualParameters;
-}
-
-async function fetchPermissions(): Promise<Permissions> {
-  const perms = await db.getPermissions();
-  const permissions: Permissions = {};
-
-  for (const p of perms) {
-    if (!permissions[p.role]) permissions[p.role] = {};
-    if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {};
-
-    permissions[p.role][p.access][p.resource] = {
-      access: p.access,
-      filter: parse(p.filter || "true"),
-    };
-    if (p.validate)
-      permissions[p.role][p.access][p.resource].validate = parse(p.validate);
-  }
-
-  return permissions;
-}
-
-async function fetchFiles(): Promise<Files> {
-  const res = await db.getFiles();
-  const files = {};
-
-  for (const r of res) {
-    const id = r["filename"] || r["_id"].toString();
-    files[id] = {};
-    files[id].length = r["length"];
-  }
-
-  return files;
-}
-
-async function fetchUsers(): Promise<Users> {
-  const _users = await db.getUsers();
-  const users = {};
-
-  for (const user of _users) {
-    users[user._id] = {
-      password: user.password,
-      salt: user.salt,
-      roles: user.roles.split(",").map((s) => s.trim()),
-    };
-  }
-
-  return users;
-}
-
-async function fetchConfig(): Promise<[Config, UiConfig]> {
-  const conf = await db.getConfig();
-
-  conf.sort((a, b) => (a.id > b.id ? 1 : a.id < b.id ? -1 : 0));
-
-  const ui = {
-    filters: {},
-    device: {},
-    index: {},
-    overview: {},
-    pageSize: null,
-  };
-  const _config = {};
-
-  for (const c of conf) {
-    // Evaluate expressions to simplify them
-    const val = expression.evaluate(c.value);
-    _config[c.id] = val;
-    if (c.id.startsWith("ui.")) {
-      const keys = c.id.split(".");
-      if (!(keys[1] in ui)) continue;
-      // remove the first key(ui)
-      keys.shift();
-      let ref = ui;
-      while (keys.length > 1) {
-        const k = keys.shift();
-        if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {};
-        ref = ref[k];
-      }
-      ref[keys[0]] = val;
-    }
-  }
-
-  return [_config, ui];
-}
-
-async function refresh(): Promise<void> {
-  if (!nextRefresh) {
-    await new Promise((resolve) => setTimeout(resolve, 20));
-    return refresh();
-  }
-
-  const now = Date.now();
-
-  if (now < nextRefresh) return;
-
-  nextRefresh = 0;
-
-  const dbHash = await cache.get("presets_hash");
-
-  if (currentSnapshot && dbHash === currentSnapshot) {
-    nextRefresh = now + (REFRESH - (now % REFRESH));
       return;
     }
 
-  const lockToken = await cache.acquireLock("presets_hash_lock", 5000);
+    const now = Date.now();
+    if (now < this.nextRefresh) return;
+    this.nextRefresh = 0;
 
-  const res = await Promise.all([
+    const dbHash = await get(this.cacheKey);
-    fetchPresets(),
-    fetchProvisions(),
-    fetchVirtualParameters(),
-    fetchFiles(),
-    fetchPermissions(),
-    fetchUsers(),
-    fetchConfig(),
-  ]);
 
-  const snapshot = {
+    if (this.currentRevision && dbHash === this.currentRevision) {
-    presets: res[0],
+      this.nextRefresh = now + (REFRESH - (now % REFRESH));
-    provisions: res[1],
+      return;
-    virtualParameters: res[2],
+    }
-    files: res[3],
-    permissions: res[4],
-    users: res[5],
-    config: res[6][0],
-    ui: res[6][1],
-  };
 
-  if (currentSnapshot) {
+    const lockToken = await acquireLock(this.cacheKey + "_lock", 5000);
-    const h = currentSnapshot;
+
-    const s = snapshots.get(h);
+    const [hash, snapshot] = await this.callback();
+
+    if (this.currentRevision) {
+      const r = this.currentRevision;
+      const s = this.snapshots.get(r);
       setTimeout(() => {
-      if (snapshots.get(h) === s) snapshots.delete(h);
+        if (this.snapshots.get(r) === s) this.snapshots.delete(r);
       }, EVICT_TIMEOUT).unref();
     }
 
-  currentSnapshot = computeHash(snapshot);
+    this.currentRevision = hash;
-  snapshots.set(currentSnapshot, snapshot);
+    this.snapshots.set(hash, snapshot);
 
     if (lockToken) {
-    if (currentSnapshot !== dbHash)
+      if (hash !== dbHash) await set(this.cacheKey, hash, 300);
-      await cache.set("presets_hash", currentSnapshot, 300);
+      await releaseLock(this.cacheKey + "_lock", lockToken);
-    await cache.releaseLock("presets_hash_lock", lockToken);
     }
 
-  nextRefresh = now + (REFRESH - (now % REFRESH));
+    this.nextRefresh = now + (REFRESH - (now % REFRESH));
   }
-
-export async function getCurrentSnapshot(): Promise<string> {
-  if (Date.now() > nextRefresh) await refresh();
-  return currentSnapshot;
-}
-
-export function hasSnapshot(hash: string): boolean {
-  return snapshots.has(hash);
-}
-
-export function getPresets(snapshotKey: string): Preset[] {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.presets;
-}
-
-export function getProvisions(snapshotKey: string): Provisions {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.provisions;
-}
-
-export function getVirtualParameters(snapshotKey: string): VirtualParameters {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.virtualParameters;
-}
-
-export function getFiles(snapshotKey: string): Files {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.files;
-}
-
-export function getConfig(
-  snapshotKey: string,
-  key: string,
-  context: Record<string, unknown>,
-  now: number,
-  cb?: (e: Expression) => Expression
-): string | number | boolean | null {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  const oldOpts = {
-    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
-    "cwmp.debug": "DEBUG",
-    "cwmp.retryDelay": "RETRY_DELAY",
-    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
-    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
-    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
-    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
-    "cwmp.cookiesPath": "COOKIES_PATH",
-    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
-    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
-    "cwmp.connectionRequestAllowBasicAuth":
-      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
-    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
-    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
-    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
-  };
-
-  if (!(key in snapshot.config)) {
-    if (key in oldOpts) {
-      let id;
-      if (context?.["id"]) {
-        id = context["id"];
-      } else if (cb) {
-        id = cb(["PARAM", "DeviceID.ID"]);
-        if (Array.isArray(id)) id = null;
-      }
-      return config.get(oldOpts[key], id);
-    }
-    return null;
-  }
-
-  const v = expression.evaluate(snapshot.config[key], context, now, cb);
-  return Array.isArray(v) ? null : v;
-}
-
-export function getConfigExpression(
-  snapshotKey: string,
-  key: string
-): Expression {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.config[key];
-}
-
-export function getUsers(snapshotKey: string): Users {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.users;
-}
-
-export function getPermissions(snapshotKey: string): Permissions {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.permissions;
-}
-
-export function getUiConfig(snapshotKey: string): UiConfig {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.ui;
 }

lib/nbi.ts

@@ -17,10 +17,11 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Collection, GridFSBucket, ObjectId } from "mongodb";
+import { Collection, ObjectId } from "mongodb";
 import * as vm from "vm";
 import * as config from "./config";
-import { onConnect, optimizeProjection, collections } from "./db";
+import { filesBucket, collections } from "./db/db";
+import { optimizeProjection } from "./db/util";
 import * as query from "./query";
 import * as apiFunctions from "./api-functions";
 import { IncomingMessage, ServerResponse } from "http";
@@ -28,7 +29,7 @@ import * as cache from "./cache";
 import { version as VERSION } from "../package.json";
 import { ping } from "./ping";
 import * as logger from "./logger";
-import { flattenDevice } from "./mongodb-functions";
+import { flattenDevice } from "./ui/db";
 import { getRequestOrigin } from "./forwarded";
 
 const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
@@ -46,12 +47,6 @@ const VIRTUAL_PARAMETERS_REGEX =
   /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
 const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
 
-let filesBucket: GridFSBucket;
-
-onConnect(async (db) => {
-  filesBucket = new GridFSBucket(db);
-});
-
 async function getBody(request: IncomingMessage): Promise<Buffer> {
   const chunks: Buffer[] = [];
   let readableEnded = false;
@@ -112,12 +107,12 @@ async function handler(
       await collections.presets.replaceOne({ _id: presetName }, preset, {
         upsert: true,
       });
-      await cache.del("presets_hash");
+      await cache.del("cwmp-local-cache-hash");
       response.writeHead(200);
       response.end();
     } else if (request.method === "DELETE") {
       await collections.presets.deleteOne({ _id: presetName });
-      await cache.del("presets_hash");
+      await cache.del("cwmp-local-cache-hash");
       response.writeHead(200);
       response.end();
     } else {
@@ -139,12 +134,12 @@ async function handler(
       await collections.objects.replaceOne({ _id: objectName }, object, {
         upsert: true,
       });
-      await cache.del("presets_hash");
+      await cache.del("cwmp-local-cache-hash");
       response.writeHead(200);
       response.end();
     } else if (request.method === "DELETE") {
       await collections.objects.deleteOne({ _id: objectName });
-      await cache.del("presets_hash");
+      await cache.del("cwmp-local-cache-hash");
       response.writeHead(200);
       response.end();
     } else {
@@ -172,12 +167,12 @@ async function handler(
       await collections.provisions.replaceOne({ _id: provisionName }, object, {
         upsert: true,
       });
-      await cache.del("presets_hash");
+      await cache.del("cwmp-local-cache-hash");
       response.writeHead(200);
       response.end();
     } else if (request.method === "DELETE") {
       await collections.provisions.deleteOne({ _id: provisionName });
-      await cache.del("presets_hash");
+      await cache.del("cwmp-local-cache-hash");
       response.writeHead(200);
       response.end();
     } else {
@@ -207,14 +202,14 @@ async function handler(
         object,
         { upsert: true }
       );
-      await cache.del("presets_hash");
+      await cache.del("cwmp-local-cache-hash");
       response.writeHead(200);
       response.end();
     } else if (request.method === "DELETE") {
       await collections.virtualParameters.deleteOne({
         _id: virtualParameterName,
       });
-      await cache.del("presets_hash");
+      await cache.del("cwmp-local-cache-hash");
       response.writeHead(200);
       response.end();
     } else {

lib/session.ts

@@ -19,7 +19,7 @@
 
 import * as device from "./device";
 import * as sandbox from "./sandbox";
-import * as localCache from "./local-cache";
+import * as localCache from "./cwmp/local-cache";
 import * as defaultProvisions from "./default-provisions";
 import { estimateGpnCount } from "./gpn-heuristic";
 import Path from "./common/path";
@@ -3004,7 +3004,7 @@ export async function deserialize(
 
   sessionContext.deviceData = deviceData;
   // Ensure cache is populated
-  await localCache.getCurrentSnapshot();
+  await localCache.getRevision();
 
   return sessionContext;
 }

lib/ui.ts

@@ -29,9 +29,9 @@ import * as config from "./config";
 import api from "./ui/api";
 import Authorizer from "./common/authorizer";
 import * as logger from "./logger";
-import * as localCache from "./local-cache";
+import * as localCache from "./ui/local-cache";
 import { PermissionSet } from "./types";
-import { authLocal } from "./ui/api-functions";
+import { authLocal } from "./api-functions";
 import * as init from "./init";
 import { version as VERSION } from "../package.json";
 import memoize from "./common/memoize";
@@ -68,7 +68,7 @@ koa.on("error", (err, ctx) => {
 });
 
 koa.use(async (ctx, next) => {
-  const configSnapshot = await localCache.getCurrentSnapshot();
+  const configSnapshot = await localCache.getRevision();
   ctx.state.configSnapshot = configSnapshot;
   ctx.set("X-Config-Snapshot", configSnapshot);
   ctx.set("GenieACS-Version", VERSION);

lib/ui/api-functions.ts

@@ -1,211 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { ObjectId } from "mongodb";
-import * as db from "./db";
-import { del } from "../cache";
-import { getUsers } from "../local-cache";
-import { hashPassword } from "../auth";
-import {
-  insertTasks,
-  awaitSessionEnd,
-  awaitSessionStart,
-  connectionRequest,
-  deleteDevice,
-} from "../api-functions";
-import { Task } from "../types";
-
-async function deleteFault(id): Promise<void> {
-  const deviceId = id.split(":", 1)[0];
-  const channel = id.slice(deviceId.length + 1);
-
-  await Promise.all([
-    db.deleteFault(id),
-    channel.startsWith("task_")
-      ? db.deleteTask(new ObjectId(channel.slice(5)))
-      : null,
-  ]);
-
-  await del(`${deviceId}_tasks_faults_operations`);
-}
-
-export async function deleteResource(
-  resource: string,
-  id: string
-): Promise<void> {
-  switch (resource) {
-    case "devices":
-      await deleteDevice(id);
-      break;
-
-    case "files":
-      await db.deleteFile(id);
-      break;
-
-    case "faults":
-      await deleteFault(id);
-      break;
-
-    case "provisions":
-      await db.deleteProvision(id);
-      break;
-
-    case "presets":
-      await db.deletePreset(id);
-      break;
-
-    case "virtualParameters":
-      await db.deleteVirtualParameter(id);
-      break;
-
-    case "config":
-      await db.deleteConfig(id);
-      break;
-
-    case "permissions":
-      await db.deletePermission(id);
-      break;
-
-    case "users":
-      await db.deleteUser(id);
-      break;
-  }
-
-  await del("presets_hash");
-}
-
-export async function postTasks(
-  deviceId: string,
-  tasks: Task[],
-  timeout: number,
-  device: Record<string, { value?: [boolean | number | string, string] }>
-): Promise<{ connectionRequest: string; tasks: any[] }> {
-  for (const task of tasks) {
-    delete task._id;
-    task["device"] = deviceId;
-  }
-
-  tasks = await insertTasks(tasks);
-  const statuses = tasks.map((t) => {
-    return { _id: t._id, status: "pending" };
-  });
-  const lastInform = Date.now();
-  const notInSession = await awaitSessionEnd(deviceId, 30000);
-  if (!notInSession) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
-
-  await del(`${deviceId}_tasks_faults_operations`);
-
-  const status = await connectionRequest(deviceId, device);
-
-  if (status) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: status,
-      tasks: statuses,
-    };
-  }
-
-  const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
-  if (!sessionStarted) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: "No contact from CPE",
-      tasks: statuses,
-    };
-  }
-
-  const sessionEnded = await awaitSessionEnd(deviceId, 120000);
-  if (!sessionEnded) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
-
-  const promises: Promise<number>[] = [];
-  for (const s of statuses) {
-    promises.push(db.count("tasks", ["=", ["PARAM", "_id"], s._id]));
-    promises.push(
-      db.count("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
-    );
-  }
-
-  const res = await Promise.all(promises);
-  for (const [i, r] of statuses.entries()) {
-    if (!res[i * 2]) {
-      r.status = "done";
-    } else if (res[i * 2 + 1]) {
-      r.status = "fault";
-      r["fault"] = res[i * 2 + 1][0];
-    }
-  }
-
-  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-
-  return { connectionRequest: "OK", tasks: statuses };
-}
-
-export async function putResource(
-  resource: string,
-  id: string,
-  data: Record<string, unknown>
-): Promise<void> {
-  if (resource === "presets") {
-    await db.putPreset(id, data);
-  } else if (resource === "provisions") {
-    await db.putProvision(id, data);
-  } else if (resource === "virtualParameters") {
-    await db.putVirtualParameter(id, data);
-  } else if (resource === "config") {
-    await db.putConfig(id, data);
-  } else if (resource === "permissions") {
-    await db.putPermission(id, data);
-  } else if (resource === "users") {
-    delete data["password"];
-    delete data["salt"];
-    await db.putUser(id, data);
-  }
-
-  await del("presets_hash");
-}
-
-export function authLocal(
-  snapshot: string,
-  username: string,
-  password: string
-): Promise<boolean> {
-  return new Promise((resolve, reject) => {
-    const users = getUsers(snapshot);
-    const user = users[username];
-    if (!user?.password) return void resolve(null);
-    hashPassword(password, user.salt)
-      .then((hash) => {
-        if (hash === user.password) resolve(true);
-        else resolve(false);
-      })
-      .catch(reject);
-  });
-}

lib/ui/api.ts

@@ -20,17 +20,17 @@
 import { PassThrough } from "stream";
 import Router from "koa-router";
 import * as db from "./db";
-import * as apiFunctions from "./api-functions";
+import * as apiFunctions from "../api-functions";
 import { evaluate, and, extractParams } from "../common/expression/util";
 import { parse } from "../common/expression/parser";
 import * as logger from "../logger";
-import { getConfig } from "../local-cache";
+import { getConfig } from "../ui/local-cache";
-import { QueryOptions, Expression } from "../types";
+import { Expression } from "../types";
 import { generateSalt, hashPassword } from "../auth";
 import { del } from "../cache";
 import Authorizer from "../common/authorizer";
 import { ping } from "../ping";
-import { decodeTag } from "../common";
+import { decodeTag } from "../util";
 import { stringify as yamlStringify } from "../common/yaml";
 
 const router = new Router();
@@ -178,7 +178,7 @@ for (const [resource, flags] of Object.entries(resources)) {
 
   router.get(`/${resource}`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
-    const options: QueryOptions = {};
+    const options: Parameters<typeof db.query>[2] = {};
     let filter: Expression = authorizer.getFilter(resource, 2);
     if (ctx.request.query.filter)
       filter = and(filter, parse(singleParam(ctx.request.query.filter)));
@@ -231,7 +231,7 @@ for (const [resource, flags] of Object.entries(resources)) {
   // CSV download
   router.get(`/${resource}.csv`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
-    const options: QueryOptions = { projection: {} };
+    const options: Parameters<typeof db.query>[2] = { projection: {} };
     let filter: Expression = authorizer.getFilter(resource, 2);
     if (ctx.request.query.filter)
       filter = and(filter, parse(singleParam(ctx.request.query.filter)));
@@ -712,7 +712,7 @@ router.put("/users/:id/password", async (ctx) => {
   const password = await hashPassword(newPassword, salt);
   await db.putUser(username, { password, salt });
 
-  await del("presets_hash");
+  await del("ui-local-cache-hash");
 
   logger.accessInfo(log);
   ctx.body = "";

lib/ui/db.ts

@@ -17,19 +17,362 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Collection, GridFSBucket, ObjectId } from "mongodb";
+import { encodeTag } from "../util";
-import { Script } from "vm";
-import { collections, onConnect, optimizeProjection } from "../db";
-import * as mongodbFunctions from "../mongodb-functions";
 import { evaluate } from "../common/expression/util";
-import { QueryOptions, Expression } from "../types";
+import { Expression, Fault, Task } from "../types";
+import { collections, filesBucket } from "../db/db";
+import { convertOldPrecondition, optimizeProjection } from "../db/util";
+import * as MongoTypes from "../db/types";
+import { parse, parseList, stringify } from "../common/expression/parser";
+import { toMongoQuery } from "../db/synth";
+import { Collection, ObjectId, WithoutId } from "mongodb";
+import { Script } from "vm";
 import { Readable } from "stream";
 
-let filesBucket: GridFSBucket;
+function processDeviceProjection(
+  projection: Record<string, 1>
+): Record<string, 1> {
+  if (!projection) return projection;
+  const p = {};
+  for (const [k, v] of Object.entries(projection)) {
+    if (k === "DeviceID.ID") {
+      p["_id"] = 1;
+    } else if (k.startsWith("DeviceID")) {
+      p["_deviceId._SerialNumber"] = v;
+      p["_deviceId._OUI"] = v;
+      p["_deviceId._ProductClass"] = v;
+      p["_deviceId._Manufacturer"] = v;
+    } else if (k.startsWith("Tags")) {
+      p["_tags"] = v;
+    } else if (k.startsWith("Events")) {
+      p["_lastInform"] = v;
+      p["_registered"] = v;
+      p["_lastBoot"] = v;
+      p["_lastBootstrap"] = v;
+    } else {
+      p[k] = v;
+    }
+  }
 
-onConnect(async (db) => {
+  return p;
-  filesBucket = new GridFSBucket(db);
+}
-});
+
+function processDeviceSort(
+  sort: Record<string, number>
+): Record<string, number> {
+  if (!sort) return sort;
+  const s = {};
+  for (const [k, v] of Object.entries(sort)) {
+    if (k === "DeviceID.ID") s["_id"] = v;
+    else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v;
+    else if (k === "Events.Inform") s["_lastInform"] = v;
+    else if (k === "Events.Registered") s["_registered"] = v;
+    else if (k === "Events.1_BOOT") s["_lastBoot"] = v;
+    else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v;
+    else s[`${k}._value`] = v;
+  }
+
+  return s;
+}
+
+function parseDate(d: Date): number | string {
+  const n = +d;
+  return isNaN(n) ? "" + d : n;
+}
+
+interface FlatAttributes {
+  object?: boolean;
+  objectTimestamp?: number;
+  writable?: boolean;
+  writableTimestamp?: number;
+  value?: [string | number | boolean, string];
+  valueTimestamp?: number;
+  notification?: number;
+  notificationTimestamp?: number;
+  accessList?: string[];
+  accessListTimestamp?: number;
+}
+
+interface FlatDevice {
+  [param: string]: FlatAttributes;
+}
+
+export function flattenDevice(device: Record<string, unknown>): FlatDevice {
+  function recursive(
+    input,
+    root: string,
+    output: FlatDevice,
+    timestamp: number
+  ): void {
+    for (const [name, tree] of Object.entries(input)) {
+      if (!root) {
+        if (name === "_lastInform") {
+          output["Events.Inform"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_registered") {
+          output["Events.Registered"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_lastBoot") {
+          output["Events.1_BOOT"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_lastBootstrap") {
+          output["Events.0_BOOTSTRAP"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_id") {
+          output["DeviceID.ID"] = {
+            value: [tree as string, "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_deviceId") {
+          output["DeviceID.Manufacturer"] = {
+            value: [tree["_Manufacturer"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.OUI"] = {
+            value: [tree["_OUI"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.ProductClass"] = {
+            value: [tree["_ProductClass"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.SerialNumber"] = {
+            value: [tree["_SerialNumber"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_tags") {
+          output["Tags"] = {
+            writable: false,
+            writableTimestamp: timestamp,
+            object: true,
+            objectTimestamp: timestamp,
+          };
+
+          for (const t of tree as string[]) {
+            output[`Tags.${encodeTag(t)}`] = {
+              value: [true, "xsd:boolean"],
+              valueTimestamp: timestamp,
+              writable: true,
+              writableTimestamp: timestamp,
+              object: false,
+              objectTimestamp: timestamp,
+            };
+          }
+        }
+      }
+
+      if (name.startsWith("_")) continue;
+
+      let childrenTimestamp = timestamp;
+
+      if (!root) childrenTimestamp = +(input["_timestamp"] || 1);
+      else if (+input["_timestamp"] > timestamp)
+        childrenTimestamp = +input["_timestamp"];
+
+      const attrs: FlatAttributes = {};
+      if (tree["_value"] != null) {
+        attrs.value = [
+          tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
+          tree["_type"],
+        ];
+        attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
+        attrs.object = false;
+        attrs.objectTimestamp = childrenTimestamp;
+      } else if (tree["_object"] != null) {
+        attrs.object = tree["_object"];
+        attrs.objectTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_writable"] != null) {
+        attrs.writable = tree["_writable"];
+        attrs.writableTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_notification"] != null) {
+        attrs.notification = tree["_notification"];
+        attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      if (tree["_accessList"] != null) {
+        attrs.accessList = tree["_accessList"];
+        attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      const r = root ? `${root}.${name}` : name;
+      output[r] = attrs;
+
+      if (attrs.object || tree["object"] == null)
+        recursive(tree, r, output, childrenTimestamp);
+    }
+  }
+
+  const newDevice: FlatDevice = {};
+  const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime();
+  recursive(device, "", newDevice, timestamp);
+  return newDevice;
+}
+
+function flattenFault(fault: unknown): Fault {
+  const f = Object.assign({}, fault) as Fault;
+  if (f.timestamp) f.timestamp = +f.timestamp;
+  if (f["expiry"]) f["expiry"] = +f["expiry"];
+  return f as Fault;
+}
+
+function flattenTask(task: unknown): Task {
+  const t = Object.assign({}, task) as Task;
+  t._id = "" + t._id;
+  if (t["timestamp"]) t["timestamp"] = +t["timestamp"];
+  if (t.expiry) t.expiry = +t.expiry;
+  return t;
+}
+
+function flattenPreset(
+  preset: Record<string, unknown>
+): Record<string, unknown> {
+  const p = Object.assign({}, preset);
+  if (p.precondition) {
+    try {
+      // Try parse to check expression validity
+      parse(p.precondition as string);
+    } catch (error) {
+      p.precondition = convertOldPrecondition(
+        JSON.parse(p.precondition as string)
+      );
+      p.precondition = (p.precondition as string).length
+        ? stringify(p.precondition as Expression)
+        : "";
+    }
+  }
+
+  if (p.events) {
+    const e = [];
+    for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`);
+    p.events = e.join(", ");
+  }
+
+  const provision = p.configurations[0];
+  if (
+    (p.configurations as any[]).length === 1 &&
+    provision.type === "provision" &&
+    provision.name &&
+    provision.name.length
+  ) {
+    p.provision = provision.name;
+    p.provisionArgs = provision.args
+      ? provision.args.map((a) => stringify(a)).join(", ")
+      : "";
+  }
+
+  delete p.configurations;
+  return p;
+}
+
+function flattenFile(file: Record<string, unknown>): Record<string, unknown> {
+  const f = {};
+  f["_id"] = file["_id"];
+  if (file.metadata) {
+    f["metadata.fileType"] = file["metadata"]["fileType"] || "";
+    f["metadata.oui"] = file["metadata"]["oui"] || "";
+    f["metadata.productClass"] = file["metadata"]["productClass"] || "";
+    f["metadata.version"] = file["metadata"]["version"] || "";
+  }
+  return f;
+}
+
+function preProcessPreset(data: Record<string, unknown>): MongoTypes.Preset {
+  const preset = Object.assign({}, data);
+
+  if (!preset.precondition) preset.precondition = "";
+  // Try parse to check expression validity
+  parse(preset.precondition as string);
+
+  preset.weight = parseInt(preset.weight as string) || 0;
+
+  const events = {};
+  if (preset.events) {
+    for (let e of (preset.events as string).split(",")) {
+      let v = true;
+      e = e.trim();
+      if (e.startsWith("-")) {
+        v = false;
+        e = e.slice(1).trim();
+      }
+      if (e) events[e] = v;
+    }
+  }
+
+  preset.events = events;
+
+  if (!preset.provision) throw new Error("Invalid preset provision");
+
+  const configuration = {
+    type: "provision",
+    name: preset.provision,
+    args: null,
+  };
+
+  if (preset.provisionArgs)
+    configuration.args = parseList(preset.provisionArgs as string);
+
+  delete preset.provision;
+  delete preset.provisionArgs;
+  preset.configurations = [configuration];
+  return preset as unknown as MongoTypes.Preset;
+}
+
+interface QueryOptions {
+  projection?: any;
+  skip?: number;
+  limit?: number;
+  sort?: {
+    [param: string]: number;
+  };
+}
 
 export async function* query(
   resource: string,
@@ -38,7 +381,7 @@ export async function* query(
 ): AsyncGenerator<any, void, undefined> {
   options = options || {};
   filter = evaluate(filter, null, Date.now());
-  const q = mongodbFunctions.toMongoQuery(filter, resource);
+  const q = toMongoQuery(filter, resource);
   if (!q) return;
 
   const collection = collections[resource] as Collection<any>;
@@ -46,7 +389,7 @@ export async function* query(
   if (options.projection) {
     let projection = options.projection;
     if (resource === "devices")
-      projection = mongodbFunctions.processDeviceProjection(options.projection);
+      projection = processDeviceProjection(options.projection);
 
     if (resource === "presets") projection.configurations = 1;
     projection = optimizeProjection(projection);
@@ -67,25 +410,25 @@ export async function* query(
         {}
       );
 
-    if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
+    if (resource === "devices") s = processDeviceSort(s);
     cursor.sort(s);
   }
 
   for await (let doc of cursor) {
-    if (resource === "devices") doc = mongodbFunctions.flattenDevice(doc);
+    if (resource === "devices") doc = flattenDevice(doc);
-    else if (resource === "faults") doc = mongodbFunctions.flattenFault(doc);
+    else if (resource === "faults") doc = flattenFault(doc);
-    else if (resource === "tasks") doc = mongodbFunctions.flattenTask(doc);
+    else if (resource === "tasks") doc = flattenTask(doc);
-    else if (resource === "presets") doc = mongodbFunctions.flattenPreset(doc);
+    else if (resource === "presets") doc = flattenPreset(doc);
-    else if (resource === "files") doc = mongodbFunctions.flattenFile(doc);
+    else if (resource === "files") doc = flattenFile(doc);
 
     yield doc;
   }
 }
 
 export function count(resource: string, filter: Expression): Promise<number> {
-  const collection = collections[resource] as Collection<unknown>;
+  const collection = collections[resource] as Collection<any>;
   filter = evaluate(filter, null, Date.now());
-  const q = mongodbFunctions.toMongoQuery(filter, resource);
+  const q = toMongoQuery(filter, resource);
   if (!q) return Promise.resolve(0);
   return collection.countDocuments(q);
 }
@@ -110,34 +453,21 @@ export async function updateDeviceTags(
   await collections.devices.updateOne({ _id: deviceId }, object);
 }
 
-async function putResource(resource, id, object): Promise<void> {
+export async function putPreset(
-  const collection = collections[resource] as Collection<unknown>;
-  await collection.replaceOne({ _id: id }, object, { upsert: true });
-}
-
-async function deleteResource(
-  resource: string,
-  id: string | ObjectId
-): Promise<void> {
-  const collection = collections[resource] as Collection<unknown>;
-  await collection.deleteOne({ _id: id });
-}
-
-export function putPreset(
   id: string,
   object: Record<string, unknown>
 ): Promise<void> {
-  object = mongodbFunctions.preProcessPreset(object);
+  const p = preProcessPreset(object);
-  return putResource("presets", id, object);
+  await collections.presets.replaceOne({ _id: id }, p, { upsert: true });
 }
 
-export function deletePreset(id: string): Promise<void> {
+export async function deletePreset(id: string): Promise<void> {
-  return deleteResource("presets", id);
+  await collections.presets.deleteOne({ _id: id });
 }
 
-export function putProvision(
+export async function putProvision(
   id: string,
-  object: Record<string, unknown>
+  object: { script: string }
 ): Promise<void> {
   if (!object.script) object.script = "";
   try {
@@ -153,16 +483,18 @@ export function putProvision(
     }
     return Promise.reject(err);
   }
-  return putResource("provisions", id, object);
+  await collections.provisions.replaceOne({ _id: id }, object, {
+    upsert: true,
+  });
 }
 
-export function deleteProvision(id: string): Promise<void> {
+export async function deleteProvision(id: string): Promise<void> {
-  return deleteResource("provisions", id);
+  await collections.provisions.deleteOne({ _id: id });
 }
 
-export function putVirtualParameter(
+export async function putVirtualParameter(
   id: string,
-  object: Record<string, unknown>
+  object: { script: string }
 ): Promise<void> {
   if (!object.script) object.script = "";
   try {
@@ -178,38 +510,42 @@ export function putVirtualParameter(
     }
     return Promise.reject(err);
   }
-  return putResource("virtualParameters", id, object);
+  await collections.virtualParameters.replaceOne({ _id: id }, object, {
+    upsert: true,
+  });
 }
 
-export function deleteVirtualParameter(id: string): Promise<void> {
+export async function deleteVirtualParameter(id: string): Promise<void> {
-  return deleteResource("virtualParameters", id);
+  await collections.virtualParameters.deleteOne({ _id: id });
 }
 
-export function putConfig(
+export async function putConfig(
   id: string,
-  object: Record<string, unknown>
+  object: WithoutId<MongoTypes.Config>
 ): Promise<void> {
-  return putResource("config", id, object);
+  await collections.config.replaceOne({ _id: id }, object, { upsert: true });
 }
 
-export function deleteConfig(id: string): Promise<void> {
+export async function deleteConfig(id: string): Promise<void> {
-  return deleteResource("config", id);
+  await collections.config.deleteOne({ _id: id });
 }
 
-export function putPermission(
+export async function putPermission(
   id: string,
-  object: Record<string, unknown>
+  object: WithoutId<MongoTypes.Permission>
 ): Promise<void> {
-  return putResource("permissions", id, object);
+  await collections.permissions.replaceOne({ _id: id }, object, {
+    upsert: true,
+  });
 }
 
-export function deletePermission(id: string): Promise<void> {
+export async function deletePermission(id: string): Promise<void> {
-  return deleteResource("permissions", id);
+  await collections.permissions.deleteOne({ _id: id });
 }
 
 export async function putUser(
   id: string,
-  object: Record<string, unknown>
+  object: Partial<WithoutId<MongoTypes.User>>
 ): Promise<void> {
   // update instead of replace to keep the password if not set by user
   await collections.users.updateOne(
@@ -219,8 +555,8 @@ export async function putUser(
   );
 }
 
-export function deleteUser(id: string): Promise<void> {
+export async function deleteUser(id: string): Promise<void> {
-  return deleteResource("users", id);
+  await collections.users.deleteOne({ _id: id });
 }
 
 export function downloadFile(filename: string): Readable {
@@ -267,10 +603,10 @@ export async function deleteFile(filename: string): Promise<void> {
   await filesBucket.delete(filename as any);
 }
 
-export function deleteFault(id: string): Promise<void> {
+export async function deleteFault(id: string): Promise<void> {
-  return deleteResource("faults", id);
+  await collections.faults.deleteOne({ _id: id });
 }
 
-export function deleteTask(id: ObjectId): Promise<void> {
+export async function deleteTask(id: ObjectId): Promise<void> {
-  return deleteResource("tasks", id);
+  await collections.tasks.deleteOne({ _id: id });
 }

lib/ui/local-cache.ts

@@ -0,0 +1,187 @@
+import * as crypto from "crypto";
+import * as config from "../config";
+import { collections } from "../db/db";
+import * as expression from "../common/expression/util";
+import { parse } from "../common/expression/parser";
+import { Expression, Users, Permissions, Config, UiConfig } from "../types";
+import { LocalCache } from "../local-cache";
+
+interface Snapshot {
+  permissions: Permissions;
+  users: Users;
+  config: Config;
+  ui: UiConfig;
+}
+
+async function fetchPermissions(): Promise<[string, Permissions]> {
+  const perms = await collections.permissions.find().toArray();
+  perms.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto
+    .createHash("md5")
+    .update(JSON.stringify(perms))
+    .digest("hex");
+  const permissions: Permissions = {};
+
+  for (const p of perms) {
+    if (!permissions[p.role]) permissions[p.role] = {};
+    if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {};
+
+    permissions[p.role][p.access][p.resource] = {
+      access: p.access,
+      filter: parse(p.filter || "true"),
+    };
+    if (p.validate)
+      permissions[p.role][p.access][p.resource].validate = parse(p.validate);
+  }
+
+  return [h, permissions];
+}
+
+async function fetchUsers(): Promise<[string, Users]> {
+  const _users = await collections.users.find().toArray();
+  _users.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto
+    .createHash("md5")
+    .update(JSON.stringify(_users))
+    .digest("hex");
+  const users = {};
+
+  for (const user of _users) {
+    users[user._id] = {
+      password: user.password,
+      salt: user.salt,
+      roles: user.roles.split(",").map((s) => s.trim()),
+    };
+  }
+
+  return [h, users];
+}
+
+async function fetchConfig(): Promise<[string, Config, UiConfig]> {
+  const conf = await collections.config.find().toArray();
+  conf.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(conf)).digest("hex");
+
+  const ui = {
+    filters: {},
+    device: {},
+    index: {},
+    overview: {},
+    pageSize: null,
+  };
+
+  const _config = {};
+
+  for (const c of conf) {
+    // Evaluate expressions to simplify them
+    const val = expression.evaluate(parse(c.value));
+    _config[c._id] = val;
+    if (c._id.startsWith("ui.")) {
+      const keys = c._id.split(".");
+      if (!(keys[1] in ui)) continue;
+      // remove the first key(ui)
+      keys.shift();
+      let ref = ui;
+      while (keys.length > 1) {
+        const k = keys.shift();
+        if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {};
+        ref = ref[k];
+      }
+      ref[keys[0]] = val;
+    }
+  }
+
+  return [h, _config, ui];
+}
+
+const localCache = new LocalCache<Snapshot>("ui-local-cache-hash", refresh);
+
+async function refresh(): Promise<[string, Snapshot]> {
+  const res = await Promise.all([
+    fetchPermissions(),
+    fetchUsers(),
+    fetchConfig(),
+  ]);
+
+  const h = crypto.createHash("md5");
+  for (const r of res) h.update(r[0]);
+
+  const snapshot = {
+    permissions: res[0][1],
+    users: res[1][1],
+    config: res[2][1],
+    ui: res[2][2],
+  };
+
+  return [h.digest("hex"), snapshot];
+}
+
+export async function getRevision(): Promise<string> {
+  return await localCache.getRevision();
+}
+
+export function getConfig(
+  revision: string,
+  key: string,
+  context: Record<string, unknown>,
+  now: number,
+  cb?: (e: Expression) => Expression
+): string | number | boolean | null {
+  const snapshot = localCache.get(revision);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  const oldOpts = {
+    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
+    "cwmp.debug": "DEBUG",
+    "cwmp.retryDelay": "RETRY_DELAY",
+    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
+    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
+    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
+    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
+    "cwmp.cookiesPath": "COOKIES_PATH",
+    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
+    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
+    "cwmp.connectionRequestAllowBasicAuth":
+      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
+    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
+    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
+  };
+
+  if (!(key in snapshot.config)) {
+    if (key in oldOpts) {
+      let id;
+      if (context?.["id"]) {
+        id = context["id"];
+      } else if (cb) {
+        id = cb(["PARAM", "DeviceID.ID"]);
+        if (Array.isArray(id)) id = null;
+      }
+      return config.get(oldOpts[key], id);
+    }
+    return null;
+  }
+
+  const v = expression.evaluate(snapshot.config[key], context, now, cb);
+  return Array.isArray(v) ? null : v;
+}
+
+export function getConfigExpression(revision: string, key: string): Expression {
+  const snapshot = localCache.get(revision);
+  return snapshot.config[key];
+}
+
+export function getUsers(revision: string): Users {
+  const snapshot = localCache.get(revision);
+  return snapshot.users;
+}
+
+export function getPermissions(revision: string): Permissions {
+  const snapshot = localCache.get(revision);
+  return snapshot.permissions;
+}
+
+export function getUiConfig(revision: string): UiConfig {
+  const snapshot = localCache.get(revision);
+  return snapshot.ui;
+}

lib/util.ts

@@ -17,6 +17,8 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+import { EventEmitter } from "events";
+
 export function generateDeviceId(
   deviceIdStruct: Record<string, string>
 ): string {
@@ -61,3 +63,27 @@ export function encodeTag(tag: string): string {
 export function decodeTag(tag: string): string {
   return decodeURIComponent(tag.replace(/0x(?=[0-9A-Z]{2})/g, "%"));
 }
+
+export function once(
+  emitter: EventEmitter,
+  event: string,
+  timeout: number
+): Promise<unknown[]> {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      reject(new Error(`Event ${event} timed out after ${timeout} ms`));
+    }, timeout);
+
+    emitter.once(event, (...args: unknown[]) => {
+      clearTimeout(timer);
+      resolve(args);
+    });
+  });
+}
+
+export function setTimeoutPromise(delay: number, ref = true): Promise<void> {
+  return new Promise((resolve) => {
+    const timerId = setTimeout(resolve, delay);
+    if (!ref) timerId.unref();
+  });
+}

test/db.ts

@@ -1,8 +1,9 @@
 import ava from "ava";
 import { EJSON } from "bson";
 import { Filter } from "mongodb";
-import * as mongodbFunctions from "../lib/mongodb-functions";
 import { stringify, parse } from "../lib/common/expression/parser";
+import { convertOldPrecondition } from "../lib/db/util";
+import { toMongoQuery } from "../lib/db/synth";
 
 ava("convertOldPrecondition", (t) => {
   const tests = [
@@ -53,20 +54,14 @@ ava("convertOldPrecondition", (t) => {
   t.plan(tests.length + 2 * shouldFailTests.length);
   for (const test of tests) {
     t.is(
-      stringify(
+      stringify(convertOldPrecondition(test[0] as Record<string, unknown>)),
-        mongodbFunctions.convertOldPrecondition(
-          test[0] as Record<string, unknown>
-        )
-      ),
       test[1]
     );
   }
 
   for (const test of shouldFailTests) {
     const func = (): void => {
-      mongodbFunctions.convertOldPrecondition(
+      convertOldPrecondition(test[0] as Record<string, unknown>);
-        test[0] as Record<string, unknown>
-      );
     };
     const error = t.throws(func, { instanceOf: Error });
     t.is(error.message, test[1]);
@@ -153,7 +148,7 @@ ava("toMongoQuery", async (t) => {
 
   for (const [expStr, expect] of queries) {
     const exp = parse(expStr);
-    let query = mongodbFunctions.toMongoQuery(exp, "devices");
+    let query = toMongoQuery(exp, "devices");
     if (query) query = EJSON.serialize(query);
     t.deepEqual(query, expect);
   }
@@ -167,7 +162,7 @@ ava("toMongoQuery", async (t) => {
 
   for (const [expStr, err] of failQueries) {
     const exp = parse(expStr);
-    t.throws(() => mongodbFunctions.toMongoQuery(exp, "devices"), {
+    t.throws(() => toMongoQuery(exp, "devices"), {
       message: err,
     });
   }

test/util.ts

@@ -1,5 +1,5 @@
 import ava from "ava";
-import * as common from "../lib/common";
+import * as common from "../lib/util";
 
 ava("generateDeviceId", (t) => {
   const space = [" ", "%20"];

ui/components/tags.ts

@@ -22,7 +22,7 @@ import { m } from "../components";
 import * as notifications from "../notifications";
 import * as store from "../store";
 import { getIcon } from "../icons";
-import { decodeTag } from "../../lib/common";
+import { decodeTag } from "../../lib/util";
 
 const component: ClosureComponent = (): Component => {
   return {

ui/filter-component.ts

@@ -22,7 +22,7 @@ import { parse, stringify, map } from "../lib/common/expression/parser";
 import memoize from "../lib/common/memoize";
 import Autocomplete from "./autocomplete-compnent";
 import * as smartQuery from "./smart-query";
-import { validQuery } from "../lib/mongodb-functions";
+import { validQuery } from "../lib/db/synth";
 import { Expression } from "../lib/types";
 
 const getAutocomplete = memoize((resource) => {

ui/smart-query.ts

@@ -19,7 +19,7 @@
 
 import config from "./config";
 import { Expression } from "../lib/types";
-import { encodeTag } from "../lib/common";
+import { encodeTag } from "../lib/util";
 
 const resources = {
   devices: {},

ui/store.ts

@@ -21,7 +21,7 @@ import m from "mithril";
 import { stringify } from "../lib/common/expression/parser";
 import { or, and, evaluate } from "../lib/common/expression/util";
 import memoize from "../lib/common/memoize";
-import { QueryOptions, Expression, Task } from "../lib/types";
+import { Expression, Task } from "../lib/types";
 import * as notifications from "./notifications";
 import { configSnapshot, genieacsVersion } from "./config";
 import { QueueTask } from "./task-queue";
@@ -342,7 +342,7 @@ function inferQuery(resourceType, queryResponse): void {
 export function fetch(
   resourceType: string,
   filter: Expression,
-  options: QueryOptions = {}
+  options: { limit?: number; sort?: { [param: string]: number } } = {}
 ): QueryResponse {
   const filterStr = memoizedStringify(filter);
   const sort = Object.assign({}, options.sort);