diff --git a/.eslintrc.json b/.eslintrc.json index c305923..75420eb 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -21,7 +21,6 @@ "no-buffer-constructor": "error", "prefer-const": ["error", { "destructuring": "all" }], "eqeqeq": ["error", "always", { "null": "ignore" }], - "no-return-await": "error", "@typescript-eslint/no-explicit-any": 0, "@typescript-eslint/no-use-before-define": [ "error", diff --git a/bin/genieacs-cwmp.ts b/bin/genieacs-cwmp.ts index 6809b2c..2aa5b5d 100755 --- a/bin/genieacs-cwmp.ts +++ b/bin/genieacs-cwmp.ts @@ -22,7 +22,7 @@ import * as logger from "../lib/logger"; import * as cluster from "../lib/cluster"; import * as server from "../lib/server"; import * as cwmp from "../lib/cwmp"; -import * as db from "../lib/db"; +import * as db from "../lib/db/db"; import * as extensions from "../lib/extensions"; import { version as VERSION } from "../package.json"; diff --git a/bin/genieacs-fs.ts b/bin/genieacs-fs.ts index 240ef4e..97a4ad7 100755 --- a/bin/genieacs-fs.ts +++ b/bin/genieacs-fs.ts @@ -22,7 +22,7 @@ import * as logger from "../lib/logger"; import * as cluster from "../lib/cluster"; import * as server from "../lib/server"; import { listener } from "../lib/fs"; -import * as db from "../lib/db"; +import * as db from "../lib/db/db"; import { version as VERSION } from "../package.json"; logger.init("fs", VERSION); diff --git a/bin/genieacs-nbi.ts b/bin/genieacs-nbi.ts index 20e36bb..b334014 100755 --- a/bin/genieacs-nbi.ts +++ b/bin/genieacs-nbi.ts @@ -22,7 +22,7 @@ import * as logger from "../lib/logger"; import * as cluster from "../lib/cluster"; import * as server from "../lib/server"; import { listener } from "../lib/nbi"; -import * as db from "../lib/db"; +import * as db from "../lib/db/db"; import * as extensions from "../lib/extensions"; import { version as VERSION } from "../package.json"; diff --git a/bin/genieacs-ui.ts b/bin/genieacs-ui.ts index 7cd93af..03f5345 100755 --- a/bin/genieacs-ui.ts +++ b/bin/genieacs-ui.ts @@ -23,7 +23,7 @@ import * as cluster from "../lib/cluster"; import * as server from "../lib/server"; import { listener } from "../lib/ui"; import * as extensions from "../lib/extensions"; -import * as db from "../lib/db"; +import * as db from "../lib/db/db"; import { version as VERSION } from "../package.json"; logger.init("ui", VERSION); diff --git a/lib/api-functions.ts b/lib/api-functions.ts index 4c2922d..a5d5e88 100644 --- a/lib/api-functions.ts +++ b/lib/api-functions.ts @@ -17,21 +17,42 @@ * along with GenieACS. If not, see . */ -import { collections } from "./db"; -import * as common from "./common"; +import { ObjectId } from "mongodb"; +import { collections } from "./db/db"; +import { + count, + deleteConfig, + deleteFault, + deleteFile, + deletePermission, + deletePreset, + deleteProvision, + deleteTask, + deleteUser, + deleteVirtualParameter, + putConfig, + putPermission, + putPreset, + putProvision, + putUser, + putVirtualParameter, +} from "./ui/db"; +import * as common from "./util"; import * as cache from "./cache"; import { - getCurrentSnapshot, + getRevision, getConfig, getConfigExpression, -} from "./local-cache"; + getUsers, +} from "./ui/local-cache"; import { httpConnectionRequest, udpConnectionRequest, } from "./connection-request"; import { Expression, Task } from "./types"; -import { flattenDevice } from "./mongodb-functions"; import { evaluate } from "./common/expression/util"; +import { hashPassword } from "./auth"; +import { flattenDevice } from "./ui/db"; export async function connectionRequest( deviceId: string, @@ -103,7 +124,7 @@ export async function connectionRequest( return exp; }; - const snapshot = await getCurrentSnapshot(); + const snapshot = await getRevision(); const now = Date.now(); const UDP_CONNECTION_REQUEST_PORT = +getConfig( snapshot, @@ -350,3 +371,176 @@ export async function deleteDevice(deviceId: string): Promise { cache.del(`${deviceId}_tasks_faults_operations`), ]); } + +export async function deleteResource( + resource: string, + id: string +): Promise { + if (resource === "devices") { + await deleteDevice(id); + } else if (resource === "files") { + await deleteFile(id); + await cache.del("cwmp-local-cache-hash"); + } else if (resource === "faults") { + const deviceId = id.split(":", 1)[0]; + const channel = id.slice(deviceId.length + 1); + const proms = [deleteFault(id)]; + if (channel.startsWith("task_")) + proms.push(deleteTask(new ObjectId(channel.slice(5)))); + await Promise.all(proms); + await cache.del(`${deviceId}_tasks_faults_operations`); + } else if (resource === "provisions") { + await deleteProvision(id); + await cache.del("cwmp-local-cache-hash"); + } else if (resource === "presets") { + await deletePreset(id); + await cache.del("cwmp-local-cache-hash"); + } else if (resource === "virtualParameters") { + await deleteVirtualParameter(id); + await cache.del("cwmp-local-cache-hash"); + } else if (resource === "config") { + await deleteConfig(id); + await Promise.all([ + cache.del("ui-local-cache-hash"), + cache.del("cwmp-local-cache-hash"), + ]); + } else if (resource === "permissions") { + await deletePermission(id); + await cache.del("ui-local-cache-hash"); + } else if (resource === "users") { + await deleteUser(id); + await cache.del("ui-local-cache-hash"); + } else { + throw new Error(`Unknown resource ${resource}`); + } +} + +export async function postTasks( + deviceId: string, + tasks: Task[], + timeout: number, + device: Record +): Promise<{ connectionRequest: string; tasks: any[] }> { + for (const task of tasks) { + delete task._id; + task["device"] = deviceId; + } + + tasks = await insertTasks(tasks); + const statuses = tasks.map((t) => { + return { _id: t._id, status: "pending" }; + }); + const lastInform = Date.now(); + const notInSession = await awaitSessionEnd(deviceId, 30000); + if (!notInSession) { + await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id)))); + return { + connectionRequest: "Session took too long to complete", + tasks: statuses, + }; + } + + await cache.del(`${deviceId}_tasks_faults_operations`); + + const status = await connectionRequest(deviceId, device); + + if (status) { + await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id)))); + return { + connectionRequest: status, + tasks: statuses, + }; + } + + const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout); + if (!sessionStarted) { + await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id)))); + return { + connectionRequest: "No contact from CPE", + tasks: statuses, + }; + } + + const sessionEnded = await awaitSessionEnd(deviceId, 120000); + if (!sessionEnded) { + await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id)))); + return { + connectionRequest: "Session took too long to complete", + tasks: statuses, + }; + } + + const promises: Promise[] = []; + for (const s of statuses) { + promises.push(count("tasks", ["=", ["PARAM", "_id"], s._id])); + promises.push( + count("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`]) + ); + } + + const res = await Promise.all(promises); + for (const [i, r] of statuses.entries()) { + if (!res[i * 2]) { + r.status = "done"; + } else if (res[i * 2 + 1]) { + r.status = "fault"; + r["fault"] = res[i * 2 + 1][0]; + } + } + + await Promise.all(statuses.map((t) => deleteTask(new ObjectId(t._id)))); + + return { connectionRequest: "OK", tasks: statuses }; +} + +// TODO Implement validation +export async function putResource( + resource: string, + id: string, + data: any +): Promise { + if (resource === "presets") { + await putPreset(id, data); + await cache.del("cwmp-local-cache-hash"); + } else if (resource === "provisions") { + await putProvision(id, data); + await cache.del("cwmp-local-cache-hash"); + } else if (resource === "virtualParameters") { + await putVirtualParameter(id, data); + await cache.del("cwmp-local-cache-hash"); + } else if (resource === "config") { + await putConfig(id, data); + await Promise.all([ + cache.del("ui-local-cache-hash"), + cache.del("cwmp-local-cache-hash"), + ]); + } else if (resource === "permissions") { + await putPermission(id, data); + await cache.del("ui-local-cache-hash"); + } else if (resource === "users") { + delete data["password"]; + delete data["salt"]; + await putUser(id, data); + await cache.del("ui-local-cache-hash"); + } else { + throw new Error(`Unknown resource ${resource}`); + } +} + +export function authLocal( + snapshot: string, + username: string, + password: string +): Promise { + return new Promise((resolve, reject) => { + const users = getUsers(snapshot); + const user = users[username]; + if (!user?.password) return void resolve(null); + hashPassword(password, user.salt) + .then((hash) => { + if (hash === user.password) resolve(true); + else resolve(false); + }) + .catch(reject); + }); +} diff --git a/lib/cache.ts b/lib/cache.ts index 200c3eb..0dfe5bb 100644 --- a/lib/cache.ts +++ b/lib/cache.ts @@ -17,28 +17,19 @@ * along with GenieACS. If not, see . */ -import { Collection } from "mongodb"; -import { onConnect } from "./db"; +import { collections } from "./db/db"; import * as config from "./config"; -import { Cache } from "./mongodb-types"; const CLOCK_SKEW_TOLERANCE = 30000; const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL"); -let cacheCollection: Collection; - -onConnect(async (db) => { - cacheCollection = db.collection("cache"); - await cacheCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 }); -}); - export async function get(key: string): Promise { - const res = await cacheCollection.findOne({ _id: key }); + const res = await collections.cache.findOne({ _id: key }); return res?.value; } export async function del(key: string): Promise { - await cacheCollection.deleteOne({ _id: key }); + await collections.cache.deleteOne({ _id: key }); } export async function set( @@ -50,7 +41,7 @@ export async function set( const expire = new Date( timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000 ); - await cacheCollection.replaceOne( + await collections.cache.replaceOne( { _id: key }, { value, expire, timestamp }, { upsert: true } @@ -58,7 +49,7 @@ export async function set( } export async function pop(key: string): Promise { - const res = await cacheCollection.findOneAndDelete({ _id: key }); + const res = await collections.cache.findOneAndDelete({ _id: key }); return res.value?.value; } @@ -70,7 +61,7 @@ export async function acquireLock( ): Promise { try { const now = Date.now(); - const r = await cacheCollection.findOneAndUpdate( + const r = await collections.cache.findOneAndUpdate( { _id: lockName, value: token }, { $set: { @@ -97,7 +88,7 @@ export async function releaseLock( lockName: string, token: string ): Promise { - const res = await cacheCollection.deleteOne({ + const res = await collections.cache.deleteOne({ _id: lockName, value: token, }); diff --git a/lib/cwmp.ts b/lib/cwmp.ts index f60a4ec..bb4a45a 100644 --- a/lib/cwmp.ts +++ b/lib/cwmp.ts @@ -22,7 +22,7 @@ import * as crypto from "crypto"; import { Socket } from "net"; import * as auth from "./auth"; import * as config from "./config"; -import * as common from "./common"; +import { generateDeviceId, once, setTimeoutPromise } from "./util"; import * as soap from "./soap"; import * as session from "./session"; import { @@ -31,8 +31,19 @@ import { extractParams, } from "./common/expression/util"; import * as cache from "./cache"; -import * as localCache from "./local-cache"; -import * as db from "./db"; +import * as localCache from "./cwmp/local-cache"; +import { + clearTasks, + deleteFault, + deleteOperation, + fetchDevice, + getDueTasks, + getFaults, + getOperations, + saveDevice, + saveFault, + saveOperation, +} from "./cwmp/db"; import * as logger from "./logger"; import * as scheduling from "./scheduling"; import Path from "./common/path"; @@ -59,7 +70,6 @@ import { parseXmlDeclaration } from "./xml-parser"; import * as debug from "./debug"; import { getRequestOrigin } from "./forwarded"; import { getSocketEndpoints } from "./server"; -import { EventEmitter } from "events"; const gzipPromisified = promisify(zlib.gzip); const deflatePromisified = promisify(zlib.deflate); @@ -773,7 +783,7 @@ async function endSession(sessionContext: SessionContext): Promise { const promises = []; promises.push( - db.saveDevice( + saveDevice( sessionContext.deviceId, sessionContext.deviceData, sessionContext.new, @@ -786,14 +796,14 @@ async function endSession(sessionContext: SessionContext): Promise { saveCache = true; if (sessionContext.operations[k]) { promises.push( - db.saveOperation( + saveOperation( sessionContext.deviceId, k, sessionContext.operations[k] ) ); } else { - promises.push(db.deleteOperation(sessionContext.deviceId, k)); + promises.push(deleteOperation(sessionContext.deviceId, k)); } } } @@ -801,7 +811,7 @@ async function endSession(sessionContext: SessionContext): Promise { if (sessionContext.doneTasks?.length) { saveCache = true; promises.push( - db.clearTasks(sessionContext.deviceId, sessionContext.doneTasks) + clearTasks(sessionContext.deviceId, sessionContext.doneTasks) ); } @@ -811,10 +821,10 @@ async function endSession(sessionContext: SessionContext): Promise { if (sessionContext.faults[k]) { sessionContext.faults[k].retries = sessionContext.retries[k]; promises.push( - db.saveFault(sessionContext.deviceId, k, sessionContext.faults[k]) + saveFault(sessionContext.deviceId, k, sessionContext.faults[k]) ); } else { - promises.push(db.deleteFault(sessionContext.deviceId, k)); + promises.push(deleteFault(sessionContext.deviceId, k)); } } } @@ -890,30 +900,6 @@ async function sendAcsRequest( return writeResponse(sessionContext, res); } -function once( - emitter: EventEmitter, - event: string, - timeout: number -): Promise { - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - reject(new Error(`Event ${event} timed out after ${timeout} ms`)); - }, timeout); - - emitter.once(event, (...args: unknown[]) => { - clearTimeout(timer); - resolve(args); - }); - }); -} - -function setTimeoutPromise(delay: number, ref = true): Promise { - return new Promise((resolve) => { - const timerId = setTimeout(resolve, delay); - if (!ref) timerId.unref(); - }); -} - // When socket closes, store active sessions in cache export async function onConnection(socket: Socket): Promise { await once(socket, "close", 300000); @@ -966,7 +952,7 @@ export async function onConnection(socket: Socket): Promise { export async function onClientError(err: Error, socket: Socket): Promise { const remoteAddress = getSocketEndpoints(socket).remoteAddress; - const cacheSnapshot = await localCache.getCurrentSnapshot(); + const cacheSnapshot = await localCache.getRevision(); const debugEnabled = !!localCache.getConfig( cacheSnapshot, "cwmp.debug", @@ -1021,9 +1007,9 @@ async function getDueTasksAndFaultsAndOperations( } const res2 = await Promise.all([ - db.getDueTasks(deviceId, timestamp), - db.getFaults(deviceId), - db.getOperations(deviceId), + getDueTasks(deviceId, timestamp), + getFaults(deviceId), + getOperations(deviceId), ]); return { tasks: res2[0][0], @@ -1383,7 +1369,7 @@ async function clientError( debugEnabled = sessionContext.debug; deviceId = sessionContext.deviceId; } else { - const cacheSnapshot = await localCache.getCurrentSnapshot(); + const cacheSnapshot = await localCache.getRevision(); debugEnabled = !!localCache.getConfig( cacheSnapshot, "cwmp.debug", @@ -1637,9 +1623,9 @@ async function listenerAsync( } stats.initiatedSessions += 1; - const deviceId = common.generateDeviceId(rpc.cpeRequest.deviceId); + const deviceId = generateDeviceId(rpc.cpeRequest.deviceId); - const cacheSnapshot = await localCache.getCurrentSnapshot(); + const cacheSnapshot = await localCache.getRevision(); const _sessionContext = session.init( deviceId, @@ -1679,7 +1665,7 @@ async function listenerAsync( } } - const parameters = await db.fetchDevice( + const parameters = await fetchDevice( _sessionContext.deviceId, _sessionContext.timestamp ); diff --git a/lib/db.ts b/lib/cwmp/db.ts similarity index 81% rename from lib/db.ts rename to lib/cwmp/db.ts index 6f2f67d..9df0ba4 100644 --- a/lib/db.ts +++ b/lib/cwmp/db.ts @@ -1,53 +1,16 @@ -/** - * Copyright 2013-2019 GenieACS Inc. - * - * This file is part of GenieACS. - * - * GenieACS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * GenieACS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with GenieACS. If not, see . - */ - -import { MongoClient, ObjectId, Collection, Db } from "mongodb"; -import { get } from "./config"; -import { decodeTag, encodeTag, escapeRegExp } from "./common"; -import { parse } from "./common/expression/parser"; +import { decodeTag, encodeTag, escapeRegExp } from "../util"; import { DeviceData, Attributes, SessionFault, Task, Operation, - Expression, -} from "./types"; -import Path from "./common/path"; -import * as MongoTypes from "./mongodb-types"; - -export const collections = { - devices: null as Collection, - presets: null as Collection, - objects: null as Collection, - provisions: null as Collection, - virtualParameters: null as Collection, - faults: null as Collection, - tasks: null as Collection, - files: null as Collection, - operations: null as Collection, - permissions: null as Collection, - users: null as Collection, - config: null as Collection, -}; - -let clientPromise: Promise; +} from "../types"; +import Path from "../common/path"; +import { collections } from "../db/db"; +import { optimizeProjection } from "../db/util"; +import * as MongoTypes from "../db/types"; +import { ObjectId } from "mongodb"; const INVALID_PATH_SUFFIX = "__invalid"; @@ -57,64 +20,6 @@ function compareAccessLists(list1: string[], list2: string[]): boolean { return true; } -const onConnectCallbacks: ((db: Db) => Promise)[] = []; - -export function onConnect(callback: (db: Db) => Promise): void { - onConnectCallbacks.push(callback); -} - -export async function connect(): Promise { - clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL")); - - const client = await clientPromise; - const db = client.db(); - await Promise.all(onConnectCallbacks.map((c) => c(db))); -} - -onConnect(async (db) => { - collections.tasks = db.collection("tasks"); - await collections.tasks.createIndex({ device: 1, timestamp: 1 }); - - collections.devices = db.collection("devices"); - collections.presets = db.collection("presets"); - collections.objects = db.collection("objects"); - collections.files = db.collection("fs.files"); - collections.provisions = db.collection("provisions"); - collections.virtualParameters = db.collection("virtualParameters"); - collections.faults = db.collection("faults"); - collections.operations = db.collection("operations"); - collections.permissions = db.collection("permissions"); - collections.users = db.collection("users"); - collections.config = db.collection("config"); -}); - -export async function disconnect(): Promise { - if (clientPromise != null) await (await clientPromise).close(); -} - -// Optimize projection by removing overlaps -// This can modify the object -export function optimizeProjection(obj: { [path: string]: 1 }): { - [path: string]: 1; -} { - if (obj[""]) return { "": obj[""] }; - - const keys = Object.keys(obj).sort(); - if (keys.length <= 1) return obj; - - for (let i = 1; i < keys.length; ++i) { - const a = keys[i - 1]; - const b = keys[i]; - if (b.startsWith(a)) { - if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") { - delete obj[b]; - keys.splice(i--, 1); - } - } - } - return obj; -} - export async function fetchDevice( id: string, timestamp: number @@ -822,56 +727,3 @@ export async function deleteOperation( ): Promise { await collections.operations.deleteOne({ _id: `${deviceId}:${commandKey}` }); } - -export async function getPresets(): Promise[]> { - return collections.presets.find().toArray(); -} - -export async function getObjects(): Promise[]> { - return collections.objects.find().toArray(); -} - -export async function getProvisions(): Promise[]> { - return collections.provisions.find().toArray(); -} - -export async function getVirtualParameters(): Promise[]> { - return collections.virtualParameters.find().toArray(); -} - -export function getFiles(): Promise[]> { - return collections.files.find().toArray(); -} - -export async function getConfig(): Promise< - { id: string; value: Expression }[] -> { - const res = await collections.config.find().toArray(); - return res.map((c) => ({ - id: c["_id"], - value: parse(c["value"]), - })); -} - -interface Permission { - role: string; - resource: string; - access: number; - filter: string; - validate: string; -} - -export async function getPermissions(): Promise { - return collections.permissions.find().toArray() as unknown as Permission[]; -} - -interface User { - _id: string; - password: string; - salt: string; - roles: string; -} - -export async function getUsers(): Promise { - return collections.users.find().toArray() as unknown as User[]; -} diff --git a/lib/cwmp/local-cache.ts b/lib/cwmp/local-cache.ts new file mode 100644 index 0000000..8d787bf --- /dev/null +++ b/lib/cwmp/local-cache.ts @@ -0,0 +1,359 @@ +import * as vm from "vm"; +import * as crypto from "crypto"; +import * as config from "../config"; +import { collections } from "../db/db"; +import { convertOldPrecondition } from "../db/util"; +import * as logger from "../logger"; +import * as scheduling from "../scheduling"; +import * as expression from "../common/expression/util"; +import { parse } from "../common/expression/parser"; +import { + Preset, + Expression, + Provisions, + VirtualParameters, + Files, + Config, +} from "../types"; +import { LocalCache } from "../local-cache"; + +interface Snapshot { + presets: Preset[]; + provisions: Provisions; + virtualParameters: VirtualParameters; + files: Files; + config: Config; +} + +function flattenObject>( + src: T, + prefix = "", + dst = {} as T +): T { + const t: Record = {}; + t["hi"] = "hi"; + for (const k of Object.keys(src)) { + const v = src[k]; + if (typeof v === "object" && !Array.isArray(v)) + flattenObject(v as T, `${prefix}${k}.`, dst); + else dst[`${prefix}${k}`] = v; + } + return dst; +} + +async function fetchPresets(): Promise<[string, Preset[]]> { + const res = await collections.presets.find().toArray(); + let objects = await collections.objects.find().toArray(); + res.sort((a, b) => (a._id > b._id ? 1 : -1)); + objects.sort((a, b) => (a._id > b._id ? 1 : -1)); + const h = crypto + .createHash("md5") + .update(JSON.stringify(res)) + .update(JSON.stringify(objects)) + .digest("hex"); + + objects = objects.map((obj) => { + // Flatten object + obj = flattenObject(obj); + + // If no keys are defined, consider all parameters as keys to keep the + // same behavior from v1.0 + if (!obj["_keys"]?.length) + obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_")); + + return obj; + }); + + res.sort((a, b) => { + if (a["weight"] === b["weight"]) + return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0; + else return a["weight"] - b["weight"]; + }); + + const presets = [] as Preset[]; + for (const preset of res) { + let schedule: { md5: string; duration: number; schedule: any } = null; + if (preset["schedule"]) { + const parts = preset["schedule"].trim().split(/\s+/); + schedule = { + md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"), + duration: null, + schedule: null, + }; + + try { + schedule.duration = +parts.shift() * 1000; + schedule.schedule = scheduling.parseCron(parts.join(" ")); + } catch (err) { + logger.warn({ + message: "Invalid preset schedule", + preset: preset["_id"], + schedule: preset["schedule"], + }); + schedule.schedule = false; + } + } + + const events = preset["events"] || {}; + let precondition = true as Expression; + if (preset["precondition"]) { + try { + precondition = parse(preset["precondition"]); + } catch (error) { + precondition = convertOldPrecondition( + JSON.parse(preset["precondition"]) + ); + } + + // Simplify expression + precondition = expression.evaluate(precondition); + } + + const _provisions: Preset["provisions"] = []; + + // Generate provisions from the old configuration format + for (const c of preset.configurations) { + switch (c.type) { + case "age": + _provisions.push(["refresh", c.name, +c.age]); + break; + + case "value": + _provisions.push(["value", c.name, c.value]); + break; + + case "add_tag": + _provisions.push(["tag", c.tag, true]); + break; + + case "delete_tag": + _provisions.push(["tag", c.tag, false]); + break; + + case "provision": + _provisions.push([c.name, ...(c.args || [])]); + break; + + case "add_object": + for (const obj of objects) { + if (obj["_id"] === c.object) { + const alias = obj["_keys"] + .map((k) => `${k}:${JSON.stringify(obj[k])}`) + .join(","); + const p = `${c.name}.[${alias}]`; + _provisions.push(["instances", p, 1]); + + for (const k in obj) { + if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1)) + _provisions.push(["value", `${p}.${k}`, obj[k]]); + } + } + } + + break; + + case "delete_object": + for (const obj of objects) { + if (obj["_id"] === c.object) { + const alias = obj["_keys"] + .map((k) => `${k}:${JSON.stringify(obj[k])}`) + .join(","); + const p = `${c.name}.[${alias}]`; + _provisions.push(["instances", p, 0]); + } + } + + break; + + default: { + const exhaustiveCheck: never = c; + throw new Error( + `Unknown configuration type ${exhaustiveCheck["type"]}` + ); + } + } + } + + presets.push({ + name: preset["_id"], + channel: (preset["channel"] as string) || "default", + schedule: schedule, + events: events, + precondition: precondition, + provisions: _provisions, + }); + } + + return [h, presets]; +} + +async function fetchProvisions(): Promise<[string, Provisions]> { + const res = await collections.provisions.find().toArray(); + res.sort((a, b) => (a._id > b._id ? 1 : -1)); + const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex"); + + const provisions = {}; + for (const r of res) { + provisions[r._id] = {}; + provisions[r._id].md5 = crypto + .createHash("md5") + .update(r.script) + .digest("hex"); + provisions[r._id].script = new vm.Script( + `"use strict";(function(){\n${r.script}\n})();`, + { filename: r._id, lineOffset: -1, timeout: 50 } + ); + } + + return [h, provisions]; +} + +async function fetchVirtualParameters(): Promise<[string, VirtualParameters]> { + const res = await collections.virtualParameters.find().toArray(); + res.sort((a, b) => (a._id > b._id ? 1 : -1)); + const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex"); + + const virtualParameters = {}; + for (const r of res) { + virtualParameters[r._id] = {}; + virtualParameters[r._id].md5 = crypto + .createHash("md5") + .update(r.script) + .digest("hex"); + virtualParameters[r._id].script = new vm.Script( + `"use strict";(function(){\n${r.script}\n})();`, + { filename: r._id, lineOffset: -1, timeout: 50 } + ); + } + + return [h, virtualParameters]; +} + +async function fetchFiles(): Promise<[string, Files]> { + const res = await collections.files.find().toArray(); + res.sort((a, b) => (a._id > b._id ? 1 : -1)); + const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex"); + const files = {}; + + for (const r of res) { + const id = r.filename || r._id.toString(); + files[id] = {}; + files[id].length = r.length; + } + + return [h, files]; +} + +async function fetchConfig(): Promise<[string, Config]> { + const conf = await collections.config.find().toArray(); + conf.sort((a, b) => (a._id > b._id ? 1 : -1)); + const h = crypto.createHash("md5").update(JSON.stringify(conf)).digest("hex"); + + const _config = {}; + + for (const c of conf) { + // Evaluate expressions to simplify them + _config[c._id] = expression.evaluate(parse(c.value)); + } + + return [h, _config]; +} + +const localCache = new LocalCache("cwmp-local-cache-hash", refresh); + +async function refresh(): Promise<[string, Snapshot]> { + const res = await Promise.all([ + fetchPresets(), + fetchProvisions(), + fetchVirtualParameters(), + fetchFiles(), + fetchConfig(), + ]); + + const h = crypto.createHash("md5"); + for (const r of res) h.update(r[0]); + + const snapshot = { + presets: res[0][1], + provisions: res[1][1], + virtualParameters: res[2][1], + files: res[3][1], + config: res[4][1], + }; + + return [h.digest("hex"), snapshot]; +} + +export async function getRevision(): Promise { + return await localCache.getRevision(); +} + +export function getPresets(revision: string): Preset[] { + return localCache.get(revision).presets; +} + +export function getProvisions(revision: string): Provisions { + return localCache.get(revision).provisions; +} + +export function getVirtualParameters(revision: string): VirtualParameters { + return localCache.get(revision).virtualParameters; +} + +export function getFiles(revision: string): Files { + return localCache.get(revision).files; +} + +export function getConfig( + snapshotKey: string, + key: string, + context: Record, + now: number, + cb?: (e: Expression) => Expression +): string | number | boolean | null { + const snapshot = localCache.get(snapshotKey); + if (!snapshot) throw new Error("Cache snapshot does not exist"); + + const oldOpts = { + "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT", + "cwmp.debug": "DEBUG", + "cwmp.retryDelay": "RETRY_DELAY", + "cwmp.sessionTimeout": "SESSION_TIMEOUT", + "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT", + "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL", + "cwmp.gpvBatchSize": "GPV_BATCH_SIZE", + "cwmp.cookiesPath": "COOKIES_PATH", + "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS", + "cwmp.booleanLiteral": "BOOLEAN_LITERAL", + "cwmp.connectionRequestAllowBasicAuth": + "CONNECTION_REQUEST_ALLOW_BASIC_AUTH", + "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS", + "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD", + "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT", + }; + + if (!(key in snapshot.config)) { + if (key in oldOpts) { + let id; + if (context?.["id"]) { + id = context["id"]; + } else if (cb) { + id = cb(["PARAM", "DeviceID.ID"]); + if (Array.isArray(id)) id = null; + } + return config.get(oldOpts[key], id); + } + return null; + } + + const v = expression.evaluate(snapshot.config[key], context, now, cb); + return Array.isArray(v) ? null : v; +} + +export function getConfigExpression( + snapshotKey: string, + key: string +): Expression { + const snapshot = localCache.get(snapshotKey); + return snapshot.config[key]; +} diff --git a/lib/db/db.ts b/lib/db/db.ts new file mode 100644 index 0000000..d9965ad --- /dev/null +++ b/lib/db/db.ts @@ -0,0 +1,73 @@ +/** + * Copyright 2013-2019 GenieACS Inc. + * + * This file is part of GenieACS. + * + * GenieACS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * GenieACS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with GenieACS. If not, see . + */ + +import { MongoClient, Collection, GridFSBucket } from "mongodb"; +import { get } from "../config"; +import * as MongoTypes from "./types"; + +export let filesBucket: GridFSBucket; + +export const collections = { + devices: null as Collection, + presets: null as Collection, + objects: null as Collection, + provisions: null as Collection, + virtualParameters: null as Collection, + faults: null as Collection, + tasks: null as Collection, + files: null as Collection, + operations: null as Collection, + permissions: null as Collection, + users: null as Collection, + config: null as Collection, + cache: null as Collection, +}; + +let clientPromise: Promise; + +export async function connect(): Promise { + clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL")); + + const client = await clientPromise; + const db = client.db(); + + collections.tasks = db.collection("tasks"); + collections.devices = db.collection("devices"); + collections.presets = db.collection("presets"); + collections.objects = db.collection("objects"); + collections.files = db.collection("fs.files"); + collections.provisions = db.collection("provisions"); + collections.virtualParameters = db.collection("virtualParameters"); + collections.faults = db.collection("faults"); + collections.operations = db.collection("operations"); + collections.permissions = db.collection("permissions"); + collections.users = db.collection("users"); + collections.config = db.collection("config"); + collections.cache = db.collection("cache"); + filesBucket = new GridFSBucket(db); + + await Promise.all([ + collections.tasks.createIndex({ device: 1, timestamp: 1 }), + collections.cache.createIndex({ expire: 1 }, { expireAfterSeconds: 0 }), + ]); +} + +export async function disconnect(): Promise { + if (clientPromise != null) await (await clientPromise).close(); +} diff --git a/lib/mongodb-functions.ts b/lib/db/synth.ts similarity index 60% rename from lib/mongodb-functions.ts rename to lib/db/synth.ts index 0267e3a..0b69809 100644 --- a/lib/mongodb-functions.ts +++ b/lib/db/synth.ts @@ -1,461 +1,16 @@ import { Filter } from "mongodb"; import { EJSON } from "bson"; -import { - parse, - stringify, - parseList, - parseLikePattern, -} from "./common/expression/parser"; -import { Expression, Fault, Task } from "./types"; -import { decodeTag, encodeTag } from "./common"; +import { parseLikePattern } from "../common/expression/parser"; +import { Expression } from "../types"; +import { decodeTag } from "../util"; import { SynthContextBase, likeDisjoint, likeImplies, -} from "./common/expression/synth"; -import normalize from "./common/expression/normalize"; -import { Clause } from "./common/expression/synth"; -import * as BI from "./common/expression/bigint"; - -export function processDeviceProjection( - projection: Record -): Record { - if (!projection) return projection; - const p = {}; - for (const [k, v] of Object.entries(projection)) { - if (k === "DeviceID.ID") { - p["_id"] = 1; - } else if (k.startsWith("DeviceID")) { - p["_deviceId._SerialNumber"] = v; - p["_deviceId._OUI"] = v; - p["_deviceId._ProductClass"] = v; - p["_deviceId._Manufacturer"] = v; - } else if (k.startsWith("Tags")) { - p["_tags"] = v; - } else if (k.startsWith("Events")) { - p["_lastInform"] = v; - p["_registered"] = v; - p["_lastBoot"] = v; - p["_lastBootstrap"] = v; - } else { - p[k] = v; - } - } - - return p; -} - -export function processDeviceSort( - sort: Record -): Record { - if (!sort) return sort; - const s = {}; - for (const [k, v] of Object.entries(sort)) { - if (k === "DeviceID.ID") s["_id"] = v; - else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v; - else if (k === "Events.Inform") s["_lastInform"] = v; - else if (k === "Events.Registered") s["_registered"] = v; - else if (k === "Events.1_BOOT") s["_lastBoot"] = v; - else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v; - else s[`${k}._value`] = v; - } - - return s; -} - -function parseDate(d: Date): number | string { - const n = +d; - return isNaN(n) ? "" + d : n; -} - -interface FlatAttributes { - object?: boolean; - objectTimestamp?: number; - writable?: boolean; - writableTimestamp?: number; - value?: [string | number | boolean, string]; - valueTimestamp?: number; - notification?: number; - notificationTimestamp?: number; - accessList?: string[]; - accessListTimestamp?: number; -} - -interface FlatDevice { - [param: string]: FlatAttributes; -} - -export function flattenDevice(device: Record): FlatDevice { - function recursive( - input, - root: string, - output: FlatDevice, - timestamp: number - ): void { - for (const [name, tree] of Object.entries(input)) { - if (!root) { - if (name === "_lastInform") { - output["Events.Inform"] = { - value: [parseDate(tree as Date), "xsd:dateTime"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp, - }; - } else if (name === "_registered") { - output["Events.Registered"] = { - value: [parseDate(tree as Date), "xsd:dateTime"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp, - }; - } else if (name === "_lastBoot") { - output["Events.1_BOOT"] = { - value: [parseDate(tree as Date), "xsd:dateTime"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp, - }; - } else if (name === "_lastBootstrap") { - output["Events.0_BOOTSTRAP"] = { - value: [parseDate(tree as Date), "xsd:dateTime"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp, - }; - } else if (name === "_id") { - output["DeviceID.ID"] = { - value: [tree as string, "xsd:string"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp, - }; - } else if (name === "_deviceId") { - output["DeviceID.Manufacturer"] = { - value: [tree["_Manufacturer"], "xsd:string"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp, - }; - output["DeviceID.OUI"] = { - value: [tree["_OUI"], "xsd:string"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp, - }; - output["DeviceID.ProductClass"] = { - value: [tree["_ProductClass"], "xsd:string"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp, - }; - output["DeviceID.SerialNumber"] = { - value: [tree["_SerialNumber"], "xsd:string"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp, - }; - } else if (name === "_tags") { - output["Tags"] = { - writable: false, - writableTimestamp: timestamp, - object: true, - objectTimestamp: timestamp, - }; - - for (const t of tree as string[]) { - output[`Tags.${encodeTag(t)}`] = { - value: [true, "xsd:boolean"], - valueTimestamp: timestamp, - writable: true, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp, - }; - } - } - } - - if (name.startsWith("_")) continue; - - let childrenTimestamp = timestamp; - - if (!root) childrenTimestamp = +(input["_timestamp"] || 1); - else if (+input["_timestamp"] > timestamp) - childrenTimestamp = +input["_timestamp"]; - - const attrs: FlatAttributes = {}; - if (tree["_value"] != null) { - attrs.value = [ - tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"], - tree["_type"], - ]; - attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp); - attrs.object = false; - attrs.objectTimestamp = childrenTimestamp; - } else if (tree["_object"] != null) { - attrs.object = tree["_object"]; - attrs.objectTimestamp = childrenTimestamp; - } - - if (tree["_writable"] != null) { - attrs.writable = tree["_writable"]; - attrs.writableTimestamp = childrenTimestamp; - } - - if (tree["_notification"] != null) { - attrs.notification = tree["_notification"]; - attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1; - } - - if (tree["_accessList"] != null) { - attrs.accessList = tree["_accessList"]; - attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1; - } - - const r = root ? `${root}.${name}` : name; - output[r] = attrs; - - if (attrs.object || tree["object"] == null) - recursive(tree, r, output, childrenTimestamp); - } - } - - const newDevice: FlatDevice = {}; - const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime(); - recursive(device, "", newDevice, timestamp); - return newDevice; -} - -export function flattenFault(fault: unknown): Fault { - const f = Object.assign({}, fault) as Fault; - if (f.timestamp) f.timestamp = +f.timestamp; - if (f["expiry"]) f["expiry"] = +f["expiry"]; - return f as Fault; -} - -export function flattenTask(task: unknown): Task { - const t = Object.assign({}, task) as Task; - t._id = "" + t._id; - if (t["timestamp"]) t["timestamp"] = +t["timestamp"]; - if (t.expiry) t.expiry = +t.expiry; - return t; -} - -export function convertOldPrecondition( - query: Record -): Expression { - function recursive(_query): Expression { - const expressions: Expression[] = []; - for (const [k, v] of Object.entries(_query)) { - if (k[0] === "$") { - if (k === "$and") { - const and: Expression = ["AND"]; - for (const vv of Object.values(v)) and.push(recursive(vv)); - expressions.push(and); - } else if (k === "$or") { - const or: Expression = ["OR"]; - for (const vv of Object.values(v)) or.push(recursive(vv)); - expressions.push(or); - } else { - throw new Error(`Operator ${k} not supported`); - } - } else if (k === "_tags") { - if (typeof v === "object") { - if (Array.isArray(v)) throw new Error(`Invalid type`); - const conjs: Expression[] = []; - for (const [op, val] of Object.entries(v)) { - if (op === "$ne") { - if (typeof v["$ne"] !== "string") - throw new Error("Only string values are allowed for _tags"); - conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]); - } else if (op === "$eq") { - if (typeof v["$eq"] !== "string") - throw new Error("Only string values are allowed for _tags"); - conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]); - } else { - throw new Error(`Invalid tag query`); - } - } - if (conjs.length === 1) expressions.push(conjs[0]); - else if (conjs.length > 1) expressions.push(["AND", ...conjs]); - } else { - expressions.push([ - "IS NOT NULL", - ["PARAM", `Tags.${encodeTag(v as string)}`], - ]); - } - } else if (k.startsWith("Tags.")) { - let exists: boolean; - if (typeof v === "boolean") exists = v; - else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"]; - else if (v.hasOwnProperty("$ne")) exists = !v["$ne"]; - else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"]; - else throw new Error(`Invalid tag query`); - - expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]); - } else if (typeof v === "object") { - if (Array.isArray(v)) throw new Error(`Invalid type`); - - const exps: Expression[] = []; - for (const [kk, vv] of Object.entries(v)) { - if (kk === "$eq") { - exps.push(["=", ["PARAM", k], vv]); - } else if (kk === "$ne") { - exps.push([ - "OR", - ["<>", ["PARAM", k], vv], - ["IS NULL", ["PARAM", k]], - ]); - } else if (kk === "$lt") { - exps.push(["<", ["PARAM", k], vv]); - } else if (kk === "$lte") { - exps.push(["<=", ["PARAM", k], vv]); - } else if (kk === "$gt") { - exps.push([">", ["PARAM", k], vv]); - } else if (kk === "$gte") { - exps.push([">=", ["PARAM", k], vv]); - } else { - throw new Error(`Operator ${kk} not supported`); - } - if (!["string", "number", "boolean"].includes(typeof vv)) - throw new Error(`Invalid value for ${kk} operator`); - } - if (exps.length === 1) { - expressions.push(exps[0]); - } else if (exps.length > 1) { - const and: Expression = ["AND"]; - expressions.push(and.concat(exps)); - } - } else { - expressions.push(["=", ["PARAM", k], v]); - } - } - if (expressions.length === 1) return expressions[0]; - if (expressions.length === 0) return true; - return ["AND", ...expressions]; - } - - // empty filter - if (!Object.keys(query).length) return true; - - return recursive(query); -} - -export function flattenPreset( - preset: Record -): Record { - const p = Object.assign({}, preset); - if (p.precondition) { - try { - // Try parse to check expression validity - parse(p.precondition as string); - } catch (error) { - p.precondition = convertOldPrecondition( - JSON.parse(p.precondition as string) - ); - p.precondition = (p.precondition as string).length - ? stringify(p.precondition as Expression) - : ""; - } - } - - if (p.events) { - const e = []; - for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`); - p.events = e.join(", "); - } - - const provision = p.configurations[0]; - if ( - (p.configurations as any[]).length === 1 && - provision.type === "provision" && - provision.name && - provision.name.length - ) { - p.provision = provision.name; - p.provisionArgs = provision.args - ? provision.args.map((a) => stringify(a)).join(", ") - : ""; - } - - delete p.configurations; - return p; -} - -export function flattenFile( - file: Record -): Record { - const f = {}; - f["_id"] = file["_id"]; - if (file.metadata) { - f["metadata.fileType"] = file["metadata"]["fileType"] || ""; - f["metadata.oui"] = file["metadata"]["oui"] || ""; - f["metadata.productClass"] = file["metadata"]["productClass"] || ""; - f["metadata.version"] = file["metadata"]["version"] || ""; - } - return f; -} - -export function preProcessPreset( - data: Record -): Record { - const preset = Object.assign({}, data); - - if (!preset.precondition) preset.precondition = ""; - // Try parse to check expression validity - parse(preset.precondition as string); - - preset.weight = parseInt(preset.weight as string) || 0; - - const events = {}; - if (preset.events) { - for (let e of (preset.events as string).split(",")) { - let v = true; - e = e.trim(); - if (e.startsWith("-")) { - v = false; - e = e.slice(1).trim(); - } - if (e) events[e] = v; - } - } - - preset.events = events; - - if (!preset.provision) throw new Error("Invalid preset provision"); - - const configuration = { - type: "provision", - name: preset.provision, - args: null, - }; - - if (preset.provisionArgs) - configuration.args = parseList(preset.provisionArgs as string); - - delete preset.provision; - delete preset.provisionArgs; - preset.configurations = [configuration]; - return preset; -} + Clause, +} from "../common/expression/synth"; +import normalize from "../common/expression/normalize"; +import * as BI from "../common/expression/bigint"; type Minterm = number[]; diff --git a/lib/mongodb-types.ts b/lib/db/types.ts similarity index 83% rename from lib/mongodb-types.ts rename to lib/db/types.ts index 7b512f5..fc8ae04 100644 --- a/lib/mongodb-types.ts +++ b/lib/db/types.ts @@ -1,5 +1,5 @@ import { ObjectId } from "mongodb"; -import { Expression, FaultStruct } from "./types"; +import { Expression, FaultStruct } from "../types"; export interface Fault { _id: string; @@ -115,13 +115,21 @@ export interface Device { _timestamp?: Date; } +type Configuration = + | { type: "age"; name: string; age: number } + | { type: "value"; name: string; value: boolean | number | string } + | { type: "add_tag"; tag: string } + | { type: "delete_tag"; tag: string } + | { type: "add_object"; name: string; object: string } + | { type: "delete_object"; name: string; object: string } + | { type: "provision"; name: string; args?: Expression[] }; + export interface Preset { _id: string; weight: number; - channels: string; - provisions: string; + channel: string; events: Record; - configurations: { type: string; name: string; args: Expression[] }[]; + configurations: Configuration[]; } export interface Object { @@ -155,7 +163,8 @@ export interface Permission { role: string; resource: string; access: 1 | 2 | 3; - validate: string; + filter?: string; + validate?: string; } export interface User { diff --git a/lib/db/util.ts b/lib/db/util.ts new file mode 100644 index 0000000..23f1c47 --- /dev/null +++ b/lib/db/util.ts @@ -0,0 +1,123 @@ +import { Expression } from "../types"; +import { encodeTag } from "../util"; + +// Optimize projection by removing overlaps +// This can modify the object +export function optimizeProjection(obj: { [path: string]: 1 }): { + [path: string]: 1; +} { + if (obj[""]) return { "": obj[""] }; + + const keys = Object.keys(obj).sort(); + if (keys.length <= 1) return obj; + + for (let i = 1; i < keys.length; ++i) { + const a = keys[i - 1]; + const b = keys[i]; + if (b.startsWith(a)) { + if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") { + delete obj[b]; + keys.splice(i--, 1); + } + } + } + return obj; +} + +export function convertOldPrecondition(q: Record): Expression { + function recursive(_query): Expression { + const expressions: Expression[] = []; + for (const [k, v] of Object.entries(_query)) { + if (k[0] === "$") { + if (k === "$and") { + const and: Expression = ["AND"]; + for (const vv of Object.values(v)) and.push(recursive(vv)); + expressions.push(and); + } else if (k === "$or") { + const or: Expression = ["OR"]; + for (const vv of Object.values(v)) or.push(recursive(vv)); + expressions.push(or); + } else { + throw new Error(`Operator ${k} not supported`); + } + } else if (k === "_tags") { + if (typeof v === "object") { + if (Array.isArray(v)) throw new Error(`Invalid type`); + const conjs: Expression[] = []; + for (const [op, val] of Object.entries(v)) { + if (op === "$ne") { + if (typeof v["$ne"] !== "string") + throw new Error("Only string values are allowed for _tags"); + conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]); + } else if (op === "$eq") { + if (typeof v["$eq"] !== "string") + throw new Error("Only string values are allowed for _tags"); + conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]); + } else { + throw new Error(`Invalid tag query`); + } + } + if (conjs.length === 1) expressions.push(conjs[0]); + else if (conjs.length > 1) expressions.push(["AND", ...conjs]); + } else { + expressions.push([ + "IS NOT NULL", + ["PARAM", `Tags.${encodeTag(v as string)}`], + ]); + } + } else if (k.startsWith("Tags.")) { + let exists: boolean; + if (typeof v === "boolean") exists = v; + else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"]; + else if (v.hasOwnProperty("$ne")) exists = !v["$ne"]; + else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"]; + else throw new Error(`Invalid tag query`); + + expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]); + } else if (typeof v === "object") { + if (Array.isArray(v)) throw new Error(`Invalid type`); + + const exps: Expression[] = []; + for (const [kk, vv] of Object.entries(v)) { + if (kk === "$eq") { + exps.push(["=", ["PARAM", k], vv]); + } else if (kk === "$ne") { + exps.push([ + "OR", + ["<>", ["PARAM", k], vv], + ["IS NULL", ["PARAM", k]], + ]); + } else if (kk === "$lt") { + exps.push(["<", ["PARAM", k], vv]); + } else if (kk === "$lte") { + exps.push(["<=", ["PARAM", k], vv]); + } else if (kk === "$gt") { + exps.push([">", ["PARAM", k], vv]); + } else if (kk === "$gte") { + exps.push([">=", ["PARAM", k], vv]); + } else { + throw new Error(`Operator ${kk} not supported`); + } + if (!["string", "number", "boolean"].includes(typeof vv)) + throw new Error(`Invalid value for ${kk} operator`); + } + if (exps.length === 1) { + expressions.push(exps[0]); + } else if (exps.length > 1) { + const and: Expression = ["AND"]; + expressions.push(and.concat(exps)); + } + } else { + expressions.push(["=", ["PARAM", k], v]); + } + } + if (expressions.length === 1) return expressions[0]; + if (expressions.length === 0) return true; + return ["AND", ...expressions]; + } + + // empty filter + if (!Object.keys(q).length) return true; + + return recursive(q); +} diff --git a/lib/fs.ts b/lib/fs.ts index 6e58335..4ee84ee 100644 --- a/lib/fs.ts +++ b/lib/fs.ts @@ -20,18 +20,11 @@ import * as url from "url"; import { IncomingMessage, ServerResponse } from "http"; import { PassThrough, pipeline, Readable } from "stream"; -import { GridFSBucket } from "mongodb"; -import { onConnect, collections } from "./db"; +import { filesBucket, collections } from "./db/db"; import * as logger from "./logger"; import { getRequestOrigin } from "./forwarded"; import memoize from "./common/memoize"; -let filesBucket: GridFSBucket; - -onConnect(async (db) => { - filesBucket = new GridFSBucket(db); -}); - const getFile = memoize( async ( uploadDate: number, diff --git a/lib/init.ts b/lib/init.ts index 2ccda41..4d48ad1 100644 --- a/lib/init.ts +++ b/lib/init.ts @@ -17,9 +17,16 @@ * along with GenieACS. If not, see . */ -import * as localCache from "./local-cache"; +import { getRevision, getUiConfig, getUsers } from "./ui/local-cache"; import { generateSalt, hashPassword } from "./auth"; -import * as db from "./ui/db"; +import { collections } from "./db/db"; +import { + putConfig, + putPermission, + putPreset, + putProvision, + putUser, +} from "./ui/db"; import { del } from "./cache"; interface Status { @@ -84,14 +91,16 @@ declare("Device.ManagementServer.PeriodicInformTime", {value: daily}, {value: in `.trim(); export async function getStatus(): Promise { - const configSnapshot = await localCache.getCurrentSnapshot(); - const users = localCache.getUsers(configSnapshot); - const presets = localCache.getPresets(configSnapshot); - const ui = localCache.getUiConfig(configSnapshot); + const [configSnapshot, presetCount] = await Promise.all([ + getRevision(), + collections.presets.countDocuments(), + ]); + const users = getUsers(configSnapshot); + const ui = getUiConfig(configSnapshot); return { users: !Object.keys(users).length, - presets: !presets.length, + presets: !presetCount, filters: !Object.keys(ui["filters"]).length, device: !Object.keys(ui["device"]).length, index: !Object.keys(ui["index"]).length, @@ -395,7 +404,7 @@ export async function seed(options: Record): Promise { if (resources["permissions"]) { for (const p of resources["permissions"]) { p["_id"] = `${p["role"]}:${p["resource"]}:${p["access"]}`; - proms.push(db.putPermission(p["_id"], p)); + proms.push(putPermission(p["_id"], p)); } } @@ -406,21 +415,21 @@ export async function seed(options: Record): Promise { u["roles"] = (u["roles"] || []).join(","); u["_id"] = u["username"]; delete u["username"]; - proms.push(db.putUser(u["_id"], u)); + proms.push(putUser(u["_id"], u)); } } if (resources["provisions"]) { for (const p of resources["provisions"]) - proms.push(db.putProvision(p["_id"], p)); + proms.push(putProvision(p["_id"], p)); } if (resources["presets"]) - for (const p of resources["presets"]) proms.push(db.putPreset(p["_id"], p)); + for (const p of resources["presets"]) proms.push(putPreset(p["_id"], p)); if (resources["config"]) - for (const c of resources["config"]) proms.push(db.putConfig(c["_id"], c)); + for (const c of resources["config"]) proms.push(putConfig(c["_id"], c)); await proms; - return del("presets_hash"); + await Promise.all([del("ui-local-cache-hash"), del("cwmp-local-cache-hash")]); } diff --git a/lib/local-cache.ts b/lib/local-cache.ts index d620a8e..1bb67da 100644 --- a/lib/local-cache.ts +++ b/lib/local-cache.ts @@ -17,534 +17,75 @@ * along with GenieACS. If not, see . */ -import * as vm from "vm"; -import * as crypto from "crypto"; -import * as config from "./config"; -import * as db from "./db"; -import * as cache from "./cache"; -import { convertOldPrecondition } from "./mongodb-functions"; -import * as logger from "./logger"; -import * as scheduling from "./scheduling"; -import * as expression from "./common/expression/util"; -import { parse } from "./common/expression/parser"; -import { - Preset, - Expression, - Provisions, - VirtualParameters, - Files, - Users, - Permissions, - Config, - UiConfig, -} from "./types"; - -interface Snapshot { - presets: Preset[]; - provisions: Provisions; - virtualParameters: VirtualParameters; - files: Files; - permissions: Permissions; - users: Users; - config: Config; - ui: UiConfig; -} +import { setTimeoutPromise } from "./util"; +import { acquireLock, get, releaseLock, set } from "./cache"; const REFRESH = 5000; const EVICT_TIMEOUT = 120000; -const snapshots = new Map(); -let currentSnapshot: string = null; -let nextRefresh = 1; +export class LocalCache { + private nextRefresh = 1; + private currentRevision: string = null; + private snapshots: Map = new Map(); -function computeHash(snapshot): string { - // MD5 hash for presets, provisions, virtual parameters for detecting changes - const h = crypto.createHash("md5"); - for (const p of snapshot.presets) { - h.update(JSON.stringify(p.name)); - h.update(JSON.stringify(p.channel)); - h.update(JSON.stringify(p.schedule)); - h.update(JSON.stringify(p.events)); - h.update(JSON.stringify(p.precondition)); - h.update(JSON.stringify(p.provisions)); + constructor( + private cacheKey: string, + private callback: () => Promise<[string, T]> + ) {} + + async getRevision(): Promise { + if (Date.now() > this.nextRefresh) await this.refresh(); + return this.currentRevision; } - let keys; - - keys = Object.keys(snapshot.provisions).sort(); - h.update(JSON.stringify(keys)); - for (const k of keys) h.update(snapshot.provisions[k].md5); - - keys = Object.keys(snapshot.virtualParameters).sort(); - h.update(JSON.stringify(keys)); - for (const k of keys) h.update(snapshot.virtualParameters[k].md5); - - keys = Object.keys(snapshot.config).sort(); - h.update(JSON.stringify(keys)); - for (const k of keys) h.update(JSON.stringify(snapshot.config[k])); - - keys = Object.keys(snapshot.files).sort(); - h.update(JSON.stringify(keys)); - for (const k of keys) h.update(JSON.stringify(snapshot.files[k])); - - keys = Object.keys(snapshot.users).sort(); - h.update(JSON.stringify(keys)); - for (const k of keys) h.update(JSON.stringify(snapshot.users[k])); - - const roles = Object.keys(snapshot.permissions).sort(); - h.update(JSON.stringify(roles)); - for (const r of roles) { - const levels = Object.keys(snapshot.permissions[r]).sort(); - h.update(JSON.stringify(levels)); - for (const l of levels) { - keys = Object.keys(snapshot.permissions[r][l]).sort(); - h.update(JSON.stringify(keys)); - for (const k of keys) - h.update(JSON.stringify(snapshot.permissions[r][l][k])); - } + hasRevision(revision: string): boolean { + return this.snapshots.has(revision); } - return h.digest("hex"); -} - -function flattenObject( - src: Record, - prefix = "", - dst = {} -): Record { - for (const k of Object.keys(src)) { - const v = src[k]; - if (typeof v === "object" && !Array.isArray(v)) - flattenObject(v as Record, `${prefix}${k}.`, dst); - else dst[`${prefix}${k}`] = v; + get(revision: string): T { + const snapshot = this.snapshots.get(revision); + if (!snapshot) throw new Error("Cache snapshot does not exist"); + return snapshot; } - return dst; -} -async function fetchPresets(): Promise { - const res = await db.getPresets(); - let objects = await db.getObjects(); - - objects = objects.map((obj) => { - // Flatten object - obj = flattenObject(obj); - - // If no keys are defined, consider all parameters as keys to keep the - // same behavior from v1.0 - if (!obj["_keys"]?.length) - obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_")); - - return obj; - }); - - res.sort((a, b) => { - if (a["weight"] === b["weight"]) - return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0; - else return a["weight"] - b["weight"]; - }); - - const presets = [] as Preset[]; - for (const preset of res) { - let schedule: { md5: string; duration: number; schedule: any } = null; - if (preset["schedule"]) { - const parts = preset["schedule"].trim().split(/\s+/); - schedule = { - md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"), - duration: null, - schedule: null, - }; - - try { - schedule.duration = +parts.shift() * 1000; - schedule.schedule = scheduling.parseCron(parts.join(" ")); - } catch (err) { - logger.warn({ - message: "Invalid preset schedule", - preset: preset["_id"], - schedule: preset["schedule"], - }); - schedule.schedule = false; - } + async refresh(): Promise { + if (!this.nextRefresh) { + await setTimeoutPromise(20); + await this.refresh(); + return; } - const events = preset["events"] || {}; - let precondition = true as Expression; - if (preset["precondition"]) { - try { - precondition = parse(preset["precondition"]); - } catch (error) { - precondition = convertOldPrecondition( - JSON.parse(preset["precondition"]) - ); - } + const now = Date.now(); + if (now < this.nextRefresh) return; + this.nextRefresh = 0; - // Simplify expression - precondition = expression.evaluate(precondition); + const dbHash = await get(this.cacheKey); + + if (this.currentRevision && dbHash === this.currentRevision) { + this.nextRefresh = now + (REFRESH - (now % REFRESH)); + return; } - const _provisions = preset["provisions"] || []; + const lockToken = await acquireLock(this.cacheKey + "_lock", 5000); - // Generate provisions from the old configuration format - for (const c of preset["configurations"]) { - switch (c.type) { - case "age": - _provisions.push(["refresh", c.name, +c.age]); - break; + const [hash, snapshot] = await this.callback(); - case "value": - _provisions.push(["value", c.name, c.value]); - break; - - case "add_tag": - _provisions.push(["tag", c.tag, true]); - break; - - case "delete_tag": - _provisions.push(["tag", c.tag, false]); - break; - - case "provision": - _provisions.push([c.name].concat(c.args || [])); - break; - - case "add_object": - for (const obj of objects) { - if (obj["_id"] === c.object) { - const alias = obj["_keys"] - .map((k) => `${k}:${JSON.stringify(obj[k])}`) - .join(","); - const p = `${c.name}.[${alias}]`; - _provisions.push(["instances", p, 1]); - - for (const k in obj) { - if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1)) - _provisions.push(["value", `${p}.${k}`, obj[k]]); - } - } - } - - break; - - case "delete_object": - for (const obj of objects) { - if (obj["_id"] === c.object) { - const alias = obj["_keys"] - .map((k) => `${k}:${JSON.stringify(obj[k])}`) - .join(","); - const p = `${c.name}.[${alias}]`; - _provisions.push(["instances", p, 0]); - } - } - - break; - - default: - throw new Error(`Unknown configuration type ${c.type}`); - } + if (this.currentRevision) { + const r = this.currentRevision; + const s = this.snapshots.get(r); + setTimeout(() => { + if (this.snapshots.get(r) === s) this.snapshots.delete(r); + }, EVICT_TIMEOUT).unref(); } - presets.push({ - name: preset["_id"], - channel: (preset["channel"] as string) || "default", - schedule: schedule, - events: events, - precondition: precondition, - provisions: _provisions, - }); - } + this.currentRevision = hash; + this.snapshots.set(hash, snapshot); - return presets; -} - -async function fetchProvisions(): Promise { - const res = await db.getProvisions(); - - const provisions = {}; - for (const r of res) { - provisions[r["_id"]] = {}; - provisions[r["_id"]].md5 = crypto - .createHash("md5") - .update(r["script"]) - .digest("hex"); - provisions[r["_id"]].script = new vm.Script( - `"use strict";(function(){\n${r["script"]}\n})();`, - { filename: r["_id"], lineOffset: -1, timeout: 50 } - ); - } - - return provisions; -} - -async function fetchVirtualParameters(): Promise { - const res = await db.getVirtualParameters(); - - const virtualParameters = {}; - for (const r of res) { - virtualParameters[r["_id"]] = {}; - virtualParameters[r["_id"]].md5 = crypto - .createHash("md5") - .update(r["script"]) - .digest("hex"); - virtualParameters[r["_id"]].script = new vm.Script( - `"use strict";(function(){\n${r["script"]}\n})();`, - { filename: r["_id"], lineOffset: -1, timeout: 50 } - ); - } - - return virtualParameters; -} - -async function fetchPermissions(): Promise { - const perms = await db.getPermissions(); - const permissions: Permissions = {}; - - for (const p of perms) { - if (!permissions[p.role]) permissions[p.role] = {}; - if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {}; - - permissions[p.role][p.access][p.resource] = { - access: p.access, - filter: parse(p.filter || "true"), - }; - if (p.validate) - permissions[p.role][p.access][p.resource].validate = parse(p.validate); - } - - return permissions; -} - -async function fetchFiles(): Promise { - const res = await db.getFiles(); - const files = {}; - - for (const r of res) { - const id = r["filename"] || r["_id"].toString(); - files[id] = {}; - files[id].length = r["length"]; - } - - return files; -} - -async function fetchUsers(): Promise { - const _users = await db.getUsers(); - const users = {}; - - for (const user of _users) { - users[user._id] = { - password: user.password, - salt: user.salt, - roles: user.roles.split(",").map((s) => s.trim()), - }; - } - - return users; -} - -async function fetchConfig(): Promise<[Config, UiConfig]> { - const conf = await db.getConfig(); - - conf.sort((a, b) => (a.id > b.id ? 1 : a.id < b.id ? -1 : 0)); - - const ui = { - filters: {}, - device: {}, - index: {}, - overview: {}, - pageSize: null, - }; - const _config = {}; - - for (const c of conf) { - // Evaluate expressions to simplify them - const val = expression.evaluate(c.value); - _config[c.id] = val; - if (c.id.startsWith("ui.")) { - const keys = c.id.split("."); - if (!(keys[1] in ui)) continue; - // remove the first key(ui) - keys.shift(); - let ref = ui; - while (keys.length > 1) { - const k = keys.shift(); - if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {}; - ref = ref[k]; - } - ref[keys[0]] = val; + if (lockToken) { + if (hash !== dbHash) await set(this.cacheKey, hash, 300); + await releaseLock(this.cacheKey + "_lock", lockToken); } + + this.nextRefresh = now + (REFRESH - (now % REFRESH)); } - - return [_config, ui]; -} - -async function refresh(): Promise { - if (!nextRefresh) { - await new Promise((resolve) => setTimeout(resolve, 20)); - return refresh(); - } - - const now = Date.now(); - - if (now < nextRefresh) return; - - nextRefresh = 0; - - const dbHash = await cache.get("presets_hash"); - - if (currentSnapshot && dbHash === currentSnapshot) { - nextRefresh = now + (REFRESH - (now % REFRESH)); - return; - } - - const lockToken = await cache.acquireLock("presets_hash_lock", 5000); - - const res = await Promise.all([ - fetchPresets(), - fetchProvisions(), - fetchVirtualParameters(), - fetchFiles(), - fetchPermissions(), - fetchUsers(), - fetchConfig(), - ]); - - const snapshot = { - presets: res[0], - provisions: res[1], - virtualParameters: res[2], - files: res[3], - permissions: res[4], - users: res[5], - config: res[6][0], - ui: res[6][1], - }; - - if (currentSnapshot) { - const h = currentSnapshot; - const s = snapshots.get(h); - setTimeout(() => { - if (snapshots.get(h) === s) snapshots.delete(h); - }, EVICT_TIMEOUT).unref(); - } - - currentSnapshot = computeHash(snapshot); - snapshots.set(currentSnapshot, snapshot); - - if (lockToken) { - if (currentSnapshot !== dbHash) - await cache.set("presets_hash", currentSnapshot, 300); - await cache.releaseLock("presets_hash_lock", lockToken); - } - - nextRefresh = now + (REFRESH - (now % REFRESH)); -} - -export async function getCurrentSnapshot(): Promise { - if (Date.now() > nextRefresh) await refresh(); - return currentSnapshot; -} - -export function hasSnapshot(hash: string): boolean { - return snapshots.has(hash); -} - -export function getPresets(snapshotKey: string): Preset[] { - const snapshot = snapshots.get(snapshotKey); - if (!snapshot) throw new Error("Cache snapshot does not exist"); - return snapshot.presets; -} - -export function getProvisions(snapshotKey: string): Provisions { - const snapshot = snapshots.get(snapshotKey); - if (!snapshot) throw new Error("Cache snapshot does not exist"); - return snapshot.provisions; -} - -export function getVirtualParameters(snapshotKey: string): VirtualParameters { - const snapshot = snapshots.get(snapshotKey); - if (!snapshot) throw new Error("Cache snapshot does not exist"); - return snapshot.virtualParameters; -} - -export function getFiles(snapshotKey: string): Files { - const snapshot = snapshots.get(snapshotKey); - if (!snapshot) throw new Error("Cache snapshot does not exist"); - return snapshot.files; -} - -export function getConfig( - snapshotKey: string, - key: string, - context: Record, - now: number, - cb?: (e: Expression) => Expression -): string | number | boolean | null { - const snapshot = snapshots.get(snapshotKey); - if (!snapshot) throw new Error("Cache snapshot does not exist"); - - const oldOpts = { - "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT", - "cwmp.debug": "DEBUG", - "cwmp.retryDelay": "RETRY_DELAY", - "cwmp.sessionTimeout": "SESSION_TIMEOUT", - "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT", - "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL", - "cwmp.gpvBatchSize": "GPV_BATCH_SIZE", - "cwmp.cookiesPath": "COOKIES_PATH", - "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS", - "cwmp.booleanLiteral": "BOOLEAN_LITERAL", - "cwmp.connectionRequestAllowBasicAuth": - "CONNECTION_REQUEST_ALLOW_BASIC_AUTH", - "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS", - "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD", - "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT", - }; - - if (!(key in snapshot.config)) { - if (key in oldOpts) { - let id; - if (context?.["id"]) { - id = context["id"]; - } else if (cb) { - id = cb(["PARAM", "DeviceID.ID"]); - if (Array.isArray(id)) id = null; - } - return config.get(oldOpts[key], id); - } - return null; - } - - const v = expression.evaluate(snapshot.config[key], context, now, cb); - return Array.isArray(v) ? null : v; -} - -export function getConfigExpression( - snapshotKey: string, - key: string -): Expression { - const snapshot = snapshots.get(snapshotKey); - if (!snapshot) throw new Error("Cache snapshot does not exist"); - - return snapshot.config[key]; -} - -export function getUsers(snapshotKey: string): Users { - const snapshot = snapshots.get(snapshotKey); - if (!snapshot) throw new Error("Cache snapshot does not exist"); - - return snapshot.users; -} - -export function getPermissions(snapshotKey: string): Permissions { - const snapshot = snapshots.get(snapshotKey); - if (!snapshot) throw new Error("Cache snapshot does not exist"); - - return snapshot.permissions; -} - -export function getUiConfig(snapshotKey: string): UiConfig { - const snapshot = snapshots.get(snapshotKey); - if (!snapshot) throw new Error("Cache snapshot does not exist"); - - return snapshot.ui; } diff --git a/lib/nbi.ts b/lib/nbi.ts index cf2bf69..d7adf8c 100644 --- a/lib/nbi.ts +++ b/lib/nbi.ts @@ -17,10 +17,11 @@ * along with GenieACS. If not, see . */ -import { Collection, GridFSBucket, ObjectId } from "mongodb"; +import { Collection, ObjectId } from "mongodb"; import * as vm from "vm"; import * as config from "./config"; -import { onConnect, optimizeProjection, collections } from "./db"; +import { filesBucket, collections } from "./db/db"; +import { optimizeProjection } from "./db/util"; import * as query from "./query"; import * as apiFunctions from "./api-functions"; import { IncomingMessage, ServerResponse } from "http"; @@ -28,7 +29,7 @@ import * as cache from "./cache"; import { version as VERSION } from "../package.json"; import { ping } from "./ping"; import * as logger from "./logger"; -import { flattenDevice } from "./mongodb-functions"; +import { flattenDevice } from "./ui/db"; import { getRequestOrigin } from "./forwarded"; const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/; @@ -46,12 +47,6 @@ const VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/; const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/; -let filesBucket: GridFSBucket; - -onConnect(async (db) => { - filesBucket = new GridFSBucket(db); -}); - async function getBody(request: IncomingMessage): Promise { const chunks: Buffer[] = []; let readableEnded = false; @@ -112,12 +107,12 @@ async function handler( await collections.presets.replaceOne({ _id: presetName }, preset, { upsert: true, }); - await cache.del("presets_hash"); + await cache.del("cwmp-local-cache-hash"); response.writeHead(200); response.end(); } else if (request.method === "DELETE") { await collections.presets.deleteOne({ _id: presetName }); - await cache.del("presets_hash"); + await cache.del("cwmp-local-cache-hash"); response.writeHead(200); response.end(); } else { @@ -139,12 +134,12 @@ async function handler( await collections.objects.replaceOne({ _id: objectName }, object, { upsert: true, }); - await cache.del("presets_hash"); + await cache.del("cwmp-local-cache-hash"); response.writeHead(200); response.end(); } else if (request.method === "DELETE") { await collections.objects.deleteOne({ _id: objectName }); - await cache.del("presets_hash"); + await cache.del("cwmp-local-cache-hash"); response.writeHead(200); response.end(); } else { @@ -172,12 +167,12 @@ async function handler( await collections.provisions.replaceOne({ _id: provisionName }, object, { upsert: true, }); - await cache.del("presets_hash"); + await cache.del("cwmp-local-cache-hash"); response.writeHead(200); response.end(); } else if (request.method === "DELETE") { await collections.provisions.deleteOne({ _id: provisionName }); - await cache.del("presets_hash"); + await cache.del("cwmp-local-cache-hash"); response.writeHead(200); response.end(); } else { @@ -207,14 +202,14 @@ async function handler( object, { upsert: true } ); - await cache.del("presets_hash"); + await cache.del("cwmp-local-cache-hash"); response.writeHead(200); response.end(); } else if (request.method === "DELETE") { await collections.virtualParameters.deleteOne({ _id: virtualParameterName, }); - await cache.del("presets_hash"); + await cache.del("cwmp-local-cache-hash"); response.writeHead(200); response.end(); } else { diff --git a/lib/session.ts b/lib/session.ts index f548c9d..b6ec7c8 100644 --- a/lib/session.ts +++ b/lib/session.ts @@ -19,7 +19,7 @@ import * as device from "./device"; import * as sandbox from "./sandbox"; -import * as localCache from "./local-cache"; +import * as localCache from "./cwmp/local-cache"; import * as defaultProvisions from "./default-provisions"; import { estimateGpnCount } from "./gpn-heuristic"; import Path from "./common/path"; @@ -3004,7 +3004,7 @@ export async function deserialize( sessionContext.deviceData = deviceData; // Ensure cache is populated - await localCache.getCurrentSnapshot(); + await localCache.getRevision(); return sessionContext; } diff --git a/lib/ui.ts b/lib/ui.ts index 588a0aa..832800e 100644 --- a/lib/ui.ts +++ b/lib/ui.ts @@ -29,9 +29,9 @@ import * as config from "./config"; import api from "./ui/api"; import Authorizer from "./common/authorizer"; import * as logger from "./logger"; -import * as localCache from "./local-cache"; +import * as localCache from "./ui/local-cache"; import { PermissionSet } from "./types"; -import { authLocal } from "./ui/api-functions"; +import { authLocal } from "./api-functions"; import * as init from "./init"; import { version as VERSION } from "../package.json"; import memoize from "./common/memoize"; @@ -68,7 +68,7 @@ koa.on("error", (err, ctx) => { }); koa.use(async (ctx, next) => { - const configSnapshot = await localCache.getCurrentSnapshot(); + const configSnapshot = await localCache.getRevision(); ctx.state.configSnapshot = configSnapshot; ctx.set("X-Config-Snapshot", configSnapshot); ctx.set("GenieACS-Version", VERSION); diff --git a/lib/ui/api-functions.ts b/lib/ui/api-functions.ts deleted file mode 100644 index 140732b..0000000 --- a/lib/ui/api-functions.ts +++ /dev/null @@ -1,211 +0,0 @@ -/** - * Copyright 2013-2019 GenieACS Inc. - * - * This file is part of GenieACS. - * - * GenieACS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * GenieACS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with GenieACS. If not, see . - */ - -import { ObjectId } from "mongodb"; -import * as db from "./db"; -import { del } from "../cache"; -import { getUsers } from "../local-cache"; -import { hashPassword } from "../auth"; -import { - insertTasks, - awaitSessionEnd, - awaitSessionStart, - connectionRequest, - deleteDevice, -} from "../api-functions"; -import { Task } from "../types"; - -async function deleteFault(id): Promise { - const deviceId = id.split(":", 1)[0]; - const channel = id.slice(deviceId.length + 1); - - await Promise.all([ - db.deleteFault(id), - channel.startsWith("task_") - ? db.deleteTask(new ObjectId(channel.slice(5))) - : null, - ]); - - await del(`${deviceId}_tasks_faults_operations`); -} - -export async function deleteResource( - resource: string, - id: string -): Promise { - switch (resource) { - case "devices": - await deleteDevice(id); - break; - - case "files": - await db.deleteFile(id); - break; - - case "faults": - await deleteFault(id); - break; - - case "provisions": - await db.deleteProvision(id); - break; - - case "presets": - await db.deletePreset(id); - break; - - case "virtualParameters": - await db.deleteVirtualParameter(id); - break; - - case "config": - await db.deleteConfig(id); - break; - - case "permissions": - await db.deletePermission(id); - break; - - case "users": - await db.deleteUser(id); - break; - } - - await del("presets_hash"); -} - -export async function postTasks( - deviceId: string, - tasks: Task[], - timeout: number, - device: Record -): Promise<{ connectionRequest: string; tasks: any[] }> { - for (const task of tasks) { - delete task._id; - task["device"] = deviceId; - } - - tasks = await insertTasks(tasks); - const statuses = tasks.map((t) => { - return { _id: t._id, status: "pending" }; - }); - const lastInform = Date.now(); - const notInSession = await awaitSessionEnd(deviceId, 30000); - if (!notInSession) { - await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id)))); - return { - connectionRequest: "Session took too long to complete", - tasks: statuses, - }; - } - - await del(`${deviceId}_tasks_faults_operations`); - - const status = await connectionRequest(deviceId, device); - - if (status) { - await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id)))); - return { - connectionRequest: status, - tasks: statuses, - }; - } - - const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout); - if (!sessionStarted) { - await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id)))); - return { - connectionRequest: "No contact from CPE", - tasks: statuses, - }; - } - - const sessionEnded = await awaitSessionEnd(deviceId, 120000); - if (!sessionEnded) { - await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id)))); - return { - connectionRequest: "Session took too long to complete", - tasks: statuses, - }; - } - - const promises: Promise[] = []; - for (const s of statuses) { - promises.push(db.count("tasks", ["=", ["PARAM", "_id"], s._id])); - promises.push( - db.count("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`]) - ); - } - - const res = await Promise.all(promises); - for (const [i, r] of statuses.entries()) { - if (!res[i * 2]) { - r.status = "done"; - } else if (res[i * 2 + 1]) { - r.status = "fault"; - r["fault"] = res[i * 2 + 1][0]; - } - } - - await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id)))); - - return { connectionRequest: "OK", tasks: statuses }; -} - -export async function putResource( - resource: string, - id: string, - data: Record -): Promise { - if (resource === "presets") { - await db.putPreset(id, data); - } else if (resource === "provisions") { - await db.putProvision(id, data); - } else if (resource === "virtualParameters") { - await db.putVirtualParameter(id, data); - } else if (resource === "config") { - await db.putConfig(id, data); - } else if (resource === "permissions") { - await db.putPermission(id, data); - } else if (resource === "users") { - delete data["password"]; - delete data["salt"]; - await db.putUser(id, data); - } - - await del("presets_hash"); -} - -export function authLocal( - snapshot: string, - username: string, - password: string -): Promise { - return new Promise((resolve, reject) => { - const users = getUsers(snapshot); - const user = users[username]; - if (!user?.password) return void resolve(null); - hashPassword(password, user.salt) - .then((hash) => { - if (hash === user.password) resolve(true); - else resolve(false); - }) - .catch(reject); - }); -} diff --git a/lib/ui/api.ts b/lib/ui/api.ts index 1395c1b..312c436 100644 --- a/lib/ui/api.ts +++ b/lib/ui/api.ts @@ -20,17 +20,17 @@ import { PassThrough } from "stream"; import Router from "koa-router"; import * as db from "./db"; -import * as apiFunctions from "./api-functions"; +import * as apiFunctions from "../api-functions"; import { evaluate, and, extractParams } from "../common/expression/util"; import { parse } from "../common/expression/parser"; import * as logger from "../logger"; -import { getConfig } from "../local-cache"; -import { QueryOptions, Expression } from "../types"; +import { getConfig } from "../ui/local-cache"; +import { Expression } from "../types"; import { generateSalt, hashPassword } from "../auth"; import { del } from "../cache"; import Authorizer from "../common/authorizer"; import { ping } from "../ping"; -import { decodeTag } from "../common"; +import { decodeTag } from "../util"; import { stringify as yamlStringify } from "../common/yaml"; const router = new Router(); @@ -178,7 +178,7 @@ for (const [resource, flags] of Object.entries(resources)) { router.get(`/${resource}`, async (ctx) => { const authorizer: Authorizer = ctx.state.authorizer; - const options: QueryOptions = {}; + const options: Parameters[2] = {}; let filter: Expression = authorizer.getFilter(resource, 2); if (ctx.request.query.filter) filter = and(filter, parse(singleParam(ctx.request.query.filter))); @@ -231,7 +231,7 @@ for (const [resource, flags] of Object.entries(resources)) { // CSV download router.get(`/${resource}.csv`, async (ctx) => { const authorizer: Authorizer = ctx.state.authorizer; - const options: QueryOptions = { projection: {} }; + const options: Parameters[2] = { projection: {} }; let filter: Expression = authorizer.getFilter(resource, 2); if (ctx.request.query.filter) filter = and(filter, parse(singleParam(ctx.request.query.filter))); @@ -712,7 +712,7 @@ router.put("/users/:id/password", async (ctx) => { const password = await hashPassword(newPassword, salt); await db.putUser(username, { password, salt }); - await del("presets_hash"); + await del("ui-local-cache-hash"); logger.accessInfo(log); ctx.body = ""; diff --git a/lib/ui/db.ts b/lib/ui/db.ts index e960b6c..c8ab83d 100644 --- a/lib/ui/db.ts +++ b/lib/ui/db.ts @@ -17,19 +17,362 @@ * along with GenieACS. If not, see . */ -import { Collection, GridFSBucket, ObjectId } from "mongodb"; -import { Script } from "vm"; -import { collections, onConnect, optimizeProjection } from "../db"; -import * as mongodbFunctions from "../mongodb-functions"; +import { encodeTag } from "../util"; import { evaluate } from "../common/expression/util"; -import { QueryOptions, Expression } from "../types"; +import { Expression, Fault, Task } from "../types"; +import { collections, filesBucket } from "../db/db"; +import { convertOldPrecondition, optimizeProjection } from "../db/util"; +import * as MongoTypes from "../db/types"; +import { parse, parseList, stringify } from "../common/expression/parser"; +import { toMongoQuery } from "../db/synth"; +import { Collection, ObjectId, WithoutId } from "mongodb"; +import { Script } from "vm"; import { Readable } from "stream"; -let filesBucket: GridFSBucket; +function processDeviceProjection( + projection: Record +): Record { + if (!projection) return projection; + const p = {}; + for (const [k, v] of Object.entries(projection)) { + if (k === "DeviceID.ID") { + p["_id"] = 1; + } else if (k.startsWith("DeviceID")) { + p["_deviceId._SerialNumber"] = v; + p["_deviceId._OUI"] = v; + p["_deviceId._ProductClass"] = v; + p["_deviceId._Manufacturer"] = v; + } else if (k.startsWith("Tags")) { + p["_tags"] = v; + } else if (k.startsWith("Events")) { + p["_lastInform"] = v; + p["_registered"] = v; + p["_lastBoot"] = v; + p["_lastBootstrap"] = v; + } else { + p[k] = v; + } + } -onConnect(async (db) => { - filesBucket = new GridFSBucket(db); -}); + return p; +} + +function processDeviceSort( + sort: Record +): Record { + if (!sort) return sort; + const s = {}; + for (const [k, v] of Object.entries(sort)) { + if (k === "DeviceID.ID") s["_id"] = v; + else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v; + else if (k === "Events.Inform") s["_lastInform"] = v; + else if (k === "Events.Registered") s["_registered"] = v; + else if (k === "Events.1_BOOT") s["_lastBoot"] = v; + else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v; + else s[`${k}._value`] = v; + } + + return s; +} + +function parseDate(d: Date): number | string { + const n = +d; + return isNaN(n) ? "" + d : n; +} + +interface FlatAttributes { + object?: boolean; + objectTimestamp?: number; + writable?: boolean; + writableTimestamp?: number; + value?: [string | number | boolean, string]; + valueTimestamp?: number; + notification?: number; + notificationTimestamp?: number; + accessList?: string[]; + accessListTimestamp?: number; +} + +interface FlatDevice { + [param: string]: FlatAttributes; +} + +export function flattenDevice(device: Record): FlatDevice { + function recursive( + input, + root: string, + output: FlatDevice, + timestamp: number + ): void { + for (const [name, tree] of Object.entries(input)) { + if (!root) { + if (name === "_lastInform") { + output["Events.Inform"] = { + value: [parseDate(tree as Date), "xsd:dateTime"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp, + }; + } else if (name === "_registered") { + output["Events.Registered"] = { + value: [parseDate(tree as Date), "xsd:dateTime"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp, + }; + } else if (name === "_lastBoot") { + output["Events.1_BOOT"] = { + value: [parseDate(tree as Date), "xsd:dateTime"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp, + }; + } else if (name === "_lastBootstrap") { + output["Events.0_BOOTSTRAP"] = { + value: [parseDate(tree as Date), "xsd:dateTime"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp, + }; + } else if (name === "_id") { + output["DeviceID.ID"] = { + value: [tree as string, "xsd:string"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp, + }; + } else if (name === "_deviceId") { + output["DeviceID.Manufacturer"] = { + value: [tree["_Manufacturer"], "xsd:string"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp, + }; + output["DeviceID.OUI"] = { + value: [tree["_OUI"], "xsd:string"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp, + }; + output["DeviceID.ProductClass"] = { + value: [tree["_ProductClass"], "xsd:string"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp, + }; + output["DeviceID.SerialNumber"] = { + value: [tree["_SerialNumber"], "xsd:string"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp, + }; + } else if (name === "_tags") { + output["Tags"] = { + writable: false, + writableTimestamp: timestamp, + object: true, + objectTimestamp: timestamp, + }; + + for (const t of tree as string[]) { + output[`Tags.${encodeTag(t)}`] = { + value: [true, "xsd:boolean"], + valueTimestamp: timestamp, + writable: true, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp, + }; + } + } + } + + if (name.startsWith("_")) continue; + + let childrenTimestamp = timestamp; + + if (!root) childrenTimestamp = +(input["_timestamp"] || 1); + else if (+input["_timestamp"] > timestamp) + childrenTimestamp = +input["_timestamp"]; + + const attrs: FlatAttributes = {}; + if (tree["_value"] != null) { + attrs.value = [ + tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"], + tree["_type"], + ]; + attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp); + attrs.object = false; + attrs.objectTimestamp = childrenTimestamp; + } else if (tree["_object"] != null) { + attrs.object = tree["_object"]; + attrs.objectTimestamp = childrenTimestamp; + } + + if (tree["_writable"] != null) { + attrs.writable = tree["_writable"]; + attrs.writableTimestamp = childrenTimestamp; + } + + if (tree["_notification"] != null) { + attrs.notification = tree["_notification"]; + attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1; + } + + if (tree["_accessList"] != null) { + attrs.accessList = tree["_accessList"]; + attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1; + } + + const r = root ? `${root}.${name}` : name; + output[r] = attrs; + + if (attrs.object || tree["object"] == null) + recursive(tree, r, output, childrenTimestamp); + } + } + + const newDevice: FlatDevice = {}; + const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime(); + recursive(device, "", newDevice, timestamp); + return newDevice; +} + +function flattenFault(fault: unknown): Fault { + const f = Object.assign({}, fault) as Fault; + if (f.timestamp) f.timestamp = +f.timestamp; + if (f["expiry"]) f["expiry"] = +f["expiry"]; + return f as Fault; +} + +function flattenTask(task: unknown): Task { + const t = Object.assign({}, task) as Task; + t._id = "" + t._id; + if (t["timestamp"]) t["timestamp"] = +t["timestamp"]; + if (t.expiry) t.expiry = +t.expiry; + return t; +} + +function flattenPreset( + preset: Record +): Record { + const p = Object.assign({}, preset); + if (p.precondition) { + try { + // Try parse to check expression validity + parse(p.precondition as string); + } catch (error) { + p.precondition = convertOldPrecondition( + JSON.parse(p.precondition as string) + ); + p.precondition = (p.precondition as string).length + ? stringify(p.precondition as Expression) + : ""; + } + } + + if (p.events) { + const e = []; + for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`); + p.events = e.join(", "); + } + + const provision = p.configurations[0]; + if ( + (p.configurations as any[]).length === 1 && + provision.type === "provision" && + provision.name && + provision.name.length + ) { + p.provision = provision.name; + p.provisionArgs = provision.args + ? provision.args.map((a) => stringify(a)).join(", ") + : ""; + } + + delete p.configurations; + return p; +} + +function flattenFile(file: Record): Record { + const f = {}; + f["_id"] = file["_id"]; + if (file.metadata) { + f["metadata.fileType"] = file["metadata"]["fileType"] || ""; + f["metadata.oui"] = file["metadata"]["oui"] || ""; + f["metadata.productClass"] = file["metadata"]["productClass"] || ""; + f["metadata.version"] = file["metadata"]["version"] || ""; + } + return f; +} + +function preProcessPreset(data: Record): MongoTypes.Preset { + const preset = Object.assign({}, data); + + if (!preset.precondition) preset.precondition = ""; + // Try parse to check expression validity + parse(preset.precondition as string); + + preset.weight = parseInt(preset.weight as string) || 0; + + const events = {}; + if (preset.events) { + for (let e of (preset.events as string).split(",")) { + let v = true; + e = e.trim(); + if (e.startsWith("-")) { + v = false; + e = e.slice(1).trim(); + } + if (e) events[e] = v; + } + } + + preset.events = events; + + if (!preset.provision) throw new Error("Invalid preset provision"); + + const configuration = { + type: "provision", + name: preset.provision, + args: null, + }; + + if (preset.provisionArgs) + configuration.args = parseList(preset.provisionArgs as string); + + delete preset.provision; + delete preset.provisionArgs; + preset.configurations = [configuration]; + return preset as unknown as MongoTypes.Preset; +} + +interface QueryOptions { + projection?: any; + skip?: number; + limit?: number; + sort?: { + [param: string]: number; + }; +} export async function* query( resource: string, @@ -38,7 +381,7 @@ export async function* query( ): AsyncGenerator { options = options || {}; filter = evaluate(filter, null, Date.now()); - const q = mongodbFunctions.toMongoQuery(filter, resource); + const q = toMongoQuery(filter, resource); if (!q) return; const collection = collections[resource] as Collection; @@ -46,7 +389,7 @@ export async function* query( if (options.projection) { let projection = options.projection; if (resource === "devices") - projection = mongodbFunctions.processDeviceProjection(options.projection); + projection = processDeviceProjection(options.projection); if (resource === "presets") projection.configurations = 1; projection = optimizeProjection(projection); @@ -67,25 +410,25 @@ export async function* query( {} ); - if (resource === "devices") s = mongodbFunctions.processDeviceSort(s); + if (resource === "devices") s = processDeviceSort(s); cursor.sort(s); } for await (let doc of cursor) { - if (resource === "devices") doc = mongodbFunctions.flattenDevice(doc); - else if (resource === "faults") doc = mongodbFunctions.flattenFault(doc); - else if (resource === "tasks") doc = mongodbFunctions.flattenTask(doc); - else if (resource === "presets") doc = mongodbFunctions.flattenPreset(doc); - else if (resource === "files") doc = mongodbFunctions.flattenFile(doc); + if (resource === "devices") doc = flattenDevice(doc); + else if (resource === "faults") doc = flattenFault(doc); + else if (resource === "tasks") doc = flattenTask(doc); + else if (resource === "presets") doc = flattenPreset(doc); + else if (resource === "files") doc = flattenFile(doc); yield doc; } } export function count(resource: string, filter: Expression): Promise { - const collection = collections[resource] as Collection; + const collection = collections[resource] as Collection; filter = evaluate(filter, null, Date.now()); - const q = mongodbFunctions.toMongoQuery(filter, resource); + const q = toMongoQuery(filter, resource); if (!q) return Promise.resolve(0); return collection.countDocuments(q); } @@ -110,34 +453,21 @@ export async function updateDeviceTags( await collections.devices.updateOne({ _id: deviceId }, object); } -async function putResource(resource, id, object): Promise { - const collection = collections[resource] as Collection; - await collection.replaceOne({ _id: id }, object, { upsert: true }); -} - -async function deleteResource( - resource: string, - id: string | ObjectId -): Promise { - const collection = collections[resource] as Collection; - await collection.deleteOne({ _id: id }); -} - -export function putPreset( +export async function putPreset( id: string, object: Record ): Promise { - object = mongodbFunctions.preProcessPreset(object); - return putResource("presets", id, object); + const p = preProcessPreset(object); + await collections.presets.replaceOne({ _id: id }, p, { upsert: true }); } -export function deletePreset(id: string): Promise { - return deleteResource("presets", id); +export async function deletePreset(id: string): Promise { + await collections.presets.deleteOne({ _id: id }); } -export function putProvision( +export async function putProvision( id: string, - object: Record + object: { script: string } ): Promise { if (!object.script) object.script = ""; try { @@ -153,16 +483,18 @@ export function putProvision( } return Promise.reject(err); } - return putResource("provisions", id, object); + await collections.provisions.replaceOne({ _id: id }, object, { + upsert: true, + }); } -export function deleteProvision(id: string): Promise { - return deleteResource("provisions", id); +export async function deleteProvision(id: string): Promise { + await collections.provisions.deleteOne({ _id: id }); } -export function putVirtualParameter( +export async function putVirtualParameter( id: string, - object: Record + object: { script: string } ): Promise { if (!object.script) object.script = ""; try { @@ -178,38 +510,42 @@ export function putVirtualParameter( } return Promise.reject(err); } - return putResource("virtualParameters", id, object); + await collections.virtualParameters.replaceOne({ _id: id }, object, { + upsert: true, + }); } -export function deleteVirtualParameter(id: string): Promise { - return deleteResource("virtualParameters", id); +export async function deleteVirtualParameter(id: string): Promise { + await collections.virtualParameters.deleteOne({ _id: id }); } -export function putConfig( +export async function putConfig( id: string, - object: Record + object: WithoutId ): Promise { - return putResource("config", id, object); + await collections.config.replaceOne({ _id: id }, object, { upsert: true }); } -export function deleteConfig(id: string): Promise { - return deleteResource("config", id); +export async function deleteConfig(id: string): Promise { + await collections.config.deleteOne({ _id: id }); } -export function putPermission( +export async function putPermission( id: string, - object: Record + object: WithoutId ): Promise { - return putResource("permissions", id, object); + await collections.permissions.replaceOne({ _id: id }, object, { + upsert: true, + }); } -export function deletePermission(id: string): Promise { - return deleteResource("permissions", id); +export async function deletePermission(id: string): Promise { + await collections.permissions.deleteOne({ _id: id }); } export async function putUser( id: string, - object: Record + object: Partial> ): Promise { // update instead of replace to keep the password if not set by user await collections.users.updateOne( @@ -219,8 +555,8 @@ export async function putUser( ); } -export function deleteUser(id: string): Promise { - return deleteResource("users", id); +export async function deleteUser(id: string): Promise { + await collections.users.deleteOne({ _id: id }); } export function downloadFile(filename: string): Readable { @@ -267,10 +603,10 @@ export async function deleteFile(filename: string): Promise { await filesBucket.delete(filename as any); } -export function deleteFault(id: string): Promise { - return deleteResource("faults", id); +export async function deleteFault(id: string): Promise { + await collections.faults.deleteOne({ _id: id }); } -export function deleteTask(id: ObjectId): Promise { - return deleteResource("tasks", id); +export async function deleteTask(id: ObjectId): Promise { + await collections.tasks.deleteOne({ _id: id }); } diff --git a/lib/ui/local-cache.ts b/lib/ui/local-cache.ts new file mode 100644 index 0000000..efd8d8d --- /dev/null +++ b/lib/ui/local-cache.ts @@ -0,0 +1,187 @@ +import * as crypto from "crypto"; +import * as config from "../config"; +import { collections } from "../db/db"; +import * as expression from "../common/expression/util"; +import { parse } from "../common/expression/parser"; +import { Expression, Users, Permissions, Config, UiConfig } from "../types"; +import { LocalCache } from "../local-cache"; + +interface Snapshot { + permissions: Permissions; + users: Users; + config: Config; + ui: UiConfig; +} + +async function fetchPermissions(): Promise<[string, Permissions]> { + const perms = await collections.permissions.find().toArray(); + perms.sort((a, b) => (a._id > b._id ? 1 : -1)); + const h = crypto + .createHash("md5") + .update(JSON.stringify(perms)) + .digest("hex"); + const permissions: Permissions = {}; + + for (const p of perms) { + if (!permissions[p.role]) permissions[p.role] = {}; + if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {}; + + permissions[p.role][p.access][p.resource] = { + access: p.access, + filter: parse(p.filter || "true"), + }; + if (p.validate) + permissions[p.role][p.access][p.resource].validate = parse(p.validate); + } + + return [h, permissions]; +} + +async function fetchUsers(): Promise<[string, Users]> { + const _users = await collections.users.find().toArray(); + _users.sort((a, b) => (a._id > b._id ? 1 : -1)); + const h = crypto + .createHash("md5") + .update(JSON.stringify(_users)) + .digest("hex"); + const users = {}; + + for (const user of _users) { + users[user._id] = { + password: user.password, + salt: user.salt, + roles: user.roles.split(",").map((s) => s.trim()), + }; + } + + return [h, users]; +} + +async function fetchConfig(): Promise<[string, Config, UiConfig]> { + const conf = await collections.config.find().toArray(); + conf.sort((a, b) => (a._id > b._id ? 1 : -1)); + const h = crypto.createHash("md5").update(JSON.stringify(conf)).digest("hex"); + + const ui = { + filters: {}, + device: {}, + index: {}, + overview: {}, + pageSize: null, + }; + + const _config = {}; + + for (const c of conf) { + // Evaluate expressions to simplify them + const val = expression.evaluate(parse(c.value)); + _config[c._id] = val; + if (c._id.startsWith("ui.")) { + const keys = c._id.split("."); + if (!(keys[1] in ui)) continue; + // remove the first key(ui) + keys.shift(); + let ref = ui; + while (keys.length > 1) { + const k = keys.shift(); + if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {}; + ref = ref[k]; + } + ref[keys[0]] = val; + } + } + + return [h, _config, ui]; +} + +const localCache = new LocalCache("ui-local-cache-hash", refresh); + +async function refresh(): Promise<[string, Snapshot]> { + const res = await Promise.all([ + fetchPermissions(), + fetchUsers(), + fetchConfig(), + ]); + + const h = crypto.createHash("md5"); + for (const r of res) h.update(r[0]); + + const snapshot = { + permissions: res[0][1], + users: res[1][1], + config: res[2][1], + ui: res[2][2], + }; + + return [h.digest("hex"), snapshot]; +} + +export async function getRevision(): Promise { + return await localCache.getRevision(); +} + +export function getConfig( + revision: string, + key: string, + context: Record, + now: number, + cb?: (e: Expression) => Expression +): string | number | boolean | null { + const snapshot = localCache.get(revision); + if (!snapshot) throw new Error("Cache snapshot does not exist"); + + const oldOpts = { + "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT", + "cwmp.debug": "DEBUG", + "cwmp.retryDelay": "RETRY_DELAY", + "cwmp.sessionTimeout": "SESSION_TIMEOUT", + "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT", + "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL", + "cwmp.gpvBatchSize": "GPV_BATCH_SIZE", + "cwmp.cookiesPath": "COOKIES_PATH", + "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS", + "cwmp.booleanLiteral": "BOOLEAN_LITERAL", + "cwmp.connectionRequestAllowBasicAuth": + "CONNECTION_REQUEST_ALLOW_BASIC_AUTH", + "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS", + "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD", + "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT", + }; + + if (!(key in snapshot.config)) { + if (key in oldOpts) { + let id; + if (context?.["id"]) { + id = context["id"]; + } else if (cb) { + id = cb(["PARAM", "DeviceID.ID"]); + if (Array.isArray(id)) id = null; + } + return config.get(oldOpts[key], id); + } + return null; + } + + const v = expression.evaluate(snapshot.config[key], context, now, cb); + return Array.isArray(v) ? null : v; +} + +export function getConfigExpression(revision: string, key: string): Expression { + const snapshot = localCache.get(revision); + return snapshot.config[key]; +} + +export function getUsers(revision: string): Users { + const snapshot = localCache.get(revision); + return snapshot.users; +} + +export function getPermissions(revision: string): Permissions { + const snapshot = localCache.get(revision); + return snapshot.permissions; +} + +export function getUiConfig(revision: string): UiConfig { + const snapshot = localCache.get(revision); + return snapshot.ui; +} diff --git a/lib/common.ts b/lib/util.ts similarity index 76% rename from lib/common.ts rename to lib/util.ts index 08e0947..ee9b46a 100644 --- a/lib/common.ts +++ b/lib/util.ts @@ -17,6 +17,8 @@ * along with GenieACS. If not, see . */ +import { EventEmitter } from "events"; + export function generateDeviceId( deviceIdStruct: Record ): string { @@ -61,3 +63,27 @@ export function encodeTag(tag: string): string { export function decodeTag(tag: string): string { return decodeURIComponent(tag.replace(/0x(?=[0-9A-Z]{2})/g, "%")); } + +export function once( + emitter: EventEmitter, + event: string, + timeout: number +): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`Event ${event} timed out after ${timeout} ms`)); + }, timeout); + + emitter.once(event, (...args: unknown[]) => { + clearTimeout(timer); + resolve(args); + }); + }); +} + +export function setTimeoutPromise(delay: number, ref = true): Promise { + return new Promise((resolve) => { + const timerId = setTimeout(resolve, delay); + if (!ref) timerId.unref(); + }); +} diff --git a/test/mongodb-functions.ts b/test/db.ts similarity index 92% rename from test/mongodb-functions.ts rename to test/db.ts index f35c09c..374bb51 100644 --- a/test/mongodb-functions.ts +++ b/test/db.ts @@ -1,8 +1,9 @@ import ava from "ava"; import { EJSON } from "bson"; import { Filter } from "mongodb"; -import * as mongodbFunctions from "../lib/mongodb-functions"; import { stringify, parse } from "../lib/common/expression/parser"; +import { convertOldPrecondition } from "../lib/db/util"; +import { toMongoQuery } from "../lib/db/synth"; ava("convertOldPrecondition", (t) => { const tests = [ @@ -53,20 +54,14 @@ ava("convertOldPrecondition", (t) => { t.plan(tests.length + 2 * shouldFailTests.length); for (const test of tests) { t.is( - stringify( - mongodbFunctions.convertOldPrecondition( - test[0] as Record - ) - ), + stringify(convertOldPrecondition(test[0] as Record)), test[1] ); } for (const test of shouldFailTests) { const func = (): void => { - mongodbFunctions.convertOldPrecondition( - test[0] as Record - ); + convertOldPrecondition(test[0] as Record); }; const error = t.throws(func, { instanceOf: Error }); t.is(error.message, test[1]); @@ -153,7 +148,7 @@ ava("toMongoQuery", async (t) => { for (const [expStr, expect] of queries) { const exp = parse(expStr); - let query = mongodbFunctions.toMongoQuery(exp, "devices"); + let query = toMongoQuery(exp, "devices"); if (query) query = EJSON.serialize(query); t.deepEqual(query, expect); } @@ -167,7 +162,7 @@ ava("toMongoQuery", async (t) => { for (const [expStr, err] of failQueries) { const exp = parse(expStr); - t.throws(() => mongodbFunctions.toMongoQuery(exp, "devices"), { + t.throws(() => toMongoQuery(exp, "devices"), { message: err, }); } diff --git a/test/common.ts b/test/util.ts similarity index 96% rename from test/common.ts rename to test/util.ts index c33e082..a704a0f 100644 --- a/test/common.ts +++ b/test/util.ts @@ -1,5 +1,5 @@ import ava from "ava"; -import * as common from "../lib/common"; +import * as common from "../lib/util"; ava("generateDeviceId", (t) => { const space = [" ", "%20"]; diff --git a/ui/components/tags.ts b/ui/components/tags.ts index 113225c..1dd0f04 100644 --- a/ui/components/tags.ts +++ b/ui/components/tags.ts @@ -22,7 +22,7 @@ import { m } from "../components"; import * as notifications from "../notifications"; import * as store from "../store"; import { getIcon } from "../icons"; -import { decodeTag } from "../../lib/common"; +import { decodeTag } from "../../lib/util"; const component: ClosureComponent = (): Component => { return { diff --git a/ui/filter-component.ts b/ui/filter-component.ts index cd28c47..2924eaf 100644 --- a/ui/filter-component.ts +++ b/ui/filter-component.ts @@ -22,7 +22,7 @@ import { parse, stringify, map } from "../lib/common/expression/parser"; import memoize from "../lib/common/memoize"; import Autocomplete from "./autocomplete-compnent"; import * as smartQuery from "./smart-query"; -import { validQuery } from "../lib/mongodb-functions"; +import { validQuery } from "../lib/db/synth"; import { Expression } from "../lib/types"; const getAutocomplete = memoize((resource) => { diff --git a/ui/smart-query.ts b/ui/smart-query.ts index eacdfb5..325eb53 100644 --- a/ui/smart-query.ts +++ b/ui/smart-query.ts @@ -19,7 +19,7 @@ import config from "./config"; import { Expression } from "../lib/types"; -import { encodeTag } from "../lib/common"; +import { encodeTag } from "../lib/util"; const resources = { devices: {}, diff --git a/ui/store.ts b/ui/store.ts index 8c977ce..5d77730 100644 --- a/ui/store.ts +++ b/ui/store.ts @@ -21,7 +21,7 @@ import m from "mithril"; import { stringify } from "../lib/common/expression/parser"; import { or, and, evaluate } from "../lib/common/expression/util"; import memoize from "../lib/common/memoize"; -import { QueryOptions, Expression, Task } from "../lib/types"; +import { Expression, Task } from "../lib/types"; import * as notifications from "./notifications"; import { configSnapshot, genieacsVersion } from "./config"; import { QueueTask } from "./task-queue"; @@ -342,7 +342,7 @@ function inferQuery(resourceType, queryResponse): void { export function fetch( resourceType: string, filter: Expression, - options: QueryOptions = {} + options: { limit?: number; sort?: { [param: string]: number } } = {} ): QueryResponse { const filterStr = memoizedStringify(filter); const sort = Object.assign({}, options.sort);