Refactor db and local-cache code
Separated cwmp and ui code into distinct files to enhance clarity and maintainability with varied internal data representations.
This commit is contained in:
Zaid Humaimidi committed
1 parent
493f1d2f37
commit
c3954c277f
31 files changed
+1542
-1530
No files matched your search
+48
-507
@@ -17,534 +17,75 @@
|
||||
* along with GenieACS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import * as vm from "vm";
|
||||
import * as crypto from "crypto";
|
||||
import * as config from "./config";
|
||||
import * as db from "./db";
|
||||
import * as cache from "./cache";
|
||||
import { convertOldPrecondition } from "./mongodb-functions";
|
||||
import * as logger from "./logger";
|
||||
import * as scheduling from "./scheduling";
|
||||
import * as expression from "./common/expression/util";
|
||||
import { parse } from "./common/expression/parser";
|
||||
import {
|
||||
Preset,
|
||||
Expression,
|
||||
Provisions,
|
||||
VirtualParameters,
|
||||
Files,
|
||||
Users,
|
||||
Permissions,
|
||||
Config,
|
||||
UiConfig,
|
||||
} from "./types";
|
||||
|
||||
interface Snapshot {
|
||||
presets: Preset[];
|
||||
provisions: Provisions;
|
||||
virtualParameters: VirtualParameters;
|
||||
files: Files;
|
||||
permissions: Permissions;
|
||||
users: Users;
|
||||
config: Config;
|
||||
ui: UiConfig;
|
||||
}
|
||||
import { setTimeoutPromise } from "./util";
|
||||
import { acquireLock, get, releaseLock, set } from "./cache";
|
||||
|
||||
const REFRESH = 5000;
|
||||
const EVICT_TIMEOUT = 120000;
|
||||
|
||||
const snapshots = new Map<string, Snapshot>();
|
||||
let currentSnapshot: string = null;
|
||||
let nextRefresh = 1;
|
||||
export class LocalCache<T> {
|
||||
private nextRefresh = 1;
|
||||
private currentRevision: string = null;
|
||||
private snapshots: Map<string, T> = new Map();
|
||||
|
||||
function computeHash(snapshot): string {
|
||||
// MD5 hash for presets, provisions, virtual parameters for detecting changes
|
||||
const h = crypto.createHash("md5");
|
||||
for (const p of snapshot.presets) {
|
||||
h.update(JSON.stringify(p.name));
|
||||
h.update(JSON.stringify(p.channel));
|
||||
h.update(JSON.stringify(p.schedule));
|
||||
h.update(JSON.stringify(p.events));
|
||||
h.update(JSON.stringify(p.precondition));
|
||||
h.update(JSON.stringify(p.provisions));
|
||||
constructor(
|
||||
private cacheKey: string,
|
||||
private callback: () => Promise<[string, T]>
|
||||
) {}
|
||||
|
||||
async getRevision(): Promise<string> {
|
||||
if (Date.now() > this.nextRefresh) await this.refresh();
|
||||
return this.currentRevision;
|
||||
}
|
||||
|
||||
let keys;
|
||||
|
||||
keys = Object.keys(snapshot.provisions).sort();
|
||||
h.update(JSON.stringify(keys));
|
||||
for (const k of keys) h.update(snapshot.provisions[k].md5);
|
||||
|
||||
keys = Object.keys(snapshot.virtualParameters).sort();
|
||||
h.update(JSON.stringify(keys));
|
||||
for (const k of keys) h.update(snapshot.virtualParameters[k].md5);
|
||||
|
||||
keys = Object.keys(snapshot.config).sort();
|
||||
h.update(JSON.stringify(keys));
|
||||
for (const k of keys) h.update(JSON.stringify(snapshot.config[k]));
|
||||
|
||||
keys = Object.keys(snapshot.files).sort();
|
||||
h.update(JSON.stringify(keys));
|
||||
for (const k of keys) h.update(JSON.stringify(snapshot.files[k]));
|
||||
|
||||
keys = Object.keys(snapshot.users).sort();
|
||||
h.update(JSON.stringify(keys));
|
||||
for (const k of keys) h.update(JSON.stringify(snapshot.users[k]));
|
||||
|
||||
const roles = Object.keys(snapshot.permissions).sort();
|
||||
h.update(JSON.stringify(roles));
|
||||
for (const r of roles) {
|
||||
const levels = Object.keys(snapshot.permissions[r]).sort();
|
||||
h.update(JSON.stringify(levels));
|
||||
for (const l of levels) {
|
||||
keys = Object.keys(snapshot.permissions[r][l]).sort();
|
||||
h.update(JSON.stringify(keys));
|
||||
for (const k of keys)
|
||||
h.update(JSON.stringify(snapshot.permissions[r][l][k]));
|
||||
}
|
||||
hasRevision(revision: string): boolean {
|
||||
return this.snapshots.has(revision);
|
||||
}
|
||||
|
||||
return h.digest("hex");
|
||||
}
|
||||
|
||||
function flattenObject(
|
||||
src: Record<string, unknown>,
|
||||
prefix = "",
|
||||
dst = {}
|
||||
): Record<string, unknown> {
|
||||
for (const k of Object.keys(src)) {
|
||||
const v = src[k];
|
||||
if (typeof v === "object" && !Array.isArray(v))
|
||||
flattenObject(v as Record<string, unknown>, `${prefix}${k}.`, dst);
|
||||
else dst[`${prefix}${k}`] = v;
|
||||
get(revision: string): T {
|
||||
const snapshot = this.snapshots.get(revision);
|
||||
if (!snapshot) throw new Error("Cache snapshot does not exist");
|
||||
return snapshot;
|
||||
}
|
||||
return dst;
|
||||
}
|
||||
|
||||
async function fetchPresets(): Promise<Preset[]> {
|
||||
const res = await db.getPresets();
|
||||
let objects = await db.getObjects();
|
||||
|
||||
objects = objects.map((obj) => {
|
||||
// Flatten object
|
||||
obj = flattenObject(obj);
|
||||
|
||||
// If no keys are defined, consider all parameters as keys to keep the
|
||||
// same behavior from v1.0
|
||||
if (!obj["_keys"]?.length)
|
||||
obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
|
||||
|
||||
return obj;
|
||||
});
|
||||
|
||||
res.sort((a, b) => {
|
||||
if (a["weight"] === b["weight"])
|
||||
return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
|
||||
else return a["weight"] - b["weight"];
|
||||
});
|
||||
|
||||
const presets = [] as Preset[];
|
||||
for (const preset of res) {
|
||||
let schedule: { md5: string; duration: number; schedule: any } = null;
|
||||
if (preset["schedule"]) {
|
||||
const parts = preset["schedule"].trim().split(/\s+/);
|
||||
schedule = {
|
||||
md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
|
||||
duration: null,
|
||||
schedule: null,
|
||||
};
|
||||
|
||||
try {
|
||||
schedule.duration = +parts.shift() * 1000;
|
||||
schedule.schedule = scheduling.parseCron(parts.join(" "));
|
||||
} catch (err) {
|
||||
logger.warn({
|
||||
message: "Invalid preset schedule",
|
||||
preset: preset["_id"],
|
||||
schedule: preset["schedule"],
|
||||
});
|
||||
schedule.schedule = false;
|
||||
}
|
||||
async refresh(): Promise<void> {
|
||||
if (!this.nextRefresh) {
|
||||
await setTimeoutPromise(20);
|
||||
await this.refresh();
|
||||
return;
|
||||
}
|
||||
|
||||
const events = preset["events"] || {};
|
||||
let precondition = true as Expression;
|
||||
if (preset["precondition"]) {
|
||||
try {
|
||||
precondition = parse(preset["precondition"]);
|
||||
} catch (error) {
|
||||
precondition = convertOldPrecondition(
|
||||
JSON.parse(preset["precondition"])
|
||||
);
|
||||
}
|
||||
const now = Date.now();
|
||||
if (now < this.nextRefresh) return;
|
||||
this.nextRefresh = 0;
|
||||
|
||||
// Simplify expression
|
||||
precondition = expression.evaluate(precondition);
|
||||
const dbHash = await get(this.cacheKey);
|
||||
|
||||
if (this.currentRevision && dbHash === this.currentRevision) {
|
||||
this.nextRefresh = now + (REFRESH - (now % REFRESH));
|
||||
return;
|
||||
}
|
||||
|
||||
const _provisions = preset["provisions"] || [];
|
||||
const lockToken = await acquireLock(this.cacheKey + "_lock", 5000);
|
||||
|
||||
// Generate provisions from the old configuration format
|
||||
for (const c of preset["configurations"]) {
|
||||
switch (c.type) {
|
||||
case "age":
|
||||
_provisions.push(["refresh", c.name, +c.age]);
|
||||
break;
|
||||
const [hash, snapshot] = await this.callback();
|
||||
|
||||
case "value":
|
||||
_provisions.push(["value", c.name, c.value]);
|
||||
break;
|
||||
|
||||
case "add_tag":
|
||||
_provisions.push(["tag", c.tag, true]);
|
||||
break;
|
||||
|
||||
case "delete_tag":
|
||||
_provisions.push(["tag", c.tag, false]);
|
||||
break;
|
||||
|
||||
case "provision":
|
||||
_provisions.push([c.name].concat(c.args || []));
|
||||
break;
|
||||
|
||||
case "add_object":
|
||||
for (const obj of objects) {
|
||||
if (obj["_id"] === c.object) {
|
||||
const alias = obj["_keys"]
|
||||
.map((k) => `${k}:${JSON.stringify(obj[k])}`)
|
||||
.join(",");
|
||||
const p = `${c.name}.[${alias}]`;
|
||||
_provisions.push(["instances", p, 1]);
|
||||
|
||||
for (const k in obj) {
|
||||
if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
|
||||
_provisions.push(["value", `${p}.${k}`, obj[k]]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case "delete_object":
|
||||
for (const obj of objects) {
|
||||
if (obj["_id"] === c.object) {
|
||||
const alias = obj["_keys"]
|
||||
.map((k) => `${k}:${JSON.stringify(obj[k])}`)
|
||||
.join(",");
|
||||
const p = `${c.name}.[${alias}]`;
|
||||
_provisions.push(["instances", p, 0]);
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new Error(`Unknown configuration type ${c.type}`);
|
||||
}
|
||||
if (this.currentRevision) {
|
||||
const r = this.currentRevision;
|
||||
const s = this.snapshots.get(r);
|
||||
setTimeout(() => {
|
||||
if (this.snapshots.get(r) === s) this.snapshots.delete(r);
|
||||
}, EVICT_TIMEOUT).unref();
|
||||
}
|
||||
|
||||
presets.push({
|
||||
name: preset["_id"],
|
||||
channel: (preset["channel"] as string) || "default",
|
||||
schedule: schedule,
|
||||
events: events,
|
||||
precondition: precondition,
|
||||
provisions: _provisions,
|
||||
});
|
||||
}
|
||||
this.currentRevision = hash;
|
||||
this.snapshots.set(hash, snapshot);
|
||||
|
||||
return presets;
|
||||
}
|
||||
|
||||
async function fetchProvisions(): Promise<Provisions> {
|
||||
const res = await db.getProvisions();
|
||||
|
||||
const provisions = {};
|
||||
for (const r of res) {
|
||||
provisions[r["_id"]] = {};
|
||||
provisions[r["_id"]].md5 = crypto
|
||||
.createHash("md5")
|
||||
.update(r["script"])
|
||||
.digest("hex");
|
||||
provisions[r["_id"]].script = new vm.Script(
|
||||
`"use strict";(function(){\n${r["script"]}\n})();`,
|
||||
{ filename: r["_id"], lineOffset: -1, timeout: 50 }
|
||||
);
|
||||
}
|
||||
|
||||
return provisions;
|
||||
}
|
||||
|
||||
async function fetchVirtualParameters(): Promise<VirtualParameters> {
|
||||
const res = await db.getVirtualParameters();
|
||||
|
||||
const virtualParameters = {};
|
||||
for (const r of res) {
|
||||
virtualParameters[r["_id"]] = {};
|
||||
virtualParameters[r["_id"]].md5 = crypto
|
||||
.createHash("md5")
|
||||
.update(r["script"])
|
||||
.digest("hex");
|
||||
virtualParameters[r["_id"]].script = new vm.Script(
|
||||
`"use strict";(function(){\n${r["script"]}\n})();`,
|
||||
{ filename: r["_id"], lineOffset: -1, timeout: 50 }
|
||||
);
|
||||
}
|
||||
|
||||
return virtualParameters;
|
||||
}
|
||||
|
||||
async function fetchPermissions(): Promise<Permissions> {
|
||||
const perms = await db.getPermissions();
|
||||
const permissions: Permissions = {};
|
||||
|
||||
for (const p of perms) {
|
||||
if (!permissions[p.role]) permissions[p.role] = {};
|
||||
if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {};
|
||||
|
||||
permissions[p.role][p.access][p.resource] = {
|
||||
access: p.access,
|
||||
filter: parse(p.filter || "true"),
|
||||
};
|
||||
if (p.validate)
|
||||
permissions[p.role][p.access][p.resource].validate = parse(p.validate);
|
||||
}
|
||||
|
||||
return permissions;
|
||||
}
|
||||
|
||||
async function fetchFiles(): Promise<Files> {
|
||||
const res = await db.getFiles();
|
||||
const files = {};
|
||||
|
||||
for (const r of res) {
|
||||
const id = r["filename"] || r["_id"].toString();
|
||||
files[id] = {};
|
||||
files[id].length = r["length"];
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
async function fetchUsers(): Promise<Users> {
|
||||
const _users = await db.getUsers();
|
||||
const users = {};
|
||||
|
||||
for (const user of _users) {
|
||||
users[user._id] = {
|
||||
password: user.password,
|
||||
salt: user.salt,
|
||||
roles: user.roles.split(",").map((s) => s.trim()),
|
||||
};
|
||||
}
|
||||
|
||||
return users;
|
||||
}
|
||||
|
||||
async function fetchConfig(): Promise<[Config, UiConfig]> {
|
||||
const conf = await db.getConfig();
|
||||
|
||||
conf.sort((a, b) => (a.id > b.id ? 1 : a.id < b.id ? -1 : 0));
|
||||
|
||||
const ui = {
|
||||
filters: {},
|
||||
device: {},
|
||||
index: {},
|
||||
overview: {},
|
||||
pageSize: null,
|
||||
};
|
||||
const _config = {};
|
||||
|
||||
for (const c of conf) {
|
||||
// Evaluate expressions to simplify them
|
||||
const val = expression.evaluate(c.value);
|
||||
_config[c.id] = val;
|
||||
if (c.id.startsWith("ui.")) {
|
||||
const keys = c.id.split(".");
|
||||
if (!(keys[1] in ui)) continue;
|
||||
// remove the first key(ui)
|
||||
keys.shift();
|
||||
let ref = ui;
|
||||
while (keys.length > 1) {
|
||||
const k = keys.shift();
|
||||
if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {};
|
||||
ref = ref[k];
|
||||
}
|
||||
ref[keys[0]] = val;
|
||||
if (lockToken) {
|
||||
if (hash !== dbHash) await set(this.cacheKey, hash, 300);
|
||||
await releaseLock(this.cacheKey + "_lock", lockToken);
|
||||
}
|
||||
|
||||
this.nextRefresh = now + (REFRESH - (now % REFRESH));
|
||||
}
|
||||
|
||||
return [_config, ui];
|
||||
}
|
||||
|
||||
async function refresh(): Promise<void> {
|
||||
if (!nextRefresh) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
return refresh();
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
|
||||
if (now < nextRefresh) return;
|
||||
|
||||
nextRefresh = 0;
|
||||
|
||||
const dbHash = await cache.get("presets_hash");
|
||||
|
||||
if (currentSnapshot && dbHash === currentSnapshot) {
|
||||
nextRefresh = now + (REFRESH - (now % REFRESH));
|
||||
return;
|
||||
}
|
||||
|
||||
const lockToken = await cache.acquireLock("presets_hash_lock", 5000);
|
||||
|
||||
const res = await Promise.all([
|
||||
fetchPresets(),
|
||||
fetchProvisions(),
|
||||
fetchVirtualParameters(),
|
||||
fetchFiles(),
|
||||
fetchPermissions(),
|
||||
fetchUsers(),
|
||||
fetchConfig(),
|
||||
]);
|
||||
|
||||
const snapshot = {
|
||||
presets: res[0],
|
||||
provisions: res[1],
|
||||
virtualParameters: res[2],
|
||||
files: res[3],
|
||||
permissions: res[4],
|
||||
users: res[5],
|
||||
config: res[6][0],
|
||||
ui: res[6][1],
|
||||
};
|
||||
|
||||
if (currentSnapshot) {
|
||||
const h = currentSnapshot;
|
||||
const s = snapshots.get(h);
|
||||
setTimeout(() => {
|
||||
if (snapshots.get(h) === s) snapshots.delete(h);
|
||||
}, EVICT_TIMEOUT).unref();
|
||||
}
|
||||
|
||||
currentSnapshot = computeHash(snapshot);
|
||||
snapshots.set(currentSnapshot, snapshot);
|
||||
|
||||
if (lockToken) {
|
||||
if (currentSnapshot !== dbHash)
|
||||
await cache.set("presets_hash", currentSnapshot, 300);
|
||||
await cache.releaseLock("presets_hash_lock", lockToken);
|
||||
}
|
||||
|
||||
nextRefresh = now + (REFRESH - (now % REFRESH));
|
||||
}
|
||||
|
||||
export async function getCurrentSnapshot(): Promise<string> {
|
||||
if (Date.now() > nextRefresh) await refresh();
|
||||
return currentSnapshot;
|
||||
}
|
||||
|
||||
export function hasSnapshot(hash: string): boolean {
|
||||
return snapshots.has(hash);
|
||||
}
|
||||
|
||||
export function getPresets(snapshotKey: string): Preset[] {
|
||||
const snapshot = snapshots.get(snapshotKey);
|
||||
if (!snapshot) throw new Error("Cache snapshot does not exist");
|
||||
return snapshot.presets;
|
||||
}
|
||||
|
||||
export function getProvisions(snapshotKey: string): Provisions {
|
||||
const snapshot = snapshots.get(snapshotKey);
|
||||
if (!snapshot) throw new Error("Cache snapshot does not exist");
|
||||
return snapshot.provisions;
|
||||
}
|
||||
|
||||
export function getVirtualParameters(snapshotKey: string): VirtualParameters {
|
||||
const snapshot = snapshots.get(snapshotKey);
|
||||
if (!snapshot) throw new Error("Cache snapshot does not exist");
|
||||
return snapshot.virtualParameters;
|
||||
}
|
||||
|
||||
export function getFiles(snapshotKey: string): Files {
|
||||
const snapshot = snapshots.get(snapshotKey);
|
||||
if (!snapshot) throw new Error("Cache snapshot does not exist");
|
||||
return snapshot.files;
|
||||
}
|
||||
|
||||
export function getConfig(
|
||||
snapshotKey: string,
|
||||
key: string,
|
||||
context: Record<string, unknown>,
|
||||
now: number,
|
||||
cb?: (e: Expression) => Expression
|
||||
): string | number | boolean | null {
|
||||
const snapshot = snapshots.get(snapshotKey);
|
||||
if (!snapshot) throw new Error("Cache snapshot does not exist");
|
||||
|
||||
const oldOpts = {
|
||||
"cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
|
||||
"cwmp.debug": "DEBUG",
|
||||
"cwmp.retryDelay": "RETRY_DELAY",
|
||||
"cwmp.sessionTimeout": "SESSION_TIMEOUT",
|
||||
"cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
|
||||
"cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
|
||||
"cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
|
||||
"cwmp.cookiesPath": "COOKIES_PATH",
|
||||
"cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
|
||||
"cwmp.booleanLiteral": "BOOLEAN_LITERAL",
|
||||
"cwmp.connectionRequestAllowBasicAuth":
|
||||
"CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
|
||||
"cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
|
||||
"cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
|
||||
"cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
|
||||
};
|
||||
|
||||
if (!(key in snapshot.config)) {
|
||||
if (key in oldOpts) {
|
||||
let id;
|
||||
if (context?.["id"]) {
|
||||
id = context["id"];
|
||||
} else if (cb) {
|
||||
id = cb(["PARAM", "DeviceID.ID"]);
|
||||
if (Array.isArray(id)) id = null;
|
||||
}
|
||||
return config.get(oldOpts[key], id);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const v = expression.evaluate(snapshot.config[key], context, now, cb);
|
||||
return Array.isArray(v) ? null : v;
|
||||
}
|
||||
|
||||
export function getConfigExpression(
|
||||
snapshotKey: string,
|
||||
key: string
|
||||
): Expression {
|
||||
const snapshot = snapshots.get(snapshotKey);
|
||||
if (!snapshot) throw new Error("Cache snapshot does not exist");
|
||||
|
||||
return snapshot.config[key];
|
||||
}
|
||||
|
||||
export function getUsers(snapshotKey: string): Users {
|
||||
const snapshot = snapshots.get(snapshotKey);
|
||||
if (!snapshot) throw new Error("Cache snapshot does not exist");
|
||||
|
||||
return snapshot.users;
|
||||
}
|
||||
|
||||
export function getPermissions(snapshotKey: string): Permissions {
|
||||
const snapshot = snapshots.get(snapshotKey);
|
||||
if (!snapshot) throw new Error("Cache snapshot does not exist");
|
||||
|
||||
return snapshot.permissions;
|
||||
}
|
||||
|
||||
export function getUiConfig(snapshotKey: string): UiConfig {
|
||||
const snapshot = snapshots.get(snapshotKey);
|
||||
if (!snapshot) throw new Error("Cache snapshot does not exist");
|
||||
|
||||
return snapshot.ui;
|
||||
}
|
||||
Reference in new issue
Block a user