rodrigo/genieacs
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor DB code to use promises

Browse Source
This commit is contained in:
Zaid Humaimidi committed 2023-08-09 19:51:38 -07:00
1 parent 3169e829e7
commit 58488ebc3d
12 files changed
+3814 -1054
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lib/ui/api-functions.ts
+5 -5
View File
@@ -145,19 +145,19 @@ export async function postTasks(
};
}
const promises = [];
const promises: Promise<number>[] = [];
for (const s of statuses) {
promises.push(db.query("tasks", ["=", ["PARAM", "_id"], s._id]));
promises.push(db.count("tasks", ["=", ["PARAM", "_id"], s._id]));
promises.push(
db.query("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
db.count("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
);
}
const res = await Promise.all(promises);
for (const [i, r] of statuses.entries()) {
if (res[i * 2].length === 0) {
if (!res[i * 2]) {
r.status = "done";
} else if (res[i * 2 + 1].length === 1) {
} else if (res[i * 2 + 1]) {
r.status = "fault";
r["fault"] = res[i * 2 + 1][0];
}
lib/ui/api.ts
+24 -35
View File
@@ -221,16 +221,11 @@ for (const [resource, flags] of Object.entries(resources)) {
let c = 0;
ctx.body.write("[\n");
db.query(resource, filter, options, (obj) => {
for await (const obj of db.query(resource, filter, options))
ctx.body.write((c++ ? "," : "") + JSON.stringify(obj) + "\n");
})
.then(() => {
ctx.body.end("]");
logger.accessInfo(log);
})
.catch((err) => {
ctx.body.emit("error", err);
});
ctx.body.end("]");
logger.accessInfo(log);
});
// CSV download
@@ -289,7 +284,7 @@ for (const [resource, flags] of Object.entries(resources)) {
ctx.body.write(
Object.keys(columns).map((k) => `"${k.replace(/"/, '""')}"`) + "\n"
);
db.query(resource, filter, options, (obj) => {
for await (const obj of db.query(resource, filter, options)) {
const arr = Object.values(columns).map((exp) => {
const v = evaluate(exp, obj, null, (e) => {
if (Array.isArray(e)) {
@@ -331,14 +326,9 @@ for (const [resource, flags] of Object.entries(resources)) {
return v;
});
ctx.body.write(arr.join(",") + "\n");
})
.then(() => {
ctx.body.end();
logger.accessInfo(log);
})
.catch((err) => {
ctx.body.emit("error", err);
});
}
ctx.body.end();
logger.accessInfo(log);
});
router.head(`/${resource}/:id`, async (ctx) => {
@@ -359,9 +349,9 @@ for (const [resource, flags] of Object.entries(resources)) {
return void (ctx.status = 403);
}
const res = await db.query(resource, filter);
const count = await db.count(resource, filter);
if (!res.length) return void (ctx.status = 404);
if (!count) return void (ctx.status = 404);
logger.accessInfo(log);
ctx.body = "";
@@ -385,12 +375,12 @@ for (const [resource, flags] of Object.entries(resources)) {
return void (ctx.status = 403);
}
const res = await db.query(resource, filter);
const { value: res } = await db.query(resource, filter).next();
if (!res.length) return void (ctx.status = 404);
if (!res) return void (ctx.status = 404);
logger.accessInfo(log);
ctx.body = res[0];
ctx.body = res;
});
if (flags & RESOURCE_DELETE) {
@@ -411,10 +401,10 @@ for (const [resource, flags] of Object.entries(resources)) {
logUnauthorizedWarning(log);
return void (ctx.status = 403);
}
const res = await db.query(resource, filter);
if (!res.length) return void (ctx.status = 404);
const { value: res } = await db.query(resource, filter).next();
if (!res) return void (ctx.status = 404);
const validate = authorizer.getValidator(resource, res[0]);
const validate = authorizer.getValidator(resource, res);
if (!validate("delete")) {
logUnauthorizedWarning(log);
return void (ctx.status = 403);
@@ -560,9 +550,8 @@ router.post("/devices/:id/tasks", async (ctx) => {
logUnauthorizedWarning(log);
return void (ctx.status = 403);
}
const devices = await db.query("devices", filter);
if (!devices.length) return void (ctx.status = 404);
const device = devices[0];
const { value: device } = await db.query("devices", filter).next();
if (!device) return void (ctx.status = 404);
const validate = authorizer.getValidator("devices", device);
for (const t of ctx.request.body) {
@@ -636,10 +625,10 @@ router.post("/devices/:id/tags", async (ctx) => {
logUnauthorizedWarning(log);
return void (ctx.status = 403);
}
const res = await db.query("devices", filter);
if (!res.length) return void (ctx.status = 404);
const { value: res } = await db.query("devices", filter).next();
if (!res) return void (ctx.status = 404);
const validate = authorizer.getValidator("devices", res[0]);
const validate = authorizer.getValidator("devices", res);
if (!validate("tags", ctx.request.body)) {
logUnauthorizedWarning(log);
return void (ctx.status = 403);
@@ -707,12 +696,12 @@ router.put("/users/:id/password", async (ctx) => {
["PARAM", RESOURCE_IDS.users],
username,
]);
const res = await db.query("users", filter);
if (!res.length) return void (ctx.status = 404);
const { value: res } = await db.query("users", filter).next();
if (!res) return void (ctx.status = 404);
const newPassword = ctx.request.body.newPassword;
if (ctx.state.user) {
const validate = authorizer.getValidator("users", res[0]);
const validate = authorizer.getValidator("users", res);
if (!validate("password", { password: newPassword })) {
logUnauthorizedWarning(log);
return void (ctx.status = 403);
lib/ui/db.ts
+69 -137
View File
@@ -17,121 +17,73 @@
* along with GenieACS. If not, see <http://www.gnu.org/licenses/>.
*/
import { Db, GridFSBucket, ObjectId } from "mongodb";
import { Collection, GridFSBucket, ObjectId } from "mongodb";
import { Script } from "vm";
import { onConnect, optimizeProjection } from "../db";
import { collections, onConnect, optimizeProjection } from "../db";
import * as mongodbFunctions from "../mongodb-functions";
import { evaluate } from "../common/expression/util";
import { QueryOptions, Expression } from "../types";
import { Readable } from "stream";
const RESOURCE_COLLECTION = {
files: "fs.files",
};
let filesBucket: GridFSBucket;
let db: Db;
onConnect(async (_db) => {
db = _db;
onConnect(async (db) => {
filesBucket = new GridFSBucket(db);
});
export function query(
export async function* query(
resource: string,
filter: Expression,
options?: QueryOptions
): Promise<any[]>;
export function query(
resource: string,
filter: Expression,
options: QueryOptions,
callback: (doc: any) => void
): Promise<void>;
export function query(
resource: string,
filter: Expression,
options?: QueryOptions,
callback?: (doc: any) => void
): Promise<void | any[]> {
): AsyncGenerator<any, void, undefined> {
options = options || {};
filter = evaluate(filter, null, Date.now());
const q = mongodbFunctions.toMongoQuery(filter, resource);
if (!q) return Promise.resolve([]);
if (!q) return;
return new Promise((resolve, reject) => {
const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
const cursor = collection.find(q);
if (options.projection) {
let projection = options.projection;
if (resource === "devices") {
projection = mongodbFunctions.processDeviceProjection(
options.projection
);
}
const collection = collections[resource] as Collection<any>;
const cursor = collection.find(q);
if (options.projection) {
let projection = options.projection;
if (resource === "devices")
projection = mongodbFunctions.processDeviceProjection(options.projection);
if (resource === "presets") projection.configurations = 1;
projection = optimizeProjection(projection);
cursor.project(projection);
}
if (resource === "presets") projection.configurations = 1;
projection = optimizeProjection(projection);
cursor.project(projection);
}
if (resource === "users") cursor.project({ password: 0, salt: 0 });
if (resource === "users") cursor.project({ password: 0, salt: 0 });
if (options.skip) cursor.skip(options.skip);
if (options.limit) cursor.limit(options.limit);
if (options.skip) cursor.skip(options.skip);
if (options.limit) cursor.limit(options.limit);
if (options.sort) {
let s = Object.entries(options.sort)
.sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
.reduce(
(obj, [k, v]) =>
Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
{}
);
if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
cursor.sort(s);
}
if (!callback) {
cursor.toArray((err, docs) => {
if (err) return reject(err);
if (resource === "devices")
return resolve(docs.map((d) => mongodbFunctions.flattenDevice(d)));
else if (resource === "faults")
return resolve(docs.map((d) => mongodbFunctions.flattenFault(d)));
else if (resource === "tasks")
return resolve(docs.map((d) => mongodbFunctions.flattenTask(d)));
else if (resource === "presets")
return resolve(docs.map((d) => mongodbFunctions.flattenPreset(d)));
else if (resource === "files")
return resolve(docs.map((d) => mongodbFunctions.flattenFile(d)));
return resolve(docs);
});
} else {
cursor.forEach(
(doc) => {
if (resource === "devices")
callback(mongodbFunctions.flattenDevice(doc));
else if (resource === "faults")
callback(mongodbFunctions.flattenFault(doc));
else if (resource === "tasks")
callback(mongodbFunctions.flattenTask(doc));
else if (resource === "presets")
callback(mongodbFunctions.flattenPreset(doc));
else if (resource === "files")
callback(mongodbFunctions.flattenFile(doc));
else callback(doc);
},
(err) => {
if (err) reject(err);
else resolve();
}
if (options.sort) {
let s = Object.entries(options.sort)
.sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
.reduce(
(obj, [k, v]) =>
Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
{}
);
}
});
if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
cursor.sort(s);
}
for await (let doc of cursor) {
if (resource === "devices") doc = mongodbFunctions.flattenDevice(doc);
else if (resource === "faults") doc = mongodbFunctions.flattenFault(doc);
else if (resource === "tasks") doc = mongodbFunctions.flattenTask(doc);
else if (resource === "presets") doc = mongodbFunctions.flattenPreset(doc);
else if (resource === "files") doc = mongodbFunctions.flattenFile(doc);
yield doc;
}
}
export function count(resource: string, filter: Expression): Promise<number> {
const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
const collection = collections[resource] as Collection<unknown>;
filter = evaluate(filter, null, Date.now());
const q = mongodbFunctions.toMongoQuery(filter, resource);
if (!q) return Promise.resolve(0);
@@ -150,37 +102,25 @@ export async function updateDeviceTags(
if (onOff) add.push(tag);
else pull.push(tag);
}
const collection = db.collection("devices");
const object = {};
if (add?.length) object["$addToSet"] = { _tags: { $each: add } };
if (pull?.length) object["$pullAll"] = { _tags: pull };
await collection.updateOne({ _id: deviceId }, object);
await collections.devices.updateOne({ _id: deviceId }, object);
}
function putResource(resource, id, object): Promise<void> {
return new Promise((resolve, reject) => {
const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
collection.replaceOne({ _id: id }, object, { upsert: true }, (err) => {
if (err) return void reject(err);
resolve();
});
});
async function putResource(resource, id, object): Promise<void> {
const collection = collections[resource] as Collection<unknown>;
await collection.replaceOne({ _id: id }, object, { upsert: true });
}
function deleteResource(
async function deleteResource(
resource: string,
id: string | ObjectId
): Promise<void> {
return new Promise((resolve, reject) => {
const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
collection.deleteOne({ _id: id }, (err) => {
if (err) return void reject(err);
resolve();
});
});
const collection = collections[resource] as Collection<unknown>;
await collection.deleteOne({ _id: id });
}
export function putPreset(
@@ -267,23 +207,16 @@ export function deletePermission(id: string): Promise<void> {
return deleteResource("permissions", id);
}
export function putUser(
export async function putUser(
id: string,
object: Record<string, unknown>
): Promise<void> {
return new Promise((resolve, reject) => {
const collection = db.collection("users");
// update instead of replace to keep the password if not set by user
collection.updateOne(
{ _id: id },
{ $set: object },
{ upsert: true },
(err) => {
if (err) return void reject(err);
resolve();
}
);
});
// update instead of replace to keep the password if not set by user
await collections.users.updateOne(
{ _id: id },
{ $set: object },
{ upsert: true }
);
}
export function deleteUser(id: string): Promise<void> {
@@ -291,8 +224,7 @@ export function deleteUser(id: string): Promise<void> {
}
export function downloadFile(filename: string): Readable {
const bucket = new GridFSBucket(db);
return bucket.openDownloadStreamByName(filename);
return filesBucket.openDownloadStreamByName(filename);
}
export function putFile(
@@ -301,8 +233,7 @@ export function putFile(
contentStream: Readable
): Promise<void> {
return new Promise((resolve, reject) => {
const bucket = new GridFSBucket(db);
const uploadStream = bucket.openUploadStreamWithId(
const uploadStream = filesBucket.openUploadStreamWithId(
filename as unknown as ObjectId,
filename,
{
@@ -310,8 +241,15 @@ export function putFile(
}
);
let readableEnded = false;
contentStream.on("end", () => {
readableEnded = true;
});
contentStream.on("close", () => {
if (!contentStream.readableEnded)
// In Node versions prior to 15, the stream will not emit an error if the
// connection is closed before the stream is finished.
// For Node 12.9+ we can just use stream.readableEnded
if (!readableEnded)
uploadStream.destroy(new Error("Stream closed prematurely"));
});
@@ -325,14 +263,8 @@ export function putFile(
});
}
export function deleteFile(filename: string): Promise<void> {
return new Promise((resolve, reject) => {
const bucket = new GridFSBucket(db);
bucket.delete(filename as any, (err) => {
if (err) return void reject(err);
resolve();
});
});
export async function deleteFile(filename: string): Promise<void> {
await filesBucket.delete(filename as any);
}
export function deleteFault(id: string): Promise<void> {