rodrigo/genieacs
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor expression to Mongo query translation

Browse Source 
The previous approach was simplistic and could result in inefficient queries
in certain edge cases, notably causing MongoDB to not utilize existing indexes
when querying by last inform timestamps given upper and lower bounds. The new
implementation leverages our existing expression minimizer code and can
therefore generate more efficient queries or at least queries that can take
advantage of existing indexes.
This commit is contained in:
Zaid Humaimidi committed 2023-08-09 19:48:49 -07:00
1 parent 947f96299d
commit 3169e829e7
37 files changed
+2175 -1632
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lib/ui/api.ts
+2 -2
View File
@@ -21,8 +21,8 @@ import { PassThrough } from "stream";
import Router from "koa-router";
import * as db from "./db";
import * as apiFunctions from "./api-functions";
import { evaluate, and, extractParams } from "../common/expression";
import { parse } from "../common/expression-parser";
import { evaluate, and, extractParams } from "../common/expression/util";
import { parse } from "../common/expression/parser";
import * as logger from "../logger";
import { getConfig } from "../local-cache";
import { QueryOptions, Expression } from "../types";
lib/ui/db.ts
+7 -40
View File
@@ -21,10 +21,9 @@ import { Db, GridFSBucket, ObjectId } from "mongodb";
import { Script } from "vm";
import { onConnect, optimizeProjection } from "../db";
import * as mongodbFunctions from "../mongodb-functions";
import * as expression from "../common/expression";
import { evaluate } from "../common/expression/util";
import { QueryOptions, Expression } from "../types";
import { Readable } from "stream";
import { minimize } from "../common/boolean-expression";
const RESOURCE_COLLECTION = {
files: "fs.files",
@@ -54,28 +53,9 @@ export function query(
callback?: (doc: any) => void
): Promise<void | any[]> {
options = options || {};
let q;
filter = expression.evaluate(filter, null, Date.now());
filter = minimize(filter, true);
if (Array.isArray(filter)) {
if (resource === "devices") {
filter = mongodbFunctions.processDeviceFilter(filter);
} else if (resource === "tasks") {
filter = mongodbFunctions.processTasksFilter(filter);
} else if (resource === "faults") {
filter = mongodbFunctions.processFaultsFilter(filter);
} else if (resource === "users") {
// Protect against brute force, and dictionary attacks
const params = expression.extractParams(filter);
if (params.includes("password") || params.includes("salt"))
return Promise.reject(new Error("Invalid users filter"));
}
q = mongodbFunctions.filterToMongoQuery(filter);
} else if (!filter) {
return Promise.resolve([]);
}
filter = evaluate(filter, null, Date.now());
const q = mongodbFunctions.toMongoQuery(filter, resource);
if (!q) return Promise.resolve([]);
return new Promise((resolve, reject) => {
const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
@@ -152,22 +132,9 @@ export function query(
export function count(resource: string, filter: Expression): Promise<number> {
const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
let q: Parameters<typeof collection.countDocuments>[0];
filter = expression.evaluate(filter, null, Date.now());
filter = minimize(filter, true);
if (Array.isArray(filter)) {
if (resource === "devices")
filter = mongodbFunctions.processDeviceFilter(filter);
else if (resource === "tasks")
filter = mongodbFunctions.processTasksFilter(filter);
else if (resource === "faults")
filter = mongodbFunctions.processFaultsFilter(filter);
q = mongodbFunctions.filterToMongoQuery(filter);
} else if (!filter) {
return Promise.resolve(0);
}
filter = evaluate(filter, null, Date.now());
const q = mongodbFunctions.toMongoQuery(filter, resource);
if (!q) return Promise.resolve(0);
return collection.countDocuments(q);
}