Refactor expression to Mongo query translation
The previous approach was simplistic and could result in inefficient queries in certain edge cases, notably causing MongoDB to not utilize existing indexes when querying by last inform timestamps given upper and lower bounds. The new implementation leverages our existing expression minimizer code and can therefore generate more efficient queries or at least queries that can take advantage of existing indexes.
This commit is contained in:
Zaid Humaimidi committed
1 parent
947f96299d
commit
3169e829e7
37 files changed
+2175
-1632
No files matched your search
+644
-245
@@ -1,248 +1,21 @@
|
||||
/**
|
||||
* Copyright 2013-2019 GenieACS Inc.
|
||||
*
|
||||
* This file is part of GenieACS.
|
||||
*
|
||||
* GenieACS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* GenieACS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with GenieACS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { ObjectId } from "mongodb";
|
||||
import { map, parse, stringify, parseList } from "./common/expression-parser";
|
||||
import { likePatternToRegExp } from "./common/expression";
|
||||
import { Filter } from "mongodb";
|
||||
import { EJSON } from "bson";
|
||||
import {
|
||||
parse,
|
||||
stringify,
|
||||
parseList,
|
||||
parseLikePattern,
|
||||
} from "./common/expression/parser";
|
||||
import { Expression, Fault, Task } from "./types";
|
||||
import { decodeTag, encodeTag } from "./common";
|
||||
|
||||
const isArray = Array.isArray;
|
||||
|
||||
export function processDeviceFilter(filter: Expression): Expression {
|
||||
return map(filter, (exp) => {
|
||||
if (!isArray(exp)) return exp;
|
||||
|
||||
if (exp[0] === "PARAM") {
|
||||
const p = exp[1];
|
||||
if (p === "DeviceID.ID") return ["PARAM", "_id"];
|
||||
else if (p === "DeviceID") return ["PARAM", "_deviceId"];
|
||||
else if (p.startsWith("DeviceID."))
|
||||
return ["PARAM", "_deviceId._" + p.slice(9)];
|
||||
else if (p === "Events.Inform") return ["PARAM", "_lastInform"];
|
||||
else if (p === "Events.Registered") return ["PARAM", "_registered"];
|
||||
else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"];
|
||||
else if (p === "Events.1_BOOT") return ["PARAM", "_lastBoot"];
|
||||
} else if (
|
||||
isArray(exp[1]) &&
|
||||
exp[1][0] === "PARAM" &&
|
||||
exp[1][1].startsWith("Tags.")
|
||||
) {
|
||||
const t = decodeTag(exp[1][1].slice(5));
|
||||
if (exp[0] === "IS NULL") return ["_tags", t, false];
|
||||
else if (exp[0] === "IS NOT NULL") return ["_tags", t, true];
|
||||
else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true];
|
||||
else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true];
|
||||
} else if (
|
||||
[
|
||||
"=",
|
||||
"<>",
|
||||
">",
|
||||
">=",
|
||||
"<",
|
||||
"<=",
|
||||
"LIKE",
|
||||
"NOT LIKE",
|
||||
"IS NULL",
|
||||
"IS NOT NULL",
|
||||
].includes(exp[0])
|
||||
) {
|
||||
let e = map(exp, (ee) => {
|
||||
if (
|
||||
isArray(ee) &&
|
||||
ee[0] === "PARAM" &&
|
||||
typeof ee[1] === "string" &&
|
||||
!ee[1].startsWith("_")
|
||||
)
|
||||
return ["PARAM", `${ee[1]}._value`];
|
||||
return ee;
|
||||
});
|
||||
|
||||
if (typeof e[2] === "number") {
|
||||
const alt = (e as any[]).slice();
|
||||
alt[2] = new Date(e[2]);
|
||||
if (exp[0] === "<>" || exp[0] === "NOT LIKE") e = ["AND", e, alt];
|
||||
else e = ["OR", e, alt];
|
||||
}
|
||||
return e;
|
||||
}
|
||||
return exp;
|
||||
});
|
||||
}
|
||||
|
||||
export function processTasksFilter(filter: Expression): Expression {
|
||||
return map(filter, (exp) => {
|
||||
if (!isArray(exp)) return exp;
|
||||
if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
|
||||
const e = exp.slice();
|
||||
if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectId(e[2]);
|
||||
else if (e[1][0] === "PARAM" && e[1][1] === "timestamp")
|
||||
e[2] = new Date(e[2]);
|
||||
else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
|
||||
e[2] = new Date(e[2]);
|
||||
return e;
|
||||
}
|
||||
return exp;
|
||||
});
|
||||
}
|
||||
|
||||
export function processFaultsFilter(filter: Expression): Expression {
|
||||
return map(filter, (exp) => {
|
||||
if (!isArray(exp)) return exp;
|
||||
if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
|
||||
const e = exp.slice();
|
||||
if (e[1][0] === "PARAM" && e[1][1] === "timestamp") e[2] = new Date(e[2]);
|
||||
else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
|
||||
e[2] = new Date(e[2]);
|
||||
return e;
|
||||
}
|
||||
return exp;
|
||||
});
|
||||
}
|
||||
|
||||
export function filterToMongoQuery(exp: Expression): Record<string, unknown> {
|
||||
function recursive(filter, negate): Record<string, unknown> {
|
||||
const op = filter[0];
|
||||
|
||||
if (op === "AND") {
|
||||
filter = filter.filter((f) => f !== true);
|
||||
if (filter.length === 2) return recursive(filter[1], negate);
|
||||
}
|
||||
|
||||
if (op === "OR") {
|
||||
filter = filter.filter((f) => f !== false);
|
||||
if (filter.length === 2) return recursive(filter[1], negate);
|
||||
}
|
||||
|
||||
if ((!negate && op === "AND") || (negate && op === "OR"))
|
||||
return { $and: filter.slice(1).map((f) => recursive(f, negate)) };
|
||||
|
||||
if ((!negate && op === "OR") || (negate && op === "AND"))
|
||||
return { $or: filter.slice(1).map((f) => recursive(f, negate)) };
|
||||
|
||||
if (op === "NOT") return recursive(filter[1], !negate);
|
||||
|
||||
if (isArray(filter[2]))
|
||||
throw new Error(`Invalid RHS operand of ${op} clause`);
|
||||
|
||||
if (op === "LIKE" || op === "NOT LIKE") {
|
||||
if (op === "NOT LIKE") negate = !negate;
|
||||
let param;
|
||||
let flags;
|
||||
if (!Array.isArray(filter[1]))
|
||||
throw new Error(`Invalid LHS operand of ${op} clause`);
|
||||
if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
|
||||
if (filter[2] !== filter[2].toUpperCase())
|
||||
throw new Error(`Invalid RHS operand of ${op} clause`);
|
||||
param = filter[1][2][1];
|
||||
flags = "i";
|
||||
} else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
|
||||
if (filter[2] !== filter[2].toLowerCase())
|
||||
throw new Error(`Invalid RHS operand of ${op} clause`);
|
||||
param = filter[1][2][1];
|
||||
flags = "i";
|
||||
} else if (filter[1][0] === "PARAM") {
|
||||
param = filter[1][1];
|
||||
flags = "";
|
||||
} else {
|
||||
throw new Error(`Invalid LHS operand of ${op} clause`);
|
||||
}
|
||||
const r = likePatternToRegExp(filter[2], filter[3], flags);
|
||||
if (negate) return { [param]: { $nin: [r, null] } };
|
||||
else return { [param]: r };
|
||||
}
|
||||
|
||||
if (op === "_tags") {
|
||||
let t = filter[2];
|
||||
if (negate) t = !t;
|
||||
if (t) return { _tags: filter[1] };
|
||||
else return { _tags: { $ne: filter[1] } };
|
||||
}
|
||||
|
||||
if (!isArray(filter[1]) || filter[1][0] !== "PARAM")
|
||||
throw new Error(`Invalid LHS operand of ${op} clause`);
|
||||
|
||||
if (op === "IS NULL") {
|
||||
const p = filter[1][1];
|
||||
return { [p]: null };
|
||||
}
|
||||
|
||||
if (op === "IS NOT NULL") {
|
||||
const p = filter[1][1];
|
||||
return { [p]: { $ne: null } };
|
||||
}
|
||||
|
||||
if (isArray(filter[2]) || filter[2] == null)
|
||||
throw new Error(`Invalid RHS operand of ${op} clause`);
|
||||
|
||||
if (op === "=") {
|
||||
const p = filter[1][1];
|
||||
const v = filter[2];
|
||||
if (!negate) return { [p]: v };
|
||||
else return { [p]: { $nin: [v, null] } };
|
||||
}
|
||||
|
||||
if (op === "<>") {
|
||||
const p = filter[1][1];
|
||||
const v = filter[2];
|
||||
if (negate) return { [p]: v };
|
||||
else return { [p]: { $nin: [v, null] } };
|
||||
}
|
||||
|
||||
if (op === ">") {
|
||||
const p = filter[1][1];
|
||||
const v = filter[2];
|
||||
if (negate) return { [p]: { $lte: v } };
|
||||
else return { [p]: { $gt: v } };
|
||||
}
|
||||
|
||||
if (op === ">=") {
|
||||
const p = filter[1][1];
|
||||
const v = filter[2];
|
||||
if (negate) return { [p]: { $lt: v } };
|
||||
else return { [p]: { $gte: v } };
|
||||
}
|
||||
|
||||
if (op === "<") {
|
||||
const p = filter[1][1];
|
||||
const v = filter[2];
|
||||
if (negate) return { [p]: { $gte: v } };
|
||||
else return { [p]: { $lt: v } };
|
||||
}
|
||||
|
||||
if (op === "<=") {
|
||||
const p = filter[1][1];
|
||||
const v = filter[2];
|
||||
if (negate) return { [p]: { $gt: v } };
|
||||
else return { [p]: { $lte: v } };
|
||||
}
|
||||
|
||||
throw new Error(`Unrecognized operator ${op}`);
|
||||
}
|
||||
|
||||
if (!isArray(exp)) {
|
||||
if (exp === true) return {};
|
||||
throw new Error("Primitives are not valid queries");
|
||||
}
|
||||
|
||||
return recursive(exp, false);
|
||||
}
|
||||
import {
|
||||
SynthContextBase,
|
||||
likeDisjoint,
|
||||
likeImplies,
|
||||
} from "./common/expression/synth";
|
||||
import normalize from "./common/expression/normalize";
|
||||
import { Clause } from "./common/expression/synth";
|
||||
import * as BI from "./common/expression/bigint";
|
||||
|
||||
export function processDeviceProjection(
|
||||
projection: Record<string, 1>
|
||||
@@ -506,7 +279,7 @@ export function convertOldPrecondition(
|
||||
}
|
||||
} else if (k === "_tags") {
|
||||
if (typeof v === "object") {
|
||||
if (isArray(v)) throw new Error(`Invalid type`);
|
||||
if (Array.isArray(v)) throw new Error(`Invalid type`);
|
||||
const conjs: Expression[] = [];
|
||||
for (const [op, val] of Object.entries(v)) {
|
||||
if (op === "$ne") {
|
||||
@@ -539,7 +312,7 @@ export function convertOldPrecondition(
|
||||
|
||||
expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
|
||||
} else if (typeof v === "object") {
|
||||
if (isArray(v)) throw new Error(`Invalid type`);
|
||||
if (Array.isArray(v)) throw new Error(`Invalid type`);
|
||||
|
||||
const exps: Expression[] = [];
|
||||
for (const [kk, vv] of Object.entries(v)) {
|
||||
@@ -683,3 +456,629 @@ export function preProcessPreset(
|
||||
preset.configurations = [configuration];
|
||||
return preset;
|
||||
}
|
||||
|
||||
type Minterm = number[];
|
||||
|
||||
function getParam(exp: Expression, collection: string): string {
|
||||
if (!Array.isArray(exp))
|
||||
throw new Error("Left-hand operand must be a parameter");
|
||||
|
||||
if (exp[0] !== "PARAM")
|
||||
throw new Error("Left-hand operand must be a parameter");
|
||||
|
||||
if (typeof exp[1] !== "string")
|
||||
throw new Error("Left-hand operand must be a parameter");
|
||||
|
||||
const p = exp[1];
|
||||
if (collection === "devices") {
|
||||
if (p === "DeviceID.ID") return "_id";
|
||||
else if (p === "DeviceID") return "_deviceId";
|
||||
else if (p.startsWith("DeviceID.")) return "_deviceId._" + p.slice(9);
|
||||
else if (p === "Events.Inform") return "_lastInform";
|
||||
else if (p === "Events.Registered") return "_registered";
|
||||
else if (p === "Events.0_BOOTSTRAP") return "_lastBootstrap";
|
||||
else if (p === "Events.1_BOOT") return "_lastBoot";
|
||||
else if (!p.endsWith("._value") && !p.startsWith("Tags."))
|
||||
return `${p}._value`;
|
||||
}
|
||||
|
||||
return p;
|
||||
}
|
||||
|
||||
function getTypes(parameter: string, collection: string): string[] {
|
||||
if (collection === "devices") {
|
||||
if (parameter === "_id") return ["string"];
|
||||
if (parameter === "_lastInform") return ["date"];
|
||||
if (parameter === "_registered") return ["date"];
|
||||
if (parameter === "_lastBootstrap") return ["date"];
|
||||
if (parameter === "_lastBoot") return ["date"];
|
||||
if (parameter.startsWith("_deviceId.")) return ["string"];
|
||||
if (parameter === "Reboot._value") return ["date"];
|
||||
if (parameter === "FactoryReset._value") return ["date"];
|
||||
if (parameter.endsWith("_timestamp")) return ["date"];
|
||||
if (parameter.startsWith("Downloads.")) {
|
||||
if (parameter.endsWith("Download._value")) return ["date"];
|
||||
if (parameter.endsWith("Time._value")) return ["date"];
|
||||
if (parameter.endsWith("Name._value")) return ["string"];
|
||||
if (parameter.endsWith("Type._value")) return ["string"];
|
||||
}
|
||||
if (parameter.endsWith("_value"))
|
||||
return ["bool", "number", "date", "string"];
|
||||
} else if (collection === "tasks") {
|
||||
if (parameter === "_id") return ["oid"];
|
||||
if (parameter === "timestamp") return ["date"];
|
||||
if (parameter === "expiry") return ["date"];
|
||||
if (parameter === "name") return ["string"];
|
||||
if (parameter === "device") return ["string"];
|
||||
} else if (collection === "faults") {
|
||||
if (parameter === "_id") return ["string"];
|
||||
if (parameter === "timestamp") return ["date"];
|
||||
if (parameter === "expiry") return ["date"];
|
||||
if (parameter === "code") return ["string"];
|
||||
if (parameter === "retries") return ["number"];
|
||||
if (parameter === "channel") return ["string"];
|
||||
if (parameter === "device") return ["string"];
|
||||
if (parameter === "message") return ["string"];
|
||||
} else if (collection === "users") {
|
||||
if (parameter === "_id") return ["string"];
|
||||
if (parameter === "roles") return ["string"];
|
||||
else if (parameter === "password")
|
||||
throw new Error("Cannot query restricted parameters");
|
||||
else if (parameter === "salt")
|
||||
throw new Error("Cannot query restricted parameters");
|
||||
} else if (collection === "config") {
|
||||
if (parameter === "_id") return ["string"];
|
||||
else if (parameter === "value") return ["string"];
|
||||
} else if (collection === "files") {
|
||||
if (parameter === "_id") return ["string"];
|
||||
if (parameter === "metadata.fileType") return ["string"];
|
||||
if (parameter === "metadata.oui") return ["string"];
|
||||
if (parameter === "metadata.productClass") return ["string"];
|
||||
if (parameter === "metadata.version") return ["string"];
|
||||
} else if (collection === "permissions") {
|
||||
if (parameter === "_id") return ["string"];
|
||||
if (parameter === "role") return ["string"];
|
||||
if (parameter === "resource") return ["devices"];
|
||||
if (parameter === "access") return ["number"];
|
||||
if (parameter === "filter") return ["string"];
|
||||
if (parameter === "validate") return ["string"];
|
||||
} else if (collection === "presets") {
|
||||
if (parameter === "_id") return ["string"];
|
||||
if (parameter === "weight") return ["number"];
|
||||
if (parameter === "channel") return ["string"];
|
||||
if (parameter === "precondition") return ["string"];
|
||||
if (parameter.startsWith("events.")) return ["bool"];
|
||||
} else if (collection === "provisions") {
|
||||
if (parameter === "_id") return ["string"];
|
||||
if (parameter === "script") return ["string"];
|
||||
} else if (collection === "virtualParameters") {
|
||||
if (parameter === "_id") return ["string"];
|
||||
if (parameter === "script") return ["string"];
|
||||
}
|
||||
|
||||
if (parameter === "_id") return ["oid", "string"];
|
||||
return ["bool", "number", "date", "string"];
|
||||
}
|
||||
|
||||
function roundOid(oid: string, roundUp: boolean): string {
|
||||
const match = (oid.match(/^[0-9a-f]*/)?.[0] ?? "").slice(0, 24);
|
||||
let num = BI.BigInt("0x0" + match);
|
||||
let lastChar = 0;
|
||||
if (oid.length > match.length) lastChar += oid.charCodeAt(match.length);
|
||||
if (match.length < 24) lastChar -= 48;
|
||||
if (lastChar > 0 && roundUp) num++;
|
||||
num <<= BI.BigInt(4 * (24 - match.length));
|
||||
if (lastChar < 0 && !roundUp) --num;
|
||||
const str = num.toString(16);
|
||||
if (str.length > 24 || str.startsWith("-")) return null;
|
||||
return str.padStart(24, "0");
|
||||
}
|
||||
|
||||
function groupBy<T, K>(
|
||||
input: T[],
|
||||
callback: (item: T) => K
|
||||
): Iterable<[K, T[]]> {
|
||||
const groups = new Map<K, T[]>();
|
||||
for (const item of input) {
|
||||
const key = callback(item);
|
||||
let arr = groups.get(key);
|
||||
if (!arr) groups.set(key, (arr = []));
|
||||
arr.push(item);
|
||||
}
|
||||
|
||||
return groups.entries();
|
||||
}
|
||||
|
||||
abstract class MongoClause {
|
||||
abstract readonly parameter: string;
|
||||
abstract toQuery(truthy: boolean): Filter<unknown>;
|
||||
toString(): string {
|
||||
return JSON.stringify(this.toQuery(true));
|
||||
}
|
||||
}
|
||||
|
||||
class MongoClauseArray extends MongoClause {
|
||||
constructor(
|
||||
public readonly parameter: string,
|
||||
public readonly value: string
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
toQuery(truthy: boolean): Filter<unknown> {
|
||||
if (truthy) return { [this.parameter]: { $eq: this.value } };
|
||||
else return { [this.parameter]: { $ne: this.value } };
|
||||
}
|
||||
}
|
||||
|
||||
class MongoClauseCompare<T> extends MongoClause {
|
||||
constructor(
|
||||
public readonly parameter: string,
|
||||
public readonly op: "$eq" | "$gt" | "$lt" | "$gte" | "$lte",
|
||||
public readonly value: T,
|
||||
public readonly type: "" | "bool" | "number" | "string" | "date" | "oid"
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
toQuery(truthy: boolean): Filter<unknown> {
|
||||
let v: any = this.value;
|
||||
if (this.type === "date" && typeof v === "number")
|
||||
v = { $date: new Date(v).toISOString() };
|
||||
if (this.type === "oid" && typeof v === "string") v = { $oid: v };
|
||||
|
||||
if (truthy) return { [this.parameter]: { [this.op]: v } };
|
||||
else if (this.op === "$eq") return { [this.parameter]: { $ne: v } };
|
||||
else return { [this.parameter]: { $not: { [this.op]: v } } };
|
||||
}
|
||||
}
|
||||
|
||||
class MongoClauseType extends MongoClause {
|
||||
constructor(public readonly parameter: string, public readonly type: string) {
|
||||
super();
|
||||
}
|
||||
|
||||
toQuery(truthy: boolean): Filter<unknown> {
|
||||
if (truthy) return { [this.parameter]: { $type: this.type } };
|
||||
else return { [this.parameter]: { $not: { $type: this.type } } };
|
||||
}
|
||||
}
|
||||
|
||||
class MongoClauseLike extends MongoClause {
|
||||
readonly pattern: string[];
|
||||
constructor(
|
||||
public readonly parameter: string,
|
||||
pat: string,
|
||||
esc: string,
|
||||
public readonly caseSensitive: boolean
|
||||
) {
|
||||
super();
|
||||
this.pattern = parseLikePattern(pat, esc);
|
||||
}
|
||||
|
||||
toQuery(truthy: boolean): Filter<unknown> {
|
||||
const convChars = {
|
||||
"-": "\\-",
|
||||
"/": "\\/",
|
||||
"\\": "\\/",
|
||||
"^": "\\^",
|
||||
$: "\\$",
|
||||
"*": "\\*",
|
||||
"+": "\\+",
|
||||
"?": "\\?",
|
||||
".": "\\.",
|
||||
"(": "\\(",
|
||||
")": "\\)",
|
||||
"|": "\\|",
|
||||
"[": "\\[",
|
||||
"]": "\\]",
|
||||
"{": "\\{",
|
||||
"}": "\\}",
|
||||
"\\%": ".*",
|
||||
"\\_": ".",
|
||||
};
|
||||
|
||||
const chars = this.pattern.map((c) => convChars[c] ?? c);
|
||||
chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
|
||||
const l = chars.length - 1;
|
||||
chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
|
||||
const pattern = chars.join("");
|
||||
const options = this.caseSensitive ? "s" : "is";
|
||||
if (truthy) {
|
||||
return { [this.parameter]: { $regularExpression: { options, pattern } } };
|
||||
} else {
|
||||
return {
|
||||
[this.parameter]: {
|
||||
$not: { $regularExpression: { options, pattern } },
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class MongoSynthContext extends SynthContextBase<Clause, MongoClause> {
|
||||
constructor(private readonly collection: string) {
|
||||
super();
|
||||
}
|
||||
|
||||
getMinterms(clause: Clause, res: true | false | null): number[][] {
|
||||
const exp = clause.expression();
|
||||
if (!Array.isArray(exp)) throw new Error("Invalid query expression");
|
||||
|
||||
if (res === null) {
|
||||
const minterms: number[][] = [];
|
||||
for (const dep of clause.getNullables()) {
|
||||
const e = dep.operand.expression();
|
||||
const param = getParam(e, this.collection);
|
||||
if (param.startsWith("Tags.") && this.collection === "devices") {
|
||||
const t = decodeTag(param.slice(5));
|
||||
const c = new MongoClauseArray("_tags", t);
|
||||
minterms.push([this.getVar(c) << 1]);
|
||||
continue;
|
||||
}
|
||||
const c = new MongoClauseCompare(param, "$eq", null, "");
|
||||
minterms.push([(this.getVar(c) << 1) ^ 1]);
|
||||
}
|
||||
return minterms;
|
||||
}
|
||||
|
||||
if ([">", "<", "="].includes(exp[0])) {
|
||||
const param = getParam(exp[1], this.collection);
|
||||
|
||||
let rhs = exp[2];
|
||||
if (typeof rhs === "boolean") rhs = +rhs;
|
||||
else if (typeof rhs !== "string" && typeof rhs !== "number")
|
||||
throw new Error(`Right-hand operand must be a literal value`);
|
||||
|
||||
if (
|
||||
param.startsWith("Tags.") &&
|
||||
this.collection === "devices" &&
|
||||
exp[0] === "="
|
||||
) {
|
||||
const t = decodeTag(param.slice(5));
|
||||
const c = new MongoClauseArray("_tags", t);
|
||||
if (typeof rhs === "string") rhs = 2;
|
||||
if (exp[0] === "=") {
|
||||
if ((rhs === 1) !== res) return [];
|
||||
} else if (exp[0] === ">") {
|
||||
if ((res && rhs >= 1) || (!res && rhs < 1)) return [];
|
||||
} else if (exp[0] === "<") {
|
||||
if ((res && rhs <= 1) || (!res && rhs > 1)) return [];
|
||||
}
|
||||
return [[(this.getVar(c) << 1) ^ 1]];
|
||||
}
|
||||
|
||||
let op: "$gt" | "$lt" | "$eq" | "$gte" | "$lte";
|
||||
if (exp[0] === "=") op = "$eq";
|
||||
else if (exp[0] === ">") op = res ? "$gt" : "$lte";
|
||||
else if (exp[0] === "<") op = res ? "$lt" : "$gte";
|
||||
|
||||
const possibleTypes = new Set(getTypes(param, this.collection));
|
||||
const clauses: MongoClause[] = [];
|
||||
|
||||
if (typeof rhs === "number") {
|
||||
if (possibleTypes.has("number"))
|
||||
clauses.push(new MongoClauseCompare(param, op, rhs, "number"));
|
||||
|
||||
if (possibleTypes.has("date"))
|
||||
clauses.push(new MongoClauseCompare(param, op, rhs, "date"));
|
||||
if (possibleTypes.has("bool") && (rhs === 0 || rhs === 1))
|
||||
clauses.push(new MongoClauseCompare(param, op, !!rhs, "bool"));
|
||||
} else if (typeof rhs === "string") {
|
||||
if (possibleTypes.has("string"))
|
||||
clauses.push(new MongoClauseCompare(param, op, rhs, "string"));
|
||||
|
||||
if (possibleTypes.has("oid")) {
|
||||
const oid = roundOid(rhs, op === "$lt" || op === "$lte");
|
||||
if (oid && (op !== "$eq" || oid === rhs))
|
||||
clauses.push(new MongoClauseCompare(param, op, oid, "oid"));
|
||||
}
|
||||
}
|
||||
|
||||
if (op === "$eq" && !res) {
|
||||
clauses.push(new MongoClauseCompare(param, "$eq", null, ""));
|
||||
return [clauses.map((c) => this.getVar(c) << 1)];
|
||||
}
|
||||
|
||||
// In the following clauses we could use $type operator, but we want the
|
||||
// final query to use comparison operators to check for type because the
|
||||
// $type operator in MongoDB doesn't use indexes
|
||||
if (typeof rhs === "number") {
|
||||
if (possibleTypes.has("bool")) {
|
||||
if (
|
||||
(rhs > 1 && (op === "$lt" || op === "$lte")) ||
|
||||
(rhs < 0 && (op === "$gt" || op === "$gte"))
|
||||
)
|
||||
clauses.push(new MongoClauseCompare(param, "$gte", false, "bool"));
|
||||
}
|
||||
|
||||
if (op === "$gt" || op === "$gte") {
|
||||
if (possibleTypes.has("string"))
|
||||
clauses.push(new MongoClauseCompare(param, "$gte", "", "string"));
|
||||
|
||||
if (possibleTypes.has("oid")) {
|
||||
clauses.push(
|
||||
new MongoClauseCompare(
|
||||
param,
|
||||
"$gte",
|
||||
"000000000000000000000000",
|
||||
"oid"
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
} else if (typeof rhs === "string") {
|
||||
if (op === "$lt" || op === "$lte") {
|
||||
if (possibleTypes.has("bool"))
|
||||
clauses.push(new MongoClauseCompare(param, "$gte", false, "bool"));
|
||||
|
||||
if (possibleTypes.has("number")) {
|
||||
clauses.push(new MongoClauseCompare(param, "$gte", 0, "number"));
|
||||
clauses.push(new MongoClauseCompare(param, "$lt", 0, "number"));
|
||||
}
|
||||
if (possibleTypes.has("date")) {
|
||||
clauses.push(new MongoClauseCompare(param, "$gte", 0, "date"));
|
||||
clauses.push(new MongoClauseCompare(param, "$lt", 0, "date"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return clauses.map((c) => [(this.getVar(c) << 1) ^ 1]);
|
||||
} else if (exp[0] === "LIKE") {
|
||||
const pat = exp[2];
|
||||
if (typeof pat !== "string")
|
||||
throw new Error("Right-hand operand of 'LIKE' must be a string");
|
||||
let p = exp[1];
|
||||
let caseSensitive = true;
|
||||
if (
|
||||
Array.isArray(p) &&
|
||||
p[0] === "FUNC" &&
|
||||
["UPPER", "LOWER"].includes(p[1])
|
||||
) {
|
||||
if (p[1] === "UPPER" && pat !== pat.toUpperCase())
|
||||
return res ? [] : [[]];
|
||||
if (p[1] === "LOWER" && pat !== pat.toLowerCase())
|
||||
return res ? [] : [[]];
|
||||
caseSensitive = false;
|
||||
p = p[2];
|
||||
}
|
||||
|
||||
const param = getParam(p, this.collection);
|
||||
const c = new MongoClauseLike(param, pat, exp[3], caseSensitive);
|
||||
if (res) return [[(this.getVar(c) << 1) ^ 1]];
|
||||
const typeClase = new MongoClauseType(param, "string");
|
||||
const r = [[this.getVar(c) << 1, (this.getVar(typeClase) << 1) ^ 1]];
|
||||
return r;
|
||||
} else {
|
||||
throw new Error("Invalid query expression");
|
||||
}
|
||||
}
|
||||
|
||||
getDcSet(minterms: Minterm[]): number[][] {
|
||||
const dcSet: number[][] = [];
|
||||
|
||||
const vars = new Set(minterms.flat().map((l) => l >> 1));
|
||||
const clauses = Array.from(vars).map((v) => this.getClause(v));
|
||||
|
||||
for (const [parameter, clauses2] of groupBy(clauses, (c) => c.parameter)) {
|
||||
const comparisons = clauses2.filter(
|
||||
(c) => c instanceof MongoClauseCompare && c.value !== null
|
||||
) as MongoClauseCompare<unknown>[];
|
||||
|
||||
for (const [type, clauses3] of groupBy(comparisons, (c) => c.type)) {
|
||||
const isType = this.getVar(new MongoClauseType(parameter, type));
|
||||
const values = new Set(clauses3.map((c) => c.value));
|
||||
const valuesSorted = Array.from(values).sort((a, b) =>
|
||||
a > b ? 1 : -1
|
||||
);
|
||||
for (const [i, v] of valuesSorted.entries()) {
|
||||
const eq = this.getVar(
|
||||
new MongoClauseCompare(parameter, "$eq", v, type)
|
||||
);
|
||||
const gt = this.getVar(
|
||||
new MongoClauseCompare(parameter, "$gt", v, type)
|
||||
);
|
||||
const lt = this.getVar(
|
||||
new MongoClauseCompare(parameter, "$lt", v, type)
|
||||
);
|
||||
const gte = this.getVar(
|
||||
new MongoClauseCompare(parameter, "$gte", v, type)
|
||||
);
|
||||
const lte = this.getVar(
|
||||
new MongoClauseCompare(parameter, "$lte", v, type)
|
||||
);
|
||||
|
||||
if (type === "bool") {
|
||||
if (v === false) dcSet.push([(lt << 1) ^ 1]);
|
||||
else if (v === true) dcSet.push([(gt << 1) ^ 1]);
|
||||
} else if (type === "string") {
|
||||
if (v === "") dcSet.push([(lt << 1) ^ 1]);
|
||||
} else if (type === "oid") {
|
||||
if (v < "000000000000000000000000") dcSet.push([(lte << 1) ^ 1]);
|
||||
else if (v === "000000000000000000000000")
|
||||
dcSet.push([(lt << 1) ^ 1]);
|
||||
else if (v > "ffffffffffffffffffffffff")
|
||||
dcSet.push([(gte << 1) ^ 1]);
|
||||
else if (v === "ffffffffffffffffffffffff")
|
||||
dcSet.push([(gt << 1) ^ 1]);
|
||||
}
|
||||
dcSet.push([(lt << 1) ^ 1, (gte << 1) ^ 1]);
|
||||
dcSet.push([(gt << 1) ^ 1, (gte << 1) ^ 0]);
|
||||
dcSet.push([(eq << 1) ^ 0, (lt << 1) ^ 0, (lte << 1) ^ 1]);
|
||||
dcSet.push([(eq << 1) ^ 1, (gte << 1) ^ 0]);
|
||||
dcSet.push([(eq << 1) ^ 1, (gt << 1) ^ 1]);
|
||||
|
||||
if (i === 0) {
|
||||
dcSet.push([(isType << 1) ^ 0, (gte << 1) ^ 1]);
|
||||
dcSet.push([(isType << 1) ^ 0, (lt << 1) ^ 1]);
|
||||
} else {
|
||||
const gt2 = this.getVar(
|
||||
new MongoClauseCompare(
|
||||
parameter,
|
||||
"$gt",
|
||||
valuesSorted[i - 1],
|
||||
type
|
||||
)
|
||||
);
|
||||
const lte2 = this.getVar(
|
||||
new MongoClauseCompare(
|
||||
parameter,
|
||||
"$lte",
|
||||
valuesSorted[i - 1],
|
||||
type
|
||||
)
|
||||
);
|
||||
dcSet.push([(gt2 << 1) ^ 0, (gte << 1) ^ 1]);
|
||||
dcSet.push([(gt2 << 1) ^ 0, (lte2 << 1) ^ 0, (lt << 1) ^ 1]);
|
||||
}
|
||||
|
||||
if (i === valuesSorted.length - 1)
|
||||
dcSet.push([(isType << 1) ^ 1, (gt << 1) ^ 0, (lte << 1) ^ 0]);
|
||||
}
|
||||
}
|
||||
|
||||
const likes = clauses2.filter(
|
||||
(c) => c instanceof MongoClauseLike
|
||||
) as MongoClauseLike[];
|
||||
if (likes.length) {
|
||||
const isType = this.getVar(new MongoClauseType(parameter, "string"));
|
||||
for (let i1 = 0; i1 < likes.length; ++i1) {
|
||||
const l1 = likes[i1];
|
||||
let p1 = l1.pattern;
|
||||
dcSet.push([(this.getVar(l1) << 1) ^ 1, (isType << 1) ^ 0]);
|
||||
for (let i2 = i1 + 1; i2 < likes.length; ++i2) {
|
||||
const l2 = likes[i2];
|
||||
let p2 = l2.pattern;
|
||||
if (!l1.caseSensitive || !l2.caseSensitive) {
|
||||
p1 = p1.map((c) => c.toLowerCase());
|
||||
p2 = p2.map((c) => c.toLowerCase());
|
||||
}
|
||||
if (likeDisjoint(p1, p2)) {
|
||||
dcSet.push([
|
||||
(this.getVar(l1) << 1) ^ 1,
|
||||
(this.getVar(l2) << 1) ^ 1,
|
||||
]);
|
||||
} else if (
|
||||
(!l1.caseSensitive || l2.caseSensitive) &&
|
||||
likeImplies(p1, p2)
|
||||
) {
|
||||
dcSet.push([
|
||||
(this.getVar(l1) << 1) ^ 0,
|
||||
(this.getVar(l2) << 1) ^ 1,
|
||||
]);
|
||||
} else if (
|
||||
(!l2.caseSensitive || l1.caseSensitive) &&
|
||||
likeImplies(p2, p1)
|
||||
) {
|
||||
dcSet.push([
|
||||
(this.getVar(l1) << 1) ^ 1,
|
||||
(this.getVar(l2) << 1) ^ 0,
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const isNull = this.getVar(
|
||||
new MongoClauseCompare(parameter, "$eq", null, "")
|
||||
);
|
||||
|
||||
const types = getTypes(parameter, this.collection).map((t) =>
|
||||
this.getVar(new MongoClauseType(parameter, t))
|
||||
);
|
||||
|
||||
dcSet.push([(isNull << 1) ^ 0, ...types.map((t) => (t << 1) ^ 0)]);
|
||||
for (let i = 0; i < types.length; ++i) {
|
||||
const t1 = types[i];
|
||||
dcSet.push([(t1 << 1) ^ 1, (isNull << 1) ^ 1]);
|
||||
for (let j = i + 1; j < types.length; ++j) {
|
||||
const t2 = types[j];
|
||||
dcSet.push([(t1 << 1) ^ 1, (t2 << 1) ^ 1]);
|
||||
}
|
||||
}
|
||||
|
||||
if (parameter === "_id") dcSet.push([(isNull << 1) ^ 1]);
|
||||
}
|
||||
return dcSet;
|
||||
}
|
||||
|
||||
canRaise(i: number, s: Set<number>): boolean {
|
||||
if (!(i & 1)) return true;
|
||||
const c = this.getClause(i >> 1);
|
||||
if (c instanceof MongoClauseCompare) {
|
||||
for (const j of s) {
|
||||
if (j === i) continue;
|
||||
const c2 = this.getClause(j >> 1);
|
||||
if (c2.parameter !== c.parameter) continue;
|
||||
if (!(j & 1)) return false;
|
||||
if (c2 instanceof MongoClauseType) return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
toQuery(minterms: Minterm[]): Filter<unknown> {
|
||||
const or = [];
|
||||
|
||||
for (const minterm of minterms) {
|
||||
const query = {};
|
||||
loop: for (const clause of minterm) {
|
||||
if (!minterm.length) return {};
|
||||
const negate = !!(clause & 1);
|
||||
const c = this.getClause(clause >> 1);
|
||||
const q = c.toQuery(negate);
|
||||
if (Object.keys(q).length !== 1)
|
||||
throw new Error("Invalid query expression");
|
||||
const [param, value] = Object.entries(q)[0];
|
||||
if (Object.getPrototypeOf(value).constructor !== Object)
|
||||
throw new Error("Invalid query expression");
|
||||
const dests = [query, ...(query["$and"] ?? [])];
|
||||
for (const dest of dests) {
|
||||
if (!(param in dest)) {
|
||||
dest[param] = value;
|
||||
continue loop;
|
||||
}
|
||||
|
||||
let src = value;
|
||||
let dst = dest[param];
|
||||
|
||||
if (src["$not"] && dst["$not"]) {
|
||||
src = src["$not"];
|
||||
dst = dst["$not"];
|
||||
}
|
||||
|
||||
if (!Object.keys(src).some((k) => k in dst)) {
|
||||
Object.assign(dst, src);
|
||||
continue loop;
|
||||
}
|
||||
}
|
||||
|
||||
query["$and"] ??= [];
|
||||
query["$and"].push({ [param]: value });
|
||||
}
|
||||
or.push(query);
|
||||
}
|
||||
|
||||
if (or.length === 1) return or[0];
|
||||
return { $or: or };
|
||||
}
|
||||
}
|
||||
|
||||
export function toMongoQuery(
|
||||
exp: Expression,
|
||||
resource: string
|
||||
): Filter<unknown> | false {
|
||||
exp = normalize(exp);
|
||||
const clause = Clause.fromExpression(normalize(exp));
|
||||
const context = new MongoSynthContext(resource);
|
||||
const minterms = clause.true(context);
|
||||
const minimized = context.minimize(minterms);
|
||||
if (!minimized.length) return false;
|
||||
return EJSON.deserialize(context.toQuery(minimized));
|
||||
}
|
||||
|
||||
export function validQuery(exp: Expression, resource: string): void {
|
||||
const clause = Clause.fromExpression(normalize(exp));
|
||||
const context = new MongoSynthContext(resource);
|
||||
clause.true(context);
|
||||
}
|
||||
Reference in new issue
Block a user