From 3169e829e781e7f7e187e76a2e38c0430394fd0d Mon Sep 17 00:00:00 2001 From: Zaid Humaimidi Date: Thu, 6 Jul 2023 15:17:52 -0700 Subject: [PATCH] Refactor expression to Mongo query translation The previous approach was simplistic and could result in inefficient queries in certain edge cases, notably causing MongoDB to not utilize existing indexes when querying by last inform timestamps given upper and lower bounds. The new implementation leverages our existing expression minimizer code and can therefore generate more efficient queries or at least queries that can take advantage of existing indexes. --- lib/api-functions.ts | 2 +- lib/common/authorizer.ts | 2 +- lib/common/boolean-expression.ts | 1234 ----------------- lib/common/{ => expression}/bigint.ts | 0 lib/common/expression/normalize.ts | 492 +++++++ .../parser.ts} | 7 +- lib/common/expression/synth.ts | 914 ++++++++++++ .../{expression.ts => expression/util.ts} | 4 +- lib/connection-request.ts | 2 +- lib/cwmp.ts | 6 +- lib/db.ts | 2 +- lib/local-cache.ts | 4 +- lib/mongodb-functions.ts | 889 ++++++++---- lib/query.ts | 2 +- lib/ui/api.ts | 4 +- lib/ui/db.ts | 47 +- npm-shrinkwrap.json | 14 +- package.json | 2 +- test/mongodb-functions.ts | 100 +- test/{boolean-expression.ts => synth.ts} | 4 +- ui/components/all-parameters.ts | 2 +- ui/components/container.ts | 2 +- ui/components/parameter-table.ts | 2 +- ui/components/parameter.ts | 4 +- ui/config-page.ts | 2 +- ui/devices-page.ts | 7 +- ui/faults-page.ts | 2 +- ui/files-page.ts | 2 +- ui/filter-component.ts | 33 +- ui/permissions-page.ts | 2 +- ui/pie-chart-component.ts | 2 +- ui/presets-page.ts | 2 +- ui/provisions-page.ts | 2 +- ui/store.ts | 6 +- ui/ui-config-component.ts | 2 +- ui/users-page.ts | 2 +- ui/virtual-parameters-page.ts | 2 +- 37 files changed, 2175 insertions(+), 1632 deletions(-) delete mode 100644 lib/common/boolean-expression.ts rename lib/common/{ => expression}/bigint.ts (100%) create mode 100644 lib/common/expression/normalize.ts rename lib/common/{expression-parser.ts => expression/parser.ts} (98%) create mode 100644 lib/common/expression/synth.ts rename lib/common/{expression.ts => expression/util.ts} (99%) rename test/{boolean-expression.ts => synth.ts} (96%) diff --git a/lib/api-functions.ts b/lib/api-functions.ts index 63927cf..adf5f97 100644 --- a/lib/api-functions.ts +++ b/lib/api-functions.ts @@ -31,7 +31,7 @@ import { } from "./connection-request"; import { Expression, Task } from "./types"; import { flattenDevice } from "./mongodb-functions"; -import { evaluate } from "./common/expression"; +import { evaluate } from "./common/expression/util"; export async function connectionRequest( deviceId: string, diff --git a/lib/common/authorizer.ts b/lib/common/authorizer.ts index 0d7929c..3bb7eab 100644 --- a/lib/common/authorizer.ts +++ b/lib/common/authorizer.ts @@ -18,7 +18,7 @@ */ import { PermissionSet, Expression } from "../types"; -import { evaluate, or } from "./expression"; +import { evaluate, or } from "./expression/util"; export default class Authorizer { private declare permissionSets: PermissionSet[]; diff --git a/lib/common/boolean-expression.ts b/lib/common/boolean-expression.ts deleted file mode 100644 index 08127ad..0000000 --- a/lib/common/boolean-expression.ts +++ /dev/null @@ -1,1234 +0,0 @@ -/** - * Copyright 2013-2020 GenieACS Inc. - * - * This file is part of GenieACS. - * - * GenieACS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * GenieACS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with GenieACS. If not, see . - */ - -import * as BI from "./bigint"; -import { espresso, complement, tautology } from "espresso-iisojs"; -import { Expression } from "../types"; -import { map } from "./expression-parser"; -import { and, evaluateCallback } from "./expression"; - -const ZERO = BI.BigInt(0); -const ONE = BI.BigInt(1); -const TWO = BI.BigInt(2); -const NEGATIVE_ONE = BI.BigInt(-1); - -class Indeterminates { - public declare map: Map; - public declare sortedKeys: string[]; - - public constructor(str?: string) { - this.map = new Map(); - if (str) { - this.map.set(str, 1); - this.sortedKeys = [str]; - } else { - this.sortedKeys = []; - } - } - - public reciprocal(): Indeterminates { - const res = new Indeterminates(); - res.sortedKeys = this.sortedKeys; - res.map = new Map(); - for (const [k, v] of this.map) res.map.set(k, 0 - v); - return res; - } - - public static multiply( - indeterminates1: Indeterminates, - indeterminates2: Indeterminates - ): Indeterminates { - const res = new Indeterminates(); - res.sortedKeys = indeterminates1.sortedKeys.slice(); - res.map = new Map(indeterminates1.map); - - for (const [key, val] of indeterminates2.map) { - const v = res.map.get(key); - if (!v) { - res.map.set(key, val); - res.sortedKeys.push(key); - } else { - const v2 = val + v; - if (!v2) { - res.map.delete(key); - res.sortedKeys = res.sortedKeys.filter((s) => s !== key); - } else { - res.map.set(key, v2); - } - } - } - - res.sortedKeys.sort((a, b) => { - if (a.length !== b.length) return b.length - a.length; - else if (a > b) return 1; - else if (a < b) return -1; - return 0; - }); - - return res; - } - - public static compare(a: Indeterminates, b: Indeterminates): number { - if (a.sortedKeys.length !== b.sortedKeys.length) - return b.sortedKeys.length - a.sortedKeys.length; - for (let i = 0; i < a.sortedKeys.length; ++i) { - const k1 = a.sortedKeys[i]; - const w1 = a.map.get(k1); - const k2 = b.sortedKeys[i]; - const w2 = b.map.get(k2); - if (w1 !== w2) return w2 - w1; - if (k1.length > k2.length) return -1; - else if (k1.length < k2.length) return 1; - else if (k1 > k2) return 1; - else if (k1 < k2) return -1; - } - return 0; - } -} - -interface Term { - indeterminates: Indeterminates; - coefficientNumerator: BI.bigint; - coefficientDenominator: BI.bigint; -} - -function findGcd(a: BI.bigint, b: BI.bigint): BI.bigint { - while (BI.ne(b, ZERO)) { - const t = b; - b = BI.rem(a, b); - a = t; - } - return a; -} - -class Polynomial { - public declare terms: Term[]; - - public constructor(terms: Term[]) { - this.terms = terms; - } - - public static simplifyTerms(terms: Term[]): Term[] { - const ts = terms - .slice() - .sort((a: Term, b: Term) => - Indeterminates.compare(a.indeterminates, b.indeterminates) - ); - - for (let i = 1; i < ts.length; ++i) { - const t1 = ts[i - 1]; - const t2 = ts[i]; - if (Indeterminates.compare(t1.indeterminates, t2.indeterminates) === 0) { - const numerator = BI.add( - BI.mul(t1.coefficientNumerator, t2.coefficientDenominator), - BI.mul(t2.coefficientNumerator, t1.coefficientDenominator) - ); - - const denominator = BI.mul( - t1.coefficientDenominator, - t2.coefficientDenominator - ); - - const gcd = findGcd(numerator, denominator); - - ts[i] = { - indeterminates: t2.indeterminates, - coefficientNumerator: BI.div(numerator, gcd), - coefficientDenominator: BI.div(denominator, gcd), - }; - ts[i - 1] = { - indeterminates: t1.indeterminates, - coefficientNumerator: ZERO, - coefficientDenominator: t1.coefficientDenominator, - }; - } - } - return ts.filter((v) => BI.ne(v.coefficientNumerator, ZERO)); - } - - public static fromIndeterminate(indeterminate: Expression): Polynomial { - const indeterminates = new Indeterminates(JSON.stringify(indeterminate)); - const terms = [ - { - indeterminates: indeterminates, - coefficientNumerator: ONE, - coefficientDenominator: ONE, - }, - ]; - return new Polynomial(terms); - } - - public static fromConstant(constant: number): Polynomial { - const [int, frac] = Math.abs(constant).toString(2).split(".", 2); - let numerator = BI.BigInt("0b" + int); - if (constant < 0) numerator = BI.mul(numerator, NEGATIVE_ONE); - let denominator = ONE; - if (frac) { - denominator = BI.exp(TWO, BI.BigInt(frac.length)); - numerator = BI.add( - BI.mul(numerator, denominator), - BI.BigInt("0b" + frac) - ); - } - - const terms = [ - { - indeterminates: new Indeterminates(), - coefficientNumerator: numerator, - coefficientDenominator: denominator, - }, - ]; - - return new Polynomial(terms); - } - - public negation(): Polynomial { - const terms = this.terms.map((t) => ({ - indeterminates: t.indeterminates, - coefficientNumerator: BI.mul(t.coefficientNumerator, NEGATIVE_ONE), - coefficientDenominator: t.coefficientDenominator, - })); - - return new Polynomial(terms); - } - - public reciprocal(): Polynomial { - const terms = this.terms.map((t) => ({ - indeterminates: t.indeterminates.reciprocal(), - coefficientNumerator: t.coefficientDenominator, - coefficientDenominator: t.coefficientNumerator, - })); - return new Polynomial(terms); - } - - public constant(): Polynomial { - const terms = this.terms.filter((t) => !t.indeterminates.sortedKeys.length); - return new Polynomial(terms); - } - - public add(rhs: Polynomial): Polynomial { - return new Polynomial( - Polynomial.simplifyTerms(this.terms.concat(rhs.terms)) - ); - } - - public subtract(rhs: Polynomial): Polynomial { - return this.add(rhs.negation()); - } - - public multiply(rhs: Polynomial): Polynomial { - const terms: Term[] = []; - - for (const t1 of this.terms) { - for (const t2 of rhs.terms) { - const numerator = BI.mul( - t1.coefficientNumerator, - t2.coefficientNumerator - ); - const denominator = BI.mul( - t1.coefficientDenominator, - t2.coefficientDenominator - ); - const gcd = findGcd(numerator, denominator); - - terms.push({ - indeterminates: Indeterminates.multiply( - t1.indeterminates, - t2.indeterminates - ), - coefficientNumerator: BI.div(numerator, gcd), - coefficientDenominator: BI.div(denominator, gcd), - }); - } - } - - return new Polynomial(Polynomial.simplifyTerms(terms)); - } - - public divide(rhs: Polynomial): Polynomial { - return this.multiply(rhs.reciprocal()); - } - - public toString(): string { - const add: string[] = []; - for (const t of this.terms) { - const coefficient = - BI.toNumber(t.coefficientNumerator) / - BI.toNumber(t.coefficientDenominator); - - const mul: string[] = []; - if (t.indeterminates.sortedKeys.length) { - for (const k of t.indeterminates.sortedKeys) { - const w = t.indeterminates.map.get(k); - for (let i = Math.abs(w); i > 0; --i) { - if (w > 0) mul.push(k); - else mul.push(`["/",1,${k}]`); - } - } - - if (coefficient !== 1) mul.push(coefficient.toString()); - - if (mul.length > 1) add.push(`["*",${mul.join(",")}]`); - else add.push(mul["0"]); - } else { - add.push(coefficient.toString()); - } - } - - if (!add.length) return "0"; - else if (add.length === 1) return add[0]; - else return `["+",${add.join(",")}]`; - } -} - -type Minterm = number[]; - -abstract class BoolExprSynth { - public abstract true(context: SynthContext): Minterm[]; - public abstract false(context: SynthContext): Minterm[]; - public abstract null(context: SynthContext): Minterm[]; -} - -class TrueSynth extends BoolExprSynth { - public true(): Minterm[] { - return [[]]; - } - public false(): Minterm[] { - return []; - } - public null(): Minterm[] { - return []; - } -} - -class FalseSynth extends BoolExprSynth { - public true(): Minterm[] { - return []; - } - public false(): Minterm[] { - return [[]]; - } - public null(): Minterm[] { - return []; - } -} - -class NullSynth extends BoolExprSynth { - public true(): Minterm[] { - return []; - } - public false(): Minterm[] { - return []; - } - public null(): Minterm[] { - return [[]]; - } -} - -class VarSynth extends BoolExprSynth { - private declare exp: Expression; - private declare negate: boolean; - public constructor(exp: Expression) { - super(); - this.negate = false; - if (Array.isArray(exp)) { - const op = exp[0]; - if (op === "<>") { - exp = exp.slice(); - exp[0] = "="; - this.negate = true; - } else if (op === ">=") { - exp = exp.slice(); - exp[0] = "<"; - this.negate = true; - } else if (op === "<=") { - exp = exp.slice(); - exp[0] = ">"; - this.negate = true; - } else if (op === "NOT LIKE") { - exp = exp.slice(); - exp[0] = "LIKE"; - this.negate = true; - } - } - this.exp = exp; - } - public true(context: SynthContext): Minterm[] { - const v = context.getVariable(this.exp); - return [[(v << 2) ^ (this.negate ? 1 : 3)]]; - } - public false(context: SynthContext): Minterm[] { - const v = context.getVariable(this.exp); - return [[(v << 2) ^ (this.negate ? 3 : 1)]]; - } - public null(context: SynthContext): Minterm[] { - const v = context.getVariable(this.exp); - return [[v << 2, (v << 2) ^ 2]]; - } -} - -class NotSynth extends BoolExprSynth { - private declare exprSynth: BoolExprSynth; - public constructor(e: BoolExprSynth) { - super(); - this.exprSynth = e; - } - public true(context: SynthContext): Minterm[] { - return this.exprSynth.false(context); - } - public false(context: SynthContext): Minterm[] { - return this.exprSynth.true(context); - } - public null(context: SynthContext): Minterm[] { - return this.exprSynth.null(context); - } -} - -class IsNullSynth extends BoolExprSynth { - private declare exprSynth: BoolExprSynth; - public constructor(e: BoolExprSynth) { - super(); - this.exprSynth = e; - } - public true(context: SynthContext): Minterm[] { - return this.exprSynth.null(context); - } - public false(context: SynthContext): Minterm[] { - return [...this.exprSynth.true(context), ...this.exprSynth.false(context)]; - } - public null(): Minterm[] { - return []; - } -} - -class OrSynth extends BoolExprSynth { - private declare exprSynths: BoolExprSynth[]; - public constructor(...e: BoolExprSynth[]) { - super(); - this.exprSynths = e.filter((ee) => !(ee instanceof FalseSynth)); - const unpacked: BoolExprSynth[] = []; - this.exprSynths = this.exprSynths.filter((ee) => { - if (ee instanceof OrSynth) { - unpacked.push(...ee.exprSynths); - return false; - } - return true; - }); - this.exprSynths.push(...unpacked); - } - public true(context: SynthContext): Minterm[] { - if (this.exprSynths.length === 0) return []; - if (this.exprSynths.length === 1) return this.exprSynths[0].true(context); - if (this.exprSynths.some((e) => e instanceof TrueSynth)) return [[]]; - return this.exprSynths.map((e) => e.true(context)).flat(); - } - public false(context: SynthContext): Minterm[] { - if (this.exprSynths.length === 0) return [[]]; - if (this.exprSynths.length === 1) return this.exprSynths[0].false(context); - if ( - this.exprSynths.some( - (e) => e instanceof TrueSynth || e instanceof NullSynth - ) - ) - return []; - return complement( - this.exprSynths.map((e) => complement(e.false(context))).flat() - ); - } - public null(context: SynthContext): Minterm[] { - if (this.exprSynths.length === 0) return []; - if (this.exprSynths.length === 1) return this.exprSynths[0].null(context); - const n = this.exprSynths.map((e) => e.null(context)).flat(); - const t = this.exprSynths.map((e) => e.true(context)).flat(); - return complement([...complement(n), ...t]); - } -} - -class AndSynth extends BoolExprSynth { - private declare exprSynths: BoolExprSynth[]; - public constructor(...e: BoolExprSynth[]) { - super(); - this.exprSynths = e.filter((ee) => !(ee instanceof TrueSynth)); - const unpacked: BoolExprSynth[] = []; - this.exprSynths = this.exprSynths.filter((ee) => { - if (ee instanceof AndSynth) { - unpacked.push(...ee.exprSynths); - return false; - } - return true; - }); - this.exprSynths.push(...unpacked); - } - public true(context: SynthContext): Minterm[] { - if (this.exprSynths.length === 0) return [[]]; - if (this.exprSynths.length === 1) return this.exprSynths[0].true(context); - if ( - this.exprSynths.some( - (e) => e instanceof FalseSynth || e instanceof NullSynth - ) - ) - return []; - return complement( - this.exprSynths.map((e) => complement(e.true(context))).flat() - ); - } - public false(context: SynthContext): Minterm[] { - if (this.exprSynths.length === 0) return []; - if (this.exprSynths.length === 1) return this.exprSynths[0].false(context); - if (this.exprSynths.some((e) => e instanceof FalseSynth)) return [[]]; - return this.exprSynths.map((e) => e.false(context)).flat(); - } - public null(context: SynthContext): Minterm[] { - if (this.exprSynths.length === 0) return []; - if (this.exprSynths.length === 1) return this.exprSynths[0].null(context); - const n = this.exprSynths.map((e) => e.null(context)).flat(); - const f = this.exprSynths.map((e) => e.false(context)).flat(); - return complement([...complement(n), ...f]); - } -} - -class CaseSynth extends BoolExprSynth { - private declare exprSynths: BoolExprSynth[]; - public constructor(e: BoolExprSynth[]) { - super(); - this.exprSynths = e; - } - public true(context: SynthContext): Minterm[] { - const minterms: Minterm[] = []; - const cumulative: Minterm[] = []; - for (let i = 0; i < this.exprSynths.length; i += 2) { - const w = this.exprSynths[i].true(context); - const t = this.exprSynths[i + 1].true(context); - minterms.push( - ...complement([...cumulative, ...complement(w), ...complement(t)]) - ); - if (i < this.exprSynths.length - 2) { - cumulative.push( - ...complement([ - ...this.exprSynths[i].false(context), - ...this.exprSynths[i].null(context), - ]) - ); - } - } - return minterms; - } - public false(context: SynthContext): Minterm[] { - const minterms: Minterm[] = []; - const cumulative: Minterm[] = []; - for (let i = 0; i < this.exprSynths.length; i += 2) { - const w = this.exprSynths[i].true(context); - const t = this.exprSynths[i + 1].false(context); - minterms.push( - ...complement([...cumulative, ...complement(w), ...complement(t)]) - ); - if (i < this.exprSynths.length - 2) { - cumulative.push( - ...complement([ - ...this.exprSynths[i].false(context), - ...this.exprSynths[i].null(context), - ]) - ); - } - } - return minterms; - } - public null(context: SynthContext): Minterm[] { - const minterms: Minterm[] = []; - const cumulative: Minterm[] = []; - for (let i = 0; i < this.exprSynths.length; i += 2) { - const w = this.exprSynths[i].true(context); - const t = this.exprSynths[i + 1].null(context); - minterms.push( - ...complement([...cumulative, ...complement(w), ...complement(t)]) - ); - cumulative.push( - ...complement([ - ...this.exprSynths[i].false(context), - ...this.exprSynths[i].null(context), - ]) - ); - } - minterms.push(...complement([...cumulative])); - return minterms; - } -} - -const ADDITIVE_IDENTITY = Polynomial.fromConstant(0); -const MULTIPLICATIVE_IDENTITY = Polynomial.fromConstant(1); - -const SWAPPED_OPS = { - "=": "=", - "<>": "<>", - ">": "<", - ">=": "<=", - "<": ">", - "<=": ">=", -}; - -function normalizeCallback(exp: Expression): Expression { - if (!Array.isArray(exp)) return exp; - const op = exp[0]; - - if (op === "FUNC" && exp[1] === "COALESCE") { - const res: Expression[] = ["CASE"]; - for (let i = 2; i < exp.length; ++i) - res.push(normalizeCallback(["IS NOT NULL", exp[i]]), exp[i]); - return normalizeCallback(res); - } - - if (op === "CASE") { - const res = [] as [Expression, Expression][]; - for (let i = 1; i < exp.length; i += 2) { - let w = exp[i]; - if (w instanceof Polynomial) w = JSON.parse(w.toString()); - if (!Array.isArray(w) && !w) continue; - const t = exp[i + 1]; - if (!Array.isArray(t) || t[0] !== "CASE") { - res.push([w, t]); - continue; - } - for (let j = 1; j < t.length; j += 2) res.push([and(w, t[j]), t[j + 1]]); - res.push([w, null]); - if (!Array.isArray(w) && w) break; - } - while (res[res.length - 1][1] == null) res.pop(); - return ["CASE", ...res.flat()]; - } - - const permutations: Map = new Map(); - for (const [i, e] of exp.entries()) { - if (!Array.isArray(e)) continue; - if (e[0] !== "CASE") continue; - const perms: [Expression, Expression][] = []; - for (let j = 1; j < e.length; j += 2) perms.push([e[j], e[j + 1]]); - permutations.set(i, perms); - } - - if (permutations.size) { - let res: [Expression, Expression][] = [[true, exp]]; - for (const [i, perms] of permutations) { - const res2: [Expression, Expression][] = []; - for (const [w, t] of perms) { - res2.push( - ...res.map((r) => { - const e = (r[1] as Expression[]).slice(); - e[i] = t; - return [and(w, r[0]), e] as [Expression, Expression]; - }) - ); - } - res = res2; - } - - for (const r of res) r[1] = normalizeCallback(r[1]); - - if (res[0][0] === true) return res[0][1]; - while (res[res.length - 1][1] == null) res.pop(); - - return ["CASE", ...res.flat()]; - } - - function toPolynomial(e: Expression): Polynomial { - if (e == null) return null; - if (e instanceof Polynomial) return e; - if (typeof e === "number") return Polynomial.fromConstant(e); - if (typeof e === "string") - return Polynomial.fromConstant(parseFloat(e) || 0); - if (typeof e === "boolean") return Polynomial.fromConstant(+e); - return Polynomial.fromIndeterminate(e); - } - - if (op === "+") { - const args: Polynomial[] = []; - for (let i = 1; i < exp.length; ++i) { - const p = toPolynomial(exp[i]); - if (p == null) return null; - args.push(p); - } - return args.reduce( - (previousValue, currentValue) => previousValue.add(currentValue), - ADDITIVE_IDENTITY - ) as unknown as Expression; - } else if (op === "*") { - const args: Polynomial[] = []; - for (let i = 1; i < exp.length; ++i) { - const p = toPolynomial(exp[i]); - if (p == null) return null; - args.push(p); - } - return args.reduce( - (previousValue, currentValue) => previousValue.multiply(currentValue), - MULTIPLICATIVE_IDENTITY - ) as unknown as Expression; - } else if (op === "-") { - const args: Polynomial[] = []; - for (let i = 1; i < exp.length; ++i) { - const p = toPolynomial(exp[i]); - if (p == null) return null; - args.push(p); - } - return args.reduce((previousValue, currentValue) => - previousValue.subtract(currentValue) - ) as unknown as Expression; - } else if (op === "/") { - const args: Polynomial[] = []; - for (let i = 1; i < exp.length; ++i) { - const p = toPolynomial(exp[i]); - if (p == null) return null; - args.push(p); - } - return args.reduce((previousValue, currentValue) => - previousValue.divide(currentValue) - ) as unknown as Expression; - } else if (["=", "<>", ">", ">=", "<", "<="].includes(op)) { - if (exp[1] == null || exp[2] == null) return null; - let lhs: Polynomial, rhs: Polynomial; - if (exp[1] instanceof Polynomial) lhs = exp[1]; - else if (typeof exp[1] === "number") lhs = Polynomial.fromConstant(exp[1]); - - if (exp[2] instanceof Polynomial) rhs = exp[2]; - else if (typeof exp[2] === "number") rhs = Polynomial.fromConstant(exp[2]); - - if (lhs || rhs) { - if (!lhs) lhs = Polynomial.fromIndeterminate(exp[1]); - if (!rhs) rhs = Polynomial.fromIndeterminate(exp[2]); - - lhs = lhs.subtract(rhs); - rhs = lhs.constant().negation(); - lhs = lhs.add(rhs); - - if (!lhs.terms.length) { - exp = [op, JSON.parse(lhs.toString()), JSON.parse(rhs.toString())]; - - if (op === "=") return exp[1] === exp[2]; - else if (op === "<>") return exp[1] !== exp[2]; - else if (op === ">") return exp[1] > exp[2]; - else if (op === ">=") return exp[1] >= exp[2]; - else if (op === "<") return exp[1] < exp[2]; - else if (op === "<=") return exp[1] <= exp[2]; - } else { - let flipOp = 1; - - const n = lhs.terms[0].coefficientNumerator; - const d = lhs.terms[0].coefficientDenominator; - - if (BI.lt(n, ZERO) || BI.lt(d, ZERO)) flipOp *= -1; - - const reciprocal = new Polynomial([ - { - indeterminates: new Indeterminates(), - coefficientNumerator: d, - coefficientDenominator: n, - }, - ]); - - lhs = lhs.multiply(reciprocal); - rhs = rhs.multiply(reciprocal); - - const keys = lhs.terms[0].indeterminates.sortedKeys; - let invert = lhs.terms[0].indeterminates.map.get(keys[0]) < 0 ? -1 : 0; - - for (const t of lhs.terms) - for (const v of t.indeterminates.map.values()) invert += v; - - if (invert < 0) { - flipOp *= -1; - lhs = lhs.reciprocal(); - rhs = rhs.reciprocal(); - } - - if (flipOp < 0) exp = [SWAPPED_OPS[op], lhs, rhs]; - else exp = [op, lhs, rhs]; - } - } - } - - // Restore polynomial expressions - exp = exp.map((e) => - e instanceof Polynomial ? JSON.parse(e.toString()) : e - ); - - exp = evaluateCallback(exp); - - return exp; -} - -export function normalize(expr: Expression): Expression { - expr = map(expr, normalizeCallback); - if (expr instanceof Polynomial) { - expr = JSON.parse(expr.toString()); - } else if (Array.isArray(expr) && expr[0] === "CASE") { - expr = expr.map((e) => - e instanceof Polynomial ? JSON.parse(e.toString()) : e - ); - } - return expr; -} - -function toBoolExprSynth(e: Expression | BoolExprSynth): BoolExprSynth { - if (e instanceof BoolExprSynth) return e; - if (Array.isArray(e)) { - if (e[0] === "CASE") - return new CaseSynth(e.slice(1).map((ee) => toBoolExprSynth(ee))); - return new VarSynth(e); - } - if (e == null) return new NullSynth(); - if (e) return new TrueSynth(); - return new FalseSynth(); -} - -function sopToExpression(sop: number[][], context: SynthContext): Expression { - if (!sop.length) return false; - const res: Expression[] = []; - for (const s of sop) { - if (!s.length) return true; - const conjs: Expression[] = []; - for (const i of s) { - let expr = context.getExpression(i >>> 2); - if (!(i & 1) !== !(i & 2)) expr = ["NOT", expr]; - if (Array.isArray(expr) && expr[0] === "NOT" && Array.isArray(expr[1])) { - const e: Expression[] = expr[1]; - if (e[0] === "IS NULL") expr = ["IS NOT NULL", ...e.slice(1)]; - else if (e[0] === "LIKE") expr = ["NOT LIKE", ...e.slice(1)]; - else if (e[0] === "=") expr = ["<>", ...e.slice(1)]; - else if (e[0] === "<>") expr = ["=", ...e.slice(1)]; - else if (e[0] === ">") expr = ["<=", ...e.slice(1)]; - else if (e[0] === ">=") expr = ["<", ...e.slice(1)]; - else if (e[0] === "<") expr = [">=", ...e.slice(1)]; - else if (e[0] === "<=") expr = [">", ...e.slice(1)]; - else if (e[0] === "NOT") expr = e[1]; - else expr = ["NOT", e]; - } - conjs.push(expr); - } - if (conjs.length > 1) res.push(["AND" as any].concat(conjs)); - else res.push(conjs[0]); - } - if (res.length > 1) return ["OR" as any].concat(res); - return res[0]; -} - -function findIsNullDeps(exp: Expression): Expression[] { - if (!Array.isArray(exp)) return []; - const op = exp[0]; - if (op === "IS NULL" || op === "IS NOT NULL") return []; - - if (op === "FUNC") { - if (exp[1] === "NOW") { - return []; - } else if (exp[1] === "LOWER" || exp[1] === "UPPER") { - return findIsNullDeps(exp[2]); - } else if (exp[1] === "ROUND") { - return exp - .slice(2, 4) - .map((e) => findIsNullDeps(e)) - .flat(); - } - } else if (op !== "PARAM") { - return exp - .slice(1) - .map((e) => findIsNullDeps(e)) - .flat(); - } - return [exp]; -} - -class SynthContext { - private declare _dcSet: number[][]; - private declare _variables: Map; - private declare _expressions: Map; - private declare _isNullRelations: Map; - private declare _isNullVars: Set; - private declare _comparisons: Map< - string, - Map - >; - - constructor() { - this._dcSet = []; - this._variables = new Map(); - this._expressions = new Map(); - this._isNullRelations = new Map(); - this._isNullVars = new Set(); - this._comparisons = new Map(); - } - - getVariable(exp: Expression): number { - const expStr = JSON.stringify(exp); - if (this._variables.has(expStr)) return this._variables.get(expStr); - const v = this._variables.size; - this._variables.set(expStr, v); - this._expressions.set(v, exp); - this.generateIsNullDc(v); - this.generateComparisionDc(v); - - if ( - expStr === '["IS NULL",["PARAM","DeviceID.ID"]]' || - expStr === '["IS NULL",["PARAM","_id"]]' - ) - this._dcSet.push([(v << 2) ^ 3]); - - return v; - } - - private generateIsNullDc(v: number): void { - this._isNullRelations.set(v, []); - const exp = this._expressions.get(v); - - const isNull = new Set( - findIsNullDeps(exp).map((e) => this.getVariable(["IS NULL", e])) - ); - - if (!isNull.size) return; - - for (const n of isNull) { - this._isNullVars.add(n); - this._isNullRelations.get(n).push(v); - this._isNullRelations.get(v).push(n); - this._dcSet.push([(n << 2) ^ 3, (v << 2) ^ 1]); - this._dcSet.push([(n << 2) ^ 3, (v << 2) ^ 3]); - } - this._dcSet.push( - [...isNull].map((n) => (n << 2) ^ 2).concat([(v << 2) ^ 0, (v << 2) ^ 2]) - ); - } - - private generateComparisionDc(v: number): void { - const exp = this._expressions.get(v); - if (!Array.isArray(exp)) return; - const op = exp[0]; - if (![">", "<", "="].includes(op)) return; - const rhs = exp[2]; - const rhsStr = JSON.stringify(rhs); - const lhsStr = JSON.stringify(exp[1]); - let allComps = this._comparisons.get(lhsStr); - if (!allComps) this._comparisons.set(lhsStr, (allComps = new Map())); - if (allComps.has(rhsStr)) return; - - const comp = { - eq: -1, - gt: -1, - lt: -1, - }; - - allComps.set(rhsStr, comp); - - comp.eq = this.getVariable(["=", exp[1], exp[2]]); - comp.gt = this.getVariable([">", exp[1], exp[2]]); - comp.lt = this.getVariable(["<", exp[1], exp[2]]); - - this._dcSet.push([ - (comp.eq << 2) ^ 1, - (comp.gt << 2) ^ 1, - (comp.lt << 2) ^ 1, - ]); - this._dcSet.push([(comp.eq << 2) ^ 3, (comp.gt << 2) ^ 3]); - this._dcSet.push([(comp.eq << 2) ^ 3, (comp.lt << 2) ^ 3]); - this._dcSet.push([(comp.gt << 2) ^ 3, (comp.lt << 2) ^ 3]); - - const type1 = typeof exp[2]; - if (!["boolean", "number", "string"].includes(type1)) return; - - for (const [rhs2Str, comp2] of allComps) { - if (comp2 === comp) continue; - const rhs2 = JSON.parse(rhs2Str); - const type2 = typeof rhs2; - - if (!["boolean", "number", "string"].includes(type2)) continue; - let cmp = 0; - if (type1 === type2) cmp = rhs > rhs2 ? 1 : -1; - else if (type1 === "string") cmp = 1; - else if (type2 === "string") cmp = -1; - else cmp = +rhs - +rhs2; - - const c1 = cmp > 0 ? comp : comp2; - const c2 = cmp > 0 ? comp2 : comp; - - // This is the minimum clauses required if all relavent vars - // were included in the DC set. - // this._dcSet.push([(c1.eq << 2) ^ 3, (c2.eq << 2) ^ 3]); - // this._dcSet.push([(c1.lt << 2) ^ 1, (c2.gt << 2) ^ 1]); - - // But we use non-minimal set because intermediate vars - // between any two may not be present in the DC set. - this._dcSet.push([(c1.lt << 2) ^ 1, (c2.gt << 2) ^ 1]); - this._dcSet.push([(c1.lt << 2) ^ 1, (c2.eq << 2) ^ 3]); - this._dcSet.push([(c1.lt << 2) ^ 1, (c2.lt << 2) ^ 3]); - this._dcSet.push([(c1.eq << 2) ^ 3, (c2.gt << 2) ^ 1]); - this._dcSet.push([(c1.eq << 2) ^ 3, (c2.eq << 2) ^ 3]); - this._dcSet.push([(c1.eq << 2) ^ 3, (c2.lt << 2) ^ 3]); - this._dcSet.push([(c1.gt << 2) ^ 3, (c2.gt << 2) ^ 1]); - this._dcSet.push([(c1.gt << 2) ^ 3, (c2.eq << 2) ^ 3]); - this._dcSet.push([(c1.gt << 2) ^ 3, (c2.lt << 2) ^ 3]); - } - } - - getExpression(v: number): Expression { - return this._expressions.get(v); - } - - getIsNullRelations(v: number): number[] { - return this._isNullRelations.get(v); - } - - isIsNull(v: number): boolean { - return this._isNullVars.has(v); - } - - getDcSet(minterms?: Minterm[]): number[][] { - if (!minterms) return this._dcSet; - const vars = new Set(minterms.flat().map((v) => v >> 2)); - - for (const comps of this._comparisons.values()) { - for (const comp of comps.values()) { - const c = Object.values(comp).filter((v) => !vars.has(v)); - if (c.length === 1) vars.add(c[0]); - } - } - - for (const n of this._isNullVars) - if (this._isNullRelations.get(n).some((v) => vars.has(v))) vars.add(n); - - const dcSet = this._dcSet.filter((m) => m.every((v) => vars.has(v >> 2))); - - return dcSet; - } -} - -function sanitizeMinterms( - minterms: number[][], - context: SynthContext -): number[][] { - const res = [] as number[][]; - loop: for (const m of minterms) { - const merged: Map = new Map(); - for (const i of m) - merged.set(i >> 2, (merged.get(i >> 2) || 0) | (1 << (i & 3))); - const minterm: number[] = []; - const perms: number[][] = []; - for (const [k, v] of merged) { - if (v === 0b1010) continue loop; - const isNullVars = context.getIsNullRelations(k); - const t = k << 2; - if (v === 0b0101) { - if (isNullVars.length === 1) minterm.push((isNullVars[0] << 2) ^ 3); - else perms.push(isNullVars.map((n) => (n << 2) ^ 3)); - } else if (v === 0b0001) { - perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 3]); - } else if (v === 0b0100) { - perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 1]); - } else if (v & 0b1000) { - minterm.push(t ^ 3); - } else if (v & 0b0010) { - minterm.push(t ^ 1); - } - } - let ms = [minterm]; - while (perms.length) { - const newMs: number[][] = []; - const perm = perms.pop(); - for (const p of perm) newMs.push(...ms.map((mm) => [...mm, p])); - ms = newMs; - } - res.push(...ms); - } - return res; -} - -function boolExprSynthToExpression(boolExpr: BoolExprSynth): Expression { - const context = new SynthContext(); - let minterms = boolExpr.true(context); - minterms = sanitizeMinterms(minterms, context); - const canRaise = getCanRaiseCallback(context); - const dcSet = context.getDcSet(minterms); - minterms = espresso(minterms, dcSet, { canRaise }); - return sopToExpression(minterms, context); -} - -function mapCallback(exp: Expression | BoolExprSynth): Expression { - if (!Array.isArray(exp)) return exp as Expression; - if (exp[0] === "CASE") { - exp = exp.slice(); - for (let i = 1; i < exp.length; i += 2) exp[i] = toBoolExprSynth(exp[i]); - return exp; - } - - const op = exp[0]; - if (op === "IS NULL") - return new IsNullSynth(toBoolExprSynth(exp[1])) as unknown as Expression; - - if (op === "IS NOT NULL") { - return new NotSynth( - new IsNullSynth(toBoolExprSynth(exp[1])) - ) as unknown as Expression; - } else if (op === "NOT") { - return new NotSynth(toBoolExprSynth(exp[1])) as unknown as Expression; - } else if (op === "OR") { - return new OrSynth( - ...exp.slice(1).map((a) => toBoolExprSynth(a)) - ) as unknown as Expression; - } else if (op === "AND") { - return new AndSynth( - ...exp.slice(1).map((a) => toBoolExprSynth(a)) - ) as unknown as Expression; - } - for (let i = 1; i < exp.length; ++i) { - if (exp[i] instanceof BoolExprSynth) { - exp[i] = boolExprSynthToExpression(exp[i]); - } else if (Array.isArray(exp[i]) && exp[i][0] === "CASE") { - for (let j = 2; j < exp[i].length; j += 2) { - if (exp[i][j] instanceof BoolExprSynth) - exp[i][j] = boolExprSynthToExpression(exp[i][j]); - } - } - } - return exp; -} - -function getCanRaiseCallback( - context: SynthContext -): (idx: number, set: Set) => boolean { - return (idx: number, set: Set): boolean => { - const i = idx >> 2; - const vars = context.getIsNullRelations(i); - if (!context.isIsNull(i)) return !(idx & 1) || !set.has(idx ^ 3); - - for (const k of vars) { - if (set.has((k << 2) ^ 1)) continue; - if (set.has((k << 2) ^ 3)) continue; - if (set.has((k << 2) ^ 0)) return false; - if (set.has((k << 2) ^ 2)) return false; - } - return true; - }; -} - -export function minimize(expr: Expression, boolean = false): Expression { - expr = normalize(expr); - expr = map(expr, mapCallback); - if (Array.isArray(expr) && expr[0] === "CASE") { - if (!boolean) { - const context = new SynthContext(); - const whens = expr.filter((e, i) => i % 2).map((e) => e.true(context)); - const caseDcSet: Minterm[] = []; - const canRaise = getCanRaiseCallback(context); - const res: Expression = ["CASE"]; - for (let i = 1; i < expr.length; i += 2) { - let minterms = sanitizeMinterms(whens[(i - 1) / 2], context); - const dcSet = context.getDcSet(minterms.concat(caseDcSet)); - minterms = espresso(minterms, dcSet, { canRaise }); - if (!minterms.length) continue; - const w = sopToExpression(minterms, context); - let t = expr[i + 1]; - if (t instanceof BoolExprSynth) t = boolExprSynthToExpression(t); - res.push(w, t); - if (w === true) break; - caseDcSet.push(...minterms); - } - while (res[res.length - 1] == null) res.splice(-2); - if (res.length < 3) return null; - return res; - } - expr = toBoolExprSynth(expr) as unknown as Expression; - } - - if (boolean) expr = toBoolExprSynth(expr) as unknown as Expression; - - if (expr instanceof BoolExprSynth) expr = boolExprSynthToExpression(expr); - return expr; -} - -export function unionDiff( - expr1: Expression, - expr2: Expression -): [Expression, Expression] { - expr2 = normalize(expr2); - - if (!expr2) return [expr1, false]; - - expr2 = map(expr2, mapCallback); - - if (!expr1) { - if (Array.isArray(expr2) && expr2[0] === "CASE") - expr2 = toBoolExprSynth(expr2) as unknown as Expression; - if (expr2 instanceof BoolExprSynth) - expr2 = boolExprSynthToExpression(expr2); - return [expr2, expr2]; - } - - const b2 = toBoolExprSynth(expr2); - expr1 = normalize(expr1); - expr1 = map(expr1, mapCallback); - const b1 = toBoolExprSynth(expr1); - - const context = new SynthContext(); - - const expr2Minterms = b2.true(context); - const expr1Minterms = b1.true(context); - const expr1NullMinterms = b1.null(context); - const expr1FalseMinterms = b1.false(context); - - const unionMinterms = sanitizeMinterms( - [...expr1Minterms, ...expr2Minterms], - context - ); - - const diffMinterms = sanitizeMinterms( - complement([ - ...complement([...expr1NullMinterms, ...expr1FalseMinterms]), - ...complement(expr2Minterms), - ]), - context - ); - - const canRaise = getCanRaiseCallback(context); - - const union = espresso(unionMinterms, context.getDcSet(unionMinterms), { - canRaise, - }); - - const diff = espresso(diffMinterms, context.getDcSet(diffMinterms), { - canRaise, - }); - - return [sopToExpression(union, context), sopToExpression(diff, context)]; -} - -export function covers(expr1: Expression, expr2: Expression): boolean { - expr2 = normalize(expr2); - if (!expr2) return true; - expr1 = normalize(expr1); - if (!Array.isArray(expr1)) return !!expr1; - - expr1 = map(expr1, mapCallback); - const b1 = toBoolExprSynth(expr1); - expr2 = map(expr2, mapCallback); - const b2 = toBoolExprSynth(expr2); - - const context = new SynthContext(); - const expr1Minterms = b1.true(context); - const expr2Minterms = b2.true(context); - - return tautology([ - ...complement(expr2Minterms), - ...context.getDcSet(), - ...expr1Minterms, - ]); -} diff --git a/lib/common/bigint.ts b/lib/common/expression/bigint.ts similarity index 100% rename from lib/common/bigint.ts rename to lib/common/expression/bigint.ts diff --git a/lib/common/expression/normalize.ts b/lib/common/expression/normalize.ts new file mode 100644 index 0000000..70efeea --- /dev/null +++ b/lib/common/expression/normalize.ts @@ -0,0 +1,492 @@ +import * as BI from "./bigint"; +import { Expression } from "../../types"; +import { map } from "./parser"; +import { and, evaluateCallback } from "./util"; + +const ZERO = BI.BigInt(0); +const ONE = BI.BigInt(1); +const TWO = BI.BigInt(2); +const NEGATIVE_ONE = BI.BigInt(-1); + +class Indeterminates { + public declare map: Map; + public declare sortedKeys: string[]; + + public constructor(str?: string) { + this.map = new Map(); + if (str) { + this.map.set(str, 1); + this.sortedKeys = [str]; + } else { + this.sortedKeys = []; + } + } + + public reciprocal(): Indeterminates { + const res = new Indeterminates(); + res.sortedKeys = this.sortedKeys; + res.map = new Map(); + for (const [k, v] of this.map) res.map.set(k, 0 - v); + return res; + } + + public static multiply( + indeterminates1: Indeterminates, + indeterminates2: Indeterminates + ): Indeterminates { + const res = new Indeterminates(); + res.sortedKeys = indeterminates1.sortedKeys.slice(); + res.map = new Map(indeterminates1.map); + + for (const [key, val] of indeterminates2.map) { + const v = res.map.get(key); + if (!v) { + res.map.set(key, val); + res.sortedKeys.push(key); + } else { + const v2 = val + v; + if (!v2) { + res.map.delete(key); + res.sortedKeys = res.sortedKeys.filter((s) => s !== key); + } else { + res.map.set(key, v2); + } + } + } + + res.sortedKeys.sort((a, b) => { + if (a.length !== b.length) return b.length - a.length; + else if (a > b) return 1; + else if (a < b) return -1; + return 0; + }); + + return res; + } + + public static compare(a: Indeterminates, b: Indeterminates): number { + if (a.sortedKeys.length !== b.sortedKeys.length) + return b.sortedKeys.length - a.sortedKeys.length; + for (let i = 0; i < a.sortedKeys.length; ++i) { + const k1 = a.sortedKeys[i]; + const w1 = a.map.get(k1); + const k2 = b.sortedKeys[i]; + const w2 = b.map.get(k2); + if (w1 !== w2) return w2 - w1; + if (k1.length > k2.length) return -1; + else if (k1.length < k2.length) return 1; + else if (k1 > k2) return 1; + else if (k1 < k2) return -1; + } + return 0; + } +} + +interface Term { + indeterminates: Indeterminates; + coefficientNumerator: BI.bigint; + coefficientDenominator: BI.bigint; +} + +function findGcd(a: BI.bigint, b: BI.bigint): BI.bigint { + while (BI.ne(b, ZERO)) { + const t = b; + b = BI.rem(a, b); + a = t; + } + return a; +} + +class Polynomial { + public declare terms: Term[]; + + public constructor(terms: Term[]) { + this.terms = terms; + } + + public static simplifyTerms(terms: Term[]): Term[] { + const ts = terms + .slice() + .sort((a: Term, b: Term) => + Indeterminates.compare(a.indeterminates, b.indeterminates) + ); + + for (let i = 1; i < ts.length; ++i) { + const t1 = ts[i - 1]; + const t2 = ts[i]; + if (Indeterminates.compare(t1.indeterminates, t2.indeterminates) === 0) { + const numerator = BI.add( + BI.mul(t1.coefficientNumerator, t2.coefficientDenominator), + BI.mul(t2.coefficientNumerator, t1.coefficientDenominator) + ); + + const denominator = BI.mul( + t1.coefficientDenominator, + t2.coefficientDenominator + ); + + const gcd = findGcd(numerator, denominator); + + ts[i] = { + indeterminates: t2.indeterminates, + coefficientNumerator: BI.div(numerator, gcd), + coefficientDenominator: BI.div(denominator, gcd), + }; + ts[i - 1] = { + indeterminates: t1.indeterminates, + coefficientNumerator: ZERO, + coefficientDenominator: t1.coefficientDenominator, + }; + } + } + return ts.filter((v) => BI.ne(v.coefficientNumerator, ZERO)); + } + + public static fromIndeterminate(indeterminate: Expression): Polynomial { + const indeterminates = new Indeterminates(JSON.stringify(indeterminate)); + const terms = [ + { + indeterminates: indeterminates, + coefficientNumerator: ONE, + coefficientDenominator: ONE, + }, + ]; + return new Polynomial(terms); + } + + public static fromConstant(constant: number): Polynomial { + const [int, frac] = Math.abs(constant).toString(2).split(".", 2); + let numerator = BI.BigInt("0b" + int); + if (constant < 0) numerator = BI.mul(numerator, NEGATIVE_ONE); + let denominator = ONE; + if (frac) { + denominator = BI.exp(TWO, BI.BigInt(frac.length)); + numerator = BI.add( + BI.mul(numerator, denominator), + BI.BigInt("0b" + frac) + ); + } + + const terms = [ + { + indeterminates: new Indeterminates(), + coefficientNumerator: numerator, + coefficientDenominator: denominator, + }, + ]; + + return new Polynomial(terms); + } + + public negation(): Polynomial { + const terms = this.terms.map((t) => ({ + indeterminates: t.indeterminates, + coefficientNumerator: BI.mul(t.coefficientNumerator, NEGATIVE_ONE), + coefficientDenominator: t.coefficientDenominator, + })); + + return new Polynomial(terms); + } + + public reciprocal(): Polynomial { + const terms = this.terms.map((t) => ({ + indeterminates: t.indeterminates.reciprocal(), + coefficientNumerator: t.coefficientDenominator, + coefficientDenominator: t.coefficientNumerator, + })); + return new Polynomial(terms); + } + + public constant(): Polynomial { + const terms = this.terms.filter((t) => !t.indeterminates.sortedKeys.length); + return new Polynomial(terms); + } + + public add(rhs: Polynomial): Polynomial { + return new Polynomial( + Polynomial.simplifyTerms(this.terms.concat(rhs.terms)) + ); + } + + public subtract(rhs: Polynomial): Polynomial { + return this.add(rhs.negation()); + } + + public multiply(rhs: Polynomial): Polynomial { + const terms: Term[] = []; + + for (const t1 of this.terms) { + for (const t2 of rhs.terms) { + const numerator = BI.mul( + t1.coefficientNumerator, + t2.coefficientNumerator + ); + const denominator = BI.mul( + t1.coefficientDenominator, + t2.coefficientDenominator + ); + const gcd = findGcd(numerator, denominator); + + terms.push({ + indeterminates: Indeterminates.multiply( + t1.indeterminates, + t2.indeterminates + ), + coefficientNumerator: BI.div(numerator, gcd), + coefficientDenominator: BI.div(denominator, gcd), + }); + } + } + + return new Polynomial(Polynomial.simplifyTerms(terms)); + } + + public divide(rhs: Polynomial): Polynomial { + return this.multiply(rhs.reciprocal()); + } + + public toString(): string { + const add: string[] = []; + for (const t of this.terms) { + const coefficient = + BI.toNumber(t.coefficientNumerator) / + BI.toNumber(t.coefficientDenominator); + + const mul: string[] = []; + if (t.indeterminates.sortedKeys.length) { + for (const k of t.indeterminates.sortedKeys) { + const w = t.indeterminates.map.get(k); + for (let i = Math.abs(w); i > 0; --i) { + if (w > 0) mul.push(k); + else mul.push(`["/",1,${k}]`); + } + } + + if (coefficient !== 1) mul.push(coefficient.toString()); + + if (mul.length > 1) add.push(`["*",${mul.join(",")}]`); + else add.push(mul["0"]); + } else { + add.push(coefficient.toString()); + } + } + + if (!add.length) return "0"; + else if (add.length === 1) return add[0]; + else return `["+",${add.join(",")}]`; + } +} + +const ADDITIVE_IDENTITY = Polynomial.fromConstant(0); +const MULTIPLICATIVE_IDENTITY = Polynomial.fromConstant(1); + +const SWAPPED_OPS = { + "=": "=", + "<>": "<>", + ">": "<", + ">=": "<=", + "<": ">", + "<=": ">=", +}; + +function normalizeCallback(exp: Expression): Expression { + if (!Array.isArray(exp)) return exp; + const op = exp[0]; + + if (op === "FUNC" && exp[1] === "COALESCE") { + const res: Expression[] = ["CASE"]; + for (let i = 2; i < exp.length; ++i) + res.push(normalizeCallback(["IS NOT NULL", exp[i]]), exp[i]); + return normalizeCallback(res); + } + + if (op === "CASE") { + const res = [] as [Expression, Expression][]; + for (let i = 1; i < exp.length; i += 2) { + let w = exp[i]; + if (w instanceof Polynomial) w = JSON.parse(w.toString()); + if (!Array.isArray(w) && !w) continue; + const t = exp[i + 1]; + if (!Array.isArray(t) || t[0] !== "CASE") { + res.push([w, t]); + continue; + } + for (let j = 1; j < t.length; j += 2) res.push([and(w, t[j]), t[j + 1]]); + res.push([w, null]); + if (!Array.isArray(w) && w) break; + } + while (res[res.length - 1][1] == null) res.pop(); + return ["CASE", ...res.flat()]; + } + + const permutations: Map = new Map(); + for (const [i, e] of exp.entries()) { + if (!Array.isArray(e)) continue; + if (e[0] !== "CASE") continue; + const perms: [Expression, Expression][] = []; + for (let j = 1; j < e.length; j += 2) perms.push([e[j], e[j + 1]]); + perms.push([true, null]); + permutations.set(i, perms); + } + + if (permutations.size) { + let res: [Expression, Expression][] = [[true, exp]]; + for (const [i, perms] of permutations) { + const res2: [Expression, Expression][] = []; + for (const [w, t] of perms) { + res2.push( + ...res.map((r) => { + const e = (r[1] as Expression[]).slice(); + e[i] = t; + return [and(w, r[0]), e] as [Expression, Expression]; + }) + ); + if (!Array.isArray(w) && w) break; + } + res = res2; + } + + for (const r of res) r[1] = normalizeCallback(r[1]); + + if (res[0][0] === true) return res[0][1]; + while (res[res.length - 1][1] == null) res.pop(); + + return ["CASE", ...res.flat()]; + } + + function toPolynomial(e: Expression): Polynomial { + if (e == null) return null; + if (e instanceof Polynomial) return e; + if (typeof e === "number") return Polynomial.fromConstant(e); + if (typeof e === "string") + return Polynomial.fromConstant(parseFloat(e) || 0); + if (typeof e === "boolean") return Polynomial.fromConstant(+e); + return Polynomial.fromIndeterminate(e); + } + + if (op === "+") { + const args: Polynomial[] = []; + for (let i = 1; i < exp.length; ++i) { + const p = toPolynomial(exp[i]); + if (p == null) return null; + args.push(p); + } + return args.reduce( + (previousValue, currentValue) => previousValue.add(currentValue), + ADDITIVE_IDENTITY + ) as unknown as Expression; + } else if (op === "*") { + const args: Polynomial[] = []; + for (let i = 1; i < exp.length; ++i) { + const p = toPolynomial(exp[i]); + if (p == null) return null; + args.push(p); + } + return args.reduce( + (previousValue, currentValue) => previousValue.multiply(currentValue), + MULTIPLICATIVE_IDENTITY + ) as unknown as Expression; + } else if (op === "-") { + const args: Polynomial[] = []; + for (let i = 1; i < exp.length; ++i) { + const p = toPolynomial(exp[i]); + if (p == null) return null; + args.push(p); + } + return args.reduce((previousValue, currentValue) => + previousValue.subtract(currentValue) + ) as unknown as Expression; + } else if (op === "/") { + const args: Polynomial[] = []; + for (let i = 1; i < exp.length; ++i) { + const p = toPolynomial(exp[i]); + if (p == null) return null; + args.push(p); + } + return args.reduce((previousValue, currentValue) => + previousValue.divide(currentValue) + ) as unknown as Expression; + } else if (["=", "<>", ">", ">=", "<", "<="].includes(op)) { + if (exp[1] == null || exp[2] == null) return null; + let lhs: Polynomial, rhs: Polynomial; + if (exp[1] instanceof Polynomial) lhs = exp[1]; + else if (typeof exp[1] === "number") lhs = Polynomial.fromConstant(exp[1]); + + if (exp[2] instanceof Polynomial) rhs = exp[2]; + else if (typeof exp[2] === "number") rhs = Polynomial.fromConstant(exp[2]); + + if (lhs || rhs) { + if (!lhs) lhs = Polynomial.fromIndeterminate(exp[1]); + if (!rhs) rhs = Polynomial.fromIndeterminate(exp[2]); + + lhs = lhs.subtract(rhs); + rhs = lhs.constant().negation(); + lhs = lhs.add(rhs); + + if (!lhs.terms.length) { + exp = [op, JSON.parse(lhs.toString()), JSON.parse(rhs.toString())]; + + if (op === "=") return exp[1] === exp[2]; + else if (op === "<>") return exp[1] !== exp[2]; + else if (op === ">") return exp[1] > exp[2]; + else if (op === ">=") return exp[1] >= exp[2]; + else if (op === "<") return exp[1] < exp[2]; + else if (op === "<=") return exp[1] <= exp[2]; + } else { + let flipOp = 1; + + const n = lhs.terms[0].coefficientNumerator; + const d = lhs.terms[0].coefficientDenominator; + + if (BI.lt(n, ZERO) || BI.lt(d, ZERO)) flipOp *= -1; + + const reciprocal = new Polynomial([ + { + indeterminates: new Indeterminates(), + coefficientNumerator: d, + coefficientDenominator: n, + }, + ]); + + lhs = lhs.multiply(reciprocal); + rhs = rhs.multiply(reciprocal); + + const keys = lhs.terms[0].indeterminates.sortedKeys; + let invert = lhs.terms[0].indeterminates.map.get(keys[0]) < 0 ? -1 : 0; + + for (const t of lhs.terms) + for (const v of t.indeterminates.map.values()) invert += v; + + if (invert < 0) { + flipOp *= -1; + lhs = lhs.reciprocal(); + rhs = rhs.reciprocal(); + } + + if (flipOp < 0) exp = [SWAPPED_OPS[op], lhs, rhs]; + else exp = [op, lhs, rhs]; + } + } + } + + // Restore polynomial expressions + exp = exp.map((e) => + e instanceof Polynomial ? JSON.parse(e.toString()) : e + ); + + exp = evaluateCallback(exp); + + return exp; +} + +export default function normalize(expr: Expression): Expression { + expr = map(expr, normalizeCallback); + if (expr instanceof Polynomial) { + expr = JSON.parse(expr.toString()); + } else if (Array.isArray(expr) && expr[0] === "CASE") { + expr = expr.map((e) => + e instanceof Polynomial ? JSON.parse(e.toString()) : e + ); + } + return expr; +} diff --git a/lib/common/expression-parser.ts b/lib/common/expression/parser.ts similarity index 98% rename from lib/common/expression-parser.ts rename to lib/common/expression/parser.ts index 9802f4d..596fd8b 100644 --- a/lib/common/expression-parser.ts +++ b/lib/common/expression/parser.ts @@ -18,7 +18,7 @@ */ import parsimmon from "parsimmon"; -import { Expression } from "../types"; +import { Expression } from "../../types"; // Turn escaped characters into real ones (e.g. "\\n" becomes "\n"). function interpretEscapes(str): string { @@ -40,10 +40,7 @@ function interpretEscapes(str): string { }); } -export function map( - exp: Expression, - callback: (e: Expression) => Expression -): Expression { +export function map(exp: Expression, callback: (e: Expression) => T): T { if (!Array.isArray(exp)) return callback(exp); let clone; diff --git a/lib/common/expression/synth.ts b/lib/common/expression/synth.ts new file mode 100644 index 0000000..2814434 --- /dev/null +++ b/lib/common/expression/synth.ts @@ -0,0 +1,914 @@ +import { espresso, complement, tautology } from "espresso-iisojs"; +import { Expression } from "../../types"; +import normalize from "./normalize"; +import { map, parseLikePattern } from "./parser"; + +type Minterm = number[]; + +export abstract class SynthContextBase< + T extends { toString: () => string } = unknown, + U = unknown +> { + public variables = new Map(); + protected clauses = new Map(); + + protected getVar(c: U): number { + const str = c.toString(); + let idx = this.variables.get(str); + if (idx == null) { + idx = this.variables.size; + this.variables.set(str, idx); + this.clauses.set(idx, c); + } + return idx; + } + + protected getClause(v: number): U { + return this.clauses.get(v); + } + + abstract getMinterms(exp: T, res: true | false | null): Minterm[]; + abstract getDcSet(minterms: Minterm[]): Minterm[]; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + canRaise(idx: number, set: Set): boolean { + return true; + } + + sanitizeMinterms(minterms: Minterm[]): Minterm[] { + return minterms; + } + + minimize(minterms: Minterm[], dcSet: Minterm[] = []): Minterm[] { + minterms = this.sanitizeMinterms(minterms); + const canRaise = this.canRaise.bind(this); + return espresso(minterms, [...this.getDcSet(minterms), ...dcSet], { + canRaise, + }); + } +} + +function* findIsNullDeps(exp: Expression): IterableIterator { + if (!Array.isArray(exp)) return; + const op = exp[0]; + if (op === "IS NULL" || op === "IS NOT NULL") return; + + if (op === "FUNC") { + if (exp[1] === "NOW") { + return; + } else if (exp[1] === "LOWER" || exp[1] === "UPPER") { + yield* findIsNullDeps(exp[2]); + } else if (exp[1] === "ROUND") { + for (const e of exp.slice(2, 4)) yield* findIsNullDeps(e); + return; + } + } else if (op !== "PARAM") { + for (const e of exp.slice(1)) yield* findIsNullDeps(e); + return; + } + yield exp; +} + +export abstract class Clause { + private _isNullable: Set; + protected _expression: Expression; + abstract true(context: SynthContextBase): Minterm[]; + abstract false(context: SynthContextBase): Minterm[]; + abstract null(context: SynthContextBase): Minterm[]; + expression(): Expression { + if (this._expression !== undefined) return this._expression; + const context = createSynthContext(); + const minterms = this.true(context); + const minimized = context.minimize(minterms); + this._expression = context.toExpression(minimized) as Expression; + return this._expression; + } + isBoolean(): boolean { + return true; + } + isNullable(c: Clause.IsNull): boolean { + if (!this._isNullable) { + this._isNullable = new Set( + [...this.getNullables()].map((n) => n.toString()) + ); + } + return this._isNullable.has(c.toString()); + } + *getNullables(): IterableIterator { + const exp = this.expression(); + for (const e of findIsNullDeps(exp)) { + if (e === exp) yield new Clause.IsNull(this); + else yield new Clause.IsNull(new Clause.Exp(e)); + } + } + toString(): string { + return JSON.stringify(this.expression()); + } +} + +// eslint-disable-next-line @typescript-eslint/no-namespace +export namespace Clause { + export class Not extends Clause { + constructor(public operand: Clause) { + super(); + } + true(context: SynthContextBase): Minterm[] { + return this.operand.false(context); + } + false(context: SynthContextBase): Minterm[] { + return this.operand.true(context); + } + null(context: SynthContextBase): Minterm[] { + return this.operand.null(context); + } + } + + export class Or extends Clause { + constructor(public operands: Clause[]) { + super(); + } + true(context: SynthContextBase): Minterm[] { + const minterms: Minterm[] = []; + for (const s of this.operands) { + const m = s.true(context); + if (m.length === 1 && !m[0].length) return [[]]; + minterms.push(...m); + } + return minterms; + } + false(context: SynthContextBase): Minterm[] { + const minterms: Minterm[][] = []; + const operands = [...this.operands]; + for (const e of operands) { + if (e instanceof Or) { + operands.push(...e.operands); + continue; + } + const m = e.false(context); + if (!m.length) return []; + minterms.push(m); + } + if (!minterms.length) return [[]]; + if (minterms.length === 1) return minterms[0]; + return complement(minterms.map((m) => complement(m)).flat()); + } + null(context: SynthContextBase): Minterm[] { + const trueMinterms: Minterm[] = []; + const operands = [...this.operands]; + for (const s of operands) { + if (s instanceof Or) { + operands.push(...s.operands); + continue; + } + const t = s.true(context); + if (t.length === 1 && !t[0].length) return []; + trueMinterms.push(...t); + } + + const nullMinterms: Minterm[] = []; + for (const s of operands) { + const n = s.null(context); + if (n.length === 1 && !n[0].length && !trueMinterms.length) return [[]]; + nullMinterms.push(...n); + } + + if (!trueMinterms.length) return nullMinterms; + return complement([...complement(nullMinterms), ...trueMinterms]); + } + } + + export class And extends Clause { + constructor(public operands: Clause[]) { + super(); + } + true(context: SynthContextBase): Minterm[] { + const minterms: Minterm[][] = []; + const operands = [...this.operands]; + for (const s of operands) { + if (s instanceof And) { + operands.push(...s.operands); + continue; + } + const m = s.true(context); + if (!m.length) return []; + if (m.length === 1 && !m[0].length) continue; + minterms.push(m); + } + if (!minterms.length) return [[]]; + if (minterms.length === 1) return minterms[0]; + return complement(minterms.map((m) => complement(m)).flat()); + } + false(context: SynthContextBase): Minterm[] { + const minterms: Minterm[] = []; + for (const s of this.operands) { + const m = s.false(context); + if (m.length === 1 && !m[0].length) return [[]]; + minterms.push(...m); + } + return minterms; + } + null(context: SynthContextBase): Minterm[] { + const falseMinterms: Minterm[] = []; + const operands = [...this.operands]; + for (const s of operands) { + if (s instanceof And) { + operands.push(...s.operands); + continue; + } + const f = s.false(context); + if (f.length === 1 && !f[0].length) return []; + falseMinterms.push(...f); + } + + const nullMinterms: Minterm[] = []; + for (const s of operands) { + const n = s.null(context); + if (n.length === 1 && !n[0].length && !falseMinterms.length) + return [[]]; + nullMinterms.push(...n); + } + + if (!falseMinterms.length) return nullMinterms; + return complement([...complement(nullMinterms), ...falseMinterms]); + } + } + + export class Case extends Clause { + constructor(public clauses: Clause[]) { + super(); + } + true(context: SynthContextBase): Minterm[] { + const minterms: Minterm[] = []; + const cumulative: Minterm[] = []; + for (let i = 0; i < this.clauses.length; i += 2) { + const w = this.clauses[i].true(context); + const t = this.clauses[i + 1].true(context); + minterms.push( + ...complement([...cumulative, ...complement(w), ...complement(t)]) + ); + if (i < this.clauses.length - 2) { + cumulative.push( + ...complement([ + ...this.clauses[i].false(context), + ...this.clauses[i].null(context), + ]) + ); + } + } + return minterms; + } + false(context: SynthContextBase): Minterm[] { + const minterms: Minterm[] = []; + const cumulative: Minterm[] = []; + for (let i = 0; i < this.clauses.length; i += 2) { + const w = this.clauses[i].true(context); + const t = this.clauses[i + 1].false(context); + minterms.push( + ...complement([...cumulative, ...complement(w), ...complement(t)]) + ); + if (i < this.clauses.length - 2) { + cumulative.push( + ...complement([ + ...this.clauses[i].false(context), + ...this.clauses[i].null(context), + ]) + ); + } + } + return minterms; + } + null(context: SynthContextBase): Minterm[] { + const minterms: Minterm[] = []; + const cumulative: Minterm[] = []; + for (let i = 0; i < this.clauses.length; i += 2) { + const w = this.clauses[i].true(context); + const t = this.clauses[i + 1].null(context); + minterms.push( + ...complement([...cumulative, ...complement(w), ...complement(t)]) + ); + cumulative.push( + ...complement([ + ...this.clauses[i].false(context), + ...this.clauses[i].null(context), + ]) + ); + } + minterms.push(...complement([...cumulative])); + return minterms; + } + expression(): Expression { + if (this._expression != null) return this._expression; + if (this.isBoolean()) { + this._expression = new Clause.Not(new Clause.Not(this)).expression(); + return this._expression; + } + const context = createSynthContext(); + const cases: { when: Minterm[]; then: Expression }[] = []; + for (let i = 0; i < this.clauses.length; i += 2) { + let minterms = this.clauses[i].true(context); + const then = this.clauses[i + 1].expression(); + if ( + cases.length && + JSON.stringify(then) === JSON.stringify(cases[cases.length - 1].then) + ) + minterms.push(...cases.pop().when); + minterms = context.minimize( + minterms, + cases.flatMap((c) => c.when) + ); + if (!minterms.length) continue; + cases.push({ when: minterms, then }); + if (minterms.length === 1 && !minterms[0].length) break; + } + while (cases[cases.length - 1].then == null) cases.pop(); + if (!cases.length) return null; + this._expression = [ + "CASE", + ...cases.flatMap((c) => [context.toExpression(c.when), c.then]), + ]; + return this._expression; + } + + isBoolean(): boolean { + for (let i = 1; i < this.clauses.length; i += 2) + if (!this.clauses[i].isBoolean()) return false; + return true; + } + } + + export class IsNull extends Clause { + private _boolean: Clause; + constructor(public operand: Clause) { + super(); + this._boolean = new Not(new Not(this)); + } + true(context: SynthContextBase): Minterm[] { + return this.operand.null(context); + } + false(context: SynthContextBase): Minterm[] { + return complement(this.true(context)); + } + null(): Minterm[] { + return []; + } + isBoolean(): boolean { + return true; + } + *getNullables(): IterableIterator { + // Never returns null + } + expression(): Expression { + const nullables = [...this.operand.getNullables()]; + if (nullables.length === 1 && nullables[0].operand === this.operand) + return ["IS NULL", this.operand.expression()]; + return super.expression(); + } + } + + export class Exp extends Clause { + constructor(public exp: Expression) { + super(); + } + true(context: SynthContextBase): Minterm[] { + if (Array.isArray(this.exp)) return context.getMinterms(this, true); + if (this.exp) return [[]]; + return []; + } + false(context: SynthContextBase): Minterm[] { + if (Array.isArray(this.exp)) return context.getMinterms(this, false); + if (!this.exp && this.exp != null) return [[]]; + return []; + } + null(context: SynthContextBase): Minterm[] { + if (Array.isArray(this.exp)) return context.getMinterms(this, null); + if (this.exp == null) return [[]]; + return []; + } + expression(): Expression { + return this.exp; + } + isBoolean(): boolean { + return this.exp === true || this.exp === false || this.exp == null; + } + } + + export class Compare extends Clause { + constructor( + public lhs: Clause, + public op: ">" | "<" | "=", + public rhs: boolean | number | string + ) { + super(); + } + true(context: SynthContextBase): Minterm[] { + return context.getMinterms(this, true); + } + false(context: SynthContextBase): Minterm[] { + return context.getMinterms(this, false); + } + null(context: SynthContextBase): Minterm[] { + return this.lhs.null(context); + } + *getNullables(): IterableIterator { + yield* this.lhs.getNullables(); + } + isBoolean(): boolean { + return true; + } + expression(): Expression { + return [this.op, this.lhs.expression(), this.rhs]; + } + } + + export class Like extends Clause { + public readonly caseSensitive: boolean; + public readonly contradiction: boolean; + public readonly pattern: string[]; + public readonly lhs: Clause; + + constructor(lhs: Clause, public rhs: string, public esc?: string) { + super(); + const exp = lhs.expression(); + let caseSensitive = true; + let contradiction = false; + if (Array.isArray(exp)) { + if (exp[0] === "FUNC") { + const func = exp[1]; + if (func === "UPPER" || func === "LOWER") { + const p = func === "UPPER" ? rhs.toUpperCase() : rhs.toLowerCase(); + if (p === rhs) caseSensitive = false; + else contradiction = true; + lhs = new Exp(exp[2]); + } + } + } + this.lhs = lhs; + this.pattern = parseLikePattern(rhs, esc); + this.caseSensitive = caseSensitive; + this.contradiction = contradiction; + } + true(context: SynthContextBase): Minterm[] { + return context.getMinterms(this, true); + } + false(context: SynthContextBase): Minterm[] { + return context.getMinterms(this, false); + } + null(context: SynthContextBase): Minterm[] { + return this.lhs.null(context); + } + isBoolean(): boolean { + return true; + } + isNullable(c: IsNull): boolean { + return this.lhs.isNullable(c); + } + getNullables(): IterableIterator { + return this.lhs.getNullables(); + } + expression(): Expression { + let lhs = this.lhs.expression(); + if (this.contradiction) { + if (this.rhs === this.rhs.toLocaleUpperCase()) + lhs = ["FUNC", "LOWER", lhs]; + else lhs = ["FUNC", "UPPER", lhs]; + } else if (!this.caseSensitive) { + if (this.rhs === this.rhs.toLocaleUpperCase()) + lhs = ["FUNC", "UPPER", lhs]; + else lhs = ["FUNC", "LOWER", lhs]; + } + return ["LIKE", lhs, this.rhs, ...(this.esc ? [this.esc] : [])]; + } + } + + export function fromExpression(exp: Expression): Clause { + const res = map(exp, (e) => { + if (!Array.isArray(e)) return new Clause.Exp(e); + let op = e[0]; + let negate = true; + if (op === "NOT LIKE") op = "LIKE"; + else if (op === "IS NOT NULL") op = "IS NULL"; + else if (op === "<>") op = "="; + else if (op === ">=") op = "<"; + else if (op === "<=") op = ">"; + else negate = false; + + let clause: Clause; + if (op === "IS NULL") { + clause = new Clause.IsNull(e[1]); + } else if (op === "NOT") { + clause = new Clause.Not(e[1]); + } else if (op === "OR") { + clause = new Clause.Or(e.slice(1)); + } else if (op === "AND") { + clause = new Clause.And(e.slice(1)); + } else if (op === "CASE") { + clause = new Clause.Case(e.slice(1)); + } else if (op === "LIKE") { + const rhs = e[2] instanceof Exp ? e[2].expression() : null; + const esc = e[3] instanceof Exp ? e[3].expression() : null; + if (typeof rhs === "string" && typeof esc === "string") + clause = new Clause.Like(e[1], rhs, esc); + else if (typeof rhs === "string" && esc == null) + clause = new Clause.Like(e[1], rhs); + } else if (op === ">" || op === "<" || op === "=") { + const rhs = e[2] instanceof Exp ? e[2].expression() : null; + if (["boolean", "number", "string"].includes(typeof rhs)) { + clause = new Clause.Compare( + e[1], + op, + rhs as boolean | number | string + ); + } + } + + if (!clause) { + const args = e.slice(1).map((a) => (a as Clause).expression()); + clause = new Clause.Exp([op, ...args]); + } + + if (negate) clause = new Clause.Not(clause); + return clause; + }); + + return res; + } +} + +function groupBy( + input: T[], + callback: (item: T) => K +): Iterable<[K, T[]]> { + const groups = new Map(); + for (const item of input) { + const key = callback(item); + let arr = groups.get(key); + if (!arr) groups.set(key, (arr = [])); + arr.push(item); + } + + return groups.entries(); +} + +export class SynthContext extends SynthContextBase { + constructor() { + super(); + } + + getMinterms(clause: Clause, res: true | false | null): number[][] { + const v = this.getVar(clause); + if (res === true) return [[(v << 2) ^ 3]]; + else if (res === false) return [[(v << 2) ^ 1]]; + else return [[v << 2, (v << 2) ^ 2]]; + } + + getDcSet(minterms: Minterm[]): number[][] { + const dcSet: number[][] = []; + + const whitelist = new Set([...minterms.flat()].map((v) => v >> 2)); + + const allClauses = [...whitelist].map((v) => this.getClause(v)); + const comparisons = allClauses.filter( + (c) => c instanceof Clause.Compare + ) as Clause.Compare[]; + + // Comparisons + for (const [, clauses] of groupBy(comparisons, (c) => + JSON.stringify(c.lhs) + )) { + const lhs = clauses[0].lhs; + const values = new Set(clauses.map((c) => c.rhs)); + const valuesSorted = [...values].sort((a, b) => { + const ta = typeof a; + const tb = typeof b; + if (ta === tb) return a > b ? 1 : -1; + else if (ta === "string") return 1; + else if (tb === "string") return -1; + return +a - +b; + }); + + for (const [i, v] of valuesSorted.entries()) { + const eq = this.getVar(new Clause.Compare(lhs, "=", v)); + const gt = this.getVar(new Clause.Compare(lhs, ">", v)); + const lt = this.getVar(new Clause.Compare(lhs, "<", v)); + + dcSet.push([(eq << 2) ^ 3, (gt << 2) ^ 3]); + dcSet.push([(lt << 2) ^ 3, (gt << 2) ^ 3]); + dcSet.push([(lt << 2) ^ 3, (eq << 2) ^ 3]); + dcSet.push([(lt << 2) ^ 1, (eq << 2) ^ 1, (gt << 2) ^ 1]); + + const negEquivOp = [lt, eq, gt].filter((o) => !whitelist.has(o)); + if (negEquivOp.length === 1) whitelist.add(negEquivOp[0]); + + for (let j = 0; j < i; j++) { + const eq2 = this.getVar( + new Clause.Compare(lhs, "=", valuesSorted[j]) + ); + const gt2 = this.getVar( + new Clause.Compare(lhs, ">", valuesSorted[j]) + ); + const lt2 = this.getVar( + new Clause.Compare(lhs, "<", valuesSorted[j]) + ); + + // This is the minimum clauses required if all relavent vars + // were included in the DC set. + // dcSet.push([(eq << 2) ^ 3, (eq2 << 2) ^ 3]); + // dcSet.push([(lt << 2) ^ 1, (gt2 << 2) ^ 1]); + + // But we use non-minimal set because intermediate vars + // between any two may not be present in the DC set. + dcSet.push([(gt2 << 2) ^ 1, (lt << 2) ^ 1]); + dcSet.push([(eq2 << 2) ^ 3, (lt << 2) ^ 1]); + dcSet.push([(lt2 << 2) ^ 3, (lt << 2) ^ 1]); + dcSet.push([(gt2 << 2) ^ 1, (gt << 2) ^ 3]); + dcSet.push([(gt2 << 2) ^ 1, (eq << 2) ^ 3]); + dcSet.push([(eq2 << 2) ^ 3, (gt << 2) ^ 3]); + dcSet.push([(eq2 << 2) ^ 3, (eq << 2) ^ 3]); + dcSet.push([(lt2 << 2) ^ 3, (gt << 2) ^ 3]); + dcSet.push([(lt2 << 2) ^ 3, (eq << 2) ^ 3]); + } + } + } + + // LIKE + const likes = allClauses.filter( + (c) => c instanceof Clause.Like + ) as Clause.Like[]; + + for (const [, clauses] of groupBy(likes, (c) => JSON.stringify(c.lhs))) { + for (let i1 = 0; i1 < clauses.length; ++i1) { + const l1 = clauses[i1]; + if (l1.contradiction) { + dcSet.push([(this.getVar(l1) << 2) ^ 3]); + continue; + } + for (let i2 = i1 + 1; i2 < clauses.length; ++i2) { + const l2 = clauses[i2]; + if (l2.contradiction) continue; + let p1 = l1.pattern; + let p2 = l2.pattern; + if (!l1.caseSensitive || !l2.caseSensitive) { + p1 = p1.map((c) => c.toLowerCase()); + p2 = p2.map((c) => c.toLowerCase()); + } + if (likeDisjoint(p1, p2)) { + dcSet.push([ + (this.getVar(l1) << 2) ^ 3, + (this.getVar(l2) << 2) ^ 3, + ]); + } else if ( + (!l1.caseSensitive || l2.caseSensitive) && + likeImplies(p1, p2) + ) { + dcSet.push([ + (this.getVar(l1) << 2) ^ 2, + (this.getVar(l2) << 2) ^ 3, + ]); + dcSet.push([ + (this.getVar(l1) << 2) ^ 1, + (this.getVar(l2) << 2) ^ 0, + ]); + } else if ( + (!l2.caseSensitive || l1.caseSensitive) && + likeImplies(p2, p1) + ) { + dcSet.push([ + (this.getVar(l1) << 2) ^ 3, + (this.getVar(l2) << 2) ^ 2, + ]); + dcSet.push([ + (this.getVar(l1) << 2) ^ 0, + (this.getVar(l2) << 2) ^ 1, + ]); + } + } + } + } + + for (const v of whitelist) { + const clause = this.getClause(v); + const nullables = [...clause.getNullables()].map((c) => this.getVar(c)); + if (nullables.length) { + dcSet.push([ + ...nullables.map((n) => (n << 2) ^ 2), + (v << 2) ^ 0, + (v << 2) ^ 2, + ]); + for (const n of nullables) { + dcSet.push([(n << 2) ^ 3, (v << 2) ^ 1]); + dcSet.push([(n << 2) ^ 3, (v << 2) ^ 3]); + whitelist.add(n); + } + } + + if (clause instanceof Clause.IsNull) { + const str = clause.operand.toString(); + if (str === '["PARAM","DeviceID.ID"]' || str === '["PARAM","_id"]') + dcSet.push([(v << 2) ^ 3]); + } + } + return dcSet.filter((m) => m.every((v) => whitelist.has(v >> 2))); + } + + canRaise(idx: number, set: Set): boolean { + const clause = this.getClause(idx >> 2); + if (clause instanceof Clause.IsNull) { + for (const i of set) { + if (i === idx || i & 1) continue; + const c = this.getClause(i >> 2); + if (c.isNullable(clause)) return false; + } + return true; + } + return !(idx & 1) || !set.has(idx ^ 3); + } + + sanitizeMinterms(minterms: Minterm[]): Minterm[] { + const res = [] as number[][]; + + loop: for (const m of minterms) { + const merged: Map = new Map(); + for (const i of m) + merged.set(i >> 2, (merged.get(i >> 2) || 0) | (1 << (i & 3))); + const minterm: number[] = []; + const perms: number[][] = []; + for (const [k, v] of merged) { + if (v === 0b1010) continue loop; + const isNullVars = [...this.clauses.get(k).getNullables()].map((c) => + this.getVar(c) + ); + const t = k << 2; + if (v === 0b0101) { + if (isNullVars.length === 1) minterm.push((isNullVars[0] << 2) ^ 3); + else perms.push(isNullVars.map((n) => (n << 2) ^ 3)); + } else if (v === 0b0001) { + perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 3]); + } else if (v === 0b0100) { + perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 1]); + } else if (v & 0b1000) { + minterm.push(t ^ 3); + } else if (v & 0b0010) { + minterm.push(t ^ 1); + } + } + let ms = [minterm]; + while (perms.length) { + const newMs: number[][] = []; + const perm = perms.pop(); + for (const p of perm) newMs.push(...ms.map((mm) => [...mm, p])); + ms = newMs; + } + res.push(...ms); + } + return res; + } + + toExpression(sop: number[][]): Expression { + if (!sop.length) return false; + const res: Expression[] = []; + for (const s of sop) { + if (!s.length) return true; + const conjs: Expression[] = []; + for (const i of s) { + let expr = this.getClause(i >>> 2).expression() as Expression; + if (!(i & 1) !== !(i & 2)) expr = ["NOT", expr]; + if ( + Array.isArray(expr) && + expr[0] === "NOT" && + Array.isArray(expr[1]) + ) { + const e: Expression[] = expr[1]; + if (e[0] === "IS NULL") expr = ["IS NOT NULL", ...e.slice(1)]; + else if (e[0] === "LIKE") expr = ["NOT LIKE", ...e.slice(1)]; + else if (e[0] === "=") expr = ["<>", ...e.slice(1)]; + else if (e[0] === "<>") expr = ["=", ...e.slice(1)]; + else if (e[0] === ">") expr = ["<=", ...e.slice(1)]; + else if (e[0] === ">=") expr = ["<", ...e.slice(1)]; + else if (e[0] === "<") expr = [">=", ...e.slice(1)]; + else if (e[0] === "<=") expr = [">", ...e.slice(1)]; + else if (e[0] === "NOT") expr = e[1]; + } + conjs.push(expr); + } + if (conjs.length > 1) res.push(["AND", ...conjs]); + else res.push(conjs[0]); + } + if (res.length > 1) return ["OR", ...res]; + return res[0]; + } +} + +// Classes aren't hoisted in JS but functions are. This function is used to +// create a new SynthContext instance from inside the Clause class. +export function createSynthContext(): SynthContext { + return new SynthContext(); +} + +export function likeImplies(pat1: string[], pat2: string[]): boolean { + let backtrack: [number, number] = null; + + for (let i1 = 0, i2 = 0; ; ++i1, ++i2) { + while (i1 < pat1.length && pat1[i1] === "\\%") backtrack = [i1++, i2]; + + if (i2 >= pat2.length) return i1 >= pat1.length; + + const c = i1 < pat1.length ? pat1[i1] : null; + + if (c !== pat2[i2] && c !== "\\_") { + if (!backtrack) return false; + [i1, i2] = backtrack; + ++backtrack[1]; + } + } +} + +export function likeDisjoint(pat1: string[], pat2: string[]): boolean { + const left1Idx = pat1.indexOf("\\%"); + const left2Idx = pat2.indexOf("\\%"); + const right1Idx = pat1.lastIndexOf("\\%"); + const right2Idx = pat2.lastIndexOf("\\%"); + + const left1 = pat1.slice(0, left1Idx !== -1 ? left1Idx : pat1.length); + const left2 = pat2.slice(0, left2Idx !== -1 ? left2Idx : pat2.length); + const right1 = pat1.slice(right1Idx !== -1 ? right1Idx + 1 : 0).reverse(); + const right2 = pat2.slice(right2Idx !== -1 ? right2Idx + 1 : 0).reverse(); + + for (let i = 0; i < Math.min(left1.length, left2.length); ++i) { + if (left1[i] !== left2[i] && left1[i] !== "\\_" && left2[i] !== "\\_") + return true; + } + + for (let i = 0; i < Math.min(right1.length, right2.length); ++i) { + if (right1[i] !== right2[i] && right1[i] !== "\\_" && right2[i] !== "\\_") + return true; + } + + if (pat1.length === left1.length) + return pat2.filter((c) => c !== "\\%").length > pat1.length; + else if (pat2.length === left2.length) + return pat1.filter((c) => c !== "\\%").length > pat2.length; + + return false; +} + +export function minimize(expr: Expression, boolean = false): Expression { + let synth = Clause.fromExpression(normalize(expr)); + if (boolean) synth = new Clause.Not(new Clause.Not(synth)); + return synth.expression(); +} + +export function unionDiff( + expr1: Expression, + expr2: Expression +): [Expression, Expression] { + expr2 = normalize(expr2); + + if (!expr2) return [expr1, false]; + + const synth2 = Clause.fromExpression(expr2); + + if (!expr1) { + const e = synth2.expression(); + return [e, e]; + } + + expr1 = normalize(expr1); + const synth1 = Clause.fromExpression(expr1); + + const context = new SynthContext(); + + const expr2Minterms = synth2.true(context); + const expr1Minterms = synth1.true(context); + const expr1NullMinterms = synth1.null(context); + const expr1FalseMinterms = synth1.false(context); + + const union = context.minimize([...expr1Minterms, ...expr2Minterms]); + + const diff = context.minimize( + complement([ + ...complement([...expr1NullMinterms, ...expr1FalseMinterms]), + ...complement(expr2Minterms), + ]) + ); + + return [context.toExpression(union), context.toExpression(diff)]; +} + +export function covers(expr1: Expression, expr2: Expression): boolean { + expr2 = normalize(expr2); + if (!expr2) return true; + expr1 = normalize(expr1); + if (!Array.isArray(expr1)) return !!expr1; + + const synt1 = Clause.fromExpression(expr1); + const synt2 = Clause.fromExpression(expr2); + + const context = new SynthContext(); + const expr1Minterms = synt1.true(context); + const expr2Minterms = synt2.true(context); + + return tautology([ + ...complement(expr2Minterms), + ...context.getDcSet([...expr2Minterms, ...expr1Minterms]), + ...expr1Minterms, + ]); +} diff --git a/lib/common/expression.ts b/lib/common/expression/util.ts similarity index 99% rename from lib/common/expression.ts rename to lib/common/expression/util.ts index 8693ea1..45cad04 100644 --- a/lib/common/expression.ts +++ b/lib/common/expression/util.ts @@ -17,8 +17,8 @@ * along with GenieACS. If not, see . */ -import { map, mapAsync, parseLikePattern } from "./expression-parser"; -import { Expression } from "../types"; +import { map, mapAsync, parseLikePattern } from "./parser"; +import { Expression } from "../../types"; const isArray = Array.isArray; diff --git a/lib/connection-request.ts b/lib/connection-request.ts index 84bd257..322885c 100644 --- a/lib/connection-request.ts +++ b/lib/connection-request.ts @@ -21,7 +21,7 @@ import * as crypto from "crypto"; import * as dgram from "dgram"; import { URL } from "url"; import * as http from "http"; -import { evaluateAsync } from "./common/expression"; +import { evaluateAsync } from "./common/expression/util"; import { Expression } from "./types"; import * as auth from "./auth"; import * as extensions from "./extensions"; diff --git a/lib/cwmp.ts b/lib/cwmp.ts index 2305cef..70f2591 100644 --- a/lib/cwmp.ts +++ b/lib/cwmp.ts @@ -25,7 +25,11 @@ import * as config from "./config"; import * as common from "./common"; import * as soap from "./soap"; import * as session from "./session"; -import { evaluateAsync, evaluate, extractParams } from "./common/expression"; +import { + evaluateAsync, + evaluate, + extractParams, +} from "./common/expression/util"; import * as cache from "./cache"; import * as localCache from "./local-cache"; import * as db from "./db"; diff --git a/lib/db.ts b/lib/db.ts index cfd8bc5..4f9ad16 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -20,7 +20,7 @@ import { MongoClient, ObjectId, Collection, Db } from "mongodb"; import { get } from "./config"; import { decodeTag, encodeTag, escapeRegExp } from "./common"; -import { parse } from "./common/expression-parser"; +import { parse } from "./common/expression/parser"; import { DeviceData, Attributes, diff --git a/lib/local-cache.ts b/lib/local-cache.ts index eb3e62f..d620a8e 100644 --- a/lib/local-cache.ts +++ b/lib/local-cache.ts @@ -25,8 +25,8 @@ import * as cache from "./cache"; import { convertOldPrecondition } from "./mongodb-functions"; import * as logger from "./logger"; import * as scheduling from "./scheduling"; -import * as expression from "./common/expression"; -import { parse } from "./common/expression-parser"; +import * as expression from "./common/expression/util"; +import { parse } from "./common/expression/parser"; import { Preset, Expression, diff --git a/lib/mongodb-functions.ts b/lib/mongodb-functions.ts index 0339dbf..0267e3a 100644 --- a/lib/mongodb-functions.ts +++ b/lib/mongodb-functions.ts @@ -1,248 +1,21 @@ -/** - * Copyright 2013-2019 GenieACS Inc. - * - * This file is part of GenieACS. - * - * GenieACS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * GenieACS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with GenieACS. If not, see . - */ - -import { ObjectId } from "mongodb"; -import { map, parse, stringify, parseList } from "./common/expression-parser"; -import { likePatternToRegExp } from "./common/expression"; +import { Filter } from "mongodb"; +import { EJSON } from "bson"; +import { + parse, + stringify, + parseList, + parseLikePattern, +} from "./common/expression/parser"; import { Expression, Fault, Task } from "./types"; import { decodeTag, encodeTag } from "./common"; - -const isArray = Array.isArray; - -export function processDeviceFilter(filter: Expression): Expression { - return map(filter, (exp) => { - if (!isArray(exp)) return exp; - - if (exp[0] === "PARAM") { - const p = exp[1]; - if (p === "DeviceID.ID") return ["PARAM", "_id"]; - else if (p === "DeviceID") return ["PARAM", "_deviceId"]; - else if (p.startsWith("DeviceID.")) - return ["PARAM", "_deviceId._" + p.slice(9)]; - else if (p === "Events.Inform") return ["PARAM", "_lastInform"]; - else if (p === "Events.Registered") return ["PARAM", "_registered"]; - else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"]; - else if (p === "Events.1_BOOT") return ["PARAM", "_lastBoot"]; - } else if ( - isArray(exp[1]) && - exp[1][0] === "PARAM" && - exp[1][1].startsWith("Tags.") - ) { - const t = decodeTag(exp[1][1].slice(5)); - if (exp[0] === "IS NULL") return ["_tags", t, false]; - else if (exp[0] === "IS NOT NULL") return ["_tags", t, true]; - else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true]; - else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true]; - } else if ( - [ - "=", - "<>", - ">", - ">=", - "<", - "<=", - "LIKE", - "NOT LIKE", - "IS NULL", - "IS NOT NULL", - ].includes(exp[0]) - ) { - let e = map(exp, (ee) => { - if ( - isArray(ee) && - ee[0] === "PARAM" && - typeof ee[1] === "string" && - !ee[1].startsWith("_") - ) - return ["PARAM", `${ee[1]}._value`]; - return ee; - }); - - if (typeof e[2] === "number") { - const alt = (e as any[]).slice(); - alt[2] = new Date(e[2]); - if (exp[0] === "<>" || exp[0] === "NOT LIKE") e = ["AND", e, alt]; - else e = ["OR", e, alt]; - } - return e; - } - return exp; - }); -} - -export function processTasksFilter(filter: Expression): Expression { - return map(filter, (exp) => { - if (!isArray(exp)) return exp; - if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) { - const e = exp.slice(); - if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectId(e[2]); - else if (e[1][0] === "PARAM" && e[1][1] === "timestamp") - e[2] = new Date(e[2]); - else if (e[1][0] === "PARAM" && e[1][1] === "expiry") - e[2] = new Date(e[2]); - return e; - } - return exp; - }); -} - -export function processFaultsFilter(filter: Expression): Expression { - return map(filter, (exp) => { - if (!isArray(exp)) return exp; - if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) { - const e = exp.slice(); - if (e[1][0] === "PARAM" && e[1][1] === "timestamp") e[2] = new Date(e[2]); - else if (e[1][0] === "PARAM" && e[1][1] === "expiry") - e[2] = new Date(e[2]); - return e; - } - return exp; - }); -} - -export function filterToMongoQuery(exp: Expression): Record { - function recursive(filter, negate): Record { - const op = filter[0]; - - if (op === "AND") { - filter = filter.filter((f) => f !== true); - if (filter.length === 2) return recursive(filter[1], negate); - } - - if (op === "OR") { - filter = filter.filter((f) => f !== false); - if (filter.length === 2) return recursive(filter[1], negate); - } - - if ((!negate && op === "AND") || (negate && op === "OR")) - return { $and: filter.slice(1).map((f) => recursive(f, negate)) }; - - if ((!negate && op === "OR") || (negate && op === "AND")) - return { $or: filter.slice(1).map((f) => recursive(f, negate)) }; - - if (op === "NOT") return recursive(filter[1], !negate); - - if (isArray(filter[2])) - throw new Error(`Invalid RHS operand of ${op} clause`); - - if (op === "LIKE" || op === "NOT LIKE") { - if (op === "NOT LIKE") negate = !negate; - let param; - let flags; - if (!Array.isArray(filter[1])) - throw new Error(`Invalid LHS operand of ${op} clause`); - if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") { - if (filter[2] !== filter[2].toUpperCase()) - throw new Error(`Invalid RHS operand of ${op} clause`); - param = filter[1][2][1]; - flags = "i"; - } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") { - if (filter[2] !== filter[2].toLowerCase()) - throw new Error(`Invalid RHS operand of ${op} clause`); - param = filter[1][2][1]; - flags = "i"; - } else if (filter[1][0] === "PARAM") { - param = filter[1][1]; - flags = ""; - } else { - throw new Error(`Invalid LHS operand of ${op} clause`); - } - const r = likePatternToRegExp(filter[2], filter[3], flags); - if (negate) return { [param]: { $nin: [r, null] } }; - else return { [param]: r }; - } - - if (op === "_tags") { - let t = filter[2]; - if (negate) t = !t; - if (t) return { _tags: filter[1] }; - else return { _tags: { $ne: filter[1] } }; - } - - if (!isArray(filter[1]) || filter[1][0] !== "PARAM") - throw new Error(`Invalid LHS operand of ${op} clause`); - - if (op === "IS NULL") { - const p = filter[1][1]; - return { [p]: null }; - } - - if (op === "IS NOT NULL") { - const p = filter[1][1]; - return { [p]: { $ne: null } }; - } - - if (isArray(filter[2]) || filter[2] == null) - throw new Error(`Invalid RHS operand of ${op} clause`); - - if (op === "=") { - const p = filter[1][1]; - const v = filter[2]; - if (!negate) return { [p]: v }; - else return { [p]: { $nin: [v, null] } }; - } - - if (op === "<>") { - const p = filter[1][1]; - const v = filter[2]; - if (negate) return { [p]: v }; - else return { [p]: { $nin: [v, null] } }; - } - - if (op === ">") { - const p = filter[1][1]; - const v = filter[2]; - if (negate) return { [p]: { $lte: v } }; - else return { [p]: { $gt: v } }; - } - - if (op === ">=") { - const p = filter[1][1]; - const v = filter[2]; - if (negate) return { [p]: { $lt: v } }; - else return { [p]: { $gte: v } }; - } - - if (op === "<") { - const p = filter[1][1]; - const v = filter[2]; - if (negate) return { [p]: { $gte: v } }; - else return { [p]: { $lt: v } }; - } - - if (op === "<=") { - const p = filter[1][1]; - const v = filter[2]; - if (negate) return { [p]: { $gt: v } }; - else return { [p]: { $lte: v } }; - } - - throw new Error(`Unrecognized operator ${op}`); - } - - if (!isArray(exp)) { - if (exp === true) return {}; - throw new Error("Primitives are not valid queries"); - } - - return recursive(exp, false); -} +import { + SynthContextBase, + likeDisjoint, + likeImplies, +} from "./common/expression/synth"; +import normalize from "./common/expression/normalize"; +import { Clause } from "./common/expression/synth"; +import * as BI from "./common/expression/bigint"; export function processDeviceProjection( projection: Record @@ -506,7 +279,7 @@ export function convertOldPrecondition( } } else if (k === "_tags") { if (typeof v === "object") { - if (isArray(v)) throw new Error(`Invalid type`); + if (Array.isArray(v)) throw new Error(`Invalid type`); const conjs: Expression[] = []; for (const [op, val] of Object.entries(v)) { if (op === "$ne") { @@ -539,7 +312,7 @@ export function convertOldPrecondition( expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]); } else if (typeof v === "object") { - if (isArray(v)) throw new Error(`Invalid type`); + if (Array.isArray(v)) throw new Error(`Invalid type`); const exps: Expression[] = []; for (const [kk, vv] of Object.entries(v)) { @@ -683,3 +456,629 @@ export function preProcessPreset( preset.configurations = [configuration]; return preset; } + +type Minterm = number[]; + +function getParam(exp: Expression, collection: string): string { + if (!Array.isArray(exp)) + throw new Error("Left-hand operand must be a parameter"); + + if (exp[0] !== "PARAM") + throw new Error("Left-hand operand must be a parameter"); + + if (typeof exp[1] !== "string") + throw new Error("Left-hand operand must be a parameter"); + + const p = exp[1]; + if (collection === "devices") { + if (p === "DeviceID.ID") return "_id"; + else if (p === "DeviceID") return "_deviceId"; + else if (p.startsWith("DeviceID.")) return "_deviceId._" + p.slice(9); + else if (p === "Events.Inform") return "_lastInform"; + else if (p === "Events.Registered") return "_registered"; + else if (p === "Events.0_BOOTSTRAP") return "_lastBootstrap"; + else if (p === "Events.1_BOOT") return "_lastBoot"; + else if (!p.endsWith("._value") && !p.startsWith("Tags.")) + return `${p}._value`; + } + + return p; +} + +function getTypes(parameter: string, collection: string): string[] { + if (collection === "devices") { + if (parameter === "_id") return ["string"]; + if (parameter === "_lastInform") return ["date"]; + if (parameter === "_registered") return ["date"]; + if (parameter === "_lastBootstrap") return ["date"]; + if (parameter === "_lastBoot") return ["date"]; + if (parameter.startsWith("_deviceId.")) return ["string"]; + if (parameter === "Reboot._value") return ["date"]; + if (parameter === "FactoryReset._value") return ["date"]; + if (parameter.endsWith("_timestamp")) return ["date"]; + if (parameter.startsWith("Downloads.")) { + if (parameter.endsWith("Download._value")) return ["date"]; + if (parameter.endsWith("Time._value")) return ["date"]; + if (parameter.endsWith("Name._value")) return ["string"]; + if (parameter.endsWith("Type._value")) return ["string"]; + } + if (parameter.endsWith("_value")) + return ["bool", "number", "date", "string"]; + } else if (collection === "tasks") { + if (parameter === "_id") return ["oid"]; + if (parameter === "timestamp") return ["date"]; + if (parameter === "expiry") return ["date"]; + if (parameter === "name") return ["string"]; + if (parameter === "device") return ["string"]; + } else if (collection === "faults") { + if (parameter === "_id") return ["string"]; + if (parameter === "timestamp") return ["date"]; + if (parameter === "expiry") return ["date"]; + if (parameter === "code") return ["string"]; + if (parameter === "retries") return ["number"]; + if (parameter === "channel") return ["string"]; + if (parameter === "device") return ["string"]; + if (parameter === "message") return ["string"]; + } else if (collection === "users") { + if (parameter === "_id") return ["string"]; + if (parameter === "roles") return ["string"]; + else if (parameter === "password") + throw new Error("Cannot query restricted parameters"); + else if (parameter === "salt") + throw new Error("Cannot query restricted parameters"); + } else if (collection === "config") { + if (parameter === "_id") return ["string"]; + else if (parameter === "value") return ["string"]; + } else if (collection === "files") { + if (parameter === "_id") return ["string"]; + if (parameter === "metadata.fileType") return ["string"]; + if (parameter === "metadata.oui") return ["string"]; + if (parameter === "metadata.productClass") return ["string"]; + if (parameter === "metadata.version") return ["string"]; + } else if (collection === "permissions") { + if (parameter === "_id") return ["string"]; + if (parameter === "role") return ["string"]; + if (parameter === "resource") return ["devices"]; + if (parameter === "access") return ["number"]; + if (parameter === "filter") return ["string"]; + if (parameter === "validate") return ["string"]; + } else if (collection === "presets") { + if (parameter === "_id") return ["string"]; + if (parameter === "weight") return ["number"]; + if (parameter === "channel") return ["string"]; + if (parameter === "precondition") return ["string"]; + if (parameter.startsWith("events.")) return ["bool"]; + } else if (collection === "provisions") { + if (parameter === "_id") return ["string"]; + if (parameter === "script") return ["string"]; + } else if (collection === "virtualParameters") { + if (parameter === "_id") return ["string"]; + if (parameter === "script") return ["string"]; + } + + if (parameter === "_id") return ["oid", "string"]; + return ["bool", "number", "date", "string"]; +} + +function roundOid(oid: string, roundUp: boolean): string { + const match = (oid.match(/^[0-9a-f]*/)?.[0] ?? "").slice(0, 24); + let num = BI.BigInt("0x0" + match); + let lastChar = 0; + if (oid.length > match.length) lastChar += oid.charCodeAt(match.length); + if (match.length < 24) lastChar -= 48; + if (lastChar > 0 && roundUp) num++; + num <<= BI.BigInt(4 * (24 - match.length)); + if (lastChar < 0 && !roundUp) --num; + const str = num.toString(16); + if (str.length > 24 || str.startsWith("-")) return null; + return str.padStart(24, "0"); +} + +function groupBy( + input: T[], + callback: (item: T) => K +): Iterable<[K, T[]]> { + const groups = new Map(); + for (const item of input) { + const key = callback(item); + let arr = groups.get(key); + if (!arr) groups.set(key, (arr = [])); + arr.push(item); + } + + return groups.entries(); +} + +abstract class MongoClause { + abstract readonly parameter: string; + abstract toQuery(truthy: boolean): Filter; + toString(): string { + return JSON.stringify(this.toQuery(true)); + } +} + +class MongoClauseArray extends MongoClause { + constructor( + public readonly parameter: string, + public readonly value: string + ) { + super(); + } + + toQuery(truthy: boolean): Filter { + if (truthy) return { [this.parameter]: { $eq: this.value } }; + else return { [this.parameter]: { $ne: this.value } }; + } +} + +class MongoClauseCompare extends MongoClause { + constructor( + public readonly parameter: string, + public readonly op: "$eq" | "$gt" | "$lt" | "$gte" | "$lte", + public readonly value: T, + public readonly type: "" | "bool" | "number" | "string" | "date" | "oid" + ) { + super(); + } + + toQuery(truthy: boolean): Filter { + let v: any = this.value; + if (this.type === "date" && typeof v === "number") + v = { $date: new Date(v).toISOString() }; + if (this.type === "oid" && typeof v === "string") v = { $oid: v }; + + if (truthy) return { [this.parameter]: { [this.op]: v } }; + else if (this.op === "$eq") return { [this.parameter]: { $ne: v } }; + else return { [this.parameter]: { $not: { [this.op]: v } } }; + } +} + +class MongoClauseType extends MongoClause { + constructor(public readonly parameter: string, public readonly type: string) { + super(); + } + + toQuery(truthy: boolean): Filter { + if (truthy) return { [this.parameter]: { $type: this.type } }; + else return { [this.parameter]: { $not: { $type: this.type } } }; + } +} + +class MongoClauseLike extends MongoClause { + readonly pattern: string[]; + constructor( + public readonly parameter: string, + pat: string, + esc: string, + public readonly caseSensitive: boolean + ) { + super(); + this.pattern = parseLikePattern(pat, esc); + } + + toQuery(truthy: boolean): Filter { + const convChars = { + "-": "\\-", + "/": "\\/", + "\\": "\\/", + "^": "\\^", + $: "\\$", + "*": "\\*", + "+": "\\+", + "?": "\\?", + ".": "\\.", + "(": "\\(", + ")": "\\)", + "|": "\\|", + "[": "\\[", + "]": "\\]", + "{": "\\{", + "}": "\\}", + "\\%": ".*", + "\\_": ".", + }; + + const chars = this.pattern.map((c) => convChars[c] ?? c); + chars[0] = chars[0] === ".*" ? "" : "^" + chars[0]; + const l = chars.length - 1; + chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$"; + const pattern = chars.join(""); + const options = this.caseSensitive ? "s" : "is"; + if (truthy) { + return { [this.parameter]: { $regularExpression: { options, pattern } } }; + } else { + return { + [this.parameter]: { + $not: { $regularExpression: { options, pattern } }, + }, + }; + } + } +} + +class MongoSynthContext extends SynthContextBase { + constructor(private readonly collection: string) { + super(); + } + + getMinterms(clause: Clause, res: true | false | null): number[][] { + const exp = clause.expression(); + if (!Array.isArray(exp)) throw new Error("Invalid query expression"); + + if (res === null) { + const minterms: number[][] = []; + for (const dep of clause.getNullables()) { + const e = dep.operand.expression(); + const param = getParam(e, this.collection); + if (param.startsWith("Tags.") && this.collection === "devices") { + const t = decodeTag(param.slice(5)); + const c = new MongoClauseArray("_tags", t); + minterms.push([this.getVar(c) << 1]); + continue; + } + const c = new MongoClauseCompare(param, "$eq", null, ""); + minterms.push([(this.getVar(c) << 1) ^ 1]); + } + return minterms; + } + + if ([">", "<", "="].includes(exp[0])) { + const param = getParam(exp[1], this.collection); + + let rhs = exp[2]; + if (typeof rhs === "boolean") rhs = +rhs; + else if (typeof rhs !== "string" && typeof rhs !== "number") + throw new Error(`Right-hand operand must be a literal value`); + + if ( + param.startsWith("Tags.") && + this.collection === "devices" && + exp[0] === "=" + ) { + const t = decodeTag(param.slice(5)); + const c = new MongoClauseArray("_tags", t); + if (typeof rhs === "string") rhs = 2; + if (exp[0] === "=") { + if ((rhs === 1) !== res) return []; + } else if (exp[0] === ">") { + if ((res && rhs >= 1) || (!res && rhs < 1)) return []; + } else if (exp[0] === "<") { + if ((res && rhs <= 1) || (!res && rhs > 1)) return []; + } + return [[(this.getVar(c) << 1) ^ 1]]; + } + + let op: "$gt" | "$lt" | "$eq" | "$gte" | "$lte"; + if (exp[0] === "=") op = "$eq"; + else if (exp[0] === ">") op = res ? "$gt" : "$lte"; + else if (exp[0] === "<") op = res ? "$lt" : "$gte"; + + const possibleTypes = new Set(getTypes(param, this.collection)); + const clauses: MongoClause[] = []; + + if (typeof rhs === "number") { + if (possibleTypes.has("number")) + clauses.push(new MongoClauseCompare(param, op, rhs, "number")); + + if (possibleTypes.has("date")) + clauses.push(new MongoClauseCompare(param, op, rhs, "date")); + if (possibleTypes.has("bool") && (rhs === 0 || rhs === 1)) + clauses.push(new MongoClauseCompare(param, op, !!rhs, "bool")); + } else if (typeof rhs === "string") { + if (possibleTypes.has("string")) + clauses.push(new MongoClauseCompare(param, op, rhs, "string")); + + if (possibleTypes.has("oid")) { + const oid = roundOid(rhs, op === "$lt" || op === "$lte"); + if (oid && (op !== "$eq" || oid === rhs)) + clauses.push(new MongoClauseCompare(param, op, oid, "oid")); + } + } + + if (op === "$eq" && !res) { + clauses.push(new MongoClauseCompare(param, "$eq", null, "")); + return [clauses.map((c) => this.getVar(c) << 1)]; + } + + // In the following clauses we could use $type operator, but we want the + // final query to use comparison operators to check for type because the + // $type operator in MongoDB doesn't use indexes + if (typeof rhs === "number") { + if (possibleTypes.has("bool")) { + if ( + (rhs > 1 && (op === "$lt" || op === "$lte")) || + (rhs < 0 && (op === "$gt" || op === "$gte")) + ) + clauses.push(new MongoClauseCompare(param, "$gte", false, "bool")); + } + + if (op === "$gt" || op === "$gte") { + if (possibleTypes.has("string")) + clauses.push(new MongoClauseCompare(param, "$gte", "", "string")); + + if (possibleTypes.has("oid")) { + clauses.push( + new MongoClauseCompare( + param, + "$gte", + "000000000000000000000000", + "oid" + ) + ); + } + } + } else if (typeof rhs === "string") { + if (op === "$lt" || op === "$lte") { + if (possibleTypes.has("bool")) + clauses.push(new MongoClauseCompare(param, "$gte", false, "bool")); + + if (possibleTypes.has("number")) { + clauses.push(new MongoClauseCompare(param, "$gte", 0, "number")); + clauses.push(new MongoClauseCompare(param, "$lt", 0, "number")); + } + if (possibleTypes.has("date")) { + clauses.push(new MongoClauseCompare(param, "$gte", 0, "date")); + clauses.push(new MongoClauseCompare(param, "$lt", 0, "date")); + } + } + } + + return clauses.map((c) => [(this.getVar(c) << 1) ^ 1]); + } else if (exp[0] === "LIKE") { + const pat = exp[2]; + if (typeof pat !== "string") + throw new Error("Right-hand operand of 'LIKE' must be a string"); + let p = exp[1]; + let caseSensitive = true; + if ( + Array.isArray(p) && + p[0] === "FUNC" && + ["UPPER", "LOWER"].includes(p[1]) + ) { + if (p[1] === "UPPER" && pat !== pat.toUpperCase()) + return res ? [] : [[]]; + if (p[1] === "LOWER" && pat !== pat.toLowerCase()) + return res ? [] : [[]]; + caseSensitive = false; + p = p[2]; + } + + const param = getParam(p, this.collection); + const c = new MongoClauseLike(param, pat, exp[3], caseSensitive); + if (res) return [[(this.getVar(c) << 1) ^ 1]]; + const typeClase = new MongoClauseType(param, "string"); + const r = [[this.getVar(c) << 1, (this.getVar(typeClase) << 1) ^ 1]]; + return r; + } else { + throw new Error("Invalid query expression"); + } + } + + getDcSet(minterms: Minterm[]): number[][] { + const dcSet: number[][] = []; + + const vars = new Set(minterms.flat().map((l) => l >> 1)); + const clauses = Array.from(vars).map((v) => this.getClause(v)); + + for (const [parameter, clauses2] of groupBy(clauses, (c) => c.parameter)) { + const comparisons = clauses2.filter( + (c) => c instanceof MongoClauseCompare && c.value !== null + ) as MongoClauseCompare[]; + + for (const [type, clauses3] of groupBy(comparisons, (c) => c.type)) { + const isType = this.getVar(new MongoClauseType(parameter, type)); + const values = new Set(clauses3.map((c) => c.value)); + const valuesSorted = Array.from(values).sort((a, b) => + a > b ? 1 : -1 + ); + for (const [i, v] of valuesSorted.entries()) { + const eq = this.getVar( + new MongoClauseCompare(parameter, "$eq", v, type) + ); + const gt = this.getVar( + new MongoClauseCompare(parameter, "$gt", v, type) + ); + const lt = this.getVar( + new MongoClauseCompare(parameter, "$lt", v, type) + ); + const gte = this.getVar( + new MongoClauseCompare(parameter, "$gte", v, type) + ); + const lte = this.getVar( + new MongoClauseCompare(parameter, "$lte", v, type) + ); + + if (type === "bool") { + if (v === false) dcSet.push([(lt << 1) ^ 1]); + else if (v === true) dcSet.push([(gt << 1) ^ 1]); + } else if (type === "string") { + if (v === "") dcSet.push([(lt << 1) ^ 1]); + } else if (type === "oid") { + if (v < "000000000000000000000000") dcSet.push([(lte << 1) ^ 1]); + else if (v === "000000000000000000000000") + dcSet.push([(lt << 1) ^ 1]); + else if (v > "ffffffffffffffffffffffff") + dcSet.push([(gte << 1) ^ 1]); + else if (v === "ffffffffffffffffffffffff") + dcSet.push([(gt << 1) ^ 1]); + } + dcSet.push([(lt << 1) ^ 1, (gte << 1) ^ 1]); + dcSet.push([(gt << 1) ^ 1, (gte << 1) ^ 0]); + dcSet.push([(eq << 1) ^ 0, (lt << 1) ^ 0, (lte << 1) ^ 1]); + dcSet.push([(eq << 1) ^ 1, (gte << 1) ^ 0]); + dcSet.push([(eq << 1) ^ 1, (gt << 1) ^ 1]); + + if (i === 0) { + dcSet.push([(isType << 1) ^ 0, (gte << 1) ^ 1]); + dcSet.push([(isType << 1) ^ 0, (lt << 1) ^ 1]); + } else { + const gt2 = this.getVar( + new MongoClauseCompare( + parameter, + "$gt", + valuesSorted[i - 1], + type + ) + ); + const lte2 = this.getVar( + new MongoClauseCompare( + parameter, + "$lte", + valuesSorted[i - 1], + type + ) + ); + dcSet.push([(gt2 << 1) ^ 0, (gte << 1) ^ 1]); + dcSet.push([(gt2 << 1) ^ 0, (lte2 << 1) ^ 0, (lt << 1) ^ 1]); + } + + if (i === valuesSorted.length - 1) + dcSet.push([(isType << 1) ^ 1, (gt << 1) ^ 0, (lte << 1) ^ 0]); + } + } + + const likes = clauses2.filter( + (c) => c instanceof MongoClauseLike + ) as MongoClauseLike[]; + if (likes.length) { + const isType = this.getVar(new MongoClauseType(parameter, "string")); + for (let i1 = 0; i1 < likes.length; ++i1) { + const l1 = likes[i1]; + let p1 = l1.pattern; + dcSet.push([(this.getVar(l1) << 1) ^ 1, (isType << 1) ^ 0]); + for (let i2 = i1 + 1; i2 < likes.length; ++i2) { + const l2 = likes[i2]; + let p2 = l2.pattern; + if (!l1.caseSensitive || !l2.caseSensitive) { + p1 = p1.map((c) => c.toLowerCase()); + p2 = p2.map((c) => c.toLowerCase()); + } + if (likeDisjoint(p1, p2)) { + dcSet.push([ + (this.getVar(l1) << 1) ^ 1, + (this.getVar(l2) << 1) ^ 1, + ]); + } else if ( + (!l1.caseSensitive || l2.caseSensitive) && + likeImplies(p1, p2) + ) { + dcSet.push([ + (this.getVar(l1) << 1) ^ 0, + (this.getVar(l2) << 1) ^ 1, + ]); + } else if ( + (!l2.caseSensitive || l1.caseSensitive) && + likeImplies(p2, p1) + ) { + dcSet.push([ + (this.getVar(l1) << 1) ^ 1, + (this.getVar(l2) << 1) ^ 0, + ]); + } + } + } + } + + const isNull = this.getVar( + new MongoClauseCompare(parameter, "$eq", null, "") + ); + + const types = getTypes(parameter, this.collection).map((t) => + this.getVar(new MongoClauseType(parameter, t)) + ); + + dcSet.push([(isNull << 1) ^ 0, ...types.map((t) => (t << 1) ^ 0)]); + for (let i = 0; i < types.length; ++i) { + const t1 = types[i]; + dcSet.push([(t1 << 1) ^ 1, (isNull << 1) ^ 1]); + for (let j = i + 1; j < types.length; ++j) { + const t2 = types[j]; + dcSet.push([(t1 << 1) ^ 1, (t2 << 1) ^ 1]); + } + } + + if (parameter === "_id") dcSet.push([(isNull << 1) ^ 1]); + } + return dcSet; + } + + canRaise(i: number, s: Set): boolean { + if (!(i & 1)) return true; + const c = this.getClause(i >> 1); + if (c instanceof MongoClauseCompare) { + for (const j of s) { + if (j === i) continue; + const c2 = this.getClause(j >> 1); + if (c2.parameter !== c.parameter) continue; + if (!(j & 1)) return false; + if (c2 instanceof MongoClauseType) return false; + } + } + return true; + } + + toQuery(minterms: Minterm[]): Filter { + const or = []; + + for (const minterm of minterms) { + const query = {}; + loop: for (const clause of minterm) { + if (!minterm.length) return {}; + const negate = !!(clause & 1); + const c = this.getClause(clause >> 1); + const q = c.toQuery(negate); + if (Object.keys(q).length !== 1) + throw new Error("Invalid query expression"); + const [param, value] = Object.entries(q)[0]; + if (Object.getPrototypeOf(value).constructor !== Object) + throw new Error("Invalid query expression"); + const dests = [query, ...(query["$and"] ?? [])]; + for (const dest of dests) { + if (!(param in dest)) { + dest[param] = value; + continue loop; + } + + let src = value; + let dst = dest[param]; + + if (src["$not"] && dst["$not"]) { + src = src["$not"]; + dst = dst["$not"]; + } + + if (!Object.keys(src).some((k) => k in dst)) { + Object.assign(dst, src); + continue loop; + } + } + + query["$and"] ??= []; + query["$and"].push({ [param]: value }); + } + or.push(query); + } + + if (or.length === 1) return or[0]; + return { $or: or }; + } +} + +export function toMongoQuery( + exp: Expression, + resource: string +): Filter | false { + exp = normalize(exp); + const clause = Clause.fromExpression(normalize(exp)); + const context = new MongoSynthContext(resource); + const minterms = clause.true(context); + const minimized = context.minimize(minterms); + if (!minimized.length) return false; + return EJSON.deserialize(context.toQuery(minimized)); +} + +export function validQuery(exp: Expression, resource: string): void { + const clause = Clause.fromExpression(normalize(exp)); + const context = new MongoSynthContext(resource); + clause.true(context); +} diff --git a/lib/query.ts b/lib/query.ts index 164a79c..da95583 100644 --- a/lib/query.ts +++ b/lib/query.ts @@ -132,7 +132,7 @@ export function expand( if (v && (v["$ne"] != null || v["$not"] != null)) { if (Object.keys(v).length > 1) throw new Error("Cannot mix $ne or $not with other operators"); - for (const c of conditions) newQuery["$an"].push(c); + for (const c of conditions) newQuery["$and"].push(c); } else { newQuery["$and"].push({ $or: conditions }); } diff --git a/lib/ui/api.ts b/lib/ui/api.ts index b3d4178..c21919e 100644 --- a/lib/ui/api.ts +++ b/lib/ui/api.ts @@ -21,8 +21,8 @@ import { PassThrough } from "stream"; import Router from "koa-router"; import * as db from "./db"; import * as apiFunctions from "./api-functions"; -import { evaluate, and, extractParams } from "../common/expression"; -import { parse } from "../common/expression-parser"; +import { evaluate, and, extractParams } from "../common/expression/util"; +import { parse } from "../common/expression/parser"; import * as logger from "../logger"; import { getConfig } from "../local-cache"; import { QueryOptions, Expression } from "../types"; diff --git a/lib/ui/db.ts b/lib/ui/db.ts index b9f6b38..13a1159 100644 --- a/lib/ui/db.ts +++ b/lib/ui/db.ts @@ -21,10 +21,9 @@ import { Db, GridFSBucket, ObjectId } from "mongodb"; import { Script } from "vm"; import { onConnect, optimizeProjection } from "../db"; import * as mongodbFunctions from "../mongodb-functions"; -import * as expression from "../common/expression"; +import { evaluate } from "../common/expression/util"; import { QueryOptions, Expression } from "../types"; import { Readable } from "stream"; -import { minimize } from "../common/boolean-expression"; const RESOURCE_COLLECTION = { files: "fs.files", @@ -54,28 +53,9 @@ export function query( callback?: (doc: any) => void ): Promise { options = options || {}; - let q; - filter = expression.evaluate(filter, null, Date.now()); - filter = minimize(filter, true); - - if (Array.isArray(filter)) { - if (resource === "devices") { - filter = mongodbFunctions.processDeviceFilter(filter); - } else if (resource === "tasks") { - filter = mongodbFunctions.processTasksFilter(filter); - } else if (resource === "faults") { - filter = mongodbFunctions.processFaultsFilter(filter); - } else if (resource === "users") { - // Protect against brute force, and dictionary attacks - const params = expression.extractParams(filter); - if (params.includes("password") || params.includes("salt")) - return Promise.reject(new Error("Invalid users filter")); - } - - q = mongodbFunctions.filterToMongoQuery(filter); - } else if (!filter) { - return Promise.resolve([]); - } + filter = evaluate(filter, null, Date.now()); + const q = mongodbFunctions.toMongoQuery(filter, resource); + if (!q) return Promise.resolve([]); return new Promise((resolve, reject) => { const collection = db.collection(RESOURCE_COLLECTION[resource] || resource); @@ -152,22 +132,9 @@ export function query( export function count(resource: string, filter: Expression): Promise { const collection = db.collection(RESOURCE_COLLECTION[resource] || resource); - let q: Parameters[0]; - filter = expression.evaluate(filter, null, Date.now()); - filter = minimize(filter, true); - - if (Array.isArray(filter)) { - if (resource === "devices") - filter = mongodbFunctions.processDeviceFilter(filter); - else if (resource === "tasks") - filter = mongodbFunctions.processTasksFilter(filter); - else if (resource === "faults") - filter = mongodbFunctions.processFaultsFilter(filter); - q = mongodbFunctions.filterToMongoQuery(filter); - } else if (!filter) { - return Promise.resolve(0); - } - + filter = evaluate(filter, null, Date.now()); + const q = mongodbFunctions.toMongoQuery(filter, resource); + if (!q) return Promise.resolve(0); return collection.countDocuments(q); } diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 594b1f8..9541d3d 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -10,7 +10,7 @@ "license": "AGPL-3.0", "dependencies": { "@breejs/later": "^4.1.0", - "espresso-iisojs": "^1.0.4", + "espresso-iisojs": "^1.0.7", "iconv-lite": "^0.6.3", "ipaddr.js": "^2.0.1", "jsonwebtoken": "^8.5.1", @@ -3041,9 +3041,9 @@ } }, "node_modules/espresso-iisojs": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/espresso-iisojs/-/espresso-iisojs-1.0.4.tgz", - "integrity": "sha512-QmdiGu97ky2pLjnBOmyTVqDQMrTOFVnJKF4Aj/dw6QoOwa29+Hk2sTmy8GQxnMW7f8av2hRgqI6mmWXdhi1jVw==" + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/espresso-iisojs/-/espresso-iisojs-1.0.7.tgz", + "integrity": "sha512-yHEY+CC9wqEyA1N+KC03xmyN+lFPZdpXaLUxf6vrofE0cMXXIkOg08tXXPmq5fnOoLgxRfm5ikrmYD8/gIhRig==" }, "node_modules/esprima": { "version": "4.0.1", @@ -10177,9 +10177,9 @@ } }, "espresso-iisojs": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/espresso-iisojs/-/espresso-iisojs-1.0.4.tgz", - "integrity": "sha512-QmdiGu97ky2pLjnBOmyTVqDQMrTOFVnJKF4Aj/dw6QoOwa29+Hk2sTmy8GQxnMW7f8av2hRgqI6mmWXdhi1jVw==" + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/espresso-iisojs/-/espresso-iisojs-1.0.7.tgz", + "integrity": "sha512-yHEY+CC9wqEyA1N+KC03xmyN+lFPZdpXaLUxf6vrofE0cMXXIkOg08tXXPmq5fnOoLgxRfm5ikrmYD8/gIhRig==" }, "esprima": { "version": "4.0.1", diff --git a/package.json b/package.json index e30247b..a326ccb 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ }, "dependencies": { "@breejs/later": "^4.1.0", - "espresso-iisojs": "^1.0.4", + "espresso-iisojs": "^1.0.7", "iconv-lite": "^0.6.3", "ipaddr.js": "^2.0.1", "jsonwebtoken": "^8.5.1", diff --git a/test/mongodb-functions.ts b/test/mongodb-functions.ts index 0983b59..f35c09c 100644 --- a/test/mongodb-functions.ts +++ b/test/mongodb-functions.ts @@ -1,6 +1,8 @@ import ava from "ava"; +import { EJSON } from "bson"; +import { Filter } from "mongodb"; import * as mongodbFunctions from "../lib/mongodb-functions"; -import { stringify, parse } from "../lib/common/expression-parser"; +import { stringify, parse } from "../lib/common/expression/parser"; ava("convertOldPrecondition", (t) => { const tests = [ @@ -71,21 +73,31 @@ ava("convertOldPrecondition", (t) => { } }); -ava("filterToMongoQuery", async (t) => { - const queries: [string, Record][] = [ +ava("toMongoQuery", async (t) => { + const queries: [string, Filter | false][] = [ ["true", {}], - ["Tags.tag1 = true", { _tags: "tag1" }], - ["Tags.tag1 <> false", { _tags: "tag1" }], + ["Tags.tag1 = true", { _tags: { $eq: "tag1" } }], + ["Tags.tag1 <> false", { _tags: { $eq: "tag1" } }], ["Tags.tag1 IS NULL", { _tags: { $ne: "tag1" } }], - ["Tags.tag1 = 123", { "Tags.tag1": 123 }], - ["Param1 = 'value1'", { "Param1._value": "value1" }], - ["Param1 <> 'value1'", { "Param1._value": { $nin: ["value1", null] } }], + ["Tags.tag1 = 123", false], + ["Param1 = 'value1'", { "Param1._value": { $eq: "value1" } }], + [ + "Param1 <> 'value1'", + { + "Param1._value": { $ne: "value1" }, + $and: [{ "Param1._value": { $ne: null } }], + }, + ], [ "Param1 <> 1657844103524", { + "Param1._value": { $ne: 1657844103524 }, + $and: [ - { "Param1._value": { $nin: [1657844103524, null] } }, - { "Param1._value": { $nin: [new Date(1657844103524), null] } }, + { + "Param1._value": { $ne: { $date: "2022-07-15T00:15:03.524Z" } }, + }, + { "Param1._value": { $ne: null } }, ], }, ], @@ -93,66 +105,70 @@ ava("filterToMongoQuery", async (t) => { "Param1 = 1657844103524", { $or: [ - { "Param1._value": 1657844103524 }, - { "Param1._value": new Date(1657844103524) }, + { "Param1._value": { $eq: { $date: "2022-07-15T00:15:03.524Z" } } }, + { "Param1._value": { $eq: 1657844103524 } }, ], }, ], ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }], ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }], - ["Param1 LIKE 'value'", { "Param1._value": /^value$/ }], - ["LOWER(Param1) LIKE 'value'", { "Param1._value": /^value$/i }], + [ + "Param1 LIKE 'value'", + { + "Param1._value": { + $regularExpression: { options: "s", pattern: "^value$" }, + }, + }, + ], + + [ + "LOWER(Param1) LIKE 'value'", + { + "Param1._value": { + $regularExpression: { options: "is", pattern: "^value$" }, + }, + }, + ], [ "Param1 <> 'value2' OR NOT (Param2 = 'value1' OR Param1 < 'value2')", { $or: [ - { "Param1._value": { $nin: ["value2", null] } }, { - $and: [ - { "Param2._value": { $nin: ["value1", null] } }, - { "Param1._value": { $gte: "value2" } }, - ], + "Param2._value": { $ne: null }, + $and: [{ "Param2._value": { $ne: "value1" } }], + "Param1._value": { $eq: "value2" }, + }, + { + "Param1._value": { $ne: "value2" }, + $and: [{ "Param1._value": { $ne: null } }], }, ], }, ], [ "Param1 <> 'value2' OR Param1 IS NULL", - { - $or: [ - { "Param1._value": { $nin: ["value2", null] } }, - { "Param1._value": null }, - ], - }, + { "Param1._value": { $ne: "value2" } }, ], ]; for (const [expStr, expect] of queries) { const exp = parse(expStr); - const query = mongodbFunctions.filterToMongoQuery( - mongodbFunctions.processDeviceFilter(exp) - ); + let query = mongodbFunctions.toMongoQuery(exp, "devices"); + if (query) query = EJSON.serialize(query); t.deepEqual(query, expect); } const failQueries: [any, string][] = [ - ["Param1 = Param2", "Invalid RHS operand of = clause"], - ["Param1 LIKE Param2", "Invalid RHS operand of LIKE clause"], - ["1 LIKE '1'", "Invalid LHS operand of LIKE clause"], - ["1 = 2", "Invalid LHS operand of = clause"], - ["1 IS NULL", "Invalid LHS operand of IS NULL clause"], - ["false", "Primitives are not valid queries"], - ["UPPER(Param1) LIKE 'value'", "Invalid RHS operand of LIKE clause"], + ["Param1 = Param2", "Right-hand operand must be a literal value"], + ["Param1 LIKE Param2", "Right-hand operand of 'LIKE' must be a string"], + ["NOW() = 1", "Left-hand operand must be a parameter"], + ["param{param2} = 1", "Left-hand operand must be a parameter"], ]; for (const [expStr, err] of failQueries) { const exp = parse(expStr); - t.throws( - () => - mongodbFunctions.filterToMongoQuery( - mongodbFunctions.processDeviceFilter(exp) - ), - { message: err } - ); + t.throws(() => mongodbFunctions.toMongoQuery(exp, "devices"), { + message: err, + }); } }); diff --git a/test/boolean-expression.ts b/test/synth.ts similarity index 96% rename from test/boolean-expression.ts rename to test/synth.ts index 5db12b7..c2a2f61 100644 --- a/test/boolean-expression.ts +++ b/test/synth.ts @@ -1,6 +1,6 @@ import ava, { TestInterface } from "ava"; -import { minimize, unionDiff } from "../lib/common/boolean-expression"; -import { parse, stringify } from "../lib/common/expression-parser"; +import { minimize, unionDiff } from "../lib/common/expression/synth"; +import { parse, stringify } from "../lib/common/expression/parser"; import initSqlJs from "sql.js/dist/sql-asm.js"; const test = ava as TestInterface<{ diff --git a/ui/components/all-parameters.ts b/ui/components/all-parameters.ts index 376df1e..2b59b7f 100644 --- a/ui/components/all-parameters.ts +++ b/ui/components/all-parameters.ts @@ -20,7 +20,7 @@ import { ClosureComponent, Component } from "mithril"; import { m } from "../components"; import * as taskQueue from "../task-queue"; -import { parse } from "../../lib/common/expression-parser"; +import { parse } from "../../lib/common/expression/parser"; import memoize from "../../lib/common/memoize"; import { getIcon } from "../icons"; import { evaluateExpression } from "../store"; diff --git a/ui/components/container.ts b/ui/components/container.ts index a130432..7c7c95c 100644 --- a/ui/components/container.ts +++ b/ui/components/container.ts @@ -18,7 +18,7 @@ */ import { Attributes, ClosureComponent, Component } from "mithril"; -import { map } from "../../lib/common/expression-parser"; +import { map } from "../../lib/common/expression/parser"; import memoize from "../../lib/common/memoize"; import { Expression } from "../../lib/types"; import { m } from "../components"; diff --git a/ui/components/parameter-table.ts b/ui/components/parameter-table.ts index 42644f1..706f872 100644 --- a/ui/components/parameter-table.ts +++ b/ui/components/parameter-table.ts @@ -21,7 +21,7 @@ import { ClosureComponent, Component } from "mithril"; import { m } from "../components"; import * as taskQueue from "../task-queue"; import { evaluateExpression } from "../store"; -import * as expressionParser from "../../lib/common/expression-parser"; +import * as expressionParser from "../../lib/common/expression/parser"; import { getIcon } from "../icons"; const component: ClosureComponent = (): Component => { diff --git a/ui/components/parameter.ts b/ui/components/parameter.ts index 1c6cb1c..ed96fb2 100644 --- a/ui/components/parameter.ts +++ b/ui/components/parameter.ts @@ -21,14 +21,14 @@ import { ClosureComponent, Component, VnodeDOM } from "mithril"; import { m } from "../components"; import * as taskQueue from "../task-queue"; import * as store from "../store"; -import * as expression from "../../lib/common/expression"; +import { evaluate } from "../../lib/common/expression/util"; import memoize from "../../lib/common/memoize"; import timeAgo from "../timeago"; import { getIcon } from "../icons"; const evaluateParam = memoize((exp, obj, now: number) => { let timestamp = now; - exp = expression.evaluate(exp, null, now, (e) => { + exp = evaluate(exp, null, now, (e) => { if (!Array.isArray(e)) return e; for (let i = 1; i < e.length; ++i) { if ( diff --git a/ui/config-page.ts b/ui/config-page.ts index 6f4edf2..00e14bf 100644 --- a/ui/config-page.ts +++ b/ui/config-page.ts @@ -23,7 +23,7 @@ import * as notifications from "./notifications"; import putFormComponent from "./put-form-component"; import uiConfigComponent from "./ui-config-component"; import * as overlay from "./overlay"; -import { parse, stringify } from "../lib/common/expression-parser"; +import { parse, stringify } from "../lib/common/expression/parser"; import { loadCodeMirror, loadYaml } from "./dynamic-loader"; import { ClosureComponent, Component, Children } from "mithril"; import { getIcon } from "./icons"; diff --git a/ui/devices-page.ts b/ui/devices-page.ts index 7e3f236..59a7026 100644 --- a/ui/devices-page.ts +++ b/ui/devices-page.ts @@ -25,11 +25,10 @@ import filterComponent from "./filter-component"; import * as store from "./store"; import { queueTask, stageDownload } from "./task-queue"; import * as notifications from "./notifications"; -import { parse, stringify } from "../lib/common/expression-parser"; -import { evaluate, extractParams } from "../lib/common/expression"; +import { parse, stringify, map } from "../lib/common/expression/parser"; +import { evaluate, extractParams } from "../lib/common/expression/util"; import memoize from "../lib/common/memoize"; import * as smartQuery from "./smart-query"; -import * as expressionParser from "../lib/common/expression-parser"; const PAGE_SIZE = config.ui.pageSize || 10; @@ -54,7 +53,7 @@ const getDownloadUrl = memoize((filter, indexParameters) => { }); const unpackSmartQuery = memoize((query) => { - return expressionParser.map(query, (e) => { + return map(query, (e) => { if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "Q") return smartQuery.unpack("devices", e[2], e[3]); return e; diff --git a/ui/faults-page.ts b/ui/faults-page.ts index fe51966..1895dc0 100644 --- a/ui/faults-page.ts +++ b/ui/faults-page.ts @@ -26,7 +26,7 @@ import * as store from "./store"; import * as notifications from "./notifications"; import memoize from "../lib/common/memoize"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; import { stringify as yamlStringify } from "../lib/common/yaml"; const PAGE_SIZE = config.ui.pageSize || 10; diff --git a/ui/files-page.ts b/ui/files-page.ts index bf30cca..f59f80d 100644 --- a/ui/files-page.ts +++ b/ui/files-page.ts @@ -28,7 +28,7 @@ import putFormComponent from "./put-form-component"; import indexTableComponent from "./index-table-component"; import * as overlay from "./overlay"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; const PAGE_SIZE = config.ui.pageSize || 10; diff --git a/ui/filter-component.ts b/ui/filter-component.ts index f3b2385..cd28c47 100644 --- a/ui/filter-component.ts +++ b/ui/filter-component.ts @@ -18,13 +18,12 @@ */ import m, { ClosureComponent } from "mithril"; -import { parse, stringify, map } from "../lib/common/expression-parser"; +import { parse, stringify, map } from "../lib/common/expression/parser"; import memoize from "../lib/common/memoize"; import Autocomplete from "./autocomplete-compnent"; import * as smartQuery from "./smart-query"; -import { filterToMongoQuery } from "../lib/mongodb-functions"; +import { validQuery } from "../lib/mongodb-functions"; import { Expression } from "../lib/types"; -import { normalize } from "../lib/common/boolean-expression"; const getAutocomplete = memoize((resource) => { const labels = smartQuery.getLabels(resource); @@ -42,16 +41,6 @@ const getAutocomplete = memoize((resource) => { return autocomplete; }); -function validateQuery(q: Expression): void { - q = normalize(q); - if (Array.isArray(q) && q[0] === "CASE") { - q = q.slice(1).filter((a) => Array.isArray(a)); - if (q.length > 1) q = ["AND", ...q]; - else q = q[0]; - } - if (Array.isArray(q)) filterToMongoQuery(q); -} - function parseFilter(resource, f): Expression { let exp; if (/^[\s0-9a-zA-Z]+:/.test(f)) { @@ -62,16 +51,16 @@ function parseFilter(resource, f): Expression { exp = parse(f); } + const unpacked = map(exp, (e) => { + if (Array.isArray(e) && e[0] === "FUNC") { + if (e[1] === "Q") return smartQuery.unpack(resource, e[2], e[3]); + else if (e[1] === "NOW") return Date.now(); + } + return e; + }); + // Throws exception if invalid Mongo query - validateQuery( - map(exp, (e) => { - if (Array.isArray(e) && e[0] === "FUNC") { - if (e[1] === "Q") return smartQuery.unpack(resource, e[2], e[3]); - else if (e[1] === "NOW") return Date.now(); - } - return e; - }) - ); + validQuery(unpacked, resource); return exp; } diff --git a/ui/permissions-page.ts b/ui/permissions-page.ts index bb8333b..77062b6 100644 --- a/ui/permissions-page.ts +++ b/ui/permissions-page.ts @@ -26,7 +26,7 @@ import putFormComponent from "./put-form-component"; import indexTableComponent from "./index-table-component"; import * as overlay from "./overlay"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; import filterComponent from "./filter-component"; import { Children, ClosureComponent, Component } from "mithril"; import { getIcon } from "./icons"; diff --git a/ui/pie-chart-component.ts b/ui/pie-chart-component.ts index 55dbe95..abaf3a5 100644 --- a/ui/pie-chart-component.ts +++ b/ui/pie-chart-component.ts @@ -19,7 +19,7 @@ import { ClosureComponent, Component, Children } from "mithril"; import { m } from "./components"; -import { stringify } from "../lib/common/expression-parser"; +import { stringify } from "../lib/common/expression/parser"; import memoize from "../lib/common/memoize"; const memoizedStringify = memoize(stringify); diff --git a/ui/presets-page.ts b/ui/presets-page.ts index 528df89..ef1754f 100644 --- a/ui/presets-page.ts +++ b/ui/presets-page.ts @@ -28,7 +28,7 @@ import putFormComponent from "./put-form-component"; import indexTableComponent from "./index-table-component"; import memoize from "../lib/common/memoize"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; const PAGE_SIZE = config.ui.pageSize || 10; diff --git a/ui/provisions-page.ts b/ui/provisions-page.ts index a797831..3cdec52 100644 --- a/ui/provisions-page.ts +++ b/ui/provisions-page.ts @@ -28,7 +28,7 @@ import putFormComponent from "./put-form-component"; import indexTableComponent from "./index-table-component"; import * as overlay from "./overlay"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; import { loadCodeMirror } from "./dynamic-loader"; const PAGE_SIZE = config.ui.pageSize || 10; diff --git a/ui/store.ts b/ui/store.ts index 1107bec..8d51040 100644 --- a/ui/store.ts +++ b/ui/store.ts @@ -18,15 +18,15 @@ */ import m from "mithril"; -import { stringify } from "../lib/common/expression-parser"; -import { or, and, evaluate } from "../lib/common/expression"; +import { stringify } from "../lib/common/expression/parser"; +import { or, and, evaluate } from "../lib/common/expression/util"; import memoize from "../lib/common/memoize"; import { QueryOptions, Expression } from "../lib/types"; import * as notifications from "./notifications"; import { configSnapshot, genieacsVersion } from "./config"; import { QueueTask } from "./task-queue"; import { PingResult } from "../lib/ping"; -import { unionDiff, covers } from "../lib/common/boolean-expression"; +import { unionDiff, covers } from "../lib/common/expression/synth"; const memoizedStringify = memoize(stringify); const memoizedEvaluate = memoize(evaluate); diff --git a/ui/ui-config-component.ts b/ui/ui-config-component.ts index 0ed6ca6..b0b4779 100644 --- a/ui/ui-config-component.ts +++ b/ui/ui-config-component.ts @@ -23,7 +23,7 @@ import * as store from "./store"; import { yaml } from "./dynamic-loader"; import * as configFunctions from "./config-functions"; import codeEditorComponent from "./code-editor-component"; -import { parse } from "../lib/common/expression-parser"; +import { parse } from "../lib/common/expression/parser"; function putActionHandler(prefix: string[], dataYaml: string): Promise { return new Promise((resolve, reject) => { diff --git a/ui/users-page.ts b/ui/users-page.ts index ee8abaf..77b2fbd 100644 --- a/ui/users-page.ts +++ b/ui/users-page.ts @@ -26,7 +26,7 @@ import putFormComponent from "./put-form-component"; import indexTableComponent from "./index-table-component"; import * as overlay from "./overlay"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; import filterComponent from "./filter-component"; import changePasswordComponent from "./change-password-component"; import { Children, ClosureComponent, Component } from "mithril"; diff --git a/ui/virtual-parameters-page.ts b/ui/virtual-parameters-page.ts index f728ed4..1642664 100644 --- a/ui/virtual-parameters-page.ts +++ b/ui/virtual-parameters-page.ts @@ -28,7 +28,7 @@ import putFormComponent from "./put-form-component"; import indexTableComponent from "./index-table-component"; import * as overlay from "./overlay"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; import { loadCodeMirror } from "./dynamic-loader"; const PAGE_SIZE = config.ui.pageSize || 10;