diff --git a/lib/api-functions.ts b/lib/api-functions.ts index 63927cf..adf5f97 100644 --- a/lib/api-functions.ts +++ b/lib/api-functions.ts @@ -31,7 +31,7 @@ import { } from "./connection-request"; import { Expression, Task } from "./types"; import { flattenDevice } from "./mongodb-functions"; -import { evaluate } from "./common/expression"; +import { evaluate } from "./common/expression/util"; export async function connectionRequest( deviceId: string, diff --git a/lib/common/authorizer.ts b/lib/common/authorizer.ts index 0d7929c..3bb7eab 100644 --- a/lib/common/authorizer.ts +++ b/lib/common/authorizer.ts @@ -18,7 +18,7 @@ */ import { PermissionSet, Expression } from "../types"; -import { evaluate, or } from "./expression"; +import { evaluate, or } from "./expression/util"; export default class Authorizer { private declare permissionSets: PermissionSet[]; diff --git a/lib/common/boolean-expression.ts b/lib/common/boolean-expression.ts deleted file mode 100644 index 08127ad..0000000 --- a/lib/common/boolean-expression.ts +++ /dev/null @@ -1,1234 +0,0 @@ -/** - * Copyright 2013-2020 GenieACS Inc. - * - * This file is part of GenieACS. - * - * GenieACS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * GenieACS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with GenieACS. If not, see . - */ - -import * as BI from "./bigint"; -import { espresso, complement, tautology } from "espresso-iisojs"; -import { Expression } from "../types"; -import { map } from "./expression-parser"; -import { and, evaluateCallback } from "./expression"; - -const ZERO = BI.BigInt(0); -const ONE = BI.BigInt(1); -const TWO = BI.BigInt(2); -const NEGATIVE_ONE = BI.BigInt(-1); - -class Indeterminates { - public declare map: Map; - public declare sortedKeys: string[]; - - public constructor(str?: string) { - this.map = new Map(); - if (str) { - this.map.set(str, 1); - this.sortedKeys = [str]; - } else { - this.sortedKeys = []; - } - } - - public reciprocal(): Indeterminates { - const res = new Indeterminates(); - res.sortedKeys = this.sortedKeys; - res.map = new Map(); - for (const [k, v] of this.map) res.map.set(k, 0 - v); - return res; - } - - public static multiply( - indeterminates1: Indeterminates, - indeterminates2: Indeterminates - ): Indeterminates { - const res = new Indeterminates(); - res.sortedKeys = indeterminates1.sortedKeys.slice(); - res.map = new Map(indeterminates1.map); - - for (const [key, val] of indeterminates2.map) { - const v = res.map.get(key); - if (!v) { - res.map.set(key, val); - res.sortedKeys.push(key); - } else { - const v2 = val + v; - if (!v2) { - res.map.delete(key); - res.sortedKeys = res.sortedKeys.filter((s) => s !== key); - } else { - res.map.set(key, v2); - } - } - } - - res.sortedKeys.sort((a, b) => { - if (a.length !== b.length) return b.length - a.length; - else if (a > b) return 1; - else if (a < b) return -1; - return 0; - }); - - return res; - } - - public static compare(a: Indeterminates, b: Indeterminates): number { - if (a.sortedKeys.length !== b.sortedKeys.length) - return b.sortedKeys.length - a.sortedKeys.length; - for (let i = 0; i < a.sortedKeys.length; ++i) { - const k1 = a.sortedKeys[i]; - const w1 = a.map.get(k1); - const k2 = b.sortedKeys[i]; - const w2 = b.map.get(k2); - if (w1 !== w2) return w2 - w1; - if (k1.length > k2.length) return -1; - else if (k1.length < k2.length) return 1; - else if (k1 > k2) return 1; - else if (k1 < k2) return -1; - } - return 0; - } -} - -interface Term { - indeterminates: Indeterminates; - coefficientNumerator: BI.bigint; - coefficientDenominator: BI.bigint; -} - -function findGcd(a: BI.bigint, b: BI.bigint): BI.bigint { - while (BI.ne(b, ZERO)) { - const t = b; - b = BI.rem(a, b); - a = t; - } - return a; -} - -class Polynomial { - public declare terms: Term[]; - - public constructor(terms: Term[]) { - this.terms = terms; - } - - public static simplifyTerms(terms: Term[]): Term[] { - const ts = terms - .slice() - .sort((a: Term, b: Term) => - Indeterminates.compare(a.indeterminates, b.indeterminates) - ); - - for (let i = 1; i < ts.length; ++i) { - const t1 = ts[i - 1]; - const t2 = ts[i]; - if (Indeterminates.compare(t1.indeterminates, t2.indeterminates) === 0) { - const numerator = BI.add( - BI.mul(t1.coefficientNumerator, t2.coefficientDenominator), - BI.mul(t2.coefficientNumerator, t1.coefficientDenominator) - ); - - const denominator = BI.mul( - t1.coefficientDenominator, - t2.coefficientDenominator - ); - - const gcd = findGcd(numerator, denominator); - - ts[i] = { - indeterminates: t2.indeterminates, - coefficientNumerator: BI.div(numerator, gcd), - coefficientDenominator: BI.div(denominator, gcd), - }; - ts[i - 1] = { - indeterminates: t1.indeterminates, - coefficientNumerator: ZERO, - coefficientDenominator: t1.coefficientDenominator, - }; - } - } - return ts.filter((v) => BI.ne(v.coefficientNumerator, ZERO)); - } - - public static fromIndeterminate(indeterminate: Expression): Polynomial { - const indeterminates = new Indeterminates(JSON.stringify(indeterminate)); - const terms = [ - { - indeterminates: indeterminates, - coefficientNumerator: ONE, - coefficientDenominator: ONE, - }, - ]; - return new Polynomial(terms); - } - - public static fromConstant(constant: number): Polynomial { - const [int, frac] = Math.abs(constant).toString(2).split(".", 2); - let numerator = BI.BigInt("0b" + int); - if (constant < 0) numerator = BI.mul(numerator, NEGATIVE_ONE); - let denominator = ONE; - if (frac) { - denominator = BI.exp(TWO, BI.BigInt(frac.length)); - numerator = BI.add( - BI.mul(numerator, denominator), - BI.BigInt("0b" + frac) - ); - } - - const terms = [ - { - indeterminates: new Indeterminates(), - coefficientNumerator: numerator, - coefficientDenominator: denominator, - }, - ]; - - return new Polynomial(terms); - } - - public negation(): Polynomial { - const terms = this.terms.map((t) => ({ - indeterminates: t.indeterminates, - coefficientNumerator: BI.mul(t.coefficientNumerator, NEGATIVE_ONE), - coefficientDenominator: t.coefficientDenominator, - })); - - return new Polynomial(terms); - } - - public reciprocal(): Polynomial { - const terms = this.terms.map((t) => ({ - indeterminates: t.indeterminates.reciprocal(), - coefficientNumerator: t.coefficientDenominator, - coefficientDenominator: t.coefficientNumerator, - })); - return new Polynomial(terms); - } - - public constant(): Polynomial { - const terms = this.terms.filter((t) => !t.indeterminates.sortedKeys.length); - return new Polynomial(terms); - } - - public add(rhs: Polynomial): Polynomial { - return new Polynomial( - Polynomial.simplifyTerms(this.terms.concat(rhs.terms)) - ); - } - - public subtract(rhs: Polynomial): Polynomial { - return this.add(rhs.negation()); - } - - public multiply(rhs: Polynomial): Polynomial { - const terms: Term[] = []; - - for (const t1 of this.terms) { - for (const t2 of rhs.terms) { - const numerator = BI.mul( - t1.coefficientNumerator, - t2.coefficientNumerator - ); - const denominator = BI.mul( - t1.coefficientDenominator, - t2.coefficientDenominator - ); - const gcd = findGcd(numerator, denominator); - - terms.push({ - indeterminates: Indeterminates.multiply( - t1.indeterminates, - t2.indeterminates - ), - coefficientNumerator: BI.div(numerator, gcd), - coefficientDenominator: BI.div(denominator, gcd), - }); - } - } - - return new Polynomial(Polynomial.simplifyTerms(terms)); - } - - public divide(rhs: Polynomial): Polynomial { - return this.multiply(rhs.reciprocal()); - } - - public toString(): string { - const add: string[] = []; - for (const t of this.terms) { - const coefficient = - BI.toNumber(t.coefficientNumerator) / - BI.toNumber(t.coefficientDenominator); - - const mul: string[] = []; - if (t.indeterminates.sortedKeys.length) { - for (const k of t.indeterminates.sortedKeys) { - const w = t.indeterminates.map.get(k); - for (let i = Math.abs(w); i > 0; --i) { - if (w > 0) mul.push(k); - else mul.push(`["/",1,${k}]`); - } - } - - if (coefficient !== 1) mul.push(coefficient.toString()); - - if (mul.length > 1) add.push(`["*",${mul.join(",")}]`); - else add.push(mul["0"]); - } else { - add.push(coefficient.toString()); - } - } - - if (!add.length) return "0"; - else if (add.length === 1) return add[0]; - else return `["+",${add.join(",")}]`; - } -} - -type Minterm = number[]; - -abstract class BoolExprSynth { - public abstract true(context: SynthContext): Minterm[]; - public abstract false(context: SynthContext): Minterm[]; - public abstract null(context: SynthContext): Minterm[]; -} - -class TrueSynth extends BoolExprSynth { - public true(): Minterm[] { - return [[]]; - } - public false(): Minterm[] { - return []; - } - public null(): Minterm[] { - return []; - } -} - -class FalseSynth extends BoolExprSynth { - public true(): Minterm[] { - return []; - } - public false(): Minterm[] { - return [[]]; - } - public null(): Minterm[] { - return []; - } -} - -class NullSynth extends BoolExprSynth { - public true(): Minterm[] { - return []; - } - public false(): Minterm[] { - return []; - } - public null(): Minterm[] { - return [[]]; - } -} - -class VarSynth extends BoolExprSynth { - private declare exp: Expression; - private declare negate: boolean; - public constructor(exp: Expression) { - super(); - this.negate = false; - if (Array.isArray(exp)) { - const op = exp[0]; - if (op === "<>") { - exp = exp.slice(); - exp[0] = "="; - this.negate = true; - } else if (op === ">=") { - exp = exp.slice(); - exp[0] = "<"; - this.negate = true; - } else if (op === "<=") { - exp = exp.slice(); - exp[0] = ">"; - this.negate = true; - } else if (op === "NOT LIKE") { - exp = exp.slice(); - exp[0] = "LIKE"; - this.negate = true; - } - } - this.exp = exp; - } - public true(context: SynthContext): Minterm[] { - const v = context.getVariable(this.exp); - return [[(v << 2) ^ (this.negate ? 1 : 3)]]; - } - public false(context: SynthContext): Minterm[] { - const v = context.getVariable(this.exp); - return [[(v << 2) ^ (this.negate ? 3 : 1)]]; - } - public null(context: SynthContext): Minterm[] { - const v = context.getVariable(this.exp); - return [[v << 2, (v << 2) ^ 2]]; - } -} - -class NotSynth extends BoolExprSynth { - private declare exprSynth: BoolExprSynth; - public constructor(e: BoolExprSynth) { - super(); - this.exprSynth = e; - } - public true(context: SynthContext): Minterm[] { - return this.exprSynth.false(context); - } - public false(context: SynthContext): Minterm[] { - return this.exprSynth.true(context); - } - public null(context: SynthContext): Minterm[] { - return this.exprSynth.null(context); - } -} - -class IsNullSynth extends BoolExprSynth { - private declare exprSynth: BoolExprSynth; - public constructor(e: BoolExprSynth) { - super(); - this.exprSynth = e; - } - public true(context: SynthContext): Minterm[] { - return this.exprSynth.null(context); - } - public false(context: SynthContext): Minterm[] { - return [...this.exprSynth.true(context), ...this.exprSynth.false(context)]; - } - public null(): Minterm[] { - return []; - } -} - -class OrSynth extends BoolExprSynth { - private declare exprSynths: BoolExprSynth[]; - public constructor(...e: BoolExprSynth[]) { - super(); - this.exprSynths = e.filter((ee) => !(ee instanceof FalseSynth)); - const unpacked: BoolExprSynth[] = []; - this.exprSynths = this.exprSynths.filter((ee) => { - if (ee instanceof OrSynth) { - unpacked.push(...ee.exprSynths); - return false; - } - return true; - }); - this.exprSynths.push(...unpacked); - } - public true(context: SynthContext): Minterm[] { - if (this.exprSynths.length === 0) return []; - if (this.exprSynths.length === 1) return this.exprSynths[0].true(context); - if (this.exprSynths.some((e) => e instanceof TrueSynth)) return [[]]; - return this.exprSynths.map((e) => e.true(context)).flat(); - } - public false(context: SynthContext): Minterm[] { - if (this.exprSynths.length === 0) return [[]]; - if (this.exprSynths.length === 1) return this.exprSynths[0].false(context); - if ( - this.exprSynths.some( - (e) => e instanceof TrueSynth || e instanceof NullSynth - ) - ) - return []; - return complement( - this.exprSynths.map((e) => complement(e.false(context))).flat() - ); - } - public null(context: SynthContext): Minterm[] { - if (this.exprSynths.length === 0) return []; - if (this.exprSynths.length === 1) return this.exprSynths[0].null(context); - const n = this.exprSynths.map((e) => e.null(context)).flat(); - const t = this.exprSynths.map((e) => e.true(context)).flat(); - return complement([...complement(n), ...t]); - } -} - -class AndSynth extends BoolExprSynth { - private declare exprSynths: BoolExprSynth[]; - public constructor(...e: BoolExprSynth[]) { - super(); - this.exprSynths = e.filter((ee) => !(ee instanceof TrueSynth)); - const unpacked: BoolExprSynth[] = []; - this.exprSynths = this.exprSynths.filter((ee) => { - if (ee instanceof AndSynth) { - unpacked.push(...ee.exprSynths); - return false; - } - return true; - }); - this.exprSynths.push(...unpacked); - } - public true(context: SynthContext): Minterm[] { - if (this.exprSynths.length === 0) return [[]]; - if (this.exprSynths.length === 1) return this.exprSynths[0].true(context); - if ( - this.exprSynths.some( - (e) => e instanceof FalseSynth || e instanceof NullSynth - ) - ) - return []; - return complement( - this.exprSynths.map((e) => complement(e.true(context))).flat() - ); - } - public false(context: SynthContext): Minterm[] { - if (this.exprSynths.length === 0) return []; - if (this.exprSynths.length === 1) return this.exprSynths[0].false(context); - if (this.exprSynths.some((e) => e instanceof FalseSynth)) return [[]]; - return this.exprSynths.map((e) => e.false(context)).flat(); - } - public null(context: SynthContext): Minterm[] { - if (this.exprSynths.length === 0) return []; - if (this.exprSynths.length === 1) return this.exprSynths[0].null(context); - const n = this.exprSynths.map((e) => e.null(context)).flat(); - const f = this.exprSynths.map((e) => e.false(context)).flat(); - return complement([...complement(n), ...f]); - } -} - -class CaseSynth extends BoolExprSynth { - private declare exprSynths: BoolExprSynth[]; - public constructor(e: BoolExprSynth[]) { - super(); - this.exprSynths = e; - } - public true(context: SynthContext): Minterm[] { - const minterms: Minterm[] = []; - const cumulative: Minterm[] = []; - for (let i = 0; i < this.exprSynths.length; i += 2) { - const w = this.exprSynths[i].true(context); - const t = this.exprSynths[i + 1].true(context); - minterms.push( - ...complement([...cumulative, ...complement(w), ...complement(t)]) - ); - if (i < this.exprSynths.length - 2) { - cumulative.push( - ...complement([ - ...this.exprSynths[i].false(context), - ...this.exprSynths[i].null(context), - ]) - ); - } - } - return minterms; - } - public false(context: SynthContext): Minterm[] { - const minterms: Minterm[] = []; - const cumulative: Minterm[] = []; - for (let i = 0; i < this.exprSynths.length; i += 2) { - const w = this.exprSynths[i].true(context); - const t = this.exprSynths[i + 1].false(context); - minterms.push( - ...complement([...cumulative, ...complement(w), ...complement(t)]) - ); - if (i < this.exprSynths.length - 2) { - cumulative.push( - ...complement([ - ...this.exprSynths[i].false(context), - ...this.exprSynths[i].null(context), - ]) - ); - } - } - return minterms; - } - public null(context: SynthContext): Minterm[] { - const minterms: Minterm[] = []; - const cumulative: Minterm[] = []; - for (let i = 0; i < this.exprSynths.length; i += 2) { - const w = this.exprSynths[i].true(context); - const t = this.exprSynths[i + 1].null(context); - minterms.push( - ...complement([...cumulative, ...complement(w), ...complement(t)]) - ); - cumulative.push( - ...complement([ - ...this.exprSynths[i].false(context), - ...this.exprSynths[i].null(context), - ]) - ); - } - minterms.push(...complement([...cumulative])); - return minterms; - } -} - -const ADDITIVE_IDENTITY = Polynomial.fromConstant(0); -const MULTIPLICATIVE_IDENTITY = Polynomial.fromConstant(1); - -const SWAPPED_OPS = { - "=": "=", - "<>": "<>", - ">": "<", - ">=": "<=", - "<": ">", - "<=": ">=", -}; - -function normalizeCallback(exp: Expression): Expression { - if (!Array.isArray(exp)) return exp; - const op = exp[0]; - - if (op === "FUNC" && exp[1] === "COALESCE") { - const res: Expression[] = ["CASE"]; - for (let i = 2; i < exp.length; ++i) - res.push(normalizeCallback(["IS NOT NULL", exp[i]]), exp[i]); - return normalizeCallback(res); - } - - if (op === "CASE") { - const res = [] as [Expression, Expression][]; - for (let i = 1; i < exp.length; i += 2) { - let w = exp[i]; - if (w instanceof Polynomial) w = JSON.parse(w.toString()); - if (!Array.isArray(w) && !w) continue; - const t = exp[i + 1]; - if (!Array.isArray(t) || t[0] !== "CASE") { - res.push([w, t]); - continue; - } - for (let j = 1; j < t.length; j += 2) res.push([and(w, t[j]), t[j + 1]]); - res.push([w, null]); - if (!Array.isArray(w) && w) break; - } - while (res[res.length - 1][1] == null) res.pop(); - return ["CASE", ...res.flat()]; - } - - const permutations: Map = new Map(); - for (const [i, e] of exp.entries()) { - if (!Array.isArray(e)) continue; - if (e[0] !== "CASE") continue; - const perms: [Expression, Expression][] = []; - for (let j = 1; j < e.length; j += 2) perms.push([e[j], e[j + 1]]); - permutations.set(i, perms); - } - - if (permutations.size) { - let res: [Expression, Expression][] = [[true, exp]]; - for (const [i, perms] of permutations) { - const res2: [Expression, Expression][] = []; - for (const [w, t] of perms) { - res2.push( - ...res.map((r) => { - const e = (r[1] as Expression[]).slice(); - e[i] = t; - return [and(w, r[0]), e] as [Expression, Expression]; - }) - ); - } - res = res2; - } - - for (const r of res) r[1] = normalizeCallback(r[1]); - - if (res[0][0] === true) return res[0][1]; - while (res[res.length - 1][1] == null) res.pop(); - - return ["CASE", ...res.flat()]; - } - - function toPolynomial(e: Expression): Polynomial { - if (e == null) return null; - if (e instanceof Polynomial) return e; - if (typeof e === "number") return Polynomial.fromConstant(e); - if (typeof e === "string") - return Polynomial.fromConstant(parseFloat(e) || 0); - if (typeof e === "boolean") return Polynomial.fromConstant(+e); - return Polynomial.fromIndeterminate(e); - } - - if (op === "+") { - const args: Polynomial[] = []; - for (let i = 1; i < exp.length; ++i) { - const p = toPolynomial(exp[i]); - if (p == null) return null; - args.push(p); - } - return args.reduce( - (previousValue, currentValue) => previousValue.add(currentValue), - ADDITIVE_IDENTITY - ) as unknown as Expression; - } else if (op === "*") { - const args: Polynomial[] = []; - for (let i = 1; i < exp.length; ++i) { - const p = toPolynomial(exp[i]); - if (p == null) return null; - args.push(p); - } - return args.reduce( - (previousValue, currentValue) => previousValue.multiply(currentValue), - MULTIPLICATIVE_IDENTITY - ) as unknown as Expression; - } else if (op === "-") { - const args: Polynomial[] = []; - for (let i = 1; i < exp.length; ++i) { - const p = toPolynomial(exp[i]); - if (p == null) return null; - args.push(p); - } - return args.reduce((previousValue, currentValue) => - previousValue.subtract(currentValue) - ) as unknown as Expression; - } else if (op === "/") { - const args: Polynomial[] = []; - for (let i = 1; i < exp.length; ++i) { - const p = toPolynomial(exp[i]); - if (p == null) return null; - args.push(p); - } - return args.reduce((previousValue, currentValue) => - previousValue.divide(currentValue) - ) as unknown as Expression; - } else if (["=", "<>", ">", ">=", "<", "<="].includes(op)) { - if (exp[1] == null || exp[2] == null) return null; - let lhs: Polynomial, rhs: Polynomial; - if (exp[1] instanceof Polynomial) lhs = exp[1]; - else if (typeof exp[1] === "number") lhs = Polynomial.fromConstant(exp[1]); - - if (exp[2] instanceof Polynomial) rhs = exp[2]; - else if (typeof exp[2] === "number") rhs = Polynomial.fromConstant(exp[2]); - - if (lhs || rhs) { - if (!lhs) lhs = Polynomial.fromIndeterminate(exp[1]); - if (!rhs) rhs = Polynomial.fromIndeterminate(exp[2]); - - lhs = lhs.subtract(rhs); - rhs = lhs.constant().negation(); - lhs = lhs.add(rhs); - - if (!lhs.terms.length) { - exp = [op, JSON.parse(lhs.toString()), JSON.parse(rhs.toString())]; - - if (op === "=") return exp[1] === exp[2]; - else if (op === "<>") return exp[1] !== exp[2]; - else if (op === ">") return exp[1] > exp[2]; - else if (op === ">=") return exp[1] >= exp[2]; - else if (op === "<") return exp[1] < exp[2]; - else if (op === "<=") return exp[1] <= exp[2]; - } else { - let flipOp = 1; - - const n = lhs.terms[0].coefficientNumerator; - const d = lhs.terms[0].coefficientDenominator; - - if (BI.lt(n, ZERO) || BI.lt(d, ZERO)) flipOp *= -1; - - const reciprocal = new Polynomial([ - { - indeterminates: new Indeterminates(), - coefficientNumerator: d, - coefficientDenominator: n, - }, - ]); - - lhs = lhs.multiply(reciprocal); - rhs = rhs.multiply(reciprocal); - - const keys = lhs.terms[0].indeterminates.sortedKeys; - let invert = lhs.terms[0].indeterminates.map.get(keys[0]) < 0 ? -1 : 0; - - for (const t of lhs.terms) - for (const v of t.indeterminates.map.values()) invert += v; - - if (invert < 0) { - flipOp *= -1; - lhs = lhs.reciprocal(); - rhs = rhs.reciprocal(); - } - - if (flipOp < 0) exp = [SWAPPED_OPS[op], lhs, rhs]; - else exp = [op, lhs, rhs]; - } - } - } - - // Restore polynomial expressions - exp = exp.map((e) => - e instanceof Polynomial ? JSON.parse(e.toString()) : e - ); - - exp = evaluateCallback(exp); - - return exp; -} - -export function normalize(expr: Expression): Expression { - expr = map(expr, normalizeCallback); - if (expr instanceof Polynomial) { - expr = JSON.parse(expr.toString()); - } else if (Array.isArray(expr) && expr[0] === "CASE") { - expr = expr.map((e) => - e instanceof Polynomial ? JSON.parse(e.toString()) : e - ); - } - return expr; -} - -function toBoolExprSynth(e: Expression | BoolExprSynth): BoolExprSynth { - if (e instanceof BoolExprSynth) return e; - if (Array.isArray(e)) { - if (e[0] === "CASE") - return new CaseSynth(e.slice(1).map((ee) => toBoolExprSynth(ee))); - return new VarSynth(e); - } - if (e == null) return new NullSynth(); - if (e) return new TrueSynth(); - return new FalseSynth(); -} - -function sopToExpression(sop: number[][], context: SynthContext): Expression { - if (!sop.length) return false; - const res: Expression[] = []; - for (const s of sop) { - if (!s.length) return true; - const conjs: Expression[] = []; - for (const i of s) { - let expr = context.getExpression(i >>> 2); - if (!(i & 1) !== !(i & 2)) expr = ["NOT", expr]; - if (Array.isArray(expr) && expr[0] === "NOT" && Array.isArray(expr[1])) { - const e: Expression[] = expr[1]; - if (e[0] === "IS NULL") expr = ["IS NOT NULL", ...e.slice(1)]; - else if (e[0] === "LIKE") expr = ["NOT LIKE", ...e.slice(1)]; - else if (e[0] === "=") expr = ["<>", ...e.slice(1)]; - else if (e[0] === "<>") expr = ["=", ...e.slice(1)]; - else if (e[0] === ">") expr = ["<=", ...e.slice(1)]; - else if (e[0] === ">=") expr = ["<", ...e.slice(1)]; - else if (e[0] === "<") expr = [">=", ...e.slice(1)]; - else if (e[0] === "<=") expr = [">", ...e.slice(1)]; - else if (e[0] === "NOT") expr = e[1]; - else expr = ["NOT", e]; - } - conjs.push(expr); - } - if (conjs.length > 1) res.push(["AND" as any].concat(conjs)); - else res.push(conjs[0]); - } - if (res.length > 1) return ["OR" as any].concat(res); - return res[0]; -} - -function findIsNullDeps(exp: Expression): Expression[] { - if (!Array.isArray(exp)) return []; - const op = exp[0]; - if (op === "IS NULL" || op === "IS NOT NULL") return []; - - if (op === "FUNC") { - if (exp[1] === "NOW") { - return []; - } else if (exp[1] === "LOWER" || exp[1] === "UPPER") { - return findIsNullDeps(exp[2]); - } else if (exp[1] === "ROUND") { - return exp - .slice(2, 4) - .map((e) => findIsNullDeps(e)) - .flat(); - } - } else if (op !== "PARAM") { - return exp - .slice(1) - .map((e) => findIsNullDeps(e)) - .flat(); - } - return [exp]; -} - -class SynthContext { - private declare _dcSet: number[][]; - private declare _variables: Map; - private declare _expressions: Map; - private declare _isNullRelations: Map; - private declare _isNullVars: Set; - private declare _comparisons: Map< - string, - Map - >; - - constructor() { - this._dcSet = []; - this._variables = new Map(); - this._expressions = new Map(); - this._isNullRelations = new Map(); - this._isNullVars = new Set(); - this._comparisons = new Map(); - } - - getVariable(exp: Expression): number { - const expStr = JSON.stringify(exp); - if (this._variables.has(expStr)) return this._variables.get(expStr); - const v = this._variables.size; - this._variables.set(expStr, v); - this._expressions.set(v, exp); - this.generateIsNullDc(v); - this.generateComparisionDc(v); - - if ( - expStr === '["IS NULL",["PARAM","DeviceID.ID"]]' || - expStr === '["IS NULL",["PARAM","_id"]]' - ) - this._dcSet.push([(v << 2) ^ 3]); - - return v; - } - - private generateIsNullDc(v: number): void { - this._isNullRelations.set(v, []); - const exp = this._expressions.get(v); - - const isNull = new Set( - findIsNullDeps(exp).map((e) => this.getVariable(["IS NULL", e])) - ); - - if (!isNull.size) return; - - for (const n of isNull) { - this._isNullVars.add(n); - this._isNullRelations.get(n).push(v); - this._isNullRelations.get(v).push(n); - this._dcSet.push([(n << 2) ^ 3, (v << 2) ^ 1]); - this._dcSet.push([(n << 2) ^ 3, (v << 2) ^ 3]); - } - this._dcSet.push( - [...isNull].map((n) => (n << 2) ^ 2).concat([(v << 2) ^ 0, (v << 2) ^ 2]) - ); - } - - private generateComparisionDc(v: number): void { - const exp = this._expressions.get(v); - if (!Array.isArray(exp)) return; - const op = exp[0]; - if (![">", "<", "="].includes(op)) return; - const rhs = exp[2]; - const rhsStr = JSON.stringify(rhs); - const lhsStr = JSON.stringify(exp[1]); - let allComps = this._comparisons.get(lhsStr); - if (!allComps) this._comparisons.set(lhsStr, (allComps = new Map())); - if (allComps.has(rhsStr)) return; - - const comp = { - eq: -1, - gt: -1, - lt: -1, - }; - - allComps.set(rhsStr, comp); - - comp.eq = this.getVariable(["=", exp[1], exp[2]]); - comp.gt = this.getVariable([">", exp[1], exp[2]]); - comp.lt = this.getVariable(["<", exp[1], exp[2]]); - - this._dcSet.push([ - (comp.eq << 2) ^ 1, - (comp.gt << 2) ^ 1, - (comp.lt << 2) ^ 1, - ]); - this._dcSet.push([(comp.eq << 2) ^ 3, (comp.gt << 2) ^ 3]); - this._dcSet.push([(comp.eq << 2) ^ 3, (comp.lt << 2) ^ 3]); - this._dcSet.push([(comp.gt << 2) ^ 3, (comp.lt << 2) ^ 3]); - - const type1 = typeof exp[2]; - if (!["boolean", "number", "string"].includes(type1)) return; - - for (const [rhs2Str, comp2] of allComps) { - if (comp2 === comp) continue; - const rhs2 = JSON.parse(rhs2Str); - const type2 = typeof rhs2; - - if (!["boolean", "number", "string"].includes(type2)) continue; - let cmp = 0; - if (type1 === type2) cmp = rhs > rhs2 ? 1 : -1; - else if (type1 === "string") cmp = 1; - else if (type2 === "string") cmp = -1; - else cmp = +rhs - +rhs2; - - const c1 = cmp > 0 ? comp : comp2; - const c2 = cmp > 0 ? comp2 : comp; - - // This is the minimum clauses required if all relavent vars - // were included in the DC set. - // this._dcSet.push([(c1.eq << 2) ^ 3, (c2.eq << 2) ^ 3]); - // this._dcSet.push([(c1.lt << 2) ^ 1, (c2.gt << 2) ^ 1]); - - // But we use non-minimal set because intermediate vars - // between any two may not be present in the DC set. - this._dcSet.push([(c1.lt << 2) ^ 1, (c2.gt << 2) ^ 1]); - this._dcSet.push([(c1.lt << 2) ^ 1, (c2.eq << 2) ^ 3]); - this._dcSet.push([(c1.lt << 2) ^ 1, (c2.lt << 2) ^ 3]); - this._dcSet.push([(c1.eq << 2) ^ 3, (c2.gt << 2) ^ 1]); - this._dcSet.push([(c1.eq << 2) ^ 3, (c2.eq << 2) ^ 3]); - this._dcSet.push([(c1.eq << 2) ^ 3, (c2.lt << 2) ^ 3]); - this._dcSet.push([(c1.gt << 2) ^ 3, (c2.gt << 2) ^ 1]); - this._dcSet.push([(c1.gt << 2) ^ 3, (c2.eq << 2) ^ 3]); - this._dcSet.push([(c1.gt << 2) ^ 3, (c2.lt << 2) ^ 3]); - } - } - - getExpression(v: number): Expression { - return this._expressions.get(v); - } - - getIsNullRelations(v: number): number[] { - return this._isNullRelations.get(v); - } - - isIsNull(v: number): boolean { - return this._isNullVars.has(v); - } - - getDcSet(minterms?: Minterm[]): number[][] { - if (!minterms) return this._dcSet; - const vars = new Set(minterms.flat().map((v) => v >> 2)); - - for (const comps of this._comparisons.values()) { - for (const comp of comps.values()) { - const c = Object.values(comp).filter((v) => !vars.has(v)); - if (c.length === 1) vars.add(c[0]); - } - } - - for (const n of this._isNullVars) - if (this._isNullRelations.get(n).some((v) => vars.has(v))) vars.add(n); - - const dcSet = this._dcSet.filter((m) => m.every((v) => vars.has(v >> 2))); - - return dcSet; - } -} - -function sanitizeMinterms( - minterms: number[][], - context: SynthContext -): number[][] { - const res = [] as number[][]; - loop: for (const m of minterms) { - const merged: Map = new Map(); - for (const i of m) - merged.set(i >> 2, (merged.get(i >> 2) || 0) | (1 << (i & 3))); - const minterm: number[] = []; - const perms: number[][] = []; - for (const [k, v] of merged) { - if (v === 0b1010) continue loop; - const isNullVars = context.getIsNullRelations(k); - const t = k << 2; - if (v === 0b0101) { - if (isNullVars.length === 1) minterm.push((isNullVars[0] << 2) ^ 3); - else perms.push(isNullVars.map((n) => (n << 2) ^ 3)); - } else if (v === 0b0001) { - perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 3]); - } else if (v === 0b0100) { - perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 1]); - } else if (v & 0b1000) { - minterm.push(t ^ 3); - } else if (v & 0b0010) { - minterm.push(t ^ 1); - } - } - let ms = [minterm]; - while (perms.length) { - const newMs: number[][] = []; - const perm = perms.pop(); - for (const p of perm) newMs.push(...ms.map((mm) => [...mm, p])); - ms = newMs; - } - res.push(...ms); - } - return res; -} - -function boolExprSynthToExpression(boolExpr: BoolExprSynth): Expression { - const context = new SynthContext(); - let minterms = boolExpr.true(context); - minterms = sanitizeMinterms(minterms, context); - const canRaise = getCanRaiseCallback(context); - const dcSet = context.getDcSet(minterms); - minterms = espresso(minterms, dcSet, { canRaise }); - return sopToExpression(minterms, context); -} - -function mapCallback(exp: Expression | BoolExprSynth): Expression { - if (!Array.isArray(exp)) return exp as Expression; - if (exp[0] === "CASE") { - exp = exp.slice(); - for (let i = 1; i < exp.length; i += 2) exp[i] = toBoolExprSynth(exp[i]); - return exp; - } - - const op = exp[0]; - if (op === "IS NULL") - return new IsNullSynth(toBoolExprSynth(exp[1])) as unknown as Expression; - - if (op === "IS NOT NULL") { - return new NotSynth( - new IsNullSynth(toBoolExprSynth(exp[1])) - ) as unknown as Expression; - } else if (op === "NOT") { - return new NotSynth(toBoolExprSynth(exp[1])) as unknown as Expression; - } else if (op === "OR") { - return new OrSynth( - ...exp.slice(1).map((a) => toBoolExprSynth(a)) - ) as unknown as Expression; - } else if (op === "AND") { - return new AndSynth( - ...exp.slice(1).map((a) => toBoolExprSynth(a)) - ) as unknown as Expression; - } - for (let i = 1; i < exp.length; ++i) { - if (exp[i] instanceof BoolExprSynth) { - exp[i] = boolExprSynthToExpression(exp[i]); - } else if (Array.isArray(exp[i]) && exp[i][0] === "CASE") { - for (let j = 2; j < exp[i].length; j += 2) { - if (exp[i][j] instanceof BoolExprSynth) - exp[i][j] = boolExprSynthToExpression(exp[i][j]); - } - } - } - return exp; -} - -function getCanRaiseCallback( - context: SynthContext -): (idx: number, set: Set) => boolean { - return (idx: number, set: Set): boolean => { - const i = idx >> 2; - const vars = context.getIsNullRelations(i); - if (!context.isIsNull(i)) return !(idx & 1) || !set.has(idx ^ 3); - - for (const k of vars) { - if (set.has((k << 2) ^ 1)) continue; - if (set.has((k << 2) ^ 3)) continue; - if (set.has((k << 2) ^ 0)) return false; - if (set.has((k << 2) ^ 2)) return false; - } - return true; - }; -} - -export function minimize(expr: Expression, boolean = false): Expression { - expr = normalize(expr); - expr = map(expr, mapCallback); - if (Array.isArray(expr) && expr[0] === "CASE") { - if (!boolean) { - const context = new SynthContext(); - const whens = expr.filter((e, i) => i % 2).map((e) => e.true(context)); - const caseDcSet: Minterm[] = []; - const canRaise = getCanRaiseCallback(context); - const res: Expression = ["CASE"]; - for (let i = 1; i < expr.length; i += 2) { - let minterms = sanitizeMinterms(whens[(i - 1) / 2], context); - const dcSet = context.getDcSet(minterms.concat(caseDcSet)); - minterms = espresso(minterms, dcSet, { canRaise }); - if (!minterms.length) continue; - const w = sopToExpression(minterms, context); - let t = expr[i + 1]; - if (t instanceof BoolExprSynth) t = boolExprSynthToExpression(t); - res.push(w, t); - if (w === true) break; - caseDcSet.push(...minterms); - } - while (res[res.length - 1] == null) res.splice(-2); - if (res.length < 3) return null; - return res; - } - expr = toBoolExprSynth(expr) as unknown as Expression; - } - - if (boolean) expr = toBoolExprSynth(expr) as unknown as Expression; - - if (expr instanceof BoolExprSynth) expr = boolExprSynthToExpression(expr); - return expr; -} - -export function unionDiff( - expr1: Expression, - expr2: Expression -): [Expression, Expression] { - expr2 = normalize(expr2); - - if (!expr2) return [expr1, false]; - - expr2 = map(expr2, mapCallback); - - if (!expr1) { - if (Array.isArray(expr2) && expr2[0] === "CASE") - expr2 = toBoolExprSynth(expr2) as unknown as Expression; - if (expr2 instanceof BoolExprSynth) - expr2 = boolExprSynthToExpression(expr2); - return [expr2, expr2]; - } - - const b2 = toBoolExprSynth(expr2); - expr1 = normalize(expr1); - expr1 = map(expr1, mapCallback); - const b1 = toBoolExprSynth(expr1); - - const context = new SynthContext(); - - const expr2Minterms = b2.true(context); - const expr1Minterms = b1.true(context); - const expr1NullMinterms = b1.null(context); - const expr1FalseMinterms = b1.false(context); - - const unionMinterms = sanitizeMinterms( - [...expr1Minterms, ...expr2Minterms], - context - ); - - const diffMinterms = sanitizeMinterms( - complement([ - ...complement([...expr1NullMinterms, ...expr1FalseMinterms]), - ...complement(expr2Minterms), - ]), - context - ); - - const canRaise = getCanRaiseCallback(context); - - const union = espresso(unionMinterms, context.getDcSet(unionMinterms), { - canRaise, - }); - - const diff = espresso(diffMinterms, context.getDcSet(diffMinterms), { - canRaise, - }); - - return [sopToExpression(union, context), sopToExpression(diff, context)]; -} - -export function covers(expr1: Expression, expr2: Expression): boolean { - expr2 = normalize(expr2); - if (!expr2) return true; - expr1 = normalize(expr1); - if (!Array.isArray(expr1)) return !!expr1; - - expr1 = map(expr1, mapCallback); - const b1 = toBoolExprSynth(expr1); - expr2 = map(expr2, mapCallback); - const b2 = toBoolExprSynth(expr2); - - const context = new SynthContext(); - const expr1Minterms = b1.true(context); - const expr2Minterms = b2.true(context); - - return tautology([ - ...complement(expr2Minterms), - ...context.getDcSet(), - ...expr1Minterms, - ]); -} diff --git a/lib/common/bigint.ts b/lib/common/expression/bigint.ts similarity index 100% rename from lib/common/bigint.ts rename to lib/common/expression/bigint.ts diff --git a/lib/common/expression/normalize.ts b/lib/common/expression/normalize.ts new file mode 100644 index 0000000..70efeea --- /dev/null +++ b/lib/common/expression/normalize.ts @@ -0,0 +1,492 @@ +import * as BI from "./bigint"; +import { Expression } from "../../types"; +import { map } from "./parser"; +import { and, evaluateCallback } from "./util"; + +const ZERO = BI.BigInt(0); +const ONE = BI.BigInt(1); +const TWO = BI.BigInt(2); +const NEGATIVE_ONE = BI.BigInt(-1); + +class Indeterminates { + public declare map: Map; + public declare sortedKeys: string[]; + + public constructor(str?: string) { + this.map = new Map(); + if (str) { + this.map.set(str, 1); + this.sortedKeys = [str]; + } else { + this.sortedKeys = []; + } + } + + public reciprocal(): Indeterminates { + const res = new Indeterminates(); + res.sortedKeys = this.sortedKeys; + res.map = new Map(); + for (const [k, v] of this.map) res.map.set(k, 0 - v); + return res; + } + + public static multiply( + indeterminates1: Indeterminates, + indeterminates2: Indeterminates + ): Indeterminates { + const res = new Indeterminates(); + res.sortedKeys = indeterminates1.sortedKeys.slice(); + res.map = new Map(indeterminates1.map); + + for (const [key, val] of indeterminates2.map) { + const v = res.map.get(key); + if (!v) { + res.map.set(key, val); + res.sortedKeys.push(key); + } else { + const v2 = val + v; + if (!v2) { + res.map.delete(key); + res.sortedKeys = res.sortedKeys.filter((s) => s !== key); + } else { + res.map.set(key, v2); + } + } + } + + res.sortedKeys.sort((a, b) => { + if (a.length !== b.length) return b.length - a.length; + else if (a > b) return 1; + else if (a < b) return -1; + return 0; + }); + + return res; + } + + public static compare(a: Indeterminates, b: Indeterminates): number { + if (a.sortedKeys.length !== b.sortedKeys.length) + return b.sortedKeys.length - a.sortedKeys.length; + for (let i = 0; i < a.sortedKeys.length; ++i) { + const k1 = a.sortedKeys[i]; + const w1 = a.map.get(k1); + const k2 = b.sortedKeys[i]; + const w2 = b.map.get(k2); + if (w1 !== w2) return w2 - w1; + if (k1.length > k2.length) return -1; + else if (k1.length < k2.length) return 1; + else if (k1 > k2) return 1; + else if (k1 < k2) return -1; + } + return 0; + } +} + +interface Term { + indeterminates: Indeterminates; + coefficientNumerator: BI.bigint; + coefficientDenominator: BI.bigint; +} + +function findGcd(a: BI.bigint, b: BI.bigint): BI.bigint { + while (BI.ne(b, ZERO)) { + const t = b; + b = BI.rem(a, b); + a = t; + } + return a; +} + +class Polynomial { + public declare terms: Term[]; + + public constructor(terms: Term[]) { + this.terms = terms; + } + + public static simplifyTerms(terms: Term[]): Term[] { + const ts = terms + .slice() + .sort((a: Term, b: Term) => + Indeterminates.compare(a.indeterminates, b.indeterminates) + ); + + for (let i = 1; i < ts.length; ++i) { + const t1 = ts[i - 1]; + const t2 = ts[i]; + if (Indeterminates.compare(t1.indeterminates, t2.indeterminates) === 0) { + const numerator = BI.add( + BI.mul(t1.coefficientNumerator, t2.coefficientDenominator), + BI.mul(t2.coefficientNumerator, t1.coefficientDenominator) + ); + + const denominator = BI.mul( + t1.coefficientDenominator, + t2.coefficientDenominator + ); + + const gcd = findGcd(numerator, denominator); + + ts[i] = { + indeterminates: t2.indeterminates, + coefficientNumerator: BI.div(numerator, gcd), + coefficientDenominator: BI.div(denominator, gcd), + }; + ts[i - 1] = { + indeterminates: t1.indeterminates, + coefficientNumerator: ZERO, + coefficientDenominator: t1.coefficientDenominator, + }; + } + } + return ts.filter((v) => BI.ne(v.coefficientNumerator, ZERO)); + } + + public static fromIndeterminate(indeterminate: Expression): Polynomial { + const indeterminates = new Indeterminates(JSON.stringify(indeterminate)); + const terms = [ + { + indeterminates: indeterminates, + coefficientNumerator: ONE, + coefficientDenominator: ONE, + }, + ]; + return new Polynomial(terms); + } + + public static fromConstant(constant: number): Polynomial { + const [int, frac] = Math.abs(constant).toString(2).split(".", 2); + let numerator = BI.BigInt("0b" + int); + if (constant < 0) numerator = BI.mul(numerator, NEGATIVE_ONE); + let denominator = ONE; + if (frac) { + denominator = BI.exp(TWO, BI.BigInt(frac.length)); + numerator = BI.add( + BI.mul(numerator, denominator), + BI.BigInt("0b" + frac) + ); + } + + const terms = [ + { + indeterminates: new Indeterminates(), + coefficientNumerator: numerator, + coefficientDenominator: denominator, + }, + ]; + + return new Polynomial(terms); + } + + public negation(): Polynomial { + const terms = this.terms.map((t) => ({ + indeterminates: t.indeterminates, + coefficientNumerator: BI.mul(t.coefficientNumerator, NEGATIVE_ONE), + coefficientDenominator: t.coefficientDenominator, + })); + + return new Polynomial(terms); + } + + public reciprocal(): Polynomial { + const terms = this.terms.map((t) => ({ + indeterminates: t.indeterminates.reciprocal(), + coefficientNumerator: t.coefficientDenominator, + coefficientDenominator: t.coefficientNumerator, + })); + return new Polynomial(terms); + } + + public constant(): Polynomial { + const terms = this.terms.filter((t) => !t.indeterminates.sortedKeys.length); + return new Polynomial(terms); + } + + public add(rhs: Polynomial): Polynomial { + return new Polynomial( + Polynomial.simplifyTerms(this.terms.concat(rhs.terms)) + ); + } + + public subtract(rhs: Polynomial): Polynomial { + return this.add(rhs.negation()); + } + + public multiply(rhs: Polynomial): Polynomial { + const terms: Term[] = []; + + for (const t1 of this.terms) { + for (const t2 of rhs.terms) { + const numerator = BI.mul( + t1.coefficientNumerator, + t2.coefficientNumerator + ); + const denominator = BI.mul( + t1.coefficientDenominator, + t2.coefficientDenominator + ); + const gcd = findGcd(numerator, denominator); + + terms.push({ + indeterminates: Indeterminates.multiply( + t1.indeterminates, + t2.indeterminates + ), + coefficientNumerator: BI.div(numerator, gcd), + coefficientDenominator: BI.div(denominator, gcd), + }); + } + } + + return new Polynomial(Polynomial.simplifyTerms(terms)); + } + + public divide(rhs: Polynomial): Polynomial { + return this.multiply(rhs.reciprocal()); + } + + public toString(): string { + const add: string[] = []; + for (const t of this.terms) { + const coefficient = + BI.toNumber(t.coefficientNumerator) / + BI.toNumber(t.coefficientDenominator); + + const mul: string[] = []; + if (t.indeterminates.sortedKeys.length) { + for (const k of t.indeterminates.sortedKeys) { + const w = t.indeterminates.map.get(k); + for (let i = Math.abs(w); i > 0; --i) { + if (w > 0) mul.push(k); + else mul.push(`["/",1,${k}]`); + } + } + + if (coefficient !== 1) mul.push(coefficient.toString()); + + if (mul.length > 1) add.push(`["*",${mul.join(",")}]`); + else add.push(mul["0"]); + } else { + add.push(coefficient.toString()); + } + } + + if (!add.length) return "0"; + else if (add.length === 1) return add[0]; + else return `["+",${add.join(",")}]`; + } +} + +const ADDITIVE_IDENTITY = Polynomial.fromConstant(0); +const MULTIPLICATIVE_IDENTITY = Polynomial.fromConstant(1); + +const SWAPPED_OPS = { + "=": "=", + "<>": "<>", + ">": "<", + ">=": "<=", + "<": ">", + "<=": ">=", +}; + +function normalizeCallback(exp: Expression): Expression { + if (!Array.isArray(exp)) return exp; + const op = exp[0]; + + if (op === "FUNC" && exp[1] === "COALESCE") { + const res: Expression[] = ["CASE"]; + for (let i = 2; i < exp.length; ++i) + res.push(normalizeCallback(["IS NOT NULL", exp[i]]), exp[i]); + return normalizeCallback(res); + } + + if (op === "CASE") { + const res = [] as [Expression, Expression][]; + for (let i = 1; i < exp.length; i += 2) { + let w = exp[i]; + if (w instanceof Polynomial) w = JSON.parse(w.toString()); + if (!Array.isArray(w) && !w) continue; + const t = exp[i + 1]; + if (!Array.isArray(t) || t[0] !== "CASE") { + res.push([w, t]); + continue; + } + for (let j = 1; j < t.length; j += 2) res.push([and(w, t[j]), t[j + 1]]); + res.push([w, null]); + if (!Array.isArray(w) && w) break; + } + while (res[res.length - 1][1] == null) res.pop(); + return ["CASE", ...res.flat()]; + } + + const permutations: Map = new Map(); + for (const [i, e] of exp.entries()) { + if (!Array.isArray(e)) continue; + if (e[0] !== "CASE") continue; + const perms: [Expression, Expression][] = []; + for (let j = 1; j < e.length; j += 2) perms.push([e[j], e[j + 1]]); + perms.push([true, null]); + permutations.set(i, perms); + } + + if (permutations.size) { + let res: [Expression, Expression][] = [[true, exp]]; + for (const [i, perms] of permutations) { + const res2: [Expression, Expression][] = []; + for (const [w, t] of perms) { + res2.push( + ...res.map((r) => { + const e = (r[1] as Expression[]).slice(); + e[i] = t; + return [and(w, r[0]), e] as [Expression, Expression]; + }) + ); + if (!Array.isArray(w) && w) break; + } + res = res2; + } + + for (const r of res) r[1] = normalizeCallback(r[1]); + + if (res[0][0] === true) return res[0][1]; + while (res[res.length - 1][1] == null) res.pop(); + + return ["CASE", ...res.flat()]; + } + + function toPolynomial(e: Expression): Polynomial { + if (e == null) return null; + if (e instanceof Polynomial) return e; + if (typeof e === "number") return Polynomial.fromConstant(e); + if (typeof e === "string") + return Polynomial.fromConstant(parseFloat(e) || 0); + if (typeof e === "boolean") return Polynomial.fromConstant(+e); + return Polynomial.fromIndeterminate(e); + } + + if (op === "+") { + const args: Polynomial[] = []; + for (let i = 1; i < exp.length; ++i) { + const p = toPolynomial(exp[i]); + if (p == null) return null; + args.push(p); + } + return args.reduce( + (previousValue, currentValue) => previousValue.add(currentValue), + ADDITIVE_IDENTITY + ) as unknown as Expression; + } else if (op === "*") { + const args: Polynomial[] = []; + for (let i = 1; i < exp.length; ++i) { + const p = toPolynomial(exp[i]); + if (p == null) return null; + args.push(p); + } + return args.reduce( + (previousValue, currentValue) => previousValue.multiply(currentValue), + MULTIPLICATIVE_IDENTITY + ) as unknown as Expression; + } else if (op === "-") { + const args: Polynomial[] = []; + for (let i = 1; i < exp.length; ++i) { + const p = toPolynomial(exp[i]); + if (p == null) return null; + args.push(p); + } + return args.reduce((previousValue, currentValue) => + previousValue.subtract(currentValue) + ) as unknown as Expression; + } else if (op === "/") { + const args: Polynomial[] = []; + for (let i = 1; i < exp.length; ++i) { + const p = toPolynomial(exp[i]); + if (p == null) return null; + args.push(p); + } + return args.reduce((previousValue, currentValue) => + previousValue.divide(currentValue) + ) as unknown as Expression; + } else if (["=", "<>", ">", ">=", "<", "<="].includes(op)) { + if (exp[1] == null || exp[2] == null) return null; + let lhs: Polynomial, rhs: Polynomial; + if (exp[1] instanceof Polynomial) lhs = exp[1]; + else if (typeof exp[1] === "number") lhs = Polynomial.fromConstant(exp[1]); + + if (exp[2] instanceof Polynomial) rhs = exp[2]; + else if (typeof exp[2] === "number") rhs = Polynomial.fromConstant(exp[2]); + + if (lhs || rhs) { + if (!lhs) lhs = Polynomial.fromIndeterminate(exp[1]); + if (!rhs) rhs = Polynomial.fromIndeterminate(exp[2]); + + lhs = lhs.subtract(rhs); + rhs = lhs.constant().negation(); + lhs = lhs.add(rhs); + + if (!lhs.terms.length) { + exp = [op, JSON.parse(lhs.toString()), JSON.parse(rhs.toString())]; + + if (op === "=") return exp[1] === exp[2]; + else if (op === "<>") return exp[1] !== exp[2]; + else if (op === ">") return exp[1] > exp[2]; + else if (op === ">=") return exp[1] >= exp[2]; + else if (op === "<") return exp[1] < exp[2]; + else if (op === "<=") return exp[1] <= exp[2]; + } else { + let flipOp = 1; + + const n = lhs.terms[0].coefficientNumerator; + const d = lhs.terms[0].coefficientDenominator; + + if (BI.lt(n, ZERO) || BI.lt(d, ZERO)) flipOp *= -1; + + const reciprocal = new Polynomial([ + { + indeterminates: new Indeterminates(), + coefficientNumerator: d, + coefficientDenominator: n, + }, + ]); + + lhs = lhs.multiply(reciprocal); + rhs = rhs.multiply(reciprocal); + + const keys = lhs.terms[0].indeterminates.sortedKeys; + let invert = lhs.terms[0].indeterminates.map.get(keys[0]) < 0 ? -1 : 0; + + for (const t of lhs.terms) + for (const v of t.indeterminates.map.values()) invert += v; + + if (invert < 0) { + flipOp *= -1; + lhs = lhs.reciprocal(); + rhs = rhs.reciprocal(); + } + + if (flipOp < 0) exp = [SWAPPED_OPS[op], lhs, rhs]; + else exp = [op, lhs, rhs]; + } + } + } + + // Restore polynomial expressions + exp = exp.map((e) => + e instanceof Polynomial ? JSON.parse(e.toString()) : e + ); + + exp = evaluateCallback(exp); + + return exp; +} + +export default function normalize(expr: Expression): Expression { + expr = map(expr, normalizeCallback); + if (expr instanceof Polynomial) { + expr = JSON.parse(expr.toString()); + } else if (Array.isArray(expr) && expr[0] === "CASE") { + expr = expr.map((e) => + e instanceof Polynomial ? JSON.parse(e.toString()) : e + ); + } + return expr; +} diff --git a/lib/common/expression-parser.ts b/lib/common/expression/parser.ts similarity index 98% rename from lib/common/expression-parser.ts rename to lib/common/expression/parser.ts index 9802f4d..596fd8b 100644 --- a/lib/common/expression-parser.ts +++ b/lib/common/expression/parser.ts @@ -18,7 +18,7 @@ */ import parsimmon from "parsimmon"; -import { Expression } from "../types"; +import { Expression } from "../../types"; // Turn escaped characters into real ones (e.g. "\\n" becomes "\n"). function interpretEscapes(str): string { @@ -40,10 +40,7 @@ function interpretEscapes(str): string { }); } -export function map( - exp: Expression, - callback: (e: Expression) => Expression -): Expression { +export function map(exp: Expression, callback: (e: Expression) => T): T { if (!Array.isArray(exp)) return callback(exp); let clone; diff --git a/lib/common/expression/synth.ts b/lib/common/expression/synth.ts new file mode 100644 index 0000000..2814434 --- /dev/null +++ b/lib/common/expression/synth.ts @@ -0,0 +1,914 @@ +import { espresso, complement, tautology } from "espresso-iisojs"; +import { Expression } from "../../types"; +import normalize from "./normalize"; +import { map, parseLikePattern } from "./parser"; + +type Minterm = number[]; + +export abstract class SynthContextBase< + T extends { toString: () => string } = unknown, + U = unknown +> { + public variables = new Map(); + protected clauses = new Map(); + + protected getVar(c: U): number { + const str = c.toString(); + let idx = this.variables.get(str); + if (idx == null) { + idx = this.variables.size; + this.variables.set(str, idx); + this.clauses.set(idx, c); + } + return idx; + } + + protected getClause(v: number): U { + return this.clauses.get(v); + } + + abstract getMinterms(exp: T, res: true | false | null): Minterm[]; + abstract getDcSet(minterms: Minterm[]): Minterm[]; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + canRaise(idx: number, set: Set): boolean { + return true; + } + + sanitizeMinterms(minterms: Minterm[]): Minterm[] { + return minterms; + } + + minimize(minterms: Minterm[], dcSet: Minterm[] = []): Minterm[] { + minterms = this.sanitizeMinterms(minterms); + const canRaise = this.canRaise.bind(this); + return espresso(minterms, [...this.getDcSet(minterms), ...dcSet], { + canRaise, + }); + } +} + +function* findIsNullDeps(exp: Expression): IterableIterator { + if (!Array.isArray(exp)) return; + const op = exp[0]; + if (op === "IS NULL" || op === "IS NOT NULL") return; + + if (op === "FUNC") { + if (exp[1] === "NOW") { + return; + } else if (exp[1] === "LOWER" || exp[1] === "UPPER") { + yield* findIsNullDeps(exp[2]); + } else if (exp[1] === "ROUND") { + for (const e of exp.slice(2, 4)) yield* findIsNullDeps(e); + return; + } + } else if (op !== "PARAM") { + for (const e of exp.slice(1)) yield* findIsNullDeps(e); + return; + } + yield exp; +} + +export abstract class Clause { + private _isNullable: Set; + protected _expression: Expression; + abstract true(context: SynthContextBase): Minterm[]; + abstract false(context: SynthContextBase): Minterm[]; + abstract null(context: SynthContextBase): Minterm[]; + expression(): Expression { + if (this._expression !== undefined) return this._expression; + const context = createSynthContext(); + const minterms = this.true(context); + const minimized = context.minimize(minterms); + this._expression = context.toExpression(minimized) as Expression; + return this._expression; + } + isBoolean(): boolean { + return true; + } + isNullable(c: Clause.IsNull): boolean { + if (!this._isNullable) { + this._isNullable = new Set( + [...this.getNullables()].map((n) => n.toString()) + ); + } + return this._isNullable.has(c.toString()); + } + *getNullables(): IterableIterator { + const exp = this.expression(); + for (const e of findIsNullDeps(exp)) { + if (e === exp) yield new Clause.IsNull(this); + else yield new Clause.IsNull(new Clause.Exp(e)); + } + } + toString(): string { + return JSON.stringify(this.expression()); + } +} + +// eslint-disable-next-line @typescript-eslint/no-namespace +export namespace Clause { + export class Not extends Clause { + constructor(public operand: Clause) { + super(); + } + true(context: SynthContextBase): Minterm[] { + return this.operand.false(context); + } + false(context: SynthContextBase): Minterm[] { + return this.operand.true(context); + } + null(context: SynthContextBase): Minterm[] { + return this.operand.null(context); + } + } + + export class Or extends Clause { + constructor(public operands: Clause[]) { + super(); + } + true(context: SynthContextBase): Minterm[] { + const minterms: Minterm[] = []; + for (const s of this.operands) { + const m = s.true(context); + if (m.length === 1 && !m[0].length) return [[]]; + minterms.push(...m); + } + return minterms; + } + false(context: SynthContextBase): Minterm[] { + const minterms: Minterm[][] = []; + const operands = [...this.operands]; + for (const e of operands) { + if (e instanceof Or) { + operands.push(...e.operands); + continue; + } + const m = e.false(context); + if (!m.length) return []; + minterms.push(m); + } + if (!minterms.length) return [[]]; + if (minterms.length === 1) return minterms[0]; + return complement(minterms.map((m) => complement(m)).flat()); + } + null(context: SynthContextBase): Minterm[] { + const trueMinterms: Minterm[] = []; + const operands = [...this.operands]; + for (const s of operands) { + if (s instanceof Or) { + operands.push(...s.operands); + continue; + } + const t = s.true(context); + if (t.length === 1 && !t[0].length) return []; + trueMinterms.push(...t); + } + + const nullMinterms: Minterm[] = []; + for (const s of operands) { + const n = s.null(context); + if (n.length === 1 && !n[0].length && !trueMinterms.length) return [[]]; + nullMinterms.push(...n); + } + + if (!trueMinterms.length) return nullMinterms; + return complement([...complement(nullMinterms), ...trueMinterms]); + } + } + + export class And extends Clause { + constructor(public operands: Clause[]) { + super(); + } + true(context: SynthContextBase): Minterm[] { + const minterms: Minterm[][] = []; + const operands = [...this.operands]; + for (const s of operands) { + if (s instanceof And) { + operands.push(...s.operands); + continue; + } + const m = s.true(context); + if (!m.length) return []; + if (m.length === 1 && !m[0].length) continue; + minterms.push(m); + } + if (!minterms.length) return [[]]; + if (minterms.length === 1) return minterms[0]; + return complement(minterms.map((m) => complement(m)).flat()); + } + false(context: SynthContextBase): Minterm[] { + const minterms: Minterm[] = []; + for (const s of this.operands) { + const m = s.false(context); + if (m.length === 1 && !m[0].length) return [[]]; + minterms.push(...m); + } + return minterms; + } + null(context: SynthContextBase): Minterm[] { + const falseMinterms: Minterm[] = []; + const operands = [...this.operands]; + for (const s of operands) { + if (s instanceof And) { + operands.push(...s.operands); + continue; + } + const f = s.false(context); + if (f.length === 1 && !f[0].length) return []; + falseMinterms.push(...f); + } + + const nullMinterms: Minterm[] = []; + for (const s of operands) { + const n = s.null(context); + if (n.length === 1 && !n[0].length && !falseMinterms.length) + return [[]]; + nullMinterms.push(...n); + } + + if (!falseMinterms.length) return nullMinterms; + return complement([...complement(nullMinterms), ...falseMinterms]); + } + } + + export class Case extends Clause { + constructor(public clauses: Clause[]) { + super(); + } + true(context: SynthContextBase): Minterm[] { + const minterms: Minterm[] = []; + const cumulative: Minterm[] = []; + for (let i = 0; i < this.clauses.length; i += 2) { + const w = this.clauses[i].true(context); + const t = this.clauses[i + 1].true(context); + minterms.push( + ...complement([...cumulative, ...complement(w), ...complement(t)]) + ); + if (i < this.clauses.length - 2) { + cumulative.push( + ...complement([ + ...this.clauses[i].false(context), + ...this.clauses[i].null(context), + ]) + ); + } + } + return minterms; + } + false(context: SynthContextBase): Minterm[] { + const minterms: Minterm[] = []; + const cumulative: Minterm[] = []; + for (let i = 0; i < this.clauses.length; i += 2) { + const w = this.clauses[i].true(context); + const t = this.clauses[i + 1].false(context); + minterms.push( + ...complement([...cumulative, ...complement(w), ...complement(t)]) + ); + if (i < this.clauses.length - 2) { + cumulative.push( + ...complement([ + ...this.clauses[i].false(context), + ...this.clauses[i].null(context), + ]) + ); + } + } + return minterms; + } + null(context: SynthContextBase): Minterm[] { + const minterms: Minterm[] = []; + const cumulative: Minterm[] = []; + for (let i = 0; i < this.clauses.length; i += 2) { + const w = this.clauses[i].true(context); + const t = this.clauses[i + 1].null(context); + minterms.push( + ...complement([...cumulative, ...complement(w), ...complement(t)]) + ); + cumulative.push( + ...complement([ + ...this.clauses[i].false(context), + ...this.clauses[i].null(context), + ]) + ); + } + minterms.push(...complement([...cumulative])); + return minterms; + } + expression(): Expression { + if (this._expression != null) return this._expression; + if (this.isBoolean()) { + this._expression = new Clause.Not(new Clause.Not(this)).expression(); + return this._expression; + } + const context = createSynthContext(); + const cases: { when: Minterm[]; then: Expression }[] = []; + for (let i = 0; i < this.clauses.length; i += 2) { + let minterms = this.clauses[i].true(context); + const then = this.clauses[i + 1].expression(); + if ( + cases.length && + JSON.stringify(then) === JSON.stringify(cases[cases.length - 1].then) + ) + minterms.push(...cases.pop().when); + minterms = context.minimize( + minterms, + cases.flatMap((c) => c.when) + ); + if (!minterms.length) continue; + cases.push({ when: minterms, then }); + if (minterms.length === 1 && !minterms[0].length) break; + } + while (cases[cases.length - 1].then == null) cases.pop(); + if (!cases.length) return null; + this._expression = [ + "CASE", + ...cases.flatMap((c) => [context.toExpression(c.when), c.then]), + ]; + return this._expression; + } + + isBoolean(): boolean { + for (let i = 1; i < this.clauses.length; i += 2) + if (!this.clauses[i].isBoolean()) return false; + return true; + } + } + + export class IsNull extends Clause { + private _boolean: Clause; + constructor(public operand: Clause) { + super(); + this._boolean = new Not(new Not(this)); + } + true(context: SynthContextBase): Minterm[] { + return this.operand.null(context); + } + false(context: SynthContextBase): Minterm[] { + return complement(this.true(context)); + } + null(): Minterm[] { + return []; + } + isBoolean(): boolean { + return true; + } + *getNullables(): IterableIterator { + // Never returns null + } + expression(): Expression { + const nullables = [...this.operand.getNullables()]; + if (nullables.length === 1 && nullables[0].operand === this.operand) + return ["IS NULL", this.operand.expression()]; + return super.expression(); + } + } + + export class Exp extends Clause { + constructor(public exp: Expression) { + super(); + } + true(context: SynthContextBase): Minterm[] { + if (Array.isArray(this.exp)) return context.getMinterms(this, true); + if (this.exp) return [[]]; + return []; + } + false(context: SynthContextBase): Minterm[] { + if (Array.isArray(this.exp)) return context.getMinterms(this, false); + if (!this.exp && this.exp != null) return [[]]; + return []; + } + null(context: SynthContextBase): Minterm[] { + if (Array.isArray(this.exp)) return context.getMinterms(this, null); + if (this.exp == null) return [[]]; + return []; + } + expression(): Expression { + return this.exp; + } + isBoolean(): boolean { + return this.exp === true || this.exp === false || this.exp == null; + } + } + + export class Compare extends Clause { + constructor( + public lhs: Clause, + public op: ">" | "<" | "=", + public rhs: boolean | number | string + ) { + super(); + } + true(context: SynthContextBase): Minterm[] { + return context.getMinterms(this, true); + } + false(context: SynthContextBase): Minterm[] { + return context.getMinterms(this, false); + } + null(context: SynthContextBase): Minterm[] { + return this.lhs.null(context); + } + *getNullables(): IterableIterator { + yield* this.lhs.getNullables(); + } + isBoolean(): boolean { + return true; + } + expression(): Expression { + return [this.op, this.lhs.expression(), this.rhs]; + } + } + + export class Like extends Clause { + public readonly caseSensitive: boolean; + public readonly contradiction: boolean; + public readonly pattern: string[]; + public readonly lhs: Clause; + + constructor(lhs: Clause, public rhs: string, public esc?: string) { + super(); + const exp = lhs.expression(); + let caseSensitive = true; + let contradiction = false; + if (Array.isArray(exp)) { + if (exp[0] === "FUNC") { + const func = exp[1]; + if (func === "UPPER" || func === "LOWER") { + const p = func === "UPPER" ? rhs.toUpperCase() : rhs.toLowerCase(); + if (p === rhs) caseSensitive = false; + else contradiction = true; + lhs = new Exp(exp[2]); + } + } + } + this.lhs = lhs; + this.pattern = parseLikePattern(rhs, esc); + this.caseSensitive = caseSensitive; + this.contradiction = contradiction; + } + true(context: SynthContextBase): Minterm[] { + return context.getMinterms(this, true); + } + false(context: SynthContextBase): Minterm[] { + return context.getMinterms(this, false); + } + null(context: SynthContextBase): Minterm[] { + return this.lhs.null(context); + } + isBoolean(): boolean { + return true; + } + isNullable(c: IsNull): boolean { + return this.lhs.isNullable(c); + } + getNullables(): IterableIterator { + return this.lhs.getNullables(); + } + expression(): Expression { + let lhs = this.lhs.expression(); + if (this.contradiction) { + if (this.rhs === this.rhs.toLocaleUpperCase()) + lhs = ["FUNC", "LOWER", lhs]; + else lhs = ["FUNC", "UPPER", lhs]; + } else if (!this.caseSensitive) { + if (this.rhs === this.rhs.toLocaleUpperCase()) + lhs = ["FUNC", "UPPER", lhs]; + else lhs = ["FUNC", "LOWER", lhs]; + } + return ["LIKE", lhs, this.rhs, ...(this.esc ? [this.esc] : [])]; + } + } + + export function fromExpression(exp: Expression): Clause { + const res = map(exp, (e) => { + if (!Array.isArray(e)) return new Clause.Exp(e); + let op = e[0]; + let negate = true; + if (op === "NOT LIKE") op = "LIKE"; + else if (op === "IS NOT NULL") op = "IS NULL"; + else if (op === "<>") op = "="; + else if (op === ">=") op = "<"; + else if (op === "<=") op = ">"; + else negate = false; + + let clause: Clause; + if (op === "IS NULL") { + clause = new Clause.IsNull(e[1]); + } else if (op === "NOT") { + clause = new Clause.Not(e[1]); + } else if (op === "OR") { + clause = new Clause.Or(e.slice(1)); + } else if (op === "AND") { + clause = new Clause.And(e.slice(1)); + } else if (op === "CASE") { + clause = new Clause.Case(e.slice(1)); + } else if (op === "LIKE") { + const rhs = e[2] instanceof Exp ? e[2].expression() : null; + const esc = e[3] instanceof Exp ? e[3].expression() : null; + if (typeof rhs === "string" && typeof esc === "string") + clause = new Clause.Like(e[1], rhs, esc); + else if (typeof rhs === "string" && esc == null) + clause = new Clause.Like(e[1], rhs); + } else if (op === ">" || op === "<" || op === "=") { + const rhs = e[2] instanceof Exp ? e[2].expression() : null; + if (["boolean", "number", "string"].includes(typeof rhs)) { + clause = new Clause.Compare( + e[1], + op, + rhs as boolean | number | string + ); + } + } + + if (!clause) { + const args = e.slice(1).map((a) => (a as Clause).expression()); + clause = new Clause.Exp([op, ...args]); + } + + if (negate) clause = new Clause.Not(clause); + return clause; + }); + + return res; + } +} + +function groupBy( + input: T[], + callback: (item: T) => K +): Iterable<[K, T[]]> { + const groups = new Map(); + for (const item of input) { + const key = callback(item); + let arr = groups.get(key); + if (!arr) groups.set(key, (arr = [])); + arr.push(item); + } + + return groups.entries(); +} + +export class SynthContext extends SynthContextBase { + constructor() { + super(); + } + + getMinterms(clause: Clause, res: true | false | null): number[][] { + const v = this.getVar(clause); + if (res === true) return [[(v << 2) ^ 3]]; + else if (res === false) return [[(v << 2) ^ 1]]; + else return [[v << 2, (v << 2) ^ 2]]; + } + + getDcSet(minterms: Minterm[]): number[][] { + const dcSet: number[][] = []; + + const whitelist = new Set([...minterms.flat()].map((v) => v >> 2)); + + const allClauses = [...whitelist].map((v) => this.getClause(v)); + const comparisons = allClauses.filter( + (c) => c instanceof Clause.Compare + ) as Clause.Compare[]; + + // Comparisons + for (const [, clauses] of groupBy(comparisons, (c) => + JSON.stringify(c.lhs) + )) { + const lhs = clauses[0].lhs; + const values = new Set(clauses.map((c) => c.rhs)); + const valuesSorted = [...values].sort((a, b) => { + const ta = typeof a; + const tb = typeof b; + if (ta === tb) return a > b ? 1 : -1; + else if (ta === "string") return 1; + else if (tb === "string") return -1; + return +a - +b; + }); + + for (const [i, v] of valuesSorted.entries()) { + const eq = this.getVar(new Clause.Compare(lhs, "=", v)); + const gt = this.getVar(new Clause.Compare(lhs, ">", v)); + const lt = this.getVar(new Clause.Compare(lhs, "<", v)); + + dcSet.push([(eq << 2) ^ 3, (gt << 2) ^ 3]); + dcSet.push([(lt << 2) ^ 3, (gt << 2) ^ 3]); + dcSet.push([(lt << 2) ^ 3, (eq << 2) ^ 3]); + dcSet.push([(lt << 2) ^ 1, (eq << 2) ^ 1, (gt << 2) ^ 1]); + + const negEquivOp = [lt, eq, gt].filter((o) => !whitelist.has(o)); + if (negEquivOp.length === 1) whitelist.add(negEquivOp[0]); + + for (let j = 0; j < i; j++) { + const eq2 = this.getVar( + new Clause.Compare(lhs, "=", valuesSorted[j]) + ); + const gt2 = this.getVar( + new Clause.Compare(lhs, ">", valuesSorted[j]) + ); + const lt2 = this.getVar( + new Clause.Compare(lhs, "<", valuesSorted[j]) + ); + + // This is the minimum clauses required if all relavent vars + // were included in the DC set. + // dcSet.push([(eq << 2) ^ 3, (eq2 << 2) ^ 3]); + // dcSet.push([(lt << 2) ^ 1, (gt2 << 2) ^ 1]); + + // But we use non-minimal set because intermediate vars + // between any two may not be present in the DC set. + dcSet.push([(gt2 << 2) ^ 1, (lt << 2) ^ 1]); + dcSet.push([(eq2 << 2) ^ 3, (lt << 2) ^ 1]); + dcSet.push([(lt2 << 2) ^ 3, (lt << 2) ^ 1]); + dcSet.push([(gt2 << 2) ^ 1, (gt << 2) ^ 3]); + dcSet.push([(gt2 << 2) ^ 1, (eq << 2) ^ 3]); + dcSet.push([(eq2 << 2) ^ 3, (gt << 2) ^ 3]); + dcSet.push([(eq2 << 2) ^ 3, (eq << 2) ^ 3]); + dcSet.push([(lt2 << 2) ^ 3, (gt << 2) ^ 3]); + dcSet.push([(lt2 << 2) ^ 3, (eq << 2) ^ 3]); + } + } + } + + // LIKE + const likes = allClauses.filter( + (c) => c instanceof Clause.Like + ) as Clause.Like[]; + + for (const [, clauses] of groupBy(likes, (c) => JSON.stringify(c.lhs))) { + for (let i1 = 0; i1 < clauses.length; ++i1) { + const l1 = clauses[i1]; + if (l1.contradiction) { + dcSet.push([(this.getVar(l1) << 2) ^ 3]); + continue; + } + for (let i2 = i1 + 1; i2 < clauses.length; ++i2) { + const l2 = clauses[i2]; + if (l2.contradiction) continue; + let p1 = l1.pattern; + let p2 = l2.pattern; + if (!l1.caseSensitive || !l2.caseSensitive) { + p1 = p1.map((c) => c.toLowerCase()); + p2 = p2.map((c) => c.toLowerCase()); + } + if (likeDisjoint(p1, p2)) { + dcSet.push([ + (this.getVar(l1) << 2) ^ 3, + (this.getVar(l2) << 2) ^ 3, + ]); + } else if ( + (!l1.caseSensitive || l2.caseSensitive) && + likeImplies(p1, p2) + ) { + dcSet.push([ + (this.getVar(l1) << 2) ^ 2, + (this.getVar(l2) << 2) ^ 3, + ]); + dcSet.push([ + (this.getVar(l1) << 2) ^ 1, + (this.getVar(l2) << 2) ^ 0, + ]); + } else if ( + (!l2.caseSensitive || l1.caseSensitive) && + likeImplies(p2, p1) + ) { + dcSet.push([ + (this.getVar(l1) << 2) ^ 3, + (this.getVar(l2) << 2) ^ 2, + ]); + dcSet.push([ + (this.getVar(l1) << 2) ^ 0, + (this.getVar(l2) << 2) ^ 1, + ]); + } + } + } + } + + for (const v of whitelist) { + const clause = this.getClause(v); + const nullables = [...clause.getNullables()].map((c) => this.getVar(c)); + if (nullables.length) { + dcSet.push([ + ...nullables.map((n) => (n << 2) ^ 2), + (v << 2) ^ 0, + (v << 2) ^ 2, + ]); + for (const n of nullables) { + dcSet.push([(n << 2) ^ 3, (v << 2) ^ 1]); + dcSet.push([(n << 2) ^ 3, (v << 2) ^ 3]); + whitelist.add(n); + } + } + + if (clause instanceof Clause.IsNull) { + const str = clause.operand.toString(); + if (str === '["PARAM","DeviceID.ID"]' || str === '["PARAM","_id"]') + dcSet.push([(v << 2) ^ 3]); + } + } + return dcSet.filter((m) => m.every((v) => whitelist.has(v >> 2))); + } + + canRaise(idx: number, set: Set): boolean { + const clause = this.getClause(idx >> 2); + if (clause instanceof Clause.IsNull) { + for (const i of set) { + if (i === idx || i & 1) continue; + const c = this.getClause(i >> 2); + if (c.isNullable(clause)) return false; + } + return true; + } + return !(idx & 1) || !set.has(idx ^ 3); + } + + sanitizeMinterms(minterms: Minterm[]): Minterm[] { + const res = [] as number[][]; + + loop: for (const m of minterms) { + const merged: Map = new Map(); + for (const i of m) + merged.set(i >> 2, (merged.get(i >> 2) || 0) | (1 << (i & 3))); + const minterm: number[] = []; + const perms: number[][] = []; + for (const [k, v] of merged) { + if (v === 0b1010) continue loop; + const isNullVars = [...this.clauses.get(k).getNullables()].map((c) => + this.getVar(c) + ); + const t = k << 2; + if (v === 0b0101) { + if (isNullVars.length === 1) minterm.push((isNullVars[0] << 2) ^ 3); + else perms.push(isNullVars.map((n) => (n << 2) ^ 3)); + } else if (v === 0b0001) { + perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 3]); + } else if (v === 0b0100) { + perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 1]); + } else if (v & 0b1000) { + minterm.push(t ^ 3); + } else if (v & 0b0010) { + minterm.push(t ^ 1); + } + } + let ms = [minterm]; + while (perms.length) { + const newMs: number[][] = []; + const perm = perms.pop(); + for (const p of perm) newMs.push(...ms.map((mm) => [...mm, p])); + ms = newMs; + } + res.push(...ms); + } + return res; + } + + toExpression(sop: number[][]): Expression { + if (!sop.length) return false; + const res: Expression[] = []; + for (const s of sop) { + if (!s.length) return true; + const conjs: Expression[] = []; + for (const i of s) { + let expr = this.getClause(i >>> 2).expression() as Expression; + if (!(i & 1) !== !(i & 2)) expr = ["NOT", expr]; + if ( + Array.isArray(expr) && + expr[0] === "NOT" && + Array.isArray(expr[1]) + ) { + const e: Expression[] = expr[1]; + if (e[0] === "IS NULL") expr = ["IS NOT NULL", ...e.slice(1)]; + else if (e[0] === "LIKE") expr = ["NOT LIKE", ...e.slice(1)]; + else if (e[0] === "=") expr = ["<>", ...e.slice(1)]; + else if (e[0] === "<>") expr = ["=", ...e.slice(1)]; + else if (e[0] === ">") expr = ["<=", ...e.slice(1)]; + else if (e[0] === ">=") expr = ["<", ...e.slice(1)]; + else if (e[0] === "<") expr = [">=", ...e.slice(1)]; + else if (e[0] === "<=") expr = [">", ...e.slice(1)]; + else if (e[0] === "NOT") expr = e[1]; + } + conjs.push(expr); + } + if (conjs.length > 1) res.push(["AND", ...conjs]); + else res.push(conjs[0]); + } + if (res.length > 1) return ["OR", ...res]; + return res[0]; + } +} + +// Classes aren't hoisted in JS but functions are. This function is used to +// create a new SynthContext instance from inside the Clause class. +export function createSynthContext(): SynthContext { + return new SynthContext(); +} + +export function likeImplies(pat1: string[], pat2: string[]): boolean { + let backtrack: [number, number] = null; + + for (let i1 = 0, i2 = 0; ; ++i1, ++i2) { + while (i1 < pat1.length && pat1[i1] === "\\%") backtrack = [i1++, i2]; + + if (i2 >= pat2.length) return i1 >= pat1.length; + + const c = i1 < pat1.length ? pat1[i1] : null; + + if (c !== pat2[i2] && c !== "\\_") { + if (!backtrack) return false; + [i1, i2] = backtrack; + ++backtrack[1]; + } + } +} + +export function likeDisjoint(pat1: string[], pat2: string[]): boolean { + const left1Idx = pat1.indexOf("\\%"); + const left2Idx = pat2.indexOf("\\%"); + const right1Idx = pat1.lastIndexOf("\\%"); + const right2Idx = pat2.lastIndexOf("\\%"); + + const left1 = pat1.slice(0, left1Idx !== -1 ? left1Idx : pat1.length); + const left2 = pat2.slice(0, left2Idx !== -1 ? left2Idx : pat2.length); + const right1 = pat1.slice(right1Idx !== -1 ? right1Idx + 1 : 0).reverse(); + const right2 = pat2.slice(right2Idx !== -1 ? right2Idx + 1 : 0).reverse(); + + for (let i = 0; i < Math.min(left1.length, left2.length); ++i) { + if (left1[i] !== left2[i] && left1[i] !== "\\_" && left2[i] !== "\\_") + return true; + } + + for (let i = 0; i < Math.min(right1.length, right2.length); ++i) { + if (right1[i] !== right2[i] && right1[i] !== "\\_" && right2[i] !== "\\_") + return true; + } + + if (pat1.length === left1.length) + return pat2.filter((c) => c !== "\\%").length > pat1.length; + else if (pat2.length === left2.length) + return pat1.filter((c) => c !== "\\%").length > pat2.length; + + return false; +} + +export function minimize(expr: Expression, boolean = false): Expression { + let synth = Clause.fromExpression(normalize(expr)); + if (boolean) synth = new Clause.Not(new Clause.Not(synth)); + return synth.expression(); +} + +export function unionDiff( + expr1: Expression, + expr2: Expression +): [Expression, Expression] { + expr2 = normalize(expr2); + + if (!expr2) return [expr1, false]; + + const synth2 = Clause.fromExpression(expr2); + + if (!expr1) { + const e = synth2.expression(); + return [e, e]; + } + + expr1 = normalize(expr1); + const synth1 = Clause.fromExpression(expr1); + + const context = new SynthContext(); + + const expr2Minterms = synth2.true(context); + const expr1Minterms = synth1.true(context); + const expr1NullMinterms = synth1.null(context); + const expr1FalseMinterms = synth1.false(context); + + const union = context.minimize([...expr1Minterms, ...expr2Minterms]); + + const diff = context.minimize( + complement([ + ...complement([...expr1NullMinterms, ...expr1FalseMinterms]), + ...complement(expr2Minterms), + ]) + ); + + return [context.toExpression(union), context.toExpression(diff)]; +} + +export function covers(expr1: Expression, expr2: Expression): boolean { + expr2 = normalize(expr2); + if (!expr2) return true; + expr1 = normalize(expr1); + if (!Array.isArray(expr1)) return !!expr1; + + const synt1 = Clause.fromExpression(expr1); + const synt2 = Clause.fromExpression(expr2); + + const context = new SynthContext(); + const expr1Minterms = synt1.true(context); + const expr2Minterms = synt2.true(context); + + return tautology([ + ...complement(expr2Minterms), + ...context.getDcSet([...expr2Minterms, ...expr1Minterms]), + ...expr1Minterms, + ]); +} diff --git a/lib/common/expression.ts b/lib/common/expression/util.ts similarity index 99% rename from lib/common/expression.ts rename to lib/common/expression/util.ts index 8693ea1..45cad04 100644 --- a/lib/common/expression.ts +++ b/lib/common/expression/util.ts @@ -17,8 +17,8 @@ * along with GenieACS. If not, see . */ -import { map, mapAsync, parseLikePattern } from "./expression-parser"; -import { Expression } from "../types"; +import { map, mapAsync, parseLikePattern } from "./parser"; +import { Expression } from "../../types"; const isArray = Array.isArray; diff --git a/lib/connection-request.ts b/lib/connection-request.ts index 84bd257..322885c 100644 --- a/lib/connection-request.ts +++ b/lib/connection-request.ts @@ -21,7 +21,7 @@ import * as crypto from "crypto"; import * as dgram from "dgram"; import { URL } from "url"; import * as http from "http"; -import { evaluateAsync } from "./common/expression"; +import { evaluateAsync } from "./common/expression/util"; import { Expression } from "./types"; import * as auth from "./auth"; import * as extensions from "./extensions"; diff --git a/lib/cwmp.ts b/lib/cwmp.ts index 2305cef..70f2591 100644 --- a/lib/cwmp.ts +++ b/lib/cwmp.ts @@ -25,7 +25,11 @@ import * as config from "./config"; import * as common from "./common"; import * as soap from "./soap"; import * as session from "./session"; -import { evaluateAsync, evaluate, extractParams } from "./common/expression"; +import { + evaluateAsync, + evaluate, + extractParams, +} from "./common/expression/util"; import * as cache from "./cache"; import * as localCache from "./local-cache"; import * as db from "./db"; diff --git a/lib/db.ts b/lib/db.ts index cfd8bc5..4f9ad16 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -20,7 +20,7 @@ import { MongoClient, ObjectId, Collection, Db } from "mongodb"; import { get } from "./config"; import { decodeTag, encodeTag, escapeRegExp } from "./common"; -import { parse } from "./common/expression-parser"; +import { parse } from "./common/expression/parser"; import { DeviceData, Attributes, diff --git a/lib/local-cache.ts b/lib/local-cache.ts index eb3e62f..d620a8e 100644 --- a/lib/local-cache.ts +++ b/lib/local-cache.ts @@ -25,8 +25,8 @@ import * as cache from "./cache"; import { convertOldPrecondition } from "./mongodb-functions"; import * as logger from "./logger"; import * as scheduling from "./scheduling"; -import * as expression from "./common/expression"; -import { parse } from "./common/expression-parser"; +import * as expression from "./common/expression/util"; +import { parse } from "./common/expression/parser"; import { Preset, Expression, diff --git a/lib/mongodb-functions.ts b/lib/mongodb-functions.ts index 0339dbf..0267e3a 100644 --- a/lib/mongodb-functions.ts +++ b/lib/mongodb-functions.ts @@ -1,248 +1,21 @@ -/** - * Copyright 2013-2019 GenieACS Inc. - * - * This file is part of GenieACS. - * - * GenieACS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * GenieACS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with GenieACS. If not, see . - */ - -import { ObjectId } from "mongodb"; -import { map, parse, stringify, parseList } from "./common/expression-parser"; -import { likePatternToRegExp } from "./common/expression"; +import { Filter } from "mongodb"; +import { EJSON } from "bson"; +import { + parse, + stringify, + parseList, + parseLikePattern, +} from "./common/expression/parser"; import { Expression, Fault, Task } from "./types"; import { decodeTag, encodeTag } from "./common"; - -const isArray = Array.isArray; - -export function processDeviceFilter(filter: Expression): Expression { - return map(filter, (exp) => { - if (!isArray(exp)) return exp; - - if (exp[0] === "PARAM") { - const p = exp[1]; - if (p === "DeviceID.ID") return ["PARAM", "_id"]; - else if (p === "DeviceID") return ["PARAM", "_deviceId"]; - else if (p.startsWith("DeviceID.")) - return ["PARAM", "_deviceId._" + p.slice(9)]; - else if (p === "Events.Inform") return ["PARAM", "_lastInform"]; - else if (p === "Events.Registered") return ["PARAM", "_registered"]; - else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"]; - else if (p === "Events.1_BOOT") return ["PARAM", "_lastBoot"]; - } else if ( - isArray(exp[1]) && - exp[1][0] === "PARAM" && - exp[1][1].startsWith("Tags.") - ) { - const t = decodeTag(exp[1][1].slice(5)); - if (exp[0] === "IS NULL") return ["_tags", t, false]; - else if (exp[0] === "IS NOT NULL") return ["_tags", t, true]; - else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true]; - else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true]; - } else if ( - [ - "=", - "<>", - ">", - ">=", - "<", - "<=", - "LIKE", - "NOT LIKE", - "IS NULL", - "IS NOT NULL", - ].includes(exp[0]) - ) { - let e = map(exp, (ee) => { - if ( - isArray(ee) && - ee[0] === "PARAM" && - typeof ee[1] === "string" && - !ee[1].startsWith("_") - ) - return ["PARAM", `${ee[1]}._value`]; - return ee; - }); - - if (typeof e[2] === "number") { - const alt = (e as any[]).slice(); - alt[2] = new Date(e[2]); - if (exp[0] === "<>" || exp[0] === "NOT LIKE") e = ["AND", e, alt]; - else e = ["OR", e, alt]; - } - return e; - } - return exp; - }); -} - -export function processTasksFilter(filter: Expression): Expression { - return map(filter, (exp) => { - if (!isArray(exp)) return exp; - if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) { - const e = exp.slice(); - if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectId(e[2]); - else if (e[1][0] === "PARAM" && e[1][1] === "timestamp") - e[2] = new Date(e[2]); - else if (e[1][0] === "PARAM" && e[1][1] === "expiry") - e[2] = new Date(e[2]); - return e; - } - return exp; - }); -} - -export function processFaultsFilter(filter: Expression): Expression { - return map(filter, (exp) => { - if (!isArray(exp)) return exp; - if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) { - const e = exp.slice(); - if (e[1][0] === "PARAM" && e[1][1] === "timestamp") e[2] = new Date(e[2]); - else if (e[1][0] === "PARAM" && e[1][1] === "expiry") - e[2] = new Date(e[2]); - return e; - } - return exp; - }); -} - -export function filterToMongoQuery(exp: Expression): Record { - function recursive(filter, negate): Record { - const op = filter[0]; - - if (op === "AND") { - filter = filter.filter((f) => f !== true); - if (filter.length === 2) return recursive(filter[1], negate); - } - - if (op === "OR") { - filter = filter.filter((f) => f !== false); - if (filter.length === 2) return recursive(filter[1], negate); - } - - if ((!negate && op === "AND") || (negate && op === "OR")) - return { $and: filter.slice(1).map((f) => recursive(f, negate)) }; - - if ((!negate && op === "OR") || (negate && op === "AND")) - return { $or: filter.slice(1).map((f) => recursive(f, negate)) }; - - if (op === "NOT") return recursive(filter[1], !negate); - - if (isArray(filter[2])) - throw new Error(`Invalid RHS operand of ${op} clause`); - - if (op === "LIKE" || op === "NOT LIKE") { - if (op === "NOT LIKE") negate = !negate; - let param; - let flags; - if (!Array.isArray(filter[1])) - throw new Error(`Invalid LHS operand of ${op} clause`); - if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") { - if (filter[2] !== filter[2].toUpperCase()) - throw new Error(`Invalid RHS operand of ${op} clause`); - param = filter[1][2][1]; - flags = "i"; - } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") { - if (filter[2] !== filter[2].toLowerCase()) - throw new Error(`Invalid RHS operand of ${op} clause`); - param = filter[1][2][1]; - flags = "i"; - } else if (filter[1][0] === "PARAM") { - param = filter[1][1]; - flags = ""; - } else { - throw new Error(`Invalid LHS operand of ${op} clause`); - } - const r = likePatternToRegExp(filter[2], filter[3], flags); - if (negate) return { [param]: { $nin: [r, null] } }; - else return { [param]: r }; - } - - if (op === "_tags") { - let t = filter[2]; - if (negate) t = !t; - if (t) return { _tags: filter[1] }; - else return { _tags: { $ne: filter[1] } }; - } - - if (!isArray(filter[1]) || filter[1][0] !== "PARAM") - throw new Error(`Invalid LHS operand of ${op} clause`); - - if (op === "IS NULL") { - const p = filter[1][1]; - return { [p]: null }; - } - - if (op === "IS NOT NULL") { - const p = filter[1][1]; - return { [p]: { $ne: null } }; - } - - if (isArray(filter[2]) || filter[2] == null) - throw new Error(`Invalid RHS operand of ${op} clause`); - - if (op === "=") { - const p = filter[1][1]; - const v = filter[2]; - if (!negate) return { [p]: v }; - else return { [p]: { $nin: [v, null] } }; - } - - if (op === "<>") { - const p = filter[1][1]; - const v = filter[2]; - if (negate) return { [p]: v }; - else return { [p]: { $nin: [v, null] } }; - } - - if (op === ">") { - const p = filter[1][1]; - const v = filter[2]; - if (negate) return { [p]: { $lte: v } }; - else return { [p]: { $gt: v } }; - } - - if (op === ">=") { - const p = filter[1][1]; - const v = filter[2]; - if (negate) return { [p]: { $lt: v } }; - else return { [p]: { $gte: v } }; - } - - if (op === "<") { - const p = filter[1][1]; - const v = filter[2]; - if (negate) return { [p]: { $gte: v } }; - else return { [p]: { $lt: v } }; - } - - if (op === "<=") { - const p = filter[1][1]; - const v = filter[2]; - if (negate) return { [p]: { $gt: v } }; - else return { [p]: { $lte: v } }; - } - - throw new Error(`Unrecognized operator ${op}`); - } - - if (!isArray(exp)) { - if (exp === true) return {}; - throw new Error("Primitives are not valid queries"); - } - - return recursive(exp, false); -} +import { + SynthContextBase, + likeDisjoint, + likeImplies, +} from "./common/expression/synth"; +import normalize from "./common/expression/normalize"; +import { Clause } from "./common/expression/synth"; +import * as BI from "./common/expression/bigint"; export function processDeviceProjection( projection: Record @@ -506,7 +279,7 @@ export function convertOldPrecondition( } } else if (k === "_tags") { if (typeof v === "object") { - if (isArray(v)) throw new Error(`Invalid type`); + if (Array.isArray(v)) throw new Error(`Invalid type`); const conjs: Expression[] = []; for (const [op, val] of Object.entries(v)) { if (op === "$ne") { @@ -539,7 +312,7 @@ export function convertOldPrecondition( expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]); } else if (typeof v === "object") { - if (isArray(v)) throw new Error(`Invalid type`); + if (Array.isArray(v)) throw new Error(`Invalid type`); const exps: Expression[] = []; for (const [kk, vv] of Object.entries(v)) { @@ -683,3 +456,629 @@ export function preProcessPreset( preset.configurations = [configuration]; return preset; } + +type Minterm = number[]; + +function getParam(exp: Expression, collection: string): string { + if (!Array.isArray(exp)) + throw new Error("Left-hand operand must be a parameter"); + + if (exp[0] !== "PARAM") + throw new Error("Left-hand operand must be a parameter"); + + if (typeof exp[1] !== "string") + throw new Error("Left-hand operand must be a parameter"); + + const p = exp[1]; + if (collection === "devices") { + if (p === "DeviceID.ID") return "_id"; + else if (p === "DeviceID") return "_deviceId"; + else if (p.startsWith("DeviceID.")) return "_deviceId._" + p.slice(9); + else if (p === "Events.Inform") return "_lastInform"; + else if (p === "Events.Registered") return "_registered"; + else if (p === "Events.0_BOOTSTRAP") return "_lastBootstrap"; + else if (p === "Events.1_BOOT") return "_lastBoot"; + else if (!p.endsWith("._value") && !p.startsWith("Tags.")) + return `${p}._value`; + } + + return p; +} + +function getTypes(parameter: string, collection: string): string[] { + if (collection === "devices") { + if (parameter === "_id") return ["string"]; + if (parameter === "_lastInform") return ["date"]; + if (parameter === "_registered") return ["date"]; + if (parameter === "_lastBootstrap") return ["date"]; + if (parameter === "_lastBoot") return ["date"]; + if (parameter.startsWith("_deviceId.")) return ["string"]; + if (parameter === "Reboot._value") return ["date"]; + if (parameter === "FactoryReset._value") return ["date"]; + if (parameter.endsWith("_timestamp")) return ["date"]; + if (parameter.startsWith("Downloads.")) { + if (parameter.endsWith("Download._value")) return ["date"]; + if (parameter.endsWith("Time._value")) return ["date"]; + if (parameter.endsWith("Name._value")) return ["string"]; + if (parameter.endsWith("Type._value")) return ["string"]; + } + if (parameter.endsWith("_value")) + return ["bool", "number", "date", "string"]; + } else if (collection === "tasks") { + if (parameter === "_id") return ["oid"]; + if (parameter === "timestamp") return ["date"]; + if (parameter === "expiry") return ["date"]; + if (parameter === "name") return ["string"]; + if (parameter === "device") return ["string"]; + } else if (collection === "faults") { + if (parameter === "_id") return ["string"]; + if (parameter === "timestamp") return ["date"]; + if (parameter === "expiry") return ["date"]; + if (parameter === "code") return ["string"]; + if (parameter === "retries") return ["number"]; + if (parameter === "channel") return ["string"]; + if (parameter === "device") return ["string"]; + if (parameter === "message") return ["string"]; + } else if (collection === "users") { + if (parameter === "_id") return ["string"]; + if (parameter === "roles") return ["string"]; + else if (parameter === "password") + throw new Error("Cannot query restricted parameters"); + else if (parameter === "salt") + throw new Error("Cannot query restricted parameters"); + } else if (collection === "config") { + if (parameter === "_id") return ["string"]; + else if (parameter === "value") return ["string"]; + } else if (collection === "files") { + if (parameter === "_id") return ["string"]; + if (parameter === "metadata.fileType") return ["string"]; + if (parameter === "metadata.oui") return ["string"]; + if (parameter === "metadata.productClass") return ["string"]; + if (parameter === "metadata.version") return ["string"]; + } else if (collection === "permissions") { + if (parameter === "_id") return ["string"]; + if (parameter === "role") return ["string"]; + if (parameter === "resource") return ["devices"]; + if (parameter === "access") return ["number"]; + if (parameter === "filter") return ["string"]; + if (parameter === "validate") return ["string"]; + } else if (collection === "presets") { + if (parameter === "_id") return ["string"]; + if (parameter === "weight") return ["number"]; + if (parameter === "channel") return ["string"]; + if (parameter === "precondition") return ["string"]; + if (parameter.startsWith("events.")) return ["bool"]; + } else if (collection === "provisions") { + if (parameter === "_id") return ["string"]; + if (parameter === "script") return ["string"]; + } else if (collection === "virtualParameters") { + if (parameter === "_id") return ["string"]; + if (parameter === "script") return ["string"]; + } + + if (parameter === "_id") return ["oid", "string"]; + return ["bool", "number", "date", "string"]; +} + +function roundOid(oid: string, roundUp: boolean): string { + const match = (oid.match(/^[0-9a-f]*/)?.[0] ?? "").slice(0, 24); + let num = BI.BigInt("0x0" + match); + let lastChar = 0; + if (oid.length > match.length) lastChar += oid.charCodeAt(match.length); + if (match.length < 24) lastChar -= 48; + if (lastChar > 0 && roundUp) num++; + num <<= BI.BigInt(4 * (24 - match.length)); + if (lastChar < 0 && !roundUp) --num; + const str = num.toString(16); + if (str.length > 24 || str.startsWith("-")) return null; + return str.padStart(24, "0"); +} + +function groupBy( + input: T[], + callback: (item: T) => K +): Iterable<[K, T[]]> { + const groups = new Map(); + for (const item of input) { + const key = callback(item); + let arr = groups.get(key); + if (!arr) groups.set(key, (arr = [])); + arr.push(item); + } + + return groups.entries(); +} + +abstract class MongoClause { + abstract readonly parameter: string; + abstract toQuery(truthy: boolean): Filter; + toString(): string { + return JSON.stringify(this.toQuery(true)); + } +} + +class MongoClauseArray extends MongoClause { + constructor( + public readonly parameter: string, + public readonly value: string + ) { + super(); + } + + toQuery(truthy: boolean): Filter { + if (truthy) return { [this.parameter]: { $eq: this.value } }; + else return { [this.parameter]: { $ne: this.value } }; + } +} + +class MongoClauseCompare extends MongoClause { + constructor( + public readonly parameter: string, + public readonly op: "$eq" | "$gt" | "$lt" | "$gte" | "$lte", + public readonly value: T, + public readonly type: "" | "bool" | "number" | "string" | "date" | "oid" + ) { + super(); + } + + toQuery(truthy: boolean): Filter { + let v: any = this.value; + if (this.type === "date" && typeof v === "number") + v = { $date: new Date(v).toISOString() }; + if (this.type === "oid" && typeof v === "string") v = { $oid: v }; + + if (truthy) return { [this.parameter]: { [this.op]: v } }; + else if (this.op === "$eq") return { [this.parameter]: { $ne: v } }; + else return { [this.parameter]: { $not: { [this.op]: v } } }; + } +} + +class MongoClauseType extends MongoClause { + constructor(public readonly parameter: string, public readonly type: string) { + super(); + } + + toQuery(truthy: boolean): Filter { + if (truthy) return { [this.parameter]: { $type: this.type } }; + else return { [this.parameter]: { $not: { $type: this.type } } }; + } +} + +class MongoClauseLike extends MongoClause { + readonly pattern: string[]; + constructor( + public readonly parameter: string, + pat: string, + esc: string, + public readonly caseSensitive: boolean + ) { + super(); + this.pattern = parseLikePattern(pat, esc); + } + + toQuery(truthy: boolean): Filter { + const convChars = { + "-": "\\-", + "/": "\\/", + "\\": "\\/", + "^": "\\^", + $: "\\$", + "*": "\\*", + "+": "\\+", + "?": "\\?", + ".": "\\.", + "(": "\\(", + ")": "\\)", + "|": "\\|", + "[": "\\[", + "]": "\\]", + "{": "\\{", + "}": "\\}", + "\\%": ".*", + "\\_": ".", + }; + + const chars = this.pattern.map((c) => convChars[c] ?? c); + chars[0] = chars[0] === ".*" ? "" : "^" + chars[0]; + const l = chars.length - 1; + chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$"; + const pattern = chars.join(""); + const options = this.caseSensitive ? "s" : "is"; + if (truthy) { + return { [this.parameter]: { $regularExpression: { options, pattern } } }; + } else { + return { + [this.parameter]: { + $not: { $regularExpression: { options, pattern } }, + }, + }; + } + } +} + +class MongoSynthContext extends SynthContextBase { + constructor(private readonly collection: string) { + super(); + } + + getMinterms(clause: Clause, res: true | false | null): number[][] { + const exp = clause.expression(); + if (!Array.isArray(exp)) throw new Error("Invalid query expression"); + + if (res === null) { + const minterms: number[][] = []; + for (const dep of clause.getNullables()) { + const e = dep.operand.expression(); + const param = getParam(e, this.collection); + if (param.startsWith("Tags.") && this.collection === "devices") { + const t = decodeTag(param.slice(5)); + const c = new MongoClauseArray("_tags", t); + minterms.push([this.getVar(c) << 1]); + continue; + } + const c = new MongoClauseCompare(param, "$eq", null, ""); + minterms.push([(this.getVar(c) << 1) ^ 1]); + } + return minterms; + } + + if ([">", "<", "="].includes(exp[0])) { + const param = getParam(exp[1], this.collection); + + let rhs = exp[2]; + if (typeof rhs === "boolean") rhs = +rhs; + else if (typeof rhs !== "string" && typeof rhs !== "number") + throw new Error(`Right-hand operand must be a literal value`); + + if ( + param.startsWith("Tags.") && + this.collection === "devices" && + exp[0] === "=" + ) { + const t = decodeTag(param.slice(5)); + const c = new MongoClauseArray("_tags", t); + if (typeof rhs === "string") rhs = 2; + if (exp[0] === "=") { + if ((rhs === 1) !== res) return []; + } else if (exp[0] === ">") { + if ((res && rhs >= 1) || (!res && rhs < 1)) return []; + } else if (exp[0] === "<") { + if ((res && rhs <= 1) || (!res && rhs > 1)) return []; + } + return [[(this.getVar(c) << 1) ^ 1]]; + } + + let op: "$gt" | "$lt" | "$eq" | "$gte" | "$lte"; + if (exp[0] === "=") op = "$eq"; + else if (exp[0] === ">") op = res ? "$gt" : "$lte"; + else if (exp[0] === "<") op = res ? "$lt" : "$gte"; + + const possibleTypes = new Set(getTypes(param, this.collection)); + const clauses: MongoClause[] = []; + + if (typeof rhs === "number") { + if (possibleTypes.has("number")) + clauses.push(new MongoClauseCompare(param, op, rhs, "number")); + + if (possibleTypes.has("date")) + clauses.push(new MongoClauseCompare(param, op, rhs, "date")); + if (possibleTypes.has("bool") && (rhs === 0 || rhs === 1)) + clauses.push(new MongoClauseCompare(param, op, !!rhs, "bool")); + } else if (typeof rhs === "string") { + if (possibleTypes.has("string")) + clauses.push(new MongoClauseCompare(param, op, rhs, "string")); + + if (possibleTypes.has("oid")) { + const oid = roundOid(rhs, op === "$lt" || op === "$lte"); + if (oid && (op !== "$eq" || oid === rhs)) + clauses.push(new MongoClauseCompare(param, op, oid, "oid")); + } + } + + if (op === "$eq" && !res) { + clauses.push(new MongoClauseCompare(param, "$eq", null, "")); + return [clauses.map((c) => this.getVar(c) << 1)]; + } + + // In the following clauses we could use $type operator, but we want the + // final query to use comparison operators to check for type because the + // $type operator in MongoDB doesn't use indexes + if (typeof rhs === "number") { + if (possibleTypes.has("bool")) { + if ( + (rhs > 1 && (op === "$lt" || op === "$lte")) || + (rhs < 0 && (op === "$gt" || op === "$gte")) + ) + clauses.push(new MongoClauseCompare(param, "$gte", false, "bool")); + } + + if (op === "$gt" || op === "$gte") { + if (possibleTypes.has("string")) + clauses.push(new MongoClauseCompare(param, "$gte", "", "string")); + + if (possibleTypes.has("oid")) { + clauses.push( + new MongoClauseCompare( + param, + "$gte", + "000000000000000000000000", + "oid" + ) + ); + } + } + } else if (typeof rhs === "string") { + if (op === "$lt" || op === "$lte") { + if (possibleTypes.has("bool")) + clauses.push(new MongoClauseCompare(param, "$gte", false, "bool")); + + if (possibleTypes.has("number")) { + clauses.push(new MongoClauseCompare(param, "$gte", 0, "number")); + clauses.push(new MongoClauseCompare(param, "$lt", 0, "number")); + } + if (possibleTypes.has("date")) { + clauses.push(new MongoClauseCompare(param, "$gte", 0, "date")); + clauses.push(new MongoClauseCompare(param, "$lt", 0, "date")); + } + } + } + + return clauses.map((c) => [(this.getVar(c) << 1) ^ 1]); + } else if (exp[0] === "LIKE") { + const pat = exp[2]; + if (typeof pat !== "string") + throw new Error("Right-hand operand of 'LIKE' must be a string"); + let p = exp[1]; + let caseSensitive = true; + if ( + Array.isArray(p) && + p[0] === "FUNC" && + ["UPPER", "LOWER"].includes(p[1]) + ) { + if (p[1] === "UPPER" && pat !== pat.toUpperCase()) + return res ? [] : [[]]; + if (p[1] === "LOWER" && pat !== pat.toLowerCase()) + return res ? [] : [[]]; + caseSensitive = false; + p = p[2]; + } + + const param = getParam(p, this.collection); + const c = new MongoClauseLike(param, pat, exp[3], caseSensitive); + if (res) return [[(this.getVar(c) << 1) ^ 1]]; + const typeClase = new MongoClauseType(param, "string"); + const r = [[this.getVar(c) << 1, (this.getVar(typeClase) << 1) ^ 1]]; + return r; + } else { + throw new Error("Invalid query expression"); + } + } + + getDcSet(minterms: Minterm[]): number[][] { + const dcSet: number[][] = []; + + const vars = new Set(minterms.flat().map((l) => l >> 1)); + const clauses = Array.from(vars).map((v) => this.getClause(v)); + + for (const [parameter, clauses2] of groupBy(clauses, (c) => c.parameter)) { + const comparisons = clauses2.filter( + (c) => c instanceof MongoClauseCompare && c.value !== null + ) as MongoClauseCompare[]; + + for (const [type, clauses3] of groupBy(comparisons, (c) => c.type)) { + const isType = this.getVar(new MongoClauseType(parameter, type)); + const values = new Set(clauses3.map((c) => c.value)); + const valuesSorted = Array.from(values).sort((a, b) => + a > b ? 1 : -1 + ); + for (const [i, v] of valuesSorted.entries()) { + const eq = this.getVar( + new MongoClauseCompare(parameter, "$eq", v, type) + ); + const gt = this.getVar( + new MongoClauseCompare(parameter, "$gt", v, type) + ); + const lt = this.getVar( + new MongoClauseCompare(parameter, "$lt", v, type) + ); + const gte = this.getVar( + new MongoClauseCompare(parameter, "$gte", v, type) + ); + const lte = this.getVar( + new MongoClauseCompare(parameter, "$lte", v, type) + ); + + if (type === "bool") { + if (v === false) dcSet.push([(lt << 1) ^ 1]); + else if (v === true) dcSet.push([(gt << 1) ^ 1]); + } else if (type === "string") { + if (v === "") dcSet.push([(lt << 1) ^ 1]); + } else if (type === "oid") { + if (v < "000000000000000000000000") dcSet.push([(lte << 1) ^ 1]); + else if (v === "000000000000000000000000") + dcSet.push([(lt << 1) ^ 1]); + else if (v > "ffffffffffffffffffffffff") + dcSet.push([(gte << 1) ^ 1]); + else if (v === "ffffffffffffffffffffffff") + dcSet.push([(gt << 1) ^ 1]); + } + dcSet.push([(lt << 1) ^ 1, (gte << 1) ^ 1]); + dcSet.push([(gt << 1) ^ 1, (gte << 1) ^ 0]); + dcSet.push([(eq << 1) ^ 0, (lt << 1) ^ 0, (lte << 1) ^ 1]); + dcSet.push([(eq << 1) ^ 1, (gte << 1) ^ 0]); + dcSet.push([(eq << 1) ^ 1, (gt << 1) ^ 1]); + + if (i === 0) { + dcSet.push([(isType << 1) ^ 0, (gte << 1) ^ 1]); + dcSet.push([(isType << 1) ^ 0, (lt << 1) ^ 1]); + } else { + const gt2 = this.getVar( + new MongoClauseCompare( + parameter, + "$gt", + valuesSorted[i - 1], + type + ) + ); + const lte2 = this.getVar( + new MongoClauseCompare( + parameter, + "$lte", + valuesSorted[i - 1], + type + ) + ); + dcSet.push([(gt2 << 1) ^ 0, (gte << 1) ^ 1]); + dcSet.push([(gt2 << 1) ^ 0, (lte2 << 1) ^ 0, (lt << 1) ^ 1]); + } + + if (i === valuesSorted.length - 1) + dcSet.push([(isType << 1) ^ 1, (gt << 1) ^ 0, (lte << 1) ^ 0]); + } + } + + const likes = clauses2.filter( + (c) => c instanceof MongoClauseLike + ) as MongoClauseLike[]; + if (likes.length) { + const isType = this.getVar(new MongoClauseType(parameter, "string")); + for (let i1 = 0; i1 < likes.length; ++i1) { + const l1 = likes[i1]; + let p1 = l1.pattern; + dcSet.push([(this.getVar(l1) << 1) ^ 1, (isType << 1) ^ 0]); + for (let i2 = i1 + 1; i2 < likes.length; ++i2) { + const l2 = likes[i2]; + let p2 = l2.pattern; + if (!l1.caseSensitive || !l2.caseSensitive) { + p1 = p1.map((c) => c.toLowerCase()); + p2 = p2.map((c) => c.toLowerCase()); + } + if (likeDisjoint(p1, p2)) { + dcSet.push([ + (this.getVar(l1) << 1) ^ 1, + (this.getVar(l2) << 1) ^ 1, + ]); + } else if ( + (!l1.caseSensitive || l2.caseSensitive) && + likeImplies(p1, p2) + ) { + dcSet.push([ + (this.getVar(l1) << 1) ^ 0, + (this.getVar(l2) << 1) ^ 1, + ]); + } else if ( + (!l2.caseSensitive || l1.caseSensitive) && + likeImplies(p2, p1) + ) { + dcSet.push([ + (this.getVar(l1) << 1) ^ 1, + (this.getVar(l2) << 1) ^ 0, + ]); + } + } + } + } + + const isNull = this.getVar( + new MongoClauseCompare(parameter, "$eq", null, "") + ); + + const types = getTypes(parameter, this.collection).map((t) => + this.getVar(new MongoClauseType(parameter, t)) + ); + + dcSet.push([(isNull << 1) ^ 0, ...types.map((t) => (t << 1) ^ 0)]); + for (let i = 0; i < types.length; ++i) { + const t1 = types[i]; + dcSet.push([(t1 << 1) ^ 1, (isNull << 1) ^ 1]); + for (let j = i + 1; j < types.length; ++j) { + const t2 = types[j]; + dcSet.push([(t1 << 1) ^ 1, (t2 << 1) ^ 1]); + } + } + + if (parameter === "_id") dcSet.push([(isNull << 1) ^ 1]); + } + return dcSet; + } + + canRaise(i: number, s: Set): boolean { + if (!(i & 1)) return true; + const c = this.getClause(i >> 1); + if (c instanceof MongoClauseCompare) { + for (const j of s) { + if (j === i) continue; + const c2 = this.getClause(j >> 1); + if (c2.parameter !== c.parameter) continue; + if (!(j & 1)) return false; + if (c2 instanceof MongoClauseType) return false; + } + } + return true; + } + + toQuery(minterms: Minterm[]): Filter { + const or = []; + + for (const minterm of minterms) { + const query = {}; + loop: for (const clause of minterm) { + if (!minterm.length) return {}; + const negate = !!(clause & 1); + const c = this.getClause(clause >> 1); + const q = c.toQuery(negate); + if (Object.keys(q).length !== 1) + throw new Error("Invalid query expression"); + const [param, value] = Object.entries(q)[0]; + if (Object.getPrototypeOf(value).constructor !== Object) + throw new Error("Invalid query expression"); + const dests = [query, ...(query["$and"] ?? [])]; + for (const dest of dests) { + if (!(param in dest)) { + dest[param] = value; + continue loop; + } + + let src = value; + let dst = dest[param]; + + if (src["$not"] && dst["$not"]) { + src = src["$not"]; + dst = dst["$not"]; + } + + if (!Object.keys(src).some((k) => k in dst)) { + Object.assign(dst, src); + continue loop; + } + } + + query["$and"] ??= []; + query["$and"].push({ [param]: value }); + } + or.push(query); + } + + if (or.length === 1) return or[0]; + return { $or: or }; + } +} + +export function toMongoQuery( + exp: Expression, + resource: string +): Filter | false { + exp = normalize(exp); + const clause = Clause.fromExpression(normalize(exp)); + const context = new MongoSynthContext(resource); + const minterms = clause.true(context); + const minimized = context.minimize(minterms); + if (!minimized.length) return false; + return EJSON.deserialize(context.toQuery(minimized)); +} + +export function validQuery(exp: Expression, resource: string): void { + const clause = Clause.fromExpression(normalize(exp)); + const context = new MongoSynthContext(resource); + clause.true(context); +} diff --git a/lib/query.ts b/lib/query.ts index 164a79c..da95583 100644 --- a/lib/query.ts +++ b/lib/query.ts @@ -132,7 +132,7 @@ export function expand( if (v && (v["$ne"] != null || v["$not"] != null)) { if (Object.keys(v).length > 1) throw new Error("Cannot mix $ne or $not with other operators"); - for (const c of conditions) newQuery["$an"].push(c); + for (const c of conditions) newQuery["$and"].push(c); } else { newQuery["$and"].push({ $or: conditions }); } diff --git a/lib/ui/api.ts b/lib/ui/api.ts index b3d4178..c21919e 100644 --- a/lib/ui/api.ts +++ b/lib/ui/api.ts @@ -21,8 +21,8 @@ import { PassThrough } from "stream"; import Router from "koa-router"; import * as db from "./db"; import * as apiFunctions from "./api-functions"; -import { evaluate, and, extractParams } from "../common/expression"; -import { parse } from "../common/expression-parser"; +import { evaluate, and, extractParams } from "../common/expression/util"; +import { parse } from "../common/expression/parser"; import * as logger from "../logger"; import { getConfig } from "../local-cache"; import { QueryOptions, Expression } from "../types"; diff --git a/lib/ui/db.ts b/lib/ui/db.ts index b9f6b38..13a1159 100644 --- a/lib/ui/db.ts +++ b/lib/ui/db.ts @@ -21,10 +21,9 @@ import { Db, GridFSBucket, ObjectId } from "mongodb"; import { Script } from "vm"; import { onConnect, optimizeProjection } from "../db"; import * as mongodbFunctions from "../mongodb-functions"; -import * as expression from "../common/expression"; +import { evaluate } from "../common/expression/util"; import { QueryOptions, Expression } from "../types"; import { Readable } from "stream"; -import { minimize } from "../common/boolean-expression"; const RESOURCE_COLLECTION = { files: "fs.files", @@ -54,28 +53,9 @@ export function query( callback?: (doc: any) => void ): Promise { options = options || {}; - let q; - filter = expression.evaluate(filter, null, Date.now()); - filter = minimize(filter, true); - - if (Array.isArray(filter)) { - if (resource === "devices") { - filter = mongodbFunctions.processDeviceFilter(filter); - } else if (resource === "tasks") { - filter = mongodbFunctions.processTasksFilter(filter); - } else if (resource === "faults") { - filter = mongodbFunctions.processFaultsFilter(filter); - } else if (resource === "users") { - // Protect against brute force, and dictionary attacks - const params = expression.extractParams(filter); - if (params.includes("password") || params.includes("salt")) - return Promise.reject(new Error("Invalid users filter")); - } - - q = mongodbFunctions.filterToMongoQuery(filter); - } else if (!filter) { - return Promise.resolve([]); - } + filter = evaluate(filter, null, Date.now()); + const q = mongodbFunctions.toMongoQuery(filter, resource); + if (!q) return Promise.resolve([]); return new Promise((resolve, reject) => { const collection = db.collection(RESOURCE_COLLECTION[resource] || resource); @@ -152,22 +132,9 @@ export function query( export function count(resource: string, filter: Expression): Promise { const collection = db.collection(RESOURCE_COLLECTION[resource] || resource); - let q: Parameters[0]; - filter = expression.evaluate(filter, null, Date.now()); - filter = minimize(filter, true); - - if (Array.isArray(filter)) { - if (resource === "devices") - filter = mongodbFunctions.processDeviceFilter(filter); - else if (resource === "tasks") - filter = mongodbFunctions.processTasksFilter(filter); - else if (resource === "faults") - filter = mongodbFunctions.processFaultsFilter(filter); - q = mongodbFunctions.filterToMongoQuery(filter); - } else if (!filter) { - return Promise.resolve(0); - } - + filter = evaluate(filter, null, Date.now()); + const q = mongodbFunctions.toMongoQuery(filter, resource); + if (!q) return Promise.resolve(0); return collection.countDocuments(q); } diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 594b1f8..9541d3d 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -10,7 +10,7 @@ "license": "AGPL-3.0", "dependencies": { "@breejs/later": "^4.1.0", - "espresso-iisojs": "^1.0.4", + "espresso-iisojs": "^1.0.7", "iconv-lite": "^0.6.3", "ipaddr.js": "^2.0.1", "jsonwebtoken": "^8.5.1", @@ -3041,9 +3041,9 @@ } }, "node_modules/espresso-iisojs": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/espresso-iisojs/-/espresso-iisojs-1.0.4.tgz", - "integrity": "sha512-QmdiGu97ky2pLjnBOmyTVqDQMrTOFVnJKF4Aj/dw6QoOwa29+Hk2sTmy8GQxnMW7f8av2hRgqI6mmWXdhi1jVw==" + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/espresso-iisojs/-/espresso-iisojs-1.0.7.tgz", + "integrity": "sha512-yHEY+CC9wqEyA1N+KC03xmyN+lFPZdpXaLUxf6vrofE0cMXXIkOg08tXXPmq5fnOoLgxRfm5ikrmYD8/gIhRig==" }, "node_modules/esprima": { "version": "4.0.1", @@ -10177,9 +10177,9 @@ } }, "espresso-iisojs": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/espresso-iisojs/-/espresso-iisojs-1.0.4.tgz", - "integrity": "sha512-QmdiGu97ky2pLjnBOmyTVqDQMrTOFVnJKF4Aj/dw6QoOwa29+Hk2sTmy8GQxnMW7f8av2hRgqI6mmWXdhi1jVw==" + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/espresso-iisojs/-/espresso-iisojs-1.0.7.tgz", + "integrity": "sha512-yHEY+CC9wqEyA1N+KC03xmyN+lFPZdpXaLUxf6vrofE0cMXXIkOg08tXXPmq5fnOoLgxRfm5ikrmYD8/gIhRig==" }, "esprima": { "version": "4.0.1", diff --git a/package.json b/package.json index e30247b..a326ccb 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ }, "dependencies": { "@breejs/later": "^4.1.0", - "espresso-iisojs": "^1.0.4", + "espresso-iisojs": "^1.0.7", "iconv-lite": "^0.6.3", "ipaddr.js": "^2.0.1", "jsonwebtoken": "^8.5.1", diff --git a/test/mongodb-functions.ts b/test/mongodb-functions.ts index 0983b59..f35c09c 100644 --- a/test/mongodb-functions.ts +++ b/test/mongodb-functions.ts @@ -1,6 +1,8 @@ import ava from "ava"; +import { EJSON } from "bson"; +import { Filter } from "mongodb"; import * as mongodbFunctions from "../lib/mongodb-functions"; -import { stringify, parse } from "../lib/common/expression-parser"; +import { stringify, parse } from "../lib/common/expression/parser"; ava("convertOldPrecondition", (t) => { const tests = [ @@ -71,21 +73,31 @@ ava("convertOldPrecondition", (t) => { } }); -ava("filterToMongoQuery", async (t) => { - const queries: [string, Record][] = [ +ava("toMongoQuery", async (t) => { + const queries: [string, Filter | false][] = [ ["true", {}], - ["Tags.tag1 = true", { _tags: "tag1" }], - ["Tags.tag1 <> false", { _tags: "tag1" }], + ["Tags.tag1 = true", { _tags: { $eq: "tag1" } }], + ["Tags.tag1 <> false", { _tags: { $eq: "tag1" } }], ["Tags.tag1 IS NULL", { _tags: { $ne: "tag1" } }], - ["Tags.tag1 = 123", { "Tags.tag1": 123 }], - ["Param1 = 'value1'", { "Param1._value": "value1" }], - ["Param1 <> 'value1'", { "Param1._value": { $nin: ["value1", null] } }], + ["Tags.tag1 = 123", false], + ["Param1 = 'value1'", { "Param1._value": { $eq: "value1" } }], + [ + "Param1 <> 'value1'", + { + "Param1._value": { $ne: "value1" }, + $and: [{ "Param1._value": { $ne: null } }], + }, + ], [ "Param1 <> 1657844103524", { + "Param1._value": { $ne: 1657844103524 }, + $and: [ - { "Param1._value": { $nin: [1657844103524, null] } }, - { "Param1._value": { $nin: [new Date(1657844103524), null] } }, + { + "Param1._value": { $ne: { $date: "2022-07-15T00:15:03.524Z" } }, + }, + { "Param1._value": { $ne: null } }, ], }, ], @@ -93,66 +105,70 @@ ava("filterToMongoQuery", async (t) => { "Param1 = 1657844103524", { $or: [ - { "Param1._value": 1657844103524 }, - { "Param1._value": new Date(1657844103524) }, + { "Param1._value": { $eq: { $date: "2022-07-15T00:15:03.524Z" } } }, + { "Param1._value": { $eq: 1657844103524 } }, ], }, ], ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }], ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }], - ["Param1 LIKE 'value'", { "Param1._value": /^value$/ }], - ["LOWER(Param1) LIKE 'value'", { "Param1._value": /^value$/i }], + [ + "Param1 LIKE 'value'", + { + "Param1._value": { + $regularExpression: { options: "s", pattern: "^value$" }, + }, + }, + ], + + [ + "LOWER(Param1) LIKE 'value'", + { + "Param1._value": { + $regularExpression: { options: "is", pattern: "^value$" }, + }, + }, + ], [ "Param1 <> 'value2' OR NOT (Param2 = 'value1' OR Param1 < 'value2')", { $or: [ - { "Param1._value": { $nin: ["value2", null] } }, { - $and: [ - { "Param2._value": { $nin: ["value1", null] } }, - { "Param1._value": { $gte: "value2" } }, - ], + "Param2._value": { $ne: null }, + $and: [{ "Param2._value": { $ne: "value1" } }], + "Param1._value": { $eq: "value2" }, + }, + { + "Param1._value": { $ne: "value2" }, + $and: [{ "Param1._value": { $ne: null } }], }, ], }, ], [ "Param1 <> 'value2' OR Param1 IS NULL", - { - $or: [ - { "Param1._value": { $nin: ["value2", null] } }, - { "Param1._value": null }, - ], - }, + { "Param1._value": { $ne: "value2" } }, ], ]; for (const [expStr, expect] of queries) { const exp = parse(expStr); - const query = mongodbFunctions.filterToMongoQuery( - mongodbFunctions.processDeviceFilter(exp) - ); + let query = mongodbFunctions.toMongoQuery(exp, "devices"); + if (query) query = EJSON.serialize(query); t.deepEqual(query, expect); } const failQueries: [any, string][] = [ - ["Param1 = Param2", "Invalid RHS operand of = clause"], - ["Param1 LIKE Param2", "Invalid RHS operand of LIKE clause"], - ["1 LIKE '1'", "Invalid LHS operand of LIKE clause"], - ["1 = 2", "Invalid LHS operand of = clause"], - ["1 IS NULL", "Invalid LHS operand of IS NULL clause"], - ["false", "Primitives are not valid queries"], - ["UPPER(Param1) LIKE 'value'", "Invalid RHS operand of LIKE clause"], + ["Param1 = Param2", "Right-hand operand must be a literal value"], + ["Param1 LIKE Param2", "Right-hand operand of 'LIKE' must be a string"], + ["NOW() = 1", "Left-hand operand must be a parameter"], + ["param{param2} = 1", "Left-hand operand must be a parameter"], ]; for (const [expStr, err] of failQueries) { const exp = parse(expStr); - t.throws( - () => - mongodbFunctions.filterToMongoQuery( - mongodbFunctions.processDeviceFilter(exp) - ), - { message: err } - ); + t.throws(() => mongodbFunctions.toMongoQuery(exp, "devices"), { + message: err, + }); } }); diff --git a/test/boolean-expression.ts b/test/synth.ts similarity index 96% rename from test/boolean-expression.ts rename to test/synth.ts index 5db12b7..c2a2f61 100644 --- a/test/boolean-expression.ts +++ b/test/synth.ts @@ -1,6 +1,6 @@ import ava, { TestInterface } from "ava"; -import { minimize, unionDiff } from "../lib/common/boolean-expression"; -import { parse, stringify } from "../lib/common/expression-parser"; +import { minimize, unionDiff } from "../lib/common/expression/synth"; +import { parse, stringify } from "../lib/common/expression/parser"; import initSqlJs from "sql.js/dist/sql-asm.js"; const test = ava as TestInterface<{ diff --git a/ui/components/all-parameters.ts b/ui/components/all-parameters.ts index 376df1e..2b59b7f 100644 --- a/ui/components/all-parameters.ts +++ b/ui/components/all-parameters.ts @@ -20,7 +20,7 @@ import { ClosureComponent, Component } from "mithril"; import { m } from "../components"; import * as taskQueue from "../task-queue"; -import { parse } from "../../lib/common/expression-parser"; +import { parse } from "../../lib/common/expression/parser"; import memoize from "../../lib/common/memoize"; import { getIcon } from "../icons"; import { evaluateExpression } from "../store"; diff --git a/ui/components/container.ts b/ui/components/container.ts index a130432..7c7c95c 100644 --- a/ui/components/container.ts +++ b/ui/components/container.ts @@ -18,7 +18,7 @@ */ import { Attributes, ClosureComponent, Component } from "mithril"; -import { map } from "../../lib/common/expression-parser"; +import { map } from "../../lib/common/expression/parser"; import memoize from "../../lib/common/memoize"; import { Expression } from "../../lib/types"; import { m } from "../components"; diff --git a/ui/components/parameter-table.ts b/ui/components/parameter-table.ts index 42644f1..706f872 100644 --- a/ui/components/parameter-table.ts +++ b/ui/components/parameter-table.ts @@ -21,7 +21,7 @@ import { ClosureComponent, Component } from "mithril"; import { m } from "../components"; import * as taskQueue from "../task-queue"; import { evaluateExpression } from "../store"; -import * as expressionParser from "../../lib/common/expression-parser"; +import * as expressionParser from "../../lib/common/expression/parser"; import { getIcon } from "../icons"; const component: ClosureComponent = (): Component => { diff --git a/ui/components/parameter.ts b/ui/components/parameter.ts index 1c6cb1c..ed96fb2 100644 --- a/ui/components/parameter.ts +++ b/ui/components/parameter.ts @@ -21,14 +21,14 @@ import { ClosureComponent, Component, VnodeDOM } from "mithril"; import { m } from "../components"; import * as taskQueue from "../task-queue"; import * as store from "../store"; -import * as expression from "../../lib/common/expression"; +import { evaluate } from "../../lib/common/expression/util"; import memoize from "../../lib/common/memoize"; import timeAgo from "../timeago"; import { getIcon } from "../icons"; const evaluateParam = memoize((exp, obj, now: number) => { let timestamp = now; - exp = expression.evaluate(exp, null, now, (e) => { + exp = evaluate(exp, null, now, (e) => { if (!Array.isArray(e)) return e; for (let i = 1; i < e.length; ++i) { if ( diff --git a/ui/config-page.ts b/ui/config-page.ts index 6f4edf2..00e14bf 100644 --- a/ui/config-page.ts +++ b/ui/config-page.ts @@ -23,7 +23,7 @@ import * as notifications from "./notifications"; import putFormComponent from "./put-form-component"; import uiConfigComponent from "./ui-config-component"; import * as overlay from "./overlay"; -import { parse, stringify } from "../lib/common/expression-parser"; +import { parse, stringify } from "../lib/common/expression/parser"; import { loadCodeMirror, loadYaml } from "./dynamic-loader"; import { ClosureComponent, Component, Children } from "mithril"; import { getIcon } from "./icons"; diff --git a/ui/devices-page.ts b/ui/devices-page.ts index 7e3f236..59a7026 100644 --- a/ui/devices-page.ts +++ b/ui/devices-page.ts @@ -25,11 +25,10 @@ import filterComponent from "./filter-component"; import * as store from "./store"; import { queueTask, stageDownload } from "./task-queue"; import * as notifications from "./notifications"; -import { parse, stringify } from "../lib/common/expression-parser"; -import { evaluate, extractParams } from "../lib/common/expression"; +import { parse, stringify, map } from "../lib/common/expression/parser"; +import { evaluate, extractParams } from "../lib/common/expression/util"; import memoize from "../lib/common/memoize"; import * as smartQuery from "./smart-query"; -import * as expressionParser from "../lib/common/expression-parser"; const PAGE_SIZE = config.ui.pageSize || 10; @@ -54,7 +53,7 @@ const getDownloadUrl = memoize((filter, indexParameters) => { }); const unpackSmartQuery = memoize((query) => { - return expressionParser.map(query, (e) => { + return map(query, (e) => { if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "Q") return smartQuery.unpack("devices", e[2], e[3]); return e; diff --git a/ui/faults-page.ts b/ui/faults-page.ts index fe51966..1895dc0 100644 --- a/ui/faults-page.ts +++ b/ui/faults-page.ts @@ -26,7 +26,7 @@ import * as store from "./store"; import * as notifications from "./notifications"; import memoize from "../lib/common/memoize"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; import { stringify as yamlStringify } from "../lib/common/yaml"; const PAGE_SIZE = config.ui.pageSize || 10; diff --git a/ui/files-page.ts b/ui/files-page.ts index bf30cca..f59f80d 100644 --- a/ui/files-page.ts +++ b/ui/files-page.ts @@ -28,7 +28,7 @@ import putFormComponent from "./put-form-component"; import indexTableComponent from "./index-table-component"; import * as overlay from "./overlay"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; const PAGE_SIZE = config.ui.pageSize || 10; diff --git a/ui/filter-component.ts b/ui/filter-component.ts index f3b2385..cd28c47 100644 --- a/ui/filter-component.ts +++ b/ui/filter-component.ts @@ -18,13 +18,12 @@ */ import m, { ClosureComponent } from "mithril"; -import { parse, stringify, map } from "../lib/common/expression-parser"; +import { parse, stringify, map } from "../lib/common/expression/parser"; import memoize from "../lib/common/memoize"; import Autocomplete from "./autocomplete-compnent"; import * as smartQuery from "./smart-query"; -import { filterToMongoQuery } from "../lib/mongodb-functions"; +import { validQuery } from "../lib/mongodb-functions"; import { Expression } from "../lib/types"; -import { normalize } from "../lib/common/boolean-expression"; const getAutocomplete = memoize((resource) => { const labels = smartQuery.getLabels(resource); @@ -42,16 +41,6 @@ const getAutocomplete = memoize((resource) => { return autocomplete; }); -function validateQuery(q: Expression): void { - q = normalize(q); - if (Array.isArray(q) && q[0] === "CASE") { - q = q.slice(1).filter((a) => Array.isArray(a)); - if (q.length > 1) q = ["AND", ...q]; - else q = q[0]; - } - if (Array.isArray(q)) filterToMongoQuery(q); -} - function parseFilter(resource, f): Expression { let exp; if (/^[\s0-9a-zA-Z]+:/.test(f)) { @@ -62,16 +51,16 @@ function parseFilter(resource, f): Expression { exp = parse(f); } + const unpacked = map(exp, (e) => { + if (Array.isArray(e) && e[0] === "FUNC") { + if (e[1] === "Q") return smartQuery.unpack(resource, e[2], e[3]); + else if (e[1] === "NOW") return Date.now(); + } + return e; + }); + // Throws exception if invalid Mongo query - validateQuery( - map(exp, (e) => { - if (Array.isArray(e) && e[0] === "FUNC") { - if (e[1] === "Q") return smartQuery.unpack(resource, e[2], e[3]); - else if (e[1] === "NOW") return Date.now(); - } - return e; - }) - ); + validQuery(unpacked, resource); return exp; } diff --git a/ui/permissions-page.ts b/ui/permissions-page.ts index bb8333b..77062b6 100644 --- a/ui/permissions-page.ts +++ b/ui/permissions-page.ts @@ -26,7 +26,7 @@ import putFormComponent from "./put-form-component"; import indexTableComponent from "./index-table-component"; import * as overlay from "./overlay"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; import filterComponent from "./filter-component"; import { Children, ClosureComponent, Component } from "mithril"; import { getIcon } from "./icons"; diff --git a/ui/pie-chart-component.ts b/ui/pie-chart-component.ts index 55dbe95..abaf3a5 100644 --- a/ui/pie-chart-component.ts +++ b/ui/pie-chart-component.ts @@ -19,7 +19,7 @@ import { ClosureComponent, Component, Children } from "mithril"; import { m } from "./components"; -import { stringify } from "../lib/common/expression-parser"; +import { stringify } from "../lib/common/expression/parser"; import memoize from "../lib/common/memoize"; const memoizedStringify = memoize(stringify); diff --git a/ui/presets-page.ts b/ui/presets-page.ts index 528df89..ef1754f 100644 --- a/ui/presets-page.ts +++ b/ui/presets-page.ts @@ -28,7 +28,7 @@ import putFormComponent from "./put-form-component"; import indexTableComponent from "./index-table-component"; import memoize from "../lib/common/memoize"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; const PAGE_SIZE = config.ui.pageSize || 10; diff --git a/ui/provisions-page.ts b/ui/provisions-page.ts index a797831..3cdec52 100644 --- a/ui/provisions-page.ts +++ b/ui/provisions-page.ts @@ -28,7 +28,7 @@ import putFormComponent from "./put-form-component"; import indexTableComponent from "./index-table-component"; import * as overlay from "./overlay"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; import { loadCodeMirror } from "./dynamic-loader"; const PAGE_SIZE = config.ui.pageSize || 10; diff --git a/ui/store.ts b/ui/store.ts index 1107bec..8d51040 100644 --- a/ui/store.ts +++ b/ui/store.ts @@ -18,15 +18,15 @@ */ import m from "mithril"; -import { stringify } from "../lib/common/expression-parser"; -import { or, and, evaluate } from "../lib/common/expression"; +import { stringify } from "../lib/common/expression/parser"; +import { or, and, evaluate } from "../lib/common/expression/util"; import memoize from "../lib/common/memoize"; import { QueryOptions, Expression } from "../lib/types"; import * as notifications from "./notifications"; import { configSnapshot, genieacsVersion } from "./config"; import { QueueTask } from "./task-queue"; import { PingResult } from "../lib/ping"; -import { unionDiff, covers } from "../lib/common/boolean-expression"; +import { unionDiff, covers } from "../lib/common/expression/synth"; const memoizedStringify = memoize(stringify); const memoizedEvaluate = memoize(evaluate); diff --git a/ui/ui-config-component.ts b/ui/ui-config-component.ts index 0ed6ca6..b0b4779 100644 --- a/ui/ui-config-component.ts +++ b/ui/ui-config-component.ts @@ -23,7 +23,7 @@ import * as store from "./store"; import { yaml } from "./dynamic-loader"; import * as configFunctions from "./config-functions"; import codeEditorComponent from "./code-editor-component"; -import { parse } from "../lib/common/expression-parser"; +import { parse } from "../lib/common/expression/parser"; function putActionHandler(prefix: string[], dataYaml: string): Promise { return new Promise((resolve, reject) => { diff --git a/ui/users-page.ts b/ui/users-page.ts index ee8abaf..77b2fbd 100644 --- a/ui/users-page.ts +++ b/ui/users-page.ts @@ -26,7 +26,7 @@ import putFormComponent from "./put-form-component"; import indexTableComponent from "./index-table-component"; import * as overlay from "./overlay"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; import filterComponent from "./filter-component"; import changePasswordComponent from "./change-password-component"; import { Children, ClosureComponent, Component } from "mithril"; diff --git a/ui/virtual-parameters-page.ts b/ui/virtual-parameters-page.ts index f728ed4..1642664 100644 --- a/ui/virtual-parameters-page.ts +++ b/ui/virtual-parameters-page.ts @@ -28,7 +28,7 @@ import putFormComponent from "./put-form-component"; import indexTableComponent from "./index-table-component"; import * as overlay from "./overlay"; import * as smartQuery from "./smart-query"; -import { map, parse, stringify } from "../lib/common/expression-parser"; +import { map, parse, stringify } from "../lib/common/expression/parser"; import { loadCodeMirror } from "./dynamic-loader"; const PAGE_SIZE = config.ui.pageSize || 10;