From f3068c5355f233162c3d8ba9c1bda510eb040254 Mon Sep 17 00:00:00 2001 From: Zaid Humaimidi Date: Sat, 24 Feb 2024 04:26:40 -0800 Subject: [PATCH] Improve pagination/sorting --- lib/common/expression/pagination.ts | 148 ++++++++++++++++++++++++++++ lib/common/expression/synth.ts | 86 +++++++++++++--- npm-shrinkwrap.json | 14 +-- package.json | 4 +- test/pagination.ts | 120 ++++++++++++++++++++++ ui/store.ts | 108 +++++++------------- 6 files changed, 381 insertions(+), 99 deletions(-) create mode 100644 lib/common/expression/pagination.ts create mode 100644 test/pagination.ts diff --git a/lib/common/expression/pagination.ts b/lib/common/expression/pagination.ts new file mode 100644 index 0000000..462fac8 --- /dev/null +++ b/lib/common/expression/pagination.ts @@ -0,0 +1,148 @@ +import { complement } from "espresso-iisojs"; +import { Expression } from "../../types"; +import normalize from "./normalize"; +import { Clause, SynthContext } from "./synth"; +import { and, or } from "./util"; + +type Bookmark = Record; + +type Minterm = number[]; + +export function toBookmark( + sort: Record, + row: unknown, +): Bookmark { + const bookmark: Bookmark = {}; + for (const param of Object.keys(sort)) { + let v = row[param]; + if (v != null && typeof v === "object") v = v.value?.[0]; + bookmark[param] = v; + } + return bookmark; +} + +export function bookmarkToExpression( + bookmark: Bookmark, + sort: Record, +): Expression { + return Object.entries(sort) + .reverse() + .reduce((cur, kv) => { + const [param, asc] = kv; + if (asc < 0) { + if (bookmark[param] == null) { + return or( + ["IS NOT NULL", ["PARAM", param]], + and(["IS NULL", ["PARAM", param]], cur), + ); + } + let f = null; + f = or(f, [">", ["PARAM", param], bookmark[param]]); + return or(f, and(["=", ["PARAM", param], bookmark[param]], cur)); + } else { + let f: Expression = ["IS NULL", ["PARAM", param]]; + if (bookmark[param] == null) return and(f, cur); + f = or(f, ["<", ["PARAM", param], bookmark[param]]); + return or(f, and(["=", ["PARAM", param], bookmark[param]], cur)); + } + }, true as Expression); +} + +function getCover( + context: SynthContext, + minterm: Minterm, + allSort: [string, number][], +): Minterm[] { + if (!allSort.length) return [[]]; + const [param, sort] = allSort[0]; + + const cov: Minterm = []; + const nextCov: Minterm = []; + + if (sort > 0) { + const lhs = new Clause.Exp(["PARAM", param]); + const isNull = context.getVar(new Clause.IsNull(lhs)); + cov.push((isNull << 2) ^ 2); + } + + for (const m of minterm) { + const clause = context.getClause(m >>> 2); + if (clause instanceof Clause.IsNull) { + const lhs = clause.operand.expression(); + if (!Array.isArray(lhs) || lhs[0] !== "PARAM") continue; + if (lhs[1] !== param) continue; + nextCov.push(m); + if (sort < 0 && m & 1) cov.push(m, m ^ 1); + } else if (clause instanceof Clause.Compare) { + const lhs = clause.lhs.expression(); + if (!Array.isArray(lhs) || lhs[0] !== "PARAM") continue; + if (lhs[1] !== param) continue; + if (!(m & 1) && sort > 0) continue; + nextCov.push(m); + + const negate = (m ^ (m >> 1)) & 1; + + if (sort > 0) { + if ( + (clause.op === "=" && !negate) || + (clause.op === ">" && !negate) || + (clause.op === "<" && negate) + ) { + const c = new Clause.Compare(clause.lhs, ">", clause.rhs); + const v = context.getVar(c); + cov.push((v << 2) ^ 3); + } + } else if (sort < 0) { + if ( + (clause.op === "=" && !negate) || + (clause.op === ">" && negate) || + (clause.op === "<" && !negate) + ) { + const c = new Clause.Compare(clause.lhs, "<", clause.rhs); + const v = context.getVar(c); + cov.push((v << 2) ^ 0); + } + } + } + } + + const next = getCover(context, minterm, allSort.slice(1)); + + return [cov, ...next.map((n) => [...nextCov, ...n])]; +} + +export function paginate( + fetched: Expression, + toFetch: Expression, + sort: Record, +): [Expression, Expression] { + fetched = normalize(fetched); + if (!fetched) return [false, toFetch]; + + toFetch = normalize(toFetch); + if (!toFetch) return [false, toFetch]; + + const synth1 = Clause.fromExpression(fetched); + const synth2 = Clause.fromExpression(toFetch); + + const context = new SynthContext(); + + const expr1Minterms = synth1.true(context); + const expr2MintermsC = complement(synth2.true(context)); + + const gaps = context.sanitizeMinterms( + complement([ + ...expr1Minterms, + ...expr2MintermsC, + ...context.getDcSet([...expr1Minterms, ...expr2MintermsC]), + ]), + ); + + const cover = gaps.flatMap((m) => getCover(context, m, Object.entries(sort))); + const minterms1 = context.minimize(complement([...cover, ...expr2MintermsC])); + const minterms2 = context.minimize( + complement([...minterms1, ...expr2MintermsC]), + ); + + return [context.toExpression(minterms1), context.toExpression(minterms2)]; +} diff --git a/lib/common/expression/synth.ts b/lib/common/expression/synth.ts index 66dc24c..bd94b09 100644 --- a/lib/common/expression/synth.ts +++ b/lib/common/expression/synth.ts @@ -12,7 +12,7 @@ export abstract class SynthContextBase< public variables = new Map(); protected clauses = new Map(); - protected getVar(c: U): number { + public getVar(c: U): number { const str = c.toString(); let idx = this.variables.get(str); if (idx == null) { @@ -23,7 +23,7 @@ export abstract class SynthContextBase< return idx; } - protected getClause(v: number): U { + public getClause(v: number): U { return this.clauses.get(v); } @@ -34,6 +34,16 @@ export abstract class SynthContextBase< return true; } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + canLower(idx: number, set: Set): boolean { + return true; + } + + bias(a: number, b: number): number { + // Bias towards 1 then lower index + return (b ^ 1) - (a ^ 1); + } + sanitizeMinterms(minterms: Minterm[]): Minterm[] { return minterms; } @@ -41,9 +51,18 @@ export abstract class SynthContextBase< minimize(minterms: Minterm[], dcSet: Minterm[] = []): Minterm[] { minterms = this.sanitizeMinterms(minterms); const canRaise = this.canRaise.bind(this); - return espresso(minterms, [...this.getDcSet(minterms), ...dcSet], { - canRaise, - }); + const canLower = this.canLower.bind(this); + const bias = this.bias.bind(this); + + return espresso( + minterms, + [...this.getDcSet([...minterms, ...dcSet]), ...dcSet], + { + canRaise, + canLower, + bias, + }, + ); } } @@ -726,6 +745,18 @@ export class SynthContext extends SynthContextBase { return !(idx & 1) || !set.has(idx ^ 3); } + canLower(idx: number, set: Set): boolean { + if (idx & 1) return true; + const clause = this.getClause(idx >> 2); + if (clause instanceof Clause.IsNull) return true; + return set.has(idx ^ 3) || set.has(idx ^ 1); + } + + bias(a: number, b: number): number { + // Bias towards 1 then true + return ((b & 3) ^ 3) - ((a & 3) ^ 3); + } + sanitizeMinterms(minterms: Minterm[]): Minterm[] { const res = [] as number[][]; @@ -736,8 +767,18 @@ export class SynthContext extends SynthContextBase { const minterm: number[] = []; const perms: number[][] = []; for (const [k, v] of merged) { - if (v === 0b1010) continue loop; - const isNullVars = [...this.clauses.get(k).getNullables()].map((c) => + if ((v & 0b0011) === 0b0011) continue loop; + if ((v & 0b1100) === 0b1100) continue loop; + const clause = this.clauses.get(k); + if (!clause) throw new Error("Invalid literal"); + if (clause instanceof Clause.IsNull) { + if (v === 0b0100) minterm.push((k << 2) ^ 2); + else if (v === 0b1000) minterm.push((k << 2) ^ 3); + else throw new Error("Invalid literal"); + continue; + } + if ((v & 0b1010) === 0b1010) continue loop; + const isNullVars = [...clause.getNullables()].map((c) => this.getVar(c), ); const t = k << 2; @@ -773,7 +814,25 @@ export class SynthContext extends SynthContextBase { if (!s.length) return true; const conjs: Expression[] = []; for (const i of s) { - let expr = this.getClause(i >>> 2).expression() as Expression; + const clause = this.getClause(i >>> 2); + if (!clause) throw new Error("Invalid literal"); + if (clause instanceof Clause.IsNull) { + if (!(i & 2)) throw new Error("Invalid literal"); + } else if (!(i & 1)) { + // Should never be reached if minimized correctly + const isNullVars = [...clause.getNullables()].map((c) => + this.getVar(c), + ); + conjs.push( + this.toExpression([ + [i ^ 3], + ...isNullVars.map((n) => [(n << 2) ^ 3]), + ]), + ); + continue; + } + + let expr = clause.expression() as Expression; if (!(i & 1) !== !(i & 2)) expr = ["NOT", expr]; if ( Array.isArray(expr) && @@ -882,16 +941,11 @@ export function unionDiff( const expr2Minterms = synth2.true(context); const expr1Minterms = synth1.true(context); - const expr1NullMinterms = synth1.null(context); - const expr1FalseMinterms = synth1.false(context); const union = context.minimize([...expr1Minterms, ...expr2Minterms]); const diff = context.minimize( - complement([ - ...complement([...expr1NullMinterms, ...expr1FalseMinterms]), - ...complement(expr2Minterms), - ]), + complement([...expr1Minterms, ...complement(expr2Minterms)]), ); return [context.toExpression(union), context.toExpression(diff)]; @@ -911,8 +965,8 @@ export function covers(expr1: Expression, expr2: Expression): boolean { const expr2Minterms = synt2.true(context); return tautology([ - ...complement(expr2Minterms), + ...context.sanitizeMinterms(complement(expr2Minterms)), ...context.getDcSet([...expr2Minterms, ...expr1Minterms]), - ...expr1Minterms, + ...context.sanitizeMinterms(expr1Minterms), ]); } diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 83c844a..003c93c 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -11,7 +11,7 @@ "dependencies": { "@breejs/later": "^4.1.0", "bson": "^4.7.2", - "espresso-iisojs": "^1.0.7", + "espresso-iisojs": "^1.0.8", "iconv-lite": "^0.6.3", "ipaddr.js": "^2.0.1", "jsonwebtoken": "^8.5.1", @@ -3055,9 +3055,9 @@ } }, "node_modules/espresso-iisojs": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/espresso-iisojs/-/espresso-iisojs-1.0.7.tgz", - "integrity": "sha512-yHEY+CC9wqEyA1N+KC03xmyN+lFPZdpXaLUxf6vrofE0cMXXIkOg08tXXPmq5fnOoLgxRfm5ikrmYD8/gIhRig==" + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/espresso-iisojs/-/espresso-iisojs-1.0.8.tgz", + "integrity": "sha512-S3D62BA/jBUCIQJ3VlBGMSxGwPyyV7ttduiJvyaD4dWyhiC/9TnJTaiurx4r8bqcYZ1J0KELliub4xo8neKjSA==" }, "node_modules/esquery": { "version": "1.5.0", @@ -7297,9 +7297,9 @@ } }, "espresso-iisojs": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/espresso-iisojs/-/espresso-iisojs-1.0.7.tgz", - "integrity": "sha512-yHEY+CC9wqEyA1N+KC03xmyN+lFPZdpXaLUxf6vrofE0cMXXIkOg08tXXPmq5fnOoLgxRfm5ikrmYD8/gIhRig==" + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/espresso-iisojs/-/espresso-iisojs-1.0.8.tgz", + "integrity": "sha512-S3D62BA/jBUCIQJ3VlBGMSxGwPyyV7ttduiJvyaD4dWyhiC/9TnJTaiurx4r8bqcYZ1J0KELliub4xo8neKjSA==" }, "esquery": { "version": "1.5.0", diff --git a/package.json b/package.json index 24ac031..eca86c2 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "dependencies": { "@breejs/later": "^4.1.0", "bson": "^4.7.2", - "espresso-iisojs": "^1.0.7", + "espresso-iisojs": "^1.0.8", "iconv-lite": "^0.6.3", "ipaddr.js": "^2.0.1", "jsonwebtoken": "^8.5.1", @@ -63,7 +63,7 @@ "node": ">=12.13.0" }, "scripts": { - "test": "esbuild --log-level=warning --bundle --platform=node --target=node18 --packages=external --outdir=test test/*.ts && node --test test/*.js && rm test/*.js", + "test": "esbuild --log-level=warning --bundle --platform=node --target=node18 --packages=external --sourcemap=inline --outdir=test test/*.ts && node --test --enable-source-maps test/*.js && rm test/*.js", "lint": "esbuild build/lint.ts --bundle --platform=node --target=node16 --packages=external | node", "build": "esbuild build/build.ts --bundle --platform=node --target=node12 --packages=external | node" } diff --git a/test/pagination.ts b/test/pagination.ts new file mode 100644 index 0000000..6a55939 --- /dev/null +++ b/test/pagination.ts @@ -0,0 +1,120 @@ +import test from "node:test"; +import assert from "node:assert"; +import initSqlJs from "sql.js/dist/sql-asm.js"; +import { parse, stringify } from "../lib/common/expression/parser.ts"; +import { + bookmarkToExpression, + paginate, + toBookmark, +} from "../lib/common/expression/pagination.ts"; +import { Expression } from "../lib/types.ts"; +import { covers, minimize } from "../lib/common/expression/synth.ts"; + +const VALUES = [null, -1, false, "a"]; +const PARAMS = ["param1", "param2"]; + +let db; + +async function query(filter: string): Promise<{ id: string }[]> { + if (!db) { + const sql = await initSqlJs(); + db = new sql.Database(); + + db.run(`CREATE TABLE test (id INTEGER PRIMARY KEY, ${PARAMS.join(", ")})`); + + const stmt = db.prepare( + `INSERT INTO test (${PARAMS.join(", ")}) VALUES (${PARAMS.map( + () => "?", + ).join(", ")})`, + ); + const count = VALUES.length ** PARAMS.length; + for (let i = 0; i < count; ++i) { + const values: (boolean | number | string)[] = []; + for (let j = 0; j < PARAMS.length; ++j) + values.push(VALUES[Math.trunc(i / VALUES.length ** j) % VALUES.length]); + stmt.run(values); + } + stmt.free(); + } + + const res = db.exec(`SELECT * FROM test WHERE ${filter}`); + if (!res.length) return []; + return res[0].values.map((row) => + Object.fromEntries(row.map((v, i) => [res[0].columns[i], v])), + ); +} + +function getAllSortOrders(columns: string[]): Array> { + const sortOrders: Array> = []; + + function generateOrders( + remaining: string[], + current: Record, + ): void { + if (remaining.length === 0) { + sortOrders.push({ ...current }); + return; + } + + for (const column of remaining) { + const newRemaining = remaining.filter((c) => c !== column); + current[column] = -1; + generateOrders(newRemaining, current); + current[column] = 1; + generateOrders(newRemaining, current); + delete current[column]; + } + } + + generateOrders(columns, {}); + return sortOrders; +} + +async function testPaginate( + q1: Expression, + q2: Expression, + sort: Record, +): Promise { + const orderBy = Object.entries(sort) + .map(([k, v]) => `${k} ${v > 0 ? "ASC" : "DESC"}`) + .join(", "); + + const allMatches = await query(`${stringify(q2)} ORDER BY ${orderBy}`); + const [fulfilled, diff] = paginate(q1, q2, sort); + assert.ok(covers(q1, fulfilled)); + + const fulfilledMatches = await query( + `${stringify(fulfilled)} ORDER BY ${orderBy}`, + ); + + const diffMatches = await query(`${stringify(diff)} ORDER BY ${orderBy}`); + assert.deepStrictEqual(allMatches, [...fulfilledMatches, ...diffMatches]); + + if (allMatches.length === fulfilledMatches.length) return; + + const nextMatches = allMatches.slice( + fulfilledMatches.length, + fulfilledMatches.length + 1, + ); + const bookmark = toBookmark(sort, nextMatches[nextMatches.length - 1]); + const bookmarkFilter = bookmarkToExpression(bookmark, sort); + const capped = ["AND", q2, bookmarkFilter]; + + assert.ok(!covers(q1, capped)); + const cappedMatches = await query( + `(${stringify(capped)}) ORDER BY ${orderBy}`, + ); + assert.deepStrictEqual(cappedMatches, [...fulfilledMatches, ...nextMatches]); + const min = minimize(["OR", q1, capped]); + await testPaginate(min, q2, sort); +} + +void test("paginate", async () => { + const cases: [string, string][] = [["param2 < 'a'", "param1 >= 'a'"]]; + const params = ["id", ...PARAMS]; + const sortOrders = getAllSortOrders(params); + + for (const [q1, q2] of cases) + for (const sort of sortOrders) + await testPaginate(parse(q1), parse(q2), sort); +}); diff --git a/ui/store.ts b/ui/store.ts index 10d89ec..1fcc1e0 100644 --- a/ui/store.ts +++ b/ui/store.ts @@ -7,7 +7,12 @@ import * as notifications from "./notifications.ts"; import { configSnapshot, genieacsVersion } from "./config.ts"; import { QueueTask } from "./task-queue.ts"; import { PingResult } from "../lib/ping.ts"; -import { unionDiff, covers } from "../lib/common/expression/synth.ts"; +import { unionDiff } from "../lib/common/expression/synth.ts"; +import { + bookmarkToExpression, + paginate, + toBookmark, +} from "../lib/common/expression/pagination.ts"; const memoizedStringify = memoize(stringify); const memoizedEvaluate = memoize(evaluate); @@ -26,6 +31,7 @@ const queries = { fulfilling: new WeakSet(), accessed: new WeakMap(), value: new WeakMap(), + unsatisfied: new WeakMap(), }; interface Resources { @@ -224,46 +230,11 @@ export function count(resourceType: string, filter: Expression): QueryResponse { return queryResponse; } -function limitFilter( - filter: Expression, - sort: Record, - bookmark, -): Expression { - const sortSort = (a, b): number => Math.abs(b[1]) - Math.abs(a[1]); - const arr = Object.entries(sort).sort(sortSort).reverse(); - return and( - filter, - arr.reduce((cur, kv) => { - const [param, asc] = kv; - if (asc <= 0) { - if (bookmark[param] == null) { - return or( - ["IS NOT NULL", ["PARAM", param]], - and(["IS NULL", ["PARAM", param]], cur), - ); - } - let f = null; - f = or(f, [">", ["PARAM", param], bookmark[param]]); - return or(f, and(["=", ["PARAM", param], bookmark[param]], cur)); - } else { - let f: Expression = ["IS NULL", ["PARAM", param]]; - if (bookmark[param] == null) return and(f, cur); - f = or(f, ["<", ["PARAM", param], bookmark[param]]); - return or(f, and(["=", ["PARAM", param], bookmark[param]], cur)); - } - }, true as Expression), - ); -} - function compareFunction(sort: { [param: string]: number; }): (a: any, b: any) => number { - const sortEntries = Object.entries(sort).sort( - (a, b) => Math.abs(b[1]) - Math.abs(a[1]), - ); - return (a, b) => { - for (const [param, asc] of sortEntries) { + for (const [param, asc] of Object.entries(sort)) { let v1 = a[param]; let v2 = b[param]; if (v1 != null && typeof v1 === "object") { @@ -306,24 +277,6 @@ function findMatches(resourceType, filter, sort, limit): any[] { return value; } -function inferQuery(resourceType, queryResponse): void { - const limit = queries.limit.get(queryResponse); - let filter = queries.filter.get(queryResponse); - filter = unpackExpression(filter); - const bookmark = queries.bookmark.get(queryResponse); - const sort = queries.sort.get(queryResponse); - if (bookmark || !limit) { - if (bookmark) filter = limitFilter(filter, sort, bookmark); - if (covers(resources[resourceType].combinedFilter, filter)) - queries.fulfilled.set(queryResponse, fulfillTimestamp); - } - - queries.value.set( - queryResponse, - findMatches(resourceType, filter, sort, limit), - ); -} - export function fetch( resourceType: string, filter: Expression, @@ -331,7 +284,6 @@ export function fetch( ): QueryResponse { const filterStr = memoizedStringify(filter); const sort = Object.assign({}, options.sort); - for (const [k, v] of Object.entries(sort)) sort[k] += Math.sign(v); const limit = options.limit || 0; if (resourceType === "devices") @@ -347,7 +299,16 @@ export function fetch( queries.filter.set(queryResponse, filter); queries.limit.set(queryResponse, limit); queries.sort.set(queryResponse, sort); - inferQuery(resourceType, queryResponse); + const [satisfied, diff] = paginate( + resources[resourceType].combinedFilter, + unpackExpression(filter), + sort, + ); + const matches = findMatches(resourceType, satisfied, sort, limit); + queries.value.set(queryResponse, matches); + if (!diff || matches.length >= limit) + queries.fulfilled.set(queryResponse, fulfillTimestamp); + else queries.unsatisfied.set(queryResponse, diff); return queryResponse; } @@ -412,11 +373,18 @@ export function fulfill(accessTimestamp: number): void { queries.fulfilling.add(queryResponse); toFetchAll[resourceType] = toFetchAll[resourceType] || []; toFetchAll[resourceType].push(queryResponse); - const limit = queries.limit.get(queryResponse); + let limit = queries.limit.get(queryResponse); const sort = queries.sort.get(queryResponse); if (limit) { let filter = queries.filter.get(queryResponse); filter = unpackExpression(filter); + + const unsatisfied = queries.unsatisfied.get(queryResponse); + if (unsatisfied) { + limit -= queries.value.get(queryResponse).length; + filter = unsatisfied; + } + allPromises.push( xhrRequest({ method: "GET", @@ -431,20 +399,9 @@ export function fulfill(accessTimestamp: number): void { }), background: true, }).then((res) => { + queries.unsatisfied.delete(queryResponse); if ((res as any[]).length) { - // Generate bookmark object - const bm = Object.keys(sort).reduce((b, k) => { - if (res[0][k] != null) { - if (typeof res[0][k] === "object") { - if (res[0][k].value != null) b[k] = res[0][k].value[0]; - } else { - b[k] = res[0][k]; - } - } - - return b; - }, {}); - queries.bookmark.set(queryResponse, bm); + queries.bookmark.set(queryResponse, toBookmark(sort, res[0])); } else { queries.bookmark.delete(queryResponse); } @@ -467,7 +424,8 @@ export function fulfill(accessTimestamp: number): void { filter = memoizedEvaluate(filter, null, fulfillTimestamp + clockSkew); const bookmark = queries.bookmark.get(queryResponse); const sort = queries.sort.get(queryResponse); - if (bookmark) filter = limitFilter(filter, sort, bookmark); + if (bookmark) + filter = and(filter, bookmarkToExpression(bookmark, sort)); combinedFilter = or(combinedFilter, filter); } @@ -487,7 +445,8 @@ export function fulfill(accessTimestamp: number): void { const limit = queries.limit.get(queryResponse); const bookmark = queries.bookmark.get(queryResponse); const sort = queries.sort.get(queryResponse); - if (bookmark) filter = limitFilter(filter, sort, bookmark); + if (bookmark) + filter = and(filter, bookmarkToExpression(bookmark, sort)); queries.value.set( queryResponse, @@ -540,7 +499,8 @@ export function fulfill(accessTimestamp: number): void { const limit = queries.limit.get(queryResponse); const bookmark = queries.bookmark.get(queryResponse); const sort = queries.sort.get(queryResponse); - if (bookmark) filter = limitFilter(filter, sort, bookmark); + if (bookmark) + filter = and(filter, bookmarkToExpression(bookmark, sort)); queries.value.set( queryResponse,