Improve pagination/sorting

ui/store.ts

@@ -7,7 +7,12 @@ import * as notifications from "./notifications.ts";
 import { configSnapshot, genieacsVersion } from "./config.ts";
 import { QueueTask } from "./task-queue.ts";
 import { PingResult } from "../lib/ping.ts";
-import { unionDiff, covers } from "../lib/common/expression/synth.ts";
+import { unionDiff } from "../lib/common/expression/synth.ts";
+import {
+  bookmarkToExpression,
+  paginate,
+  toBookmark,
+} from "../lib/common/expression/pagination.ts";
 
 const memoizedStringify = memoize(stringify);
 const memoizedEvaluate = memoize(evaluate);
@@ -26,6 +31,7 @@ const queries = {
   fulfilling: new WeakSet(),
   accessed: new WeakMap(),
   value: new WeakMap(),
+  unsatisfied: new WeakMap(),
 };
 
 interface Resources {
@@ -224,46 +230,11 @@ export function count(resourceType: string, filter: Expression): QueryResponse {
   return queryResponse;
 }
 
-function limitFilter(
-  filter: Expression,
-  sort: Record<string, number>,
-  bookmark,
-): Expression {
-  const sortSort = (a, b): number => Math.abs(b[1]) - Math.abs(a[1]);
-  const arr = Object.entries(sort).sort(sortSort).reverse();
-  return and(
-    filter,
-    arr.reduce((cur, kv) => {
-      const [param, asc] = kv;
-      if (asc <= 0) {
-        if (bookmark[param] == null) {
-          return or(
-            ["IS NOT NULL", ["PARAM", param]],
-            and(["IS NULL", ["PARAM", param]], cur),
-          );
-        }
-        let f = null;
-        f = or(f, [">", ["PARAM", param], bookmark[param]]);
-        return or(f, and(["=", ["PARAM", param], bookmark[param]], cur));
-      } else {
-        let f: Expression = ["IS NULL", ["PARAM", param]];
-        if (bookmark[param] == null) return and(f, cur);
-        f = or(f, ["<", ["PARAM", param], bookmark[param]]);
-        return or(f, and(["=", ["PARAM", param], bookmark[param]], cur));
-      }
-    }, true as Expression),
-  );
-}
-
 function compareFunction(sort: {
   [param: string]: number;
 }): (a: any, b: any) => number {
-  const sortEntries = Object.entries(sort).sort(
-    (a, b) => Math.abs(b[1]) - Math.abs(a[1]),
-  );
-
   return (a, b) => {
-    for (const [param, asc] of sortEntries) {
+    for (const [param, asc] of Object.entries(sort)) {
       let v1 = a[param];
       let v2 = b[param];
       if (v1 != null && typeof v1 === "object") {
@@ -306,24 +277,6 @@ function findMatches(resourceType, filter, sort, limit): any[] {
   return value;
 }
 
-function inferQuery(resourceType, queryResponse): void {
-  const limit = queries.limit.get(queryResponse);
-  let filter = queries.filter.get(queryResponse);
-  filter = unpackExpression(filter);
-  const bookmark = queries.bookmark.get(queryResponse);
-  const sort = queries.sort.get(queryResponse);
-  if (bookmark || !limit) {
-    if (bookmark) filter = limitFilter(filter, sort, bookmark);
-    if (covers(resources[resourceType].combinedFilter, filter))
-      queries.fulfilled.set(queryResponse, fulfillTimestamp);
-  }
-
-  queries.value.set(
-    queryResponse,
-    findMatches(resourceType, filter, sort, limit),
-  );
-}
-
 export function fetch(
   resourceType: string,
   filter: Expression,
@@ -331,7 +284,6 @@ export function fetch(
 ): QueryResponse {
   const filterStr = memoizedStringify(filter);
   const sort = Object.assign({}, options.sort);
-  for (const [k, v] of Object.entries(sort)) sort[k] += Math.sign(v);
 
   const limit = options.limit || 0;
   if (resourceType === "devices")
@@ -347,7 +299,16 @@ export function fetch(
   queries.filter.set(queryResponse, filter);
   queries.limit.set(queryResponse, limit);
   queries.sort.set(queryResponse, sort);
-  inferQuery(resourceType, queryResponse);
+  const [satisfied, diff] = paginate(
+    resources[resourceType].combinedFilter,
+    unpackExpression(filter),
+    sort,
+  );
+  const matches = findMatches(resourceType, satisfied, sort, limit);
+  queries.value.set(queryResponse, matches);
+  if (!diff || matches.length >= limit)
+    queries.fulfilled.set(queryResponse, fulfillTimestamp);
+  else queries.unsatisfied.set(queryResponse, diff);
   return queryResponse;
 }
 
@@ -412,11 +373,18 @@ export function fulfill(accessTimestamp: number): void {
         queries.fulfilling.add(queryResponse);
         toFetchAll[resourceType] = toFetchAll[resourceType] || [];
         toFetchAll[resourceType].push(queryResponse);
-        const limit = queries.limit.get(queryResponse);
+        let limit = queries.limit.get(queryResponse);
         const sort = queries.sort.get(queryResponse);
         if (limit) {
           let filter = queries.filter.get(queryResponse);
           filter = unpackExpression(filter);
+
+          const unsatisfied = queries.unsatisfied.get(queryResponse);
+          if (unsatisfied) {
+            limit -= queries.value.get(queryResponse).length;
+            filter = unsatisfied;
+          }
+
           allPromises.push(
             xhrRequest({
               method: "GET",
@@ -431,20 +399,9 @@ export function fulfill(accessTimestamp: number): void {
                 }),
               background: true,
             }).then((res) => {
+              queries.unsatisfied.delete(queryResponse);
               if ((res as any[]).length) {
-                // Generate bookmark object
-                const bm = Object.keys(sort).reduce((b, k) => {
-                  if (res[0][k] != null) {
-                    if (typeof res[0][k] === "object") {
-                      if (res[0][k].value != null) b[k] = res[0][k].value[0];
-                    } else {
-                      b[k] = res[0][k];
-                    }
-                  }
-
-                  return b;
-                }, {});
-                queries.bookmark.set(queryResponse, bm);
+                queries.bookmark.set(queryResponse, toBookmark(sort, res[0]));
               } else {
                 queries.bookmark.delete(queryResponse);
               }
@@ -467,7 +424,8 @@ export function fulfill(accessTimestamp: number): void {
           filter = memoizedEvaluate(filter, null, fulfillTimestamp + clockSkew);
           const bookmark = queries.bookmark.get(queryResponse);
           const sort = queries.sort.get(queryResponse);
-          if (bookmark) filter = limitFilter(filter, sort, bookmark);
+          if (bookmark)
+            filter = and(filter, bookmarkToExpression(bookmark, sort));
           combinedFilter = or(combinedFilter, filter);
         }
 
@@ -487,7 +445,8 @@ export function fulfill(accessTimestamp: number): void {
             const limit = queries.limit.get(queryResponse);
             const bookmark = queries.bookmark.get(queryResponse);
             const sort = queries.sort.get(queryResponse);
-            if (bookmark) filter = limitFilter(filter, sort, bookmark);
+            if (bookmark)
+              filter = and(filter, bookmarkToExpression(bookmark, sort));
 
             queries.value.set(
               queryResponse,
@@ -540,7 +499,8 @@ export function fulfill(accessTimestamp: number): void {
               const limit = queries.limit.get(queryResponse);
               const bookmark = queries.bookmark.get(queryResponse);
               const sort = queries.sort.get(queryResponse);
-              if (bookmark) filter = limitFilter(filter, sort, bookmark);
+              if (bookmark)
+                filter = and(filter, bookmarkToExpression(bookmark, sort));
 
               queries.value.set(
                 queryResponse,