diff --git a/client/devices-page.js b/client/devices-page.js index a508715..6ca53f4 100644 --- a/client/devices-page.js +++ b/client/devices-page.js @@ -288,7 +288,7 @@ const component = { let devs = store.fetch( "devices", vnode.attrs.filter == null ? true : vnode.attrs.filter, - vnode.state.showCount || 10 + { limit: vnode.state.showCount || 10 } ); let count = store.count( "devices", diff --git a/client/faults-page.js b/client/faults-page.js index 6653d8e..a80294f 100644 --- a/client/faults-page.js +++ b/client/faults-page.js @@ -160,7 +160,7 @@ const component = { let faults = store.fetch( "faults", vnode.attrs.filter == null ? true : vnode.attrs.filter, - vnode.state.showCount || 10 + { limit: vnode.state.showCount || 10 } ); let count = store.count( "faults", diff --git a/client/store.js b/client/store.js index dc2025b..6663f1e 100644 --- a/client/store.js +++ b/client/store.js @@ -13,11 +13,11 @@ let evaluateExpressionCache = new WeakMap(); const queries = { filter: new WeakMap(), - last: new WeakMap(), + bookmark: new WeakMap(), limit: new WeakMap(), + sort: new WeakMap(), fulfilled: new WeakMap(), fulfilling: new WeakSet(), - mapper: new WeakMap(), accessed: new WeakMap(), value: new WeakMap() }; @@ -39,11 +39,7 @@ class QueryResponse { get value() { queries.accessed.set(this, Date.now()); - const mapper = queries.mapper.get(this); - const value = queries.value.get(this); - - if (mapper) return mapper(queries.value.get(this)); - else return value; + return queries.value.get(this); } } @@ -69,13 +65,89 @@ function count(resourceType, filter) { return queryResponse; } -function limitFilter(resourceType, filter, last) { - if (resourceType === "devices") - return expression.and(filter, ["<=", ["PARAM", "DeviceID.ID"], last]); - else return expression.and(filter, ["<=", ["PARAM", "_id"], last]); +function limitFilter(filter, sort, bookmark) { + const sortSort = (a, b) => Math.abs(b[1]) - Math.abs(a[1]); + const arr = Object.entries(sort) + .sort(sortSort) + .reverse(); + return expression.and( + filter, + arr.reduce((cur, kv) => { + const [param, asc] = kv; + if (asc <= 0) { + if (bookmark[param] == null) + return expression.or( + ["IS NOT NULL", ["PARAM", param]], + expression.and(["IS NULL", ["PARAM", param]], cur) + ); + + let f = null; + + if (typeof bookmark[param] !== "string") + f = expression.or(f, [">=", ["PARAM", param], ""]); + + f = expression.or(f, [">", ["PARAM", param], bookmark[param]]); + return expression.or( + f, + expression.and(["=", ["PARAM", param], bookmark[param]], cur) + ); + } else { + let f = ["IS NULL", ["PARAM", param]]; + if (bookmark[param] == null) return expression.and(f, cur); + + if (typeof bookmark[param] !== "number") { + f = expression.or(f, [">=", ["PARAM", param], 0]); + f = expression.or(f, ["<", ["PARAM", param], 0]); + } + + f = expression.or(f, ["<", ["PARAM", param], bookmark[param]]); + + return expression.or( + f, + expression.and(["=", ["PARAM", param], bookmark[param]], cur) + ); + } + }, true) + ); } -function findMatches(resourceType, filter, limit) { +function compareFunction(sort) { + const sortEntries = Object.entries(sort).sort( + (a, b) => Math.abs(b[1]) - Math.abs(a[1]) + ); + + return (a, b) => { + for (const [param, asc] of sortEntries) { + let v1 = a[param]; + let v2 = b[param]; + if (v1 != null && typeof v1 === "object") + if (v1.value) v1 = v1.value[0]; + else v1 = null; + + if (v2 != null && typeof v2 === "object") + if (v2.value) v2 = v2.value[0]; + else v2 = null; + + if (v1 > v2) { + return asc; + } else if (v1 < v2) { + return asc * -1; + } else if (v1 !== v2) { + const w = { + null: 1, + number: 2, + string: 3 + }; + const w1 = w[v1 == null ? "null" : typeof v1] || 4; + const w2 = w[v2 == null ? "null" : typeof v2] || 4; + return Math.max(-1, Math.min(1, w1 - w2)) * asc; + } + } + return 0; + }; +} + +function findMatches(resourceType, filter, sort, limit) { // Handle "tag =" and "tag <>" special cases if (resourceType === "devices") filter = expressionParser.map(filter, e => { @@ -91,11 +163,11 @@ function findMatches(resourceType, filter, limit) { }); let value = []; - for (let [id, obj] of resources[resourceType].objects.entries()) + for (let obj of resources[resourceType].objects.values()) if (expression.evaluate(filter, obj, fulfillTimestamp)) - value.push(id); + value.push(obj); - value = value.sort(); + value = value.sort(compareFunction(sort)); if (limit) value = value.slice(0, limit); return value; @@ -105,9 +177,10 @@ function inferQuery(resourceType, queryResponse) { const limit = queries.limit.get(queryResponse); let filter = queries.filter.get(queryResponse); filter = unpackExpression(filter); - let last = queries.last.get(queryResponse); - if (last || !limit) { - if (last) filter = limitFilter(resourceType, filter, last); + let bookmark = queries.bookmark.get(queryResponse); + const sort = queries.sort.get(queryResponse); + if (bookmark || !limit) { + if (bookmark) filter = limitFilter(filter, sort, bookmark); if ( resources[resourceType].combinedFilter && expression.subset(filter, resources[resourceType].combinedFilter) @@ -115,12 +188,21 @@ function inferQuery(resourceType, queryResponse) { queries.fulfilled.set(queryResponse, fulfillTimestamp); } - queries.value.set(queryResponse, findMatches(resourceType, filter, limit)); + queries.value.set( + queryResponse, + findMatches(resourceType, filter, sort, limit) + ); } -function fetch(resourceType, filter, limit = 0) { +function fetch(resourceType, filter, options = {}) { const filterStr = funcCache.get(expression.stringify, filter); - const key = `${limit}:${filterStr}`; + const sort = options.sort || {}; + const limit = options.limit || 0; + if (resourceType === "devices") + sort["DeviceID.ID"] = sort["DeviceID.ID"] || 1; + else sort["_id"] = sort["_id"] || 1; + + let key = `${filterStr}:${limit}:${JSON.stringify(sort)}`; let queryResponse = resources[resourceType].fetch.get(key); if (queryResponse) return queryResponse; @@ -128,9 +210,7 @@ function fetch(resourceType, filter, limit = 0) { resources[resourceType].fetch.set(key, queryResponse); queries.filter.set(queryResponse, filter); queries.limit.set(queryResponse, limit); - queries.mapper.set(queryResponse, list => - list.map(x => resources[resourceType].objects.get(x)) - ); + queries.sort.set(queryResponse, sort); inferQuery(resourceType, queryResponse); return queryResponse; } @@ -204,6 +284,7 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { toFetchAll[resourceType] = toFetchAll[resourceType] || []; toFetchAll[resourceType].push(queryResponse); const limit = queries.limit.get(queryResponse); + const sort = queries.sort.get(queryResponse); if (limit) allPromises.push( new Promise((resolve, reject) => { @@ -219,18 +300,26 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { filter: funcCache.get(expression.stringify, filter), limit: 1, skip: limit - 1, - projection: "_id" + sort: JSON.stringify(sort), + projection: Object.keys(sort).join(",") }) }) .then(res => { - if (res.length) - if (resourceType === "devices") - queries.last.set( - queryResponse, - res[0]["DeviceID.ID"].value[0] - ); - else queries.last.set(queryResponse, res[0]["_id"]); - else queries.last.delete(queryResponse); + if (res.length) { + // Generate bookmark object + let bm = Object.keys(sort).reduce((b, k) => { + if (res[0][k] != null) + if (typeof res[0][k] === "object") { + if (res[0][k].value != null) + b[k] = res[0][k].value[0]; + } else b[k] = res[0][k]; + + return b; + }, {}); + queries.bookmark.set(queryResponse, bm); + } else { + queries.bookmark.delete(queryResponse); + } resolve(); }) .catch(reject); @@ -249,8 +338,10 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { toFetch = toFetch.filter(queryResponse => { let filter = queries.filter.get(queryResponse); filter = unpackExpression(filter); - let last = queries.last.get(queryResponse); - if (last) filter = limitFilter(resourceType, filter, last); + const limit = queries.limit.get(queryResponse); + const bookmark = queries.bookmark.get(queryResponse); + const sort = queries.sort.get(queryResponse); + if (bookmark) filter = limitFilter(filter, sort, bookmark); if ( resources[resourceType].combinedFilter && @@ -258,7 +349,7 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { ) { queries.value.set( queryResponse, - findMatches(resourceType, filter) + findMatches(resourceType, filter, sort, limit) ); queries.fulfilled.set(queryResponse, fulfillTimestamp); queries.fulfilling.delete(queryResponse); @@ -325,12 +416,14 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { for (let queryResponse of toFetch) { let filter = queries.filter.get(queryResponse); filter = unpackExpression(filter); - let last = queries.last.get(queryResponse); - if (last) filter = limitFilter(resourceType, filter, last); + const limit = queries.limit.get(queryResponse); + const bookmark = queries.bookmark.get(queryResponse); + const sort = queries.sort.get(queryResponse); + if (bookmark) filter = limitFilter(filter, sort, bookmark); queries.value.set( queryResponse, - findMatches(resourceType, filter) + findMatches(resourceType, filter, sort, limit) ); queries.fulfilled.set(queryResponse, fulfillTimestamp); queries.fulfilling.delete(queryResponse); diff --git a/common/expression-cnf.js b/common/expression-cnf.js index d2ea2ce..27c28f9 100644 --- a/common/expression-cnf.js +++ b/common/expression-cnf.js @@ -48,7 +48,7 @@ function booleanCnf(exp) { let m = mutuallyExclusive.get(lhs); if (!m) mutuallyExclusive.set(lhs, (m = new Set())); m.add(f); - if (rhs || key !== "null") m.add(getVariable(lhs, "null")); + if (rhs || op !== "null") m.add(getVariable(lhs, "null")); } return f; } @@ -66,7 +66,7 @@ function booleanCnf(exp) { for (let i = 1; i < range.length; ++i) if ((from == null || range[i] > from) && (to == null || range[i] <= to)) { vars.push(`${lhs}:${range[i - 1]}`); - vars.push(`${lhs}:${range[i]}`); + if (range[i] !== to) vars.push(`${lhs}:${range[i]}`); vars.push(`${lhs}:${range[i - 1]}--${range[i]}`); } diff --git a/server/api.js b/server/api.js index ff472db..40479e7 100644 --- a/server/api.js +++ b/server/api.js @@ -58,6 +58,8 @@ for (let [resource, flags] of Object.entries(resources)) { filter = expression.parse(ctx.request.query.filter); if (ctx.request.query.limit) options.limit = +ctx.request.query.limit; if (ctx.request.query.skip) options.skip = +ctx.request.query.skip; + if (ctx.request.query.sort) + options.sort = JSON.parse(ctx.request.query.sort); if (ctx.request.query.projection) options.projection = ctx.request.query.projection .split(",") diff --git a/server/db.js b/server/db.js index f0fa4b8..6afed3c 100644 --- a/server/db.js +++ b/server/db.js @@ -56,7 +56,19 @@ function query(resource, filter, options, callback) { ); if (options.skip) cursor = cursor.skip(options.skip); if (options.limit) cursor = cursor.limit(options.limit); - cursor = cursor.sort({ _id: 1 }); + + if (options.sort) { + let s = Object.entries(options.sort) + .sort((a, b) => Math.abs(b[1]) - Math.abs(a[1])) + .reduce( + (obj, [k, v]) => + Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }), + {} + ); + + if (resource === "devices") s = mongodbFunctions.processDeviceSort(s); + cursor = cursor.sort(s); + } if (!callback) cursor.toArray((err, docs) => {