From a49cbc5e408afe93d8b3b07e9e32612a6e74a897 Mon Sep 17 00:00:00 2001 From: Zaid Abdulla Date: Mon, 13 Aug 2018 19:29:40 -0700 Subject: [PATCH] Implement CSV download --- client/components/container.js | 6 +- client/components/device-faults.js | 2 +- client/components/overview-dot.js | 6 +- client/components/parameter-list.js | 44 +++++---------- client/components/parameter-table.js | 33 ++++++----- client/components/parameter.js | 63 ++++++++++++++++----- client/components/summon-button.js | 24 ++++---- client/devices-page.js | 66 +++++++++++++--------- client/faults-page.js | 57 +++++++++++++------ client/filter-component.js | 50 ++++++++++------- client/overview-page.js | 6 +- client/store.js | 41 ++++---------- common/authorizer.js | 8 +-- common/expression.js | 3 +- common/func-cache.js | 29 ---------- common/memoize.js | 47 ++++++++++++++++ config.json | 4 +- server/api.js | 84 +++++++++++++++++++++++++++- server/db.js | 12 +++- server/mongodb-functions.js | 21 +++++++ 20 files changed, 391 insertions(+), 215 deletions(-) delete mode 100644 common/func-cache.js create mode 100644 common/memoize.js diff --git a/client/components/container.js b/client/components/container.js index 19603c3..9d1713a 100644 --- a/client/components/container.js +++ b/client/components/container.js @@ -4,12 +4,14 @@ import m from "mithril"; import * as components from "../components"; import * as store from "../store"; import * as expression from "../../common/expression"; -import * as funcCache from "../../common/func-cache"; +import memoize from "../../common/memoize"; + +const memoizedParse = memoize(expression.parse); const component = { view: vnode => { if (vnode.attrs.filter) { - const filter = funcCache.get(expression.parse, vnode.attrs.filter); + const filter = memoizedParse(vnode.attrs.filter); if (!store.evaluateExpression(filter, vnode.attrs.device)) return; } diff --git a/client/components/device-faults.js b/client/components/device-faults.js index 74145bb..2eff14a 100644 --- a/client/components/device-faults.js +++ b/client/components/device-faults.js @@ -26,7 +26,7 @@ const component = { m("td", f["code"]), m("td", f["message"]), m("td", f["retries"]), - m("td", f["timestamp"]), + m("td", new Date(f["timestamp"]).toLocaleString()), m( "td", m( diff --git a/client/components/overview-dot.js b/client/components/overview-dot.js index 918e927..6789df4 100644 --- a/client/components/overview-dot.js +++ b/client/components/overview-dot.js @@ -3,8 +3,10 @@ import m from "mithril"; import config from "../config"; import * as store from "../store"; -import * as funcCache from "../../common/func-cache"; import * as expression from "../../common/expression"; +import memoize from "../../common/memoize"; + +const memoizedParse = memoize(expression.parse); const CHARTS = config.ui.overview.charts; @@ -13,7 +15,7 @@ const component = { const device = vnode.attrs.device; const chart = CHARTS[vnode.attrs.chart]; for (let slice of Object.values(chart.slices)) { - const filter = funcCache.get(expression.parse, slice.filter); + const filter = memoizedParse(slice.filter); if (store.evaluateExpression(filter, device)) { const dot = m( "svg", diff --git a/client/components/parameter-list.js b/client/components/parameter-list.js index 77a9e31..194fdb0 100644 --- a/client/components/parameter-list.js +++ b/client/components/parameter-list.js @@ -2,47 +2,29 @@ import m from "mithril"; import * as components from "../components"; -import * as store from "../store"; -import * as funcCache from "../../common/func-cache"; -import * as expression from "../../common/expression"; +import memoize from "../../common/memoize"; -const parseParameter = funcCache.getter(p => { - p = expression.parse(p); - if (Array.isArray(p) && p[0] === "PARAM") p = p[1]; - return p; -}); +const getChildAttrs = memoize((attrs, device) => + Object.assign({}, attrs, { device: device }) +); const component = { - oninit: vnode => { - vnode.state.parameters = Object.values(vnode.attrs.parameters).map( - parameter => - Object.assign({}, parameter, { - parameter: parseParameter(parameter.parameter) - }) - ); - }, view: vnode => { const device = vnode.attrs.device; - const rows = []; - for (let parameter of vnode.state.parameters) { - let p = store.evaluateExpression(parameter.parameter, device); - - if (!(p in device)) continue; - rows.push( + const rows = Object.values(vnode.attrs.parameters).map(parameter => + m( + "tr", + m("th", parameter.label), m( - "tr", - m("th", parameter.label), + "td", m( - "td", - m( - components.get(parameter.type || "parameter"), - Object.assign({}, parameter, { device: device, parameter: p }) - ) + components.get(parameter.type || "parameter"), + getChildAttrs(parameter, device) ) ) - ); - } + ) + ); return m("table.parameter-list", rows); } diff --git a/client/components/parameter-table.js b/client/components/parameter-table.js index 84bb939..f406b8a 100644 --- a/client/components/parameter-table.js +++ b/client/components/parameter-table.js @@ -5,21 +5,18 @@ import * as components from "../components"; import * as taskQueue from "../task-queue"; import * as store from "../store"; import * as expression from "../../common/expression"; -import * as funcCache from "../../common/func-cache"; - -const parseParameter = funcCache.getter(p => { - p = expression.parse(p); - if (Array.isArray(p) && p[0] === "PARAM") p = p[1]; - return p; -}); +import * as expressionParser from "../../common/expression-parser"; const component = { oninit: vnode => { - vnode.state.object = parseParameter(vnode.attrs.parameter); + let obj = expression.parse(vnode.attrs.parameter); + if (!Array.isArray(obj) || !obj[0] === "PARAM") + throw new Error("Object must be a parameter path"); + vnode.state.object = obj[1]; vnode.state.parameters = Object.values(vnode.attrs.childParameters).map( parameter => { return Object.assign({}, parameter, { - parameter: parseParameter(parameter.parameter) + parameter: expression.parse(parameter.parameter) }); } ); @@ -46,22 +43,24 @@ const component = { const rows = []; for (let i of instances) { - const row = parameters.map(p => - m( + const row = parameters.map(p => { + const param = expressionParser.map(p.parameter, e => { + if (Array.isArray(e) && e[0] === "PARAM") + return ["PARAM", ["||", i, ".", e[1]]]; + return e; + }); + return m( "td", m( components.get(p.type || "parameter"), Object.assign({}, p, { device: device, - parameter: `${i}.${store.evaluateExpression( - p.parameter, - device - )}`, + parameter: expressionParser.stringify(param), label: "" }) ) - ) - ); + ); + }); if (device[i].writable === true) row.push( diff --git a/client/components/parameter.js b/client/components/parameter.js index 0b339ec..0871252 100644 --- a/client/components/parameter.js +++ b/client/components/parameter.js @@ -2,19 +2,53 @@ import m from "mithril"; import * as taskQueue from "../task-queue"; +import * as store from "../store"; +import * as expression from "../../common/expression"; +import memoize from "../../common/memoize"; + +const memoizedParse = memoize(expression.parse); + +const evaluateParam = memoize((exp, obj, now) => { + let timestamp = now; + let params = new Set(); + let value = expression.evaluate( + memoizedParse(exp), + obj, + now, + e => { + if (Array.isArray(e)) { + if (e[0] === "PARAM") { + params.add(e[1]); + if (!Array.isArray(e[1])) { + let p = obj[e[1]]; + if (p && p.valueTimestamp) + timestamp = Math.min(timestamp, p.valueTimestamp); + } + } + if (e[0] === "FUNC" && e[1] === "DATE_STRING" && !Array.isArray(e[2])) + return new Date(e[2]).toLocaleString(); + } + return e; + } + ); + + let parameter = params.size === 1 ? params.values().next().value : null; + if (Array.isArray(parameter)) parameter = null; + return { value, timestamp, parameter }; +}); const component = { view: vnode => { const device = vnode.attrs.device; - const parameter = vnode.attrs.parameter; - const param = device[parameter]; - if (!param || !param.value) return m("span.na", "N/A"); - let value = param.value[0]; - if (param.value[1] === "xsd:dateTime") - value = new Date(value).toLocaleString(); + + let { value, timestamp, parameter } = evaluateParam( + vnode.attrs.parameter, + vnode.attrs.device, + store.getTimestamp() + ); let edit; - if (param.writable) + if (device[parameter] && device[parameter].writable) edit = m( "button", { @@ -23,19 +57,20 @@ const component = { taskQueue.stageSpv({ name: "setParameterValues", device: device["DeviceID.ID"].value[0], - parameterValues: [[parameter, param.value[0], param.value[1]]] + parameterValues: [ + [ + parameter, + device[parameter].value[0], + device[parameter].value[1] + ] + ] }); } }, "✎" ); - return m( - "span.parameter-value", - { title: param.valueTimestamp }, - value, - edit - ); + return m("span.parameter-value", { title: timestamp }, value, edit); } }; diff --git a/client/components/summon-button.js b/client/components/summon-button.js index f379049..8afdc4b 100644 --- a/client/components/summon-button.js +++ b/client/components/summon-button.js @@ -4,21 +4,12 @@ import m from "mithril"; import * as taskQueue from "../task-queue"; import * as store from "../store"; import * as notifications from "../notifications"; -import * as funcCache from "../../common/func-cache"; import * as expression from "../../common/expression"; +import memoize from "../../common/memoize"; -const parseParameter = funcCache.getter(p => { - p = expression.parse(p); - if (Array.isArray(p) && p[0] === "PARAM") p = p[1]; - return p; -}); +const memoizedParse = memoize(expression.parse); const component = { - oninit: vnode => { - vnode.state.parameters = Object.values(vnode.attrs.parameters).map( - parameter => parseParameter(parameter) - ); - }, view: vnode => { const device = vnode.attrs.device; @@ -28,9 +19,14 @@ const component = { title: "Initiate session and refresh basic parameters", onclick: e => { e.target.disabled = true; - const params = vnode.state.parameters.map(p => - store.evaluateExpression(p, device) - ); + const params = Object.values(vnode.attrs.parameters) + .map(p => { + const exp = memoizedParse(p); + if (!Array.isArray(exp) || !exp[0] == "PARAM") return null; + return store.evaluateExpression(exp[1], device); + }) + .filter(exp => !!exp); + const task = { name: "getParameterValues", parameterNames: params, diff --git a/client/devices-page.js b/client/devices-page.js index 6ca53f4..685cf8a 100644 --- a/client/devices-page.js +++ b/client/devices-page.js @@ -9,20 +9,31 @@ import * as components from "./components"; import * as taskQueue from "./task-queue"; import * as notifications from "./notifications"; import * as expression from "../common/expression"; +import memoize from "../common/memoize"; + +const memoizedParse = memoize(expression.parse); + +const getDownloadUrl = memoize((filter, indexParameters) => { + const columns = {}; + for (const p of indexParameters) columns[p.label] = p.parameter; + return `/api/devices.csv?${m.buildQueryString({ + filter: filter, + columns: JSON.stringify(columns) + })}`; +}); + +const getChildAttrs = memoize((attrs, device) => + Object.assign({}, attrs, { device: device }) +); function init(args) { return new Promise((resolve, reject) => { if (!window.authorizer.hasAccess("devices", 2)) return reject(new Error("You are not authorized to view this page")); - let filter = null; - if (args.filter != null) filter = expression.parse(`${args.filter}`); - let indexParameters = Object.values(config.ui.index).map(p => { - let param = expression.parse(p.parameter); - if (Array.isArray(param) && param[0] === "PARAM") param = param[1]; - return Object.assign({}, p, { parameter: param }); - }); - resolve({ filter: filter, indexParameters: indexParameters }); + const filter = args.filter; + const indexParameters = Object.values(config.ui.index); + resolve({ filter, indexParameters }); }); } @@ -31,7 +42,8 @@ function renderTable( parameters, total, selected, - showMoreCallback + showMoreCallback, + downloadUrl ) { const devices = devicesResponse.value; const selectAll = m("input", { @@ -75,10 +87,7 @@ function renderTable( }, m("td", checkbox), parameters.map(p => { - const attrs = Object.assign({}, p, { - device: device, - parameter: store.evaluateExpression(p.parameter, device) - }); + const attrs = getChildAttrs(p, device); const comp = m(components.get(attrs.type || "parameter"), attrs); return m("td", comp); }), @@ -119,6 +128,9 @@ function renderTable( ) ); + if (downloadUrl) + footerElements.push(m("a.download-csv", { href: downloadUrl }, "Download")); + let tfoot = m( "tfoot", m("tr", m("td", { colspan: labels.length }, footerElements)) @@ -280,20 +292,16 @@ const component = { } function onFilterChanged(filter) { - let params = {}; - if (filter != null) params.filter = expression.stringify(filter); - m.route.set("/devices", params); + m.route.set("/devices", { filter }); } - let devs = store.fetch( - "devices", - vnode.attrs.filter == null ? true : vnode.attrs.filter, - { limit: vnode.state.showCount || 10 } - ); - let count = store.count( - "devices", - vnode.attrs.filter == null ? true : vnode.attrs.filter - ); + const filter = vnode.attrs.filter + ? memoizedParse(vnode.attrs.filter) + : true; + let devs = store.fetch("devices", filter, { + limit: vnode.state.showCount || 10 + }); + let count = store.count("devices", filter); let selected = new Set(); if (vnode.state.selected) @@ -302,6 +310,11 @@ const component = { selected.add(d["DeviceID.ID"].value[0]); vnode.state.selected = selected; + const downloadUrl = getDownloadUrl( + vnode.attrs.filter, + vnode.attrs.indexParameters + ); + return [ m("h1", "Listing devices"), m(filterComponent, { @@ -314,7 +327,8 @@ const component = { vnode.attrs.indexParameters, count.value, selected, - showMore + showMore, + downloadUrl ), renderActions(selected) ]; diff --git a/client/faults-page.js b/client/faults-page.js index a80294f..d2bd054 100644 --- a/client/faults-page.js +++ b/client/faults-page.js @@ -6,6 +6,22 @@ import filterComponent from "./filter-component"; import * as store from "./store"; import * as notifications from "./notifications"; import * as expression from "../common/expression"; +import memoize from "../common/memoize"; + +const memoizedParse = memoize(expression.parse); + +const getDownloadUrl = memoize(filter => { + return `/api/faults.csv?${m.buildQueryString({ + filter: filter, + columns: JSON.stringify({ + Device: "device", + Channel: "channel", + Code: "code", + Retries: "retries", + Timestamp: "DATE_STRING(timestamp)" + }) + })}`; +}); function init(args) { if (!window.authorizer.hasAccess("faults", 2)) @@ -13,12 +29,17 @@ function init(args) { new Error("You are not authorized to view this page") ); - let filter = null; - if (args.filter != null) filter = expression.parse(`${args.filter}`); - return Promise.resolve({ filter: filter }); + const filter = args.filter; + return Promise.resolve({ filter }); } -function renderTable(faultsResponse, total, selected, showMoreCallback) { +function renderTable( + faultsResponse, + total, + selected, + showMoreCallback, + downloadUrl +) { const faults = faultsResponse.value; const selectAll = m("input", { type: "checkbox", @@ -101,6 +122,9 @@ function renderTable(faultsResponse, total, selected, showMoreCallback) { ) ); + if (downloadUrl) + footerElements.push(m("a.download-csv", { href: downloadUrl }, "Download")); + let tfoot = m( "tfoot", m("tr", m("td", { colspan: labels.length }, footerElements)) @@ -152,20 +176,17 @@ const component = { } function onFilterChanged(filter) { - let params = {}; - if (filter != null) params.filter = expression.stringify(filter); - m.route.set("/faults", params); + m.route.set("/faults", { filter }); } - let faults = store.fetch( - "faults", - vnode.attrs.filter == null ? true : vnode.attrs.filter, - { limit: vnode.state.showCount || 10 } - ); - let count = store.count( - "faults", - vnode.attrs.filter == null ? true : vnode.attrs.filter - ); + const filter = vnode.attrs.filter + ? memoizedParse(vnode.attrs.filter) + : true; + + let faults = store.fetch("faults", filter, { + limit: vnode.state.showCount || 10 + }); + let count = store.count("faults", filter); let selected = new Set(); if (vnode.state.selected) @@ -173,6 +194,8 @@ const component = { if (vnode.state.selected.has(f["_id"])) selected.add(f["_id"]); vnode.state.selected = selected; + const downloadUrl = getDownloadUrl(vnode.attrs.filter); + return [ m("h1", "Listing faults"), m(filterComponent, { @@ -186,7 +209,7 @@ const component = { filter: vnode.attrs.filter, onChange: onFilterChanged }), - renderTable(faults, count.value, selected, showMore) + renderTable(faults, count.value, selected, showMore, downloadUrl) ]; } }; diff --git a/client/filter-component.js b/client/filter-component.js index 34c4df7..f2e3e56 100644 --- a/client/filter-component.js +++ b/client/filter-component.js @@ -1,8 +1,20 @@ "use strict"; import m from "mithril"; - import * as expression from "../common/expression"; +import memoize from "../common/memoize"; + +const splitFilter = memoize(filter => { + if (!filter) return [""]; + const list = []; + const f = expression.parse(filter); + if (Array.isArray(f) && f[0] === "AND") + for (let ff of f.slice(1)) list.push(expression.stringify(ff)); + else list.push(expression.stringify(f)); + + list.push(""); + return list; +}); const component = { view: vnode => { @@ -12,25 +24,15 @@ const component = { predefined.map(f => m("option", { value: `${f.parameter} = ` })) ); - if (vnode.attrs.filter !== vnode.state.filter) { - vnode.state.filterList = []; + if (!vnode.state.filterList || vnode.attrs.filter !== vnode.state.filter) { + vnode.state.filterInvalid = 0; vnode.state.filter = vnode.attrs.filter; - if (vnode.attrs.filter != null) - if (vnode.attrs.filter[0] === "AND") - for (let i = 1; i < vnode.attrs.filter.length; ++i) - vnode.state.filterList.push( - expression.stringify(vnode.attrs.filter[i]) - ); - else - vnode.state.filterList.push(expression.stringify(vnode.attrs.filter)); - vnode.state.filterList.push(""); + vnode.state.filterList = splitFilter(vnode.attrs.filter); } function onChange() { vnode.state.filterInvalid = 0; - vnode.state.filterList = vnode.state.filterList.filter( - f => f && f.trim() - ); + vnode.state.filterList = vnode.state.filterList.filter(f => f); let filter = vnode.state.filterList.map((f, idx) => { try { return expression.parse(f); @@ -38,11 +40,18 @@ const component = { vnode.state.filterInvalid |= 1 << idx; } }); - if (filter.length) filter = ["AND"].concat(filter); - else filter = null; - vnode.state.filterList.push(""); - if (!vnode.state.filterInvalid) vnode.attrs.onChange(filter); + + if (!vnode.state.filterInvalid) { + delete vnode.state.filter; + if (filter.length === 0) { + vnode.attrs.onChange(""); + } else { + if (filter.length > 1) filter = ["AND"].concat(filter); + else filter = filter[0]; + vnode.attrs.onChange(expression.stringify(filter)); + } + } } return m( @@ -55,7 +64,8 @@ const component = { class: `${(vnode.state.filterInvalid >> idx) & 1 ? "error" : ""}`, value: fltr, onchange: e => { - vnode.state.filterList[idx] = e.target.value; + vnode.state.filterList = vnode.state.filterList.slice(); + vnode.state.filterList[idx] = e.target.value.trim(); onChange(); } }); diff --git a/client/overview-page.js b/client/overview-page.js index 302cc50..b0c4b29 100644 --- a/client/overview-page.js +++ b/client/overview-page.js @@ -4,8 +4,10 @@ import m from "mithril"; import config from "./config"; import * as store from "./store"; import pieChartComponent from "./pie-chart-component"; -import * as funcCache from "../common/func-cache"; import * as expression from "../common/expression"; +import memoize from "../common/memoize"; + +const memoizedParse = memoize(expression.parse); const GROUPS = config.ui.overview.groups; const CHARTS = {}; @@ -19,7 +21,7 @@ function queryCharts(charts) { charts[chartName] = chart = Object.assign({}, chart); chart.slices = Object.assign({}, chart.slices); for (let [sliceName, slice] of Object.entries(chart.slices)) { - const filter = funcCache.get(expression.parse, slice.filter); + const filter = memoizedParse(slice.filter); chart.slices[sliceName] = slice = Object.assign({}, slice); slice.count = store.count("devices", filter); } diff --git a/client/store.js b/client/store.js index 83b7e84..ccc259b 100644 --- a/client/store.js +++ b/client/store.js @@ -4,13 +4,13 @@ import m from "mithril"; import config from "./config"; import * as expressionParser from "../common/expression-parser"; import * as expression from "../common/expression"; -import * as funcCache from "../common/func-cache"; +import memoize from "../common/memoize"; + +const memoizedStringify = memoize(expression.stringify); +const memoizedEvaluate = memoize(expression.evaluate); let fulfillTimestamp = 0; -let unpackExpressionCache = new WeakMap(); -let evaluateExpressionCache = new WeakMap(); - const queries = { filter: new WeakMap(), bookmark: new WeakMap(), @@ -45,16 +45,12 @@ class QueryResponse { function unpackExpression(exp) { if (!Array.isArray(exp)) return exp; - let e = unpackExpressionCache.get(exp); - if (e === undefined) { - e = expression.evaluate(e, null, fulfillTimestamp); - unpackExpressionCache.set(exp, e); - } + const e = memoizedEvaluate(exp, null, fulfillTimestamp); return e; } function count(resourceType, filter) { - const filterStr = funcCache.get(expression.stringify, filter); + const filterStr = memoizedStringify(filter); let queryResponse = resources[resourceType].count.get(filterStr); if (queryResponse) return queryResponse; @@ -195,7 +191,7 @@ function inferQuery(resourceType, queryResponse) { } function fetch(resourceType, filter, options = {}) { - const filterStr = funcCache.get(expression.stringify, filter); + const filterStr = memoizedStringify(filter); const sort = options.sort || {}; const limit = options.limit || 0; if (resourceType === "devices") @@ -219,9 +215,6 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { let updated = false; if (_fulfillTimestamp > fulfillTimestamp) { - unpackExpressionCache = new WeakMap(); - evaluateExpressionCache = new WeakMap(); - for (let resource of Object.values(resources)) resource.combinedFilter = null; fulfillTimestamp = _fulfillTimestamp; @@ -250,7 +243,7 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { url: `/api/${resourceType}/?` + m.buildQueryString({ - filter: funcCache.get(expression.stringify, filter) + filter: memoizedStringify(filter) }), extract: xhr => +xhr.getResponseHeader("x-total-count"), background: true @@ -295,7 +288,7 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { url: `/api/${resourceType}/?` + m.buildQueryString({ - filter: funcCache.get(expression.stringify, filter), + filter: memoizedStringify(filter), limit: 1, skip: limit - 1, sort: JSON.stringify(sort), @@ -382,10 +375,7 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { url: `/api/${resourceType}/?` + m.buildQueryString({ - filter: funcCache.get( - expression.stringify, - combinedFilterDiff - ) + filter: memoizedStringify(combinedFilterDiff) }) }) .then(res => { @@ -488,16 +478,7 @@ function deleteResource(resourceType, id) { function evaluateExpression(exp, obj) { if (!Array.isArray(exp)) return exp; - - let exps = evaluateExpressionCache.get(exp); - if (!exps) evaluateExpressionCache.set(exp, (exps = new WeakMap())); - - if (!exps.has(obj)) { - let v = expression.evaluate(exp, obj, fulfillTimestamp); - exps.set(obj, v); - } - - return exps.get(obj); + return memoizedEvaluate(exp, obj, fulfillTimestamp); } function logIn(username, password) { diff --git a/common/authorizer.js b/common/authorizer.js index 03e0122..0288142 100644 --- a/common/authorizer.js +++ b/common/authorizer.js @@ -10,7 +10,7 @@ const validators = { class Authorizer { constructor(permissionSets) { this.permissionSets = permissionSets; - this.validtorCache = new WeakMap(); + this.validatorCache = new WeakMap(); this.hasAccessCache = {}; } @@ -34,8 +34,8 @@ class Authorizer { } getValidator(resourceType, resource) { - if (this.validtorCache.has(resource)) - return this.validtorCache.get(resource); + if (this.validatorCache.has(resource)) + return this.validatorCache.get(resource); let funcs = {}; @@ -70,7 +70,7 @@ class Authorizer { return valid; }; - this.validtorCache.set(resource, validator); + this.validatorCache.set(resource, validator); return validator; } } diff --git a/common/expression.js b/common/expression.js index 290ee7a..77256dc 100644 --- a/common/expression.js +++ b/common/expression.js @@ -36,7 +36,7 @@ function reduce(exp, callback) { return exp; } -function evaluate(exp, obj, now) { +function evaluate(exp, obj, now, cb) { function getRegExp(pat, esc) { const k = `${esc || ""}:${pat}`; let c = regExpCache.get(exp); @@ -46,6 +46,7 @@ function evaluate(exp, obj, now) { } return expressionParser.map(exp, e => { + if (cb) e = cb(e); if (!isArray(e)) return e; if (e[0] === "FUNC") { diff --git a/common/func-cache.js b/common/func-cache.js deleted file mode 100644 index 4c9f081..0000000 --- a/common/func-cache.js +++ /dev/null @@ -1,29 +0,0 @@ -"use strict"; - -let cache1 = new WeakMap(); -let cache2 = new WeakMap(); - -function get(func, k) { - let c1 = cache1.get(func); - if (!c1) cache1.set(func, (c1 = new Map())); - if (c1.has(k)) return c1.get(k); - const c2 = cache2.get(func); - const v = c2 && c2.has(k) ? c2.get(k) : func(k); - c1.set(k, v); - return v; -} - -function getter(func) { - return k => get(func, k); -} - -function purge() { - cache2 = cache1; - cache1 = new WeakMap(); -} - -setInterval(purge, 120000); - -exports.get = get; -exports.getter = getter; -exports.purge = purge; diff --git a/common/memoize.js b/common/memoize.js new file mode 100644 index 0000000..9a13007 --- /dev/null +++ b/common/memoize.js @@ -0,0 +1,47 @@ +"use strict"; + +let cache1 = new Map(); +let cache2 = new Map(); +const keys = new WeakMap(); + +function getKey(obj) { + if (obj === null) return "null"; + else if (obj === undefined) return "undefined"; + + const t = typeof obj; + if (t === "number" || t === "boolean" || t === "string") return `${t}:${obj}`; + if (t !== "function" && t !== "object") + throw new Error(`Cannot memoize ${t} arguments`); + + let k = keys.get(obj); + if (!k) { + const rnd = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER); + k = `${t}:${rnd.toString(36)}`; + keys.set(obj, k); + } + return k; +} + +function memoize(func) { + return (...args) => { + const key = JSON.stringify(args.map(getKey)) + getKey(func); + + if (cache1.has(key)) return cache1.get(key); + + let r; + if (cache2.has(key)) r = cache2.get(key); + else r = func(...args); + cache1.set(key, r); + return r; + }; +} + +const interval = setInterval(() => { + cache2 = cache1; + cache1 = new Map(); +}, 120000); + +// Don't hold Node.js process +if (interval.unref) interval.unref(); + +module.exports = memoize; diff --git a/config.json b/config.json index e83fbce..32dfac5 100644 --- a/config.json +++ b/config.json @@ -95,7 +95,7 @@ }, { "label": "Last inform", - "parameter": "Events.Inform", + "parameter": "DATE_STRING(Events.Inform)", "type": "container", "element": "span.inform", "components": [ @@ -161,7 +161,7 @@ "type": "container", "label": "Last inform", "element": "span.inform", - "parameter": "Events.Inform", + "parameter": "DATE_STRING(Events.Inform)", "components": [ { "type": "parameter" diff --git a/server/api.js b/server/api.js index 1774e1e..9d4eb0b 100644 --- a/server/api.js +++ b/server/api.js @@ -3,7 +3,7 @@ const zlib = require("zlib"); const Router = require("koa-router"); const PassThrough = require("stream").PassThrough; - +const config = require("./config"); const db = require("./db"); const apiFunctions = require("./api-functions"); const expression = require("../common/expression"); @@ -102,6 +102,88 @@ for (let [resource, flags] of Object.entries(resources)) { stream.end("]"); }); + // CSV download + router.get(`/${resource}.csv`, async (ctx, next) => { + let options = {}; + let filter = true; + if (ctx.request.query.filter) + filter = expression.parse(ctx.request.query.filter); + if (ctx.request.query.limit) options.limit = +ctx.request.query.limit; + if (ctx.request.query.skip) options.skip = +ctx.request.query.skip; + if (ctx.request.query.sort) + options.sort = JSON.parse(ctx.request.query.sort); + + if (!ctx.state.authorizer.hasAccess(resource, 2)) return next(); + + const columns = JSON.parse(ctx.request.query.columns); + const now = Date.now(); + for (let [k, v] of Object.entries(columns)) + columns[k] = expression.evaluate( + expression.parse(v), + null, + now + ); + + // Exclude temporary tasks and faults + if (resource === "tasks" || resource === "faults") + filter = expression.and(filter, [ + "NOT", + ["<", ["PARAM", "expiry"], Date.now() + 60000] + ]); + + let stream; + switch (ctx.acceptsEncodings("gzip", "deflate", "identity")) { + case "gzip": + stream = zlib.createGzip(); + ctx.set("Content-Encoding", "gzip"); + break; + case "deflate": + stream = zlib.createDeflate(); + ctx.set("Content-Encoding", "deflate"); + break; + default: + stream = new PassThrough(); + break; + } + + ctx.body = stream; + ctx.type = "text/csv"; + + stream.write( + Object.keys(columns).map(k => `"${k.replace(/"/, '""')}"`) + "\n" + ); + await db.query(resource, filter, options, obj => { + let arr = Object.values(columns).map(exp => { + let v = expression.evaluate(exp, obj, null, e => { + if (Array.isArray(e)) + if (e[0] === "PARAM") { + if (resource === "devices") { + if (e[1] === "Tags") { + let tags = []; + for (let p in obj) + if (p.startsWith("Tags.")) tags.push(p.slice(5)); + + return tags.join(", "); + } + } + } else if (e[0] === "FUNC") { + if (e[1] === "DATE_STRING") + if (e[2] && !Array.isArray(e[2])) + return new Date(e[2]).toISOString(); + } + + return e; + }); + + if (Array.isArray(v) || v == null) return ""; + if (typeof v === "string") return `"${v.replace(/"/g, '""')}"`; + return v; + }); + stream.write(arr.join(",") + "\n"); + }); + stream.end(); + }); + router.head(`/${resource}/:id`, async (ctx, next) => { let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id]; if (!ctx.state.authorizer.hasAccess(resource, 2)) return next(); diff --git a/server/db.js b/server/db.js index f1eb19a..dafaf92 100644 --- a/server/db.js +++ b/server/db.js @@ -35,7 +35,7 @@ function getClient(db) { function query(resource, filter, options, callback) { options = options || {}; let q; - filter = expression.evaluate(filter, null, null, Date.now()); + filter = expression.evaluate(filter, null, Date.now()); if (Array.isArray(filter)) { if (resource === "devices") @@ -79,6 +79,10 @@ function query(resource, filter, options, callback) { if (err) return reject(err); if (resource === "devices") docs = docs.map(d => mongodbFunctions.flattenDevice(d)); + else if (resource === "faults") + docs = docs.map(d => mongodbFunctions.flattenFault(d)); + else if (resource === "tasks") + docs = docs.map(d => mongodbFunctions.flattenTask(d)); return resolve(docs); }); else @@ -86,6 +90,10 @@ function query(resource, filter, options, callback) { doc => { if (resource === "devices") doc = mongodbFunctions.flattenDevice(doc); + else if (resource === "faults") + doc = mongodbFunctions.flattenFault(doc); + else if (resource === "tasks") + doc = mongodbFunctions.flattenTask(doc); callback(doc); }, err => { @@ -99,7 +107,7 @@ function query(resource, filter, options, callback) { function count(resource, filter) { let q; - filter = expression.evaluate(filter, null, null, Date.now()); + filter = expression.evaluate(filter, null, Date.now()); if (Array.isArray(filter)) { if (resource === "devices") diff --git a/server/mongodb-functions.js b/server/mongodb-functions.js index f0eb680..8be074e 100644 --- a/server/mongodb-functions.js +++ b/server/mongodb-functions.js @@ -337,6 +337,25 @@ function flattenDevice(device) { return newDevice; } +function flattenFault(fault) { + const f = Object.assign({}, fault); + if (f.timestamp) f.timestamp = +f.timestamp; + if (f.expiry) f.expiry = +f.expiry; + if (f.detail) { + for (let [k, v] of Object.entries(f.detail)) fault[`detail.${k}`] = v; + delete f.detail; + } + return f; +} + +function flattenTask(task) { + let t = Object.assign({}, task); + t._id = "" + t._id; + if (t.timestamp) t.timestamp = +t.timestamp; + if (t.expiry) t.expiry = +t.expiry; + return t; +} + exports.processDeviceFilter = processDeviceFilter; exports.processTasksFilter = processTasksFilter; exports.processFaultsFilter = processFaultsFilter; @@ -344,3 +363,5 @@ exports.filterToMongoQuery = filterToMongoQuery; exports.processDeviceProjection = processDeviceProjection; exports.processDeviceSort = processDeviceSort; exports.flattenDevice = flattenDevice; +exports.flattenFault = flattenFault; +exports.flattenTask = flattenTask;