Implement CSV download

client/components/container.js

@@ -4,12 +4,14 @@ import m from "mithril";
 import * as components from "../components";
 import * as store from "../store";
 import * as expression from "../../common/expression";
-import * as funcCache from "../../common/func-cache";
+import memoize from "../../common/memoize";
+
+const memoizedParse = memoize(expression.parse);
 
 const component = {
   view: vnode => {
     if (vnode.attrs.filter) {
-      const filter = funcCache.get(expression.parse, vnode.attrs.filter);
+      const filter = memoizedParse(vnode.attrs.filter);
       if (!store.evaluateExpression(filter, vnode.attrs.device)) return;
     }
 

client/components/device-faults.js

@@ -26,7 +26,7 @@ const component = {
         m("td", f["code"]),
         m("td", f["message"]),
         m("td", f["retries"]),
-        m("td", f["timestamp"]),
+        m("td", new Date(f["timestamp"]).toLocaleString()),
         m(
           "td",
           m(

client/components/overview-dot.js

@@ -3,8 +3,10 @@
 import m from "mithril";
 import config from "../config";
 import * as store from "../store";
-import * as funcCache from "../../common/func-cache";
 import * as expression from "../../common/expression";
+import memoize from "../../common/memoize";
+
+const memoizedParse = memoize(expression.parse);
 
 const CHARTS = config.ui.overview.charts;
 
@@ -13,7 +15,7 @@ const component = {
     const device = vnode.attrs.device;
     const chart = CHARTS[vnode.attrs.chart];
     for (let slice of Object.values(chart.slices)) {
-      const filter = funcCache.get(expression.parse, slice.filter);
+      const filter = memoizedParse(slice.filter);
       if (store.evaluateExpression(filter, device)) {
         const dot = m(
           "svg",

client/components/parameter-list.js

@@ -2,34 +2,17 @@
 
 import m from "mithril";
 import * as components from "../components";
-import * as store from "../store";
+import memoize from "../../common/memoize";
-import * as funcCache from "../../common/func-cache";
-import * as expression from "../../common/expression";
 
-const parseParameter = funcCache.getter(p => {
+const getChildAttrs = memoize((attrs, device) =>
-  p = expression.parse(p);
+  Object.assign({}, attrs, { device: device })
-  if (Array.isArray(p) && p[0] === "PARAM") p = p[1];
+);
-  return p;
-});
 
 const component = {
-  oninit: vnode => {
-    vnode.state.parameters = Object.values(vnode.attrs.parameters).map(
-      parameter =>
-        Object.assign({}, parameter, {
-          parameter: parseParameter(parameter.parameter)
-        })
-    );
-  },
   view: vnode => {
     const device = vnode.attrs.device;
 
-    const rows = [];
+    const rows = Object.values(vnode.attrs.parameters).map(parameter =>
-    for (let parameter of vnode.state.parameters) {
-      let p = store.evaluateExpression(parameter.parameter, device);
-
-      if (!(p in device)) continue;
-      rows.push(
       m(
         "tr",
         m("th", parameter.label),
@@ -37,12 +20,11 @@ const component = {
           "td",
           m(
             components.get(parameter.type || "parameter"),
-              Object.assign({}, parameter, { device: device, parameter: p })
+            getChildAttrs(parameter, device)
           )
         )
       )
     );
-    }
 
     return m("table.parameter-list", rows);
   }

client/components/parameter-table.js

@@ -5,21 +5,18 @@ import * as components from "../components";
 import * as taskQueue from "../task-queue";
 import * as store from "../store";
 import * as expression from "../../common/expression";
-import * as funcCache from "../../common/func-cache";
+import * as expressionParser from "../../common/expression-parser";
-
-const parseParameter = funcCache.getter(p => {
-  p = expression.parse(p);
-  if (Array.isArray(p) && p[0] === "PARAM") p = p[1];
-  return p;
-});
 
 const component = {
   oninit: vnode => {
-    vnode.state.object = parseParameter(vnode.attrs.parameter);
+    let obj = expression.parse(vnode.attrs.parameter);
+    if (!Array.isArray(obj) || !obj[0] === "PARAM")
+      throw new Error("Object must be a parameter path");
+    vnode.state.object = obj[1];
     vnode.state.parameters = Object.values(vnode.attrs.childParameters).map(
       parameter => {
         return Object.assign({}, parameter, {
-          parameter: parseParameter(parameter.parameter)
+          parameter: expression.parse(parameter.parameter)
         });
       }
     );
@@ -46,22 +43,24 @@ const component = {
 
     const rows = [];
     for (let i of instances) {
-      const row = parameters.map(p =>
+      const row = parameters.map(p => {
-        m(
+        const param = expressionParser.map(p.parameter, e => {
+          if (Array.isArray(e) && e[0] === "PARAM")
+            return ["PARAM", ["||", i, ".", e[1]]];
+          return e;
+        });
+        return m(
           "td",
           m(
             components.get(p.type || "parameter"),
             Object.assign({}, p, {
               device: device,
-              parameter: `${i}.${store.evaluateExpression(
+              parameter: expressionParser.stringify(param),
-                p.parameter,
-                device
-              )}`,
               label: ""
             })
           )
-        )
         );
+      });
 
       if (device[i].writable === true)
         row.push(

client/components/parameter.js

@@ -2,19 +2,53 @@
 
 import m from "mithril";
 import * as taskQueue from "../task-queue";
+import * as store from "../store";
+import * as expression from "../../common/expression";
+import memoize from "../../common/memoize";
+
+const memoizedParse = memoize(expression.parse);
+
+const evaluateParam = memoize((exp, obj, now) => {
+  let timestamp = now;
+  let params = new Set();
+  let value = expression.evaluate(
+    memoizedParse(exp),
+    obj,
+    now,
+    e => {
+      if (Array.isArray(e)) {
+        if (e[0] === "PARAM") {
+          params.add(e[1]);
+          if (!Array.isArray(e[1])) {
+            let p = obj[e[1]];
+            if (p && p.valueTimestamp)
+              timestamp = Math.min(timestamp, p.valueTimestamp);
+          }
+        }
+        if (e[0] === "FUNC" && e[1] === "DATE_STRING" && !Array.isArray(e[2]))
+          return new Date(e[2]).toLocaleString();
+      }
+      return e;
+    }
+  );
+
+  let parameter = params.size === 1 ? params.values().next().value : null;
+  if (Array.isArray(parameter)) parameter = null;
+  return { value, timestamp, parameter };
+});
 
 const component = {
   view: vnode => {
     const device = vnode.attrs.device;
-    const parameter = vnode.attrs.parameter;
+
-    const param = device[parameter];
+    let { value, timestamp, parameter } = evaluateParam(
-    if (!param || !param.value) return m("span.na", "N/A");
+      vnode.attrs.parameter,
-    let value = param.value[0];
+      vnode.attrs.device,
-    if (param.value[1] === "xsd:dateTime")
+      store.getTimestamp()
-      value = new Date(value).toLocaleString();
+    );
 
     let edit;
-    if (param.writable)
+    if (device[parameter] && device[parameter].writable)
       edit = m(
         "button",
         {
@@ -23,19 +57,20 @@ const component = {
             taskQueue.stageSpv({
               name: "setParameterValues",
               device: device["DeviceID.ID"].value[0],
-              parameterValues: [[parameter, param.value[0], param.value[1]]]
+              parameterValues: [
+                [
+                  parameter,
+                  device[parameter].value[0],
+                  device[parameter].value[1]
+                ]
+              ]
             });
           }
         },
         "✎"
       );
 
-    return m(
+    return m("span.parameter-value", { title: timestamp }, value, edit);
-      "span.parameter-value",
-      { title: param.valueTimestamp },
-      value,
-      edit
-    );
   }
 };
 

client/components/summon-button.js

@@ -4,21 +4,12 @@ import m from "mithril";
 import * as taskQueue from "../task-queue";
 import * as store from "../store";
 import * as notifications from "../notifications";
-import * as funcCache from "../../common/func-cache";
 import * as expression from "../../common/expression";
+import memoize from "../../common/memoize";
 
-const parseParameter = funcCache.getter(p => {
+const memoizedParse = memoize(expression.parse);
-  p = expression.parse(p);
-  if (Array.isArray(p) && p[0] === "PARAM") p = p[1];
-  return p;
-});
 
 const component = {
-  oninit: vnode => {
-    vnode.state.parameters = Object.values(vnode.attrs.parameters).map(
-      parameter => parseParameter(parameter)
-    );
-  },
   view: vnode => {
     const device = vnode.attrs.device;
 
@@ -28,9 +19,14 @@ const component = {
         title: "Initiate session and refresh basic parameters",
         onclick: e => {
           e.target.disabled = true;
-          const params = vnode.state.parameters.map(p =>
+          const params = Object.values(vnode.attrs.parameters)
-            store.evaluateExpression(p, device)
+            .map(p => {
-          );
+              const exp = memoizedParse(p);
+              if (!Array.isArray(exp) || !exp[0] == "PARAM") return null;
+              return store.evaluateExpression(exp[1], device);
+            })
+            .filter(exp => !!exp);
+
           const task = {
             name: "getParameterValues",
             parameterNames: params,

client/devices-page.js

@@ -9,20 +9,31 @@ import * as components from "./components";
 import * as taskQueue from "./task-queue";
 import * as notifications from "./notifications";
 import * as expression from "../common/expression";
+import memoize from "../common/memoize";
+
+const memoizedParse = memoize(expression.parse);
+
+const getDownloadUrl = memoize((filter, indexParameters) => {
+  const columns = {};
+  for (const p of indexParameters) columns[p.label] = p.parameter;
+  return `/api/devices.csv?${m.buildQueryString({
+    filter: filter,
+    columns: JSON.stringify(columns)
+  })}`;
+});
+
+const getChildAttrs = memoize((attrs, device) =>
+  Object.assign({}, attrs, { device: device })
+);
 
 function init(args) {
   return new Promise((resolve, reject) => {
     if (!window.authorizer.hasAccess("devices", 2))
       return reject(new Error("You are not authorized to view this page"));
 
-    let filter = null;
+    const filter = args.filter;
-    if (args.filter != null) filter = expression.parse(`${args.filter}`);
+    const indexParameters = Object.values(config.ui.index);
-    let indexParameters = Object.values(config.ui.index).map(p => {
+    resolve({ filter, indexParameters });
-      let param = expression.parse(p.parameter);
-      if (Array.isArray(param) && param[0] === "PARAM") param = param[1];
-      return Object.assign({}, p, { parameter: param });
-    });
-    resolve({ filter: filter, indexParameters: indexParameters });
   });
 }
 
@@ -31,7 +42,8 @@ function renderTable(
   parameters,
   total,
   selected,
-  showMoreCallback
+  showMoreCallback,
+  downloadUrl
 ) {
   const devices = devicesResponse.value;
   const selectAll = m("input", {
@@ -75,10 +87,7 @@ function renderTable(
         },
         m("td", checkbox),
         parameters.map(p => {
-          const attrs = Object.assign({}, p, {
+          const attrs = getChildAttrs(p, device);
-            device: device,
-            parameter: store.evaluateExpression(p.parameter, device)
-          });
           const comp = m(components.get(attrs.type || "parameter"), attrs);
           return m("td", comp);
         }),
@@ -119,6 +128,9 @@ function renderTable(
     )
   );
 
+  if (downloadUrl)
+    footerElements.push(m("a.download-csv", { href: downloadUrl }, "Download"));
+
   let tfoot = m(
     "tfoot",
     m("tr", m("td", { colspan: labels.length }, footerElements))
@@ -280,20 +292,16 @@ const component = {
     }
 
     function onFilterChanged(filter) {
-      let params = {};
+      m.route.set("/devices", { filter });
-      if (filter != null) params.filter = expression.stringify(filter);
-      m.route.set("/devices", params);
     }
 
-    let devs = store.fetch(
+    const filter = vnode.attrs.filter
-      "devices",
+      ? memoizedParse(vnode.attrs.filter)
-      vnode.attrs.filter == null ? true : vnode.attrs.filter,
+      : true;
-      { limit: vnode.state.showCount || 10 }
+    let devs = store.fetch("devices", filter, {
-    );
+      limit: vnode.state.showCount || 10
-    let count = store.count(
+    });
-      "devices",
+    let count = store.count("devices", filter);
-      vnode.attrs.filter == null ? true : vnode.attrs.filter
-    );
 
     let selected = new Set();
     if (vnode.state.selected)
@@ -302,6 +310,11 @@ const component = {
           selected.add(d["DeviceID.ID"].value[0]);
     vnode.state.selected = selected;
 
+    const downloadUrl = getDownloadUrl(
+      vnode.attrs.filter,
+      vnode.attrs.indexParameters
+    );
+
     return [
       m("h1", "Listing devices"),
       m(filterComponent, {
@@ -314,7 +327,8 @@ const component = {
         vnode.attrs.indexParameters,
         count.value,
         selected,
-        showMore
+        showMore,
+        downloadUrl
       ),
       renderActions(selected)
     ];

client/faults-page.js

@@ -6,6 +6,22 @@ import filterComponent from "./filter-component";
 import * as store from "./store";
 import * as notifications from "./notifications";
 import * as expression from "../common/expression";
+import memoize from "../common/memoize";
+
+const memoizedParse = memoize(expression.parse);
+
+const getDownloadUrl = memoize(filter => {
+  return `/api/faults.csv?${m.buildQueryString({
+    filter: filter,
+    columns: JSON.stringify({
+      Device: "device",
+      Channel: "channel",
+      Code: "code",
+      Retries: "retries",
+      Timestamp: "DATE_STRING(timestamp)"
+    })
+  })}`;
+});
 
 function init(args) {
   if (!window.authorizer.hasAccess("faults", 2))
@@ -13,12 +29,17 @@ function init(args) {
       new Error("You are not authorized to view this page")
     );
 
-  let filter = null;
+  const filter = args.filter;
-  if (args.filter != null) filter = expression.parse(`${args.filter}`);
+  return Promise.resolve({ filter });
-  return Promise.resolve({ filter: filter });
 }
 
-function renderTable(faultsResponse, total, selected, showMoreCallback) {
+function renderTable(
+  faultsResponse,
+  total,
+  selected,
+  showMoreCallback,
+  downloadUrl
+) {
   const faults = faultsResponse.value;
   const selectAll = m("input", {
     type: "checkbox",
@@ -101,6 +122,9 @@ function renderTable(faultsResponse, total, selected, showMoreCallback) {
     )
   );
 
+  if (downloadUrl)
+    footerElements.push(m("a.download-csv", { href: downloadUrl }, "Download"));
+
   let tfoot = m(
     "tfoot",
     m("tr", m("td", { colspan: labels.length }, footerElements))
@@ -152,20 +176,17 @@ const component = {
     }
 
     function onFilterChanged(filter) {
-      let params = {};
+      m.route.set("/faults", { filter });
-      if (filter != null) params.filter = expression.stringify(filter);
-      m.route.set("/faults", params);
     }
 
-    let faults = store.fetch(
+    const filter = vnode.attrs.filter
-      "faults",
+      ? memoizedParse(vnode.attrs.filter)
-      vnode.attrs.filter == null ? true : vnode.attrs.filter,
+      : true;
-      { limit: vnode.state.showCount || 10 }
+
-    );
+    let faults = store.fetch("faults", filter, {
-    let count = store.count(
+      limit: vnode.state.showCount || 10
-      "faults",
+    });
-      vnode.attrs.filter == null ? true : vnode.attrs.filter
+    let count = store.count("faults", filter);
-    );
 
     let selected = new Set();
     if (vnode.state.selected)
@@ -173,6 +194,8 @@ const component = {
         if (vnode.state.selected.has(f["_id"])) selected.add(f["_id"]);
     vnode.state.selected = selected;
 
+    const downloadUrl = getDownloadUrl(vnode.attrs.filter);
+
     return [
       m("h1", "Listing faults"),
       m(filterComponent, {
@@ -186,7 +209,7 @@ const component = {
         filter: vnode.attrs.filter,
         onChange: onFilterChanged
       }),
-      renderTable(faults, count.value, selected, showMore)
+      renderTable(faults, count.value, selected, showMore, downloadUrl)
     ];
   }
 };

client/filter-component.js

@@ -1,8 +1,20 @@
 "use strict";
 
 import m from "mithril";
-
 import * as expression from "../common/expression";
+import memoize from "../common/memoize";
+
+const splitFilter = memoize(filter => {
+  if (!filter) return [""];
+  const list = [];
+  const f = expression.parse(filter);
+  if (Array.isArray(f) && f[0] === "AND")
+    for (let ff of f.slice(1)) list.push(expression.stringify(ff));
+  else list.push(expression.stringify(f));
+
+  list.push("");
+  return list;
+});
 
 const component = {
   view: vnode => {
@@ -12,25 +24,15 @@ const component = {
       predefined.map(f => m("option", { value: `${f.parameter} = ` }))
     );
 
-    if (vnode.attrs.filter !== vnode.state.filter) {
+    if (!vnode.state.filterList || vnode.attrs.filter !== vnode.state.filter) {
-      vnode.state.filterList = [];
+      vnode.state.filterInvalid = 0;
       vnode.state.filter = vnode.attrs.filter;
-      if (vnode.attrs.filter != null)
+      vnode.state.filterList = splitFilter(vnode.attrs.filter);
-        if (vnode.attrs.filter[0] === "AND")
-          for (let i = 1; i < vnode.attrs.filter.length; ++i)
-            vnode.state.filterList.push(
-              expression.stringify(vnode.attrs.filter[i])
-            );
-        else
-          vnode.state.filterList.push(expression.stringify(vnode.attrs.filter));
-      vnode.state.filterList.push("");
     }
 
     function onChange() {
       vnode.state.filterInvalid = 0;
-      vnode.state.filterList = vnode.state.filterList.filter(
+      vnode.state.filterList = vnode.state.filterList.filter(f => f);
-        f => f && f.trim()
-      );
       let filter = vnode.state.filterList.map((f, idx) => {
         try {
           return expression.parse(f);
@@ -38,11 +40,18 @@ const component = {
           vnode.state.filterInvalid |= 1 << idx;
         }
       });
-      if (filter.length) filter = ["AND"].concat(filter);
-      else filter = null;
-
       vnode.state.filterList.push("");
-      if (!vnode.state.filterInvalid) vnode.attrs.onChange(filter);
+
+      if (!vnode.state.filterInvalid) {
+        delete vnode.state.filter;
+        if (filter.length === 0) {
+          vnode.attrs.onChange("");
+        } else {
+          if (filter.length > 1) filter = ["AND"].concat(filter);
+          else filter = filter[0];
+          vnode.attrs.onChange(expression.stringify(filter));
+        }
+      }
     }
 
     return m(
@@ -55,7 +64,8 @@ const component = {
             class: `${(vnode.state.filterInvalid >> idx) & 1 ? "error" : ""}`,
             value: fltr,
             onchange: e => {
-              vnode.state.filterList[idx] = e.target.value;
+              vnode.state.filterList = vnode.state.filterList.slice();
+              vnode.state.filterList[idx] = e.target.value.trim();
               onChange();
             }
           });

client/overview-page.js

@@ -4,8 +4,10 @@ import m from "mithril";
 import config from "./config";
 import * as store from "./store";
 import pieChartComponent from "./pie-chart-component";
-import * as funcCache from "../common/func-cache";
 import * as expression from "../common/expression";
+import memoize from "../common/memoize";
+
+const memoizedParse = memoize(expression.parse);
 
 const GROUPS = config.ui.overview.groups;
 const CHARTS = {};
@@ -19,7 +21,7 @@ function queryCharts(charts) {
     charts[chartName] = chart = Object.assign({}, chart);
     chart.slices = Object.assign({}, chart.slices);
     for (let [sliceName, slice] of Object.entries(chart.slices)) {
-      const filter = funcCache.get(expression.parse, slice.filter);
+      const filter = memoizedParse(slice.filter);
       chart.slices[sliceName] = slice = Object.assign({}, slice);
       slice.count = store.count("devices", filter);
     }

client/store.js

@@ -4,13 +4,13 @@ import m from "mithril";
 import config from "./config";
 import * as expressionParser from "../common/expression-parser";
 import * as expression from "../common/expression";
-import * as funcCache from "../common/func-cache";
+import memoize from "../common/memoize";
+
+const memoizedStringify = memoize(expression.stringify);
+const memoizedEvaluate = memoize(expression.evaluate);
 
 let fulfillTimestamp = 0;
 
-let unpackExpressionCache = new WeakMap();
-let evaluateExpressionCache = new WeakMap();
-
 const queries = {
   filter: new WeakMap(),
   bookmark: new WeakMap(),
@@ -45,16 +45,12 @@ class QueryResponse {
 
 function unpackExpression(exp) {
   if (!Array.isArray(exp)) return exp;
-  let e = unpackExpressionCache.get(exp);
+  const e = memoizedEvaluate(exp, null, fulfillTimestamp);
-  if (e === undefined) {
-    e = expression.evaluate(e, null, fulfillTimestamp);
-    unpackExpressionCache.set(exp, e);
-  }
   return e;
 }
 
 function count(resourceType, filter) {
-  const filterStr = funcCache.get(expression.stringify, filter);
+  const filterStr = memoizedStringify(filter);
   let queryResponse = resources[resourceType].count.get(filterStr);
   if (queryResponse) return queryResponse;
 
@@ -195,7 +191,7 @@ function inferQuery(resourceType, queryResponse) {
 }
 
 function fetch(resourceType, filter, options = {}) {
-  const filterStr = funcCache.get(expression.stringify, filter);
+  const filterStr = memoizedStringify(filter);
   const sort = options.sort || {};
   const limit = options.limit || 0;
   if (resourceType === "devices")
@@ -219,9 +215,6 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
   let updated = false;
 
   if (_fulfillTimestamp > fulfillTimestamp) {
-    unpackExpressionCache = new WeakMap();
-    evaluateExpressionCache = new WeakMap();
-
     for (let resource of Object.values(resources))
       resource.combinedFilter = null;
     fulfillTimestamp = _fulfillTimestamp;
@@ -250,7 +243,7 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
               url:
                 `/api/${resourceType}/?` +
                 m.buildQueryString({
-                  filter: funcCache.get(expression.stringify, filter)
+                  filter: memoizedStringify(filter)
                 }),
               extract: xhr => +xhr.getResponseHeader("x-total-count"),
               background: true
@@ -295,7 +288,7 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
                 url:
                   `/api/${resourceType}/?` +
                   m.buildQueryString({
-                    filter: funcCache.get(expression.stringify, filter),
+                    filter: memoizedStringify(filter),
                     limit: 1,
                     skip: limit - 1,
                     sort: JSON.stringify(sort),
@@ -382,10 +375,7 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
                 url:
                   `/api/${resourceType}/?` +
                   m.buildQueryString({
-                    filter: funcCache.get(
+                    filter: memoizedStringify(combinedFilterDiff)
-                      expression.stringify,
-                      combinedFilterDiff
-                    )
                   })
               })
                 .then(res => {
@@ -488,16 +478,7 @@ function deleteResource(resourceType, id) {
 
 function evaluateExpression(exp, obj) {
   if (!Array.isArray(exp)) return exp;
-
+  return memoizedEvaluate(exp, obj, fulfillTimestamp);
-  let exps = evaluateExpressionCache.get(exp);
-  if (!exps) evaluateExpressionCache.set(exp, (exps = new WeakMap()));
-
-  if (!exps.has(obj)) {
-    let v = expression.evaluate(exp, obj, fulfillTimestamp);
-    exps.set(obj, v);
-  }
-
-  return exps.get(obj);
 }
 
 function logIn(username, password) {

common/authorizer.js

@@ -10,7 +10,7 @@ const validators = {
 class Authorizer {
   constructor(permissionSets) {
     this.permissionSets = permissionSets;
-    this.validtorCache = new WeakMap();
+    this.validatorCache = new WeakMap();
     this.hasAccessCache = {};
   }
 
@@ -34,8 +34,8 @@ class Authorizer {
   }
 
   getValidator(resourceType, resource) {
-    if (this.validtorCache.has(resource))
+    if (this.validatorCache.has(resource))
-      return this.validtorCache.get(resource);
+      return this.validatorCache.get(resource);
 
     let funcs = {};
 
@@ -70,7 +70,7 @@ class Authorizer {
       return valid;
     };
 
-    this.validtorCache.set(resource, validator);
+    this.validatorCache.set(resource, validator);
     return validator;
   }
 }

common/expression.js

@@ -36,7 +36,7 @@ function reduce(exp, callback) {
   return exp;
 }
 
-function evaluate(exp, obj, now) {
+function evaluate(exp, obj, now, cb) {
   function getRegExp(pat, esc) {
     const k = `${esc || ""}:${pat}`;
     let c = regExpCache.get(exp);
@@ -46,6 +46,7 @@ function evaluate(exp, obj, now) {
   }
 
   return expressionParser.map(exp, e => {
+    if (cb) e = cb(e);
     if (!isArray(e)) return e;
 
     if (e[0] === "FUNC") {

common/func-cache.js

@@ -1,29 +0,0 @@
-"use strict";
-
-let cache1 = new WeakMap();
-let cache2 = new WeakMap();
-
-function get(func, k) {
-  let c1 = cache1.get(func);
-  if (!c1) cache1.set(func, (c1 = new Map()));
-  if (c1.has(k)) return c1.get(k);
-  const c2 = cache2.get(func);
-  const v = c2 && c2.has(k) ? c2.get(k) : func(k);
-  c1.set(k, v);
-  return v;
-}
-
-function getter(func) {
-  return k => get(func, k);
-}
-
-function purge() {
-  cache2 = cache1;
-  cache1 = new WeakMap();
-}
-
-setInterval(purge, 120000);
-
-exports.get = get;
-exports.getter = getter;
-exports.purge = purge;

common/memoize.js

@@ -0,0 +1,47 @@
+"use strict";
+
+let cache1 = new Map();
+let cache2 = new Map();
+const keys = new WeakMap();
+
+function getKey(obj) {
+  if (obj === null) return "null";
+  else if (obj === undefined) return "undefined";
+
+  const t = typeof obj;
+  if (t === "number" || t === "boolean" || t === "string") return `${t}:${obj}`;
+  if (t !== "function" && t !== "object")
+    throw new Error(`Cannot memoize ${t} arguments`);
+
+  let k = keys.get(obj);
+  if (!k) {
+    const rnd = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER);
+    k = `${t}:${rnd.toString(36)}`;
+    keys.set(obj, k);
+  }
+  return k;
+}
+
+function memoize(func) {
+  return (...args) => {
+    const key = JSON.stringify(args.map(getKey)) + getKey(func);
+
+    if (cache1.has(key)) return cache1.get(key);
+
+    let r;
+    if (cache2.has(key)) r = cache2.get(key);
+    else r = func(...args);
+    cache1.set(key, r);
+    return r;
+  };
+}
+
+const interval = setInterval(() => {
+  cache2 = cache1;
+  cache1 = new Map();
+}, 120000);
+
+// Don't hold Node.js process
+if (interval.unref) interval.unref();
+
+module.exports = memoize;

config.json

@@ -95,7 +95,7 @@
       },
       {
         "label": "Last inform",
-        "parameter": "Events.Inform",
+        "parameter": "DATE_STRING(Events.Inform)",
         "type": "container",
         "element": "span.inform",
         "components": [
@@ -161,7 +161,7 @@
             "type": "container",
             "label": "Last inform",
             "element": "span.inform",
-            "parameter": "Events.Inform",
+            "parameter": "DATE_STRING(Events.Inform)",
             "components": [
               {
                 "type": "parameter"

server/api.js

@@ -3,7 +3,7 @@
 const zlib = require("zlib");
 const Router = require("koa-router");
 const PassThrough = require("stream").PassThrough;
-
+const config = require("./config");
 const db = require("./db");
 const apiFunctions = require("./api-functions");
 const expression = require("../common/expression");
@@ -102,6 +102,88 @@ for (let [resource, flags] of Object.entries(resources)) {
     stream.end("]");
   });
 
+  // CSV download
+  router.get(`/${resource}.csv`, async (ctx, next) => {
+    let options = {};
+    let filter = true;
+    if (ctx.request.query.filter)
+      filter = expression.parse(ctx.request.query.filter);
+    if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
+    if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
+    if (ctx.request.query.sort)
+      options.sort = JSON.parse(ctx.request.query.sort);
+
+    if (!ctx.state.authorizer.hasAccess(resource, 2)) return next();
+
+    const columns = JSON.parse(ctx.request.query.columns);
+    const now = Date.now();
+    for (let [k, v] of Object.entries(columns))
+      columns[k] = expression.evaluate(
+        expression.parse(v),
+        null,
+        now
+      );
+
+    // Exclude temporary tasks and faults
+    if (resource === "tasks" || resource === "faults")
+      filter = expression.and(filter, [
+        "NOT",
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+      ]);
+
+    let stream;
+    switch (ctx.acceptsEncodings("gzip", "deflate", "identity")) {
+      case "gzip":
+        stream = zlib.createGzip();
+        ctx.set("Content-Encoding", "gzip");
+        break;
+      case "deflate":
+        stream = zlib.createDeflate();
+        ctx.set("Content-Encoding", "deflate");
+        break;
+      default:
+        stream = new PassThrough();
+        break;
+    }
+
+    ctx.body = stream;
+    ctx.type = "text/csv";
+
+    stream.write(
+      Object.keys(columns).map(k => `"${k.replace(/"/, '""')}"`) + "\n"
+    );
+    await db.query(resource, filter, options, obj => {
+      let arr = Object.values(columns).map(exp => {
+        let v = expression.evaluate(exp, obj, null, e => {
+          if (Array.isArray(e))
+            if (e[0] === "PARAM") {
+              if (resource === "devices") {
+                if (e[1] === "Tags") {
+                  let tags = [];
+                  for (let p in obj)
+                    if (p.startsWith("Tags.")) tags.push(p.slice(5));
+
+                  return tags.join(", ");
+                }
+              }
+            } else if (e[0] === "FUNC") {
+              if (e[1] === "DATE_STRING")
+                if (e[2] && !Array.isArray(e[2]))
+                  return new Date(e[2]).toISOString();
+            }
+
+          return e;
+        });
+
+        if (Array.isArray(v) || v == null) return "";
+        if (typeof v === "string") return `"${v.replace(/"/g, '""')}"`;
+        return v;
+      });
+      stream.write(arr.join(",") + "\n");
+    });
+    stream.end();
+  });
+
   router.head(`/${resource}/:id`, async (ctx, next) => {
     let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
     if (!ctx.state.authorizer.hasAccess(resource, 2)) return next();

server/db.js

@@ -35,7 +35,7 @@ function getClient(db) {
 function query(resource, filter, options, callback) {
   options = options || {};
   let q;
-  filter = expression.evaluate(filter, null, null, Date.now());
+  filter = expression.evaluate(filter, null, Date.now());
 
   if (Array.isArray(filter)) {
     if (resource === "devices")
@@ -79,6 +79,10 @@ function query(resource, filter, options, callback) {
           if (err) return reject(err);
           if (resource === "devices")
             docs = docs.map(d => mongodbFunctions.flattenDevice(d));
+          else if (resource === "faults")
+            docs = docs.map(d => mongodbFunctions.flattenFault(d));
+          else if (resource === "tasks")
+            docs = docs.map(d => mongodbFunctions.flattenTask(d));
           return resolve(docs);
         });
       else
@@ -86,6 +90,10 @@ function query(resource, filter, options, callback) {
           doc => {
             if (resource === "devices")
               doc = mongodbFunctions.flattenDevice(doc);
+            else if (resource === "faults")
+              doc = mongodbFunctions.flattenFault(doc);
+            else if (resource === "tasks")
+              doc = mongodbFunctions.flattenTask(doc);
             callback(doc);
           },
           err => {
@@ -99,7 +107,7 @@ function query(resource, filter, options, callback) {
 
 function count(resource, filter) {
   let q;
-  filter = expression.evaluate(filter, null, null, Date.now());
+  filter = expression.evaluate(filter, null, Date.now());
 
   if (Array.isArray(filter)) {
     if (resource === "devices")

server/mongodb-functions.js

@@ -337,6 +337,25 @@ function flattenDevice(device) {
   return newDevice;
 }
 
+function flattenFault(fault) {
+  const f = Object.assign({}, fault);
+  if (f.timestamp) f.timestamp = +f.timestamp;
+  if (f.expiry) f.expiry = +f.expiry;
+  if (f.detail) {
+    for (let [k, v] of Object.entries(f.detail)) fault[`detail.${k}`] = v;
+    delete f.detail;
+  }
+  return f;
+}
+
+function flattenTask(task) {
+  let t = Object.assign({}, task);
+  t._id = "" + t._id;
+  if (t.timestamp) t.timestamp = +t.timestamp;
+  if (t.expiry) t.expiry = +t.expiry;
+  return t;
+}
+
 exports.processDeviceFilter = processDeviceFilter;
 exports.processTasksFilter = processTasksFilter;
 exports.processFaultsFilter = processFaultsFilter;
@@ -344,3 +363,5 @@ exports.filterToMongoQuery = filterToMongoQuery;
 exports.processDeviceProjection = processDeviceProjection;
 exports.processDeviceSort = processDeviceSort;
 exports.flattenDevice = flattenDevice;
+exports.flattenFault = flattenFault;
+exports.flattenTask = flattenTask;