Generalize store quering functions

client/app.js

@@ -3,41 +3,67 @@
 import m from "mithril";
 
 import layout from "./layout";
+import * as store from "./store";
 import * as overviewPage from "./overview-page";
 import * as devicesPage from "./devices-page";
+import * as devicePage from "./device-page";
 import * as errorPage from "./error-page";
 
 let state;
 
+let lastRenderTimestamp = 0;
+let pageVisitTimestamp = 0;
+let fulfillTimeout = null;
+
+function fulfill() {
+  clearTimeout(fulfillTimeout);
+  fulfillTimeout = setTimeout(() => {
+    store.fulfill(lastRenderTimestamp, pageVisitTimestamp).then(updated => {
+      if (updated) m.redraw();
+    });
+  }, 100);
+}
+
 function pagify(pageName, page) {
   const component = {
     render: () => {
-      if (state && state.error) return m(errorPage.component, state);
-      else return m(layout, { page: pageName }, m(page.component, state));
+      lastRenderTimestamp = Date.now();
+      let c;
+      if (state && state.error) c = m(errorPage.component, state);
+      else c = m(layout, { page: pageName }, m(page.component, state));
+      fulfill();
+      return c;
     }
   };
 
-  if (page.init)
-    component.onmatch = args => {
-      return new Promise(resolve => {
-        page
-          .init(args)
-          .then(st => {
-            if (!st) return m.route.set("/");
-            state = st;
-            resolve();
-          })
-          .catch(err => {
-            state = { error: err.message };
-            resolve();
-          });
-      });
-    };
+  component.onmatch = args => {
+    pageVisitTimestamp = Date.now();
+    if (!page.init) {
+      fulfill();
+      return;
+    }
+
+    return new Promise(resolve => {
+      page
+        .init(args)
+        .then(st => {
+          if (!st) return m.route.set("/");
+          state = st;
+          resolve();
+          fulfill();
+        })
+        .catch(err => {
+          state = { error: err.message };
+          resolve();
+        });
+    });
+  };
 
   return component;
 }
 
 m.route(document.body, "/overview", {
   "/overview": pagify("overview", overviewPage),
-  "/devices": pagify("devices", devicesPage)
+  "/devices": pagify("devices", devicesPage),
+  "/devices/:id": pagify("devices", devicePage)
 });

client/components.js

@@ -0,0 +1,16 @@
+"use strict";
+
+import parameter from "./components/parameter";
+import parameterList from "./components/parameter-list";
+import parameterTable from "./components/parameter-table";
+import overviewDot from "./components/overview-dot";
+
+function get(name) {
+  if (name === "parameter") return parameter;
+  else if (name === "parameter-list") return parameterList;
+  else if (name === "parameter-table") return parameterTable;
+  else if (name === "overview-dot") return overviewDot;
+  else throw new Error(`No such component '${name}'`);
+}
+
+export { get };

client/components/overview-dot.js

@@ -0,0 +1,37 @@
+"use strict";
+
+import m from "mithril";
+import * as config from "../config";
+import * as store from "../store";
+
+const CHARTS = config.get("ui.overview.charts");
+
+const component = {
+  view: vnode => {
+    const device = vnode.attrs.device;
+    const chart = CHARTS[vnode.attrs.chart];
+    for (let slice of Object.values(chart.slices)) {
+      const filter = store.unpackFilter(slice.filter);
+      if (filter.test(device)) {
+        const dot = m(
+          "svg",
+          {
+            width: "1em",
+            height: "1em",
+            xmlns: "http://www.w3.org/2000/svg",
+            "xmlns:xlink": "http://www.w3.org/1999/xlink"
+          },
+          m("circle", {
+            cx: "0.5em",
+            cy: "0.5em",
+            r: "0.4em",
+            fill: slice.color
+          })
+        );
+        return m("span.overview-dot", dot, `${slice.label}`);
+      }
+    }
+  }
+};
+
+export default component;

client/components/parameter-list.js

@@ -0,0 +1,27 @@
+"use strict";
+
+import m from "mithril";
+import * as components from "../components";
+
+const component = {
+  view: vnode => {
+    const device = vnode.attrs.device;
+    const parameters = vnode.attrs.parameters;
+    const filtered = Object.values(parameters).filter(
+      p => p.parameter in device
+    );
+    const rows = filtered.map(p =>
+      m(
+        "tr",
+        m("th", p.label),
+        m(
+          "td",
+          m(components.get("parameter"), Object.assign({ device: device }, p))
+        )
+      )
+    );
+    return m("table.parameter-list", rows);
+  }
+};
+
+export default component;

client/components/parameter-table.js

@@ -0,0 +1,42 @@
+"use strict";
+
+import m from "mithril";
+import * as components from "../components";
+
+const component = {
+  view: vnode => {
+    const device = vnode.attrs.device;
+    const object = vnode.attrs.object;
+    const parameters = vnode.attrs.parameters;
+
+    if (!device[object]) return null;
+
+    const instances = new Set();
+    for (let p in device)
+      if (p.startsWith(object))
+        instances.add(p.slice(0, p.indexOf(".", object.length + 1)));
+
+    const thead = m(
+      "thead",
+      m("tr", Object.values(parameters).map(p => m("th", p.label)))
+    );
+    const rows = [];
+    for (let i of instances) {
+      const row = Object.values(parameters).map(p =>
+        m(
+          "td",
+          m(
+            components.get("parameter"),
+            Object.assign({ device: device }, p, {
+              parameter: `${i}.${p.parameter}`
+            })
+          )
+        )
+      );
+      rows.push(m("tr", row));
+    }
+    return m("table.parameter-table", thead, m("tbody", rows));
+  }
+};
+
+export default component;

client/components/parameter.js

@@ -0,0 +1,30 @@
+"use strict";
+
+import m from "mithril";
+import * as components from "../components";
+
+const component = {
+  view: vnode => {
+    const param = vnode.attrs.device[vnode.attrs.parameter];
+    if (!param || !param.value) return m("span.na", "N/A");
+    let value = param.value[0];
+    if (param.value[1] === "xsd:dateTime")
+      value = new Date(value).toISOString();
+
+    let meta;
+    if (vnode.attrs.meta) {
+      const metaAttrs = Object.assign(
+        { device: vnode.attrs.device, parameter: vnode.attrs.parameter },
+        vnode.attrs.meta
+      );
+      meta = m("span", [
+        " (",
+        m(components.get(metaAttrs.type), metaAttrs),
+        ")"
+      ]);
+    }
+    return [m("span", { title: param.valueTimestamp }, value), meta];
+  }
+};
+
+export default component;

client/config.js

@@ -1,69 +1,40 @@
 "use strict";
 
-export default {
-  pageSize: 10,
-  filter: [
-    {
-      label: "Serial number",
-      parameter: "DeviceID.SerialNumber",
-      type: "string"
-    },
-    {
-      label: "Product class",
-      parameter: "DeviceID.ProductClass",
-      type: "string"
-    },
-    {
-      label: "Tag",
-      parameter: "tag",
-      type: "string"
-    }
-  ],
-  index: [
-    {
-      label: "Serial number",
-      parameter: "DeviceID.SerialNumber"
-    },
-    {
-      label: "Product class",
-      parameter: "DeviceID.ProductClass"
-    },
-    {
-      label: "Software version",
-      parameter: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
-    },
-    {
-      label: "MAC",
-      parameter:
-        "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress"
-    }
-  ],
-  overview: {
-    online: {
-      label: "Online devices",
-      charts: {
-        all: {
-          label: "All devices",
-          slices: {
-            onlineNow: {
-              label: "Online now",
-              color: "#31a354",
-              filter: "Events.Inform > NOW() - 5 * 60 * 1000"
-            },
-            past24: {
-              label: "Past 24 hours",
-              color: "#addd8e",
-              filter:
-                "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)"
-            },
-            others: {
-              label: "Others",
-              color: "#f7fcb9",
-              filter: "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)"
-            }
-          }
+import * as Filter from "../common/filter";
+
+const configFile = window.clientConfig;
+
+const cache = {};
+
+function parse(name, value) {
+  if (/^ui\.overview\.charts\.[^.]+\.slices\.[^.]+\.filter$/.test(name))
+    return new Filter(value);
+
+  return value;
+}
+
+function get(name) {
+  let v = cache[name];
+  if (v === undefined) {
+    v = configFile[name];
+    if (v != null) {
+      v = parse(name, v);
+    } else {
+      const prefix = `${name}.`;
+      for (let k of Object.keys(configFile))
+        if (k.startsWith(prefix)) {
+          v = v || {};
+          let ar = k.slice(prefix.length).split(".");
+          let ref = v;
+          for (let a of ar.slice(0, -1)) ref = ref[a] = ref[a] || {};
+          ref[ar[ar.length - 1]] = parse(k, configFile[k]);
         }
-      }
     }
+    if (v) cache[name] = v;
+    else cache[name] = null;
   }
-};
+
+  return v;
+}
+
+export { get };

client/css/app.css

@@ -244,6 +244,42 @@ button {
   }
 }
 
+table.parameter-list {
+  text-align: left;
+  & th {
+    padding-right: 1em;
+  }
+}
+
+table.parameter-table {
+  border-spacing: 0;
+  text-align: left;
+
+  & th {
+    color: var(--color4);
+    border-bottom: 2px solid var(--color4);
+    margin: 0;
+    padding: 0.2em 0.5em;
+  }
+
+  & td {
+    margin: 0;
+    padding: 0.2em 0.5em;
+  }
+}
+
+.overview-dot {
+  & > svg {
+    margin-bottom: -0.125em;
+    margin-left: 0.2em;
+    margin-right: 0.2em;
+    & > circle {
+      stroke: var(--color5);
+      stroke-width: 0.08em;
+    }
+  }
+}
+
 .pie-chart {
   margin: 20px;
   text-align: center;

client/device-page.js

@@ -0,0 +1,41 @@
+"use strict";
+
+import m from "mithril";
+
+import * as config from "./config";
+import Filter from "../common/filter";
+import * as store from "./store";
+import * as components from "./components";
+
+function init(args) {
+  return new Promise(resolve => {
+    resolve({
+      deviceId: args.id,
+      deviceFilter: new Filter(["=", "DeviceID.ID", args.id])
+    });
+  });
+}
+
+const component = {
+  view: vnode => {
+    document.title = `${vnode.attrs.deviceId} - Devices - GenieACS`;
+
+    let dev = store.fetch("devices", vnode.attrs.deviceFilter).value;
+    if (!dev.length) return "Loading";
+    const conf = config.get("ui.device");
+    const cmps = [];
+
+    for (let c of Object.values(conf))
+      cmps.push(
+        m(components.get(c["type"]), Object.assign({ device: dev[0] }, c))
+      );
+
+    return [
+      m("h1", vnode.attrs.deviceId),
+      m("span", dev[0]["InternetGatewayDevice.DeviceInfo.SerialNumber"]),
+      cmps
+    ];
+  }
+};
+
+export { init, component };

client/devices-page.js

@@ -2,78 +2,50 @@
 
 import m from "mithril";
 
-import config from "./config";
+import * as config from "./config";
 import filterComponent from "./filter-component";
 import * as filterParser from "../common/filter-parser";
-
-function load(filter, limit) {
-  return new Promise((resolve, reject) => {
-    m
-      .request({
-        url: "/api/devices/",
-        data: { filter: filter, limit: limit }
-      })
-      .then(res => {
-        resolve(res);
-      })
-      .catch(err => {
-        err.message = err.message || "Unknown error";
-        reject(err);
-      });
-  });
-}
-
-function count(filter, limit) {
-  return new Promise((resolve, reject) => {
-    function extract(xhr) {
-      return +xhr.getResponseHeader("x-total-count");
-    }
-    m
-      .request({
-        method: "HEAD",
-        url:
-          "/api/devices/?" +
-          m.buildQueryString({ filter: filter, limit: limit }),
-        extract: extract
-      })
-      .then(res => {
-        resolve(res);
-      })
-      .catch(err => {
-        err.message = err.message || "Unknown error";
-        reject(err);
-      });
-  });
-}
+import Filter from "../common/filter";
+import * as store from "./store";
+import * as components from "./components";
 
 function init(args) {
-  return new Promise((resolve, reject) => {
-    let filter;
-    if (args.filter) filter = filterParser.parse(args.filter);
-
-    Promise.all([count(args.filter), load(args.filter, config.pageSize)])
-      .then(res => {
-        resolve({ filter: filter, total: res[0], devices: res[1] });
-      })
-      .catch(reject);
+  return new Promise(resolve => {
+    let filter = new Filter(args.filter);
+    resolve({ filter: filter });
   });
 }
 
 function renderTable(devices, parameters, total, showMoreCallback) {
   let labels = [];
   for (let param of parameters) labels.push(m("th", param.label));
+  labels.push(m("th"));
 
   let rows = [];
   for (let device of devices)
     rows.push(
       m(
         "tr",
-        parameters.map(p => {
-          if (!device[p.parameter] || !device[p.parameter].value)
-            return m("td.na");
-          if (!device[p.parameter].value[0] === "") return m("td.blank");
-          return m("td", device[p.parameter].value[0]);
-        })
+        parameters
+          .map(p => {
+            const attrs = Object.assign({ device: device }, p);
+            const comp = m(components.get("parameter"), attrs);
+            return m("td", comp);
+          })
+          .concat(
+            m(
+              "td.table-row-links",
+              m(
+                "a",
+                {
+                  href: `#!/devices/${encodeURIComponent(
+                    device["DeviceID.ID"].value[0]
+                  )}`
+                },
+                "Show"
+              )
+            )
+          )
       )
     );
 
@@ -84,7 +56,7 @@ function renderTable(devices, parameters, total, showMoreCallback) {
 
   let tfoot = m(
     "tfoot",
-    m("tr", m("td", { colspan: labels.length }, footerElements))
+    m("tr", m("td", { colspan: labels.length + 1 }, footerElements))
   );
 
   return m("table.table", [
@@ -99,18 +71,8 @@ const component = {
     document.title = "Devices - GenieACS";
 
     function showMore() {
-      const lastDevice = vnode.attrs.devices[vnode.attrs.devices.length - 1];
-      let f = filterParser.and(vnode.attrs.filter, [
-        ">",
-        "DeviceID.ID",
-        lastDevice["DeviceID.ID"].value[0]
-      ]);
-
-      load(filterParser.stringify(f), config.pageSize).then(res => {
-        vnode.attrs.devices = vnode.attrs.devices.concat(res);
-        if (res.length < config.pageSize)
-          vnode.attrs.total = vnode.attrs.devices.length;
-      });
+      vnode.state.showCount = (vnode.state.showCount || 10) + 10;
+      m.redraw();
     }
 
     function onFilterChanged(filter) {
@@ -119,16 +81,23 @@ const component = {
       m.route.set("/devices", params);
     }
 
+    let devs = store.fetch(
+      "devices",
+      vnode.attrs.filter,
+      vnode.state.showCount || 10
+    );
+    let count = store.count("devices", vnode.attrs.filter);
+
     return [
       m("h1", "Listing devices"),
       m(filterComponent, {
-        filter: vnode.attrs.filter,
+        filter: vnode.attrs.filter ? vnode.attrs.filter.ast : null,
         onChange: onFilterChanged
       }),
       renderTable(
-        vnode.attrs.devices,
-        config.index,
-        vnode.attrs.total,
+        devs.value,
+        Object.values(config.get("ui.index")),
+        count.value,
         showMore
       )
     ];

client/filter-component.js

@@ -2,18 +2,21 @@
 
 import m from "mithril";
 
-import config from "./config";
+import * as config from "./config";
 import * as filterParser from "../common/filter-parser";
 
 const component = {
   view: vnode => {
     let a = m(
       "datalist#filters",
-      config.filter.map(f => m("option", { value: `${f.parameter} = ` }))
+      Object.values(config.get("ui.filters")).map(f =>
+        m("option", { value: `${f.parameter} = ` })
+      )
     );
 
-    if (!vnode.state.filterList) {
+    if (vnode.attrs.filter !== vnode.state.filter) {
       vnode.state.filterList = [];
+      vnode.state.filter = vnode.attrs.filter;
       if (vnode.attrs.filter)
         if (vnode.attrs.filter[0] === "AND")
           for (let i = 1; i < vnode.attrs.filter.length; ++i)
@@ -32,16 +35,18 @@ const component = {
       vnode.state.filterList = vnode.state.filterList.filter(
         f => f && f.trim()
       );
-      let filter = vnode.state.filterList.map((f, idx) => {
+      let filterAst = vnode.state.filterList.map((f, idx) => {
         try {
           return filterParser.parse(f);
         } catch (err) {
           vnode.state.filterInvalid |= 1 << idx;
         }
       });
+      if (filterAst.length) filterAst = ["AND"].concat(filterAst);
+      else filterAst = null;
+
       vnode.state.filterList.push("");
-      if (!vnode.state.filterInvalid)
-        vnode.attrs.onChange(["AND"].concat(filter));
+      if (!vnode.state.filterInvalid) vnode.attrs.onChange(filterAst);
     }
 
     return m(

client/overview-page.js

@@ -1,93 +1,44 @@
 "use strict";
 
 import m from "mithril";
-import config from "./config";
+import * as config from "./config";
+import * as store from "./store";
 import pieChartComponent from "./pie-chart-component";
-import * as filterParser from "../common/filter-parser";
 
-function count(filter, limit) {
-  return new Promise((resolve, reject) => {
-    function extract(xhr) {
-      return +xhr.getResponseHeader("x-total-count");
-    }
-    m
-      .request({
-        method: "HEAD",
-        url:
-          "/api/devices/?" +
-          m.buildQueryString({ filter: filter, limit: limit }),
-        extract: extract
-      })
-      .then(res => {
-        resolve(res);
-      })
-      .catch(err => {
-        err.message = err.message || "Unknown error";
-        reject(err);
-      });
-  });
-}
+const GROUPS = config.get("ui.overview.groups");
+const CHARTS = {};
+for (let group of Object.values(GROUPS))
+  for (let chartName of Object.values(group["charts"]))
+    CHARTS[chartName] = config.get("ui.overview.charts")[chartName];
 
-function queryCharts(groups) {
-  const now = Date.now();
-  function evalNow(exp) {
-    if (exp[0] == "FUNC" && exp[1] === "NOW" && exp.length == 2) return now;
-  }
-
-  let calls = [];
-  groups = Object.assign({}, groups);
-  for (let [groupName, group] of Object.entries(groups)) {
-    groups[groupName] = group = Object.assign({}, group);
-    for (let [chartName, chart] of Object.entries(group.charts)) {
-      group.charts[chartName] = chart = Object.assign({}, chart);
-      for (let [sliceName, slice] of Object.entries(chart.slices)) {
-        chart.slices[sliceName] = slice = Object.assign({}, slice);
-        calls.push(
-          new Promise((resolve, reject) => {
-            let f = filterParser.map(filterParser.parse(slice.filter), evalNow);
-            count(filterParser.stringify(f))
-              .then(c => {
-                slice.count = c;
-                resolve();
-              })
-              .catch(reject);
-          })
-        );
-      }
+function queryCharts(charts) {
+  charts = Object.assign({}, charts);
+  for (let [chartName, chart] of Object.entries(charts)) {
+    charts[chartName] = chart = Object.assign({}, chart);
+    for (let [sliceName, slice] of Object.entries(chart.slices)) {
+      chart.slices[sliceName] = slice = Object.assign({}, slice);
+      slice.count = store.count("devices", slice.filter);
     }
   }
-
-  return new Promise((resolve, reject) => {
-    Promise.all(calls)
-      .then(() => {
-        resolve(groups);
-      })
-      .catch(reject);
-  });
+  return charts;
 }
 
 const init = function() {
-  return new Promise((resolve, reject) => {
-    queryCharts(config.overview)
-      .then(charts => {
-        resolve({ charts: charts });
-      })
-      .catch(err => {
-        err.message = err.message || "Unknown error";
-        reject(err);
-      });
+  return new Promise(resolve => {
+    resolve({ charts: queryCharts(CHARTS) });
   });
 };
 
 const component = {
-  view: vnode => {
+  view: () => {
     document.title = "Overview - GenieACS";
     let children = [];
-    for (let group of Object.values(vnode.attrs.charts)) {
+    for (let group of Object.values(GROUPS)) {
       if (group.label) children.push(m("h1", group.label));
 
       let groupChildren = [];
-      for (let chart of Object.values(group.charts)) {
+      for (let chartName of Object.values(group.charts)) {
+        const chart = CHARTS[chartName];
         let chartChildren = [];
         if (chart.label) chartChildren.push(m("h2", chart.label));
 

client/pie-chart-component.js

@@ -5,7 +5,7 @@ import m from "mithril";
 const drawChart = function(chartData) {
   let slices = chartData.slices;
   let total = Array.from(Object.values(chartData.slices)).reduce(
-    (a, s) => a + s.count,
+    (a, s) => a + (s.count.value || 0),
     0
   );
   let legend = [];
@@ -17,7 +17,7 @@ const drawChart = function(chartData) {
   let endX, endY;
 
   for (let slice of Object.values(slices)) {
-    let percent = total > 0 ? slice.count / total : 0;
+    let percent = total > 0 ? (slice.count.value || 0) / total : 0;
     legend.push(
       m(".legend-line", [
         m("span.color", {
@@ -28,10 +28,10 @@ const drawChart = function(chartData) {
           "a",
           {
             href: `/#!/devices/?${m.buildQueryString({
-              filter: slice.filter
+              filter: slice.filter.toString()
             })}`
           },
-          slice.count
+          slice.count.value || 0
         ),
         ` (${(percent * 100).toFixed(2)}%)`
       ])
@@ -68,7 +68,7 @@ const drawChart = function(chartData) {
           "a",
           {
             href: `/#!/devices/?${m.buildQueryString({
-              filter: slice.filter
+              filter: slice.filter.toString()
             })}`
           },
           [

client/store.js

@@ -0,0 +1,307 @@
+"use strict";
+
+import m from "mithril";
+import Filter from "../common/filter";
+import * as config from "./config";
+
+let fulfillTimestamp = 0;
+
+let unpackedFilters = new WeakMap();
+
+const queries = {
+  filter: new WeakMap(),
+  last: new WeakMap(),
+  limit: new WeakMap(),
+  fulfilled: new WeakMap(),
+  fulfilling: new WeakSet(),
+  mapper: new WeakMap(),
+  accessed: new WeakMap(),
+  value: new WeakMap()
+};
+
+const resources = {};
+for (let r of ["devices"])
+  resources[r] = {
+    objects: new Map(),
+    count: new Map(),
+    fetch: new Map(),
+    combinedFilter: null
+  };
+
+class QueryResponse {
+  get fulfilled() {
+    queries.accessed.set(this, Date.now());
+    return !!queries.fulfilled.get(this);
+  }
+
+  get value() {
+    queries.accessed.set(this, Date.now());
+    const mapper = queries.mapper.get(this);
+    const value = queries.value.get(this);
+
+    if (mapper) return mapper(queries.value.get(this));
+    else return value;
+  }
+}
+
+function unpackFilter(filter) {
+  let f = unpackedFilters.get(filter);
+  if (!f) {
+    f = f.evaluateExpressions(fulfillTimestamp);
+    unpackedFilters.set(filter, f);
+  }
+  return f;
+}
+
+function count(resourceType, filter) {
+  let queryResponse = resources[resourceType].count.get(filter.toString());
+  if (queryResponse) return queryResponse;
+
+  queryResponse = new QueryResponse();
+
+  resources[resourceType].count.set(filter.toString(), queryResponse);
+  queries.filter.set(queryResponse, filter);
+  return queryResponse;
+}
+
+function limitFilter(resourceType, filter, last) {
+  if (resourceType === "devices")
+    return filter.and(new Filter(["<=", "DeviceID.ID", last]));
+  else return filter.and(new Filter(["<=", "_id", last]));
+}
+
+function findMatches(resourceType, filter, limit) {
+  let value = [];
+  for (let [id, obj] of resources[resourceType].objects.entries())
+    if (filter.test(obj)) value.push(id);
+
+  value = value.sort();
+  if (limit) value = value.slice(0, limit);
+
+  return value;
+}
+
+function inferQuery(resourceType, queryResponse) {
+  const limit = queries.limit.get(queryResponse);
+  let filter = queries.filter.get(queryResponse);
+  filter = unpackFilter(filter);
+  let last = queries.last.get(queryResponse);
+  if (last || !limit) {
+    if (last) filter = limitFilter(resourceType, filter, last);
+    if (
+      resources[resourceType].combinedFilter &&
+      filter.subset(resources[resourceType].combinedFilter)
+    )
+      queries.fulfilled.set(queryResponse, fulfillTimestamp);
+  }
+
+  queries.value.set(queryResponse, findMatches(resourceType, filter, limit));
+}
+
+function fetch(resourceType, filter, limit = 0) {
+  const key = `${limit}:${filter.toString()}`;
+  let queryResponse = resources[resourceType].fetch.get(key);
+  if (queryResponse) return queryResponse;
+
+  queryResponse = new QueryResponse();
+  resources[resourceType].fetch.set(key, queryResponse);
+  queries.filter.set(queryResponse, filter);
+  queries.limit.set(queryResponse, limit);
+  queries.mapper.set(queryResponse, list =>
+    list.map(x => resources[resourceType].objects.get(x))
+  );
+  inferQuery(resourceType, queryResponse);
+  return queryResponse;
+}
+
+function fulfill(accessTimestamp, _fulfillTimestamp) {
+  let updated = false;
+
+  if (_fulfillTimestamp > fulfillTimestamp) {
+    unpackedFilters = new WeakMap();
+    for (let resource of Object.values(resources))
+      resource.combinedFilter = null;
+    fulfillTimestamp = _fulfillTimestamp;
+  }
+
+  const allPromises = [];
+
+  for (let [resourceType, resource] of Object.entries(resources))
+    for (let queryResponse of resource.count.values()) {
+      if (queries.fulfilling.has(queryResponse)) continue;
+      if (
+        queries.accessed.get(queryResponse) >= accessTimestamp &&
+        !(fulfillTimestamp <= queries.fulfilled.get(queryResponse))
+      ) {
+        queries.fulfilling.add(queryResponse);
+        allPromises.push(
+          new Promise((resolve, reject) => {
+            updated = true;
+            let filter = queries.filter.get(queryResponse);
+            filter = unpackFilter(filter);
+            m
+              .request({
+                method: "HEAD",
+                url:
+                  `/api/${resourceType}/?` +
+                  m.buildQueryString({
+                    filter: filter.toString()
+                  }),
+                extract: xhr => +xhr.getResponseHeader("x-total-count"),
+                background: true
+              })
+              .then(c => {
+                queries.value.set(queryResponse, c);
+                queries.fulfilled.set(queryResponse, fulfillTimestamp);
+                queries.fulfilling.delete(queryResponse);
+                resolve();
+              })
+              .catch(err => reject(err));
+          })
+        );
+      }
+    }
+
+  let toFetchAll = {};
+
+  for (let [resourceType, resource] of Object.entries(resources))
+    for (let queryResponse of resource.fetch.values()) {
+      if (queries.fulfilling.has(queryResponse)) continue;
+
+      if (
+        queries.accessed.get(queryResponse) >= accessTimestamp &&
+        !(fulfillTimestamp <= queries.fulfilled.get(queryResponse))
+      ) {
+        queries.fulfilling.add(queryResponse);
+        toFetchAll[resourceType] = toFetchAll[resourceType] || [];
+        toFetchAll[resourceType].push(queryResponse);
+        const limit = queries.limit.get(queryResponse);
+        if (limit)
+          allPromises.push(
+            new Promise((resolve, reject) => {
+              updated = true;
+              let filter = queries.filter.get(queryResponse);
+              filter = unpackFilter(filter);
+              m
+                .request({
+                  method: "GET",
+                  url:
+                    `/api/${resourceType}/?` +
+                    m.buildQueryString({
+                      filter: filter.toString(),
+                      limit: 1,
+                      skip: limit - 1,
+                      projection: "_id"
+                    })
+                })
+                .then(res => {
+                  if (res.length)
+                    if (resourceType === "devices")
+                      queries.last.set(
+                        queryResponse,
+                        res[0]["DeviceID.ID"].value[0]
+                      );
+                    else queries.last.set(queryResponse, res[0]["_id"]);
+                  else queries.last.delete(queryResponse);
+                  resolve();
+                })
+                .catch(reject);
+            })
+          );
+      }
+    }
+
+  return new Promise((resolve, reject) => {
+    Promise.all(allPromises)
+      .then(() => {
+        const allPromises2 = [];
+        for (let [resourceType, toFetch] of Object.entries(toFetchAll)) {
+          let combinedFilter = null;
+
+          toFetch = toFetch.filter(queryResponse => {
+            let filter = queries.filter.get(queryResponse);
+            filter = unpackFilter(filter);
+            let last = queries.last.get(queryResponse);
+            if (last) filter = limitFilter(resourceType, filter, last);
+
+            if (
+              resources[resourceType].combinedFilter &&
+              filter.subset(resources[resourceType].combinedFilter)
+            ) {
+              queries.value.set(
+                queryResponse,
+                findMatches(resourceType, filter)
+              );
+              queries.fulfilled.set(queryResponse, fulfillTimestamp);
+              queries.fulfilling.delete(queryResponse);
+              return false;
+            }
+
+            combinedFilter = filter.or(combinedFilter);
+            return true;
+          });
+
+          if (!combinedFilter) return resolve(updated);
+
+          updated = true;
+          let combinedFilterDiff = combinedFilter;
+          if (resources[resourceType].combinedFilter)
+            combinedFilterDiff = combinedFilterDiff.and(
+              resources[resourceType].combinedFilter.not()
+            );
+          resources[resourceType].combinedFilter = combinedFilter.or(
+            resources[resourceType].combinedFilter
+          );
+
+          allPromises2.push(
+            new Promise((resolve2, reject2) => {
+              m
+                .request({
+                  method: "GET",
+                  url:
+                    `/api/${resourceType}/?` +
+                    m.buildQueryString({
+                      filter: combinedFilterDiff.toString()
+                    })
+                })
+                .then(res => {
+                  for (let r of res)
+                    if (resourceType === "devices")
+                      resources.devices.objects.set(
+                        r["DeviceID.ID"].value[0],
+                        r
+                      );
+                    else resources[resourceType].objects.set(r["_id"], r);
+
+                  for (let queryResponse of toFetch) {
+                    let filter = queries.filter.get(queryResponse);
+                    filter = unpackFilter(filter);
+                    let last = queries.last.get(queryResponse);
+                    if (last) filter = limitFilter(resourceType, filter, last);
+
+                    queries.value.set(
+                      queryResponse,
+                      findMatches(resourceType, filter)
+                    );
+                    queries.fulfilled.set(queryResponse, fulfillTimestamp);
+                    queries.fulfilling.delete(queryResponse);
+                  }
+                  resolve2();
+                })
+                .catch(reject2);
+            })
+          );
+        }
+        Promise.all(allPromises2)
+          .then(() => resolve(updated))
+          .catch(reject);
+      })
+      .catch(reject);
+  });
+}
+
+function getTimestamp() {
+  return fulfillTimestamp;
+}
+
+export { count, fetch, fulfill, unpackFilter, getTimestamp };

common/filter-cnf.js

@@ -0,0 +1,138 @@
+"use strict";
+
+function collectRanges(filter, ranges = new Map()) {
+  if (["AND", "OR", "NOT"].includes(filter[0])) {
+    for (let i = 1; i < filter.length; ++i) collectRanges(filter[i], ranges);
+  } else if ([">", ">=", "<", "<=", "=", "<>"].includes(filter[0])) {
+    const key = `${filter[1]}:${typeof filter[2]}`;
+    let s = ranges.get(key);
+    if (!s) ranges.set(key, (s = new Set()));
+    s.add(filter[2]);
+  }
+  return ranges;
+}
+
+function or(cnf1, cnf2) {
+  if (cnf1.length === 0) return cnf2;
+  else if (cnf2.length === 0) return cnf1;
+
+  const res = [];
+  for (let i = 0; i < cnf1.length; ++i)
+    for (let j = i; j < cnf2.length; ++j) res.push(cnf1[i].concat(cnf2[j]));
+
+  return res;
+}
+
+function booleanCnf(filter) {
+  const ranges = collectRanges(filter);
+  for (let [k, v] of ranges)
+    if (k.endsWith(":number"))
+      ranges.set(k, Array.from(v).sort((a, b) => a - b));
+    else ranges.set(k, Array.from(v).sort());
+
+  const expressions = new Map();
+  const mutuallyExclusive = new Map();
+
+  function getIsNullVariable(param) {
+    const key = `${param}:null`;
+    let f = expressions.get(key);
+    if (!f) {
+      expressions.set(key, (f = expressions.size + 1));
+      let m = mutuallyExclusive.get(param);
+      if (!m) mutuallyExclusive.set(param, (m = new Set()));
+      m.add(f);
+    }
+    return f;
+  }
+
+  function getComparisonVariables(param, from, fromInclusive, to, toInclusive) {
+    const type = from != null ? typeof from : typeof to;
+    const range = ranges.get(`${param}:${type}`);
+    const vars = [];
+
+    if (from == null) vars.push(`${param}:--${range[0]}`);
+    if (to == null) vars.push(`${param}:${range[range.length - 1]}--`);
+    if (fromInclusive) vars.push(`${param}:${from}`);
+    if (toInclusive) vars.push(`${param}:${to}`);
+
+    for (let i = 1; i < range.length; ++i)
+      if ((from == null || range[i] > from) && (to == null || range[i] <= to)) {
+        vars.push(`${param}:${range[i - 1]}`);
+        vars.push(`${param}:${range[i]}`);
+        vars.push(`${param}:${range[i - 1]}--${range[i]}`);
+      }
+
+    return vars.map(key => {
+      let f = expressions.get(key);
+      if (!f) {
+        expressions.set(key, (f = expressions.size + 1));
+        let m = mutuallyExclusive.get(param);
+        if (!m) mutuallyExclusive.set(param, (m = new Set()));
+        m.add(f);
+      }
+      return f;
+    });
+  }
+
+  const comparisonOperators = {
+    "=": v => [v, true, v, true],
+    ">": v => [v, false, null, null],
+    ">=": v => [v, true, null, null],
+    "<": v => [null, null, v, false],
+    "<=": v => [null, null, v, true]
+  };
+
+  function recursive(clause, negate) {
+    const op = clause[0];
+
+    if ((op === "AND" && !negate) || (op === "OR" && negate)) {
+      let res = [];
+      for (let i = 1; i < clause.length; ++i)
+        res = res.concat(recursive(clause[i], negate));
+      return res;
+    } else if ((op === "OR" && !negate) || (op === "AND" && negate)) {
+      let res = [];
+      for (let i = 1; i < clause.length; ++i)
+        res = or(res, recursive(clause[i], negate));
+      return res;
+    } else if (op === "NOT") {
+      return recursive(clause[1], !negate);
+    } else if (op in comparisonOperators) {
+      const vars = getComparisonVariables(
+        clause[1],
+        ...comparisonOperators[op](clause[2])
+      );
+      if (negate) return vars.map(x => [0 - x]);
+      else return [vars];
+    } else if (op === "<>") {
+      const vars = getComparisonVariables(
+        clause[1],
+        ...comparisonOperators["="](clause[2])
+      );
+      if (!negate) return vars.map(x => [0 - x]);
+      else return [vars];
+    } else if (op === "IS NULL") {
+      const v = getIsNullVariable(clause[1]);
+      if (negate) return [[0 - v]];
+      else return [[v]];
+    } else if (op === "IS NOT NULL") {
+      const v = getIsNullVariable(clause[1]);
+      if (!negate) return [[0 - v]];
+      else return [[v]];
+    } else {
+      throw new Error(`Unknown operator ${op}`);
+    }
+  }
+
+  const cnf = recursive(filter, false);
+
+  for (let m of mutuallyExclusive.values()) {
+    let ar = Array.from(m);
+    for (let i = 0; i < ar.length; ++i)
+      for (let j = i + 1; j < ar.length; ++j) cnf.push([0 - ar[i], 0 - ar[j]]);
+  }
+
+  return { vars: expressions.size, clauses: cnf };
+}
+
+exports.booleanCnf = booleanCnf;

common/filter-parser.js

@@ -205,10 +205,6 @@ function parseExpression(exp) {
   return lang.ValueExpression.tryParse(exp);
 }
 
-function parseParameter(p) {
-  return lang.Parameter.tryParse(p);
-}
-
 function stringify(filter) {
   function value(v) {
     if (Array.isArray(v))
@@ -271,54 +267,13 @@ function map(filter, callback) {
       }
     }
 
-  return callback(clone || filter) || clone || filter;
-}
+  let r = callback(clone || filter);
+  if (r != null) return r;
 
-function and(filter1, filter2) {
-  if (!filter1) return filter2;
-  else if (!filter2) return filter1;
-
-  let res = ["AND"];
-
-  if (filter1[0] === "AND") res = res.concat(filter1.slice(1));
-  else res.push(filter1);
-
-  if (filter2[0] === "AND") res = res.concat(filter2.slice(1));
-  else res.push(filter2);
-
-  return res;
-}
-
-function evaluateExpressions(filter, now = Date.now()) {
-  return map(filter, exp => {
-    if (exp[0] === "FUNC" && exp[1] === "NOW") {
-      return now;
-    } else if (exp[0] === "*") {
-      let v = exp[1];
-      for (let i = 2; i < exp.length; ++i) v *= exp[i];
-      return v;
-    } else if (exp[0] === "/") {
-      let v = exp[1];
-      for (let i = 2; i < exp.length; ++i) v /= exp[i];
-      return v;
-    } else if (exp[0] === "+") {
-      let v = exp[1];
-      for (let i = 2; i < exp.length; ++i) v += exp[i];
-      return v;
-    } else if (exp[0] === "-") {
-      let v = exp[1];
-      for (let i = 2; i < exp.length; ++i) v -= exp[i];
-      return v;
-    } else if (exp[0] === "||") {
-      return exp.slice(1).join("");
-    }
-  });
+  return clone || filter;
 }
 
 exports.parse = parse;
 exports.parseExpression = parseExpression;
-exports.parseParameter = parseParameter;
 exports.stringify = stringify;
 exports.map = map;
-exports.and = and;
-exports.evaluateExpressions = evaluateExpressions;

common/filter.js

@@ -0,0 +1,157 @@
+"use strict";
+
+const filterParser = require("./filter-parser");
+const filterCnf = require("./filter-cnf");
+const satSolver = require("./sat-solver");
+
+function* permute(arr) {
+  if (arr.length <= 1) {
+    for (let i = 0; i < arr[0]; ++i) yield [i];
+    return;
+  }
+
+  let slc = arr.slice(1);
+  for (let i = 0; i < arr[0]; ++i)
+    for (let innerArr of permute(slc)) yield [i].concat(innerArr);
+}
+
+class Filter {
+  constructor(str) {
+    if (Array.isArray(str)) this.ast = str;
+    else if (str instanceof Filter) this.ast = str.ast;
+    else if (!str) this.ast = null;
+    else this.ast = filterParser.parse(str);
+  }
+
+  toString() {
+    if (!this.string)
+      if (this.ast) this.string = filterParser.stringify(this.ast);
+      else this.string = "";
+
+    return this.string;
+  }
+
+  or(fltr) {
+    if (!fltr || !this.ast) return this;
+
+    let f = ["OR"];
+
+    if (this.ast[0] === "OR") f = f.concat(this.ast.slice(1));
+    else f.push(this.ast);
+
+    if (fltr.ast[0] === "OR") f = f.concat(fltr.ast.slice(1));
+    else f.push(fltr.ast);
+
+    return new Filter(f);
+  }
+
+  and(fltr) {
+    if (!fltr) return this;
+    else if (!this.ast) return fltr;
+
+    let f = ["AND"];
+
+    if (this.ast[0] === "AND") f = f.concat(this.ast.slice(1));
+    else f.push(this.ast);
+
+    if (fltr.ast[0] === "AND") f = f.concat(fltr.ast.slice(1));
+    else f.push(fltr.ast);
+
+    return new Filter(f);
+  }
+
+  not() {
+    if (this.ast[0] === "NOT") return new Filter(this.ast[1]);
+    else return new Filter(["NOT", this.ast]);
+  }
+
+  evaluateExpressions(now = Date.now()) {
+    if (!this.ast) return this;
+    let ast = filterParser.map(this.ast, exp => {
+      if (exp[0] === "FUNC" && exp[1] === "NOW") {
+        return now;
+      } else if (exp[0] === "*") {
+        let v = exp[1];
+        for (let i = 2; i < exp.length; ++i) v *= exp[i];
+        return v;
+      } else if (exp[0] === "/") {
+        let v = exp[1];
+        for (let i = 2; i < exp.length; ++i) v /= exp[i];
+        return v;
+      } else if (exp[0] === "+") {
+        let v = exp[1];
+        for (let i = 2; i < exp.length; ++i) v += exp[i];
+        return v;
+      } else if (exp[0] === "-") {
+        let v = exp[1];
+        for (let i = 2; i < exp.length; ++i) v -= exp[i];
+        return v;
+      } else if (exp[0] === "||") {
+        return exp.slice(1).join("");
+      }
+    });
+
+    if (ast === this.ast) return this;
+    else return new Filter(ast);
+  }
+
+  subset(fltr) {
+    if (!fltr.ast) return true;
+    else if (!this.ast) return false;
+
+    const f = this.not()
+      .or(fltr)
+      .not();
+    const { vars, clauses } = filterCnf.booleanCnf(f.ast);
+    return !satSolver.naiveDpll(clauses, vars);
+  }
+
+  test(obj) {
+    if (!this.ast) return true;
+    return filterParser.map(this.ast, exp => {
+      const op = exp[0];
+
+      if (op === "AND") {
+        for (let i = 1; i < exp.length; ++i) if (!exp[i]) return false;
+        return true;
+      } else if (op === "OR") {
+        for (let i = 1; i < exp.length; ++i) if (exp[i]) return true;
+        return false;
+      } else if (op === "NOT") {
+        return !exp[1];
+      } else if (op === "=") {
+        const val =
+          typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
+        return val === exp[2];
+      } else if (op === "<>") {
+        const val =
+          typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
+        return val !== exp[2];
+      } else if (op === ">=") {
+        const val =
+          typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
+        return typeof val === typeof exp[2] && val >= exp[2];
+      } else if (op === ">") {
+        const val =
+          typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
+        return typeof val === typeof exp[2] && val > exp[2];
+      } else if (op === "<=") {
+        const val =
+          typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
+        return typeof val === typeof exp[2] && val <= exp[2];
+      } else if (op === "<") {
+        const val =
+          typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
+        return typeof val === typeof exp[2] && val < exp[2];
+      } else if (op === "IS NULL") {
+        return obj[exp[1]] == null;
+      } else if (op === "IS NOT NULL") {
+        return obj[exp[1]] != null;
+      } else {
+        throw new Error(`Unsupported operator ${op}`);
+      }
+    });
+  }
+}
+
+module.exports = Filter;

common/sat-solver.js

@@ -0,0 +1,175 @@
+/**
+ * Naive DPLL solver for propositional logic adapted from
+ * https://github.com/tammet/logictools
+ *
+ * Origianl code by Tanel Tammet released under the MIT license.
+ *
+ * Check whether a clause set is satisfiable or contradictory, using a naive
+ * implementation of the dpll (Davis-Putnam-Logemann-Loveland) procedure
+ * combining truth table search with the resolution-like unit propagation.
+ *
+ * The standard pure-literal rule is not implemented in this naive version,
+ * since it is not really crucial for the algorithm.
+ *
+ * The code is very similar to the pure table solver, except we use a
+ * unit-propagating, derived-varlist-returing satisfiableAt.
+ */
+"use strict";
+
+function satisfiableAt(clauses, varVals, varnr, val) {
+  if (varnr !== 0) varVals[varnr] = val;
+
+  const propRes = unitPropagate(clauses, varVals);
+  if (propRes === 1) {
+    varVals[varnr] = 0;
+    return true;
+  }
+
+  // if result false (-1) unitPropagate restores the varVals state itself
+  if (propRes === -1) {
+    varVals[varnr] = 0;
+    return false;
+  }
+
+  // find next unassigned var to split
+  let nextVar = 0;
+  for (let i = 1; i < varVals.length; i++)
+    if (varVals[i] === 0) {
+      nextVar = i;
+      break;
+    }
+
+  if (nextVar === 0) throw new Error("Error in satisfiableAt");
+
+  if (
+    satisfiableAt(clauses, varVals, nextVar, 1) ||
+    satisfiableAt(clauses, varVals, nextVar, -1)
+  )
+    // not necessary to restore split var: search is halted anyway
+    return true;
+
+  // restore split var
+  varVals[nextVar] = 0;
+
+  // restore derived vars
+  for (let i = 0; i < propRes.length; i++) varVals[propRes[i]] = 0;
+
+  return false;
+}
+
+/**
+ * Check whether clauses are satisfiable under the varVals assignment, without
+ * search.
+ *
+ * Iterate through all clauses and check whether
+ *
+ *  - all literals in the clause are assigned false: undo derived units (set
+ *    values in varVals to 0) and return false immediately
+ *
+ *  - a clause contains a literal assigned true: skip further actions on this
+ *    clause
+ *
+ *  - a single literal in the clause is unassigned, all the rest are assigned
+ *    false: derive this literal as a unit clause.
+ *
+ * In case at least one literal is derived during the process above, iterate
+ * the whole process until no more new literals are derived.
+ *
+ * Returns:
+ *
+ *   1: all clauses were assigned true, hence the clause set is satisfiable.
+ *   The whole search will terminate after that.
+ *
+ *   -1: clause set is contradictory. Derived units have been restored.
+ *
+ *   a list of derived variables:  the status of the clause set is
+ *   undetermined. The values of variables on the list will be restored (set to
+ *   0) during backtracking.
+ */
+function unitPropagate(clauses, varVals) {
+  // all vars with a value derived in the forthcoming main loop
+  const derivedVars = [];
+
+  // the main loop is run, deriving new unit clauses, until either:
+  // - the clause set is detected to be unsatisfiable
+  // - the clause set is detected to be satisfiable
+  // - there are no more derived unit clauses
+
+  for (;;) {
+    let allClausesFound = true; // set to false if we find an undetermined clause
+    const derivedQueue = []; // derived units (literals) during one iteration
+    for (let i = 0; i < clauses.length; i++) {
+      const clause = clauses[i];
+      let clauseVal = 0;
+      let unassignedCount = 0; // count unassigned vars in clause
+      let unassignedLit = 0; // 0 means none found for this clause
+
+      for (let j = 0; j < clause.length; j++) {
+        let lit = clause[j];
+        let nr, polarity;
+        if (lit < 0) {
+          nr = 0 - lit;
+          polarity = -1;
+        } else {
+          nr = lit;
+          polarity = 1;
+        }
+
+        if (varVals[nr] === polarity) {
+          clauseVal = 1;
+          break;
+        } else if (varVals[nr] === 0) {
+          unassignedCount++;
+          unassignedLit = lit;
+        }
+      }
+
+      if (clauseVal === 1) continue; // clause is subsumed
+
+      // clause is not subsumed by varVals
+      if (unassignedCount === 0) {
+        for (let j = 0; j < derivedVars.length; j++)
+          varVals[derivedVars[j]] = 0; // restore derived
+        return -1;
+      } else if (unassignedCount === 1) {
+        // unassignedLit is a derived literal
+        derivedQueue.push(unassignedLit);
+      }
+
+      if (unassignedCount !== 0) allClausesFound = false;
+    }
+
+    // if all clauses were subsumed or all the variables in a clause had a value
+    // no need to restore varVals: whole search finished
+    if (allClausesFound) return 1;
+
+    // in case there are any derived units present, store to varVals and
+    // iterate the main loop
+    if (derivedQueue.length === 0) break;
+
+    // some units were derived: iterate the main loop
+    for (let j = 0; j < derivedQueue.length; j++) {
+      let lit = derivedQueue[j];
+      let nr, polarity;
+      if (lit < 0) {
+        nr = 0 - lit;
+        polarity = -1;
+      } else {
+        nr = lit;
+        polarity = 1;
+      }
+      varVals[nr] = polarity;
+      derivedVars.push(nr);
+    }
+  }
+
+  return derivedVars;
+}
+
+function naiveDpll(clauses, maxvarnr) {
+  // variable values are 0 if not set, 1 if positive, -1 if negative
+  const varVals = new Int32Array(maxvarnr + 1);
+  return satisfiableAt(clauses, varVals, 0, 0);
+}
+
+exports.naiveDpll = naiveDpll;

config.json

@@ -0,0 +1,82 @@
+{
+  "server.port": 3000,
+  "server.nbi": "http://127.0.0.1:7557/",
+  "ui.pageSize": 10,
+  "ui.filters.1.label": "Serial number",
+  "ui.filters.1.parameter": "DeviceID.SerialNumber",
+  "ui.filters.1.type": "string",
+  "ui.filters.2.label": "Product class",
+  "ui.filters.2.parameter": "DeviceID.ProductClass",
+  "ui.filters.2.type": "string",
+  "ui.filters.3.label": "Tag",
+  "ui.filters.3.parameter": "tag",
+  "ui.filters.3.type": "string",
+  "ui.index.1.label": "Serial number",
+  "ui.index.1.parameter": "DeviceID.SerialNumber",
+  "ui.index.2.label": "Product class",
+  "ui.index.2.parameter": "DeviceID.ProductClass",
+  "ui.index.3.label": "Software version",
+  "ui.index.3.parameter": "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
+  "ui.index.4.label": "MAC",
+  "ui.index.4.parameter": "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress",
+  "ui.index.4.label": "IP",
+  "ui.index.4.parameter": "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
+  "ui.index.5.label": "SSID",
+  "ui.index.5.parameter": "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
+  "ui.index.6.label": "Last inform",
+  "ui.index.6.parameter": "Events.Inform",
+  "ui.index.6.meta.type": "overview-dot",
+  "ui.index.6.meta.chart": "online",
+  "ui.overview.groups.online.label": "",
+  "ui.overview.groups.online.charts.1": "online",
+  "ui.overview.charts.online.label": "Online status",
+  "ui.overview.charts.online.slices.onlineNow.label": "Online now",
+  "ui.overview.charts.online.slices.onlineNow.color": "#31a354",
+  "ui.overview.charts.online.slices.onlineNow.filter": "Events.Inform > NOW() - 5 * 60 * 1000",
+  "ui.overview.charts.online.slices.past24.label": "Past 24 hours",
+  "ui.overview.charts.online.slices.past24.color": "#addd8e",
+  "ui.overview.charts.online.slices.past24.filter": "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)",
+  "ui.overview.charts.online.slices.others.label": "Others",
+  "ui.overview.charts.online.slices.others.color": "#f7fcb9",
+  "ui.overview.charts.online.slices.others.filter": "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)",
+  "ui.device.1.type": "parameter-list",
+  "ui.device.1.parameters.1.label": "Last inform",
+  "ui.device.1.parameters.1.parameter": "Events.Inform",
+  "ui.device.1.parameters.1.meta.type": "overview-dot",
+  "ui.device.1.parameters.1.meta.chart": "online",
+  "ui.device.1.parameters.2.label": "Last boot",
+  "ui.device.1.parameters.2.parameter": "Events.1_BOOT",
+  "ui.device.1.parameters.3.label": "Last bootstrap",
+  "ui.device.1.parameters.3.parameter": "Events.0_BOOTSTRAP",
+  "ui.device.1.parameters.4.label": "Registered",
+  "ui.device.1.parameters.4.parameter": "Events.Registered",
+  "ui.device.2.type": "parameter-list",
+  "ui.device.2.parameters.1.label": "Serial number",
+  "ui.device.2.parameters.1.parameter": "InternetGatewayDevice.DeviceInfo.SerialNumber",
+  "ui.device.2.parameters.2.label": "Product class",
+  "ui.device.2.parameters.2.parameter": "InternetGatewayDevice.DeviceInfo.ProductClass",
+  "ui.device.2.parameters.3.label": "OUI",
+  "ui.device.2.parameters.3.parameter": "InternetGatewayDevice.DeviceInfo.ManufacturerOUI",
+  "ui.device.2.parameters.4.label": "Manufacturer",
+  "ui.device.2.parameters.4.parameter": "InternetGatewayDevice.DeviceInfo.Manufacturer",
+  "ui.device.2.parameters.5.label": "Hardware version",
+  "ui.device.2.parameters.5.parameter": "InternetGatewayDevice.DeviceInfo.HardwareVersion",
+  "ui.device.2.parameters.6.label": "Software version",
+  "ui.device.2.parameters.6.parameter": "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
+  "ui.device.2.parameters.7.label": "MAC",
+  "ui.device.2.parameters.7.parameter": "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress",
+  "ui.device.2.parameters.8.label": "IP",
+  "ui.device.2.parameters.8.parameter": "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
+  "ui.device.2.parameters.8.label": "WLAN SSID",
+  "ui.device.2.parameters.8.parameter": "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
+  "ui.device.2.parameters.9.label": "WLAN passphrase",
+  "ui.device.2.parameters.9.parameter": "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase",
+  "ui.device.3.type": "parameter-table",
+  "ui.device.3.object": "InternetGatewayDevice.LANDevice.1.Hosts.Host",
+  "ui.device.3.parameters.1.label": "Host name",
+  "ui.device.3.parameters.1.parameter": "HostName",
+  "ui.device.3.parameters.2.label": "IP address",
+  "ui.device.3.parameters.2.parameter": "IPAddress",
+  "ui.device.3.parameters.3.label": "MAC address",
+  "ui.device.3.parameters.3.parameter": "MACAddress"
+}

package.json

@@ -4,7 +4,7 @@
   "description": "",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "lint": "eslint --fix common/*.js server/*.js client/*.js;prettier --write common/*.js server/*.js client/*.js client/css/*.css",
+    "lint": "eslint --fix common/*.js server/*.js client/*.js client/components/*.js;prettier --write common/*.js server/*.js client/*.js client/components/*.js client/css/*.css",
     "build": "npm run lint;webpack;postcss -c postcss.config.js client/css/app.css -o public/app.css",
     "start": "node server/server"
   },

public/index.html

@@ -1,10 +0,0 @@
-<html>
-  <head>
-    <title>GenieACS</title>
-    <link rel="shortcut icon" type="image/png" href="favicon.png" />
-    <link rel="stylesheet" href="app.css">
-  </head>
-  <body>
-    <script src="app.js"></script>
-  </body>
-</html>

server/api-functions.js

@@ -80,11 +80,10 @@ function count(resource, filter, limit) {
   return new Promise((resolve, reject) => {
     let qs = {};
     if (filter) {
-      let f = filterParser.parse(filter);
-      f = filterParser.evaluateExpressions(f);
-      f = unpackTimestamps(f);
+      filter = filter.evaluateExpressions();
+      let ast = unpackTimestamps(filter.ast);
 
-      let q = filterToQuery(f);
+      let q = filterToQuery(ast);
       if (resource === "devices") q = device.transposeQuery(q);
       qs.query = JSON.stringify(q);
     }
@@ -92,7 +91,7 @@ function count(resource, filter, limit) {
     if (limit) qs.limit = limit;
 
     let options = url.parse(
-      `${config.GENIEACS_NBI}${resource}?${querystring.stringify(qs)}`
+      `${config.get("server.nbi")}${resource}?${querystring.stringify(qs)}`
     );
 
     options.method = "HEAD";
@@ -112,26 +111,28 @@ function count(resource, filter, limit) {
   });
 }
 
-function query(resource, filter, limit, callback) {
+function query(resource, filter, limit, skip, projection, callback) {
   return new Promise((resolve, reject) => {
     let ret;
     if (!callback) ret = [];
     let qs = {};
     if (filter) {
-      let f = filterParser.parse(filter);
-      f = filterParser.evaluateExpressions(f);
-      f = unpackTimestamps(f);
+      filter = filter.evaluateExpressions();
+      let ast = unpackTimestamps(filter.ast);
 
-      let q = filterToQuery(f);
+      let q = filterToQuery(ast);
       if (resource === "devices") q = device.transposeQuery(q);
 
       qs.query = JSON.stringify(q);
     }
 
     if (limit) qs.limit = limit;
+    if (skip) qs.skip = skip;
+    if (projection) qs.projection = projection;
+    qs.sort = JSON.stringify({ _id: 1 });
 
     let options = url.parse(
-      `${config.GENIEACS_NBI}${resource}?${querystring.stringify(qs)}`
+      `${config.get("server.nbi")}${resource}?${querystring.stringify(qs)}`
     );
 
     let _http = options.protocol === "https:" ? https : http;

server/api.js

@@ -3,6 +3,7 @@
 const Router = require("koa-router");
 
 const apiFunctions = require("./api-functions");
+const Filter = require("../common/filter");
 
 const router = new Router();
 
@@ -17,7 +18,7 @@ const resources = {
 for (let [resource, update] of Object.entries(resources)) {
   router.head(`/${resource}`, async ctx => {
     let filter, limit;
-    if (ctx.request.query.filter) filter = ctx.request.query.filter;
+    if (ctx.request.query.filter) filter = new Filter(ctx.request.query.filter);
     if (ctx.request.query.limit) limit = +ctx.request.query.limit;
 
     let count = await apiFunctions.count(resource, filter, limit);
@@ -26,22 +27,29 @@ for (let [resource, update] of Object.entries(resources)) {
   });
 
   router.get(`/${resource}`, async ctx => {
-    let filter, limit;
-    if (ctx.request.query.filter) filter = ctx.request.query.filter;
+    let filter, limit, skip;
+    if (ctx.request.query.filter) filter = new Filter(ctx.request.query.filter);
     if (ctx.request.query.limit) limit = +ctx.request.query.limit;
+    if (ctx.request.query.skip) skip = +ctx.request.query.skip;
 
-    ctx.body = await apiFunctions.query(resource, filter, limit);
+    ctx.body = await apiFunctions.query(
+      resource,
+      filter,
+      limit,
+      skip,
+      ctx.request.query.projection
+    );
   });
 
   router.head(`/${resource}/:id`, async (ctx, next) => {
-    let filter = `DeviceID.ID = "${ctx.params.id}"`;
+    let filter = new Filter(`DeviceID.ID = "${ctx.params.id}"`);
     let res = await apiFunctions.query(resource, filter);
     if (!res.length) return next();
     ctx.body = "";
   });
 
   router.get(`/${resource}/:id`, async (ctx, next) => {
-    let filter = `DeviceID.ID = "${ctx.params.id}"`;
+    let filter = new Filter(`DeviceID.ID = "${ctx.params.id}"`);
     let res = await apiFunctions.query(resource, filter);
     if (!res.length) return next();
     ctx.body = res[0];

server/config.js

@@ -1,6 +1,65 @@
 "use strict";
 
 const filterParser = require("../common/filter-parser.js");
+const Filter = require("../common/filter.js");
 
-exports.PORT = 3000;
-exports.GENIEACS_NBI = "http://127.0.0.1:7557/";
+const configFile = require("../config.json");
+
+const cache = {};
+
+function parse(name, value) {
+  if (/^ui\.overview\.charts\.[^.]+\.slices\.[^.]+\.filter$/.test(name))
+    return new Filter(value);
+
+  return value;
+}
+
+function get(name) {
+  let v = cache[name];
+  if (v === undefined) {
+    v = configFile[name];
+    if (v != null) {
+      v = parse(name, v);
+    } else {
+      const prefix = `${name}.`;
+      for (let k of Object.keys(configFile))
+        if (k.startsWith(prefix)) {
+          v = v || {};
+          let ar = k.slice(prefix.length).split(".");
+          let ref = v;
+          for (let a of ar.slice(0, -1)) ref = ref[a] = ref[a] || {};
+          ref[ar[ar.length - 1]] = parse(k, configFile[k]);
+        }
+    }
+    if (v) cache[name] = v;
+    else cache[name] = null;
+  }
+
+  return v;
+}
+
+function getClientConfig() {
+  const regex = [
+    /^ui\.pageSize$/,
+    /^ui\.filters\.[^.]+\.label$/,
+    /^ui\.filters\.[^.]+\.parameter$/,
+    /^ui\.filters\.[^.]+\.type$/,
+    /^ui\.index\./,
+    /^ui\.overview\.groups\./,
+    /^ui\.overview\.charts\./,
+    /^ui\.device/
+  ];
+
+  const clientConfig = {};
+  for (let [k, v] of Object.entries(configFile))
+    for (let r of regex)
+      if (r.test(k)) {
+        clientConfig[k] = v;
+        break;
+      }
+
+  return clientConfig;
+}
+
+exports.get = get;
+exports.getClientConfig = getClientConfig;

server/device.js

@@ -5,7 +5,7 @@ function recursive(input, root, output, timestamp) {
     if (root.length === 0)
       if (name === "_lastInform") {
         output["Events.Inform"] = {
-          value: [timestamp, "xsd:dateTime"],
+          value: [Date.parse(tree), "xsd:dateTime"],
           valueTimestamp: timestamp,
           writable: false,
           writableTimestamp: timestamp,
@@ -14,7 +14,7 @@ function recursive(input, root, output, timestamp) {
         };
       } else if (name === "_registered") {
         output["Events.Registered"] = {
-          value: [timestamp, "xsd:dateTime"],
+          value: [Date.parse(tree), "xsd:dateTime"],
           valueTimestamp: timestamp,
           writable: false,
           writableTimestamp: timestamp,
@@ -23,7 +23,7 @@ function recursive(input, root, output, timestamp) {
         };
       } else if (name === "_lastBoot") {
         output["Events.1_BOOT"] = {
-          value: [timestamp, "xsd:dateTime"],
+          value: [Date.parse(tree), "xsd:dateTime"],
           valueTimestamp: timestamp,
           writable: false,
           writableTimestamp: timestamp,
@@ -32,7 +32,7 @@ function recursive(input, root, output, timestamp) {
         };
       } else if (name === "_lastBootstrap") {
         output["Events.0_BOOTSTRAP"] = {
-          value: [timestamp, "xsd:dateTime"],
+          value: [Date.parse(tree), "xsd:dateTime"],
           valueTimestamp: timestamp,
           writable: false,
           writableTimestamp: timestamp,

server/server.js

@@ -12,10 +12,28 @@ const router = new Router();
 
 router.use("/api", api.routes(), api.allowedMethods());
 
+router.get("/", async ctx => {
+  ctx.body = `
+  <html>
+    <head>
+      <title>GenieACS</title>
+      <link rel="shortcut icon" type="image/png" href="favicon.png" />
+      <link rel="stylesheet" href="app.css">
+    </head>
+    <body>
+      <script>
+        window.clientConfig = ${JSON.stringify(config.getClientConfig())};
+      </script>
+      <script src="app.js"></script>
+    </body>
+  </html>
+  `;
+});
+
 koa.use(router.routes());
 koa.use(koaStatic("./public"));
 
-koa.listen(config.PORT, () => {
+koa.listen(config.get("server.port"), () => {
   // eslint-disable-next-line no-console
-  console.log(`Server listening on port ${config.PORT}`);
+  console.log(`Server listening on port ${config.get("server.port")}`);
 });