Fetch directly from db instead of API

client/store.js

@@ -71,7 +71,7 @@ function count(resourceType, filter) {
 
 function limitFilter(resourceType, filter, last) {
   if (resourceType === "devices")
-    return expression.and(filter, ["<=", "DeviceID.ID", last]);
+    return expression.and(filter, ["<=", ["PARAM", "DeviceID.ID"], last]);
   else return expression.and(filter, ["<=", ["PARAM", "_id"], last]);
 }
 
@@ -83,11 +83,10 @@ function findMatches(resourceType, filter, limit) {
         Array.isArray(e) &&
         Array.isArray(e[1]) &&
         e[1][0] === "PARAM" &&
-        e[1][0] === "tag"
+        e[1][1] === "tag"
       )
-        if (e[0] === "=") return ["=", ["PARAM", `Tags.${e[2]}`], true];
-        else if (e[0] === "<>")
-          return ["NOT", ["=", ["PARAM", `Tags.${e[2]}`], true]];
+        if (e[0] === "=") return ["IS NOT NULL", ["PARAM", `Tags.${e[2]}`]];
+        else if (e[0] === "<>") return ["IS NULL", ["PARAM", `Tags.${e[2]}`]];
       return e;
     });
 

common/expression.js

@@ -50,10 +50,12 @@ function evaluate(exp, obj, now) {
       }
     } else if (e[0] === "PARAM") {
       if (e[1] == null) return null;
-      if (obj && !isArray(e[1]))
-        if (typeof obj[e[1]] === "object")
-          return obj[e[1]].value ? obj[e[1]].value[0] : null;
-        else return obj[e[1]];
+      if (obj && !isArray(e[1])) {
+        let v = obj[e[1]];
+        if (typeof v === "object") v = v.value ? v.value[0] : null;
+        if (v == null) return null;
+        return v;
+      }
     } else if (e[0] === "AND") {
       return reduce(e, (a, b) => {
         if (!isArray(a)) return a ? b : a;
@@ -69,10 +71,12 @@ function evaluate(exp, obj, now) {
       else if (e[1][0] === "NOT") return e[1][1];
     } else if (e[0] === "IS NULL") {
       if (isArray(e[1])) return e;
-      return e[1] == null;
+      else if (e[1] == null) return true;
+      else return null;
     } else if (e[0] === "IS NOT NULL") {
       if (isArray(e[1])) return e;
-      return e[1] != null;
+      else if (e[1] != null) return true;
+      else return null;
     } else if (e[0] === "=") {
       if (isArray(e[1]) || isArray(e[2])) return e;
       if (e[1] == null || e[2] == null) return null;

server/api-functions.js

@@ -3,29 +3,11 @@
 const http = require("http");
 const https = require("https");
 const url = require("url");
-const querystring = require("querystring");
 
 const config = require("./config");
-const device = require("./device");
-const expressionParser = require("../common/expression-parser");
+const db = require("./db");
 const expression = require("../common/expression");
 
-const commaAscii = ",".charCodeAt(0);
-const newLineAscii = "\n".charCodeAt(0);
-
-function unpackTimestamps(filter) {
-  if (!Array.isArray(filter)) return filter;
-  return expressionParser.map(filter, exp => {
-    if (["=", "<>", ">=", "<=", "<", ">"].includes(exp[0]))
-      if (Array.isArray(exp) && typeof exp[2] === "number") {
-        let alt = exp.slice();
-        alt[2] = new Date(alt[2]).toJSON();
-        return ["OR", exp, alt];
-      }
-    return exp;
-  });
-}
-
 function filterToMongoQuery(filter, negate = false, res = {}) {
   const op = filter[0];
 
@@ -84,136 +66,6 @@ function filterToMongoQuery(filter, negate = false, res = {}) {
   return res;
 }
 
-function count(resource, filter, limit) {
-  return new Promise((resolve, reject) => {
-    let qs = {};
-    filter = expression.evaluate(filter, null, Date.now());
-    filter = unpackTimestamps(filter);
-
-    let q = {};
-    if (Array.isArray(filter)) q = filterToMongoQuery(filter);
-    else if (filter != null && !filter) return resolve(0);
-
-    if (resource === "devices") q = device.transposeQuery(q);
-    qs.query = JSON.stringify(q);
-
-    if (limit) qs.limit = limit;
-
-    let options = url.parse(
-      `${config.server.nbi}${resource}?${querystring.stringify(qs)}`
-    );
-
-    options.method = "HEAD";
-
-    let _http = options.protocol === "https:" ? https : http;
-
-    _http
-      .request(options, res => {
-        if (res.statusCode !== 200) {
-          reject(new Error(`Unexpected status code ${res.statusCode}`));
-          res.resume();
-          return;
-        }
-        resolve(+res.headers["total"]);
-      })
-      .end();
-  });
-}
-
-function query(resource, filter, limit, skip, projection, callback) {
-  return new Promise((resolve, reject) => {
-    let ret;
-    if (!callback) ret = [];
-    let qs = {};
-    filter = expression.evaluate(filter, null, Date.now());
-    filter = unpackTimestamps(filter);
-
-    let q = {};
-    if (Array.isArray(filter)) q = filterToMongoQuery(filter);
-    else if (filter != null && !filter) return resolve(ret);
-
-    if (resource === "devices") q = device.transposeQuery(q);
-    qs.query = JSON.stringify(q);
-
-    if (limit) qs.limit = limit;
-    if (skip) qs.skip = skip;
-    if (projection) qs.projection = projection;
-    qs.sort = JSON.stringify({ _id: 1 });
-
-    let options = url.parse(
-      `${config.server.nbi}${resource}?${querystring.stringify(qs)}`
-    );
-
-    let _http = options.protocol === "https:" ? https : http;
-
-    _http.get(options, res => {
-      if (res.statusCode !== 200) {
-        reject(new Error(`Unexpected status code ${res.statusCode}`));
-        res.resume();
-        return;
-      }
-      let chunks = [];
-      let bytes = 0;
-      res.on("data", chunk => {
-        let i;
-        while ((i = chunk.indexOf(newLineAscii)) !== -1) {
-          let buf;
-          if (chunk[chunk.length - 2] === commaAscii) {
-            i -= 1;
-            buf = new Buffer(bytes + i);
-            chunk.copy(buf, buf.length - i, 0, i);
-            chunk = chunk.slice(i + 2);
-          } else {
-            buf = new Buffer(bytes + i);
-            chunk.copy(buf, buf.length - i, 0, i);
-            chunk = chunk.slice(i + 1);
-          }
-
-          let o = 0;
-          for (let c of chunks) {
-            c.copy(buf, o, 0, c.length);
-            o += c.length;
-          }
-
-          if (buf.length > 1) {
-            let obj = JSON.parse(buf);
-            if (resource === "devices") obj = device.transpose(obj);
-            if (ret) ret.push(obj);
-            else callback(obj);
-          }
-
-          bytes = 0;
-          chunks = [];
-        }
-        bytes += chunk.length;
-        chunks.push(chunk);
-      });
-
-      res.on("end", () => {
-        let buf = new Buffer(bytes);
-        let o = 0;
-        for (let c of chunks) {
-          c.copy(buf, o, 0, c.length);
-          o += c.length;
-        }
-        if (buf.length > 1) {
-          let obj = JSON.parse(buf);
-          if (resource === "devices") obj = device.transpose(obj);
-
-          if (ret) ret.push(obj);
-          else callback(obj);
-        }
-        resolve(ret);
-      });
-
-      res.on("aborted", () => {
-        res.removeAllListeners("end");
-        reject(new Error("Timeout"));
-      });
-    });
-  });
-}
-
 function deleteResource(resource, id) {
   return new Promise((resolve, reject) => {
     let options = url.parse(
@@ -237,7 +89,7 @@ function deleteResource(resource, id) {
 
 function postTask(task, connectionRequest, callback) {
   let options = url.parse(
-    `${config.server.nbi}devices/${task.device}/tasks${
+    `${config.server.nbi}devices/${encodeURIComponent(task.device)}/tasks${
       connectionRequest ? "?connection_request" : ""
     }`
   );
@@ -316,9 +168,9 @@ function postTasks(deviceId, tasks) {
 
         let promises2 = [];
         for (let s of statuses) {
-          promises2.push(query("tasks", ["=", ["PARAM", "_id"], s._id]));
+          promises2.push(db.query("tasks", ["=", ["PARAM", "_id"], s._id]));
           promises2.push(
-            query("faults", [
+            db.query("faults", [
               "=",
               ["PARAM", "_id"],
               `${deviceId}:task_${s._id}`
@@ -355,9 +207,9 @@ function updateTags(deviceId, tags) {
       ([tag, onOff]) =>
         new Promise((resolve, reject) => {
           const options = url.parse(
-            `${config.server.nbi}devices/${deviceId}/tags/${encodeURIComponent(
-              tag
-            )}`
+            `${config.server.nbi}devices/${encodeURIComponent(
+              deviceId
+            )}/tags/${encodeURIComponent(tag)}`
           );
           if (onOff) options.method = "POST";
           else options.method = "DELETE";
@@ -431,8 +283,6 @@ function ping(host) {
   });
 }
 
-exports.query = query;
-exports.count = count;
 exports.postTasks = postTasks;
 exports.deleteResource = deleteResource;
 exports.updateTags = updateTags;

server/api.js

@@ -9,7 +9,6 @@ const expression = require("../common/expression");
 const router = new Router();
 
 const RESOURCE_DELETE = 1 << 0;
-const RESOURCE_DB = 1 << 1;
 
 const RESOURCE_IDS = {
   devices: "DeviceID.ID",
@@ -34,10 +33,8 @@ const resources = {
 for (let [resource, flags] of Object.entries(resources)) {
   router.head(`/${resource}`, async (ctx, next) => {
     let filter = true;
-    let limit;
     if (ctx.request.query.filter)
       filter = expression.parse(ctx.request.query.filter);
-    if (ctx.request.query.limit) limit = +ctx.request.query.limit;
 
     if (!ctx.state.authorizer.hasAccess(resource, 1)) return next();
 
@@ -48,22 +45,23 @@ for (let [resource, flags] of Object.entries(resources)) {
         ["<", ["PARAM", "expiry"], Date.now() + 60000]
       ]);
 
-    let count;
-    if (flags & RESOURCE_DB) count = await db.count(resource, filter, limit);
-    else count = await apiFunctions.count(resource, filter, limit);
+    let count = await db.count(resource, filter);
 
     ctx.set("X-Total-Count", count);
     ctx.body = "";
   });
 
   router.get(`/${resource}`, async (ctx, next) => {
+    let options = {};
     let filter = true;
-    let limit, skip, projection;
     if (ctx.request.query.filter)
       filter = expression.parse(ctx.request.query.filter);
-    if (ctx.request.query.limit) limit = +ctx.request.query.limit;
-    if (ctx.request.query.skip) skip = +ctx.request.query.skip;
-    if (ctx.request.query.projection) projection = ctx.request.query.projection;
+    if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
+    if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
+    if (ctx.request.query.projection)
+      options.projection = ctx.request.query.projection
+        .split(",")
+        .reduce((obj, k) => Object.assign(obj, { [k]: 1 }), {});
 
     if (!ctx.state.authorizer.hasAccess(resource, 2)) return next();
 
@@ -74,11 +72,7 @@ for (let [resource, flags] of Object.entries(resources)) {
         ["<", ["PARAM", "expiry"], Date.now() + 60000]
       ]);
 
-    let res;
-    if (flags & RESOURCE_DB)
-      res = await db.query(resource, filter, limit, skip);
-    else
-      res = await apiFunctions.query(resource, filter, limit, skip, projection);
+    let res = await db.query(resource, filter, options);
 
     ctx.body = res;
   });
@@ -87,9 +81,7 @@ for (let [resource, flags] of Object.entries(resources)) {
     let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
     if (!ctx.state.authorizer.hasAccess(resource, 2)) return next();
 
-    let res;
-    if (flags & RESOURCE_DB) res = await db.query(resource, filter);
-    else res = await apiFunctions.query(resource, filter);
+    let res = await db.query(resource, filter);
 
     if (!res.length) return next();
     ctx.body = "";
@@ -99,20 +91,19 @@ for (let [resource, flags] of Object.entries(resources)) {
     let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
     if (!ctx.state.authorizer.hasAccess(resource, 2)) return next();
 
-    let res;
-    if (flags & RESOURCE_DB) res = await db.query(resource, filter);
-    else res = await apiFunctions.query(resource, filter);
+    let res = await db.query(resource, filter);
+
     if (!res.length) return next();
     ctx.body = res[0];
   });
 
   // TODO add PUT, PATCH routes
-  if (flags & RESOURCE_DELETE && !(flags & RESOURCE_DB))
+  if (flags & RESOURCE_DELETE)
     router.delete(`/${resource}/:id`, async (ctx, next) => {
       const authorizer = ctx.state.authorizer;
       let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
       if (!authorizer.hasAccess(resource, 2)) return next();
-      let res = await apiFunctions.query(resource, filter);
+      let res = await db.query(resource, filter);
       if (!res.length) return next();
 
       const validate = authorizer.getValidator(resource, res[0]);
@@ -127,7 +118,7 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
   const authorizer = ctx.state.authorizer;
   let filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
   if (!authorizer.hasAccess("devices", 2)) return next();
-  let devices = await apiFunctions.query("devices", filter);
+  let devices = await db.query("devices", filter);
   if (!devices.length) return next();
 
   const validate = authorizer.getValidator("devices", devices[0]);
@@ -143,7 +134,7 @@ router.post("/devices/:id/tags", async (ctx, next) => {
   const authorizer = ctx.state.authorizer;
   let filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
   if (!authorizer.hasAccess("devices", 2)) return next();
-  let res = await apiFunctions.query("devices", filter);
+  let res = await db.query("devices", filter);
   if (!res.length) return next();
 
   const validate = authorizer.getValidator("devices", res[0]);

server/db.js

@@ -1,43 +1,75 @@
 "use strict";
 
-const MongoClient = require("mongodb").MongoClient;
+const mongodb = require("mongodb");
 const config = require("./config");
-const apiFunctions = require("./api-functions");
+const mongodbFunctions = require("./mongodb-functions");
+const expression = require("../common/expression");
 
-let _client = null;
+const _client = {};
 
-function getClient() {
-  if (_client) return Promise.resolve(_client);
+const RESOURCE_DB = {
+  devices: "genieacs",
+  faults: "genieacs",
+  tasks: "genieacs",
+  presets: "genieacs",
+  provisions: "genieacs",
+  files: "genieacs"
+};
+
+function getClient(db) {
+  if (_client[db]) return Promise.resolve(_client[db]);
 
   return new Promise((resolve, reject) => {
     const CONNECTION_URL = config.server.mongodbConnectionUrl;
-    MongoClient.connect(CONNECTION_URL, (err, client) => {
+    mongodb.MongoClient.connect(CONNECTION_URL, (err, client) => {
       if (err) return reject(err);
-      resolve((_client = client.db("genieacs-ui")));
+      resolve((_client[db] = client.db(db)));
     });
   });
 }
 
-function query(resource, filter, limit, skip, callback) {
+function query(resource, filter, options, callback) {
+  options = options || {};
   let q;
-  if (Array.isArray(filter)) q = apiFunctions.filterToMongoQuery(filter);
-  else if (filter != null && !filter) return Promise.resolve([]);
+  filter = expression.evaluate(filter, null, null, Date.now());
+
+  if (Array.isArray(filter)) {
+    if (resource === "devices")
+      filter = mongodbFunctions.processDeviceFilter(filter);
+    else if (resource === "tasks")
+      filter = mongodbFunctions.processTasksFilter(filter);
+    else if (resource === "faults")
+      filter = mongodbFunctions.processFaultsFilter(filter);
+    q = mongodbFunctions.filterToMongoQuery(filter);
+  } else if (filter != null && !filter) {
+    return Promise.resolve([]);
+  }
 
   return new Promise((resolve, reject) => {
-    getClient().then(client => {
+    getClient(RESOURCE_DB[resource]).then(client => {
       const collection = client.collection(resource);
+      let cursor = collection.find(
+        q,
+        resource === "Devices"
+          ? mongodbFunctions.processDeviceProjection(options.projection)
+          : options.projection
+      );
+      if (options.skip) cursor = cursor.skip(options.skip);
+      if (options.limit) cursor = cursor.limit(options.limit);
+      cursor = cursor.sort({ _id: 1 });
+
       if (!callback)
-        collection
-          .find(q)
-          .skip(skip || 0)
-          .limit(limit || 0)
-          .toArray((err, docs) => {
+        cursor.toArray((err, docs) => {
           if (err) return reject(err);
+          if (resource === "devices")
+            docs = docs.map(d => mongodbFunctions.flattenDevice(d));
           return resolve(docs);
         });
       else
-        collection.find(q).forEach(
+        cursor.forEach(
           doc => {
+            if (resource === "devices")
+              doc = mongodbFunctions.flattenDevice(doc);
             callback(doc);
           },
           err => {
@@ -51,11 +83,22 @@ function query(resource, filter, limit, skip, callback) {
 
 function count(resource, filter) {
   let q;
-  if (Array.isArray(filter)) q = apiFunctions.filterToMongoQuery(filter);
-  else if (filter != null && !filter) return Promise.resolve([]);
+  filter = expression.evaluate(filter, null, null, Date.now());
+
+  if (Array.isArray(filter)) {
+    if (resource === "devices")
+      filter = mongodbFunctions.processDeviceFilter(filter);
+    else if (resource === "tasks")
+      filter = mongodbFunctions.processTasksFilter(filter);
+    else if (resource === "faults")
+      filter = mongodbFunctions.processFaultsFilter(filter);
+    q = mongodbFunctions.filterToMongoQuery(filter);
+  } else if (filter != null && !filter) {
+    return Promise.resolve([]);
+  }
 
   return new Promise((resolve, reject) => {
-    getClient().then(client => {
+    getClient(RESOURCE_DB[resource]).then(client => {
       const collection = client.collection(resource);
       collection.find(q).count((err, c) => {
         if (err) reject(err);

server/device.js

@@ -1,191 +0,0 @@
-"use strict";
-
-function recursive(input, root, output, timestamp) {
-  for (let [name, tree] of Object.entries(input)) {
-    if (root.length === 0)
-      if (name === "_lastInform") {
-        output["Events.Inform"] = {
-          value: [Date.parse(tree), "xsd:dateTime"],
-          valueTimestamp: timestamp,
-          writable: false,
-          writableTimestamp: timestamp,
-          object: false,
-          objectTimestamp: timestamp
-        };
-      } else if (name === "_registered") {
-        output["Events.Registered"] = {
-          value: [Date.parse(tree), "xsd:dateTime"],
-          valueTimestamp: timestamp,
-          writable: false,
-          writableTimestamp: timestamp,
-          object: false,
-          objectTimestamp: timestamp
-        };
-      } else if (name === "_lastBoot") {
-        output["Events.1_BOOT"] = {
-          value: [Date.parse(tree), "xsd:dateTime"],
-          valueTimestamp: timestamp,
-          writable: false,
-          writableTimestamp: timestamp,
-          object: false,
-          objectTimestamp: timestamp
-        };
-      } else if (name === "_lastBootstrap") {
-        output["Events.0_BOOTSTRAP"] = {
-          value: [Date.parse(tree), "xsd:dateTime"],
-          valueTimestamp: timestamp,
-          writable: false,
-          writableTimestamp: timestamp,
-          object: false,
-          objectTimestamp: timestamp
-        };
-      } else if (name === "_id") {
-        output["DeviceID.ID"] = {
-          value: [tree, "xsd:string"],
-          valueTimestamp: timestamp,
-          writable: false,
-          writableTimestamp: timestamp,
-          object: false,
-          objectTimestamp: timestamp
-        };
-      } else if (name === "_deviceId") {
-        output["DeviceID.Manufacturer"] = {
-          value: [tree["_Manufacturer"], "xsd:string"],
-          valueTimestamp: timestamp,
-          writable: false,
-          writableTimestamp: timestamp,
-          object: false,
-          objectTimestamp: timestamp
-        };
-        output["DeviceID.OUI"] = {
-          value: [tree["_OUI"], "xsd:string"],
-          valueTimestamp: timestamp,
-          writable: false,
-          writableTimestamp: timestamp,
-          object: false,
-          objectTimestamp: timestamp
-        };
-        output["DeviceID.ProductClass"] = {
-          value: [tree["_ProductClass"], "xsd:string"],
-          valueTimestamp: timestamp,
-          writable: false,
-          writableTimestamp: timestamp,
-          object: false,
-          objectTimestamp: timestamp
-        };
-        output["DeviceID.SerialNumber"] = {
-          value: [tree["_SerialNumber"], "xsd:string"],
-          valueTimestamp: timestamp,
-          writable: false,
-          writableTimestamp: timestamp,
-          object: false,
-          objectTimestamp: timestamp
-        };
-      } else if (name === "_tags") {
-        output["Tags"] = {
-          writable: true,
-          writableTimestamp: timestamp,
-          object: true,
-          objectTimestamp: timestamp
-        };
-
-        for (let t of tree)
-          output[`Tags.${t}`] = {
-            value: [true, "xsd:boolean"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp
-          };
-      }
-
-    if (name.startsWith("_")) continue;
-
-    let childrenTimestamp = timestamp;
-
-    if (root.length === 0) childrenTimestamp = input["_timestamp"] || 1;
-    else if (input["_timestamp"] > timestamp)
-      childrenTimestamp = input["_timestamp"];
-
-    const attrs = {};
-    if (tree["_value"] != null) {
-      attrs["value"] = [tree["_value"], tree["_type"]];
-      attrs["valueTimestamp"] = tree["_timestamp"] || childrenTimestamp;
-      attrs["object"] = false;
-      attrs["objectTimestamp"] = childrenTimestamp;
-    } else if (tree["_object"] != null) {
-      attrs["object"] = tree["_object"];
-      attrs["objectTimestamp"] = childrenTimestamp;
-    }
-
-    if (tree["_writable"] != null) {
-      attrs["writable"] = tree["_writable"];
-      attrs["writableTimestamp"] = childrenTimestamp;
-    }
-
-    let r = root.concat(name);
-    output[r.join(".")] = attrs;
-
-    if (attrs["object"]) recursive(tree, r, output, childrenTimestamp);
-  }
-}
-
-function transpose(device) {
-  const newDevice = {};
-  const timestamp = new Date(device["_lastInform"] || 1).getTime();
-  recursive(device, [], newDevice, timestamp);
-  return newDevice;
-}
-
-function transposeQuery(query) {
-  const newQuery = {};
-
-  for (let [key, val] of Object.entries(query))
-    if (key.startsWith("$")) {
-      newQuery[key] = val.map(v => transposeQuery(v));
-    } else if (key === "DeviceID.ID") {
-      newQuery["_id"] = val;
-    } else if (key === "DeviceID.SerialNumber") {
-      newQuery["_deviceId._SerialNumber"] = val;
-    } else if (key === "DeviceID.OUI") {
-      newQuery["_deviceId._OUI"] = val;
-    } else if (key === "DeviceID.ProductClass") {
-      newQuery["_deviceId._ProductClass"] = val;
-    } else if (key === "DeviceID.Manufacturer") {
-      newQuery["_deviceId._Manufacturer"] = val;
-    } else if (key.startsWith("Tags.")) {
-      if (val["$exists"] === true) {
-        newQuery["_tags"] = newQuery["_tags"] || {};
-        newQuery["_tags"]["$all"] = newQuery["_tags"]["$all"] || [];
-        newQuery["_tags"]["$all"].push(key.slice(5));
-      } else if (val["$exists"] === false) {
-        newQuery["_tags"] = newQuery["_tags"] || {};
-        newQuery["_tags"]["$nin"] = newQuery["_tags"]["$nin"] || [];
-        newQuery["_tags"]["$nin"].push(key.slice(5));
-      }
-    } else if (key === "tag") {
-      if (typeof val === "string") {
-        newQuery["_tags"] = newQuery["_tags"] || {};
-        newQuery["_tags"]["$all"] = newQuery["_tags"]["$all"] || [];
-        newQuery["_tags"]["$all"].push(val);
-      } else if (val["$eq"]) {
-        newQuery["_tags"] = newQuery["_tags"] || {};
-        newQuery["_tags"]["$all"] = newQuery["_tags"]["$all"] || [];
-        newQuery["_tags"]["$all"].push(val["$eq"]);
-      } else if (val["$ne"]) {
-        newQuery["_tags"] = newQuery["_tags"] || {};
-        newQuery["_tags"]["$nin"] = newQuery["_tags"]["$nin"] || [];
-        newQuery["_tags"]["$nin"].push(val["$ne"]);
-      }
-    } else if (key === "Events.Inform") {
-      newQuery["_lastInform"] = val;
-    } else {
-      newQuery[key] = val;
-    }
-
-  return newQuery;
-}
-
-exports.transpose = transpose;
-exports.transposeQuery = transposeQuery;

server/mongodb-functions.js

@@ -0,0 +1,320 @@
+"use strict";
+const mongodb = require("mongodb");
+const expressionParser = require("../common/expression-parser");
+
+function processDeviceFilter(filter) {
+  return expressionParser.map(filter, exp => {
+    if (!Array.isArray(exp)) return exp;
+
+    if (exp[0] === "PARAM") {
+      const p = exp[1];
+      if (p === "DeviceID.ID") return ["PARAM", "_id"];
+      else if (p === "DeviceID") return ["PARAM", "_deviceId"];
+      else if (p.startsWith("DeviceID."))
+        return ["PARAM", "_deviceId._" + p.slice(9)];
+      else if (p === "Tags") return ["PARAM", "_tags"];
+      else if (p === "tag") return ["PARAM", "_tags"];
+      else if (p === "Events.Inform") return ["PARAM", "_lastInform"];
+      else if (p === "Events.Registered") return ["PARAM", "_registered"];
+      else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"];
+      else if (p === "Events.1_BOOT") return ["PARAM", "_lastBoot"];
+    } else if (
+      Array.isArray(exp[1]) &&
+      exp[1][0] === "PARAM" &&
+      exp[1][1].startsWith("Tags.")
+    ) {
+      const t = exp[1][1].slice(5);
+      if (exp[0] === "IS NULL") return ["<>", ["PARAM", "_tags"], t];
+      else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t];
+      else if (exp[0] === "=" && exp[2] === true)
+        return ["=", ["PARAM", "_tags"], t];
+    } else if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
+      let e = exp.slice();
+      if (
+        Array.isArray(exp[1]) &&
+        exp[1][0] === "PARAM" &&
+        typeof exp[1][1] === "string" &&
+        !exp[1][1].startsWith("_")
+      )
+        e[1] = ["PARAM", `${exp[1][1]}._value`];
+      if (typeof e[2] === "number") {
+        let alt = e.slice();
+        alt[2] = new Date(e[2]);
+        e = ["OR", e, alt];
+      }
+      return e;
+    }
+    return exp;
+  });
+}
+
+function processTasksFilter(filter) {
+  return expressionParser.map(filter, exp => {
+    if (!Array.isArray(exp)) return exp;
+    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
+      let e = exp.slice();
+      if (e[1][0] === "PARAM" && e[1][1] === "_id")
+        e[2] = new mongodb.ObjectID(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "timestamp")
+        e[2] = new Date(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
+        e[2] = new Date(e[2]);
+      return e;
+    }
+    return exp;
+  });
+}
+
+function processFaultsFilter(filter) {
+  return expressionParser.map(filter, exp => {
+    if (!Array.isArray(exp)) return exp;
+    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
+      let e = exp.slice();
+      if (e[1][0] === "PARAM" && e[1][1] === "timestamp") e[2] = new Date(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
+        e[2] = new Date(e[2]);
+      return e;
+    }
+    return exp;
+  });
+}
+
+function filterToMongoQuery(filter, negate = false, res = {}) {
+  const op = filter[0];
+
+  if ((!negate && op === "AND") || (negate && op === "OR")) {
+    res["$and"] = res["$and"] || [];
+    for (let i = 1; i < filter.length; ++i)
+      res["$and"].push(filterToMongoQuery(filter[i], negate));
+  } else if ((!negate && op === "OR") || (negate && op === "AND")) {
+    res["$or"] = res["$or"] || [];
+
+    for (let i = 1; i < filter.length; ++i)
+      res["$or"].push(filterToMongoQuery(filter[i], negate));
+  } else if (op === "NOT") {
+    filterToMongoQuery(filter[1], !negate, res);
+  } else if (op === "=") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p["$ne"] = filter[2];
+    else p["$eq"] = filter[2];
+  } else if (op === "<>") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$ne"] = filter[2];
+    p["$exists"] = true;
+  } else if (op === ">") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$gt"] = filter[2];
+  } else if (op === ">=") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$gte"] = filter[2];
+  } else if (op === "<") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$lt"] = filter[2];
+  } else if (op === "<=") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$lte"] = filter[2];
+  } else if (op === "IS NULL") {
+    const param = filter[1][1];
+    res[param] = { $exists: negate };
+  } else if (op === "IS NOT NULL") {
+    const param = filter[1][1];
+    res[param] = { $exists: !negate };
+  } else {
+    throw new Error(`Unrecognized operator ${op}`);
+  }
+
+  return res;
+}
+
+function processDeviceProjection(projection) {
+  if (!projection) return projection;
+  const p = {};
+  for (const [k, v] of Object.entries(projection))
+    if (k === "DeviceID.ID") {
+      p["_id"] = 1;
+    } else if (k.startsWith("DeviceID")) {
+      p["_deviceId._SerialNumber"] = v;
+      p["_deviceId._OUI"] = v;
+      p["_deviceId._ProductClass"] = v;
+      p["_deviceId._Manufacturer"] = v;
+    } else if (k.startsWith("Tags")) {
+      p["_tags"] = v;
+    } else if (k.startsWith("Events")) {
+      p["_lastInform"] = v;
+      p["_registered"] = v;
+      p["_lastBoot"] = v;
+      p["_lastBootstrap"] = v;
+    } else {
+      p[k] = v;
+    }
+
+  return p;
+}
+
+function processDeviceSort(sort) {
+  if (!sort) return sort;
+  const s = {};
+  for (const [k, v] of Object.entries(sort))
+    if (k === "DeviceID.ID") s["_id"] = v;
+    else if (k === "Events.Inform") s["_lastInform"] = v;
+    else s[k] = v;
+
+  return s;
+}
+
+function flattenDevice(device) {
+  function recursive(input, root, output, timestamp) {
+    for (let [name, tree] of Object.entries(input)) {
+      if (root.length === 0)
+        if (name === "_lastInform") {
+          output["Events.Inform"] = {
+            value: [Date.parse(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_registered") {
+          output["Events.Registered"] = {
+            value: [Date.parse(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_lastBoot") {
+          output["Events.1_BOOT"] = {
+            value: [Date.parse(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_lastBootstrap") {
+          output["Events.0_BOOTSTRAP"] = {
+            value: [Date.parse(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_id") {
+          output["DeviceID.ID"] = {
+            value: [tree, "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_deviceId") {
+          output["DeviceID.Manufacturer"] = {
+            value: [tree["_Manufacturer"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+          output["DeviceID.OUI"] = {
+            value: [tree["_OUI"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+          output["DeviceID.ProductClass"] = {
+            value: [tree["_ProductClass"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+          output["DeviceID.SerialNumber"] = {
+            value: [tree["_SerialNumber"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_tags") {
+          output["Tags"] = {
+            writable: true,
+            writableTimestamp: timestamp,
+            object: true,
+            objectTimestamp: timestamp
+          };
+
+          for (let t of tree)
+            output[`Tags.${t}`] = {
+              value: [true, "xsd:boolean"],
+              valueTimestamp: timestamp,
+              writable: false,
+              writableTimestamp: timestamp,
+              object: false,
+              objectTimestamp: timestamp
+            };
+        }
+
+      if (name.startsWith("_")) continue;
+
+      let childrenTimestamp = timestamp;
+
+      if (root.length === 0) childrenTimestamp = +(input["_timestamp"] || 1);
+      else if (+input["_timestamp"] > timestamp)
+        childrenTimestamp = +input["_timestamp"];
+
+      const attrs = {};
+      if (tree["_value"] != null) {
+        attrs["value"] = [tree["_value"], tree["_type"]];
+        attrs["valueTimestamp"] = +(tree["_timestamp"] || childrenTimestamp);
+        attrs["object"] = false;
+        attrs["objectTimestamp"] = childrenTimestamp;
+      } else if (tree["_object"] != null) {
+        attrs["object"] = tree["_object"];
+        attrs["objectTimestamp"] = childrenTimestamp;
+      }
+
+      if (tree["_writable"] != null) {
+        attrs["writable"] = tree["_writable"];
+        attrs["writableTimestamp"] = childrenTimestamp;
+      }
+
+      let r = root.concat(name);
+      output[r.join(".")] = attrs;
+
+      if (attrs["object"]) recursive(tree, r, output, childrenTimestamp);
+    }
+  }
+
+  const newDevice = {};
+  const timestamp = new Date(device["_lastInform"] || 1).getTime();
+  recursive(device, [], newDevice, timestamp);
+  return newDevice;
+}
+
+exports.processDeviceFilter = processDeviceFilter;
+exports.processTasksFilter = processTasksFilter;
+exports.processFaultsFilter = processFaultsFilter;
+exports.filterToMongoQuery = filterToMongoQuery;
+exports.processDeviceProjection = processDeviceProjection;
+exports.processDeviceSort = processDeviceSort;
+exports.flattenDevice = flattenDevice;