From db900622a0e2c5a8a846d094c0fefffc69c3624a Mon Sep 17 00:00:00 2001 From: Zaid Abdulla Date: Fri, 20 Jul 2018 17:08:13 -0700 Subject: [PATCH] Fetch directly from db instead of API --- client/store.js | 9 +- common/expression.js | 16 +- server/api-functions.js | 164 +----------------- server/api.js | 41 ++--- server/db.js | 89 +++++++--- server/device.js | 191 --------------------- server/mongodb-functions.js | 320 ++++++++++++++++++++++++++++++++++++ 7 files changed, 423 insertions(+), 407 deletions(-) delete mode 100644 server/device.js create mode 100644 server/mongodb-functions.js diff --git a/client/store.js b/client/store.js index 02ba2c3..dc2025b 100644 --- a/client/store.js +++ b/client/store.js @@ -71,7 +71,7 @@ function count(resourceType, filter) { function limitFilter(resourceType, filter, last) { if (resourceType === "devices") - return expression.and(filter, ["<=", "DeviceID.ID", last]); + return expression.and(filter, ["<=", ["PARAM", "DeviceID.ID"], last]); else return expression.and(filter, ["<=", ["PARAM", "_id"], last]); } @@ -83,11 +83,10 @@ function findMatches(resourceType, filter, limit) { Array.isArray(e) && Array.isArray(e[1]) && e[1][0] === "PARAM" && - e[1][0] === "tag" + e[1][1] === "tag" ) - if (e[0] === "=") return ["=", ["PARAM", `Tags.${e[2]}`], true]; - else if (e[0] === "<>") - return ["NOT", ["=", ["PARAM", `Tags.${e[2]}`], true]]; + if (e[0] === "=") return ["IS NOT NULL", ["PARAM", `Tags.${e[2]}`]]; + else if (e[0] === "<>") return ["IS NULL", ["PARAM", `Tags.${e[2]}`]]; return e; }); diff --git a/common/expression.js b/common/expression.js index 14e6c53..1e8d84f 100644 --- a/common/expression.js +++ b/common/expression.js @@ -50,10 +50,12 @@ function evaluate(exp, obj, now) { } } else if (e[0] === "PARAM") { if (e[1] == null) return null; - if (obj && !isArray(e[1])) - if (typeof obj[e[1]] === "object") - return obj[e[1]].value ? obj[e[1]].value[0] : null; - else return obj[e[1]]; + if (obj && !isArray(e[1])) { + let v = obj[e[1]]; + if (typeof v === "object") v = v.value ? v.value[0] : null; + if (v == null) return null; + return v; + } } else if (e[0] === "AND") { return reduce(e, (a, b) => { if (!isArray(a)) return a ? b : a; @@ -69,10 +71,12 @@ function evaluate(exp, obj, now) { else if (e[1][0] === "NOT") return e[1][1]; } else if (e[0] === "IS NULL") { if (isArray(e[1])) return e; - return e[1] == null; + else if (e[1] == null) return true; + else return null; } else if (e[0] === "IS NOT NULL") { if (isArray(e[1])) return e; - return e[1] != null; + else if (e[1] != null) return true; + else return null; } else if (e[0] === "=") { if (isArray(e[1]) || isArray(e[2])) return e; if (e[1] == null || e[2] == null) return null; diff --git a/server/api-functions.js b/server/api-functions.js index e41cd92..52794ce 100644 --- a/server/api-functions.js +++ b/server/api-functions.js @@ -3,29 +3,11 @@ const http = require("http"); const https = require("https"); const url = require("url"); -const querystring = require("querystring"); const config = require("./config"); -const device = require("./device"); -const expressionParser = require("../common/expression-parser"); +const db = require("./db"); const expression = require("../common/expression"); -const commaAscii = ",".charCodeAt(0); -const newLineAscii = "\n".charCodeAt(0); - -function unpackTimestamps(filter) { - if (!Array.isArray(filter)) return filter; - return expressionParser.map(filter, exp => { - if (["=", "<>", ">=", "<=", "<", ">"].includes(exp[0])) - if (Array.isArray(exp) && typeof exp[2] === "number") { - let alt = exp.slice(); - alt[2] = new Date(alt[2]).toJSON(); - return ["OR", exp, alt]; - } - return exp; - }); -} - function filterToMongoQuery(filter, negate = false, res = {}) { const op = filter[0]; @@ -84,136 +66,6 @@ function filterToMongoQuery(filter, negate = false, res = {}) { return res; } -function count(resource, filter, limit) { - return new Promise((resolve, reject) => { - let qs = {}; - filter = expression.evaluate(filter, null, Date.now()); - filter = unpackTimestamps(filter); - - let q = {}; - if (Array.isArray(filter)) q = filterToMongoQuery(filter); - else if (filter != null && !filter) return resolve(0); - - if (resource === "devices") q = device.transposeQuery(q); - qs.query = JSON.stringify(q); - - if (limit) qs.limit = limit; - - let options = url.parse( - `${config.server.nbi}${resource}?${querystring.stringify(qs)}` - ); - - options.method = "HEAD"; - - let _http = options.protocol === "https:" ? https : http; - - _http - .request(options, res => { - if (res.statusCode !== 200) { - reject(new Error(`Unexpected status code ${res.statusCode}`)); - res.resume(); - return; - } - resolve(+res.headers["total"]); - }) - .end(); - }); -} - -function query(resource, filter, limit, skip, projection, callback) { - return new Promise((resolve, reject) => { - let ret; - if (!callback) ret = []; - let qs = {}; - filter = expression.evaluate(filter, null, Date.now()); - filter = unpackTimestamps(filter); - - let q = {}; - if (Array.isArray(filter)) q = filterToMongoQuery(filter); - else if (filter != null && !filter) return resolve(ret); - - if (resource === "devices") q = device.transposeQuery(q); - qs.query = JSON.stringify(q); - - if (limit) qs.limit = limit; - if (skip) qs.skip = skip; - if (projection) qs.projection = projection; - qs.sort = JSON.stringify({ _id: 1 }); - - let options = url.parse( - `${config.server.nbi}${resource}?${querystring.stringify(qs)}` - ); - - let _http = options.protocol === "https:" ? https : http; - - _http.get(options, res => { - if (res.statusCode !== 200) { - reject(new Error(`Unexpected status code ${res.statusCode}`)); - res.resume(); - return; - } - let chunks = []; - let bytes = 0; - res.on("data", chunk => { - let i; - while ((i = chunk.indexOf(newLineAscii)) !== -1) { - let buf; - if (chunk[chunk.length - 2] === commaAscii) { - i -= 1; - buf = new Buffer(bytes + i); - chunk.copy(buf, buf.length - i, 0, i); - chunk = chunk.slice(i + 2); - } else { - buf = new Buffer(bytes + i); - chunk.copy(buf, buf.length - i, 0, i); - chunk = chunk.slice(i + 1); - } - - let o = 0; - for (let c of chunks) { - c.copy(buf, o, 0, c.length); - o += c.length; - } - - if (buf.length > 1) { - let obj = JSON.parse(buf); - if (resource === "devices") obj = device.transpose(obj); - if (ret) ret.push(obj); - else callback(obj); - } - - bytes = 0; - chunks = []; - } - bytes += chunk.length; - chunks.push(chunk); - }); - - res.on("end", () => { - let buf = new Buffer(bytes); - let o = 0; - for (let c of chunks) { - c.copy(buf, o, 0, c.length); - o += c.length; - } - if (buf.length > 1) { - let obj = JSON.parse(buf); - if (resource === "devices") obj = device.transpose(obj); - - if (ret) ret.push(obj); - else callback(obj); - } - resolve(ret); - }); - - res.on("aborted", () => { - res.removeAllListeners("end"); - reject(new Error("Timeout")); - }); - }); - }); -} - function deleteResource(resource, id) { return new Promise((resolve, reject) => { let options = url.parse( @@ -237,7 +89,7 @@ function deleteResource(resource, id) { function postTask(task, connectionRequest, callback) { let options = url.parse( - `${config.server.nbi}devices/${task.device}/tasks${ + `${config.server.nbi}devices/${encodeURIComponent(task.device)}/tasks${ connectionRequest ? "?connection_request" : "" }` ); @@ -316,9 +168,9 @@ function postTasks(deviceId, tasks) { let promises2 = []; for (let s of statuses) { - promises2.push(query("tasks", ["=", ["PARAM", "_id"], s._id])); + promises2.push(db.query("tasks", ["=", ["PARAM", "_id"], s._id])); promises2.push( - query("faults", [ + db.query("faults", [ "=", ["PARAM", "_id"], `${deviceId}:task_${s._id}` @@ -355,9 +207,9 @@ function updateTags(deviceId, tags) { ([tag, onOff]) => new Promise((resolve, reject) => { const options = url.parse( - `${config.server.nbi}devices/${deviceId}/tags/${encodeURIComponent( - tag - )}` + `${config.server.nbi}devices/${encodeURIComponent( + deviceId + )}/tags/${encodeURIComponent(tag)}` ); if (onOff) options.method = "POST"; else options.method = "DELETE"; @@ -431,8 +283,6 @@ function ping(host) { }); } -exports.query = query; -exports.count = count; exports.postTasks = postTasks; exports.deleteResource = deleteResource; exports.updateTags = updateTags; diff --git a/server/api.js b/server/api.js index 37ec195..ff472db 100644 --- a/server/api.js +++ b/server/api.js @@ -9,7 +9,6 @@ const expression = require("../common/expression"); const router = new Router(); const RESOURCE_DELETE = 1 << 0; -const RESOURCE_DB = 1 << 1; const RESOURCE_IDS = { devices: "DeviceID.ID", @@ -34,10 +33,8 @@ const resources = { for (let [resource, flags] of Object.entries(resources)) { router.head(`/${resource}`, async (ctx, next) => { let filter = true; - let limit; if (ctx.request.query.filter) filter = expression.parse(ctx.request.query.filter); - if (ctx.request.query.limit) limit = +ctx.request.query.limit; if (!ctx.state.authorizer.hasAccess(resource, 1)) return next(); @@ -48,22 +45,23 @@ for (let [resource, flags] of Object.entries(resources)) { ["<", ["PARAM", "expiry"], Date.now() + 60000] ]); - let count; - if (flags & RESOURCE_DB) count = await db.count(resource, filter, limit); - else count = await apiFunctions.count(resource, filter, limit); + let count = await db.count(resource, filter); ctx.set("X-Total-Count", count); ctx.body = ""; }); router.get(`/${resource}`, async (ctx, next) => { + let options = {}; let filter = true; - let limit, skip, projection; if (ctx.request.query.filter) filter = expression.parse(ctx.request.query.filter); - if (ctx.request.query.limit) limit = +ctx.request.query.limit; - if (ctx.request.query.skip) skip = +ctx.request.query.skip; - if (ctx.request.query.projection) projection = ctx.request.query.projection; + if (ctx.request.query.limit) options.limit = +ctx.request.query.limit; + if (ctx.request.query.skip) options.skip = +ctx.request.query.skip; + if (ctx.request.query.projection) + options.projection = ctx.request.query.projection + .split(",") + .reduce((obj, k) => Object.assign(obj, { [k]: 1 }), {}); if (!ctx.state.authorizer.hasAccess(resource, 2)) return next(); @@ -74,11 +72,7 @@ for (let [resource, flags] of Object.entries(resources)) { ["<", ["PARAM", "expiry"], Date.now() + 60000] ]); - let res; - if (flags & RESOURCE_DB) - res = await db.query(resource, filter, limit, skip); - else - res = await apiFunctions.query(resource, filter, limit, skip, projection); + let res = await db.query(resource, filter, options); ctx.body = res; }); @@ -87,9 +81,7 @@ for (let [resource, flags] of Object.entries(resources)) { let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id]; if (!ctx.state.authorizer.hasAccess(resource, 2)) return next(); - let res; - if (flags & RESOURCE_DB) res = await db.query(resource, filter); - else res = await apiFunctions.query(resource, filter); + let res = await db.query(resource, filter); if (!res.length) return next(); ctx.body = ""; @@ -99,20 +91,19 @@ for (let [resource, flags] of Object.entries(resources)) { let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id]; if (!ctx.state.authorizer.hasAccess(resource, 2)) return next(); - let res; - if (flags & RESOURCE_DB) res = await db.query(resource, filter); - else res = await apiFunctions.query(resource, filter); + let res = await db.query(resource, filter); + if (!res.length) return next(); ctx.body = res[0]; }); // TODO add PUT, PATCH routes - if (flags & RESOURCE_DELETE && !(flags & RESOURCE_DB)) + if (flags & RESOURCE_DELETE) router.delete(`/${resource}/:id`, async (ctx, next) => { const authorizer = ctx.state.authorizer; let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id]; if (!authorizer.hasAccess(resource, 2)) return next(); - let res = await apiFunctions.query(resource, filter); + let res = await db.query(resource, filter); if (!res.length) return next(); const validate = authorizer.getValidator(resource, res[0]); @@ -127,7 +118,7 @@ router.post("/devices/:id/tasks", async (ctx, next) => { const authorizer = ctx.state.authorizer; let filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id]; if (!authorizer.hasAccess("devices", 2)) return next(); - let devices = await apiFunctions.query("devices", filter); + let devices = await db.query("devices", filter); if (!devices.length) return next(); const validate = authorizer.getValidator("devices", devices[0]); @@ -143,7 +134,7 @@ router.post("/devices/:id/tags", async (ctx, next) => { const authorizer = ctx.state.authorizer; let filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id]; if (!authorizer.hasAccess("devices", 2)) return next(); - let res = await apiFunctions.query("devices", filter); + let res = await db.query("devices", filter); if (!res.length) return next(); const validate = authorizer.getValidator("devices", res[0]); diff --git a/server/db.js b/server/db.js index 5147001..f0fa4b8 100644 --- a/server/db.js +++ b/server/db.js @@ -1,43 +1,75 @@ "use strict"; -const MongoClient = require("mongodb").MongoClient; +const mongodb = require("mongodb"); const config = require("./config"); -const apiFunctions = require("./api-functions"); +const mongodbFunctions = require("./mongodb-functions"); +const expression = require("../common/expression"); -let _client = null; +const _client = {}; -function getClient() { - if (_client) return Promise.resolve(_client); +const RESOURCE_DB = { + devices: "genieacs", + faults: "genieacs", + tasks: "genieacs", + presets: "genieacs", + provisions: "genieacs", + files: "genieacs" +}; + +function getClient(db) { + if (_client[db]) return Promise.resolve(_client[db]); return new Promise((resolve, reject) => { const CONNECTION_URL = config.server.mongodbConnectionUrl; - MongoClient.connect(CONNECTION_URL, (err, client) => { + mongodb.MongoClient.connect(CONNECTION_URL, (err, client) => { if (err) return reject(err); - resolve((_client = client.db("genieacs-ui"))); + resolve((_client[db] = client.db(db))); }); }); } -function query(resource, filter, limit, skip, callback) { +function query(resource, filter, options, callback) { + options = options || {}; let q; - if (Array.isArray(filter)) q = apiFunctions.filterToMongoQuery(filter); - else if (filter != null && !filter) return Promise.resolve([]); + filter = expression.evaluate(filter, null, null, Date.now()); + + if (Array.isArray(filter)) { + if (resource === "devices") + filter = mongodbFunctions.processDeviceFilter(filter); + else if (resource === "tasks") + filter = mongodbFunctions.processTasksFilter(filter); + else if (resource === "faults") + filter = mongodbFunctions.processFaultsFilter(filter); + q = mongodbFunctions.filterToMongoQuery(filter); + } else if (filter != null && !filter) { + return Promise.resolve([]); + } return new Promise((resolve, reject) => { - getClient().then(client => { + getClient(RESOURCE_DB[resource]).then(client => { const collection = client.collection(resource); + let cursor = collection.find( + q, + resource === "Devices" + ? mongodbFunctions.processDeviceProjection(options.projection) + : options.projection + ); + if (options.skip) cursor = cursor.skip(options.skip); + if (options.limit) cursor = cursor.limit(options.limit); + cursor = cursor.sort({ _id: 1 }); + if (!callback) - collection - .find(q) - .skip(skip || 0) - .limit(limit || 0) - .toArray((err, docs) => { - if (err) return reject(err); - return resolve(docs); - }); + cursor.toArray((err, docs) => { + if (err) return reject(err); + if (resource === "devices") + docs = docs.map(d => mongodbFunctions.flattenDevice(d)); + return resolve(docs); + }); else - collection.find(q).forEach( + cursor.forEach( doc => { + if (resource === "devices") + doc = mongodbFunctions.flattenDevice(doc); callback(doc); }, err => { @@ -51,11 +83,22 @@ function query(resource, filter, limit, skip, callback) { function count(resource, filter) { let q; - if (Array.isArray(filter)) q = apiFunctions.filterToMongoQuery(filter); - else if (filter != null && !filter) return Promise.resolve([]); + filter = expression.evaluate(filter, null, null, Date.now()); + + if (Array.isArray(filter)) { + if (resource === "devices") + filter = mongodbFunctions.processDeviceFilter(filter); + else if (resource === "tasks") + filter = mongodbFunctions.processTasksFilter(filter); + else if (resource === "faults") + filter = mongodbFunctions.processFaultsFilter(filter); + q = mongodbFunctions.filterToMongoQuery(filter); + } else if (filter != null && !filter) { + return Promise.resolve([]); + } return new Promise((resolve, reject) => { - getClient().then(client => { + getClient(RESOURCE_DB[resource]).then(client => { const collection = client.collection(resource); collection.find(q).count((err, c) => { if (err) reject(err); diff --git a/server/device.js b/server/device.js deleted file mode 100644 index 28da8d0..0000000 --- a/server/device.js +++ /dev/null @@ -1,191 +0,0 @@ -"use strict"; - -function recursive(input, root, output, timestamp) { - for (let [name, tree] of Object.entries(input)) { - if (root.length === 0) - if (name === "_lastInform") { - output["Events.Inform"] = { - value: [Date.parse(tree), "xsd:dateTime"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp - }; - } else if (name === "_registered") { - output["Events.Registered"] = { - value: [Date.parse(tree), "xsd:dateTime"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp - }; - } else if (name === "_lastBoot") { - output["Events.1_BOOT"] = { - value: [Date.parse(tree), "xsd:dateTime"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp - }; - } else if (name === "_lastBootstrap") { - output["Events.0_BOOTSTRAP"] = { - value: [Date.parse(tree), "xsd:dateTime"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp - }; - } else if (name === "_id") { - output["DeviceID.ID"] = { - value: [tree, "xsd:string"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp - }; - } else if (name === "_deviceId") { - output["DeviceID.Manufacturer"] = { - value: [tree["_Manufacturer"], "xsd:string"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp - }; - output["DeviceID.OUI"] = { - value: [tree["_OUI"], "xsd:string"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp - }; - output["DeviceID.ProductClass"] = { - value: [tree["_ProductClass"], "xsd:string"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp - }; - output["DeviceID.SerialNumber"] = { - value: [tree["_SerialNumber"], "xsd:string"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp - }; - } else if (name === "_tags") { - output["Tags"] = { - writable: true, - writableTimestamp: timestamp, - object: true, - objectTimestamp: timestamp - }; - - for (let t of tree) - output[`Tags.${t}`] = { - value: [true, "xsd:boolean"], - valueTimestamp: timestamp, - writable: false, - writableTimestamp: timestamp, - object: false, - objectTimestamp: timestamp - }; - } - - if (name.startsWith("_")) continue; - - let childrenTimestamp = timestamp; - - if (root.length === 0) childrenTimestamp = input["_timestamp"] || 1; - else if (input["_timestamp"] > timestamp) - childrenTimestamp = input["_timestamp"]; - - const attrs = {}; - if (tree["_value"] != null) { - attrs["value"] = [tree["_value"], tree["_type"]]; - attrs["valueTimestamp"] = tree["_timestamp"] || childrenTimestamp; - attrs["object"] = false; - attrs["objectTimestamp"] = childrenTimestamp; - } else if (tree["_object"] != null) { - attrs["object"] = tree["_object"]; - attrs["objectTimestamp"] = childrenTimestamp; - } - - if (tree["_writable"] != null) { - attrs["writable"] = tree["_writable"]; - attrs["writableTimestamp"] = childrenTimestamp; - } - - let r = root.concat(name); - output[r.join(".")] = attrs; - - if (attrs["object"]) recursive(tree, r, output, childrenTimestamp); - } -} - -function transpose(device) { - const newDevice = {}; - const timestamp = new Date(device["_lastInform"] || 1).getTime(); - recursive(device, [], newDevice, timestamp); - return newDevice; -} - -function transposeQuery(query) { - const newQuery = {}; - - for (let [key, val] of Object.entries(query)) - if (key.startsWith("$")) { - newQuery[key] = val.map(v => transposeQuery(v)); - } else if (key === "DeviceID.ID") { - newQuery["_id"] = val; - } else if (key === "DeviceID.SerialNumber") { - newQuery["_deviceId._SerialNumber"] = val; - } else if (key === "DeviceID.OUI") { - newQuery["_deviceId._OUI"] = val; - } else if (key === "DeviceID.ProductClass") { - newQuery["_deviceId._ProductClass"] = val; - } else if (key === "DeviceID.Manufacturer") { - newQuery["_deviceId._Manufacturer"] = val; - } else if (key.startsWith("Tags.")) { - if (val["$exists"] === true) { - newQuery["_tags"] = newQuery["_tags"] || {}; - newQuery["_tags"]["$all"] = newQuery["_tags"]["$all"] || []; - newQuery["_tags"]["$all"].push(key.slice(5)); - } else if (val["$exists"] === false) { - newQuery["_tags"] = newQuery["_tags"] || {}; - newQuery["_tags"]["$nin"] = newQuery["_tags"]["$nin"] || []; - newQuery["_tags"]["$nin"].push(key.slice(5)); - } - } else if (key === "tag") { - if (typeof val === "string") { - newQuery["_tags"] = newQuery["_tags"] || {}; - newQuery["_tags"]["$all"] = newQuery["_tags"]["$all"] || []; - newQuery["_tags"]["$all"].push(val); - } else if (val["$eq"]) { - newQuery["_tags"] = newQuery["_tags"] || {}; - newQuery["_tags"]["$all"] = newQuery["_tags"]["$all"] || []; - newQuery["_tags"]["$all"].push(val["$eq"]); - } else if (val["$ne"]) { - newQuery["_tags"] = newQuery["_tags"] || {}; - newQuery["_tags"]["$nin"] = newQuery["_tags"]["$nin"] || []; - newQuery["_tags"]["$nin"].push(val["$ne"]); - } - } else if (key === "Events.Inform") { - newQuery["_lastInform"] = val; - } else { - newQuery[key] = val; - } - - return newQuery; -} - -exports.transpose = transpose; -exports.transposeQuery = transposeQuery; diff --git a/server/mongodb-functions.js b/server/mongodb-functions.js new file mode 100644 index 0000000..a4751b1 --- /dev/null +++ b/server/mongodb-functions.js @@ -0,0 +1,320 @@ +"use strict"; +const mongodb = require("mongodb"); +const expressionParser = require("../common/expression-parser"); + +function processDeviceFilter(filter) { + return expressionParser.map(filter, exp => { + if (!Array.isArray(exp)) return exp; + + if (exp[0] === "PARAM") { + const p = exp[1]; + if (p === "DeviceID.ID") return ["PARAM", "_id"]; + else if (p === "DeviceID") return ["PARAM", "_deviceId"]; + else if (p.startsWith("DeviceID.")) + return ["PARAM", "_deviceId._" + p.slice(9)]; + else if (p === "Tags") return ["PARAM", "_tags"]; + else if (p === "tag") return ["PARAM", "_tags"]; + else if (p === "Events.Inform") return ["PARAM", "_lastInform"]; + else if (p === "Events.Registered") return ["PARAM", "_registered"]; + else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"]; + else if (p === "Events.1_BOOT") return ["PARAM", "_lastBoot"]; + } else if ( + Array.isArray(exp[1]) && + exp[1][0] === "PARAM" && + exp[1][1].startsWith("Tags.") + ) { + const t = exp[1][1].slice(5); + if (exp[0] === "IS NULL") return ["<>", ["PARAM", "_tags"], t]; + else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t]; + else if (exp[0] === "=" && exp[2] === true) + return ["=", ["PARAM", "_tags"], t]; + } else if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) { + let e = exp.slice(); + if ( + Array.isArray(exp[1]) && + exp[1][0] === "PARAM" && + typeof exp[1][1] === "string" && + !exp[1][1].startsWith("_") + ) + e[1] = ["PARAM", `${exp[1][1]}._value`]; + if (typeof e[2] === "number") { + let alt = e.slice(); + alt[2] = new Date(e[2]); + e = ["OR", e, alt]; + } + return e; + } + return exp; + }); +} + +function processTasksFilter(filter) { + return expressionParser.map(filter, exp => { + if (!Array.isArray(exp)) return exp; + if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) { + let e = exp.slice(); + if (e[1][0] === "PARAM" && e[1][1] === "_id") + e[2] = new mongodb.ObjectID(e[2]); + else if (e[1][0] === "PARAM" && e[1][1] === "timestamp") + e[2] = new Date(e[2]); + else if (e[1][0] === "PARAM" && e[1][1] === "expiry") + e[2] = new Date(e[2]); + return e; + } + return exp; + }); +} + +function processFaultsFilter(filter) { + return expressionParser.map(filter, exp => { + if (!Array.isArray(exp)) return exp; + if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) { + let e = exp.slice(); + if (e[1][0] === "PARAM" && e[1][1] === "timestamp") e[2] = new Date(e[2]); + else if (e[1][0] === "PARAM" && e[1][1] === "expiry") + e[2] = new Date(e[2]); + return e; + } + return exp; + }); +} + +function filterToMongoQuery(filter, negate = false, res = {}) { + const op = filter[0]; + + if ((!negate && op === "AND") || (negate && op === "OR")) { + res["$and"] = res["$and"] || []; + for (let i = 1; i < filter.length; ++i) + res["$and"].push(filterToMongoQuery(filter[i], negate)); + } else if ((!negate && op === "OR") || (negate && op === "AND")) { + res["$or"] = res["$or"] || []; + + for (let i = 1; i < filter.length; ++i) + res["$or"].push(filterToMongoQuery(filter[i], negate)); + } else if (op === "NOT") { + filterToMongoQuery(filter[1], !negate, res); + } else if (op === "=") { + const param = filter[1][1]; + let p = (res[param] = res[param] || {}); + if (negate) p["$ne"] = filter[2]; + else p["$eq"] = filter[2]; + } else if (op === "<>") { + const param = filter[1][1]; + let p = (res[param] = res[param] || {}); + if (negate) p = p["$not"] = p["$not"] || {}; + p["$ne"] = filter[2]; + p["$exists"] = true; + } else if (op === ">") { + const param = filter[1][1]; + let p = (res[param] = res[param] || {}); + if (negate) p = p["$not"] = p["$not"] || {}; + p["$gt"] = filter[2]; + } else if (op === ">=") { + const param = filter[1][1]; + let p = (res[param] = res[param] || {}); + if (negate) p = p["$not"] = p["$not"] || {}; + p["$gte"] = filter[2]; + } else if (op === "<") { + const param = filter[1][1]; + let p = (res[param] = res[param] || {}); + if (negate) p = p["$not"] = p["$not"] || {}; + p["$lt"] = filter[2]; + } else if (op === "<=") { + const param = filter[1][1]; + let p = (res[param] = res[param] || {}); + if (negate) p = p["$not"] = p["$not"] || {}; + p["$lte"] = filter[2]; + } else if (op === "IS NULL") { + const param = filter[1][1]; + res[param] = { $exists: negate }; + } else if (op === "IS NOT NULL") { + const param = filter[1][1]; + res[param] = { $exists: !negate }; + } else { + throw new Error(`Unrecognized operator ${op}`); + } + + return res; +} + +function processDeviceProjection(projection) { + if (!projection) return projection; + const p = {}; + for (const [k, v] of Object.entries(projection)) + if (k === "DeviceID.ID") { + p["_id"] = 1; + } else if (k.startsWith("DeviceID")) { + p["_deviceId._SerialNumber"] = v; + p["_deviceId._OUI"] = v; + p["_deviceId._ProductClass"] = v; + p["_deviceId._Manufacturer"] = v; + } else if (k.startsWith("Tags")) { + p["_tags"] = v; + } else if (k.startsWith("Events")) { + p["_lastInform"] = v; + p["_registered"] = v; + p["_lastBoot"] = v; + p["_lastBootstrap"] = v; + } else { + p[k] = v; + } + + return p; +} + +function processDeviceSort(sort) { + if (!sort) return sort; + const s = {}; + for (const [k, v] of Object.entries(sort)) + if (k === "DeviceID.ID") s["_id"] = v; + else if (k === "Events.Inform") s["_lastInform"] = v; + else s[k] = v; + + return s; +} + +function flattenDevice(device) { + function recursive(input, root, output, timestamp) { + for (let [name, tree] of Object.entries(input)) { + if (root.length === 0) + if (name === "_lastInform") { + output["Events.Inform"] = { + value: [Date.parse(tree), "xsd:dateTime"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp + }; + } else if (name === "_registered") { + output["Events.Registered"] = { + value: [Date.parse(tree), "xsd:dateTime"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp + }; + } else if (name === "_lastBoot") { + output["Events.1_BOOT"] = { + value: [Date.parse(tree), "xsd:dateTime"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp + }; + } else if (name === "_lastBootstrap") { + output["Events.0_BOOTSTRAP"] = { + value: [Date.parse(tree), "xsd:dateTime"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp + }; + } else if (name === "_id") { + output["DeviceID.ID"] = { + value: [tree, "xsd:string"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp + }; + } else if (name === "_deviceId") { + output["DeviceID.Manufacturer"] = { + value: [tree["_Manufacturer"], "xsd:string"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp + }; + output["DeviceID.OUI"] = { + value: [tree["_OUI"], "xsd:string"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp + }; + output["DeviceID.ProductClass"] = { + value: [tree["_ProductClass"], "xsd:string"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp + }; + output["DeviceID.SerialNumber"] = { + value: [tree["_SerialNumber"], "xsd:string"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp + }; + } else if (name === "_tags") { + output["Tags"] = { + writable: true, + writableTimestamp: timestamp, + object: true, + objectTimestamp: timestamp + }; + + for (let t of tree) + output[`Tags.${t}`] = { + value: [true, "xsd:boolean"], + valueTimestamp: timestamp, + writable: false, + writableTimestamp: timestamp, + object: false, + objectTimestamp: timestamp + }; + } + + if (name.startsWith("_")) continue; + + let childrenTimestamp = timestamp; + + if (root.length === 0) childrenTimestamp = +(input["_timestamp"] || 1); + else if (+input["_timestamp"] > timestamp) + childrenTimestamp = +input["_timestamp"]; + + const attrs = {}; + if (tree["_value"] != null) { + attrs["value"] = [tree["_value"], tree["_type"]]; + attrs["valueTimestamp"] = +(tree["_timestamp"] || childrenTimestamp); + attrs["object"] = false; + attrs["objectTimestamp"] = childrenTimestamp; + } else if (tree["_object"] != null) { + attrs["object"] = tree["_object"]; + attrs["objectTimestamp"] = childrenTimestamp; + } + + if (tree["_writable"] != null) { + attrs["writable"] = tree["_writable"]; + attrs["writableTimestamp"] = childrenTimestamp; + } + + let r = root.concat(name); + output[r.join(".")] = attrs; + + if (attrs["object"]) recursive(tree, r, output, childrenTimestamp); + } + } + + const newDevice = {}; + const timestamp = new Date(device["_lastInform"] || 1).getTime(); + recursive(device, [], newDevice, timestamp); + return newDevice; +} + +exports.processDeviceFilter = processDeviceFilter; +exports.processTasksFilter = processTasksFilter; +exports.processFaultsFilter = processFaultsFilter; +exports.filterToMongoQuery = filterToMongoQuery; +exports.processDeviceProjection = processDeviceProjection; +exports.processDeviceSort = processDeviceSort; +exports.flattenDevice = flattenDevice;