Merge remote-tracking branch 'ui/master'

lib/ui/api-functions.js

@@ -0,0 +1,372 @@
+"use strict";
+
+const http = require("http");
+const https = require("https");
+const url = require("url");
+
+const config = require("./config");
+const db = require("./db");
+const mongodbFunctions = require("./mongodb-functions");
+
+function filterToMongoQuery(filter, negate = false, res = {}) {
+  const op = filter[0];
+
+  if ((!negate && op === "AND") || (negate && op === "OR")) {
+    res["$and"] = res["$and"] || [];
+    for (let i = 1; i < filter.length; ++i)
+      res["$and"].push(filterToMongoQuery(filter[i], negate));
+  } else if ((!negate && op === "OR") || (negate && op === "AND")) {
+    res["$or"] = res["$or"] || [];
+
+    for (let i = 1; i < filter.length; ++i)
+      res["$or"].push(filterToMongoQuery(filter[i], negate));
+  } else if (op === "NOT") {
+    filterToMongoQuery(filter[1], !negate, res);
+  } else if (op === "=") {
+    const param = filter[1][1];
+    const p = (res[param] = res[param] || {});
+    if (negate) p["$ne"] = filter[2];
+    else p["$eq"] = filter[2];
+  } else if (op === "<>") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$ne"] = filter[2];
+    p["$exists"] = true;
+  } else if (op === ">") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$gt"] = filter[2];
+  } else if (op === ">=") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$gte"] = filter[2];
+  } else if (op === "<") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$lt"] = filter[2];
+  } else if (op === "<=") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$lte"] = filter[2];
+  } else if (op === "IS NULL") {
+    const param = filter[1][1];
+    res[param] = { $exists: negate };
+  } else if (op === "IS NOT NULL") {
+    const param = filter[1][1];
+    res[param] = { $exists: !negate };
+  } else {
+    throw new Error(`Unrecognized operator ${op}`);
+  }
+
+  return res;
+}
+
+function deleteResource(resource, id) {
+  return new Promise((resolve, reject) => {
+    const resourceUrl =
+      resource === "virtualParameters" ? "virtual_parameters" : resource;
+    const options = url.parse(
+      `${config.server.nbi}${resourceUrl}/${encodeURIComponent(id)}`
+    );
+
+    options.method = "DELETE";
+
+    const _http = options.protocol === "https:" ? https : http;
+
+    _http
+      .request(options, res => {
+        res.resume();
+        if (res.statusCode === 200) resolve(true);
+        else if (res.statusCode === 404) resolve(false);
+        else reject(new Error(`Unexpected status code ${res.statusCode}`));
+      })
+      .end();
+  });
+}
+
+function postTask(task, connectionRequest, callback) {
+  const options = url.parse(
+    `${config.server.nbi}devices/${encodeURIComponent(task.device)}/tasks${
+      connectionRequest ? "?connection_request" : ""
+    }`
+  );
+  options.method = "POST";
+  const _http = options.protocol === "https:" ? https : http;
+
+  _http
+    .request(options, res => {
+      if (res.statusCode !== 200 && res.statusCode !== 202) {
+        callback(new Error(`Unexpected status code ${res.statusCode}`));
+        res.resume();
+        return;
+      }
+      const chunks = [];
+      let bytes = 0;
+      res.on("data", chunk => {
+        chunks.push(chunk);
+        bytes += chunk.length;
+      });
+
+      res.on("end", () => {
+        const buf = Buffer.allocUnsafe(bytes);
+        let o = 0;
+        for (const c of chunks) {
+          c.copy(buf, o, 0, c.length);
+          o += c.length;
+        }
+        try {
+          const t = JSON.parse(buf);
+          let connectionRequestStatus;
+          if (connectionRequest) {
+            connectionRequestStatus = "OK";
+            if (
+              res.statusCode === 202 &&
+              res.statusMessage !== "Task queued but not processed" &&
+              res.statusMessage !== "Task faulted"
+            )
+              connectionRequestStatus = res.statusMessage;
+          }
+          callback(null, t, connectionRequestStatus);
+        } catch (err) {
+          callback(err);
+        }
+      });
+    })
+    .end(JSON.stringify(task));
+}
+
+function postTasks(deviceId, tasks) {
+  return new Promise((resolve, reject) => {
+    let connectionRequestStatus;
+    const promises = [];
+    for (const [idx, task] of tasks.entries()) {
+      task.device = deviceId;
+      promises.push(
+        new Promise((res, rej) => {
+          const conReq = idx === tasks.length - 1;
+          delete task._id;
+          task.expiry = 5;
+          postTask(task, conReq, (err, t, crs) => {
+            if (err) return void rej(err);
+            if (conReq) connectionRequestStatus = crs;
+            res({ _id: t._id, status: "pending" });
+          });
+        })
+      );
+    }
+
+    Promise.all(promises)
+      .then(statuses => {
+        if (connectionRequestStatus !== "OK") {
+          return void resolve({
+            connectionRequest: connectionRequestStatus,
+            tasks: statuses
+          });
+        }
+
+        const promises2 = [];
+        for (const s of statuses) {
+          promises2.push(db.query("tasks", ["=", ["PARAM", "_id"], s._id]));
+          promises2.push(
+            db.query("faults", [
+              "=",
+              ["PARAM", "_id"],
+              `${deviceId}:task_${s._id}`
+            ])
+          );
+        }
+
+        Promise.all(promises2)
+          .then(res => {
+            for (const [i, r] of statuses.entries()) {
+              if (res[i * 2].length === 0) {
+                r.status = "done";
+              } else if (res[i * 2 + 1].length === 1) {
+                r.status = "fault";
+                r.fault = res[i * 2 + 1][0];
+              }
+              deleteResource("tasks", r._id);
+            }
+
+            resolve({
+              connectionRequest: connectionRequestStatus,
+              tasks: statuses
+            });
+          })
+          .catch(reject);
+      })
+      .catch(reject);
+  });
+}
+
+function updateTags(deviceId, tags) {
+  return Promise.all(
+    Object.entries(tags).map(
+      ([tag, onOff]) =>
+        new Promise((resolve, reject) => {
+          const options = url.parse(
+            `${config.server.nbi}devices/${encodeURIComponent(
+              deviceId
+            )}/tags/${encodeURIComponent(tag)}`
+          );
+          if (onOff) options.method = "POST";
+          else options.method = "DELETE";
+          const _http = options.protocol === "https:" ? https : http;
+
+          _http
+            .request(options, res => {
+              res.resume();
+              if (res.statusCode === 200) resolve();
+              else
+                reject(new Error(`Unexpected status code ${res.statusCode}`));
+            })
+            .end();
+        })
+    )
+  );
+}
+
+function ping(host) {
+  return new Promise((resolve, reject) => {
+    const options = url.parse(
+      `${config.server.nbi}ping/${encodeURIComponent(host)}`
+    );
+
+    const _http = options.protocol === "https:" ? https : http;
+    _http
+      .request(options, res => {
+        if (res.statusCode === 404) {
+          res.resume();
+          return void resolve({});
+        }
+
+        if (res.statusCode !== 200) {
+          res.resume();
+          return void reject(
+            new Error(`Unexpected status code ${res.statusCode}`)
+          );
+        }
+
+        const chunks = [];
+        let bytes = 0;
+        res.on("data", chunk => {
+          chunks.push(chunk);
+          bytes += chunk.length;
+        });
+
+        res.on("end", () => {
+          const buf = Buffer.allocUnsafe(bytes);
+          let o = 0;
+          for (const c of chunks) {
+            c.copy(buf, o, 0, c.length);
+            o += c.length;
+          }
+          const m = buf
+            .toString()
+            .match(
+              /(\d) packets transmitted, (\d) received, ([\d.%]+) packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/
+            );
+          if (!m)
+            return void reject(new Error("Could not parse ping response"));
+
+          resolve({
+            packetsTransmitted: +m[1],
+            packetsReceived: +m[2],
+            packetLoss: m[3],
+            min: +m[4],
+            avg: +m[5],
+            max: +m[6],
+            mdev: +m[7]
+          });
+        });
+      })
+      .end();
+  });
+}
+
+function putResource(resource, id, data) {
+  return new Promise((resolve, reject) => {
+    const resourceUrl =
+      resource === "virtualParameters" ? "virtual_parameters" : resource;
+    const options = url.parse(
+      `${config.server.nbi}${resourceUrl}/${encodeURIComponent(id)}`
+    );
+
+    options.method = "PUT";
+
+    const _http = options.protocol === "https:" ? https : http;
+
+    if (resource === "presets") data = mongodbFunctions.preProcessPreset(data);
+
+    if (resource === "provisions")
+      data = mongodbFunctions.preProcessProvision(data);
+
+    if (resource === "virtualParameters")
+      data = mongodbFunctions.preProcessVirtualParameters(data);
+
+    const body = typeof data !== "string" ? JSON.stringify(data) : data;
+    _http
+      .request(options, res => {
+        const chunks = [];
+        let bytes = 0;
+        res.on("data", chunk => {
+          chunks.push(chunk);
+          bytes += chunk.length;
+        });
+
+        res.on("end", () => {
+          if (res.statusCode === 200) {
+            return void resolve();
+          } else if (res.statusCode !== 400) {
+            return void reject(
+              new Error(`Unexpected status code ${res.statusCode}`)
+            );
+          }
+          const buf = Buffer.allocUnsafe(bytes);
+          let o = 0;
+          for (const c of chunks) {
+            c.copy(buf, o, 0, c.length);
+            o += c.length;
+          }
+          resolve(buf.toString());
+        });
+      })
+      .end(body);
+  });
+}
+
+function putFile(id, metadata, fileStream) {
+  return new Promise((resolve, reject) => {
+    const options = url.parse(
+      `${config.server.nbi}files/${encodeURIComponent(id)}`
+    );
+
+    options.method = "PUT";
+    options.headers = Object.assign({}, metadata);
+    const _http = options.protocol === "https:" ? https : http;
+    const req = _http.request(options, res => {
+      res.resume();
+      if (res.statusCode === 201) {
+        return void resolve();
+      } else {
+        return void reject(
+          new Error(`Unexpected status code ${res.statusCode}`)
+        );
+      }
+    });
+    fileStream.pipe(req);
+  });
+}
+
+exports.postTasks = postTasks;
+exports.deleteResource = deleteResource;
+exports.putResource = putResource;
+exports.updateTags = updateTags;
+exports.filterToMongoQuery = filterToMongoQuery;
+exports.ping = ping;
+exports.putFile = putFile;

lib/ui/api.js

@@ -0,0 +1,485 @@
+"use strict";
+
+const stream = require("stream");
+const Router = require("koa-router");
+const db = require("./db");
+const apiFunctions = require("./api-functions");
+const expression = require("../common/expression");
+const logger = require("./logger");
+
+const router = new Router();
+
+function logUnauthorizedWarning(log) {
+  log.message += " not authorized";
+  logger.accessWarn(log);
+}
+
+const RESOURCE_DELETE = 1 << 0;
+const RESOURCE_PUT = 1 << 1;
+
+const RESOURCE_IDS = {
+  devices: "DeviceID.ID",
+  presets: "_id",
+  provisions: "_id",
+  files: "_id",
+  virtualParameters: "_id",
+  faults: "_id",
+  tasks: "_id"
+};
+
+const resources = {
+  devices: 0 | RESOURCE_DELETE,
+  presets: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  provisions: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  files: 0 | RESOURCE_DELETE,
+  virtualParameters: 0 | RESOURCE_DELETE | RESOURCE_PUT,
+  faults: 0 | RESOURCE_DELETE,
+  tasks: 0
+};
+
+router.get(`/devices/:id.csv`, async ctx => {
+  const log = {
+    message: "Query device (CSV)",
+    context: ctx,
+    id: ctx.params.id
+  };
+
+  const filter = ["=", ["PARAM", RESOURCE_IDS.devices], ctx.params.id];
+
+  if (!ctx.state.authorizer.hasAccess("devices", 2)) {
+    logUnauthorizedWarning(log);
+    return void (ctx.status = 404);
+  }
+
+  const res = await db.query("devices", filter);
+  if (!res[0]) return void (ctx.status = 404);
+
+  ctx.type = "text/csv";
+  ctx.attachment(
+    `device-${ctx.params.id}-${new Date()
+      .toISOString()
+      .replace(/[:.]/g, "")}.csv`
+  );
+
+  ctx.body = new stream.PassThrough();
+  ctx.body.write(
+    "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp\n"
+  );
+
+  for (const k of Object.keys(res[0]).sort()) {
+    const p = res[0][k];
+    const row = [
+      k,
+      p.object,
+      p.objectTimestamp,
+      p.writable,
+      p.writableTimestamp,
+      p.value != null ? `"${p.value[0].toString().replace(/"/g, '""')}"` : "",
+      p.value != null ? p.value[1] : "",
+      p.valueTimestamp
+    ];
+    ctx.body.write(row.map(r => (r != null ? r : "")).join(",") + "\n");
+  }
+  ctx.body.end();
+  logger.accessInfo(log);
+});
+
+for (const [resource, flags] of Object.entries(resources)) {
+  router.head(`/${resource}`, async (ctx, next) => {
+    let filter = true;
+    if (ctx.request.query.filter)
+      filter = expression.parse(ctx.request.query.filter);
+
+    const log = {
+      message: `Count ${resource}`,
+      context: ctx,
+      filter: ctx.request.query.filter
+    };
+
+    if (!ctx.state.authorizer.hasAccess(resource, 1)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    // Exclude temporary tasks and faults
+    if (resource === "tasks" || resource === "faults") {
+      filter = expression.and(filter, [
+        "NOT",
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+      ]);
+    }
+
+    const count = await db.count(resource, filter);
+
+    ctx.set("X-Total-Count", count);
+    ctx.body = "";
+
+    log.count = count;
+    logger.accessInfo(log);
+  });
+
+  router.get(`/${resource}`, async (ctx, next) => {
+    const options = {};
+    let filter = true;
+    if (ctx.request.query.filter)
+      filter = expression.parse(ctx.request.query.filter);
+    if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
+    if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
+    if (ctx.request.query.sort)
+      options.sort = JSON.parse(ctx.request.query.sort);
+    if (ctx.request.query.projection) {
+      options.projection = ctx.request.query.projection
+        .split(",")
+        .reduce((obj, k) => Object.assign(obj, { [k]: 1 }), {});
+    }
+
+    const log = {
+      message: `Query ${resource}`,
+      context: ctx,
+      filter: ctx.request.query.filter,
+      limit: options.limit,
+      skip: options.skip,
+      sort: options.sort,
+      projection: options.projection
+    };
+
+    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    // Exclude temporary tasks and faults
+    if (resource === "tasks" || resource === "faults") {
+      filter = expression.and(filter, [
+        "NOT",
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+      ]);
+    }
+
+    ctx.body = new stream.PassThrough();
+    ctx.type = "application/json";
+
+    let c = 0;
+    ctx.body.write("[\n");
+    await db.query(resource, filter, options, obj => {
+      ctx.body.write((c++ ? "," : "") + JSON.stringify(obj) + "\n");
+    });
+    ctx.body.end("]");
+
+    logger.accessInfo(log);
+  });
+
+  // CSV download
+  router.get(`/${resource}.csv`, async (ctx, next) => {
+    const options = { projection: {} };
+    let filter = true;
+    if (ctx.request.query.filter)
+      filter = expression.parse(ctx.request.query.filter);
+    if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
+    if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
+    if (ctx.request.query.sort)
+      options.sort = JSON.parse(ctx.request.query.sort);
+
+    const log = {
+      message: `Query ${resource} (CSV)`,
+      context: ctx,
+      filter: ctx.request.query.filter,
+      limit: options.limit,
+      skip: options.skip,
+      sort: options.sort
+    };
+
+    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    const columns = JSON.parse(ctx.request.query.columns);
+    const now = Date.now();
+
+    for (const [k, v] of Object.entries(columns)) {
+      const e = expression.evaluate(expression.parse(v), null, now);
+      columns[k] = e;
+      for (const p of expression.extractParams(e)) options.projection[p] = 1;
+    }
+
+    // Exclude temporary tasks and faults
+    if (resource === "tasks" || resource === "faults") {
+      filter = expression.and(filter, [
+        "NOT",
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+      ]);
+    }
+
+    ctx.body = new stream.PassThrough();
+    ctx.type = "text/csv";
+    ctx.attachment(
+      `${resource}-${new Date(now).toISOString().replace(/[:.]/g, "")}.csv`
+    );
+
+    ctx.body.write(
+      Object.keys(columns).map(k => `"${k.replace(/"/, '""')}"`) + "\n"
+    );
+    await db.query(resource, filter, options, obj => {
+      const arr = Object.values(columns).map(exp => {
+        const v = expression.evaluate(exp, obj, null, e => {
+          if (Array.isArray(e)) {
+            if (e[0] === "PARAM") {
+              if (resource === "devices") {
+                if (e[1] === "Tags") {
+                  const tags = [];
+                  for (const p in obj)
+                    if (p.startsWith("Tags.")) tags.push(p.slice(5));
+
+                  return tags.join(", ");
+                }
+              }
+            } else if (e[0] === "FUNC") {
+              if (e[1] === "DATE_STRING") {
+                if (e[2] && !Array.isArray(e[2]))
+                  return new Date(e[2]).toISOString();
+              }
+            }
+          }
+
+          return e;
+        });
+
+        if (Array.isArray(v) || v == null) return "";
+        if (typeof v === "string") return `"${v.replace(/"/g, '""')}"`;
+        return v;
+      });
+      ctx.body.write(arr.join(",") + "\n");
+    });
+    ctx.body.end();
+
+    logger.accessInfo(log);
+  });
+
+  router.head(`/${resource}/:id`, async (ctx, next) => {
+    const log = {
+      message: `Count ${resource}`,
+      context: ctx,
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
+    };
+
+    const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
+    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    const res = await db.query(resource, filter);
+
+    if (!res.length) return void next();
+
+    logger.accessInfo(log);
+    ctx.body = "";
+  });
+
+  router.get(`/${resource}/:id`, async (ctx, next) => {
+    const log = {
+      message: `Query ${resource}`,
+      context: ctx,
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
+    };
+
+    const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
+    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+      logUnauthorizedWarning(log);
+      return void next();
+    }
+
+    const res = await db.query(resource, filter);
+
+    if (!res.length) return void next();
+
+    logger.accessInfo(log);
+    ctx.body = res[0];
+  });
+
+  // TODO add PUT, PATCH routes
+  if (flags & RESOURCE_DELETE) {
+    router.delete(`/${resource}/:id`, async (ctx, next) => {
+      const log = {
+        message: `Delete ${resource}`,
+        context: ctx,
+        id: ctx.params.id
+      };
+
+      const authorizer = ctx.state.authorizer;
+      const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
+      if (!authorizer.hasAccess(resource, 3)) {
+        logUnauthorizedWarning(log);
+        return void next();
+      }
+      const res = await db.query(resource, filter);
+      if (!res.length) return void next();
+
+      const validate = authorizer.getValidator(resource, res[0]);
+      if (!validate("delete")) {
+        logUnauthorizedWarning(log);
+        return void (ctx.status = 403);
+      }
+
+      await apiFunctions.deleteResource(resource, ctx.params.id);
+
+      logger.accessInfo(log);
+
+      ctx.body = "";
+    });
+  }
+
+  if (flags & RESOURCE_PUT) {
+    router.put(`/${resource}/:id`, async (ctx, next) => {
+      const id = ctx.params.id;
+
+      const log = {
+        message: `Put ${resource}`,
+        context: ctx,
+        id: id
+      };
+
+      const authorizer = ctx.state.authorizer;
+      if (!authorizer.hasAccess(resource, 3)) {
+        logUnauthorizedWarning(log);
+        return void next();
+      }
+
+      const obj = ctx.request.body;
+
+      const validate = authorizer.getValidator(resource, obj);
+      if (!validate("put")) {
+        logUnauthorizedWarning(log);
+        return void (ctx.status = 403);
+      }
+
+      const err = await apiFunctions.putResource(resource, id, obj);
+
+      if (err) {
+        log.message += " failed";
+        logger.accessWarn(log);
+        ctx.body = err;
+        return void (ctx.status = 400);
+      }
+
+      logger.accessInfo(log);
+
+      ctx.body = "";
+    });
+  }
+}
+
+router.put("/files/:id", async (ctx, next) => {
+  const resource = "files";
+  const id = ctx.params.id;
+
+  const log = {
+    message: `Upload ${resource}`,
+    context: ctx,
+    id: id
+  };
+
+  const authorizer = ctx.state.authorizer;
+  if (!authorizer.hasAccess(resource, 3)) {
+    logUnauthorizedWarning(log);
+    return void next();
+  }
+
+  const metadata = {
+    fileType: ctx.request.headers["metadata.filetype"] || "",
+    oui: ctx.request.headers["metadata.oui"] || "",
+    productClass: ctx.request.headers["metadata.productclass"] || "",
+    version: ctx.request.headers["metadata.version"] || ""
+  };
+
+  const validate = authorizer.getValidator(resource, metadata);
+  if (!validate("put")) {
+    logUnauthorizedWarning(log);
+    return void (ctx.status = 403);
+  }
+
+  const err = await apiFunctions.putFile(id, metadata, ctx.req);
+
+  if (err) {
+    log.message += " failed";
+    logger.accessWarn(log);
+    ctx.body = err;
+    return void (ctx.status = 400);
+  }
+
+  log.metadata = metadata;
+  logger.accessInfo(log);
+
+  ctx.body = "";
+});
+
+router.post("/devices/:id/tasks", async (ctx, next) => {
+  const log = {
+    message: "Commit tasks",
+    context: ctx,
+    deviceId: ctx.params.id
+  };
+
+  const authorizer = ctx.state.authorizer;
+  const filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
+  if (!authorizer.hasAccess("devices", 3)) {
+    logUnauthorizedWarning(log);
+    return void next();
+  }
+  const devices = await db.query("devices", filter);
+  if (!devices.length) return void next();
+
+  const validate = authorizer.getValidator("devices", devices[0]);
+  for (const t of ctx.request.body) {
+    if (!validate("task", t)) {
+      logUnauthorizedWarning(log);
+      return void (ctx.status = 403);
+    }
+  }
+
+  const res = await apiFunctions.postTasks(ctx.params.id, ctx.request.body);
+
+  log.tasks = res.tasks.map(t => t._id).join(",");
+
+  logger.accessInfo(log);
+
+  ctx.set("Connection-Request", res.connectionRequest);
+  ctx.body = res.tasks;
+});
+
+router.post("/devices/:id/tags", async (ctx, next) => {
+  const log = {
+    message: "Update tags",
+    context: ctx,
+    deviceId: ctx.params.id,
+    tags: ctx.request.body
+  };
+
+  const authorizer = ctx.state.authorizer;
+  const filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
+  if (!authorizer.hasAccess("devices", 3)) {
+    logUnauthorizedWarning(log);
+    return void next();
+  }
+  const res = await db.query("devices", filter);
+  if (!res.length) return void next();
+
+  const validate = authorizer.getValidator("devices", res[0]);
+  if (!validate("tags", ctx.request.body)) {
+    logUnauthorizedWarning(log);
+    return void (ctx.status = 403);
+  }
+
+  await apiFunctions.updateTags(ctx.params.id, ctx.request.body);
+
+  logger.accessInfo(log);
+
+  ctx.body = "";
+});
+
+router.get("/ping/:host", async ctx => {
+  ctx.body = await apiFunctions.ping(ctx.params.host);
+});
+
+module.exports = router;

lib/ui/cluster.js

@@ -0,0 +1,110 @@
+"use strict";
+
+const cluster = require("cluster");
+const config = require("./config");
+const logger = require("./logger");
+
+const VERSION = require("../../package.json")["version"];
+
+let respawnTimestamp = 0;
+let tooManyCrashesTimestamp = 0;
+
+function restartWorker(worker, code, signal) {
+  const msg = {
+    message: "Worker died",
+    pid: worker.process.pid
+  };
+
+  if (code != null) msg.exitCode = code;
+
+  if (signal != null) msg.signal = signal;
+
+  logger.error(msg);
+
+  const now = Date.now();
+  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  if (respawnTimestamp === now) {
+    tooManyCrashesTimestamp = now;
+    cluster.fork();
+    return;
+  }
+
+  if (now - tooManyCrashesTimestamp > 60000) {
+    process.exitCode = 1;
+    cluster.removeListener("exit", restartWorker);
+    for (const pid in cluster.workers) cluster.workers[pid].kill();
+
+    logger.error({
+      message: "Too many crashes, exiting",
+      pid: process.pid
+    });
+    return;
+  }
+
+  setTimeout(() => {
+    if (process.exitCode) return;
+    cluster.fork();
+  }, respawnTimestamp - now);
+}
+
+function start() {
+  logger.init(VERSION);
+
+  logger.info({
+    message: "GenieACS UI starting",
+    pid: process.pid,
+    version: VERSION
+  });
+
+  cluster.on("listening", (worker, address) => {
+    logger.info({
+      message: "Worker listening",
+      pid: worker.process.pid,
+      address: address.address,
+      port: address.port
+    });
+  });
+
+  cluster.on("exit", restartWorker);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid
+    });
+
+    cluster.removeListener("exit", restartWorker);
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid
+    });
+
+    cluster.removeListener("exit", restartWorker);
+    for (const pid in cluster.workers) cluster.workers[pid].kill();
+  });
+
+  cluster.setupMaster({ exec: "./server" });
+
+  let workerCount = config.server.workers;
+
+  if (!workerCount) workerCount = Math.max(2, require("os").cpus().length);
+
+  for (let i = 0; i < workerCount; ++i) cluster.fork();
+}
+
+process.on("uncaughtException", err => {
+  logger.error({
+    message: "Uncaught exception in master process, exiting",
+    exception: err,
+    pid: process.pid
+  });
+  process.exitCode = 1;
+  cluster.removeListener("exit", restartWorker);
+  for (const pid in cluster.workers) cluster.workers[pid].kill();
+});
+
+// Starting GenieACS UI
+start();

lib/ui/config.js

@@ -0,0 +1,36 @@
+"use strict";
+
+const fs = require("fs");
+const configFile = JSON.parse(fs.readFileSync("./config-ui.json"));
+
+const config = loadConfig(configFile);
+
+function getClientConfig() {
+  return Object.assign({}, config, {
+    server: null,
+    auth: null,
+    permissions: null
+  });
+}
+
+function loadConfig(conf, root = {}) {
+  for (const [key, value] of Object.entries(conf)) {
+    const keys = key.split(".");
+    let ref = root;
+    while (keys.length > 1) {
+      const k = keys.shift();
+      if (typeof ref[k] !== "object") ref[k] = {};
+      ref = ref[k];
+    }
+
+    if (typeof value === "object") {
+      if (typeof ref[keys[0]] !== "object") ref[keys[0]] = {};
+      loadConfig(value, ref[keys[0]]);
+    } else {
+      ref[keys[0]] = value;
+    }
+  }
+  return root;
+}
+
+module.exports = Object.assign({}, config, { getClientConfig });

lib/ui/db.js

@@ -0,0 +1,212 @@
+"use strict";
+
+const mongodb = require("mongodb");
+const config = require("./config");
+const mongodbFunctions = require("./mongodb-functions");
+const expression = require("../common/expression");
+
+const CACHE_TTL = 300000;
+
+let _clientPromise = null;
+
+const RESOURCE_DB = {
+  devices: "genieacs",
+  faults: "genieacs",
+  tasks: "genieacs",
+  presets: "genieacs",
+  provisions: "genieacs",
+  virtualParameters: "genieacs",
+  files: "genieacs",
+  cache: "genieacs-ui"
+};
+
+const RESOURCE_COLLECTION = {
+  files: "fs.files"
+};
+
+function ensureIndexes(client) {
+  client
+    .db(RESOURCE_DB["cache"])
+    .collection("cache")
+    .createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
+}
+
+function getClient() {
+  if (!_clientPromise) {
+    _clientPromise = new Promise((resolve, reject) => {
+      const CONNECTION_URL = config.server.mongodbConnectionUrl;
+      mongodb.MongoClient.connect(
+        CONNECTION_URL,
+        { useNewUrlParser: true },
+        (err, client) => {
+          if (err) return void reject(err);
+          ensureIndexes(client);
+          resolve(client);
+        }
+      );
+    });
+  }
+
+  return _clientPromise;
+}
+
+function cache(key, valueGetter, ttl) {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const collection = client.db(RESOURCE_DB["cache"]).collection("cache");
+        collection.findOne({ _id: key }, (err, doc) => {
+          if (err) return void reject(err);
+          if (doc != null) return void resolve(JSON.parse(doc.value));
+          valueGetter()
+            .then(res => {
+              const expire = Date.now() + (ttl || CACHE_TTL);
+              const cacheDoc = {
+                _id: key,
+                value: JSON.stringify(res),
+                expire: new Date(expire)
+              };
+              collection.updateOne(
+                { _id: key },
+                { $set: cacheDoc },
+                { upsert: true },
+                err => {
+                  if (err) reject(err);
+                  else resolve(res);
+                }
+              );
+            })
+            .catch(reject);
+        });
+      })
+      .catch(reject);
+  });
+}
+
+function query(resource, filter, options, callback) {
+  options = options || {};
+  let q;
+  filter = expression.evaluate(filter, null, Date.now());
+
+  if (Array.isArray(filter)) {
+    if (resource === "devices")
+      filter = mongodbFunctions.processDeviceFilter(filter);
+    else if (resource === "tasks")
+      filter = mongodbFunctions.processTasksFilter(filter);
+    else if (resource === "faults")
+      filter = mongodbFunctions.processFaultsFilter(filter);
+    q = mongodbFunctions.filterToMongoQuery(filter);
+  } else if (filter != null && !filter) {
+    return Promise.resolve([]);
+  }
+
+  return new Promise((resolve, reject) => {
+    getClient().then(client => {
+      const collection = client
+        .db(RESOURCE_DB[resource])
+        .collection(RESOURCE_COLLECTION[resource] || resource);
+      const cursor = collection.find(q);
+      if (options.projection) {
+        cursor.project(
+          resource === "devices"
+            ? mongodbFunctions.processDeviceProjection(options.projection)
+            : options.projection
+        );
+      }
+      if (options.skip) cursor.skip(options.skip);
+      if (options.limit) cursor.limit(options.limit);
+
+      if (options.sort) {
+        let s = Object.entries(options.sort)
+          .sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
+          .reduce(
+            (obj, [k, v]) =>
+              Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
+            {}
+          );
+
+        if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
+        cursor.sort(s);
+      }
+
+      if (!callback) {
+        cursor.toArray((err, docs) => {
+          if (err) return reject(err);
+          if (resource === "devices")
+            docs = docs.map(d => mongodbFunctions.flattenDevice(d));
+          else if (resource === "faults")
+            docs = docs.map(d => mongodbFunctions.flattenFault(d));
+          else if (resource === "tasks")
+            docs = docs.map(d => mongodbFunctions.flattenTask(d));
+          else if (resource === "presets")
+            docs = docs.map(d => mongodbFunctions.flattenPreset(d));
+          else if (resource === "files")
+            docs = docs.map(d => mongodbFunctions.flattenFile(d));
+          return resolve(docs);
+        });
+      } else {
+        cursor.forEach(
+          doc => {
+            if (resource === "devices")
+              doc = mongodbFunctions.flattenDevice(doc);
+            else if (resource === "faults")
+              doc = mongodbFunctions.flattenFault(doc);
+            else if (resource === "tasks")
+              doc = mongodbFunctions.flattenTask(doc);
+            else if (resource === "presets")
+              doc = mongodbFunctions.flattenPreset(doc);
+            else if (resource === "files")
+              doc = mongodbFunctions.flattenFile(doc);
+            callback(doc);
+          },
+          err => {
+            if (err) reject(err);
+            else resolve();
+          }
+        );
+      }
+    });
+  });
+}
+
+function count(resource, filter) {
+  let q;
+  filter = expression.evaluate(filter, null, Date.now());
+
+  if (Array.isArray(filter)) {
+    if (resource === "devices")
+      filter = mongodbFunctions.processDeviceFilter(filter);
+    else if (resource === "tasks")
+      filter = mongodbFunctions.processTasksFilter(filter);
+    else if (resource === "faults")
+      filter = mongodbFunctions.processFaultsFilter(filter);
+    q = mongodbFunctions.filterToMongoQuery(filter);
+  } else if (filter != null && !filter) {
+    return Promise.resolve(0);
+  }
+
+  return new Promise((resolve, reject) => {
+    getClient().then(client => {
+      const collection = client
+        .db(RESOURCE_DB[resource])
+        .collection(RESOURCE_COLLECTION[resource] || resource);
+      collection.find(q).count((err, c) => {
+        if (err) reject(err);
+        else resolve(c);
+      });
+    });
+  });
+}
+
+function disconnect() {
+  if (_clientPromise) {
+    _clientPromise.then(client => {
+      client.close();
+    });
+  }
+}
+
+exports.query = query;
+exports.count = count;
+exports.cache = cache;
+exports.disconnect = disconnect;

lib/ui/logger.js

@@ -0,0 +1,249 @@
+"use strict";
+
+const fs = require("fs");
+const os = require("os");
+
+const config = require("./config");
+
+const REOPEN_EVERY = 60000;
+
+const LOG_FORMAT = config.logger.logFormat;
+const ACCESS_LOG_FORMAT = config.logger.accessLogFormat || LOG_FORMAT;
+
+const defaultMeta = {};
+
+let LOG_SYSTEMD = false;
+let ACCESS_LOG_SYSTEMD = false;
+
+let LOG_FILE, ACCESS_LOG_FILE;
+
+let logStream = fs.createWriteStream(null, { fd: process.stderr.fd });
+let logStat = fs.fstatSync(logStream.fd);
+let accessLogStream = fs.createWriteStream(null, { fd: process.stdout.fd });
+let accessLogStat = fs.fstatSync(accessLogStream.fd);
+
+// Reopen if original files have been moved (e.g. logrotate)
+function reopen() {
+  let counter = 1;
+
+  if (LOG_FILE) {
+    ++counter;
+    fs.stat(LOG_FILE, (err, stat) => {
+      if (err && !err.message.startsWith("ENOENT:")) throw err;
+
+      if (!(stat && stat.dev === logStat.dev && stat.ino === logStat.ino)) {
+        logStream.end();
+        logStream = fs.createWriteStream(null, {
+          fd: fs.openSync(LOG_FILE, "a")
+        });
+        logStat = fs.fstatSync(logStream.fd);
+      }
+
+      if (--counter === 0)
+        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+    });
+  }
+
+  if (ACCESS_LOG_FILE) {
+    ++counter;
+    fs.stat(ACCESS_LOG_FILE, (err, stat) => {
+      if (err && !err.message.startsWith("ENOENT:")) throw err;
+
+      if (
+        !(
+          stat &&
+          stat.dev === accessLogStat.dev &&
+          stat.ino === accessLogStat.ino
+        )
+      ) {
+        accessLogStream.end();
+        accessLogStream = fs.createWriteStream(null, {
+          fd: fs.openSync(ACCESS_LOG_FILE, "a")
+        });
+        accessLogStat = fs.fstatSync(accessLogStream.fd);
+      }
+
+      if (--counter === 0)
+        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+    });
+  }
+
+  if (--counter === 0)
+    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+}
+
+function init(version) {
+  defaultMeta.hostname = os.hostname();
+  defaultMeta.pid = process.pid;
+  defaultMeta.name = "genieacs-ui";
+  defaultMeta.version = version;
+
+  LOG_FILE = config.logger.logFile;
+  ACCESS_LOG_FILE = config.logger.accessLogFile;
+
+  if (LOG_FILE) {
+    logStream = fs.createWriteStream(null, { fd: fs.openSync(LOG_FILE, "a") });
+    logStat = fs.fstatSync(logStream.fd);
+  }
+
+  if (ACCESS_LOG_FILE) {
+    accessLogStream = fs.createWriteStream(null, {
+      fd: fs.openSync(ACCESS_LOG_FILE, "a")
+    });
+    accessLogStat = fs.fstatSync(accessLogStream.fd);
+  }
+
+  // Determine if logs are going to journald
+  const JOURNAL_STREAM = process.env["JOURNAL_STREAM"];
+
+  if (JOURNAL_STREAM) {
+    let [dev, inode] = JOURNAL_STREAM.split(":");
+    dev = parseInt(dev);
+    inode = parseInt(inode);
+
+    LOG_SYSTEMD = logStat.dev === dev && logStat.ino === inode;
+    ACCESS_LOG_SYSTEMD =
+      accessLogStat.dev === dev && accessLogStat.ino === inode;
+  }
+
+  if (LOG_FILE || ACCESS_LOG_FILE)
+    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+}
+
+function close() {
+  accessLogStream.end();
+  logStream.end();
+}
+
+function flatten(details) {
+  if (details.exception) {
+    details.exceptionName = details.exception.name;
+    details.exceptionMessage = details.exception.message;
+    details.exceptionStack = details.exception.stack;
+    delete details.exception;
+  }
+
+  if (details.context) {
+    details.remoteAddress = details.context.request.ip;
+    if (details.context.state.user)
+      details.user = details.context.state.user.username;
+    delete details.context;
+  }
+
+  for (const [k, v] of Object.entries(details))
+    if (v == null) delete details[k];
+
+  return details;
+}
+
+function formatJson(details, systemd) {
+  if (systemd) {
+    let severity = "";
+    if (details.severity === "info") severity = "<6>";
+    else if (details.severity === "warn") severity = "<4>";
+    else if (details.severity === "error") severity = "<3>";
+
+    return `${severity}${JSON.stringify(flatten(details))}${os.EOL}`;
+  }
+
+  return `${JSON.stringify(flatten(details))}${os.EOL}`;
+}
+
+function formatSimple(details, systemd) {
+  flatten(details);
+
+  let remote = "";
+  if (details.remoteAddress) {
+    if (details.user) remote = `${details.user}@${details.remoteAddress}: `;
+    else remote = `${details.remoteAddress}: `;
+  }
+
+  const keys = Object.keys(details);
+  const skip = {
+    user: 1,
+    remoteAddress: 1,
+    severity: 1,
+    timestamp: 1,
+    message: 1
+  };
+
+  let meta = "";
+
+  const kv = [];
+  for (const k of keys)
+    if (!skip[k]) kv.push(`${k}=${JSON.stringify(details[k])}`);
+
+  if (kv.length) meta = `; ${kv.join(" ")}`;
+
+  if (systemd) {
+    let severity = "";
+    if (details.severity === "info") severity = "<6>";
+    else if (details.severity === "warn") severity = "<4>";
+    else if (details.severity === "error") severity = "<3>";
+
+    return `${severity}${remote}${details.message}${meta}${os.EOL}`;
+  }
+
+  return `${details.timestamp} [${details.severity.toUpperCase()}] ${remote}${
+    details.message
+  }${meta}${os.EOL}`;
+}
+
+function log(details) {
+  details.timestamp = new Date().toISOString();
+  if (LOG_FORMAT === "json") {
+    details = Object.assign({}, defaultMeta, details);
+    logStream.write(formatJson(details, LOG_SYSTEMD));
+  } else {
+    logStream.write(formatSimple(details, LOG_SYSTEMD));
+  }
+}
+
+function info(details) {
+  details.severity = "info";
+  log(details);
+}
+
+function warn(details) {
+  details.severity = "warn";
+  log(details);
+}
+
+function error(details) {
+  details.severity = "error";
+  log(details);
+}
+
+function accessLog(details) {
+  details.timestamp = new Date().toISOString();
+  if (ACCESS_LOG_FORMAT === "json") {
+    Object.assign(details, defaultMeta);
+    accessLogStream.write(formatJson(details, ACCESS_LOG_SYSTEMD));
+  } else {
+    accessLogStream.write(formatSimple(details, ACCESS_LOG_SYSTEMD));
+  }
+}
+
+function accessInfo(details) {
+  details.severity = "info";
+  accessLog(details);
+}
+
+function accessWarn(details) {
+  details.severity = "warn";
+  accessLog(details);
+}
+
+function accessError(details) {
+  details.severity = "error";
+  accessLog(details);
+}
+
+exports.init = init;
+exports.close = close;
+exports.info = info;
+exports.warn = warn;
+exports.error = error;
+exports.accessInfo = accessInfo;
+exports.accessWarn = accessWarn;
+exports.accessError = accessError;

lib/ui/mongodb-functions.js

@@ -0,0 +1,550 @@
+"use strict";
+const mongodb = require("mongodb");
+const expressionParser = require("../common/expression-parser");
+const expression = require("../common/expression");
+
+function processDeviceFilter(filter) {
+  return expressionParser.map(filter, exp => {
+    if (!Array.isArray(exp)) return exp;
+
+    if (exp[0] === "PARAM") {
+      const p = exp[1];
+      if (p === "DeviceID.ID") return ["PARAM", "_id"];
+      else if (p === "DeviceID") return ["PARAM", "_deviceId"];
+      else if (p.startsWith("DeviceID."))
+        return ["PARAM", "_deviceId._" + p.slice(9)];
+      else if (p === "Tags") return ["PARAM", "_tags"];
+      else if (p === "tag") return ["PARAM", "_tags"];
+      else if (p === "Events.Inform") return ["PARAM", "_lastInform"];
+      else if (p === "Events.Registered") return ["PARAM", "_registered"];
+      else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"];
+      else if (p === "Events.1_BOOT") return ["PARAM", "_lastBoot"];
+    } else if (
+      Array.isArray(exp[1]) &&
+      exp[1][0] === "PARAM" &&
+      exp[1][1].startsWith("Tags.")
+    ) {
+      const t = exp[1][1].slice(5);
+      if (exp[0] === "IS NULL") return ["<>", ["PARAM", "_tags"], t];
+      else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t];
+      else if (exp[0] === "=" && exp[2] === true)
+        return ["=", ["PARAM", "_tags"], t];
+    } else if (
+      ["=", "<>", ">", ">=", "<", "<=", "LIKE", "NOT LIKE"].includes(exp[0])
+    ) {
+      let e = expressionParser.map(exp, ee => {
+        if (
+          Array.isArray(ee) &&
+          ee[0] === "PARAM" &&
+          typeof ee[1] === "string" &&
+          !ee[1].startsWith("_")
+        )
+          return ["PARAM", `${ee[1]}._value`];
+        return ee;
+      });
+
+      if (typeof e[2] === "number") {
+        const alt = e.slice();
+        alt[2] = new Date(e[2]);
+        e = ["OR", e, alt];
+      }
+      return e;
+    }
+    return exp;
+  });
+}
+
+function processTasksFilter(filter) {
+  return expressionParser.map(filter, exp => {
+    if (!Array.isArray(exp)) return exp;
+    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
+      const e = exp.slice();
+      if (e[1][0] === "PARAM" && e[1][1] === "_id")
+        e[2] = new mongodb.ObjectID(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "timestamp")
+        e[2] = new Date(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
+        e[2] = new Date(e[2]);
+      return e;
+    }
+    return exp;
+  });
+}
+
+function processFaultsFilter(filter) {
+  return expressionParser.map(filter, exp => {
+    if (!Array.isArray(exp)) return exp;
+    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
+      const e = exp.slice();
+      if (e[1][0] === "PARAM" && e[1][1] === "timestamp") e[2] = new Date(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
+        e[2] = new Date(e[2]);
+      return e;
+    }
+    return exp;
+  });
+}
+
+function filterToMongoQuery(filter, negate = false, res = {}) {
+  const op = filter[0];
+
+  if ((!negate && op === "AND") || (negate && op === "OR")) {
+    res["$and"] = res["$and"] || [];
+    for (let i = 1; i < filter.length; ++i)
+      res["$and"].push(filterToMongoQuery(filter[i], negate));
+  } else if ((!negate && op === "OR") || (negate && op === "AND")) {
+    res["$or"] = res["$or"] || [];
+
+    for (let i = 1; i < filter.length; ++i)
+      res["$or"].push(filterToMongoQuery(filter[i], negate));
+  } else if (op === "NOT") {
+    filterToMongoQuery(filter[1], !negate, res);
+  } else if (op === "=") {
+    const param = filter[1][1];
+    const p = (res[param] = res[param] || {});
+    if (negate) p["$ne"] = filter[2];
+    else p["$eq"] = filter[2];
+  } else if (op === "<>") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$ne"] = filter[2];
+    p["$exists"] = true;
+  } else if (op === ">") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$gt"] = filter[2];
+  } else if (op === ">=") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$gte"] = filter[2];
+  } else if (op === "<") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$lt"] = filter[2];
+  } else if (op === "<=") {
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
+    if (negate) p = p["$not"] = p["$not"] || {};
+    p["$lte"] = filter[2];
+  } else if (op === "IS NULL") {
+    const param = filter[1][1];
+    res[param] = { $exists: negate };
+  } else if (op === "IS NOT NULL") {
+    const param = filter[1][1];
+    res[param] = { $exists: !negate };
+  } else if (op === "LIKE" || op === "NOT LIKE") {
+    if (op === "NOT LIKE") negate = !negate;
+    let param;
+    let flags;
+    if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
+      if (filter[2] !== filter[2].toUpperCase()) {
+        throw new Error(
+          "Cannot compare UPPER() against non upper case pattern"
+        );
+      }
+      param = filter[1][2][1];
+      flags = "i";
+    } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
+      if (filter[2] !== filter[2].toLowerCase()) {
+        throw new Error(
+          "Cannot compare LOWER() against non lower case pattern"
+        );
+      }
+      param = filter[1][2][1];
+      flags = "i";
+    } else {
+      param = filter[1][1];
+      flags = "";
+    }
+    const r = expression.likePatternToRegExp(filter[2], filter[3], flags);
+    if (negate) res[param] = { $not: r };
+    else res[param] = r;
+  } else {
+    throw new Error(`Unrecognized operator ${op}`);
+  }
+
+  return res;
+}
+
+function processDeviceProjection(projection) {
+  if (!projection) return projection;
+  const p = {};
+  for (const [k, v] of Object.entries(projection)) {
+    if (k === "DeviceID.ID") {
+      p["_id"] = 1;
+    } else if (k.startsWith("DeviceID")) {
+      p["_deviceId._SerialNumber"] = v;
+      p["_deviceId._OUI"] = v;
+      p["_deviceId._ProductClass"] = v;
+      p["_deviceId._Manufacturer"] = v;
+    } else if (k.startsWith("Tags")) {
+      p["_tags"] = v;
+    } else if (k.startsWith("Events")) {
+      p["_lastInform"] = v;
+      p["_registered"] = v;
+      p["_lastBoot"] = v;
+      p["_lastBootstrap"] = v;
+    } else {
+      p[k] = v;
+    }
+  }
+
+  return p;
+}
+
+function processDeviceSort(sort) {
+  if (!sort) return sort;
+  const s = {};
+  for (const [k, v] of Object.entries(sort)) {
+    if (k === "DeviceID.ID") s["_id"] = v;
+    else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v;
+    else if (k === "Events.Inform") s["_lastInform"] = v;
+    else if (k === "Events.Registered") s["_registered"] = v;
+    else if (k === "Events.1_BOOT") s["_lastBoot"] = v;
+    else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v;
+    else s[`${k}._value`] = v;
+  }
+
+  return s;
+}
+
+function parseDate(d) {
+  const n = +d;
+  return isNaN(n) ? "" + d : n;
+}
+
+function flattenDevice(device) {
+  function recursive(input, root, output, timestamp) {
+    for (const [name, tree] of Object.entries(input)) {
+      if (!root) {
+        if (name === "_lastInform") {
+          output["Events.Inform"] = {
+            value: [parseDate(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_registered") {
+          output["Events.Registered"] = {
+            value: [parseDate(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_lastBoot") {
+          output["Events.1_BOOT"] = {
+            value: [parseDate(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_lastBootstrap") {
+          output["Events.0_BOOTSTRAP"] = {
+            value: [parseDate(tree), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_id") {
+          output["DeviceID.ID"] = {
+            value: [tree, "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_deviceId") {
+          output["DeviceID.Manufacturer"] = {
+            value: [tree["_Manufacturer"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+          output["DeviceID.OUI"] = {
+            value: [tree["_OUI"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+          output["DeviceID.ProductClass"] = {
+            value: [tree["_ProductClass"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+          output["DeviceID.SerialNumber"] = {
+            value: [tree["_SerialNumber"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp
+          };
+        } else if (name === "_tags") {
+          output["Tags"] = {
+            writable: true,
+            writableTimestamp: timestamp,
+            object: true,
+            objectTimestamp: timestamp
+          };
+
+          for (const t of tree) {
+            output[`Tags.${t}`] = {
+              value: [true, "xsd:boolean"],
+              valueTimestamp: timestamp,
+              writable: false,
+              writableTimestamp: timestamp,
+              object: false,
+              objectTimestamp: timestamp
+            };
+          }
+        }
+      }
+
+      if (name.startsWith("_")) continue;
+
+      let childrenTimestamp = timestamp;
+
+      if (!root) childrenTimestamp = +(input["_timestamp"] || 1);
+      else if (+input["_timestamp"] > timestamp)
+        childrenTimestamp = +input["_timestamp"];
+
+      const attrs = {};
+      if (tree["_value"] != null) {
+        attrs["value"] = [
+          tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
+          tree["_type"]
+        ];
+        attrs["valueTimestamp"] = +(tree["_timestamp"] || childrenTimestamp);
+        attrs["object"] = false;
+        attrs["objectTimestamp"] = childrenTimestamp;
+      } else if (tree["_object"] != null) {
+        attrs["object"] = tree["_object"];
+        attrs["objectTimestamp"] = childrenTimestamp;
+      }
+
+      if (tree["_writable"] != null) {
+        attrs["writable"] = tree["_writable"];
+        attrs["writableTimestamp"] = childrenTimestamp;
+      }
+
+      const r = root ? `${root}.${name}` : name;
+      output[r] = attrs;
+
+      if (attrs["object"] || tree["object"] == null)
+        recursive(tree, r, output, childrenTimestamp);
+    }
+  }
+
+  const newDevice = {};
+  const timestamp = new Date(device["_lastInform"] || 1).getTime();
+  recursive(device, "", newDevice, timestamp);
+  return newDevice;
+}
+
+function flattenFault(fault) {
+  const f = Object.assign({}, fault);
+  if (f.timestamp) f.timestamp = +f.timestamp;
+  if (f.expiry) f.expiry = +f.expiry;
+  return f;
+}
+
+function flattenTask(task) {
+  const t = Object.assign({}, task);
+  t._id = "" + t._id;
+  if (t.timestamp) t.timestamp = +t.timestamp;
+  if (t.expiry) t.expiry = +t.expiry;
+  return t;
+}
+
+function mongoQueryToFilter(query) {
+  const expressions = [];
+  function recursive(_query) {
+    for (const [k, v] of Object.entries(_query)) {
+      if (k[0] === "$") {
+        if (k === "$and") for (const vv of Object.values(v)) recursive(vv);
+        else throw new Error(`Operator ${k} not supported`);
+      } else if (k === "_tags") {
+        if (typeof v === "object") {
+          if (Array.isArray(v)) throw new Error(`Invalid type`);
+
+          for (const [kk, vv] of Object.entries(v)) {
+            if (kk === "$ne")
+              expressions.push(["IS NULL", ["PARAM", `Tags.${vv}`]]);
+            else throw new Error(`Operator ${kk} not supported`);
+          }
+        } else {
+          expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v}`]]);
+        }
+      } else if (k.startsWith("Tags.")) {
+        expressions.push([
+          v["$exists"] ? "IS NOT NULL" : "IS NULL",
+          ["PARAM", k]
+        ]);
+      } else if (typeof v === "object") {
+        if (Array.isArray(v)) throw new Error(`Invalid type`);
+
+        for (const [kk, vv] of Object.entries(v)) {
+          let op;
+          switch (kk) {
+            case "$eq":
+              op = "=";
+              break;
+            case "$ne":
+              op = "<>";
+              break;
+            case "$lt":
+              op = "<";
+              break;
+            case "$lte":
+              op = "<=";
+              break;
+            case "$gt":
+              op = ">";
+              break;
+            case "$gte":
+              op = ">=";
+              break;
+            default:
+              throw new Error(`Operator ${kk} not supported`);
+          }
+          expressions.push([op, ["PARAM", k], vv]);
+        }
+      } else {
+        expressions.push(["=", ["PARAM", k], v]);
+      }
+    }
+  }
+  recursive(query);
+
+  if (expressions.length > 1) return ["AND"].concat(expressions);
+
+  if (expressions.length === 1) return expressions[0];
+
+  return expressions;
+}
+
+function flattenPreset(preset) {
+  const p = Object.assign({}, preset);
+  if (p.precondition) {
+    p.precondition = mongoQueryToFilter(JSON.parse(p.precondition));
+    p.precondition = p.precondition.length
+      ? expression.stringify(p.precondition)
+      : "";
+  }
+
+  if (p.events) {
+    const e = [];
+    for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`);
+    p.events = e.join(", ");
+  }
+
+  const provision = p.configurations[0];
+  if (
+    p.configurations.length !== 1 ||
+    provision.type !== "provision" ||
+    !provision.name ||
+    !provision.name.length
+  )
+    throw new Error("Invalid preset provision");
+  p.provision = provision.name;
+  p.provisionArgs = provision.args
+    ? JSON.stringify(provision.args).slice(1, -1)
+    : "";
+  delete p.configurations;
+  return p;
+}
+
+function flattenFile(file) {
+  const f = {};
+  f["_id"] = file["_id"];
+  if (file.metadata) {
+    f["metadata.fileType"] = file.metadata.fileType || "";
+    f["metadata.oui"] = file.metadata.oui || "";
+    f["metadata.productClass"] = file.metadata.productClass || "";
+    f["metadata.version"] = file.metadata.version || "";
+  }
+  return f;
+}
+
+function preProcessPreset(data) {
+  const preset = Object.assign({}, data);
+  preset.precondition = preset.precondition
+    ? filterToMongoQuery(expression.parse(preset.precondition))
+    : {};
+  preset.precondition = JSON.stringify(preset.precondition);
+
+  preset.weight = parseInt(preset.weight) || 0;
+
+  const events = {};
+  if (preset.events) {
+    for (let e of preset.events.split(",")) {
+      let v = true;
+      e = e.trim();
+      if (e.startsWith("-")) {
+        v = false;
+        e = e.slice(1).trim();
+      }
+      if (e) events[e] = v;
+    }
+  }
+
+  preset.events = events;
+
+  if (!preset.provision) throw new Error("Invalid preset provision");
+
+  const configuration = {
+    type: "provision",
+    name: preset.provision
+  };
+
+  if (preset.provisionArgs)
+    configuration.args = JSON.parse(`[${preset.provisionArgs}]`);
+
+  delete preset.provision;
+  delete preset.provisionArgs;
+  preset.configurations = [configuration];
+  return preset;
+}
+
+function preProcessProvision(data) {
+  return data.script;
+}
+
+function preProcessVirtualParameters(data) {
+  return data.script;
+}
+
+exports.processDeviceFilter = processDeviceFilter;
+exports.processTasksFilter = processTasksFilter;
+exports.processFaultsFilter = processFaultsFilter;
+exports.filterToMongoQuery = filterToMongoQuery;
+exports.processDeviceProjection = processDeviceProjection;
+exports.processDeviceSort = processDeviceSort;
+exports.flattenDevice = flattenDevice;
+exports.flattenFault = flattenFault;
+exports.flattenTask = flattenTask;
+exports.flattenPreset = flattenPreset;
+exports.flattenFile = flattenFile;
+exports.preProcessPreset = preProcessPreset;
+exports.preProcessProvision = preProcessProvision;
+exports.preProcessVirtualParameters = preProcessVirtualParameters;

lib/ui/server.js

@@ -0,0 +1,220 @@
+"use strict";
+
+const http = require("http");
+const cluster = require("cluster");
+const Koa = require("koa");
+const Router = require("koa-router");
+const koaStatic = require("koa-static");
+const koaCompress = require("koa-compress");
+const koaBodyParser = require("koa-bodyparser");
+const koaJwt = require("koa-jwt");
+const jwt = require("jsonwebtoken");
+
+const config = require("./config");
+const api = require("./api");
+const Authorizer = require("../common/authorizer");
+const expression = require("../common/expression");
+const logger = require("./logger");
+const db = require("./db");
+
+const VERSION = require("../../package.json")["version"];
+logger.init("server", VERSION);
+
+const koa = new Koa();
+const router = new Router();
+
+const JWT_SECRET = config.server.jwtSecret;
+const JWT_COOKIE = "genieacs-ui-jwt";
+
+function getPermissionSets(roles) {
+  const allPermissions = config.permissions;
+  const permissionSets = roles.map(role => {
+    return Object.values(allPermissions[role] || {}).map(p => {
+      p = Object.assign({}, p);
+      for (let [k, v] of Object.entries(p)) {
+        p[k] = v = Object.assign({}, v);
+        if (
+          [
+            "devices",
+            "faults",
+            "presets",
+            "provisions",
+            "virtualParameters",
+            "files"
+          ].includes(k)
+        ) {
+          if (v.filter == null || v.filter === "") v.filter = true;
+          else v.filter = expression.parse(v.filter);
+        }
+      }
+      return p;
+    });
+  });
+  return permissionSets;
+}
+
+function authSimple(username, password) {
+  const user = config.auth.simple.users[username];
+  if (user && user.password === password) {
+    const roles = user.roles.split(",").map(s => s.trim());
+    return Promise.resolve(roles);
+  }
+  return Promise.resolve(null);
+}
+
+koa.on("error", async err => {
+  throw err;
+});
+
+koa.use(
+  koaJwt({
+    secret: JWT_SECRET,
+    passthrough: true,
+    cookie: JWT_COOKIE
+  })
+);
+
+koa.use(async (ctx, next) => {
+  if (ctx.state.user && ctx.state.user.roles) {
+    ctx.state.authorizer = new Authorizer(
+      getPermissionSets(ctx.state.user.roles)
+    );
+  } else {
+    ctx.state.authorizer = new Authorizer([]);
+  }
+
+  return next();
+});
+
+router.post("/login", async ctx => {
+  const username = ctx.request.body.username;
+  const password = ctx.request.body.password;
+
+  const log = {
+    message: "Log in",
+    context: ctx,
+    username: username
+  };
+
+  function success(roles, method) {
+    log.method = method;
+    const token = jwt.sign({ username, roles }, JWT_SECRET);
+    ctx.cookies.set(JWT_COOKIE, token);
+    ctx.body = JSON.stringify(token);
+    logger.accessInfo(log);
+  }
+
+  function failure() {
+    ctx.status = 400;
+    ctx.body = "Incorrect username or password";
+    log.message += " failed";
+    logger.accessWarn(log);
+  }
+
+  const roles = await authSimple(username, password);
+
+  if (roles) return void success(roles, "simple");
+
+  failure();
+});
+
+router.post("/logout", async ctx => {
+  ctx.cookies.set(JWT_COOKIE); // Delete cookie
+  ctx.body = "";
+
+  logger.accessInfo({
+    message: "Log out",
+    context: ctx
+  });
+});
+
+koa.use(async (ctx, next) => {
+  if (ctx.request.type === "application/octet-stream")
+    ctx.disableBodyParser = true;
+
+  return next();
+});
+
+koa.use(koaBodyParser());
+router.use("/api", api.routes(), api.allowedMethods());
+
+router.get("/", async ctx => {
+  let permissionSets = [];
+  if (ctx.state.user && ctx.state.user.roles)
+    permissionSets = getPermissionSets(ctx.state.user.roles);
+
+  ctx.body = `
+  <html>
+    <head>
+      <title>GenieACS</title>
+      <link rel="shortcut icon" type="image/png" href="favicon.png" />
+      <link rel="stylesheet" href="app.css">
+    </head>
+    <body>
+      <script>
+        window.clientConfig = ${JSON.stringify(config.getClientConfig())};
+        window.username = ${JSON.stringify(
+          ctx.state.user ? ctx.state.user.username : ""
+        )};
+        window.permissionSets = ${JSON.stringify(permissionSets)};
+      </script>
+      <script src="app.js"></script>
+    </body>
+  </html>
+  `;
+});
+
+koa.use(
+  koaCompress({
+    flush: require("zlib").Z_SYNC_FLUSH
+  })
+);
+
+const server = http
+  .createServer(koa.callback())
+  .listen(config.server.port, config.server.interface);
+
+koa.use(async (ctx, next) => {
+  await next();
+  if (!server.listening) ctx.set("Connection", "close");
+});
+
+koa.use(router.routes());
+koa.use(koaStatic("./public"));
+
+function closeServer(timeout) {
+  return new Promise(resolve => {
+    // prevent new sockets to connect and close server eventually.
+    server.close(resolve);
+    setTimeout(resolve, timeout).unref();
+  });
+}
+
+function exit() {
+  setTimeout(() => {
+    process.exit(1);
+  }, 30000).unref();
+
+  closeServer(20000).then(() => {
+    db.disconnect();
+    if (cluster.worker) cluster.worker.disconnect();
+    logger.close();
+  });
+}
+
+process.on("unhandledRejection", err => {
+  throw err;
+});
+
+process.on("uncaughtException", err => {
+  logger.error({
+    message: "Uncaught exception",
+    exception: err,
+    pid: process.pid
+  });
+  process.exitCode = 1;
+  exit();
+});
+
+process.on("SIGINT", exit);
+process.on("SIGTERM", exit);