From 75a5a42296ae1b392cec386ab7845731e174be1a Mon Sep 17 00:00:00 2001
From: Zaid Abdulla <zaid@genieacs.com>
Date: Sat, 29 Sep 2018 12:46:11 -0700
Subject: [PATCH] Optimize CSV download

---
 server/api.js               | 24 ++++++++----------------
 server/db.js                | 19 ++++++++++---------
 server/mongodb-functions.js |  3 ++-
 3 files changed, 20 insertions(+), 26 deletions(-)

diff --git a/server/api.js b/server/api.js
index 71c36b7..dd1cd3c 100644
--- a/server/api.js
+++ b/server/api.js
@@ -139,7 +139,7 @@ for (let [resource, flags] of Object.entries(resources)) {
 
   // CSV download
   router.get(`/${resource}.csv`, async (ctx, next) => {
-    let options = {};
+    let options = { projection: {} };
     let filter = true;
     if (ctx.request.query.filter)
       filter = expression.parse(ctx.request.query.filter);
@@ -154,8 +154,7 @@ for (let [resource, flags] of Object.entries(resources)) {
       filter: ctx.request.query.filter,
       limit: options.limit,
       skip: options.skip,
-      sort: options.sort,
-      projection: options.projection
+      sort: options.sort
     };
 
     if (!ctx.state.authorizer.hasAccess(resource, 2)) {
@@ -165,12 +164,16 @@ for (let [resource, flags] of Object.entries(resources)) {
 
     const columns = JSON.parse(ctx.request.query.columns);
     const now = Date.now();
-    for (let [k, v] of Object.entries(columns))
-      columns[k] = expression.evaluate(
+
+    for (let [k, v] of Object.entries(columns)) {
+      const e = expression.evaluate(
         expression.parse(v),
         null,
         now
       );
+      columns[k] = e;
+      for (let p of expression.extractParams(e)) options.projection[p] = 1;
+    }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults")
@@ -392,17 +395,6 @@ router.put("/files/:id", async (ctx, next) => {
   log.metadata = metadata;
   logger.accessInfo(log);
 
-  db.putAudit({
-    username: ctx.state.user.username,
-    action: "put",
-    objectType: resource,
-    objectId: id
-  }).catch(err => {
-    setTimeout(() => {
-      throw err;
-    }, 0);
-  });
-
   ctx.body = "";
 });
 
diff --git a/server/db.js b/server/db.js
index 0d1dfaa..153160b 100644
--- a/server/db.js
+++ b/server/db.js
@@ -60,14 +60,15 @@ function query(resource, filter, options, callback) {
       const collection = client
         .db(RESOURCE_DB[resource])
         .collection(RESOURCE_COLLECTION[resource] || resource);
-      let cursor = collection.find(
-        q,
-        resource === "Devices"
-          ? mongodbFunctions.processDeviceProjection(options.projection)
-          : options.projection
-      );
-      if (options.skip) cursor = cursor.skip(options.skip);
-      if (options.limit) cursor = cursor.limit(options.limit);
+      const cursor = collection.find(q);
+      if (options.projection)
+        cursor.project(
+          resource === "devices"
+            ? mongodbFunctions.processDeviceProjection(options.projection)
+            : options.projection
+        );
+      if (options.skip) cursor.skip(options.skip);
+      if (options.limit) cursor.limit(options.limit);
 
       if (options.sort) {
         let s = Object.entries(options.sort)
@@ -79,7 +80,7 @@ function query(resource, filter, options, callback) {
           );
 
         if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
-        cursor = cursor.sort(s);
+        cursor.sort(s);
       }
 
       if (!callback)
diff --git a/server/mongodb-functions.js b/server/mongodb-functions.js
index 53c4c3c..80a626d 100644
--- a/server/mongodb-functions.js
+++ b/server/mongodb-functions.js
@@ -332,7 +332,8 @@ function flattenDevice(device) {
       let r = root ? `${root}.${name}` : name;
       output[r] = attrs;
 
-      if (attrs["object"]) recursive(tree, r, output, childrenTimestamp);
+      if (attrs["object"] || tree["object"] == null)
+        recursive(tree, r, output, childrenTimestamp);
     }
   }