Add missing exception handling in NBI

Resolves: #148

lib/api-functions.coffee

@@ -84,7 +84,7 @@ connectionRequest = (deviceId, callback) ->
     )
 
     request.on('socket', (socket) ->
-      socket.setTimeout(2000)
+      socket.setTimeout(config.get('CONNECTION_REQUEST_TIMEOUT', deviceId))
       socket.on('timeout', () ->
         request.abort()
       )
@@ -130,15 +130,19 @@ connectionRequest = (deviceId, callback) ->
 
 watchTask = (taskId, timeout, callback) ->
   setTimeout( () ->
-    db.tasksCollection.findOne({_id : taskId}, {'_id' : 1}, (err, task) ->
+    db.tasksCollection.findOne({_id : taskId}, {'_id' : 1, 'fault' : 1}, (err, task) ->
+      return callback(err) if err
+
       if task
         timeout -= 500
-        if timeout < 0
+        if task.fault?
-          callback('timeout')
+          callback(null, 'fault')
+        else if timeout <= 0
+          callback(null, 'timeout')
         else
           watchTask(taskId, timeout, callback)
       else
-        callback(err)
+        callback(null, 'completed')
     )
   , 500)
 

lib/config.coffee

@@ -50,6 +50,7 @@ options = {
   GET_PARAMETER_NAMES_DEPTH_THRESHOLD : {type : 'int', default : 0},
   TASK_PARAMETERS_BATCH_SIZE : {type : 'int', default : 32},
   COOKIES_PATH : {type : 'string'},
+  CONNECTION_REQUEST_TIMEOUT : {type : 'int', default : 2000},
 
   # Libxml related configuration
   XML_PARSE_RECOVER : {type : 'bool'},

lib/custom-commands.coffee

@@ -15,22 +15,31 @@
 # along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
 ###
 
+path = require 'path'
 config = require './config'
 common = require './common'
 
 
+FILES = {}
+getFile = (filename) ->
+  if filename not of FILES
+    FILES[filename] = require(path.resolve(config.get('CONFIG_DIR'), "custom_commands/#{filename}"))
+
+  return FILES[filename]
+
+
 exports.execute = (deviceId, command, callback) ->
   args = command.split(/\s+/)
   f = args.shift()
   c = args.shift()
 
-  file = require "../config/custom_commands/#{f}"
+  file = getFile(f)
   file[c](deviceId, args.join(' '), callback)
 
 
 exports.getFileCommands = (filename) ->
   commands = []
-  f = require("../config/custom_commands/#{filename}")
+  f = getFile(filename)
   for k,v of f
     commands.push(k)
   return commands

lib/cwmp.coffee

@@ -173,7 +173,7 @@ inform = (currentRequest, cwmpRequest) ->
         for k of reference
           continue if k[0] == '_'
           if not actual[k]?._timestamp? or actual[k]._timestamp < lastBootstrap
-            _tasks.push({device: currentRequest.session.deviceId, name : 'refreshObject', objectName : reference[k]._path})
+            _tasks.push({device: currentRequest.session.deviceId, name : 'refreshObject', objectName : reference[k]._path, timestamp : now})
           else
             traverse(reference[k], actual[k])
 
@@ -185,8 +185,9 @@ inform = (currentRequest, cwmpRequest) ->
         deviceCustomCommands[k] = v._timestamp for k,v of device._customCommands
 
         for cmd in customCommands.getDeviceCustomCommandNames(currentRequest.session.deviceId)
-          if not (deviceCustomCommands[cmd]?._timestamp < lastBootstrap)
+          if not deviceCustomCommands[cmd]? or deviceCustomCommands[cmd] < lastBootstrap
-            _tasks.push({device: currentRequest.session.deviceId, name: 'customCommand', command: "#{cmd} init"})
+            # Increment timestamp by one millisecond to ensure it runs after any refresh task
+            _tasks.push({device: currentRequest.session.deviceId, name: 'customCommand', command: "#{cmd} init", timestamp: new Date(now.getTime() + 1)})
           delete deviceCustomCommands[cmd]
 
         for cmd of deviceCustomCommands
@@ -210,10 +211,11 @@ inform = (currentRequest, cwmpRequest) ->
         db.devicesCollection.insert({_id : currentRequest.session.deviceId, _registered : now, _deviceId : deviceIdDetails}, (err) ->
           throw err if err
           util.log("#{currentRequest.session.deviceId}: New device registered")
-          _tasks.push({device: currentRequest.session.deviceId, name : 'refreshObject', objectName : ''})
+          _tasks.push({device: currentRequest.session.deviceId, name : 'refreshObject', objectName : '', timestamp : now})
 
           for cmd in customCommands.getDeviceCustomCommandNames(currentRequest.session.deviceId)
-            _tasks.push({device: currentRequest.session.deviceId, name: 'customCommand', command: "#{cmd} init"})
+            # Increment timestamp by one millisecond to ensure it runs after any refresh task
+            _tasks.push({device: currentRequest.session.deviceId, name: 'customCommand', command: "#{cmd} init", timestamp: new Date(now.getTime() + 1)})
 
           apiFunctions.insertTasks(_tasks, {}, () ->
             updateAndRespond()

lib/nbi.coffee

@@ -61,6 +61,10 @@ DEVICE_PRESET_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/preset\/?$/
 DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/?$/
 
 
+errorToString = (err) ->
+  "#{err.name}: #{err.message}"
+
+
 listener = (request, response) ->
   chunks = []
   bytes = 0
@@ -94,7 +98,7 @@ listener = (request, response) ->
           )
           if err
             response.writeHead(500)
-            response.end(err)
+            response.end(errorToString(err))
             return
           response.writeHead(200)
           response.end()
@@ -106,8 +110,14 @@ listener = (request, response) ->
           )
           if err
             response.writeHead(500)
-            response.end(err)
+            response.end(errorToString(err))
             return
+
+          if removedCount == 0
+            response.writeHead(404)
+            response.end()
+            return
+
           response.writeHead(200)
           response.end()
         )
@@ -126,7 +136,7 @@ listener = (request, response) ->
           )
           if err
             response.writeHead(500)
-            response.end(err)
+            response.end(errorToString(err))
             return
           response.writeHead(200)
           response.end()
@@ -138,8 +148,14 @@ listener = (request, response) ->
           )
           if err
             response.writeHead(500)
-            response.end(err)
+            response.end(errorToString(err))
             return
+
+          if removedCount == 0
+            response.writeHead(404)
+            response.end()
+            return
+
           response.writeHead(200)
           response.end()
         )
@@ -151,26 +167,38 @@ listener = (request, response) ->
       deviceId = querystring.unescape(r[1])
       tag = querystring.unescape(r[2])
       if request.method == 'POST'
-        db.devicesCollection.update({'_id' : deviceId}, {'$addToSet' : {'_tags' : tag}}, {safe: true}, (err) ->
+        db.devicesCollection.update({'_id' : deviceId}, {'$addToSet' : {'_tags' : tag}}, {safe: true}, (err, updatedCount) ->
           db.redisClient.del("#{deviceId}_presets_hash", (err) ->
             throw err if err
           )
           if err
             response.writeHead(500)
-            response.end(err)
+            response.end(errorToString(err))
             return
+
+          if updatedCount == 0
+            response.writeHead(404)
+            response.end()
+            return
+
           response.writeHead(200)
           response.end()
         )
       else if request.method == 'DELETE'
-        db.devicesCollection.update({'_id' : deviceId}, {'$pull' : {'_tags' : tag}}, {safe: true}, (err) ->
+        db.devicesCollection.update({'_id' : deviceId}, {'$pull' : {'_tags' : tag}}, {safe: true}, (err, updatedCount) ->
           db.redisClient.del("#{deviceId}_presets_hash", (err) ->
             throw err if err
           )
           if err
             response.writeHead(500)
-            response.end(err)
+            response.end(errorToString(err))
             return
+
+          if updatedCount == 0
+            response.writeHead(404)
+            response.end()
+            return
+
           response.writeHead(200)
           response.end()
         )
@@ -190,22 +218,37 @@ listener = (request, response) ->
               )
               if err
                 response.writeHead(500)
-                response.end(err)
+                response.end(errorToString(err))
                 return
 
               if urlParts.query.connection_request?
                 apiFunctions.connectionRequest(deviceId, (err) ->
                   if err
-                    response.writeHead(202, {'Content-Type' : 'application/json'})
+                    response.writeHead(202, err.message, {'Content-Type' : 'application/json'})
                     response.end(JSON.stringify(task))
                   else
-                    apiFunctions.watchTask(task._id, config.get('DEVICE_ONLINE_THRESHOLD', deviceId), (err) ->
+                    apiFunctions.watchTask(task._id, config.get('DEVICE_ONLINE_THRESHOLD', deviceId), (err, status) ->
                       if err
-                        response.writeHead(202, {'Content-Type' : 'application/json'})
+                        response.writeHead(500)
-                        response.end(JSON.stringify(task))
+                        response.end(errorToString(err))
                         return
-                      response.writeHead(200, {'Content-Type' : 'application/json'})
+
-                      response.end(JSON.stringify(task))
+                      if status is 'timeout'
+                        response.writeHead(202, 'Task queued but not processed', {'Content-Type' : 'application/json'})
+                        response.end(JSON.stringify(task))
+                      else if status is 'fault'
+                        db.tasksCollection.findOne({_id : task._id}, (err, task) ->
+                          if err
+                            response.writeHead(500)
+                            response.end(errorToString(err))
+                            return
+
+                          response.writeHead(202, 'Task faulted', {'Content-Type' : 'application/json'})
+                          response.end(JSON.stringify(task))
+                        )
+                      else
+                        response.writeHead(200, {'Content-Type' : 'application/json'})
+                        response.end(JSON.stringify(task))
                     )
                 )
               else
@@ -218,7 +261,7 @@ listener = (request, response) ->
           apiFunctions.connectionRequest(deviceId, (err) ->
             if err
               response.writeHead 504
-              response.end()
+              response.end(errorToString(err))
               return
             response.writeHead 200
             response.end()
@@ -247,21 +290,39 @@ listener = (request, response) ->
       action = r[2]
       if not action? or action is '/'
         if request.method == 'DELETE'
-          db.tasksCollection.remove({'_id' : taskId}, (err, removedCount) ->
+          db.tasksCollection.findOne({'_id' : taskId}, {'device' : 1}, (err, task) ->
-            if err
+            throw err if err
-              response.writeHead(500)
+
-              response.end(err)
+            if not task
+              response.writeHead(404)
+              response.end()
               return
-            response.writeHead(200)
+
-            response.end()
+            deviceId = task.device
+            db.tasksCollection.remove({'_id' : taskId}, (err, removedCount) ->
+              throw err if err
+
+              db.redisClient.del("#{deviceId}_presets_hash", (err) ->
+                throw err if err
+
+                response.writeHead(200)
+                response.end()
+              )
+            )
           )
         else
           response.writeHead 405, {'Allow': 'PUT DELETE'}
           response.end('405 Method Not Allowed')
       else if action is '/retry'
         if request.method == 'POST'
-          db.tasksCollection.update({_id : taskId}, {$unset : {fault : 1}, $set : {timestamp : new Date()}}, (err, count) ->
+          db.tasksCollection.update({_id : taskId}, {$unset : {fault : 1}, $set : {timestamp : new Date()}}, (err, updatedCount) ->
             # TODO need to invalidate presets hash for the device
+            throw err if err
+            if updatedCount == 0
+              response.writeHead(404)
+              response.end()
+              return
+
             response.writeHead(200)
             response.end()
           )
@@ -283,6 +344,11 @@ listener = (request, response) ->
 
         gs = new mongodb.GridStore(db.mongoDb, filename, 'w', {metadata : metadata})
         gs.open((err, gs) ->
+          if err
+            response.writeHead(500)
+            response.end(errorToString(err))
+            return
+
           gs.write(body, (err, res) ->
             throw err if err
             gs.close((err) ->
@@ -294,6 +360,11 @@ listener = (request, response) ->
         )
       else if request.method == 'DELETE'
         mongodb.GridStore.unlink(db.mongoDb, filename, (err) ->
+          if err
+            response.writeHead(500)
+            response.end(errorToString(err))
+            return
+
           response.writeHead(200)
           response.end()
         )
@@ -303,9 +374,9 @@ listener = (request, response) ->
     else if PING_REGEX.test(urlParts.pathname)
       host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1])
       require('child_process').exec("ping -w 1 -i 0.2 -c 3 #{host}", (err, stdout, stderr) ->
-        if err?
+        if err
           response.writeHead(404, {'Cache-Control' : 'no-cache'})
-          response.end()
+          response.end(errorToString(err))
           return
         response.writeHead(200, {'Content-Type' : 'text/plain', 'Cache-Control' : 'no-cache'})
         response.end(stdout)
@@ -320,7 +391,7 @@ listener = (request, response) ->
       apiFunctions.deleteDevice(deviceId, (err) ->
         if err
           response.writeHead(500)
-          response.end(err.message)
+          response.end(errorToString(err))
           return
         response.writeHead(200)
         response.end()
@@ -343,7 +414,7 @@ listener = (request, response) ->
             q = JSON.parse(urlParts.query.query)
           catch err
             response.writeHead(400)
-            response.end(err.toString())
+            response.end(errorToString(err))
             return
         else
           q = {}
@@ -366,14 +437,26 @@ listener = (request, response) ->
           sort = {}
           for k,v of s
             if aliases[k]?
-              sort[a] = v for a in aliases[k]
+              for a in aliases[k]
+                if a[a.lastIndexOf('.')+1] != '_' and collectionName is 'devices'
+                  sort["#{a}._value"] = v
+                else
+                  sort[a] = v
             else
-              sort[k] = v
+              if k[k.lastIndexOf('.') + 1] != '_' and collectionName is 'devices'
+                sort["#{k}._value"] = v
+              else
+                sort[k] = v
           cur.sort(sort)
 
         cur.skip(parseInt(urlParts.query.skip)) if urlParts.query.skip?
         cur.limit(parseInt(urlParts.query.limit)) if urlParts.query.limit?
         cur.count((err, total) ->
+          if err
+            response.writeHead(500)
+            response.end(errorToString(err))
+            return
+
           response.writeHead(200, {'Content-Type' : 'application/json', 'total' : total})
           if request.method is 'HEAD'
             response.end()
@@ -381,6 +464,8 @@ listener = (request, response) ->
           response.write("[\n")
           i = 0
           cur.each((err, item) ->
+            throw err if err
+
             if item is null
               response.end("\n]")
             else

lib/normalize.coffee

@@ -68,7 +68,7 @@ normalizers.default = (input, normType) ->
 
     r = stringToRegexp(input)
     if r isnt false
-      vals.push({'$regex' : r})
+      vals.push(r)
 
     return vals
   return input
@@ -90,7 +90,7 @@ normalizers.date = (input, normType) ->
 normalizers.string = (input, normType) ->
   if normType is 'query'
     if (m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input))
-      return [{'$regex' : new RegExp(m[1], m[2])}, input]
+      return [new RegExp(m[1], m[2]), input]
   return input
 
 
@@ -124,7 +124,7 @@ normalizers.mac = (input, normType) ->
   if input.length == 17
     return [input, input.toLowerCase()]
 
-  return {'$regex' : new RegExp(input, 'i')}
+  return new RegExp(input, 'i')
 
 
 exports.normalize = (path, value, normType) ->

lib/presets.coffee

@@ -294,11 +294,19 @@ exports.processDevicePreset = (deviceId, devicePreset, callback) ->
 
 
 getObjectHash = (object) ->
-  return object._id if not object._keys? or object._keys.length == 0
+  if object._keys?.length
-  hash = ''
+    keys = object._keys
-  for k in object._keys
+  else
-    hash += "#{k}=#{object[k]}"
+    keys = (k for k in Object.keys(object) when k[0] != '_')
-  return hash
+
+  keys.sort()
+
+  hash = []
+  for k in keys
+    hash.push(k)
+    hash.push(common.getParamValueFromPath(object, k))
+
+  return JSON.stringify(hash)
 
 
 accumulateConfigurations = (presets, objects) ->

lib/query.coffee

@@ -51,8 +51,17 @@ expandValue = (param, value) ->
   while i < indices.length
     obj = {}
     for i in [0...keys.length]
+      if common.typeOf(values[i][indices[i]]) is common.REGEXP_TYPE
+        if keys[i] == '$ne'
+          obj['$not'] = values[i][indices[i]]
+          continue
+      else if keys[i] == '$not' and common.typeOf(values[i][indices[i]]) isnt common.OBJECT_TYPE
+        # Only a regex or decoument (object) are allowed within a $not operator.
+        # This is needed in order to discard string from which a regex is generated.
+        continue
+
       obj[keys[i]] = values[i][indices[i]]
-    objs.push(obj)
+    objs.push(obj) if Object.keys(obj).length
 
     for i in [0...indices.length]
       indices[i] += 1
@@ -75,7 +84,7 @@ permute = (param, val, aliases) ->
     values = expandValue(k, val)
     if k[k.lastIndexOf('.') + 1] != '_'
       k += '._value'
-  
+
     for v in values
       obj = {}
       obj[k] = v
@@ -95,10 +104,12 @@ expand = (query, aliases) ->
     else
       conditions = permute(k, v, aliases)
       if conditions.length > 1
-        if new_query['$and']?
+        new_query['$and'] ?= []
-          new_query['$and'].push({'$or' : conditions})
+        if v?['$ne']? or v?['$nin']? or v?['$not']?
+          for c in conditions
+            new_query['$and'].push(c)
         else
-          new_query['$and'] = [{'$or' : conditions}]
+          new_query['$and'].push({'$or' : conditions})
       else
         common.extend(new_query, conditions[0])
 

lib/server.coffee

@@ -51,7 +51,7 @@ listener = (httpRequest, httpResponse) ->
       )
 
       httpResponse.writeHead(500, {'Connection' : 'close'})
-      httpResponse.end()
+      httpResponse.end("#{err.name}: #{err.message}")
     catch err2
       util.error("#{new Date().toISOString()} - #{err2.stack}\n")
   )

lib/tasks.coffee

@@ -322,31 +322,39 @@ this.download = (task, methodResponse, callback) ->
     return
 
   if methodResponse.type isnt 'DownloadResponse'
-    db.filesCollection.findOne({_id : mongodb.ObjectID(String(task.file))}, (err, file) ->
+    if task.url?
-      if not file?
+        methodRequest = {
-        callback('File not found')
+          type : 'Download',
-        return
+          fileType : task.fileType or '1 Firmware Upgrade Image',
-      else if err?
+          url : task.url,
-        callback(err)
+          username: task.username,
-        return
+          password: task.password,
+          successUrl : task.successUrl,
+          failureUrl : task.failureUrl
+        }
+        callback(null, STATUS_OK, methodRequest)
+    else
+      db.filesCollection.findOne({_id : mongodb.ObjectID(String(task.file))}, (err, file) ->
+        if err or not file?
+          return callback(err or new Error('File not found'))
 
-      l = {
+        l = {
-        protocol : if config.get('FS_SSL') then 'https' else 'http',
+          protocol : if config.get('FS_SSL') then 'https' else 'http',
-        hostname : config.get('FS_IP'),
+          hostname : config.get('FS_IP'),
-        port : config.get('FS_PORT'),
+          port : config.get('FS_PORT'),
-        pathname : encodeURIComponent(file.filename)
+          pathname : encodeURIComponent(file.filename)
-      }
+        }
 
-      methodRequest = {
+        methodRequest = {
-        type : 'Download',
+          type : 'Download',
-        fileType : file.metadata.fileType,
+          fileType : file.metadata.fileType,
-        fileSize : file.length,
+          fileSize : file.length,
-        url : url.format(l),
+          url : url.format(l),
-        successUrl : task.successUrl,
+          successUrl : task.successUrl,
-        failureUrl : task.failureUrl
+          failureUrl : task.failureUrl
-      }
+        }
-      callback(null, STATUS_OK, methodRequest)
+        callback(null, STATUS_OK, methodRequest)
-    )
+      )
   else
     callback(null, STATUS_COMPLETED)
 
@@ -354,8 +362,8 @@ this.download = (task, methodResponse, callback) ->
 this.customCommand = (task, methodResponse, callback) ->
   # TODO implement timeout
   customCommands.execute(task.device, task.command, (err, value) ->
-    if err?
+    if err
-      task.fault = err
+      task.fault = {name : err.name, message : err.message}
       callback(null, STATUS_FAULT)
     else
       commandName = task.command.split(' ', 2)[0]

lib/update-device.coffee

@@ -53,8 +53,8 @@ commitUpdates = (deviceId, deviceUpdates, commitQueue, callback) ->
   processBatch = (updatesBatch, cb) ->
     if updatesBatch.informEvents?
       update['$set']['_lastInform'] = now
-      update['$set']['_lastBoot'] if '1 BOOT' in updatesBatch.informEvents
+      update['$set']['_lastBoot'] = now if '1 BOOT' in updatesBatch.informEvents
-      update['$set']['_lastBootstrap'] if '0 BOOTSTRAP' in updatesBatch.informEvents
+      update['$set']['_lastBootstrap'] = now if '0 BOOTSTRAP' in updatesBatch.informEvents
 
     if updatesBatch.parameterValues?
       for p in updatesBatch.parameterValues

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.0.4",
+  "version": "1.0.7",
   "description": "A blazing fast TR-069 auto configuration server (ACS) built with Node.js, Redis, and MongoDB",
   "repository": {
     "type": "git",
@@ -12,7 +12,7 @@
     "ACS"
   ],
   "author": "Zaid Abdulla",
-  "license": "AGPL3",
+  "license": "AGPL-3.0",
   "readmeFilename": "README.md",
   "preferGlobal": true,
   "directories": {