Add missing exception handling in NBI

Resolves: #148

.gitignore

@@ -1,3 +1,4 @@
 *~
 *.js
 node_modules
+debug/*

README.md

@@ -19,10 +19,10 @@ Install [Node.js](http://nodejs.org/), [Redis](http://redis.io/), and [MongoDB](
 
 You may need to modify the configuration files under "config" directory (in /lib/node_modules/genieacs/config) depending on your setup.
 
-Alternatively, you can use git to get the latest development version (not recommended for production use):
+Alternatively, you can install from source by cloning the git repository:
 
     cd /opt
-    git clone https://github.com/zaidka/genieacs.git
+    git clone https://github.com/zaidka/genieacs.git --branch v1.0
     cd genieacs
     npm install
     npm run configure
@@ -54,13 +54,12 @@ You may now proceed with installing [GenieACS GUI front end](https://github.com/
 
 ## Support
 
-Documentation is still work in progress. Feel free to contact me if you require any assistance in installation, configuration, or using the APIs.
+The [Users mailing list](http://lists.genieacs.com) is a good place to get guidance and help from the community. Head on over and join the conversation! In addition, the [wiki](https://github.com/zaidka/genieacs/wiki) provides useful documentation and tips from GenieACS users.
 
-You may submit bug reports or feature requests [here](https://github.com/zaidka/genieacs/issues).
+You may submit bug reports or feature requests [here](https://github.com/zaidka/genieacs/issues). For device interoperability issues, please consult the mailing list first — it's likely that a workaround already exists.
 
-For commercial support options, please contact me.
+For commercial support options and professional services, please visit [genieacs.com](https://genieacs.com).
 
 ## Contributing
 
 Contributions are welcome. Fork this repo and open a pull request and wait for feedback. You can also contribute by enhancing the documentation in the [wiki section](https://github.com/zaidka/genieacs/wiki).
-

config/parameters-sample.json

@@ -8,12 +8,20 @@
   "_deviceId._Manufacturer" : {"alias" : "summary.manufacturer"},
   "InternetGatewayDevice.DeviceInfo.HardwareVersion" : {"alias" : "summary.hardwareVersion"},
   "InternetGatewayDevice.DeviceInfo.SoftwareVersion" : {"alias" : "summary.softwareVersion"},
+  "Device.DeviceInfo.HardwareVersion" : {"alias" : "summary.hardwareVersion"},
+  "Device.DeviceInfo.SoftwareVersion" : {"alias" : "summary.softwareVersion"},
 
   "InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
   "InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
   "InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
   "InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
+  "Device.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
+  "Device.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
+  "Device.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
+  "Device.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
 
   "InternetGatewayDevice.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.SSID" : {"alias" : "summary.wlanSsid"},
-  "InternetGatewayDevice.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.PreSharedKey.1.KeyPassphrase" : {"alias" : "summary.wlanPassphrase"}
+  "InternetGatewayDevice.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.PreSharedKey.1.KeyPassphrase" : {"alias" : "summary.wlanPassphrase"},
+  "Device.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.SSID" : {"alias" : "summary.wlanSsid"},
+  "Device.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.PreSharedKey.1.KeyPassphrase" : {"alias" : "summary.wlanPassphrase"}
 }

lib/api-functions.coffee

@@ -84,7 +84,7 @@ connectionRequest = (deviceId, callback) ->
     )
 
     request.on('socket', (socket) ->
-      socket.setTimeout(2000)
+      socket.setTimeout(config.get('CONNECTION_REQUEST_TIMEOUT', deviceId))
       socket.on('timeout', () ->
         request.abort()
       )
@@ -117,6 +117,9 @@ connectionRequest = (deviceId, callback) ->
           authString = auth.digest(username, password, uri.path, 'GET', null, authHeader)
 
         conReq(connectionRequestUrl, authString, (statusCode, authHeader) ->
+          if statusCode == 0
+            # Workaround for some devices unexpectedly closing the connection
+            return conReq(connectionRequestUrl, authString, (statusCode) -> callback(statusToError(statusCode)))
           callback(statusToError(statusCode))
         )
       else
@@ -127,15 +130,19 @@ connectionRequest = (deviceId, callback) ->
 
 watchTask = (taskId, timeout, callback) ->
   setTimeout( () ->
-    db.tasksCollection.findOne({_id : taskId}, {'_id' : 1}, (err, task) ->
+    db.tasksCollection.findOne({_id : taskId}, {'_id' : 1, 'fault' : 1}, (err, task) ->
+      return callback(err) if err
+
       if task
         timeout -= 500
-        if timeout < 0
-          callback('timeout')
+        if task.fault?
+          callback(null, 'fault')
+        else if timeout <= 0
+          callback(null, 'timeout')
         else
           watchTask(taskId, timeout, callback)
       else
-        callback(err)
+        callback(null, 'completed')
     )
   , 500)
 

lib/common.coffee

@@ -82,10 +82,22 @@ exports.parseDeviceId = (deviceId) ->
 
 
 exports.extend = (obj, mixin) ->
-  obj[name] = method for name, method of mixin        
+  obj[name] = method for name, method of mixin
   obj
 
 
+exports.flattenObject = (object) ->
+  newObj = {}
+  f = (obj, prefix) ->
+    for k, v of obj
+      if typeof(v) is 'object'
+        f(v, "#{prefix}#{k}.")
+      else
+        newObj["#{prefix}#{k}"] = v
+  f(object, '')
+  return newObj
+
+
 exports.getParamValueFromPath = (obj, path) ->
   pp = path.split('.')
   ref = obj

lib/config.coffee

@@ -50,6 +50,7 @@ options = {
   GET_PARAMETER_NAMES_DEPTH_THRESHOLD : {type : 'int', default : 0},
   TASK_PARAMETERS_BATCH_SIZE : {type : 'int', default : 32},
   COOKIES_PATH : {type : 'string'},
+  CONNECTION_REQUEST_TIMEOUT : {type : 'int', default : 2000},
 
   # Libxml related configuration
   XML_PARSE_RECOVER : {type : 'bool'},

lib/custom-commands.coffee

@@ -15,22 +15,31 @@
 # along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
 ###
 
+path = require 'path'
 config = require './config'
 common = require './common'
 
 
+FILES = {}
+getFile = (filename) ->
+  if filename not of FILES
+    FILES[filename] = require(path.resolve(config.get('CONFIG_DIR'), "custom_commands/#{filename}"))
+
+  return FILES[filename]
+
+
 exports.execute = (deviceId, command, callback) ->
   args = command.split(/\s+/)
   f = args.shift()
   c = args.shift()
 
-  file = require "../config/custom_commands/#{f}"
+  file = getFile(f)
   file[c](deviceId, args.join(' '), callback)
 
 
 exports.getFileCommands = (filename) ->
   commands = []
-  f = require("../config/custom_commands/#{filename}")
+  f = getFile(filename)
   for k,v of f
     commands.push(k)
   return commands

lib/cwmp.coffee

@@ -173,7 +173,7 @@ inform = (currentRequest, cwmpRequest) ->
         for k of reference
           continue if k[0] == '_'
           if not actual[k]?._timestamp? or actual[k]._timestamp < lastBootstrap
-            _tasks.push({device: currentRequest.session.deviceId, name : 'refreshObject', objectName : reference[k]._path})
+            _tasks.push({device: currentRequest.session.deviceId, name : 'refreshObject', objectName : reference[k]._path, timestamp : now})
           else
             traverse(reference[k], actual[k])
 
@@ -185,8 +185,9 @@ inform = (currentRequest, cwmpRequest) ->
         deviceCustomCommands[k] = v._timestamp for k,v of device._customCommands
 
         for cmd in customCommands.getDeviceCustomCommandNames(currentRequest.session.deviceId)
-          if not (deviceCustomCommands[cmd]?._timestamp < lastBootstrap)
-            _tasks.push({device: currentRequest.session.deviceId, name: 'customCommand', command: "#{cmd} init"})
+          if not deviceCustomCommands[cmd]? or deviceCustomCommands[cmd] < lastBootstrap
+            # Increment timestamp by one millisecond to ensure it runs after any refresh task
+            _tasks.push({device: currentRequest.session.deviceId, name: 'customCommand', command: "#{cmd} init", timestamp: new Date(now.getTime() + 1)})
           delete deviceCustomCommands[cmd]
 
         for cmd of deviceCustomCommands
@@ -210,10 +211,11 @@ inform = (currentRequest, cwmpRequest) ->
         db.devicesCollection.insert({_id : currentRequest.session.deviceId, _registered : now, _deviceId : deviceIdDetails}, (err) ->
           throw err if err
           util.log("#{currentRequest.session.deviceId}: New device registered")
-          _tasks.push({device: currentRequest.session.deviceId, name : 'refreshObject', objectName : ''})
+          _tasks.push({device: currentRequest.session.deviceId, name : 'refreshObject', objectName : '', timestamp : now})
 
           for cmd in customCommands.getDeviceCustomCommandNames(currentRequest.session.deviceId)
-            _tasks.push({device: currentRequest.session.deviceId, name: 'customCommand', command: "#{cmd} init"})
+            # Increment timestamp by one millisecond to ensure it runs after any refresh task
+            _tasks.push({device: currentRequest.session.deviceId, name: 'customCommand', command: "#{cmd} init", timestamp: new Date(now.getTime() + 1)})
 
           apiFunctions.insertTasks(_tasks, {}, () ->
             updateAndRespond()

lib/nbi.coffee

@@ -61,6 +61,10 @@ DEVICE_PRESET_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/preset\/?$/
 DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/?$/
 
 
+errorToString = (err) ->
+  "#{err.name}: #{err.message}"
+
+
 listener = (request, response) ->
   chunks = []
   bytes = 0
@@ -94,7 +98,7 @@ listener = (request, response) ->
           )
           if err
             response.writeHead(500)
-            response.end(err)
+            response.end(errorToString(err))
             return
           response.writeHead(200)
           response.end()
@@ -106,8 +110,14 @@ listener = (request, response) ->
           )
           if err
             response.writeHead(500)
-            response.end(err)
+            response.end(errorToString(err))
             return
+
+          if removedCount == 0
+            response.writeHead(404)
+            response.end()
+            return
+
           response.writeHead(200)
           response.end()
         )
@@ -126,7 +136,7 @@ listener = (request, response) ->
           )
           if err
             response.writeHead(500)
-            response.end(err)
+            response.end(errorToString(err))
             return
           response.writeHead(200)
           response.end()
@@ -138,8 +148,14 @@ listener = (request, response) ->
           )
           if err
             response.writeHead(500)
-            response.end(err)
+            response.end(errorToString(err))
             return
+
+          if removedCount == 0
+            response.writeHead(404)
+            response.end()
+            return
+
           response.writeHead(200)
           response.end()
         )
@@ -151,26 +167,38 @@ listener = (request, response) ->
       deviceId = querystring.unescape(r[1])
       tag = querystring.unescape(r[2])
       if request.method == 'POST'
-        db.devicesCollection.update({'_id' : deviceId}, {'$addToSet' : {'_tags' : tag}}, {safe: true}, (err) ->
+        db.devicesCollection.update({'_id' : deviceId}, {'$addToSet' : {'_tags' : tag}}, {safe: true}, (err, updatedCount) ->
           db.redisClient.del("#{deviceId}_presets_hash", (err) ->
             throw err if err
           )
           if err
             response.writeHead(500)
-            response.end(err)
+            response.end(errorToString(err))
             return
+
+          if updatedCount == 0
+            response.writeHead(404)
+            response.end()
+            return
+
           response.writeHead(200)
           response.end()
         )
       else if request.method == 'DELETE'
-        db.devicesCollection.update({'_id' : deviceId}, {'$pull' : {'_tags' : tag}}, {safe: true}, (err) ->
+        db.devicesCollection.update({'_id' : deviceId}, {'$pull' : {'_tags' : tag}}, {safe: true}, (err, updatedCount) ->
           db.redisClient.del("#{deviceId}_presets_hash", (err) ->
             throw err if err
           )
           if err
             response.writeHead(500)
-            response.end(err)
+            response.end(errorToString(err))
             return
+
+          if updatedCount == 0
+            response.writeHead(404)
+            response.end()
+            return
+
           response.writeHead(200)
           response.end()
         )
@@ -190,27 +218,42 @@ listener = (request, response) ->
               )
               if err
                 response.writeHead(500)
-                response.end(err)
+                response.end(errorToString(err))
                 return
 
               if urlParts.query.connection_request?
                 apiFunctions.connectionRequest(deviceId, (err) ->
                   if err
-                    response.writeHead(202)
-                    response.end()
+                    response.writeHead(202, err.message, {'Content-Type' : 'application/json'})
+                    response.end(JSON.stringify(task))
                   else
-                    apiFunctions.watchTask(task._id, config.get('DEVICE_ONLINE_THRESHOLD', deviceId), (err) ->
+                    apiFunctions.watchTask(task._id, config.get('DEVICE_ONLINE_THRESHOLD', deviceId), (err, status) ->
                       if err
-                        response.writeHead(202)
-                        response.end()
+                        response.writeHead(500)
+                        response.end(errorToString(err))
                         return
-                      response.writeHead(200)
-                      response.end()
+
+                      if status is 'timeout'
+                        response.writeHead(202, 'Task queued but not processed', {'Content-Type' : 'application/json'})
+                        response.end(JSON.stringify(task))
+                      else if status is 'fault'
+                        db.tasksCollection.findOne({_id : task._id}, (err, task) ->
+                          if err
+                            response.writeHead(500)
+                            response.end(errorToString(err))
+                            return
+
+                          response.writeHead(202, 'Task faulted', {'Content-Type' : 'application/json'})
+                          response.end(JSON.stringify(task))
+                        )
+                      else
+                        response.writeHead(200, {'Content-Type' : 'application/json'})
+                        response.end(JSON.stringify(task))
                     )
                 )
               else
-                response.writeHead(202)
-                response.end()
+                response.writeHead(202, {'Content-Type' : 'application/json'})
+                response.end(JSON.stringify(task))
             )
           )
         else if urlParts.query.connection_request?
@@ -218,7 +261,7 @@ listener = (request, response) ->
           apiFunctions.connectionRequest(deviceId, (err) ->
             if err
               response.writeHead 504
-              response.end()
+              response.end(errorToString(err))
               return
             response.writeHead 200
             response.end()
@@ -244,24 +287,42 @@ listener = (request, response) ->
     else if TASKS_REGEX.test(urlParts.pathname)
       r = TASKS_REGEX.exec(urlParts.pathname)
       taskId = mongodb.ObjectID(querystring.unescape(r[1]))
-      action = try querystring.unescape(r[2])
+      action = r[2]
       if not action? or action is '/'
         if request.method == 'DELETE'
-          db.tasksCollection.remove({'_id' : taskId}, (err, removedCount) ->
-            if err
-              response.writeHead(500)
-              response.end(err)
+          db.tasksCollection.findOne({'_id' : taskId}, {'device' : 1}, (err, task) ->
+            throw err if err
+
+            if not task
+              response.writeHead(404)
+              response.end()
               return
-            response.writeHead(200)
-            response.end()
+
+            deviceId = task.device
+            db.tasksCollection.remove({'_id' : taskId}, (err, removedCount) ->
+              throw err if err
+
+              db.redisClient.del("#{deviceId}_presets_hash", (err) ->
+                throw err if err
+
+                response.writeHead(200)
+                response.end()
+              )
+            )
           )
         else
           response.writeHead 405, {'Allow': 'PUT DELETE'}
           response.end('405 Method Not Allowed')
       else if action is '/retry'
         if request.method == 'POST'
-          db.tasksCollection.update({_id : taskId}, {$unset : {fault : 1}, $set : {timestamp : new Date()}}, (err, count) ->
+          db.tasksCollection.update({_id : taskId}, {$unset : {fault : 1}, $set : {timestamp : new Date()}}, (err, updatedCount) ->
             # TODO need to invalidate presets hash for the device
+            throw err if err
+            if updatedCount == 0
+              response.writeHead(404)
+              response.end()
+              return
+
             response.writeHead(200)
             response.end()
           )
@@ -283,6 +344,11 @@ listener = (request, response) ->
 
         gs = new mongodb.GridStore(db.mongoDb, filename, 'w', {metadata : metadata})
         gs.open((err, gs) ->
+          if err
+            response.writeHead(500)
+            response.end(errorToString(err))
+            return
+
           gs.write(body, (err, res) ->
             throw err if err
             gs.close((err) ->
@@ -294,6 +360,11 @@ listener = (request, response) ->
         )
       else if request.method == 'DELETE'
         mongodb.GridStore.unlink(db.mongoDb, filename, (err) ->
+          if err
+            response.writeHead(500)
+            response.end(errorToString(err))
+            return
+
           response.writeHead(200)
           response.end()
         )
@@ -303,9 +374,9 @@ listener = (request, response) ->
     else if PING_REGEX.test(urlParts.pathname)
       host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1])
       require('child_process').exec("ping -w 1 -i 0.2 -c 3 #{host}", (err, stdout, stderr) ->
-        if err?
+        if err
           response.writeHead(404, {'Cache-Control' : 'no-cache'})
-          response.end()
+          response.end(errorToString(err))
           return
         response.writeHead(200, {'Content-Type' : 'text/plain', 'Cache-Control' : 'no-cache'})
         response.end(stdout)
@@ -320,7 +391,7 @@ listener = (request, response) ->
       apiFunctions.deleteDevice(deviceId, (err) ->
         if err
           response.writeHead(500)
-          response.end(err.message)
+          response.end(errorToString(err))
           return
         response.writeHead(200)
         response.end()
@@ -339,10 +410,16 @@ listener = (request, response) ->
 
       func = (aliases) ->
         if urlParts.query.query?
-          q = JSON.parse(urlParts.query.query)
+          try
+            q = JSON.parse(urlParts.query.query)
+          catch err
+            response.writeHead(400)
+            response.end(errorToString(err))
+            return
         else
           q = {}
         q = query.expand(q, aliases) if collectionName is 'devices'
+        q = query.substituteObjectId(q) if collectionName is 'tasks'
 
         if urlParts.query.projection?
           projection = {}
@@ -360,14 +437,26 @@ listener = (request, response) ->
           sort = {}
           for k,v of s
             if aliases[k]?
-              sort[a] = v for a in aliases[k]
+              for a in aliases[k]
+                if a[a.lastIndexOf('.')+1] != '_' and collectionName is 'devices'
+                  sort["#{a}._value"] = v
+                else
+                  sort[a] = v
             else
-              sort[k] = v
+              if k[k.lastIndexOf('.') + 1] != '_' and collectionName is 'devices'
+                sort["#{k}._value"] = v
+              else
+                sort[k] = v
           cur.sort(sort)
 
         cur.skip(parseInt(urlParts.query.skip)) if urlParts.query.skip?
         cur.limit(parseInt(urlParts.query.limit)) if urlParts.query.limit?
         cur.count((err, total) ->
+          if err
+            response.writeHead(500)
+            response.end(errorToString(err))
+            return
+
           response.writeHead(200, {'Content-Type' : 'application/json', 'total' : total})
           if request.method is 'HEAD'
             response.end()
@@ -375,6 +464,8 @@ listener = (request, response) ->
           response.write("[\n")
           i = 0
           cur.each((err, item) ->
+            throw err if err
+
             if item is null
               response.end("\n]")
             else

lib/normalize.coffee

@@ -27,7 +27,7 @@ common = require './common'
 parameters = require './parameters'
 
 
-stringToRegexp = (input) ->
+stringToRegexp = (input, flags) ->
   if (input.indexOf('*') == -1)
     return false
 
@@ -44,7 +44,7 @@ stringToRegexp = (input) ->
 
   output = output.replace(/[\*]/, '.*')
 
-  return new RegExp(output)
+  return new RegExp(output, flags)
 
 
 normalizers = {}
@@ -68,7 +68,7 @@ normalizers.default = (input, normType) ->
 
     r = stringToRegexp(input)
     if r isnt false
-      vals.push({'$regex' : r})
+      vals.push(r)
 
     return vals
   return input
@@ -90,7 +90,7 @@ normalizers.date = (input, normType) ->
 normalizers.string = (input, normType) ->
   if normType is 'query'
     if (m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input))
-      return [{'$regex' : new RegExp(m[1], m[2])}, input]
+      return [new RegExp(m[1], m[2]), input]
   return input
 
 
@@ -117,14 +117,14 @@ normalizers.mac = (input, normType) ->
 
   if input.indexOf('*') != -1
     input = (colonizeMac(a) for a in input.split('*')).join('*')
-    return stringToRegexp(input, false)
+    return stringToRegexp(input, 'i')
 
   input = colonizeMac(input)
 
   if input.length == 17
-    return input
+    return [input, input.toLowerCase()]
 
-  return {'$regex' : new RegExp(input)}
+  return new RegExp(input, 'i')
 
 
 exports.normalize = (path, value, normType) ->

lib/presets.coffee

@@ -54,12 +54,21 @@ exports.calculatePresetsHash = (presets, objects) ->
 
 
 matchObject = (object, param) ->
-  return false if not object._keys? or object._keys.length == 0
-  for k in object._keys
-    return false if not param[k]?
-    v = common.matchType(param[k]._value, object[k])
-    if param[k]._value != v
+  flatObject = common.flattenObject(object)
+  if object._keys?.length
+    keys = object._keys
+  else
+    # Consider all parameters as keys if none are defined
+    keys = (k for k in Object.keys(flatObject) when k[0] != '_')
+
+  for k in keys
+    p = common.getParamValueFromPath(param, k)
+    return false if not p?
+    v = common.matchType(p._value, flatObject[k])
+
+    if p._value != common.matchType(p._value, flatObject[k])
       return false
+
   return true
 
 
@@ -146,15 +155,7 @@ exports.getDevicePreset = (deviceId, presets, objects, aliases, callback) ->
           devicePreset.objects[c.name][c.object].current ?= {}
           for k,p of param
             continue if k[0] == '_'
-            if p._name?
-              if p._name == c.object
-                devicePreset.objects[c.name][c.object].current[k] = p
-            else if matchObject(objects[c.object], p)
-              u = {}
-              u["#{c.name}.#{k}._name"] = c.object
-              db.devicesCollection.update({'_id' : deviceId}, {'$set' : u}, {}, (err, count) ->
-                throw err if err
-              )
+            if matchObject(objects[c.object], p)
               devicePreset.objects[c.name][c.object].current[k] = p
 
         when 'delete_object'
@@ -165,15 +166,7 @@ exports.getDevicePreset = (deviceId, presets, objects, aliases, callback) ->
           devicePreset.objects[c.name][c.object].current ?= {}
           for k,p of param
             continue if k[0] == '_'
-            if p._name?
-              if p._name == c.object
-                devicePreset.objects[c.name][c.object].current[k] = p
-            else if matchObject(objects[c.object], p)
-              u = {}
-              u["#{c.name}.#{k}._name"] = c.object
-              db.devicesCollection.update({'_id' : deviceId}, {'$set' : u}, {}, (err, count) ->
-                throw err if err
-              )
+            if matchObject(objects[c.object], p)
               devicePreset.objects[c.name][c.object].current[k] = p
 
         when 'custom_command_value'
@@ -250,20 +243,21 @@ exports.processDevicePreset = (deviceId, devicePreset, callback) ->
     for objectName, objectDetails of object
       continue if not objectDetails.current?
       if objectDetails.preset?
+        flatObject = common.flattenObject(objectDetails.preset)
         if Object.keys(objectDetails.current).length > 0
           for i, obj of objectDetails.current
-            for paramName, paramValue of objectDetails.preset
+            for paramName, paramValue of flatObject
               continue if paramName[0] == '_'
-              if obj[paramName]
-                currentValue = obj[paramName]._value
-                presetValue = common.matchType(currentValue, objectDetails.preset[paramName])
-                if currentValue != presetValue
-                  setParameterValues.push(["#{parameterPath}.#{i}.#{paramName}", presetValue, obj[paramName]._type])
+              currentValue = common.getParamValueFromPath(obj, paramName)
+              if currentValue?
+                presetValue = common.matchType(currentValue._value, flatObject[paramName])
+                if currentValue._value != presetValue
+                  setParameterValues.push(["#{parameterPath}.#{i}.#{paramName}", presetValue, currentValue._type])
         else
           vals = []
-          for k,v of objectDetails.preset
+          for k,v of flatObject
             vals.push([k, v]) if k[0] != '_'
-          taskList.push({device : deviceId, name : 'addObject', objectName : parameterPath, parameterValues : vals, instanceName : objectName})
+          taskList.push({device : deviceId, name : 'addObject', objectName : parameterPath, parameterValues : vals})
       else if Object.keys(objectDetails.current).length > 0
         for i, obj of objectDetails.current
           taskList.push({device : deviceId, name : 'deleteObject', objectName : "#{parameterPath}.#{i}"})
@@ -300,11 +294,19 @@ exports.processDevicePreset = (deviceId, devicePreset, callback) ->
 
 
 getObjectHash = (object) ->
-  return object._id if not object._keys? or object._keys.length == 0
-  hash = ''
-  for k in object._keys
-    hash += "#{k}=#{object[k]}"
-  return hash
+  if object._keys?.length
+    keys = object._keys
+  else
+    keys = (k for k in Object.keys(object) when k[0] != '_')
+
+  keys.sort()
+
+  hash = []
+  for k in keys
+    hash.push(k)
+    hash.push(common.getParamValueFromPath(object, k))
+
+  return JSON.stringify(hash)
 
 
 accumulateConfigurations = (presets, objects) ->
@@ -320,6 +322,9 @@ accumulateConfigurations = (presets, objects) ->
           "object_#{c.name}_#{objectHash}"
         when 'software_version'
           'software_version'
+        when 'custom_command_age', 'custom_command_value'
+          [filename, commandName] = c.command.split(' ', 2)
+          "#{c.type}_#{filename}_#{commandName}"
         else
           "#{c.type}_#{c.name}"
 

lib/query.coffee

@@ -22,6 +22,7 @@
 
 common = require './common'
 normalize = require './normalize'
+mongodb = require 'mongodb'
 
 
 expandValue = (param, value) ->
@@ -50,8 +51,17 @@ expandValue = (param, value) ->
   while i < indices.length
     obj = {}
     for i in [0...keys.length]
+      if common.typeOf(values[i][indices[i]]) is common.REGEXP_TYPE
+        if keys[i] == '$ne'
+          obj['$not'] = values[i][indices[i]]
+          continue
+      else if keys[i] == '$not' and common.typeOf(values[i][indices[i]]) isnt common.OBJECT_TYPE
+        # Only a regex or decoument (object) are allowed within a $not operator.
+        # This is needed in order to discard string from which a regex is generated.
+        continue
+
       obj[keys[i]] = values[i][indices[i]]
-    objs.push(obj)
+    objs.push(obj) if Object.keys(obj).length
 
     for i in [0...indices.length]
       indices[i] += 1
@@ -74,7 +84,7 @@ permute = (param, val, aliases) ->
     values = expandValue(k, val)
     if k[k.lastIndexOf('.') + 1] != '_'
       k += '._value'
-  
+
     for v in values
       obj = {}
       obj[k] = v
@@ -94,14 +104,41 @@ expand = (query, aliases) ->
     else
       conditions = permute(k, v, aliases)
       if conditions.length > 1
-        if new_query['$and']?
-          new_query['$and'].push({'$or' : conditions})
+        new_query['$and'] ?= []
+        if v?['$ne']? or v?['$nin']? or v?['$not']?
+          for c in conditions
+            new_query['$and'].push(c)
         else
-          new_query['$and'] = [{'$or' : conditions}]
+          new_query['$and'].push({'$or' : conditions})
       else
         common.extend(new_query, conditions[0])
 
   return new_query
 
+# Replace _id string values with ObjectID type
+substituteObjectId = (query) ->
+  for k, v of query
+    if k[0] == '$' # logical operator
+      for i in [0...v.length]
+        substituteObjectId(v[i])
+    else if k == '_id'
+      if common.typeOf(v) is common.STRING_TYPE
+        query[k] = mongodb.ObjectID(v) if v.length == 24
+      else if common.typeOf(v) is common.OBJECT_TYPE
+        for kk, vv of v
+          switch kk # operator
+            when '$in', '$nin'
+              for i in [0...vv.length]
+                vv[i] = mongodb.ObjectID(vv[i]) if vv[i].length == 24
+            when '$eq', '$gt', '$gte', '$lt', '$lte', '$ne'
+              v[kk] = mongodb.ObjectID(vv) if vv.length == 24
+            when '$exists', '$type' then
+              # ignore
+            else
+              throw new Error('Operator not supported')
+
+  return query
+
 
 exports.expand = expand
+exports.substituteObjectId = substituteObjectId

lib/server.coffee

@@ -51,7 +51,7 @@ listener = (httpRequest, httpResponse) ->
       )
 
       httpResponse.writeHead(500, {'Connection' : 'close'})
-      httpResponse.end()
+      httpResponse.end("#{err.name}: #{err.message}")
     catch err2
       util.error("#{new Date().toISOString()} - #{err2.stack}\n")
   )

lib/tasks.coffee

@@ -43,6 +43,7 @@ common = require './common'
 db = require './db'
 mongodb = require 'mongodb'
 customCommands = require './custom-commands'
+url = require 'url'
 
 exports.STATUS_OK = STATUS_OK = 1
 exports.STATUS_FAULT = STATUS_FAULT = 2
@@ -220,7 +221,6 @@ this.addObject = (task, methodResponse, callback) ->
       task.session.subtask = {device : task.device, name : 'getParameterNames', parameterPath : "#{task.objectName}.#{task.session.instanceNumber}.", nextLevel : false}
       task.session.appliedParameterValues = []
       task.session.parameterNames = []
-      allDeviceUpdates.instanceName = [["#{task.objectName}.#{task.session.instanceNumber}", task.instanceName]] if task.instanceName?
     else
       callback(null, STATUS_OK, {type : 'AddObject', objectName : "#{task.objectName}."})
       return
@@ -251,7 +251,7 @@ this.addObject = (task, methodResponse, callback) ->
             if deviceUpdates and deviceUpdates.parameterValues?
               for p1 in deviceUpdates.parameterValues
                 for p2 in task.parameterValues
-                  if common.endsWith(p1[0], ".#{p2[0]}")
+                  if p1[0] == "#{task.objectName}.#{task.session.instanceNumber}.#{p2[0]}"
                     t = if p2[2] then p2[2] else p1[2]
                     v = common.matchType(p1[1], p2[1])
                     # TODO only include if writable
@@ -296,7 +296,7 @@ this.reboot = (task, methodResponse, callback) ->
     task.fault = methodResponse
     callback(null, STATUS_FAULT)
     return
-  
+
   if methodResponse.type isnt 'RebootResponse'
     callback(null, STATUS_OK, {type : 'Reboot'})
   else
@@ -322,24 +322,39 @@ this.download = (task, methodResponse, callback) ->
     return
 
   if methodResponse.type isnt 'DownloadResponse'
-    db.filesCollection.findOne({_id : mongodb.ObjectID(String(task.file))}, (err, file) ->
-      if not file?
-        callback('File not found')
-        return
-      else if err?
-        callback(err)
-        return
+    if task.url?
+        methodRequest = {
+          type : 'Download',
+          fileType : task.fileType or '1 Firmware Upgrade Image',
+          url : task.url,
+          username: task.username,
+          password: task.password,
+          successUrl : task.successUrl,
+          failureUrl : task.failureUrl
+        }
+        callback(null, STATUS_OK, methodRequest)
+    else
+      db.filesCollection.findOne({_id : mongodb.ObjectID(String(task.file))}, (err, file) ->
+        if err or not file?
+          return callback(err or new Error('File not found'))
 
-      methodRequest = {
-        type : 'Download',
-        fileType : file.metadata.fileType,
-        fileSize : file.length,
-        url : "http://#{config.get('FS_IP')}:#{config.get('FS_PORT')}/#{encodeURIComponent(file.filename)}",
-        successUrl : task.successUrl,
-        failureUrl : task.failureUrl
-      }
-      callback(null, STATUS_OK, methodRequest)
-    )
+        l = {
+          protocol : if config.get('FS_SSL') then 'https' else 'http',
+          hostname : config.get('FS_IP'),
+          port : config.get('FS_PORT'),
+          pathname : encodeURIComponent(file.filename)
+        }
+
+        methodRequest = {
+          type : 'Download',
+          fileType : file.metadata.fileType,
+          fileSize : file.length,
+          url : url.format(l),
+          successUrl : task.successUrl,
+          failureUrl : task.failureUrl
+        }
+        callback(null, STATUS_OK, methodRequest)
+      )
   else
     callback(null, STATUS_COMPLETED)
 
@@ -347,8 +362,8 @@ this.download = (task, methodResponse, callback) ->
 this.customCommand = (task, methodResponse, callback) ->
   # TODO implement timeout
   customCommands.execute(task.device, task.command, (err, value) ->
-    if err?
-      task.fault = err
+    if err
+      task.fault = {name : err.name, message : err.message}
       callback(null, STATUS_FAULT)
     else
       commandName = task.command.split(' ', 2)[0]

lib/update-device.coffee

@@ -53,8 +53,8 @@ commitUpdates = (deviceId, deviceUpdates, commitQueue, callback) ->
   processBatch = (updatesBatch, cb) ->
     if updatesBatch.informEvents?
       update['$set']['_lastInform'] = now
-      update['$set']['_lastBoot'] if '1 BOOT' in updatesBatch.informEvents
-      update['$set']['_lastBootstrap'] if '0 BOOTSTRAP' in updatesBatch.informEvents
+      update['$set']['_lastBoot'] = now if '1 BOOT' in updatesBatch.informEvents
+      update['$set']['_lastBootstrap'] = now if '0 BOOTSTRAP' in updatesBatch.informEvents
 
     if updatesBatch.parameterValues?
       for p in updatesBatch.parameterValues
@@ -75,10 +75,6 @@ commitUpdates = (deviceId, deviceUpdates, commitQueue, callback) ->
       for p in updatesBatch.deletedObjects
         update['$unset'][p] = 1
 
-    if updatesBatch.instanceName?
-      for i in updatesBatch.instanceName
-        update['$set']["#{i[0]}._name"] = i[1]
-
     if updatesBatch.parameterNames?
       presentPaths ?= {}
       projection ?= {}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.0.1",
+  "version": "1.0.7",
   "description": "A blazing fast TR-069 auto configuration server (ACS) built with Node.js, Redis, and MongoDB",
   "repository": {
     "type": "git",
@@ -12,7 +12,7 @@
     "ACS"
   ],
   "author": "Zaid Abdulla",
-  "license": "AGPL3",
+  "license": "AGPL-3.0",
   "readmeFilename": "README.md",
   "preferGlobal": true,
   "directories": {
@@ -20,13 +20,13 @@
     "bin": "bin"
   },
   "dependencies": {
-    "libxmljs": "~0.11.0",
-    "redis": "~0.12.1",
-    "hiredis": "~0.1.17",
-    "mongodb": "~1.4.8"
+    "libxmljs": "0.11.0 - 0.14.x",
+    "redis": "^0.12.1",
+    "hiredis": "0.1.17 - 0.4.x",
+    "mongodb": "^1.4.8"
   },
   "devDependencies": {
-    "coffee-script": "~1.7.1"
+    "coffee-script": "^1.7.1"
   },
   "engines": {
     "node": ">=0.10.0 <0.12"