1.0.4

Fix SyntaxError on the inform message

.gitignore

@@ -1,3 +1,4 @@
 *~
 *.js
 node_modules
+debug/*

README.md

@@ -19,10 +19,10 @@ Install [Node.js](http://nodejs.org/), [Redis](http://redis.io/), and [MongoDB](
 
 You may need to modify the configuration files under "config" directory (in /lib/node_modules/genieacs/config) depending on your setup.
 
-Alternatively, you can use git to get the latest development version (not recommended for production use):
+Alternatively, you can install from source by cloning the git repository:
 
     cd /opt
-    git clone https://github.com/zaidka/genieacs.git
+    git clone https://github.com/zaidka/genieacs.git --branch v1.0
     cd genieacs
     npm install
     npm run configure
@@ -54,13 +54,12 @@ You may now proceed with installing [GenieACS GUI front end](https://github.com/
 
 ## Support
 
-Documentation is still work in progress. Feel free to contact me if you require any assistance in installation, configuration, or using the APIs.
+The [Users mailing list](http://lists.genieacs.com) is a good place to get guidance and help from the community. Head on over and join the conversation! In addition, the [wiki](https://github.com/zaidka/genieacs/wiki) provides useful documentation and tips from GenieACS users.
 
-You may submit bug reports or feature requests [here](https://github.com/zaidka/genieacs/issues).
+You may submit bug reports or feature requests [here](https://github.com/zaidka/genieacs/issues). For device interoperability issues, please consult the mailing list first — it's likely that a workaround already exists.
 
-For commercial support options, please contact me.
+For commercial support options and professional services, please visit [genieacs.com](https://genieacs.com).
 
 ## Contributing
 
 Contributions are welcome. Fork this repo and open a pull request and wait for feedback. You can also contribute by enhancing the documentation in the [wiki section](https://github.com/zaidka/genieacs/wiki).
-

bin/genieacs-cwmp

@@ -2,9 +2,8 @@
 var path = require('path');
 var fs = require('fs');
 
-var dir = path.join(path.dirname(fs.realpathSync(__filename)), '../lib/');
+var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.title = 'genieacs-cwmp';
 process.chdir(dir);
 
-require(path.join(dir, './cwmp'));
+cluster = require(path.resolve(dir, 'lib/cluster'));
-
+cluster.start('cwmp');

bin/genieacs-fs

@@ -2,9 +2,8 @@
 var path = require('path');
 var fs = require('fs');
 
-var dir = path.join(path.dirname(fs.realpathSync(__filename)), '../lib/');
+var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.title = 'genieacs-fs';
 process.chdir(dir);
 
-require(path.join(dir, './fs'));
+cluster = require(path.resolve(dir, 'lib/cluster'));
-
+cluster.start('fs');

bin/genieacs-nbi

@@ -2,9 +2,8 @@
 var path = require('path');
 var fs = require('fs');
 
-var dir = path.join(path.dirname(fs.realpathSync(__filename)), '../lib/');
+var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.title = 'genieacs-nbi';
 process.chdir(dir);
 
-require(path.join(dir, './nbi'));
+cluster = require(path.resolve(dir, 'lib/cluster'));
-
+cluster.start('nbi');

config/config-sample.json

@@ -1,16 +1,15 @@
 {
-  "DATABASE_NAME" : "genieacs",
+  "MONGODB_CONNECTION_URL" : "mongodb://127.0.0.1/genieacs",
-  "MONGODB_SOCKET" : "/tmp/mongodb-27017.sock",
+  "REDIS_PORT" : "6379",
-  "REDIS_SOCKET" : "6379",
+  "REDIS_HOST" : "127.0.0.1",
-  "ACS_INTERFACE" : "0.0.0.0",
+  "CWMP_INTERFACE" : "0.0.0.0",
-  "ACS_PORT" : 7547,
+  "CWMP_PORT" : 7547,
-  "ACS_HTTPS_INTERFACE" : "0.0.0.0",
+  "CWMP_SSL" : false,
-  "ACS_HTTPS_PORT" : 7548,
+  "NBI_INTERFACE" : "0.0.0.0",
-  "API_INTERFACE" : "0.0.0.0",
+  "NBI_PORT" : 7557,
-  "API_PORT" : 7557,
+  "FS_INTERFACE" : "0.0.0.0",
-  "FILES_INTERFACE" : "0.0.0.0",
+  "FS_PORT" : 7567,
-  "FILES_PORT" : 7567,
+  "FS_IP" : "192.168.0.1",
-  "FILES_IP" : "192.168.0.1",
   "LOG_INFORMS" : true,
-  "DEBUG_DEVICES" : {}
+  "DEBUG" : false
 }

config/custom_commands/greenpacket_frequencies.js

@@ -1,3 +1,25 @@
+/*
+ * Copyright 2013 Fanoos Telecom
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
 var net = require('net');
 var PORT = 23;
 var USERNAME = 'mt7109';
@@ -47,7 +69,7 @@ var telnetExecute = function(command, connection, prompt, callback) {
 }
 
 var getDeviceIp = function(deviceId, callback) {
-  var db = require('../../db');
+  var db = require('../../lib/db');
   var URL = require('url');
 
   db.devicesCollection.findOne({_id : deviceId}, {'InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value' : 1}, function(err, device) {

config/custom_commands/seowon_frequencies.js

@@ -1,3 +1,25 @@
+/*
+ * Copyright 2013 Fanoos Telecom
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
 // Change frequency plans for Seowon devices.
 // Args format: 3532500 5000,3537500 5000,3542500 5000,3547500 5000,3552500 5000,3557500 5000,3505000 10000,3515000 10000,3525000 10000,3535000 10000,3545000 10000,3555000 10000
 var net = require('net');
@@ -68,7 +90,7 @@ var findFrequency = function(haystack, needle) {
 }
 
 var getDeviceIp = function(deviceId, callback) {
-  var db = require('../../db');
+  var db = require('../../lib/db');
   var URL = require('url');
 
   db.devicesCollection.findOne({_id : deviceId}, {'InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value' : 1}, function(err, device) {

config/httpscert-sample.crt

@@ -1,17 +0,0 @@
------BEGIN CERTIFICATE-----
-MIICqTCCAhICCQDSrolRw1qPBzANBgkqhkiG9w0BAQUFADCBmDELMAkGA1UEBhMC
-SVExFTATBgNVBAgMDFN1bGF5bWFuaXlhaDEVMBMGA1UEBwwMU3VsYXltYW5peWFo
-MQ8wDQYDVQQKDAZGYW5vb3MxEzARBgNVBAsMClRlY2hub2xvZ3kxDjAMBgNVBAMM
-BWdlbmllMSUwIwYJKoZIhvcNAQkBFhZ6YWlkLmFiZHVsbGFAZmFub29zLmlxMB4X
-DTEzMDIxNzEyNTYzN1oXDTE0MDIxNzEyNTYzN1owgZgxCzAJBgNVBAYTAklRMRUw
-EwYDVQQIDAxTdWxheW1hbml5YWgxFTATBgNVBAcMDFN1bGF5bWFuaXlhaDEPMA0G
-A1UECgwGRmFub29zMRMwEQYDVQQLDApUZWNobm9sb2d5MQ4wDAYDVQQDDAVnZW5p
-ZTElMCMGCSqGSIb3DQEJARYWemFpZC5hYmR1bGxhQGZhbm9vcy5pcTCBnzANBgkq
-hkiG9w0BAQEFAAOBjQAwgYkCgYEAxtHC3jlOoVsZQ9I8Wgd+8NkEaRfc01lRIAzj
-eWuUO/xNY54EUDHecRxwolHtoGkkZAp4kKR3uUVdAo9uXmNT2CeeZ5lfRoa8WmgG
-Yd6PExsdc4ZeKcw20XrKtQICqmU6rGlLERMFX97ejQh3eUmqwbzNaXiURxcKGdeK
-2C/NxoUCAwEAATANBgkqhkiG9w0BAQUFAAOBgQAYFw6dy/z74ahnA5QpuJOjjTrt
-AJaL9iuL1xS6TFfzne9UMIT+fFJsgSy6xx3TGkvbZJTnCzJN7I2uqmAhMoGFy7Ea
-svCCkj8U5qkjqgwwi7IbwQ7fXxTVdPqUlkIdNB2YLDELksxCX+ZzGQPiFB8WrSoO
-OBvB+hhPFpgp3aO2AQ==
------END CERTIFICATE-----

config/httpscert-sample.key

@@ -1,15 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIICXgIBAAKBgQDG0cLeOU6hWxlD0jxaB37w2QRpF9zTWVEgDON5a5Q7/E1jngRQ
-Md5xHHCiUe2gaSRkCniQpHe5RV0Cj25eY1PYJ55nmV9GhrxaaAZh3o8TGx1zhl4p
-zDbResq1AgKqZTqsaUsREwVf3t6NCHd5SarBvM1peJRHFwoZ14rYL83GhQIDAQAB
-AoGAR4NpuW8ZhlZzNv7m8Gcj1NPqjzajGm5VDNc19xWoAw+UcpGC8amOXlcheHFJ
-rNpE2asehMBzw/NcKjCXPWIy5LnGmttJMN2EbT2h/+uxBIM5IUc+joF744xF4C0t
-5zrtqQYJxC7sBLX2vf7nY6iuovUM7fHEwACyV3Rpw3zdZf0CQQDiqW+ugLKcfS/c
-6hkGTxeYWF4gUJTf3ke3au8nyMmUhyBwOuP4kpO72mK8bMLtjZRk3g6YChA4qH41
-nfXUXIh7AkEA4I2/zj4GDjjspDR+/tX0iUHQk1Rm2CRmRXXFJV1f688OLODCcOFK
-Ms/+qnbe41wXo1NUxCqVrnPwEh3MVl88/wJBANUTpXqtX7s+8ETECTxmE+9Se0Hz
-ntCJXG/v5d1Ho0Cueu8mXXMrEBiTpygtMl4xL2RXX+7L3w30T5lF4nh9eIsCQQDY
-DmsbdL/81TaTc9MxPQlxOzsFVrMxiixA7uTGY8ZULD/2XDeZxprOsyPUPlWMquIv
-ipFHwbWNSg4WUEd4X2BfAkEAytze+0mCd0WCsKbKwfTQnmJ3bK7JQywtC03llWyA
-Om3Vl61Syaa7M2hl5rLvitZMvU1Wd0s8raOXN8RUuRXZBw==
------END RSA PRIVATE KEY-----

config/parameters-sample.json

@@ -8,12 +8,20 @@
   "_deviceId._Manufacturer" : {"alias" : "summary.manufacturer"},
   "InternetGatewayDevice.DeviceInfo.HardwareVersion" : {"alias" : "summary.hardwareVersion"},
   "InternetGatewayDevice.DeviceInfo.SoftwareVersion" : {"alias" : "summary.softwareVersion"},
+  "Device.DeviceInfo.HardwareVersion" : {"alias" : "summary.hardwareVersion"},
+  "Device.DeviceInfo.SoftwareVersion" : {"alias" : "summary.softwareVersion"},
 
   "InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
   "InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
   "InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
   "InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
+  "Device.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
+  "Device.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
+  "Device.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
+  "Device.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
 
   "InternetGatewayDevice.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.SSID" : {"alias" : "summary.wlanSsid"},
-  "InternetGatewayDevice.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.PreSharedKey.1.KeyPassphrase" : {"alias" : "summary.wlanPassphrase"}
+  "InternetGatewayDevice.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.PreSharedKey.1.KeyPassphrase" : {"alias" : "summary.wlanPassphrase"},
+  "Device.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.SSID" : {"alias" : "summary.wlanSsid"},
+  "Device.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.PreSharedKey.1.KeyPassphrase" : {"alias" : "summary.wlanPassphrase"}
 }

lib/api-functions.coffee

@@ -1,3 +1,43 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This file incorporates work covered by the following copyright and
+# permission notice:
+#
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
 config = require './config'
 db = require './db'
 http = require 'http'
@@ -11,6 +51,17 @@ connectionRequest = (deviceId, callback) ->
   # ensure socket is reused in case of digest authentication
   agent = new http.Agent({maxSockets : 1})
 
+  statusToError = (statusCode) ->
+    switch statusCode
+      when 200, 204
+        null
+      when 401
+        new Error('Incorrect connection request credentials')
+      when 0
+        new Error('Device is offline')
+      else
+        new Error("Unexpected response code from device: #{statusCode}")
+
   conReq = (url, authString, callback) ->
     options = URL.parse(url)
     options.agent = agent
@@ -19,7 +70,7 @@ connectionRequest = (deviceId, callback) ->
       options.headers = {'Authorization' : authString}
 
     request = http.get(options, (res) ->
-      if res.statusCode == 401
+      if res.statusCode == 401 and res.headers['www-authenticate']?
         authHeader = auth.parseAuthHeader(res.headers['www-authenticate'])
       callback(res.statusCode, authHeader)
       # don't need body, go ahead and emit events to free up the socket for possible reuse
@@ -39,11 +90,21 @@ connectionRequest = (deviceId, callback) ->
       )
     )
 
-  db.devicesCollection.findOne({_id : deviceId}, {'InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value' : 1}, (err, device)->
+  proj = {
+    'Device.ManagementServer.ConnectionRequestURL._value' : 1,
+    'InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value' : 1
+  }
+
+  db.devicesCollection.findOne({_id : deviceId}, proj, (err, device)->
     if err
       callback(err)
       return
+
+    if device.Device? # TR-181 data model
+      connectionRequestUrl = device.Device.ManagementServer.ConnectionRequestURL._value
+    else # TR-098 data model
       connectionRequestUrl = device.InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value
+
     # for testing
     #connectionRequestUrl = connectionRequestUrl.replace(/^(http:\/\/)([0-9\.]+)(\:[0-9]+\/[a-zA-Z0-9]+\/?$)/, '$110.1.1.254$3')
     conReq(connectionRequestUrl, null, (statusCode, authHeader) ->
@@ -56,10 +117,13 @@ connectionRequest = (deviceId, callback) ->
           authString = auth.digest(username, password, uri.path, 'GET', null, authHeader)
 
         conReq(connectionRequestUrl, authString, (statusCode, authHeader) ->
-          callback()
+          if statusCode == 0
+            # Workaround for some devices unexpectedly closing the connection
+            return conReq(connectionRequestUrl, authString, (statusCode) -> callback(statusToError(statusCode)))
+          callback(statusToError(statusCode))
         )
       else
-        callback()
+        callback(statusToError(statusCode))
     )
   )
 
@@ -169,8 +233,21 @@ insertTasks = (tasks, aliases, callback) ->
     )
 
 
+deleteDevice = (deviceId, callback) ->
+  db.tasksCollection.remove({'device' : deviceId}, (err, removed) ->
+    return callback(err) if err
+    db.devicesCollection.remove({'_id' : deviceId}, (err, removed) ->
+      return callback(err) if err
+      db.redisClient.del("#{deviceId}_presets_hash", "#{deviceId}_inform_hash", (err) ->
+        callback(err)
+      )
+    )
+  )
+
+
 exports.addAliases = addAliases
 exports.sanitizeTask = sanitizeTask
 exports.connectionRequest = connectionRequest
 exports.watchTask = watchTask
 exports.insertTasks = insertTasks
+exports.deleteDevice = deleteDevice

lib/auth.coffee

@@ -1,3 +1,20 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
 crypto = require 'crypto'
 
 
@@ -7,11 +24,11 @@ exports.parseAuthHeader = (authHeader) ->
   res['method'] = authHeader.slice(0, i)
 
   options = authHeader.slice(i + 1)
-  regex = /([a-z]+)="([a-zA-Z0-9\/\.@\-,=]+)"/g
+  regex = /([a-z]+)="([a-zA-Z0-9\/\.@\-\+,= ]+)"/g
   while (r = regex.exec(options)) != null
     res[r[1]] = r[2]
 
-  regex = /([a-z]+)=([a-zA-Z0-9\/\.@\-]+)/g
+  regex = /([a-z]+)=([a-zA-Z0-9\/\.@\-\+]+)/g
   while (r = regex.exec(options)) != null
     res[r[1]] = r[2]
   return res

lib/cluster.coffee

@@ -0,0 +1,50 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
+config = require './config'
+cluster = require 'cluster'
+util = require 'util'
+
+# Keep current configuration to pass that to workers as environment variables
+ENVIRONMENT_VARIABLES = {}
+for k, v of config.allConfig
+  ENVIRONMENT_VARIABLES["GENIEACS_#{k}"] = v
+
+
+cluster.on('listening', (worker, address) ->
+  util.log("Worker #{worker.process.pid} listening to #{address.address}:#{address.port}")
+)
+
+cluster.on('exit', (worker, code, signal) ->
+  util.log("Worker #{worker.process.pid} died (#{worker.process.exitCode})")
+  cluster.fork(ENVIRONMENT_VARIABLES)
+)
+
+
+start = (service) ->
+  workerProcesses = config.get("#{service.toUpperCase()}_WORKER_PROCESSES")
+
+  cluster.setupMaster({
+      exec : 'lib/server',
+      args : [service]
+  })
+
+  for [0 ... workerProcesses]
+    cluster.fork(ENVIRONMENT_VARIABLES)
+
+
+exports.start = start

lib/common.coffee

@@ -1,3 +1,28 @@
+###
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
+buffer = require 'buffer'
+querystring = require 'querystring'
+
 UNDEFINED_TYPE = '[object Undefined]'
 NULL_TYPE = '[object Null]'
 BOOLEAN_TYPE = '[object Boolean]'
@@ -28,11 +53,11 @@ exports.arrayToHash = (arr) ->
   return hash
 
 
-exports.getDeviceId = (deviceIdStruct) ->
+exports.generateDeviceId = (deviceIdStruct) ->
   # Percent escaping function. Escapes everything except alphanumerics and underscore
   esc = (str) ->
     str.replace(/[^A-Za-z0-9_]/g, (chr) ->
-      buf = new require('buffer').Buffer(chr)
+      buf = new buffer.Buffer(chr, 'utf8')
       rep = ''
       rep += "%#{b.toString(16).toUpperCase()}" for b in buf
       return rep
@@ -45,11 +70,34 @@ exports.getDeviceId = (deviceIdStruct) ->
   return "#{esc(deviceIdStruct['OUI'])}-#{esc(deviceIdStruct['SerialNumber'])}"
 
 
+exports.parseDeviceId = (deviceId) ->
+  parts = deviceId.split('-')
+  ret = {oui : querystring.unescape(parts[0])}
+  if parts.length == 3
+    ret.productClass = querystring.unescape(parts[1])
+    ret.serialNumber = querystring.unescape(parts[2])
+  else
+    ret.serialNumber = querystring.unescape(parts[1])
+  return ret
+
+
 exports.extend = (obj, mixin) ->
   obj[name] = method for name, method of mixin
   obj
 
 
+exports.flattenObject = (object) ->
+  newObj = {}
+  f = (obj, prefix) ->
+    for k, v of obj
+      if typeof(v) is 'object'
+        f(v, "#{prefix}#{k}.")
+      else
+        newObj["#{prefix}#{k}"] = v
+  f(object, '')
+  return newObj
+
+
 exports.getParamValueFromPath = (obj, path) ->
   pp = path.split('.')
   ref = obj

lib/config.coffee

@@ -1,45 +1,171 @@
-exports.DATABASE_NAME = 'genieacs'
+###
-exports.ACS_PORT = 7547 # CWMP port as assigned by IANA
+# Copyright 2013, 2014  Zaid Abdulla
-exports.ACS_HTTPS_PORT = 7548
+#
-exports.ACS_INTERFACE = '0.0.0.0'
+# GenieACS is free software: you can redistribute it and/or modify
-exports.ACS_HTTPS_INTERFACE = '0.0.0.0'
+# it under the terms of the GNU Affero General Public License as
-exports.API_PORT = 7557
+# published by the Free Software Foundation, either version 3 of the
-exports.API_INTERFACE = '0.0.0.0'
+# License, or (at your option) any later version.
-exports.FILES_PORT = 7567
+#
-exports.FILES_INTERFACE = '0.0.0.0'
+# GenieACS is distributed in the hope that it will be useful,
-exports.FILES_IP = '172.240.21.2' # Used when sending download requests to devices
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
-exports.CACHE_DURATION = 60 # in seconds
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-exports.PRESETS_CACHE_DURATION = 86400
+# GNU Affero General Public License for more details.
-exports.PRESETS_TIME_PADDING = 1
+#
-exports.MONGODB_SOCKET = '/tmp/mongodb-27017.sock'
+# You should have received a copy of the GNU Affero General Public License
-exports.REDIS_SOCKET = '/tmp/redis.sock'
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-exports.WORKER_RESPAWN_TIME = 60000
+###
-exports.LOG_INFORMS = false
-exports.DEBUG_DEVICES = {} # {'202BC1-BM632w-8KA8WA1151100043' : true}
-exports.DEVICE_ONLINE_THRESHOLD = 4000
-exports.RETRY_DELAY = 300
-exports.IGNORE_XML_NAMESPACES = false # Traverse XML using element's local name only
 
-exports.MONGODB_OPTIONS = {
+path = require 'path'
-  db : {
+common = require './common'
-    w : 1
+
-    wtimeout : 60000
+options = {
-  }
+  CONFIG_DIR : {type : 'string', default : 'config'},
-  server : {
+  MONGODB_CONNECTION_URL : {type : 'string', default : 'mongodb://127.0.0.1/genieacs'},
-    auto_reconnect : true
+  REDIS_PORT : {type : 'int', default : 6379},
-  }
+  REDIS_HOST : {type : 'string', default : '127.0.0.1'},
+  REDIS_DB : {type : 'int', default : 0},
+
+  CWMP_WORKER_PROCESSES : {type : 'int', default : 4},
+  CWMP_PORT : {type : 'int', default : 7547},
+  CWMP_INTERFACE : {type : 'string', default : '0.0.0.0'},
+  CWMP_SSL : {type : 'bool', default : false},
+
+  NBI_WORKER_PROCESSES : {type : 'int', default : 2},
+  NBI_PORT : {type : 'int', default : 7557},
+  NBI_INTERFACE : {type : 'string', default : '0.0.0.0'},
+  NBI_SSL : {type : 'bool', default : false},
+
+  FS_WORKER_PROCESSES : {type : 'int', default : 2},
+  FS_PORT : {type : 'int', default : 7567},
+  FS_INTERFACE : {type : 'string', default : '0.0.0.0'},
+  FS_SSL : {type : 'bool', default : false},
+  FS_IP : {type : 'string', default : '192.168.0.1'},
+
+  PRESETS_CACHE_DURATION : {type : 'int', default : 86400},
+  LOG_INFORMS : {type : 'bool', default : true},
+  DEBUG : {type : 'bool', default : false},
+  RETRY_DELAY : {type : 'int', default : 300},
+  IGNORE_XML_NAMESPACES : {type : 'bool', default : false},
+  SESSION_TIMEOUT : {type : 'int', default : 30},
+  GET_PARAMETER_NAMES_DEPTH_THRESHOLD : {type : 'int', default : 0},
+  TASK_PARAMETERS_BATCH_SIZE : {type : 'int', default : 32},
+  COOKIES_PATH : {type : 'string'},
+
+  # Libxml related configuration
+  XML_PARSE_RECOVER : {type : 'bool'},
+  XML_PARSE_NOENT : {type : 'bool'},
+  XML_PARSE_NOBLANKS : {type : 'bool'},
+  XML_PARSE_NSCLEAN : {type : 'bool'},
+  XML_PARSE_NOCDATA : {type : 'bool'},
+  XML_PARSE_IGNORE_ENC : {type : 'bool'},
+
+  # Should probably never be changed
+  PRESETS_TIME_PADDING : {type : 'int', default : 1},
+  DEVICE_ONLINE_THRESHOLD : {type : 'int', default : 4000}
 }
 
-# load configuration
+allConfig = {}
-c = require('../config/config')
+
-for k, v of c
+
-  exports[k] = v
+setConfig = (name, value, commandLineArgument) ->
+  return true if allConfig[name]?
+
+  cast = (val, type) ->
+    switch type
+      when 'int'
+        Number(val)
+      when 'bool'
+        String(val).trim().toLowerCase() in ['true', 'on', 'yes', '1']
+      when 'string'
+        String(val)
+      else
+        null
+
+  _value = null
+
+  for optionName, optionDetails of options
+    n = optionName
+    if commandLineArgument
+      n = n.toLowerCase().replace(/_/g, '-')
+
+    if name == n
+      _value = cast(value, optionDetails.type)
+      n = optionName
+    else if common.startsWith(name, "#{n}-")
+      _value = cast(value, optionDetails.type)
+      n = "#{optionName}-#{name[optionName.length+1..]}"
+
+    if _value?
+      allConfig[n] = _value
+      # Save as environmnet variable to pass on to any child processes
+      process.env[n] = _value
+      return true
+
+  return false
+
+
+# Command line arguments
+exports.argv = []
+argv = process.argv[2..]
+while argv.length
+  arg = argv.shift()
+  if arg[0] == '-'
+    v = argv.shift()
+    exports.argv[arg] = v
+    setConfig(arg[2..], v, true)
+  else
+    exports.argv.push(arg)
+
+
+# Environment variable
+for k, v of process.env
+  continue if k.lastIndexOf('GENIEACS_', 0) != 0
+  k = k[9..] # remove "GENIEACS_" prefix
+  setConfig(k, v)
+
+
+# Find config dir
+if exports.argv['--config-dir']?
+  allConfig.CONFIG_DIR = exports.argv['--config-dir']
+else if process.env['GENIEACS_CONFIG_DIR']?
+  allConfig.CONFIG_DIR = process.env['GENIEACS_CONFIG_DIR']
+else
+  allConfig.CONFIG_DIR = options.CONFIG_DIR.default
+
+
+# Configuration file
+for k, v of require(path.resolve(allConfig.CONFIG_DIR, 'config'))
+  setConfig(k, v)
+
+
+# Defaults
+for k, v of options
+  setConfig(k, v.default) if v.default?
+
+
+get = (option, deviceId) ->
+  if deviceId?
+    name = "#{option}-#{deviceId}"
+    v = allConfig[name]
+    return v if v?
+    i = name.lastIndexOf('-')
+    v = allConfig[name[0...i]]
+    return v if v?
+    i = name.lastIndexOf('-', i-1)
+    v = allConfig[name[0...i]]
+    return v if v? or i == -1
+
+  return allConfig[option]
+
 
 # load parameter configurations
-exports.PARAMETERS = require('../config/parameters')
+exports.PARAMETERS = require(path.resolve(allConfig.CONFIG_DIR, 'parameters'))
 
 # load authentication scripts
-exports.auth = require('../config/auth')
+exports.auth = require(path.resolve(allConfig.CONFIG_DIR, 'auth'))
 
-exports.CUSTOM_COMMANDS = require('../config/custom_commands')
+exports.CUSTOM_COMMANDS = require(path.resolve(allConfig.CONFIG_DIR, 'custom_commands'))
 
+exports.get = get
+
+exports.allConfig = allConfig

lib/custom-commands.coffee

@@ -1,3 +1,20 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
 config = require './config'
 common = require './common'
 

lib/cwmp.coffee

@@ -1,23 +1,56 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This file incorporates work covered by the following copyright and
+# permission notice:
+#
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
 config = require './config'
 common = require './common'
 util = require 'util'
-http = require 'http'
-https = require 'https'
 soap = require './soap'
 tasks = require './tasks'
-normalize = require('./normalize').normalize
 db = require './db'
 presets = require './presets'
 mongodb = require 'mongodb'
-fs = require 'fs'
 apiFunctions = require './api-functions'
 customCommands = require './custom-commands'
 crypto = require 'crypto'
-parameters = require './parameters'
 zlib = require 'zlib'
-
+updateDevice = require './update-device'
-OBJECT_REGEX = /\.$/
-INSTANCE_REGEX = /\.[\d]+\.$/
 
 
 # Used to reject new TR-069 sessions when under load
@@ -25,19 +58,14 @@ holdUntil = Date.now()
 
 
 writeResponse = (currentRequest, res) ->
-  data = new Buffer(res.data)
+  if config.get('DEBUG', currentRequest.session.deviceId)
-  if data.length == 0
+    dump = "# RESPONSE #{new Date(Date.now())}\n" + JSON.stringify(res.headers) + "\n#{res.data}\n\n"
-    # no more requests. terminate TCP connection
+    require('fs').appendFile("./debug/#{currentRequest.session.deviceId}.dump", dump, (err) ->
-    res.headers['Connection'] = 'close'
-
-  if config.DEBUG_DEVICES[currentRequest.deviceId]
-    dump = "# RESPONSE #{new Date(Date.now())}\n" + JSON.stringify(res.headers) + "\n#{data}\n\n"
-    fs = require('fs').appendFile("../debug/#{currentRequest.deviceId}.dump", dump, (err) ->
       throw err if err
     )
 
   # respond using the same content-encoding as the request
-  if currentRequest.httpRequest.headers['content-encoding']? and data.length > 0
+  if currentRequest.httpRequest.headers['content-encoding']? and res.data.length > 0
     switch currentRequest.httpRequest.headers['content-encoding']
       when 'gzip'
         res.headers['Content-Encoding'] = 'gzip'
@@ -47,95 +75,23 @@ writeResponse = (currentRequest, res) ->
         compress = zlib.deflate
 
   if compress?
-    compress(data, (err, data) ->
+    compress(res.data, (err, data) ->
       res.headers['Content-Length'] = data.length
       currentRequest.httpResponse.writeHead(res.code, res.headers)
       currentRequest.httpResponse.end(data)
     )
   else
-    res.headers['Content-Length'] = data.length
+    res.headers['Content-Length'] = res.data.length
     currentRequest.httpResponse.writeHead(res.code, res.headers)
-    currentRequest.httpResponse.end(data)
+    currentRequest.httpResponse.end(res.data)
 
 
-updateDevice = (currentRequest, actions, callback) ->
+endSession = (currentRequest) ->
-  if not actions?
+  db.redisClient.del("session_#{currentRequest.sessionId}", (err, res) ->
-    callback() if callback?
-    return
-
-  now = new Date(Date.now())
-  updates = {}
-  deletes = {}
-
-  if actions.parameterValues?
-    for p in actions.parameterValues
-      origValue = p[1]
-      v = normalize(p[0], origValue)
-
-      path = if common.endsWith(p[0], '.') then p[0] else "#{p[0]}."
-      if v == origValue
-        deletes["#{path}_orig"] = 1
-      else
-        updates["#{path}_orig"] = origValue
-      updates["#{path}_value"] = v
-      updates["#{path}_timestamp"] = now
-      if p[2]?
-        updates["#{path}_type"] = p[2]
-      else
-        deletes["#{path}_type"] = 1
-
-  if actions.deletedObjects?
-    for p in actions.deletedObjects
-      deletes[p] = 1
-
-  if actions.instanceName?
-    for i in actions.instanceName
-      updates["#{i[0]}._name"] = i[1]
-
-  if actions.parameterNames?
-    for p in actions.parameterNames
-      if OBJECT_REGEX.test(p[0])
-        path = p[0]
-        if INSTANCE_REGEX.test(p[0])
-          updates["#{path}_instance"] = true
-        else
-          updates["#{path}_object"] = true
-      else
-        path = "#{p[0]}."
-      updates["#{path}_writable"] = p[1] if p[1]?
-      updates["#{path}_timestamp"] = now
-
-      # Add parameter (sans any dot suffix) if has an alias
-      if (a = parameters.getAlias(pp = path.slice(0, -1)))?
-        newAliases = {} if not newAliases?
-        newAliases[pp] = a
-
-  if actions.customCommands?
-    for p in actions.customCommands
-      commandName = p[0]
-      updates["_customCommands.#{commandName}._value"] = p[1]
-      updates["_customCommands.#{commandName}._timestamp"] = now
-
-  if actions.set?
-    common.extend(updates, actions.set)
-
-  if Object.keys(updates).length > 0 or Object.keys(deletes).length > 0
-    db.devicesCollection.update({'_id' : currentRequest.deviceId}, {'$set' : updates, '$unset' : deletes}, {}, (err, count) ->
-      throw err if err
-      # Clear aliases cache if there's a new aliased parameter that has not been included in aliases cache
-      if newAliases?
-        db.getAliases((aliases) ->
-          for p, a of newAliases
-            if not aliases[a]? or p not in aliases[a]
-              db.redisClient.del('aliases', (err, res) ->
     throw err if err
+    res = soap.response(null)
+    writeResponse(currentRequest, res)
   )
-              break
-        )
-      callback?(err)
-    )
-  else
-    callback?()
 
 
 inform = (currentRequest, cwmpRequest) ->
@@ -146,42 +102,49 @@ inform = (currentRequest, cwmpRequest) ->
 
   now = new Date(Date.now())
 
-  util.log("#{currentRequest.deviceId}: Inform (#{cwmpRequest.methodRequest.event}); retry count #{cwmpRequest.methodRequest.retryCount}") if config.LOG_INFORMS
+  if config.get('LOG_INFORMS', currentRequest.session.deviceId)
+    util.log("#{currentRequest.session.deviceId}: Inform (#{cwmpRequest.methodRequest.event}); retry count #{cwmpRequest.methodRequest.retryCount}")
 
   parameterNames = (p[0] for p in cwmpRequest.methodRequest.parameterList)
 
   informHash = crypto.createHash('md5').update(JSON.stringify(parameterNames)).update(JSON.stringify(config.CUSTOM_COMMANDS)).digest('hex')
 
-  actions = {parameterValues : cwmpRequest.methodRequest.parameterList, set : {}}
+  deviceUpdates = {parameterValues : cwmpRequest.methodRequest.parameterList, informEvents : cwmpRequest.methodRequest.event}
-  actions.set._lastInform = now
-  actions.set._lastBoot = now if '1 BOOT' in cwmpRequest.methodRequest.event
 
+  # Delete all pending tasks on bootstrap (e.g. factory reset)
   if '0 BOOTSTRAP' in cwmpRequest.methodRequest.event
-    actions.set._lastBootstrap = lastBootstrap = now
+    lastBootstrap = now
-    # ignore all pending tasks
+    db.tasksCollection.remove({device : currentRequest.session.deviceId}, (err, removed) ->
-    db.tasksCollection.remove({device : currentRequest.deviceId}, (err, removed) ->
       throw err if err
     )
 
-  db.redisClient.get("#{currentRequest.deviceId}_inform_hash", (err, oldInformHash) ->
+  db.redisClient.get("#{currentRequest.session.deviceId}_inform_hash", (err, oldInformHash) ->
     throw err if err
 
     updateAndRespond = () ->
-      updateDevice(currentRequest, actions, (err) ->
+      updateDevice.commitUpdates(currentRequest.session.deviceId, deviceUpdates, false, (err) ->
+        throw err if err
+        db.redisClient.setex("session_#{currentRequest.sessionId}", currentRequest.session.sessionTimeout, JSON.stringify(currentRequest.session), (err) ->
           throw err if err
           res = soap.response({
             id : cwmpRequest.id,
             methodResponse : {type : 'InformResponse'},
-          cwmpVersion : cwmpRequest.cwmpVersion,
+            cwmpVersion : cwmpRequest.cwmpVersion
-          cookies : {deviceId : currentRequest.deviceId}
           })
+
+          if !!cookiesPath = config.get('COOKIES_PATH', currentRequest.session.deviceId)
+            res.headers['Set-Cookie'] = "session=#{currentRequest.sessionId}; Path=#{cookiesPath}"
+          else
+            res.headers['Set-Cookie'] = "session=#{currentRequest.sessionId}"
+
           writeResponse(currentRequest, res)
         )
+      )
 
     if oldInformHash == informHash and not lastBootstrap?
       return updateAndRespond()
 
-    db.redisClient.setex("#{currentRequest.deviceId}_inform_hash", config.PRESETS_CACHE_DURATION, informHash, (err, res) ->
+    db.redisClient.setex("#{currentRequest.session.deviceId}_inform_hash", config.get('PRESETS_CACHE_DURATION', currentRequest.session.deviceId), informHash, (err, res) ->
       throw err if err
     )
 
@@ -200,7 +163,7 @@ inform = (currentRequest, cwmpRequest) ->
         path += p + '.'
         projection[path + '_timestamp'] = 1
 
-    db.devicesCollection.findOne({'_id' : currentRequest.deviceId}, projection, (err, device) ->
+    db.devicesCollection.findOne({'_id' : currentRequest.session.deviceId}, projection, (err, device) ->
       throw err if err
       lastBootstrap ?= device?._lastBootstrap
       _tasks = []
@@ -210,7 +173,7 @@ inform = (currentRequest, cwmpRequest) ->
         for k of reference
           continue if k[0] == '_'
           if not actual[k]?._timestamp? or actual[k]._timestamp < lastBootstrap
-            _tasks.push({device: currentRequest.deviceId, name : 'refreshObject', objectName : reference[k]._path})
+            _tasks.push({device: currentRequest.session.deviceId, name : 'refreshObject', objectName : reference[k]._path})
           else
             traverse(reference[k], actual[k])
 
@@ -221,18 +184,18 @@ inform = (currentRequest, cwmpRequest) ->
         deviceCustomCommands = {}
         deviceCustomCommands[k] = v._timestamp for k,v of device._customCommands
 
-        for cmd in customCommands.getDeviceCustomCommandNames(currentRequest.deviceId)
+        for cmd in customCommands.getDeviceCustomCommandNames(currentRequest.session.deviceId)
           if not (deviceCustomCommands[cmd]?._timestamp < lastBootstrap)
-            _tasks.push({device: currentRequest.deviceId, name: 'customCommand', command: "#{cmd} init"})
+            _tasks.push({device: currentRequest.session.deviceId, name: 'customCommand', command: "#{cmd} init"})
           delete deviceCustomCommands[cmd]
 
         for cmd of deviceCustomCommands
-          actions.deletedObjects ?= []
+          deviceUpdates.deletedObjects ?= []
-          actions.deletedObjects.push("_customCommands.#{cmd}")
+          deviceUpdates.deletedObjects.push("_customCommands.#{cmd}")
 
         # clear presets hash if parameters are potentially modified
         if _tasks.length > 0
-          db.redisClient.del("#{currentRequest.deviceId}_presets_hash", (err) ->
+          db.redisClient.del("#{currentRequest.session.deviceId}_presets_hash", (err) ->
             throw err if err
           )
 
@@ -244,13 +207,13 @@ inform = (currentRequest, cwmpRequest) ->
         for k, v of cwmpRequest.methodRequest.deviceId
           deviceIdDetails["_#{k}"] = v
 
-        db.devicesCollection.insert({_id : currentRequest.deviceId, _registered : now, _deviceId : deviceIdDetails}, (err) ->
+        db.devicesCollection.insert({_id : currentRequest.session.deviceId, _registered : now, _deviceId : deviceIdDetails}, (err) ->
           throw err if err
-          util.log("#{currentRequest.deviceId}: New device registered")
+          util.log("#{currentRequest.session.deviceId}: New device registered")
-          _tasks.push({device: currentRequest.deviceId, name : 'refreshObject', objectName : ''})
+          _tasks.push({device: currentRequest.session.deviceId, name : 'refreshObject', objectName : ''})
 
-          for cmd in customCommands.getDeviceCustomCommandNames(currentRequest.deviceId)
+          for cmd in customCommands.getDeviceCustomCommandNames(currentRequest.session.deviceId)
-            _tasks.push({device: currentRequest.deviceId, name: 'customCommand', command: "#{cmd} init"})
+            _tasks.push({device: currentRequest.session.deviceId, name: 'customCommand', command: "#{cmd} init"})
 
           apiFunctions.insertTasks(_tasks, {}, () ->
             updateAndRespond()
@@ -264,9 +227,6 @@ runTask = (currentRequest, task, methodResponse) ->
   timeDiff = process.hrtime()
   tasks.task(task, methodResponse, (err, status, methodRequest, deviceUpdates) ->
     throw err if err
-    # TODO handle error
-    updateDevice(currentRequest, deviceUpdates, (err) ->
-      throw err if err
 
     timeDiff = process.hrtime(timeDiff)[0] + 1
     if timeDiff > 3 # in seconds
@@ -277,25 +237,33 @@ runTask = (currentRequest, task, methodResponse) ->
     switch status & ~tasks.STATUS_SAVE
       when tasks.STATUS_OK
         f = () ->
-            db.redisClient.setex(String(task._id), config.CACHE_DURATION, JSON.stringify(task), (err) ->
+          db.redisClient.setex(String(task._id), currentRequest.session.sessionTimeout, JSON.stringify(task), (err) ->
             throw err if err
             res = soap.response({
               id : task._id,
               methodRequest : methodRequest,
-                cwmpVersion : currentRequest.cwmpVersion
+              cwmpVersion : currentRequest.session.cwmpVersion
             })
             writeResponse(currentRequest, res)
+            updateSessionExpiry(currentRequest)
           )
 
         if save
+          updateDevice.commitUpdates(currentRequest.session.deviceId, deviceUpdates, true, (err) ->
+            throw err if err
             db.tasksCollection.update({_id : mongodb.ObjectID(String(task._id))}, {$set : {session : task.session}}, (err) ->
               throw err if err
               f()
             )
+          )
         else
+          updateDevice.queueUpdates(currentRequest.session.deviceId, deviceUpdates, (err) ->
             f()
+          )
       when tasks.STATUS_COMPLETED
-          util.log("#{currentRequest.deviceId}: Completed task #{task.name}(#{task._id})")
+        util.log("#{currentRequest.session.deviceId}: Completed task #{task.name}(#{task._id})")
+        updateDevice.commitUpdates(currentRequest.session.deviceId, deviceUpdates, true, (err) ->
+          throw err if err
           db.tasksCollection.remove({'_id' : mongodb.ObjectID(String(task._id))}, (err, removed) ->
             throw err if err
             db.redisClient.del(String(task._id), (err, res) ->
@@ -303,9 +271,10 @@ runTask = (currentRequest, task, methodResponse) ->
               nextTask(currentRequest)
             )
           )
+        )
       when tasks.STATUS_FAULT
-          retryAfter = config.RETRY_DELAY * Math.pow(2, task.retries ? 0)
+        retryAfter = config.get('RETRY_DELAY', currentRequest.session.deviceId) * Math.pow(2, task.retries ? 0)
-          util.log("#{currentRequest.deviceId}: Fault response for task #{task._id}. Retrying after #{retryAfter} seconds.")
+        util.log("#{currentRequest.session.deviceId}: Fault response for task #{task._id}. Retrying after #{retryAfter} seconds.")
         taskUpdate = {fault : task.fault, timestamp : new Date(Date.now() + retryAfter * 1000)}
         if save
           taskUpdate.session = task.session
@@ -317,7 +286,6 @@ runTask = (currentRequest, task, methodResponse) ->
       else
         throw new Error('Unknown task status')
   )
-  )
 
 
 isTaskExpired = (task) ->
@@ -325,41 +293,41 @@ isTaskExpired = (task) ->
 
 
 assertPresets = (currentRequest) ->
-  db.redisClient.mget("#{currentRequest.deviceId}_presets_hash", 'presets_hash', (err, res) ->
+  db.redisClient.mget("#{currentRequest.session.deviceId}_presets_hash", 'presets_hash', (err, res) ->
     throw err if err
     devicePresetsHash = res[0]
     presetsHash = res[1]
     if devicePresetsHash? and devicePresetsHash == presetsHash
-      # no discrepancy, return empty response
+      # no discrepancy, end the session
-      res = soap.response(null)
+      endSession(currentRequest)
-      writeResponse(currentRequest, res)
     else
+      PRESETS_CACHE_DURATION = config.get('PRESETS_CACHE_DURATION', currentRequest.session.deviceId)
       db.getPresetsObjectsAliases((allPresets, allObjects, allAliases) ->
         if not presetsHash?
           presetsHash = presets.calculatePresetsHash(allPresets, allObjects)
-          db.redisClient.setex('presets_hash', config.PRESETS_CACHE_DURATION, presetsHash, (err, res) ->
+          db.redisClient.setex('presets_hash', PRESETS_CACHE_DURATION, presetsHash, (err, res) ->
             throw err if err
           )
 
-        presets.getDevicePreset(currentRequest.deviceId, allPresets, allObjects, allAliases, (devicePreset) ->
+        presets.getDevicePreset(currentRequest.session.deviceId, allPresets, allObjects, allAliases, (devicePreset) ->
-          presets.processDevicePreset(currentRequest.deviceId, devicePreset, (taskList, addTags, deleteTags, expiry) ->
+          presets.processDevicePreset(currentRequest.session.deviceId, devicePreset, (taskList, addTags, deleteTags, expiry) ->
-            db.redisClient.setex("#{currentRequest.deviceId}_presets_hash", Math.floor(Math.max(1, expiry - config.PRESETS_TIME_PADDING)), presetsHash, (err, res) ->
+            db.redisClient.setex("#{currentRequest.session.deviceId}_presets_hash", Math.floor(Math.max(1, expiry - config.get('PRESETS_TIME_PADDING'))), presetsHash, (err, res) ->
               throw err if err
             )
 
             if addTags.length + deleteTags.length + taskList.length
-              util.log("#{currentRequest.deviceId}: Presets discrepancy found")
+              util.log("#{currentRequest.session.deviceId}: Presets discrepancy found")
 
             if addTags.length + deleteTags.length
-              util.log("#{currentRequest.deviceId}: Updating tags +(#{addTags}) -(#{deleteTags})")
+              util.log("#{currentRequest.session.deviceId}: Updating tags +(#{addTags}) -(#{deleteTags})")
 
             if deleteTags.length
-              db.devicesCollection.update({'_id' : currentRequest.deviceId}, {'$pull' : {'_tags' : {'$in' : deleteTags}}}, {}, (err, count) ->
+              db.devicesCollection.update({'_id' : currentRequest.session.deviceId}, {'$pull' : {'_tags' : {'$in' : deleteTags}}}, {}, (err, count) ->
                 throw err if err
               )
 
             if addTags.length
-              db.devicesCollection.update({'_id' : currentRequest.deviceId}, {'$addToSet' : {'_tags' : {'$each' : addTags}}}, {}, (err, count) ->
+              db.devicesCollection.update({'_id' : currentRequest.session.deviceId}, {'$addToSet' : {'_tags' : {'$each' : addTags}}}, {}, (err, count) ->
                 throw err if err
               )
 
@@ -368,12 +336,11 @@ assertPresets = (currentRequest) ->
               apiFunctions.insertTasks(taskList, allAliases, (err, taskList) ->
                 throw err if err
                 task = taskList[0]
-                util.log("#{currentRequest.deviceId}: Started task #{task.name}(#{task._id})")
+                util.log("#{currentRequest.session.deviceId}: Started task #{task.name}(#{task._id})")
                 runTask(currentRequest, task, {})
               )
             else
-              res = soap.response(null)
+              endSession(currentRequest)
-              writeResponse(currentRequest, res)
           )
         )
       )
@@ -382,7 +349,7 @@ assertPresets = (currentRequest) ->
 
 nextTask = (currentRequest) ->
   now = new Date()
-  cur = db.tasksCollection.find({'device' : currentRequest.deviceId, timestamp : {$lte : now}}).sort(['timestamp']).limit(1)
+  cur = db.tasksCollection.find({'device' : currentRequest.session.deviceId, timestamp : {$lte : now}}).sort(['timestamp']).limit(1)
   cur.nextObject((err, task) ->
     throw err if err
 
@@ -390,15 +357,43 @@ nextTask = (currentRequest) ->
       # no more tasks, check presets discrepancy
       assertPresets(currentRequest)
     else if isTaskExpired(task)
-      util.log("#{currentRequest.deviceId}: Task is expired #{task.name}(#{task._id})")
+      util.log("#{currentRequest.session.deviceId}: Task is expired #{task.name}(#{task._id})")
       db.tasksCollection.remove({'_id' : mongodb.ObjectID(String(task._id))}, {safe: true}, (err, removed) ->
         throw err if err
         nextTask(currentRequest)
       )
     else
-      util.log("#{currentRequest.deviceId}: Started task #{task.name}(#{task._id})")
+      util.log("#{currentRequest.session.deviceId}: Started task #{task.name}(#{task._id})")
+      updateDevice.clearUpdatesQueue(currentRequest.session.deviceId, (err) ->
+        throw err if err
         runTask(currentRequest, task, {})
       )
+  )
+
+
+getSession = (httpRequest, callback) ->
+  # Separation by comma is important as some devices don't comform to standard
+  COOKIE_REGEX = /\s*([a-zA-Z0-9\-_]+?)\s*=\s*"?([a-zA-Z0-9\-_]*?)"?\s*(,|;|$)/g
+  while match = COOKIE_REGEX.exec(httpRequest.headers.cookie)
+    sessionId = match[2] if match[1] == 'session'
+
+  return callback() if not sessionId?
+
+  db.redisClient.get("session_#{sessionId}", (err, res) ->
+    throw err if err
+    return callback(sessionId, JSON.parse(res))
+  )
+
+
+updateSessionExpiry = (currentRequest) ->
+  db.redisClient.expire("session_#{currentRequest.sessionId}", currentRequest.session.sessionTimeout, (err, res) ->
+    throw err if err
+    if not res
+      # Resave session in the very rare case when session cache expires while processing request
+      db.redisClient.setex("session_#{currentRequest.sessionId}", currentRequest.session.sessionTimeout, JSON.stringify(currentRequest.session), (err) ->
+        throw err if err
+      )
+  )
 
 
 listener = (httpRequest, httpResponse) ->
@@ -422,14 +417,13 @@ listener = (httpRequest, httpResponse) ->
 
   chunks = []
   bytes = 0
-  cookies = {}
 
   stream.on('data', (chunk) ->
     chunks.push(chunk)
     bytes += chunk.length
   )
 
-  httpRequest.getBody = (encoding) ->
+  httpRequest.getBody = () ->
     # Write all chunks into a Buffer
     body = new Buffer(bytes)
     offset = 0
@@ -437,27 +431,38 @@ listener = (httpRequest, httpResponse) ->
       chunk.copy(body, offset, 0, chunk.length)
       offset += chunk.length
     )
-
+    return body
-    #Return encoded (default to UTF8) string
-    return body.toString(encoding || 'utf8', 0, body.byteLength)
 
   stream.on('end', () ->
-    cwmpRequest = soap.request(httpRequest)
+    getSession(httpRequest, (sessionId, session) ->
+      cwmpRequest = soap.request(httpRequest, session?.cwmpVersion)
+      if not session?
+        if cwmpRequest.methodRequest?.type isnt 'Inform'
+          httpResponse.writeHead(400)
+          httpResponse.end('Session is expired')
+          return
 
-    currentRequest = {
+        deviceId = common.generateDeviceId(cwmpRequest.methodRequest.deviceId)
-      httpRequest : httpRequest
+        sessionId = crypto.randomBytes(8).toString('hex')
-      httpResponse : httpResponse
+
-      cwmpVersion : cwmpRequest.cwmpVersion
+        session = {
+          deviceId : deviceId,
+          cwmpVersion : cwmpRequest.cwmpVersion,
+          sessionTimeout : cwmpRequest.sessionTimeout ? config.get('SESSION_TIMEOUT', deviceId)
         }
 
-    if cwmpRequest.methodRequest?.deviceId?
+        httpRequest.connection.setTimeout(session.sessionTimeout * 1000)
-      currentRequest.deviceId = common.getDeviceId(cwmpRequest.methodRequest.deviceId)
-    else
-      currentRequest.deviceId = cwmpRequest.cookies.deviceId
 
-    if config.DEBUG_DEVICES[currentRequest.deviceId]
+      currentRequest = {
+        httpRequest : httpRequest,
+        httpResponse : httpResponse,
+        sessionId : sessionId,
+        session : session
+      }
+
+      if config.get('DEBUG', currentRequest.session.deviceId)
         dump = "# REQUEST #{new Date(Date.now())}\n" + JSON.stringify(httpRequest.headers) + "\n#{httpRequest.getBody()}\n\n"
-      require('fs').appendFile("../debug/#{currentRequest.deviceId}.dump", dump, (err) ->
+        require('fs').appendFile("./debug/#{currentRequest.session.deviceId}.dump", dump, (err) ->
           throw err if err
         )
 
@@ -465,60 +470,53 @@ listener = (httpRequest, httpResponse) ->
         if cwmpRequest.methodRequest.type is 'Inform'
           inform(currentRequest, cwmpRequest)
         else if cwmpRequest.methodRequest.type is 'GetRPCMethods'
-        util.log("#{currentRequest.deviceId}: GetRPCMethods")
+          util.log("#{currentRequest.session.deviceId}: GetRPCMethods")
           res = soap.response({
             id : cwmpRequest.id,
             methodResponse : {type : 'GetRPCMethodsResponse', methodList : ['Inform', 'GetRPCMethods', 'TransferComplete', 'RequestDownload']},
-          cwmpVersion : currentRequest.cwmpVersion,
+            cwmpVersion : currentRequest.session.cwmpVersion
-          cookies : cookies
           })
           writeResponse(currentRequest, res)
+          updateSessionExpiry(currentRequest)
         else if cwmpRequest.methodRequest.type is 'TransferComplete'
           # do nothing
-        util.log("#{currentRequest.deviceId}: Transfer complete")
+          util.log("#{currentRequest.session.deviceId}: Transfer complete")
           res = soap.response({
             id : cwmpRequest.id,
             methodResponse : {type : 'TransferCompleteResponse'},
-          cwmpVersion : currentRequest.cwmpVersion,
+            cwmpVersion : currentRequest.session.cwmpVersion
-          cookies : cookies
           })
           writeResponse(currentRequest, res)
+          updateSessionExpiry(currentRequest)
         else if cwmpRequest.methodRequest.type is 'RequestDownload'
           requestDownloadResponse = () ->
             res = soap.response({
               id : cwmpRequest.id,
               methodResponse : {type : 'RequestDownloadResponse'},
-            cwmpVersion : currentRequest.cwmpVersion,
+              cwmpVersion : currentRequest.session.cwmpVersion
-            cookies : cookies
             })
             writeResponse(currentRequest, res)
+            updateSessionExpiry(currentRequest)
           fileType = cwmpRequest.methodRequest.fileType
-        util.log("#{currentRequest.deviceId}: RequestDownload (#{fileType})")
+          util.log("#{currentRequest.session.deviceId}: RequestDownload (#{fileType})")
           if fileType isnt '1 Firmware Upgrade Image'
             # Only supporting firmware upgrade for now
             return requestDownloadResponse()
 
           db.getPresetsObjectsAliases((allPresets, allObjects, allAliases) ->
-          presets.getDevicePreset(currentRequest.deviceId, allPresets, allObjects, allAliases, (devicePreset) ->
+            presets.getDevicePreset(currentRequest.session.deviceId, allPresets, allObjects, allAliases, (devicePreset) ->
               presetSoftwareVersion = devicePreset.softwareVersion?.preset
               currentSoftwareVersion = devicePreset.softwareVersion?.current._value
               if presetSoftwareVersion? and presetSoftwareVersion != currentSoftwareVersion
-              projection = {
+                deviceIdStruct = common.parseDeviceId(currentRequest.session.deviceId)
-                '_deviceId._Manufacturer' : 1,
+                db.filesCollection.findOne({'metadata.fileType' : '1 Firmware Upgrade Image', 'metadata.oui' : deviceIdStruct.oui, 'metadata.productClass' : deviceIdStruct.productClass, 'metadata.version' : presetSoftwareVersion}, {_id : 1}, (err, file) ->
-                '_deviceId._ProductClass' : 1,
-              }
-              db.devicesCollection.findOne({'_id' : currentRequest.deviceId}, projection, (err, device) ->
-                throw err if err
-                manufacturer = device._deviceId._Manufacturer
-                productClass = device._deviceId._ProductClass
-                db.filesCollection.findOne({'metadata.fileType' : '1 Firmware Upgrade Image', 'metadata.manufacturer' : manufacturer, 'metadata.productClass' : productClass, 'metadata.version' : presetSoftwareVersion}, {_id : 1}, (err, file) ->
                   throw err if err
                   if not file?
-                    util.error("#{currentRequest.deviceId}: Firmware image not found (#{presetSoftwareVersion})")
+                    util.error("#{currentRequest.session.deviceId}: Firmware image not found (#{presetSoftwareVersion})")
                     return requestDownloadResponse()
 
                   task = {
-                    device : currentRequest.deviceId,
+                    device : currentRequest.session.deviceId,
                     name : 'download',
                     file : file['_id']
                   }
@@ -527,7 +525,6 @@ listener = (httpRequest, httpResponse) ->
                     return requestDownloadResponse()
                   )
                 )
-              )
               else
                 return requestDownloadResponse()
             )
@@ -547,10 +544,8 @@ listener = (httpRequest, httpResponse) ->
       else if cwmpRequest.fault?
         taskId = cwmpRequest.id
         if not taskId
-        # Fault not related to a task. return empty response.
+          # Fault not related to a task. End the session.
-        res = soap.response(null)
+          return endSession(currentRequest)
-        writeResponse(currentRequest, res)
-        return
 
         db.getTask(taskId, (err, task) ->
           throw err if err
@@ -563,36 +558,7 @@ listener = (httpRequest, httpResponse) ->
         # cpe sent empty response. start sending acs requests
         nextTask(currentRequest)
     )
-
-cluster = require 'cluster'
-numCPUs = require('os').cpus().length
-
-if cluster.isMaster
-  db.redisClient.flushdb()
-  cluster.on('listening', (worker, address) ->
-    util.log("Worker #{worker.process.pid} listening to #{address.address}:#{address.port}")
   )
 
-  cluster.on('exit', (worker, code, signal) ->
-    util.log("Worker #{worker.process.pid} died (#{worker.process.exitCode})")
-    setTimeout(()->
-      cluster.fork()
-    , config.WORKER_RESPAWN_TIME)
-  )
 
-  for [1 .. numCPUs]
+exports.listener = listener
-    cluster.fork()
-else
-  options = {
-    key: fs.readFileSync('../config/httpscert.key'),
-    cert: fs.readFileSync('../config/httpscert.crt')
-  }
-
-  httpServer = http.createServer(listener)
-  httpsServer = https.createServer(options, listener)
-
-  # wait until DB connections are established
-  setTimeout(() ->
-    httpServer.listen(config.ACS_PORT, config.ACS_INTERFACE)
-    httpsServer.listen(config.ACS_HTTPS_PORT, config.ACS_HTTPS_INTERFACE)
-  , 1000)

lib/db.coffee

@@ -1,42 +1,116 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
 config = require './config'
 mongodb = require 'mongodb'
-redisClient = require('redis').createClient(config.REDIS_SOCKET)
+redis = require('redis')
 parameters = require './parameters'
 
+redisClient = null
 tasksCollection = null
 devicesCollection = null
 presetsCollection = null
 objectsCollection = null
 
-mongodb.MongoClient.connect("mongodb://#{config.MONGODB_SOCKET}/#{config.DATABASE_NAME}", config.MONGODB_OPTIONS, (err, db) ->
+
+# Only flush redis DB if aliases or custom commands config has changed
+flushRedis = (callback) ->
+  redisClient.get('config_hash', (err, res) ->
+    return callback(err) if err
+
+    hash = require('crypto')
+      .createHash('md5')
+      .update(JSON.stringify(config.PARAMETERS))
+      .update(JSON.stringify(config.CUSTOM_COMMANDS))
+      .digest('hex')
+
+    if res == hash
+      return callback()
+
+    redisClient.flushdb((err) ->
+      return callback(err) if err
+      redisClient.set('config_hash', hash, (err, res) ->
+        return callback(err)
+      )
+    )
+  )
+
+
+connect = (callback) ->
+  callbackCounter = 6
+  mongodb.MongoClient.connect(config.get('MONGODB_CONNECTION_URL'), {db:{w:1},server:{autoReconnect:true}}, (err, db) ->
+    return callback(err) if err
     exports.mongoDb = db
     db.collection('tasks', (err, collection) ->
-    throw new Error(err) if err?
       exports.tasksCollection = tasksCollection = collection
-    collection.ensureIndex({device: 1, timestamp: 1}, (err) ->
+      collection?.ensureIndex({device: 1, timestamp: 1}, (err) ->
       )
+
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
     )
 
     db.collection('devices', (err, collection) ->
-    throw new Error(err) if err?
       exports.devicesCollection  = devicesCollection = collection
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
     )
 
     db.collection('presets', (err, collection) ->
-    throw new Error(err) if err?
       exports.presetsCollection = presetsCollection = collection
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
     )
 
     db.collection('objects', (err, collection) ->
-    throw new Error(err) if err?
       exports.objectsCollection = objectsCollection = collection
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
     )
 
     db.collection('fs.files', (err, collection) ->
-    throw new Error(err) if err?
       exports.filesCollection = filesCollection = collection
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
+    )
+
+    exports.redisClient = redisClient = redis.createClient(config.get('REDIS_PORT'), config.get('REDIS_HOST'))
+    redisClient.select(config.get('REDIS_DB'), (err) ->
+      if err
+        callbackCounter = 0
+        return callback(err)
+
+      flushRedis((err) ->
+        if --callbackCounter == 0 or err
+          callbackCounter = 0
+          return callback(err)
       )
     )
+  )
+
+
+disconnect = () ->
+  exports.mongoDb.close()
+  redisClient.quit()
 
 
 getTask = (taskId, callback) ->
@@ -100,7 +174,7 @@ getCached = (name, valueCallback, valueExpiry, callback) ->
 
 
 getAliases = (callback) ->
-  getCached('aliases', parameters.compileAliases, config.PRESETS_CACHE_DURATION, (err, res) ->
+  getCached('aliases', parameters.compileAliases, config.get('PRESETS_CACHE_DURATION'), (err, res) ->
     throw err if err
     callback(res)
   )
@@ -120,7 +194,7 @@ getPresetsObjectsAliases = (callback) ->
         presetsCollection.find().toArray((err, res) ->
           callback(err, res)
         )
-      , config.PRESETS_CACHE_DURATION, (err, res) ->
+      , config.get('PRESETS_CACHE_DURATION'), (err, res) ->
         throw err if err
         presets = res
         callback(presets, objects, aliases) if objects and aliases
@@ -134,14 +208,14 @@ getPresetsObjectsAliases = (callback) ->
           objs[r._id] = r for r in res
           callback(null, objs)
         )
-      , config.PRESETS_CACHE_DURATION, (err, res) ->
+      , config.get('PRESETS_CACHE_DURATION'), (err, res) ->
         throw err if err
         objects = res
         callback(presets, objects, aliases) if presets and aliases
       )
 
     if not aliases
-      getCached('aliases', parameters.compileAliases, config.PRESETS_CACHE_DURATION, (err, res) ->
+      getCached('aliases', parameters.compileAliases, config.get('PRESETS_CACHE_DURATION'), (err, res) ->
         throw err if err
         aliases = res
         callback(presets, objects, aliases) if presets and objects
@@ -149,7 +223,8 @@ getPresetsObjectsAliases = (callback) ->
   )
 
 
-exports.redisClient = redisClient
 exports.getTask = getTask
 exports.getPresetsObjectsAliases = getPresetsObjectsAliases
 exports.getAliases = getAliases
+exports.connect = connect
+exports.disconnect = disconnect

lib/fs.coffee

@@ -1,39 +1,37 @@
+###
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
 config = require './config'
-util = require 'util'
-http = require 'http'
 url = require 'url'
 mongodb = require 'mongodb'
 querystring = require 'querystring'
-
+db = require './db'
-db = null
-mongodb.MongoClient.connect("mongodb://#{config.MONGODB_SOCKET}/#{config.DATABASE_NAME}", config.MONGODB_OPTIONS, (err, _db) ->
-  db = _db
-)
 
 
-cluster = require 'cluster'
+listener = (request, response) ->
-numCPUs = require('os').cpus().length
-
-if cluster.isMaster
-  cluster.on('listening', (worker, address) ->
-    util.log("Worker #{worker.process.pid} listening to #{address.address}:#{address.port}")
-  )
-
-  cluster.on('exit', (worker, code, signal) ->
-    util.log("Worker #{worker.process.pid} died (#{worker.process.exitCode})")
-    setTimeout(()->
-      cluster.fork()
-    , config.WORKER_RESPAWN_TIME)
-  )
-
-  for i in [1 .. numCPUs]
-    cluster.fork()
-else
-  server = http.createServer((request, response) ->
   urlParts = url.parse(request.url, true)
   if request.method == 'GET'
     filename = querystring.unescape(urlParts.pathname.substring(1))
-      gs = new mongodb.GridStore(db, filename, 'r', {})
+    gs = new mongodb.GridStore(db.mongoDb, filename, 'r', {})
     gs.open((err, gs) ->
       if err
         response.writeHead(404)
@@ -46,6 +44,6 @@ else
   else
     response.writeHead(405, {'Allow': 'GET'})
     response.end('405 Method Not Allowed')
-  )
 
-  server.listen config.FILES_PORT, config.FILES_INTERFACE
+
+exports.listener = listener

lib/mongo-query.coffee

@@ -1,3 +1,43 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This file incorporates work covered by the following copyright and
+# permission notice:
+#
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
 common = require './common'
 
 
@@ -93,6 +133,7 @@ optimizeProjection = (obj) ->
         delete obj[b]
         keys.splice(i--, 1)
     ++ i
+  return
 
 
 exports.test = test

lib/nbi.coffee

@@ -1,6 +1,44 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This file incorporates work covered by the following copyright and
+# permission notice:
+#
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
 config = require './config'
-util = require 'util'
-http = require 'http'
 url = require 'url'
 common = require './common'
 db = require './db'
@@ -16,31 +54,14 @@ TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-\_\%]+)(\/[a-zA-Z_]*)?$/
 TAGS_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/tags\/([a-zA-Z0-9\-\_\%]+)\/?$/
 PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-\_\%]+)\/?$/
 OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-\_\%]+)\/?$/
-FILES_REGEX = /^\/files\/([a-zA-Z0-9\-\_\%\ \.\/\(\)]+)\/?$/
+FILES_REGEX = /^\/files\/([a-zA-Z0-9\%\!\*\'\(\)\;\:\@\&\=\+\$\,\?\#\[\]\-\_\.\~]+)\/?$/
 PING_REGEX = /^\/ping\/([a-zA-Z0-9\-\_\.]+)\/?$/
 QUERY_REGEX = /^\/([a-zA-Z0-9_]+s)\/?$/
 DEVICE_PRESET_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/preset\/?$/
+DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/?$/
 
 
-cluster = require 'cluster'
+listener = (request, response) ->
-numCPUs = require('os').cpus().length
-
-if cluster.isMaster
-  cluster.on('listening', (worker, address) ->
-    util.log("Worker #{worker.process.pid} listening to #{address.address}:#{address.port}")
-  )
-
-  cluster.on('exit', (worker, code, signal) ->
-    util.log("Worker #{worker.process.pid} died (#{worker.process.exitCode})")
-    setTimeout(()->
-      cluster.fork()
-    , config.WORKER_RESPAWN_TIME)
-  )
-
-  for [1 .. numCPUs]
-    cluster.fork()
-else
-  server = http.createServer (request, response) ->
   chunks = []
   bytes = 0
 
@@ -48,7 +69,7 @@ else
     chunks.push(chunk)
     bytes += chunk.length
 
-    request.getBody = (encoding) ->
+  request.getBody = () ->
     # Write all chunks into a Buffer
     body = new Buffer(bytes)
     offset = 0
@@ -56,9 +77,7 @@ else
       chunk.copy(body, offset, 0, chunk.length)
       offset += chunk.length
     )
-
+    return body
-      # Return encoded (default to UTF8) string
-      return body.toString(encoding || 'utf8', 0, body.byteLength)
 
   request.addListener 'end', () ->
     body = request.getBody()
@@ -161,7 +180,7 @@ else
     else if DEVICE_TASKS_REGEX.test(urlParts.pathname)
       if request.method == 'POST'
         deviceId = querystring.unescape(DEVICE_TASKS_REGEX.exec(urlParts.pathname)[1])
-          if body
+        if body.length
           task = JSON.parse(body)
           task.device = deviceId
           db.getAliases((aliases) ->
@@ -177,21 +196,21 @@ else
               if urlParts.query.connection_request?
                 apiFunctions.connectionRequest(deviceId, (err) ->
                   if err
-                      response.writeHead(202)
+                    response.writeHead(202, {'Content-Type' : 'application/json'})
-                      response.end()
+                    response.end(JSON.stringify(task))
                   else
-                      apiFunctions.watchTask(task._id, config.DEVICE_ONLINE_THRESHOLD, (err) ->
+                    apiFunctions.watchTask(task._id, config.get('DEVICE_ONLINE_THRESHOLD', deviceId), (err) ->
                       if err
-                          response.writeHead(202)
+                        response.writeHead(202, {'Content-Type' : 'application/json'})
-                          response.end()
+                        response.end(JSON.stringify(task))
                         return
-                        response.writeHead(200)
+                      response.writeHead(200, {'Content-Type' : 'application/json'})
-                        response.end()
+                      response.end(JSON.stringify(task))
                     )
                 )
               else
-                  response.writeHead(202)
+                response.writeHead(202, {'Content-Type' : 'application/json'})
-                  response.end()
+                response.end(JSON.stringify(task))
             )
           )
         else if urlParts.query.connection_request?
@@ -225,7 +244,7 @@ else
     else if TASKS_REGEX.test(urlParts.pathname)
       r = TASKS_REGEX.exec(urlParts.pathname)
       taskId = mongodb.ObjectID(querystring.unescape(r[1]))
-        action = try querystring.unescape(r[2])
+      action = r[2]
       if not action? or action is '/'
         if request.method == 'DELETE'
           db.tasksCollection.remove({'_id' : taskId}, (err, removedCount) ->
@@ -257,22 +276,22 @@ else
       if request.method == 'PUT'
         metadata = {
           fileType : request.headers.filetype,
-            manufacturer : request.headers.manufacturer,
+          oui : request.headers.oui,
           productClass : request.headers.productclass,
           version : request.headers.version,
         }
 
         gs = new mongodb.GridStore(db.mongoDb, filename, 'w', {metadata : metadata})
         gs.open((err, gs) ->
-            gs.write(request.getBody('binary'), (err, res) ->
+          gs.write(body, (err, res) ->
             throw err if err
             gs.close((err) ->
               throw err if err
-              )
               response.writeHead(201)
               response.end()
             )
           )
+        )
       else if request.method == 'DELETE'
         mongodb.GridStore.unlink(db.mongoDb, filename, (err) ->
           response.writeHead(200)
@@ -291,6 +310,21 @@ else
         response.writeHead(200, {'Content-Type' : 'text/plain', 'Cache-Control' : 'no-cache'})
         response.end(stdout)
       )
+    else if DELETE_DEVICE_REGEX.test(urlParts.pathname)
+      if request.method isnt 'DELETE'
+        response.writeHead(405, {'Allow' : 'DELETE'})
+        response.end('405 Method Not Allowed')
+        return
+
+      deviceId = querystring.unescape(DELETE_DEVICE_REGEX.exec(urlParts.pathname)[1])
+      apiFunctions.deleteDevice(deviceId, (err) ->
+        if err
+          response.writeHead(500)
+          response.end(err.message)
+          return
+        response.writeHead(200)
+        response.end()
+      )
     else if QUERY_REGEX.test(urlParts.pathname)
       collectionName = QUERY_REGEX.exec(urlParts.pathname)[1]
       if request.method not in ['GET', 'HEAD']
@@ -305,10 +339,16 @@ else
 
       func = (aliases) ->
         if urlParts.query.query?
+          try
             q = JSON.parse(urlParts.query.query)
+          catch err
+            response.writeHead(400)
+            response.end(err.toString())
+            return
         else
           q = {}
         q = query.expand(q, aliases) if collectionName is 'devices'
+        q = query.substituteObjectId(q) if collectionName is 'tasks'
 
         if urlParts.query.projection?
           projection = {}
@@ -360,4 +400,5 @@ else
       response.writeHead 404
       response.end('404 Not Found')
 
-  server.listen config.API_PORT, config.API_INTERFACE
+
+exports.listener = listener

lib/normalize.coffee

@@ -1,27 +1,50 @@
+###
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
+# Credit to Netanel Rubin from Check Point Software Technologies for discovering
+# vulnerability caused by use of eval to generate regex from user input.
+
 common = require './common'
 parameters = require './parameters'
 
 
-stringToRegexp = (input) ->
+stringToRegexp = (input, flags) ->
   if (input.indexOf('*') == -1)
     return false
 
   output = input.replace(/[\[\]\\\^\$\.\|\?\+\(\)]/, "\\$&")
   if output[0] == '*'
-    prefix = '/'
     output = output.replace(/^\*+/g, '')
   else
-    prefix = '/^'
+    output = '^' + output
 
   if output[output.length - 1] == '*'
-    suffix = '/'
     output = output.replace(/\*+$/g, '')
   else
-    suffix = '$/'
+    output = output + '$'
 
   output = output.replace(/[\*]/, '.*')
 
-  return eval(prefix + output + suffix)
+  return new RegExp(output, flags)
 
 
 normalizers = {}
@@ -66,9 +89,9 @@ normalizers.date = (input, normType) ->
 
 normalizers.string = (input, normType) ->
   if normType is 'query'
-    if /^\/(.*?)\/(g?i?m?y?)$/.test(input)
+    if (m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input))
-      return [{'$regex' : eval(input)}, input]
+      return [{'$regex' : new RegExp(m[1], m[2])}, input]
-  input
+  return input
 
 
 colonizeMac = (input) ->
@@ -94,14 +117,14 @@ normalizers.mac = (input, normType) ->
 
   if input.indexOf('*') != -1
     input = (colonizeMac(a) for a in input.split('*')).join('*')
-    return stringToRegexp(input, false)
+    return stringToRegexp(input, 'i')
 
   input = colonizeMac(input)
 
   if input.length == 17
-    return input
+    return [input, input.toLowerCase()]
 
-  return {'$regex' : eval('/' + input + '/')}
+  return {'$regex' : new RegExp(input, 'i')}
 
 
 exports.normalize = (path, value, normType) ->

lib/parameters.coffee

@@ -1,3 +1,20 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
 config = require './config'
 db = require './db'
 
@@ -160,7 +177,7 @@ compileAliases = (callback) ->
   counter = 0
   splitIds(4, (batches) ->
     if not batches?
-      return callback(null, aliases, config.ALIASES_CACHE)
+      return callback(null, aliases)
 
     for batch in batches
       options = {
@@ -184,7 +201,7 @@ compileAliases = (callback) ->
             aliases[alias].push(id)
 
         if ++counter == batches.length
-          callback?(null, aliases, config.ALIASES_CACHE)
+          callback?(null, aliases)
       )
     return
   )

lib/presets.coffee

@@ -1,3 +1,43 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This file incorporates work covered by the following copyright and
+# permission notice:
+#
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
 config = require './config'
 common = require './common'
 db = require './db'
@@ -14,12 +54,21 @@ exports.calculatePresetsHash = (presets, objects) ->
 
 
 matchObject = (object, param) ->
-  return false if not object._keys? or object._keys.length == 0
+  flatObject = common.flattenObject(object)
-  for k in object._keys
+  if object._keys?.length
-    return false if not param[k]?
+    keys = object._keys
-    v = common.matchType(param[k]._value, object[k])
+  else
-    if param[k]._value != v
+    # Consider all parameters as keys if none are defined
+    keys = (k for k in Object.keys(flatObject) when k[0] != '_')
+
+  for k in keys
+    p = common.getParamValueFromPath(param, k)
+    return false if not p?
+    v = common.matchType(p._value, flatObject[k])
+
+    if p._value != common.matchType(p._value, flatObject[k])
       return false
+
   return true
 
 
@@ -28,7 +77,11 @@ exports.getDevicePreset = (deviceId, presets, objects, aliases, callback) ->
   # only fetch relevant params
   projection = {_id : 1}
   for p in presets
-    p.precondition = query.expand(p.precondition, aliases)
+    if common.typeOf(p.precondition) is common.STRING_TYPE
+      p.precondition = query.expand(JSON.parse(p.precondition), aliases)
+    else
+      # Accept an object for backward compatiblity
+      p.precondition = query.expand(p.precondition ? {}, aliases)
     mongoQuery.projection(p.precondition, projection)
 
     for c in p.configurations
@@ -40,6 +93,7 @@ exports.getDevicePreset = (deviceId, presets, objects, aliases, callback) ->
           if commandfile of deviceCustomCommands
             projection["_customCommands.#{c.command.split(' ', 1)}"] = 1
         when 'software_version'
+          projection['Device.DeviceInfo.SoftwareVersion'] = 1
           projection['InternetGatewayDevice.DeviceInfo.SoftwareVersion'] = 1
         when 'add_tag', 'delete_tag'
           projection['_tags'] = 1
@@ -101,15 +155,7 @@ exports.getDevicePreset = (deviceId, presets, objects, aliases, callback) ->
           devicePreset.objects[c.name][c.object].current ?= {}
           for k,p of param
             continue if k[0] == '_'
-            if p._name?
+            if matchObject(objects[c.object], p)
-              if p._name == c.object
-                devicePreset.objects[c.name][c.object].current[k] = p
-            else if matchObject(objects[c.object], p)
-              u = {}
-              u["#{c.name}.#{k}._name"] = c.object
-              db.devicesCollection.update({'_id' : deviceId}, {'$set' : u}, {}, (err, count) ->
-                throw err if err
-              )
               devicePreset.objects[c.name][c.object].current[k] = p
 
         when 'delete_object'
@@ -120,15 +166,7 @@ exports.getDevicePreset = (deviceId, presets, objects, aliases, callback) ->
           devicePreset.objects[c.name][c.object].current ?= {}
           for k,p of param
             continue if k[0] == '_'
-            if p._name?
+            if matchObject(objects[c.object], p)
-              if p._name == c.object
-                devicePreset.objects[c.name][c.object].current[k] = p
-            else if matchObject(objects[c.object], p)
-              u = {}
-              u["#{c.name}.#{k}._name"] = c.object
-              db.devicesCollection.update({'_id' : deviceId}, {'$set' : u}, {}, (err, count) ->
-                throw err if err
-              )
               devicePreset.objects[c.name][c.object].current[k] = p
 
         when 'custom_command_value'
@@ -160,6 +198,9 @@ exports.getDevicePreset = (deviceId, presets, objects, aliases, callback) ->
         when 'software_version'
           devicePreset.softwareVersion ?= {}
           devicePreset.softwareVersion.preset = c.software_version
+          if device?.Device? # TR-181 data model
+            devicePreset.softwareVersion.current = common.getParamValueFromPath(device, 'Device.DeviceInfo.SoftwareVersion')
+          else # TR-098 data model
             devicePreset.softwareVersion.current = common.getParamValueFromPath(device, 'InternetGatewayDevice.DeviceInfo.SoftwareVersion')
 
         else
@@ -170,8 +211,9 @@ exports.getDevicePreset = (deviceId, presets, objects, aliases, callback) ->
 
 
 exports.processDevicePreset = (deviceId, devicePreset, callback) ->
+  PRESETS_TIME_PADDING = config.get('PRESETS_TIME_PADDING')
   now = new Date()
-  expiry = config.PRESETS_CACHE_DURATION
+  expiry = config.get('PRESETS_CACHE_DURATION', deviceId)
   getParameterValues = []
   setParameterValues = []
   addTags = []
@@ -188,7 +230,7 @@ exports.processDevicePreset = (deviceId, devicePreset, callback) ->
       setParameterValues.push([parameterPath, presetValue, parameterDetails.current._type])
     else if parameterDetails.preset.expiry?
       diff = parameterDetails.preset.expiry - (now - (parameterDetails.current._timestamp ? 0)) / 1000
-      if diff <= config.PRESETS_TIME_PADDING
+      if diff <= PRESETS_TIME_PADDING
         if parameterDetails.current._value?
           getParameterValues.push(parameterPath)
         else
@@ -201,20 +243,21 @@ exports.processDevicePreset = (deviceId, devicePreset, callback) ->
     for objectName, objectDetails of object
       continue if not objectDetails.current?
       if objectDetails.preset?
+        flatObject = common.flattenObject(objectDetails.preset)
         if Object.keys(objectDetails.current).length > 0
           for i, obj of objectDetails.current
-            for paramName, paramValue of objectDetails.preset
+            for paramName, paramValue of flatObject
               continue if paramName[0] == '_'
-              if obj[paramName]
+              currentValue = common.getParamValueFromPath(obj, paramName)
-                currentValue = obj[paramName]._value
+              if currentValue?
-                presetValue = common.matchType(currentValue, objectDetails.preset[paramName])
+                presetValue = common.matchType(currentValue._value, flatObject[paramName])
-                if currentValue != presetValue
+                if currentValue._value != presetValue
-                  setParameterValues.push(["#{parameterPath}.#{i}.#{paramName}", presetValue, obj[paramName]._type])
+                  setParameterValues.push(["#{parameterPath}.#{i}.#{paramName}", presetValue, currentValue._type])
         else
           vals = []
-          for k,v of objectDetails.preset
+          for k,v of flatObject
             vals.push([k, v]) if k[0] != '_'
-          taskList.push({device : deviceId, name : 'addObject', objectName : parameterPath, parameterValues : vals, instanceName : objectName})
+          taskList.push({device : deviceId, name : 'addObject', objectName : parameterPath, parameterValues : vals})
       else if Object.keys(objectDetails.current).length > 0
         for i, obj of objectDetails.current
           taskList.push({device : deviceId, name : 'deleteObject', objectName : "#{parameterPath}.#{i}"})
@@ -236,7 +279,7 @@ exports.processDevicePreset = (deviceId, devicePreset, callback) ->
       taskList.push({device : deviceId, name : 'customCommand', command : commandDetails.preset.valueCommand})
     else if commandDetails.preset.expiry?
       diff = commandDetails.preset.expiry - (now - (commandDetails.current._timestamp ? 0)) / 1000
-      if diff <= config.PRESETS_TIME_PADDING
+      if diff <= PRESETS_TIME_PADDING
         taskList.push({device : deviceId, name : 'customCommand', command : commandDetails.preset.expiryCommand})
       else
         expiry = Math.min(expiry, diff)
@@ -271,6 +314,9 @@ accumulateConfigurations = (presets, objects) ->
           "object_#{c.name}_#{objectHash}"
         when 'software_version'
           'software_version'
+        when 'custom_command_age', 'custom_command_value'
+          [filename, commandName] = c.command.split(' ', 2)
+          "#{c.type}_#{filename}_#{commandName}"
         else
           "#{c.type}_#{c.name}"
 

lib/query.coffee

@@ -1,5 +1,28 @@
+###
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
 common = require './common'
 normalize = require './normalize'
+mongodb = require 'mongodb'
 
 
 expandValue = (param, value) ->
@@ -81,5 +104,30 @@ expand = (query, aliases) ->
 
   return new_query
 
+# Replace _id string values with ObjectID type
+substituteObjectId = (query) ->
+  for k, v of query
+    if k[0] == '$' # logical operator
+      for i in [0...v.length]
+        substituteObjectId(v[i])
+    else if k == '_id'
+      if common.typeOf(v) is common.STRING_TYPE
+        query[k] = mongodb.ObjectID(v) if v.length == 24
+      else if common.typeOf(v) is common.OBJECT_TYPE
+        for kk, vv of v
+          switch kk # operator
+            when '$in', '$nin'
+              for i in [0...vv.length]
+                vv[i] = mongodb.ObjectID(vv[i]) if vv[i].length == 24
+            when '$eq', '$gt', '$gte', '$lt', '$lte', '$ne'
+              v[kk] = mongodb.ObjectID(vv) if vv.length == 24
+            when '$exists', '$type' then
+              # ignore
+            else
+              throw new Error('Operator not supported')
+
+  return query
+
 
 exports.expand = expand
+exports.substituteObjectId = substituteObjectId

lib/server.coffee

@@ -0,0 +1,83 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
+domain = require 'domain'
+util = require 'util'
+cluster = require 'cluster'
+config = require './config'
+db = require './db'
+
+service = config.argv[0]
+if not service?
+  throw new Error('Missing argument cwmp, fs, or nbi')
+
+networkInterface = config.get("#{service.toUpperCase()}_INTERFACE")
+port = config.get("#{service.toUpperCase()}_PORT")
+useHttps = config.get("#{service.toUpperCase()}_SSL")
+
+serviceListener = require("./#{service}").listener
+
+server = null
+
+listener = (httpRequest, httpResponse) ->
+  d = domain.create()
+
+  d.on('error', (err) ->
+    util.error("#{new Date().toISOString()} - #{err.stack}\n")
+    try
+      killTimer = setTimeout(() ->
+        process.exit(1)
+      , 30000)
+
+      killTimer.unref()
+
+      cluster.worker?.disconnect()
+      server.close(() ->
+        db.disconnect()
+      )
+
+      httpResponse.writeHead(500, {'Connection' : 'close'})
+      httpResponse.end()
+    catch err2
+      util.error("#{new Date().toISOString()} - #{err2.stack}\n")
+  )
+
+  d.add(httpRequest)
+  d.add(httpResponse)
+
+  d.run(() ->
+    serviceListener(httpRequest, httpResponse)
+  )
+
+
+if useHttps
+  path = require 'path'
+  fs = require 'fs'
+  httpsKey = path.resolve(config.get('CONFIG_DIR'), "#{service}.key")
+  httpsCert = path.resolve(config.get('CONFIG_DIR'), "#{service}.crt")
+  options = {
+    key: fs.readFileSync(httpsKey),
+    cert: fs.readFileSync(httpsCert)
+  }
+  server = require('https').createServer(options, listener)
+else
+  server = require('http').createServer(listener)
+
+db.connect((err) ->
+  throw err if err
+  server.listen(port, networkInterface)
+)

lib/soap.coffee

@@ -1,3 +1,43 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This file incorporates work covered by the following copyright and
+# permission notice:
+#
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
 config = require './config'
 libxmljs = require 'libxmljs'
 
@@ -41,9 +81,15 @@ NAMESPACES = {
   }
 }
 
+# Generate Libxmljs options
+LIBXMLJS_OPTIONS = {}
+for k, v of config.allConfig
+  if k.lastIndexOf('XML_PARSE_', 0) == 0
+    LIBXMLJS_OPTIONS[k] = v
+
 
 # Workaround for devices that don't use correct namespace declarations or prefixes
-if config.IGNORE_XML_NAMESPACES
+if config.get('IGNORE_XML_NAMESPACES')
   libxmljs.Element.prototype.__find = libxmljs.Element.prototype.find
   libxmljs.Element.prototype.find = (xpath, namespaces) ->
     # Modify xpath queries to work regardless of element's namespace
@@ -52,25 +98,6 @@ if config.IGNORE_XML_NAMESPACES
     libxmljs.Element.prototype.__find.call(this, p, namespaces)
 
 
-# Separation by comma is important as some devices don't comform to standard
-COOKIE_REGEX = /\s*([a-zA-Z0-9\-_]+?)\s*=\s*"?([a-zA-Z0-9\-_]*?)"?\s*(,|;|$)/g
-
-cookieEncode = (value) ->
-  new Buffer(value).toString('base64').replace(/\/|\+|\=/g, (c) -> if c is '/' then '_' else if c is '+' then '-' else '')
-
-cookieDecode = (encodedValue) ->
-  new Buffer(encodedValue, 'base64').toString()
-
-cookiesToObj = (cookieLine) ->
-  cookies = {}
-  while match = COOKIE_REGEX.exec(cookieLine)
-    cookies[cookieDecode(match[1])] = cookieDecode(match[2])
-  return cookies
-
-cookiesToStr = (obj) ->
-  "#{cookieEncode(cn)}=#{cookieEncode(cv)}" for cn, cv of obj
-
-
 event = (xml) ->
   e.text().trim() for e in xml.find('EventStruct/EventCode')
 
@@ -205,7 +232,7 @@ acsInform = (xml) ->
     parameterList : parameterValueList(xml.get('ParameterList')),
     deviceId : traverseXml(xml.get('DeviceId')),
     event : event(xml.get('Event')),
-    retryCount : JSON.parse(xml.get('RetryCount').text())
+    retryCount : parseInt(xml.get('RetryCount').text())
   }
 
 
@@ -264,22 +291,16 @@ fault = (xml) ->
   return traverseXml(xml)
 
 
-exports.request = (httpRequest) ->
+exports.request = (httpRequest, cwmpVersion) ->
-  cwmpRequest = {cookies: {}}
+  cwmpRequest = {cwmpVersion : cwmpVersion}
-  cwmpRequest.cookies = cookiesToObj(httpRequest.headers.cookie) if httpRequest.headers.cookie
-  cwmpRequest.cwmpVersion = cwmpRequest.cookies.cwmpVersion
 
   data = httpRequest.getBody()
 
   if data.length > 0
-    try
+    xml = libxmljs.parseXml(data, LIBXMLJS_OPTIONS)
-      xml = libxmljs.parseXml data
-    catch err
-      # some devices send invalid utf8 characters
-      xml = libxmljs.parseXml httpRequest.getBody('binary')
 
     if not cwmpRequest.cwmpVersion?
-      # cwmpVersion doesn't exist in cookies, thus it's an inform request
+      # cwmpVersion not passed, thus it's an inform request
       methodElement = xml.get('/soap-env:Envelope/soap-env:Body/*', NAMESPACES['1.0'])
       switch methodElement.namespace().href()
         when 'urn:dslforum-org:cwmp-1-0'
@@ -303,7 +324,7 @@ exports.request = (httpRequest) ->
 
     cwmpRequest.id = try xml.get('/soap-env:Envelope/soap-env:Header/cwmp:ID', NAMESPACES[cwmpRequest.cwmpVersion]).text() catch then null
 
-    if methodElement?
+    if methodElement? and not (config.get('IGNORE_XML_NAMESPACES') and methodElement.name() is 'Fault')
       switch methodElement.name()
         when 'Inform'
           cwmpRequest.methodRequest = acsInform(methodElement)
@@ -360,7 +381,6 @@ createSoapEnv = (cwmpVersion) ->
 
 exports.response = (cwmpResponse) ->
   headers = {
-    'Content-Type' : 'text/xml; charset="utf-8"',
     'Server' : SERVER_NAME,
     'SOAPServer' : SERVER_NAME,
   }
@@ -372,8 +392,6 @@ exports.response = (cwmpResponse) ->
     body = env.node('soap-env:Body')
     switch cwmpResponse.methodResponse.type
       when 'InformResponse'
-        cwmpResponse.cookies ?= {}
-        cwmpResponse.cookies.cwmpVersion = cwmpResponse.cwmpVersion
         acsInformResponse(body, cwmpResponse.methodResponse)
       when 'GetRPCMethodsResponse'
         acsGetRPCMethodsResponse(body, cwmpResponse.methodResponse)
@@ -408,10 +426,8 @@ exports.response = (cwmpResponse) ->
       else
         throw Error("Unknown method request #{cwmpResponse.methodRequest.type}")
   
-  if cwmpResponse?.cookies? and Object.keys(cwmpResponse.cookies).length > 0
-    headers['Set-Cookie'] = cookiesToStr(cwmpResponse.cookies)
-
   if env?
-    return {code: 200, headers: headers, data: env.doc().toString()}
+    headers['Content-Type'] = 'text/xml; charset="utf-8"'
+    return {code: 200, headers: headers, data: new Buffer(env.doc().toString(false))}
   else
-    return {code: 204, headers: headers, data: ''}
+    return {code: 204, headers: headers, data: new Buffer(0)}

lib/tasks.coffee

@@ -1,9 +1,49 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This file incorporates work covered by the following copyright and
+# permission notice:
+#
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
 config = require './config'
 common = require './common'
 db = require './db'
 mongodb = require 'mongodb'
 customCommands = require './custom-commands'
-BATCH_SIZE = 16
+url = require 'url'
 
 exports.STATUS_OK = STATUS_OK = 1
 exports.STATUS_FAULT = STATUS_FAULT = 2
@@ -17,7 +57,7 @@ this.refreshObject = (task, methodResponse, callback) ->
   if not task.session.subtask?
     path = task.objectName
     path += '.' if path != '' and not common.endsWith(path, '.')
-    task.session.subtask = {device : task.device, name : 'getParameterNames', parameterPath : path, nextLevel : false}
+    task.session.subtask = {device : task.device, name : 'getParameterNames', parameterPath : path}
 
   if methodResponse.faultcode?
     task.fault = methodResponse
@@ -26,12 +66,16 @@ this.refreshObject = (task, methodResponse, callback) ->
 
   if task.session.subtask.name is 'getParameterNames'
     this.getParameterNames(task.session.subtask, methodResponse, (err, status, methodRequest, deviceUpdates) =>
-      if status & STATUS_COMPLETED
+      task.session.parameterNames ?= []
-        parameterNames = []
+      if deviceUpdates?.parameterNames?
         for p in deviceUpdates.parameterNames
           if not common.endsWith(p[0], '.')
-            parameterNames.push(p[0])
+            task.session.parameterNames.push(p[0])
-        task.session.subtask = {name : 'getParameterValues', parameterNames : parameterNames}
+
+      if status & STATUS_COMPLETED
+        task.session.subtask = {device : task.device, name : 'getParameterValues', parameterNames : task.session.parameterNames}
+        delete task.session.parameterNames
+
         this.getParameterValues(task.session.subtask, {}, (err, status, methodRequest) ->
           # ignore deviceUpdates returned by firt call to getParameterValues
           callback(err, status, methodRequest, deviceUpdates)
@@ -39,7 +83,7 @@ this.refreshObject = (task, methodResponse, callback) ->
       else if status & STATUS_OK
         callback(err, STATUS_OK, methodRequest, deviceUpdates)
       else
-        throw Error('Unexpected subtask status')
+        throw new Error('Unexpected subtask status')
     )
   else if task.session.subtask.name is 'getParameterValues'
     if methodResponse.faultcode?
@@ -53,88 +97,55 @@ this.refreshObject = (task, methodResponse, callback) ->
     throw Error('Unexpected subtask name')
 
 
-findMissingParameters = (device, parameterList, prefix) ->
-  missingParameters = []
-  paths = {}
-  for param in parameterList
-    p = param[0]
-    paths[p] = true
-    i = p.indexOf('.')
-    while i != -1
-      paths[p[0...i]] = true
-      i = p.indexOf('.', i + 1)
-
-  recursive = (obj, prefix) ->
-    for k,v of obj
-      continue if k[0] == '_'
-      p = prefix + k
-
-      if paths[p]
-        recursive(v, "#{p}.")
-      else
-        missingParameters.push(p)
-
-  recursive(device, prefix)
-  return missingParameters
-
-
 this.getParameterNames = (task, methodResponse, callback) ->
   if methodResponse.faultcode?
     task.fault = methodResponse
     callback(null, STATUS_FAULT)
     return
 
+  GET_PARAMETER_NAMES_DEPTH_THRESHOLD = config.get('GET_PARAMETER_NAMES_DEPTH_THRESHOLD', task.device)
+
+  getParameterDepth = (param) ->
+    return 0 if !param
+    return (param[...-1] + '.').split('.').length - 1
+
   if methodResponse.type is 'GetParameterNamesResponse'
-    # Make sure that for each parameters, all its parents are included
+    path = task.session.queue.pop()
-    found = {}
+
+    # If parameter depth higher than the threshold, nextLevel was set to false
+    if not task.nextLevel? and getParameterDepth(path) < GET_PARAMETER_NAMES_DEPTH_THRESHOLD
       for p in methodResponse.parameterList
+        task.session.queue.push(p[0]) if p[0][-1..] == '.'
+
+    deviceUpdates = {parameterNames : methodResponse.parameterList}
+
+    # Make sure that for each parameter, all its parents are explicitly included
+    found = {}
+    found[path] = 0 if !!path
+    for p in deviceUpdates.parameterNames
       param = p[0]
       i = param.length
-      while (i = param.lastIndexOf('.', i-1)) > task.parameterPath.length
+      while (i = param.lastIndexOf('.', i-1)) > path.length
         pp = param.slice(0, i + 1)
         break if found[pp]?
         found[pp] = 0
       found[p[0]] = 1
 
     for k, v of found
-      methodResponse.parameterList.push([k]) if v == 0
+      deviceUpdates.parameterNames.push([k]) if v == 0
-
-    deviceUpdates = {parameterNames : methodResponse.parameterList}
-    path = if common.endsWith(task.parameterPath, '.') then task.parameterPath.slice(0, -1) else task.parameterPath
-    projection = {}
-    projection[path] = 1 if !!task.parameterPath
-
-    # delete nonexisting params
-    db.devicesCollection.findOne({_id : task.device}, projection, (err, device) ->
-      if device
-        root = device
-        rootPath = ''
-        if !!task.parameterPath
-          ps = path.split('.')
-          for p in ps
-            rootPath += "#{p}."
-            root = root[p]
-            break if not root?
-
-        if root
-          deviceUpdates.deletedObjects = findMissingParameters(root, methodResponse.parameterList, rootPath)
   else
-          # avoid adding and deleting the same param
+    task.session = {queue : [task.parameterPath]}
-          if rootPath isnt task.parameterPath
-            deviceUpdates.deletedObjects = [rootPath.slice(0, -1)]
 
-      # some devices don't return the root object as described in the standard. add manually to update timestamp
+  if task.session.queue.length > 0
-      deviceUpdates.parameterNames.push([task.parameterPath]) if !!task.parameterPath
+    path = task.session.queue[-1..][0]
-
-      callback(null, STATUS_COMPLETED, null, deviceUpdates)
-    )
-  else
     methodRequest = {
       type : 'GetParameterNames',
-      parameterPath : task.parameterPath,
+      parameterPath : path,
-      nextLevel : if task.nextLevel? then task.nextLevel else false
+      nextLevel : task.nextLevel ? getParameterDepth(path) < GET_PARAMETER_NAMES_DEPTH_THRESHOLD
     }
-    callback(null, STATUS_OK, methodRequest)
+    return callback(null, STATUS_OK, methodRequest, deviceUpdates)
+  else
+    return callback(null, STATUS_COMPLETED, null, deviceUpdates)
 
 
 this.getParameterValues = (task, methodResponse, callback) ->
@@ -150,7 +161,7 @@ this.getParameterValues = (task, methodResponse, callback) ->
   else if methodResponse.parameterList?
     task.session.currentIndex = task.session.nextIndex
 
-  task.session.nextIndex = Math.min(task.session.currentIndex + BATCH_SIZE, task.parameterNames.length)
+  task.session.nextIndex = Math.min(task.session.currentIndex + config.get('TASK_PARAMETERS_BATCH_SIZE', task.device), task.parameterNames.length)
   names = task.parameterNames.slice(task.session.currentIndex, task.session.nextIndex)
 
   if methodResponse.type is 'GetParameterValuesResponse'
@@ -180,7 +191,7 @@ this.setParameterValues = (task, methodResponse, callback) ->
     prevValues = task.parameterValues.slice(task.session.currentIndex, task.session.nextIndex)
     task.session.currentIndex = task.session.nextIndex
 
-  task.session.nextIndex = Math.min(task.session.currentIndex + BATCH_SIZE, task.parameterValues.length)
+  task.session.nextIndex = Math.min(task.session.currentIndex + config.get('TASK_PARAMETERS_BATCH_SIZE', task.device), task.parameterValues.length)
   values = task.parameterValues.slice(task.session.currentIndex, task.session.nextIndex)
 
   if prevValues?
@@ -206,10 +217,10 @@ this.addObject = (task, methodResponse, callback) ->
     if methodResponse.type is 'AddObjectResponse'
       status_save = STATUS_SAVE
       task.session.instanceNumber = methodResponse.instanceNumber
+      # TODO Don't specify nextLevel explicity. Also consider using refreshObject instead.
       task.session.subtask = {device : task.device, name : 'getParameterNames', parameterPath : "#{task.objectName}.#{task.session.instanceNumber}.", nextLevel : false}
       task.session.appliedParameterValues = []
       task.session.parameterNames = []
-      allDeviceUpdates.instanceName = [["#{task.objectName}.#{task.session.instanceNumber}", task.instanceName]] if task.instanceName?
     else
       callback(null, STATUS_OK, {type : 'AddObject', objectName : "#{task.objectName}."})
       return
@@ -224,7 +235,7 @@ this.addObject = (task, methodResponse, callback) ->
               task.session.parameterNames.push(p[0]) if not common.endsWith(p[0], '.')
 
           if status & STATUS_COMPLETED
-            task.session.subtask = {name : 'getParameterValues', parameterNames : task.session.parameterNames}
+            task.session.subtask = {device : task.device, name : 'getParameterValues', parameterNames : task.session.parameterNames}
             subtask()
           else if status & STATUS_OK
             # Use STATUS_SAVE to avoid adding duplicate object in case of error
@@ -240,7 +251,7 @@ this.addObject = (task, methodResponse, callback) ->
             if deviceUpdates and deviceUpdates.parameterValues?
               for p1 in deviceUpdates.parameterValues
                 for p2 in task.parameterValues
-                  if common.endsWith(p1[0], ".#{p2[0]}")
+                  if p1[0] == "#{task.objectName}.#{task.session.instanceNumber}.#{p2[0]}"
                     t = if p2[2] then p2[2] else p1[2]
                     v = common.matchType(p1[1], p2[1])
                     # TODO only include if writable
@@ -251,7 +262,7 @@ this.addObject = (task, methodResponse, callback) ->
             return callback(null, STATUS_FAULT)
 
           if status & STATUS_COMPLETED and task.session.appliedParameterValues.length > 0
-            task.session.subtask = {name : 'setParameterValues', parameterValues : task.session.appliedParameterValues}
+            task.session.subtask = {device : task.device, name : 'setParameterValues', parameterValues : task.session.appliedParameterValues}
             subtask()
           else
             callback(err, status, methodRequest, allDeviceUpdates)
@@ -319,11 +330,18 @@ this.download = (task, methodResponse, callback) ->
         callback(err)
         return
 
+      l = {
+        protocol : if config.get('FS_SSL') then 'https' else 'http',
+        hostname : config.get('FS_IP'),
+        port : config.get('FS_PORT'),
+        pathname : encodeURIComponent(file.filename)
+      }
+
       methodRequest = {
         type : 'Download',
         fileType : file.metadata.fileType,
         fileSize : file.length,
-        url : "http://#{config.FILES_IP}:#{config.FILES_PORT}/#{encodeURIComponent(file.filename)}",
+        url : url.format(l),
         successUrl : task.successUrl,
         failureUrl : task.failureUrl
       }

lib/update-device.coffee

@@ -0,0 +1,217 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
+common = require './common'
+db = require './db'
+normalize = require('./normalize').normalize
+mongoQuery = require './mongo-query'
+parameters = require './parameters'
+
+OBJECT_REGEX = /\.$/
+INSTANCE_REGEX = /\.[\d]+\.$/
+
+
+queueUpdates = (deviceId, deviceUpdates, callback) ->
+  return callback() if not deviceUpdates? or Object.keys(deviceUpdates).length == 0
+  # TODO consider setting expiry
+  db.redisClient.rpush("#{deviceId}_updates", JSON.stringify(deviceUpdates), (err, res) ->
+    return callback(err)
+  )
+
+
+clearUpdatesQueue = (deviceId, callback) ->
+  db.redisClient.del("#{deviceId}_updates", (err, res) ->
+    return callback(err)
+  )
+
+
+commitUpdates = (deviceId, deviceUpdates, commitQueue, callback) ->
+  now = new Date(Date.now())
+  update = {'$set' : {}, '$unset' : {}}
+
+  # Used to find parameters that no longer exist in device
+  presentPaths = null
+  projection = null
+
+  # Keep track of alias changes to clear cache if needed
+  newAliases = null
+
+  processBatch = (updatesBatch, cb) ->
+    if updatesBatch.informEvents?
+      update['$set']['_lastInform'] = now
+      update['$set']['_lastBoot'] if '1 BOOT' in updatesBatch.informEvents
+      update['$set']['_lastBootstrap'] if '0 BOOTSTRAP' in updatesBatch.informEvents
+
+    if updatesBatch.parameterValues?
+      for p in updatesBatch.parameterValues
+        origValue = p[1]
+        v = normalize(p[0], origValue)
+
+        path = if common.endsWith(p[0], '.') then p[0] else "#{p[0]}."
+        if v == origValue
+          update['$unset']["#{path}_orig"] = 1
+        else
+          update['$set']["#{path}_orig"] = origValue
+        update['$set']["#{path}_value"] = v
+        update['$set']["#{path}_timestamp"] = now
+        if p[2]?
+          update['$set']["#{path}_type"] = p[2]
+
+    if updatesBatch.deletedObjects?
+      for p in updatesBatch.deletedObjects
+        update['$unset'][p] = 1
+
+    if updatesBatch.parameterNames?
+      presentPaths ?= {}
+      projection ?= {}
+
+      for p in updatesBatch.parameterNames
+        if OBJECT_REGEX.test(p[0])
+          path = p[0].slice(0, -1)
+          if INSTANCE_REGEX.test(p[0])
+            update['$set']["#{path}._instance"] = true
+          else
+            update['$set']["#{path}._object"] = true
+        else
+          path = p[0]
+        update['$set']["#{path}._writable"] = p[1] if p[1]?
+        update['$set']["#{path}._timestamp"] = now
+
+        # Add parameter (sans any dot suffix) if has an alias
+        if (a = parameters.getAlias(path))?
+          newAliases ?= {}
+          newAliases[path] = a
+
+        projection[path] = 1
+
+        # Store this param path and all its parents for later look up
+        presentPaths[path] = true
+        i = p[0].indexOf('.')
+        while i != -1
+          presentPaths[p[0][0...i]] = true
+          i = p[0].indexOf('.', i + 1)
+
+    if updatesBatch.customCommands?
+      for p in updatesBatch.customCommands
+        commandName = p[0]
+        update['$set']["_customCommands.#{commandName}._value"] = p[1]
+        update['$set']["_customCommands.#{commandName}._timestamp"] = now
+
+  processBatches = (cb) ->
+    if not commitQueue
+      processBatch(deviceUpdates) if deviceUpdates?
+      return cb()
+
+    db.redisClient.lpop("#{deviceId}_updates", (err, res) ->
+      callback(err) if err
+
+      if not res?
+        processBatch(deviceUpdates) if deviceUpdates?
+        return cb()
+
+      processBatch(JSON.parse(res))
+      return processBatches(cb)
+    )
+
+  commit = () ->
+    for k of update
+      delete update[k] if Object.keys(update[k]).length == 0
+
+    if Object.keys(update).length
+      db.devicesCollection.update({'_id' : deviceId}, update, {}, (err, count) ->
+        return callback(err) if err
+
+        # Clear aliases cache if there's a new aliased parameter that has not been included in aliases cache
+        if newAliases?
+          newAliasFound = false
+          db.getAliases((aliases) ->
+            for p, a of newAliases
+              if not aliases[a]? or p not in aliases[a]
+                newAliasFound = true
+                break
+
+            return callback() if not newAliasFound
+            db.redisClient.del('aliases', (err, res) ->
+              return callback(err)
+            )
+          )
+        else
+          return callback()
+      )
+    else
+      return callback()
+
+  processBatches(() ->
+    if projection?
+      mongoQuery.optimizeProjection(projection)
+
+      # Fetch device from DB to discover and delete parameters that no longer exist
+      # While we're fetching the object, we might as well use the information
+      # to optimize our update query
+      db.devicesCollection.findOne({'_id' : deviceId}, projection, (err, device) ->
+        return callback(err) if err
+
+        recursive = (obj, prefix) ->
+          for k, v of obj
+            p = prefix + k
+
+            # Mark unset operations we want to keep
+            if update['$unset'][p]?
+              update['$unset'][p] = 2
+              continue
+
+            if k[0] == '_'
+              # Remove set operations for unmodified values for optimization
+              if update['$set'][p] == v
+                delete update['$set'][p]
+              continue
+
+            if presentPaths[p]
+              recursive(v, "#{p}.")
+            else
+              update['$unset'][p] = 2
+
+        # Scan the DB object and remove parameters that no longer exists in device
+        for path of projection
+          root = device
+          rootPath = ''
+          for p in path.split('.')
+            rootPath += "#{p}."
+            root = root[p]
+            break if not root?
+
+          if root
+            recursive(root, rootPath)
+          else
+            if not presentPaths[rootPath.slice(0, -1)]?
+              update['$unset'][rootPath.slice(0, -1)] = 2
+
+          # Remove unnecessary unset updates
+          for k, v of update['$unset']
+            if common.startsWith(k, path + '.') and v != 2
+              delete update['$unset'][k]
+
+        commit()
+      )
+    else
+      commit()
+  )
+
+
+exports.queueUpdates = queueUpdates
+exports.clearUpdatesQueue = clearUpdatesQueue
+exports.commitUpdates = commitUpdates

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "0.9.8",
+  "version": "1.0.4",
   "description": "A blazing fast TR-069 auto configuration server (ACS) built with Node.js, Redis, and MongoDB",
   "repository": {
     "type": "git",
@@ -20,13 +20,13 @@
     "bin": "bin"
   },
   "dependencies": {
-    "libxmljs": "~0.8.1",
+    "libxmljs": "0.11.0 - 0.14.x",
-    "redis": "~0.10.0",
+    "redis": "^0.12.1",
-    "hiredis": "~0.1.15",
+    "hiredis": "0.1.17 - 0.4.x",
-    "mongodb": "~1.3.23"
+    "mongodb": "^1.4.8"
   },
   "devDependencies": {
-    "coffee-script": "~1.7.1"
+    "coffee-script": "^1.7.1"
   },
   "engines": {
     "node": ">=0.10.0 <0.12"
@@ -34,7 +34,7 @@
   "scripts": {
     "prepublish": "npm run compile",
     "compile": "coffee -cb lib/*.coffee test/*.coffee tools/*.coffee",
-    "configure": "cp ./config/config-sample.json ./config/config.json; cp ./config/parameters-sample.json ./config/parameters.json; cp ./config/custom_commands-sample.json ./config/custom_commands.json; cp ./config/auth-sample.js ./config/auth.js; cp ./config/httpscert-sample.key ./config/httpscert.key; cp ./config/httpscert-sample.crt ./config/httpscert.crt",
+    "configure": "cp ./config/config-sample.json ./config/config.json; cp ./config/parameters-sample.json ./config/parameters.json; cp ./config/custom_commands-sample.json ./config/custom_commands.json; cp ./config/auth-sample.js ./config/auth.js",
     "install": "npm run configure",
     "test": "node test"
   }

test/methods.coffee

@@ -1,3 +1,20 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
 common = require '../lib/common'
 
 NAMESPACES = {

test/test.coffee

@@ -1,3 +1,20 @@
+###
+# Copyright 2013, 2014  Zaid Abdulla
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
 http = require 'http'
 libxmljs = require 'libxmljs'
 deviceTemplate = require './huawei_bm632w'