Queue device updates and commit them using fewer database queries for better performance. Partial updates caused by faulty tasks will no longer be committed in order to avoid messing the device model.
This commit is contained in:
Zaid Abdulla committed
1 parent
4b244f43e0
commit
db6eae2aee
3 files changed
+213
-142
No files matched your search
+156
-95
@@ -25,136 +25,197 @@ OBJECT_REGEX = /\.$/
|
||||
INSTANCE_REGEX = /\.[\d]+\.$/
|
||||
|
||||
|
||||
updateDevice = (deviceId, actions, callback) ->
|
||||
return callback?() if not actions?
|
||||
queueUpdates = (deviceId, deviceUpdates, callback) ->
|
||||
return callback() if not deviceUpdates? or Object.keys(deviceUpdates).length == 0
|
||||
# TODO consider setting expiry
|
||||
db.redisClient.rpush("#{deviceId}_updates", JSON.stringify(deviceUpdates), (err, res) ->
|
||||
return callback(err)
|
||||
)
|
||||
|
||||
|
||||
clearUpdatesQueue = (deviceId, callback) ->
|
||||
db.redisClient.del("#{deviceId}_updates", (err, res) ->
|
||||
return callback(err)
|
||||
)
|
||||
|
||||
|
||||
commitUpdates = (deviceId, deviceUpdates, commitQueue, callback) ->
|
||||
now = new Date(Date.now())
|
||||
update = {'$set' : {}, '$unset' : {}}
|
||||
|
||||
if actions.parameterValues?
|
||||
for p in actions.parameterValues
|
||||
origValue = p[1]
|
||||
v = normalize(p[0], origValue)
|
||||
# Used to find parameters that no longer exist in device
|
||||
presentPaths = null
|
||||
projection = null
|
||||
|
||||
path = if common.endsWith(p[0], '.') then p[0] else "#{p[0]}."
|
||||
if v == origValue
|
||||
update['$unset']["#{path}_orig"] = 1
|
||||
else
|
||||
update['$set']["#{path}_orig"] = origValue
|
||||
update['$set']["#{path}_value"] = v
|
||||
update['$set']["#{path}_timestamp"] = now
|
||||
if p[2]?
|
||||
update['$set']["#{path}_type"] = p[2]
|
||||
# Keep track of alias changes to clear cache if needed
|
||||
newAliases = null
|
||||
|
||||
if actions.deletedObjects?
|
||||
for p in actions.deletedObjects
|
||||
update['$unset'][p] = 1
|
||||
processBatch = (updatesBatch, cb) ->
|
||||
if updatesBatch.informEvents?
|
||||
update['$set']['_lastInform'] = now
|
||||
update['$set']['_lastBoot'] if '1 BOOT' in updatesBatch.informEvents
|
||||
update['$set']['_lastBootstrap'] if '0 BOOTSTRAP' in updatesBatch.informEvents
|
||||
|
||||
if actions.instanceName?
|
||||
for i in actions.instanceName
|
||||
update['$set']["#{i[0]}._name"] = i[1]
|
||||
if updatesBatch.parameterValues?
|
||||
for p in updatesBatch.parameterValues
|
||||
origValue = p[1]
|
||||
v = normalize(p[0], origValue)
|
||||
|
||||
if actions.parameterNames?
|
||||
projection = {} # fetch only necessary parameters
|
||||
presentPaths = {} # to be compared with to find no longer existing parameters
|
||||
|
||||
for p in actions.parameterNames
|
||||
if OBJECT_REGEX.test(p[0])
|
||||
path = p[0].slice(0, -1)
|
||||
if INSTANCE_REGEX.test(p[0])
|
||||
update['$set']["#{path}._instance"] = true
|
||||
path = if common.endsWith(p[0], '.') then p[0] else "#{p[0]}."
|
||||
if v == origValue
|
||||
update['$unset']["#{path}_orig"] = 1
|
||||
else
|
||||
update['$set']["#{path}._object"] = true
|
||||
else
|
||||
path = p[0]
|
||||
update['$set']["#{path}._writable"] = p[1] if p[1]?
|
||||
update['$set']["#{path}._timestamp"] = now
|
||||
update['$set']["#{path}_orig"] = origValue
|
||||
update['$set']["#{path}_value"] = v
|
||||
update['$set']["#{path}_timestamp"] = now
|
||||
if p[2]?
|
||||
update['$set']["#{path}_type"] = p[2]
|
||||
|
||||
# Add parameter (sans any dot suffix) if has an alias
|
||||
if (a = parameters.getAlias(path))?
|
||||
newAliases = {} if not newAliases?
|
||||
newAliases[path] = a
|
||||
if updatesBatch.deletedObjects?
|
||||
for p in updatesBatch.deletedObjects
|
||||
update['$unset'][p] = 1
|
||||
|
||||
projection[path] = 1
|
||||
if updatesBatch.instanceName?
|
||||
for i in updatesBatch.instanceName
|
||||
update['$set']["#{i[0]}._name"] = i[1]
|
||||
|
||||
# Store this param path and all its parents for later look up
|
||||
presentPaths[path] = true
|
||||
i = p[0].indexOf('.')
|
||||
while i != -1
|
||||
presentPaths[p[0][0...i]] = true
|
||||
i = p[0].indexOf('.', i + 1)
|
||||
if updatesBatch.parameterNames?
|
||||
presentPaths ?= {}
|
||||
projection ?= {}
|
||||
|
||||
mongoQuery.optimizeProjection(projection)
|
||||
for p in updatesBatch.parameterNames
|
||||
if OBJECT_REGEX.test(p[0])
|
||||
path = p[0].slice(0, -1)
|
||||
if INSTANCE_REGEX.test(p[0])
|
||||
update['$set']["#{path}._instance"] = true
|
||||
else
|
||||
update['$set']["#{path}._object"] = true
|
||||
else
|
||||
path = p[0]
|
||||
update['$set']["#{path}._writable"] = p[1] if p[1]?
|
||||
update['$set']["#{path}._timestamp"] = now
|
||||
|
||||
if actions.customCommands?
|
||||
for p in actions.customCommands
|
||||
commandName = p[0]
|
||||
update['$set']["_customCommands.#{commandName}._value"] = p[1]
|
||||
update['$set']["_customCommands.#{commandName}._timestamp"] = now
|
||||
# Add parameter (sans any dot suffix) if has an alias
|
||||
if (a = parameters.getAlias(path))?
|
||||
newAliases ?= {}
|
||||
newAliases[path] = a
|
||||
|
||||
if actions.set?
|
||||
common.extend(update['$set'], actions.set)
|
||||
projection[path] = 1
|
||||
|
||||
f = () ->
|
||||
# Store this param path and all its parents for later look up
|
||||
presentPaths[path] = true
|
||||
i = p[0].indexOf('.')
|
||||
while i != -1
|
||||
presentPaths[p[0][0...i]] = true
|
||||
i = p[0].indexOf('.', i + 1)
|
||||
|
||||
if updatesBatch.customCommands?
|
||||
for p in updatesBatch.customCommands
|
||||
commandName = p[0]
|
||||
update['$set']["_customCommands.#{commandName}._value"] = p[1]
|
||||
update['$set']["_customCommands.#{commandName}._timestamp"] = now
|
||||
|
||||
processBatches = (cb) ->
|
||||
if not commitQueue?
|
||||
processBatch(deviceUpdates) if deviceUpdates?
|
||||
return cb()
|
||||
|
||||
db.redisClient.lpop("#{deviceId}_updates", (err, res) ->
|
||||
callback(err) if err
|
||||
|
||||
if not res?
|
||||
processBatch(deviceUpdates) if deviceUpdates?
|
||||
return cb()
|
||||
|
||||
processBatch(JSON.parse(res))
|
||||
return processBatches(cb)
|
||||
)
|
||||
|
||||
commit = () ->
|
||||
for k of update
|
||||
delete update[k] if Object.keys(update[k]).length == 0
|
||||
|
||||
if Object.keys(update).length
|
||||
db.devicesCollection.update({'_id' : deviceId}, update, {}, (err, count) ->
|
||||
throw err if err
|
||||
return callback(err) if err
|
||||
|
||||
# Clear aliases cache if there's a new aliased parameter that has not been included in aliases cache
|
||||
if newAliases?
|
||||
newAliasFound = false
|
||||
db.getAliases((aliases) ->
|
||||
for p, a of newAliases
|
||||
if not aliases[a]? or p not in aliases[a]
|
||||
db.redisClient.del('aliases', (err, res) ->
|
||||
throw err if err
|
||||
)
|
||||
newAliasFound = true
|
||||
break
|
||||
|
||||
return callback() if not newAliasFound
|
||||
db.redisClient.del('aliases', (err, res) ->
|
||||
return callback(err)
|
||||
)
|
||||
)
|
||||
callback?(err)
|
||||
else
|
||||
return callback()
|
||||
)
|
||||
else
|
||||
callback?()
|
||||
return callback()
|
||||
|
||||
if projection?
|
||||
db.devicesCollection.findOne({'_id' : deviceId}, projection, (err, device) ->
|
||||
throw err if err
|
||||
processBatches(() ->
|
||||
if projection?
|
||||
mongoQuery.optimizeProjection(projection)
|
||||
|
||||
recursive = (obj, prefix) ->
|
||||
for k,v of obj
|
||||
p = prefix + k
|
||||
if k[0] == '_'
|
||||
# Remove set operations for unmodified values for optimization
|
||||
if update['$set'][p] == v
|
||||
delete update['$set'][p]
|
||||
continue
|
||||
# Fetch device from DB to discover and delete parameters that no longer exist
|
||||
# While we're fetching the object, we might as well use the information
|
||||
# to optimize our update query
|
||||
db.devicesCollection.findOne({'_id' : deviceId}, projection, (err, device) ->
|
||||
return callback(err) if err
|
||||
|
||||
# Remove unset operations for values that don't exist
|
||||
delete update['$unset']["#{p}._orig"] if not v._orig?
|
||||
recursive = (obj, prefix) ->
|
||||
for k, v of obj
|
||||
p = prefix + k
|
||||
|
||||
if presentPaths[p]
|
||||
recursive(v, "#{p}.")
|
||||
# Mark unset operations we want to keep
|
||||
if update['$unset'][p]?
|
||||
update['$unset'][p] = 2
|
||||
continue
|
||||
|
||||
if k[0] == '_'
|
||||
# Remove set operations for unmodified values for optimization
|
||||
if update['$set'][p] == v
|
||||
delete update['$set'][p]
|
||||
continue
|
||||
|
||||
if presentPaths[p]
|
||||
recursive(v, "#{p}.")
|
||||
else
|
||||
update['$unset'][p] = 1
|
||||
|
||||
# Scan the DB object and remove parameters that no longer exists in device
|
||||
for path of projection
|
||||
root = device
|
||||
rootPath = ''
|
||||
for p in path.split('.')
|
||||
rootPath += "#{p}."
|
||||
root = root[p]
|
||||
break if not root?
|
||||
|
||||
if root
|
||||
recursive(root, rootPath)
|
||||
else
|
||||
update['$unset'][p] = 1
|
||||
if not presentPaths[rootPath.slice(0, -1)]?
|
||||
update['$unset'][rootPath.slice(0, -1)] = 1
|
||||
|
||||
# Scan the DB object and remove parameters that no longer exists in device
|
||||
for k, v of projection
|
||||
root = device
|
||||
rootPath = ''
|
||||
for p in path.split('.')
|
||||
rootPath += "#{p}."
|
||||
root = root[p]
|
||||
break if not root?
|
||||
# Remove unnecessary unset updates
|
||||
for k, v of update['$unset']
|
||||
if common.startsWith(k, path + '.') and v != 2
|
||||
delete update['$unset'][k]
|
||||
|
||||
if root
|
||||
recursive(root, rootPath)
|
||||
else
|
||||
if not presentPaths[rootPath.slice(0, -1)]
|
||||
update['$unset'][rootPath.slice(0, -1)] = 1
|
||||
f()
|
||||
)
|
||||
else
|
||||
f()
|
||||
commit()
|
||||
)
|
||||
else
|
||||
commit()
|
||||
)
|
||||
|
||||
module.exports = updateDevice
|
||||
|
||||
exports.queueUpdates = queueUpdates
|
||||
exports.clearUpdatesQueue = clearUpdatesQueue
|
||||
exports.commitUpdates = commitUpdates
|
||||
Reference in new issue
Block a user