Improve paramter set and load routines

lib/config.coffee

@@ -49,6 +49,7 @@ options = {
   SESSION_TIMEOUT : {type : 'int', default : 30},
   GET_PARAMETER_NAMES_DEPTH_THRESHOLD : {type : 'int', default : 0},
   TASK_PARAMETERS_BATCH_SIZE : {type : 'int', default : 32},
+  MAX_DEPTH : {type: 'int', default : 16},
   COOKIES_PATH : {type : 'string'},
 
   # Libxml related configuration

lib/db.coffee

@@ -126,24 +126,24 @@ optimizeProjection = (obj) ->
 
 
 fetchDevice = (id, timestamp, patterns, callback) ->
+  MAX_DEPTH = config.get('MAX_DEPTH', id)
   res = []
   loaded = []
 
   # Build projection
   projection = {_id: 1}
-  for pattern in patterns
+  for [pattern, depth] in patterns
-    loaded.push([pattern, 1])
+    loaded.push([pattern, depth])
-    if pattern.length == 0
+    if depth & 1 and (pattern.length < 1 or pattern[0] == '*')
-      projection['_timestamp'] = 1
-    else if pattern[0] == '*'
       projection[''] = 1
-    else if pattern[0] == 'Events'
+
-      if pattern.length == 1
+    if depth & 1 and (pattern.length < 1 or pattern[0] == '*' or pattern[0] == 'Events')
       res.push([['Events'],
         {exist: timestamp, object: timestamp, writable: timestamp},
         {exist: 1, object: 1, writable: 0}])
-      else if pattern.length == 2
+
-        if pattern[1] == '*'
+    if depth & 2 and (pattern.length < 2 or pattern[0] == '*' or pattern[0] == 'Events')
+      if not pattern[1]? or pattern[1] == '*'
         projection['_registered'] = 1
         projection['_lastInform'] = 1
         projection['_lastBootstrap'] = 1
@@ -157,21 +157,20 @@ fetchDevice = (id, timestamp, patterns, callback) ->
         projection['_lastBootstrap'] = 1
       else if pattern[1] == '1_BOOT'
         projection['_lastBoot'] = 1
-    else if pattern[0] == 'DeviceID'
+
-      if pattern.length == 1
+    if depth & 1 and (pattern.length < 1 or pattern[0] == '*' or pattern[0] == 'DeviceID')
       res.push([['DeviceID'],
         {exist: timestamp, object: timestamp, writable: timestamp},
         {exist: 1, object: 1, writable: 0}])
 
-      else if pattern.length == 2
+    if depth & 2 and (pattern.length < 2 or pattern[0] == '*' or pattern[0] == 'DeviceID')
-        if pattern[1] == '*'
+      if not pattern[1]? or pattern[1] == '*'
         projection['_id'] = 1
         projection['_deviceId._Manufacturer'] = 1
         projection['_deviceId._ProductClass'] = 1
         projection['_deviceId._SerialNumber'] = 1
         projection['_deviceId._OUI'] = 1
         res.push([['DeviceID', '*'], {exist: timestamp}])
-
       else if pattern[1] == 'ID'
         projection['_id'] = 1
       else if pattern[1] == 'Manufacturer'
@@ -182,38 +181,43 @@ fetchDevice = (id, timestamp, patterns, callback) ->
         projection['_deviceId._OUI'] = 1
       else if pattern[1] == 'SerialNumber'
         projection['_deviceId._SerialNumber'] = 1
-    else if pattern[0] == 'Tags'
+
-      if pattern.length == 1
+    if depth & 1 and (pattern.length < 1 or pattern[0] == '*' or pattern[0] == 'Tags')
       res.push([['Tags'],
         {exist: timestamp, object: timestamp, writable: timestamp},
         {exist: 1, object: 1, writable: 0}])
 
-      else if pattern.length == 2
+    if depth & 2 and (pattern.length < 1 or pattern[0] == '*' or pattern[0] == 'Tags')
       res.push([['Tags', '*'], {exist: timestamp}])
       projection['_tags'] = 1
-    else
-      wildcardIndex = pattern.indexOf('*')
 
-      if wildcardIndex == -1
+    if pattern[0] not in ['Tags', 'Events', 'DeviceID']
-        s = pattern.join('.')
+      i = 0
+      while (1 << i) <= depth
+        if pattern.length <= i or pattern[i] == '*'
+          p = pattern.slice(0, i)
+          loaded.push([p, ((1 << p.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])
+          s = p.join('.')
+          projection[s] = 1
+          projection[pattern.slice(0, i - 1).concat('_timestamp').join('.')] = 1
+          break
+
+        if depth & (1 << i)
+          s = pattern.slice(0, i + 1).join('.')
           projection["#{s}._value"] = 1
           projection["#{s}._timestamp"] = 1
+          projection["#{s}._orig"] = 1
           projection["#{s}._type"] = 1
           projection["#{s}._writable"] = 1
           projection["#{s}._object"] = 1
-        projection["#{s}._orig"] = 1
 
           # Timestamp from parent is needed for writable timestamp
-        if pattern.length <= 1
+          if i <= 1
             projection['_timestamp'] = 1
           else
-          projection["#{pattern.slice(0, -1).join('.')}._timestamp"] = 1
+            projection["#{pattern.slice(0, i).join('.')}._timestamp"] = 1
-      else
+
-        p = pattern.slice(0, wildcardIndex)
+        ++ i
-        loaded.push([p, 99])
-        s = p.join('.')
-        projection[s] = 1
-        projection[pattern.slice(0, wildcardIndex - 1).concat('_timestamp').join('.')] = 1
 
   if projection['']
     projection = {}
@@ -253,8 +257,8 @@ fetchDevice = (id, timestamp, patterns, callback) ->
         t.exist = Math.max(t.exist || 0, t.object)
         v.exist = 1
 
-        if not obj['_object']
+      if not descendantsFetched and v.object == 0
-          loaded.push([path, 99])
+        loaded.push([path, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])
 
       res.push([path, t, v])
 

lib/device.js

@@ -226,26 +226,6 @@ function unpack(deviceData, path, revision) {
 }
 
 
-function invalidateSuperset(deviceData, path, attributes) {
-  let depth = 0;
-  if (attributes.exist != null)
-    depth = 99;
-
-  for (let sup of deviceData.paths.superset(path, 99)) {
-    if (sup.wildcard == path.wildcard && sup.length == path.length)
-      continue;
-
-    for (let attrName in deviceData.timestamps) {
-      let t = deviceData.timestamps[attrName].get(sup);
-      if (t <= attributes[attrName] || t <= attributes.exist) {
-        deviceData.timestamps[attrName].delete(sup);
-        deviceData.values[attrName].delete(sup);
-      }
-    }
-  }
-}
-
-
 function normalizeAttributes(timestamps, values) {
   if ('value' in values && timestamps.value >= (timestamps.object || 0)) {
     values.object = 0;
@@ -274,41 +254,9 @@ function normalizeAttributes(timestamps, values) {
 }
 
 
-function set(deviceData, path, timestamps, values) {
+function clear(deviceData, path, timestamps, toClear) {
-  path = deviceData.paths.add(path);
+  if (!toClear)
-  if (path.wildcard || values == null)
+    toClear = new Map();
-    values = {};
-
-  let toInvalidate = new Map();
-
-  if (values.exist) {
-    normalizeAttributes(timestamps, values);
-    if (timestamps.exist && path.length > 1) {
-      set(deviceData, path.slice(0, path.length - 1),
-        {exist: timestamps.exist, object: timestamps.exist},
-        {exist: 1, object: 1});
-    }
-
-    for (let attrName in timestamps) {
-      if (deviceData.values[attrName].has(path) != (attrName in values)) {
-        let inv = toInvalidate.get(path);
-        if (inv == null)
-          toInvalidate.set(path, inv = {});
-        inv[attrName] = timestamps[attrName];
-        if (!(attrName in values))
-          deviceData.values[attrName].delete(path);
-      }
-    }
-  }
-  else {
-    // Clear redundant timestamps
-    let current = inferTimestamps(deviceData, path);
-    for (let c in timestamps)
-      if (timestamps[c] <= current[c])
-        delete timestamps[c];
-      else if (c != 'exist' &&
-          (timestamps[c] <= timestamps.exist || timestamps[c] <= current.exist))
-        delete timetamps[c];
 
   // Clear redundant subsets and deleted params
   for (let sub of deviceData.paths.subset(path)) {
@@ -319,9 +267,9 @@ function set(deviceData, path, timestamps, values) {
       let existTimestamp = deviceData.timestamps.exist.get(sub);
       for (let attrName in timestamps) {
         if (timestamps[attrName] > existTimestamp) {
-            let inv = toInvalidate.get(sub);
+          let inv = toClear.get(sub);
           if (inv == null)
-              toInvalidate.set(sub, inv = {});
+            toClear.set(sub, inv = {});
           for (let attrName in deviceData.timestamps) {
             inv[attrName] = timestamps[attrName];
             deviceData.timestamps[attrName].delete(sub);
@@ -330,14 +278,16 @@ function set(deviceData, path, timestamps, values) {
           break;
         }
 
-          if (timestamps[attrName] > deviceData.timestamps[attrName].get(sub))
+        if (timestamps[attrName] > deviceData.timestamps[attrName].get(sub)) {
           if (deviceData.values[attrName].has(sub)) {
-              let inv = toInvalidate.get(sub);
+            let inv = toClear.get(sub);
             if (inv == null)
-                toInvalidate.set(sub, inv = {});
+              toClear.set(sub, inv = {});
             inv[attrName] = timestamps[attrName];
             deviceData.values[attrName].delete(sub);
           }
+          deviceData.timestamps[attrName].set(sub, timestamps[attrName]);
+        }
       }
     }
     else {
@@ -354,17 +304,82 @@ function set(deviceData, path, timestamps, values) {
       }
     }
   }
+  return toClear;
 }
 
-  toInvalidate.forEach(function(a, p) {
+
-    invalidateSuperset(deviceData, p, a);
+function clearSupersets(deviceData, path, timestamps) {
-  });
+  let depth = 0;
+  if (timestamps.exist != null)
+    depth = 99;
+
+  for (let sup of deviceData.paths.superset(path, 99)) {
+    if (sup.wildcard == path.wildcard && sup.length == path.length)
+      continue;
+
+    for (let attrName in deviceData.timestamps) {
+      let t = deviceData.timestamps[attrName].get(sup);
+      if (t <= timestamps[attrName] || t <= timestamps.exist) {
+        deviceData.timestamps[attrName].delete(sup);
+        deviceData.values[attrName].delete(sup);
+      }
+    }
+  }
+}
+
+
+function set(deviceData, path, timestamps, values, toClear) {
+  path = deviceData.paths.add(path);
+  if (path.wildcard || values == null)
+    values = {};
+
+  if (values.exist) {
+    normalizeAttributes(timestamps, values);
+    if (timestamps.exist && path.length > 1)
+      toClear = set(deviceData, path.slice(0, path.length - 1),
+        {exist: timestamps.exist, object: timestamps.exist},
+        {exist: 1, object: 1},
+        toClear);
+    else if (!toClear)
+      toClear = new Map();
+
+    // Clear attribute patterns
+    for (let attrName in timestamps) {
+      if (deviceData.values[attrName].has(path) != (attrName in values)) {
+        let inv = toClear.get(path);
+        if (inv == null)
+          toClear.set(path, inv = {});
+        inv[attrName] = timestamps[attrName];
+        if (!(attrName in values))
+          deviceData.values[attrName].delete(path);
+      }
+    }
 
     for (let attrName in timestamps) {
       deviceData.timestamps[attrName].set(path, timestamps[attrName]);
       if (attrName in values)
         deviceData.values[attrName].set(path, values[attrName]);
     }
+
+    return toClear;
+  }
+
+  // Clear redundant timestamps
+  let current = inferTimestamps(deviceData, path);
+  for (let c in timestamps)
+    if (timestamps[c] <= current[c])
+      delete timestamps[c];
+    else if (c != 'exist' &&
+        (timestamps[c] <= timestamps.exist || timestamps[c] <= current.exist))
+      delete timetamps[c];
+
+  for (let attrName in timestamps) {
+    deviceData.timestamps[attrName].set(path, timestamps[attrName]);
+    if (attrName in values)
+      deviceData.values[attrName].set(path, values[attrName]);
+  }
+
+  return clear(deviceData, path, timestamps, toClear);
 }
 
 
@@ -373,3 +388,5 @@ exports.getAliasDeclarations = getAliasDeclarations;
 exports.inferTimestamps = inferTimestamps;
 exports.unpack = unpack;
 exports.set = set;
+exports.clear = clear;
+exports.clearSupersets = clearSupersets

lib/session.coffee

@@ -77,7 +77,7 @@ loadParameters = (sessionData, callback) ->
     if not parameters?
       # Device not available in database, mark as new
       sessionData.new = true
-      loaded = [[[], 99]]
+      loaded = [[[], (1 << config.get('MAX_DEPTH', sessionData.deviceId)) - 1]]
       parameters = [
         [['DeviceID', '*'], {exist: sessionData.timestamp}],
         [['Events', '*'], {exist: sessionData.timestamp}]
@@ -89,18 +89,16 @@ loadParameters = (sessionData, callback) ->
       path = sessionData.deviceData.paths.add(p[0])
 
       if p[1]
-        l = sessionData.deviceData.loaded.get(path) || 0
+        l = sessionData.deviceData.loaded.get(path) | 0
-        sessionData.deviceData.loaded.set(path, Math.max(l, p[1]))
+        sessionData.deviceData.loaded.set(path, l | p[1])
 
-      if path.length <= 2 and path.length + p[1] > 2
+      if p[1] & 2 and (path.length < 1 or path[0] == '*' or path[0] == 'VirtualParameters')
-        if path.length < 1 or path[0] == '*' or path[0] == 'VirtualParameters'
         if path.length < 2 or path[1] == '*'
           vparams['*'] = 1
         else
           vparams[path[1]] = 1
 
-      if path.length <= 1 and path.length + p[1] > 1
+      if p[1] & 1 and (path.length < 1 or path[0] == '*' or path[0] == 'VirtualParameters')
-        if path.length < 1 or path[0] == '*' or path[0] == 'VirtualParameters'
         vparams[''] = 1
 
     for p in parameters
@@ -112,32 +110,37 @@ loadParameters = (sessionData, callback) ->
       for k, v of p[2]
         sessionData.deviceData.values[k].set(path, v, 0)
 
+    toClear = null
     # Virtual parameters
     if vparams['']
-      device.set(sessionData.deviceData, ['VirtualParameters'],
+      toClear = device.set(sessionData.deviceData, ['VirtualParameters'],
         {exist: sessionData.timestamp, object: sessionData.timestamp, writable: sessionData.timestamp},
-        {exist: 1, object: 1, writable: 0})
+        {exist: 1, object: 1, writable: 0},
+        toClear)
       delete vparams['']
 
     if vparams['*']
       for k, v of sessionData.cache.virtualParameters
-        device.set(sessionData.deviceData, ['VirtualParameters', k],
+        toClear = device.set(sessionData.deviceData, ['VirtualParameters', k],
           {exist: sessionData.timestamp, object: sessionData.timestamp},
-          {exist: 1, object: 0})
+          {exist: 1, object: 0},
+          toClear)
 
-      device.set(sessionData.deviceData, ['VirtualParameters', '*'],
+      toClear = device.set(sessionData.deviceData, ['VirtualParameters', '*'],
-        {exist: sessionData.timestamp})
+        {exist: sessionData.timestamp}, toClear)
     else
       for k of vparams
         if k of sessionData.cache.virtualParameters
-          device.set(sessionData.deviceData, ['VirtualParameters', k]
+          toClear = device.set(sessionData.deviceData, ['VirtualParameters', k]
             {exist: sessionData.timestamp, object: sessionData.timestamp},
-            {exist: 1, object: 0})
+            {exist: 1, object: 0},
+            toClear)
         else
-          device.set(sessionData.deviceData, ['VirtualParameters', k], {exist: sessionData.timestamp})
+          toClear = device.set(sessionData.deviceData, ['VirtualParameters', k], {exist: sessionData.timestamp}, toClear)
 
     delete sessionData.toLoad
-    return callback()
+
+    return clear(sessionData, toClear, callback)
   )
 
 
@@ -192,10 +195,13 @@ inform = (sessionData, rpcReq, callback) ->
     v.revision = 1 for k, v of sessionData.deviceData.timestamps
     v.revision = 1 for k, v of sessionData.deviceData.values
 
+    toClear = null
     for p in params
-      device.set(sessionData.deviceData, p[0], p[1], p[2])
+      toClear = device.set(sessionData.deviceData, p[0], p[1], p[2], toClear)
 
-    return callback(null, {type : 'InformResponse'})
+    clear(sessionData, toClear, (err) ->
+      return callback(err, {type : 'InformResponse'})
+    )
   )
 
 
@@ -323,11 +329,14 @@ runVirtualParameters = (sessionData, provisions, startRevision, endRevision, cal
       return runVirtualParameters(sessionData, provisions, startRevision, endRevision, callback)
     )
 
+  toClear = null
   if virtualParameterUpdates.length == provisions.length
     for vpu, i in virtualParameterUpdates
-      commitVirtualParameter(sessionData, vpu[0], provisions[i][1], vpu[1])
+      toClear = commitVirtualParameter(sessionData, vpu[0], provisions[i][1], vpu[1], toClear)
 
-  return callback(null, done, decs)
+  clear(sessionData, toClear, (err) ->
+    return callback(err, done, decs)
+  )
 
 
 runDeclarations = (sessionData, declarations, callback) ->
@@ -865,29 +874,27 @@ processDeclaration = (sessionData, path, _timestamps, values) ->
   return ret
 
 
-loadPath = (sessionData, path) ->
+loadPath = (sessionData, path, depth) ->
   return true if sessionData.new
-  tl = []
+  depth ?= (1 << path.length) - 1
-  for i in [path.length...0] by -1
+
-    p = path.slice(0, i)
+  # Trim trailing wildcards
-    loaded = 0
+  trimWildcard = path.length
-    iter = sessionData.deviceData.paths.superset(p)
+  -- trimWildcard while trimWildcard and path[trimWildcard - 1] != '*'
+  path = path.slice(0, trimWildcard) if trimWildcard < path.length
+
+  for i in [0..path.length] by 1
+    iter = sessionData.deviceData.paths.superset(path.slice(0, i))
 
     while sup = iter.next().value
-      if (l = sessionData.deviceData.loaded.get(sup)) > loaded
+      v = sessionData.deviceData.loaded.get(sup)
-        loaded = l
+      depth &= depth ^ sessionData.deviceData.loaded.get(sup)
 
-    if loaded
+    return true if depth == 0
-      tl = tl.slice(0, tl.length - (loaded - 1))
-      break
 
-    tl.push(p)
-
-  if not tl.length
-    return true
 
   sessionData.toLoad ?= []
-  sessionData.toLoad = sessionData.toLoad.concat(tl)
+  sessionData.toLoad.push([path, depth])
   return false
 
 
@@ -991,7 +998,27 @@ applyDeclaration = (sessionData, path, timestamps, values) ->
   return true
 
 
-commitVirtualParameter = (sessionData, name, declaration, update) ->
+clear = (sessionData, toClear, callback) ->
+  return callback() if not toClear?.size
+  MAX_DEPTH = config.get('MAX_DEPTH', sessionData.deviceId)
+
+  toClear.forEach((t, p) ->
+    if t.exist?
+      loadPath(sessionData, p, ((1 << p.length) - 1) ^ (1 << MAX_DEPTH) - 1)
+    else
+      loadPath(sessionData, p, 1 << (p.length - 1))
+  )
+
+  loadParameters(sessionData, (err) ->
+    return callback(err) if err
+
+    toClear.forEach((t, p) ->
+      device.clearSupersets(sessionData.deviceData, p, t)
+    )
+    return callback()
+  )
+
+commitVirtualParameter = (sessionData, name, declaration, update, toClear) ->
   t = {}
   v = {}
   if update.writable?
@@ -1020,7 +1047,7 @@ commitVirtualParameter = (sessionData, name, declaration, update) ->
   else if declaration.timestamps.value?
     throw new Error('Virtual parameter must provide declared attributes')
 
-  device.set(sessionData.deviceData, ['VirtualParameters', name], t, v)
+  return device.set(sessionData.deviceData, ['VirtualParameters', name], t, v, toClear)
 
 
 runExtensions = (sessionData, revision, _extensions, callback) ->
@@ -1093,14 +1120,17 @@ rpcResponse = (sessionData, id, rpcRes, callback) ->
   v.revision = revision for k, v of sessionData.deviceData.timestamps
   v.revision = revision for k, v of sessionData.deviceData.values
 
+  toClear = null
+
   switch rpcRes.type
     when 'GetParameterValuesResponse'
       return callback(new Error('Response type does not match request type')) if rpcReq.type isnt 'GetParameterValues'
 
       for p in rpcRes.parameterList
-        device.set(sessionData.deviceData, common.parsePath(p[0]),
+        toClear = device.set(sessionData.deviceData, common.parsePath(p[0]),
           {exist: timestamp, object: timestamp, value: timestamp},
-          {exist: 1, object: 0, value: p.slice(1)})
+          {exist: 1, object: 0, value: p.slice(1)},
+          toClear)
 
     when 'GetParameterNamesResponse'
       return callback(new Error('Response type does not match request type')) if rpcReq.type isnt 'GetParameterNames'
@@ -1137,16 +1167,18 @@ rpcResponse = (sessionData, id, rpcRes, callback) ->
         return al - bl
       )
 
-      for p in params
+      if rpcReq.nextLevel
-        loadPath(sessionData, p[0])
+        loadPath(sessionData, root, (1 << (root.length + 1)) - 1)
+      else
+        loadPath(sessionData, root, (1 << config.get('MAX_DEPTH', sessionData.deviceId) - 1))
 
       loadParameters(sessionData, (err) ->
         return callback(err) if err
 
         for p in params
-          device.set(sessionData.deviceData, p[0], p[1], p[2])
+          toClear = device.set(sessionData.deviceData, p[0], p[1], p[2], toClear)
 
-        return callback()
+        clear(sessionData, toClear, callback)
       )
       return
 
@@ -1154,33 +1186,36 @@ rpcResponse = (sessionData, id, rpcRes, callback) ->
       return callback(new Error('Response type does not match request type')) if rpcReq.type isnt 'SetParameterValues'
 
       for p in rpcReq.parameterList
-        device.set(sessionData.deviceData, common.parsePath(p[0]),
+        toClear = device.set(sessionData.deviceData, common.parsePath(p[0]),
           {exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
-          {exist: 1, object: 0, writable: 1, value: p.slice(1)})
+          {exist: 1, object: 0, writable: 1, value: p.slice(1)}, toClear)
 
     when 'AddObjectResponse'
-      device.set(sessionData.deviceData, common.parsePath(rpcReq.objectName + rpcRes.instanceNumber),
+      toClear = device.set(sessionData.deviceData, common.parsePath(rpcReq.objectName + rpcRes.instanceNumber),
         {exist: timestamp, object: timestamp},
-        {exist: 1, object: 1})
+        {exist: 1, object: 1},
+        toClear)
 
     when 'DeleteObjectResponse'
-      device.set(sessionData.deviceData, common.parsePath(rpcReq.objectName.slice(0, -1)),
+      toClear = device.set(sessionData.deviceData, common.parsePath(rpcReq.objectName.slice(0, -1)),
-        {exist: timestamp})
+        {exist: timestamp}, toClear)
 
     when 'RebootResponse'
-      device.set(sessionData.deviceData, common.parsePath('Reboot'),
+      toClear = device.set(sessionData.deviceData, common.parsePath('Reboot'),
         {exist: sessionData.timestamp, value: sessionData.timestamp},
-        {exist: 1, value: [sessionData.timestamp, 'xsd:unsignedInt']})
+        {exist: 1, value: [sessionData.timestamp, 'xsd:unsignedInt']},
+        toClear)
 
     when 'FactoryResetResponse'
-      device.set(sessionData.deviceData, common.parsePath('FactoryReset'),
+      toClear = device.set(sessionData.deviceData, common.parsePath('FactoryReset'),
         {exist: sessionData.timestamp, value: sessionData.timestamp},
-        {exist: 1, value: [sessionData.timestamp, 'xsd:unsignedInt']})
+        {exist: 1, value: [sessionData.timestamp, 'xsd:unsignedInt']},
+        toClear)
 
     else
       return callback(new Error('Response type not recognized'))
 
-  return callback()
+  return clear(sessionData, toClear, callback)
 
 
 rpcFault = (sessionData, id, faultResponse, callback) ->