Major restructuring and optimization

lib/cwmp.coffee

@@ -157,7 +157,7 @@ applyPresets = (currentRequest) ->
     deviceEvents = {}
     iter = currentRequest.sessionData.deviceData.paths.subset(['Events', '*'])
     while (p = iter.next().value)
-      if currentRequest.sessionData.timestamp == currentRequest.sessionData.deviceData.values.value.get(p)?[0]
+      if currentRequest.sessionData.timestamp == currentRequest.sessionData.deviceData.attributes.get(p)?.value?[1][0]
         deviceEvents[p[1]] = true
 
     parameters = {}
@@ -171,11 +171,11 @@ applyPresets = (currentRequest) ->
 
       eventsMatch = true
       for k, v of p.events
-        if !v != !deviceEvents[k.replace(' ', '_')]
+        if (!v) != (!deviceEvents[k.replace(' ', '_')])
           eventsMatch = false
           break
 
-      continue if not eventsMatch or (p.schedule and not
+      continue if (not eventsMatch) or (p.schedule and not
         (p.schedule.schedule and testSchedule(p.schedule, currentRequest.sessionData.timestamp)[0]))
 
       filteredPresets.push(p)
@@ -184,7 +184,7 @@ applyPresets = (currentRequest) ->
 
     declarations = []
     for k, v of parameters
-      declarations.push([v, {exist: 1, value: 1}])
+      declarations.push([v, 1, {value: 1}])
 
     session.rpcRequest(currentRequest.sessionData, declarations, (err, id, rpcRequest) ->
       throw err if err
@@ -194,7 +194,7 @@ applyPresets = (currentRequest) ->
 
       for k, v of parameters
         unpacked = device.unpack(currentRequest.sessionData.deviceData, v)
-        if unpacked[0] and (vv = currentRequest.sessionData.deviceData.values.value.get(unpacked[0]))?
+        if unpacked[0] and (vv = currentRequest.sessionData.deviceData.attributes.get(unpacked[0]).value?[1])?
           parameters[k] = vv[0]
 
       if whiteList?
@@ -215,12 +215,12 @@ applyPresets = (currentRequest) ->
 
 
 nextRpc = (currentRequest) ->
-  session.rpcRequest(currentRequest.sessionData, null, (err, id, rpcRequest) ->
+  session.rpcRequest(currentRequest.sessionData, null, (err, id, rpcRequest, doneProvisions) ->
     throw err if err
     if rpcRequest?
       return sendRpcRequest(currentRequest, id, rpcRequest)
 
-    for p, i in currentRequest.sessionData.provisions[0]
+    for p, i in doneProvisions or []
       if currentRequest.sessionData.faults?
         delete currentRequest.sessionData.faults[currentRequest.sessionData.channels[i]]
 
@@ -336,6 +336,7 @@ onConnection = (socket) ->
     for sessionId, sessionData of sessions
       session.serialize(sessionData, (err, sessionDataString) ->
         throw err if err
+        # TODO don't set if process is shutting down
         db.redisClient.setex("session_#{sessionId}", sessionData.timeout, sessionDataString, (err) ->
           throw err if err
         )

lib/db.coffee

@@ -138,9 +138,8 @@ fetchDevice = (id, timestamp, patterns, callback) ->
       projection[''] = 1
 
     if depth & 1 and (pattern.length < 1 or pattern[0] == '*' or pattern[0] == 'Events')
-      res.push([['Events'],
+      res.push([['Events'], timestamp,
-        {exist: timestamp, object: timestamp, writable: timestamp},
+        {object: [timestamp, 1], writable: [timestamp, 0]}])
-        {exist: 1, object: 1, writable: 0}])
 
     if depth & 2 and (pattern.length < 2 or pattern[0] == '*' or pattern[0] == 'Events')
       if not pattern[1]? or pattern[1] == '*'
@@ -148,7 +147,7 @@ fetchDevice = (id, timestamp, patterns, callback) ->
         projection['_lastInform'] = 1
         projection['_lastBootstrap'] = 1
         projection['_lastBoot'] = 1
-        res.push([['Events', '*'], {exist: timestamp}])
+        res.push([['Events', '*'], timestamp])
       else if pattern[1] == 'Registered'
         projection['_registered'] = 1
       else if pattern[1] == 'Inform'
@@ -159,9 +158,8 @@ fetchDevice = (id, timestamp, patterns, callback) ->
         projection['_lastBoot'] = 1
 
     if depth & 1 and (pattern.length < 1 or pattern[0] == '*' or pattern[0] == 'DeviceID')
-      res.push([['DeviceID'],
+      res.push([['DeviceID'], timestamp,
-        {exist: timestamp, object: timestamp, writable: timestamp},
+        {object: [timestamp, 1], writable: [timestamp, 0]}])
-        {exist: 1, object: 1, writable: 0}])
 
     if depth & 2 and (pattern.length < 2 or pattern[0] == '*' or pattern[0] == 'DeviceID')
       if not pattern[1]? or pattern[1] == '*'
@@ -170,7 +168,7 @@ fetchDevice = (id, timestamp, patterns, callback) ->
         projection['_deviceId._ProductClass'] = 1
         projection['_deviceId._SerialNumber'] = 1
         projection['_deviceId._OUI'] = 1
-        res.push([['DeviceID', '*'], {exist: timestamp}])
+        res.push([['DeviceID', '*'], timestamp])
       else if pattern[1] == 'ID'
         projection['_id'] = 1
       else if pattern[1] == 'Manufacturer'
@@ -183,12 +181,11 @@ fetchDevice = (id, timestamp, patterns, callback) ->
         projection['_deviceId._SerialNumber'] = 1
 
     if depth & 1 and (pattern.length < 1 or pattern[0] == '*' or pattern[0] == 'Tags')
-      res.push([['Tags'],
+      res.push([['Tags'], timestamp,
-        {exist: timestamp, object: timestamp, writable: timestamp},
+        {object: [timestamp, 1], writable: [timestamp, 0]}])
-        {exist: 1, object: 1, writable: 0}])
 
     if depth & 2 and (pattern.length < 1 or pattern[0] == '*' or pattern[0] == 'Tags')
-      res.push([['Tags', '*'], {exist: timestamp}])
+      res.push([['Tags', '*'], timestamp])
       projection['_tags'] = 1
 
     if pattern[0] not in ['Tags', 'Events', 'DeviceID']
@@ -235,89 +232,71 @@ fetchDevice = (id, timestamp, patterns, callback) ->
       if obj['_timestamp']?
         obj['_timestamp'] = +obj['_timestamp']
 
-      t = {}
+      t = if obj['_timestamp'] > timestamp then obj['_timestamp'] else timestamp
-      v = {}
+
+      attrs = {}
 
       if obj['_value']?
-        t.value = obj['_timestamp'] ? timestamp
+        attrs.value = [t, [obj['_value'], obj['_type']]]
-        v.value = [obj['_value'], obj['_type']]
         obj['_object'] = false
-        t.exist = Math.max(t.exist || 0, t.value)
-        v.exist = 1
 
       if obj['_writable']?
-        t.writable = timestamp or 1
+        attrs.writable = [timestamp, if obj['_writable'] then 1 else 0]
-        v.writable = if obj['_writable'] then 1 else 0
-        t.exist = Math.max(t.exist || 0, t.writable)
-        v.exist = 1
 
       if obj['_object']?
-        t.object = obj['_timestamp'] or timestamp or 1
+        attrs.object = [t, if obj['_object'] then 1 else 0]
-        v.object = if obj['_object'] then 1 else 0
-        t.exist = Math.max(t.exist || 0, t.object)
-        v.exist = 1
 
-      if not descendantsFetched and v.object == 0
+      if not descendantsFetched and attrs.object?[1] == 0
         loaded.push([path, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])
 
-      res.push([path, t, v])
+      res.push([path, t, attrs])
 
       for k, v of obj
         if k[0] != '_'
           obj['_object'] = true
-          storeParams(v, path.concat(k), obj['_timestamp'] ? 0, descendantsFetched)
+          storeParams(v, path.concat(k), obj['_timestamp'] or 1, descendantsFetched)
 
       if obj['_object'] and descendantsFetched
-        res.push([path.concat('*'), {exist: obj['_timestamp'] ? 0}])
+        res.push([path.concat('*'), obj['_timestamp'] or 1])
 
     for k, v of device
       if k == '_timestamp'
-        res.push([['*'], {exist: v}]) if not Object.keys(projection).length
+        res.push([['*'], +v]) if not Object.keys(projection).length
       else if k == '_lastInform'
-        res.push([['Events', 'Inform'],
+        res.push([['Events', 'Inform'], timestamp,
-          {exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
+          {object: [timestamp, 0], writable: [timestamp, 0], value: [+v, [+v, 'xsd:dateTime']]}])
-          {exist: 1, object: 0, writable: 0, value: [+v, 'xsd:dateTime']}])
       else if k == '_lastBoot'
-        res.push([['Events', '1_BOOT'],
+        res.push([['Events', '1_BOOT'], timestamp,
-          {exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
+          {object: [timestamp, 0], writable: [timestamp, 0], value: [+v, [+v, 'xsd:dateTime']]}])
-          {exist: 1, object: 0, writable: 0, value: [+v, 'xsd:dateTime']}])
       else if k == '_lastBootstrap'
-        res.push([['Events', '0_BOOTSTRAP'],
+        res.push([['Events', '0_BOOTSTRAP'], timestamp,
-          {exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
+          {object: [timestamp, 0], writable: [timestamp, 0], value: [+v, [+v, 'xsd:dateTime']]}])
-          {exist: 1, object: 0, writable: 0, value: [+v, 'xsd:dateTime']}])
       else if k == '_registered'
-        res.push([['Events', 'Registered'],
+        res.push([['Events', 'Registered'], timestamp,
-          {exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
+          {object: [timestamp, 0], writable: [timestamp, 0], value: [+v, [+v, 'xsd:dateTime']]}])
-          {exist: 1, object: 0, writable: 0, value: [+v, 'xsd:dateTime']}])
       else if k == '_id'
-        res.push([['DeviceID', 'ID'],
+        res.push([['DeviceID', 'ID'], timestamp,
-          {exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
+          {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v, 'xsd:string']]}])
-          {exist: 1, object: 0, writable: 0, value: [v, 'xsd:string']}])
       else if k == '_tags'
         for t in v
-          res.push([['Tags', t],
+          res.push([['Tags', t], timestamp,
-            {exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
+            {object: [timestamp, 0], writable: [timestamp, 1], value: [timestamp, [true, 'xsd:boolean']]}])
-            {exist: 1, object: 0, writable: 1, value: [true, 'xsd:boolean']}])
       else if k == '_deviceId'
         for kk, vv of v
           if kk == '_Manufacturer'
-            res.push([['DeviceID', 'Manufacturer'],
+            res.push([['DeviceID', 'Manufacturer'], timestamp,
-              {exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
+              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [vv, 'xsd:string']]}])
-              {exist: 1, object: 0, writable: 0, value: [vv, 'xsd:boolean']}])
           else if kk == '_OUI'
-            res.push([['DeviceID', 'OUI'],
+            res.push([['DeviceID', 'OUI'], timestamp,
-              {exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
+              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [vv, 'xsd:string']]}])
-              {exist: 1, object: 0, writable: 0, value: [vv, 'xsd:boolean']}])
           if kk == '_ProductClass'
-            res.push([['DeviceID', 'ProductClass'],
+            res.push([['DeviceID', 'ProductClass'], timestamp,
-              {exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
+              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [vv, 'xsd:string']]}])
-              {exist: 1, object: 0, writable: 0, value: [vv, 'xsd:boolean']}])
           if kk == '_SerialNumber'
-            res.push([['DeviceID', 'SerialNumber'],
+            res.push([['DeviceID', 'SerialNumber'], timestamp,
-              {exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
+              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [vv, 'xsd:string']]}])
-              {exist: 1, object: 0, writable: 0, value: [vv, 'xsd:boolean']}])
       else if common.typeOf(v) is common.OBJECT_TYPE
-        storeParams(v, [k], +(device['_timestamp'] ? 0), Object.keys(projection).length == 0)
+        storeParams(v, [k], +(device['_timestamp'] ? 1), Object.keys(projection).length == 0)
 
     return callback(null, res, loaded)
   )
@@ -326,102 +305,91 @@ fetchDevice = (id, timestamp, patterns, callback) ->
 saveDevice = (deviceId, deviceData, isNew, callback) ->
   update = {'$set' : {}, '$unset' : {}, '$addToSet' : {}, '$pull' : {}}
 
-  # Delete parameters that no longer exist
+  iter = deviceData.timestamps.diff()
-  iter = deviceData.values.exist.diff()
+  while diff = iter.next().value
-  while not (diff = iter.next()).done
+    continue if diff[0].wildcard != (1 << (diff[0].length - 1))
-    if diff.value[1] and not diff.value[2]
+    continue if diff[0][0] in ['Events', 'DeviceID', 'Tags']
-      update['$unset'][diff.value[0].join('.')] = 1
 
-  # Set object's children timestamps
+    if not diff[2]? and diff[1]?
-  iter = deviceData.timestamps.exist.diff()
+      update['$unset'][diff[0].slice(0, -1).concat('_timestamp').join('.')] = 1
-  while not (diff = iter.next()).done
+    else if diff[2] != diff[1]
-    continue if diff.value[0].wildcard != (1 << (diff.value[0].length - 1))
+      parent = deviceData.paths.subset(diff[0].slice(0, -1)).next().value
-    continue if diff.value[0][0] in ['Events', 'DeviceID', 'Tags']
+      if parent and (parent.length == 0 or deviceData.attributes.has(parent))
+        update['$set'][diff[0].slice(0, -1).concat('_timestamp').join('.')] = new Date(diff[2])
 
-    if not diff.value[2]? and diff.value[1]?
+  iter = deviceData.attributes.diff()
-      update['$unset'][diff.value[0].slice(0, -1).concat('_timestamp').join('.')] = 1
+  while diff = iter.next().value
-    else if diff.value[2] != diff.value[1]
+    continue if diff[1] == diff[2]
-      update['$set'][diff.value[0].slice(0, -1).concat('_timestamp').join('.')] = new Date(diff.value[2])
+    path = diff[0]
-
+    switch path[0]
-  # Set writable
+      when 'Events'
-  iter = deviceData.values.writable.diff()
+        if diff[0].length == 2 and diff[2].value?[1][0] != diff[1]?.value?[1][0]
-  while not (diff = iter.next()).done
+          t = new Date(diff[2].value[1][0])
-    continue if diff.value[0][0] in ['Events', 'DeviceID', 'Tags']
+          switch path[1]
-    if not diff.value[2]? and diff.value[1]?
+            when 'Inform'
-      update['$unset'][diff.value[0].concat('_writable').join('.')] = 1
+              update['$set']['_lastInform'] = t
-    else if diff.value[2] != diff.value[1]
+            when '1_BOOT'
-      update['$set'][diff.value[0].concat('_writable').join('.')] = Boolean(diff.value[2])
+              update['$set']['_lastBoot'] = t
-
+            when '0_BOOTSTRAP'
-  # Set object
+              update['$set']['_lastBootstrap'] = t
-  iter = deviceData.values.object.diff()
+            when 'Registered'
-  while not (diff = iter.next()).done
+              update['$set']['_registered'] = t
-    continue if diff.value[0][0] in ['Events', 'DeviceID', 'Tags']
+      when 'DeviceID'
-    if diff.value[1] != diff.value[2]
+        if diff[2].value?[1]?[0] != diff[1]?.value?[1]?[0]
-      if diff.value[2]
+          v = diff[2].value[1][0]
-        update['$set'][diff.value[0].concat('_object').join('.')] = true
+          switch path[1]
-      else if not diff.value[2]?
+            when 'ID'
-        update['$unset'][diff.value[0].concat('_object').join('.')] = 1
+              update['$set']['_id'] = v
+            when 'Manufacturer'
+              update['$set']['_deviceId._Manufacturer'] = v
+            when 'OUI'
+              update['$set']['_deviceId._OUI'] = v
+            when 'ProductClass'
+              update['$set']['_deviceId._ProductClass'] = v
+            when 'SerialNumber'
+              update['$set']['_deviceId._SerialNumber'] = v
+      when 'Tags'
+        if diff[2].value?[1][0] != diff[1].value?[1][0]
+          v = diff[2].value?[1][0]
+          if v?
+            update['$addToSet']['_tags'] ?= {'$each' : []}
+            update['$addToSet']['_tags']['$each'].push(v)
+          else
+            update['$pull']['_tags'] ?= {'$in' : []}
+            update['$pull']['_tags']['$in'].push(v)
       else
-        if deviceData.values.value.getDiff(diff.value[0])?[2]?
+        if not diff[2]
-          if diff.value[1]?
+          update['$unset'][diff[0].join('.')] = 1
-            update['$unset'][diff.value[0].concat('_object').join('.')] = 1
+          continue
-        else
-          update['$set'][diff.value[0].concat('_object').join('.')] = false
-    else if diff.value[2]?
-      vdiff = deviceData.values.value.getDiff(diff.value[0])
-      if vdiff[1] != vdiff[2]
-        if not vdiff[2]?
-          update['$set'][diff.value[0].concat('_object').join('.')] = false
-        else if not vdiff[1]?
-          update['$unset'][diff.value[0].concat('_object').join('.')] = 1
 
-  # Set value
+        for attrName of diff[2]
-  iter = deviceData.values.value.diff()
+          if diff[2][attrName][1]? or diff[1]?[attrName]?[1]?
-  while not (diff = iter.next()).done
+            switch attrName
-    if diff.value[0][0] == 'Events'
+              when 'value'
-      continue if diff.value[1]?[0] == diff.value[2][0]
+                if not diff[1] or diff[2].value[1][0] != diff[1].value?[1][0]
-      if diff.value[0][1] == 'Inform'
+                  if diff[2].value[1][1] == 'xsd:dateTime'
-        update['$set']['_lastInform'] = new Date(diff.value[2][0])
+                    update['$set'][path.concat('_value').join('.')] = new Date(diff[2].value[1][0])
-      else if diff.value[0][1] == '1_BOOT'
+                  else
-        update['$set']['_lastBoot'] = new Date(diff.value[2][0])
+                    update['$set'][path.concat('_value').join('.')] = diff[2].value[1][0]
-      else if diff.value[0][1] == '0_BOOTSTRAP'
+                if not diff[1] or diff[2].value[1][1] != diff[1].value?[1][1]
-        update['$set']['_lastBootstrap'] = new Date(diff.value[2][0])
+                  update['$set'][path.concat('_type').join('.')] = diff[2].value[1][1]
-      else if diff.value[0][1] == 'Registered'
+                if not diff[1] or diff[2].value[0] != diff[1].value?[0]
-        update['$set']['_registered'] = new Date(diff.value[2][0])
+                  update['$set'][path.concat('_timestamp').join('.')] = new Date(diff[2].value[0])
-    else if diff.value[0][0] == 'Tags'
+              when 'object'
-      continue if diff.value[1]?[0] == diff.value[2]?[0]
+                if not diff[1]?.object or diff[2].object[1] != diff[1].object?[1]
-      if diff.value[2][0]
+                  update['$set'][path.concat('_object').join('.')] = !!diff[2].object[1]
-        update['$addToSet']['_tags'] ?= {'$each' : []}
+              when 'writable'
-        update['$addToSet']['_tags']['$each'].push(p[0][1])
+                if not diff[1]?.writable or diff[2].writable[1] != diff[1].writable?[1]
-      else
+                  update['$set'][path.concat('_writable').join('.')] = !!diff[2].writable[1]
-        update['$pull']['_tags'] ?= {'$in' : []}
+
-        update['$pull']['_tags']['$in'].push(p[0][1])
+        if diff[1]
-    else if diff.value[0][0] == 'DeviceID'
+          for attrName of diff[1]
-      continue if diff.value[1]?[0] == diff.value[2][0]
+            if attrName not of diff[2]
-      if diff.value[0][1] == 'ID'
+              update['$unset'][path.concat("_#{attrName}").join('.')] = 1
-        update['$set']['_id'] = diff.value[2][0]
+              if attrName is 'value'
-      else if diff.value[0][1] == 'Manufacturer'
+                update['$unset'][path.concat('_type').join('.')] = 1
-        update['$set']['_deviceId._Manufacturer'] = diff.value[2][0]
+                update['$unset'][path.concat('_timestamp').join('.')] = 1
-      else if diff.value[0][1] == 'OUI'
-        update['$set']['_deviceId._OUI'] = diff.value[2][0]
-      else if diff.value[0][1] == 'ProductClass'
-        update['$set']['_deviceId._ProductClass'] = diff.value[2][0]
-      else if diff.value[0][1] == 'SerialNumber'
-        update['$set']['_deviceId._SerialNumber'] = diff.value[2][0]
-    else
-      tdiff = deviceData.timestamps.value.getDiff(diff.value[0])
-      if diff.value[2]?
-        if tdiff[2] != tdiff[1]
-          update['$set'][diff.value[0].concat('_timestamp').join('.')] = new Date(tdiff[2])
-          delete update['$unset'][diff.value[0].concat('_timestamp').join('.')]
-        if diff.value[2][0] != diff.value[1]?[0]
-          update['$set'][diff.value[0].concat('_value').join('.')] = diff.value[2][0]
-        if diff.value[2][1] != diff.value[1]?[1]
-          update['$set'][diff.value[0].concat('_type').join('.')] = diff.value[2][1]
 
-      else if diff.value[1]?
-        update['$unset'][diff.value[0].concat('_value').join('.')] = 1
-        update['$unset'][diff.value[0].concat('_value').join('.')] = 1
 
   # Remove empty keys
   for k of update

lib/device.js

@@ -19,31 +19,34 @@
 const common = require('./common');
 
 
+const CHANGE_FLAGS = {
+  object: 4,
+  writable: 16,
+  value: 64
+};
+
+
 function sanitizeParameterValue(parameterValue) {
   if (parameterValue[0] != null) {
     switch (parameterValue[1]) {
       case 'xsd:boolean':
         if (typeof parameterValue[0] !== 'boolean') {
-          parameterValue = parameterValue.slice();
           parameterValue[0] = !!JSON.parse(parameterValue[0]);
         }
         break;
       case 'xsd:string':
         if (typeof parameterValue[0] !== 'string') {
-          parameterValue = parameterValue.slice();
           parameterValue[0] = '' + parameterValue[0];
         }
         break;
       case 'xsd:int':
       case 'xsd:unsignedInt':
         if (typeof parameterValue[0] !== 'number') {
-          parameterValue = parameterValue.slice();
           parameterValue[0] = +parameterValue[0];
         }
         break;
       case 'xsd:dateTime':
         if (typeof parameterValue[0] !== 'number') {
-          parameterValue = parameterValue.slice();
           if (parameterValue[0].getTime() != null)
             parameterValue[0] = parameterValue[0].getTime();
           else if (isNaN(parameterValue[0]))
@@ -63,68 +66,6 @@ function sanitizeParameterValue(parameterValue) {
 }
 
 
-function inferTimestamps(deviceData, path) {
-  common.addPathMeta(path);
-  var res = {};
-
-  for (let sup of deviceData.paths.superset(path)) {
-    for (let attrName in deviceData.timestamps) {
-      let t = deviceData.timestamps[attrName].get(sup);
-      if (t && !(t < res[sup]))
-        res[attrName] = t;
-    }
-  }
-
-  for (let i = path.length - 1; i > 0; -- i) {
-    if ((path.wildcard & ((1 << i) - 1)) == 0)
-      break;
-
-    let param = null;
-    let ancestor = path.slice(0, i);
-    let ancestorTimestamp = 0;
-    for (let sup of deviceData.paths.superset(ancestor)) {
-      if (!sup.wildcard)
-        param = sup;
-
-      let t = deviceData.timestamps.exist.get(sup);
-      if (t > ancestorTimestamp)
-        ancestorTimestamp = t;
-    }
-
-    if (param != null && deviceData.values.exist.get(param)) {
-      if (deviceData.values.object.get(param) == 0) {
-        let t = deviceData.timestamps.object.get(param);
-        for (let attrName in deviceData.timestamps)
-          if (!(t <= res[attrName]))
-            res[attrName] = t;
-      }
-    }
-    else if (ancestorTimestamp) {
-      let newt = {};
-      for (let attrName in deviceData.timestamps)
-        newt[attrName] = ancestorTimestamp;
-
-      for (let sub of deviceData.paths.subset(ancestor)) {
-        if (sub.wildcard)
-          continue;
-        let subt = inferTimestamps(deviceData, sub.concat(path.slice(sub.length, path.length)));
-        for (let attrName in newt) {
-          if (!(attrName in subt))
-            delete newt[attrName];
-          else if (subt[attrName] < newt[attrName])
-            newt[attrName] = subt[attrName];
-        }
-      }
-      for (let attrName in newt)
-        if (!(newt[attrName] <= res[attrName]))
-          res[attrName] = newt[attrName];
-    }
-  }
-
-  return res;
-}
-
-
 function getAliasDeclarations(path, timestamp) {
   var decs = [];
 
@@ -141,9 +82,9 @@ function getAliasDeclarations(path, timestamp) {
     }
 
     if (prefix.length)
-      decs.push([pattern, {exist: timestamp, value: timestamp}]);
+      decs.push([pattern, timestamp, {value: timestamp}]);
     else
-      decs.push([pattern, {exist: timestamp}]);
+      decs.push([pattern, timestamp]);
   }
 
   recursive(path, []);
@@ -154,50 +95,51 @@ function getAliasDeclarations(path, timestamp) {
 
 function unpack(deviceData, path, revision) {
   let allMatches = [];
-  if (path.alias == 0) {
+  if (!path.alias) {
     for (let p of deviceData.paths.subset(path, 0))
-      if (deviceData.values.exist.get(p, revision))
+      if (deviceData.attributes.has(p, revision))
         allMatches.push(p);
-    return allMatches;
   }
+  else {
+    let wildcardPath = path.slice();
+    wildcardPath.wildcard = path.wildcard;
+    for (let i = 0; i < wildcardPath.length; ++ i)
+      if (Array.isArray(wildcardPath[i])) {
+        wildcardPath[i] = '*';
+        wildcardPath.wildcard |= 1 << i;
+      }
 
-  let wildcardPath = path.slice();
+    for (let p of deviceData.paths.subset(wildcardPath, 0))
-  wildcardPath.wildcard = path.wildcard;
+      if (deviceData.attributes.has(p, revision))
-  for (let i = 0; i < wildcardPath.length; ++ i)
+        allMatches.push(p);
-    if (Array.isArray(wildcardPath[i])) {
-      wildcardPath[i] = '*';
-      wildcardPath.wildcard |= 1 << i;
-    }
 
-  for (let p of deviceData.paths.subset(wildcardPath, 0))
+    for (let i = path.length - 1; i >= 0; -- i) {
-    if (deviceData.values.exist.get(p, revision))
+      if (Array.isArray(path[i])) {
-      allMatches.push(p);
+        for (let j = 0; j < path[i].length; j += 2) {
+          let p = wildcardPath.slice(0, i + 1).concat(path[i][j]);
+          p.alias = (wildcardPath.alias & ((1 << (i + 1)) - 1)) + path[i][j].alias << (i + 1);
+          p.wildcard = (wildcardPath.wildcard & ((1 << (i + 1)) - 1)) + path[i][j].wildcard << (i + 1);
+          let unpacked = unpack(deviceData, p, revision);
+          let filtered = [];
+          for (let up of unpacked) {
+            let attributes = deviceData.attributes.get(up, revision);
+            if (attributes && attributes.value && attributes.value[1] &&
+                sanitizeParameterValue([path[i][j + 1], attributes.value[1][1]])[0] === attributes.value[1][0]) {
+              for (let m = 0; m < allMatches.length; ++ m) {
+                let k;
+                for (k = i; k >= 0; -- k)
+                  if (allMatches[m][k] != up[k])
+                    break
 
-  for (let i = path.length - 1; i >= 0; -- i) {
+                if (k < 0) {
-    if (Array.isArray(path[i])) {
+                  filtered.push(allMatches[m]);
-      for (let j = 0; j < path[i].length; j += 2) {
+                  allMatches[m] = [];
-        let p = wildcardPath.slice(0, i + 1).concat(path[i][j]);
+                }
-        p.alias = (wildcardPath.alias & ((1 << (i + 1)) - 1)) + path[i][j].alias << (i + 1);
-        p.wildcard = (wildcardPath.wildcard & ((1 << (i + 1)) - 1)) + path[i][j].wildcard << (i + 1);
-        let unpacked = unpack(deviceData, p, revision);
-        let filtered = [];
-        for (let up of unpacked) {
-          let v = deviceData.values.value.get(up, revision);
-          if (v && sanitizeParameterValue([path[i][j + 1], v[1]])[0] == v[0]) {
-            for (let m = 0; m < allMatches.length; ++ m) {
-              let k;
-              for (k = i; k >= 0; -- k)
-                if (allMatches[m][k] != up[k])
-                  break
-
-              if (k < 0) {
-                filtered.push(allMatches[m]);
-                allMatches[m] = [];
               }
             }
           }
+          allMatches = filtered;
         }
-        allMatches = filtered;
       }
     }
   }
@@ -209,7 +151,7 @@ function unpack(deviceData, path, revision) {
 
       // Use numeric sorting for numbers
       let ia = parseInt(a), ib = parseInt(b);
-      if (ia.toString() == a && ib.toString() == b) {
+      if (ia.toString() === a && ib.toString() === b) {
         a = ia;
         b = ib;
       }
@@ -226,167 +168,221 @@ function unpack(deviceData, path, revision) {
 }
 
 
-function normalizeAttributes(timestamps, values) {
+function clear(deviceData, path, timestamp, attributes) {
-  if ('value' in values && timestamps.value >= (timestamps.object || 0)) {
+  for (let p of deviceData.paths.all(path, timestamp ? 99 : 0)) {
-    values.object = 0;
+    let t = deviceData.timestamps.get(p);
-    timestamps.object = timestamps.value;
+    if (!t)
-  }
+      continue;
 
-  if (values.object && timestamps.object > (timestamps.value || 0)) {
+    let changeFlags = 0;
-    delete values.value;
-    timestamps.value = timestamps.object;
-  }
 
-  if ('value' in values && timestamps.value >= (timestamps.exist || 0)) {
+    if (!(timestamp <= t)) {
-    values.exist = 1;
+      deviceData.timestamps.delete(p);
-    timestamps.exist = timestamps.value;
+      deviceData.attributes.delete(p);
-  }
+      changeFlags |= 3;
+    } else if (t && attributes) {
+      let currentAttributes = deviceData.attributes.get(p);
+      if (currentAttributes) {
+        let newAttrs;
+        for (let attrName in attributes) {
+          if (attrName in currentAttributes &&
+              attributes[attrName][0] > currentAttributes[attrName][0]) {
+            if (!newAttrs)
+              newAttrs = JSON.parse(JSON.stringify(currentAttributes));
+            delete newAttrs[attrName];
+            changeFlags |= CHANGE_FLAGS[attrName];
+            changeFlags |= CHANGE_FLAGS[attrName] << 1;
+          }
+        }
 
-  if ('object' in values && timestamps.object >= (timestamps.exist || 0)) {
+        if (newAttrs)
-    values.exist = 1;
+          deviceData.attributes.set(p, newAttrs);
-    timestamps.exist = timestamps.object;
+      }
-  }
+    }
 
-  if ('writable' in values && timestamps.writable >= (timestamps.exist || 0)) {
+    if (changeFlags)
-    values.exist = 1;
+      for (let sup of deviceData.paths.superset(p)) {
-    timestamps.exist = timestamps.writable;
+        let c = deviceData.trackers.get(sup);
+        for (let k in c)
+          if (c[k] & changeFlags)
+            deviceData.changes.add(k);
+      }
   }
 }
 
 
-function clear(deviceData, path, timestamps, toClear) {
+function compareEquality(a, b) {
-  if (!toClear)
+  var t = typeof a;
-    toClear = new Map();
+  if (a === null ||
+      a === undefined ||
+      t === 'number' ||
+      t === 'boolean' ||
+      t === 'string' ||
+      t === 'symbol')
+    return a === b;
 
-  // Clear redundant subsets and deleted params
+  return JSON.stringify(a) === JSON.stringify(b);
-  for (let sub of deviceData.paths.subset(path)) {
+}
-    if (path.wildcard == sub.wildcard)
-      continue;
 
-    if (deviceData.values.exist.has(sub)) {
+
-      let existTimestamp = deviceData.timestamps.exist.get(sub);
+function set(deviceData, path, timestamp, attributes, toClear) {
-      for (let attrName in timestamps) {
+  path = deviceData.paths.add(path);
-        if (timestamps[attrName] > existTimestamp) {
+
-          let inv = toClear.get(sub);
+  let changeFlags = 0;
-          if (inv == null)
+
-            toClear.set(sub, inv = {});
+  let currentTimestamp = deviceData.timestamps.get(path);
-          for (let attrName in deviceData.timestamps) {
+
-            inv[attrName] = timestamps[attrName];
+  let currentAttributes;
-            deviceData.timestamps[attrName].delete(sub);
+
-            deviceData.values[attrName].delete(sub);
+  if (path.wildcard)
+    attributes = undefined;
+  else
+    currentAttributes = deviceData.attributes.get(path);
+
+  if (attributes) {
+    if (attributes.value && attributes.value[1]
+        && attributes.value[0] >= (attributes.object ? attributes.object[0] : 0))
+      attributes.object = [attributes.value[0], 0];
+
+    if (attributes.object && attributes.object[1]
+      && attributes.object[0] >= (attributes.value ? attributes.value[0] : 0))
+      attributes.value = [attributes.object[0]];
+
+    if (currentAttributes) {
+      for (let attrName in attributes) {
+        timestamp = Math.max(timestamp, attributes[attrName][0]);
+        if (attrName in currentAttributes) {
+          if (attributes[attrName][0] > currentAttributes[attrName][0]) {
+            changeFlags |= CHANGE_FLAGS[attrName];
+            if (compareEquality(attributes[attrName][1], currentAttributes[attrName][1]))
+              changeFlags |= CHANGE_FLAGS[attrName] << 1;
           }
-          break;
+          else
+            delete attributes[attrName];
         }
+        else {
+          changeFlags |= CHANGE_FLAGS[attrName];
+          changeFlags |= CHANGE_FLAGS[attrName] << 1;
+        }
+      }
 
-        if (timestamps[attrName] > deviceData.timestamps[attrName].get(sub)) {
+      if (Object.keys(attributes).length) {
-          if (deviceData.values[attrName].has(sub)) {
+        if (attributes.object && attributes.object[1] === 0) {
-            let inv = toClear.get(sub);
+          toClear = toClear || [];
-            if (inv == null)
+          toClear.push([path.concat('*'), attributes.object[0]]);
-              toClear.set(sub, inv = {});
-            inv[attrName] = timestamps[attrName];
-            deviceData.values[attrName].delete(sub);
-          }
-          deviceData.timestamps[attrName].set(sub, timestamps[attrName]);
         }
+        for (let attrName in currentAttributes)
+          if (!(attrName in attributes))
+            attributes[attrName] = currentAttributes[attrName];
+
+        deviceData.attributes.set(path, attributes);
+      }
+
+      if (timestamp > currentTimestamp) {
+        deviceData.timestamps.set(path, timestamp);
+
+        if (path.length > 1)
+          toClear = set(deviceData, path.slice(0, path.length - 1), timestamp,
+            {object: [timestamp, 1]}, toClear);
       }
     }
-    else {
+    else if (!(timestamp <= currentTimestamp)) {
-      for (let attrName in timestamps) {
+      changeFlags |= 2;
-        if (attrName == 'exist') {
+      deviceData.timestamps.set(path, timestamp);
-          for (let an in deviceData.timestamps)
+      deviceData.attributes.set(path, attributes);
-            if (timestamps[attrName] > deviceData.timestamps[an].get(sub))
-              deviceData.timestamps[an].delete(sub);
-          continue;
-        }
 
-        if (timestamps[attrName] > deviceData.timestamps[attrName].get(sub))
+      if (path.length > 1)
-          deviceData.timestamps[attrName].delete(sub);
+        toClear = set(deviceData, path.slice(0, path.length - 1), timestamp,
-      }
+          {object: [timestamp, 1]}, toClear);
+
+      toClear = toClear || [];
+      toClear.push([path, timestamp]);
     }
   }
+  else if (!(timestamp <= currentTimestamp)) {
+    changeFlags |= 1;
+    deviceData.timestamps.set(path, timestamp);
+
+    if (currentAttributes) {
+      changeFlags |= 2;
+      deviceData.attributes.delete(path);
+      toClear = toClear || [];
+      toClear.push([path, timestamp]);
+    }
+
+    if (path.wildcard)
+      for (let sub of deviceData.paths.subset(path))
+        if (timestamp > deviceData.timestamps.get(sub)) {
+          deviceData.timestamps.delete(sub);
+          if (deviceData.attributes.delete(sub)) {
+            toClear = toClear || [];
+            toClear.push([sub, timestamp]);
+          }
+        }
+  }
+
+  if (changeFlags)
+    for (let sup of deviceData.paths.superset(path)) {
+      let c = deviceData.trackers.get(sup);
+      for (let k in c)
+        if (c[k] & changeFlags)
+          deviceData.changes.add(k);
+    }
+
   return toClear;
 }
 
 
-function clearSupersets(deviceData, path, timestamps) {
+function track(deviceData, path, marker, timestamp, attributeTimestamps, exist, attributeValues) {
-  let depth = 0;
+  path = deviceData.paths.add(path);
-  if (timestamps.exist != null)
+  let f = 0;
-    depth = 99;
 
-  for (let sup of deviceData.paths.superset(path, 99)) {
+  if (timestamp)
-    if (sup.wildcard == path.wildcard && sup.length == path.length)
+    f |= 1;
-      continue;
 
-    for (let attrName in deviceData.timestamps) {
+  if (exist)
-      let t = deviceData.timestamps[attrName].get(sup);
+    f |= 2;
-      if (t <= timestamps[attrName] || t <= timestamps.exist) {
+
-        deviceData.timestamps[attrName].delete(sup);
+  if (attributeTimestamps)
-        deviceData.values[attrName].delete(sup);
+    for (let attrName in attributeTimestamps)
-      }
+      f |= CHANGE_FLAGS[attrName];
-    }
+
+  if (attributeValues)
+    for (let attrName in attributeValues)
+      f |= CHANGE_FLAGS[attrName] << 1;
+
+  let cur = deviceData.trackers.get(path);
+  if (!cur) {
+    cur = {};
+    deviceData.trackers.set(path, cur);
   }
+
+  cur[marker] |= f;
 }
 
 
-function set(deviceData, path, timestamps, values, toClear) {
+function clearTrackers(deviceData, tracker) {
-  path = deviceData.paths.add(path);
+  if (Array.isArray(tracker)) {
-  if (path.wildcard || values == null)
+    deviceData.trackers.forEach(function(v, k) {
-    values = {};
+      for (let t of tracker)
-
+        delete v[t];
-  if (values.exist) {
+    });
-    normalizeAttributes(timestamps, values);
+    for (let t of tracker)
-    if (timestamps.exist && path.length > 1)
+      deviceData.changes.delete(t);
-      toClear = set(deviceData, path.slice(0, path.length - 1),
-        {exist: timestamps.exist, object: timestamps.exist},
-        {exist: 1, object: 1},
-        toClear);
-    else if (!toClear)
-      toClear = new Map();
-
-    // Clear attribute patterns
-    for (let attrName in timestamps) {
-      if (deviceData.values[attrName].has(path) != (attrName in values)) {
-        let inv = toClear.get(path);
-        if (inv == null)
-          toClear.set(path, inv = {});
-        inv[attrName] = timestamps[attrName];
-        if (!(attrName in values))
-          deviceData.values[attrName].delete(path);
-      }
-    }
-
-    for (let attrName in timestamps) {
-      deviceData.timestamps[attrName].set(path, timestamps[attrName]);
-      if (attrName in values)
-        deviceData.values[attrName].set(path, values[attrName]);
-    }
-
-    return toClear;
   }
-
+  else {
-  // Clear redundant timestamps
+    deviceData.trackers.forEach(function(v, k) {
-  let current = inferTimestamps(deviceData, path);
+      delete v[tracker];
-  for (let c in timestamps)
+    });
-    if (timestamps[c] <= current[c])
+    deviceData.changes.delete(tracker);
-      delete timestamps[c];
-    else if (c != 'exist' &&
-        (timestamps[c] <= timestamps.exist || timestamps[c] <= current.exist))
-      delete timetamps[c];
-
-  for (let attrName in timestamps) {
-    deviceData.timestamps[attrName].set(path, timestamps[attrName]);
-    if (attrName in values)
-      deviceData.values[attrName].set(path, values[attrName]);
   }
-
-  return clear(deviceData, path, timestamps, toClear);
 }
 
 
 exports.sanitizeParameterValue = sanitizeParameterValue;
 exports.getAliasDeclarations = getAliasDeclarations;
-exports.inferTimestamps = inferTimestamps;
 exports.unpack = unpack;
 exports.set = set;
 exports.clear = clear;
-exports.clearSupersets = clearSupersets
+exports.track = track;
+exports.clearTrackers = clearTrackers;

lib/sandbox.coffee

@@ -68,16 +68,26 @@ Object.defineProperty(sandbox.context, 'declare', {value: (decs) ->
 
   if sandbox.revision == sandbox.maxRevision + 1
     for k, v of decs
-      decT = {}
+      timestamp = 1
-      decV = {}
+      exist = null
+      attributeTimestamps = {}
+      attributeValues = {}
       for attrName, dec of v
-        if Array.isArray(dec)
+        if attrName is 'exist'
-          decT[attrName] = dec[0]
+          if Array.isArray(dec)
-          decV[attrName] = dec[1] if 1 of dec
+            timestamp = dec[0]
-        else
+            exist = dec[1] if 1 of dec
-          decT[attrName] = dec
+          else
+            timestamp = dec
+          continue
 
-      sandbox.declarations.push([common.parsePath(k), decT, decV])
+        if Array.isArray(dec)
+          attributeTimestamps[attrName] = dec[0]
+          attributeValues[attrName] = dec[1] if 1 of dec
+        else
+          attributeTimestamps[attrName] = dec
+
+      sandbox.declarations.push([common.parsePath(k), timestamp, attributeTimestamps, exist, attributeValues])
 
     throw new Exit()
 
@@ -93,9 +103,10 @@ Object.defineProperty(sandbox.context, 'declare', {value: (decs) ->
     unpacked = device.unpack(sandbox.deviceData, path, sandbox.revision)
     for param in unpacked
       r = {}
+      attrs = sandbox.deviceData.attributes.get(param, sandbox.revision)
       for attrName, dec of v
-        if (attrValue = sandbox.deviceData.values[attrName].get(param, sandbox.revision))?
+        if attrName of attrs
-          r[attrName] = [sandbox.deviceData.timestamps[attrName].get(param, sandbox.revision), attrValue]
+          r[attrName] = attrs[attrName]
 
       if single
         res[k] = r
@@ -107,6 +118,14 @@ Object.defineProperty(sandbox.context, 'declare', {value: (decs) ->
 })
 
 
+Object.defineProperty(sandbox.context, 'clear', {value: (pattern, timestamp) ->
+  if sandbox.revision == sandbox.maxRevision
+    sandbox.clear ?= []
+    sandbox.clear.push([common.parsePath(pattern), timestamp])
+  return
+})
+
+
 run = (script, globals, timestamp, deviceData, extensionsCache, startRevision, maxRevision) ->
   sandbox.timestamp = timestamp
   sandbox.deviceData = deviceData
@@ -115,6 +134,7 @@ run = (script, globals, timestamp, deviceData, extensionsCache, startRevision, m
   sandbox.maxRevision = maxRevision
   sandbox.declarations = []
   sandbox.extensions = null
+  sandbox.clear = null
 
   for k, v of globals
     sandbox.context[k] = v
@@ -122,11 +142,11 @@ run = (script, globals, timestamp, deviceData, extensionsCache, startRevision, m
   try
     ret = script.runInNewContext(sandbox.context, {displayErrors: false})
     delete sandbox.context[k] for k of sandbox.context
-    return {done: true, declarations: sandbox.declarations, returnValue: ret}
+    return {done: true, declarations: sandbox.declarations, extensions: sandbox.extensions, clear: sandbox.clear, returnValue: ret}
   catch err
     delete sandbox.context[k] for k of sandbox.context
     throw err if err not instanceof Exit
-    return {done: false, declarations: sandbox.declarations, extensions: sandbox.extensions}
+    return {done: false, declarations: sandbox.declarations, extensions: sandbox.extensions, clear: sandbox.clear}
 
 
 exports.run = run