Massive restructuring to use flat data model representation

Good luck reading this commit!
This commit is contained in:
Zaid Abdulla committed 2016-07-12 04:51:24 +03:00
1 parent 18e7e9e615
commit e5a21d2b40
7 files changed
+1138 -868

No files matched your search

+11 -1
View File
@@ -247,7 +247,7 @@ parsePath = (pattern, start, res) ->
while i < pattern.length and pattern[i] != ':' and pattern[i] != '.'
++ i
n = pattern.slice(j, i)
path.push(if n == '*' then null else n)
path.push(n)
if i >= pattern.length or pattern[i] == ':'
break
@@ -263,6 +263,15 @@ parsePath = (pattern, start, res) ->
return i
hammingWeight = (flags) ->
flags -= ((flags >> 1) & 0x55555555)
flags = (((flags >> 2) & 0x33333333) + (flags & 0x33333333))
flags = (((flags >> 4) + flags) & 0x0f0f0f0f)
flags += (flags >> 8)
flags += (flags >> 16)
return flags & 0x0000003f
exports.UNDEFINED_TYPE = UNDEFINED_TYPE
exports.NULL_TYPE = NULL_TYPE
exports.NUMBER_TYPE = NUMBER_TYPE
@@ -274,3 +283,4 @@ exports.REGEXP_TYPE = REGEXP_TYPE
exports.typeOf = typeOf
exports.pathOverlap = pathOverlap
exports.parsePath = parsePath
exports.hammingWeight = hammingWeight
+1 -1
View File
@@ -118,7 +118,7 @@ applyPresets = (currentRequest) ->
declarations = []
for k, v of parameters
declarations.push([v, 1, null, null, null, null, null, 1])
declarations.push([v, {exist: 1, value: 1}])
session.rpcRequest(currentRequest.sessionData, declarations, (err, id, rpcRequest) ->
throw err if err
+154 -100
View File
@@ -248,24 +248,28 @@ optimizeProjection = (obj) ->
fetchDevice = (id, timestamp, patterns, callback) ->
res = []
loaded = []
# Build projection
projection = {_id: 1}
for pattern in patterns
loaded.push([pattern, 1])
if pattern.length == 0
projection['_timestamp'] = 1
else if not pattern[0]?
projection[''] = 1
else if pattern[0] == 'Events'
if pattern.length == 1
res.push([['Events'], timestamp, 1, timestamp, 1, timestamp, 0])
res.push([['Events'],
{exist: timestamp, object: timestamp, writable: timestamp},
{exist: 1, object: 1, writable: 0}])
else if pattern.length == 2
if not pattern[1]?
projection['_registered'] = 1
projection['_lastInform'] = 1
projection['_lastBootstrap'] = 1
projection['_lastBoot'] = 1
res.push([['Events', null], timestamp])
res.push([['Events', '*'], {exist: timestamp}])
else if pattern[1] == 'Registered'
projection['_registered'] = 1
else if pattern[1] == 'Inform'
@@ -276,7 +280,10 @@ fetchDevice = (id, timestamp, patterns, callback) ->
projection['_lastBoot'] = 1
else if pattern[0] == 'DeviceID'
if pattern.length == 1
res.push([['DeviceID'], timestamp, 1, timestamp, 1, timestamp, 0])
res.push([['DeviceID'],
{exist: timestamp, object: timestamp, writable: timestamp},
{exist: 1, object: 1, writable: 0}])
else if pattern.length == 2
if not pattern[1]?
projection['_id'] = 1
@@ -284,7 +291,8 @@ fetchDevice = (id, timestamp, patterns, callback) ->
projection['_deviceId._ProductClass'] = 1
projection['_deviceId._SerialNumber'] = 1
projection['_deviceId._OUI'] = 1
res.push([['DeviceID', null], timestamp])
res.push([['DeviceID', '*'], {exist: timestamp}])
else if pattern[1] == 'ID'
projection['_id'] = 1
else if pattern[1] == 'Manufacturer'
@@ -297,15 +305,17 @@ fetchDevice = (id, timestamp, patterns, callback) ->
projection['_deviceId._SerialNumber'] = 1
else if pattern[0] == 'Tags'
if pattern.length == 1
res.push([['Tags'], timestamp, 1, timestamp, 1, timestamp, 0])
res.push([['Tags'],
{exist: timestamp, object: timestamp, writable: timestamp},
{exist: 1, object: 1, writable: 0}])
else if pattern.length == 2
res.push([['Tags', null], timestamp])
res.push([['Tags', '*'], {exist: timestamp}])
projection['_tags'] = 1
else
for i in [0...pattern.length] by 1
break if not pattern[i]
wildcardIndex = pattern.indexOf('*')
if i == pattern.length
if wildcardIndex == -1
s = pattern.join('.')
projection["#{s}._value"] = 1
projection["#{s}._timestamp"] = 1
@@ -320,8 +330,11 @@ fetchDevice = (id, timestamp, patterns, callback) ->
else
projection["#{pattern.slice(0, -1).join('.')}._timestamp"] = 1
else
s = pattern.slice(0, i).join('.')
p = pattern.slice(0, wildcardIndex)
loaded.push([p, 99])
s = p.join('.')
projection[s] = 1
projection[pattern.slice(0, wildcardIndex - 1).concat('_timestamp').join('.')] = 1
if projection['']
projection = {}
@@ -339,25 +352,32 @@ fetchDevice = (id, timestamp, patterns, callback) ->
if obj['_timestamp']?
obj['_timestamp'] = +obj['_timestamp']
v = [timestamp, 1]
t = {}
v = {}
if obj['_value']?
v[6] = obj['_timestamp'] ? timestamp
v[7] = [obj['_value'], obj['_type']]
t.value = obj['_timestamp'] ? timestamp
v.value = [obj['_value'], obj['_type']]
obj['_object'] = false
t.exist = Math.max(t.exist || 0, t.value)
v.exist = 1
if obj['_writable']?
v[4] = timestamp
v[5] = if obj['_writable'] then 1 else 0
t.writable = timestamp
v.writable = if obj['_writable'] then 1 else 0
t.exist = Math.max(t.exist || 0, t.writable)
v.exist = 1
if obj['_object']?
v[2] = timestamp
v[3] = if obj['_object'] then 1 else 0
t.object = obj['_timestamp'] ? timestamp
v.object = if obj['_object'] then 1 else 0
t.exist = Math.max(t.exist || 0, t.object)
v.exist = 1
if v.length
v[0] ?= timestamp ? 0
v[1] ?= 1
res.push([path].concat(v))
if not obj['_object']
loaded.push([path, 99])
res.push([path, t, v])
for k, v of obj
if k[0] != '_'
@@ -365,112 +385,146 @@ fetchDevice = (id, timestamp, patterns, callback) ->
storeParams(v, path.concat(k), obj['_timestamp'] ? 0, descendantsFetched)
if obj['_object'] and descendantsFetched
res.push([path.concat(null), obj['_timestamp'] ? 0])
res.push([path.concat('*'), {exist: obj['_timestamp'] ? 0}])
for k, v of device
if k == '_timestamp'
res.push([[null], v]) if not Object.keys(projection).length
res.push([['*'], {exist: v}]) if not Object.keys(projection).length
else if k == '_lastInform'
res.push([['Events', 'Inform'], timestamp, 1, timestamp, 0, timestamp, 0, timestamp, [v, 'xsd:dateTime']])
res.push([['Events', 'Inform'],
{exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
{exist: 1, object: 0, writable: 0, value: [+v, 'xsd:dateTime']}])
else if k == '_lastBoot'
res.push([['Events', '1_BOOT'], timestamp, 1, timestamp, 0, timestamp, 0, timestamp, [v, 'xsd:dateTime']])
res.push([['Events', '1_BOOT'],
{exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
{exist: 1, object: 0, writable: 0, value: [+v, 'xsd:dateTime']}])
else if k == '_lastBootstrap'
res.push([['Events', '0_BOOTSTRAP'], timestamp, 1, timestamp, 0, timestamp, 0, timestamp, [v, 'xsd:dateTime']])
res.push([['Events', '0_BOOTSTRAP'],
{exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
{exist: 1, object: 0, writable: 0, value: [+v, 'xsd:dateTime']}])
else if k == '_registered'
res.push([['Events', 'Registered'], timestamp, 1, timestamp, 0, timestamp, 0, timestamp, [v, 'xsd:dateTime']])
res.push([['Events', 'Registered'],
{exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
{exist: 1, object: 0, writable: 0, value: [+v, 'xsd:dateTime']}])
else if k == '_id'
res.push([['DeviceID', 'ID'], timestamp, 1, timestamp, 0, timestamp, 0, timestamp, [v, 'xsd:string']])
res.push([['DeviceID', 'ID'],
{exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
{exist: 1, object: 0, writable: 0, value: [v, 'xsd:string']}])
else if k == '_tags'
for t in v
res.push([['Tags', t], timestamp, 1, timestamp, 0, timestamp, 1, timestamp, [true, 'xsd:boolean']])
res.push([['Tags', t],
{exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
{exist: 1, object: 0, writable: 1, value: [true, 'xsd:boolean']}])
else if k == '_deviceId'
for kk, vv of v
if kk == '_Manufacturer'
res.push([['DeviceID', 'Manufacturer'], timestamp, 1, timestamp, 0, timestamp, 0, timestamp, [vv, 'xsd:string']])
res.push([['DeviceID', 'Manufacturer'],
{exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
{exist: 1, object: 0, writable: 0, value: [vv, 'xsd:boolean']}])
else if kk == '_OUI'
res.push([['DeviceID', 'OUI'], timestamp, 1, timestamp, 0, timestamp, 0, timestamp, [vv, 'xsd:string']])
res.push([['DeviceID', 'OUI'],
{exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
{exist: 1, object: 0, writable: 0, value: [vv, 'xsd:boolean']}])
if kk == '_ProductClass'
res.push([['DeviceID', 'ProductClass'], timestamp, 1, timestamp, 0, timestamp, 0, timestamp, [vv, 'xsd:string']])
res.push([['DeviceID', 'ProductClass'],
{exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
{exist: 1, object: 0, writable: 0, value: [vv, 'xsd:boolean']}])
if kk == '_SerialNumber'
res.push([['DeviceID', 'SerialNumber'], timestamp, 1, timestamp, 0, timestamp, 0, timestamp, [vv, 'xsd:string']])
res.push([['DeviceID', 'SerialNumber'],
{exist: timestamp, object: timestamp, writable: timestamp, value: timestamp},
{exist: 1, object: 0, writable: 0, value: [vv, 'xsd:boolean']}])
else if common.typeOf(v) is common.OBJECT_TYPE
storeParams(v, [k], device['_timestamp'] ? 0, projection == null)
storeParams(v, [k], +(device['_timestamp'] ? 0), projection == null)
return callback(null, res)
return callback(null, res, loaded)
)
saveDevice = (deviceId, diff, isNew, callback) ->
saveDevice = (deviceId, deviceData, isNew, callback) ->
update = {'$set' : {}, '$unset' : {}, '$addToSet' : {}, '$pull' : {}}
for p in diff
base = p[2]
current = p[3]
if p[0][0] == 'Events'
if p[0].length == 2
if p[0][1] == 'Inform'
update['$set']['_lastInform'] = new Date(current[7][0])
else if p[0][1] == '0_BOOTSTRAP'
update['$set']['_lastBootstrap'] = new Date(current[7][0])
else if p[0][1] == '1_BOOT'
update['$set']['_lastBoot'] = new Date(current[7][0])
else if p[0][1] == 'Registered'
update['$set']['_registered'] = new Date(current[7][0])
else if p[0][0] == 'Tags'
if p[0].length == 2 and p[0][1]?
if current[7][0] and not base[7]?[0]
# Delete parameters that no longer exist
iter = deviceData.values.exist.diff()
while not (diff = iter.next()).done
if diff.value[1] and not diff.value[2]
update['$unset'][diff.value[0].join('.')] = 1
# Set object's children timestamps
iter = deviceData.timestamps.exist.diff()
while not (diff = iter.next()).done
continue if diff.value[0].wildcards != (1 << (diff.value[0].length - 1))
continue if diff.value[0][0] in ['Events', 'DeviceID', 'Tags']
if not diff.value[2]? and diff.value[1]?
update['$unset'][diff.value[0].slice(0, -1).concat('_timestamp').join('.')] = 1
else if diff.value[2] != diff.value[1]
update['$set'][diff.value[0].slice(0, -1).concat('_timestamp').join('.')] = new Date(diff.value[2])
# Set writable
iter = deviceData.values.writable.diff()
while not (diff = iter.next()).done
continue if diff.value[0][0] in ['Events', 'DeviceID', 'Tags']
if not diff.value[2]? and diff.value[1]?
update['$unset'][diff.value[0].concat('_writable').join('.')] = 1
else if diff.value[2] != diff.value[1]
update['$set'][diff.value[0].concat('_writable').join('.')] = Boolean(diff.value[2])
# Set object
iter = deviceData.values.object.diff()
while not (diff = iter.next()).done
continue if diff.value[0][0] in ['Events', 'DeviceID', 'Tags']
if not diff.value[2] and diff.value[1]
update['$unset'][diff.value[0].concat('_object').join('.')] = 1
else if diff.value[2] and diff.value[2] != diff.value[1]
update['$set'][diff.value[0].concat('_object').join('.')] = true
# Set value
iter = deviceData.values.value.diff()
while not (diff = iter.next()).done
if diff.value[0][0] == 'Events'
continue if diff.value[1]?[0] == diff.value[2][0]
if diff.value[0][1] == 'Inform'
update['$set']['_lastInform'] = new Date(diff.value[2][0])
else if diff.value[0][1] == '1_BOOT'
update['$set']['_lastBoot'] = new Date(diff.value[2][0])
else if diff.value[0][1] == '0_BOOTSTRAP'
update['$set']['_lastBootstrap'] = new Date(diff.value[2][0])
else if diff.value[0][1] == 'Registered'
update['$set']['_registered'] = new Date(diff.value[2][0])
else if diff.value[0][0] == 'Tags'
continue if diff.value[1]?[0] == diff.value[2]?[0]
if diff.value[2][0]
update['$addToSet']['_tags'] ?= {'$each' : []}
update['$addToSet']['_tags']['$each'].push(p[0][1])
else if not current[7]?[0] and base[7]?[0]
else
update['$pull']['_tags'] ?= {'$in' : []}
update['$pull']['_tags']['$in'].push(p[0][1])
else if p[0][0] == 'DeviceID'
if p[0].length == 2 and current[7]?
if p[0][1] == 'ID'
update['$set']['_id'] = current[7][0]
else if p[0][1] == 'Manufacturer'
update['$set']['_deviceId._Manufacturer'] = current[7][0]
else if p[0][1] == 'OUI'
update['$set']['_deviceId._OUI'] = current[7][0]
else if p[0][1] == 'ProductClass'
update['$set']['_deviceId._ProductClass'] = current[7][0]
else if p[0][1] == 'SerialNumber'
update['$set']['_deviceId._SerialNumber'] = current[7][0]
else if p[1] == p[0].length
param = p[0].join('.')
if current[2]?
if current[3] != base[3]
if current[3]
update['$set']["#{param}._object"] = true
else if base[3]?
update['$unset']["#{param}._object"] = 1
else if base[2]?
update['$unset']["#{param}._object"] = 1
if current[4]?
if current[5] != base[5]
update['$set']["#{param}._writable"] = !!current[5]
else if base[4]?
update['$unset']["#{param}._writable"] = 1
if current[6]? and current[7]?
if current[6] != base[6]
update['$set']["#{param}._timestamp"] = new Date(current[6])
if current[7][0] != base[7]?[0] or current[7][1] != base[7]?[1]
if current[7][1] == 'xsd:dateTime'
update['$set']["#{param}._value"] = new Date(current[7][0])
else if diff.value[0][0] == 'DeviceID'
continue if diff.value[1]?[0] == diff.value[2][0]
if diff.value[0][1] == 'ID'
update['$set']['_id'] = diff.value[2][0]
else if diff.value[0][1] == 'Manufacturer'
update['$set']['_deviceId._Manufacturer'] = diff.value[2][0]
else if diff.value[0][1] == 'OUI'
update['$set']['_deviceId._OUI'] = diff.value[2][0]
else if diff.value[0][1] == 'ProductClass'
update['$set']['_deviceId._ProductClass'] = diff.value[2][0]
else if diff.value[0][1] == 'SerialNumber'
update['$set']['_deviceId._SerialNumber'] = diff.value[2][0]
else
update['$set']["#{param}._value"] = current[7][0]
update['$set']["#{param}._type"] = current[7][1]
else if base[7]?
update['$unset']["#{param}._value"] = 1
update['$unset']["#{param}._type"] = 1
else if p[1] == p[0].length - 1 and not p[0][p[1]]?
param = p[0].slice(0, p[1]).concat('_timestamp').join('.')
if current?
update['$set'][param] = new Date(current)
else if base?
update['$unset'][param] = 1
tdiff = deviceData.timestamps.value.getDiff(diff.value[0])
if diff.value[2]?
if tdiff[2] != tdiff[1]
update['$set'][diff.value[0].concat('_timestamp').join('.')] = new Date(tdiff[2])
delete update['$unset'][diff.value[0].concat('_timestamp').join('.')]
if diff.value[2][0] != diff.value[1]?[0]
update['$set'][diff.value[0].concat('_value').join('.')] = diff.value[2][0]
if diff.value[2][1] != diff.value[1]?[1]
update['$set'][diff.value[0].concat('_type').join('.')] = diff.value[2][1]
else if diff.value[1]?
update['$unset'][diff.value[0].concat('_value').join('.')] = 1
update['$unset'][diff.value[0].concat('_value').join('.')] = 1
# Remove empty keys
for k of update
-412
View File
@@ -1,412 +0,0 @@
###
# Copyright 2013-2016 Zaid Abdulla
#
# GenieACS is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# GenieACS is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with GenieACS. If not, see <http://www.gnu.org/licenses/>.
###
common = require './common'
sanitizeDeclaration = (declaration, wildcard, now) ->
if typeof declaration is 'number'
declaration = [declaration]
if declaration[7]?
declaration[7] = sanitizeParameterValue(declaration[7])
for i in [0...declaration.length] by 2 when declaration[i]?
if declaration[i] <= 0
declaration[i] = now + declaration[i]
else
declaration[i] = Math.min(now, declaration[i])
return declaration
sanitizeProperties = (properties, wildcard) ->
if not wildcard
if typeof properties is 'number'
properties = [properties]
else if properties[1]
properties[1] = 1
for i in [2...properties.length] by 2 \
when properties[i]? and properties[i + 1]?
properties[0] = Math.max(properties[0] ? 0, properties[i])
if properties[6]? and properties[7]? and not (properties[6] < properties[2])
properties[2] = properties[6]
properties[3] = 0
else if properties[2]? and properties[3] and not (properties[2] < properties[6])
properties[6] = properties[2]
properties[7] = null
else if typeof properties isnt 'number'
properties = properties[0]
return properties
sanitizeParameterValue = (parameterValue) ->
if parameterValue[0]?
switch parameterValue[1]
when 'xsd:boolean'
if typeof parameterValue[0] isnt 'boolean'
parameterValue = parameterValue.slice()
parameterValue[0] = !!JSON.parse(parameterValue[0])
when 'xsd:string'
if typeof parameterValue[0] isnt 'string'
parameterValue = parameterValue.slice()
parameterValue[0] = '' + parameterValue[0]
when 'xsd:int', 'xsd:unsignedInt'
if typeof parameterValue[0] isnt 'number'
parameterValue = parameterValue.slice()
parameterValue[0] = +parameterValue[0]
when 'xsd:dateTime'
if typeof parameterValue[0] isnt 'number'
parameterValue = parameterValue.slice()
if parameterValue[0].getTime?
parameterValue[0] = parameterValue[0].getTime()
else if isNaN(parameterValue[0])
parameterValue[0] = Date.parse(parameterValue[0])
else
parameterValue[0] = +parameterValue[0]
else
if parameterValue[1]?
throw new Error("Parameter value type \"#{parameterValue[1]}\" not recognized.")
# Ensure consistency after serialization
parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]))
return parameterValue
init = () ->
return [[[null, [0, 1]]]]
declare = (deviceData, path, declaration, now) ->
wildcard = false
for i in [0...path.length] by 1 when not path[i]?
wildcard = true
break
declaration = sanitizeDeclaration(declaration, wildcard, now)
ref = deviceData
depth = 0
for i in [0...path.length] by 1
if not (p = path[i])?
++ depth
continue
ref[1 + depth] ?= {}
ref[1 + depth][p] ?= []
ref = ref[1 + depth][p]
depth = 0
ref[0] ?= []
ref[0][depth] ?= []
if wildcard
ref[0][depth][0] = Math.max(ref[0][depth][0] ? 0, declaration[0])
else if not ref[0][depth][0]?
ref[0][depth][0] = declaration
else
for i in [0...declaration.length] by 2
if declaration[i]?
ref[0][depth][0][i] = Math.max(ref[0][depth][0][i] ? 0, declaration[i])
if declaration[i + 1]?
ref[0][depth][0][i + 1] = declaration[i + 1]
return
set = (deviceData, path, revision, properties) ->
++ revision # revisions start at layer 1
wildcard = false
for i in [0...path.length] by 1 when not path[i]?
wildcard = true
break
properties = sanitizeProperties(properties, wildcard)
ref = deviceData
depth = 0
for i in [0...path.length] by 1
if not (p = path[i])?
++ depth
continue
if not wildcard and properties[1]
# Update existing timestamp for ancestors
ref[0] ?= []
ref[0][0] ?= []
ref[0][0][revision] ?= []
if not (ref[0][0][revision][0] >= properties[0])
ref[0][0][revision][0] = properties[0]
ref[0][0][revision][1] = 1
if not (ref[0][0][revision][2] >= properties[0])
ref[0][0][revision][2] = properties[0]
ref[0][0][revision][3] = 1
ref[1 + depth] ?= {}
ref[1 + depth][p] ?= []
ref = ref[1 + depth][p]
depth = 0
ref[0] ?= []
ref[0][depth] ?= []
if wildcard
ref[0][depth][revision] = Math.max(ref[0][depth][revision] ? 0, properties)
else if not ref[0][0][revision]?
ref[0][0][revision] = properties
else
for i in [0...properties.length] by 2 \
when properties[i]?# and properties[i + 1]?
if not (properties[i] < ref[0][0][revision][i])
ref[0][0][revision][i] = properties[i]
ref[0][0][revision][i + 1] = properties[i + 1]
if properties[7]?
ref[0][0][revision][7] = sanitizeParameterValue(ref[0][0][revision][7])
return
traverse = (deviceData, pattern, revision, callback) ->
if not pattern?
pattern = []
pattern.length = 99
if not revision?
revision = 99
++ revision # First layer is for declarations
collectTimestamps = (ref, path, root, timestamps, wildcard) ->
# Skip properties of parameter we're currently at
skip = if root == path.length then 1 else 0
if ref[0]?
for i in [skip...ref[0].length] by 1
if ref[0][i]? and i <= pattern.length - path.length
p = path.slice()
p.length += i
if wildcard or i > 0
revisions = ref[0][i]
else
# Extract only timestamp from properties array
revisions = (j?[0] for j in ref[0][i])
current = -1
for j in [Math.min(revision, revisions.length - 1)..1] by -1 when revisions[j]?
current = revisions[j]; break
timestamps.push([p, root, revisions, current, current])
for i in [skip...ref.length - 1] by 1
if ref[1 + i]? and i < pattern.length - path.length
p = path.slice()
p.length += i
for k, v of ref[1+i]
if not pattern[path.length + i]? or pattern[path.length + i] == k
collectTimestamps(v, p.concat(k), root, timestamps, wildcard or i > 0)
return
processOverlaps = (timestamps, root, skip) ->
skip = 1 if not skip
for i in [skip...timestamps.length] by 1
for j in [0...i] by 1
if timestamps[i][0].length > timestamps[j][0].length
t = timestamps[j]
tt = timestamps[i]
else
t = timestamps[i]
tt = timestamps[j]
overlap = common.pathOverlap(t[0], tt[0], root)
if overlap & 1
if t[0].length == tt[0].length
tt[3 + (root - tt[1]) * 2] = Math.max(tt[3 + (root - tt[1]) * 2], t[3 + (root - t[1]) * 2])
else
tt[4 + (root - tt[1]) * 2] = Math.max(tt[4 + (root - tt[1]) * 2], t[4 + (root - t[1]) * 2])
if overlap & 2
t[3 + (root - t[1]) * 2] = Math.max(t[3 + (root - t[1]) * 2], tt[3 + (root - t[1]) * 2])
return
recursive = (ref, path, timestamps, timestampThreshold, revisionThreshold) ->
declaration = ref[0]?[0]?[0]?.slice() or []
base = ref[0]?[0]?[1]?.slice() or []
current = []
# Find timestamp and revision in which this parameter was most recently
# created in order to ignore older properties
for t in timestamps
for j in [Math.min(revision, t[2].length - 1)..1] by -1 when t[2][j]?
if not (t[2][j] <= ref[0]?[0]?[j]?[0]) and t[2][j] >= timestampThreshold
timestampThreshold = Math.max(timestampThreshold, t[2][j])
revisionThreshold = Math.max(revisionThreshold, j)
current[0] = timestampThreshold
break
# Collect current properties
if ref[0]?[0]?
for i in [revisionThreshold...Math.min(revision + 1, ref[0][0].length)] by 1 when (rev = ref[0][0][i])?
for j in [0...rev.length] by 2 when rev[j]?
if rev[j] >= timestampThreshold
current[j] = rev[j]
current[j + 1] = rev[j + 1]
else
current[j] ?= 0
# Return if parameter no longer exists
if not (declaration[1] or base[1] or current[1])
l = timestamps.length
collectTimestamps(ref, path, path.length - 1, timestamps)
processOverlaps(timestamps, path.length - 1, l)
return
tentativeTimestamp = -1
if current[2]? and not current[3]
tentativeTimestamp = current[2]
filteredTimestamps = timestamps.filter((t) ->
if t[0][path.length - 1]?
if t[0][path.length - 1]? and t[0][path.length - 1] == path[path.length - 1]
t[2 + (path.length - t[1]) * 2] = Math.max(t[2 + (path.length - t[1]) * 2], current[0])
else
return false
if t[0].length > path.length
t[3 + (path.length - t[1]) * 2] = t[1 + (path.length - t[1]) * 2]
t[4 + (path.length - t[1]) * 2] = tentativeTimestamp
return true
if not declaration[0] or t[2][0] > declaration[0]
declaration[0] = t[2][0]
return false
)
collectTimestamps(ref, path, path.length, filteredTimestamps)
processOverlaps(filteredTimestamps, path.length, 0)
if path.length < pattern.length
children = {}
for k, v of ref[1]
if not pattern[path.length]? or k == pattern[path.length]
r = recursive(v, path.concat(k), filteredTimestamps, timestampThreshold, revisionThreshold)
children[k] = r if r?
descendantTimestamps = filteredTimestamps.map((t) ->
res = [t[0], t[1]]
res[2] = t[2][0] if t[2][0]?
res[3] = t[2][1] if t[2][1]?
cur = Math.max(t[3 + (path.length - t[1]) * 2], t[4 + (path.length - t[1]) * 2])
if path.length > t[1]
t[2 + (path.length - t[1]) * 2] = Math.min(t[2 + (path.length - t[1]) * 2], cur)
if cur >= 0
res[4] = cur
return res
).filter((t) -> t.length > 1)
# Match value type of current if no type is specified
if declaration[7]? and not declaration[7][1]?
declaration[7] = sanitizeParameterValue([declaration[7][0], current[7]?[1]])
return callback(path, declaration, base, current, descendantTimestamps, children)
return recursive(deviceData, [], [], 0, 1)
collapse = (deviceData, revision, wildcard) ->
ref = deviceData
if ref[0]?
for depth, i in ref[0] when depth?.length > revision + 2
if i == 0 and not wildcard
depth[revision + 1] ?= []
for j in [revision + 2...depth.length] by 1 when depth[j]?
for k in [0...depth[j].length] by 2 when depth[j][k]?
depth[revision + 1][k] = depth[j][k]
depth[revision + 1][k + 1] = depth[j][k + 1]
else
for j in [revision + 2...depth.length] by 1 when depth[j]?
depth[revision + 1] = depth[j]
depth.length = revision + 2
for i in [1...ref.length] by 1 when ref[i]?
for k, v of ref[i]
collapse(v, revision, wildcard or i != 1)
clearDeclarations = (deviceData, prefix) ->
ref = deviceData
if prefix?
for p in prefix
ref = ref[1]?[p]
return if not ref?
if ref[0]?
for depth in ref[0] when depth?
depth[0] = null
for i in [1...ref.length] by 1 when ref[i]?
for k, v of ref[i]
clearDeclarations(v)
getPrerequisiteDeclarations = (declarations) ->
dec = [declarations[0], 1]
for i in [1...declarations.length] by 2
if declarations[i]?
dec[i] = declarations[i]
else if declarations[i + 1]?
dec[i] = 1
return [dec]
getAll = (deviceData, pattern, revision) ->
allParameters = []
traverse(deviceData, pattern, revision, (path, declaration, base, current, descendantTimestamps, children) ->
if path.length == pattern.length and current[1]
allParameters.push([path].concat(current))
)
return allParameters
exports.getPrerequisiteDeclarations = getPrerequisiteDeclarations
exports.init = init
exports.getAll = getAll
exports.declare = declare
exports.set = set
exports.traverse = traverse
exports.collapse = collapse
exports.clearDeclarations = clearDeclarations
exports.sanitizeParameterValue = sanitizeParameterValue
+277
View File
@@ -0,0 +1,277 @@
/**
* Copyright 2013-2016 Zaid Abdulla
*
* GenieACS is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* GenieACS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with GenieACS. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
function sanitizeParameterValue(parameterValue) {
if (parameterValue[0] != null) {
switch (parameterValue[1]) {
case 'xsd:boolean':
if (typeof parameterValue[0] !== 'boolean') {
parameterValue = parameterValue.slice();
parameterValue[0] = !!JSON.parse(parameterValue[0]);
}
break;
case 'xsd:string':
if (typeof parameterValue[0] !== 'string') {
parameterValue = parameterValue.slice();
parameterValue[0] = '' + parameterValue[0];
}
break;
case 'xsd:int':
case 'xsd:unsignedInt':
if (typeof parameterValue[0] !== 'number') {
parameterValue = parameterValue.slice();
parameterValue[0] = +parameterValue[0];
}
break;
case 'xsd:dateTime':
if (typeof parameterValue[0] !== 'number') {
parameterValue = parameterValue.slice();
if (parameterValue[0].getTime() != null)
parameterValue[0] = parameterValue[0].getTime();
else if (isNaN(parameterValue[0]))
parameterValue[0] = Date.parse(parameterValue[0]);
else
parameterValue[0] = +parameterValue[0];
}
break;
default:
if (parameterValue[1] != null)
throw new Error(`Parameter value type "${parameterValue[1]}" not recognized.`);
parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
}
}
return parameterValue;
}
function getAliasDeclarations(path) {
// TODO Return declarations necessary for unpacking aliases
return [];
}
function inferTimestamps(deviceData, path) {
var res = {};
for (let sup of deviceData.paths.superset(path)) {
for (let attrName in deviceData.timestamps) {
let t = deviceData.timestamps[attrName].get(sup);
if (t && !(t < res[sup]))
res[attrName] = t;
}
}
for (let i = path.length - 1; i > 0; -- i) {
if (path.wildcards & ((2 << i) - 1) === 0)
break;
let param = null;
let ancestor = path.slice(0, i);
let ancestorTimestamp = 0;
for (let sup of deviceData.paths.superset(ancestor)) {
if (!sup.wildcards)
param = sup;
let t = deviceData.timestamps.exist.get(sup);
if (t > ancestorTimestamp)
ancestorTimestamp = t;
}
if (param != null && deviceData.values.exist.get(param)) {
if (deviceData.values.object.get(param) == 0) {
let t = deviceData.timestamps.object.get(param);
for (let attrName in deviceData.timestamps)
if (!(t <= res[attrName]))
res[attrName] = t;
}
}
else if (ancestorTimestamp) {
let newt = {};
for (let attrName in deviceData.timestamps)
newt[attrName] = ancestorTimestamp;
for (let sub of deviceData.paths.subset(ancestor)) {
if (sub.wildcards)
continue;
let subt = inferTimestamps(deviceData, sub.concat(path.slice(sub.length, path.length)));
for (let attrName in newt) {
if (!(attrName in subt))
delete newt[attrName];
else if (subt[attrName] < newt[attrName])
newt[attrName] = subt[attrName];
}
}
for (let attrName in newt)
if (!(newt[attrName] <= res[attrName]))
res[attrName] = newt[attrName];
}
}
return res;
}
function unpack(deviceData, path, revision) {
// TODO implement alias unpacking
return [path];
}
function invalidateSuperset(deviceData, path, attributes) {
throw new Error('Not implemented');
}
function normalizeAttributes(timestamps, values) {
// TODO Consider moving implemention within set function
if (!values)
return;
if ('value' in values && timestamps.value >= (timestamps.object || 0)) {
values.object = 0;
timestamps.object = timestamps.value;
}
if (values.object && timestamps.object > (timestamps.value || 0)) {
delete values.value;
timestamps.value = timestamps.object;
}
if ('value' in values && timestamps.value >= (timestamps.exist || 0)) {
values.exist = 1;
timestamps.exist = timestamps.value;
}
if ('object' in values && timestamps.object >= (timestamps.exist || 0)) {
values.exist = 1;
timestamps.exist = timestamps.object;
}
if ('writable' in values && timestamps.writable >= (timestamps.exist || 0)) {
values.exist = 1;
timestamps.exist = timestamps.writable;
}
}
function set(deviceData, path, timestamps, values) {
// TODO Optimize
// TODO Allow overwriting attribute value with same timestamp
path = deviceData.paths.add(path);
if (!path.wildcards) {
normalizeAttributes(timestamps, values);
if (values.exist && path.length > 1)
set(deviceData, path.slice(0, path.length - 1),
{exist: timestamps.exist, object: timestamps.exist},
{exist: 1, object: 1});
}
var current = inferTimestamps(deviceData, path);
for (let c in current)
if (current[c] > timestamps[c] || (current[c] == timestamps[c] && path.wildcards > 0))//!deviceData.values[c].has(path)))
delete timestamps[c];
if (timestamps.exist != null) {
deviceData.timestamps.exist.set(path, timestamps.exist);
if (values != null && values.exist != null)
deviceData.values.exist.set(path, values.exist);
}
current = inferTimestamps(deviceData, path);
for (let c in current)
if (current[c] > timestamps[c] || (current[c] == timestamps[c] && path.wildcards > 0))//!deviceData.values[c].has(path)))
delete timestamps[c];
let toInvalidate = new Map();
for (let sub of deviceData.paths.subset(path, 0)) {
if (path.wildcards == sub.wildcards)
continue;
for (let attrName in timestamps) {
if (deviceData.timestamps[attrName].get(sub) <= timestamps[attrName]) {
if (sub.wildcards || !deviceData.values[attrName].has(sub))
deviceData.timestamps[attrName].delete(sub);
else if (deviceData.timestamps[attrName].get(sub) < timestamps[attrName]) {
deviceData.timestamps[attrName].delete(sub);
deviceData.values[attrName].delete(sub);
let inv = toInvalidate.get(sub) || {};
if (!timestamps[attrName] <= inv[attrName]) {
inv[attrName] = timestamps[attrName];
toInvalidate.set(sub, inv);
}
}
}
}
}
for (let attrName in timestamps) {
let existing = deviceData.timestamps[attrName].get(path);
if (existing == null || timestamps[attrName] >= existing) {
deviceData.timestamps[attrName].set(path, timestamps[attrName]);
let invalidate = false;
if (values[attrName] != null) {
if (!deviceData.values[attrName].has(path))
invalidate = true;
deviceData.values[attrName].set(path, values[attrName]);
}
else {
if (deviceData.values[attrName].has(path))
invalidate = true;
deviceData.values[attrName].delete(path)
}
if (invalidate) {
let inv = toInvalidate.get(path) || {};
if (!timestamps[attrName] <= inv[attrName]) {
inv[attrName] = timestamps[attrName];
toInvalidate.set(sub, inv);
}
}
}
}
toInvalidate.forEach(function(p, a) {
invalidateSuperset(deviceData, p, a);
});
// TODO recalculate descendant superset
// var depth = 0;
// if (timestamps.exist != null)
// depth = 99;
//
// for (let sup of deviceData.paths.superset(path, depth)) {
// if ((sup.wildcards && (3 << path.length) - 1) != path.wildcards) {
// throw new Error('Not implemented');
// }
// }
}
exports.sanitizeParameterValue = sanitizeParameterValue;
exports.getAliasDeclarations = getAliasDeclarations;
exports.inferTimestamps = inferTimestamps;
exports.unpack = unpack;
exports.set = set;
+36 -44
View File
@@ -26,6 +26,7 @@ class Exit;
sandbox = {
timestamp: null
deviceData: null
args: null
revision: null
@@ -36,6 +37,16 @@ sandbox = {
context: vm.createContext()
}
sandbox.context.Date = class
constructor: (arg) ->
if arguments.length
return new (Function.prototype.bind.apply(Date, arguments))
return new Date(sandbox.timestamp)
now: () ->
return sandbox.timestamp
sandbox.context.ext = () ->
if sandbox.extensions?.length
@@ -56,23 +67,35 @@ sandbox.context.declare = (decs) ->
if sandbox.revision == sandbox.maxRevision + 1
for k, v of decs
sandbox.declarations.push([common.parsePath(k)].concat(toIndexDeclaration(v)))
decT = {}
decV = {}
for attrName, dec of v
if Array.isArray(dec)
decT[attrName] = dec[0]
decV[attrName] = dec[1] if 1 in dec
else
decT[attrName] = dec
sandbox.declarations.push([common.parsePath(k), decT, decV])
throw new Exit()
res = {}
for k, v of decs
path = common.parsePath(k)
all = device.getAll(sandbox.deviceData, path, sandbox.revision)
res[k] = {}
for a in all
r = {}
r.exist = a[1] if v.exist?
r.type = a.slice(3, 5) if v.type?
r.writable = a.slice(5, 7) if v.writable?
r.value = a.slice(7, 9) if v.value? and a[8]?
p = a[0].join('.')
res[k] = {}
unpacked = device.unpack(sandbox.deviceData, path, sandbox.revision)
for path in unpacked
iter = sandbox.deviceData.paths.subset(path)
while (param = iter.next().value)?
if param.wildcards == 0 and sandbox.deviceData.values.exist.get(param, sandbox.revision)
r = {}
for attrName, dec of v
if (attrValue = sandbox.deviceData.values[attrName].get(param, sandbox.revision))?
r[attrName] = [sandbox.deviceData.timestamps[attrName].get(param, sandbox.revision), attrValue]
p = param.join('.')
if p == k
res[k] = r
else
@@ -83,39 +106,8 @@ sandbox.context.declare = (decs) ->
Object.freeze(sandbox.context)
# Convert from friendly declaration format
toIndexDeclaration = (keyDeclaraiton) ->
indexDeclaraiton = []
if keyDeclaraiton['exist']?
if Array.isArray(keyDeclaraiton['exist'])
indexDeclaraiton[0] = keyDeclaraiton['exist'][0]
else
indexDeclaraiton[0] = keyDeclaraiton['exist']
if keyDeclaraiton['type']?
if Array.isArray(keyDeclaraiton['type'])
indexDeclaraiton[2] = keyDeclaraiton['type'][0]
else
indexDeclaraiton[2] = keyDeclaraiton['type']
if keyDeclaraiton['writable']?
if Array.isArray(keyDeclaraiton['writable'])
indexDeclaraiton[4] = keyDeclaraiton['writable'][0]
else
indexDeclaraiton[4] = keyDeclaraiton['writable']
if keyDeclaraiton['value']?
if Array.isArray(keyDeclaraiton['value'])
indexDeclaraiton[6] = keyDeclaraiton['value'][0]
indexDeclaraiton[7] = keyDeclaraiton['value'][1]
else
indexDeclaraiton[6] = keyDeclaraiton['value']
return indexDeclaraiton
run = (script, args, deviceData, extensionsCache, startRevision, maxRevision) ->
run = (script, args, timestamp, deviceData, extensionsCache, startRevision, maxRevision) ->
sandbox.timestamp = timestamp
sandbox.deviceData = deviceData
sandbox.extensionsCache = extensionsCache
sandbox.args = args
@@ -125,7 +117,7 @@ run = (script, args, deviceData, extensionsCache, startRevision, maxRevision) ->
sandbox.extensions = null
try
ret = script.runInNewContext(sandbox.context)
ret = script.runInNewContext(sandbox.context, {displayErrors: false})
return {done: true, declarations: sandbox.declarations, returnValue: ret}
catch err
throw err if err not instanceof Exit
+659 -310
View File
File diff suppressed because it is too large. Load diff