Remove obsolete aliasing feature

This commit is contained in:
Zaid Abdulla committed 2016-09-15 20:52:28 -07:00
1 parent b9e17c9add
commit 036368172c
7 files changed
+63 -566

No files matched your search

-27
View File
@@ -1,27 +0,0 @@
{
"_lastInform" : {"alias" : "summary.lastInform", "type" : "date"},
"_lastBoot" : {"alias" : "summary.lastBoot", "type" : "date"},
"_lastBootstrap" : {"alias" : "summary.lastBootstrap", "type" : "date"},
"_deviceId._SerialNumber" : {"alias" : "summary.serialNumber"},
"_deviceId._ProductClass" : {"alias" : "summary.productClass"},
"_deviceId._OUI" : {"alias" : "summary.oui"},
"_deviceId._Manufacturer" : {"alias" : "summary.manufacturer"},
"InternetGatewayDevice.DeviceInfo.HardwareVersion" : {"alias" : "summary.hardwareVersion"},
"InternetGatewayDevice.DeviceInfo.SoftwareVersion" : {"alias" : "summary.softwareVersion"},
"Device.DeviceInfo.HardwareVersion" : {"alias" : "summary.hardwareVersion"},
"Device.DeviceInfo.SoftwareVersion" : {"alias" : "summary.softwareVersion"},
"InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
"InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
"InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
"InternetGatewayDevice.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
"Device.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
"Device.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANIPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
"Device.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.MACAddress" : {"alias" : "summary.mac", "type" : "mac"},
"Device.WANDevice./^\\d+$/.WANConnectionDevice./^\\d+$/.WANPPPConnection./^\\d+$/.ExternalIPAddress" : {"alias" : "summary.ip"},
"InternetGatewayDevice.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.SSID" : {"alias" : "summary.wlanSsid"},
"InternetGatewayDevice.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.PreSharedKey.1.KeyPassphrase" : {"alias" : "summary.wlanPassphrase"},
"Device.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.SSID" : {"alias" : "summary.wlanSsid"},
"Device.LANDevice./^\\d+$/.WLANConfiguration./^\\d+$/.PreSharedKey.1.KeyPassphrase" : {"alias" : "summary.wlanPassphrase"}
}
+6 -34
View File
@@ -147,17 +147,7 @@ watchTask = (taskId, timeout, callback) ->
, 500)
expandParam = (param, aliases) ->
params = [param]
for a,aa of aliases
if a == param or common.startsWith(a, "#{param}.")
for p in aa
params.push(p) if p[p.lastIndexOf('.') + 1] != '_'
return params
sanitizeTask = (task, aliases, callback) ->
sanitizeTask = (task, callback) ->
task.timestamp = new Date(task.timestamp ? Date.now())
if task.expiry?
if common.typeOf(task.expiry) is common.DATE_TYPE or isNaN(task.expiry)
@@ -169,8 +159,7 @@ sanitizeTask = (task, aliases, callback) ->
when 'getParameterValues'
projection = {}
for p in task.parameterNames
for pp in expandParam(p, aliases)
projection[pp] = 1
projection[p] = 1
db.devicesCollection.findOne({_id : task.device}, projection, (err, device) ->
parameterNames = []
for k of projection
@@ -183,9 +172,8 @@ sanitizeTask = (task, aliases, callback) ->
projection = {}
values = {}
for p in task.parameterValues
for pp in expandParam(p[0], aliases)
projection[pp] = 1
values[pp] = p[1]
projection[p[0]] = 1
values[p[0]] = p[1]
db.devicesCollection.findOne({_id : task.device}, projection, (err, device) ->
parameterValues = []
for k of projection
@@ -200,21 +188,7 @@ sanitizeTask = (task, aliases, callback) ->
callback(task)
addAliases = (device, aliases) ->
for k,v of aliases
for p in v
pp = p.split('.')
obj = device
for i in pp
if not obj[i]?
obj = null
break
obj = obj[i]
device[k] = obj if obj?
insertTasks = (tasks, aliases, callback) ->
insertTasks = (tasks, callback) ->
if tasks? and common.typeOf(tasks) isnt common.ARRAY_TYPE
tasks = [tasks]
else if not tasks? or tasks.length == 0
@@ -223,7 +197,7 @@ insertTasks = (tasks, aliases, callback) ->
counter = tasks.length
for task in tasks
sanitizeTask(task, aliases, (t) ->
sanitizeTask(task, (t) ->
if t.uniqueKey?
db.tasksCollection.remove({device : t.device, uniqueKey : t.uniqueKey}, (err) ->
)
@@ -250,8 +224,6 @@ deleteDevice = (deviceId, callback) ->
)
exports.addAliases = addAliases
exports.sanitizeTask = sanitizeTask
exports.connectionRequest = connectionRequest
exports.watchTask = watchTask
exports.insertTasks = insertTasks
-117
View File
@@ -109,121 +109,6 @@ disconnect = () ->
redisClient.quit()
getCached = (name, valueCallback, valueExpiry, callback) ->
redisClient.get(name, (err, res) ->
return callback(err) if err
return callback(null, JSON.parse(res)) if res?
lockAcquired = () ->
valueCallback((err, value) ->
return callback(err, value) if err
redisClient.setex(name, valueExpiry, JSON.stringify(value), (err) ->
return callback(err, value) if err
redisClient.del("lock.#{name}", (err) ->
callback(err, value)
)
)
)
redisClient.setnx("lock.#{name}", Date.now() + 30000, (err, res) ->
return callback(err) if err
if res == 1
lockAcquired()
else
setTimeout(() ->
redisClient.get("lock.#{name}", (err, timestamp) ->
return callback(err) if err
now = Date.now()
if not timestamp?
getCached(name, valueCallback, valueExpiry, callback)
else if timestamp > now
setTimeout(() ->
getCached(name, valueCallback, valueExpiry, callback)
, 1000)
else
redisClient.getset("lock.#{name}", now + 30000, (err, timestamp) ->
return callback(err) if err
if timestamp > now
setTimeout(() ->
getCached(name, valueCallback, valueExpiry, callback)
, 1000)
else
lockAcquired()
)
)
, 1000)
)
)
getAliases = (callback) ->
getCached('aliases', parameters.compileAliases, config.get('PRESETS_CACHE_DURATION'), (err, res) ->
throw err if err
callback(res)
)
getPresetsObjectsAliases = (callback) ->
redisClient.mget('presets', 'objects', 'aliases', (err, res) ->
presets = JSON.parse(res[0])
objects = JSON.parse(res[1])
aliases = JSON.parse(res[2])
if presets and objects and aliases
return callback(presets, objects, aliases)
if not presets
getCached('presets', (callback) ->
presetsCollection.find().toArray((err, res) ->
for r in res
r.precondition = JSON.parse(r.precondition)
# Generate provisions from the old configuration format
r.provisions = []
for c in r.configurations
switch c.type
when 'age'
r.provisions.push(['refresh', c.name, c.age * -1000])
when 'value'
r.provisions.push(['value', c.name, c.value])
when 'add_tag'
r.provisions.push(['tag', c.tag, true])
when 'delete_tag'
r.provisions.push(['tag', c.tag, false])
else
throw new Error("Unknown configuration type #{c.type}")
callback(err, res)
)
, config.get('PRESETS_CACHE_DURATION'), (err, res) ->
throw err if err
presets = res
callback(presets, objects, aliases) if objects and aliases
)
if not objects
getCached('objects', (callback) ->
objectsCollection.find().toArray((err, res) ->
return callback(err) if err
objs = {}
objs[r._id] = r for r in res
callback(null, objs)
)
, config.get('PRESETS_CACHE_DURATION'), (err, res) ->
throw err if err
objects = res
callback(presets, objects, aliases) if presets and aliases
)
if not aliases
getCached('aliases', parameters.compileAliases, config.get('PRESETS_CACHE_DURATION'), (err, res) ->
throw err if err
aliases = res
callback(presets, objects, aliases) if presets and objects
)
)
# Optimize projection by removing overlaps
# This modifies given object and returns it
optimizeProjection = (obj) ->
@@ -714,8 +599,6 @@ syncFaults = (deviceId, faults, callback) ->
)
exports.getPresetsObjectsAliases = getPresetsObjectsAliases
exports.getAliases = getAliases
exports.connect = connect
exports.disconnect = disconnect
exports.fetchDevice = fetchDevice
+2 -23
View File
@@ -276,8 +276,7 @@ listener = (request, response) ->
if body.length
task = JSON.parse(body)
task.device = deviceId
db.getAliases((aliases) ->
apiFunctions.insertTasks(task, aliases, (err) ->
apiFunctions.insertTasks(task, (err) ->
throw err if err
db.redisClient.del("#{deviceId}_no_tasks", (err) ->
throw err if err
@@ -317,7 +316,6 @@ listener = (request, response) ->
response.end(JSON.stringify(task))
)
)
)
else if urlParts.query.connection_request?
# no task, send connection request only
apiFunctions.connectionRequest(deviceId, (err) ->
@@ -453,7 +451,6 @@ listener = (request, response) ->
response.end('404 Not Found')
return
func = (aliases) ->
if urlParts.query.query?
try
q = JSON.parse(urlParts.query.query)
@@ -466,7 +463,7 @@ listener = (request, response) ->
switch collectionName
when 'devices'
q = query.expand(q, aliases)
q = query.expand(q)
when 'tasks'
q = query.sanitizeQueryTypes(q, {
_id: ((v) -> return new mongodb.ObjectID(v))
@@ -484,23 +481,12 @@ listener = (request, response) ->
for p in urlParts.query.projection.split(',')
p = p.trim()
projection[p] = 1
if collectionName is 'devices'
for k,v of aliases
if k == p or common.startsWith(k, "#{p}.")
projection[a] = 1 for a in v
cur = collection.find(q, projection, {batchSize : 50})
if urlParts.query.sort?
s = JSON.parse(urlParts.query.sort)
sort = {}
for k,v of s
if aliases[k]?
for a in aliases[k]
if a[a.lastIndexOf('.')+1] != '_' and collectionName is 'devices'
sort["#{a}._value"] = v
else
sort[a] = v
else
if k[k.lastIndexOf('.') + 1] != '_' and collectionName is 'devices'
sort["#{k}._value"] = v
else
@@ -521,7 +507,6 @@ listener = (request, response) ->
if item?
response.write(",\n") if i++
apiFunctions.addAliases(item, aliases) if collectionName is 'devices'
response.write(JSON.stringify(item))
if not item? or (limit? and i >= limit)
@@ -529,12 +514,6 @@ listener = (request, response) ->
)
)
if collectionName is 'devices'
db.getAliases((aliases) ->
func(aliases)
)
else
func({})
else
response.writeHead 404
response.end('404 Not Found')
-132
View File
@@ -1,132 +0,0 @@
###
# Copyright 2013 Fanoos Telecom
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
###
# Credit to Netanel Rubin from Check Point Software Technologies for discovering
# vulnerability caused by use of eval to generate regex from user input.
common = require './common'
parameters = require './parameters'
stringToRegexp = (input, flags) ->
if (input.indexOf('*') == -1)
return false
output = input.replace(/[\[\]\\\^\$\.\|\?\+\(\)]/, "\\$&")
if output[0] == '*'
output = output.replace(/^\*+/g, '')
else
output = '^' + output
if output[output.length - 1] == '*'
output = output.replace(/\*+$/g, '')
else
output = output + '$'
output = output.replace(/[\*]/, '.*')
return new RegExp(output, flags)
normalizers = {}
normalizers.default = (input, normType) ->
if normType is 'query' and common.typeOf(input) is common.STRING_TYPE
vals = []
str = normalizers.string(input, normType)
if common.typeOf(str) == common.ARRAY_TYPE
vals = vals.concat(str)
else
vals.push(str)
f = parseFloat(input)
if not isNaN(f)
vals.push(f)
d = new Date(input)
if input.length >= 8 and d.getFullYear() > 1983
vals.push(d)
r = stringToRegexp(input)
if r isnt false
vals.push({'$regex' : r})
return vals
return input
FLOAT_REGEX = /^([\-\+]?[0-9\.]+)/
normalizers.float = (input, normType) ->
res = FLOAT_REGEX.exec(input)
if res != null
parseFloat(res[1])
else
null
normalizers.date = (input, normType) ->
return new Date(input)
normalizers.string = (input, normType) ->
if normType is 'query'
if (m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input))
return [{'$regex' : new RegExp(m[1], m[2])}, input]
return input
colonizeMac = (input) ->
if (i = input.indexOf(':')) != -1
i = if (i % 2) == 0 then 0 else 1
while i < input.length
if input[i-1] != ':' and input[i-2] != ':'
if input[i] != ':' and i != 0
input = input.substring(0, i) + ':' + input.substring(i)
++i
++i
++ i
return input
else
return input.replace(/[0-9A-F]{1}(?!$)/g, '$&:?')
normalizers.mac = (input, normType) ->
if normType isnt 'query'
return input
input = input.trim().toUpperCase().replace(/[^0-9A-F\*\-\:]/g, '').replace(/[\-\:]/g, ':')
if input.indexOf('*') != -1
input = (colonizeMac(a) for a in input.split('*')).join('*')
return stringToRegexp(input, 'i')
input = colonizeMac(input)
if input.length == 17
return [input, input.toLowerCase()]
return {'$regex' : new RegExp(input, 'i')}
exports.normalize = (path, value, normType) ->
type = parameters.getType(path) ? 'default'
normalizers[type](value, normType)
-212
View File
@@ -1,212 +0,0 @@
###
# Copyright 2013, 2014 Zaid Abdulla
#
# GenieACS is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# GenieACS is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with GenieACS. If not, see <http://www.gnu.org/licenses/>.
###
config = require './config'
db = require './db'
PATH_REGEX = /\w+|(\/.+?\/\w*)/g
# Javascript represents numbers as 64-bit floating point numbers
# Integers up to 2^53 can be encoded precisely
MAX_PARAMETERS = 52
# get the index of the least significant bit
getLsb = (num) ->
i = -1
while num > 0
++ i
break if num & 1
num >>= 1
return i
createParameterAttributeFinder = (attributeName) ->
# compile attributes into data types optimized for quick lookup
attributes = []
stringSegments = {}
regexSegments = {}
paramRegex = {}
for k, v of config.PARAMETERS
continue if not v[attributeName]?
attributeIndex = attributes.push(v[attributeName]) - 1
throw new Error("There cannot be more than #{MAX_PARAMETERS} configured parameters") if attributeIndex >= MAX_PARAMETERS
segmentIndex = 0
parts = k.match(PATH_REGEX)
for i in [0 ... parts.length]
part = parts[i]
positionHash = i + parts.length * MAX_PARAMETERS
if part[0] == '/' # this is a regex part
j = part.lastIndexOf('/')
regExp = new RegExp(part[1...j], part[j+1..])
regexSegments[positionHash] ?= 0
regexSegments[positionHash] |= Math.pow(2, attributeIndex)
paramRegex[attributeIndex] ?= {}
paramRegex[attributeIndex][i] = regExp
else
stringSegments[positionHash] ?= {}
stringSegments[positionHash][part] ?= 0
stringSegments[positionHash][part] |= Math.pow(2, attributeIndex)
# Finder function
return (param) ->
parts = param.split('.')
positionHash = parts.length * MAX_PARAMETERS
stringIndices = 0 | stringSegments[positionHash]?[parts[0]]
allIndices = stringIndices | regexSegments[positionHash]
for i in [1...parts.length]
positionHash = i + parts.length * MAX_PARAMETERS
strIdx = stringSegments[positionHash]?[parts[i]]
rgxIdx = regexSegments[positionHash]
stringIndices &= strIdx
allIndices &= (strIdx | rgxIdx)
if stringIndices > 0
return attributes[getLsb(stringIndices)]
attributeIndex = 0
while allIndices > 0
if allIndices & 1
match = true
for i, r of paramRegex[attributeIndex]
if not r.test(parts[i])
match = false
break
if match
return attributes[attributeIndex]
++ attributeIndex
allIndices >>= 1
return null
splitIds = (batches, callback) ->
db.devicesCollection.count((err, count) ->
throw err if err
return callback(null) if count == 0
ids = [{}]
return callback(ids) if count <= 1000 or batches == 1
batchSize = Math.floor(count / batches)
for i in [1 ... batches]
ids[i] = {}
do (i) ->
cursor = db.devicesCollection.find({}, {_id:1}).sort({'_id':1}).skip(i * batchSize).limit(1)
cursor.nextObject((err, obj) ->
ids[i]['$gte'] = obj._id
ids[i-1]['$lt'] = obj._id
if i == batches - 1
callback(ids)
)
)
compileAliases = (callback) ->
map = () ->
recurse = (partIndex, ref) ->
part = parts[partIndex]
res = []
if part.test?
for p of ref
if part.test(p)
if partIndex == parts.length - 1
res.push(p)
else
for r in recurse(partIndex + 1, ref[p])
res.push("#{p}.#{r}")
else
if ref[part]?
if partIndex == parts.length - 1
return [part]
else
for r in recurse(partIndex + 1, ref[part])
res.push("#{part}.#{r}")
return res
for key, parts of keys
for alias in recurse(0, this)
emit(alias, key)
return
reduce = (k, v) ->
return v[0]
aliases = {}
keys = {}
for k, v of config.PARAMETERS
alias = v.alias
continue if not alias?
isStatic = true
ar = []
for p in k.match(/\w+|(\/.+?\/\w*)/g)
if p[0] == '/' # this is a regex
isStatic = false
i = p.lastIndexOf('/')
r = new RegExp(p[1...i], p[i+1..])
ar.push(r)
else
ar.push(p)
if isStatic
aliases[alias] ?= []
aliases[alias].push(k)
else
keys[k] = ar
counter = 0
splitIds(4, (batches) ->
if not batches?
return callback(null, aliases)
for batch in batches
options = {
out : {inline : 1}
jsMode : true
sort : {$natural : 1}
scope : {keys : keys, flags : {}}
query : {_id : batch} if batches.length > 1
}
db.devicesCollection.mapReduce(map, reduce, options, (err, out) ->
if err
callback?(err)
return callback = null
for o in out
alias = config.PARAMETERS[o.value].alias
aliases[alias] ?= []
id = String(o._id)
if id not in aliases[alias]
aliases[alias].push(id)
if ++counter == batches.length
callback?(null, aliases)
)
return
)
exports.getType = createParameterAttributeFinder('type')
exports.getAlias = createParameterAttributeFinder('alias')
exports.compileAliases = compileAliases
+55 -21
View File
@@ -39,18 +39,59 @@
###
common = require './common'
normalize = require './normalize'
mongodb = require 'mongodb'
expandValue = (param, value) ->
stringToRegexp = (input, flags) ->
if (input.indexOf('*') == -1)
return false
output = input.replace(/[\[\]\\\^\$\.\|\?\+\(\)]/, "\\$&")
if output[0] == '*'
output = output.replace(/^\*+/g, '')
else
output = '^' + output
if output[output.length - 1] == '*'
output = output.replace(/\*+$/g, '')
else
output = output + '$'
output = output.replace(/[\*]/, '.*')
return new RegExp(output, flags)
normalize = (input) ->
if common.typeOf(input) is common.STRING_TYPE
vals = [input]
if (m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input))
vals.push({'$regex' : new RegExp(m[1], m[2])})
f = parseFloat(input)
if not isNaN(f)
vals.push(f)
d = new Date(input)
if input.length >= 8 and d.getFullYear() > 1983
vals.push(d)
r = stringToRegexp(input)
if r isnt false
vals.push({'$regex' : r})
return vals
return input
expandValue = (value) ->
if common.typeOf(value) is common.ARRAY_TYPE
a = []
for j in value
a = a.concat(expandValue(param, j))
a = a.concat(expandValue(j))
return [a]
else if common.typeOf(value) isnt common.OBJECT_TYPE
n = normalize.normalize(param, value, 'query')
n = normalize(value)
if common.typeOf(n) isnt common.ARRAY_TYPE
return [n]
else
@@ -62,7 +103,7 @@ expandValue = (param, value) ->
values = []
for k,v of value
keys.push(k)
values.push(expandValue(param, v))
values.push(expandValue(v))
indices.push(0)
i = 0
@@ -80,38 +121,31 @@ expandValue = (param, value) ->
return objs
permute = (param, val, aliases) ->
keys = []
if aliases[param]?
for p in aliases[param]
keys.push(p)
else
keys.push(param)
permute = (param, val) ->
conditions = []
for k in keys
values = expandValue(k, val)
if k[k.lastIndexOf('.') + 1] != '_'
k += '._value'
values = expandValue(val)
if param[param.lastIndexOf('.') + 1] != '_'
param += '._value'
for v in values
obj = {}
obj[k] = v
obj[param] = v
conditions.push(obj)
return conditions
expand = (query, aliases) ->
expand = (query) ->
new_query = {}
for k,v of query
if k[0] == '$' # operator
expressions = []
for e in v
expressions.push(expand(e, aliases))
expressions.push(expand(e))
new_query[k] = expressions
else
conditions = permute(k, v, aliases)
conditions = permute(k, v)
if conditions.length > 1
new_query['$and'] ?= []
if v?['$ne']?