Update MongoDB connector

This commit is contained in:
Zaid Abdulla committed 2018-12-02 02:54:54 -08:00
1 parent ea8273556f
commit dfc8928a71
6 files changed
+168 -179

No files matched your search

+22 -12
View File
@@ -196,7 +196,8 @@ function connectionRequest(deviceId, callback) {
"CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
deviceId
);
const proj = {
const options = {
projection: {
"Device.ManagementServer.ConnectionRequestURL._value": 1,
"Device.ManagementServer.UDPConnectionRequestAddress._value": 1,
"Device.ManagementServer.ConnectionRequestUsername._value": 1,
@@ -205,9 +206,10 @@ function connectionRequest(deviceId, callback) {
"InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress._value": 1,
"InternetGatewayDevice.ManagementServer.ConnectionRequestUsername._value": 1,
"InternetGatewayDevice.ManagementServer.ConnectionRequestPassword._value": 1
}
};
db.devicesCollection.findOne({ _id: deviceId }, proj, (err, device) => {
db.devicesCollection.findOne({ _id: deviceId }, options, (err, device) => {
if (err) return void callback(err);
if (!device) return void callback(new Error("No such device"));
@@ -284,13 +286,19 @@ function connectionRequest(deviceId, callback) {
function watchTask(deviceId, taskId, timeout, callback) {
setTimeout(() => {
db.tasksCollection.findOne({ _id: taskId }, { _id: 1 }, (err, task) => {
db.tasksCollection.findOne(
{ _id: taskId },
{ projection: { _id: 1 } },
(err, task) => {
if (err) return void callback(err);
if (!task) return void callback(null, "completed");
const q = { _id: `${deviceId}:task_${taskId}` };
db.faultsCollection.findOne(q, { _id: 1 }, (err, fault) => {
db.faultsCollection.findOne(
q,
{ projection: { _id: 1 } },
(err, fault) => {
if (err) return void callback(err);
if (fault) return void callback(null, "fault");
@@ -298,8 +306,10 @@ function watchTask(deviceId, taskId, timeout, callback) {
if ((timeout -= 500) <= 0) return void callback(null, "timeout");
watchTask(deviceId, taskId, timeout, callback);
});
});
}
);
}
);
}, 500);
}
@@ -323,23 +333,23 @@ function insertTasks(tasks, callback) {
for (const task of tasks) {
sanitizeTask(task, t => {
if (t.uniqueKey) {
db.tasksCollection.remove(
db.tasksCollection.deleteOne(
{ device: t.device, uniqueKey: t.uniqueKey },
() => {}
);
}
if (--counter === 0) db.tasksCollection.insert(tasks, callback);
if (--counter === 0) db.tasksCollection.insertMany(tasks, callback);
});
}
}
function deleteDevice(deviceId, callback) {
db.tasksCollection.remove({ device: deviceId }, err => {
db.tasksCollection.deleteMany({ device: deviceId }, err => {
if (err) return void callback(err);
db.devicesCollection.remove({ _id: deviceId }, err => {
db.devicesCollection.deleteOne({ _id: deviceId }, err => {
if (err) return void callback(err);
db.faultsCollection.remove(
db.faultsCollection.deleteMany(
{
_id: {
$regex: `^${common.escapeRegExp(deviceId)}\\:`
@@ -347,7 +357,7 @@ function deleteDevice(deviceId, callback) {
},
err => {
if (err) return void callback(err);
db.operationsCollection.remove(
db.operationsCollection.deleteMany(
{
_id: {
$regex: `^${common.escapeRegExp(deviceId)}\\:`
+21 -21
View File
@@ -23,7 +23,7 @@ const config = require("./config");
const MAX_CACHE_TTL = config.get("MAX_CACHE_TTL");
let redisClient;
let mongoDb;
let mongoClient;
let mongoCollection;
let mongoTimeOffset = 0;
@@ -49,15 +49,17 @@ function connect(callback) {
mongodb.MongoClient.connect(
MONGODB_CONNECTION_URL,
(err, db) => {
{ useNewUrlParser: true },
(err, client) => {
if (err) return void callback(err);
mongoDb = db;
const db = client.db();
mongoClient = client;
mongoCollection = db.collection("cache");
mongoCollection.ensureIndex({ expire: 1 }, { expireAfterSeconds: 0 });
mongoCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
const now = Date.now();
mongoDb.command({ hostInfo: 1 }, (err, res) => {
db.command({ hostInfo: 1 }, (err, res) => {
if (err) return void callback(err);
mongoTimeOffset = res.system.currentTime.getTime() - now;
exports.get = mongoGet;
@@ -74,8 +76,7 @@ function connect(callback) {
function disconnect() {
if (redisClient) redisClient.quit();
if (mongoDb) mongoDb.close();
if (mongoClient) mongoClient.close();
}
function redisGet(key, callback) {
@@ -171,8 +172,8 @@ function mongoGet(key, callback) {
function mongoDel(key, callback) {
if (Array.isArray(key))
mongoCollection.remove({ _id: { $in: key } }, callback);
else mongoCollection.remove({ _id: key }, callback);
mongoCollection.deleteMany({ _id: { $in: key } }, callback);
else mongoCollection.deleteOne({ _id: key }, callback);
}
function mongoSet(key, value, ttl, callback) {
@@ -184,24 +185,23 @@ function mongoSet(key, value, ttl, callback) {
if (!ttl) ttl = MAX_CACHE_TTL;
const expire = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
mongoCollection.save({ _id: key, value: value, expire: expire }, callback);
mongoCollection.replaceOne(
{ _id: key },
{ _id: key, value: value, expire: expire },
{ upsert: true },
callback
);
}
function mongoPop(key, callback) {
mongoCollection.findAndModify(
{ _id: key },
null,
null,
{ remove: true },
(err, res) => {
mongoCollection.findOneAndDelete({ _id: key }, (err, res) => {
if (err || !res["value"]) return void callback(err);
if (res["value"]["expire"] > new Date(Date.now() - mongoTimeOffset))
return void callback(null, res["value"]["value"]);
callback();
}
);
});
}
function mongoLock(lockName, ttl, callback) {
@@ -211,13 +211,13 @@ function mongoLock(lockName, ttl, callback) {
function unlockOrExtend(extendTtl) {
if (!extendTtl) {
mongoCollection.remove({ _id: lockName, value: token }, (err, res) => {
mongoCollection.deleteOne({ _id: lockName, value: token }, (err, res) => {
if (err || res["result"]["n"] !== 1)
throw err || new Error("Lock expired");
});
} else {
const expire = new Date(Date.now() - mongoTimeOffset + extendTtl * 1000);
mongoCollection.update(
mongoCollection.updateOne(
{ _id: lockName, value: token },
{ expire: expire },
(err, res) => {
@@ -231,7 +231,7 @@ function mongoLock(lockName, ttl, callback) {
const expireTest = new Date(Date.now() - mongoTimeOffset);
const expireSet = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
mongoCollection.update(
mongoCollection.updateOne(
{ _id: lockName, expire: { $lte: expireTest } },
{ $set: { value: token, expire: expireSet } },
{ upsert: true },
+56 -89
View File
@@ -33,85 +33,39 @@ let tasksCollection,
operationsCollection;
function connect(callback) {
let callbackCounter = 9;
mongodb.MongoClient.connect(
config.get("MONGODB_CONNECTION_URL"),
(err, db) => {
{ useNewUrlParser: true },
(err, client) => {
if (err) return void callback(err);
exports.client = client;
const db = client.db();
exports.mongoDb = db;
db.collection("tasks", (err, collection) => {
exports.tasksCollection = tasksCollection = collection;
if (collection != null)
collection.ensureIndex({ device: 1, timestamp: 1 });
exports.tasksCollection = tasksCollection = db.collection("tasks");
tasksCollection.createIndex({ device: 1, timestamp: 1 });
if (--callbackCounter === 0 || err) {
callbackCounter = 0;
callback(err);
}
});
db.collection("devices", (err, collection) => {
exports.devicesCollection = devicesCollection = collection;
if (--callbackCounter === 0 || err) {
callbackCounter = 0;
callback(err);
}
});
db.collection("presets", (err, collection) => {
exports.presetsCollection = presetsCollection = collection;
if (--callbackCounter === 0 || err) {
callbackCounter = 0;
callback(err);
}
});
db.collection("objects", (err, collection) => {
exports.objectsCollection = objectsCollection = collection;
if (--callbackCounter === 0 || err) {
callbackCounter = 0;
callback(err);
}
});
db.collection("fs.files", (err, collection) => {
exports.filesCollection = filesCollection = collection;
if (--callbackCounter === 0 || err) {
callbackCounter = 0;
callback(err);
}
});
db.collection("provisions", (err, collection) => {
exports.provisionsCollection = provisionsCollection = collection;
if (--callbackCounter === 0 || err) {
callbackCounter = 0;
callback(err);
}
});
db.collection("virtualParameters", (err, collection) => {
exports.virtualParametersCollection = virtualParametersCollection = collection;
if (--callbackCounter === 0 || err) {
callbackCounter = 0;
callback(err);
}
});
db.collection("faults", (err, collection) => {
exports.faultsCollection = faultsCollection = collection;
if (--callbackCounter === 0 || err) {
callbackCounter = 0;
callback(err);
}
});
db.collection("operations", (err, collection) => {
exports.operationsCollection = operationsCollection = collection;
if (--callbackCounter === 0 || err) {
callbackCounter = 0;
callback(err);
}
});
exports.devicesCollection = devicesCollection = db.collection("devices");
exports.presetsCollection = presetsCollection = db.collection("presets");
exports.objectsCollection = objectsCollection = db.collection("objects");
exports.filesCollection = filesCollection = db.collection("fs.files");
exports.provisionsCollection = provisionsCollection = db.collection(
"provisions"
);
exports.virtualParametersCollection = virtualParametersCollection = db.collection(
"virtualParameters"
);
exports.faultsCollection = faultsCollection = db.collection("faults");
exports.operationsCollection = operationsCollection = db.collection(
"operations"
);
callback();
}
);
}
function disconnect() {
if (exports.mongoDb) exports.mongoDb.close();
if (exports.client) exports.client.close();
}
// Optimize projection by removing overlaps
@@ -279,12 +233,13 @@ function fetchDevice(id, timestamp, patterns, callback) {
}
}
let proj;
if (projection[""]) proj = {};
else if (!Object.keys(projection).length) proj = { _id: 1 };
else proj = projection;
const options = {};
if (!projection[""]) {
if (!Object.keys(projection).length) options.projection = { _id: 1 };
else options.projection = projection;
}
devicesCollection.findOne({ _id: id }, proj, (err, device) => {
devicesCollection.findOne({ _id: id }, options, (err, device) => {
if (err || !device) return void callback(err);
function storeParams(obj, path, ts, descendantsFetched, projTree) {
@@ -358,12 +313,14 @@ function fetchDevice(id, timestamp, patterns, callback) {
}
if (!descendantsFetched) {
if (projTree) {
for (const k of Object.keys(projTree)) {
const p = path.concat(k);
loaded.push([p, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)]);
if ((obj["_object"] || !path.length) && obj["_timestamp"])
res.push([p, obj["_timestamp"]]);
}
}
} else if ((obj["_object"] || !path.length) && obj["_timestamp"]) {
res.push([path.concat("*"), obj["_timestamp"]]);
}
@@ -740,26 +697,26 @@ function saveDevice(deviceId, deviceData, isNew, sessionTimestamp, callback) {
delete update["$pull"];
}
devicesCollection.update(
devicesCollection.updateOne(
{ _id: deviceId },
update,
{ upsert: isNew },
(err, result) => {
if (!err && result.result.n !== 1) {
if (err || result.result.n !== 1) {
return void callback(
new Error(`Device ${deviceId} not found in database`)
err || new Error(`Device ${deviceId} not found in database`)
);
}
if (update2) {
return void devicesCollection.update(
return void devicesCollection.updateOne(
{ _id: deviceId },
update2,
callback
);
}
callback(err);
callback();
}
);
}
@@ -786,17 +743,18 @@ function getFaults(deviceId, callback) {
}
function saveFault(deviceId, channel, fault, callback) {
const id = `${deviceId}:${channel}`;
fault = Object.assign({}, fault);
fault._id = `${deviceId}:${channel}`;
fault._id = id;
fault.device = deviceId;
fault.channel = channel;
fault.timestamp = new Date(fault.timestamp);
fault.provisions = JSON.stringify(fault.provisions);
faultsCollection.save(fault, callback);
faultsCollection.replaceOne({ _id: id }, fault, { upsert: true }, callback);
}
function deleteFault(deviceId, channel, callback) {
faultsCollection.remove({ _id: `${deviceId}:${channel}` }, callback);
faultsCollection.deleteOne({ _id: `${deviceId}:${channel}` }, callback);
}
function getDueTasks(deviceId, timestamp, callback) {
@@ -804,7 +762,7 @@ function getDueTasks(deviceId, timestamp, callback) {
const tasks = [];
let f;
cur.nextObject(
cur.next(
(f = (err, task) => {
if (err) return void callback(err);
@@ -835,17 +793,17 @@ function getDueTasks(deviceId, timestamp, callback) {
task.fileName = res[0].filename || res[0]._id.toString();
}
cur.nextObject(f);
cur.next(f);
});
} else {
cur.nextObject(f);
cur.next(f);
}
})
);
}
function clearTasks(deviceId, taskIds, callback) {
tasksCollection.remove(
tasksCollection.deleteMany(
{ _id: { $in: taskIds.map(id => new mongodb.ObjectID(id)) } },
callback
);
@@ -873,17 +831,26 @@ function getOperations(deviceId, callback) {
}
function saveOperation(deviceId, commandKey, operation, callback) {
const id = `${deviceId}:${commandKey}`;
operation = Object.assign({}, operation);
operation._id = `${deviceId}:${commandKey}`;
operation._id = id;
operation.timestamp = new Date(operation.timestamp);
operation.provisions = JSON.stringify(operation.provisions);
operation.retries = JSON.stringify(operation.retries);
operation.args = JSON.stringify(operation.args);
operationsCollection.save(operation, callback);
operationsCollection.replaceOne(
{ _id: id },
operation,
{ upsert: true },
callback
);
}
function deleteOperation(deviceId, commandKey, callback) {
operationsCollection.remove({ _id: `${deviceId}:${commandKey}` }, callback);
operationsCollection.deleteOne(
{ _id: `${deviceId}:${commandKey}` },
callback
);
}
function getPresets(callback) {
+14 -6
View File
@@ -27,19 +27,27 @@ function listener(request, response) {
const urlParts = url.parse(request.url, true);
if (request.method === "GET") {
const filename = querystring.unescape(urlParts.pathname.substring(1));
const gs = new mongodb.GridStore(db.mongoDb, filename, "r", {});
gs.open(err => {
db.filesCollection.findOne({ _id: filename }, (err, file) => {
if (err) {
response.writeHead(500, { Connection: "close" });
response.end(`${err.name}: ${err.message}`);
throw err;
}
if (!file) {
response.writeHead(404);
response.end();
return;
}
const stream = gs.stream(true);
response.writeHead(200, {
"Content-Type": "application/octet-stream",
"Content-Length": gs.length
"Content-Type": file.contentType || "application/octet-stream",
"Content-Length": file.length
});
stream.pipe(response);
const bucket = new mongodb.GridFSBucket(db.client.db());
const downloadStream = bucket.openDownloadStream(filename);
downloadStream.pipe(response);
});
} else {
response.writeHead(405, { Allow: "GET" });
+54 -50
View File
@@ -83,7 +83,11 @@ function listener(request, response) {
if (request.method === "PUT") {
const preset = JSON.parse(body);
preset._id = presetName;
db.presetsCollection.save(preset, err => {
db.presetsCollection.replaceOne(
{ _id: presetName },
preset,
{ upsert: true },
err => {
if (err) return void throwError(err, response);
cache.del("presets_hash", err => {
@@ -92,9 +96,10 @@ function listener(request, response) {
response.writeHead(200);
response.end();
});
});
}
);
} else if (request.method === "DELETE") {
db.presetsCollection.remove({ _id: presetName }, err => {
db.presetsCollection.deleteOne({ _id: presetName }, err => {
if (err) return void throwError(err, response);
cache.del("presets_hash", err => {
@@ -115,7 +120,11 @@ function listener(request, response) {
if (request.method === "PUT") {
const object = JSON.parse(body);
object._id = objectName;
db.objectsCollection.save(object, err => {
db.objectsCollection.replaceOne(
{ _id: objectName },
object,
{ upsert: true },
err => {
if (err) return void throwError(err, response);
cache.del("presets_hash", err => {
@@ -124,9 +133,10 @@ function listener(request, response) {
response.writeHead(200);
response.end();
});
});
}
);
} else if (request.method === "DELETE") {
db.objectsCollection.remove({ _id: objectName }, err => {
db.objectsCollection.deleteOne({ _id: objectName }, err => {
if (err) return void throwError(err, response);
cache.del("presets_hash", err => {
@@ -158,7 +168,11 @@ function listener(request, response) {
return;
}
db.provisionsCollection.save(object, err => {
db.provisionsCollection.replaceOne(
{ _id: provisionName },
object,
{ upsert: true },
err => {
if (err) return void throwError(err, response);
cache.del("presets_hash", err => {
@@ -167,9 +181,10 @@ function listener(request, response) {
response.writeHead(200);
response.end();
});
});
}
);
} else if (request.method === "DELETE") {
db.provisionsCollection.remove({ _id: provisionName }, err => {
db.provisionsCollection.deleteOne({ _id: provisionName }, err => {
if (err) return void throwError(err, response);
cache.del("presets_hash", err => {
@@ -201,7 +216,11 @@ function listener(request, response) {
return;
}
db.virtualParametersCollection.save(object, err => {
db.virtualParametersCollection.replaceOne(
{ _id: virtualParameterName },
object,
{ upsert: true },
err => {
if (err) return void throwError(err, response);
cache.del("presets_hash", err => {
@@ -210,9 +229,10 @@ function listener(request, response) {
response.writeHead(200);
response.end();
});
});
}
);
} else if (request.method === "DELETE") {
db.virtualParametersCollection.remove(
db.virtualParametersCollection.deleteOne(
{ _id: virtualParameterName },
err => {
if (err) return void throwError(err, response);
@@ -234,7 +254,7 @@ function listener(request, response) {
const deviceId = querystring.unescape(r[1]);
const tag = querystring.unescape(r[2]);
if (request.method === "POST") {
db.devicesCollection.update(
db.devicesCollection.updateOne(
{ _id: deviceId },
{ $addToSet: { _tags: tag } },
{ safe: true },
@@ -245,7 +265,7 @@ function listener(request, response) {
}
);
} else if (request.method === "DELETE") {
db.devicesCollection.update(
db.devicesCollection.updateOne(
{ _id: deviceId },
{ $pull: { _tags: tag } },
{ safe: true },
@@ -267,12 +287,12 @@ function listener(request, response) {
);
const deviceId = faultId.split(":", 1)[0];
const channel = faultId.slice(deviceId.length + 1);
db.faultsCollection.remove({ _id: faultId }, err => {
db.faultsCollection.deleteOne({ _id: faultId }, err => {
if (err) return void throwError(err, response);
if (channel.startsWith("task_")) {
const objId = new mongodb.ObjectID(channel.slice(5));
return void db.tasksCollection.remove({ _id: objId }, err => {
return void db.tasksCollection.deleteOne({ _id: objId }, err => {
if (err) return void throwError(err, response);
cache.del(`${deviceId}_tasks_faults_operations`, err => {
@@ -396,7 +416,7 @@ function listener(request, response) {
if (request.method === "DELETE") {
db.tasksCollection.findOne(
{ _id: new mongodb.ObjectID(taskId) },
{ device: 1 },
{ projection: { device: 1 } },
(err, task) => {
if (err) return void throwError(err, response);
@@ -407,12 +427,12 @@ function listener(request, response) {
}
const deviceId = task.device;
db.tasksCollection.remove(
db.tasksCollection.deleteOne(
{ _id: new mongodb.ObjectID(taskId) },
err => {
if (err) return void throwError(err, response);
db.faultsCollection.remove(
db.faultsCollection.deleteOne(
{ _id: `${deviceId}:task_${taskId}` },
err => {
if (err) return void throwError(err, response);
@@ -437,12 +457,12 @@ function listener(request, response) {
if (request.method === "POST") {
db.tasksCollection.findOne(
{ _id: new mongodb.ObjectID(taskId) },
{ device: 1 },
{ projection: { device: 1 } },
(err, task) => {
if (err) return void throwError(err, response);
const deviceId = task.device;
db.faultsCollection.remove(
db.faultsCollection.deleteOne(
{ _id: `${deviceId}:task_${taskId}` },
err => {
if (err) return void throwError(err, response);
@@ -476,25 +496,19 @@ function listener(request, response) {
productClass: request.headers.productclass,
version: request.headers.version
};
const gs = new mongodb.GridStore(db.mongoDb, filename, filename, "w", {
const bucket = new mongodb.GridFSBucket(db.client.db());
const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
metadata: metadata
});
gs.open(err => {
if (err) return void throwError(err, response);
gs.write(body, err => {
if (err) return void throwError(err, response);
gs.close(err => {
uploadStream.end(body, err => {
if (err) return void throwError(err, response);
response.writeHead(201);
response.end();
});
});
});
} else if (request.method === "DELETE") {
mongodb.GridStore.unlink(db.mongoDb, filename, err => {
const bucket = new mongodb.GridFSBucket(db.client.db());
bucket.delete(filename, err => {
if (err) return void throwError(err, response);
response.writeHead(200);
@@ -597,7 +611,7 @@ function listener(request, response) {
projection[p.trim()] = 1;
}
const cur = collection.find(q, projection, { batchSize: 50 });
const cur = collection.find(q, { projection: projection });
if (urlParts.query.sort) {
const s = JSON.parse(urlParts.query.sort);
@@ -613,9 +627,7 @@ function listener(request, response) {
if (urlParts.query.skip) cur.skip(parseInt(urlParts.query.skip));
let limit;
if (urlParts.query.limit)
cur.limit((limit = parseInt(urlParts.query.limit)));
if (urlParts.query.limit) cur.limit(parseInt(urlParts.query.limit));
cur.count(false, (err, total) => {
if (err) return void throwError(err);
@@ -632,24 +644,16 @@ function listener(request, response) {
response.write("[\n");
i = 0;
cur.each((err, item) => {
if (err) {
throwError(err);
return false;
}
if (item) {
cur.forEach(
item => {
if (i++) response.write(",\n");
response.write(JSON.stringify(item));
}
if (!item || i >= limit) {
},
err => {
if (err) return void throwError(err);
response.end("\n]");
return false;
}
return null;
});
);
});
} else {
response.writeHead(404);
+1 -1
View File
@@ -21,7 +21,7 @@
"dependencies": {
"later": "^1.2.0",
"libxmljs": "^0.18.7",
"mongodb": "^2.2.34",
"mongodb": "^3.1.10",
"seedrandom": "^2.4.3"
},
"optionalDependencies": {