From dae521ae893713f4339ce33bfc365ed9f29f04c8 Mon Sep 17 00:00:00 2001 From: Zaid Abdulla Date: Thu, 29 Oct 2020 11:31:36 -0700 Subject: [PATCH] Implement TR-069 Upload method --- bin/genieacs-fs.ts | 4 +- lib/api-functions.ts | 1 + lib/cwmp.ts | 20 ++ lib/db.ts | 30 +- lib/default-provisions.ts | 38 +++ lib/fs.ts | 96 ++++-- lib/init.ts | 1 + lib/mongodb-functions.ts | 8 + lib/session.ts | 499 +++++++++++++++++++++++++++++--- lib/soap.ts | 54 ++++ lib/types.ts | 33 ++- lib/ui/api-functions.ts | 4 + lib/ui/api.ts | 39 ++- lib/ui/db.ts | 15 + ui/components.ts | 2 + ui/components/device-actions.ts | 25 ++ ui/components/device-uploads.ts | 155 ++++++++++ ui/css/app.css | 2 +- ui/drawer-component.ts | 47 +++ ui/permissions-page.ts | 1 + ui/store.ts | 2 +- ui/task-queue.ts | 8 + 22 files changed, 1016 insertions(+), 68 deletions(-) create mode 100644 ui/components/device-uploads.ts diff --git a/bin/genieacs-fs.ts b/bin/genieacs-fs.ts index c794fa6..013772a 100755 --- a/bin/genieacs-fs.ts +++ b/bin/genieacs-fs.ts @@ -94,7 +94,9 @@ if (!cluster.worker) { const _listener = (req, res): void => { if (stopping) res.setHeader("Connection", "close"); - listener(req, res); + listener(req, res).catch((err) => { + throw err; + }); }; const initPromise = db diff --git a/lib/api-functions.ts b/lib/api-functions.ts index df8a414..91d58a4 100644 --- a/lib/api-functions.ts +++ b/lib/api-functions.ts @@ -333,5 +333,6 @@ export async function deleteDevice(deviceId: string): Promise { }, }), cache.del(`${deviceId}_tasks_faults_operations`), + db.deleteDeviceUploads(deviceId), ]); } diff --git a/lib/cwmp.ts b/lib/cwmp.ts index c7f583c..8c9b150 100644 --- a/lib/cwmp.ts +++ b/lib/cwmp.ts @@ -705,6 +705,11 @@ async function nextRpc(sessionContext: SessionContext): Promise { ["download", task.fileType, task.fileName, task.targetFileName || ""], ]); break; + case "upload": + session.addProvisions(sessionContext, `task_${task._id}`, [ + ["upload", task.fileType, task.fileName], + ]); + break; case "addObject": alias = (task.parameterValues || []) .map((p) => `${p[0]}:${JSON.stringify(p[1])}`) @@ -850,6 +855,21 @@ async function sendAcsRequest( } } + if (acsRequest.name === "Upload") { + let prefix = "" + config.get("FS_URL_PREFIX"); + + if (!prefix) { + const FS_PORT = +config.get("FS_PORT"); + const ssl = !!config.get("FS_SSL_CERT"); + const origin = getRequestOrigin(sessionContext.httpRequest); + let hostname = origin.localAddress; + if (origin.host) [hostname] = origin.host.split(":", 1); + prefix = (ssl ? "https" : "http") + `://${hostname}:${FS_PORT}/`; + } + + acsRequest.url = prefix + encodeURI(acsRequest.fileName); + } + const rpc = { id: id, acsRequest: acsRequest, diff --git a/lib/db.ts b/lib/db.ts index c87b9cf..702e37a 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -17,7 +17,7 @@ * along with GenieACS. If not, see . */ -import { MongoClient, ObjectID, Collection, Db } from "mongodb"; +import { MongoClient, ObjectID, Collection, Db, GridFSBucket } from "mongodb"; import { get } from "./config"; import { encodeTag, escapeRegExp } from "./common"; import { parse } from "./common/expression-parser"; @@ -39,10 +39,12 @@ export let tasksCollection: Collection, virtualParametersCollection: Collection, faultsCollection: Collection, filesCollection: Collection, + uploadsCollection: Collection, operationsCollection: Collection, permissionsCollection: Collection, usersCollection: Collection, - configCollection: Collection; + configCollection: Collection, + uploadsBucket: GridFSBucket; let clientPromise: Promise; @@ -77,6 +79,7 @@ onConnect(async (db) => { presetsCollection = db.collection("presets"); objectsCollection = db.collection("objects"); filesCollection = db.collection("fs.files"); + uploadsCollection = db.collection("uploads.files"); provisionsCollection = db.collection("provisions"); virtualParametersCollection = db.collection("virtualParameters"); faultsCollection = db.collection("faults"); @@ -84,6 +87,7 @@ onConnect(async (db) => { permissionsCollection = db.collection("permissions"); usersCollection = db.collection("users"); configCollection = db.collection("config"); + uploadsBucket = new GridFSBucket(db, { bucketName: "uploads" }); }); export async function disconnect(): Promise { @@ -439,6 +443,17 @@ export async function saveDevice( break; default: + if ( + diff[0].segments[0] === "Uploads" && + diff[0].segments[2] === "LastFileName" && + value1 && + value1 !== value2 + ) { + uploadsBucket.delete(value1 as any, () => { + // Ignore error due to mising files + }); + } + if (!diff[2]) { update["$unset"][path.toString()] = 1; continue; @@ -788,3 +803,14 @@ interface User { export async function getUsers(): Promise { return usersCollection.find().toArray(); } + +export async function deleteDeviceUploads(deviceId: string): Promise { + const files = await uploadsCollection + .find({ + _id: { + $regex: `^${escapeRegExp(deviceId)}\\/`, + }, + }) + .toArray(); + await Promise.all(files.map((f) => uploadsBucket.delete(f["_id"]))); +} diff --git a/lib/default-provisions.ts b/lib/default-provisions.ts index 04b1d28..d3ad166 100644 --- a/lib/default-provisions.ts +++ b/lib/default-provisions.ts @@ -229,6 +229,44 @@ export function download( return true; } +export function upload( + sessionContext: SessionContext, + provision: (string | number | boolean)[], + declarations: Declaration[] +): boolean { + if ( + provision.length !== 3 || + typeof provision[1] !== "string" || + typeof provision[2] !== "string" + ) + throw new Error("Invalid arguments"); + + const alias = [ + `FileType:${JSON.stringify(provision[1] || "")}`, + `FileName:${JSON.stringify(provision[2] || "")}`, + ].join(","); + + declarations.push({ + path: Path.parse(`Uploads.[${alias}]`), + pathGet: 1, + pathSet: 1, + attrGet: null, + attrSet: null, + defer: true, + }); + + declarations.push({ + path: Path.parse(`Uploads.[${alias}].Upload`), + pathGet: 1, + pathSet: null, + attrGet: { value: 1 }, + attrSet: { value: [sessionContext.timestamp] }, + defer: true, + }); + + return true; +} + export function instances( sessionContext: SessionContext, provision: (string | number | boolean)[], diff --git a/lib/fs.ts b/lib/fs.ts index 1d9d38f..c2c61f7 100644 --- a/lib/fs.ts +++ b/lib/fs.ts @@ -20,23 +20,42 @@ import * as url from "url"; import * as querystring from "querystring"; import { IncomingMessage, ServerResponse } from "http"; -import { Collection, GridFSBucket } from "mongodb"; -import { onConnect } from "./db"; +import { Collection, GridFSBucket, ObjectId } from "mongodb"; +import { onConnect, getOperations } from "./db"; import * as logger from "./logger"; import { getRequestOrigin } from "./forwarded"; let filesCollection: Collection; let filesBucket: GridFSBucket; +let uploadsBucket: GridFSBucket; onConnect(async (db) => { filesCollection = db.collection("fs.files"); filesBucket = new GridFSBucket(db); + uploadsBucket = new GridFSBucket(db, { bucketName: "uploads" }); }); -export function listener( +async function canUpload( + deviceId: string, + fileName: string, + timeout = Date.now() + 5000 +): Promise { + const operations = Object.values(await getOperations(deviceId)); + for (const operation of operations) { + if (operation.name === "Upload" && operation.args.fileName === fileName) + return true; + } + + if (Date.now() >= timeout) return false; + // Need to wait and retry in case upload was initiated before session was closed + await new Promise((resolve) => setTimeout(resolve, 1000)); + return canUpload(deviceId, fileName, timeout); +} + +export async function listener( request: IncomingMessage, response: ServerResponse -): void { +): Promise { const urlParts = url.parse(request.url, true); if (request.method === "GET") { const filename = querystring.unescape(urlParts.pathname.substring(1)); @@ -47,27 +66,58 @@ export function listener( remoteAddress: getRequestOrigin(request).remoteAddress, }; - filesCollection.findOne({ _id: filename }, (err, file) => { - if (err) throw err; + const file = await filesCollection.findOne({ _id: filename }); + if (!file) { + response.writeHead(404); + response.end(); + log.message += " not found"; + logger.accessError(log); + return; + } - if (!file) { - response.writeHead(404); - response.end(); - log.message += " not found"; - logger.accessError(log); - return; - } - - response.writeHead(200, { - "Content-Type": file.contentType || "application/octet-stream", - "Content-Length": file.length, - }); - - const downloadStream = filesBucket.openDownloadStreamByName(filename); - downloadStream.pipe(response); - - logger.accessInfo(log); + response.writeHead(200, { + "Content-Type": file.contentType || "application/octet-stream", + "Content-Length": file.length, }); + + const downloadStream = filesBucket.openDownloadStreamByName(filename); + downloadStream.pipe(response); + + logger.accessInfo(log); + } else if (request.method === "PUT") { + const [, deviceId, ...filePath] = urlParts.pathname + .split("/") + .map(querystring.unescape); + + const fileName = `${deviceId}/${filePath.join("/")}`; + + if (!(await canUpload(deviceId, fileName))) { + response.writeHead(403); + response.end("403 Forbidden"); + return; + } + + try { + await uploadsBucket.delete((fileName as unknown) as ObjectId); + } catch (err) { + // File not found, do nothing + } + const uploadStream = uploadsBucket.openUploadStreamWithId( + fileName, + fileName + ); + + uploadStream.on("finish", () => { + response.writeHead(200); + response.end(); + }); + + uploadStream.on("error", (err) => { + response.writeHead(500); + response.end(err.message); + }); + + request.pipe(uploadStream); } else { response.writeHead(405, { Allow: "GET" }); response.end("405 Method Not Allowed"); diff --git a/lib/init.ts b/lib/init.ts index 816963a..78b86d6 100644 --- a/lib/init.ts +++ b/lib/init.ts @@ -108,6 +108,7 @@ export async function seed(options: Record): Promise { { role: "admin", resource: "devices", access: 3, validate: "true" }, { role: "admin", resource: "faults", access: 3, validate: "true" }, { role: "admin", resource: "files", access: 3, validate: "true" }, + { role: "admin", resource: "uploads", access: 3, validate: "true" }, { role: "admin", resource: "presets", access: 3, validate: "true" }, { role: "admin", resource: "provisions", access: 3, validate: "true" }, { role: "admin", resource: "config", access: 3, validate: "true" }, diff --git a/lib/mongodb-functions.ts b/lib/mongodb-functions.ts index f933ac0..b7d1368 100644 --- a/lib/mongodb-functions.ts +++ b/lib/mongodb-functions.ts @@ -640,6 +640,14 @@ export function flattenFile( return f; } +export function flattenUpload( + file: Record +): Record { + const f = {}; + f["_id"] = file["_id"]; + return f; +} + export function preProcessPreset( data: Record ): Record { diff --git a/lib/session.ts b/lib/session.ts index abf4e66..9e8e1d3 100644 --- a/lib/session.ts +++ b/lib/session.ts @@ -53,6 +53,7 @@ import { AddObject, DeleteObject, Download, + Upload, Reboot, FactoryReset, AddObjectResponse, @@ -292,14 +293,15 @@ export async function transferComplete( }; } - const instance = operation.args.instance; - delete sessionContext.operations[commandKey]; if (!sessionContext.operationsTouched) sessionContext.operationsTouched = {}; sessionContext.operationsTouched[commandKey] = 1; if (rpcReq.faultStruct && rpcReq.faultStruct.faultCode !== "0") { - revertDownloadParameters(sessionContext, operation.args.instance); + if (operation.name === "Download") + revertDownloadParameters(sessionContext, operation.args.instance); + else if (operation.name === "Upload") + revertUploadParameters(sessionContext, operation.args.instance); const fault: Fault = { code: `cwmp.${rpcReq.faultStruct.faultCode}`, @@ -318,6 +320,44 @@ export async function transferComplete( let toClear = null; const timestamp = sessionContext.timestamp + sessionContext.iteration + 1; + if (operation.name === "Download") { + toClear = commitDownloadOperation( + sessionContext, + operation, + rpcReq, + timestamp, + toClear + ); + } else if (operation.name === "Upload") { + toClear = commitUploadOperation( + sessionContext, + operation, + rpcReq, + timestamp, + toClear + ); + } + + if (toClear) { + for (const c of toClear) + device.clear(sessionContext.deviceData, c[0], c[1], c[2], c[3]); + } + + return { + acsResponse: { name: "TransferCompleteResponse" }, + operation: operation, + fault: null, + }; +} + +function commitDownloadOperation( + sessionContext: SessionContext, + operation: Operation, + rpcReq: TransferCompleteRequest, + timestamp: number, + toClear: Clear[] +): Clear[] { + const instance = operation.args.instance; let p; p = sessionContext.deviceData.paths.add( @@ -386,16 +426,110 @@ export async function transferComplete( toClear ); + return toClear; +} + +function commitUploadOperation( + sessionContext: SessionContext, + operation: Operation, + rpcReq: TransferCompleteRequest, + timestamp: number, + toClear: Clear[] +): Clear[] { + const instance = operation.args.instance; + + let p; + + p = sessionContext.deviceData.paths.add( + Path.parse(`Uploads.${instance}.LastUpload`) + ); + toClear = device.set( + sessionContext.deviceData, + p, + timestamp, + { value: [timestamp, [operation.timestamp, "xsd:dateTime"]] }, + toClear + ); + + p = sessionContext.deviceData.paths.add( + Path.parse(`Uploads.${instance}.LastFileType`) + ); + toClear = device.set( + sessionContext.deviceData, + p, + timestamp, + { value: [timestamp, [operation.args.fileType, "xsd:string"]] }, + toClear + ); + + p = sessionContext.deviceData.paths.add( + Path.parse(`Uploads.${instance}.LastFileName`) + ); + toClear = device.set( + sessionContext.deviceData, + p, + timestamp, + { value: [timestamp, [operation.args.fileName, "xsd:string"]] }, + toClear + ); + + p = sessionContext.deviceData.paths.add( + Path.parse(`Uploads.${instance}.StartTime`) + ); + toClear = device.set( + sessionContext.deviceData, + p, + timestamp, + { value: [timestamp, [+rpcReq.startTime, "xsd:dateTime"]] }, + toClear + ); + + p = sessionContext.deviceData.paths.add( + Path.parse(`Uploads.${instance}.CompleteTime`) + ); + toClear = device.set( + sessionContext.deviceData, + p, + timestamp, + { value: [timestamp, [+rpcReq.completeTime, "xsd:dateTime"]] }, + toClear + ); + + return toClear; +} + +function revertUploadParameters( + sessionContext: SessionContext, + instance +): void { + const timestamp = sessionContext.timestamp + sessionContext.iteration + 1; + + let p; + + p = sessionContext.deviceData.paths.add( + Path.parse(`Uploads.${instance}.LastUpload`) + ); + + const lastUpload = sessionContext.deviceData.attributes.get(p); + + p = sessionContext.deviceData.paths.add( + Path.parse(`Uploads.${instance}.Upload`) + ); + + const toClear = device.set(sessionContext.deviceData, p, timestamp, { + value: [ + timestamp, + [ + lastUpload && lastUpload.value[1] ? lastUpload.value[1][0] : 0, + "xsd:dateTime", + ], + ], + }); + if (toClear) { for (const c of toClear) device.clear(sessionContext.deviceData, c[0], c[1], c[2], c[3]); } - - return { - acsResponse: { name: "TransferCompleteResponse" }, - operation: operation, - fault: null, - }; } function revertDownloadParameters( @@ -445,33 +579,62 @@ export async function timeoutOperations( for (const [commandKey, operation] of Object.entries( sessionContext.operations )) { - if (operation.name !== "Download") + if (operation.name === "Download") { + const DOWNLOAD_TIMEOUT = + +localCache.getConfig( + sessionContext.cacheSnapshot, + "cwmp.downloadTimeout", + {}, + sessionContext.timestamp, + (e) => configContextCallback(sessionContext, e) + ) * 1000; + + if (sessionContext.timestamp > operation.timestamp + DOWNLOAD_TIMEOUT) { + delete sessionContext.operations[commandKey]; + if (!sessionContext.operationsTouched) + sessionContext.operationsTouched = {}; + sessionContext.operationsTouched[commandKey] = 1; + + faults.push({ + code: "timeout", + message: "Download operation timed out", + timestamp: operation.timestamp, + }); + + operations.push(operation); + + revertDownloadParameters(sessionContext, operation.args.instance); + } + } else if (operation.name === "Upload") { + const UPLOAD_TIMEOUT = + +( + localCache.getConfig( + sessionContext.cacheSnapshot, + "cwmp.uploadTimeout", + {}, + sessionContext.timestamp, + (e) => configContextCallback(sessionContext, e) + ) || 3600 + ) * 1000; + + if (sessionContext.timestamp > operation.timestamp + UPLOAD_TIMEOUT) { + delete sessionContext.operations[commandKey]; + if (!sessionContext.operationsTouched) + sessionContext.operationsTouched = {}; + sessionContext.operationsTouched[commandKey] = 1; + + faults.push({ + code: "timeout", + message: "Upload operation timed out", + timestamp: operation.timestamp, + }); + + operations.push(operation); + + revertUploadParameters(sessionContext, operation.args.instance); + } + } else { throw new Error(`Unknown operation name ${operation.name}`); - - const DOWNLOAD_TIMEOUT = - +localCache.getConfig( - sessionContext.cacheSnapshot, - "cwmp.downloadTimeout", - {}, - sessionContext.timestamp, - (e) => configContextCallback(sessionContext, e) - ) * 1000; - - if (sessionContext.timestamp > operation.timestamp + DOWNLOAD_TIMEOUT) { - delete sessionContext.operations[commandKey]; - if (!sessionContext.operationsTouched) - sessionContext.operationsTouched = {}; - sessionContext.operationsTouched[commandKey] = 1; - - faults.push({ - code: "timeout", - message: "Download operation timed out", - timestamp: operation.timestamp, - }); - - operations.push(operation); - - revertDownloadParameters(sessionContext, operation.args.instance); } } @@ -793,6 +956,10 @@ function runDeclarations( downloadsToCreate: new InstanceSet(), downloadsValues: new Map(), downloadsDownload: new Map(), + uploadsToDelete: new Set(), + uploadsToCreate: new InstanceSet(), + uploadsValues: new Map(), + uploadsUpload: new Map(), reboot: 0, factoryReset: 0, }; @@ -1304,7 +1471,7 @@ export async function rpcRequest( } // Downloads - let index; + let index: number; for (const instance of sessionContext.syncState.downloadsToCreate) { if (index == null) { index = 0; @@ -1426,6 +1593,9 @@ export async function rpcRequest( } } + // Uploads + toClear = syncUploads(sessionContext, timestamp, toClear); + if (toClear || sessionContext.deviceData.changes.has("prerequisite")) { if (toClear) { for (const c of toClear) @@ -1503,6 +1673,129 @@ export async function rpcRequest( return rpcRequest(sessionContext, null); } +function syncUploads( + sessionContext: SessionContext, + timestamp: number, + toClear: Clear[] +): Clear[] { + let index: number; + for (const instance of sessionContext.syncState.uploadsToCreate) { + if (index == null) { + index = 0; + for (const p of sessionContext.deviceData.paths.find( + Path.parse("Uploads.*"), + false, + true + )) { + if ( + +p.segments[1] > index && + sessionContext.deviceData.attributes.has(p) + ) + index = +p.segments[1]; + } + } + + ++index; + + toClear = device.set( + sessionContext.deviceData, + Path.parse("Uploads"), + timestamp, + { object: [timestamp, 1], writable: [timestamp, 1] }, + toClear + ); + + toClear = device.set( + sessionContext.deviceData, + Path.parse(`Uploads.${index}`), + timestamp, + { object: [timestamp, 1], writable: [timestamp, 1] }, + toClear + ); + + const params = { + FileType: { + writable: 1, + value: [instance.FileType || "", "xsd:string"], + }, + FileName: { + writable: 1, + value: [instance.FileName || "", "xsd:string"], + }, + Upload: { + writable: 1, + value: [instance.Upload || 0, "xsd:dateTime"], + }, + LastFileType: { writable: 0, value: ["", "xsd:string"] }, + LastFileName: { writable: 0, value: ["", "xsd:string"] }, + LastTargetFileName: { writable: 0, value: ["", "xsd:string"] }, + LastUpload: { writable: 0, value: [0, "xsd:dateTime"] }, + StartTime: { writable: 0, value: [0, "xsd:dateTime"] }, + CompleteTime: { writable: 0, value: [0, "xsd:dateTime"] }, + }; + + for (const [k, v] of Object.entries(params)) { + toClear = device.set( + sessionContext.deviceData, + Path.parse(`Uploads.${index}.${k}`), + timestamp, + { + object: [timestamp, 0], + writable: [timestamp, v.writable as 0 | 1], + value: [timestamp, v.value as [string | number | boolean, string]], + }, + toClear + ); + } + + toClear = device.set( + sessionContext.deviceData, + Path.parse(`Uploads.${index}.*`), + timestamp, + null, + toClear + ); + } + + sessionContext.syncState.uploadsToCreate.clear(); + + for (const instance of sessionContext.syncState.uploadsToDelete) { + toClear = device.set( + sessionContext.deviceData, + instance, + timestamp, + null, + toClear + ); + for (const p of sessionContext.syncState.uploadsValues.keys()) { + if (p.segments[1] === instance.segments[1]) + sessionContext.syncState.uploadsValues.delete(p); + } + } + + sessionContext.syncState.uploadsToDelete.clear(); + + for (const [p, v] of sessionContext.syncState.uploadsValues) { + const attrs = sessionContext.deviceData.attributes.get(p); + if (attrs) { + if (attrs.writable && attrs.writable[1] && attrs.value) { + const val = device.sanitizeParameterValue([v, attrs.value[1][1]]); + if (val[0] !== attrs.value[1][0]) { + toClear = device.set( + sessionContext.deviceData, + p, + timestamp, + { value: [timestamp, val] }, + toClear + ); + } + } + } + } + + return toClear; +} + function generateGetRpcRequest( sessionContext: SessionContext ): GetParameterNames | GetParameterValues | GetParameterAttributes { @@ -1728,6 +2021,7 @@ function generateSetRpcRequest( | FactoryReset | Reboot | Download + | Upload ) & { next?: string } { const syncState = sessionContext.syncState; if (!syncState) return null; @@ -1906,6 +2200,36 @@ function generateSetRpcRequest( } } + // Uploads + for (const [p, t] of syncState.uploadsUpload) { + const attrs = deviceData.attributes.get(p); + if (!(attrs && attrs.value && t <= attrs.value[1][0])) { + const fileTypeAttrs = deviceData.attributes.get( + deviceData.paths.get(p.slice(0, -1).concat(Path.parse("FileType"))) + ); + const fileNameAttrs = deviceData.attributes.get( + deviceData.paths.get(p.slice(0, -1).concat(Path.parse("FileName"))) + ); + + if (fileNameAttrs && fileNameAttrs.value) { + return { + name: "Upload", + commandKey: generateRpcId(sessionContext), + instance: p.segments[1] as string, + fileType: fileTypeAttrs + ? fileTypeAttrs.value + ? (fileTypeAttrs.value[1][0] as string) + : null + : null, + fileName: + sessionContext.deviceId + + "/" + + (fileNameAttrs.value[1][0] as string), + }; + } + } + } + // Reboot if (syncState.reboot) { const p = sessionContext.deviceData.paths.get(Path.parse("Reboot")); @@ -2187,6 +2511,26 @@ function processDeclarations( } } break; + case "Uploads": + if ( + currentPath.length === 3 && + currentPath.wildcard === 0 && + declareAttributeValues && + declareAttributeValues.value + ) { + if (currentPath.segments[2] === "Upload") { + syncState.uploadsUpload.set( + currentPath, + declareAttributeValues.value[0] + ); + } else { + syncState.uploadsValues.set( + currentPath, + declareAttributeValues.value[0] + ); + } + } + break; case "VirtualParameters": if (currentPath.length <= 2) { let d; @@ -2357,6 +2701,10 @@ function processInstances( if (parent.length !== 1) return; instancesToDelete = sessionContext.syncState.downloadsToDelete; instancesToCreate = sessionContext.syncState.downloadsToCreate; + } else if (parent.segments[0] === "Uploads") { + if (parent.length !== 1) return; + instancesToDelete = sessionContext.syncState.uploadsToDelete; + instancesToCreate = sessionContext.syncState.uploadsToCreate; } else { instancesToDelete = sessionContext.syncState.instancesToDelete.get(parent); if (instancesToDelete == null) { @@ -2578,6 +2926,7 @@ export async function rpcResponse( "FactoryReset", "VirtualParameters", "Downloads", + "Uploads", ]) { const p = sessionContext.deviceData.paths.get(Path.parse(n)); if (p && sessionContext.deviceData.attributes.has(p)) @@ -2788,6 +3137,84 @@ export async function rpcResponse( operation.retries[channel] = sessionContext.retries[channel]; } + sessionContext.operations[rpcReq.commandKey] = operation; + if (!sessionContext.operationsTouched) + sessionContext.operationsTouched = {}; + sessionContext.operationsTouched[rpcReq.commandKey] = 1; + } + } else if (rpcRes.name === "UploadResponse") { + if (rpcReq.name !== "Upload") + throw new Error("Response name does not match request name"); + + toClear = device.set( + sessionContext.deviceData, + Path.parse(`Uploads.${rpcReq.instance}.Upload`), + timestamp + 1, + { value: [timestamp + 1, [sessionContext.timestamp, "xsd:dateTime"]] }, + toClear + ); + + if (rpcRes.status === 0) { + toClear = device.set( + sessionContext.deviceData, + Path.parse(`Uploads.${rpcReq.instance}.LastUpload`), + timestamp + 1, + { + value: [timestamp + 1, [sessionContext.timestamp, "xsd:dateTime"]], + }, + toClear + ); + + toClear = device.set( + sessionContext.deviceData, + Path.parse(`Uploads.${rpcReq.instance}.LastFileType`), + timestamp + 1, + { value: [timestamp + 1, [rpcReq.fileType, "xsd:string"]] }, + toClear + ); + + toClear = device.set( + sessionContext.deviceData, + Path.parse(`Uploads.${rpcReq.instance}.LastFileName`), + timestamp + 1, + { value: [timestamp + 1, [rpcReq.fileType, "xsd:string"]] }, + toClear + ); + + toClear = device.set( + sessionContext.deviceData, + Path.parse(`Uploads.${rpcReq.instance}.StartTime`), + timestamp + 1, + { value: [timestamp + 1, [+rpcRes.startTime, "xsd:dateTime"]] }, + toClear + ); + + toClear = device.set( + sessionContext.deviceData, + Path.parse(`Uploads.${rpcReq.instance}.CompleteTime`), + timestamp + 1, + { value: [timestamp + 1, [+rpcRes.completeTime, "xsd:dateTime"]] }, + toClear + ); + } else { + const operation = { + name: "Upload", + timestamp: sessionContext.timestamp, + provisions: sessionContext.provisions, + channels: sessionContext.channels, + retries: {}, + args: { + instance: rpcReq.instance, + fileType: rpcReq.fileType, + fileName: rpcReq.fileName, + }, + }; + + for (const channel of Object.keys(sessionContext.channels)) { + if (sessionContext.retries[channel] != null) + operation.retries[channel] = sessionContext.retries[channel]; + } + sessionContext.operations[rpcReq.commandKey] = operation; if (!sessionContext.operationsTouched) sessionContext.operationsTouched = {}; diff --git a/lib/soap.ts b/lib/soap.ts index 9cfd899..120c9c2 100644 --- a/lib/soap.ts +++ b/lib/soap.ts @@ -45,6 +45,7 @@ import { RebootResponse, FactoryResetResponse, DownloadResponse, + UploadResponse, GetRPCMethodsRequest, RequestDownloadRequest, AcsResponse, @@ -496,6 +497,24 @@ function Download(methodRequest): string { )}`; } +function Upload(methodRequest): string { + return `${ + methodRequest.commandKey || "" + }${methodRequest.fileType}${ + methodRequest.url + }${encodeEntities( + methodRequest.username || "" + )}${encodeEntities( + methodRequest.password || "" + )}${ + methodRequest.delaySeconds || "0" + }${encodeEntities( + methodRequest.successUrl || "" + )}${encodeEntities( + methodRequest.failureUrl || "" + )}`; +} + function DownloadResponse(xml: Element): DownloadResponse { let status: number, startTime: number, completeTime: number; for (const c of xml.children) { @@ -544,6 +563,35 @@ function DownloadResponse(xml: Element): DownloadResponse { }; } +function UploadResponse(xml: Element): UploadResponse { + let status: number, startTime: number, completeTime: number; + for (const c of xml.children) { + switch (c.localName) { + case "Status": + status = parseInt(c.text); + break; + case "StartTime": + startTime = Date.parse(c.text); + break; + case "CompleteTime": + completeTime = Date.parse(c.text); + break; + } + } + + if (!(status >= 0)) { + warnings.push({ message: "Invalid UploadResponse status" }); + status = 0; + } + + return { + name: "UploadResponse", + status: status, + startTime: startTime, + completeTime: completeTime, + }; +} + function Inform(xml: Element): InformRequest { let retryCount: number, evnt: string[]; let parameterList: [Path, string | number | boolean, string][]; @@ -927,6 +975,9 @@ export function request( case "DownloadResponse": rpc.cpeResponse = DownloadResponse(methodElement); break; + case "UploadResponse": + rpc.cpeResponse = UploadResponse(methodElement); + break; case "Fault": rpc.cpeFault = fault(methodElement); break; @@ -1022,6 +1073,9 @@ export function response(rpc: { case "Download": body = Download(rpc.acsRequest); break; + case "Upload": + body = Upload(rpc.acsRequest); + break; default: throw new Error( `Unknown method request ${(rpc.acsRequest as AcsRequest).name}` diff --git a/lib/types.ts b/lib/types.ts index 0defd88..7826c6d 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -121,6 +121,10 @@ export interface SyncState { downloadsToCreate: InstanceSet; downloadsValues: Map; downloadsDownload: Map; + uploadsToDelete: Set; + uploadsToCreate: InstanceSet; + uploadsValues: Map; + uploadsUpload: Map; reboot: number; factoryReset: number; } @@ -193,7 +197,7 @@ export interface Operation { instance: string; fileType: string; fileName: string; - targetFileName: string; + targetFileName?: string; }; } @@ -207,7 +211,8 @@ export type AcsRequest = | DeleteObject | FactoryReset | Reboot - | Download; + | Download + | Upload; export interface GetParameterNames { name: "GetParameterNames"; @@ -277,6 +282,20 @@ export interface Download { failureUrl?: string; } +export interface Upload { + name: "Upload"; + commandKey: string; + instance: string; + fileType: string; + fileName?: string; + url?: string; + username?: string; + password?: string; + delaySecods?: number; + successUrl?: string; + failureUrl?: string; +} + export interface SpvFault { parameterName: string; faultCode: string; @@ -305,7 +324,8 @@ export type CpeResponse = | DeleteObjectResponse | RebootResponse | FactoryResetResponse - | DownloadResponse; + | DownloadResponse + | UploadResponse; export interface GetParameterNamesResponse { name: "GetParameterNamesResponse"; @@ -357,6 +377,13 @@ export interface DownloadResponse { completeTime?: number; } +export interface UploadResponse { + name: "UploadResponse"; + status: number; + startTime?: number; + completeTime?: number; +} + export type CpeRequest = | InformRequest | TransferCompleteRequest diff --git a/lib/ui/api-functions.ts b/lib/ui/api-functions.ts index b382a45..a5ead4b 100644 --- a/lib/ui/api-functions.ts +++ b/lib/ui/api-functions.ts @@ -58,6 +58,10 @@ export async function deleteResource( await db.deleteFile(id); break; + case "uploads": + await db.deleteUpload(id); + break; + case "faults": await deleteFault(id); break; diff --git a/lib/ui/api.ts b/lib/ui/api.ts index 15f5d0c..2345bb6 100644 --- a/lib/ui/api.ts +++ b/lib/ui/api.ts @@ -49,6 +49,7 @@ const RESOURCE_IDS = { presets: "_id", provisions: "_id", files: "_id", + uploads: "_id", virtualParameters: "_id", config: "_id", permissions: "_id", @@ -62,6 +63,7 @@ const resources = { presets: 0 | RESOURCE_DELETE | RESOURCE_PUT, provisions: 0 | RESOURCE_DELETE | RESOURCE_PUT, files: 0 | RESOURCE_DELETE, + uploads: 0 | RESOURCE_DELETE, virtualParameters: 0 | RESOURCE_DELETE | RESOURCE_PUT, config: 0 | RESOURCE_DELETE | RESOURCE_PUT, permissions: 0 | RESOURCE_DELETE | RESOURCE_PUT, @@ -361,7 +363,7 @@ for (const [resource, flags] of Object.entries(resources)) { }); if (flags & RESOURCE_DELETE) { - router.delete(`/${resource}/:id`, async (ctx) => { + router.delete(`/${resource}/:id(.+)`, async (ctx) => { const authorizer: Authorizer = ctx.state.authorizer; const log = { message: `Delete ${resource}`, @@ -478,6 +480,41 @@ router.put("/files/:id", async (ctx) => { ctx.body = ""; }); +router.get("/uploads/blob/:id(.+)", async (ctx) => { + const authorizer: Authorizer = ctx.state.authorizer; + const resource = "uploads"; + const id = ctx.params.id; + + const log = { + message: `Download ${resource} blob`, + context: ctx, + id: id, + }; + 1; + + const filter = and(authorizer.getFilter(resource, 2), [ + "=", + ["PARAM", RESOURCE_IDS[resource]], + id, + ]); + + if (!authorizer.hasAccess(resource, 2)) { + logUnauthorizedWarning(log); + return void (ctx.status = 403); + } + + const res = await db.query(resource, filter); + if (!res[0]) return void (ctx.status = 404); + + ctx.type = "application/octet-stream"; + ctx.attachment(id.replace(/\//g, "_")); + ctx.body = new stream.PassThrough(); + + const downloadStream = await db.getUploadBlob(id); + downloadStream.pipe(ctx.body); + logger.accessInfo(log); +}); + router.post("/devices/:id/tasks", async (ctx) => { const authorizer: Authorizer = ctx.state.authorizer; const log = { diff --git a/lib/ui/db.ts b/lib/ui/db.ts index d452367..dc01068 100644 --- a/lib/ui/db.ts +++ b/lib/ui/db.ts @@ -28,6 +28,7 @@ import { minimize } from "../common/boolean-expression"; const RESOURCE_COLLECTION = { files: "fs.files", + uploads: "uploads.files", }; let db: Db; @@ -112,6 +113,8 @@ export function query( docs = docs.map((d) => mongodbFunctions.flattenPreset(d)); else if (resource === "files") docs = docs.map((d) => mongodbFunctions.flattenFile(d)); + else if (resource === "uploads") + docs = docs.map((d) => mongodbFunctions.flattenUpload(d)); return resolve(docs); }); } else { @@ -126,6 +129,8 @@ export function query( doc = mongodbFunctions.flattenPreset(doc); else if (resource === "files") doc = mongodbFunctions.flattenFile(doc); + else if (resource === "uploads") + doc = mongodbFunctions.flattenUpload(doc); callback(doc); }, (err) => { @@ -326,6 +331,16 @@ export function deleteFile(filename: string): Promise { }); } +export async function deleteUpload(filename: string): Promise { + const bucket = new GridFSBucket(db, { bucketName: "uploads" }); + await bucket.delete(filename as any); +} + +export async function getUploadBlob(filename: string): Promise { + const bucket = new GridFSBucket(db, { bucketName: "uploads" }); + return bucket.openDownloadStreamByName(filename); +} + export function deleteFault(id: string): Promise { return deleteResource("faults", id); } diff --git a/ui/components.ts b/ui/components.ts index 20ea419..2385307 100644 --- a/ui/components.ts +++ b/ui/components.ts @@ -39,6 +39,7 @@ import tags from "./components/tags"; import ping from "./components/ping"; import deviceLink from "./components/device-link"; import longTextComponent from "./long-text-component"; +import deviceUploads from "./components/device-uploads"; const comps = { parameter, @@ -54,6 +55,7 @@ const comps = { ping, "device-link": deviceLink, "long-text": longTextComponent, + "device-uploads": deviceUploads, }; const contextifiedComponents = new WeakMap(); diff --git a/ui/components/device-actions.ts b/ui/components/device-actions.ts index 02ed3a8..b750961 100644 --- a/ui/components/device-actions.ts +++ b/ui/components/device-actions.ts @@ -22,6 +22,7 @@ import { m } from "../components"; import * as taskQueue from "../task-queue"; import * as notifications from "../notifications"; import * as store from "../store"; +import * as overlay from "../overlay"; const component: ClosureComponent = (): Component => { return { @@ -78,6 +79,30 @@ const component: ClosureComponent = (): Component => { ) ); + buttons.push( + m( + "button.primary", + { + title: "Fetch files from device", + onclick: () => { + const deviceId = device["DeviceID.ID"].value[0]; + overlay.open(() => { + const dev = store.fetch("devices", [ + "=", + ["PARAM", "DeviceID.ID"], + deviceId, + ]); + return [ + m("h3", `Uploads from ${deviceId}`), + m("device-uploads", { device: dev.value[0] }), + ]; + }); + }, + }, + "Uploads" + ) + ); + buttons.push( m( "button.primary", diff --git a/ui/components/device-uploads.ts b/ui/components/device-uploads.ts new file mode 100644 index 0000000..d6bae96 --- /dev/null +++ b/ui/components/device-uploads.ts @@ -0,0 +1,155 @@ +/** + * Copyright 2013-2020 GenieACS Inc. + * + * This file is part of GenieACS. + * + * GenieACS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * GenieACS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with GenieACS. If not, see . + */ + +import { ClosureComponent, Component } from "mithril"; +import { m } from "../components"; +import * as store from "../store"; +import * as notifications from "../notifications"; +import { getIcon } from "../icons"; +import * as taskQueue from "../task-queue"; + +const component: ClosureComponent = (): Component => { + return { + view: (vnode) => { + const device = vnode.attrs["device"]; + const deviceId = device["DeviceID.ID"].value[0]; + const files = store.fetch("uploads", [ + "AND", + [">", ["PARAM", "_id"], `${deviceId}/`], + ["<", ["PARAM", "_id"], `${deviceId}/zzzz`], + ]); + + const uploads: Record = {}; + + for (const [k, v] of Object.entries(device)) { + if (k.startsWith("Uploads.")) { + const parts = k.split("."); + if (parts.length !== 3) continue; + uploads[parts[1]] = uploads[parts[1]] || {}; + uploads[parts[1]][parts[2]] = v["value"][0]; + } + } + + const headers = ["Filename", "Type", "Timestamp", "Status"].map((l) => + m("th", l) + ); + const thead = m("thead", m("tr", headers)); + + let minTime = 999999; + const timestamp = store.getTimestamp(); + const rows = []; + const uploadsSorted = Object.values(uploads).sort( + (a, b) => a["Upload"] - b["Upload"] + ); + + for (const u of Object.values(uploadsSorted)) { + if (!u["Upload"]) continue; + const ready = u["LastUpload"] >= u["Upload"]; + if (!ready) { + minTime = Math.min(minTime, u["Upload"] - timestamp); + rows.push([ + m("td", u["FileName"]), + m("td", u["FileType"]), + m("td", new Date(u["Upload"]).toLocaleString()), + m("td", "Waiting for upload"), + ]); + } else { + const filePath = `${deviceId}/${u["FileName"]}`; + if (!Array.isArray(files.value)) continue; + if (!files.value.some((f) => f["_id"] === filePath)) continue; + + const deleteButton = m( + "button", + { + title: "Delete file", + onclick: (e) => { + e.redraw = false; + store + .deleteResource("uploads", filePath) + .then(() => { + notifications.push("success", "File deleted"); + store.fulfill(Date.now(), Date.now()); + m.redraw(); + }) + .catch((err) => { + notifications.push("error", err.message); + store.fulfill(Date.now(), Date.now()); + }); + }, + }, + getIcon("delete-instance") + ); + + rows.push([ + m("td", u["FileName"]), + m("td", u["FileType"]), + m("td", new Date(u["LastUpload"]).toLocaleString()), + m( + "td", + m( + "a", + { + href: `api/uploads/blob/${encodeURIComponent(filePath)}`, + }, + "Ready" + ) + ), + m("td", deleteButton), + ]); + } + } + + if (minTime < 60000) { + setTimeout(() => { + store.fulfill(0, timestamp + 5000); + }, 10000); + } + + const addButton = m( + "button", + { + title: "Fetch a new file", + onclick: () => { + taskQueue.stageUpload({ + name: "upload", + devices: [deviceId], + }); + }, + }, + getIcon("add-instance") + ); + + const empty = !rows.length; + + rows.push([m("td", { colspan: headers.length }), m("td", addButton)]); + + const tbody = m( + "tbody", + rows.map((r) => m("tr", r)), + empty + ? m("tr.empty", m("td", { colspan: headers.length }, "No uploads")) + : null + ); + + return m("table.table", thead, tbody); + }, + }; +}; + +export default component; diff --git a/ui/css/app.css b/ui/css/app.css index fe005f1..f2027b0 100644 --- a/ui/css/app.css +++ b/ui/css/app.css @@ -106,7 +106,7 @@ body { & > .drawer-wrapper { position: fixed; - z-index: 10; + z-index: 101; width: 720px; margin-left: -360px; top: 0; diff --git a/ui/drawer-component.ts b/ui/drawer-component.ts index 1d1d76f..41e6922 100644 --- a/ui/drawer-component.ts +++ b/ui/drawer-component.ts @@ -184,6 +184,44 @@ function renderStagingDownload(task: StageTask): Children { ]; } +function renderStagingUpload(task: StageTask): Children { + if (!task.fileName || !task.fileType) invalid.add(task); + else invalid.delete(task); + + const typesList = [ + "", + "1 Vendor Configuration File", + "2 Vendor Log File", + ].map((t) => + m( + "option", + { disabled: !t, value: t, selected: (task.fileType || "") === t }, + t + ) + ); + + return [ + "Fetch ", + m( + "select", + { + onchange: (e) => { + task.fileType = e.target.value; + task.fileName = task.fileType.replace(/\s+/g, "_") + ".txt"; + }, + }, + typesList + ), + " as ", + m("input", { + onchange: (e) => { + task.fileName = e.target.value; + }, + value: task.fileName || "", + }), + ]; +} + function renderStaging(staging: Set): Child[] { const elements: Child[] = []; @@ -204,6 +242,7 @@ function renderStaging(staging: Set): Child[] { if (s.name === "setParameterValues") elms = renderStagingSpv(s, queueFunc, cancelFunc); else if (s.name === "download") elms = renderStagingDownload(s); + else if (s.name === "upload") elms = renderStagingUpload(s); const queue = m( "button.primary", @@ -323,6 +362,14 @@ function renderQueue(queue: Set): Child[] { m(".actions", actions) ) ); + } else if (t.name === "upload") { + details.push( + m( + `div.${t.status}`, + `Fetch file ${t.fileName} (${t.fileType})`, + m(".actions", actions) + ) + ); } else { details.push(m(`div.${t.status}`, t.name, m(".actions", actions))); } diff --git a/ui/permissions-page.ts b/ui/permissions-page.ts index e4ac703..461ae07 100644 --- a/ui/permissions-page.ts +++ b/ui/permissions-page.ts @@ -47,6 +47,7 @@ const attributes = [ "devices", "faults", "files", + "uploads", "permissions", "users", "presets", diff --git a/ui/store.ts b/ui/store.ts index 82bb02f..9899994 100644 --- a/ui/store.ts +++ b/ui/store.ts @@ -58,11 +58,11 @@ const resources: Resources = {}; for (const r of [ "devices", "faults", - "files", "presets", "provisions", "virtualParameters", "files", + "uploads", "config", "users", "permissions", diff --git a/ui/task-queue.ts b/ui/task-queue.ts index 6fd4447..84bd5d2 100644 --- a/ui/task-queue.ts +++ b/ui/task-queue.ts @@ -89,6 +89,14 @@ export function stageDownload(task: StageTask): void { staging.add(task); } +export function stageUpload(task: StageTask): void { + if (queue.size + task.devices.length > MAX_QUEUE) { + notifications.push("error", "Too many tasks in queue"); + return; + } + staging.add(task); +} + export function commit( tasks: QueueTask[], callback: (