Fix XMPP stream not getting closed properly

Always bind resource to XMPP stream
The XMPP server may not flag the bind feature as required but we're still required to bind resource to the stream in order to be able to send IQ stanzas to other clients.
2023-02-06 14:48:00 -08:00 · 2021-01-05 18:05:19 -08:00 · 2020-12-08 02:08:25 -08:00 · 2020-12-08 02:08:25 -08:00 · 2020-12-08 02:08:25 -08:00 · 2020-12-08 02:08:25 -08:00
27 changed files with 667 additions and 1057 deletions
@@ -94,9 +94,7 @@ if (!cluster.worker) {

  const _listener = (req, res): void => {
    if (stopping) res.setHeader("Connection", "close");
-    listener(req, res).catch((err) => {
-      throw err;
-    });
+    listener(req, res);
  };

  const initPromise = db
@@ -51,6 +51,8 @@ const externals = [
  "tls",
  "http",
  "https",
+  "net",
+  "events",
  "zlib",
  "crypto",
  "util",
@@ -29,10 +29,14 @@ import {
 import {
  httpConnectionRequest,
  udpConnectionRequest,
+  xmppConnectionRequest,
 } from "./connection-request";
 import { Expression, Task } from "./types";
 import { flattenDevice } from "./mongodb-functions";
 import { evaluate } from "./common/expression";
+import * as config from "../lib/config";
+
+const XMPP_CONFIGURED = !!config.get("XMPP_JID");

 export async function connectionRequest(
  deviceId: string,
@@ -44,7 +48,12 @@ export async function connectionRequest(
    device = flattenDevice(res);
  }

-  let connectionRequestUrl, udpConnectionRequestAddress, username, password;
+  let connectionRequestUrl,
+    udpConnectionRequestAddress,
+    stunEnable,
+    connReqJabberId,
+    username,
+    password;

  if (device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"]) {
    connectionRequestUrl = (device[
@@ -55,6 +64,12 @@ export async function connectionRequest(
        "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress"
      ] || {}
    ).value || [""])[0];
+    stunEnable = ((
+      device["InternetGatewayDevice.ManagementServer.STUNEnable"] || {}
+    ).value || [""])[0];
+    connReqJabberId = ((
+      device["InternetGatewayDevice.ManagementServer.ConnReqJabberID"] || {}
+    ).value || [""])[0];
    username = ((
      device[
        "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername"
@@ -72,6 +87,10 @@ export async function connectionRequest(
    udpConnectionRequestAddress = ((
      device["Device.ManagementServer.UDPConnectionRequestAddress"] || {}
    ).value || [""])[0];
+    stunEnable = ((device["Device.ManagementServer.STUNEnable"] || {})
+      .value || [""])[0];
+    connReqJabberId = ((device["Device.ManagementServer.ConnReqJabberID"] || {})
+      .value || [""])[0];
    username = ((
      device["Device.ManagementServer.ConnectionRequestUsername"] || {}
    ).value || [""])[0];
@@ -146,7 +165,7 @@ export async function connectionRequest(
  const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);

  let udpProm;
-  if (udpConnectionRequestAddress) {
+  if (udpConnectionRequestAddress && +stunEnable) {
    udpProm = udpConnectionRequest(
      udpConnectionRequestAddress,
      authExp,
@@ -156,7 +175,18 @@ export async function connectionRequest(
    );
  }

-  const status = await httpConnectionRequest(
+  let status;
+
+  if (connReqJabberId && XMPP_CONFIGURED) {
+    status = await xmppConnectionRequest(
+      connReqJabberId,
+      authExp,
+      CONNECTION_REQUEST_TIMEOUT,
+      debug,
+      deviceId
+    );
+  } else {
+    status = await httpConnectionRequest(
      connectionRequestUrl,
      authExp,
      CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
@@ -169,6 +199,7 @@ export async function connectionRequest(
      await udpProm;
      return "";
    }
+  }

  return status;
 }
@@ -333,6 +364,5 @@ export async function deleteDevice(deviceId: string): Promise<void> {
      },
    }),
    cache.del(`${deviceId}_tasks_faults_operations`),
-    db.deleteDeviceUploads(deviceId),
  ]);
 }
@@ -101,6 +101,9 @@ const options = {

  // Should probably never be changed
  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 },
+
+  XMPP_JID: { type: "string", default: "" },
+  XMPP_PASSWORD: { type: "string", default: "" },
 };

 const allConfig: { [name: string]: string | number } = {};
@@ -26,6 +26,10 @@ import { Expression } from "./types";
 import * as auth from "./auth";
 import * as extensions from "./extensions";
 import * as debug from "./debug";
+import XmppClient from "./xmpp-client";
+import * as config from "../lib/config";
+import { encodeEntities, parseAttrs, Element } from "./xml-parser";
+import * as logger from "../lib/logger";

 async function extractAuth(
  exp: Expression,
@@ -234,3 +238,84 @@ export async function udpConnectionRequest(
  }
  client.close();
 }
+
+const XMPP_JID = config.get("XMPP_JID") as string;
+const XMPP_PASSWORD = config.get("XMPP_PASSWORD") as string;
+const XMPP_RESOURCE = crypto.randomBytes(8).toString("hex");
+
+let xmppClient: XmppClient;
+
+function xmppClientOnError(err: Error): void {
+  xmppClient = null;
+  logger.error({
+    message: "XMPP exception",
+    exception: err,
+    pid: process.pid,
+  });
+}
+
+function xmppClientOnClose(): void {
+  xmppClient = null;
+}
+
+export async function xmppConnectionRequest(
+  jid: string,
+  authExp: Expression,
+  timeout: number,
+  _debug: boolean,
+  deviceId: string
+): Promise<string> {
+  if (!xmppClient) {
+    const [host, username] = XMPP_JID.split("@").reverse();
+    xmppClient = await XmppClient.connect({
+      host,
+      username,
+      resource: XMPP_RESOURCE,
+      password: XMPP_PASSWORD,
+      timeout: 120000,
+    });
+    xmppClient.on("error", xmppClientOnError);
+    xmppClient.on("close", xmppClientOnClose);
+    xmppClient.unref();
+  }
+
+  let username: string;
+  let password: string;
+
+  [username, password, authExp] = await extractAuth(authExp, null);
+  while (username != null && password != null) {
+    const msg = `<connectionRequest xmlns="urn:broadband-forum-org:cwmp:xmppConnReq-1-0"><username>${encodeEntities(
+      username
+    )}</username><password>${encodeEntities(
+      password
+    )}</password></connectionRequest>`;
+    let res: Element, rawRes: string, rawReq: string;
+    try {
+      ({ res, rawRes, rawReq } = await xmppClient.sendIqStanza(
+        XMPP_JID,
+        jid,
+        "get",
+        msg,
+        timeout
+      ));
+    } catch (err) {
+      return err.message;
+    }
+    if (_debug) {
+      debug.outgoingXmppStanza(deviceId, rawReq);
+      debug.incomingXmppStanza(deviceId, rawRes);
+    }
+    const attrs = parseAttrs(res.attrs);
+    const type = attrs.find((a) => a.name === "type");
+    if (type && type.value === "result") return "";
+    const error = res.children.find((c) => c.name === "error");
+    if (!error || !error.children[0])
+      return "Unexpected XMPP connection request response";
+    if (error.children[0].name === "service-unavailable")
+      return "Device is offline";
+    if (error.children[0].name !== "not-authorized")
+      return "Unexpected XMPP connection request response";
+    [username, password, authExp] = await extractAuth(authExp, null);
+  }
+  return "Incorrect connection request credentials";
+}
@@ -705,11 +705,6 @@ async function nextRpc(sessionContext: SessionContext): Promise<void> {
        ["download", task.fileType, task.fileName, task.targetFileName || ""],
      ]);
      break;
-    case "upload":
-      session.addProvisions(sessionContext, `task_${task._id}`, [
-        ["upload", task.fileType, task.fileName],
-      ]);
-      break;
    case "addObject":
      alias = (task.parameterValues || [])
        .map((p) => `${p[0]}:${JSON.stringify(p[1])}`)
@@ -855,21 +850,6 @@ async function sendAcsRequest(
    }
  }

-  if (acsRequest.name === "Upload") {
-    let prefix = "" + config.get("FS_URL_PREFIX");
-
-    if (!prefix) {
-      const FS_PORT = +config.get("FS_PORT");
-      const ssl = !!config.get("FS_SSL_CERT");
-      const origin = getRequestOrigin(sessionContext.httpRequest);
-      let hostname = origin.localAddress;
-      if (origin.host) [hostname] = origin.host.split(":", 1);
-      prefix = (ssl ? "https" : "http") + `://${hostname}:${FS_PORT}/`;
-    }
-
-    acsRequest.url = prefix + encodeURI(acsRequest.fileName);
-  }
-
  const rpc = {
    id: id,
    acsRequest: acsRequest,
@@ -17,7 +17,7 @@
 * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
 */

-import { MongoClient, ObjectID, Collection, Db, GridFSBucket } from "mongodb";
+import { MongoClient, ObjectID, Collection, Db } from "mongodb";
 import { get } from "./config";
 import { encodeTag, escapeRegExp } from "./common";
 import { parse } from "./common/expression-parser";
@@ -39,12 +39,10 @@ export let tasksCollection: Collection,
  virtualParametersCollection: Collection,
  faultsCollection: Collection,
  filesCollection: Collection,
-  uploadsCollection: Collection,
  operationsCollection: Collection,
  permissionsCollection: Collection,
  usersCollection: Collection,
-  configCollection: Collection,
-  uploadsBucket: GridFSBucket;
+  configCollection: Collection;

 let clientPromise: Promise<MongoClient>;

@@ -79,7 +77,6 @@ onConnect(async (db) => {
  presetsCollection = db.collection("presets");
  objectsCollection = db.collection("objects");
  filesCollection = db.collection("fs.files");
-  uploadsCollection = db.collection("uploads.files");
  provisionsCollection = db.collection("provisions");
  virtualParametersCollection = db.collection("virtualParameters");
  faultsCollection = db.collection("faults");
@@ -87,7 +84,6 @@ onConnect(async (db) => {
  permissionsCollection = db.collection("permissions");
  usersCollection = db.collection("users");
  configCollection = db.collection("config");
-  uploadsBucket = new GridFSBucket(db, { bucketName: "uploads" });
 });

 export async function disconnect(): Promise<void> {
@@ -443,17 +439,6 @@ export async function saveDevice(

        break;
      default:
-        if (
-          diff[0].segments[0] === "Uploads" &&
-          diff[0].segments[2] === "LastFileName" &&
-          value1 &&
-          value1 !== value2
-        ) {
-          uploadsBucket.delete(value1 as any, () => {
-            // Ignore error due to mising files
-          });
-        }
-
        if (!diff[2]) {
          update["$unset"][path.toString()] = 1;
          continue;
@@ -803,14 +788,3 @@ interface User {
 export async function getUsers(): Promise<User[]> {
  return usersCollection.find().toArray();
 }
-
-export async function deleteDeviceUploads(deviceId: string): Promise<void> {
-  const files = await uploadsCollection
-    .find({
-      _id: {
-        $regex: `^${escapeRegExp(deviceId)}\\/`,
-      },
-    })
-    .toArray();
-  await Promise.all(files.map((f) => uploadsBucket.delete(f["_id"])));
-}
@@ -202,3 +202,37 @@ export function outgoingUdpMessage(
    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
 }
+
+export function outgoingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function incomingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "incoming XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
@@ -229,44 +229,6 @@ export function download(
  return true;
 }

-export function upload(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
-  if (
-    provision.length !== 3 ||
-    typeof provision[1] !== "string" ||
-    typeof provision[2] !== "string"
-  )
-    throw new Error("Invalid arguments");
-
-  const alias = [
-    `FileType:${JSON.stringify(provision[1] || "")}`,
-    `FileName:${JSON.stringify(provision[2] || "")}`,
-  ].join(",");
-
-  declarations.push({
-    path: Path.parse(`Uploads.[${alias}]`),
-    pathGet: 1,
-    pathSet: 1,
-    attrGet: null,
-    attrSet: null,
-    defer: true,
-  });
-
-  declarations.push({
-    path: Path.parse(`Uploads.[${alias}].Upload`),
-    pathGet: 1,
-    pathSet: null,
-    attrGet: { value: 1 },
-    attrSet: { value: [sessionContext.timestamp] },
-    defer: true,
-  });
-
-  return true;
-}
-
 export function instances(
  sessionContext: SessionContext,
  provision: (string | number | boolean)[],
@@ -20,42 +20,23 @@
 import * as url from "url";
 import * as querystring from "querystring";
 import { IncomingMessage, ServerResponse } from "http";
-import { Collection, GridFSBucket, ObjectId } from "mongodb";
-import { onConnect, getOperations } from "./db";
+import { Collection, GridFSBucket } from "mongodb";
+import { onConnect } from "./db";
 import * as logger from "./logger";
 import { getRequestOrigin } from "./forwarded";

 let filesCollection: Collection;
 let filesBucket: GridFSBucket;
-let uploadsBucket: GridFSBucket;

 onConnect(async (db) => {
  filesCollection = db.collection("fs.files");
  filesBucket = new GridFSBucket(db);
-  uploadsBucket = new GridFSBucket(db, { bucketName: "uploads" });
 });

-async function canUpload(
-  deviceId: string,
-  fileName: string,
-  timeout = Date.now() + 5000
-): Promise<boolean> {
-  const operations = Object.values(await getOperations(deviceId));
-  for (const operation of operations) {
-    if (operation.name === "Upload" && operation.args.fileName === fileName)
-      return true;
-  }
-
-  if (Date.now() >= timeout) return false;
-  // Need to wait and retry in case upload was initiated before session was closed
-  await new Promise((resolve) => setTimeout(resolve, 1000));
-  return canUpload(deviceId, fileName, timeout);
-}
-
-export async function listener(
+export function listener(
  request: IncomingMessage,
  response: ServerResponse
-): Promise<void> {
+): void {
  const urlParts = url.parse(request.url, true);
  if (request.method === "GET") {
    const filename = querystring.unescape(urlParts.pathname.substring(1));
@@ -66,7 +47,9 @@ export async function listener(
      remoteAddress: getRequestOrigin(request).remoteAddress,
    };

-    const file = await filesCollection.findOne({ _id: filename });
+    filesCollection.findOne({ _id: filename }, (err, file) => {
+      if (err) throw err;
+
      if (!file) {
        response.writeHead(404);
        response.end();
@@ -84,40 +67,7 @@ export async function listener(
      downloadStream.pipe(response);

      logger.accessInfo(log);
-  } else if (request.method === "PUT") {
-    const [, deviceId, ...filePath] = urlParts.pathname
-      .split("/")
-      .map(querystring.unescape);
-
-    const fileName = `${deviceId}/${filePath.join("/")}`;
-
-    if (!(await canUpload(deviceId, fileName))) {
-      response.writeHead(403);
-      response.end("403 Forbidden");
-      return;
-    }
-
-    try {
-      await uploadsBucket.delete((fileName as unknown) as ObjectId);
-    } catch (err) {
-      // File not found, do nothing
-    }
-    const uploadStream = uploadsBucket.openUploadStreamWithId(
-      fileName,
-      fileName
-    );
-
-    uploadStream.on("finish", () => {
-      response.writeHead(200);
-      response.end();
    });
-
-    uploadStream.on("error", (err) => {
-      response.writeHead(500);
-      response.end(err.message);
-    });
-
-    request.pipe(uploadStream);
  } else {
    response.writeHead(405, { Allow: "GET" });
    response.end("405 Method Not Allowed");
@@ -108,7 +108,6 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
      { role: "admin", resource: "devices", access: 3, validate: "true" },
      { role: "admin", resource: "faults", access: 3, validate: "true" },
      { role: "admin", resource: "files", access: 3, validate: "true" },
-      { role: "admin", resource: "uploads", access: 3, validate: "true" },
      { role: "admin", resource: "presets", access: 3, validate: "true" },
      { role: "admin", resource: "provisions", access: 3, validate: "true" },
      { role: "admin", resource: "config", access: 3, validate: "true" },
@@ -640,14 +640,6 @@ export function flattenFile(
  return f;
 }

-export function flattenUpload(
-  file: Record<string, unknown>
-): Record<string, unknown> {
-  const f = {};
-  f["_id"] = file["_id"];
-  return f;
-}
-
 export function preProcessPreset(
  data: Record<string, unknown>
 ): Record<string, unknown> {
@@ -53,7 +53,6 @@ import {
  AddObject,
  DeleteObject,
  Download,
-  Upload,
  Reboot,
  FactoryReset,
  AddObjectResponse,
@@ -293,15 +292,14 @@ export async function transferComplete(
    };
  }

+  const instance = operation.args.instance;
+
  delete sessionContext.operations[commandKey];
  if (!sessionContext.operationsTouched) sessionContext.operationsTouched = {};
  sessionContext.operationsTouched[commandKey] = 1;

  if (rpcReq.faultStruct && rpcReq.faultStruct.faultCode !== "0") {
-    if (operation.name === "Download")
    revertDownloadParameters(sessionContext, operation.args.instance);
-    else if (operation.name === "Upload")
-      revertUploadParameters(sessionContext, operation.args.instance);

    const fault: Fault = {
      code: `cwmp.${rpcReq.faultStruct.faultCode}`,
@@ -320,44 +318,6 @@ export async function transferComplete(
  let toClear = null;
  const timestamp = sessionContext.timestamp + sessionContext.iteration + 1;

-  if (operation.name === "Download") {
-    toClear = commitDownloadOperation(
-      sessionContext,
-      operation,
-      rpcReq,
-      timestamp,
-      toClear
-    );
-  } else if (operation.name === "Upload") {
-    toClear = commitUploadOperation(
-      sessionContext,
-      operation,
-      rpcReq,
-      timestamp,
-      toClear
-    );
-  }
-
-  if (toClear) {
-    for (const c of toClear)
-      device.clear(sessionContext.deviceData, c[0], c[1], c[2], c[3]);
-  }
-
-  return {
-    acsResponse: { name: "TransferCompleteResponse" },
-    operation: operation,
-    fault: null,
-  };
-}
-
-function commitDownloadOperation(
-  sessionContext: SessionContext,
-  operation: Operation,
-  rpcReq: TransferCompleteRequest,
-  timestamp: number,
-  toClear: Clear[]
-): Clear[] {
-  const instance = operation.args.instance;
  let p;

  p = sessionContext.deviceData.paths.add(
@@ -426,110 +386,16 @@ function commitDownloadOperation(
    toClear
  );

-  return toClear;
-}
-
-function commitUploadOperation(
-  sessionContext: SessionContext,
-  operation: Operation,
-  rpcReq: TransferCompleteRequest,
-  timestamp: number,
-  toClear: Clear[]
-): Clear[] {
-  const instance = operation.args.instance;
-
-  let p;
-
-  p = sessionContext.deviceData.paths.add(
-    Path.parse(`Uploads.${instance}.LastUpload`)
-  );
-  toClear = device.set(
-    sessionContext.deviceData,
-    p,
-    timestamp,
-    { value: [timestamp, [operation.timestamp, "xsd:dateTime"]] },
-    toClear
-  );
-
-  p = sessionContext.deviceData.paths.add(
-    Path.parse(`Uploads.${instance}.LastFileType`)
-  );
-  toClear = device.set(
-    sessionContext.deviceData,
-    p,
-    timestamp,
-    { value: [timestamp, [operation.args.fileType, "xsd:string"]] },
-    toClear
-  );
-
-  p = sessionContext.deviceData.paths.add(
-    Path.parse(`Uploads.${instance}.LastFileName`)
-  );
-  toClear = device.set(
-    sessionContext.deviceData,
-    p,
-    timestamp,
-    { value: [timestamp, [operation.args.fileName, "xsd:string"]] },
-    toClear
-  );
-
-  p = sessionContext.deviceData.paths.add(
-    Path.parse(`Uploads.${instance}.StartTime`)
-  );
-  toClear = device.set(
-    sessionContext.deviceData,
-    p,
-    timestamp,
-    { value: [timestamp, [+rpcReq.startTime, "xsd:dateTime"]] },
-    toClear
-  );
-
-  p = sessionContext.deviceData.paths.add(
-    Path.parse(`Uploads.${instance}.CompleteTime`)
-  );
-  toClear = device.set(
-    sessionContext.deviceData,
-    p,
-    timestamp,
-    { value: [timestamp, [+rpcReq.completeTime, "xsd:dateTime"]] },
-    toClear
-  );
-
-  return toClear;
-}
-
-function revertUploadParameters(
-  sessionContext: SessionContext,
-  instance
-): void {
-  const timestamp = sessionContext.timestamp + sessionContext.iteration + 1;
-
-  let p;
-
-  p = sessionContext.deviceData.paths.add(
-    Path.parse(`Uploads.${instance}.LastUpload`)
-  );
-
-  const lastUpload = sessionContext.deviceData.attributes.get(p);
-
-  p = sessionContext.deviceData.paths.add(
-    Path.parse(`Uploads.${instance}.Upload`)
-  );
-
-  const toClear = device.set(sessionContext.deviceData, p, timestamp, {
-    value: [
-      timestamp,
-      [
-        lastUpload && lastUpload.value[1] ? lastUpload.value[1][0] : 0,
-        "xsd:dateTime",
-      ],
-    ],
-  });
-
  if (toClear) {
    for (const c of toClear)
      device.clear(sessionContext.deviceData, c[0], c[1], c[2], c[3]);
  }
+
+  return {
+    acsResponse: { name: "TransferCompleteResponse" },
+    operation: operation,
+    fault: null,
+  };
 }

 function revertDownloadParameters(
@@ -579,7 +445,9 @@ export async function timeoutOperations(
  for (const [commandKey, operation] of Object.entries(
    sessionContext.operations
  )) {
-    if (operation.name === "Download") {
+    if (operation.name !== "Download")
+      throw new Error(`Unknown operation name ${operation.name}`);
+
    const DOWNLOAD_TIMEOUT =
      +localCache.getConfig(
        sessionContext.cacheSnapshot,
@@ -605,37 +473,6 @@ export async function timeoutOperations(

      revertDownloadParameters(sessionContext, operation.args.instance);
    }
-    } else if (operation.name === "Upload") {
-      const UPLOAD_TIMEOUT =
-        +(
-          localCache.getConfig(
-            sessionContext.cacheSnapshot,
-            "cwmp.uploadTimeout",
-            {},
-            sessionContext.timestamp,
-            (e) => configContextCallback(sessionContext, e)
-          ) || 3600
-        ) * 1000;
-
-      if (sessionContext.timestamp > operation.timestamp + UPLOAD_TIMEOUT) {
-        delete sessionContext.operations[commandKey];
-        if (!sessionContext.operationsTouched)
-          sessionContext.operationsTouched = {};
-        sessionContext.operationsTouched[commandKey] = 1;
-
-        faults.push({
-          code: "timeout",
-          message: "Upload operation timed out",
-          timestamp: operation.timestamp,
-        });
-
-        operations.push(operation);
-
-        revertUploadParameters(sessionContext, operation.args.instance);
-      }
-    } else {
-      throw new Error(`Unknown operation name ${operation.name}`);
-    }
  }

  return { faults, operations };
@@ -956,10 +793,6 @@ function runDeclarations(
      downloadsToCreate: new InstanceSet(),
      downloadsValues: new Map(),
      downloadsDownload: new Map(),
-      uploadsToDelete: new Set(),
-      uploadsToCreate: new InstanceSet(),
-      uploadsValues: new Map(),
-      uploadsUpload: new Map(),
      reboot: 0,
      factoryReset: 0,
    };
@@ -1471,7 +1304,7 @@ export async function rpcRequest(
      }

      // Downloads
-      let index: number;
+      let index;
      for (const instance of sessionContext.syncState.downloadsToCreate) {
        if (index == null) {
          index = 0;
@@ -1593,9 +1426,6 @@ export async function rpcRequest(
        }
      }

-      // Uploads
-      toClear = syncUploads(sessionContext, timestamp, toClear);
-
      if (toClear || sessionContext.deviceData.changes.has("prerequisite")) {
        if (toClear) {
          for (const c of toClear)
@@ -1673,129 +1503,6 @@ export async function rpcRequest(
  return rpcRequest(sessionContext, null);
 }

-function syncUploads(
-  sessionContext: SessionContext,
-  timestamp: number,
-  toClear: Clear[]
-): Clear[] {
-  let index: number;
-  for (const instance of sessionContext.syncState.uploadsToCreate) {
-    if (index == null) {
-      index = 0;
-      for (const p of sessionContext.deviceData.paths.find(
-        Path.parse("Uploads.*"),
-        false,
-        true
-      )) {
-        if (
-          +p.segments[1] > index &&
-          sessionContext.deviceData.attributes.has(p)
-        )
-          index = +p.segments[1];
-      }
-    }
-
-    ++index;
-
-    toClear = device.set(
-      sessionContext.deviceData,
-      Path.parse("Uploads"),
-      timestamp,
-      { object: [timestamp, 1], writable: [timestamp, 1] },
-      toClear
-    );
-
-    toClear = device.set(
-      sessionContext.deviceData,
-      Path.parse(`Uploads.${index}`),
-      timestamp,
-      { object: [timestamp, 1], writable: [timestamp, 1] },
-      toClear
-    );
-
-    const params = {
-      FileType: {
-        writable: 1,
-        value: [instance.FileType || "", "xsd:string"],
-      },
-      FileName: {
-        writable: 1,
-        value: [instance.FileName || "", "xsd:string"],
-      },
-      Upload: {
-        writable: 1,
-        value: [instance.Upload || 0, "xsd:dateTime"],
-      },
-      LastFileType: { writable: 0, value: ["", "xsd:string"] },
-      LastFileName: { writable: 0, value: ["", "xsd:string"] },
-      LastTargetFileName: { writable: 0, value: ["", "xsd:string"] },
-      LastUpload: { writable: 0, value: [0, "xsd:dateTime"] },
-      StartTime: { writable: 0, value: [0, "xsd:dateTime"] },
-      CompleteTime: { writable: 0, value: [0, "xsd:dateTime"] },
-    };
-
-    for (const [k, v] of Object.entries(params)) {
-      toClear = device.set(
-        sessionContext.deviceData,
-        Path.parse(`Uploads.${index}.${k}`),
-        timestamp,
-        {
-          object: [timestamp, 0],
-          writable: [timestamp, v.writable as 0 | 1],
-          value: [timestamp, v.value as [string | number | boolean, string]],
-        },
-        toClear
-      );
-    }
-
-    toClear = device.set(
-      sessionContext.deviceData,
-      Path.parse(`Uploads.${index}.*`),
-      timestamp,
-      null,
-      toClear
-    );
-  }
-
-  sessionContext.syncState.uploadsToCreate.clear();
-
-  for (const instance of sessionContext.syncState.uploadsToDelete) {
-    toClear = device.set(
-      sessionContext.deviceData,
-      instance,
-      timestamp,
-      null,
-      toClear
-    );
-    for (const p of sessionContext.syncState.uploadsValues.keys()) {
-      if (p.segments[1] === instance.segments[1])
-        sessionContext.syncState.uploadsValues.delete(p);
-    }
-  }
-
-  sessionContext.syncState.uploadsToDelete.clear();
-
-  for (const [p, v] of sessionContext.syncState.uploadsValues) {
-    const attrs = sessionContext.deviceData.attributes.get(p);
-    if (attrs) {
-      if (attrs.writable && attrs.writable[1] && attrs.value) {
-        const val = device.sanitizeParameterValue([v, attrs.value[1][1]]);
-        if (val[0] !== attrs.value[1][0]) {
-          toClear = device.set(
-            sessionContext.deviceData,
-            p,
-            timestamp,
-            { value: [timestamp, val] },
-            toClear
-          );
-        }
-      }
-    }
-  }
-
-  return toClear;
-}
-
 function generateGetRpcRequest(
  sessionContext: SessionContext
 ): GetParameterNames | GetParameterValues | GetParameterAttributes {
@@ -2021,7 +1728,6 @@ function generateSetRpcRequest(
  | FactoryReset
  | Reboot
  | Download
-  | Upload
 ) & { next?: string } {
  const syncState = sessionContext.syncState;
  if (!syncState) return null;
@@ -2200,36 +1906,6 @@ function generateSetRpcRequest(
    }
  }

-  // Uploads
-  for (const [p, t] of syncState.uploadsUpload) {
-    const attrs = deviceData.attributes.get(p);
-    if (!(attrs && attrs.value && t <= attrs.value[1][0])) {
-      const fileTypeAttrs = deviceData.attributes.get(
-        deviceData.paths.get(p.slice(0, -1).concat(Path.parse("FileType")))
-      );
-      const fileNameAttrs = deviceData.attributes.get(
-        deviceData.paths.get(p.slice(0, -1).concat(Path.parse("FileName")))
-      );
-
-      if (fileNameAttrs && fileNameAttrs.value) {
-        return {
-          name: "Upload",
-          commandKey: generateRpcId(sessionContext),
-          instance: p.segments[1] as string,
-          fileType: fileTypeAttrs
-            ? fileTypeAttrs.value
-              ? (fileTypeAttrs.value[1][0] as string)
-              : null
-            : null,
-          fileName:
-            sessionContext.deviceId +
-            "/" +
-            (fileNameAttrs.value[1][0] as string),
-        };
-      }
-    }
-  }
-
  // Reboot
  if (syncState.reboot) {
    const p = sessionContext.deviceData.paths.get(Path.parse("Reboot"));
@@ -2511,26 +2187,6 @@ function processDeclarations(
          }
        }
        break;
-      case "Uploads":
-        if (
-          currentPath.length === 3 &&
-          currentPath.wildcard === 0 &&
-          declareAttributeValues &&
-          declareAttributeValues.value
-        ) {
-          if (currentPath.segments[2] === "Upload") {
-            syncState.uploadsUpload.set(
-              currentPath,
-              declareAttributeValues.value[0]
-            );
-          } else {
-            syncState.uploadsValues.set(
-              currentPath,
-              declareAttributeValues.value[0]
-            );
-          }
-        }
-        break;
      case "VirtualParameters":
        if (currentPath.length <= 2) {
          let d;
@@ -2701,10 +2357,6 @@ function processInstances(
    if (parent.length !== 1) return;
    instancesToDelete = sessionContext.syncState.downloadsToDelete;
    instancesToCreate = sessionContext.syncState.downloadsToCreate;
-  } else if (parent.segments[0] === "Uploads") {
-    if (parent.length !== 1) return;
-    instancesToDelete = sessionContext.syncState.uploadsToDelete;
-    instancesToCreate = sessionContext.syncState.uploadsToCreate;
  } else {
    instancesToDelete = sessionContext.syncState.instancesToDelete.get(parent);
    if (instancesToDelete == null) {
@@ -2926,7 +2578,6 @@ export async function rpcResponse(
        "FactoryReset",
        "VirtualParameters",
        "Downloads",
-        "Uploads",
      ]) {
        const p = sessionContext.deviceData.paths.get(Path.parse(n));
        if (p && sessionContext.deviceData.attributes.has(p))
@@ -3137,84 +2788,6 @@ export async function rpcResponse(
          operation.retries[channel] = sessionContext.retries[channel];
      }

-      sessionContext.operations[rpcReq.commandKey] = operation;
-      if (!sessionContext.operationsTouched)
-        sessionContext.operationsTouched = {};
-      sessionContext.operationsTouched[rpcReq.commandKey] = 1;
-    }
-  } else if (rpcRes.name === "UploadResponse") {
-    if (rpcReq.name !== "Upload")
-      throw new Error("Response name does not match request name");
-
-    toClear = device.set(
-      sessionContext.deviceData,
-      Path.parse(`Uploads.${rpcReq.instance}.Upload`),
-      timestamp + 1,
-      { value: [timestamp + 1, [sessionContext.timestamp, "xsd:dateTime"]] },
-      toClear
-    );
-
-    if (rpcRes.status === 0) {
-      toClear = device.set(
-        sessionContext.deviceData,
-        Path.parse(`Uploads.${rpcReq.instance}.LastUpload`),
-        timestamp + 1,
-        {
-          value: [timestamp + 1, [sessionContext.timestamp, "xsd:dateTime"]],
-        },
-        toClear
-      );
-
-      toClear = device.set(
-        sessionContext.deviceData,
-        Path.parse(`Uploads.${rpcReq.instance}.LastFileType`),
-        timestamp + 1,
-        { value: [timestamp + 1, [rpcReq.fileType, "xsd:string"]] },
-        toClear
-      );
-
-      toClear = device.set(
-        sessionContext.deviceData,
-        Path.parse(`Uploads.${rpcReq.instance}.LastFileName`),
-        timestamp + 1,
-        { value: [timestamp + 1, [rpcReq.fileType, "xsd:string"]] },
-        toClear
-      );
-
-      toClear = device.set(
-        sessionContext.deviceData,
-        Path.parse(`Uploads.${rpcReq.instance}.StartTime`),
-        timestamp + 1,
-        { value: [timestamp + 1, [+rpcRes.startTime, "xsd:dateTime"]] },
-        toClear
-      );
-
-      toClear = device.set(
-        sessionContext.deviceData,
-        Path.parse(`Uploads.${rpcReq.instance}.CompleteTime`),
-        timestamp + 1,
-        { value: [timestamp + 1, [+rpcRes.completeTime, "xsd:dateTime"]] },
-        toClear
-      );
-    } else {
-      const operation = {
-        name: "Upload",
-        timestamp: sessionContext.timestamp,
-        provisions: sessionContext.provisions,
-        channels: sessionContext.channels,
-        retries: {},
-        args: {
-          instance: rpcReq.instance,
-          fileType: rpcReq.fileType,
-          fileName: rpcReq.fileName,
-        },
-      };
-
-      for (const channel of Object.keys(sessionContext.channels)) {
-        if (sessionContext.retries[channel] != null)
-          operation.retries[channel] = sessionContext.retries[channel];
-      }
-
      sessionContext.operations[rpcReq.commandKey] = operation;
      if (!sessionContext.operationsTouched)
        sessionContext.operationsTouched = {};
@@ -45,7 +45,6 @@ import {
  RebootResponse,
  FactoryResetResponse,
  DownloadResponse,
-  UploadResponse,
  GetRPCMethodsRequest,
  RequestDownloadRequest,
  AcsResponse,
@@ -497,24 +496,6 @@ function Download(methodRequest): string {
  )}</FailureURL></cwmp:Download>`;
 }

-function Upload(methodRequest): string {
-  return `<cwmp:Upload><CommandKey>${
-    methodRequest.commandKey || ""
-  }</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
-    methodRequest.url
-  }</URL><Username>${encodeEntities(
-    methodRequest.username || ""
-  )}</Username><Password>${encodeEntities(
-    methodRequest.password || ""
-  )}</Password><DelaySeconds>${
-    methodRequest.delaySeconds || "0"
-  }</DelaySeconds><SuccessURL>${encodeEntities(
-    methodRequest.successUrl || ""
-  )}</SuccessURL><FailureURL>${encodeEntities(
-    methodRequest.failureUrl || ""
-  )}</FailureURL></cwmp:Upload>`;
-}
-
 function DownloadResponse(xml: Element): DownloadResponse {
  let status: number, startTime: number, completeTime: number;
  for (const c of xml.children) {
@@ -563,35 +544,6 @@ function DownloadResponse(xml: Element): DownloadResponse {
  };
 }

-function UploadResponse(xml: Element): UploadResponse {
-  let status: number, startTime: number, completeTime: number;
-  for (const c of xml.children) {
-    switch (c.localName) {
-      case "Status":
-        status = parseInt(c.text);
-        break;
-      case "StartTime":
-        startTime = Date.parse(c.text);
-        break;
-      case "CompleteTime":
-        completeTime = Date.parse(c.text);
-        break;
-    }
-  }
-
-  if (!(status >= 0)) {
-    warnings.push({ message: "Invalid UploadResponse status" });
-    status = 0;
-  }
-
-  return {
-    name: "UploadResponse",
-    status: status,
-    startTime: startTime,
-    completeTime: completeTime,
-  };
-}
-
 function Inform(xml: Element): InformRequest {
  let retryCount: number, evnt: string[];
  let parameterList: [Path, string | number | boolean, string][];
@@ -975,9 +927,6 @@ export function request(
    case "DownloadResponse":
      rpc.cpeResponse = DownloadResponse(methodElement);
      break;
-    case "UploadResponse":
-      rpc.cpeResponse = UploadResponse(methodElement);
-      break;
    case "Fault":
      rpc.cpeFault = fault(methodElement);
      break;
@@ -1073,9 +1022,6 @@ export function response(rpc: {
      case "Download":
        body = Download(rpc.acsRequest);
        break;
-      case "Upload":
-        body = Upload(rpc.acsRequest);
-        break;
      default:
        throw new Error(
          `Unknown method request ${(rpc.acsRequest as AcsRequest).name}`
@@ -121,10 +121,6 @@ export interface SyncState {
  downloadsToCreate: InstanceSet;
  downloadsValues: Map<Path, string | number>;
  downloadsDownload: Map<Path, number>;
-  uploadsToDelete: Set<Path>;
-  uploadsToCreate: InstanceSet;
-  uploadsValues: Map<Path, string | number>;
-  uploadsUpload: Map<Path, number>;
  reboot: number;
  factoryReset: number;
 }
@@ -197,7 +193,7 @@ export interface Operation {
    instance: string;
    fileType: string;
    fileName: string;
-    targetFileName?: string;
+    targetFileName: string;
  };
 }

@@ -211,8 +207,7 @@ export type AcsRequest =
  | DeleteObject
  | FactoryReset
  | Reboot
-  | Download
-  | Upload;
+  | Download;

 export interface GetParameterNames {
  name: "GetParameterNames";
@@ -282,20 +277,6 @@ export interface Download {
  failureUrl?: string;
 }

-export interface Upload {
-  name: "Upload";
-  commandKey: string;
-  instance: string;
-  fileType: string;
-  fileName?: string;
-  url?: string;
-  username?: string;
-  password?: string;
-  delaySecods?: number;
-  successUrl?: string;
-  failureUrl?: string;
-}
-
 export interface SpvFault {
  parameterName: string;
  faultCode: string;
@@ -324,8 +305,7 @@ export type CpeResponse =
  | DeleteObjectResponse
  | RebootResponse
  | FactoryResetResponse
-  | DownloadResponse
-  | UploadResponse;
+  | DownloadResponse;

 export interface GetParameterNamesResponse {
  name: "GetParameterNamesResponse";
@@ -377,13 +357,6 @@ export interface DownloadResponse {
  completeTime?: number;
 }

-export interface UploadResponse {
-  name: "UploadResponse";
-  status: number;
-  startTime?: number;
-  completeTime?: number;
-}
-
 export type CpeRequest =
  | InformRequest
  | TransferCompleteRequest
@@ -58,10 +58,6 @@ export async function deleteResource(
      await db.deleteFile(id);
      break;

-    case "uploads":
-      await db.deleteUpload(id);
-      break;
-
    case "faults":
      await deleteFault(id);
      break;
@@ -49,7 +49,6 @@ const RESOURCE_IDS = {
  presets: "_id",
  provisions: "_id",
  files: "_id",
-  uploads: "_id",
  virtualParameters: "_id",
  config: "_id",
  permissions: "_id",
@@ -63,7 +62,6 @@ const resources = {
  presets: 0 | RESOURCE_DELETE | RESOURCE_PUT,
  provisions: 0 | RESOURCE_DELETE | RESOURCE_PUT,
  files: 0 | RESOURCE_DELETE,
-  uploads: 0 | RESOURCE_DELETE,
  virtualParameters: 0 | RESOURCE_DELETE | RESOURCE_PUT,
  config: 0 | RESOURCE_DELETE | RESOURCE_PUT,
  permissions: 0 | RESOURCE_DELETE | RESOURCE_PUT,
@@ -363,7 +361,7 @@ for (const [resource, flags] of Object.entries(resources)) {
  });

  if (flags & RESOURCE_DELETE) {
-    router.delete(`/${resource}/:id(.+)`, async (ctx) => {
+    router.delete(`/${resource}/:id`, async (ctx) => {
      const authorizer: Authorizer = ctx.state.authorizer;
      const log = {
        message: `Delete ${resource}`,
@@ -480,41 +478,6 @@ router.put("/files/:id", async (ctx) => {
  ctx.body = "";
 });

-router.get("/uploads/blob/:id(.+)", async (ctx) => {
-  const authorizer: Authorizer = ctx.state.authorizer;
-  const resource = "uploads";
-  const id = ctx.params.id;
-
-  const log = {
-    message: `Download ${resource} blob`,
-    context: ctx,
-    id: id,
-  };
-  1;
-
-  const filter = and(authorizer.getFilter(resource, 2), [
-    "=",
-    ["PARAM", RESOURCE_IDS[resource]],
-    id,
-  ]);
-
-  if (!authorizer.hasAccess(resource, 2)) {
-    logUnauthorizedWarning(log);
-    return void (ctx.status = 403);
-  }
-
-  const res = await db.query(resource, filter);
-  if (!res[0]) return void (ctx.status = 404);
-
-  ctx.type = "application/octet-stream";
-  ctx.attachment(id.replace(/\//g, "_"));
-  ctx.body = new stream.PassThrough();
-
-  const downloadStream = await db.getUploadBlob(id);
-  downloadStream.pipe(ctx.body);
-  logger.accessInfo(log);
-});
-
 router.post("/devices/:id/tasks", async (ctx) => {
  const authorizer: Authorizer = ctx.state.authorizer;
  const log = {
@@ -28,7 +28,6 @@ import { minimize } from "../common/boolean-expression";

 const RESOURCE_COLLECTION = {
  files: "fs.files",
-  uploads: "uploads.files",
 };

 let db: Db;
@@ -113,8 +112,6 @@ export function query(
          docs = docs.map((d) => mongodbFunctions.flattenPreset(d));
        else if (resource === "files")
          docs = docs.map((d) => mongodbFunctions.flattenFile(d));
-        else if (resource === "uploads")
-          docs = docs.map((d) => mongodbFunctions.flattenUpload(d));
        return resolve(docs);
      });
    } else {
@@ -129,8 +126,6 @@ export function query(
            doc = mongodbFunctions.flattenPreset(doc);
          else if (resource === "files")
            doc = mongodbFunctions.flattenFile(doc);
-          else if (resource === "uploads")
-            doc = mongodbFunctions.flattenUpload(doc);
          callback(doc);
        },
        (err) => {
@@ -331,16 +326,6 @@ export function deleteFile(filename: string): Promise<void> {
  });
 }

-export async function deleteUpload(filename: string): Promise<void> {
-  const bucket = new GridFSBucket(db, { bucketName: "uploads" });
-  await bucket.delete(filename as any);
-}
-
-export async function getUploadBlob(filename: string): Promise<Readable> {
-  const bucket = new GridFSBucket(db, { bucketName: "uploads" });
-  return bucket.openDownloadStreamByName(filename);
-}
-
 export function deleteFault(id: string): Promise<void> {
  return deleteResource("faults", id);
 }
@@ -0,0 +1,481 @@
+/**
+ * Copyright 2013-2020  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as net from "net";
+import * as tls from "tls";
+import { EventEmitter } from "events";
+import { createHash, createHmac, randomBytes } from "crypto";
+import { parseXml, Element, parseAttrs } from "./xml-parser";
+
+function encodeBase64(str: string): string {
+  return Buffer.from(str).toString("base64");
+}
+
+function decodeBase64(str: string): string {
+  return Buffer.from(str, "base64").toString();
+}
+
+function detectStreamTag(data: string): number {
+  const i1 = data.indexOf("<stream:stream");
+  if (i1 < 0) throw new Error("Cannot detect opening stream tag");
+  const i2 = data.indexOf(">", i1);
+  if (i2 < 0) throw new Error("Cannot detect opening stream tag");
+  return i2 + 1;
+}
+
+function xmppStream<T>(
+  socket: net.Socket,
+  callback: Generator<void, T, Element>
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const onError = (err: Error): void => {
+      socket.removeListener("error", onError);
+      reject(err);
+    };
+    socket.on("error", onError);
+
+    const onData = (chunk: Buffer): void => {
+      try {
+        const str = chunk.toString("utf8");
+        const xml = parseXml(str);
+        const { value, done } = callback.next(xml.children[0]);
+        if (done) {
+          socket.removeListener("error", onError);
+          socket.removeListener("data", onData);
+          resolve(value as T);
+        }
+      } catch (err) {
+        socket.removeListener("error", onError);
+        socket.removeListener("data", onData);
+        reject(err);
+      }
+    };
+
+    socket.once("data", (chunk: Buffer) => {
+      try {
+        const str = chunk.toString("utf8");
+        const i = detectStreamTag(str);
+        const streamTagStr = str.slice(0, i);
+        const xml = parseXml(streamTagStr + "</stream:stream>");
+        callback.next(xml.children[0]);
+        chunk = chunk.slice(Buffer.byteLength(streamTagStr));
+        if (chunk.length) onData(chunk);
+        socket.on("data", onData);
+      } catch (err) {
+        socket.removeListener("error", onError);
+        socket.removeListener("data", onData);
+        reject(err);
+      }
+    });
+
+    try {
+      callback.next();
+    } catch (err) {
+      socket.removeListener("error", onError);
+      socket.removeListener("data", onData);
+      reject(err);
+    }
+  });
+}
+
+const INT_1 = Buffer.from([0, 0, 0, 1]);
+const saltedPasswordCache = {
+  password: "",
+  iterationCount: 0,
+  saltBase64: "",
+  salted: Buffer.allocUnsafe(0),
+};
+
+function saltPassword(
+  password: string,
+  saltBase64: string,
+  iteractionCount: number
+): Buffer {
+  if (
+    password === saltedPasswordCache.password &&
+    saltBase64 === saltedPasswordCache.saltBase64 &&
+    iteractionCount === saltedPasswordCache.iterationCount
+  )
+    return saltedPasswordCache.salted;
+
+  const hi = createHmac("sha1", password)
+    .update(Buffer.concat([Buffer.from(saltBase64, "base64"), INT_1]))
+    .digest();
+  let hi2: Buffer = hi;
+  for (let i = 1; i < iteractionCount; ++i) {
+    hi2 = createHmac("sha1", password).update(hi2).digest();
+    for (const [j, b] of hi2.entries()) hi[j] ^= b;
+  }
+
+  saltedPasswordCache.saltBase64 = saltBase64;
+  saltedPasswordCache.password = password;
+  saltedPasswordCache.iterationCount = iteractionCount;
+  saltedPasswordCache.salted = hi;
+
+  return hi;
+}
+
+function* loginPlain(
+  socket: net.Socket,
+  username: string,
+  password: string
+): Generator<void, void, Element> {
+  socket.write(
+    `<auth xmlns="urn:ietf:params:xml:ns:xmpp-sasl" mechanism="PLAIN">${encodeBase64(
+      `\x00${username}\x00${password}`
+    )}</auth>`
+  );
+  const res1 = yield;
+
+  if (
+    res1.name === "failure" &&
+    res1.children.some((c) => c.name === "not-authorized")
+  )
+    throw new Error("Not authorized");
+
+  if (res1.name !== "success")
+    throw new Error(`Unexpected response ${res1.name}`);
+}
+
+function* loginScram(
+  socket: net.Socket,
+  username: string,
+  password: string
+): Generator<void, void, Element> {
+  const cnonce = randomBytes(8).toString("base64");
+  const gs2Header = "n,,";
+  const clientFirstMessageBare = `n=${username},r=${cnonce}`;
+  const clientFirstMessage = gs2Header + clientFirstMessageBare;
+  socket.write(
+    `<auth xmlns="urn:ietf:params:xml:ns:xmpp-sasl" mechanism="SCRAM-SHA-1">${encodeBase64(
+      clientFirstMessage
+    )}</auth>`
+  );
+  const res1 = yield;
+  if (res1.name !== "challenge")
+    throw new Error(`Unexpected element ${res1.name}`);
+  const serverFirstMessage = decodeBase64(res1.text);
+
+  let iterationCount: number;
+  let saltBase64: string;
+  let nonce: string;
+  for (const s of serverFirstMessage.split(",")) {
+    if (s.startsWith("i=")) iterationCount = parseInt(s.slice(2));
+    else if (s.startsWith("s=")) saltBase64 = s.slice(2);
+    else if (s.startsWith("r=")) nonce = s.slice(2);
+  }
+  if (iterationCount == null || isNaN(iterationCount))
+    throw new Error("Invalid iteration count");
+  if (saltBase64 == null) throw new Error("Missing salt");
+  if (nonce == null) throw new Error("Missing nonce");
+
+  const saltedPassword = saltPassword(password, saltBase64, iterationCount);
+  const clientKey = createHmac("sha1", saltedPassword)
+    .update("Client Key")
+    .digest();
+  const storedKey = createHash("sha1").update(clientKey).digest();
+
+  const clientFinalMessageWithoutProof = `c=${encodeBase64(
+    gs2Header
+  )},r=${nonce}`;
+  const authMessage = `${clientFirstMessageBare},${serverFirstMessage},${clientFinalMessageWithoutProof}`;
+  const clientSignature = createHmac("sha1", storedKey)
+    .update(authMessage)
+    .digest();
+
+  const clientProof = Buffer.from(clientKey);
+  for (const [i, b] of clientSignature.entries()) clientProof[i] ^= b;
+
+  const clientFinalMessage = `${clientFinalMessageWithoutProof},p=${clientProof.toString(
+    "base64"
+  )}`;
+  socket.write(
+    `<response xmlns="urn:ietf:params:xml:ns:xmpp-sasl">${encodeBase64(
+      clientFinalMessage
+    )}</response>`
+  );
+  const res2 = yield;
+
+  if (
+    res2.name === "failure" &&
+    res2.children.some((c) => c.name === "not-authorized")
+  )
+    throw new Error("Not authorized");
+
+  if (res2.name !== "success")
+    throw new Error(`Unexpected response ${res2.name}`);
+
+  const serverKey = createHmac("sha1", saltedPassword)
+    .update("Server Key")
+    .digest();
+  const serverSignature = createHmac("sha1", serverKey)
+    .update(authMessage)
+    .digest("base64");
+
+  if (!decodeBase64(res2.text).endsWith(serverSignature))
+    throw new Error("Invalid server signature");
+}
+
+function* starttls(socket: net.Socket): Generator<void, void, Element> {
+  socket.write("<starttls xmlns='urn:ietf:params:xml:ns:xmpp-tls'/>");
+  const res1 = yield;
+  if (res1.name !== "proceed") throw new Error("Failed to initiate STARTTLS");
+}
+
+function* bind(
+  socket: net.Socket,
+  resource: string
+): Generator<void, void, Element> {
+  const id = randomBytes(8).toString("base64");
+  socket.write(
+    `<iq id='${id}' type='set'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'><resource>${resource}</resource></bind></iq>`
+  );
+  const res1 = yield;
+  if (res1.name !== "iq") throw new Error(`Unexpected element ${res1.name}`);
+  const attrs1 = parseAttrs(res1.attrs);
+  const idAttr = attrs1.find((a) => a.name === "id");
+  if (!idAttr || idAttr.value !== id) throw new Error("Invalid ID");
+  const typeAttr = attrs1.find((a) => a.name === "type");
+  if (!typeAttr) throw new Error("Missing type attribute");
+  if (typeAttr.value !== "result") throw new Error("Cannot bind to resource");
+}
+
+const STATUS_RESTART_STREAM = 1;
+const STATUS_STARTTLS = 2;
+
+function* init(
+  socket: net.Socket,
+  host: string,
+  username: string,
+  password: string,
+  resource: string
+): Generator<void, number, Element> {
+  socket.write(
+    `<?xml version='1.0'?><stream:stream from='${username}@${host}' to='${host}' version='1.0' xml:lang='en' xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams'>`
+  );
+  const open = yield;
+  if (open.name !== "stream:stream")
+    throw new Error(`Unexpected element ${open.name}`);
+  const features = yield;
+  if (features.name !== "stream:features")
+    throw new Error(`Unexpected element ${features.name}`);
+  for (const feature of features.children) {
+    if (feature.name === "starttls") {
+      if (feature.children.some((c) => c.name === "required")) {
+        yield* starttls(socket);
+        return STATUS_STARTTLS;
+      }
+    } else if (feature.name === "mechanisms") {
+      const mechanisms: Set<string> = new Set(
+        feature.children.map((c) => c.text)
+      );
+      if (mechanisms.has("PLAIN")) {
+        yield* loginPlain(socket, username, password);
+        return STATUS_RESTART_STREAM;
+      } else if (mechanisms.has("SCRAM-SHA-1")) {
+        yield* loginScram(socket, username, password);
+        return STATUS_RESTART_STREAM;
+      } else {
+        throw new Error("No supported SASL method");
+      }
+    } else if (feature.name === "bind") {
+      yield* bind(socket, resource);
+      return 0;
+    }
+  }
+  return 0;
+}
+
+function upgradeTls(socket: net.Socket, host: string): Promise<tls.TLSSocket> {
+  return new Promise((resolve, reject) => {
+    socket.on("error", reject);
+    const newSocket = tls.connect({ socket, host }, () => {
+      socket.removeListener("error", reject);
+      resolve(newSocket);
+    });
+  });
+}
+
+interface XmppClientOptions {
+  host: string;
+  port?: number;
+  username?: string;
+  password?: string;
+  resource?: string;
+  timeout?: number;
+}
+
+export default class XmppClient extends EventEmitter {
+  private _socket: net.Socket;
+  private _host: string;
+  private _username: string;
+  private _resource: string;
+  private _iqStanzaCallbacks: Map<
+    string,
+    (err: Error, r?: { rawRes: string; res: Element }) => void
+  >;
+
+  private constructor() {
+    super();
+    this._socket = null;
+    this._host = null;
+    this._username = null;
+    this._resource = null;
+    this._iqStanzaCallbacks = new Map();
+  }
+
+  static connect(opts: XmppClientOptions): Promise<XmppClient> {
+    return new Promise((resolve, reject) => {
+      let socket = new net.Socket();
+      socket.on("error", reject);
+
+      socket.connect(opts.port || 5222, opts.host, async () => {
+        socket.removeListener("error", reject);
+        try {
+          let status = 1;
+          while (status) {
+            if (status === STATUS_STARTTLS)
+              socket = await upgradeTls(socket, opts.host);
+            status = await xmppStream(
+              socket,
+              init(
+                socket,
+                opts.host,
+                opts.username,
+                opts.password,
+                opts.resource
+              )
+            );
+          }
+
+          const client = new XmppClient();
+          client._socket = socket;
+          client._host = opts.host;
+          client._username = opts.username;
+          client._resource = opts.resource;
+          socket.on("data", client._onData.bind(client));
+          socket.on("error", client._onError.bind(client));
+          if (opts.timeout)
+            socket.setTimeout(opts.timeout, client.close.bind(client));
+          resolve(client);
+        } catch (err) {
+          socket.destroy();
+          reject(err);
+        }
+      });
+    });
+  }
+
+  close(): void {
+    this._socket.end("</stream:stream>");
+    this._socket.removeAllListeners("data");
+    this._socket.removeAllListeners("error");
+    this.emit("close");
+  }
+
+  ref(): void {
+    this._socket.ref();
+  }
+
+  unref(): void {
+    this._socket.unref();
+  }
+
+  get host(): string {
+    return this._host;
+  }
+
+  get username(): string {
+    return this._username;
+  }
+
+  get resource(): string {
+    return this._resource;
+  }
+
+  private _onData(chunk: Buffer): void {
+    try {
+      let close = false;
+      let str = chunk.toString("utf8");
+      if (str.endsWith("</stream:stream>")) {
+        str = str.slice(0, -16);
+        close = true;
+      }
+      const xml = parseXml(str);
+      const idx = xml.children.map((c) => c.bodyIndex);
+      for (const [i, c] of xml.children.entries()) {
+        const s = str.slice(idx[i], idx[i + 1]);
+        if (c.name === "iq") {
+          const attrs = parseAttrs(c.attrs);
+          const id = attrs.find((a) => a.name === "id");
+          if (id) {
+            const cb = this._iqStanzaCallbacks.get(id.value);
+            if (cb) cb(null, { rawRes: s, res: c });
+          }
+        }
+        this.emit("stanza", c, s);
+      }
+      if (close) {
+        this._socket.removeAllListeners("data");
+        this._socket.removeAllListeners("error");
+        this.emit("close");
+      }
+    } catch (err) {
+      this._socket.removeAllListeners("data");
+      this._socket.removeAllListeners("error");
+      this.emit("error", err);
+    }
+  }
+
+  private _onError(err: Error): void {
+    this._socket.end();
+    this.emit("error", err);
+    for (const cb of this._iqStanzaCallbacks.values()) cb(err);
+  }
+
+  send(msg: string): void {
+    this._socket.write(msg);
+  }
+
+  sendIqStanza(
+    from: string,
+    to: string,
+    type: string,
+    body: string,
+    timeout = 3000
+  ): Promise<{ rawReq: string; rawRes: string; res: Element }> {
+    return new Promise((resolve, reject) => {
+      const id = randomBytes(8).toString("base64");
+      const rawReq = `<iq from="${from}" to="${to}" id="${id}" type="${type}">${body}</iq>`;
+      this.send(rawReq);
+      const t = setTimeout(() => {
+        this._iqStanzaCallbacks.delete(id);
+        reject(
+          new Error("Did not receive IQ stanza response in a timely manner")
+        );
+      }, timeout);
+      this._iqStanzaCallbacks.set(id, (err, r) => {
+        this._iqStanzaCallbacks.delete(id);
+        clearTimeout(t);
+        if (err) reject(err);
+        else resolve(Object.assign(r, { rawReq }));
+      });
+    });
+  }
+}
@@ -39,7 +39,6 @@ import tags from "./components/tags";
 import ping from "./components/ping";
 import deviceLink from "./components/device-link";
 import longTextComponent from "./long-text-component";
-import deviceUploads from "./components/device-uploads";

 const comps = {
  parameter,
@@ -55,7 +54,6 @@ const comps = {
  ping,
  "device-link": deviceLink,
  "long-text": longTextComponent,
-  "device-uploads": deviceUploads,
 };

 const contextifiedComponents = new WeakMap<ComponentTypes, ComponentTypes>();
@@ -22,7 +22,6 @@ import { m } from "../components";
 import * as taskQueue from "../task-queue";
 import * as notifications from "../notifications";
 import * as store from "../store";
-import * as overlay from "../overlay";

 const component: ClosureComponent = (): Component => {
  return {
@@ -79,30 +78,6 @@ const component: ClosureComponent = (): Component => {
        )
      );

-      buttons.push(
-        m(
-          "button.primary",
-          {
-            title: "Fetch files from device",
-            onclick: () => {
-              const deviceId = device["DeviceID.ID"].value[0];
-              overlay.open(() => {
-                const dev = store.fetch("devices", [
-                  "=",
-                  ["PARAM", "DeviceID.ID"],
-                  deviceId,
-                ]);
-                return [
-                  m("h3", `Uploads from ${deviceId}`),
-                  m("device-uploads", { device: dev.value[0] }),
-                ];
-              });
-            },
-          },
-          "Uploads"
-        )
-      );
-
      buttons.push(
        m(
          "button.primary",
@@ -1,155 +0,0 @@
-/**
- * Copyright 2013-2020  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { ClosureComponent, Component } from "mithril";
-import { m } from "../components";
-import * as store from "../store";
-import * as notifications from "../notifications";
-import { getIcon } from "../icons";
-import * as taskQueue from "../task-queue";
-
-const component: ClosureComponent = (): Component => {
-  return {
-    view: (vnode) => {
-      const device = vnode.attrs["device"];
-      const deviceId = device["DeviceID.ID"].value[0];
-      const files = store.fetch("uploads", [
-        "AND",
-        [">", ["PARAM", "_id"], `${deviceId}/`],
-        ["<", ["PARAM", "_id"], `${deviceId}/zzzz`],
-      ]);
-
-      const uploads: Record<string, any> = {};
-
-      for (const [k, v] of Object.entries(device)) {
-        if (k.startsWith("Uploads.")) {
-          const parts = k.split(".");
-          if (parts.length !== 3) continue;
-          uploads[parts[1]] = uploads[parts[1]] || {};
-          uploads[parts[1]][parts[2]] = v["value"][0];
-        }
-      }
-
-      const headers = ["Filename", "Type", "Timestamp", "Status"].map((l) =>
-        m("th", l)
-      );
-      const thead = m("thead", m("tr", headers));
-
-      let minTime = 999999;
-      const timestamp = store.getTimestamp();
-      const rows = [];
-      const uploadsSorted = Object.values(uploads).sort(
-        (a, b) => a["Upload"] - b["Upload"]
-      );
-
-      for (const u of Object.values(uploadsSorted)) {
-        if (!u["Upload"]) continue;
-        const ready = u["LastUpload"] >= u["Upload"];
-        if (!ready) {
-          minTime = Math.min(minTime, u["Upload"] - timestamp);
-          rows.push([
-            m("td", u["FileName"]),
-            m("td", u["FileType"]),
-            m("td", new Date(u["Upload"]).toLocaleString()),
-            m("td", "Waiting for upload"),
-          ]);
-        } else {
-          const filePath = `${deviceId}/${u["FileName"]}`;
-          if (!Array.isArray(files.value)) continue;
-          if (!files.value.some((f) => f["_id"] === filePath)) continue;
-
-          const deleteButton = m(
-            "button",
-            {
-              title: "Delete file",
-              onclick: (e) => {
-                e.redraw = false;
-                store
-                  .deleteResource("uploads", filePath)
-                  .then(() => {
-                    notifications.push("success", "File deleted");
-                    store.fulfill(Date.now(), Date.now());
-                    m.redraw();
-                  })
-                  .catch((err) => {
-                    notifications.push("error", err.message);
-                    store.fulfill(Date.now(), Date.now());
-                  });
-              },
-            },
-            getIcon("delete-instance")
-          );
-
-          rows.push([
-            m("td", u["FileName"]),
-            m("td", u["FileType"]),
-            m("td", new Date(u["LastUpload"]).toLocaleString()),
-            m(
-              "td",
-              m(
-                "a",
-                {
-                  href: `api/uploads/blob/${encodeURIComponent(filePath)}`,
-                },
-                "Ready"
-              )
-            ),
-            m("td", deleteButton),
-          ]);
-        }
-      }
-
-      if (minTime < 60000) {
-        setTimeout(() => {
-          store.fulfill(0, timestamp + 5000);
-        }, 10000);
-      }
-
-      const addButton = m(
-        "button",
-        {
-          title: "Fetch a new file",
-          onclick: () => {
-            taskQueue.stageUpload({
-              name: "upload",
-              devices: [deviceId],
-            });
-          },
-        },
-        getIcon("add-instance")
-      );
-
-      const empty = !rows.length;
-
-      rows.push([m("td", { colspan: headers.length }), m("td", addButton)]);
-
-      const tbody = m(
-        "tbody",
-        rows.map((r) => m("tr", r)),
-        empty
-          ? m("tr.empty", m("td", { colspan: headers.length }, "No uploads"))
-          : null
-      );
-
-      return m("table.table", thead, tbody);
-    },
-  };
-};
-
-export default component;
@@ -106,7 +106,7 @@ body {

  & > .drawer-wrapper {
    position: fixed;
-    z-index: 101;
+    z-index: 10;
    width: 720px;
    margin-left: -360px;
    top: 0;
@@ -184,122 +184,6 @@ function renderStagingDownload(task: StageTask): Children {
  ];
 }

-const uploadTypesForDevice: WeakMap<
-  Record<string, unknown>,
-  Record<string, string>
-> = new WeakMap();
-function getUploadTypesForDevice(
-  device: Record<string, unknown>
-): Record<string, string> {
-  if (uploadTypesForDevice.has(device)) return uploadTypesForDevice.get(device);
-  const instances: Set<string> = new Set();
-  const filenames: Record<string, string> = {};
-  const versions: Record<string, string> = {};
-
-  if (device) {
-    for (const [k, v] of Object.entries(device)) {
-      if (/DeviceInfo\.VendorConfigFile\.[0-9]+(\.Name|\.Version)?$/.test(k)) {
-        const p = k.split(".");
-        if (p[p.length - 1] === "Name") {
-          const n = `3 Vendor Configuration File ${p[p.length - 2]}`;
-          instances.add(n);
-          if (v["value"]) filenames[n] = v["value"][0];
-        } else if (p[p.length - 1] === "Version") {
-          const n = `3 Vendor Configuration File ${p[p.length - 2]}`;
-          instances.add(n);
-          if (v["value"]) versions[n] = v["value"][0];
-        } else {
-          const n = `3 Vendor Configuration File ${p[p.length - 1]}`;
-          instances.add(n);
-        }
-      } else if (/DeviceInfo\.VendorLogFile\.[0-9]+(\.Name)?$/.test(k)) {
-        const p = k.split(".");
-        if (p[p.length - 1] === "Name") {
-          const n = `4 Vendor Log File ${p[p.length - 2]}`;
-          instances.add(n);
-          if (v["value"]) filenames[n] = v["value"][0];
-        } else {
-          const n = `4 Vendor Log File ${p[p.length - 1]}`;
-          instances.add(n);
-        }
-      }
-    }
-  }
-
-  if (!instances.size) {
-    instances.add("1 Vendor Configuration File");
-    instances.add("2 Vendor Log File");
-  }
-
-  const res: Record<string, string> = {};
-
-  for (const i of instances) {
-    let n = filenames[i];
-    const v = versions[i];
-    if (!n) n = i.replace(/\s+/g, "_") + ".txt";
-    if (v) n = `${n}-${v}`;
-    res[i] = n;
-  }
-
-  uploadTypesForDevice.set(device, res);
-  return res;
-}
-
-function renderStagingUpload(task: StageTask): Children {
-  if (!task.fileName || !task.fileType) invalid.add(task);
-  else invalid.delete(task);
-
-  const commonTypes: Record<string, number> = {};
-
-  const filenames: Record<string, string> = {};
-
-  for (const d of task.devices) {
-    const dev = store.fetch("devices", ["=", ["PARAM", "DeviceID.ID"], d])
-      .value[0];
-    if (dev) {
-      const u = getUploadTypesForDevice(dev);
-      Object.assign(filenames, u);
-      for (const k of Object.keys(u))
-        commonTypes[k] = (commonTypes[k] || 0) + 1;
-    }
-  }
-
-  const typesList = [""];
-  typesList.push(
-    ...Object.entries(commonTypes)
-      .filter(([, c]) => c === task.devices.length)
-      .map(([k]) => k)
-      .sort()
-  );
-
-  return [
-    "Fetch ",
-    m(
-      "select",
-      {
-        onchange: (e) => {
-          task.fileType = e.target.value;
-          task.fileName = filenames[task.fileType];
-        },
-      },
-      typesList.map((t) =>
-        m(
-          "option",
-          { disabled: !t, value: t, selected: (task.fileType || "") === t },
-          t
-        )
-      )
-    ),
-    " as ",
-    m("input", {
-      onchange: (e) => {
-        task.fileName = e.target.value;
-      },
-      value: task.fileName || "",
-    }),
-  ];
-}
-
 function renderStaging(staging: Set<StageTask>): Child[] {
  const elements: Child[] = [];

@@ -320,7 +204,6 @@ function renderStaging(staging: Set<StageTask>): Child[] {
    if (s.name === "setParameterValues")
      elms = renderStagingSpv(s, queueFunc, cancelFunc);
    else if (s.name === "download") elms = renderStagingDownload(s);
-    else if (s.name === "upload") elms = renderStagingUpload(s);

    const queue = m(
      "button.primary",
@@ -440,14 +323,6 @@ function renderQueue(queue: Set<QueueTask>): Child[] {
            m(".actions", actions)
          )
        );
-      } else if (t.name === "upload") {
-        details.push(
-          m(
-            `div.${t.status}`,
-            `Fetch file ${t.fileName} (${t.fileType})`,
-            m(".actions", actions)
-          )
-        );
      } else {
        details.push(m(`div.${t.status}`, t.name, m(".actions", actions)));
      }
@@ -47,7 +47,6 @@ const attributes = [
      "devices",
      "faults",
      "files",
-      "uploads",
      "permissions",
      "users",
      "presets",
@@ -58,11 +58,11 @@ const resources: Resources = {};
 for (const r of [
  "devices",
  "faults",
+  "files",
  "presets",
  "provisions",
  "virtualParameters",
  "files",
-  "uploads",
  "config",
  "users",
  "permissions",
@@ -89,14 +89,6 @@ export function stageDownload(task: StageTask): void {
  staging.add(task);
 }

-export function stageUpload(task: StageTask): void {
-  if (queue.size + task.devices.length > MAX_QUEUE) {
-    notifications.push("error", "Too many tasks in queue");
-    return;
-  }
-  staging.add(task);
-}
-
 export function commit(
  tasks: QueueTask[],
  callback: (
Author	SHA1	Message	Date
Zaid Humaimidi	b5f689b014	Fix XMPP stream not getting closed properly	2023-02-06 14:48:00 -08:00
Zaid Abdulla	ccfbcddeda	Always bind resource to XMPP stream The XMPP server may not flag the bind feature as required but we're still required to bind resource to the stream in order to be able to send IQ stanzas to other clients.	2021-01-05 18:05:19 -08:00
Zaid Abdulla	081201cd2f	Only send UDP con req if STUNEnable is true Because even when STUN is not enabled, the param UDPConnectionRequestAddress will still hold IP of the listening interface which we can't reach.	2020-12-08 02:08:25 -08:00
Zaid Abdulla	0e0380917f	XMPP fixes	2020-12-08 02:08:25 -08:00
Zaid Abdulla	79ede00ef2	Implement XMPP connection request	2020-12-08 02:08:25 -08:00
Zaid Abdulla	82ed742a67	Convenience function to send IQ stanza messages	2020-12-08 02:08:25 -08:00
Zaid Abdulla	53de5d01e7	Initial XmppClient implementation	2020-12-08 02:08:25 -08:00