Add /api/flow/<id>/raw route for performance

webapp/main.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: GPL-3.0-only
 
 import asyncio
+import base64
 import contextlib
 import json
 
@@ -139,13 +140,30 @@ async def api_flow_get(request):
         rows = await cursor.fetchall()
         result["alert"] = [row_to_dict(f) for f in rows]
 
-    # Get associated TCP/UDP raw data
+    return JSONResponse(result, headers={"Cache-Control": "max-age=86400"})
+
+
+async def api_flow_raw_get(request):
+    flow_id = request.path_params["flow_id"]
+
+    # Query flow from database to get proto
+    cursor = await database.execute("SELECT proto FROM flow WHERE id = ?", [flow_id])
+    flow = await cursor.fetchone()
+    if not flow:
+        raise HTTPException(404)
+    basepath = "static/{}store/".format(flow["proto"].lower())
+
+    # Get associated raw data
     cursor = await database.execute(
         "SELECT server_to_client, sha256 FROM raw WHERE flow_id = ? ORDER BY count",
         [flow_id],
     )
     rows = await cursor.fetchall()
-    result["raw"] = [dict(f) for f in rows]
+    result = []
+    for r in rows:
+        with open("{}/{}/{}".format(basepath, r["sha256"][:2], r["sha256"]), "rb") as f:
+            data = base64.b64encode(f.read()).decode()
+        result.append({"server_to_client": r["server_to_client"], "data": data})
 
     return JSONResponse(result, headers={"Cache-Control": "max-age=86400"})
 
@@ -280,6 +298,7 @@ async def lifespan(app):
         Route("/", index),
         Route("/api/flow", api_flow_list),
         Route("/api/flow/{flow_id:int}", api_flow_get),
+        Route("/api/flow/{flow_id:int}/raw", api_flow_raw_get),
         Route("/api/replay-http/{flow_id:int}", api_replay_http),
         Route("/api/replay-raw/{flow_id:int}", api_replay_raw),
         Mount(

webapp/static/assets/js/api.js

@@ -61,4 +61,19 @@ export default class Api {
     const data = await response.json()
     return data
   }
+
+  /**
+   * Call API to get flow raw data from identifier
+   *
+   * @param {Number} flowId Flow identifier
+   */
+  async getFlowRaw (flowId) {
+    const response = await fetch(`api/flow/${flowId}/raw`, {})
+    if (!response.ok) {
+      throw Error('failed to get flow raw data')
+    }
+
+    const data = await response.json()
+    return data
+  }
 }

webapp/static/assets/js/flowdisplay.js

@@ -299,50 +299,37 @@ class FlowDisplay {
     })
 
     // Raw data card
-    const proto = flow.flow.proto.toLowerCase()
-    if (proto === 'tcp' || proto === 'udp') {
-      const promises = flow.raw.map(async (data) => {
-        // TODO: automatically switch to hex view when the UTF-8 is invalid
-        let d = await fetch(`static/${proto}store/${data.sha256.slice(0, 2)}/${data.sha256}`, {})
-        d = await d.arrayBuffer()
-        const byteArray = new Uint8Array(d)
+    if (['TCP', 'UDP'].includes(flow.flow.proto)) {
+      document.getElementById('display-raw').classList.remove('d-none')
+      document.getElementById('display-raw-replay').href = `api/replay-raw/${flowId}`
+
+      // Display loading indicator before sending HTTP request
+      const utf8View = document.getElementById('display-raw-utf8')
+      const hexView = document.getElementById('display-raw-hex')
+      utf8View.textContent = 'Loading...'
+      hexView.textContent = 'Loading...'
+
+      const chunks = await this.apiClient.getFlowRaw(flowId)
+      utf8View.textContent = ''
+      hexView.textContent = ''
+      chunks.forEach(chunk => {
+        const byteArray = Uint8Array.from(atob(chunk.data), c => c.charCodeAt(0))
         const utf8Decoder = new TextDecoder()
-        return {
-          data: {
-            utf8: utf8Decoder.decode(byteArray),
-            hex: this.renderHexDump(byteArray)
-          },
-          direction: data.server_to_client
-        }
+
+        const codeElUtf8 = document.createElement('code')
+        codeElUtf8.classList.add('text-white')
+        codeElUtf8.classList.toggle('bg-danger', chunk.server_to_client === 0)
+        codeElUtf8.classList.toggle('bg-success', chunk.server_to_client === 1)
+        codeElUtf8.textContent = utf8Decoder.decode(byteArray)
+        utf8View.appendChild(codeElUtf8)
+
+        const codeElHex = document.createElement('code')
+        codeElHex.classList.add('text-white')
+        codeElHex.classList.toggle('bg-danger', chunk.server_to_client === 0)
+        codeElHex.classList.toggle('bg-success', chunk.server_to_client === 1)
+        codeElHex.textContent = this.renderHexDump(byteArray)
+        hexView.appendChild(codeElHex)
       })
-      if (promises.length) {
-        document.getElementById('display-raw').classList.remove('d-none')
-        document.getElementById('display-raw-replay').href = `api/replay-raw/${flowId}`
-
-        const utf8View = document.getElementById('display-raw-utf8')
-        const hexView = document.getElementById('display-raw-hex')
-        utf8View.textContent = 'Loading...'
-        hexView.textContent = 'Loading...'
-        Promise.all(promises).then(results => {
-          utf8View.textContent = ''
-          hexView.textContent = ''
-          results.forEach(e => {
-            const codeElUtf8 = document.createElement('code')
-            codeElUtf8.classList.add('text-white')
-            codeElUtf8.classList.toggle('bg-danger', e.direction === 0)
-            codeElUtf8.classList.toggle('bg-success', e.direction === 1)
-            codeElUtf8.textContent = e.data.utf8
-            utf8View.appendChild(codeElUtf8)
-
-            const codeElHex = document.createElement('code')
-            codeElHex.classList.add('text-white')
-            codeElHex.classList.toggle('bg-danger', e.direction === 0)
-            codeElHex.classList.toggle('bg-success', e.direction === 1)
-            codeElHex.textContent = e.data.hex
-            hexView.appendChild(codeElHex)
-          })
-        })
-      }
     }
   }
 }