From 24ede8974273a6a32137d2c0a3d3a80c66e4ebf4 Mon Sep 17 00:00:00 2001
From: Milas Bowman <milas@tilt.dev>
Date: Mon, 1 Feb 2021 18:01:15 -0500
Subject: [PATCH] web: ensure snapshots are under 4MB upload limit (#4150)

Truncate logs included in the snapshot upload to avoid being rejected
by the server for > 4MB size.

The logs are truncated to 1MB as there is a substantial amount of
overhead from JSON as well as the rest of the actual UI state that
needs to fit in there as well.
---
 web/src/HUD.tsx          |  6 ++++-
 web/src/LogStore.test.ts | 47 ++++++++++++++++++++++++++++++++++++++++
 web/src/LogStore.ts      | 40 +++++++++++++++++++++++-----------
 3 files changed, 79 insertions(+), 14 deletions(-)

diff --git a/web/src/HUD.tsx b/web/src/HUD.tsx
index c63b0d7be6..a992f07b0a 100644
--- a/web/src/HUD.tsx
+++ b/web/src/HUD.tsx
@@ -56,6 +56,10 @@ type HudProps = {
   interfaceVersion: InterfaceVersion
 }
 
+// Snapshot logs are capped to 1MB (max upload size is 4MB; this ensures between the rest of state and JSON overhead
+// that the snapshot should still fit)
+const maxSnapshotLogSize = 1000 * 1000
+
 // The Main HUD view, as specified in
 // https://docs.google.com/document/d/1VNIGfpC4fMfkscboW0bjYYFJl07um_1tsFrbN-Fu3FI/edit#heading=h.l8mmnclsuxl1
 export default class HUD extends Component<HudProps, HudState> {
@@ -191,7 +195,7 @@ export default class HUD extends Component<HudProps, HudState> {
   snapshotFromState(state: HudState): Proto.webviewSnapshot {
     let view = _.cloneDeep(state.view ?? null)
     if (view && state.logStore) {
-      view.logList = state.logStore.toLogList()
+      view.logList = state.logStore.toLogList(maxSnapshotLogSize)
     }
     return {
       view: view,
diff --git a/web/src/LogStore.test.ts b/web/src/LogStore.test.ts
index 4a2739daee..f65de95b69 100644
--- a/web/src/LogStore.test.ts
+++ b/web/src/LogStore.test.ts
@@ -426,4 +426,51 @@ describe("LogStore", () => {
       "build ...... 3\nbuild 4\nbuild 5"
     )
   })
+
+  it("truncates output for snapshots", () => {
+    let logs = new LogStore()
+
+    logs.append({
+      spans: {
+        "build:1": { manifestName: "fe" },
+      },
+      segments: [newManifestSegment("build:1", "build 1\n")],
+      fromCheckpoint: 0,
+      toCheckpoint: 1,
+    })
+
+    logs.append({
+      spans: {
+        "build:2": { manifestName: "be" },
+      },
+      segments: [
+        newManifestSegment("build:2", "build 2\n"),
+        newManifestSegment("build:2", "build 3\n"),
+      ],
+      fromCheckpoint: 1,
+      toCheckpoint: 3,
+    })
+
+    logs.append({
+      spans: {
+        "": {},
+      },
+      segments: [newGlobalSegment("global line 1\n")],
+      fromCheckpoint: 3,
+      toCheckpoint: 4,
+    })
+
+    const logList = logs.toLogList(28)
+    // log should be truncated
+    expect(logList.segments?.length).toEqual(2)
+    // order should be preserved
+    expect(logList.segments![0].text).toEqual("build 3\n")
+    expect(logList.segments![1].text).toEqual("global line 1\n")
+
+    // only spans referenced by segments in the truncated output should exist
+    const spans = logList.spans as { [key: string]: Proto.webviewLogSpan }
+    expect(Object.keys(spans).length).toEqual(2)
+    expect(spans["build:2"].manifestName).toEqual("be")
+    expect(spans["_"].manifestName).toEqual("")
+  })
 })
diff --git a/web/src/LogStore.ts b/web/src/LogStore.ts
index 7c2c4a62b4..f46f02d830 100644
--- a/web/src/LogStore.ts
+++ b/web/src/LogStore.ts
@@ -111,22 +111,36 @@ class LogStore {
     return this.warningIndex[spanId] ?? []
   }
 
-  toLogList(): Proto.webviewLogList {
+  toLogList(maxSize: number | null | undefined): Proto.webviewLogList {
     let spans = {} as { [key: string]: Proto.webviewLogSpan }
-    for (let key in this.spans) {
-      spans[key] = { manifestName: this.spans[key].manifestName }
-    }
 
-    let segments = this.segments.map(
-      (segment): Proto.webviewLogSegment => {
-        let spanId = segment.spanId
-        let time = segment.time
-        let text = segment.text
-        let level = segment.level
-        let fields = segment.fields
-        return { spanId, time, text, level, fields }
+    let size = 0
+    const segments = [] as Proto.webviewLogSegment[]
+    for (let i = this.segments.length - 1; i >= 0; i--) {
+      let segment = this.segments[i]
+      size += segment.text?.length || 0
+      if (maxSize && size > maxSize) {
+        break
       }
-    )
+
+      let spanId = segment.spanId
+      if (spanId && !spans[spanId]) {
+        spans[spanId] = { manifestName: this.spans[spanId].manifestName }
+      }
+
+      segments.push({
+        spanId: spanId,
+        time: segment.time,
+        text: segment.text,
+        level: segment.level,
+        fields: segment.fields,
+      })
+    }
+
+    // caller expects segments in chronological order
+    // (iteration here was done backwards for truncation)
+    segments.reverse()
+
     return {
       spans: spans,
       segments: segments,