Limit displayable payload and add memsize

CHANGELOG.md

@@ -1,6 +1,8 @@
 # Karafka Web changelog
 
 ## 0.9.2 (Unreleased)
+- [Enhancement] Prevent explorer from displaying too big payloads (bigger than 1MB by default)
+- [Enhancement] Include memsize based on `ObjectSpace.memsize_of`.
 - [Enhancement] Improve how charts with many topics work.
 - [Enhancement] Count and display executed jobs independently from processed batches.
 - [Enhancement] Prevent karafka-web from being configured before karafka is configured.

lib/karafka/web/config.rb

@@ -173,6 +173,10 @@ class Config
         # to explicitly define routing
         setting :dlq_patterns, default: [/(dlq)|(dead_letter)/i]
 
+        # Maximum in-memory size of payload that we will render. Anything bigger than this by
+        # default will not be displayed not to hang the browser
+        setting :max_visible_payload_size, default: 1_048_576
+
         # Specific kafka settings that are tuned to operate within the Web UI interface.
         #
         # Please do not change them unless you know what you are doing as their misconfiguration

lib/karafka/web/pro/ui/controllers/explorer_controller.rb

@@ -112,6 +112,9 @@ def show(topic_id, partition_id, offset, paginate: true)
                 @message.headers
               end
 
+              memsize_base = @safe_payload.failure? ? @message.raw_payload : @safe_payload.result
+              @displayable_payload_size = ObjectSpace.memsize_of(memsize_base)
+
               # This may be off for certain views like recent view where we are interested only
               # in the most recent all the time. It does not make any sense to display pagination
               # there

lib/karafka/web/pro/ui/views/explorer/message/_metadata.erb

@@ -55,6 +55,18 @@
           )
         %>
 
+        <% unless @safe_payload.failure? %>
+          <%==
+            partial(
+              'explorer/messages/detail',
+              locals: {
+                k: 'memsize',
+                v: format_memory((@displayable_payload_size / 1024.to_f).round(4))
+              }
+            )
+          %>
+        <% end %>
+
         <%==
           partial(
             'explorer/messages/headers',

lib/karafka/web/pro/ui/views/explorer/message/_payload.erb

@@ -2,18 +2,27 @@
   <div class="row">
     <div class="col-sm-12 mb-4">
       <% if @safe_payload.failure? %>
-        <%== partial 'explorer/failed_deserialization', locals: { attribute: 'payload', error: @safe_payload.error } %>
+        <%==
+          partial(
+            'explorer/failed_deserialization',
+            locals: { attribute: 'payload', error: @safe_payload.error }
+          )
+        %>
       <% end %>
 
-      <div class="card">
-        <div class="card-body">
-          <% if @safe_payload.failure? %>
-            <pre class="m-0 p-0"><code class="wrapped json p-0 m-0"><%= @message.raw_payload %></code></pre>
-          <% else %>
-            <pre class="m-0 p-0"><code class="wrapped json p-0 m-0"><%= @safe_payload.result %></code></pre>
-          <% end %>
+      <% if @displayable_payload_size > Karafka::Web.config.ui.max_visible_payload_size %>
+        <%== partial('explorer/message/too_big_to_be_displayed') %>
+      <% else %>
+        <div class="card">
+          <div class="card-body">
+            <% if @safe_payload.failure? %>
+              <pre class="m-0 p-0"><code class="wrapped json p-0 m-0"><%= @message.raw_payload %></code></pre>
+            <% else %>
+              <pre class="m-0 p-0"><code class="wrapped json p-0 m-0"><%= @safe_payload.result %></code></pre>
+            <% end %>
+          </div>
         </div>
-      </div>
+      <% end %>
     </div>
   </div>
 <% else %>

lib/karafka/web/pro/ui/views/explorer/message/_too_big_to_be_displayed.erb

@@ -0,0 +1,9 @@
+<div class="alert alert-warning" role="alert">
+  Message payloads larger than
+  <strong>
+  <%=
+    format_memory((Karafka::Web.config.ui.max_visible_payload_size / 1024.to_f).round(4))
+  %>
+  </strong>
+  are not displayed to prevent browser performance issues. This limit ensures a smooth and responsive user experience. If you need to view larger payloads, please download the message and inspect it using an appropriate tool.
+</div>

lib/karafka/web/ui/lib/sorter.rb

@@ -33,7 +33,7 @@ def initialize(sort_query, allowed_attributes:)
             @field = '' unless allowed_attributes.include?(@field)
 
             # Things we have already seen and sorted. Prevents crashing on the circular
-            # dependencies sorting when same resources are present in different parts of the three
+            # dependencies sorting when same resources are present in different parts of the tree
             @seen = {}
           end
 

spec/lib/karafka/web/pro/ui/controllers/explorer_controller_spec.rb

@@ -547,6 +547,37 @@
       end
     end
 
+    context 'when requested message exists but is too big to be presented' do
+      before do
+        topic_name = topic
+        draw_routes do
+          topic topic_name do
+            active(false)
+            deserializers(payload: ::Karafka::Web::Deserializer.new)
+          end
+        end
+
+        # This will send a compressed message that after unpack will be bigger than what we
+        # decide to fit into memory by default
+        produce(
+          topic,
+          Fixtures.consumers_metrics_file('compressed/v1.4.0_large.msg'),
+          headers: { 'zlib' => '1' }
+        )
+        get "explorer/#{topic}/0/0"
+      end
+
+      it do
+        expect(response).to be_ok
+        expect(body).to include(breadcrumbs)
+        expect(body).not_to include('<code class="wrapped json')
+        expect(body).to include('Metadata')
+        expect(body).to include('Message payloads larger than')
+        expect(body).not_to include(pagination)
+        expect(body).not_to include(support_message)
+      end
+    end
+
     context 'when message exists but cannot be deserialized' do
       before do
         produce(topic, '{1=')