Provide search capabilities in the Explorer (#341)

CHANGELOG.md

@@ -1,12 +1,17 @@
 # Karafka Web changelog
 
-## 0.9.2 (Unreleased)
+## 0.10.0 (Unreleased)
+- **[Feature]** Provide Search capabilities in the Explorer (Pro).
+- [Enhancement] Display raw numerical timestamp alongside message time.
+- [Enhancement] Support `/topics` root redirect.
 - [Enhancement] Prevent explorer from displaying too big payloads (bigger than 1MB by default)
 - [Enhancement] Include memsize based on `ObjectSpace.memsize_of` if `#memsize_of` is available.
 - [Enhancement] Improve how charts with many topics work.
 - [Enhancement] Count and display executed jobs independently from processed batches.
 - [Enhancement] Prevent karafka-web from being configured before karafka is configured.
 - [Refactor] Namespace migrations so migrations related to each topic data are in an independent directory.
+- [Fix] Fix invalid deserialization metadata display in the per-message Explorer view.
+- [Fix] Fix a case where started page refresh would update content despite limiters being in place.
 
 ## 0.9.1 (2024-05-03)
 - [Fix] OSS `lag_stored` for not-subscribed consumers causes Web UI to crash.

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.9.2)
+    karafka-web (0.10.0)
       erubi (~> 1.4)
       karafka (>= 2.4.0, < 2.5.0)
       karafka-core (>= 2.4.0, < 2.5.0)

config/locales/pro_errors.yml

@@ -9,3 +9,19 @@ en:
       commanding.pause_timeout_format: needs to be an integer bigger than 0
       key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       commanding.kafka_format: needs to be a filled hash
+      ui.search.matchers_must_have_name_and_call: 'must respond to #name and its instance to #call'
+      ui.search.matchers_format: must be an array with matchers
+      ui.search.matchers_name_must_be_valid: all matchers names must be non-empty strings
+      ui.search.limits_format: all limits need to be integers bigger than 0
+      ui.search.timeout_format: must be at least 1 ms
+
+    search_form:
+      missing: needs to be present
+      timestamp_key_must_be_large_enough: 'must be a Kafka message timestamp with ms precision'
+      matcher_format: must match the existing matchers names
+      limit_format: must be one of the predefined limits
+      phrase_format: must be a non-empty string
+      offset_type_format: must be latest, offset or a timestamp
+      offset_format: needs to be an integer bigger than 0
+      partitions_format: needs to include "all" or partitions ids
+      timestamp_format: must be a Kafka message timestamp with ms precision

lib/karafka/web.rb

@@ -7,6 +7,8 @@
   open3
   zlib
   securerandom
+  cgi
+  uri
 ].each { |lib| require lib }
 
 module Karafka

lib/karafka/web/config.rb

@@ -162,11 +162,6 @@ class Config
         # transaction that will cause given consumer group to halt processing and wait
         setting :lso_threshold, default: 5 * 60 * 1_000
 
-        # Allows to manage visibility of payload, headers and message key in the UI
-        # In some cases you may want to limit what is being displayed due to the type of data you
-        # are dealing with
-        setting :visibility_filter, default: Ui::Models::VisibilityFilter.new
-
         # Consider any topic matching those names as a DLQ topic for the DLQ view
         # Web UI uses auto DLQ discovery based on routing but this may not be fully operable when
         # using a multi-app setup. This config allows to add extra topics if needed without having

lib/karafka/web/contracts/base.rb

@@ -10,6 +10,8 @@ class << self
           # This layer is not for users extensive feedback, thus we can easily use the minimum
           # error messaging there is.
           def configure
+            return super if block_given?
+
             super do |config|
               config.error_messages = YAML.safe_load(
                 File.read(

lib/karafka/web/pro/loader.rb

@@ -25,13 +25,19 @@ def pre_setup_all(config)
             config.instance_eval do
               setting(:commanding, default: Commanding::Config.config)
             end
+
+            # Expand UI config with extra search capabilities settings
+            config.ui.instance_eval do
+              setting(:search, default: Ui::Lib::Search::Config.config)
+            end
           end
 
           # Runs post setup features configuration operations
           #
           # @param config [Karafka::Core::Configurable::Node]
           def post_setup_all(config)
             Commanding.post_setup(config)
+            Ui::Lib::Search.post_setup(config)
           end
         end
       end

lib/karafka/web/pro/ui/app.rb

@@ -155,6 +155,13 @@ class App < Web::Ui::Base
             end
 
             r.on 'explorer' do
+              r.get String, 'search' do |topic_id|
+                # Search has it's own controller but we want to have this in the explorer routing
+                # namespace because topic search is conceptually part of the explorer
+                controller = Controllers::SearchController.new(params)
+                controller.index(topic_id)
+              end
+
               controller = Controllers::ExplorerController.new(params)
 
               r.get String, Integer, 'recent' do |topic_id, partition_id|
@@ -262,16 +269,20 @@ class App < Web::Ui::Base
             r.on 'topics' do
               controller = Controllers::TopicsController.new(params)
 
-              r.get String, 'config' do |topic_name|
-                controller.config(topic_name)
+              r.get String, 'config' do |topic_id|
+                controller.config(topic_id)
               end
 
-              r.get String, 'replication' do |topic_name|
-                controller.replication(topic_name)
+              r.get String, 'replication' do |topic_id|
+                controller.replication(topic_id)
               end
 
-              r.get String, 'distribution' do |topic_name|
-                controller.distribution(topic_name)
+              r.get String, 'distribution' do |topic_id|
+                controller.distribution(topic_id)
+              end
+
+              r.get String do |topic_id|
+                r.redirect root_path('topics', topic_id, 'config')
               end
 
               r.get do

lib/karafka/web/pro/ui/controllers/search_controller.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Handles the search requests
+          # We present this as a part of explorer scope but we use a separate controller not to
+          # mix data exploring with searching.
+          class SearchController < Web::Ui::Controllers::ClusterController
+            # Runs the search if search parameters are provided
+            # If no parameters provided, displays the search modal and info to provide search data
+            # If invalid search parameters provided, modal contains errors
+            #
+            # @param topic_id [String] topic we're interested in
+            # @note In theory search can be used to detect pieces of information within messages.
+            #   Since we allow for custom search strategies, this is not an issue because users
+            #   that need to provide only granular search can do so.
+            def index(topic_id)
+              @topic_id = topic_id
+              @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
+              @matchers = Web.config.ui.search.matchers
+              @search_criteria = !@params.current_search.empty?
+              @current_search = Lib::Search::Normalizer.call(@params.current_search)
+              # Needed when rendering found messages rows. We should always filter the messages
+              # details with the visibility filter
+              @visibility_filter = ::Karafka::Web.config.ui.visibility.filter
+              @limits = ::Karafka::Web.config.ui.search.limits.sort
+
+              # If there is search form filled, we validate it to make sure there are no errors
+              @errors = if @search_criteria
+                          Lib::Search::Contracts::Form.new.call(@current_search).errors
+                        else
+                          {}
+                        end
+
+              # If all good we run the search
+              if @search_criteria && @errors.empty?
+                found, @search_details = Lib::Search::Runner.new(
+                  @topic_id,
+                  @partitions_count,
+                  @current_search
+                ).call
+
+                @messages, last_page = Paginators::Arrays.call(
+                  found,
+                  @params.current_page
+                )
+
+                paginate(@params.current_page, !last_page)
+              end
+
+              render
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/karafka/web/pro/ui/lib/search.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          # All code needed to support the search functionality
+          # It's not in models because it does not provide "data" per se but rather fetches it
+          # from a Kafka.
+          module Search
+            class << self
+              # Validates that the UI search config is correct
+              #
+              # @param config [Karafka::Core::Configurable::Node] web config
+              def post_setup(config)
+                Search::Contracts::Config.new.validate!(config.to_h)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/karafka/web/pro/ui/lib/search/config.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Search
+            # Extra configuration for pro UI
+            class Config
+              extend ::Karafka::Core::Configurable
+
+              # Matches for messages
+              setting :matchers, default: [
+                Matchers::RawPayloadIncludes,
+                Matchers::RawKeyIncludes,
+                Matchers::RawHeaderIncludes
+              ]
+
+              # How long should we at most search before stopping (in ms)
+              # This prevents us from having a search that would basically hang the browser and
+              # never finish
+              setting :timeout, default: 30_000
+
+              # Search limits as the search runs in Puma
+              # Too big value will make the scan really slow
+              # We do not put a max limit here because those values are used in a select in
+              # the Web UI.
+              setting :limits, default: [
+                1_000,
+                10_000,
+                100_000
+              ]
+
+              configure
+            end
+          end
+        end
+      end
+    end
+  end
+end