PERF: Defer sending env to client until it's needed (#96)

* PERF: Defer sending env to client until it's needed

Currently when you go to Logster homepage, the server fetches the last
50 messages with their envs and send them to your browser. 50 messages
with their envs can be 5+ MB in size. This commit changes the server so
now it'll only serve the messages without their envs, and when you
select a message and switch to the env tab, the client will request the
env associated with the message you selected via the new endpoint
`/fetch-env/${msg.key}.json`.

* compile assets

client-app/app/components/message-info.js

@@ -46,6 +46,12 @@ export default Component.extend({
   }),
 
   actions: {
+    tabChanged(newTab) {
+      if (this.onTabChange) {
+        this.onTabChange(newTab);
+      }
+    },
+
     protect() {
       this.get("currentMessage").protect();
     },

client-app/app/components/tabbed-section.js

@@ -14,6 +14,7 @@ export default Component.extend({
     }
     this.set("selected", view);
     view.set("active", true);
+    this.onTabChange(view.name);
   },
 
   addTab(tab) {

client-app/app/controllers/index.js

@@ -12,6 +12,7 @@ export default Controller.extend({
   showFatal: true,
   search: "",
   currentMessage: Em.computed.alias("model.currentMessage"),
+  currentTab: null,
 
   showSettings: computed(function() {
     return Preload.get("patterns_enabled");
@@ -26,6 +27,16 @@ export default Controller.extend({
     return this.site.isMobile;
   }),
 
+  fetchEnv() {
+    const message = this.get("currentMessage");
+    if (message) {
+      this.set("loadingEnv", true);
+      return ajax(`/fetch-env/${message.key}.json`)
+        .then(env => message.set("env", env))
+        .always(() => this.set("loadingEnv", false));
+    }
+  },
+
   actions: {
     expandMessage(message) {
       message.expand();
@@ -38,7 +49,23 @@ export default Controller.extend({
       }
 
       message.set("selected", true);
-      this.set("currentMessage", message);
+      this.setProperties({
+        currentMessage: message,
+        loadingEnv: false
+      });
+      if (!message.env && this.currentTab === "env") {
+        this.fetchEnv();
+      }
+    },
+
+    tabChanged(newTab) {
+      this.setProperties({
+        currentTab: newTab,
+        loadingEnv: false
+      });
+      if (newTab === "env" && !this.get("currentMessage.env")) {
+        this.fetchEnv();
+      }
     },
 
     showMoreBefore() {

client-app/app/templates/components/message-info.hbs

@@ -1,5 +1,5 @@
 <div class="message-info">
-  {{#tabbed-section}}
+  {{#tabbed-section onTabChange=(action "tabChanged")}}
     {{#tab-contents name="info" hint="show info" currentMessage=currentMessage}}
       {{#if showTitle}}
         <h3>Message
@@ -16,14 +16,18 @@
       {{/if}}
       <pre>{{currentMessage.backtrace}}</pre>
     {{/tab-contents}}
-    {{#if currentMessage.env}}
-      {{#tab-contents className="env" name="env" hint="show environment" currentMessage=currentMessage}}
+    {{#tab-contents className="env" name="env" hint="show environment" currentMessage=currentMessage }}
+      {{#if currentMessage.env}}
         {{#if showTitle}}
           <h3>Env</h3>
         {{/if}}
         {{env-tab message=currentMessage}}
-      {{/tab-contents}}
-    {{/if}}
+      {{else if loadingEnv}}
+        Loading env...
+      {{else}}
+        No env for this message.
+      {{/if}}
+    {{/tab-contents}}
   {{/tabbed-section}}
 
   {{#if currentMessage}}

client-app/app/templates/index.hbs

@@ -17,8 +17,10 @@
 <div id="bottom-panel">
   {{message-info
     currentMessage=currentMessage
+    loadingEnv=loadingEnv
     removeMessage=(action "removeMessage")
     solveMessage=(action "solveMessage")
+    onTabChange=(action "tabChanged")
     actionsInMenu=actionsInMenu}}
 
   <div class="action-panel">

lib/logster/middleware/viewer.rb

@@ -156,6 +156,14 @@ def call(env)
             [200, {}, ["OK"]]
           elsif resource == "/"
             [200, { "Content-Type" => "text/html; charset=utf-8" }, [body(preload_json)]]
+          elsif resource =~ /\/fetch-env\/([0-9a-f]+)\.json$/
+            key = $1
+            env = Logster.store.get_env(key)
+            if env
+              [200, { "Content-Type" => "application/json; charset=utf-8" }, [JSON.generate(env)]]
+            else
+              not_found
+            end
           else
             not_found
           end
@@ -190,6 +198,8 @@ def serve_messages(req)
           search = (parse_regex(search) || search) if params["regex_search"] == "true"
           opts[:search] = search
         end
+        search = opts[:search]
+        opts[:with_env] = (String === search && search.size > 0) || Regexp === search
 
         payload = {
           messages: @store.latest(opts),

lib/logster/redis_store.rb

@@ -98,6 +98,7 @@ def latest(opts = {})
       before = opts[:before]
       after = opts[:after]
       search = opts[:search]
+      with_env = opts.key?(:with_env) ? opts[:with_env] : true
 
       start, finish = find_location(before, after, limit)
 
@@ -110,7 +111,7 @@ def latest(opts = {})
       begin
         keys = @redis.lrange(list_key, start, finish) || []
         break unless keys && (keys.count > 0)
-        rows = bulk_get(keys)
+        rows = bulk_get(keys, with_env: with_env)
 
         temp = []
 
@@ -204,14 +205,16 @@ def get_all_messages
       bulk_get(@redis.lrange(list_key, 0, -1))
     end
 
-    def bulk_get(message_keys)
-      envs = @redis.hmget(env_key, message_keys)
-      @redis.hmget(hash_key, message_keys).map!.with_index do |json, ind|
+    def bulk_get(message_keys, with_env: true)
+      envs = @redis.mapped_hmget(env_key, *message_keys) if with_env
+      @redis.hmget(hash_key, message_keys).map! do |json|
         message = Message.from_json(json)
-        env = envs[ind]
-        if !message.env || message.env.size == 0
-          env = env && env.size > 0 ? ::JSON.parse(env) : {}
-          message.env = env
+        if with_env
+          env = envs[message.key]
+          if !message.env || message.env.size == 0
+            env = env && env.size > 0 ? ::JSON.parse(env) : {}
+            message.env = env
+          end
         end
         message
       end

test/logster/middleware/test_viewer.rb

@@ -328,4 +328,44 @@ def test_linking_to_an_invalid_ember_component_or_template
       assert_equal(404, response.status, "#{path} should have 404'ed")
     end
   end
+
+  def test_messages_endpoint_doesnt_include_envs_when_search_term_absent
+    Logster.store.clear_all
+    env = { "b" => 1, "c" => 2 }
+    msg = Logster.store.report(Logger::INFO, "test", "something hello", env: env)
+    response = request.get("/logsie/messages.json")
+    assert_equal(200, response.status)
+    messages = JSON.parse(response.body)["messages"]
+    assert_equal(1, messages.size)
+    msg = messages.first
+    assert_equal("something hello", msg["message"])
+    assert_nil(msg["env"])
+  end
+
+  def test_messages_endpoint_includes_env_when_there_is_search_term
+    Logster.store.clear_all
+    env = { "b" => 1, "c" => 2 }
+    msg = Logster.store.report(Logger::INFO, "test", "something hello", env: env)
+    response = request.get("/logsie/messages.json?search=something")
+    assert_equal(200, response.status)
+    messages = JSON.parse(response.body)["messages"]
+    assert_equal(1, messages.size)
+    msg = messages.first
+    assert_equal("something hello", msg["message"])
+    assert_includes(msg["env"].values, 1, 2)
+  end
+
+  def test_fetch_env_returns_env_associated_with_message
+    env = { "b" => 1, "c" => 2 }
+    msg = Logster.store.report(Logger::INFO, "test", "something whatever store", env: env)
+    response = request.get("/logsie/fetch-env/#{msg.key}.json")
+    assert_equal(200, response.status)
+    res = JSON.parse(response.body)
+    assert_includes(res.values, 1, 2)
+  end
+
+  def test_fetch_env_returns_404_when_invalid_key
+    response = request.get("/logsie/fetch-env/123456abc.json")
+    assert_equal(404, response.status)
+  end
 end

website/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    logster (2.3.3)
+    logster (2.4.0)
 
 GEM
   remote: https://rubygems.org/
@@ -29,4 +29,4 @@ DEPENDENCIES
   sinatra
 
 BUNDLED WITH
-   1.17.3
+   2.0.1