From 1b6c1ba939179ca07c18e687496e702306a7024d Mon Sep 17 00:00:00 2001
From: Keegan George <kgeorge13@gmail.com>
Date: Wed, 19 Feb 2025 13:25:01 -0800
Subject: [PATCH 1/3] DEV: Add eval for summarization

---
 evals/lib/eval.rb | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/evals/lib/eval.rb b/evals/lib/eval.rb
index 280ab8713..bda0e32b6 100644
--- a/evals/lib/eval.rb
+++ b/evals/lib/eval.rb
@@ -60,6 +60,8 @@ def run(llm:)
         prompt_call(llm, **args)
       when "edit_artifact"
         edit_artifact(llm, **args)
+      when "summarization"
+        summarization(llm, **args)
       end
 
     if expected_output
@@ -312,4 +314,35 @@ def valid_javascript?(str)
   rescue StandardError
     false
   end
+
+  def summarization(llm, input:)
+    topic =
+      Topic.create!(
+        category: Category.last,
+        title: "Eval topic for topic summarization",
+        id: -99,
+        user_id: Discourse.system_user.id,
+      )
+    post = Post.create!(topic: topic, id: -99, user_id: Discourse.system_user.id, raw: input)
+
+    strategy =
+      DiscourseAi::Summarization::FoldContent.new(
+        llm.llm_proxy,
+        DiscourseAi::Summarization::Strategies::TopicSummary.new(topic),
+      )
+
+    result = []
+
+    summary =
+      DiscourseAi::TopicSummarization
+        .new(strategy, Discourse.system_user)
+        .summarize { |partial| result << partial }
+
+    summarized_text = summary.summarized_text
+
+    topic.destroy
+    post.destroy
+
+    summarized_text
+  end
 end

From 8af2ccd2b3086539747ab902d5354f0d07e4aa83 Mon Sep 17 00:00:00 2001
From: Keegan George <kgeorge13@gmail.com>
Date: Wed, 19 Feb 2025 13:30:01 -0800
Subject: [PATCH 2/3] DEV: No need to add partial to result

---
 evals/lib/eval.rb | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/evals/lib/eval.rb b/evals/lib/eval.rb
index bda0e32b6..bf16ae69a 100644
--- a/evals/lib/eval.rb
+++ b/evals/lib/eval.rb
@@ -331,13 +331,7 @@ def summarization(llm, input:)
         DiscourseAi::Summarization::Strategies::TopicSummary.new(topic),
       )
 
-    result = []
-
-    summary =
-      DiscourseAi::TopicSummarization
-        .new(strategy, Discourse.system_user)
-        .summarize { |partial| result << partial }
-
+    summary = DiscourseAi::TopicSummarization.new(strategy, Discourse.system_user).summarize
     summarized_text = summary.summarized_text
 
     topic.destroy

From 9822a33b71d0604c2c90c3255c2bab37552721b3 Mon Sep 17 00:00:00 2001
From: Keegan George <kgeorge13@gmail.com>
Date: Wed, 19 Feb 2025 15:01:40 -0800
Subject: [PATCH 3/3] DEV: Didn't actually need to create, `.new` sufficient

---
 evals/lib/eval.rb | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/evals/lib/eval.rb b/evals/lib/eval.rb
index bf16ae69a..4a62e3329 100644
--- a/evals/lib/eval.rb
+++ b/evals/lib/eval.rb
@@ -317,13 +317,13 @@ def valid_javascript?(str)
 
   def summarization(llm, input:)
     topic =
-      Topic.create!(
+      Topic.new(
         category: Category.last,
         title: "Eval topic for topic summarization",
         id: -99,
         user_id: Discourse.system_user.id,
       )
-    post = Post.create!(topic: topic, id: -99, user_id: Discourse.system_user.id, raw: input)
+    Post.new(topic: topic, id: -99, user_id: Discourse.system_user.id, raw: input)
 
     strategy =
       DiscourseAi::Summarization::FoldContent.new(
@@ -332,11 +332,6 @@ def summarization(llm, input:)
       )
 
     summary = DiscourseAi::TopicSummarization.new(strategy, Discourse.system_user).summarize
-    summarized_text = summary.summarized_text
-
-    topic.destroy
-    post.destroy
-
-    summarized_text
+    summary.summarized_text
   end
 end