Updating

scripts/exp/tools/collect.py

@@ -1,24 +1,64 @@
 import os
 import json
+from pathlib import Path
+from datetime import datetime
 
 def collect_results(dir_path) -> list[dict]:
     summary = []
-    for root, _, filies in os.walk(dir_path):
-        for file in filies:
-            if file.endswith(".json"):
+    for root, _, files in os.walk(dir_path):
+        for file in files:
+            if file.endswith("_result.json"):
+                config_name = file.replace("_result.json", "")
                 with open(os.path.join(root, file), "r") as f:
                     data = json.load(f)
-                    summary.append(data)
+                    # Extract both CV and Kaggle submission results
+                    summary.append({
+                        "config": config_name,
+                        "cv_results": data.get("cv_score", None),
+                        "kaggle_score": data.get("kaggle_score", None),
+                        "trace": data.get("trace", {})
+                    })
     return summary
 
 def generate_summary(results, output_path):
-    # First analyze the results and generate a summary
-    # For each experiment, we find the best result, the metric, and result trajectory
-    #TODO: Implement this 
-
-    # Then write the summary to the output path
+    summary = {
+        "configs": {},
+        "best_cv_result": {"config": None, "score": None},
+        "best_kaggle_result": {"config": None, "score": None},
+        "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S")
+    }
+
+    for result in results:
+        config = result["config"]
+        metrics = {
+            "cv_score": result["cv_results"],
+            "kaggle_score": result["kaggle_score"],
+            "iterations": len(result["trace"].get("steps", [])),
+            "best_model": result["trace"].get("best_model")
+        }
+
+        summary["configs"][config] = metrics
+
+        # Update best CV result
+        if (metrics["cv_score"] is not None and 
+            (summary["best_cv_result"]["score"] is None or 
+             metrics["cv_score"] > summary["best_cv_result"]["score"])):
+            summary["best_cv_result"].update({
+                "config": config,
+                "score": metrics["cv_score"]
+            })
+
+        # Update best Kaggle result
+        if (metrics["kaggle_score"] is not None and 
+            (summary["best_kaggle_result"]["score"] is None or 
+             metrics["kaggle_score"] > summary["best_kaggle_result"]["score"])):
+            summary["best_kaggle_result"].update({
+                "config": config,
+                "score": metrics["kaggle_score"]
+            })
+
     with open(output_path, "w") as f:
-        json.dump(results, f, indent = 4)
+        json.dump(summary, f, indent=4)
 
 if __name__ == "__main__":
     result_dir = os.path.join(os.getenv("EXP_DIR"), "results")

scripts/exp/tools/test_system.sh

@@ -3,19 +3,25 @@
 # Test directory setup
 TEST_DIR="test_run"
 mkdir -p "$TEST_DIR/results"
+mkdir -p "$TEST_DIR/logs"
 
-# Test 1: Environment loading
+# Test 1: Environment loading verification
 echo "Testing environment loading..."
 ./scripts/exp/tools/run_envs.sh -d scripts/exp/ablation/env -j 1 -- env | grep "if_using"
 
-# Test 2: Parallel execution
-echo "Testing parallel execution..."
+# Test 2: Run actual experiments
+echo "Running experiments with different configurations..."
 ./scripts/exp/tools/run_envs.sh -d scripts/exp/ablation/env -j 4 -- \
-    echo "Processing env with RAG setting: $if_using_vector_rag"
+    python -m rdagent.app.kaggle.loop \
+    --competition "titanic" \
+    --result_path "${TEST_DIR}/results/$(basename {} .env)_result.json"
 
 # Test 3: Result collection
-echo "Testing result collection..."
+echo "Collecting and analyzing results..."
 EXP_DIR="$TEST_DIR" python scripts/exp/tools/collect.py
 
+# Display results location
+echo "Test results available at: $TEST_DIR"
+
 # Cleanup
 rm -rf "$TEST_DIR"