Updating the env files & Upading test_system file

scripts/exp/ablation/env/basic.env

@@ -1,5 +1,5 @@
-if_using_vector_rag=False
-if_using_graph_rag=False
-if_action_choosing_based_on_UCB=False
-model_feature_selection_coder=True
-hypothesis_gen=True 
+KG_IF_USING_VECTOR_RAG=False
+KG_IF_USING_GRAPH_RAG=False
+KG_IF_ACTION_CHOOSING_BASED_ON_UCB=False
+KG_MODEL_FEATURE_SELECTION_CODER=True
+KG_HYPOTHESIS_GEN=True

scripts/exp/ablation/env/max.env

@@ -1,5 +1,5 @@
-if_using_vector_rag=True
-if_using_graph_rag=True
-if_action_choosing_based_on_UCB=True
-model_feature_selection_coder=True
-hypothesis_gen=True 
+KG_IF_USING_VECTOR_RAG=True
+KG_IF_USING_GRAPH_RAG=True
+KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True
+KG_MODEL_FEATURE_SELECTION_CODER=True
+KG_HYPOTHESIS_GEN=True 

scripts/exp/ablation/env/mini-case.env

@@ -1,5 +1,5 @@
-if_using_vector_rag=True
-if_using_graph_rag=False
-if_action_choosing_based_on_UCB=True
-model_feature_selection_coder=True
-hypothesis_gen=False 
+KG_IF_USING_VECTOR_RAG=True
+KG_IF_USING_GRAPH_RAG=False
+KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True
+KG_MODEL_FEATURE_SELECTION_CODER=True
+KG_HYPOTHESIS_GEN=False 

scripts/exp/ablation/env/pro.env

@@ -1,5 +1,5 @@
-if_using_vector_rag=True
-if_using_graph_rag=False
-if_action_choosing_based_on_UCB=True
-model_feature_selection_coder=True
-hypothesis_gen=True 
+KG_IF_USING_VECTOR_RAG=True
+KG_IF_USING_GRAPH_RAG=False
+KG_IF_ACTION_CHOOSING_BASED_ON_UCB=True
+KG_MODEL_FEATURE_SELECTION_CODER=True
+KG_HYPOTHESIS_GEN=True 

scripts/exp/tools/collect.py

@@ -1,5 +1,6 @@
 import os
 import json
+import argparse
 from pathlib import Path
 from datetime import datetime
 from rdagent.log.storage import FileStorage
@@ -56,12 +57,27 @@ def generate_summary(results, output_path):
                     "competition_name": result["competition_name"]
                 })
 
-    with open(output_path, "w") as f:
+    with open(output_path, "w") as f: 
         json.dump(summary, f, indent=4)
 
+def parse_args():
+    parser = argparse.ArgumentParser(description='Collect and summarize experiment results')
+    parser.add_argument('--log_path', type=str, required=True,
+                       help='Path to the log directory containing experiment results')
+    parser.add_argument('--output_name', type=str, default='summary.json',
+                       help='Name of the output summary file (default: summary.json)')
+    return parser.parse_args()
+
 if __name__ == "__main__":
-    sample_result_dir = Path("/home/bowen/workspace/RD-Agent/log/MAY2022_5")
-    results = collect_results(sample_result_dir )
-    generate_summary(results, os.path.join(sample_result_dir, "summary.json"))
-    print("Summary generated successfully at ", os.path.join(sample_result_dir, "summary.json"))
+    args = parse_args()
+    log_path = Path(args.log_path)
+
+    # Verify the log path exists
+    if not log_path.exists():
+        raise FileNotFoundError(f"Log path does not exist: {log_path}")
+
+    results = collect_results(log_path)
+    output_path = log_path / args.output_name
+    generate_summary(results, output_path)
+    print("Summary generated successfully at", output_path)
 

scripts/exp/tools/test_system.sh

@@ -5,23 +5,15 @@ TEST_DIR="test_run"
 mkdir -p "$TEST_DIR/results"
 mkdir -p "$TEST_DIR/logs"
 
-# Test 1: Environment loading verification
-echo "Testing environment loading..."
-./scripts/exp/tools/run_envs.sh -d scripts/exp/ablation/env -j 1 -- env | grep "if_using"
+# Define paths
+ENV_DIR="/home/v-xisenwang/RD-Agent/scripts/exp/ablation/env"
+PYTHON_SCRIPT="/home/v-xisenwang/RD-Agent/rdagent/app/kaggle/loop.py"
 
-# Test 2: Run actual experiments
-echo "Running experiments with different configurations..."
-./scripts/exp/tools/run_envs.sh -d scripts/exp/ablation/env -j 4 -- \
-    python -m rdagent.app.kaggle.loop \
-    --competition "titanic" \
-    --result_path "${TEST_DIR}/results/$(basename {} .env)_result.json"
+# Run the experiment
+echo "Running experiments..."
+dotenv run -- ./scripts/exp/tools/run_envs.sh -d "$ENV_DIR" -j 4 -- \
+    python "$PYTHON_SCRIPT" \
+    --competition "spaceship-titanic" \ 
 
-# Test 3: Result collection
-echo "Collecting and analyzing results..."
-EXP_DIR="$TEST_DIR" python scripts/exp/tools/collect.py
-
-# Display results location
-echo "Test results available at: $TEST_DIR"
-
-# Cleanup
-rm -rf "$TEST_DIR"
+# Cleanup (optional - comment out if you want to keep results)
+# rm -rf "$TEST_DIR"