Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: change log object #553

Open
wants to merge 18 commits into
base: main
Choose a base branch
from
+602 −1
Open

chore: change log object #553

Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
cf3547a
change_log_object
SunsetWolf Feb 5, 2025
120e5c1
lint code
SunsetWolf Feb 5, 2025
82f2c10
delete comments
SunsetWolf Feb 5, 2025
1f924fd
change_log_object
SunsetWolf Feb 6, 2025
69c709a
change_log_object
SunsetWolf Feb 6, 2025
54c2028
fix import test error
SunsetWolf Feb 6, 2025
5fd1a6d
update code
SunsetWolf Feb 16, 2025
6040976
update code
SunsetWolf Feb 19, 2025
330563a
fix bugs
SunsetWolf Feb 24, 2025
3c19613
skip mypy error
SunsetWolf Feb 27, 2025
f8a1e0a
skip mypy error
SunsetWolf Feb 27, 2025
cd3d69b
skip mypy error
SunsetWolf Feb 27, 2025
a2015cf
Merge branch 'main' of github.com:microsoft/RD-Agent into change_log_…
SunsetWolf Feb 27, 2025
2f1c1a2
Start the flask server before running the demo.
SunsetWolf Feb 28, 2025
dad1844
achieve front and back interaction
SunsetWolf Mar 6, 2025
4b92bb3
Merge branch 'main' of github.com:microsoft/RD-Agent into change_log_…
SunsetWolf Mar 7, 2025
d874e94
fix github-advanced-security comments
SunsetWolf Mar 7, 2025
1b46f92
fix github-advanced-security comments
SunsetWolf Mar 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
lint code
  • Loading branch information
@SunsetWolf
SunsetWolf committed Feb 5, 2025
commit 120e5c1d65e9716c2cdb3a4b1022f6683c361086
235 changes: 129 additions & 106 deletions rdagent/log/logger.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import json
import os
import pickle
import plotly
import sys
from collections import defaultdict
from contextlib import contextmanager
@@ -13,26 +12,26 @@
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, Generator, Union

import plotly
from loguru import logger

if TYPE_CHECKING:
from loguru import Record

from psutil import Process

from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.proposal import Hypothesis, HypothesisFeedback
from rdagent.core.utils import SingletonBaseClass

from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedbackDeprecated
from rdagent.components.coder.factor_coder.evaluators import FactorSingleFeedback
from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask
from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.proposal import Hypothesis, HypothesisFeedback
from rdagent.core.utils import SingletonBaseClass

from .storage import FileStorage
from .utils import LogColors, get_caller_info
from .ui.llm_st import extract_loopid_func_name
from .ui.qlib_report_figure import report_figure
from .utils import LogColors, get_caller_info


class RDAgentLog(SingletonBaseClass):
@@ -131,135 +130,159 @@ def format_pkl(self, base_path: str | Path):
lp = extract_loopid_func_name(m.tag)
lp_id = lp[0] if lp and lp[0] is not None else None
# lp_id = (lp := extract_loopid_func_name(m.tag))[0] if lp[0] is not None else None
if 'r.hypothesis generation' in m.tag:
if "r.hypothesis generation" in m.tag:
h: Hypothesis = m.content
self.msgs_for_frontend[did].append({
"tag": "research.hypothesis",
"timestamp": m.timestamp.isoformat(),
"content": {
"name_map": {
"hypothesis": "RD-Agent proposes the hypothesis⬇️",
"concise_justification": "because the reason⬇️",
"concise_observation": "based on the observation⬇️",
"concise_knowledge": "Knowledge⬇️ gained after practice",
self.msgs_for_frontend[did].append(
{
"tag": "research.hypothesis",
"timestamp": m.timestamp.isoformat(),
"content": {
"name_map": {
"hypothesis": "RD-Agent proposes the hypothesis⬇️",
"concise_justification": "because the reason⬇️",
"concise_observation": "based on the observation⬇️",
"concise_knowledge": "Knowledge⬇️ gained after practice",
},
"hypothesis": h.hypothesis,
"concise_justification": h.concise_justification,
"concise_observation": h.concise_observation,
"concise_knowledge": h.concise_knowledge,
},
"hypothesis": h.hypothesis,
"concise_justification": h.concise_justification,
"concise_observation": h.concise_observation,
"concise_knowledge": h.concise_knowledge,
}
})
)

# m.tag 中不存在 d.load_experiment, 存在 r.load_experiment
elif 'r.experiment generation' in m.tag or 'd.load_experiment' in m.tag:
if 'd.load_experiment' in m.tag:
elif "r.experiment generation" in m.tag or "d.load_experiment" in m.tag:
if "d.load_experiment" in m.tag:
tasks: list[FactorTask | ModelTask] = m.content.sub_tasks
else:
tasks: list[FactorTask | ModelTask] = m.content
if isinstance(tasks[0], FactorTask):
self.msgs_for_frontend[did].append({
"tag": "research.tasks",
"timestamp": m.timestamp.isoformat(),
"content": [
{
"name": t.factor_name,
"description": t.factor_description,
"formulation": t.factor_formulation,
"variables": t.variables,
} for t in tasks
]
})
self.msgs_for_frontend[did].append(
{
"tag": "research.tasks",
"timestamp": m.timestamp.isoformat(),
"content": [
{
"name": t.factor_name,
"description": t.factor_description,
"formulation": t.factor_formulation,
"variables": t.variables,
}
for t in tasks
],
}
)
elif isinstance(tasks[0], ModelTask):
self.msgs_for_frontend[did].append({
"tag": "research.tasks",
"timestamp": m.timestamp.isoformat(),
"content": [
{
"name": t.name,
"description": t.description,
"model_type": t.model_type,
"formulation": t.formulation,
"variables": t.variables,
} for t in tasks
]
})
self.msgs_for_frontend[did].append(
{
"tag": "research.tasks",
"timestamp": m.timestamp.isoformat(),
"content": [
{
"name": t.name,
"description": t.description,
"model_type": t.model_type,
"formulation": t.formulation,
"variables": t.variables,
}
for t in tasks
],
}
)

elif f"evo_loop_{lp_id}.evolving code" in m.tag:
ws: list[FactorFBWorkspace | ModelFBWorkspace] = [i for i in m.content]
self.msgs_for_frontend[did].append({
"tag": "evolving.codes",
"timestamp": m.timestamp.isoformat(),
"content": [
{
"target_task_name": w.target_task.name if isinstance(w.target_task, ModelTask) else w.target_task.factor_name,
"code": w.file_dict
} for w in ws if w
]
})
self.msgs_for_frontend[did].append(
{
"tag": "evolving.codes",
"timestamp": m.timestamp.isoformat(),
"content": [
{
"target_task_name": (
w.target_task.name
if isinstance(w.target_task, ModelTask)
else w.target_task.factor_name
),
"code": w.file_dict,
}
for w in ws
if w
],
}
)

elif f"evo_loop_{lp_id}.evolving feedback" in m.tag:
fl: list[FactorSingleFeedback | CoSTEERSingleFeedbackDeprecated] = [i for i in m.content]
self.msgs_for_frontend[did].append({
"tag": "evolving.feedbacks",
"timestamp": m.timestamp.isoformat(),
"content": [
{
"final_decision": f.final_decision,
"final_feedback": f.final_feedback,
"execution_feedback": f.execution_feedback,
"code_feedback": f.code_feedback,
"value_feedback": f.value_feedback if isinstance(f, CoSTEERSingleFeedbackDeprecated) else f.factor_value_feedback,
"model_shape_feedback": f.shape_feedback if isinstance(f, CoSTEERSingleFeedbackDeprecated) else None,
} for f in fl if f
]
})
self.msgs_for_frontend[did].append(
{
"tag": "evolving.feedbacks",
"timestamp": m.timestamp.isoformat(),
"content": [
{
"final_decision": f.final_decision,
"final_feedback": f.final_feedback,
"execution_feedback": f.execution_feedback,
"code_feedback": f.code_feedback,
"value_feedback": (
f.value_feedback
if isinstance(f, CoSTEERSingleFeedbackDeprecated)
else f.factor_value_feedback
),
"model_shape_feedback": (
f.shape_feedback if isinstance(f, CoSTEERSingleFeedbackDeprecated) else None
),
}
for f in fl
if f
],
}
)

elif "scenario" in m.tag:
self.msgs_for_frontend[did].append({
"tag": "feedback.config",
"timestamp": m.timestamp.isoformat(),
"content": {
"config": m.content.experiment_setting
self.msgs_for_frontend[did].append(
{
"tag": "feedback.config",
"timestamp": m.timestamp.isoformat(),
"content": {"config": m.content.experiment_setting},
}
})
)

elif "ef.Quantitative Backtesting Chart" in m.tag:
self.msgs_for_frontend[did].append({
"tag": "feedback.return_chart",
"timestamp": m.timestamp.isoformat(),
"content": {
"chart_html": plotly.io.to_html(report_figure(m.content))
self.msgs_for_frontend[did].append(
{
"tag": "feedback.return_chart",
"timestamp": m.timestamp.isoformat(),
"content": {"chart_html": plotly.io.to_html(report_figure(m.content))},
}
})
)

elif "model runner result" in m.tag or "factor runner result" in m.tag or "runner result" in m.tag:
self.msgs_for_frontend[did].append({
"tag": "feedback.metric",
"timestamp": m.timestamp.isoformat(),
"content": {
"result": m.content.result.iloc[0]
self.msgs_for_frontend[did].append(
{
"tag": "feedback.metric",
"timestamp": m.timestamp.isoformat(),
"content": {"result": m.content.result.iloc[0]},
}
})
)

elif "ef.feedback" in m.tag:
hf: HypothesisFeedback = m.content
self.msgs_for_frontend[did].append({
"tag": "feedback.hypothesis_feedback",
"timestamp": m.timestamp.isoformat(),
"content": {
"observations": hf.observations,
"hypothesis_evaluation": hf.hypothesis_evaluation,
"new_hypothesis": hf.new_hypothesis,
"decision": hf.decision,
"reason": hf.reason,
self.msgs_for_frontend[did].append(
{
"tag": "feedback.hypothesis_feedback",
"timestamp": m.timestamp.isoformat(),
"content": {
"observations": hf.observations,
"hypothesis_evaluation": hf.hypothesis_evaluation,
"new_hypothesis": hf.new_hypothesis,
"decision": hf.decision,
"reason": hf.reason,
},
}
})
)
for msgs in self.msgs_for_frontend.values():
msgs.append({
"tag": "END",
"timestamp": datetime.now(timezone.utc).isoformat(),
"content": {}
})
msgs.append({"tag": "END", "timestamp": datetime.now(timezone.utc).isoformat(), "content": {}})

return self.msgs_for_frontend