Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: change log object #553

Open
wants to merge 18 commits into
base: main
Choose a base branch
from
+602 −1
Open

chore: change log object #553

Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
cf3547a
change_log_object
SunsetWolf Feb 5, 2025
120e5c1
lint code
SunsetWolf Feb 5, 2025
82f2c10
delete comments
SunsetWolf Feb 5, 2025
1f924fd
change_log_object
SunsetWolf Feb 6, 2025
69c709a
change_log_object
SunsetWolf Feb 6, 2025
54c2028
fix import test error
SunsetWolf Feb 6, 2025
5fd1a6d
update code
SunsetWolf Feb 16, 2025
6040976
update code
SunsetWolf Feb 19, 2025
330563a
fix bugs
SunsetWolf Feb 24, 2025
3c19613
skip mypy error
SunsetWolf Feb 27, 2025
f8a1e0a
skip mypy error
SunsetWolf Feb 27, 2025
cd3d69b
skip mypy error
SunsetWolf Feb 27, 2025
a2015cf
Merge branch 'main' of github.com:microsoft/RD-Agent into change_log_…
SunsetWolf Feb 27, 2025
2f1c1a2
Start the flask server before running the demo.
SunsetWolf Feb 28, 2025
dad1844
achieve front and back interaction
SunsetWolf Mar 6, 2025
4b92bb3
Merge branch 'main' of github.com:microsoft/RD-Agent into change_log_…
SunsetWolf Mar 7, 2025
d874e94
fix github-advanced-security comments
SunsetWolf Mar 7, 2025
1b46f92
fix github-advanced-security comments
SunsetWolf Mar 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
update code
  • Loading branch information
@SunsetWolf
SunsetWolf committed Feb 16, 2025
commit 5fd1a6d9b446b4f794bccdcf856e0018e853b58e
2 changes: 2 additions & 0 deletions rdagent/log/logger.py
View file
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.utils import SingletonBaseClass

from .ui.utils import format_pkl
from .storage import FileStorage
from .utils import LogColors, get_caller_info

@@ -131,6 +132,7 @@ def log_object(self, obj: object, *, tag: str = "") -> None:
return

logp = self.storage.log(obj, name=tag, save_type="pkl")
format_pkl(obj=obj, tag=tag, log_trace_path=self.log_trace_path)

file_handler_id = logger.add(
self.log_trace_path / tag.replace(".", "/") / "common_logs.log", format=self.file_format
303 changes: 150 additions & 153 deletions rdagent/log/ui/utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,182 +1,179 @@
import typing
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path

import plotly

from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedbackDeprecated
from rdagent.components.coder.factor_coder.evaluators import FactorSingleFeedback
from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask
from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
from rdagent.core.proposal import Hypothesis, HypothesisFeedback
from rdagent.log.storage import FileStorage
from rdagent.log.ui.llm_st import extract_evoid, extract_loopid_func_name
from rdagent.log.ui.qlib_report_figure import report_figure

if typing.TYPE_CHECKING:
from rdagent.components.coder.CoSTEER.evaluators import (
CoSTEERSingleFeedbackDeprecated,
)
from rdagent.components.coder.factor_coder.evaluators import FactorSingleFeedback
from rdagent.components.coder.factor_coder.factor import (
FactorFBWorkspace,
FactorTask,
)
from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
from rdagent.core.experiment import Experiment
from rdagent.core.proposal import Hypothesis, HypothesisFeedback


msgs_for_frontend = defaultdict(list)


def format_pkl(base_path: str | Path):
base_path = Path(base_path)
dir2id = {dir_name.name: idx for idx, dir_name in enumerate(base_path.iterdir())}
for dn, did in dir2id.items():
fs = FileStorage(base_path / dn)
for m in fs.iter_msg():
lp = extract_loopid_func_name(m.tag)
lp_id = lp[0] if lp and lp[0] is not None else None
# lp_id = (lp := extract_loopid_func_name(m.tag))[0] if lp[0] is not None else None
if "r.hypothesis generation" in m.tag:
h: Hypothesis = m.content
msgs_for_frontend[did].append(
{
"tag": "research.hypothesis",
"timestamp": m.timestamp.isoformat(),
"content": {
"name_map": {
"hypothesis": "RD-Agent proposes the hypothesis⬇️",
"concise_justification": "because the reason⬇️",
"concise_observation": "based on the observation⬇️",
"concise_knowledge": "Knowledge⬇️ gained after practice",
},
"hypothesis": h.hypothesis,
"concise_justification": h.concise_justification,
"concise_observation": h.concise_observation,
"concise_knowledge": h.concise_knowledge,
},
}
)

# m.tag 中不存在 d.load_experiment, 存在 r.load_experiment
elif "r.experiment generation" in m.tag or "d.load_experiment" in m.tag:
if "d.load_experiment" in m.tag:
tasks: list[FactorTask | ModelTask] = m.content.sub_tasks
else:
tasks: list[FactorTask | ModelTask] = m.content
if isinstance(tasks[0], FactorTask):
msgs_for_frontend[did].append(
def format_pkl(obj: object, tag: str = "", log_trace_path: str = None):

ts = datetime.now(timezone.utc).isoformat()
lp = extract_loopid_func_name(tag)
lp_id = lp[0] if lp and lp[0] is not None else None

if "r.hypothesis generation" in tag:
h: Hypothesis = obj
msgs_for_frontend[log_trace_path].append(
{
"tag": "research.hypothesis",
"timestamp": ts,
"content": {
"name_map": {
"hypothesis": "RD-Agent proposes the hypothesis⬇️",
"concise_justification": "because the reason⬇️",
"concise_observation": "based on the observation⬇️",
"concise_knowledge": "Knowledge⬇️ gained after practice",
},
"hypothesis": h.hypothesis,
"concise_justification": h.concise_justification,
"concise_observation": h.concise_observation,
"concise_knowledge": h.concise_knowledge,
},
}
)

elif "r.experiment generation" in tag or "d.load_experiment" in tag:
if "d.load_experiment" in tag:
if isinstance(obj, Experiment):
tasks: list[FactorTask | ModelTask] = obj.sub_tasks
else:
tasks: list[FactorTask | ModelTask] = obj
if isinstance(tasks[0], FactorTask):
msgs_for_frontend[log_trace_path].append(
{
"tag": "research.tasks",
"timestamp": ts,
"content": [
{
"tag": "research.tasks",
"timestamp": m.timestamp.isoformat(),
"content": [
{
"name": t.factor_name,
"description": t.factor_description,
"formulation": t.factor_formulation,
"variables": t.variables,
}
for t in tasks
],
"name": t.factor_name,
"description": t.factor_description,
"formulation": t.factor_formulation,
"variables": t.variables,
}
)
elif isinstance(tasks[0], ModelTask):
msgs_for_frontend[did].append(
for t in tasks
],
}
)
elif isinstance(tasks[0], ModelTask):
msgs_for_frontend[log_trace_path].append(
{
"tag": "research.tasks",
"timestamp": ts,
"content": [
{
"tag": "research.tasks",
"timestamp": m.timestamp.isoformat(),
"content": [
{
"name": t.name,
"description": t.description,
"model_type": t.model_type,
"formulation": t.formulation,
"variables": t.variables,
}
for t in tasks
],
"name": t.name,
"description": t.description,
"model_type": t.model_type,
"formulation": t.formulation,
"variables": t.variables,
}
)
for t in tasks
],
}
)

elif f"evo_loop_{lp_id}.evolving code" in m.tag:
ws: list[FactorFBWorkspace | ModelFBWorkspace] = [i for i in m.content]
msgs_for_frontend[did].append(
elif f"evo_loop_{lp_id}.evolving code" in tag:
ws: list[FactorFBWorkspace | ModelFBWorkspace] = [i for i in obj]
msgs_for_frontend[log_trace_path].append(
{
"tag": "evolving.codes",
"timestamp": ts,
"content": [
{
"tag": "evolving.codes",
"timestamp": m.timestamp.isoformat(),
"content": [
{
"target_task_name": (
w.target_task.name
if isinstance(w.target_task, ModelTask)
else w.target_task.factor_name
),
"code": w.file_dict,
}
for w in ws
if w
],
"target_task_name": (
w.target_task.name if isinstance(w.target_task, ModelTask) else w.target_task.factor_name
),
"code": w.file_dict,
}
)
for w in ws
if w
],
}
)

elif f"evo_loop_{lp_id}.evolving feedback" in m.tag:
fl: list[FactorSingleFeedback | CoSTEERSingleFeedbackDeprecated] = [i for i in m.content]
msgs_for_frontend[did].append(
elif f"evo_loop_{lp_id}.evolving feedback" in tag:
fl: list[FactorSingleFeedback | CoSTEERSingleFeedbackDeprecated] = [i for i in obj]
msgs_for_frontend[log_trace_path].append(
{
"tag": "evolving.feedbacks",
"timestamp": ts,
"content": [
{
"tag": "evolving.feedbacks",
"timestamp": m.timestamp.isoformat(),
"content": [
{
"final_decision": f.final_decision,
"final_feedback": f.final_feedback,
"execution_feedback": f.execution_feedback,
"code_feedback": f.code_feedback,
"value_feedback": (
f.value_feedback
if isinstance(f, CoSTEERSingleFeedbackDeprecated)
else f.factor_value_feedback
),
"model_shape_feedback": (
f.shape_feedback if isinstance(f, CoSTEERSingleFeedbackDeprecated) else None
),
}
for f in fl
if f
],
"final_decision": f.final_decision,
"final_feedback": f.final_feedback,
"execution_feedback": f.execution_feedback,
"code_feedback": f.code_feedback,
"value_feedback": (
f.value_feedback
if isinstance(f, CoSTEERSingleFeedbackDeprecated)
else f.factor_value_feedback
),
"model_shape_feedback": (
f.shape_feedback if isinstance(f, CoSTEERSingleFeedbackDeprecated) else None
),
}
)
for f in fl
if f
],
}
)

elif "scenario" in m.tag:
msgs_for_frontend[did].append(
{
"tag": "feedback.config",
"timestamp": m.timestamp.isoformat(),
"content": {"config": m.content.experiment_setting},
}
)
elif "scenario" in tag:
msgs_for_frontend[log_trace_path].append(
{"tag": "feedback.config", "timestamp": ts, "content": {"config": obj.experiment_setting}}
)

elif "ef.Quantitative Backtesting Chart" in m.tag:
msgs_for_frontend[did].append(
{
"tag": "feedback.return_chart",
"timestamp": m.timestamp.isoformat(),
"content": {"chart_html": plotly.io.to_html(report_figure(m.content))},
}
)
elif "ef.Quantitative Backtesting Chart" in tag:
msgs_for_frontend[log_trace_path].append(
{
"tag": "feedback.return_chart",
"timestamp": ts,
"content": {"chart_html": plotly.io.to_html(report_figure(obj))},
}
)

elif "model runner result" in m.tag or "factor runner result" in m.tag or "runner result" in m.tag:
msgs_for_frontend[did].append(
{
"tag": "feedback.metric",
"timestamp": m.timestamp.isoformat(),
"content": {"result": m.content.result.iloc[0]},
}
)
elif "model runner result" in tag or "factor runner result" in tag or "runner result" in tag:
if isinstance(obj, Experiment):
msgs_for_frontend[log_trace_path].append(
{"tag": "feedback.metric", "timestamp": ts, "content": {"result": obj.result.iloc[0]}}
)

elif "ef.feedback" in m.tag:
hf: HypothesisFeedback = m.content
msgs_for_frontend[did].append(
{
"tag": "feedback.hypothesis_feedback",
"timestamp": m.timestamp.isoformat(),
"content": {
"observations": hf.observations,
"hypothesis_evaluation": hf.hypothesis_evaluation,
"new_hypothesis": hf.new_hypothesis,
"decision": hf.decision,
"reason": hf.reason,
},
}
)
elif "ef.feedback" in tag:
hf: HypothesisFeedback = obj
msgs_for_frontend[log_trace_path].append(
{
"tag": "feedback.hypothesis_feedback",
"timestamp": ts,
"content": {
"observations": hf.observations,
"hypothesis_evaluation": hf.hypothesis_evaluation,
"new_hypothesis": hf.new_hypothesis,
"decision": hf.decision,
"reason": hf.reason,
},
}
)
for msgs in msgs_for_frontend.values():
msgs.append({"tag": "END", "timestamp": datetime.now(timezone.utc).isoformat(), "content": {}})
msgs.append({"tag": "END", "timestamp": ts, "content": {}})

return msgs_for_frontend
Loading
Oops, something went wrong.