Skip to content

Commit

Permalink
[tune] remove TuneRichReporter (ray-project#40169)
Browse files Browse the repository at this point in the history
Signed-off-by: matthewdeng <matt@anyscale.com>
  • Loading branch information
matthewdeng authored and jonathan-anyscale committed Oct 26, 2023
1 parent 56a02d1 commit da2fec8
Show file tree
Hide file tree
Showing 4 changed files with 43 additions and 163 deletions.
2 changes: 0 additions & 2 deletions doc/source/tune/api/env.rst
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,6 @@ These are the environment variables Ray Tune currently considers:
are printed. Defaults to 0 (disabled).
* **RAY_AIR_NEW_OUTPUT**: If set to 0, this disables
the `experimental new console output <https://github.com/ray-project/ray/issues/36949>`_.
* **RAY_AIR_RICH_LAYOUT**: If set to 1, this enables
the `stick table layout <https://github.com/ray-project/ray/issues/36949>`_.



Expand Down
1 change: 0 additions & 1 deletion python/ray/air/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,5 +91,4 @@
COPY_DIRECTORY_CHECKPOINTS_INSTEAD_OF_MOVING_ENV,
"RAY_AIR_FULL_TRACEBACKS",
"RAY_AIR_NEW_OUTPUT",
"RAY_AIR_RICH_LAYOUT",
}
120 changes: 8 additions & 112 deletions python/ray/tune/experimental/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
Union,
)

import contextlib
import collections
from dataclasses import dataclass
import datetime
Expand Down Expand Up @@ -813,36 +812,20 @@ def _detect_reporter(
config: Optional[Dict] = None,
progress_metrics: Optional[Union[List[str], List[Dict[str, str]]]] = None,
):
# TODO: Add JupyterNotebook and Ray Client case later.
rich_enabled = bool(int(os.environ.get("RAY_AIR_RICH_LAYOUT", "0")))
if entrypoint in {
AirEntrypoint.TUNE_RUN,
AirEntrypoint.TUNE_RUN_EXPERIMENTS,
AirEntrypoint.TUNER,
}:
if rich_enabled:
if not rich:
raise ImportError("Please run `pip install rich`. ")
reporter = TuneRichReporter(
verbosity,
num_samples=num_samples,
metric=metric,
mode=mode,
config=config,
progress_metrics=progress_metrics,
)
else:
reporter = TuneTerminalReporter(
verbosity,
num_samples=num_samples,
metric=metric,
mode=mode,
config=config,
progress_metrics=progress_metrics,
)
reporter = TuneTerminalReporter(
verbosity,
num_samples=num_samples,
metric=metric,
mode=mode,
config=config,
progress_metrics=progress_metrics,
)
else:
if rich_enabled:
logger.warning("`RAY_AIR_RICH_LAYOUT` is only effective with Tune usecase.")
reporter = TrainReporter(verbosity, progress_metrics=progress_metrics)
return reporter

Expand Down Expand Up @@ -1028,93 +1011,6 @@ def _print_heartbeat(self, trials, *sys_args, force: bool = False):
print("* The trial terminated successfully after retrying.")


class TuneRichReporter(TuneReporterBase):
_wrap_headers = True

def __init__(
self,
verbosity: AirVerbosity,
num_samples: int = 0,
metric: Optional[str] = None,
mode: Optional[str] = None,
config: Optional[Dict] = None,
progress_metrics: Optional[Union[List[str], List[Dict[str, str]]]] = None,
):
super().__init__(
verbosity=verbosity,
num_samples=num_samples,
metric=metric,
mode=mode,
config=config,
progress_metrics=progress_metrics,
)
self._live = None

# since sticky table, we can afford to do that more often.
_heartbeat_freq = 5

@contextlib.contextmanager
def with_live(self):
with rich.live.Live(
refresh_per_second=4, redirect_stdout=True, redirect_stderr=True
) as live:
self._live = live
yield
self._live = None

def _render_layout(self, heartbeat_strs: List[str], table_data: _TrialTableData):
# generate a nested table, the top table will write some basic info
# and the bottom table shows trial status.
table = rich.table.Table(
show_header=False,
show_edge=False,
show_lines=False,
)
table_basic_info = rich.table.Table(
show_header=False,
show_edge=False,
show_lines=False,
)
for s in heartbeat_strs:
table_basic_info.add_row(str(s))
table_trial = rich.table.Table(
box=rich.box.SQUARE,
expand=True,
show_header=False,
title=":glowing_star: Ray Tune Trial Status Table :glowing_star:",
)
header = table_data.header
table_data = table_data.data
for _ in header:
table_trial.add_column(overflow="fold")
table_trial.add_row(*header)
for per_status_info in table_data:
trial_infos = per_status_info.trial_infos
more_info = per_status_info.more_info
for trial_info in trial_infos:
table_trial.add_row(*[str(_) for _ in trial_info])
if more_info:
table_trial.add_row(more_info)
table.add_row(table_basic_info)
table.add_row(table_trial)

self._live.update(table)

def _print_heartbeat(self, trials, *args, force: bool = False):
if not rich:
return
if not self._live:
logger.warning(
"`print_heartbeat` is not supposed to "
"be called without `with_live` context manager."
)
return
heartbeat_strs, table_data = self._get_heartbeat(
trials, *args, force_full_output=force
)
self._render_layout(heartbeat_strs, table_data)


class TrainReporter(ProgressReporter):
# the minimal verbosity threshold at which heartbeat starts getting printed.
_heartbeat_threshold = AirVerbosity.VERBOSE
Expand Down
83 changes: 35 additions & 48 deletions python/ray/tune/tune.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import abc
import contextlib
import copy
import datetime
import logging
Expand Down Expand Up @@ -985,60 +984,48 @@ class and registered trainables.
)
break

# rich live context manager has to be called encapsulating
# the while loop. For other kind of reporters, no op.
# `ExitStack` allows us to *conditionally* apply context manager.
with contextlib.ExitStack() as stack:
from ray.tune.experimental.output import TuneRichReporter
experiment_local_path = runner._storage.experiment_local_path
experiment_dir_name = runner._storage.experiment_dir_name

experiment_local_path = runner._storage.experiment_local_path
experiment_dir_name = runner._storage.experiment_dir_name

if any(isinstance(cb, TBXLoggerCallback) for cb in callbacks):
tensorboard_path = experiment_local_path
else:
tensorboard_path = None

if air_progress_reporter and isinstance(
air_progress_reporter, TuneRichReporter
):
stack.enter_context(air_progress_reporter.with_live())
elif air_progress_reporter:
air_progress_reporter.experiment_started(
experiment_name=experiment_dir_name,
experiment_path=runner.experiment_path,
searcher_str=search_alg.__class__.__name__,
scheduler_str=scheduler.__class__.__name__,
total_num_samples=search_alg.total_samples,
tensorboard_path=tensorboard_path,
)
if any(isinstance(cb, TBXLoggerCallback) for cb in callbacks):
tensorboard_path = experiment_local_path
else:
tensorboard_path = None

if air_progress_reporter:
air_progress_reporter.experiment_started(
experiment_name=experiment_dir_name,
experiment_path=runner.experiment_path,
searcher_str=search_alg.__class__.__name__,
scheduler_str=scheduler.__class__.__name__,
total_num_samples=search_alg.total_samples,
tensorboard_path=tensorboard_path,
)

try:
while (
not runner.is_finished() and not experiment_interrupted_event.is_set()
):
runner.step()
if has_verbosity(Verbosity.V1_EXPERIMENT):
_report_progress(runner, progress_reporter)
try:
while not runner.is_finished() and not experiment_interrupted_event.is_set():
runner.step()
if has_verbosity(Verbosity.V1_EXPERIMENT):
_report_progress(runner, progress_reporter)

if air_verbosity is not None:
_report_air_progress(runner, air_progress_reporter)
except Exception:
runner.cleanup()
raise
if air_verbosity is not None:
_report_air_progress(runner, air_progress_reporter)
except Exception:
runner.cleanup()
raise

tune_taken = time.time() - tune_start
tune_taken = time.time() - tune_start

try:
runner.checkpoint(force=True, wait=True)
except Exception as e:
logger.warning(f"Trial Runner checkpointing failed: {str(e)}")
try:
runner.checkpoint(force=True, wait=True)
except Exception as e:
logger.warning(f"Trial Runner checkpointing failed: {str(e)}")

if has_verbosity(Verbosity.V1_EXPERIMENT):
_report_progress(runner, progress_reporter, done=True)
if has_verbosity(Verbosity.V1_EXPERIMENT):
_report_progress(runner, progress_reporter, done=True)

if air_verbosity is not None:
_report_air_progress(runner, air_progress_reporter, force=True)
if air_verbosity is not None:
_report_air_progress(runner, air_progress_reporter, force=True)

all_trials = runner.get_trials()

Expand Down

0 comments on commit da2fec8

Please sign in to comment.