diff --git a/docs/source/saving-and-reading-results.mdx b/docs/source/saving-and-reading-results.mdx index 2a54aeaf4..e37cfb88e 100644 --- a/docs/source/saving-and-reading-results.mdx +++ b/docs/source/saving-and-reading-results.mdx @@ -36,7 +36,7 @@ This will create a Tensorboard dashboard in a HF org set with the `--results-org option. -## Pushing results to WandB +## Pushing results to Trackio or WandB You can push the results to WandB by setting `--wandb`. This will init a WandB run and log the results. @@ -49,6 +49,15 @@ export WANDB_PROJECT="lighteval" You can find a list of variable in the [wandb documentation](https://docs.wandb.ai/guides/track/environment-variables/). +If trackio is available in your environment `pip install +lighteval[trackio]`, it will be used to log and push the results on a +huggingface dataset. Choose the dataset name and org with: + +``` +export WANDB_SPACE_ID="org/name" +``` + + ## How to load and investigate details diff --git a/pyproject.toml b/pyproject.toml index 8a5a9cf90..4100add41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,7 @@ multilingual = [ ] math = ["latex2sympy2_extended==1.0.6"] wandb = ["wandb"] +trackio = ["trackio"] [project.urls] Homepage = "https://github.com/huggingface/lighteval" diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 32a638d1e..c9a336fc1 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -130,7 +130,7 @@ def __init__( tensorboard_metric_prefix: str = "eval", public: bool = False, nanotron_run_info: "GeneralArgs" = None, - wandb: bool = False, + use_wandb: bool = False, ) -> None: """Creates all the necessary loggers for evaluation tracking.""" self.details_logger = DetailsLogger() @@ -150,7 +150,7 @@ def __init__( self.should_push_to_hub = push_to_hub self.should_save_details = save_details - self.wandb = wandb + self.use_wandb = use_wandb self.should_push_results_to_tensorboard = push_to_tensorboard self.tensorboard_repo = f"{hub_results_org}/tensorboard_logs" @@ -160,18 +160,32 @@ def __init__( self.public = public - if wandb is True: - import wandb + if use_wandb is True: + try: + import trackio as wandb + + logger.warning("Trackio was found available in your environment, using it instead of wandb") + self.wandb_project = os.environ.get("WANDB_PROJECT", None) + self.space_id = os.environ.get("WANDB_SPACE_ID", None) + + wandb_kwargs = { + "space_id": self.space_id, + } + + except ImportError: + import wandb - self.wandb_project = os.environ.get("WANDB_PROJECT", None) + self.wandb_project = os.environ.get("WANDB_PROJECT", None) + wandb.login() + wandb_kwargs = {} if self.wandb_project is None: - raise ValueError("You need to specify the project name in wandb_args") + raise ValueError("You need to specify the project name using the WANDB_PROJECT environment variable") - wandb.login() self.wandb_run = wandb.init( project=self.wandb_project, resume="allow", + **wandb_kwargs, ) @property @@ -245,7 +259,7 @@ def save(self) -> None: results_dict=results_dict, ) - if self.wandb is True: + if self.use_wandb is True: self.push_to_wandb( results_dict=results_dict, details_datasets=details_datasets, diff --git a/src/lighteval/main_accelerate.py b/src/lighteval/main_accelerate.py index ce275092c..0c0049040 100644 --- a/src/lighteval/main_accelerate.py +++ b/src/lighteval/main_accelerate.py @@ -99,7 +99,7 @@ def accelerate( # noqa C901 wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, @@ -132,7 +132,7 @@ def accelerate( # noqa C901 push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) pipeline_params = PipelineParameters( launcher_type=ParallelismManager.ACCELERATE, diff --git a/src/lighteval/main_endpoint.py b/src/lighteval/main_endpoint.py index 793a578a5..a6a29f2b5 100644 --- a/src/lighteval/main_endpoint.py +++ b/src/lighteval/main_endpoint.py @@ -101,7 +101,7 @@ def inference_endpoint( wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, @@ -128,7 +128,7 @@ def inference_endpoint( push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) parallelism_manager = ParallelismManager.NONE # since we're using inference endpoints in remote @@ -226,7 +226,7 @@ def tgi( wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, @@ -256,7 +256,7 @@ def tgi( push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) parallelism_manager = ParallelismManager.TGI @@ -358,7 +358,7 @@ def litellm( wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, @@ -388,7 +388,7 @@ def litellm( push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) parallelism_manager = ParallelismManager.NONE @@ -481,7 +481,7 @@ def inference_providers( wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, @@ -521,7 +521,7 @@ def inference_providers( push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) parallelism_manager = ParallelismManager.NONE diff --git a/src/lighteval/main_sglang.py b/src/lighteval/main_sglang.py index a10964ed5..2129a7281 100644 --- a/src/lighteval/main_sglang.py +++ b/src/lighteval/main_sglang.py @@ -92,7 +92,7 @@ def sglang( wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, @@ -121,7 +121,7 @@ def sglang( push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) pipeline_params = PipelineParameters( diff --git a/src/lighteval/main_vllm.py b/src/lighteval/main_vllm.py index ba30777d2..f5ba384ed 100644 --- a/src/lighteval/main_vllm.py +++ b/src/lighteval/main_vllm.py @@ -95,7 +95,7 @@ def vllm( wandb: Annotated[ bool, Option( - help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", + help="Push results to wandb or trackio if available. We use env variable to configure trackio or wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/, https://github.com/gradio-app/trackio", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, @@ -124,7 +124,7 @@ def vllm( push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, - wandb=wandb, + use_wandb=wandb, ) pipeline_params = PipelineParameters(