diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index e013cabc..96dfebf8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -25,9 +25,9 @@ If applicable, add screenshots to help explain your problem. **Environment (please complete the following information):** - OS: [e.g. macOS, Windows, Linux] - - Python version: [e.g. 3.9, 3.10] - - Reward Kit version: [e.g. 0.1.0, or commit SHA if from source] - - How installed: [e.g. pip, from source] +- Python version: [e.g. 3.9, 3.10] + - Eval Protocol version: [e.g. 0.1.0, or commit SHA if from source] +- How installed: [e.g. pip, from source] **Additional context** Add any other context about the problem here. For example, are you using it with a specific LLM provider, or in a particular environment? diff --git a/development/CONTRIBUTING.md b/development/CONTRIBUTING.md index 15e624e4..3d421cc8 100644 --- a/development/CONTRIBUTING.md +++ b/development/CONTRIBUTING.md @@ -360,7 +360,7 @@ uv run black eval_protocol ## Available Reward Functions -Reward Kit includes these out-of-the-box reward functions: +Eval Protocol includes these out-of-the-box reward functions: | Category | Reward Functions | |----------|-----------------| @@ -380,7 +380,7 @@ Reward Kit includes these out-of-the-box reward functions: ## Running Examples -The examples folder contains sample code for using the Reward Kit: +The examples folder contains sample code for using Eval Protocol: ```bash # Run evaluation preview example @@ -506,7 +506,7 @@ If you encounter authentication issues: 3. **Check Account ID**: Confirm that the `FIREWORKS_ACCOUNT_ID` is correct for the environment you are targeting (e.g., `pyroworks-dev` for the dev API, or your personal account ID). 4. **API Base URL**: If using `FIREWORKS_API_BASE`, ensure it points to the correct API endpoint (e.g., `https://dev.api.fireworks.ai` for development). -You can use the following snippet to check what credentials the Reward Kit is resolving: +You can use the following snippet to check what credentials Eval Protocol is resolving: ```python from eval_protocol.auth import get_fireworks_api_key, get_fireworks_account_id @@ -556,7 +556,7 @@ uv pip install dist/eval_protocol-*.whl ## Contributing Process -We welcome contributions to Reward Kit! Please follow these steps to contribute: +We welcome contributions to Eval Protocol! Please follow these steps to contribute: 1. **Find or Create an Issue**: * Look for existing issues on the [GitHub Issues page](https://github.com/fireworks-ai/eval-protocol/issues) that you'd like to work on. @@ -581,9 +581,9 @@ We welcome contributions to Reward Kit! Please follow these steps to contribute: 6. **Run Code Quality Checks**: - * Format your code: `uv run black reward_kit tests` - * Check linting: `uv run flake8 reward_kit tests` - * Check types: `uv run mypy reward_kit` + * Format your code: `uv run black eval_protocol tests` + * Check linting: `uv run flake8 eval_protocol tests` + * Check types: `uv run mypy eval_protocol` * Run pre-commit hooks: `pre-commit run --all-files` 7. **Update Documentation**: diff --git a/eval_protocol/__main__.py b/eval_protocol/__main__.py index 1e74dafc..90c78294 100644 --- a/eval_protocol/__main__.py +++ b/eval_protocol/__main__.py @@ -1,5 +1,5 @@ """ -Main entry point for running reward-kit as a module. +Main entry point for running Eval Protocol as a module. """ import sys diff --git a/eval_protocol/_version.py b/eval_protocol/_version.py index 4068d145..4a548c9a 100644 --- a/eval_protocol/_version.py +++ b/eval_protocol/_version.py @@ -51,7 +51,7 @@ def get_config() -> VersioneerConfig: cfg.VCS = "git" cfg.style = "pep440" cfg.tag_prefix = "v" - cfg.parentdir_prefix = "reward-kit-" + cfg.parentdir_prefix = "eval-protocol-" cfg.versionfile_source = "eval_protocol/_version.py" cfg.verbose = False return cfg diff --git a/eval_protocol/cli_commands/deploy.py b/eval_protocol/cli_commands/deploy.py index ad2a129a..7536969a 100644 --- a/eval_protocol/cli_commands/deploy.py +++ b/eval_protocol/cli_commands/deploy.py @@ -93,7 +93,7 @@ def _establish_local_server_and_tunnel(args): function_ref = args.function_ref local_server_port = args.local_port - log_dir = os.path.join(os.getcwd(), "logs", "reward-kit-local") + log_dir = os.path.join(os.getcwd(), "logs", "eval-protocol-local") os.makedirs(log_dir, exist_ok=True) generic_server_log_path = os.path.join(log_dir, f"generic_server_{evaluator_id}.log") @@ -270,9 +270,9 @@ def _deploy_to_gcp_cloud_run(args, current_config, gcp_config_from_yaml): gcp_sanitized_eval_id = "".join(filter(lambda char: char.isalnum() or char in ["-", "_"], args.id)) if not gcp_sanitized_eval_id: - gcp_sanitized_eval_id = "rewardkit-evaluator" + gcp_sanitized_eval_id = "evalprotocol-evaluator" secret_id_for_auth_key = f"rk-eval-{gcp_sanitized_eval_id}-authkey" - secret_labels = {"managed-by": "reward-kit", "evaluator-id": evaluator_id} + secret_labels = {"managed-by": "eval-protocol", "evaluator-id": evaluator_id} api_key_secret_version_id = ensure_gcp_secret( project_id=gcp_project_id, diff --git a/eval_protocol/cli_commands/deploy_mcp.py b/eval_protocol/cli_commands/deploy_mcp.py index dfd81a61..f71796d6 100644 --- a/eval_protocol/cli_commands/deploy_mcp.py +++ b/eval_protocol/cli_commands/deploy_mcp.py @@ -43,7 +43,7 @@ def _generate_mcp_dockerfile_content( # Base requirements for MCP servers - matching setup.py dependencies base_requirements = [ "fastmcp>=0.1.0", - # Core reward-kit dependencies from setup.py + # Core Eval Protocol dependencies from setup.py "requests>=2.25.0", "pydantic>=2.0.0", "dataclasses-json>=0.5.7", @@ -154,7 +154,7 @@ def _deploy_mcp_to_gcp_cloud_run(args, current_config, gcp_config_from_yaml): if not gcp_ar_repo_name and gcp_config_from_yaml: gcp_ar_repo_name = gcp_config_from_yaml.artifact_registry_repository if not gcp_ar_repo_name: - gcp_ar_repo_name = "reward-kit-mcp-servers" + gcp_ar_repo_name = "eval-protocol-mcp-servers" print(f"Using GCP Project: {gcp_project_id}, Region: {gcp_region}, AR Repo: {gcp_ar_repo_name}") diff --git a/eval_protocol/execution/pipeline.py b/eval_protocol/execution/pipeline.py index 0d0f864b..7e7f7c29 100644 --- a/eval_protocol/execution/pipeline.py +++ b/eval_protocol/execution/pipeline.py @@ -1,5 +1,5 @@ """ -Core evaluation execution pipeline for reward-kit. +Core evaluation execution pipeline for Eval Protocol. This module orchestrates dataset loading, model response generation (optional), and evaluation using specified reward functions. """ diff --git a/eval_protocol/gcp_tools.py b/eval_protocol/gcp_tools.py index 7078aa72..b19089ca 100644 --- a/eval_protocol/gcp_tools.py +++ b/eval_protocol/gcp_tools.py @@ -289,7 +289,7 @@ def ensure_artifact_registry_repo_exists(project_id: str, region: str, repo_name "--location", region, "--description", - "Repository for reward-kit evaluators (auto-created by reward-kit CLI)", + "Repository for Eval Protocol evaluators (auto-created by Eval Protocol CLI)", ] create_success, create_stdout, create_stderr = _run_gcloud_command(create_cmd, dry_run=dry_run) if create_success: @@ -442,7 +442,7 @@ def ensure_gcp_secret( dummy_dockerfile = 'FROM python:3.10-slim\nCMD ["echo", "hello"]' img_name = "gcr.io/my-test-project/my-test-reward-eval:latest" # Old GCR name, update for AR # Example AR image name: us-central1-docker.pkg.dev/my-test-project/my-ar-repo/my-test-reward-eval:latest - ar_img_name = "us-central1-docker.pkg.dev/my-test-project/reward-kit-images/my-test-reward-eval:latest" + ar_img_name = "us-central1-docker.pkg.dev/my-test-project/eval-protocol-images/my-test-reward-eval:latest" print(f"\n1. Simulating build and push for {ar_img_name} (dry_run=True)") build_and_push_docker_image( @@ -478,7 +478,7 @@ def ensure_gcp_secret( project_id="my-test-project", secret_id="my-test-api-key-secret", secret_value="supersecretvalue123", - labels={"managed-by": "reward-kit-test"}, + labels={"managed-by": "eval-protocol-test"}, dry_run=True, ) print("\nNote: These are placeholder executions. Real implementation requires gcloud CLI and Docker.") diff --git a/eval_protocol/mcp_agent/config.py b/eval_protocol/mcp_agent/config.py index caed6556..44b0249e 100644 --- a/eval_protocol/mcp_agent/config.py +++ b/eval_protocol/mcp_agent/config.py @@ -114,7 +114,7 @@ def model_post_init(self, __context: Any) -> None: class AppConfig(BaseModel): - """Root configuration for the RewardKit Intermediary MCP Server.""" + """Root configuration for the Eval Protocol Intermediary MCP Server.""" backends: List[BackendServerConfig] = Field( default_factory=list, diff --git a/eval_protocol/mcp_env.py b/eval_protocol/mcp_env.py index c28d6309..822de7bb 100644 --- a/eval_protocol/mcp_env.py +++ b/eval_protocol/mcp_env.py @@ -1,5 +1,5 @@ """ -MCP Environment API for reward-kit - Backward Compatibility Facade +MCP Environment API for Eval Protocol - Backward Compatibility Facade This module has been refactored into modular components for better maintainability. This file now serves as a backward compatibility facade. diff --git a/mypy.ini b/mypy.ini index 7618bf7c..182ca82b 100644 --- a/mypy.ini +++ b/mypy.ini @@ -19,7 +19,7 @@ follow_imports = skip # Specific package options [mypy.eval_protocol.*] -# Be less strict when type checking reward kit code +# Be less strict when type checking Eval Protocol code disallow_untyped_defs = False disallow_incomplete_defs = False