From b6fe9be9b27f88259e8a89d17fef2cedee249c1e Mon Sep 17 00:00:00 2001 From: wangyukai Date: Wed, 3 Sep 2025 01:55:30 +0000 Subject: [PATCH 1/7] fix style --- .github/workflows/ci.yml | 63 ++++++++++++++ .gitignore | 3 +- .gitmodules | 1 + .pre-commit-config.yaml | 58 +------------ pyproject.toml | 11 +++ requirements/test.txt | 2 + tests/conftest.py | 20 +++++ tests/function_test/e2e_test.py | 54 ++++++++++++ tests/function_test/test_evaluator.py | 112 +++++++++++++++++++++++++ tests/unit_test/test_basic.py | 33 ++++++++ tests/unit_test/test_evaluator_unit.py | 22 +++++ 11 files changed, 324 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 requirements/test.txt create mode 100644 tests/conftest.py create mode 100644 tests/function_test/e2e_test.py create mode 100644 tests/function_test/test_evaluator.py create mode 100644 tests/unit_test/test_basic.py create mode 100644 tests/unit_test/test_evaluator_unit.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..44a9a57 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,63 @@ +name: PR CI + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: [ main, master, develop ] + paths-ignore: + - '**.md' + - 'docs/**' + +concurrency: + group: pr-${{ github.event.pull_request.number }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + test: + # 如果你只用 GitHub 托管 runner: + runs-on: ubuntu-latest + if: ${{ github.event.pull_request.draft == false }} # 草稿 PR 不跑 + + strategy: + fail-fast: false + matrix: + python-version: ['3.9', '3.11'] + os: [ubuntu-latest] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-pip-${{ matrix.python-version }}- + + - name: Install deps + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi + + - name: Lint (flake8 + isort + black --check) + run: | + pip install flake8 isort black + flake8 . + isort --check-only --diff . + black --check . + + - name: Run tests + run: | + pip install pytest + pytest -q diff --git a/.gitignore b/.gitignore index d202a63..73cfe4b 100644 --- a/.gitignore +++ b/.gitignore @@ -148,4 +148,5 @@ logs/ *.png *.ckpt /results/ -checkpoints \ No newline at end of file +checkpoints +internnav/model/basemodel/LongCLIP/ diff --git a/.gitmodules b/.gitmodules index fdc4a9d..761bb22 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,4 @@ [submodule "internnav/model/basemodel/LongCLIP"] path = internnav/model/basemodel/LongCLIP url = https://github.com/beichenzbc/Long-CLIP + ignore = untracked diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9dee0f4..e1846e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,69 +26,19 @@ repos: rev: v2.2.1 hooks: - id: codespell - exclude: | - (?x)( - ^toolkits/grscenes_scripts/README.md| - ^toolkits/indoor_scenes_generation/infinigen/infinigen_examples/constraints - ) - # - repo: https://github.com/gitleaks/gitleaks - # rev: v8.24.0 - # hooks: - # - id: gitleaks + - repo: https://github.com/gitleaks/gitleaks + rev: v8.24.0 + hooks: + - id: gitleaks - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.1.0 hooks: - id: trailing-whitespace - id: check-yaml - id: end-of-file-fixer - exclude: '^(.*/lcmtypes/.*)' - id: requirements-txt-fixer - - id: double-quote-string-fixer - exclude: '^(.*/lcmtypes/.*)' - id: check-merge-conflict - id: fix-encoding-pragma args: ["--remove"] - id: mixed-line-ending args: ["--fix=lf"] - - # - repo: https://github.com/PyCQA/isort - # rev: 5.11.5 - # hooks: - # - id: isort - # - repo: https://github.com/psf/black - # rev: 22.10.0 - # hooks: - # - id: black - # args: [--line-length=79] - # - repo: https://github.com/PyCQA/flake8 - # rev: 4.0.1 - # hooks: - # - id: flake8 - # - repo: https://github.com/codespell-project/codespell - # rev: v2.2.1 - # hooks: - # - id: codespell - # exclude: | - # (?x)( - # ^toolkits/grscenes_scripts/README.md| - # ^toolkits/indoor_scenes_generation/infinigen/infinigen_examples/constraints - # ) - # - repo: https://github.com/gitleaks/gitleaks - # rev: v8.24.0 - # hooks: - # - id: gitleaks - # - repo: https://github.com/pre-commit/pre-commit-hooks - # rev: v3.1.0 - # hooks: - # - id: trailing-whitespace - # - id: check-yaml - # - id: end-of-file-fixer - # exclude: '^(.*/lcmtypes/.*)' - # - id: requirements-txt-fixer - # - id: double-quote-string-fixer - # exclude: '^(.*/lcmtypes/.*)' - # - id: check-merge-conflict - # - id: fix-encoding-pragma - # args: ["--remove"] - # - id: mixed-line-ending - # args: ["--fix=lf"] diff --git a/pyproject.toml b/pyproject.toml index c6b12da..b8f0b8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,3 +8,14 @@ lcmtypes [tool.isort] profile = "black" skip_glob = '**/lcmtypes/**' + +[tool.pytest.ini_options] +# 指定测试搜索路径（避免扫到脚本目录等） +testpaths = [ + "tests" +] +addopts = "-ra --color=yes --maxfail=1" +markers = [ + "slow: marks tests as slow", + "gpu: requires GPU" +] diff --git a/requirements/test.txt b/requirements/test.txt new file mode 100644 index 0000000..e6d0251 --- /dev/null +++ b/requirements/test.txt @@ -0,0 +1,2 @@ +pytest==7.3.1 +pytest-cov==4.0.0 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d6eca95 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,20 @@ +import pytest + + +@pytest.fixture +def tmp_cfg(tmp_path): + p = tmp_path / "config.yaml" + p.write_text("hello: world\n") + return p + + +# 按需全局 hook（例：缺 GPU 时自动跳过 gpu 标记） +def pytest_runtest_setup(item): + if "gpu" in item.keywords: + try: + import torch + + if not torch.cuda.is_available(): + pytest.skip("No CUDA for gpu-marked test") + except Exception: + pytest.skip("Torch not available") diff --git a/tests/function_test/e2e_test.py b/tests/function_test/e2e_test.py new file mode 100644 index 0000000..f9004c4 --- /dev/null +++ b/tests/function_test/e2e_test.py @@ -0,0 +1,54 @@ +import json +import os +import subprocess +import sys + +import pytest + + +def common_body(cmd_line): + with subprocess.Popen( + cmd_line, + stdin=subprocess.PIPE, + stderr=sys.stderr, + close_fds=True, + stdout=sys.stdout, + universal_newlines=True, + shell=True, + bufsize=1, + ) as cmd: + cmd.communicate() + assert cmd.returncode == 0, f'real exit code is {cmd.returncode}' + + +def update_jsonl_from_json(json_file_path, jsonl_file_path, update_item): + with open(json_file_path, 'r', encoding='utf-8') as json_file: + data = json.load(json_file) + data = {**update_item, **data} + if not isinstance(data, list): + data = [data] + with open(jsonl_file_path, 'a', encoding='utf-8') as jsonl_file: + for item in data: + json_line = json.dumps(item, ensure_ascii=False) + jsonl_file.write(json_line + '\n') + + +def teardown_function(function): + if os.path.exists('./test_result.json'): + case_info = {} + test_name = function.__name__ + case_info['case_info'] = test_name + '_' + os.environ.get('JOB_ID') + update_jsonl_from_json('./test_result.json', '../total_result.jsonl', case_info) + else: + print('Warning! There is no test_result.json') + + +"""""" """""" """""" """""" """""" """""" """""" """ +Test +""" """""" """""" """""" """""" """""" """""" """""" + + +@pytest.mark.gpu +def test_evaluator(): + start_command = 'python ./tests/function_test/test_evaluator.py' + common_body(start_command) diff --git a/tests/function_test/test_evaluator.py b/tests/function_test/test_evaluator.py new file mode 100644 index 0000000..8fdde78 --- /dev/null +++ b/tests/function_test/test_evaluator.py @@ -0,0 +1,112 @@ +''' +Test the evaluator eval logic without model involve. +The main progress: + Init => warm up => fake one action +''' + + +def main(): + from enum import Enum + + from internnav.configs.agent import AgentCfg + from internnav.configs.evaluator import ( + EnvCfg, + EvalCfg, + EvalDatasetCfg, + SceneCfg, + TaskCfg, + ) + + class runner_status_code(Enum): + NORMAL = 0 + WARM_UP = 1 + NOT_RESET = 3 + TERMINATED = 2 + STOP = 4 + + eval_cfg = EvalCfg( + agent=AgentCfg( + server_port=8087, + model_name='rdp', + ckpt_path='checkpoints/r2r/fine_tuned/rdp', + model_settings={}, + ), + env=EnvCfg( + env_type='vln_pe', + env_settings={ + 'use_fabric': False, + 'headless': True, # display option: set to False will open isaac-sim interactive window + }, + ), + task=TaskCfg( + task_name='test_evaluator', + task_settings={ + 'env_num': 2, + 'use_distributed': False, # Ray distributed framework + 'proc_num': 8, + }, + scene=SceneCfg( + scene_type='mp3d', + scene_data_dir='data/scene_data/mp3d_pe', + ), + robot_name='h1', + robot_usd_path='data/Embodiments/vln-pe/h1/h1_vln_pointcloud.usd', + camera_resolution=[256, 256], # (W,H) + camera_prim_path='torso_link/h1_pano_camera_0', + ), + dataset=EvalDatasetCfg( + dataset_type="mp3d", + dataset_settings={ + 'base_data_dir': 'data/vln_pe/raw_data/r2r', + 'split_data_types': ['val_unseen', 'val_seen'], + 'filter_stairs': False, + }, + ), + eval_settings={'save_to_json': False, 'vis_output': False}, # save result to video under logs/ + ) + print(eval_cfg) + + # cfg = get_config(eval_cfg) + # try: + # evaluator = Evaluator.init(cfg) + # except Exception as e: + # print(e) + + # print('--- VlnPeEvaluator start ---') + # obs, reset_info = evaluator.env.reset() + # for info in reset_info: + # if info is None: + # continue + # progress_log_multi_util.trace_start( + # trajectory_id=evaluator.now_path_key(info), + # ) + + # obs = evaluator.warm_up() + # evaluator.fake_obs = obs[0][evaluator.robot_name] + # action = [{evaluator.robot_name: {'stand_still': []}} for _ in range(evaluator.env_num * evaluator.proc_num)] + # obs = evaluator._obs_remove_robot_name(obs) + # evaluator.runner_status = np.full( + # (evaluator.env_num * evaluator.proc_num), + # runner_status_code.NORMAL, + # runner_status_code, + # ) + # evaluator.runner_status[[info is None for info in reset_info]] = runner_status_code.TERMINATED + + # while evaluator.env.is_running(): + + # obs, terminated = evaluator.env_step(action) + # break + + # evaluator.env.close() + + +if __name__ == '__main__': + try: + main() + except Exception as e: + print(f'exception is {e}') + import sys + import traceback + + traceback.print_exc() + sys.exit(1) diff --git a/tests/unit_test/test_basic.py b/tests/unit_test/test_basic.py new file mode 100644 index 0000000..70e51bb --- /dev/null +++ b/tests/unit_test/test_basic.py @@ -0,0 +1,33 @@ +import math + +import pytest + + +def add(a, b): + return a + b + + +def test_add_works(): + assert add(1, 2) == 3 + + +@pytest.mark.parametrize("x,expected", [(0, 0.0), (math.pi, 0.0)]) +def test_sin(x, expected): + assert math.isclose(math.sin(x), expected, abs_tol=1e-9) + + +@pytest.mark.slow +def test_slow_example(): + # 假装这里很慢 + assert sum(range(10000)) > 0 + + +@pytest.mark.gpu +def test_gpu_feature(): + pytest.importorskip("torch") + import torch + + if not torch.cuda.is_available(): + pytest.skip("No CUDA available") + x = torch.tensor([1.0], device="cuda") + assert float(x.item()) == 1.0 diff --git a/tests/unit_test/test_evaluator_unit.py b/tests/unit_test/test_evaluator_unit.py new file mode 100644 index 0000000..2c7c7eb --- /dev/null +++ b/tests/unit_test/test_evaluator_unit.py @@ -0,0 +1,22 @@ +import numpy as np +import pytest + +from internnav.evaluator.vln_pe_evaluator import transform_action_batch # 按你的真实模块路径改 + + +@pytest.mark.slow +def test_transform_action_batch_discrete(): + origin = [(np.array([0]),), (np.array([-1]),), (np.array([3]),)] + out = transform_action_batch(origin, flash=False) + assert out == [ + {'h1': {'stop': []}}, + {'h1': {'stand_still': []}}, + {'h1': {'move_by_discrete': [3]}}, + ] + + +@pytest.mark.slow +def test_transform_action_batch_flash(): + origin = [(np.array([5]),)] + out = transform_action_batch(origin, flash=True) + assert out == [{'h1': {'move_by_flash': [5]}}] From 4b9d98e53c2e18acc5af76c4f20ecf9d7afe399e Mon Sep 17 00:00:00 2001 From: Yukai Wang <58201273+kew6688@users.noreply.github.com> Date: Wed, 3 Sep 2025 15:41:28 +0800 Subject: [PATCH 2/7] [Fix] CI is ready for style check on github * test * fix ci * fix origin * fix codespell * test --- .github/workflows/ci.yml | 19 +++-- .pre-commit-config.yaml | 7 +- tests/function_test/e2e_test.py | 4 +- tests/function_test/test_challenge.py | 88 ++++++++++++++++++++ tests/function_test/test_evaluator.py | 112 -------------------------- 5 files changed, 105 insertions(+), 125 deletions(-) create mode 100644 tests/function_test/test_challenge.py delete mode 100644 tests/function_test/test_evaluator.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44a9a57..857b7b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,12 +24,15 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.11'] + python-version: ['3.10'] os: [ubuntu-latest] steps: - name: Checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -50,14 +53,16 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi - - name: Lint (flake8 + isort + black --check) + - name: Install pre-commit + run: pip install pre-commit + + - name: Run pre-commit on diff only + if: ${{ github.event_name == 'pull_request' }} run: | - pip install flake8 isort black - flake8 . - isort --check-only --diff . - black --check . + git fetch origin ${{ github.base_ref }} + pre-commit run --from-ref origin/${{ github.base_ref }} --to-ref HEAD - name: Run tests run: | pip install pytest - pytest -q + # pytest -q -W ignore diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1846e2..0e05562 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,6 @@ +exclude: | + ^internnav/model/basemodel/LongCLIP/ + repos: - repo: https://github.com/PyCQA/autoflake rev: v2.2.0 @@ -26,10 +29,6 @@ repos: rev: v2.2.1 hooks: - id: codespell - - repo: https://github.com/gitleaks/gitleaks - rev: v8.24.0 - hooks: - - id: gitleaks - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.1.0 hooks: diff --git a/tests/function_test/e2e_test.py b/tests/function_test/e2e_test.py index f9004c4..4cfe64e 100644 --- a/tests/function_test/e2e_test.py +++ b/tests/function_test/e2e_test.py @@ -49,6 +49,6 @@ def teardown_function(function): @pytest.mark.gpu -def test_evaluator(): - start_command = 'python ./tests/function_test/test_evaluator.py' +def test_challenge(): + start_command = 'python ./tests/function_test/test_challenge.py' common_body(start_command) diff --git a/tests/function_test/test_challenge.py b/tests/function_test/test_challenge.py new file mode 100644 index 0000000..7d1bba3 --- /dev/null +++ b/tests/function_test/test_challenge.py @@ -0,0 +1,88 @@ +''' +Test the evaluator eval logic without model involve. +The main progress: + Init => warm up => fake one action +''' + +import importlib.util +import subprocess +import sys + +import numpy as np + +from internnav.configs.evaluator.default_config import get_config +from internnav.evaluator import Evaluator +from internnav.utils import progress_log_multi_util + + +def main(): + from enum import Enum + + class runner_status_code(Enum): + NORMAL = 0 + WARM_UP = 1 + NOT_RESET = 3 + TERMINATED = 2 + STOP = 4 + + def load_eval_cfg(config_path, attr_name='eval_cfg'): + spec = importlib.util.spec_from_file_location("eval_config_module", config_path) + config_module = importlib.util.module_from_spec(spec) + sys.modules["eval_config_module"] = config_module + spec.loader.exec_module(config_module) + return getattr(config_module, attr_name) + + evaluator_cfg = load_eval_cfg('scripts/eval/configs/challenge_cfg.py', attr_name='eval_cfg') + cfg = get_config(evaluator_cfg) + evaluator = Evaluator.init(cfg) + + print('--- VlnPeEvaluator start ---') + obs, reset_info = evaluator.env.reset() + for info in reset_info: + if info is None: + continue + progress_log_multi_util.trace_start( + trajectory_id=evaluator.now_path_key(info), + ) + + obs = evaluator.warm_up() + evaluator.fake_obs = obs[0][evaluator.robot_name] + action = [{evaluator.robot_name: {'stand_still': []}} for _ in range(evaluator.env_num * evaluator.proc_num)] + obs = evaluator._obs_remove_robot_name(obs) + evaluator.runner_status = np.full( + (evaluator.env_num * evaluator.proc_num), + runner_status_code.NORMAL, + runner_status_code, + ) + evaluator.runner_status[[info is None for info in reset_info]] = runner_status_code.TERMINATED + + while evaluator.env.is_running(): + obs, action = evaluator.get_action(obs, action) + obs, terminated = evaluator.env_step(action) + env_term, reset_info = evaluator.terminate_ops(obs, reset_info, terminated) + break + + evaluator.env.close() + + +def start_server(): + # Start server + server_cmd = [ + sys.executable, + "internnav/agent/utils/server.py", + "--config", + 'scripts/eval/configs/challenge_cfg.py', + ] + subprocess.Popen(server_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, start_new_session=True) + + +if __name__ == '__main__': + try: + start_server() + main() + except Exception as e: + print(f'exception is {e}') + import traceback + + traceback.print_exc() + sys.exit(1) diff --git a/tests/function_test/test_evaluator.py b/tests/function_test/test_evaluator.py deleted file mode 100644 index 8fdde78..0000000 --- a/tests/function_test/test_evaluator.py +++ /dev/null @@ -1,112 +0,0 @@ -''' -Test the evaluator eval logic without model involve. -The main progress: - Init => warm up => fake one action -''' - - -def main(): - from enum import Enum - - from internnav.configs.agent import AgentCfg - from internnav.configs.evaluator import ( - EnvCfg, - EvalCfg, - EvalDatasetCfg, - SceneCfg, - TaskCfg, - ) - - class runner_status_code(Enum): - NORMAL = 0 - WARM_UP = 1 - NOT_RESET = 3 - TERMINATED = 2 - STOP = 4 - - eval_cfg = EvalCfg( - agent=AgentCfg( - server_port=8087, - model_name='rdp', - ckpt_path='checkpoints/r2r/fine_tuned/rdp', - model_settings={}, - ), - env=EnvCfg( - env_type='vln_pe', - env_settings={ - 'use_fabric': False, - 'headless': True, # display option: set to False will open isaac-sim interactive window - }, - ), - task=TaskCfg( - task_name='test_evaluator', - task_settings={ - 'env_num': 2, - 'use_distributed': False, # Ray distributed framework - 'proc_num': 8, - }, - scene=SceneCfg( - scene_type='mp3d', - scene_data_dir='data/scene_data/mp3d_pe', - ), - robot_name='h1', - robot_usd_path='data/Embodiments/vln-pe/h1/h1_vln_pointcloud.usd', - camera_resolution=[256, 256], # (W,H) - camera_prim_path='torso_link/h1_pano_camera_0', - ), - dataset=EvalDatasetCfg( - dataset_type="mp3d", - dataset_settings={ - 'base_data_dir': 'data/vln_pe/raw_data/r2r', - 'split_data_types': ['val_unseen', 'val_seen'], - 'filter_stairs': False, - }, - ), - eval_settings={'save_to_json': False, 'vis_output': False}, # save result to video under logs/ - ) - print(eval_cfg) - - # cfg = get_config(eval_cfg) - # try: - # evaluator = Evaluator.init(cfg) - # except Exception as e: - # print(e) - - # print('--- VlnPeEvaluator start ---') - # obs, reset_info = evaluator.env.reset() - # for info in reset_info: - # if info is None: - # continue - # progress_log_multi_util.trace_start( - # trajectory_id=evaluator.now_path_key(info), - # ) - - # obs = evaluator.warm_up() - # evaluator.fake_obs = obs[0][evaluator.robot_name] - # action = [{evaluator.robot_name: {'stand_still': []}} for _ in range(evaluator.env_num * evaluator.proc_num)] - # obs = evaluator._obs_remove_robot_name(obs) - # evaluator.runner_status = np.full( - # (evaluator.env_num * evaluator.proc_num), - # runner_status_code.NORMAL, - # runner_status_code, - # ) - # evaluator.runner_status[[info is None for info in reset_info]] = runner_status_code.TERMINATED - - # while evaluator.env.is_running(): - - # obs, terminated = evaluator.env_step(action) - # break - - # evaluator.env.close() - - -if __name__ == '__main__': - try: - main() - except Exception as e: - print(f'exception is {e}') - import sys - import traceback - - traceback.print_exc() - sys.exit(1) From 289a1e3eb05b44c4418c87654130557c23d45435 Mon Sep 17 00:00:00 2001 From: Yukai Wang <58201273+kew6688@users.noreply.github.com> Date: Thu, 4 Sep 2025 10:39:41 +0800 Subject: [PATCH 3/7] add tests; add ray test; add ISSUE_TEMPLATE; remove duplicate requirements; Add runner ci; * update test; add ray test * update self hosted * update ci * update ci * update ci * update ci * update ci * update * update ci; add issue template * update ci * update ci * update ci * fix server hang * fix server hang * fix server hang * fix server hang * update ci and test * update ci and test * update ci and test * Revert "update ci and test" This reverts commit 35434940ddf3f926785c5ed684f01f1a912e207d. * fix * fix server --- .github/workflows/ISSUE_TEMPLATE/1-bug.yml | 96 ++++++++++++++++ .../ISSUE_TEMPLATE/2-enhancement.yml | 39 +++++++ .../workflows/ISSUE_TEMPLATE/3-question.yml | 16 +++ .github/workflows/ci.yml | 47 ++++---- .gitmodules | 1 + requirements/agent.txt | 2 - requirements/eval.txt | 4 +- requirements/test.txt | 4 +- tests/conftest.py | 8 ++ tests/function_test/e2e_test.py | 12 ++ tests/function_test/test_challenge.py | 24 +++- tests/function_test/test_challenge_ray.py | 106 ++++++++++++++++++ tests/function_test/test_server.py | 49 ++++++++ 13 files changed, 376 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/ISSUE_TEMPLATE/1-bug.yml create mode 100644 .github/workflows/ISSUE_TEMPLATE/2-enhancement.yml create mode 100644 .github/workflows/ISSUE_TEMPLATE/3-question.yml delete mode 100644 requirements/agent.txt create mode 100644 tests/function_test/test_challenge_ray.py create mode 100644 tests/function_test/test_server.py diff --git a/.github/workflows/ISSUE_TEMPLATE/1-bug.yml b/.github/workflows/ISSUE_TEMPLATE/1-bug.yml new file mode 100644 index 0000000..5b156ca --- /dev/null +++ b/.github/workflows/ISSUE_TEMPLATE/1-bug.yml @@ -0,0 +1,96 @@ +name: "🐛 Bug report" +description: Report errors or unexpected behavior +title: "[Bug]: " +labels: ["type/bug", "triage-needed"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report, please make sure to [search for existing issues](https://github.com/InternRobotics/InternUtopia/issues) before filing a new one! + + - type: textarea + id: bug-description + attributes: + label: Bug Description + placeholder: | + A clear and concise description of what the bug is. + Try to isolate the issue to help the community to reproduce it easily and increase chances for a fast fix. + validations: + required: true + + - type: textarea + id: steps-to-reproduce + attributes: + label: Steps to Reproduce + placeholder: | + Please try to provide a minimal example to reproduce the bug. Error messages and stack traces are also helpful. + + + value: | + Please try to provide a minimal example to reproduce the bug. Error messages and stack traces are also helpful. + + + validations: + required: true + + - type: textarea + id: expected-behavior + attributes: + label: Expected Behavior + placeholder: "A clear and concise description of what you expected to happen." + validations: + required: true + + - type: textarea + id: screenshots-videos + attributes: + label: Screenshots/Videos + placeholder: "If applicable, add screenshots and/or a video to help explain your problem." + + - type: textarea + id: desktop-device + attributes: + label: Environment + placeholder: | + - OS: [e.g. Ubuntu 22.04] + - GPU/CPU [e.g. A100, RTX 4090, i9-14900K] + - GPU-driver version + value: | + - OS: [e.g. Ubuntu 22.04] + - GPU/CPU [e.g. A100, RTX 4090, i9-14900K] + - GPU-driver version + validations: + required: true + + - type: textarea + id: version + attributes: + label: Release version or Commit ID + placeholder: | + Please provide: + - a) **version number** of the release causing the issue, OR + - b) **SHA/hash** of the latest commit if working from git. You can get this by running the `git rev-parse HEAD` command on your current branch. + validations: + required: true + + - type: textarea + id: additional-context + attributes: + label: Additional Context + placeholder: "Add any other context about the problem here." diff --git a/.github/workflows/ISSUE_TEMPLATE/2-enhancement.yml b/.github/workflows/ISSUE_TEMPLATE/2-enhancement.yml new file mode 100644 index 0000000..3752c0f --- /dev/null +++ b/.github/workflows/ISSUE_TEMPLATE/2-enhancement.yml @@ -0,0 +1,39 @@ +name: "🚀 Enhancement" +description: Suggest a new feature request or improvement on the project +title: '[Enhancement]: ' +labels: + - type/enhancement + - triage-needed + +body: + - type: markdown + attributes: + value: | + A clear and concise description of what new feature or behavior you would like to see. If applicable, please describe the current behavior as well. + + - type: textarea + id: suggestion + attributes: + label: What feature or enhancement are you proposing? + validations: + required: true + + - type: textarea + id: motivation + attributes: + label: Motivation + description: What is your motivation for adding / enhancing this feature, optimally described in the form of a concrete user story or use case. + value: | + + validations: + required: false + + - type: textarea + id: additionalinfo + attributes: + label: Additional information + description: If you think that any additional information would be useful please provide them here. diff --git a/.github/workflows/ISSUE_TEMPLATE/3-question.yml b/.github/workflows/ISSUE_TEMPLATE/3-question.yml new file mode 100644 index 0000000..0b17ba7 --- /dev/null +++ b/.github/workflows/ISSUE_TEMPLATE/3-question.yml @@ -0,0 +1,16 @@ +name: "🙏 Question" +description: Ask a question +title: "[Question]: " +labels: ["type/question", "triage-needed"] +body: + - type: markdown + attributes: + value: | + Please make sure to [search for existing issues](https://github.com/InternRobotics/InternUtopia/issues) before filing a new one! + + - type: textarea + attributes: + label: Question + description: Describe your question in detail. + validations: + required: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 857b7b1..b566f50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,15 +17,14 @@ permissions: jobs: test: - # 如果你只用 GitHub 托管 runner: - runs-on: ubuntu-latest - if: ${{ github.event.pull_request.draft == false }} # 草稿 PR 不跑 + runs-on: self-hosted + if: ${{ github.event.pull_request.draft == false }} # no test on draft strategy: fail-fast: false matrix: python-version: ['3.10'] - os: [ubuntu-latest] + os: [linux] steps: - name: Checkout @@ -39,30 +38,32 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Cache pip - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }} - restore-keys: | - ${{ runner.os }}-pip-${{ matrix.python-version }}- - - - name: Install deps - run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi - - - name: Install pre-commit - run: pip install pre-commit - - name: Run pre-commit on diff only if: ${{ github.event_name == 'pull_request' }} + shell: bash -l {0} run: | + export PATH=/root/miniconda3/bin:$PATH + source /root/miniconda3/etc/profile.d/conda.sh + conda activate internutopia + git fetch origin ${{ github.base_ref }} pre-commit run --from-ref origin/${{ github.base_ref }} --to-ref HEAD - name: Run tests + shell: bash -l {0} run: | - pip install pytest - # pytest -q -W ignore + # conda + export PATH=/root/miniconda3/bin:$PATH + source /root/miniconda3/etc/profile.d/conda.sh + conda activate internutopia + + # link data + mkdir data + ln -s /cpfs/user/wangyukai/mp3d_data/vln_pe data/vln_pe + ln -s /cpfs/user/wangyukai/mp3d_data/Embodiments data/Embodiments + ln -s /cpfs/user/wangyukai/mp3d_data/scene_data data/scene_data + ln -s /cpfs/user/wangyukai/checkpoints checkpoints + + # run tests + /root/miniconda3/envs/internutopia/bin/python -c "import torch,sys;print(sys.executable);print('cuda:',torch.cuda.is_available())" + /root/miniconda3/envs/internutopia/bin/python -m pytest -q -W ignore diff --git a/.gitmodules b/.gitmodules index 761bb22..e307b6b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,4 +5,5 @@ [submodule "internnav/model/basemodel/LongCLIP"] path = internnav/model/basemodel/LongCLIP url = https://github.com/beichenzbc/Long-CLIP + commit = 3966af9ae9331666309a22128468b734db4672a7 ignore = untracked diff --git a/requirements/agent.txt b/requirements/agent.txt deleted file mode 100644 index 49135e2..0000000 --- a/requirements/agent.txt +++ /dev/null @@ -1,2 +0,0 @@ -flask -pydantic diff --git a/requirements/eval.txt b/requirements/eval.txt index 3521b85..0d8028c 100644 --- a/requirements/eval.txt +++ b/requirements/eval.txt @@ -1,3 +1,3 @@ +ansi2txt==0.2.0 pydantic>2.0 -requests -ansi2txt \ No newline at end of file +requests==2.32.3 diff --git a/requirements/test.txt b/requirements/test.txt index e6d0251..c233f57 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,2 +1,4 @@ +coverage==7.5.4 pytest==7.3.1 -pytest-cov==4.0.0 +pytest-cov==4.1.0 +tomli==2.0.1 diff --git a/tests/conftest.py b/tests/conftest.py index d6eca95..38c9556 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,3 +18,11 @@ def pytest_runtest_setup(item): pytest.skip("No CUDA for gpu-marked test") except Exception: pytest.skip("Torch not available") + if "ray" in item.keywords: + try: + import ray + + ray.init() + assert ray.is_initialized() + except Exception: + pytest.skip("ray not available") diff --git a/tests/function_test/e2e_test.py b/tests/function_test/e2e_test.py index 4cfe64e..67b48b8 100644 --- a/tests/function_test/e2e_test.py +++ b/tests/function_test/e2e_test.py @@ -48,7 +48,19 @@ def teardown_function(function): """ """""" """""" """""" """""" """""" """""" """""" +@pytest.mark.cpu +def test_server(): + start_command = 'python ./tests/function_test/test_server.py' + common_body(start_command) + + @pytest.mark.gpu def test_challenge(): start_command = 'python ./tests/function_test/test_challenge.py' common_body(start_command) + + +@pytest.mark.ray +def test_challenge_ray(): + start_command = 'python ./tests/function_test/test_challenge_ray.py' + common_body(start_command) diff --git a/tests/function_test/test_challenge.py b/tests/function_test/test_challenge.py index 7d1bba3..ab492a5 100644 --- a/tests/function_test/test_challenge.py +++ b/tests/function_test/test_challenge.py @@ -7,6 +7,7 @@ import importlib.util import subprocess import sys +import time import numpy as np @@ -66,19 +67,25 @@ def load_eval_cfg(config_path, attr_name='eval_cfg'): def start_server(): - # Start server server_cmd = [ sys.executable, "internnav/agent/utils/server.py", "--config", - 'scripts/eval/configs/challenge_cfg.py', + "scripts/eval/configs/challenge_cfg.py", ] - subprocess.Popen(server_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, start_new_session=True) + + proc = subprocess.Popen( + server_cmd, + stdout=None, + stderr=None, + ) + return proc if __name__ == '__main__': try: - start_server() + proc = start_server() + time.sleep(3) main() except Exception as e: print(f'exception is {e}') @@ -86,3 +93,12 @@ def start_server(): traceback.print_exc() sys.exit(1) + finally: + if proc and proc.poll() is None: + print("Shutting down server...") + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + print("Force killing server...") + proc.kill() diff --git a/tests/function_test/test_challenge_ray.py b/tests/function_test/test_challenge_ray.py new file mode 100644 index 0000000..2b35afb --- /dev/null +++ b/tests/function_test/test_challenge_ray.py @@ -0,0 +1,106 @@ +''' +Test the evaluator eval logic with ray, set proc_num = 4. +The main progress: + Init => warm up => one action +''' + +import importlib.util +import subprocess +import sys +import time + +import numpy as np + +from internnav.configs.evaluator.default_config import get_config +from internnav.evaluator import Evaluator +from internnav.utils import progress_log_multi_util + + +def main(): + from enum import Enum + + class runner_status_code(Enum): + NORMAL = 0 + WARM_UP = 1 + NOT_RESET = 3 + TERMINATED = 2 + STOP = 4 + + def load_eval_cfg(config_path, attr_name='eval_cfg'): + spec = importlib.util.spec_from_file_location("eval_config_module", config_path) + config_module = importlib.util.module_from_spec(spec) + sys.modules["eval_config_module"] = config_module + spec.loader.exec_module(config_module) + return getattr(config_module, attr_name) + + evaluator_cfg = load_eval_cfg('scripts/eval/configs/challenge_cfg.py', attr_name='eval_cfg') + evaluator_cfg.task.task_settings["use_distributed"] = True + evaluator_cfg.task.task_settings["proc_num"] = 4 + cfg = get_config(evaluator_cfg) + evaluator = Evaluator.init(cfg) + + print('--- VlnPeEvaluator start ---') + obs, reset_info = evaluator.env.reset() + for info in reset_info: + if info is None: + continue + progress_log_multi_util.trace_start( + trajectory_id=evaluator.now_path_key(info), + ) + + obs = evaluator.warm_up() + evaluator.fake_obs = obs[0][evaluator.robot_name] + action = [{evaluator.robot_name: {'stand_still': []}} for _ in range(evaluator.env_num * evaluator.proc_num)] + obs = evaluator._obs_remove_robot_name(obs) + evaluator.runner_status = np.full( + (evaluator.env_num * evaluator.proc_num), + runner_status_code.NORMAL, + runner_status_code, + ) + evaluator.runner_status[[info is None for info in reset_info]] = runner_status_code.TERMINATED + + while evaluator.env.is_running(): + obs, action = evaluator.get_action(obs, action) + obs, terminated = evaluator.env_step(action) + env_term, reset_info = evaluator.terminate_ops(obs, reset_info, terminated) + break + + evaluator.env.close() + + +def start_server(): + server_cmd = [ + sys.executable, + "internnav/agent/utils/server.py", + "--config", + "scripts/eval/configs/challenge_cfg.py", + ] + + proc = subprocess.Popen( + server_cmd, + stdout=None, + stderr=None, + ) + return proc + + +if __name__ == '__main__': + try: + proc = start_server() + time.sleep(3) + main() + except Exception as e: + print(f'exception is {e}') + import traceback + + traceback.print_exc() + sys.exit(1) + finally: + if proc and proc.poll() is None: + print("Shutting down server...") + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + print("Force killing server...") + proc.kill() diff --git a/tests/function_test/test_server.py b/tests/function_test/test_server.py new file mode 100644 index 0000000..5bdfcee --- /dev/null +++ b/tests/function_test/test_server.py @@ -0,0 +1,49 @@ +""" +Test if the server starts successfully and is still alive after sleep. +""" +import subprocess +import sys +import time + + +def start_server(): + server_cmd = [ + sys.executable, + "internnav/agent/utils/server.py", + "--config", + "scripts/eval/configs/challenge_cfg.py", + ] + + proc = subprocess.Popen( + server_cmd, + stdout=None, + stderr=None, + start_new_session=True, + ) + return proc + + +if __name__ == '__main__': + try: + proc = start_server() + time.sleep(5) + + # Raise if process exited + if proc.poll() is not None: + raise RuntimeError(f"❌ Server exited too early with code {proc.returncode}") + print("✅ Server is still alive after 5 seconds.") + + if proc and proc.poll() is None: + print("Shutting down server...") + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + raise RuntimeError("❌ Server failed to shut down within 10 seconds.") + + except Exception as e: + print(f'exception is {e}') + import traceback + + traceback.print_exc() + sys.exit(1) From 0b007985991c99db5e8bd1fa70692cf899fec57d Mon Sep 17 00:00:00 2001 From: Wang Yukai Date: Thu, 4 Sep 2025 02:45:46 +0000 Subject: [PATCH 4/7] fix issue --- .github/{workflows => }/ISSUE_TEMPLATE/1-bug.yml | 0 .github/{workflows => }/ISSUE_TEMPLATE/2-enhancement.yml | 0 .github/{workflows => }/ISSUE_TEMPLATE/3-question.yml | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename .github/{workflows => }/ISSUE_TEMPLATE/1-bug.yml (100%) rename .github/{workflows => }/ISSUE_TEMPLATE/2-enhancement.yml (100%) rename .github/{workflows => }/ISSUE_TEMPLATE/3-question.yml (100%) diff --git a/.github/workflows/ISSUE_TEMPLATE/1-bug.yml b/.github/ISSUE_TEMPLATE/1-bug.yml similarity index 100% rename from .github/workflows/ISSUE_TEMPLATE/1-bug.yml rename to .github/ISSUE_TEMPLATE/1-bug.yml diff --git a/.github/workflows/ISSUE_TEMPLATE/2-enhancement.yml b/.github/ISSUE_TEMPLATE/2-enhancement.yml similarity index 100% rename from .github/workflows/ISSUE_TEMPLATE/2-enhancement.yml rename to .github/ISSUE_TEMPLATE/2-enhancement.yml diff --git a/.github/workflows/ISSUE_TEMPLATE/3-question.yml b/.github/ISSUE_TEMPLATE/3-question.yml similarity index 100% rename from .github/workflows/ISSUE_TEMPLATE/3-question.yml rename to .github/ISSUE_TEMPLATE/3-question.yml From da02fde5fc42bfa8aaae665df4aaa441972c7f29 Mon Sep 17 00:00:00 2001 From: Wang Yukai Date: Thu, 4 Sep 2025 03:08:40 +0000 Subject: [PATCH 5/7] fix comments --- pyproject.toml | 1 - tests/conftest.py | 2 +- tests/unit_test/test_basic.py | 1 - tests/unit_test/test_evaluator_unit.py | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b8f0b8e..10baee6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,6 @@ profile = "black" skip_glob = '**/lcmtypes/**' [tool.pytest.ini_options] -# 指定测试搜索路径（避免扫到脚本目录等） testpaths = [ "tests" ] diff --git a/tests/conftest.py b/tests/conftest.py index 38c9556..a1c6517 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ def tmp_cfg(tmp_path): return p -# 按需全局 hook（例：缺 GPU 时自动跳过 gpu 标记） +# global hook: skip mark def pytest_runtest_setup(item): if "gpu" in item.keywords: try: diff --git a/tests/unit_test/test_basic.py b/tests/unit_test/test_basic.py index 70e51bb..cf7cd49 100644 --- a/tests/unit_test/test_basic.py +++ b/tests/unit_test/test_basic.py @@ -18,7 +18,6 @@ def test_sin(x, expected): @pytest.mark.slow def test_slow_example(): - # 假装这里很慢 assert sum(range(10000)) > 0 diff --git a/tests/unit_test/test_evaluator_unit.py b/tests/unit_test/test_evaluator_unit.py index 2c7c7eb..829da0a 100644 --- a/tests/unit_test/test_evaluator_unit.py +++ b/tests/unit_test/test_evaluator_unit.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from internnav.evaluator.vln_pe_evaluator import transform_action_batch # 按你的真实模块路径改 +from internnav.evaluator.vln_pe_evaluator import transform_action_batch @pytest.mark.slow From 2bd0db05fa369fad0a8045f17ee4efcdbc6edc53 Mon Sep 17 00:00:00 2001 From: Wang Yukai Date: Thu, 4 Sep 2025 05:24:02 +0000 Subject: [PATCH 6/7] add timeout in ci --- .github/workflows/ci.yml | 2 +- requirements/test.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b566f50..2c39dff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,4 +66,4 @@ jobs: # run tests /root/miniconda3/envs/internutopia/bin/python -c "import torch,sys;print(sys.executable);print('cuda:',torch.cuda.is_available())" - /root/miniconda3/envs/internutopia/bin/python -m pytest -q -W ignore + /root/miniconda3/envs/internutopia/bin/python -m pytest -q -W ignore --timeout=900 --timeout-method=kill diff --git a/requirements/test.txt b/requirements/test.txt index c233f57..11f4ffc 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,4 +1,5 @@ coverage==7.5.4 pytest==7.3.1 pytest-cov==4.1.0 +pytest-timeout==2.4.0 tomli==2.0.1 From fd5aecbc89d912cfdeb438c94ef8d030b1d7e514 Mon Sep 17 00:00:00 2001 From: Wang Yukai Date: Thu, 4 Sep 2025 05:27:26 +0000 Subject: [PATCH 7/7] add timeout in ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c39dff..cd17d8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,4 +66,4 @@ jobs: # run tests /root/miniconda3/envs/internutopia/bin/python -c "import torch,sys;print(sys.executable);print('cuda:',torch.cuda.is_available())" - /root/miniconda3/envs/internutopia/bin/python -m pytest -q -W ignore --timeout=900 --timeout-method=kill + /root/miniconda3/envs/internutopia/bin/python -m pytest -q -W ignore --timeout=900 --timeout-method=signal