From 132f5536021a877593944952a1c59b56d7a0163c Mon Sep 17 00:00:00 2001 From: Can Balioglu Date: Thu, 2 Nov 2023 09:10:19 -0700 Subject: [PATCH 1/4] Revise versioning --- .github/workflows/_build_doc.yaml | 6 +- .github/workflows/_build_wheel-linux.yaml | 7 +- .github/workflows/_build_wheel-macos.yaml | 7 +- .github/workflows/_build_wheels.yaml | 12 ++-- .github/workflows/_publish_doc.yaml | 6 +- .github/workflows/_test_py_devel.yaml | 67 ------------------- .github/workflows/ci_build_wheels.yaml | 7 -- .github/workflows/release.yaml | 14 ++-- CONTRIBUTING.md | 25 ++++--- INSTALL_FROM_SOURCE.md | 10 +-- README.md | 9 +-- VERSION | 2 +- fairseq2n/CMakeLists.txt | 2 +- fairseq2n/python/setup.py | 2 +- fairseq2n/python/src/fairseq2n/__init__.py | 2 +- setup.py | 18 ++--- src/fairseq2/__init__.py | 2 +- tests/common.py | 8 --- .../audio/test_waveform_to_fbank_converter.py | 6 +- tests/unit/data/data_pipeline/test_collate.py | 3 +- tests/unit/data/data_pipeline/test_concat.py | 5 -- .../data/data_pipeline/test_round_robin.py | 5 -- tests/unit/data/data_pipeline/test_sample.py | 6 +- tests/unit/data/test_collater.py | 6 +- tests/unit/data/text/test_sentencepiece.py | 6 +- tools/set-project-version.sh | 15 ++--- 26 files changed, 72 insertions(+), 186 deletions(-) delete mode 100644 .github/workflows/_test_py_devel.yaml diff --git a/.github/workflows/_build_doc.yaml b/.github/workflows/_build_doc.yaml index d4b5802a0..ef24104ba 100644 --- a/.github/workflows/_build_doc.yaml +++ b/.github/workflows/_build_doc.yaml @@ -13,7 +13,7 @@ on: py: type: string default: '3.11' - version_overwrite: + version_override: type: string default: '' @@ -48,9 +48,9 @@ jobs: --requirement fairseq2n/python/requirements-build.txt\ --requirement doc/requirements.txt - name: Overwrite project version - if: inputs.version_overwrite + if: inputs.version_override run: | - tools/set-project-version.sh ${{ inputs.version_overwrite }} + tools/set-project-version.sh ${{ inputs.version_override }} - name: Install fairseq2n run: | pip install --editable fairseq2n/python diff --git a/.github/workflows/_build_wheel-linux.yaml b/.github/workflows/_build_wheel-linux.yaml index 5707b05b4..e3fbef0c0 100644 --- a/.github/workflows/_build_wheel-linux.yaml +++ b/.github/workflows/_build_wheel-linux.yaml @@ -25,7 +25,7 @@ on: artifacts_name: type: string default: 'wheels' - version_overwrite: + version_override: type: string default: '' label_version_with_variant: @@ -69,9 +69,9 @@ jobs: run: | pip install --requirement fairseq2n/python/requirements-build.txt - name: Override project version - if: inputs.version_overwrite + if: inputs.version_override run: | - tools/set-project-version.sh ${{ inputs.version_overwrite }} + tools/set-project-version.sh ${{ inputs.version_override }} - name: Set build variant as local version label if: inputs.label_version_with_variant env: @@ -116,7 +116,6 @@ jobs: -GNinja\ -DCMAKE_BUILD_TYPE=$build_type\ -DCMAKE_CUDA_ARCHITECTURES=$cuda_archs\ - -DFAIRSEQ2N_INSTALL_STANDALONE=ON\ -DFAIRSEQ2N_PERFORM_LTO=$lto\ -DFAIRSEQ2N_SANITIZERS="${SANITIZERS/_/;}"\ -DFAIRSEQ2N_TREAT_WARNINGS_AS_ERRORS=ON\ diff --git a/.github/workflows/_build_wheel-macos.yaml b/.github/workflows/_build_wheel-macos.yaml index 60a5f7414..361b99855 100644 --- a/.github/workflows/_build_wheel-macos.yaml +++ b/.github/workflows/_build_wheel-macos.yaml @@ -19,7 +19,7 @@ on: artifacts_name: type: string default: 'wheels' - version_overwrite: + version_override: type: string default: '' @@ -50,9 +50,9 @@ jobs: run: | pip install --requirement fairseq2n/python/requirements-build.txt - name: Override project version - if: inputs.version_overwrite + if: inputs.version_override run: | - tools/set-project-version.sh ${{ inputs.version_overwrite }} + tools/set-project-version.sh ${{ inputs.version_override }} - name: Build fairseq2n working-directory: fairseq2n run: | @@ -60,7 +60,6 @@ jobs: -GNinja\ -DCMAKE_BUILD_TYPE=Release\ -DPython3_FIND_FRAMEWORK=NEVER\ - -DFAIRSEQ2N_INSTALL_STANDALONE=ON\ -DFAIRSEQ2N_PERFORM_LTO=OFF\ -DFAIRSEQ2N_TREAT_WARNINGS_AS_ERRORS=ON\ -DFAIRSEQ2N_THREAD_LIB=""\ diff --git a/.github/workflows/_build_wheels.yaml b/.github/workflows/_build_wheels.yaml index 6ee6e3b83..d6a8e5330 100644 --- a/.github/workflows/_build_wheels.yaml +++ b/.github/workflows/_build_wheels.yaml @@ -10,7 +10,7 @@ on: release_type: type: string required: true - version_overwrite: + version_override: type: string required: true @@ -30,7 +30,7 @@ jobs: py: ${{ matrix.py }} variant: ${{ matrix.variant }} sanitizers: ${{ matrix.sanitizers }} - version_overwrite: ${{ inputs.version_overwrite }} + version_override: ${{ inputs.version_override }} build_pt131_wheel-linux: name: Build wheels (pt${{ matrix.torch }}, py${{ matrix.py }}, linux-x86_64, ${{ matrix.variant }}, ${{ matrix.sanitizers }}) @@ -47,7 +47,7 @@ jobs: py: ${{ matrix.py }} variant: ${{ matrix.variant }} sanitizers: ${{ matrix.sanitizers }} - version_overwrite: ${{ inputs.version_overwrite }} + version_override: ${{ inputs.version_override }} build_pt20_wheel-linux: name: Build wheels (pt${{ matrix.torch }}, py${{ matrix.py }}, linux-x86_64, ${{ matrix.variant }}, ${{ matrix.sanitizers }}) @@ -80,7 +80,7 @@ jobs: py: ${{ matrix.py }} variant: ${{ matrix.variant }} sanitizers: ${{ matrix.sanitizers }} - version_overwrite: ${{ inputs.version_overwrite }} + version_override: ${{ inputs.version_override }} run_integration_tests: ${{ matrix.run_integration_tests }} build_pypi_wheel-linux: @@ -98,7 +98,7 @@ jobs: py: ${{ matrix.py }} variant: ${{ matrix.variant }} artifacts_name: 'pypi' - version_overwrite: ${{ inputs.version_overwrite }} + version_override: ${{ inputs.version_override }} label_version_with_variant: false # build_pypi_wheel-macos: @@ -114,4 +114,4 @@ jobs: # torch: ${{ matrix.torch }} # py: ${{ matrix.py }} # artifacts_name: 'pypi' -# version_overwrite: ${{ inputs.version_overwrite }} +# version_override: ${{ inputs.version_override }} diff --git a/.github/workflows/_publish_doc.yaml b/.github/workflows/_publish_doc.yaml index f5f347dbb..7bf3c8c58 100644 --- a/.github/workflows/_publish_doc.yaml +++ b/.github/workflows/_publish_doc.yaml @@ -37,12 +37,12 @@ jobs: doc_dir=nightly else # Ignore pre-release segment for directory name. - mmp_version=$(echo $version | grep --only-matching --extended-regexp '^([0-9]+\.)*[0-9]+' -) + mmm_version=$(echo $version | grep --only-matching --extended-regexp '^([0-9]+\.)*[0-9]+' -) - doc_dir=$mmp_version + doc_dir=$mmm_version # If we have a stable release, update the 'stable' symlink. - if [[ $version == $mmp_version ]]; then + if [[ $version == $mmm_version ]]; then ln --symbolic --no-target-directory --force $doc_dir stable fi fi diff --git a/.github/workflows/_test_py_devel.yaml b/.github/workflows/_test_py_devel.yaml deleted file mode 100644 index 7b5e47a58..000000000 --- a/.github/workflows/_test_py_devel.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -on: - workflow_call: - inputs: - torch: - type: string - required: true - py: - type: string - required: true - arch: - type: string - default: 'x86_64' - run_on_device: - type: boolean - default: false - -defaults: - run: - shell: bash - -jobs: - test: - name: Test - runs-on: ubuntu-latest - container: - image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_${{ inputs.arch }}:1-cpu - steps: - - name: Check-out the repository - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Install libsndfile - run: | - yum --assumeyes install libsndfile-devel - - name: Create the Python virtual environment - run: | - python${{ inputs.py }} -m venv ~/venv - - echo ~/venv/bin >> "$GITHUB_PATH" - - name: Install PyTorch - run: | - pip install torch==${{ inputs.torch }} - - name: Install fairseq2 - run: | - unset CI - - pip install --editable . - - name: Install pytest - run: | - pip install --requirement requirements-devel.txt - - name: Run Python tests - env: - RUN_ON_DEVICE: ${{ inputs.run_on_device }} - run: | - if [[ $RUN_ON_DEVICE != true ]]; then - dev=cpu - else - dev=cuda:0 - fi - - pytest -rP --device $dev --verbose diff --git a/.github/workflows/ci_build_wheels.yaml b/.github/workflows/ci_build_wheels.yaml index f9684c600..e6d6739d4 100644 --- a/.github/workflows/ci_build_wheels.yaml +++ b/.github/workflows/ci_build_wheels.yaml @@ -14,13 +14,6 @@ on: - 'doc/**' jobs: - test_py_devel: - name: Test Python development - uses: ./.github/workflows/_test_py_devel.yaml - with: - torch: '2.0.1' - py: '3.11' - build_wheel-linux: name: Build wheels (pt${{ matrix.torch }}, py${{ matrix.py }}, linux-x86_64, ${{ matrix.variant }}, ${{ matrix.sanitizers }}) uses: ./.github/workflows/_build_wheel-linux.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4372a80cb..a1432cc9f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -28,7 +28,7 @@ jobs: process_version: name: Process project version outputs: - version_overwrite: ${{ steps.stamp_version.outputs.version_overwrite }} + version_override: ${{ steps.stamp_version.outputs.version_override }} runs-on: ubuntu-latest steps: - name: Check-out the repository @@ -40,8 +40,8 @@ jobs: version=$(cat VERSION) if [[ $RELEASE_TYPE != nightly ]]; then - if [[ $version == *+* ]]; then - echo "A PEP 440 version with a local label cannot be released as '$RELEASE_TYPE'." >&2 + if [[ $version == *.dev* ]]; then + echo "A PEP 440 version with dev segment cannot be released as '$RELEASE_TYPE'." >&2 exit 1 fi @@ -52,8 +52,8 @@ jobs: run: | version=$(cat VERSION) - # Remove local version label and append date-stamped dev segment. - echo version_overwrite=${version%+*}.dev$(date +%Y%m%d) >> "$GITHUB_OUTPUT" + # Append date-stamped dev segment. + echo version_override=${version%.dev*}.dev$(date +%Y%m%d) >> "$GITHUB_OUTPUT" lint: name: Lint @@ -65,14 +65,14 @@ jobs: uses: ./.github/workflows/_build_wheels.yaml with: release_type: ${{ inputs.release_type }} - version_overwrite: ${{ needs.process_version.outputs.version_overwrite }} + version_override: ${{ needs.process_version.outputs.version_override }} build_doc: name: Build documentation needs: [process_version] uses: ./.github/workflows/_build_doc.yaml with: - version_overwrite: ${{ needs.process_version.outputs.version_overwrite }} + version_override: ${{ needs.process_version.outputs.version_override }} publish: name: Publish diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1dccdc7cd..e6171e5da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,21 +1,28 @@ # Contributing to fairseq2 -We want to make contributing to fairseq2 as easy and transparent as possible. -Please make sure to read this guideline carefully. +We want to make contributing to fairseq2 as easy as possible. Please make sure +to read this guideline carefully. ## Setting up Development Environment fairseq2 consists of two packages; the user-facing fairseq2 package implemented in pure Python, and the fairseq2n package that contains the C++ and CUDA -portions of the library. If fairseq2n is available as a pre-built package for -your system (check installation instructions for your operating system in -[README](.)), and if you are interested in only modifying Python portions of +portions of the library. If pre-built fairseq2n nightlies are available for your +system (check installation instructions for your operating +system in [README](.#nightlies)), and if you are interested in only modifying Python portions of fairseq2, you can use an editable pip installation as described below. Otherwise, if you are planning to work on C++ or CUDA, or if fairseq2n is not -available as a pre-built package for your system, please follow the install +available as a pre-built package for your system, please follow the installation instructions [here](INSTALL_FROM_SOURCE.md). -For an editable installation, first, clone the fairseq2 repository to your -machine: +For an editable installation, first, install a nightly build of fairseq2n (shown +for PyTorch `2.0.1` and variant `cu118`): + +```sh +pip install fairseq2n\ + --pre --upgrade --extra-index-url https://fair.pkg.atmeta.com/fairseq2/whl/nightly/pt2.0.1/cu118 +``` + +Then, clone the fairseq2 repository to your machine: ```sh git clone https://github.com/facebookresearch/fairseq2.git @@ -23,7 +30,7 @@ git clone https://github.com/facebookresearch/fairseq2.git cd fairseq2 ``` -Then, install the fairseq2 package in editable mode: +And, install the fairseq2 package in editable mode: ```sh pip install -e . diff --git a/INSTALL_FROM_SOURCE.md b/INSTALL_FROM_SOURCE.md index 69f54f518..0020d4122 100644 --- a/INSTALL_FROM_SOURCE.md +++ b/INSTALL_FROM_SOURCE.md @@ -112,7 +112,7 @@ configure the build: ```sh cd fairseq2n -cmake -GNinja -DFAIRSEQ2N_INSTALL_STANDALONE=ON -B build +cmake -GNinja -B build ``` Once the configuration step is complete, build fairseq2n using: @@ -138,7 +138,7 @@ option `ON`. When turned on, the version of the CUDA Toolkit installed on your machine and the version of CUDA that was used to build PyTorch must match: ```sh -cmake -GNinja -DFAIRSEQ2N_INSTALL_STANDALONE=ON -DFAIRSEQ2N_USE_CUDA=ON -B build +cmake -GNinja -DFAIRSEQ2N_USE_CUDA=ON -B build ``` Similar to CPU-only build, follow this command with: @@ -170,10 +170,10 @@ pip install . cd - ``` -Then, fairseq2 (note the `FAIRSEQ2N_DEVEL` environment variable): +Then, fairseq2: ```sh -FAIRSEQ2N_DEVEL=1 pip install . +pip install . ``` ### Editable Install @@ -187,7 +187,7 @@ pip install -e . cd - -FAIRSEQ2N_DEVEL=1 pip install -e . +pip install -e . ``` Optionally, you can also install the development tools (e.g. linters, diff --git a/README.md b/README.md index ad7a7bbcf..9482d9049 100644 --- a/README.md +++ b/README.md @@ -101,10 +101,11 @@ pip install fairseq2\ > [!WARNING] > fairseq2 relies on the C++ API of PyTorch which has no API/ABI compatibility -> between releases. This means you have to ensure that you **use the exact same -> PyTorch version and variant as your fairseq2 installation was built against**; -> otherwise, you might experience issues like immediate process crashes or -> spurious segfaults. +> between releases. This means you have to **first install a PyTorch version +> that exactly matches the version that fairseq2 was built against**. Otherwise, +> you might experience issues like immediate process crashes or spurious +> segfaults. For the same reason, if you upgrade your PyTorch version, you must +> also upgrade your fairseq2 installation. ### Nightlies For Linux, we also host nightly builds on FAIR's package repository. The diff --git a/VERSION b/VERSION index 9147529c7..679c772d0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.0+devel +0.2.0.dev0 diff --git a/fairseq2n/CMakeLists.txt b/fairseq2n/CMakeLists.txt index 1e7b7f79d..a51fd25a7 100644 --- a/fairseq2n/CMakeLists.txt +++ b/fairseq2n/CMakeLists.txt @@ -50,7 +50,7 @@ option(FAIRSEQ2N_INSTALL_STANDALONE #DESCRIPTION "Installs with relative rpaths." #VALUE - OFF + ON ) option(FAIRSEQ2N_PERFORM_LTO diff --git a/fairseq2n/python/setup.py b/fairseq2n/python/setup.py index c8bdfc3d1..91917daee 100644 --- a/fairseq2n/python/setup.py +++ b/fairseq2n/python/setup.py @@ -136,7 +136,7 @@ def get_inputs(self) -> List[str]: "install_cmake": install_cmake, }, name="fairseq2n", - version="0.2.0+devel", + version="0.2.0.dev0", description="FAIR Sequence Modeling Toolkit (Native)", long_description="https://github.com/facebookresearch/fairseq2", long_description_content_type="text/plain", diff --git a/fairseq2n/python/src/fairseq2n/__init__.py b/fairseq2n/python/src/fairseq2n/__init__.py index 250f41a82..b6f680ecb 100644 --- a/fairseq2n/python/src/fairseq2n/__init__.py +++ b/fairseq2n/python/src/fairseq2n/__init__.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import List, Optional, Tuple -__version__ = "0.2.0+devel" +__version__ = "0.2.0.dev0" # Keeps the shared libraries that we load using our own extended lookup logic diff --git a/setup.py b/setup.py index 54ba51ba4..4b2e4c610 100644 --- a/setup.py +++ b/setup.py @@ -4,20 +4,16 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from os import environ - from setuptools import find_packages, setup -version = "0.2.0+devel" - -# DO NOT forget to update the fallback version with each release! -fallback_fairseq2n_version = "0.1.1" +version = "0.2.0.dev0" -if "CI" in environ or "FAIRSEQ2N_DEVEL" in environ: - fairseq2n_version = version +# If this is a local development install, allow nightly fairseq2n builds to +# take precedence. +if version.endswith(".dev0"): + fairseq2n_version_spec = f">={version},<={version[:-5]}" else: - fairseq2n_version = fallback_fairseq2n_version - + fairseq2n_version_spec = f"=={version}" setup( name="fairseq2", @@ -47,7 +43,7 @@ zip_safe=False, python_requires=">=3.8", install_requires=[ - "fairseq2n==" + fairseq2n_version, + "fairseq2n" + fairseq2n_version_spec, "jiwer~=3.0", "numpy~=1.23", "overrides~=7.3", diff --git a/src/fairseq2/__init__.py b/src/fairseq2/__init__.py index 792b70428..37b1f9732 100644 --- a/src/fairseq2/__init__.py +++ b/src/fairseq2/__init__.py @@ -8,7 +8,7 @@ # We import fairseq2n to report any initialization error eagerly. import fairseq2n -__version__ = "0.2.0+devel" +__version__ = "0.2.0.dev0" # If ``True``, indicates that we are run under Sphinx. diff --git a/tests/common.py b/tests/common.py index 62ff7dd0a..e84c7ac36 100644 --- a/tests/common.py +++ b/tests/common.py @@ -60,11 +60,3 @@ def tmp_rng_seed(device: Device, seed: int = 0) -> Generator[None, None, None]: torch.manual_seed(seed) yield - - -def python_devel_only() -> bool: - """Return ``True`` if fairseq2 is installed for Python development only.""" - import fairseq2 - import fairseq2n - - return fairseq2.__version__ != fairseq2n.__version__ diff --git a/tests/unit/data/audio/test_waveform_to_fbank_converter.py b/tests/unit/data/audio/test_waveform_to_fbank_converter.py index 89c074999..07f28cb1e 100644 --- a/tests/unit/data/audio/test_waveform_to_fbank_converter.py +++ b/tests/unit/data/audio/test_waveform_to_fbank_converter.py @@ -16,15 +16,11 @@ WaveformToFbankConverter, ) from fairseq2.memory import MemoryBlock -from tests.common import assert_equal, device, python_devel_only +from tests.common import assert_equal, device TEST_OGG_PATH: Final = Path(__file__).parent.joinpath("test.ogg") -@pytest.mark.skipif( - python_devel_only(), - reason="New fairseq2n API in Python-only installation. Skipping till v0.2.", -) class TestWaveformToFbankConverter: def test_call_works(self) -> None: audio = self.get_audio() diff --git a/tests/unit/data/data_pipeline/test_collate.py b/tests/unit/data/data_pipeline/test_collate.py index d0a1fb8c0..134b6b297 100644 --- a/tests/unit/data/data_pipeline/test_collate.py +++ b/tests/unit/data/data_pipeline/test_collate.py @@ -8,11 +8,10 @@ import torch from fairseq2.data import Collater, read_sequence -from tests.common import assert_equal, device, python_devel_only +from tests.common import assert_equal, device class TestCollateOp: - @pytest.mark.skipif(python_devel_only(), reason="fairseq2n 0.2.0") @pytest.mark.parametrize("pad_to_multiple", [1, 2, 3, 8]) def test_op_works(self, pad_to_multiple: int) -> None: pad_value = 3 diff --git a/tests/unit/data/data_pipeline/test_concat.py b/tests/unit/data/data_pipeline/test_concat.py index 659bf3f8b..0b7d99149 100644 --- a/tests/unit/data/data_pipeline/test_concat.py +++ b/tests/unit/data/data_pipeline/test_concat.py @@ -8,13 +8,8 @@ from fairseq2.data import DataPipeline, DataPipelineError, read_sequence from fairseq2.data.text import read_text -from tests.common import python_devel_only -@pytest.mark.skipif( - python_devel_only(), - reason="New fairseq2n API in Python-only installation. Skipping till v0.2.", -) class TestConcatOp: def test_op_works(self) -> None: pipeline1 = read_sequence([1, 2, 3, 4]).and_return() diff --git a/tests/unit/data/data_pipeline/test_round_robin.py b/tests/unit/data/data_pipeline/test_round_robin.py index e8ba0e984..f77e112e0 100644 --- a/tests/unit/data/data_pipeline/test_round_robin.py +++ b/tests/unit/data/data_pipeline/test_round_robin.py @@ -8,7 +8,6 @@ from fairseq2.data import DataPipeline, DataPipelineError, read_sequence from fairseq2.data.text import read_text -from tests.common import python_devel_only class TestRoundRobinOp: @@ -77,10 +76,6 @@ def test_op_works_when_pipelines_have_different_lengths(self) -> None: pipeline.reset() - @pytest.mark.skipif( - python_devel_only(), - reason="New fairseq2n API in Python-only installation. Skipping till v0.2.", - ) def test_op_works_when_pipelines_have_different_lengths_stop_at_shortest( self, ) -> None: diff --git a/tests/unit/data/data_pipeline/test_sample.py b/tests/unit/data/data_pipeline/test_sample.py index 989fefc28..925f86039 100644 --- a/tests/unit/data/data_pipeline/test_sample.py +++ b/tests/unit/data/data_pipeline/test_sample.py @@ -10,15 +10,11 @@ from fairseq2.data import DataPipeline, read_sequence from fairseq2.data.text import read_text from fairseq2.utils.version import is_pt2_or_greater -from tests.common import python_devel_only, tmp_rng_seed +from tests.common import tmp_rng_seed cpu_device = torch.device("cpu") -@pytest.mark.skipif( - python_devel_only(), - reason="New fairseq2n API in Python-only installation. Skipping till v0.2.", -) @pytest.mark.skipif( not is_pt2_or_greater(), reason="Different sampling results with versions lower than PyTorch 2.0", diff --git a/tests/unit/data/test_collater.py b/tests/unit/data/test_collater.py index b5a3d568f..8be672c84 100644 --- a/tests/unit/data/test_collater.py +++ b/tests/unit/data/test_collater.py @@ -9,13 +9,9 @@ from torch.nn.functional import pad from fairseq2.data import CollateOptionsOverride, Collater -from tests.common import assert_close, assert_equal, device, python_devel_only +from tests.common import assert_close, assert_equal, device -@pytest.mark.skipif( - python_devel_only(), - reason="New fairseq2n API in Python-only installation. Skipping till v0.2.", -) class TestCollater: def test_call_works_when_input_has_only_non_composite_types(self) -> None: # fmt: off diff --git a/tests/unit/data/text/test_sentencepiece.py b/tests/unit/data/text/test_sentencepiece.py index 11a3f1e4b..72bfb6a09 100644 --- a/tests/unit/data/text/test_sentencepiece.py +++ b/tests/unit/data/text/test_sentencepiece.py @@ -18,15 +18,11 @@ SentencePieceModel, ) from fairseq2.typing import DataType -from tests.common import assert_equal, device, python_devel_only +from tests.common import assert_equal, device TEST_SPM_PATH: Final = Path(__file__).parent.joinpath("test.spm") -@pytest.mark.skipif( - python_devel_only(), - reason="New fairseq2n API in Python-only installation. Skipping till v0.2.", -) class TestSentencePieceModel: sentence: ClassVar[str] token_indices: ClassVar[List[int]] diff --git a/tools/set-project-version.sh b/tools/set-project-version.sh index 82e73a0fe..c28d33adf 100755 --- a/tools/set-project-version.sh +++ b/tools/set-project-version.sh @@ -37,9 +37,9 @@ function extract_pep_version echo "$1" } -function extract_mmp_version +function extract_mmm_version { - # Grep major, minor, and patch segments. + # Grep major, minor, and micro segments. echo "$1" | grep --only-matching --extended-regexp '^([0-9]+\.)*[0-9]+' - } @@ -52,7 +52,7 @@ if [[ $1 == -h || $1 == --help ]]; then fi pep_ver=$(extract_pep_version "$1") -mmp_ver=$(extract_mmp_version "$1") +mmm_ver=$(extract_mmm_version "$1") base=$(cd "$(dirname "$0")/.." && pwd) @@ -69,7 +69,7 @@ replace_match\ # Update fairseq2n CMake project. replace_match\ "$base/fairseq2n/CMakeLists.txt"\ - "s/VERSION .* LANGUAGES/VERSION $mmp_ver LANGUAGES/" + "s/VERSION .* LANGUAGES/VERSION $mmm_ver LANGUAGES/" # Update fairseq2n Python distribution. replace_match\ @@ -81,12 +81,5 @@ replace_match\ "$base/fairseq2n/python/src/fairseq2n/__init__.py"\ "s/^__version__ = \".*\"$/__version__ = \"$pep_ver\"/" -if [[ $pep_ver != *+devel ]]; then - # Update fairseq2n fallback version. - replace_match\ - "$base/setup.py"\ - "s/^fallback_fairseq2n_version = \".*\"$/fallback_fairseq2n_version = \"$pep_ver\"/" -fi - # Update VERSION file. echo "$pep_ver" > "$base/VERSION" From 7d1358a4516ed22ab5096d13b04dd989895c131c Mon Sep 17 00:00:00 2001 From: Can Balioglu Date: Thu, 2 Nov 2023 09:19:59 -0700 Subject: [PATCH 2/4] Update README --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9482d9049..7b0a7569b 100644 --- a/README.md +++ b/README.md @@ -101,11 +101,11 @@ pip install fairseq2\ > [!WARNING] > fairseq2 relies on the C++ API of PyTorch which has no API/ABI compatibility -> between releases. This means you have to **first install a PyTorch version -> that exactly matches the version that fairseq2 was built against**. Otherwise, -> you might experience issues like immediate process crashes or spurious -> segfaults. For the same reason, if you upgrade your PyTorch version, you must -> also upgrade your fairseq2 installation. +> between releases. This means **you have to install the fairseq2 variant that +> exactly matches your PyTorch version**. Otherwise, you might experience issues +> like immediate process crashes or spurious segfaults. For the same reason, if +> you upgrade your PyTorch version, you must also upgrade your fairseq2 +> installation. ### Nightlies For Linux, we also host nightly builds on FAIR's package repository. The From 87abfcb56c7a568a444f6aa671e14a32edae7e3d Mon Sep 17 00:00:00 2001 From: Can Balioglu Date: Thu, 2 Nov 2023 09:29:42 -0700 Subject: [PATCH 3/4] Update doc --- CONTRIBUTING.md | 21 ++++++++++++++------- INSTALL_FROM_SOURCE.md | 6 +++--- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e6171e5da..a28d3f1c3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,13 +6,12 @@ to read this guideline carefully. ## Setting up Development Environment fairseq2 consists of two packages; the user-facing fairseq2 package implemented in pure Python, and the fairseq2n package that contains the C++ and CUDA -portions of the library. If pre-built fairseq2n nightlies are available for your -system (check installation instructions for your operating -system in [README](.#nightlies)), and if you are interested in only modifying Python portions of -fairseq2, you can use an editable pip installation as described below. -Otherwise, if you are planning to work on C++ or CUDA, or if fairseq2n is not -available as a pre-built package for your system, please follow the installation -instructions [here](INSTALL_FROM_SOURCE.md). +portions of the library. If pre-built fairseq2n nightly packages are available +for your system (check [README](.#nightlies)), and if you are interested in only +modifying Python portions of fairseq2, you can use an editable pip installation +as described below. Otherwise, if you are planning to work on C++ or CUDA, or if +fairseq2n is not available as a pre-built package for your system, please follow +the installation instructions [here](INSTALL_FROM_SOURCE.md). For an editable installation, first, install a nightly build of fairseq2n (shown for PyTorch `2.0.1` and variant `cu118`): @@ -22,6 +21,14 @@ pip install fairseq2n\ --pre --upgrade --extra-index-url https://fair.pkg.atmeta.com/fairseq2/whl/nightly/pt2.0.1/cu118 ``` +> [!WARNING] +> fairseq2n relies on the C++ API of PyTorch which has no API/ABI compatibility +> between releases. This means **you have to install the fairseq2n variant that +> exactly matches your PyTorch version**. Otherwise, you might experience issues +> like immediate process crashes or spurious segfaults. For the same reason, if +> you upgrade your PyTorch version, you must also upgrade your fairseq2n +> installation. + Then, clone the fairseq2 repository to your machine: ```sh diff --git a/INSTALL_FROM_SOURCE.md b/INSTALL_FROM_SOURCE.md index 0020d4122..8148da9f9 100644 --- a/INSTALL_FROM_SOURCE.md +++ b/INSTALL_FROM_SOURCE.md @@ -5,8 +5,8 @@ want to work on the C++/CUDA code of fairseq2. > [!NOTE] > If you plan to only modify Python portions of fairseq2, and if fairseq2 -> provides a pre-built package for your system, we recommend using an editable -> pip installation as described in +> provides a pre-built nightly package for your system, we recommend using an +> editable pip installation as described in > [Contribution Guidelines](CONTRIBUTING.md#setting-up-development-environment). @@ -81,7 +81,7 @@ brew install libsndfile ### 3.2 PyTorch Follow the instructions on [pytorch.org](https://pytorch.org) to install the -desired PyTorch version. Note that fairseq2 supports only PyTorch 1.12 or +desired PyTorch version. greater. ### 3.3 CUDA From d7ef42a013d5d4db93b902deefcc46786d02d1f6 Mon Sep 17 00:00:00 2001 From: Can Balioglu Date: Thu, 2 Nov 2023 10:04:06 -0700 Subject: [PATCH 4/4] Update doc --- CONTRIBUTING.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a28d3f1c3..28f1e2a8a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,6 +50,12 @@ formatters): pip install -r requirements-devel.txt ``` +> [!NOTE] +> Any time you pull the latest fairseq2 commits from GitHub, make sure to re-run +> the fairseq2n installation command above to get the most up-to-date binary. If +> you observe runtime or test failures after the installation, it might be +> because the latest nightlies are not published yet. If the problem persists +> after about 12 hours, please create a GitHub issue. ## Testing Your Work