From 5049057beae56e9002199a24e2336fb3c2b0991d Mon Sep 17 00:00:00 2001 From: Manuel Candales Date: Fri, 17 Oct 2025 16:34:26 -0400 Subject: [PATCH] Add Metal backend CI workflow with Voxtral testing --- .../ci_commit_pins/optimum-executorch.txt | 2 +- .github/workflows/metal.yml | 191 ++++++++++++++++++ 2 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/metal.yml diff --git a/.ci/docker/ci_commit_pins/optimum-executorch.txt b/.ci/docker/ci_commit_pins/optimum-executorch.txt index e42ee83cab3..574ccb745d0 100644 --- a/.ci/docker/ci_commit_pins/optimum-executorch.txt +++ b/.ci/docker/ci_commit_pins/optimum-executorch.txt @@ -1 +1 @@ -e8f76b4295584c4328e7fd7971c131cb341c7438 +467660923a5a25e4718e1d6697b93ff1bab4e807 diff --git a/.github/workflows/metal.yml b/.github/workflows/metal.yml new file mode 100644 index 00000000000..4d33b84f5a1 --- /dev/null +++ b/.github/workflows/metal.yml @@ -0,0 +1,191 @@ +name: Test Metal Backend + +on: + pull_request: + push: + branches: + - main + - release/* + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} + cancel-in-progress: false + +jobs: + test-metal-builds: + name: test-executorch-metal-build + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + with: + runner: macos-m2-stable + python-version: '3.11' + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 + script: | + set -eux + + echo "::group::Test ExecuTorch Metal build" + PYTHON_EXECUTABLE=python CMAKE_ARGS="-DEXECUTORCH_BUILD_METAL=ON" ${CONDA_RUN} --no-capture-output ./install_executorch.sh + echo "::endgroup::" + + export-voxtral-metal-artifact: + name: export-voxtral-metal-artifact + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + secrets: inherit + with: + runner: macos-m2-stable + python-version: '3.11' + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 + secrets-env: EXECUTORCH_HF_TOKEN + upload-artifact: voxtral-metal-export + script: | + set -eux + + echo "::group::Setup Huggingface" + ${CONDA_RUN} pip install -U "huggingface_hub[cli]" accelerate + ${CONDA_RUN} huggingface-cli login --token $SECRET_EXECUTORCH_HF_TOKEN + echo "::endgroup::" + + echo "::group::Setup Optimum-ExecuTorch" + OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt) + echo "Using optimum-executorch version: ${OPTIMUM_ET_VERSION}" + ${CONDA_RUN} pip install git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION} + ${CONDA_RUN} pip install mistral-common librosa + echo "::endgroup::" + + echo "::group::Setup ExecuTorch" + PYTHON_EXECUTABLE=python ${CONDA_RUN} ./install_executorch.sh + echo "::endgroup::" + + echo "::group::Pip List" + ${CONDA_RUN} pip list + echo "::endgroup::" + + echo "::group::Export Voxtral" + ${CONDA_RUN} optimum-cli export executorch \ + --model "mistralai/Voxtral-Mini-3B-2507" \ + --task "multimodal-text-to-text" \ + --recipe "metal" \ + --dtype bfloat16 \ + --max_seq_len 1024 \ + --output_dir ./ + ${CONDA_RUN} python -m executorch.extension.audio.mel_spectrogram \ + --feature_size 128 \ + --stack_output \ + --max_audio_len 300 \ + --output_file voxtral_preprocessor.pte + + test -f model.pte + test -f aoti_metal_blob.ptd + test -f voxtral_preprocessor.pte + echo "::endgroup::" + + echo "::group::Store Voxtral Artifacts" + mkdir -p "${RUNNER_ARTIFACT_DIR}" + cp model.pte "${RUNNER_ARTIFACT_DIR}/" + cp aoti_metal_blob.ptd "${RUNNER_ARTIFACT_DIR}/" + cp voxtral_preprocessor.pte "${RUNNER_ARTIFACT_DIR}/" + ls -al "${RUNNER_ARTIFACT_DIR}" + echo "::endgroup::" + + test-voxtral-metal-e2e: + name: test-voxtral-metal-e2e + needs: export-voxtral-metal-artifact + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + with: + runner: macos-m2-stable + python-version: '3.11' + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 + download-artifact: voxtral-metal-export + script: | + set -eux + + echo "::group::Print machine info" + uname -a + if [ $(uname -s) == Darwin ]; then + sw_vers + # Print RAM in GB + RAM_BYTES=$(sysctl -n hw.memsize) + RAM_GB=$(echo "scale=2; $RAM_BYTES/1024/1024/1024" | bc) + echo "Available RAM (GB): $RAM_GB" + sysctl machdep.cpu.brand_string + sysctl machdep.cpu.core_count + # Print number of GPU cores (Apple Silicon) + if command -v system_profiler &> /dev/null; then + GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Total Number of Cores/ {print $5; exit}') + if [ -z "$GPU_CORES" ]; then + # Fallback: try to parse "Core Count" from Apple GPU section + GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Core Count/ {print $3; exit}') + fi + echo "GPU Cores: ${GPU_CORES:-Unknown}" + else + echo "system_profiler not available, cannot determine GPU cores." + fi + fi + echo "::endgroup::" + + echo "::group::Setup ExecuTorch Requirements" + CMAKE_ARGS="-DEXECUTORCH_BUILD_METAL=ON" ${CONDA_RUN} --no-capture-output ./install_requirements.sh + echo "::endgroup::" + + echo "::group::Pip List" + ${CONDA_RUN} pip list + echo "::endgroup::" + + echo "::group::Prepare Voxtral Artifacts" + cp "${RUNNER_ARTIFACT_DIR}/model.pte" . + cp "${RUNNER_ARTIFACT_DIR}/aoti_metal_blob.ptd" . + cp "${RUNNER_ARTIFACT_DIR}/voxtral_preprocessor.pte" . + TOKENIZER_URL="https://huggingface.co/mistralai/Voxtral-Mini-3B-2507/resolve/main/tekken.json" + curl -L $TOKENIZER_URL -o tekken.json + ls -al model.pte aoti_metal_blob.ptd voxtral_preprocessor.pte tekken.json + echo "::endgroup::" + + echo "::group::Create Test Audio File" + say -o call_samantha_hall.aiff "Call Samantha Hall" + afconvert -f WAVE -d LEI16 call_samantha_hall.aiff call_samantha_hall.wav + echo "::endgroup::" + + echo "::group::Build Voxtral Runner" + ${CONDA_RUN} cmake --preset llm \ + -DEXECUTORCH_BUILD_METAL=ON \ + -DCMAKE_INSTALL_PREFIX=cmake-out \ + -DCMAKE_BUILD_TYPE=Release \ + -Bcmake-out -S. + ${CONDA_RUN} cmake --build cmake-out -j$(( $(sysctl -n hw.ncpu) - 1 )) --target install --config Release + + ${CONDA_RUN} cmake -DEXECUTORCH_BUILD_METAL=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -Sexamples/models/voxtral \ + -Bcmake-out/examples/models/voxtral/ + ${CONDA_RUN} cmake --build cmake-out/examples/models/voxtral --target voxtral_runner --config Release + echo "::endgroup::" + + echo "::group::Run Voxtral Runner" + set +e + OUTPUT=$(cmake-out/examples/models/voxtral/voxtral_runner \ + --model_path model.pte \ + --data_path aoti_metal_blob.ptd \ + --tokenizer_path tekken.json \ + --audio_path call_samantha_hall.wav \ + --processor_path voxtral_preprocessor.pte \ + --temperature 0 2>&1) + EXIT_CODE=$? + set -e + + echo "$OUTPUT" + + if ! echo "$OUTPUT" | grep -iq "Samantha"; then + echo "Expected output 'Samantha' not found in output" + exit 1 + fi + + if [ $EXIT_CODE -ne 0 ]; then + echo "Unexpected exit code: $EXIT_CODE" + exit $EXIT_CODE + fi + echo "::endgroup::"