windows.yml

name: Windows (VS 2019, Python 3.11)
on:
  workflow_dispatch:
  pull_request:
  merge_group:
  push:
    branches:
      - master
      - 'releases/**'

concurrency:
  # github.ref is not unique in post-commit
  group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-windows
  cancel-in-progress: true

env:
  PYTHON_VERSION: '3.11'
  OV_BRANCH: 'releases/2024/3'
  OV_TARBALL: ''

permissions: read-all

jobs:
  openvino_download:
    outputs:
      status: ${{ steps.openvino_download.outcome }}
    timeout-minutes: 10
    defaults:
      run:
        shell: bash
    runs-on: ubuntu-20.04

    steps:
      - name: Download OpenVINO build
        id: openvino_download
        run: |
          wget ${{ env.OV_TARBALL}} --progress=bar:force:noscroll -O openvino_package.zip
          unzip -l openvino_package.zip
        continue-on-error: true

      #
      # Upload to artifacts
      #

      - name: Upload openvino package
        if: steps.openvino_download.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: openvino_package
          path: openvino_package.zip
          if-no-files-found: 'error'

  openvino_build:
    needs: [openvino_download]
    if: needs.openvino_download.outputs.status != 'success'
    timeout-minutes: 150
    defaults:
      run:
        shell: pwsh
    runs-on: windows-2019-16-core
    env:
      CMAKE_BUILD_TYPE: 'Release'
      CMAKE_GENERATOR: 'Ninja Multi-Config'
      CMAKE_CXX_COMPILER_LAUNCHER: ccache
      CMAKE_C_COMPILER_LAUNCHER: ccache
      OPENVINO_REPO: ${{ github.workspace }}\\openvino
      INSTALL_DIR: ${{ github.workspace }}\\openvino\\install
      BUILD_DIR: ${{ github.workspace }}\\openvino\\build

    steps:
      - name: git configuration
        run: git config --system core.longpaths true

      - name: Clone OpenVINO
        uses: actions/checkout@v4
        with:
          repository: 'openvinotoolkit/openvino'
          path: ${{ env.OPENVINO_REPO }}
          submodules: 'true'
          ref: ${{ env.OV_BRANCH }}

      #
      # Dependencies
      #

      - name: Setup Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'

      - name: Install python dependencies
        run: |
          # For Python API: build and wheel packaging
          python3 -m pip install -r ${env:OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt

      - name: Install build dependencies
        run: |
          Invoke-WebRequest https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip -OutFile ninja-win.zip -MaximumRetryCount 10
          Expand-Archive -Force ninja-win.zip
          # Add it to the GitHub Path so it would be available in the subsequent steps
          Add-Content -Path $env:GITHUB_PATH -Value "${{ github.workspace }}/ninja-win"

      #
      # Build
      #

      - name: Setup ccache
        uses: hendrikmuhs/ccache-action@v1.2
        with:
          max-size: "2000M"
          # Should save cache only if run in the master branch of the base repo
          # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push
          save: ${{ github.ref_name == 'master' && 'true' || 'false'  }}
          verbose: 2
          key: ccache-windows
          restore-keys: |
            ccache-windows

      - name: Configure Developer Command Prompt for Microsoft Visual C++
        uses: ilammy/msvc-dev-cmd@v1

      - name: CMake configure - OpenVINO
        run: |
          cmake -G "${{ env.CMAKE_GENERATOR }}" `
            -DENABLE_CPPLINT=OFF `
            -DBUILD_nvidia_plugin=OFF `
            -DBUILD_SHARED_LIBS=ON `
            -DENABLE_TESTS=OFF `
            -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF `
            -DENABLE_STRICT_DEPENDENCIES=OFF `
            -DENABLE_PYTHON=ON `
            -DENABLE_WHEEL=ON `
            -DENABLE_JS=OFF `
            -DENABLE_SAMPLES=OFF `
            -DENABLE_INTEL_NPU=OFF `
            -DENABLE_OV_ONNX_FRONTEND=OFF `
            -DENABLE_OV_PADDLE_FRONTEND=OFF `
            -DENABLE_OV_PYTORCH_FRONTEND=OFF `
            -DENABLE_OV_TF_FRONTEND=ON `
            -DENABLE_OV_TF_LITE_FRONTEND=OFF `
            -DENABLE_INTEL_GPU=OFF `
            -DENABLE_INTEL_NPU=OFF `
            -DCMAKE_DISABLE_FIND_PACKAGE_PkgConfig=ON `
            -S ${{ env.OPENVINO_REPO }} `
            -B ${{ env.BUILD_DIR }}

      - name: Clean ccache stats
        run: ccache --zero-stats --show-config

      - name: Cmake build - OpenVINO
        run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --verbose

      - name: Show ccache stats
        run: ccache --show-stats

      - name: Cmake install - OpenVINO
        run: |
          cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }}/ov_package -P ${{ env.BUILD_DIR }}/cmake_install.cmake
          cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }}/ov_package -DCOMPONENT=python_wheels -P ${{ env.BUILD_DIR }}/cmake_install.cmake

      - name: Pack Artifacts
        run: |
          $file=Get-ChildItem -Path "${{ env.INSTALL_DIR }}"
          $compress = @{
            Path = $file
            CompressionLevel = "Optimal"
            DestinationPath = "${{ env.BUILD_DIR }}/openvino_package.zip"
          }
          Compress-Archive @compress

      #
      # Upload build artifacts and logs
      #

      - name: Upload openvino package
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: openvino_package
          path: ${{ env.BUILD_DIR }}/openvino_package.zip
          if-no-files-found: 'error'

  openvino_tokenizers_cpack:
    name: OpenVINO tokenizers extension (BUILD_FAST_TOKENIZERS=${{ matrix.build_fast_tokenizers }}) cpack
    strategy:
      matrix:
        build_fast_tokenizers: [ON, OFF]
    needs: [ openvino_download, openvino_build ]
    if: |
      always() &&
      (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success')
    timeout-minutes: 45
    defaults:
      run:
        shell: pwsh
    runs-on: windows-latest

    env:
      OPENVINO_REPO: ${{ github.workspace }}\src\ov
      OV_INSTALL_DIR: ${{ github.workspace }}\b\ov
      OV_TOKENIZERS_INSTALL_DIR: ${{ github.workspace }}\b\ov_tk
      OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}\src\tk
      BUILD_DIR: ${{ github.workspace }}\b\tk
      CMAKE_BUILD_TYPE: 'Release'

    steps:
      - name: Clone Openvino tokenizers
        uses: actions/checkout@v4
        with:
          path: ${{ env.OPENVINO_TOKENIZERS_REPO }}

      - name: Clone Openvino
        uses: actions/checkout@v4
        with:
          repository: 'openvinotoolkit/openvino'
          path: ${{ env.OPENVINO_REPO }}
          ref: ${{ env.OV_BRANCH }}
          sparse-checkout: |
            install_build_dependencies.sh

      - name: Setup Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'

      - name: Download OpenVINO package
        uses: actions/download-artifact@v4
        with:
          name: openvino_package
          path: ${{ env.OV_INSTALL_DIR }}

      - name: Extract OpenVINO packages
        run: |
          pushd ${{ env.OV_INSTALL_DIR }}
            Expand-Archive openvino_package.zip -DestinationPath ./tmp
            mv ./tmp/*/* .
          popd

      #
      # Build
      #

      - name: CMake configure - tokenizers
        run: |
          cmake -DBUILD_FAST_TOKENIZERS="${{ matrix.build_fast_tokenizers }}" `
                -S ${{ env.OPENVINO_TOKENIZERS_REPO }} `
                -B ${{ env.BUILD_DIR }}
        env:
          OpenVINO_DIR: ${{ env.OV_INSTALL_DIR }}/runtime/cmake
          CMAKE_BUILD_PARALLEL_LEVEL: '4'

      - name: Cmake build - tokenizers
        run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --verbose
        env:
          ICU_DATA_FILTER_FILE: ${{ env.OPENVINO_TOKENIZERS_REPO }}\\src\\icu_filter_en.json

      - name: Cmake install - tokenizers
        run: |
          cmake -DCMAKE_INSTALL_PREFIX=${{ env.OV_TOKENIZERS_INSTALL_DIR }}/ov_tokenizers -P ${{ env.BUILD_DIR }}/cmake_install.cmake

      - name: Pack Artifacts
        run: |
          $file=Get-ChildItem -Path "${{ env.OV_TOKENIZERS_INSTALL_DIR }}"
          $compress = @{
            Path = $file
            CompressionLevel = "Optimal"
            DestinationPath = "${{ env.BUILD_DIR }}/ov_tokenizers.zip"
          }
          Compress-Archive @compress

      #
      # Upload build artifacts
      #

      - name: Upload openvino tokenizers package
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: openvino_tokenizers_cpack_${{ matrix.build_fast_tokenizers }}
          path: ${{ env.BUILD_DIR }}/*.zip
          if-no-files-found: 'error'

  openvino_tokenizers_wheel:
    name: OpenVINO tokenizers extension (BUILD_FAST_TOKENIZERS=${{ matrix.build_fast_tokenizers }}) wheel
    strategy:
      matrix:
        build_fast_tokenizers: [ON, OFF]
    needs: [ openvino_download, openvino_build ]
    if: |
      always() &&
      (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success')
    timeout-minutes: 25
    defaults:
      run:
        shell: pwsh
    runs-on: windows-latest

    env:
      OPENVINO_REPO: ${{ github.workspace }}\src\ov
      INSTALL_DIR: ${{ github.workspace }}\b\ov
      OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}\src\tk
      BUILD_DIR: ${{ github.workspace }}\b\tk

    steps:
      - name: Clone Openvino tokenizers
        uses: actions/checkout@v4
        with:
          path: ${{ env.OPENVINO_TOKENIZERS_REPO }}

      - name: Clone Openvino
        uses: actions/checkout@v4
        with:
          repository: 'openvinotoolkit/openvino'
          path: ${{ env.OPENVINO_REPO }}
          ref: ${{ env.OV_BRANCH }}
          sparse-checkout: |
            install_build_dependencies.sh

      - name: Setup Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'

      - name: Download OpenVINO package
        uses: actions/download-artifact@v4
        with:
          name: openvino_package
          path: ${{ env.INSTALL_DIR }}

      - name: Extract OpenVINO packages
        run: |
          pushd ${{ env.INSTALL_DIR }}
            Expand-Archive openvino_package.zip -DestinationPath ./tmp
            mv ./tmp/*/* .
          popd

      #
      # Build
      #

      - name: Build tokenizers wheel
        run: |
          python -m pip wheel -v --no-deps --wheel-dir ${env:BUILD_DIR} `
            --find-links ${env:INSTALL_DIR}/tools `
            --config-settings=override=cmake.build_path="${env:CMAKE_BUILD_DIR}" `
            --config-settings=override=cmake.options.BUILD_FAST_TOKENIZERS="${{ matrix.build_fast_tokenizers }}" `
            ${env:OPENVINO_TOKENIZERS_REPO}
        env:
          CMAKE_BUILD_PARALLEL_LEVEL: '4'
          ICU_DATA_FILTER_FILE: ${{ env.OPENVINO_TOKENIZERS_REPO }}\\src\\icu_filter_en.json
          CMAKE_BUILD_DIR: 'D:\a\b'

      #
      # Upload build artifacts
      #

      - name: Upload openvino tokenizers wheel
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: openvino_tokenizers_wheel_${{ matrix.build_fast_tokenizers }}
          path: ${{ env.BUILD_DIR }}/*.whl
          if-no-files-found: 'error'

  openvino_tokenizers_tests:
    name: OpenVINO tokenizers tests (BUILD_FAST_TOKENIZERS=${{ matrix.build_fast_tokenizers }})
    strategy:
      matrix:
        build_fast_tokenizers: [ON, OFF]
    needs: [ openvino_download, openvino_build, openvino_tokenizers_wheel]
    if: always() && needs.openvino_tokenizers_wheel.result == 'success'
    timeout-minutes: 25
    defaults:
      run:
        shell: pwsh
    runs-on: windows-latest

    env:
      OPENVINO_REPO: ${{ github.workspace }}\\openvino
      INSTALL_DIR: ${{ github.workspace }}\\openvino\\install
      OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}\\openvino_tokenizers
      BUILD_DIR: ${{ github.workspace }}\\openvino_tokenizers\\build

    steps:
      - name: Clone Openvino tokenizers sources and tests
        uses: actions/checkout@v4
        with:
          path: ${{ env.OPENVINO_TOKENIZERS_REPO }}

      - name: Setup Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'

      - name: Download tokenizers package
        uses: actions/download-artifact@v4
        with:
          name: openvino_tokenizers_wheel_${{ matrix.build_fast_tokenizers }}
          path: ${{ env.INSTALL_DIR }}\\ov_tokenizers

      - name: Download OpenVINO package
        uses: actions/download-artifact@v4
        with:
          name: openvino_package
          path: ${{ env.INSTALL_DIR }}


      - name: Extract OpenVINO packages
        run: |
          pushd ${{ env.INSTALL_DIR }}
            Expand-Archive openvino_package.zip -DestinationPath ./tmp
            mv ./tmp/*/* .
          popd

      - name: Install OpenVINO Python wheel
        if: needs.openvino_build.result == 'success'
        run: |
          # Find and install wheel
          $ovCoreWheelPath=Get-ChildItem -Path "${{ env.INSTALL_DIR }}\\tools" -Filter openvino-*.whl | % { $_.FullName }
          python3 -m pip install "$ovCoreWheelPath"

      - name: Install OpenVINO tokenizers wheel
        run: |
          # Find and install wheel
          $ovCoreWheelPath=Get-ChildItem -Path "${{ env.INSTALL_DIR }}\\ov_tokenizers" -Filter openvino_tokenizers*.whl | % { $_.FullName }
          python3 -m pip install "$ovCoreWheelPath[all]"
        env:
          PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu

      - name: Tokenizers regression tests (using openvino python modules)
        if: needs.openvino_download.outputs.status == 'success'
        run: |
          . "${{ env.INSTALL_DIR }}/setupvars.ps1"
          python3 -m pytest tokenizers_test.py
        working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests

      - name: Tokenizers regression tests
        if: needs.openvino_build.result == 'success'
        run: |
          python3 -m pytest tokenizers_test.py
        working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests