From b552193cad38aadd64074fff01a33698e624ff9f Mon Sep 17 00:00:00 2001 From: fangyangci Date: Tue, 12 May 2026 14:42:55 +0800 Subject: [PATCH 1/3] fix build_release --- .pipelines/modelkit-release-github.yml | 68 +++++++++++++++++-- scripts/download_rules.py | 7 +- .../rules/runtime_check_rules/README.md | 19 +++++- 3 files changed, 86 insertions(+), 8 deletions(-) diff --git a/.pipelines/modelkit-release-github.yml b/.pipelines/modelkit-release-github.yml index 417357e10..2f150f188 100644 --- a/.pipelines/modelkit-release-github.yml +++ b/.pipelines/modelkit-release-github.yml @@ -84,14 +84,72 @@ extends: displayName: 'Extract version from pyproject.toml' - powershell: | + Add-Type -AssemblyName System.IO.Compression.FileSystem + $artifactDir = "$(Pipeline.Workspace)" $outDir = "$(ob_outputDirectory)" + $rulesZipPath = Join-Path $outDir "rules.zip" + $rulesRootMarker = "src\winml\modelkit\analyze\rules\runtime_check_rules\" + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + $wheels = Get-ChildItem "$artifactDir" -Filter "*.whl" -Recurse - $parquets = Get-ChildItem "$artifactDir" -Filter "*.parquet" -Recurse - Write-Host "Found $($wheels.Count) wheel(s) and $($parquets.Count) parquet file(s)" - $wheels | Copy-Item -Destination $outDir - $parquets | Copy-Item -Destination $outDir + if (-not $wheels) { + throw "No wheel files found under $artifactDir" + } + $wheels | Copy-Item -Destination $outDir -Force + + $parquets = Get-ChildItem "$artifactDir" -Filter "*.parquet" -Recurse | Where-Object { + $_.FullName -like "*$rulesRootMarker*" + } + if (-not $parquets) { + throw "No runtime rule parquet files found under '$rulesRootMarker' in $artifactDir" + } + + $entries = [ordered]@{} + foreach ($parquet in $parquets) { + $fullPath = [System.IO.Path]::GetFullPath($parquet.FullName) + $markerIndex = $fullPath.IndexOf($rulesRootMarker, [System.StringComparison]::OrdinalIgnoreCase) + if ($markerIndex -lt 0) { + continue + } + $entryName = $fullPath.Substring($markerIndex + $rulesRootMarker.Length).Replace('\', '/') + if (-not $entryName) { + continue + } + if ($entries.Contains($entryName)) { + Write-Host "Skipping duplicate parquet entry '$entryName' from $fullPath" + continue + } + $entries[$entryName] = $fullPath + } + + if ($entries.Count -eq 0) { + throw "No parquet entries matched runtime rules root marker '$rulesRootMarker'" + } + + if (Test-Path $rulesZipPath) { + Remove-Item -Path $rulesZipPath -Force + } + + $zip = [System.IO.Compression.ZipFile]::Open($rulesZipPath, [System.IO.Compression.ZipArchiveMode]::Create) + try { + foreach ($entryName in $entries.Keys) { + $sourcePath = $entries[$entryName] + [System.IO.Compression.ZipFileExtensions]::CreateEntryFromFile( + $zip, + $sourcePath, + $entryName, + [System.IO.Compression.CompressionLevel]::Optimal + ) | Out-Null + } + } + finally { + $zip.Dispose() + } + + Write-Host "Found $($wheels.Count) wheel(s)" + Write-Host "Packed $($entries.Count) runtime rule parquet file(s) into $rulesZipPath" displayName: 'Stage release assets' - task: GitHubRelease@1 @@ -106,7 +164,7 @@ extends: title: 'ModelKit $(ReleaseTag)' assets: | $(ob_outputDirectory)\*.whl - $(ob_outputDirectory)\*.parquet + $(ob_outputDirectory)\rules.zip isDraft: false isPreRelease: true addChangeLog: false diff --git a/scripts/download_rules.py b/scripts/download_rules.py index d6ac55196..febdf0094 100644 --- a/scripts/download_rules.py +++ b/scripts/download_rules.py @@ -11,6 +11,9 @@ WinML-ModelKit GitHub release; see src/winml/modelkit/analyze/rules/runtime_check_rules/README.md. +This script does not download release assets. It pulls parquet files directly +from gim-home/ModelKitArtifacts/rules. + Usage: uv run python scripts/download_rules.py --account uv run python scripts/download_rules.py --account --force @@ -110,7 +113,9 @@ def _sparse_clone(clone_url: str, dest: Path) -> bool: def main() -> None: - parser = argparse.ArgumentParser(description="Download runtime check rule parquet files") + parser = argparse.ArgumentParser( + description="Download runtime check rule parquet files from gim-home/ModelKitArtifacts" + ) parser.add_argument("--force", action="store_true", help="Re-download all parquet files") parser.add_argument("--account", type=str, help="gh CLI account with access to gim-home org") args = parser.parse_args() diff --git a/src/winml/modelkit/analyze/rules/runtime_check_rules/README.md b/src/winml/modelkit/analyze/rules/runtime_check_rules/README.md index 56904a219..c03d7ce89 100644 --- a/src/winml/modelkit/analyze/rules/runtime_check_rules/README.md +++ b/src/winml/modelkit/analyze/rules/runtime_check_rules/README.md @@ -10,15 +10,27 @@ also be fetched from release assets or `ModelKitArtifacts` for source builds. ### Option 1: Download from the latest GitHub release (for source builds) If you are building from source code (for example, cloning this repo), download -the parquet assets from the latest WinML-ModelKit release. +the `rules.zip` asset from the latest WinML-ModelKit release. ```bash -gh release download --repo microsoft/WinML-ModelKit --pattern '*.parquet' --dir src/winml/modelkit/analyze/rules/runtime_check_rules +gh release download --repo microsoft/WinML-ModelKit --pattern 'rules.zip' --dir . ``` `gh release download` defaults to the latest release. Use `--tag ` to pin a specific release if you need a reproducible snapshot. +Then extract `rules.zip` into this directory: + +```powershell +Expand-Archive -Path .\rules.zip -DestinationPath src\winml\modelkit\analyze\rules\runtime_check_rules -Force +``` + +```bash +unzip -o rules.zip -d src/winml/modelkit/analyze/rules/runtime_check_rules +``` + +The zip preserves file paths relative to `runtime_check_rules/`. + ### Option 2: Download script (Microsoft internal fallback) Requires [GitHub CLI](https://cli.github.com) (`gh`) with an account that has access to `gim-home`. @@ -30,6 +42,9 @@ uv run python scripts/download_rules.py --account The script sparse-checkouts `gim-home/ModelKitArtifacts/rules` and copies all `*.parquet` files here (preserving subdirectories). +This script downloads from the internal `ModelKitArtifacts` repo, not from +WinML-ModelKit release assets. + Use `--force` to re-download all files even if they already exist locally. ### Option 3: Manual copy (Microsoft internal fallback) From 102eb7ed7c7ead2567b688357dc7220ac6aa9d0e Mon Sep 17 00:00:00 2001 From: fangyangci Date: Tue, 12 May 2026 15:16:01 +0800 Subject: [PATCH 2/3] fix build zip --- .pipelines/modelkit-official-build.yml | 40 +++++++++++++++++- .pipelines/modelkit-release-github.yml | 57 ++++---------------------- 2 files changed, 46 insertions(+), 51 deletions(-) diff --git a/.pipelines/modelkit-official-build.yml b/.pipelines/modelkit-official-build.yml index df9d805a4..1ee3fa6c2 100644 --- a/.pipelines/modelkit-official-build.yml +++ b/.pipelines/modelkit-official-build.yml @@ -85,12 +85,48 @@ extends: Write-Host "Copied $($parquets.Count) rule parquet file(s) to $destDir" displayName: 'Copy runtime check rules' + - powershell: | + Add-Type -AssemblyName System.IO.Compression.FileSystem + + $rulesRoot = "$(Build.SourcesDirectory)\src\winml\modelkit\analyze\rules\runtime_check_rules" + $outDir = "$(ob_outputDirectory)" + $rulesZipPath = Join-Path $outDir "rules.zip" + $parquets = Get-ChildItem "$rulesRoot\*.parquet" -Recurse + + if (-not $parquets) { + throw "No runtime rule parquet files found under $rulesRoot" + } + + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + if (Test-Path $rulesZipPath) { + Remove-Item -Path $rulesZipPath -Force + } + + $zip = [System.IO.Compression.ZipFile]::Open($rulesZipPath, [System.IO.Compression.ZipArchiveMode]::Create) + try { + foreach ($parquet in $parquets) { + $entryName = $parquet.FullName.Substring($rulesRoot.Length).TrimStart('\').Replace('\', '/') + [System.IO.Compression.ZipFileExtensions]::CreateEntryFromFile( + $zip, + $parquet.FullName, + $entryName, + [System.IO.Compression.CompressionLevel]::Optimal + ) | Out-Null + } + } + finally { + $zip.Dispose() + } + + Write-Host "Packed $($parquets.Count) runtime rule parquet file(s) into $rulesZipPath" + displayName: 'Create rules.zip for drop/release' + - task: PipAuthenticate@1 inputs: artifactFeeds: 'windows.ai.toolkit/Modelkit' displayName: 'Authenticate pip with Azure Artifacts' - - script: pip install --upgrade build twine packaging + - script: python -m pip install --upgrade build twine packaging displayName: 'Install build tools' # Build sdist BEFORE iKey injection so the source archive @@ -161,6 +197,6 @@ extends: Write-Host "Sdist verified - constants.py has empty placeholder" displayName: 'Verify wheel has iKey, sdist does not' - - script: twine check "$(ob_outputDirectory)\*.whl" "$(ob_outputDirectory)\*.tar.gz" + - script: python -m twine check "$(ob_outputDirectory)\*.whl" "$(ob_outputDirectory)\*.tar.gz" continueOnError: true displayName: 'Validate packages for PyPI' diff --git a/.pipelines/modelkit-release-github.yml b/.pipelines/modelkit-release-github.yml index 2f150f188..96b7ec7c6 100644 --- a/.pipelines/modelkit-release-github.yml +++ b/.pipelines/modelkit-release-github.yml @@ -84,12 +84,9 @@ extends: displayName: 'Extract version from pyproject.toml' - powershell: | - Add-Type -AssemblyName System.IO.Compression.FileSystem - $artifactDir = "$(Pipeline.Workspace)" $outDir = "$(ob_outputDirectory)" $rulesZipPath = Join-Path $outDir "rules.zip" - $rulesRootMarker = "src\winml\modelkit\analyze\rules\runtime_check_rules\" New-Item -ItemType Directory -Path $outDir -Force | Out-Null @@ -99,57 +96,19 @@ extends: } $wheels | Copy-Item -Destination $outDir -Force - $parquets = Get-ChildItem "$artifactDir" -Filter "*.parquet" -Recurse | Where-Object { - $_.FullName -like "*$rulesRootMarker*" - } - if (-not $parquets) { - throw "No runtime rule parquet files found under '$rulesRootMarker' in $artifactDir" + $rulesZipCandidates = Get-ChildItem "$artifactDir" -Filter "rules.zip" -Recurse + if (-not $rulesZipCandidates) { + throw "No rules.zip found under $artifactDir. Release requires the official build to publish rules.zip." } - $entries = [ordered]@{} - foreach ($parquet in $parquets) { - $fullPath = [System.IO.Path]::GetFullPath($parquet.FullName) - $markerIndex = $fullPath.IndexOf($rulesRootMarker, [System.StringComparison]::OrdinalIgnoreCase) - if ($markerIndex -lt 0) { - continue - } - $entryName = $fullPath.Substring($markerIndex + $rulesRootMarker.Length).Replace('\', '/') - if (-not $entryName) { - continue - } - if ($entries.Contains($entryName)) { - Write-Host "Skipping duplicate parquet entry '$entryName' from $fullPath" - continue - } - $entries[$entryName] = $fullPath - } - - if ($entries.Count -eq 0) { - throw "No parquet entries matched runtime rules root marker '$rulesRootMarker'" - } - - if (Test-Path $rulesZipPath) { - Remove-Item -Path $rulesZipPath -Force - } - - $zip = [System.IO.Compression.ZipFile]::Open($rulesZipPath, [System.IO.Compression.ZipArchiveMode]::Create) - try { - foreach ($entryName in $entries.Keys) { - $sourcePath = $entries[$entryName] - [System.IO.Compression.ZipFileExtensions]::CreateEntryFromFile( - $zip, - $sourcePath, - $entryName, - [System.IO.Compression.CompressionLevel]::Optimal - ) | Out-Null - } - } - finally { - $zip.Dispose() + if ($rulesZipCandidates.Count -gt 1) { + Write-Warning "Found $($rulesZipCandidates.Count) rules.zip files; using the first one" } + $selectedRulesZip = $rulesZipCandidates | Select-Object -First 1 + Copy-Item $selectedRulesZip.FullName -Destination $rulesZipPath -Force Write-Host "Found $($wheels.Count) wheel(s)" - Write-Host "Packed $($entries.Count) runtime rule parquet file(s) into $rulesZipPath" + Write-Host "Using build-produced rules.zip from $($selectedRulesZip.FullName)" displayName: 'Stage release assets' - task: GitHubRelease@1 From 97a81ea3793aa9645ad4643b5b4b68134c36ca78 Mon Sep 17 00:00:00 2001 From: fangyangci Date: Tue, 12 May 2026 15:33:49 +0800 Subject: [PATCH 3/3] fix type --- .pipelines/modelkit-official-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/modelkit-official-build.yml b/.pipelines/modelkit-official-build.yml index 1ee3fa6c2..2b7ca7d89 100644 --- a/.pipelines/modelkit-official-build.yml +++ b/.pipelines/modelkit-official-build.yml @@ -86,6 +86,7 @@ extends: displayName: 'Copy runtime check rules' - powershell: | + Add-Type -AssemblyName System.IO.Compression Add-Type -AssemblyName System.IO.Compression.FileSystem $rulesRoot = "$(Build.SourcesDirectory)\src\winml\modelkit\analyze\rules\runtime_check_rules" @@ -159,6 +160,7 @@ extends: # of each archive directly catches build-cache / ordering # bugs that disk-only checks would miss. - powershell: | + Add-Type -AssemblyName System.IO.Compression Add-Type -AssemblyName System.IO.Compression.FileSystem function Read-ConstantsFromArchive($archivePath) {