Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 40 additions & 2 deletions .pipelines/modelkit-official-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,49 @@ extends:
Write-Host "Copied $($parquets.Count) rule parquet file(s) to $destDir"
displayName: 'Copy runtime check rules'

- powershell: |
Add-Type -AssemblyName System.IO.Compression
Add-Type -AssemblyName System.IO.Compression.FileSystem

$rulesRoot = "$(Build.SourcesDirectory)\src\winml\modelkit\analyze\rules\runtime_check_rules"
$outDir = "$(ob_outputDirectory)"
$rulesZipPath = Join-Path $outDir "rules.zip"
$parquets = Get-ChildItem "$rulesRoot\*.parquet" -Recurse

if (-not $parquets) {
throw "No runtime rule parquet files found under $rulesRoot"
}

New-Item -ItemType Directory -Path $outDir -Force | Out-Null
if (Test-Path $rulesZipPath) {
Remove-Item -Path $rulesZipPath -Force
}

$zip = [System.IO.Compression.ZipFile]::Open($rulesZipPath, [System.IO.Compression.ZipArchiveMode]::Create)
try {
foreach ($parquet in $parquets) {
$entryName = $parquet.FullName.Substring($rulesRoot.Length).TrimStart('\').Replace('\', '/')
[System.IO.Compression.ZipFileExtensions]::CreateEntryFromFile(
$zip,
$parquet.FullName,
$entryName,
[System.IO.Compression.CompressionLevel]::Optimal
) | Out-Null
}
}
finally {
$zip.Dispose()
}

Write-Host "Packed $($parquets.Count) runtime rule parquet file(s) into $rulesZipPath"
displayName: 'Create rules.zip for drop/release'

- task: PipAuthenticate@1
inputs:
artifactFeeds: 'windows.ai.toolkit/Modelkit'
displayName: 'Authenticate pip with Azure Artifacts'

- script: pip install --upgrade build twine packaging
- script: python -m pip install --upgrade build twine packaging
displayName: 'Install build tools'

# Build sdist BEFORE iKey injection so the source archive
Expand Down Expand Up @@ -123,6 +160,7 @@ extends:
# of each archive directly catches build-cache / ordering
# bugs that disk-only checks would miss.
- powershell: |
Add-Type -AssemblyName System.IO.Compression
Add-Type -AssemblyName System.IO.Compression.FileSystem

function Read-ConstantsFromArchive($archivePath) {
Expand Down Expand Up @@ -161,6 +199,6 @@ extends:
Write-Host "Sdist verified - constants.py has empty placeholder"
displayName: 'Verify wheel has iKey, sdist does not'

- script: twine check "$(ob_outputDirectory)\*.whl" "$(ob_outputDirectory)\*.tar.gz"
- script: python -m twine check "$(ob_outputDirectory)\*.whl" "$(ob_outputDirectory)\*.tar.gz"
continueOnError: true
displayName: 'Validate packages for PyPI'
27 changes: 22 additions & 5 deletions .pipelines/modelkit-release-github.yml
Original file line number Diff line number Diff line change
Expand Up @@ -86,12 +86,29 @@ extends:
- powershell: |
$artifactDir = "$(Pipeline.Workspace)"
$outDir = "$(ob_outputDirectory)"
$rulesZipPath = Join-Path $outDir "rules.zip"

New-Item -ItemType Directory -Path $outDir -Force | Out-Null

$wheels = Get-ChildItem "$artifactDir" -Filter "*.whl" -Recurse
$parquets = Get-ChildItem "$artifactDir" -Filter "*.parquet" -Recurse
Write-Host "Found $($wheels.Count) wheel(s) and $($parquets.Count) parquet file(s)"
$wheels | Copy-Item -Destination $outDir
$parquets | Copy-Item -Destination $outDir
if (-not $wheels) {
throw "No wheel files found under $artifactDir"
}
$wheels | Copy-Item -Destination $outDir -Force

$rulesZipCandidates = Get-ChildItem "$artifactDir" -Filter "rules.zip" -Recurse
if (-not $rulesZipCandidates) {
throw "No rules.zip found under $artifactDir. Release requires the official build to publish rules.zip."
}

if ($rulesZipCandidates.Count -gt 1) {
Write-Warning "Found $($rulesZipCandidates.Count) rules.zip files; using the first one"
}
$selectedRulesZip = $rulesZipCandidates | Select-Object -First 1
Copy-Item $selectedRulesZip.FullName -Destination $rulesZipPath -Force

Write-Host "Found $($wheels.Count) wheel(s)"
Write-Host "Using build-produced rules.zip from $($selectedRulesZip.FullName)"
displayName: 'Stage release assets'

- task: GitHubRelease@1
Expand All @@ -106,7 +123,7 @@ extends:
title: 'ModelKit $(ReleaseTag)'
assets: |
$(ob_outputDirectory)\*.whl
$(ob_outputDirectory)\*.parquet
$(ob_outputDirectory)\rules.zip
isDraft: false
isPreRelease: true
addChangeLog: false
7 changes: 6 additions & 1 deletion scripts/download_rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
WinML-ModelKit GitHub release; see
src/winml/modelkit/analyze/rules/runtime_check_rules/README.md.

This script does not download release assets. It pulls parquet files directly
from gim-home/ModelKitArtifacts/rules.

Usage:
uv run python scripts/download_rules.py --account <account>
uv run python scripts/download_rules.py --account <account> --force
Expand Down Expand Up @@ -110,7 +113,9 @@ def _sparse_clone(clone_url: str, dest: Path) -> bool:


def main() -> None:
parser = argparse.ArgumentParser(description="Download runtime check rule parquet files")
parser = argparse.ArgumentParser(
description="Download runtime check rule parquet files from gim-home/ModelKitArtifacts"
)
parser.add_argument("--force", action="store_true", help="Re-download all parquet files")
parser.add_argument("--account", type=str, help="gh CLI account with access to gim-home org")
args = parser.parse_args()
Expand Down
19 changes: 17 additions & 2 deletions src/winml/modelkit/analyze/rules/runtime_check_rules/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,27 @@ also be fetched from release assets or `ModelKitArtifacts` for source builds.
### Option 1: Download from the latest GitHub release (for source builds)

If you are building from source code (for example, cloning this repo), download
the parquet assets from the latest WinML-ModelKit release.
the `rules.zip` asset from the latest WinML-ModelKit release.

```bash
gh release download --repo microsoft/WinML-ModelKit --pattern '*.parquet' --dir src/winml/modelkit/analyze/rules/runtime_check_rules
gh release download --repo microsoft/WinML-ModelKit --pattern 'rules.zip' --dir .
```

`gh release download` defaults to the latest release. Use `--tag <version>`
to pin a specific release if you need a reproducible snapshot.

Then extract `rules.zip` into this directory:

```powershell
Expand-Archive -Path .\rules.zip -DestinationPath src\winml\modelkit\analyze\rules\runtime_check_rules -Force
```

```bash
unzip -o rules.zip -d src/winml/modelkit/analyze/rules/runtime_check_rules
```

The zip preserves file paths relative to `runtime_check_rules/`.

### Option 2: Download script (Microsoft internal fallback)

Requires [GitHub CLI](https://cli.github.com) (`gh`) with an account that has access to `gim-home`.
Expand All @@ -30,6 +42,9 @@ uv run python scripts/download_rules.py --account <your_gim-home_account>
The script sparse-checkouts `gim-home/ModelKitArtifacts/rules` and copies all `*.parquet`
files here (preserving subdirectories).

This script downloads from the internal `ModelKitArtifacts` repo, not from
WinML-ModelKit release assets.

Use `--force` to re-download all files even if they already exist locally.

### Option 3: Manual copy (Microsoft internal fallback)
Expand Down
Loading