diff --git a/windows-release/azure-pipelines.yml b/windows-release/azure-pipelines.yml index c4460195..f4e08765 100644 --- a/windows-release/azure-pipelines.yml +++ b/windows-release/azure-pipelines.yml @@ -146,8 +146,6 @@ variables: IsRealSigned: false ${{ if ne(parameters.SigningDescription, '(default)') }}: SigningDescription: ${{ parameters.SigningDescription }} - ${{ else }}: - SigningDescription: '' PublishARM64: ${{ parameters.DoARM64 }} # QUEUE TIME VARIABLES # OverrideNugetVersion: '' @@ -322,7 +320,6 @@ stages: BuildToPublish: ${{ parameters.BuildToPublish }} DoEmbed: ${{ parameters.DoEmbed }} DoFreethreaded: ${{ parameters.DoFreethreaded }} - SigningCertificate: ${{ iif(eq(parameters.SigningCertificate, 'Unsigned'), '', parameters.SigningCertificate) }} - ${{ if eq(parameters.DoMSI, 'true') }}: - template: stage-publish-pythonorg.yml parameters: diff --git a/windows-release/merge-and-upload.py b/windows-release/merge-and-upload.py index 0565748f..87bbaf6a 100644 --- a/windows-release/merge-and-upload.py +++ b/windows-release/merge-and-upload.py @@ -20,18 +20,12 @@ UPLOAD_USER = os.getenv("UPLOAD_USER", "") NO_UPLOAD = os.getenv("NO_UPLOAD", "no")[:1].lower() in "yt1" LOCAL_INDEX = os.getenv("LOCAL_INDEX", "no")[:1].lower() in "yt1" -SIGN_COMMAND = os.getenv("SIGN_COMMAND", "") def find_cmd(env, exe): cmd = os.getenv(env) if cmd: - cmd = Path(cmd) - if not cmd.is_file(): - raise RuntimeError( - f"Could not find {cmd} to perform upload. Incorrect %{env}% setting." - ) - return cmd + return Path(cmd) for p in os.getenv("PATH", "").split(";"): if p: cmd = Path(p) / exe @@ -46,7 +40,6 @@ def find_cmd(env, exe): PLINK = find_cmd("PLINK", "plink.exe") PSCP = find_cmd("PSCP", "pscp.exe") -MAKECAT = find_cmd("MAKECAT", "makecat.exe") def _std_args(cmd): @@ -67,9 +60,7 @@ class RunError(Exception): pass -def _run(*args, single_cmd=False): - if single_cmd: - args = args[0] +def _run(*args): with subprocess.Popen( args, stdout=subprocess.PIPE, @@ -202,43 +193,6 @@ def calculate_uploads(): ) -def sign_json(cat_file, *files): - if not MAKECAT: - if not UPLOAD_HOST or NO_UPLOAD: - print("makecat.exe not found, but not uploading, so skip signing.") - return - raise RuntimeError("No makecat.exe found") - if not SIGN_COMMAND: - if not UPLOAD_HOST or NO_UPLOAD: - print("No signing command set, but not uploading, so skip signing.") - return - raise RuntimeError("No SIGN_COMMAND set") - - cat = Path(cat_file).absolute() - cdf = cat.with_suffix(".cdf") - cdf.parent.mkdir(parents=True, exist_ok=True) - - with open(cdf, "w", encoding="ansi") as f: - print("[CatalogHeader]", file=f) - print("Name=", cat.name, sep="", file=f) - print("ResultDir=", cat.parent, sep="", file=f) - print("PublicVersion=0x00000001", file=f) - print("CatalogVersion=2", file=f) - print("HashAlgorithms=SHA256", file=f) - print("EncodingType=", file=f) - print(file=f) - print("[CatalogFiles]", file=f) - for a in map(Path, files): - print("", a.name, "=", a.absolute(), sep="", file=f) - - _run(MAKECAT, "-v", cdf) - if not cat.is_file(): - raise FileNotFoundError(cat) - # Pass as a single arg because the command variable has its own arguments - _run(f'{SIGN_COMMAND} "{cat}"', single_cmd=True) - cdf.unlink() - - def remove_and_insert(index, new_installs): new = {(i["id"].casefold(), i["sort-version"].casefold()) for i in new_installs} to_remove = [ @@ -320,7 +274,6 @@ def find_missing_from_index(url, installs): except FileNotFoundError: pass - print(INDEX_PATH, "mtime =", INDEX_MTIME) @@ -331,20 +284,10 @@ def find_missing_from_index(url, installs): if INDEX_FILE: INDEX_FILE = Path(INDEX_FILE).absolute() - INDEX_CAT_FILE = INDEX_FILE.with_name(f"{INDEX_FILE.name}.cat") INDEX_FILE.parent.mkdir(parents=True, exist_ok=True) with open(INDEX_FILE, "w", encoding="utf-8") as f: json.dump(index, f) - sign_json(INDEX_CAT_FILE, INDEX_FILE) - INDEX_CAT_URL = f"{INDEX_URL}.cat" - INDEX_CAT_PATH = f"{INDEX_PATH}.cat" -else: - INDEX_CAT_FILE = None - INDEX_CAT_URL = None - INDEX_CAT_PATH = None - - if MANIFEST_FILE: # Use the sort-version so that the manifest name includes prerelease marks MANIFEST_FILE = Path(MANIFEST_FILE).absolute() @@ -380,36 +323,33 @@ def find_missing_from_index(url, installs): print("Expecting mtime", INDEX_MTIME, "but saw", mtime) sys.exit(1) -TO_PURGE = [i["url"] for i, *_ in UPLOADS] - -if MANIFEST_FILE: - print("Uploading", MANIFEST_FILE, "to", MANIFEST_URL) - upload_ssh(MANIFEST_FILE, MANIFEST_PATH) - TO_PURGE.append(MANIFEST_URL) -if INDEX_FILE: - print("Uploading", INDEX_FILE, "to", INDEX_URL) - upload_ssh(INDEX_FILE, INDEX_PATH) - TO_PURGE.append(INDEX_URL) - -if INDEX_CAT_FILE: - print("Uploading", INDEX_CAT_FILE, "to", INDEX_CAT_URL) - upload_ssh(INDEX_CAT_FILE, INDEX_CAT_PATH) - TO_PURGE.append(INDEX_CAT_URL) - -# Calculate directory parents for all files -TO_PURGE.extend({i.rpartition("/")[0] + "/" for i in TO_PURGE}) - -print("Purging", len(TO_PURGE), "uploaded files, indexes and directories") - -for i in TO_PURGE: - purge(i) - -if INDEX_URL: - missing = find_missing_from_index(INDEX_URL, [i for i, *_ in UPLOADS]) - if missing: - print("##[error]Lost a race with another publish step!") - print("Index at", INDEX_URL, "does not contain installs:") - for m in missing: - print(m["id"], m["sort-version"]) - sys.exit(1) +if not NO_UPLOAD: + if MANIFEST_FILE: + print("Uploading", MANIFEST_FILE, "to", MANIFEST_URL) + upload_ssh(MANIFEST_FILE, MANIFEST_PATH) + + if INDEX_FILE: + print("Uploading", INDEX_FILE, "to", INDEX_URL) + upload_ssh(INDEX_FILE, INDEX_PATH) + + print("Purging", len(UPLOADS), "uploaded files") + parents = set() + for i, *_ in UPLOADS: + purge(i["url"]) + parents.add(i["url"].rpartition("/")[0] + "/") + for i in parents: + purge(i) + if MANIFEST_URL: + purge(MANIFEST_URL) + purge(MANIFEST_URL.rpartition("/")[0] + "/") + if INDEX_URL: + purge(INDEX_URL) + purge(INDEX_URL.rpartition("/")[0] + "/") + missing = find_missing_from_index(INDEX_URL, [i for i, *_ in UPLOADS]) + if missing: + print("##[error]Lost a race with another publish step!") + print("Index at", INDEX_URL, "does not contain installs:") + for m in missing: + print(m["id"], m["sort-version"]) + sys.exit(1) diff --git a/windows-release/sign-files.yml b/windows-release/sign-files.yml index 7661e0ea..6809d9c9 100644 --- a/windows-release/sign-files.yml +++ b/windows-release/sign-files.yml @@ -6,87 +6,33 @@ parameters: ExtractDir: '' SigningCertificate: '' ExportCommand: '' - ExportLegacyCommand: '' ContinueOnError: false - InstallTool: true - InstallLegacyTool: false AzureServiceConnectionName: 'Python Signing' steps: -- ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}: - - ${{ if eq(parameters.InstallTool, 'true') }}: - - powershell: | - # Install sign tool - dotnet tool install --global --prerelease sign - $signtool = (gcm sign -EA SilentlyContinue).Source - if (-not $signtool) { - $signtool = (gi "${env:USERPROFILE}\.dotnet\tools\sign.exe").FullName - } - $signargs = 'code artifact-signing -v Information ' + ` - '-fd sha256 -t http://timestamp.acs.microsoft.com -td sha256 ' + ` - "-ase ""${env:ASE}"" -asa ""${env:ASA}"" -ascp ""${env:ASCP}"" " + ` - "-act azure-cli -d ""${env:DESCRIPTION}""" - - Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool" - Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs" - if ($env:EXPORT_COMMAND) { - $signcmd = """$signtool"" $signargs" - Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd" - } - workingDirectory: $(Build.BinariesDirectory) - displayName: 'Install Azure Artifact Signing tools' - env: - ASE: $(TrustedSigningUri) - ASA: $(TrustedSigningAccount) - ASCP: $(TrustedSigningCertificateName) - DESCRIPTION: $(SigningDescription) - EXPORT_COMMAND: ${{ parameters.ExportCommand }} - - - ${{ if eq(parameters.InstallLegacyTool, 'true') }}: - - powershell: | - git clone https://github.com/python/cpython-bin-deps --revision fb06137dccc43ed5b030cdd9e3560990b37f39da --depth 1 --progress -v "signtool" - - $signtool = gi signtool\x64\signtool.exe - $dlib = gi signtool\azure_trusted_signing\x64\Azure.CodeSigning.Dlib.dll - Write-Host "##vso[task.setvariable variable=MAKECAT]$(gi signtool\x64\makecat.exe)" - - ConvertTo-Json @{ - Endpoint=$env:ASE; - CodeSigningAccountName=$env:ASA; - CertificateProfileName=$env:ASCP; - # Only allow Azure CLI credentials - ExcludeCredentials=@( - "ManagedIdentityCredential", - "WorkloadIdentityCredential", - "SharedTokenCacheCredential", - "EnvironmentCredential", - "VisualStudioCredential", - "VisualStudioCodeCredential", - "AzurePowerShellCredential", - "AzureDeveloperCliCredential", - "InteractiveBrowserCredential" - ); - } | Out-File signtool\metadata.json -Encoding ascii - Write-Host "##vso[task.setvariable variable=SIGNTOOL_METADATA]$(gi signtool\metadata.json)" +- ${{ if parameters.SigningCertificate }}: + - powershell: | + # Install sign tool + dotnet tool install --global --prerelease sign + $signtool = (gcm sign -EA SilentlyContinue).Source + if (-not $signtool) { + $signtool = (gi "${env:USERPROFILE}\.dotnet\tools\sign.exe").FullName + } + $signargs = 'code trusted-signing -v Information ' + ` + '-fd sha256 -t http://timestamp.acs.microsoft.com -td sha256 ' + ` + '-tse "$(TrustedSigningUri)" -tsa "$(TrustedSigningAccount)" -tscp "$(TrustedSigningCertificateName)" ' + ` + '-d "$(SigningDescription)" ' - $signargs = 'sign /v /fd sha256 /tr http://timestamp.acs.microsoft.com /td sha256 ' + ` - "/dlib ""$dlib"" /dmdf ""$(gi signtool\metadata.json)"" " + ` - "/d ""${env:DESCRIPTION}""" - Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool" - Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs" - - if ($env:EXPORT_COMMAND) { - $signcmd = """$signtool"" $signargs" - Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd" - } - workingDirectory: $(Pipeline.Workspace) - displayName: 'Download signtool binaries' - env: - ASE: $(TrustedSigningUri) - ASA: $(TrustedSigningAccount) - ASCP: $(TrustedSigningCertificateName) - DESCRIPTION: $(SigningDescription) - EXPORT_COMMAND: ${{ parameters.ExportLegacyCommand }} + Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool" + Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs" + if ($env:EXPORT_COMMAND) { + $signcmd = """$signtool"" $signargs" + Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd" + } + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Install Trusted Signing tools' + env: + EXPORT_COMMAND: ${{ parameters.ExportCommand }} - ${{ if parameters.AzureServiceConnectionName }}: # We sign in once with the AzureCLI task, as it uses OIDC to obtain a @@ -99,28 +45,28 @@ steps: scriptType: 'ps' scriptLocation: 'inlineScript' inlineScript: | - "##vso[task.setvariable variable=__AZURE_CLIENT_ID;issecret=true]${env:servicePrincipalId}" - "##vso[task.setvariable variable=__AZURE_ID_TOKEN;issecret=true]${env:idToken}" - "##vso[task.setvariable variable=__AZURE_TENANT_ID;issecret=true]${env:tenantId}" + "##vso[task.setvariable variable=AZURE_CLIENT_ID;issecret=true]${env:servicePrincipalId}" + "##vso[task.setvariable variable=AZURE_ID_TOKEN;issecret=true]${env:idToken}" + "##vso[task.setvariable variable=AZURE_TENANT_ID;issecret=true]${env:tenantId}" addSpnToEnvironment: true - powershell: > az login --service-principal - -u $(__AZURE_CLIENT_ID) - --tenant $(__AZURE_TENANT_ID) + -u $(AZURE_CLIENT_ID) + --tenant $(AZURE_TENANT_ID) --allow-no-subscriptions - --federated-token $(__AZURE_ID_TOKEN) + --federated-token $(AZURE_ID_TOKEN) displayName: 'Authenticate signing tools (2/2)' - ${{ if parameters.Include }}: - powershell: | - if ($env:EXCLUDE) { - $files = (dir ($env:INCLUDE -split ',').Trim() -Exclude ($env:EXCLUDE -split ',').Trim() -File) + if ("${{ parameters.Exclude }}") { + $files = (dir ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }} -File) } else { - $files = (dir ($env:INCLUDE -split ',').Trim() -File) + $files = (dir ${{ parameters.Include }} -File) } if ($env:FILTER) { - ($env:FILTER -split ';').Trim() -join "`n" | Out-File __filelist.txt -Encoding utf8 + ($env:FILTER -split ';') -join "`n" | Out-File __filelist.txt -Encoding utf8 } else { "*" | Out-File __filelist.txt -Encoding utf8 } @@ -136,37 +82,31 @@ steps: continueOnError: true workingDirectory: ${{ parameters.WorkingDir }} env: - INCLUDE: ${{ parameters.Include }} - EXCLUDE: ${{ parameters.Exclude }} TRUSTED_SIGNING_CMD: $(__TrustedSigningCmd) TRUSTED_SIGNING_ARGS: $(__TrustedSigningArgs) ${{ if parameters.Filter }}: FILTER: ${{ parameters.Filter }} - - ${{ if parameters.ExtractDir }}: - - powershell: | - if ($env:EXCLUDE) { - $files = (dir ($env:INCLUDE -split ',').Trim() -Exclude ($env:EXCLUDE -split ',').Trim() -File) - } else { - $files = (dir ($env:INCLUDE -split ',').Trim() -File) - } - $c = $files | %{ (Get-AuthenticodeSignature $_).SignerCertificate } | ?{ $_ -ne $null } | select -First 1 - if (-not $c) { - Write-Host "Failed to find certificate for ${{ parameters.SigningCertificate }}" - exit - } +- ${{ if parameters.ExtractDir }}: + - powershell: | + if ("${{ parameters.Exclude }}") { + $files = (dir ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }} -File) + } else { + $files = (dir ${{ parameters.Include }} -File) + } + $c = $files | %{ (Get-AuthenticodeSignature $_).SignerCertificate } | ?{ $_ -ne $null } | select -First 1 + if (-not $c) { + Write-Host "Failed to find certificate for ${{ parameters.SigningCertificate }}" + exit + } - $d = mkdir $env:EXTRACT_DIR -Force - $cf = "$d\cert.cer" - [IO.File]::WriteAllBytes($cf, $c.RawData) - $csha = (Get-FileHash $cf -Algorithm SHA256).Hash.ToLower() + $d = mkdir "${{ parameters.ExtractDir }}" -Force + $cf = "$d\cert.cer" + [IO.File]::WriteAllBytes($cf, $c.RawData) + $csha = (Get-FileHash $cf -Algorithm SHA256).Hash.ToLower() - $info = @{ Subject=$c.Subject; SHA256=$csha; } - $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" - displayName: "Extract certificate info" - workingDirectory: ${{ parameters.WorkingDir }} - env: - INCLUDE: ${{ parameters.Include }} - EXCLUDE: ${{ parameters.Exclude }} - EXTRACT_DIR: ${{ parameters.ExtractDir }} + $info = @{ Subject=$c.Subject; SHA256=$csha; } + $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" + displayName: "Extract certificate info" + workingDirectory: ${{ parameters.WorkingDir }} diff --git a/windows-release/stage-publish-pymanager.yml b/windows-release/stage-publish-pymanager.yml index f951b95f..f24f5073 100644 --- a/windows-release/stage-publish-pymanager.yml +++ b/windows-release/stage-publish-pymanager.yml @@ -3,7 +3,6 @@ parameters: DoFreethreaded: false DoEmbed: false HashAlgorithms: ['SHA256'] - SigningCertificate: '' Artifacts: - name: win32 @@ -32,14 +31,6 @@ jobs: variables: - group: PythonOrgPublish - - ${{ if eq(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}: - - group: CPythonSign - - ${{ if eq(parameters.SigningCertificate, 'TestSign') }}: - - group: CPythonTestSign - # Override the SigningDescription here, since we're only signing the feed - # and not the actual binaries. - - name: SigningDescription - value: "Python $(Build.BuildNumber)" workspace: clean: all @@ -66,29 +57,18 @@ jobs: artifact: pymanager_${{ a.name }} targetPath: $(Build.BinariesDirectory)\${{ a.name }} - - ${{ if eq(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}: - - task: DownloadSecureFile@1 - name: sshkey - inputs: - secureFile: pydotorg-ssh.ppk - displayName: 'Download PuTTY key' + - task: DownloadSecureFile@1 + name: sshkey + inputs: + secureFile: pydotorg-ssh.ppk + displayName: 'Download PuTTY key' - powershell: | - git clone https://github.com/python/cpython-bin-deps --revision 9f9e6fc31a55406ee5ff0198ea47bbb445eeb942 --depth 1 --progress -v "putty" + git clone https://github.com/python/cpython-bin-deps --branch putty --single-branch --depth 1 --progress -v "putty" "##vso[task.prependpath]$(gi putty)" workingDirectory: $(Pipeline.Workspace) displayName: 'Download PuTTY binaries' - # Use the template just to configure the signing tool. - # This will set MAKECAT and SIGN_COMMAND to be injected into later build steps - - template: sign-files.yml - parameters: - Include: "" - InstallTool: false - InstallLegacyTool: true - ExportLegacyCommand: SIGN_COMMAND - SigningCertificate: ${{ parameters.SigningCertificate }} - - powershell: | if ($env:FILENAME) { "##vso[task.setvariable variable=_PyManagerIndexFilename]${env:FILENAME}" @@ -101,38 +81,38 @@ jobs: FILENAME: $(PyManagerIndexFilename) displayName: 'Infer index filename' - - ${{ if ne(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}: - - powershell: | - "Preparing following packages:" - (dir "__install__.*.json").FullName - (dir "*\__install__.json").FullName - python "$(Build.SourcesDirectory)\windows-release\merge-and-upload.py" - workingDirectory: $(Build.BinariesDirectory) - displayName: 'Produce uploadable ZIPs (no upload)' - env: - NO_UPLOAD: 1 - INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' - INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' - MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' + - powershell: | + "Uploading following packages:" + (dir "__install__.*.json").FullName + (dir "*\__install__.json").FullName + python "$(Build.SourcesDirectory)\windows-release\merge-and-upload.py" + workingDirectory: $(Build.BinariesDirectory) + condition: and(succeeded(), ne(variables['IsRealSigned'], 'true')) + displayName: 'Produce uploadable ZIPs' + env: + NO_UPLOAD: 1 + INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' + INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' + MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' - - ${{ else }}: - - powershell: | - "Uploading following packages:" - (dir "__install__.*.json").FullName - (dir "*\__install__.json").FullName - python "$(Build.SourcesDirectory)\windows-release\merge-and-upload.py" - workingDirectory: $(Build.BinariesDirectory) - displayName: 'Upload ZIPs' - env: - INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' - INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' - MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' - UPLOAD_URL_PREFIX: $(PyDotOrgUrlPrefix) - UPLOAD_PATH_PREFIX: $(PyDotOrgUploadPathPrefix) - UPLOAD_HOST: $(PyDotOrgServer) - UPLOAD_HOST_KEY: $(PyDotOrgHostKey) - UPLOAD_USER: $(PyDotOrgUsername) - UPLOAD_KEYFILE: $(sshkey.secureFilePath) + - powershell: | + "Uploading following packages:" + (dir "__install__.*.json").FullName + (dir "*\__install__.json").FullName + python "$(Build.SourcesDirectory)\windows-release\merge-and-upload.py" + workingDirectory: $(Build.BinariesDirectory) + condition: and(succeeded(), eq(variables['IsRealSigned'], 'true')) + displayName: 'Upload ZIPs' + env: + INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' + INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' + MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' + UPLOAD_URL_PREFIX: $(PyDotOrgUrlPrefix) + UPLOAD_PATH_PREFIX: $(PyDotOrgUploadPathPrefix) + UPLOAD_HOST: $(PyDotOrgServer) + UPLOAD_HOST_KEY: $(PyDotOrgHostKey) + UPLOAD_USER: $(PyDotOrgUsername) + UPLOAD_KEYFILE: $(sshkey.secureFilePath) - ${{ each alg in parameters.HashAlgorithms }}: - powershell: |