From 4dfca9fd421f6506e162af6348fad13de2fbb5fb Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 29 Apr 2025 13:22:57 +0100 Subject: [PATCH 1/2] Fixes #9 Implement publish to python.org --- ci/release.yml | 29 ++++++- ci/upload.py | 202 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 ci/upload.py diff --git a/ci/release.yml b/ci/release.yml index 36361b9..51f14a8 100644 --- a/ci/release.yml +++ b/ci/release.yml @@ -279,7 +279,30 @@ jobs: Get-AppxPackage PythonSoftwareFoundation.PythonManager | Remove-AppxPackage displayName: 'Remove MSIX' - - ${{ if and(eq(parameters.Publish, 'true'), eq(parameters.Sign, 'true')) }}: + - ${{ if eq(parameters.Publish, 'true') }}: + - task: DownloadSecureFile@1 + name: sshkey + inputs: + secureFile: pydotorg-ssh.ppk + displayName: 'Download PuTTY key' + - powershell: | - Write-Host "TODO: Publish packages" - displayName: 'TODO: Publish packages' + git clone https://github.com/python/cpython-bin-deps --branch putty --single-branch --depth 1 --progress -v "putty" + "##vso[task.prependpath]$(gi putty)" + workingDirectory: $(Pipeline.Workspace) + displayName: 'Download PuTTY binaries' + + - powershell: | + python ci\upload.py + displayName: 'Publish packages' + env: + UPLOAD_URL: $(PyDotOrgUrlPrefix)python/pymanager + UPLOAD_DIR: $(DIST_DIR) + UPLOAD_URL_PREFIX: $(PyDotOrgUrlPrefix) + UPLOAD_PATH_PREFIX: $(PyDotOrgUploadPathPrefix) + UPLOAD_HOST: $(PyDotOrgServer) + UPLOAD_HOST_KEY: $(PyDotOrgHostKey) + UPLOAD_USER: $(PyDotOrgUsername) + UPLOAD_KEYFILE: $(sshkey.secureFilePath) + ${{ if ne(parameters.Sign, 'true') }}: + NO_UPLOAD: 1 diff --git a/ci/upload.py b/ci/upload.py new file mode 100644 index 0000000..3c78454 --- /dev/null +++ b/ci/upload.py @@ -0,0 +1,202 @@ +import os +import subprocess +import sys +from pathlib import Path +from xml.etree import ElementTree as ET + +UPLOAD_URL_PREFIX = os.getenv("UPLOAD_URL_PREFIX", "https://www.python.org/ftp/") +UPLOAD_PATH_PREFIX = os.getenv("UPLOAD_PATH_PREFIX", "/srv/www.python.org/ftp/") +UPLOAD_URL = os.getenv("UPLOAD_URL") +UPLOAD_DIR = os.getenv("UPLOAD_DIR") +# A version will be inserted before the extension later on +MANIFEST_FILE = os.getenv("MANIFEST_FILE") +UPLOAD_HOST = os.getenv("UPLOAD_HOST", "") +UPLOAD_HOST_KEY = os.getenv("UPLOAD_HOST_KEY", "") +UPLOAD_KEYFILE = os.getenv("UPLOAD_KEYFILE", "") +UPLOAD_USER = os.getenv("UPLOAD_USER", "") +NO_UPLOAD = os.getenv("NO_UPLOAD", "no")[:1].lower() in "yt1" + + +if not UPLOAD_URL: + print("##[error]Cannot upload without UPLOAD_URL") + sys.exit(1) + + +def find_cmd(env, exe): + cmd = os.getenv(env) + if cmd: + return Path(cmd) + for p in os.getenv("PATH", "").split(";"): + if p: + cmd = Path(p) / exe + if cmd.is_file(): + return cmd + if UPLOAD_HOST: + raise RuntimeError( + f"Could not find {exe} to perform upload. Try setting %{env}% or %PATH%" + ) + print(f"Did not find {exe}, but not uploading anyway.") + + +PLINK = find_cmd("PLINK", "plink.exe") +PSCP = find_cmd("PSCP", "pscp.exe") + + +def _std_args(cmd): + if not cmd: + raise RuntimeError("Cannot upload because command is missing") + all_args = [cmd, "-batch"] + if UPLOAD_HOST_KEY: + all_args.append("-hostkey") + all_args.append(UPLOAD_HOST_KEY) + if UPLOAD_KEYFILE: + all_args.append("-noagent") + all_args.append("-i") + all_args.append(UPLOAD_KEYFILE) + return all_args + + +class RunError(Exception): + pass + + +def _run(*args): + with subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="ascii", + errors="replace", + ) as p: + out, _ = p.communicate(None) + if out: + print(out.encode("ascii", "replace").decode("ascii")) + if p.returncode: + raise RunError(p.returncode, out) + + +def call_ssh(*args, allow_fail=True): + if not UPLOAD_HOST or NO_UPLOAD or LOCAL_INDEX: + print("Skipping", args, "because UPLOAD_HOST is missing") + return + try: + _run(*_std_args(PLINK), f"{UPLOAD_USER}@{UPLOAD_HOST}", *args) + except RunError: + if not allow_fail: + raise + + +def upload_ssh(source, dest): + if not UPLOAD_HOST or NO_UPLOAD or LOCAL_INDEX: + print("Skipping upload of", source, "because UPLOAD_HOST is missing") + return + _run(*_std_args(PSCP), source, f"{UPLOAD_USER}@{UPLOAD_HOST}:{dest}") + call_ssh(f"chgrp downloads {dest} && chmod g-x,o+r {dest}") + + +def download_ssh(source, dest): + if not UPLOAD_HOST: + print("Skipping download of", source, "because UPLOAD_HOST is missing") + return + Path(dest).parent.mkdir(exist_ok=True, parents=True) + _run(*_std_args(PSCP), f"{UPLOAD_USER}@{UPLOAD_HOST}:{source}", dest) + + +def ls_ssh(dest): + if not UPLOAD_HOST or LOCAL_INDEX: + print("Skipping ls of", dest, "because UPLOAD_HOST is missing") + return + try: + _run(*_std_args(PSCP), "-ls", f"{UPLOAD_USER}@{UPLOAD_HOST}:{dest}") + except RunError as ex: + if not ex.args[1].rstrip().endswith("No such file or directory"): + raise + print(dest, "was not found") + + +def url2path(url): + if not UPLOAD_URL_PREFIX: + raise ValueError("%UPLOAD_URL_PREFIX% was not set") + if not url: + raise ValueError("Unexpected empty URL") + if not url.startswith(UPLOAD_URL_PREFIX): + if LOCAL_INDEX: + return url + raise ValueError(f"Unexpected URL: {url}") + return UPLOAD_PATH_PREFIX + url[len(UPLOAD_URL_PREFIX) :] + + +def validate_appinstaller(file, uploads): + NS = {} + with open(file, "r", encoding="utf-8") as f: + NS = dict(e for _, e in ET.iterparse(f, events=("start-ns",))) + for k, v in NS.items(): + ET.register_namespace(k, v) + NS["x"] = NS[""] + + with open(file, "r", encoding="utf-8") as f: + xml = ET.parse(f) + + self_uri = xml.find(".[@Uri]", NS).get("Uri") + if not self_uri: + print("##[error]Empty Uri attribute in appinstaller file") + sys.exit(2) + if not any( + u.casefold() == self_uri.casefold() and f == file + for f, u, _ in uploads + ): + print("##[error]Uri", self_uri, "in appinstaller file is not where " + "the appinstaller file is being uploaded.") + sys.exit(2) + + main = xml.find("x:MainPackage[@Uri]", NS) + if main is None: + print("##[error]No MainPackage element with Uri in appinstaller file") + sys.exit(2) + package_uri = main.get("Uri") + if not package_uri: + print("##[error]Empty Mainpackage.Uri attribute in appinstaller file") + sys.exit(2) + if package_uri.casefold() not in [u.casefold() for _, u, _ in uploads]: + print("##[error]Uri", package_uri, "in appinstaller file is not being uploaded") + sys.exit(2) + + print(file, "checked:") + print("-", package_uri, "is part of this upload") + print("-", self_uri, "is the destination of this file") + print() + + +def purge(url): + if not UPLOAD_HOST or NO_UPLOAD: + print("Skipping purge of", url, "because UPLOAD_HOST is missing") + return + with urlopen(Request(url, method="PURGE", headers={"Fastly-Soft-Purge": 1})) as r: + r.read() + + +UPLOAD_DIR = Path(UPLOAD_DIR).absolute() +UPLOAD_URL = UPLOAD_URL.rstrip("/") + "/" + +UPLOADS = [] + +for pat in ("python-manager-*.msix", "python-manager-*.msi", "pymanager.appinstaller"): + for f in UPLOAD_DIR.glob(pat): + u = UPLOAD_URL + f.name + UPLOADS.append((f, u, url2path(u))) + +print("Planned uploads:") +for f, u, p in UPLOADS: + print(f"{f} -> {p}") + print(f" Final URL: {u}") +print() + +for f, *_ in UPLOADS: + if f.match("*.appinstaller"): + validate_appinstaller(f, UPLOADS) + +for f, u, p in UPLOADS: + print("Upload", f, "to", p) + upload_ssh(f, p) + print("Purge", u) + purge(u) From 4802e55b74440690b6b98b2de74357ef1429dd90 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 29 Apr 2025 13:25:01 +0100 Subject: [PATCH 2/2] Don't touch secure file in test builds --- ci/release.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ci/release.yml b/ci/release.yml index 51f14a8..daf6d0a 100644 --- a/ci/release.yml +++ b/ci/release.yml @@ -280,11 +280,12 @@ jobs: displayName: 'Remove MSIX' - ${{ if eq(parameters.Publish, 'true') }}: - - task: DownloadSecureFile@1 - name: sshkey - inputs: - secureFile: pydotorg-ssh.ppk - displayName: 'Download PuTTY key' + - ${{ if eq(parameters.Sign, 'true') }}: + - task: DownloadSecureFile@1 + name: sshkey + inputs: + secureFile: pydotorg-ssh.ppk + displayName: 'Download PuTTY key' - powershell: | git clone https://github.com/python/cpython-bin-deps --branch putty --single-branch --depth 1 --progress -v "putty"