diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index b37eb727955909..b969e6d13e61a3 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -41,7 +41,7 @@ jobs: - true - false llvm: - - 16 + - 18 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -94,6 +94,7 @@ jobs: - name: Native Windows if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | + choco upgrade llvm -y choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -102,27 +103,31 @@ jobs: - name: Emulated Windows if: runner.os == 'Windows' && matrix.architecture == 'ARM64' run: | + choco upgrade llvm -y choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} - name: Native macOS if: runner.os == 'macOS' run: | + brew update brew install llvm@${{ matrix.llvm }} SDKROOT="$(xcrun --show-sdk-path)" \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} make all --jobs 4 ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553) - name: Native Linux if: runner.os == 'Linux' && matrix.architecture == 'x86_64' run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }} make all --jobs 4 ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553) - name: Emulated Linux if: runner.os == 'Linux' && matrix.architecture != 'x86_64' run: | @@ -139,6 +144,6 @@ jobs: CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \ CPP="$CC --preprocess" \ HOSTRUNNER=qemu-${{ matrix.architecture }} \ - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst new file mode 100644 index 00000000000000..051295541ab7e2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst @@ -0,0 +1 @@ +Update JIT compilation to use LLVM 18 diff --git a/Python/jit.c b/Python/jit.c index 8782adb847cfd6..75ec4fb9756eb7 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -234,6 +234,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) assert((int64_t)value < (1LL << 31)); *loc32 = (uint32_t)value; continue; + case HoleKind_ARM64_RELOC_BRANCH26: case HoleKind_IMAGE_REL_ARM64_BRANCH26: case HoleKind_R_AARCH64_CALL26: case HoleKind_R_AARCH64_JUMP26: @@ -278,6 +279,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21: case HoleKind_IMAGE_REL_ARM64_PAGEBASE_REL21: case HoleKind_R_AARCH64_ADR_GOT_PAGE: + case HoleKind_R_AARCH64_ADR_PREL_PG_HI21: // 21-bit count of pages between this page and an absolute address's // page... I know, I know, it's weird. Pairs nicely with // ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below). @@ -341,6 +343,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) case HoleKind_ARM64_RELOC_PAGEOFF12: case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12A: case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L: + case HoleKind_R_AARCH64_ADD_ABS_LO12_NC: case HoleKind_R_AARCH64_LD64_GOT_LO12_NC: // 12-bit low part of an absolute address. Pairs nicely with // ARM64_RELOC_GOT_LOAD_PAGE21 (above). diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 04a6c0780bf972..7b33f99d23f75d 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -23,17 +23,21 @@ sudo ./llvm.sh 16 ### macOS -Install LLVM 16 with [Homebrew](https://brew.sh): +Install LLVM 18 with [Homebrew](https://brew.sh): ```sh -brew install llvm@16 +brew install llvm@18 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. ### Windows -Install LLVM 16 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=16), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Install LLVM 18 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=18), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** + +### Dev Containers + +If you are working CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 40 base image includes LLVM 18 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 603bbef59ba2e6..74a048ccbfcc58 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -7,7 +7,7 @@ import subprocess import typing -_LLVM_VERSION = 16 +_LLVM_VERSION = 18 _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\s+") _P = typing.ParamSpec("_P") diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 045fd502a03c12..6aef887e12475b 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -2,6 +2,7 @@ import typing HoleKind: typing.TypeAlias = typing.Literal[ + "ARM64_RELOC_BRANCH26", "ARM64_RELOC_GOT_LOAD_PAGE21", "ARM64_RELOC_GOT_LOAD_PAGEOFF12", "ARM64_RELOC_PAGE21", @@ -16,8 +17,10 @@ "IMAGE_REL_I386_REL32", "R_AARCH64_ABS64", "R_AARCH64_ADR_GOT_PAGE", + "R_AARCH64_ADR_PREL_PG_HI21", "R_AARCH64_CALL26", "R_AARCH64_JUMP26", + "R_AARCH64_ADD_ABS_LO12_NC", "R_AARCH64_LD64_GOT_LO12_NC", "R_AARCH64_MOVW_UABS_G0_NC", "R_AARCH64_MOVW_UABS_G1_NC", @@ -68,12 +71,12 @@ class _COFFSymbol(typing.TypedDict): class _ELFSymbol(typing.TypedDict): - Name: dict[typing.Literal["Value"], str] + Name: dict[typing.Literal["Name"], str] Value: int class _MachOSymbol(typing.TypedDict): - Name: dict[typing.Literal["Value"], str] + Name: dict[typing.Literal["Name"], str] Value: int @@ -99,7 +102,7 @@ class ELFSection(typing.TypedDict): Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]] SectionData: dict[typing.Literal["Bytes"], list[int]] Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]] - Type: dict[typing.Literal["Value"], str] + Type: dict[typing.Literal["Name"], str] class MachOSection(typing.TypedDict): diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 243bb3dd134f70..f8ecffcf3ddda2 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -203,7 +203,9 @@ def process_relocations(self, *, alignment: int = 1) -> None: """Fix up all GOT and internal relocations for this stencil group.""" for hole in self.code.holes.copy(): if ( - hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"} + hole.kind in { + "R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26" + } and hole.value is HoleValue.ZERO ): self.code.pad(alignment) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 66db358679239e..91734b36b4ab1b 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -221,7 +221,7 @@ def _handle_relocation( case { "Offset": offset, "Symbol": s, - "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind}, + "Type": {"Name": "IMAGE_REL_I386_DIR32" as kind}, }: offset += base value, symbol = self._unwrap_dllimport(s) @@ -230,7 +230,7 @@ def _handle_relocation( "Offset": offset, "Symbol": s, "Type": { - "Value": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind + "Name": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind }, }: offset += base @@ -242,7 +242,7 @@ def _handle_relocation( "Offset": offset, "Symbol": s, "Type": { - "Value": "IMAGE_REL_ARM64_BRANCH26" + "Name": "IMAGE_REL_ARM64_BRANCH26" | "IMAGE_REL_ARM64_PAGEBASE_REL21" | "IMAGE_REL_ARM64_PAGEOFFSET_12A" | "IMAGE_REL_ARM64_PAGEOFFSET_12L" as kind @@ -262,7 +262,7 @@ class _ELF( def _handle_section( self, section: _schema.ELFSection, group: _stencils.StencilGroup ) -> None: - section_type = section["Type"]["Value"] + section_type = section["Type"]["Name"] flags = {flag["Name"] for flag in section["Flags"]["Flags"]} if section_type == "SHT_RELA": assert "SHF_INFO_LINK" in flags, flags @@ -290,7 +290,7 @@ def _handle_section( for wrapped_symbol in section["Symbols"]: symbol = wrapped_symbol["Symbol"] offset = len(stencil.body) + symbol["Value"] - name = symbol["Name"]["Value"] + name = symbol["Name"]["Name"] name = name.removeprefix(self.prefix) group.symbols[name] = value, offset stencil.body.extend(section["SectionData"]["Bytes"]) @@ -312,9 +312,9 @@ def _handle_relocation( case { "Addend": addend, "Offset": offset, - "Symbol": {"Value": s}, + "Symbol": {"Name": s}, "Type": { - "Value": "R_AARCH64_ADR_GOT_PAGE" + "Name": "R_AARCH64_ADR_GOT_PAGE" | "R_AARCH64_LD64_GOT_LO12_NC" | "R_X86_64_GOTPCREL" | "R_X86_64_GOTPCRELX" @@ -327,8 +327,8 @@ def _handle_relocation( case { "Addend": addend, "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": kind}, + "Symbol": {"Name": s}, + "Type": {"Name": kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -371,7 +371,7 @@ def _handle_section( for wrapped_symbol in section["Symbols"]: symbol = wrapped_symbol["Symbol"] offset = symbol["Value"] - start_address - name = symbol["Name"]["Value"] + name = symbol["Name"]["Name"] name = name.removeprefix(self.prefix) group.symbols[name] = value, offset assert "Relocations" in section @@ -387,9 +387,9 @@ def _handle_relocation( match relocation: case { "Offset": offset, - "Symbol": {"Value": s}, + "Symbol": {"Name": s}, "Type": { - "Value": "ARM64_RELOC_GOT_LOAD_PAGE21" + "Name": "ARM64_RELOC_GOT_LOAD_PAGE21" | "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind }, }: @@ -399,8 +399,8 @@ def _handle_relocation( addend = 0 case { "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, + "Symbol": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -410,13 +410,13 @@ def _handle_relocation( ) case { "Offset": offset, - "Section": {"Value": s}, - "Type": {"Value": "X86_64_RELOC_SIGNED" as kind}, + "Section": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_SIGNED" as kind}, } | { "Offset": offset, - "Symbol": {"Value": s}, + "Symbol": {"Name": s}, "Type": { - "Value": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind + "Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind }, }: offset += base @@ -427,12 +427,12 @@ def _handle_relocation( ) case { "Offset": offset, - "Section": {"Value": s}, - "Type": {"Value": kind}, + "Section": {"Name": s}, + "Type": {"Name": kind}, } | { "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": kind}, + "Symbol": {"Name": s}, + "Type": {"Name": kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -446,13 +446,12 @@ def _handle_relocation( def get_target(host: str) -> _COFF | _ELF | _MachO: """Build a _Target for the given host "triple" and options.""" if re.fullmatch(r"aarch64-apple-darwin.*", host): - args = ["-mcmodel=large"] - return _MachO(host, alignment=8, args=args, prefix="_") + return _MachO(host, alignment=8, prefix="_") if re.fullmatch(r"aarch64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] return _COFF(host, alignment=8, args=args) if re.fullmatch(r"aarch64-.*-linux-gnu", host): - args = ["-mcmodel=large"] + args = ["-fpic"] return _ELF(host, alignment=8, args=args) if re.fullmatch(r"i686-pc-windows-msvc", host): args = ["-DPy_NO_ENABLE_SHARED"] @@ -463,5 +462,6 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: args = ["-fms-runtime-lib=dll"] return _COFF(host, args=args) if re.fullmatch(r"x86_64-.*-linux-gnu", host): - return _ELF(host) + args = ["-fpic"] + return _ELF(host, args=args) raise ValueError(host)