From 6be4e483e030b3a94c24c4f981be40d1f613d063 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sat, 26 Jul 2025 15:37:34 +0000 Subject: [PATCH 1/7] Update main snapshot to 2025-07-23 --- README.md | 2 +- docs/upstreaming.md | 8 +--- schemes/main/build/build-target-toolchain.sh | 5 +- schemes/main/manifest.json | 2 +- tools/build/build-toolchain | 50 ++++++++++++++++++++ tools/build/build-toolchain.sh | 23 --------- tools/build/build_support/actions.py | 23 ++++++--- tools/build/build_support/llvm_tools.py | 3 +- tools/build/ci.sh | 2 +- 9 files changed, 76 insertions(+), 42 deletions(-) create mode 100755 tools/build/build-toolchain delete mode 100755 tools/build/build-toolchain.sh diff --git a/README.md b/README.md index 7c378d8e..4dfe941c 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ $ ./tools/build/install-build-sdk.sh main # Checkout the Swift repositories in the parent directory (swiftwasm-source) and apply patches $ ./tools/git-swift-workspace --scheme main # Build the toolchain (this will take a while) -$ ./tools/build/build-toolchain.sh main +$ ./tools/build/build-toolchain main ``` See [SwiftWasm book](https://book.swiftwasm.org/contribution-guide/how-to-build-toolchain.html) for more details about dependencies you need to install and how to build on Docker. diff --git a/docs/upstreaming.md b/docs/upstreaming.md index fa0a8c42..52fdb914 100644 --- a/docs/upstreaming.md +++ b/docs/upstreaming.md @@ -65,13 +65,7 @@ If it's first time to build or patch files are changed, you need to checkout the If you want to build the whole toolchain, run the following command: ``` -./tools/build/build-toolchain.sh -``` - -If you already built the compiler and want to build only the standard library for WebAssembly, run the following command: - -``` -./tools/build/build-toolchain.sh --skip-build-host-toolchain +./tools/build/build-toolchain --scheme main ``` ### Edit diff --git a/schemes/main/build/build-target-toolchain.sh b/schemes/main/build/build-target-toolchain.sh index 8e53202f..60c66fb6 100755 --- a/schemes/main/build/build-target-toolchain.sh +++ b/schemes/main/build/build-target-toolchain.sh @@ -137,7 +137,8 @@ main() { # NOTE: The llvm-cmake-options is a workaround for the issue on amazonlinux2 # See https://github.com/apple/swift/commit/40c7268e8f7d402b27e3ad16a84180e07c37f92c - "$SOURCE_PATH/swift/utils/build-script" \ + # NOTE: Add llvm-bin directory to PATH so that wasmstdlib.py can find FileCheck during tests + env PATH="$OPTIONS_LLVM_BIN:$OPTIONS_SWIFT_BIN:$PATH" "$SOURCE_PATH/swift/utils/build-script" \ --build-subdir=WebAssembly \ --release \ --skip-build-llvm \ @@ -145,11 +146,13 @@ main() { --skip-build-cmark \ --skip-build-benchmarks \ --skip-early-swift-driver \ + --wasmkit \ --build-wasm-stdlib \ --skip-test-wasm-stdlib \ --native-swift-tools-path="$OPTIONS_SWIFT_BIN" \ --native-clang-tools-path="$OPTIONS_CLANG_BIN" \ --native-llvm-tools-path="$OPTIONS_LLVM_BIN" \ + --build-runtime-with-host-compiler \ --extra-cmake-options="\ -DSWIFT_STDLIB_TRACING=NO \ -DSWIFT_STDLIB_HAS_ASL=NO \ diff --git a/schemes/main/manifest.json b/schemes/main/manifest.json index 47605521..f851a218 100644 --- a/schemes/main/manifest.json +++ b/schemes/main/manifest.json @@ -1,5 +1,5 @@ { - "base-tag": "swift-DEVELOPMENT-SNAPSHOT-2025-07-18-a", + "base-tag": "swift-DEVELOPMENT-SNAPSHOT-2025-07-23-a", "icu4c": [], "libxml2": [ "https://github.com/swiftwasm/libxml2-wasm/releases/download/2.0.0/libxml2-wasm32-unknown-wasi.tar.gz", diff --git a/tools/build/build-toolchain b/tools/build/build-toolchain new file mode 100755 index 00000000..93a25182 --- /dev/null +++ b/tools/build/build-toolchain @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import sys +import argparse +import pathlib +import subprocess +from build_support.actions import derive_options_from_args, REPO_ROOT, DownloadBaseSnapshotAction + + +def main(): + parser = argparse.ArgumentParser(description='Build the Swift toolchain') + options = derive_options_from_args(sys.argv[1:], parser) + + # Calculate paths + repo_path = pathlib.Path(REPO_ROOT) + source_path = repo_path.parent + tools_build_path = pathlib.Path(__file__).parent + + build_dir = source_path / "build" + cross_compiler_destdir = pathlib.Path(DownloadBaseSnapshotAction.toolchain_path(options)) + + # Install base toolchain + print("=====> Installing base toolchain") + subprocess.check_call([ + str(tools_build_path / "install-base-toolchain"), + "--scheme", options.scheme + ]) + + # Build LLVM tools + print("=====> Building LLVM tools") + subprocess.check_call([ + str(tools_build_path / "build-llvm-tools"), + "--toolchain", str(cross_compiler_destdir) + ]) + + # Build target toolchain + print("=====> Building target toolchain") + scheme_build_script = repo_path / "schemes" / options.scheme / "build" / "build-target-toolchain.sh" + + subprocess.check_call([ + str(scheme_build_script), + "--llvm-bin", str(build_dir / "llvm-tools" / "bin"), + "--clang-bin", str(cross_compiler_destdir / "usr" / "bin"), + "--swift-bin", str(cross_compiler_destdir / "usr" / "bin"), + "--scheme", options.scheme + ]) + + +if __name__ == '__main__': + main() diff --git a/tools/build/build-toolchain.sh b/tools/build/build-toolchain.sh deleted file mode 100755 index 1823d9a3..00000000 --- a/tools/build/build-toolchain.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -eux -o pipefail - -REPO_PATH="$(cd "$(dirname $0)/../.." && pwd)" -SOURCE_PATH="$REPO_PATH/.." -SCHEME="${1:?"scheme is not specified"}" - -TOOLS_BUILD_PATH="$(cd "$(dirname "$0")" && pwd)" - -BUILD_DIR="$SOURCE_PATH/build" -PACKAGING_DIR="$BUILD_DIR/Packaging" -CROSS_COMPILER_DESTDIR=$PACKAGING_DIR/base-snapshot - -"$TOOLS_BUILD_PATH/install-base-toolchain" --scheme "$SCHEME" -"$TOOLS_BUILD_PATH/build-llvm-tools" --toolchain "$CROSS_COMPILER_DESTDIR" - -"$REPO_PATH/schemes/$SCHEME/build/build-target-toolchain.sh" \ - --llvm-bin "$BUILD_DIR/llvm-tools/bin" \ - --clang-bin "$CROSS_COMPILER_DESTDIR/usr/bin" \ - --swift-bin "$CROSS_COMPILER_DESTDIR/usr/bin" \ - --scheme "$SCHEME" - diff --git a/tools/build/build_support/actions.py b/tools/build/build_support/actions.py index fb16c6d1..126dce2b 100644 --- a/tools/build/build_support/actions.py +++ b/tools/build/build_support/actions.py @@ -114,16 +114,25 @@ def run(self): self.system('git', '-C', f'../{repo}', 'checkout', rev) class DownloadBaseSnapshotAction(Action): + + @staticmethod + def toolchains_dir(): + return os.path.join(REPO_ROOT, '..', 'build', 'Packaging', 'base-toolchain') + + @staticmethod + def toolchain_path(options): + return os.path.join(DownloadBaseSnapshotAction.toolchains_dir(), options.tag) + def run(self): platform_info = platform.PlatformInfo.derive() snapshot_url, tarball_name = platform_info.snapshot_url(self.options.swift_org_download_channel, self.options.tag) extension = platform_info.package_extension - tarball_path = os.path.join('..', 'build', 'Packaging', f'base-snapshot.{extension}') - base_snapshot_dir = os.path.join('..', 'build', 'Packaging', 'base-snapshot') + base_toolchain_dir = DownloadBaseSnapshotAction.toolchain_path(self.options) + tarball_path = os.path.join(os.path.dirname(base_toolchain_dir), tarball_name) - if os.path.exists(os.path.join(base_snapshot_dir, 'usr')): - print(f"=====> Base snapshot '{base_snapshot_dir}' already exists") + if os.path.exists(os.path.join(base_toolchain_dir, 'usr')): + print(f"=====> Base snapshot '{base_toolchain_dir}' already exists") return if not os.path.exists(tarball_path): @@ -132,15 +141,15 @@ def run(self): self.system('curl', '--fail', '-L', '-o', tarball_path, snapshot_url) print(f"=====> Unpacking base snapshot {tarball_name}") - os.makedirs(base_snapshot_dir, exist_ok=True) + os.makedirs(base_toolchain_dir, exist_ok=True) if extension == "tar.gz": - self.system('tar', '-C', base_snapshot_dir, '--strip-components', '1', '-xzf', tarball_path) + self.system('tar', '-C', base_toolchain_dir, '--strip-components', '1', '-xzf', tarball_path) elif extension == "pkg": import tempfile with tempfile.TemporaryDirectory() as tmpdir: self.system('xar', '-xf', tarball_path, '-C', tmpdir) old_cwd = os.getcwd() - os.chdir(base_snapshot_dir) + os.chdir(base_toolchain_dir) pkg_name = tarball_name.replace(".pkg", "-package.pkg") self.system('cpio', '-i', '-I', os.path.join(tmpdir, pkg_name, 'Payload')) os.chdir(old_cwd) diff --git a/tools/build/build_support/llvm_tools.py b/tools/build/build_support/llvm_tools.py index eb47da43..837082c3 100644 --- a/tools/build/build_support/llvm_tools.py +++ b/tools/build/build_support/llvm_tools.py @@ -8,4 +8,5 @@ # These tools are used by Swift test suite 'llvm-nm', 'FileCheck', -] \ No newline at end of file + 'split-file', +] diff --git a/tools/build/ci.sh b/tools/build/ci.sh index f75c44d9..584e19d7 100755 --- a/tools/build/ci.sh +++ b/tools/build/ci.sh @@ -34,6 +34,6 @@ install_sccache_if_needed() { install_sccache_if_needed export PATH="$SCCACHE_INSTALL_PATH:$PATH" -$TOOLS_BUILD_PATH/build-toolchain.sh "$SCHEME" +$TOOLS_BUILD_PATH/build-toolchain --scheme "$SCHEME" $TOOLS_BUILD_PATH/package-toolchain --scheme "$SCHEME" --daily-snapshot From 36dc4c10571776cea97e4d3445b28b5cbaf89d63 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sun, 27 Jul 2025 04:03:01 +0000 Subject: [PATCH 2/7] Include not tool as some embedded tests require it --- tools/build/build_support/llvm_tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/build/build_support/llvm_tools.py b/tools/build/build_support/llvm_tools.py index 837082c3..6e2f7e0e 100644 --- a/tools/build/build_support/llvm_tools.py +++ b/tools/build/build_support/llvm_tools.py @@ -9,4 +9,5 @@ 'llvm-nm', 'FileCheck', 'split-file', + 'not', ] From 49698aaa86b1831162bb723bd90c56f289d0cb49 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sun, 27 Jul 2025 05:53:44 +0000 Subject: [PATCH 3/7] Fix e2e test driver to use new base-toolchain path --- .github/workflows/build-toolchain.yml | 5 +-- tools/build/run-e2e-test | 54 +++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) create mode 100755 tools/build/run-e2e-test diff --git a/.github/workflows/build-toolchain.yml b/.github/workflows/build-toolchain.yml index c18685d5..e0b05aed 100644 --- a/.github/workflows/build-toolchain.yml +++ b/.github/workflows/build-toolchain.yml @@ -265,10 +265,7 @@ jobs: if: ${{ matrix.run_e2e_test }} run: | docker exec swiftwasm-ci-buildbot /bin/bash -lc \ - "./llvm-project/llvm/utils/lit/lit.py \ - --param swift-sdk=wasm32-unknown-wasi \ - --param swift-sdks-path=./swift-sdk-generator/Bundles \ - --param scheme=${{ matrix.scheme }} ./swiftwasm-build/test -vv" + "./swiftwasm-build/tools/build/run-e2e-test --scheme ${{ matrix.scheme }}" - name: Cleanup docker volume if: ${{ matrix.container != null }} diff --git a/tools/build/run-e2e-test b/tools/build/run-e2e-test new file mode 100755 index 00000000..1615d9f2 --- /dev/null +++ b/tools/build/run-e2e-test @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import sys +import argparse +import pathlib +import subprocess +import os +from build_support.actions import derive_options_from_args, REPO_ROOT, DownloadBaseSnapshotAction + + +def main(): + parser = argparse.ArgumentParser( + description='Run e2e integration tests for Swift SDK') + parser.add_argument('--swift-sdk', default='wasm32-unknown-wasi', + help='Swift SDK identifier (default: wasm32-unknown-wasi)') + parser.add_argument('args', nargs=argparse.REMAINDER, + help='Extra arguments to pass to lit.py') + options = derive_options_from_args(sys.argv[1:], parser) + + extra_lit_args = options.args + if len(extra_lit_args) > 0 and extra_lit_args[0] == "--": + extra_lit_args = extra_lit_args[1:] + + repo_path = pathlib.Path(REPO_ROOT) + base_toolchain_path = DownloadBaseSnapshotAction.toolchain_path(options) + lit_py_path = repo_path.parent / "llvm-project" / "llvm" / "utils" / "lit" / "lit.py" + swift_sdks_path = repo_path.parent / "swift-sdk-generator" / "Bundles" + + # Build lit command + lit_cmd = [ + str(lit_py_path), + "--param", f"swift-sdk={options.swift_sdk}", + "--param", f"scheme={options.scheme}", + "--param", f"base-toolchain-path={base_toolchain_path}", + "--param", f"swift-sdks-path={swift_sdks_path}", + ] + test_dir = repo_path / "test" + lit_cmd.append(str(test_dir)) + lit_cmd.extend(extra_lit_args) + + print("=====> Running e2e integration tests") + print(f"Command: {' '.join(lit_cmd)}") + + # Run the tests + try: + subprocess.check_call(lit_cmd) + print("=====> e2e tests completed successfully") + except subprocess.CalledProcessError as e: + print(f"=====> e2e tests failed with exit code {e.returncode}") + sys.exit(e.returncode) + + +if __name__ == '__main__': + main() From 6d3a5dd7a518791c9c5df7d98632441b813b65d0 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sun, 27 Jul 2025 08:31:36 +0000 Subject: [PATCH 4/7] Clean up no longer needed Foundation and XCTest build steps --- schemes/main/build/build-foundation.sh | 55 -------------------- schemes/main/build/build-target-toolchain.sh | 49 +++-------------- schemes/main/build/build-xctest.sh | 39 -------------- schemes/main/build/run-test.sh | 7 +-- schemes/main/manifest.json | 5 +- tools/build/package-toolchain | 13 +++-- tools/build/run-e2e-test | 2 +- 7 files changed, 20 insertions(+), 150 deletions(-) delete mode 100755 schemes/main/build/build-foundation.sh delete mode 100755 schemes/main/build/build-xctest.sh diff --git a/schemes/main/build/build-foundation.sh b/schemes/main/build/build-foundation.sh deleted file mode 100755 index f34f9625..00000000 --- a/schemes/main/build/build-foundation.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -ex -DESTINATION_TOOLCHAIN=$1 -LLVM_BIN_DIR=$2 -CLANG_BIN_DIR=$3 -SWIFT_BIN_DIR=$4 -WASI_SYSROOT_PATH=$5 -TRIPLE="$6" - -SOURCE_PATH="$(cd "$(dirname $0)/../../../.." && pwd)" -BUILD_SDK_PATH="$SOURCE_PATH/build-sdk" -LIBXML2_PATH="$BUILD_SDK_PATH/libxml2-$TRIPLE" - -FOUNDATION_BUILD="$SOURCE_PATH/build/WebAssembly/foundation-$TRIPLE" - -swift_extra_flags="" -c_extra_flags="" -if [[ "$TRIPLE" == "wasm32-unknown-wasip1-threads" ]]; then - swift_extra_flags="-Xcc -matomics -Xcc -mbulk-memory -Xcc -mthread-model -Xcc posix -Xcc -pthread -Xcc -ftls-model=local-exec" - c_extra_flags="-mthread-model posix -pthread -ftls-model=local-exec" -fi - -cmake -G Ninja \ - -D CMAKE_BUILD_TYPE="Release" \ - -D CMAKE_SYSROOT="$WASI_SYSROOT_PATH" \ - -D CMAKE_Swift_COMPILER="$SWIFT_BIN_DIR/swiftc" \ - -D CMAKE_STAGING_PREFIX="$DESTINATION_TOOLCHAIN/usr" \ - -D CMAKE_SYSTEM_NAME=WASI \ - -D CMAKE_SYSTEM_PROCESSOR=wasm32 \ - -D CMAKE_C_COMPILER_TARGET="$TRIPLE" \ - -D CMAKE_CXX_COMPILER_TARGET="$TRIPLE" \ - -D CMAKE_Swift_COMPILER_TARGET="$TRIPLE" \ - -D CMAKE_C_COMPILER="$CLANG_BIN_DIR/clang" \ - -D CMAKE_CXX_COMPILER="$CLANG_BIN_DIR/clang++" \ - -D CMAKE_AR="$LLVM_BIN_DIR/llvm-ar" \ - -D CMAKE_RANLIB="$LLVM_BIN_DIR/llvm-ranlib" \ - -D LIBXML2_INCLUDE_DIR="$LIBXML2_PATH/include/libxml2" \ - -D LIBXML2_LIBRARY="$LIBXML2_PATH/lib" \ - -D BUILD_SHARED_LIBS=OFF \ - -D FOUNDATION_BUILD_TOOLS=OFF \ - -D CMAKE_Swift_COMPILER_FORCED=ON \ - -D CMAKE_C_COMPILER_FORCED=ON \ - -D CMAKE_CXX_COMPILER_FORCED=ON \ - -D CMAKE_Swift_FLAGS="-sdk $WASI_SYSROOT_PATH -resource-dir $DESTINATION_TOOLCHAIN/usr/lib/swift_static $swift_extra_flags" \ - -D CMAKE_C_FLAGS="-resource-dir $DESTINATION_TOOLCHAIN/usr/lib/swift_static/clang -B $LLVM_BIN_DIR $c_extra_flags" \ - -D CMAKE_CXX_FLAGS="-fno-exceptions -resource-dir $DESTINATION_TOOLCHAIN/usr/lib/swift_static/clang -B $LLVM_BIN_DIR $c_extra_flags" \ - -D _SwiftCollections_SourceDIR="$SOURCE_PATH/swift-collections" \ - -D _SwiftFoundation_SourceDIR="$SOURCE_PATH/swift-foundation" \ - -D _SwiftFoundationICU_SourceDIR="$SOURCE_PATH/swift-foundation-icu" \ - -D SwiftFoundation_MACRO="$SWIFT_BIN_DIR/../lib/swift/host/plugins" \ - -B "$FOUNDATION_BUILD" \ - "${SOURCE_PATH}/swift-corelibs-foundation" - -cmake --build "$FOUNDATION_BUILD" -cmake --install "$FOUNDATION_BUILD" diff --git a/schemes/main/build/build-target-toolchain.sh b/schemes/main/build/build-target-toolchain.sh index 60c66fb6..e79748e9 100755 --- a/schemes/main/build/build-target-toolchain.sh +++ b/schemes/main/build/build-target-toolchain.sh @@ -40,48 +40,16 @@ build_target_toolchain() { env DESTDIR="$TRIPLE_DESTDIR" \ cmake --install "$TARGET_BUILD_ROOT/$STDLIB_PRODUCT-$HOST_SUFFIX" --prefix /usr - local swift_testing_build_dir="$TARGET_BUILD_ROOT/wasmswiftsdk-$HOST_SUFFIX/swift-testing/$TRIPLE" - # TODO: Remove this check once we build swift-testing for +threads target - if [[ -d "$swift_testing_build_dir" ]]; then - env DESTDIR="$TRIPLE_DESTDIR" \ - cmake --install "$swift_testing_build_dir" --prefix /usr - fi + env DESTDIR="$TRIPLE_DESTDIR" \ + cmake --install "$TARGET_BUILD_ROOT/wasmswiftsdk-$HOST_SUFFIX/swift-testing/$TRIPLE" --prefix /usr + env DESTDIR="$TRIPLE_DESTDIR" \ + cmake --install "$TARGET_BUILD_ROOT/wasmswiftsdk-$HOST_SUFFIX/foundation/$TRIPLE" --prefix /usr + env DESTDIR="$TRIPLE_DESTDIR" \ + cmake --install "$TARGET_BUILD_ROOT/wasmswiftsdk-$HOST_SUFFIX/xctest/$TRIPLE" --prefix /usr rm -rf "$TRIPLE_DESTDIR/usr/lib/swift_static/clang/lib/$COMPILER_RT_OS_DIR" # XXX: Is this the right way to install compiler-rt? cp -R "$TARGET_BUILD_ROOT/wasi-sysroot/$CLANG_MULTIARCH_TRIPLE/lib/$COMPILER_RT_OS_DIR" "$TRIPLE_DESTDIR/usr/lib/swift_static/clang/lib/$COMPILER_RT_OS_DIR" - - # FIXME: Clang resource directory installation is not the best way currently. - # We currently have two copies of compiler headers copied from the base toolchain in - # lib/swift/clang and lib/swift_static/clang. This is because the Swift CMake build - # system installs the compiler headers from the native tools path when not building - # tools including clang compiler. This is not ideal but then where should we bring - # the compiler headers from? If we use the headers beside the base toolchain, clang - # driver will not be able to find libclang_rt.builtins-wasm32.a because it is not - # a part of the base toolchain. We need to find a better way to handle this. - local CLANG_VERSION - CLANG_VERSION="$(basename "$($CLANG_BIN_DIR/clang -print-resource-dir)")" - mkdir -p "$TRIPLE_DESTDIR/usr/lib/clang/$CLANG_VERSION/lib" - ln -sf "../../../swift_static/clang/lib/$COMPILER_RT_OS_DIR" "$TRIPLE_DESTDIR/usr/lib/clang/$CLANG_VERSION/lib/$COMPILER_RT_OS_DIR" -} - -build_target_corelibs() { - local LLVM_BIN_DIR="$1" - local CLANG_BIN_DIR="$2" - local SWIFT_BIN_DIR="$3" - local TRIPLE="$4" - local SHORT_TRIPLE="$5" - - local TRIPLE_DESTDIR="$TARGET_TOOLCHAIN_DESTDIR/$TRIPLE" - local CORELIBS_ARGS=( - "$TRIPLE_DESTDIR" - "$LLVM_BIN_DIR" - "$CLANG_BIN_DIR" - "$SWIFT_BIN_DIR" - "$WASI_SYSROOT_PATH/$SHORT_TRIPLE" - ) - "$SCHEMES_BUILD_PATH/build-foundation.sh" "${CORELIBS_ARGS[@]}" "$TRIPLE" - "$SCHEMES_BUILD_PATH/build-xctest.sh" "${CORELIBS_ARGS[@]}" "$TRIPLE" } main() { @@ -171,13 +139,10 @@ main() { "$OPTIONS_SWIFT_BIN" ) - build_target_toolchain "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasi" "wasi-wasm32" "wasm32-wasi" "wasmstdlib" "wasi" + build_target_toolchain "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasip1" "wasip1-wasm32" "wasm32-wasip1" "wasmstdlib" "wasip1" build_target_toolchain "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasip1-threads" "wasip1-threads-wasm32" "wasm32-wasip1-threads" "wasmthreadsstdlib" "wasip1" rsync -av "$WASI_SYSROOT_PATH/" "$PACKAGING_DIR/wasi-sysroot/" - - build_target_corelibs "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasi" "wasm32-wasi" - build_target_corelibs "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasip1-threads" "wasm32-wasip1-threads" } main "$@" diff --git a/schemes/main/build/build-xctest.sh b/schemes/main/build/build-xctest.sh deleted file mode 100755 index 472cc664..00000000 --- a/schemes/main/build/build-xctest.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -ex -DESTINATION_TOOLCHAIN=$1 -LLVM_BIN_DIR=$2 -CLANG_BIN_DIR=$3 -SWIFT_BIN_DIR=$4 -WASI_SYSROOT_PATH=$5 -TRIPLE="$6" - -SOURCE_PATH="$(cd "$(dirname $0)/../../../../" && pwd)" -SCHEME_BUILD_PATH="$(cd "$(dirname $0)" && pwd)" - -BUILD_DIR="$SOURCE_PATH/build/WebAssembly/xctest-$TRIPLE" - -mkdir -p $BUILD_DIR -cd $BUILD_DIR - -swift_extra_flags="" -if [[ "$TRIPLE" == "wasm32-unknown-wasip1-threads" ]]; then - swift_extra_flags="-Xcc -matomics -Xcc -mbulk-memory -Xcc -mthread-model -Xcc posix -Xcc -pthread -Xcc -ftls-model=local-exec" -fi - -cmake -G Ninja \ - -DCMAKE_BUILD_TYPE="Release" \ - -DCMAKE_SYSROOT="$WASI_SYSROOT_PATH" \ - -DCMAKE_STAGING_PREFIX="$DESTINATION_TOOLCHAIN/usr" \ - -DCMAKE_Swift_COMPILER="$SWIFT_BIN_DIR/swiftc" \ - -DCMAKE_TOOLCHAIN_FILE="$SCHEME_BUILD_PATH/toolchain-wasi.cmake" \ - -DTRIPLE="$TRIPLE" \ - -DLLVM_BIN="$LLVM_BIN_DIR" \ - -DCLANG_BIN="$CLANG_BIN_DIR" \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_Swift_COMPILER_FORCED=ON \ - -DCMAKE_Swift_FLAGS="-sdk $WASI_SYSROOT_PATH -resource-dir $DESTINATION_TOOLCHAIN/usr/lib/swift_static $swift_extra_flags" \ - -DSWIFT_FOUNDATION_PATH=$DESTINATION_TOOLCHAIN/usr/lib/swift_static/wasi/wasm32 \ - "${SOURCE_PATH}/swift-corelibs-xctest" - -ninja -v -ninja -v install diff --git a/schemes/main/build/run-test.sh b/schemes/main/build/run-test.sh index aa92d6f8..701e6c60 100755 --- a/schemes/main/build/run-test.sh +++ b/schemes/main/build/run-test.sh @@ -2,9 +2,4 @@ set -euxo pipefail -SOURCE_PATH="$(cd "$(dirname "$0")/../../../.." && pwd)" -BUILD_DIR="$SOURCE_PATH/build" -TARGET_BUILD_ROOT="$BUILD_DIR/WebAssembly" - -HOST_SUFFIX=$(find "$TARGET_BUILD_ROOT" -name "wasmstdlib-*" -exec basename {} \; | sed 's/wasmstdlib-//') -env "PATH=$TARGET_BUILD_ROOT/llvm-$HOST_SUFFIX/bin:$BUILD_DIR/llvm-tools/bin:$PATH" "LIT_FILTER_OUT=(IRGen/|embedded/)" ninja check-swift-wasi-wasm32-custom -C "$TARGET_BUILD_ROOT/wasmstdlib-$HOST_SUFFIX" +echo "Skipping test because we now run tests in build-script" diff --git a/schemes/main/manifest.json b/schemes/main/manifest.json index f851a218..8867f7f9 100644 --- a/schemes/main/manifest.json +++ b/schemes/main/manifest.json @@ -1,9 +1,6 @@ { "base-tag": "swift-DEVELOPMENT-SNAPSHOT-2025-07-23-a", "icu4c": [], - "libxml2": [ - "https://github.com/swiftwasm/libxml2-wasm/releases/download/2.0.0/libxml2-wasm32-unknown-wasi.tar.gz", - "https://github.com/swiftwasm/libxml2-wasm/releases/download/2.0.0/libxml2-wasm32-unknown-wasip1-threads.tar.gz" - ], + "libxml2": [], "swift-org-download-channel": "development" } diff --git a/tools/build/package-toolchain b/tools/build/package-toolchain index 9fe19402..4110c2aa 100755 --- a/tools/build/package-toolchain +++ b/tools/build/package-toolchain @@ -51,7 +51,10 @@ def copy_icu_libs(scheme, build_sdk_path, dist_toolchain_path, target_triple): shutil.copy(os.path.join(icu_lib_dir, lib), dest_path) -def copy_libxml2_libs(build_sdk_path, dist_toolchain_path, target_triple): +def copy_libxml2_libs(scheme, build_sdk_path, dist_toolchain_path, target_triple): + if scheme not in ["release-6.0", "release-6.1"]: + # Recent channels install libxml2 in wasi-sysroot by default + return import shutil lib_dir = os.path.join(build_sdk_path, f"libxml2-{target_triple}", 'lib') if not os.path.exists(lib_dir): @@ -83,7 +86,7 @@ class PackageSwiftSDKAction(Action): def run(self): build_sdk_path = os.path.join('..', 'build-sdk') copy_icu_libs(self.options.scheme, build_sdk_path, self.target_toolchain_path, self.target_triple) - copy_libxml2_libs(build_sdk_path, self.target_toolchain_path, self.target_triple) + copy_libxml2_libs(self.options.scheme, build_sdk_path, self.target_toolchain_path, self.target_triple) self.make_swift_sdk( self.target_toolchain_path, self.wasi_sysroot_path, @@ -207,8 +210,12 @@ def main(): toolchain_channel = derive_toolchain_channel(options.scheme) triples = [ - ["wasm32-unknown-wasi", "wasm32-wasi"], + ["wasm32-unknown-wasip1", "wasm32-wasip1"], ] + if options.scheme in ["release-6.0", "release-6.1"]: + triples = [ + ["wasm32-unknown-wasi", "wasm32-wasi"], + ] if options.scheme not in ["release-6.0"]: triples += [ ["wasm32-unknown-wasip1-threads", "wasm32-wasip1-threads"] diff --git a/tools/build/run-e2e-test b/tools/build/run-e2e-test index 1615d9f2..018d0a6f 100755 --- a/tools/build/run-e2e-test +++ b/tools/build/run-e2e-test @@ -11,7 +11,7 @@ from build_support.actions import derive_options_from_args, REPO_ROOT, DownloadB def main(): parser = argparse.ArgumentParser( description='Run e2e integration tests for Swift SDK') - parser.add_argument('--swift-sdk', default='wasm32-unknown-wasi', + parser.add_argument('--swift-sdk', default='wasm32-unknown-wasip1', help='Swift SDK identifier (default: wasm32-unknown-wasi)') parser.add_argument('args', nargs=argparse.REMAINDER, help='Extra arguments to pass to lit.py') From 4708c2af2ada74dff1b3224e86a6f13150eccedc Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sun, 27 Jul 2025 08:56:03 +0000 Subject: [PATCH 5/7] Fix libxml2 installation and default Swift SDK selection in E2E tests --- schemes/main/build/build-target-toolchain.sh | 6 ++++ tools/build/run-e2e-test | 31 +++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/schemes/main/build/build-target-toolchain.sh b/schemes/main/build/build-target-toolchain.sh index e79748e9..77f0c901 100755 --- a/schemes/main/build/build-target-toolchain.sh +++ b/schemes/main/build/build-target-toolchain.sh @@ -50,6 +50,12 @@ build_target_toolchain() { rm -rf "$TRIPLE_DESTDIR/usr/lib/swift_static/clang/lib/$COMPILER_RT_OS_DIR" # XXX: Is this the right way to install compiler-rt? cp -R "$TARGET_BUILD_ROOT/wasi-sysroot/$CLANG_MULTIARCH_TRIPLE/lib/$COMPILER_RT_OS_DIR" "$TRIPLE_DESTDIR/usr/lib/swift_static/clang/lib/$COMPILER_RT_OS_DIR" + + # FIXME: The following is a workaround for the issue with wrong libxml2.a path + local LIBXML2_PATH_TO_FIX="$TARGET_BUILD_ROOT/wasi-sysroot/$CLANG_MULTIARCH_TRIPLE/lib/libxml2.a" + if [[ -f "$LIBXML2_PATH_TO_FIX" ]]; then + cp "$LIBXML2_PATH_TO_FIX" "$TRIPLE_DESTDIR/usr/lib/swift_static/wasi/libxml2.a" + fi } main() { diff --git a/tools/build/run-e2e-test b/tools/build/run-e2e-test index 018d0a6f..374c48c6 100755 --- a/tools/build/run-e2e-test +++ b/tools/build/run-e2e-test @@ -4,15 +4,37 @@ import sys import argparse import pathlib import subprocess -import os from build_support.actions import derive_options_from_args, REPO_ROOT, DownloadBaseSnapshotAction +def default_swift_sdk(options, swift_sdks_path: pathlib.Path): + """Return the default target triple based on the scheme.""" + import json + for artifact_bundle in swift_sdks_path.iterdir(): + if not artifact_bundle.is_dir() or not artifact_bundle.name.endswith('.artifactbundle'): + # Skip if not ending with .artifactbundle + continue + info_json = artifact_bundle / 'info.json' + if not info_json.is_file(): + continue + try: + with info_json.open('r') as f: + info = json.load(f) + + sdk_ids = info.get('artifacts', {}).keys() + for sdk_id in sdk_ids: + if sdk_id.endswith("wasm32-unknown-wasip1") or sdk_id.endswith("wasm32-unknown-wasi"): + return sdk_id + except Exception as e: + print(f"Error reading {info_json}: {e}", file=sys.stderr) + continue + + def main(): parser = argparse.ArgumentParser( description='Run e2e integration tests for Swift SDK') - parser.add_argument('--swift-sdk', default='wasm32-unknown-wasip1', - help='Swift SDK identifier (default: wasm32-unknown-wasi)') + parser.add_argument('--swift-sdk', default=None, + help='Swift SDK identifier (default: wasm32-unknown-wasip1)') parser.add_argument('args', nargs=argparse.REMAINDER, help='Extra arguments to pass to lit.py') options = derive_options_from_args(sys.argv[1:], parser) @@ -25,11 +47,12 @@ def main(): base_toolchain_path = DownloadBaseSnapshotAction.toolchain_path(options) lit_py_path = repo_path.parent / "llvm-project" / "llvm" / "utils" / "lit" / "lit.py" swift_sdks_path = repo_path.parent / "swift-sdk-generator" / "Bundles" + swift_sdk = options.swift_sdk or default_swift_sdk(options, swift_sdks_path) # Build lit command lit_cmd = [ str(lit_py_path), - "--param", f"swift-sdk={options.swift_sdk}", + "--param", f"swift-sdk={swift_sdk}", "--param", f"scheme={options.scheme}", "--param", f"base-toolchain-path={base_toolchain_path}", "--param", f"swift-sdks-path={swift_sdks_path}", From a51e06ccd670aa03230e10fd1e31d416e0f61e23 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sun, 27 Jul 2025 10:44:18 +0000 Subject: [PATCH 6/7] Fix artifactbundle upload for wasm32-unknown-wasip1 --- .github/workflows/build-toolchain.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-toolchain.yml b/.github/workflows/build-toolchain.yml index e0b05aed..c1aaa9f1 100644 --- a/.github/workflows/build-toolchain.yml +++ b/.github/workflows/build-toolchain.yml @@ -185,8 +185,12 @@ jobs: ${{ matrix.container }} docker exec swiftwasm-ci-buildbot /bin/bash -lc 'env; cp -r /source/* /home/build-user/; ./swiftwasm-build/tools/build/ci.sh ${{ matrix.scheme }}' + - name: Extract installable archive from Docker container (wasm32-unknown-wasip1) + if: ${{ matrix.container != null && matrix.scheme != 'release-6.0' && matrix.scheme != 'release-6.1' }} + run: | + docker cp swiftwasm-ci-buildbot:/home/build-user/swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-wasm32-unknown-wasip1.artifactbundle.zip . - name: Extract installable archive from Docker container (wasm32-unknown-wasi) - if: ${{ matrix.container != null }} + if: ${{ matrix.container != null && !(matrix.scheme != 'release-6.0' && matrix.scheme != 'release-6.1') }} run: | docker cp swiftwasm-ci-buildbot:/home/build-user/swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-wasm32-unknown-wasi.artifactbundle.zip . # release-6.0 doesn't have wasm32-unknown-wasip1-threads SDK support yet @@ -214,6 +218,11 @@ jobs: path: build-cache key: ${{ steps.cache_key.outputs.SCCACHE_KEY }} + - name: Upload Swift SDK artifact bundle (wasm32-unknown-wasip1) + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.scheme }}-wasm32-unknown-wasip1-artifactbundle + path: swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-wasm32-unknown-wasip1.artifactbundle.zip - name: Upload Swift SDK artifact bundle (wasm32-unknown-wasi) uses: actions/upload-artifact@v4 with: From cd9d85d03a4f8f1d9af230cb42017e18056f24e5 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sun, 27 Jul 2025 11:55:02 +0000 Subject: [PATCH 7/7] un-XFAIL: test/swift-sdk/foundation/ProcessInfo.swift --- test/swift-sdk/foundation/ProcessInfo.swift | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/swift-sdk/foundation/ProcessInfo.swift b/test/swift-sdk/foundation/ProcessInfo.swift index 74053dd1..1b1ef371 100644 --- a/test/swift-sdk/foundation/ProcessInfo.swift +++ b/test/swift-sdk/foundation/ProcessInfo.swift @@ -1,8 +1,6 @@ // RUN: %{target_simple_swift_build} // RUN: %{wasm_run} --dir %t.dir::/tmp %t.dir/.build/debug/Check.wasm | %{FileCheck} %s // REQUIRES: FileCheck && scheme=main -// XFAIL: scheme=main -// https://github.com/swiftlang/swift-foundation/pull/1447 import Foundation