Cross-compile llbuild and the Swift package manager for Android with …

…the trunk toolchain on the CI
finagolfin · Nov 23, 2021 · 5d6a511 · 5d6a511
1 parent 69bc222
commit 5d6a511
Show file tree

Hide file tree

Showing 3 changed files with 105 additions and 3 deletions.
diff --git a/.github/workflows/sdks.yml b/.github/workflows/sdks.yml
@@ -87,7 +87,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/swift-${{ matrix.version }}-android-${{ matrix.arch }}-*-sdk.tar.xz
-          key: ${{ matrix.version }}-${{ steps.version.outputs.latest }}-${{ matrix.arch }}-ndk-23b-sdk
+          key: ${{ matrix.version }}-${{ steps.version.outputs.latest }}-${{ matrix.arch }}-latest-sdk
       - name: Clone
         uses: actions/checkout@v2
         with:
@@ -100,6 +100,9 @@ jobs:
         run: |
           cd sdk-config
           sudo apt install ninja-build
+          if [ ${{ matrix.version }} = 'trunk' ]; then
+            export BUILD_SWIFT_PM=1
+          fi
           swift get-packages-and-swift-source.swift
 
           SDK_NAME=$(ls | grep swift-${{ matrix.version }}-android-${{ matrix.arch }})
@@ -108,7 +111,12 @@ jobs:
           tar xf ~/$SWIFT_TAG-ubuntu20.04.tar.gz
 
           if [ ${{ matrix.version }} = 'trunk' ]; then
-            EXTRA_FLAGS="--skip-early-swift-driver"
+            EXTRA_FLAGS="-b -p --install-llbuild --skip-early-swift-driver"
+
+            wget -q https://patch-diff.githubusercontent.com/raw/apple/swift/pull/36917.diff
+            cd swift
+            patch -p1 < ../36917.diff
+            cd ..
           fi
 
           ./swift/utils/build-script -RA --skip-build-cmark --build-llvm=0 --android --android-ndk $ANDROID_NDK_LATEST_HOME --android-arch ${{ matrix.arch }} --android-api-level $ANDROID_API_LEVEL --android-icu-uc $SDK/usr/lib/libicuuc.so --android-icu-uc-include $SDK/usr/include --android-icu-i18n $SDK/usr/lib/libicui18n.so --android-icu-i18n-include $SDK/usr/include --android-icu-data $SDK/usr/lib/libicudata.so --build-swift-tools=0 --native-swift-tools-path=`pwd`/$SWIFT_TAG-ubuntu20.04/usr/bin --native-clang-tools-path=`pwd`/$SWIFT_TAG-ubuntu20.04/usr/bin --cross-compile-hosts=android-${{ matrix.arch }} --cross-compile-deps-path=$SDK --skip-local-build --xctest --install-swift --install-libdispatch --install-foundation --install-xctest --install-destdir=$SDK --swift-install-components='clang-resource-dir-symlink;license;stdlib;sdk-overlay' $EXTRA_FLAGS

diff --git a/get-packages-and-swift-source.swift b/get-packages-and-swift-source.swift
@@ -1,11 +1,20 @@
 import Foundation
 
 // The Termux packages to download and unpack
-let termuxPackages = ["libicu", "libicu-static", "libandroid-spawn", "libcurl", "libxml2"]
+var termuxPackages = ["libicu", "libicu-static", "libandroid-spawn", "libcurl", "libxml2"]
 
 let swiftRepos = ["llvm-project", "swift", "swift-corelibs-libdispatch",
                   "swift-corelibs-foundation", "swift-corelibs-xctest"]
 
+let extraSwiftRepos = ["swift-llbuild", "swift-package-manager", "swift-driver",
+                       "swift-tools-support-core", "swift-argument-parser", "swift-crypto",
+                       "Yams"]
+let renameRepos = ["swift-llbuild" : "llbuild", "swift-package-manager" : "swiftpm", "Yams" : "yams"]
+let repoTags = ["swift-argument-parser" : "1.0.1", "swift-crypto" : "1.1.5", "Yams" : "4.0.2"]
+if ProcessInfo.processInfo.environment["BUILD_SWIFT_PM"] != nil {
+  termuxPackages += ["ncurses", "libsqlite"]
+}
+
 guard let SWIFT_TAG = ProcessInfo.processInfo.environment["SWIFT_TAG"] else {
   fatalError("You must specify a SWIFT_TAG environment variable.")
 }
@@ -229,4 +238,16 @@ for repo in swiftRepos {
   }
 }
 
+if ProcessInfo.processInfo.environment["BUILD_SWIFT_PM"] != nil {
+  for repo in extraSwiftRepos {
+    let tag = repoTags[repo] ?? SWIFT_TAG
+    _ = runCommand("curl", with: ["-L", "-O",
+              "https://github.com/\(repo == "Yams" ? "jpsim" : "apple")/\(repo)/archive/refs/tags/\(tag).tar.gz"])
+    _ = runCommand("tar", with: ["xf", "\(tag).tar.gz"])
+    try fmd.moveItem(atPath: cwd.appendingPathComponent("\(repo)-\(tag)"),
+                     toPath: cwd.appendingPathComponent(renameRepos[repo] ?? repo))
+    try fmd.removeItem(atPath: cwd.appendingPathComponent("\(tag).tar.gz"))
+  }
+}
+
 _ = runCommand("git", with: ["apply", "swift-android-\(swiftBranch == "RELEASE" ? swiftVersion : (swiftVersion == "" ? "trunk" : "devel")).patch"])
diff --git a/swift-android-trunk.patch b/swift-android-trunk.patch
@@ -80,6 +80,19 @@ index a5fb6cb780..1f1db7513d 100755
          fi
      else
          local host_install_destdir="${INSTALL_DESTDIR}"
+diff --git a/swift/utils/swift_build_support/swift_build_support/products/product.py b/swift/utils/swift_build_support/swift_build_support/products/product.py
+index 405f4795461..3af29a27937 100644
+--- a/swift/utils/swift_build_support/swift_build_support/products/product.py
++++ b/swift/utils/swift_build_support/swift_build_support/products/product.py
+@@ -229,7 +229,7 @@ class Product(object):
+                 # This assumes that all hosts are merged to the lipo.
+                 return self.args.install_destdir
+             else:
+-                return '%s/%s' % (self.args.install_destdir, host_target)
++                return self.args.install_destdir
+         else:
+             return self.args.install_destdir
+
 diff --git a/swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt b/swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt
 index 016bf294..5c42986a 100644
 --- a/swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt
@@ -96,3 +109,63 @@ index 016bf294..5c42986a 100644
  endif()
 
 
+diff --git a/llbuild/CMakeLists.txt b/llbuild/CMakeLists.txt
+index 766e38e7..ad4d369f 100644
+--- a/llbuild/CMakeLists.txt
++++ b/llbuild/CMakeLists.txt
+@@ -158,6 +158,7 @@ elseif(MSVC)
+ else ()
+   add_compile_options(
+     # Enable additional Clang warnings.
++    "$<$<COMPILE_LANGUAGE:CXX>:-D_FORTIFY_SOURCE=0>"
+     "$<$<COMPILE_LANGUAGE:CXX>:-fno-rtti>"
+     "$<$<COMPILE_LANGUAGE:CXX>:-fno-exceptions>"
+     "$<$<COMPILE_LANGUAGE:CXX>:-Wbool-conversion>"
+diff --git a/llbuild/lib/llvm/Support/CMakeLists.txt b/llbuild/lib/llvm/Support/CMakeLists.txt
+index 9d5a714b..3f8f8c1d 100644
+--- a/llbuild/lib/llvm/Support/CMakeLists.txt
++++ b/llbuild/lib/llvm/Support/CMakeLists.txt
+@@ -66,4 +66,15 @@ endif()
+
+ if(${CMAKE_SYSTEM_NAME} MATCHES "Android|Darwin|Linux")
+   target_link_libraries(llvmSupport PRIVATE curses)
++if(${CMAKE_SYSTEM_NAME} MATCHES "Android")
++  target_link_libraries(llvmSupport PRIVATE android-spawn)
++  list(GET CMAKE_FIND_ROOT_PATH 0 SPAWN_DIR)
++  target_include_directories(llvmSupport
++    PUBLIC
++    ${SPAWN_DIR}/usr/include
++  )
++  target_link_directories(llvmSupport PUBLIC
++    ${SPAWN_DIR}/usr/lib
++  )
++endif()
+ endif()
+diff --git a/llbuild/unittests/CMakeLists.txt b/llbuild/unittests/CMakeLists.txt
+index 92a1ee38..52273afb 100644
+--- a/llbuild/unittests/CMakeLists.txt
++++ b/llbuild/unittests/CMakeLists.txt
+@@ -6,9 +6,9 @@ function(add_llbuild_unittest test_dirname)
+ endfunction()
+
+ add_subdirectory(Basic)
+-add_subdirectory(CAS)
++#add_subdirectory(CAS)
+ add_subdirectory(CAPI)
+ add_subdirectory(Core)
+ add_subdirectory(Evo)
+ add_subdirectory(BuildSystem)
+-add_subdirectory(Ninja)
++#add_subdirectory(Ninja)
+diff --git a/swiftpm/Utilities/bootstrap b/swiftpm/Utilities/bootstrap
+index 085056de..1ded1a90 100755
+--- a/swiftpm/Utilities/bootstrap
++++ b/swiftpm/Utilities/bootstrap
+@@ -764,6 +764,7 @@ def get_swiftpm_flags(args):
+     if 'ANDROID_DATA' in os.environ or (args.cross_compile_hosts and re.match(
+         'android-', args.cross_compile_hosts)):
+         build_flags.extend(["-Xswiftc", "-Xcc", "-Xswiftc", "-U_GNU_SOURCE"])
++        build_flags.extend(["-Xlinker", "-landroid-spawn"])
+
+     if platform.system() == "OpenBSD":
+         build_flags.extend(["-Xcc", "-I/usr/local/include"])