Skip to content

Commit

Permalink
Yet more improvements to native library builds (#2150)
Browse files Browse the repository at this point in the history
* Native/CMake: Configure all binutils tools in toolchain files for good measure.

* Native/Vkd3d: Fix source repository URL to match submodule.

* Native/Vkd3d: Clean up build script and build for linux-arm and linux-arm64.

* Native: Don't install pkg-config explicitly in workflows.

It's already preinstalled.

* Native/wgpu-native: Add a clarifying comment to the workflow.

* Native/Vkd3d: Use x86_64-w64-mingw32-widl as IDL compiler.

Should avoid having to install all of Wine.

* Native/Vkd3d: Temporarily patch out the Vulkan library check in configure.ac.

* Native/Vkd3d: Perform configure.ac patching before invoking autogen.sh.

* New binaries for Vkd3d on Linux 6.5.0-1018-azure #19~22.04.2-Ubuntu SMP Thu Mar 21 16:45:46 UTC 2024

* Revert "Native/Vkd3d: Fix source repository URL to match submodule."

This reverts commit d5949f0.

---------

Co-authored-by: The Silk.NET Automaton <9011267+dotnet-bot@users.noreply.github.com>
  • Loading branch information
alexrp and dotnet-bot committed Apr 23, 2024
1 parent f3087ca commit 0e8e039
Show file tree
Hide file tree
Showing 15 changed files with 147 additions and 75 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/glfw.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ jobs:
sudo dpkg --add-architecture armhf
sudo apt update
sudo apt install -y gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf
sudo apt install -y extra-cmake-modules pkg-config wayland-protocols
sudo apt install -y extra-cmake-modules wayland-protocols
for arch in amd64 arm64 armhf; do
sudo apt install -y libwayland-dev:$arch libxcursor-dev:$arch libxi-dev:$arch
sudo apt install -y libxinerama-dev:$arch libxrandr-dev:$arch libxkbcommon-dev:$arch
Expand Down
39 changes: 25 additions & 14 deletions .github/workflows/vkd3d.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ on:
- build/submodules/vkd3d
- build/submodules/SPIRV-Tools
- build/submodules/Vulkan-Headers
- build/submodules/vkd3d-no-ms-abi.patch
- "build/cmake/*"
- build/nuke/Native/Core.cs
- build/nuke/Native/Vkd3d.cs
- .github/workflows/vkd3d.yml
Expand All @@ -21,22 +21,33 @@ jobs:
fail-fast: false
matrix:
env:
- os: ubuntu-latest
- os: ubuntu-22.04
name: Linux
nuke_invoke: ./build.sh
extras: |
sudo apt-get update
wget http://mirrors.kernel.org/ubuntu/pool/main/v/vulkan-loader/libvulkan1_1.3.239.0-1_amd64.deb
sudo apt install ./libvulkan1_1.3.239.0-1_amd64.deb
wget http://mirrors.kernel.org/ubuntu/pool/main/v/vulkan-loader/libvulkan-dev_1.3.239.0-1_amd64.deb
sudo apt install ./libvulkan-dev_1.3.239.0-1_amd64.deb
sudo apt-get install -y xorg-dev spirv-headers wine64-development-tools cmake build-essential
wget http://mirrors.kernel.org/ubuntu/pool/universe/s/spirv-tools/spirv-tools_2020.1-2_amd64.deb
sudo apt install ./spirv-tools_2020.1-2_amd64.deb
# We need to adjust APT sources for multiarch. Use the ones corresponding to
# Ubuntu 22.04 with appropriate `arch` values. ports.ubuntu.com is required
# for armhf and arm64.
sudo tee /etc/apt/sources.list << EOF
deb [arch=amd64] http://archive.ubuntu.com/ubuntu jammy main universe
deb [arch=amd64] http://archive.ubuntu.com/ubuntu jammy-backports main universe
deb [arch=amd64] http://archive.ubuntu.com/ubuntu jammy-security main universe
deb [arch=amd64] http://archive.ubuntu.com/ubuntu jammy-updates main universe
deb [arch=arm64,armhf] http://ports.ubuntu.com jammy main universe
deb [arch=arm64,armhf] http://ports.ubuntu.com jammy-backports main universe
deb [arch=arm64,armhf] http://ports.ubuntu.com jammy-security main universe
deb [arch=arm64,armhf] http://ports.ubuntu.com jammy-updates main universe
deb [arch=amd64] http://security.ubuntu.com/ubuntu jammy main universe
deb [arch=amd64] http://security.ubuntu.com/ubuntu jammy-backports main universe
deb [arch=amd64] http://security.ubuntu.com/ubuntu jammy-security main universe
deb [arch=amd64] http://security.ubuntu.com/ubuntu jammy-updates main universe
EOF
sudo dpkg --add-architecture arm64
sudo dpkg --add-architecture armhf
sudo apt update
sudo apt install -y gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf
sudo apt install -y g++-aarch64-linux-gnu g++-arm-linux-gnueabihf
sudo apt install -y mingw-w64-tools
name: ${{ matrix.env.name }} Build
runs-on: ${{ matrix.env.os }}
steps:
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/vulkan-loader.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ jobs:
sudo dpkg --add-architecture armhf
sudo apt update
sudo apt install -y gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf
sudo apt install -y pkg-config
for arch in amd64 arm64 armhf; do
sudo apt install -y libx11-xcb-dev:$arch libxkbcommon-dev:$arch libwayland-dev:$arch libxrandr-dev:$arch
done
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/wgpu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ jobs:
sudo dpkg --add-architecture armhf
sudo apt update
sudo apt install -y gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf
# These are needed because Rust tooling is weird about cross-linking.
sudo apt install -y libgcc-11-dev libgcc-11-dev-arm64-cross libgcc-11-dev-armhf-cross
- os: windows-2022
name: Windows
Expand Down
16 changes: 14 additions & 2 deletions build/cmake/aarch64-linux-gnu.cmake
Original file line number Diff line number Diff line change
@@ -1,12 +1,24 @@
set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR aarch64)
set(CMAKE_SYSTEM_NAME "Linux")
set(CMAKE_SYSTEM_PROCESSOR "aarch64")

set(CMAKE_ASM_COMPILER "/usr/bin/aarch64-linux-gnu-as")
set(CMAKE_C_COMPILER "/usr/bin/aarch64-linux-gnu-gcc")
set(CMAKE_CXX_COMPILER "/usr/bin/aarch64-linux-gnu-g++")

set(CMAKE_ADDR2LINE "/usr/bin/aarch64-linux-gnu-addr2line")
set(CMAKE_AR "/usr/bin/aarch64-linux-gnu-ar")
set(CMAKE_LINKER "/usr/bin/aarch64-linux-gnu-ld")
set(CMAKE_NM "/usr/bin/aarch64-linux-gnu-nm")
set(CMAKE_OBJCOPY "/usr/bin/aarch64-linux-gnu-objcopy")
set(CMAKE_OBJDUMP "/usr/bin/aarch64-linux-gnu-objdump")
set(CMAKE_RANLIB "/usr/bin/aarch64-linux-gnu-ranlib")
set(CMAKE_READELF "/usr/bin/aarch64-linux-gnu-readelf")
set(CMAKE_STRIP "/usr/bin/aarch64-linux-gnu-strip")

set(PKG_CONFIG_EXECUTABLE "/usr/bin/pkg-config")
set(ENV{PKG_CONFIG_LIBDIR} "/usr/lib/aarch64-linux-gnu/pkgconfig")
set(ENV{PKG_CONFIG_PATH} "/usr/lib/pkgconfig:/usr/share/pkgconfig")

set(CMAKE_FIND_ROOT_PATH "/usr/aarch64-linux-gnu")
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)
Expand Down
16 changes: 14 additions & 2 deletions build/cmake/aarch64-w64-mingw32.cmake
Original file line number Diff line number Diff line change
@@ -1,10 +1,22 @@
set(CMAKE_SYSTEM_NAME Windows)
set(CMAKE_SYSTEM_PROCESSOR aarch64)
set(CMAKE_SYSTEM_NAME "Windows")
set(CMAKE_SYSTEM_PROCESSOR "aarch64")

set(CMAKE_ASM_COMPILER "/opt/llvm-mingw-msvcrt/bin/aarch64-w64-mingw32-as")
set(CMAKE_C_COMPILER "/opt/llvm-mingw-msvcrt/bin/aarch64-w64-mingw32-gcc")
set(CMAKE_CXX_COMPILER "/opt/llvm-mingw-msvcrt/bin/aarch64-w64-mingw32-g++")
set(CMAKE_RC_COMPILER "/opt/llvm-mingw-msvcrt/bin/aarch64-w64-mingw32-windres")

set(CMAKE_ADDR2LINE "/usr/bin/aarch64-w64-mingw32-addr2line")
set(CMAKE_AR "/usr/bin/aarch64-w64-mingw32-ar")
set(CMAKE_DLLTOOL "/usr/bin/aarch64-w64-mingw32-dlltool")
set(CMAKE_LINKER "/opt/llvm-mingw-msvcrt/bin/aarch64-w64-mingw32-ld")
set(CMAKE_NM "/usr/bin/aarch64-w64-mingw32-nm")
set(CMAKE_OBJCOPY "/usr/bin/aarch64-w64-mingw32-objcopy")
set(CMAKE_OBJDUMP "/usr/bin/aarch64-w64-mingw32-objdump")
set(CMAKE_RANLIB "/usr/bin/aarch64-w64-mingw32-ranlib")
set(CMAKE_READELF "/usr/bin/aarch64-w64-mingw32-readelf")
set(CMAKE_STRIP "/usr/bin/aarch64-w64-mingw32-strip")

set(CMAKE_FIND_ROOT_PATH "/opt/llvm-mingw-msvcrt/aarch64-w64-mingw32")
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM BOTH)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
Expand Down
16 changes: 14 additions & 2 deletions build/cmake/arm-linux-gnueabihf.cmake
Original file line number Diff line number Diff line change
@@ -1,12 +1,24 @@
set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR armv7l)
set(CMAKE_SYSTEM_NAME "Linux")
set(CMAKE_SYSTEM_PROCESSOR "armv7l")

set(CMAKE_ASM_COMPILER "/usr/bin/arm-linux-gnueabihf-as")
set(CMAKE_C_COMPILER "/usr/bin/arm-linux-gnueabihf-gcc")
set(CMAKE_CXX_COMPILER "/usr/bin/arm-linux-gnueabihf-g++")

set(CMAKE_ADDR2LINE "/usr/bin/arm-linux-gnueabihf-addr2line")
set(CMAKE_AR "/usr/bin/arm-linux-gnueabihf-ar")
set(CMAKE_LINKER "/usr/bin/arm-linux-gnueabihf-ld")
set(CMAKE_NM "/usr/bin/arm-linux-gnueabihf-nm")
set(CMAKE_OBJCOPY "/usr/bin/arm-linux-gnueabihf-objcopy")
set(CMAKE_OBJDUMP "/usr/bin/arm-linux-gnueabihf-objdump")
set(CMAKE_RANLIB "/usr/bin/arm-linux-gnueabihf-ranlib")
set(CMAKE_READELF "/usr/bin/arm-linux-gnueabihf-readelf")
set(CMAKE_STRIP "/usr/bin/arm-linux-gnueabihf-strip")

set(PKG_CONFIG_EXECUTABLE "/usr/bin/pkg-config")
set(ENV{PKG_CONFIG_LIBDIR} "/usr/lib/arm-linux-gnueabihf/pkgconfig")
set(ENV{PKG_CONFIG_PATH} "/usr/lib/pkgconfig:/usr/share/pkgconfig")

set(CMAKE_FIND_ROOT_PATH "/usr/arm-linux-gnueabihf")
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)
Expand Down
16 changes: 14 additions & 2 deletions build/cmake/x86_64-linux-gnu.cmake
Original file line number Diff line number Diff line change
@@ -1,12 +1,24 @@
set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR x86_64)
set(CMAKE_SYSTEM_NAME "Linux")
set(CMAKE_SYSTEM_PROCESSOR "x86_64")

set(CMAKE_ASM_COMPILER "/usr/bin/x86_64-linux-gnu-as")
set(CMAKE_C_COMPILER "/usr/bin/x86_64-linux-gnu-gcc")
set(CMAKE_CXX_COMPILER "/usr/bin/x86_64-linux-gnu-g++")

set(CMAKE_ADDR2LINE "/usr/bin/x86_64-linux-gnu-addr2line")
set(CMAKE_AR "/usr/bin/x86_64-linux-gnu-ar")
set(CMAKE_LINKER "/usr/bin/x86_64-linux-gnu-ld")
set(CMAKE_NM "/usr/bin/x86_64-linux-gnu-nm")
set(CMAKE_OBJCOPY "/usr/bin/x86_64-linux-gnu-objcopy")
set(CMAKE_OBJDUMP "/usr/bin/x86_64-linux-gnu-objdump")
set(CMAKE_RANLIB "/usr/bin/x86_64-linux-gnu-ranlib")
set(CMAKE_READELF "/usr/bin/x86_64-linux-gnu-readelf")
set(CMAKE_STRIP "/usr/bin/x86_64-linux-gnu-strip")

set(PKG_CONFIG_EXECUTABLE "/usr/bin/pkg-config")
set(ENV{PKG_CONFIG_LIBDIR} "/usr/lib/x86_64-linux-gnu/pkgconfig")
set(ENV{PKG_CONFIG_PATH} "/usr/lib/pkgconfig:/usr/share/pkgconfig")

set(CMAKE_FIND_ROOT_PATH "/usr/x86_64-linux-gnu")
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)
Expand Down
111 changes: 63 additions & 48 deletions build/nuke/Native/Vkd3d.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
partial class Build {
AbsolutePath Vkd3dPath => RootDirectory / "build" / "submodules" / "vkd3d";
AbsolutePath SPIRVToolsPath => RootDirectory / "build" / "submodules" / "SPIRV-Tools";
AbsolutePath VulkanHeadersPath => RootDirectory / "build" / "submodules" / "Vulkan-Headers";

Target Vkd3d => CommonTarget
(
Expand All @@ -37,64 +38,78 @@ partial class Build {
(
() =>
{
if(!RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
throw new PlatformNotSupportedException("This task only runs on Linux!");
}
var runtimes = RootDirectory / "src" / "Native" / "Silk.NET.Vkd3d.Native" / "runtimes";
var vkd3dBuild = SPIRVToolsPath / "build";
EnsureCleanDirectory(vkd3dBuild);
InheritedShell($"./git-sync-deps", SPIRVToolsPath / "utils").AssertZeroExitCode();
{ //SPIRV-Tools
//Sync the external deps
InheritedShell($"./git-sync-deps", SPIRVToolsPath / "utils").AssertZeroExitCode();
// Get rid of the Vulkan library check since we will not be needing it.
File.WriteAllText(
Vkd3dPath / "configure.ac",
File.ReadAllText(Vkd3dPath / "configure.ac")
.Replace("[VKD3D_CHECK_VULKAN]", "[]"));
//Make the build scripts, with shared libs enabled, and tests disabled
InheritedShell($"cmake .. -DBUILD_SHARED_LIBS=1 -DSPIRV_SKIP_TESTS=ON", vkd3dBuild).AssertZeroExitCode();
InheritedShell($"./autogen.sh", Vkd3dPath).AssertZeroExitCode();
//Compile SPIRV-Tools
InheritedShell($"cmake --build . --config Release {JobsArg}", vkd3dBuild).AssertZeroExitCode();
//Run `strip -g` on the shared library file to remove debug info and shrink it from ~30mb down to only ~5.5mb
InheritedShell($"strip -g libSPIRV-Tools-shared.so", vkd3dBuild / "source").AssertZeroExitCode();
var spirvToolsBuild = SPIRVToolsPath / "build";
var vkd3dBuild = Vkd3dPath / "build";
var vkd3dShaderCompiler = RootDirectory / "src" / "Microsoft" / "Vkd3dCompiler";
var runtimes = RootDirectory / "src" / "Native" / "Silk.NET.Vkd3d.Native" / "runtimes";
//Copy the resulting SPIRV-Tools shared library to the runtimes folder
CopyFile(vkd3dBuild / "source" / "libSPIRV-Tools-shared.so", runtimes / "linux-x64" / "native" / "libSPIRV-Tools-shared.so", FileExistsPolicy.Overwrite);
foreach (var (triple, rid) in new[]
{
("x86_64-linux-gnu", "linux-x64"),
("arm-linux-gnueabihf", "linux-arm"),
("aarch64-linux-gnu", "linux-arm64"),
})
{
// SPIRV-Tools
{
EnsureCleanDirectory(spirvToolsBuild);
InheritedShell($"cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DSPIRV_SKIP_EXECUTABLES=ON {GetCMakeToolchainFlag(triple)}", spirvToolsBuild).AssertZeroExitCode();
InheritedShell($"cmake --build . --config Release{JobsArg}", spirvToolsBuild).AssertZeroExitCode();
InheritedShell($"{triple}-strip --strip-unneeded source/libSPIRV-Tools-shared.so", spirvToolsBuild).AssertZeroExitCode();
CopyFile(spirvToolsBuild / "source" / "libSPIRV-Tools-shared.so", runtimes / rid / "native" / "libSPIRV-Tools-shared.so", FileExistsPolicy.Overwrite);
CopyFile(spirvToolsBuild / "source" / "libSPIRV-Tools-shared.so", vkd3dShaderCompiler / "libSPIRV-Tools-shared.so", FileExistsPolicy.Overwrite);
}
// Vkd3d
{
EnsureCleanDirectory(vkd3dBuild);
// We only need to configure Vkd3d; we include its sources directly.
InheritedShell($"./configure --prefix={vkd3dBuild} --disable-static --host={triple} --disable-tests --disable-doxygen-doc --with-spirv-tools WIDL=x86_64-w64-mingw32-widl CPPFLAGS=\"-I {VulkanHeadersPath / "include"} -I {SPIRVToolsPath / "external" / "spirv-headers" / "include"} -DNDEBUG -DVKD3D_NO_DEBUG_MESSAGES -DVKD3D_NO_TRACE_MESSAGES\" PKG_CONFIG_PATH={spirvToolsBuild}", Vkd3dPath).AssertZeroExitCode();
// Invoke widl for some headers that we need.
foreach (var name in new[]
{
"vkd3d_d3d12",
"vkd3d_d3d12sdklayers",
"vkd3d_d3dcommon",
"vkd3d_dxgibase",
"vkd3d_dxgiformat",
})
{
InheritedShell($"make include/{name}.h", Vkd3dPath).AssertZeroExitCode();
}
}
// d3dcompile_vkd3d
{
// Note that the glibc version should match the one used to build SPIRV-Tools. Since we
// currently build on Ubuntu 22.04, that's glibc 2.35.
InheritedShell($"zig build -Doptimize=ReleaseFast -Dtarget={triple}.2.35 --verbose", vkd3dShaderCompiler).AssertZeroExitCode();
CopyFile(vkd3dShaderCompiler / "zig-out" / "lib" / "libd3dcompile_vkd3d.so", runtimes / rid / "native" / "libd3dcompile_vkd3d.so", FileExistsPolicy.Overwrite);
}
}
{ //Vkd3d
var dest = Vkd3dPath / "dest";
var @out = Vkd3dPath / "build";
EnsureCleanDirectory(@out);
EnsureCleanDirectory(dest);
//Run autogen
InheritedShell($"./autogen.sh", Vkd3dPath).AssertZeroExitCode();
//Run configure to make a non-debug build, with no trace messages, with a prefix of /usr and with spirv-tools
InheritedShell($"./configure CPPFLAGS=\"-DNDEBUG -DVKD3D_NO_TRACE_MESSAGES -fPIC\" --prefix=/usr --with-spirv-tools --disable-doxygen-pdf", Vkd3dPath).AssertZeroExitCode();
//Build vkd3d
InheritedShell($"make {JobsArg}", Vkd3dPath).AssertZeroExitCode();
//Install vkd3d to the dest folder
InheritedShell($"make DESTDIR=\"{Vkd3dPath.ToString().TrimEnd('/')}/dest\" install", Vkd3dPath).AssertZeroExitCode();
var vkd3dShaderCompiler = RootDirectory / "src" / "Microsoft" / "Vkd3dCompiler";
//Copy libvkd3d-shader.a
CopyFile(@dest / "usr" / "lib" / "libvkd3d-shader.a", vkd3dShaderCompiler / "libvkd3d-shader.a");
//Copy libvkd3d-shader.la
CopyFile(@dest / "usr" / "lib" / "libvkd3d-shader.la", vkd3dShaderCompiler / "libvkd3d-shader.la");
//Copy libSPIRV-Tools-shared.so
CopyFile(vkd3dBuild / "source" / "libSPIRV-Tools-shared.so", vkd3dShaderCompiler / "libSPIRV-Tools-shared.so");
//Build the shader compiler
InheritedShell($"zig build -Doptimize=ReleaseFast -Dtarget=x86_64-linux-gnu --verbose", vkd3dShaderCompiler).AssertZeroExitCode();
//Copy the resulting shader compiler to the native output
CopyFile(vkd3dShaderCompiler / "zig-out" / "lib" / "libd3dcompile_vkd3d.so", runtimes / "linux-x64" / "native" / "libd3dcompile_vkd3d.so", FileExistsPolicy.Overwrite);
}
Git("checkout HEAD configure.ac", Vkd3dPath);
PrUpdatedNativeBinary("Vkd3d");
}
Expand Down
4 changes: 1 addition & 3 deletions src/Microsoft/Vkd3dCompiler/build.zig
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,7 @@ pub fn build(b: *std.Build) void {

vkd3d_compiler.linkLibC();

// vkd3d_compiler.linkSystemLibrary2("vkd3d-shader", .{ .preferred_link_mode = .static });

//Since we statically link with vkd3d-shader, we need to dynamically link against SPIRV-Tools ourselves
//Since we include vkd3d-shader sources, we need to dynamically link against SPIRV-Tools ourselves
vkd3d_compiler.linkSystemLibrary("SPIRV-Tools-shared");

vkd3d_compiler.addIncludePath(.{ .path = vulkan_header_folder ++ "include/" });
Expand Down
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.

0 comments on commit 0e8e039

Please sign in to comment.