Switch to LLVM-based MinGW toolchain (#14329)

.github/workflows/windows.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Install compiler
         run: |
           sudo apt-get update && sudo apt-get install -y gettext
-          sudo ./util/buildbot/download_toolchain.sh i686 /usr
+          sudo ./util/buildbot/download_toolchain.sh /usr
 
       - name: Build
         run: |
@@ -56,7 +56,7 @@ jobs:
       - name: Install compiler
         run: |
           sudo apt-get update && sudo apt-get install -y gettext
-          sudo ./util/buildbot/download_toolchain.sh x86_64 /usr
+          sudo ./util/buildbot/download_toolchain.sh /usr
 
       - name: Build
         run: |

src/main.cpp

@@ -67,12 +67,14 @@ extern "C" {
 #error Minetest cannot be built without exceptions or RTTI
 #endif
 
-#if defined(__MINGW32__) && !defined(__MINGW64__) && !defined(__clang__) && \
-	(__GNUC__ < 11 || (__GNUC__ == 11 && __GNUC_MINOR__ < 1))
-// see e.g. https://github.com/minetest/minetest/issues/10137
-#warning ==================================
-#warning 32-bit MinGW gcc before 11.1 has known issues with crashes on thread exit, you should upgrade.
-#warning ==================================
+#if defined(__MINGW32__) && !defined(__clang__)
+// see https://github.com/minetest/minetest/issues/14140 or
+// https://github.com/minetest/minetest/issues/10137 for one of the various issues we had
+#error ==================================
+#error MinGW gcc has a broken TLS implementation and is not supported for building \
+	Minetest. Look at testTLS() in test_threading.cpp and see for yourself. \
+	Please use a clang-based compiler or alternatively MSVC.
+#error ==================================
 #endif
 
 #define DEBUGFILE "debug.txt"
@@ -435,6 +437,13 @@ static void print_version(std::ostream &os)
 	os << "Using " << LUAJIT_VERSION << std::endl;
 #else
 	os << "Using " << LUA_RELEASE << std::endl;
+#endif
+#if defined(__clang__)
+	os << "Built by Clang " << __clang_major__ << "." << __clang_minor__ << std::endl;
+#elif defined(__GNUC__)
+	os << "Built by GCC " << __GNUC__ << "." << __GNUC_MINOR__ << std::endl;
+#elif defined(_MSC_VER)
+	os << "Built by MSVC " << (_MSC_VER / 100) << "." << (_MSC_VER % 100) << std::endl;
 #endif
 	os << "Running on " << porting::get_sysinfo() << std::endl;
 	os << g_build_info << std::endl;

util/buildbot/buildwin32.sh

@@ -13,42 +13,36 @@ libdir=$builddir/libs
 
 source $topdir/common.sh
 
-# Test which win32 compiler is present
-command -v i686-w64-mingw32-gcc >/dev/null &&
-	compiler=i686-w64-mingw32-gcc
-command -v i686-w64-mingw32-gcc-posix >/dev/null &&
-	compiler=i686-w64-mingw32-gcc-posix
+compiler=i686-w64-mingw32-clang
 
-if [ -z "$compiler" ]; then
-	echo "Unable to determine which MinGW compiler to use"
+if ! command -v "$compiler" >/dev/null; then
+	echo "Unable to find $compiler"
 	exit 1
 fi
-toolchain_file=$topdir/toolchain_${compiler/-gcc/}.cmake
+toolchain_file=$topdir/toolchain_${compiler%-*}.cmake
 echo "Using $toolchain_file"
 
-find_runtime_dlls i686-w64-mingw32
+find_runtime_dlls ${compiler%-*}
 
 # Get stuff
 irrlicht_version=$(cat $topdir/../../misc/irrlichtmt_tag.txt)
 
 mkdir -p $libdir
 
-# 'ucrt' just points to rebuilt versions after a toolchain change
-
 cd $libdir
 libhost="http://minetest.kitsunemimi.pw"
-download "https://github.com/minetest/irrlicht/releases/download/$irrlicht_version/win32.zip" irrlicht-$irrlicht_version-win32.zip
-download "$libhost/zlib-$zlib_version-win32.zip"
-download "$libhost/ucrt/zstd-$zstd_version-win32.zip"
-download "$libhost/ucrt/libogg-$ogg_version-win32.zip"
-download "$libhost/ucrt/libvorbis-$vorbis_version-win32.zip"
-download "$libhost/curl-$curl_version-win32.zip"
+download "https://github.com/minetest/irrlicht/releases/download/$irrlicht_version/win32-llvm.zip" irrlicht-$irrlicht_version-win32.zip
+download "$libhost/llvm/zlib-$zlib_version-win32.zip"
+download "$libhost/llvm/zstd-$zstd_version-win32.zip"
+download "$libhost/llvm/libogg-$ogg_version-win32.zip"
+download "$libhost/llvm/libvorbis-$vorbis_version-win32.zip"
+download "$libhost/llvm/curl-$curl_version-win32.zip"
 download "$libhost/ucrt/gettext-$gettext_version-win32.zip"
-download "$libhost/freetype-$freetype_version-win32.zip"
-download "$libhost/sqlite3-$sqlite3_version-win32.zip"
-download "$libhost/luajit-$luajit_version-win32.zip"
-download "$libhost/ucrt/libleveldb-$leveldb_version-win32.zip" leveldb-$leveldb_version-win32.zip
-download "$libhost/openal-soft-$openal_version-win32.zip"
+download "$libhost/llvm/freetype-$freetype_version-win32.zip"
+download "$libhost/llvm/sqlite3-$sqlite3_version-win32.zip"
+download "$libhost/llvm/luajit-$luajit_version-win32.zip"
+download "$libhost/llvm/libleveldb-$leveldb_version-win32.zip"
+download "$libhost/llvm/openal-soft-$openal_version-win32.zip"
 
 # Set source dir, downloading Minetest as needed
 get_sources

util/buildbot/buildwin64.sh

@@ -13,42 +13,36 @@ libdir=$builddir/libs
 
 source $topdir/common.sh
 
-# Test which win64 compiler is present
-command -v x86_64-w64-mingw32-gcc >/dev/null &&
-	compiler=x86_64-w64-mingw32-gcc
-command -v x86_64-w64-mingw32-gcc-posix >/dev/null &&
-	compiler=x86_64-w64-mingw32-gcc-posix
+compiler=x86_64-w64-mingw32-clang
 
-if [ -z "$compiler" ]; then
-	echo "Unable to determine which MinGW compiler to use"
+if ! command -v "$compiler" >/dev/null; then
+	echo "Unable to find $compiler"
 	exit 1
 fi
-toolchain_file=$topdir/toolchain_${compiler/-gcc/}.cmake
+toolchain_file=$topdir/toolchain_${compiler%-*}.cmake
 echo "Using $toolchain_file"
 
-find_runtime_dlls x86_64-w64-mingw32
+find_runtime_dlls ${compiler%-*}
 
 # Get stuff
 irrlicht_version=$(cat $topdir/../../misc/irrlichtmt_tag.txt)
 
 mkdir -p $libdir
 
-# 'ucrt' just points to rebuilt versions after a toolchain change
-
 cd $libdir
 libhost="http://minetest.kitsunemimi.pw"
-download "https://github.com/minetest/irrlicht/releases/download/$irrlicht_version/win64.zip" irrlicht-$irrlicht_version-win64.zip
-download "$libhost/zlib-$zlib_version-win64.zip"
-download "$libhost/ucrt/zstd-$zstd_version-win64.zip"
-download "$libhost/ucrt/libogg-$ogg_version-win64.zip"
-download "$libhost/ucrt/libvorbis-$vorbis_version-win64.zip"
-download "$libhost/curl-$curl_version-win64.zip"
+download "https://github.com/minetest/irrlicht/releases/download/$irrlicht_version/win64-llvm.zip" irrlicht-$irrlicht_version-win64.zip
+download "$libhost/llvm/zlib-$zlib_version-win64.zip"
+download "$libhost/llvm/zstd-$zstd_version-win64.zip"
+download "$libhost/llvm/libogg-$ogg_version-win64.zip"
+download "$libhost/llvm/libvorbis-$vorbis_version-win64.zip"
+download "$libhost/llvm/curl-$curl_version-win64.zip"
 download "$libhost/ucrt/gettext-$gettext_version-win64.zip"
-download "$libhost/freetype-$freetype_version-win64.zip"
-download "$libhost/sqlite3-$sqlite3_version-win64.zip"
-download "$libhost/luajit-$luajit_version-win64.zip"
-download "$libhost/ucrt/libleveldb-$leveldb_version-win64.zip" leveldb-$leveldb_version-win64.zip
-download "$libhost/openal-soft-$openal_version-win64.zip"
+download "$libhost/llvm/freetype-$freetype_version-win64.zip"
+download "$libhost/llvm/sqlite3-$sqlite3_version-win64.zip"
+download "$libhost/llvm/luajit-$luajit_version-win64.zip"
+download "$libhost/llvm/libleveldb-$leveldb_version-win64.zip"
+download "$libhost/llvm/openal-soft-$openal_version-win64.zip"
 
 # Set source dir, downloading Minetest as needed
 get_sources

util/buildbot/common.sh

@@ -9,9 +9,9 @@ curl_version=8.5.0
 gettext_version=0.20.2
 freetype_version=2.13.2
 sqlite3_version=3.44.2
-luajit_version=20231211
+luajit_version=20240125
 leveldb_version=1.23
-zlib_version=1.3
+zlib_version=1.3.1
 zstd_version=1.5.5
 
 download () {
@@ -48,11 +48,11 @@ get_sources () {
 # sets $runtime_dlls
 find_runtime_dlls () {
 	local triple=$1
-	# Try to find runtime DLLs in various paths (varies by distribution, sigh)
+	# Try to find runtime DLLs in various paths
 	local tmp=$(dirname "$(command -v $compiler)")/..
 	runtime_dlls=
-	for name in lib{gcc_,stdc++-,winpthread-}'*'.dll; do
-		for dir in $tmp/$triple/{bin,lib} $tmp/lib/gcc/$triple/*; do
+	for name in lib{clang_rt,c++,unwind,winpthread-}'*'.dll; do
+		for dir in $tmp/$triple/{bin,lib}; do
 			[ -d "$dir" ] || continue
 			local file=$(echo $dir/$name)
 			[ -f "$file" ] && { runtime_dlls+="$file;"; break; }
@@ -65,14 +65,20 @@ find_runtime_dlls () {
 	fi
 }
 
-add_cmake_libs () {
-	local irr_dlls=$(echo $libdir/irrlicht/lib/*.dll | tr ' ' ';')
-	local vorbis_dlls=$(echo $libdir/libvorbis/bin/libvorbis{,file}-*.dll | tr ' ' ';')
-	local gettext_dlls=$(echo $libdir/gettext/bin/lib{intl,iconv}-*.dll | tr ' ' ';')
+_dlls () {
+	for f in "$@"; do
+		if [ ! -e "$f" ]; then
+			echo "Could not find $f" >&2
+		elif [[ -f "$f" && "$f" == *.dll ]]; then
+			printf '%s;' "$f"
+		fi
+	done
+}
 
+add_cmake_libs () {
 	cmake_args+=(
 		-DCMAKE_PREFIX_PATH=$libdir/irrlicht
-		-DIRRLICHT_DLL="$irr_dlls"
+		-DIRRLICHT_DLL="$(_dlls $libdir/irrlicht/lib/*)"
 
 		-DZLIB_INCLUDE_DIR=$libdir/zlib/include
 		-DZLIB_LIBRARY=$libdir/zlib/lib/libz.dll.a
@@ -87,37 +93,37 @@ add_cmake_libs () {
 
 		-DOGG_INCLUDE_DIR=$libdir/libogg/include
 		-DOGG_LIBRARY=$libdir/libogg/lib/libogg.dll.a
-		-DOGG_DLL=$libdir/libogg/bin/libogg-0.dll
+		-DOGG_DLL="$(_dlls $libdir/libogg/bin/*)"
 
 		-DVORBIS_INCLUDE_DIR=$libdir/libvorbis/include
 		-DVORBIS_LIBRARY=$libdir/libvorbis/lib/libvorbis.dll.a
-		-DVORBIS_DLL="$vorbis_dlls"
+		-DVORBIS_DLL="$(_dlls $libdir/libvorbis/bin/libvorbis{,file}[-.]*)"
 		-DVORBISFILE_LIBRARY=$libdir/libvorbis/lib/libvorbisfile.dll.a
 
 		-DOPENAL_INCLUDE_DIR=$libdir/openal/include/AL
 		-DOPENAL_LIBRARY=$libdir/openal/lib/libOpenAL32.dll.a
 		-DOPENAL_DLL=$libdir/openal/bin/OpenAL32.dll
 
-		-DCURL_DLL=$libdir/curl/bin/libcurl-4.dll
+		-DCURL_DLL="$(_dlls $libdir/curl/bin/libcurl*)"
 		-DCURL_INCLUDE_DIR=$libdir/curl/include
 		-DCURL_LIBRARY=$libdir/curl/lib/libcurl.dll.a
 
 		-DGETTEXT_MSGFMT=`command -v msgfmt`
-		-DGETTEXT_DLL="$gettext_dlls"
+		-DGETTEXT_DLL="$(_dlls $libdir/gettext/bin/lib{intl,iconv}*)"
 		-DGETTEXT_INCLUDE_DIR=$libdir/gettext/include
 		-DGETTEXT_LIBRARY=$libdir/gettext/lib/libintl.dll.a
 
 		-DFREETYPE_INCLUDE_DIR_freetype2=$libdir/freetype/include/freetype2
 		-DFREETYPE_INCLUDE_DIR_ft2build=$libdir/freetype/include/freetype2
 		-DFREETYPE_LIBRARY=$libdir/freetype/lib/libfreetype.dll.a
-		-DFREETYPE_DLL=$libdir/freetype/bin/libfreetype-6.dll
+		-DFREETYPE_DLL="$(_dlls $libdir/freetype/bin/libfreetype*)"
 
 		-DSQLITE3_INCLUDE_DIR=$libdir/sqlite3/include
 		-DSQLITE3_LIBRARY=$libdir/sqlite3/lib/libsqlite3.dll.a
-		-DSQLITE3_DLL=$libdir/sqlite3/bin/libsqlite3-0.dll
+		-DSQLITE3_DLL="$(_dlls $libdir/sqlite3/bin/libsqlite*)"
 
-		-DLEVELDB_INCLUDE_DIR=$libdir/leveldb/include
-		-DLEVELDB_LIBRARY=$libdir/leveldb/lib/libleveldb.dll.a
-		-DLEVELDB_DLL=$libdir/leveldb/bin/libleveldb.dll
+		-DLEVELDB_INCLUDE_DIR=$libdir/libleveldb/include
+		-DLEVELDB_LIBRARY=$libdir/libleveldb/lib/libleveldb.dll.a
+		-DLEVELDB_DLL=$libdir/libleveldb/bin/libleveldb.dll
 	)
 }

util/buildbot/download_toolchain.sh

@@ -1,18 +1,17 @@
 #!/bin/bash
 set -e
 topdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-if [[ -z "$1" || -z "$2" ]]; then
-	echo "Usage: $0 <i686 | x86_64> <dest path>"
+if [ -z "$1" ]; then
+	echo "Usage: $0 <dest path>"
 	exit 1
 fi
 
-# our current toolchain:
-# binutils 2.41 + GCC 13.2.0 + Mingw-w64 11.0.1 with UCRT enabled and winpthreads support
-# built from source on Ubuntu 22.04, so should work on any similarily up-to-date distro
-ver=13.2.0
-os=ubuntu22.04
-name="mingw-w64-${1}_${ver}_${os}.tar.xz"
-wget "http://minetest.kitsunemimi.pw/$name" -O "$name"
+# key points:
+# * Clang + LLD + libc++ instead of GCC + binutils + stdc++
+# * Mingw-w64 with UCRT enabled and winpthreads support
+# why are we avoiding GCC? -> Thread Local Storage (TLS) is totally broken
+name=llvm-mingw-20231128-ucrt-ubuntu-20.04-x86_64.tar.xz
+wget "https://github.com/mstorsjo/llvm-mingw/releases/download/20231128/$name" -O "$name"
 sha256sum -w -c <(grep -F "$name" "$topdir/sha256sums.txt")
-tar -xaf "$name" -C "$2"
+tar -xaf "$name" -C "$1" --strip-components=1
 rm -f "$name"

util/buildbot/sha256sums.txt

@@ -1,26 +1,25 @@
-c6759580175dee6c3673bb0544f0aca855f76b415b441db2b949fe9e2af4e6ee  curl-8.5.0-win32.zip
-a99ebdccad524f3738fa3a6a9d1dcabc39cb668f97790638d77b4bb96ea3edca  curl-8.5.0-win64.zip
-d70c9886526513a2c8a7962815fb425f296ab934239470a03ea350944169a7ac  freetype-2.13.2-win32.zip
-06aa20c71724e832874baa296d047aa866db2c336e26aa49e4faa72e559414a6  freetype-2.13.2-win64.zip
+753dc38c591e078eae6a0a6b25f69826211256f444f3691a170670d8a12988f9  curl-8.5.0-win32.zip
+aa86abc3eb054d74d5fe15996f281cf84230a61b4ab7b3a702ab7dbb71e1203f  curl-8.5.0-win64.zip
+3e9d7bbca953b96dfd65acc28baaa87e8881aab29809ba03b9c9aefe3d071189  freetype-2.13.2-win32.zip
+acf901e93aedbcfa92eb3aab1def252676af845b1747ca5c3e7c5866576168cc  freetype-2.13.2-win64.zip
 41b10766de2773f0f0851fde16b363024685e0397f4bb2e5cd2a7be196960a01  gettext-0.20.2-win32.zip
 1ceed167ff16fea944f76ab6ea2969160c71a67419259b17c9c523e7a01eb883  gettext-0.20.2-win64.zip
-faa09cd5c3790fdad3fcb43ba1d5c5102492a7b88b9301de59cecb92af37c162  irrlicht-1.9.0mt14-win32.zip
-a5724e994f417b04e43bc9dd54adf6f32050e39b2cc1a79acdf361a9d3972ece  irrlicht-1.9.0mt14-win64.zip
-6d49348215916ff355187fec808d0847450f70e45fe2719f45af9eb61c047358  leveldb-1.23-win32.zip
-30c680277320bdda130b238d0adc30c3c59e7522dc008d677893ebfaea22f28b  leveldb-1.23-win64.zip
-d58b67954f3f552fba5e315ed476c38b230d0cf53445fe07dc733e72f8ba7dc2  libogg-1.3.5-win32.zip
-2083cceb79b648cd500afe8b71c56170481f309cb6abd950195cdd13570e03dd  libogg-1.3.5-win64.zip
-1ce1c71e1dfdd99f47c93614a521ec0797d8fb55fb3fc07b67937ea7c6f76cca  libvorbis-1.3.7-win32.zip
-1c6fe4aa1c38079f2917e17e6b5acd7505331236c426e3b86054efccec6cee1c  libvorbis-1.3.7-win64.zip
-1c9b9580d869ee57b8c30a083d0e9a737310c1bb5e376b05fba483bac99eb2e1  luajit-20231211-win32.zip
-3b42a31887ad7901f83a9f5b5faa4745ce95c7e95a7d8fd569d603fc95573ea5  luajit-20231211-win64.zip
-9f0cfab8ca089d48be7a59f85d5fd5648f18f54c91d7ac6c31b281ba5e90852a  mingw-w64-i686_13.2.0_ubuntu22.04.tar.xz
-93bc9f04d43a023358d1ae2b76dec42d3d79baecd452402ee9fb3ee21945fdfe  mingw-w64-x86_64_13.2.0_ubuntu22.04.tar.xz
-34c4e6826a8e0dc4f7a49f7e4e4d54676f89a20fe781bad876795b857f7c5395  openal-soft-1.23.1-win32.zip
-4b9c9a7f42aa8f7e6d26347ad61c55a32a2b11e4f02b8562542bcd132b0c7115  openal-soft-1.23.1-win64.zip
-082dfee313c7e29e48ff798503acb286a4542c315618d5d3b33fc2bbed4170a5  sqlite3-3.44.2-win32.zip
-e8fda50178f1371c52f85ac19a0998d797ad6b2439f1da87c49a1f44ba33649c  sqlite3-3.44.2-win64.zip
-3c5abd40e9492c834651d995db6bbf0f57a7579d091d2d03110293b95e9b039a  zlib-1.3-win32.zip
-f63d9a38c2ee56fa1e95a486224c274412cb5b3275734c1da53b0a68a7e8c654  zlib-1.3-win64.zip
-7508d714dbed4e1b1340cfb13ea77ef631746dad99ac97434171f2f4dd64d94b  zstd-1.5.5-win32.zip
-30353afddb459974c4e90c4eb3fbf975951247cf310fa5f40208806e275776fa  zstd-1.5.5-win64.zip
+6b37e3f734c48caa7ba8756abed63c5171bf54f4f679a1dc3869ec2bd94be48d  irrlicht-1.9.0mt14-win32.zip
+5aca6d711881d3d6a4d5da4b53b2cf0e73a0c113b8776730bbfa3f3bc4aebea7  irrlicht-1.9.0mt14-win64.zip
+f54e9a577e2db47ed28f4a01e74181d2c607627c551d30f48263e01b59e84f67  libleveldb-1.23-win32.zip
+2f039848a4e6c05a2347fe5a7fa63c430dd08d1bc88235645a863c859e14f5f8  libleveldb-1.23-win64.zip
+0df94afb8efa361cceb132ecf9491720afbc45ba844a7b1c94607295829b53ca  libogg-1.3.5-win32.zip
+5c4acb4c99429a04b5e69650719b2eb17616bf52837d2372a0f859952eebce48  libogg-1.3.5-win64.zip
+456ece10a2be4247b27fbe88f88ddd54aae604736a6b76ba9a922b602fe40f40  libvorbis-1.3.7-win32.zip
+57f4db02da00556895bb63fafa6e46b5f7dac87c25fde27af4315f56a1aa7a86  libvorbis-1.3.7-win64.zip
+0f21ff3be90311092fe32e0e30878ef3ae9d9437b8d9ac25ef279e0d84e9bb8e  llvm-mingw-20231128-ucrt-ubuntu-20.04-x86_64.tar.xz
+da6ad10632cf172992158e9ea0977a87914b5d5de93a972c3430b6a412237556  luajit-20240125-win32.zip
+2b1dabe83d478b398cf9226d96de7fa62c973365c4aea70d27ba5782fb49d2d0  luajit-20240125-win64.zip
+e2443451fe5c2066eb564c64b8a1762738a88b7fd749c8b5907fed45c785497b  openal-soft-1.23.1-win32.zip
+cb041445a118469caefbad2647470cb8571c8337bce2adc07634011ab5625417  openal-soft-1.23.1-win64.zip
+326701086a0ed66e09a9f3ec4d971654c13b6bd79cfdd079c947ecdcd6409525  sqlite3-3.44.2-win32.zip
+b2d474e3625f8f426b6cc5c0ecac831a1de46f7d1027bf4a9f6267b0b0411d42  sqlite3-3.44.2-win64.zip
+8af10515d57dbfee5d2106cd66cafa2adeb4270d4c6047ccbf7e8b5d2d50681c  zlib-1.3.1-win32.zip
+ad43f5d23052590c65633530743e5d622cc76b33c109072e6fd7b487aff56bca  zlib-1.3.1-win64.zip
+3564dabbe17ec4ecae1fb9a78fe48d9f7c71e2b1166456f6ee27e52fd9c84357  zstd-1.5.5-win32.zip
+e61b1f327ce2d836d1f8ca00c40ac77d3ab5309135851c98229bbdf82b060ae5  zstd-1.5.5-win64.zip

util/buildbot/toolchain_i686-w64-mingw32.cmake

@@ -2,8 +2,8 @@
 SET(CMAKE_SYSTEM_NAME Windows)
 
 # which compilers to use for C and C++
-SET(CMAKE_C_COMPILER i686-w64-mingw32-gcc)
-SET(CMAKE_CXX_COMPILER i686-w64-mingw32-g++)
+SET(CMAKE_C_COMPILER i686-w64-mingw32-clang)
+SET(CMAKE_CXX_COMPILER i686-w64-mingw32-clang++)
 SET(CMAKE_RC_COMPILER i686-w64-mingw32-windres)
 
 # here is the target environment located