partial merge bitcoin#19683: Pin clang search paths for darwin host

Skipped bitcoin@8033110
kwvg · Jul 20, 2021 · 173e6ca · 173e6ca
1 parent 5f71d0b
commit 173e6ca
Show file tree

Hide file tree

Showing 3 changed files with 137 additions and 34 deletions.
diff --git a/depends/Makefile b/depends/Makefile
@@ -89,6 +89,8 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null)
 
 ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+# Make sure that cache is invalidated when switching between system and
+# depends-managed, pinned clang
 build_id_string+=system_clang
 $(host_arch)_$(host_os)_id_string+=system_clang
 endif
@@ -113,12 +115,6 @@ $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain)
 
 include funcs.mk
 
-binutils_path=$($($(host_arch)_$(host_os)_native_binutils)_prefixbin)
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin)
-else
-toolchain_path=
-endif
 final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in)
 final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))
 $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
@@ -129,15 +125,39 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
 	$(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); )
 	$(AT)touch $@
 
+# $PATH is not preserved between ./configure and make by convention. Its
+# modification and overriding at ./configure time is (as I understand it)
+# supposed to be captured by the AC_{PROG_{,OBJ}CXX,PATH_{PROG,TOOL}} macros,
+# which will expand the program names to their full absolute paths. The notable
+# exception is command line overriding: ./configure CC=clang, which skips the
+# program name expansion step, and works because the user implicitly indicates
+# with CC=clang that clang will be available in $PATH at all times, and is most
+# likely part of the user's system.
+#
+# Therefore, when we "seed the autoconf cache"/"override well-known program
+# vars" by setting AR=<blah> in our config.site, either one of two things needs
+# to be true for the build system to work correctly:
+#
+#   1. If we refer to the program by name (e.g. AR=riscv64-gnu-linux-ar), the
+#      tool needs to be available in $PATH at all times.
+#
+#   2. If the tool is _**not**_ expected to be available in $PATH at all times
+#      (such as is the case for our native_cctools binutils tools), it needs to
+#      be referred to by its absolute path, such as would be output by the
+#      AC_PATH_{PROG,TOOL} macros.
+#
+# Minor note: it is also okay to refer to tools by their absolute path even if
+# we expect them to be available in $PATH at all times, more specificity does
+# not hurt.
 $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id)
 	$(AT)@mkdir -p $(@D)
 	$(AT)sed -e 's|@HOST@|$(host)|' \
-            -e 's|@CC@|$(toolchain_path)$(host_CC)|' \
-            -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \
-            -e 's|@AR@|$(binutils_path)$(host_AR)|' \
-            -e 's|@RANLIB@|$(binutils_path)$(host_RANLIB)|' \
-            -e 's|@NM@|$(binutils_path)$(host_NM)|' \
-            -e 's|@STRIP@|$(binutils_path)$(host_STRIP)|' \
+            -e 's|@CC@|$(host_CC)|' \
+            -e 's|@CXX@|$(host_CXX)|' \
+            -e 's|@AR@|$(host_AR)|' \
+            -e 's|@RANLIB@|$(host_RANLIB)|' \
+            -e 's|@NM@|$(host_NM)|' \
+            -e 's|@STRIP@|$(host_STRIP)|' \
             -e 's|@build_os@|$(build_os)|' \
             -e 's|@host_os@|$(host_os)|' \
             -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \

diff --git a/depends/funcs.mk b/depends/funcs.mk
@@ -1,17 +1,23 @@
 define int_vars
 #Set defaults for vars which may be overridden per-package
-$(1)_cc=$($($(1)_type)_CC)
-$(1)_cxx=$($($(1)_type)_CXX)
-$(1)_objc=$($($(1)_type)_OBJC)
-$(1)_objcxx=$($($(1)_type)_OBJCXX)
-$(1)_ar=$($($(1)_type)_AR)
-$(1)_ranlib=$($($(1)_type)_RANLIB)
-$(1)_libtool=$($($(1)_type)_LIBTOOL)
-$(1)_nm=$($($(1)_type)_NM)
-$(1)_cflags=$($($(1)_type)_CFLAGS) $($($(1)_type)_$(release_type)_CFLAGS)
-$(1)_cxxflags=$($($(1)_type)_CXXFLAGS) $($($(1)_type)_$(release_type)_CXXFLAGS)
-$(1)_ldflags=$($($(1)_type)_LDFLAGS) $($($(1)_type)_$(release_type)_LDFLAGS) -L$($($(1)_type)_prefix)/lib
-$(1)_cppflags=$($($(1)_type)_CPPFLAGS) $($($(1)_type)_$(release_type)_CPPFLAGS) -I$($($(1)_type)_prefix)/include
+$(1)_cc=$$($$($(1)_type)_CC)
+$(1)_cxx=$$($$($(1)_type)_CXX)
+$(1)_objc=$$($$($(1)_type)_OBJC)
+$(1)_objcxx=$$($$($(1)_type)_OBJCXX)
+$(1)_ar=$$($$($(1)_type)_AR)
+$(1)_ranlib=$$($$($(1)_type)_RANLIB)
+$(1)_libtool=$$($$($(1)_type)_LIBTOOL)
+$(1)_nm=$$($$($(1)_type)_NM)
+$(1)_cflags=$$($$($(1)_type)_CFLAGS) \
+            $$($$($(1)_type)_$$(release_type)_CFLAGS)
+$(1)_cxxflags=$$($$($(1)_type)_CXXFLAGS) \
+              $$($$($(1)_type)_$$(release_type)_CXXFLAGS)
+$(1)_ldflags=$$($$($(1)_type)_LDFLAGS) \
+             $$($$($(1)_type)_$$(release_type)_LDFLAGS) \
+             -L$$($($(1)_type)_prefix)/lib
+$(1)_cppflags=$$($$($(1)_type)_CPPFLAGS) \
+              $$($$($(1)_type)_$$(release_type)_CPPFLAGS) \
+              -I$$($$($(1)_type)_prefix)/include
 $(1)_recipe_hash:=
 endef
 

diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk
@@ -6,6 +6,49 @@ LD64_VERSION=530
 
 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
 
+darwin_native_binutils=native_cctools
+
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang
+# from llvm.org
+
+# The native_cctools package is what provides clang when FORCE_USE_SYSTEM_CLANG
+# is empty
+darwin_native_toolchain=native_cctools
+
+clang_prog=$(build_prefix)/bin/clang
+clangxx_prog=$(clang_prog)++
+
+clang_resource_dir=$(build_prefix)/lib/clang/$(native_cctools_clang_version)
+else
+# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's
+# system
+
+darwin_native_toolchain=
+
+# We can't just use $(shell command -v clang) because GNU Make handles builtins
+# in a special way and doesn't know that `command` is a POSIX-standard builtin
+# prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90.
+# At the time of writing, GNU Make v4.2.1 is still being used in supported
+# distro releases.
+#
+# Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html
+clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang")
+clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++")
+
+clang_resource_dir=$(shell clang -print-resource-dir)
+endif
+
+cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL
+
+# Make-only lowercase function
+lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
+
+# For well-known tools provided by cctools, make sure that their well-known
+# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG}
+# would.
+$(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(host)-$(call lc,$(TOOL))))
+
 # Flag explanations:
 #
 #     -mlinker-version
@@ -18,7 +61,7 @@ OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-
 #         Explicitly point to our binaries (e.g. cctools) so that they are
 #         ensured to be found and preferred over other possibilities.
 #
-#     -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1
+#     -stdlib=libc++ -nostdinc++ -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1
 #
 #         Forces clang to use the libc++ headers from our SDK and completely
 #         forget about the libc++ headers from the standard directories
@@ -28,8 +71,49 @@ OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-
 #         https://reviews.llvm.org/D64089, we should use that instead. Read the
 #         differential summary there for more details.
 #
-darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin
-darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1
+#     -Xclang -*system<path_a> \
+#     -Xclang -*system<path_b> \
+#     -Xclang -*system<path_c> ...
+#
+#         Adds path_a, path_b, and path_c to the bottom of clang's list of
+#         include search paths. This is used to explicitly specify the list of
+#         system include search paths and its ordering, rather than rely on
+#         clang's autodetection routine. This routine has been shown to:
+#             1. Fail to pickup libc++ headers in $SYSROOT/usr/include/c++/v1
+#                when clang was built manually (see: https://github.com/bitcoin/bitcoin/pull/17919#issuecomment-656785034)
+#             2. Fail to pickup C headers in $SYSROOT/usr/include when
+#                C_INCLUDE_DIRS was specified at configure time (see: https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9)
+#
+#         Talking directly to cc1 with -Xclang here grants us access to specify
+#         more granular categories for these system include search paths, and we
+#         can use the correct categories that these search paths would have been
+#         placed in if the autodetection routine had worked correctly. (see:
+#         https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9#the-treatment)
+#
+#         Furthermore, it places these search paths after any "non-Xclang"
+#         specified search paths. This prevents any additional clang options or
+#         environment variables from coming after or in between these system
+#         include search paths, as that would be wrong in general but would also
+#         break #include_next's.
+#
+darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
+              -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
+              -u LIBRARY_PATH \
+            $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
+              -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
+              --sysroot=$(OSX_SDK) \
+              -Xclang -internal-externc-isystem$(clang_resource_dir)/include \
+              -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
+darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
+               -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
+               -u LIBRARY_PATH \
+             $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
+               -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
+               --sysroot=$(OSX_SDK) \
+               -stdlib=libc++ -nostdinc++ \
+               -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 \
+               -Xclang -internal-externc-isystem$(clang_resource_dir)/include \
+               -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
 
 darwin_CFLAGS=-pipe
 darwin_CXXFLAGS=$(darwin_CFLAGS)
@@ -39,10 +123,3 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)
 
 darwin_debug_CFLAGS=-O1
 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS)
-
-darwin_native_binutils=native_cctools
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-darwin_native_toolchain=native_cctools
-else
-darwin_native_toolchain=
-endif