mk: Add IMPLICIT_DEPENDS and stricter check-shlibs.

This helps to expose potential issues where buildlink does not have the
full graph of dependencies due to inclusion guards, and libraries may be
buildlinked but are only build dependencies, causing runtime failures
that are not caught by the regular check-shlibs.

The situation was muddied slightly by our builds of GCC defaulting to
search for includes and libraries under LOCALBASE, but that has been
resolved by enhancing the custom specs file used by gcc-libs to remove
those paths.

This isn't ready for upstream yet as it may cause false positives due to
buildlink's incomplete graph, but we want to be alerted to potential
problems so that we can investigate and fix, and we would rather see
false positives than potentially let broken packages out undetected.

mk/buildlink3/bsd.buildlink3.mk

@@ -226,9 +226,46 @@ _SYS_VARS.bl3+=		${v}.${p}
 .  endfor
 .endfor
 
-# By default, every package receives a full dependency.
+# Set BUILDLINK_DEFAULT_DEPMETHOD for each package.  Iterate through the tree,
+# and if a package has explicitly set BUILDLINK_DEPMETHOD then use it as the
+# default for its dependencies, otherwise default to "full".
+#
+# If a package was previously marked "build" but is later encountered inside a
+# "full" stack then "full" must take precedence.
+#
+_stack_:=	bottom
+.for _pkg_ in ${BUILDLINK_TREE}
+.  if !${_pkg_:M-*}
+     # Some packages erroneously have multiple values for BUILDLINK_DEPMETHOD,
+     # so resort to matching and "full" takes precedence over "build".
+.    if ${BUILDLINK_DEPMETHOD.${_pkg_}:U:Mfull}
+_stack_:=	full ${_stack_}
+.    elif ${BUILDLINK_DEPMETHOD.${_pkg_}:U:Mbuild}
+_stack_:=	build ${_stack_}
+.    elif ${_stack_} == "bottom"
+_stack_:=	full ${_stack_}
+.    else
+_stack_:=	${_stack_:[1]} ${_stack_}
+.    endif
+.  else
+.    if !defined(BUILDLINK_DEFAULT_DEPMETHOD.${_pkg_:S/^-//}) || \
+	(${BUILDLINK_DEFAULT_DEPMETHOD.${_pkg_:S/^-//}} == "build" && \
+	 ${_stack_:[1]} == full)
+BUILDLINK_DEFAULT_DEPMETHOD.${_pkg_:S/^-//}:=	${_stack_:[1]}
+.    endif
+_stack_:=	${_stack_:[2..-1]}
+.  endif
+.endfor
+.if ${_stack_} != "bottom"
+.  error "The above loop through BUILDLINK_TREE failed to balance"
+.endif
+
+# Now set BUILDLINK_DEPMETHOD for each package based on its default.  Note that
+# we can't do this in one loop above because we need to detect when
+# BUILDLINK_DEPMETHOD has been set by a package and not as part of the loop.
+#
 .for _pkg_ in ${_BLNK_PACKAGES}
-BUILDLINK_DEPMETHOD.${_pkg_}?=	full
+BUILDLINK_DEPMETHOD.${_pkg_}?=	${BUILDLINK_DEFAULT_DEPMETHOD.${_pkg_}}
 .endfor
 
 # _BLNK_DEPENDS contains all of the elements of _BLNK_PACKAGES for which
@@ -253,10 +290,13 @@ _BLNK_DEPENDS+=	${_pkg_}
 # "build", and if any of that list is "full" then we use a full dependency
 # on <pkg>, otherwise we use a build dependency on <pkg>.
 #
-_BLNK_ADD_TO.DEPENDS=		# empty
-_BLNK_ADD_TO.BUILD_DEPENDS=	# empty
-_BLNK_ADD_TO.ABI_DEPENDS=	# empty
-_BLNK_ADD_TO.BUILD_ABI_DEPENDS=	# empty
+_BLNK_ADD_TO.DEPENDS=			# empty
+_BLNK_ADD_TO.BUILD_DEPENDS=		# empty
+_BLNK_ADD_TO.ABI_DEPENDS=		# empty
+_BLNK_ADD_TO.BUILD_ABI_DEPENDS=		# empty
+_BLNK_ADD_TO.IMPLICIT_DEPENDS=		# empty
+_BLNK_ADD_TO.IMPLICIT_BUILD_DEPENDS=	# empty
+
 .for _pkg_ in ${_BLNK_DEPENDS}
 .  if !empty(BUILDLINK_DEPMETHOD.${_pkg_}:Mfull)
 _BLNK_DEPMETHOD.${_pkg_}=	_BLNK_ADD_TO.DEPENDS
@@ -265,6 +305,23 @@ _BLNK_ABIMETHOD.${_pkg_}=	_BLNK_ADD_TO.ABI_DEPENDS
 _BLNK_DEPMETHOD.${_pkg_}=	_BLNK_ADD_TO.BUILD_DEPENDS
 _BLNK_ABIMETHOD.${_pkg_}=	_BLNK_ADD_TO.BUILD_ABI_DEPENDS
 .  endif
+.endfor
+
+.for _pkg_ in ${_BLNK_PACKAGES}
+.  if !${_BLNK_DEPENDS:M${_pkg_}} && \
+      !defined(IGNORE_PKG.${_pkg_}) && \
+      ${USE_BUILTIN.${_pkg_}:U:tl} == "no"
+.    if ${BUILDLINK_DEPMETHOD.${_pkg_}} == "full"
+_BLNK_DEPMETHOD.${_pkg_}=	_BLNK_ADD_TO.IMPLICIT_DEPENDS
+_BLNK_ABIMETHOD.${_pkg_}=	_BLNK_ADD_TO.IMPLICIT_ABI_DEPENDS
+.    elif ${BUILDLINK_DEPMETHOD.${_pkg_}} == "build"
+_BLNK_DEPMETHOD.${_pkg_}=	_BLNK_ADD_TO.IMPLICIT_BUILD_DEPENDS
+_BLNK_ABIMETHOD.${_pkg_}=	_BLNK_ADD_TO.IMPLICIT_BUILD_ABI_DEPENDS
+.    endif
+.  endif
+.endfor
+
+.for _pkg_ in ${_BLNK_PACKAGES:O:u}
 .  if defined(BUILDLINK_API_DEPENDS.${_pkg_}) && \
       defined(BUILDLINK_PKGSRCDIR.${_pkg_})
 .    for _depend_ in ${BUILDLINK_API_DEPENDS.${_pkg_}}
@@ -282,11 +339,12 @@ ${_BLNK_ABIMETHOD.${_pkg_}}+=	${_abi_}:${BUILDLINK_PKGSRCDIR.${_pkg_}}
 .    endfor
 .  endif
 .endfor
-.for _depmethod_ in DEPENDS BUILD_DEPENDS ABI_DEPENDS BUILD_ABI_DEPENDS
+
+.for _depmethod_ in DEPENDS BUILD_DEPENDS ABI_DEPENDS BUILD_ABI_DEPENDS IMPLICIT_DEPENDS IMPLICIT_BUILD_DEPENDS
 .  if !empty(_BLNK_ADD_TO.${_depmethod_})
 ${_depmethod_}+=	${_BLNK_ADD_TO.${_depmethod_}}
 .  endif
-.endfor	# _BLNK_DEPENDS
+.endfor
 
 ###
 ### BEGIN: after the barrier

mk/check/check-shlibs-elf.awk

@@ -84,20 +84,15 @@ function check_pkg(DSO, pkg, found) {
 	}
 	if (pkg == "")
 		return 0
-	found=0
 	while ((getline < depends_file) > 0) {
 		if ($3 == pkg) {
-			found=1
-			if ($1 != "full")
-				continue
-			close(depends_file)
-			return 0
+			if ($1 == "full" || $1 == "implicit-full") {
+				close(depends_file)
+				return 0
+			}
 		}
 	}
-	if (found)
-		print DSO ": " pkg " is not a runtime dependency"
-	# Not yet:
-	# print DSO ": " pkg " is not a dependency"
+	print DSO ": " pkg " is not a runtime dependency"
 	close(depends_file)
 }
 

mk/pkgformat/pkg/depends.mk

@@ -65,19 +65,21 @@ _LIST_DEPENDS_CMD=	\
 			" " \
 			" "${TOOL_DEPENDS:Q} \
 			" "${BUILD_DEPENDS:Q} \
-			" "${DEPENDS:Q}
+			" "${DEPENDS:Q} \
+			" "${IMPLICIT_BUILD_DEPENDS:Q} \
+			" "${IMPLICIT_DEPENDS:Q}
 
 _LIST_DEPENDS_CMD.bootstrap=	\
 	${PKGSRC_SETENV} AWK=${AWK:Q} PKG_ADMIN=${PKG_ADMIN:Q} \
 		PKGSRCDIR=${PKGSRCDIR:Q} PWD_CMD=${PWD_CMD:Q} SED=${SED:Q} \
 		${SH} ${PKGSRCDIR}/mk/pkgformat/pkg/list-dependencies \
-			" "${BOOTSTRAP_DEPENDS:Q} " " " " " " " "
+			" "${BOOTSTRAP_DEPENDS:Q} " " " " " " " " " " " "
 
 _LIST_DEPENDS_CMD.test=	\
 	${PKGSRC_SETENV} AWK=${AWK:Q} PKG_ADMIN=${PKG_ADMIN:Q} \
 		PKGSRCDIR=${PKGSRCDIR:Q} PWD_CMD=${PWD_CMD:Q} SED=${SED:Q} \
 		${SH} ${PKGSRCDIR}/mk/pkgformat/pkg/list-dependencies \
-			" " " "${TEST_DEPENDS:Q} " " " " " "
+			" " " "${TEST_DEPENDS:Q} " " " " " " " " " "
 
 _RESOLVE_DEPENDS_CMD=	\
 	${PKGSRC_SETENV} _PKG_DBDIR=${_PKG_DBDIR:Q} PKG_INFO=${PKG_INFO:Q} \
@@ -88,7 +90,9 @@ _RESOLVE_DEPENDS_CMD=	\
 			" " \
 			" "${TOOL_DEPENDS:Q} \
 			" "${BUILD_DEPENDS:Q} \
-			" "${DEPENDS:Q}
+			" "${DEPENDS:Q} \
+			" "${IMPLICIT_BUILD_DEPENDS:Q} \
+			" "${IMPLICIT_DEPENDS:Q}
 
 # _DEPENDS_INSTALL_CMD checks whether the package $pattern is installed,
 #	and installs it if necessary.
@@ -101,6 +105,7 @@ _RESOLVE_DEPENDS_CMD=	\
 #
 _DEPENDS_INSTALL_CMD=							\
 	case $$type in							\
+	implicit*)	continue;;					\
 	bootstrap)	Type=Bootstrap;;				\
 	tool)		Type=Tool;;					\
 	build)		Type=Build;;					\

mk/pkgformat/pkg/list-dependencies

@@ -58,8 +58,8 @@ print_entries() {
 	done
 }
 
-if [ $# != 5 ]; then
-	echo "usage: list-dependencies bootstrap_depends test_depends tool_depends build_depends depends" 1>&2
+if [ $# != 7 ]; then
+	echo "usage: list-dependencies bootstrap_depends test_depends tool_depends build_depends depends implicit_build_depends implicit_depends" 1>&2
 	exit 1
 fi
 
@@ -68,3 +68,5 @@ print_entries test "$2"
 print_entries tool "$3"
 print_entries build "$4"
 print_entries full "$5"
+print_entries implicit-build "$6"
+print_entries implicit-full "$7"

mk/pkgformat/pkg/reduce-resolved-depends.awk

@@ -65,6 +65,8 @@ BEGIN {
 		}
 		if ($1 != "full" &&
 		    $1 != "build" &&
+		    $1 != "implicit-full" &&
+		    $1 != "implicit-build" &&
 		    $1 != "tool" &&
 		    $1 != "bootstrap") {
 			print "ERROR: [" PROGNAME "] invalid dependency line " $0 | ERRCAT
@@ -82,6 +84,8 @@ BEGIN {
 		if (type[i] == "full" && checked_full[pkg[i]] != 1) {
 			checked_full[pkg[i]] = 1
 			checked_build[pkg[i]] = 1
+			checked_implicit_full[pkg[i]] = 1
+			checked_implicit_build[pkg[i]] = 1
 			checked_tool[pkg[i]] = 1
 			checked_bootstrap[pkg[i]] = 1
 			print_line[i] = 1
@@ -142,6 +146,18 @@ BEGIN {
 		}
 	}
 
+	# Just pass through implicit-* for now.
+	for (i = 0; i < lines; ++i) {
+		if (type[i] == "implicit-build" && checked_implicit_build[pkg[i]] != 1) {
+			checked_implicit_build[pkg[i]] = 1
+			print_line[i] = 1
+		}
+		if (type[i] == "implicit-full" && checked_implicit_full[pkg[i]] != 1) {
+			checked_implicit_full[pkg[i]] = 1
+			print_line[i] = 1
+		}
+	}
+
 	for (i = 0; i < lines; ++i) {
 		if (print_line[i] == 1)
 			printf("%s\t%s\t%s\n", type[i], pattern[i], pkg[i])

mk/pkgformat/pkg/resolve-dependencies

@@ -33,7 +33,7 @@ find_best() {
 	case $1 in
 	bootstrap|tool)
 		${HOST_PKG_INFO} -E "$2" || ${TRUE};;
-	build|full)
+	build|full|implicit-build|implicit-full)
 		${PKG_INFO} -E "$2" || ${TRUE};;
 	esac
 }