diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 16f20f5..6945985 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -11,6 +11,6 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Run shellcheck - run: shellcheck cc.sh test/run.sh + run: shellcheck cc.sh test/*.sh benchmark/*.sh - name: Run tests run: sh test/run.sh diff --git a/benchmark/benchmark.sh b/benchmark/benchmark.sh new file mode 100755 index 0000000..7079590 --- /dev/null +++ b/benchmark/benchmark.sh @@ -0,0 +1,109 @@ +#!/bin/sh + +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +REPO_DIR=$(CDPATH='' cd -- "$SCRIPT_DIR/.." && pwd) +CC_SH="$REPO_DIR/cc.sh" + +if [ ! -f "$CC_SH" ]; then + echo "Cannot find cc.sh at $CC_SH" >&2 + exit 1 +fi + +export SPACK_COMPILER_WRAPPER_PATH="$SCRIPT_DIR" +export SPACK_DEBUG_LOG_DIR="/tmp" +export SPACK_DEBUG_LOG_ID="bench" +export SPACK_SHORT_SPEC="py-torch@2.11.0%gcc@14.2.0 arch=linux-ubuntu24.04-aarch64" +export SPACK_SYSTEM_DIRS="/usr/*|/lib/*" +export SPACK_CXX="true" +export SPACK_CC="true" +export SPACK_FC="true" +export SPACK_F77="true" +export SPACK_CC_LINKER_ARG="-Wl," +export SPACK_CC_RPATH_ARG="-Wl,-rpath," +export SPACK_CXX_LINKER_ARG="-Wl," +export SPACK_CXX_RPATH_ARG="-Wl,-rpath," +export SPACK_FC_LINKER_ARG="-Wl," +export SPACK_FC_RPATH_ARG="-Wl,-rpath," +export SPACK_F77_LINKER_ARG="-Wl," +export SPACK_F77_RPATH_ARG="-Wl,-rpath," +export SPACK_CXX_HAS_FRANDOM_SEED=true +export SPACK_DTAGS_TO_ADD="--enable-new-dtags" +export SPACK_DTAGS_TO_STRIP="--disable-new-dtags" + +_P="/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-aarch64" + +export SPACK_MANAGED_DIRS="$_P/*" + +# Hash generator: produces a fake 32-char hex from an index. +_h() { + printf '%032x' "$1" +} + +# Build large dependency lists to stress the wrapper. +_store_pkgs="python-3.13.2 py-numpy-2.4.1 py-setuptools-80.7.1 py-pyyaml-6.0.2 openblas-0.3.33 cuda-12.9.1 py-pip-24.2 py-wheel-0.43.0 py-cython-3.0.11 py-pybind11-2.13.6 py-typing-extensions-4.12.2 py-sympy-1.13.3 py-mpmath-1.3.0 py-filelock-3.16.1 py-jinja2-3.1.4 py-markupsafe-2.1.5 py-networkx-3.4.2 py-fsspec-2024.10.0 py-packaging-24.1 py-six-1.16.0 zlib-ng-2.2.2 bzip2-1.0.8 xz-5.4.6 zstd-1.5.6 openssl-3.3.2 sqlite-3.46.1 ncurses-6.5 readline-8.2 libffi-3.4.6 expat-2.6.4 gdbm-1.24 util-linux-uuid-2.40.2 tar-1.35 gettext-0.22.5 libiconv-1.17 pcre2-10.44 ca-certificates-2024 cmake-3.31.0 ninja-1.12.1" + +_dep_pkgs="libxfixes-6.0.1 libxdamage-1.1.6 libxshmfence-1.3.2 libxxf86vm-1.1.5 libxcursor-1.2.3 libxcomposite-0.4.6 libxinerama-1.1.5 libxrandr-1.5.4 libxrender-0.9.11 libxext-1.3.6 libxi-1.8.2 libxtst-1.2.5 libxkbcommon-1.7.0 libxkbfile-1.1.3 libxmu-1.2.1 libxt-1.3.0 libxaw-1.0.16 libxpm-3.5.17 libxft-2.3.8 libxss-1.2.4 libx11-1.8.10 libxcb-1.17.0 libxau-1.0.11 libxdmcp-1.1.5 libxres-1.2.2 libxv-1.0.12 libxvmc-1.0.14 libpciaccess-0.18.1 libdrm-2.4.123 mesa-24.2.5 libglvnd-1.7.0 libglx-1.7.0 libegl-1.7.0 libgles-1.7.0 wayland-1.23.1 xorgproto-2024.1 xtrans-1.5.1 fontconfig-2.15.0 freetype-2.13.3 harfbuzz-10.0.1 libpng-1.6.44 libjpeg-turbo-3.0.4 libtiff-4.7.0 libwebp-1.4.0 giflib-5.2.2 librsvg-2.59.0 cairo-1.18.2 pixman-0.43.4 pango-1.54.0 fribidi-1.0.16 graphite2-1.3.14" + +_idx=0 +_store_include="" +_store_link="" +for _pkg in $_store_pkgs; do + _hash=$(_h $_idx) + _store_include="$_store_include$_P/$_pkg-$_hash/include:" + _store_link="$_store_link$_P/$_pkg-$_hash/lib:" + _idx=$((_idx + 1)) +done + +_dep_include="" +_dep_link="" +for _pkg in $_dep_pkgs; do + _hash=$(_h $_idx) + _dep_include="$_dep_include$_P/$_pkg-$_hash/include:" + _dep_link="$_dep_link$_P/$_pkg-$_hash/lib:" + _idx=$((_idx + 1)) +done + +export SPACK_STORE_INCLUDE_DIRS="${_store_include%:}" +export SPACK_STORE_LINK_DIRS="${_store_link%:}" +export SPACK_STORE_RPATH_DIRS="$SPACK_STORE_LINK_DIRS" + +export SPACK_INCLUDE_DIRS="${_dep_include%:}" +export SPACK_LINK_DIRS="${_dep_link%:}" +export SPACK_RPATH_DIRS="$SPACK_LINK_DIRS" + +export SPACK_COMPILER_IMPLICIT_RPATHS="\ +$_P/gcc-16.1.0-a30491defb3c2a8a14f43c8c93ec2ef2/lib" + +export SPACK_COMPILER_EXTRA_RPATHS="\ +$_P/gcc-runtime-stuff-16.1.0-6bce6552b41bbeed94e4faee72dcbe96/lib" + +export SPACK_CFLAGS="" +export SPACK_CXXFLAGS="" +export SPACK_FFLAGS="" +export SPACK_CPPFLAGS="" +export SPACK_LDFLAGS="" +export SPACK_LDLIBS="" +export SPACK_ALWAYS_CFLAGS="" +export SPACK_ALWAYS_CXXFLAGS="" +export SPACK_ALWAYS_CPPFLAGS="" +export SPACK_ALWAYS_FFLAGS="" +export SPACK_TARGET_ARGS_CC="" +export SPACK_TARGET_ARGS_CXX="" +export SPACK_TARGET_ARGS_FORTRAN="" +export SPACK_COMPILER_FLAGS_KEEP="" +export SPACK_COMPILER_FLAGS_REPLACE="" + +# Warmup +"$SCRIPT_DIR/g++" -c foo.c -o foo.o + +# Do a 1000 runs +N=1000 +_start=$(date +%s) +_i=0 +while [ $_i -lt $N ]; do + "$SCRIPT_DIR/g++" -c foo.c -o foo.o + _i=$((_i + 1)) +done +_end=$(date +%s) +_elapsed=$((_end - _start)) +printf '%s runs in %ss (avg %sms)\n' "$N" "$_elapsed" "$((_elapsed * 1000 / N))" diff --git a/benchmark/g++ b/benchmark/g++ new file mode 120000 index 0000000..ca40160 --- /dev/null +++ b/benchmark/g++ @@ -0,0 +1 @@ +../cc.sh \ No newline at end of file diff --git a/cc.sh b/cc.sh index 4156eed..1523379 100755 --- a/cc.sh +++ b/cc.sh @@ -24,6 +24,8 @@ # other separators, we set and reset it. unset IFS +# BEGIN list functions + # Separator for lists whose names end with `_list`. # We pick the alarm bell character, which is highly unlikely to # conflict with anything. This is a literal bell character (which @@ -94,23 +96,6 @@ setsep() { esac } -# prepend LISTNAME ELEMENT -# -# Prepend ELEMENT to the list stored in the variable LISTNAME. -# Handles empty lists and single-element lists. -prepend() { - varname="$1" - elt="$2" - - if empty "$varname"; then - eval "$varname=\"\${elt}\"" - else - # Get the appropriate separator for the list we're appending to. - setsep "$varname" - eval "$varname=\"\${elt}${sep}\${$varname}\"" - fi -} - # append LISTNAME ELEMENT [SEP] # # Append ELEMENT to the list stored in the variable LISTNAME, @@ -129,43 +114,76 @@ append() { fi } -# extend LISTNAME1 LISTNAME2 [PREFIX] +# extend DST_LISTNAME SRC_LISTNAME [PREFIX] # -# Append the elements stored in the variable LISTNAME2 -# to the list stored in LISTNAME1. +# Append the elements stored in the variable SRC_LISTNAME +# to the list stored in DST_LISTNAME. # If PREFIX is provided, prepend it to each element. extend() { - # Figure out the appropriate IFS for the list we're reading. - setsep "$2" - if [ "$sep" != " " ]; then - IFS="$sep" - fi - eval "for elt in \${$2}; do append $1 \"$3\${elt}\"; done" + _dst="$1" + _src="$2" + _prefix="$3" + + # Turn source list into positional parameters + setsep "$_src" + [ "$sep" != " " ] && IFS="$sep" + eval "set -- \${$_src}" unset IFS + + [ $# -eq 0 ] && return + + setsep "$_dst"; _dst_sep="$sep" + + if [ -z "$_prefix" ]; then + # Fast concatenation when no prefix is needed + IFS="$_dst_sep"; _ext_str="$*"; unset IFS + else + _ext_str="${_prefix}$1" + shift + for elt; do + _ext_str="${_ext_str}${_dst_sep}${_prefix}${elt}" + done + fi + + eval "$_dst=\"\${$_dst:+\${$_dst}$_dst_sep}\${_ext_str}\"" } -# preextend LISTNAME1 LISTNAME2 [PREFIX] +# preextend DST_LISTNAME SRC_LISTNAME [PREFIX] # -# Prepend the elements stored in the list at LISTNAME2 -# to the list at LISTNAME1, preserving order. +# Prepend the elements stored in the list at SRC_LISTNAME +# to the list at DST_LISTNAME, preserving order. # If PREFIX is provided, prepend it to each element. preextend() { - # Figure out the appropriate IFS for the list we're reading. - setsep "$2" - if [ "$sep" != " " ]; then - IFS="$sep" - fi + _dst="$1" + _src="$2" + _prefix="$3" + + # Turn source list into positional parameters + setsep "$_src" + [ "$sep" != " " ] && IFS="$sep" + eval "set -- \${$_src}" + unset IFS - # first, reverse the list to prepend - _reversed_list="" - eval "for elt in \${$2}; do prepend _reversed_list \"$3\${elt}\"; done" + [ $# -eq 0 ] && return - # prepend reversed list to preextend in order - IFS="${lsep}" - for elt in $_reversed_list; do prepend "$1" "$3${elt}"; done - unset IFS + setsep "$_dst"; _dst_sep="$sep" + + if [ -z "$_prefix" ]; then + # Fast concatenation when no prefix is needed + IFS="$_dst_sep"; _ext_str="$*"; unset IFS + else + _ext_str="${_prefix}$1" + shift + for elt; do + _ext_str="${_ext_str}${_dst_sep}${_prefix}${elt}" + done + fi + + eval "$_dst=\"\${_ext_str}\${$_dst:+$_dst_sep\${$_dst}}\"" } +# END list functions + execute() { # dump the full command if the caller supplies SPACK_TEST_COMMAND=dump-args if [ -n "${SPACK_TEST_COMMAND=}" ]; then @@ -977,7 +995,7 @@ esac if [ -n "$SPACK_CCACHE_BINARY" ]; then case "$lang_flags" in C|CXX) # ccache only supports C languages - prepend full_command_list "${SPACK_CCACHE_BINARY}" + full_command_list="${SPACK_CCACHE_BINARY}${lsep}${full_command_list}" # workaround for stage being a temp folder # see #3761#issuecomment-294352232 export CCACHE_NOHASHDIR=yes diff --git a/test/run.sh b/test/run.sh old mode 100644 new mode 100755 index a59b0a9..3edb659 --- a/test/run.sh +++ b/test/run.sh @@ -1,10 +1,13 @@ #!/bin/sh +# shellcheck disable=SC2034 # vars are passed by name to functions sourced from cc.sh +# shellcheck disable=SC2154 # lsep/sep are defined by the sourced cc.sh block # # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) # # Exercises ../cc.sh via SPACK_TEST_COMMAND=dump-args/dump-mode/dump-env-. +# Also unit-tests the list-manipulation primitives extracted from cc.sh. # # Run all tests: sh test/run.sh # Run a single test: sh test/run.sh test_modes @@ -21,12 +24,21 @@ if [ ! -f "$CC_SH" ]; then fi WRAPPER_DIR=$(mktemp -d) -trap 'rm -rf "$WRAPPER_DIR"' EXIT INT TERM for name in cc c++ cpp fc ld; do ln -s "$CC_SH" "$WRAPPER_DIR/$name" done +# Extract list-manipulation functions from cc.sh for unit testing. +FUNCS_SH=$(mktemp) +trap 'rm -rf "$WRAPPER_DIR" "$FUNCS_SH"' EXIT INT TERM + +awk '/^# BEGIN list functions$/{flag=1; next} /^# END list functions$/{flag=0} flag' \ + "$CC_SH" > "$FUNCS_SH" + +# shellcheck disable=SC1090 +. "$FUNCS_SH" + REAL_CC=/bin/mycc # --------------------------------------------------------------------------- @@ -132,6 +144,32 @@ expect_not_contains() { fi } +# expect_eq LABEL ACTUAL EXPECTED +expect_eq() { + if [ "$2" != "$3" ]; then + # Render the bell separator visibly in error output. + _exp=$(printf '%s' "$3" | tr "$lsep" '|') + _act=$(printf '%s' "$2" | tr "$lsep" '|') + fail "$1: expected '$_exp', got '$_act'" + fi +} + +# expect_true LABEL CMD... +expect_true() { + _label="$1"; shift + if ! "$@"; then + fail "$_label: expected success, got failure" + fi +} + +# expect_false LABEL CMD... +expect_false() { + _label="$1"; shift + if "$@"; then + fail "$_label: expected failure, got success" + fi +} + # ------------------- # Wrapper environment # ------------------- @@ -1056,11 +1094,202 @@ hello.c') expect_contains frandom_seed_user_passthrough "$_out" '-frandom-seed=custom' } +# --------------------------------------------------------------------------- +# List-ops unit tests (set +u: the sourced list functions use optional $3) +# --------------------------------------------------------------------------- + +test_empty() { + unset tvar || true + expect_true empty_unset empty tvar + tvar='' + expect_true empty_empty empty tvar + tvar='x' + expect_false empty_nonempty empty tvar + tvar=' ' + expect_false empty_space empty tvar + unset tvar +} + +test_setsep() { + setsep foo_dirs; expect_eq setsep_dirs "$sep" ':' + setsep FOO_DIRS; expect_eq setsep_DIRS "$sep" ':' + setsep MYPATH; expect_eq setsep_PATH "$sep" ':' + setsep MYPATHS; expect_eq setsep_PATHS "$sep" ':' + setsep foo_list; expect_eq setsep_list "$sep" "$lsep" + setsep whatever; expect_eq setsep_other "$sep" ' ' +} + +test_append() { + # _list (lsep separator) + tgt_list='' + append tgt_list a + expect_eq append_list_empty "$tgt_list" "a" + append tgt_list b + expect_eq append_list_two "$tgt_list" "a${lsep}b" + append tgt_list c + expect_eq append_list_three "$tgt_list" "a${lsep}b${lsep}c" + + # _dirs (colon separator) + tgt_dirs='' + append tgt_dirs /a + append tgt_dirs /b + expect_eq append_dirs "$tgt_dirs" "/a:/b" + + # default (space separator) + tgt_other='' + append tgt_other x + append tgt_other y + expect_eq append_default "$tgt_other" "x y" +} + +test_extend_empty_source() { + src_list='' + tgt_list='existing' + extend tgt_list src_list + expect_eq extend_empty_src "$tgt_list" 'existing' +} + +test_extend_single_into_empty() { + src_list='a' + tgt_list='' + extend tgt_list src_list + expect_eq extend_single "$tgt_list" 'a' +} + +test_extend_multi_into_empty() { + src_list="a${lsep}b${lsep}c" + tgt_list='' + extend tgt_list src_list + expect_eq extend_multi_empty "$tgt_list" "a${lsep}b${lsep}c" +} + +test_extend_multi_into_nonempty() { + src_list="b${lsep}c" + tgt_list='a' + extend tgt_list src_list + expect_eq extend_multi_nonempty "$tgt_list" "a${lsep}b${lsep}c" +} + +test_extend_prefix() { + src_list="b${lsep}c" + tgt_list='a' + extend tgt_list src_list '-I' + expect_eq extend_prefix "$tgt_list" "a${lsep}-Ib${lsep}-Ic" +} + +test_extend_cross_separator() { + # Source uses ':' (dirs), target uses lsep (list). + src_dirs='a:b:c' + tgt_list='' + extend tgt_list src_dirs + expect_eq extend_dirs_to_list_empty "$tgt_list" "a${lsep}b${lsep}c" + + tgt_list='x' + extend tgt_list src_dirs '-L' + expect_eq extend_dirs_to_list_nonempty "$tgt_list" "x${lsep}-La${lsep}-Lb${lsep}-Lc" +} + +test_extend_default_separator() { + src_other='a b c' + tgt_other='x' + extend tgt_other src_other + expect_eq extend_default_sep "$tgt_other" 'x a b c' +} + +test_preextend_empty_source() { + src_list='' + tgt_list='existing' + preextend tgt_list src_list + expect_eq preextend_empty_src "$tgt_list" 'existing' +} + +test_preextend_single_into_empty() { + src_list='a' + tgt_list='' + preextend tgt_list src_list + expect_eq preextend_single "$tgt_list" 'a' +} + +test_preextend_multi_into_empty() { + # The original reversed-prepend logic existed to preserve source order. + src_list="a${lsep}b${lsep}c" + tgt_list='' + preextend tgt_list src_list + expect_eq preextend_multi_empty "$tgt_list" "a${lsep}b${lsep}c" +} + +test_preextend_multi_into_nonempty() { + src_list="a${lsep}b" + tgt_list="c${lsep}d" + preextend tgt_list src_list + expect_eq preextend_multi_nonempty "$tgt_list" "a${lsep}b${lsep}c${lsep}d" +} + +test_preextend_prefix() { + src_list="a${lsep}b" + tgt_list='c' + preextend tgt_list src_list '-I' + expect_eq preextend_prefix "$tgt_list" "-Ia${lsep}-Ib${lsep}c" +} + +test_preextend_cross_separator() { + src_dirs='a:b:c' + tgt_list='x' + preextend tgt_list src_dirs + expect_eq preextend_dirs_to_list "$tgt_list" "a${lsep}b${lsep}c${lsep}x" +} + +test_lsep_prepend_pattern() { + # The inline replacement for the old prepend helper, as used at + # 'full_command_list="${SPACK_CCACHE_BINARY}${lsep}${full_command_list}"'. + tgt_list='' + append tgt_list compiler + append tgt_list -O2 + tgt_list="ccache${lsep}${tgt_list}" + + IFS="$lsep" + # shellcheck disable=SC2086 + set -- $tgt_list + unset IFS + expect_eq prepend_pattern_count "$#" 3 + expect_eq prepend_pattern_first "$1" 'ccache' + expect_eq prepend_pattern_mid "$2" 'compiler' + expect_eq prepend_pattern_last "$3" '-O2' +} + # --------------------------------------------------------------------------- # Runner # --------------------------------------------------------------------------- -all_tests=' +# List-ops tests need set +u because the sourced functions use optional $3. +list_ops_tests=' +test_empty +test_setsep +test_append +test_extend_empty_source +test_extend_single_into_empty +test_extend_multi_into_empty +test_extend_multi_into_nonempty +test_extend_prefix +test_extend_cross_separator +test_extend_default_separator +test_preextend_empty_source +test_preextend_single_into_empty +test_preextend_multi_into_empty +test_preextend_multi_into_nonempty +test_preextend_prefix +test_preextend_cross_separator +test_lsep_prepend_pattern +' + +is_list_ops_test() { + case "$list_ops_tests" in + *"$1"*) return 0 ;; + *) return 1 ;; + esac +} + +wrapper_tests=' test_no_wrapper_environment test_modes test_expected_args @@ -1078,6 +1307,8 @@ test_frandom_seed_not_added_without_env test_frandom_seed_filters_args ' +all_tests="$wrapper_tests $list_ops_tests" + if [ $# -gt 0 ]; then tests_to_run="$*" else @@ -1086,7 +1317,11 @@ fi for t in $tests_to_run; do start_test "$t" - "$t" + if is_list_ops_test "$t"; then + set +u; "$t"; set -u + else + "$t" + fi end_test done