From f918fec3ac1bc34eaedc73126c0b2cbcf429d9d3 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Thu, 4 Sep 2025 09:33:56 +0900 Subject: [PATCH 01/23] ci: add dynamic build/test matrix (DYNAMIC=0/1) and restrict push trigger to stable-ghc-9.14 --- .github/workflows/ci.yml | 59 +++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c4b65f3c27..0a62582985f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,30 +1,45 @@ -name: CI +name: Nix CI -# Trigger the workflow on push or pull request, but only for the master branch on: pull_request: - types: - - opened - - synchronize + types: [opened, synchronize] push: - branches: [master] - + branches: [stable-ghc-9.14] workflow_dispatch: jobs: cabal: - name: ${{ matrix.plat }} / ghc ${{ matrix.ghc }} - runs-on: "${{ fromJSON('{\"x86_64-linux\": \"ubuntu-24.04\", \"aarch64-linux\": \"ubuntu-24.04-arm\", \"x86_64-darwin\": \"macos-latest\", \"aarch64-darwin\": \"macos-latest\"}')[matrix.plat] }}" - strategy: fail-fast: false matrix: - plat: - - x86_64-linux - # - aarch64-linux # disabled: waiting for devx images to be fixed - # - x86_64-darwin # disabled: waiting for devx images to be fixed - - aarch64-darwin - ghc: ['98'] # bootstrapping compiler + include: + - plat: x86_64-linux + runner: ubuntu-24.04 + ghc: '98' + dynamic: 0 + - plat: x86_64-linux + runner: ubuntu-24.04 + ghc: '98' + dynamic: 1 + # - plat: aarch64-linux # disabled: waiting for devx images to be fixed + # runner: ubuntu-24.04-arm + # ghc: '98' + # dynamic: 0 + # - plat: aarch64-linux + # runner: ubuntu-24.04-arm + # ghc: '98' + # dynamic: 1 + - plat: aarch64-darwin + runner: macos-latest + ghc: '98' + dynamic: 0 + - plat: aarch64-darwin + runner: macos-latest + ghc: '98' + dynamic: 1 + + name: "${{ matrix.plat }} / ghc ${{ matrix.ghc }} / dynamic=${{ matrix.dynamic }}" + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -50,25 +65,25 @@ jobs: # shell: devx {0} # run: ./configure - - name: Build the bindist + - name: Build the bindist (dynamic=${{ matrix.dynamic }}) shell: devx {0} - run: make CABAL=$PWD/_build/stage0/bin/cabal + run: make CABAL=$PWD/_build/stage0/bin/cabal DYNAMIC=${{ matrix.dynamic }} - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: ${{ matrix.plat }}-bindist + name: ${{ matrix.plat }}-dynamic${{ matrix.dynamic }}-bindist path: _build/bindist - - name: Run the testsuite + - name: Run the testsuite (dynamic=${{ matrix.dynamic }}) shell: devx {0} - run: make test CABAL=$PWD/_build/stage0/bin/cabal + run: make test CABAL=$PWD/_build/stage0/bin/cabal DYNAMIC=${{ matrix.dynamic }} - name: Upload test results uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} # upload test results even if the testsuite failed to pass with: - name: ${{ matrix.plat }}-testsuite-results + name: ${{ matrix.plat }}-dynamic${{ matrix.dynamic }}-testsuite-results path: | _build/test-perf.csv _build/test-summary.txt From 6131ef0a87aa7845b4e1f0b0c27fda8a996dbb3d Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Thu, 4 Sep 2025 09:36:30 +0900 Subject: [PATCH 02/23] ci: trigger also on stable-master branch --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a62582985f..6da30bfba00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: pull_request: types: [opened, synchronize] push: - branches: [stable-ghc-9.14] + branches: [stable-ghc-9.14, stable-master] workflow_dispatch: jobs: From 9d770e44b41d10ded26f3afaacb03f3fa9b20937 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Fri, 12 Sep 2025 10:59:44 +0900 Subject: [PATCH 03/23] Add the geenrated stage2.settings file to .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f6d6b0bc900..3393551070c 100644 --- a/.gitignore +++ b/.gitignore @@ -259,3 +259,4 @@ ghc.nix/ # clangd .clangd dist-newstyle/ +cabal.project.stage2.settings From 86f0f5764426b31eabe8fbfb7962eb05e7f9668a Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Fri, 12 Sep 2025 11:00:06 +0900 Subject: [PATCH 04/23] makefile: Stop rebuilding everything when running the test-suite --- Makefile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 20837cbea96..454f3e18e11 100644 --- a/Makefile +++ b/Makefile @@ -1033,13 +1033,11 @@ CANONICAL_TEST_HC_OPTS = \ -Werror=compat -dno-debug-output # Build timeout utility (needed for some tests) if not already built. -.PHONY: testsuite-timeout -testsuite-timeout: +testsuite/timeout/install-inplace/bin/timeout: $(MAKE) -C testsuite/timeout - # --- Test Target --- -test: _build/bindist testsuite-timeout +test: $(TEST_GHC) $(TEST_GHC_PKG) $(TEST_HP2PS) $(TEST_HPC) $(TEST_RUN_GHC) testsuite/timeout/install-inplace/bin/timeout @echo "::group::Running tests with THREADS=$(THREADS)" >&2 # If any required tool is missing, testsuite logic will skip related tests. TEST_HC='$(TEST_GHC)' \ From 66e260ca8b5580083fa99fc1056c5af146fbbe92 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 15 Sep 2025 10:31:18 +0900 Subject: [PATCH 05/23] Do not auto-link unlit (which is effectively a .c only utility. --- utils/hp2ps/hp2ps.cabal | 7 +++++++ utils/unlit/unlit.cabal | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/utils/hp2ps/hp2ps.cabal b/utils/hp2ps/hp2ps.cabal index 482ea772b93..ea071266a64 100644 --- a/utils/hp2ps/hp2ps.cabal +++ b/utils/hp2ps/hp2ps.cabal @@ -23,3 +23,10 @@ Executable hp2ps HpFile.c Marks.c Scale.c TraceElement.c Axes.c Dimensions.c Key.c PsFile.c Shade.c Utilities.c + ghc-options: + -- We do _not_ want to auto-link this against rts, base, ... + -- otherwise we end up with broken dependencies. E.g. unlit + -- depending on libHSrts, which then depends on the libHSrts + -- implementation sublib, due to the -u,... flags from the + -- rts. + -no-auto-link-packages \ No newline at end of file diff --git a/utils/unlit/unlit.cabal b/utils/unlit/unlit.cabal index 770567be3a8..696516dd8f5 100644 --- a/utils/unlit/unlit.cabal +++ b/utils/unlit/unlit.cabal @@ -10,5 +10,12 @@ Executable unlit cc-options: -DFS_NAMESPACE=rts Default-Language: Haskell2010 Main-Is: unlit.c + ghc-options: + -- We do _not_ want to auto-link this against rts, base, ... + -- otherwise we end up with broken dependencies. E.g. unlit + -- depending on libHSrts, which then depends on the libHSrts + -- implementation sublib, due to the -u,... flags from the + -- rts. + -no-auto-link-packages build-depends: rts-fs From b103d8246b96302a01ff99919932681080064d92 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 15 Sep 2025 16:03:33 +0900 Subject: [PATCH 06/23] compiler/rts: prelink archive threshold --- compiler/GHC/Driver/DynFlags.hs | 4 + compiler/GHC/Driver/Session.hs | 11 +- compiler/cbits/cutils.c | 12 ++ rts/Linker.c | 217 ++++++++++++++++++++++++++++++++ rts/RtsFlags.c | 23 ++++ rts/include/rts/Flags.h | 2 + 6 files changed, 268 insertions(+), 1 deletion(-) diff --git a/compiler/GHC/Driver/DynFlags.hs b/compiler/GHC/Driver/DynFlags.hs index 4377094a98c..13786cc03e7 100644 --- a/compiler/GHC/Driver/DynFlags.hs +++ b/compiler/GHC/Driver/DynFlags.hs @@ -196,6 +196,9 @@ data DynFlags = DynFlags { enableTimeStats :: Bool, -- ^ Enable RTS timing statistics? ghcHeapSize :: Maybe Int, -- ^ The heap size to set. + -- | Threshold (in bytes) above which GHCi will pre-link archives (ld -r) + -- before loading. A value of 0 disables pre-linking. + ghciPrelinkArchiveThreshold :: Maybe Integer, maxRelevantBinds :: Maybe Int, -- ^ Maximum number of bindings from the type envt -- to show in type error messages @@ -583,6 +586,7 @@ defaultDynFlags mySettings = enableTimeStats = False, ghcHeapSize = Nothing, + ghciPrelinkArchiveThreshold = Nothing, importPaths = ["."], mainModuleNameIs = mAIN_NAME, diff --git a/compiler/GHC/Driver/Session.hs b/compiler/GHC/Driver/Session.hs index 0ca58c33866..38e9a8c5e3a 100644 --- a/compiler/GHC/Driver/Session.hs +++ b/compiler/GHC/Driver/Session.hs @@ -295,6 +295,8 @@ import qualified Data.Map as Map import qualified Data.Set as Set import Data.Word import System.FilePath +-- no env tweaks needed here +import Data.Int (Int64) import Text.ParserCombinators.ReadP hiding (char) import Text.ParserCombinators.ReadP as R @@ -884,7 +886,7 @@ parseDynamicFlagsFull activeFlags cmdline logger dflags0 args = do map ((rdr . ppr . getLoc &&& unLoc) . errMsg) $ errs -- check for disabled flags in safe haskell - let (dflags2, sh_warns) = safeFlagCheck cmdline dflags1 + let (dflags2, sh_warns) = safeFlagCheck cmdline dflags1 theWays = ways dflags2 unless (allowed_combination theWays) $ liftIO $ @@ -899,6 +901,10 @@ parseDynamicFlagsFull activeFlags cmdline logger dflags0 args = do Just x -> liftIO (setHeapSize x) _ -> return () + case ghciPrelinkArchiveThreshold dflags3 of + Just n -> liftIO (setGhciPrelinkArchiveThreshold (fromIntegral (n :: Integer))) + _ -> return () + liftIO $ setUnsafeGlobalDynFlags dflags3 -- create message envelopes using final DynFlags: #23402 @@ -1075,6 +1081,8 @@ dynamic_flags_deps = [ -- RTS options ------------------------------------------------------------- , make_ord_flag defFlag "H" (HasArg (\s -> upd (\d -> d { ghcHeapSize = Just $ fromIntegral (decodeSize s)}))) + , make_ord_flag defGhcFlag "ghci-prelink-archive-threshold" + (HasArg (\s -> upd (\d -> d { ghciPrelinkArchiveThreshold = Just (decodeSize s) }))) , make_ord_flag defFlag "Rghc-timing" (NoArg (upd (\d -> d { enableTimeStats = True }))) @@ -3808,6 +3816,7 @@ decodeSize str foreign import ccall unsafe "setHeapSize" setHeapSize :: Int -> IO () foreign import ccall unsafe "enableTimingStats" enableTimingStats :: IO () +foreign import ccall unsafe "setGhciPrelinkArchiveThreshold" setGhciPrelinkArchiveThreshold :: Int64 -> IO () outputFile :: DynFlags -> Maybe String outputFile dflags diff --git a/compiler/cbits/cutils.c b/compiler/cbits/cutils.c index fb7e3f7335b..0091dfa767e 100644 --- a/compiler/cbits/cutils.c +++ b/compiler/cbits/cutils.c @@ -7,6 +7,11 @@ places in the GHC library. #include +/* Prototype for FFI-callable helper */ +void enableTimingStats( void ); +void setHeapSize( HsInt size ); +void setGhciPrelinkArchiveThreshold( HsInt64 bytes ); + void enableTimingStats( void ) /* called from the driver */ { @@ -22,3 +27,10 @@ setHeapSize( HsInt size ) RtsFlags.GcFlags.maxHeapSize = RtsFlags.GcFlags.heapSizeSuggestion; } } + +/* Configure GHCi pre-link archive threshold (in bytes). 0 disables. */ +void +setGhciPrelinkArchiveThreshold( HsInt64 bytes ) +{ + RtsFlags.MiscFlags.linkerPrelinkArchiveThreshold = (StgInt64) bytes; +} diff --git a/rts/Linker.c b/rts/Linker.c index dc9a3a336c6..3ee3545f865 100644 --- a/rts/Linker.c +++ b/rts/Linker.c @@ -35,6 +35,7 @@ #include "PathUtils.h" #include "CheckUnload.h" // createOCSectionIndices #include "ReportMemoryMap.h" +#include "xxhash.h" #if !defined(mingw32_HOST_OS) && defined(HAVE_SIGNAL_H) #include "posix/Signals.h" @@ -444,6 +445,176 @@ void initLinker (void) initLinker_(1); } +// Helper to pre-link big archives into a temporary object file so the +// internal linker can load a single .o instead of many members. +#if defined(OBJFORMAT_ELF) || defined(OBJFORMAT_MACHO) +#include +#include +#include +#include +#include + +static const char *ghci_basename_posix(const char *p) +{ + const char *last = p; + if (!p) return p; + for (const char *s = p; *s; ++s) { + if (*s == '/') last = s + 1; + } + return last; +} + +static const char *ghci_tmpdir(void) +{ + const char *t = getenv("TMPDIR"); + return t && *t ? t : "/tmp"; +} + +static bool ghci_read_file_prefix(const char *path, char *buf, size_t bufsz) +{ + FILE *f = fopen(path, "rb"); + if (!f) return false; + size_t n = fread(buf, 1, bufsz - 1, f); + buf[n] = '\0'; + fclose(f); + return true; +} + +static bool ghci_compute_cache_key(const char *path, char out_hex[65]) +{ + FILE *f = fopen(path, "rb"); + if (!f) return false; + XXH64_state_t* st = XXH64_createState(); + if (!st) { fclose(f); return false; } + if (XXH64_reset(st, 0) != XXH_OK) { XXH64_freeState(st); fclose(f); return false; } + unsigned char buf[256 * 1024]; + size_t n; + while ((n = fread(buf, 1, sizeof(buf), f)) > 0) { + if (XXH64_update(st, buf, n) != XXH_OK) { + XXH64_freeState(st); fclose(f); return false; + } + } + fclose(f); + XXH64_hash_t hv = XXH64_digest(st); + XXH64_freeState(st); + // render as 16-char hex (zero-padded) + // use unsigned long long to satisfy printf on most platforms + unsigned long long v = (unsigned long long)hv; + // ensure buffer large enough; we reserved 65 earlier + snprintf(out_hex, 65, "%016llx", v); + return true; +} + +static pathchar *ghci_prelink_archive_to_tmp(pathchar *archivePath, size_t threshold) +{ + struct_stat st; + if (pathstat(archivePath, &st) != 0) return NULL; + if (threshold == 0 || (size_t)st.st_size < threshold) return NULL; + + // Build cache target name using SHA-256 and include basename for readability + char hex[65]; + if (!ghci_compute_cache_key((const char *)archivePath, hex)) { + return NULL; + } + + const char *tmpdir = ghci_tmpdir(); + const char *base = ghci_basename_posix((const char *)archivePath); + int target_needed = snprintf(NULL, 0, "%s/ghc-prelink-%s-%s.o", tmpdir, base, hex); + pathchar *target = stgMallocBytes((target_needed + 1) * pathsize, "ghci_prelink(target)"); + snprintf((char *)target, target_needed + 1, "%s/ghc-prelink-%s-%s.o", tmpdir, base, hex); + + // If cached object exists, use it + struct stat sb; + if (stat((const char *)target, &sb) == 0 && sb.st_size > 0) { + return target; + } + + // Cross-process lock using a directory + char lock_path[PATH_MAX]; + snprintf(lock_path, sizeof(lock_path), "%s/ghc-prelink-%s-%s.building", tmpdir, base, hex); + if (mkdir(lock_path, 0700) == 0) { + // We are the builder + int tmp_needed = snprintf(NULL, 0, "%s/ghc-prelink-%d-%s.tmp.o", tmpdir, (int)getpid(), base); + pathchar *tmp_out = stgMallocBytes((tmp_needed + 1) * pathsize, "ghci_prelink(tmp)"); + snprintf((char *)tmp_out, tmp_needed + 1, "%s/ghc-prelink-%d-%s.tmp.o", tmpdir, (int)getpid(), base); + + // Log for diagnostics + char log_path[PATH_MAX]; + snprintf(log_path, sizeof(log_path), "%s/ghc-prelink-%s.log", tmpdir, hex); + + const char *cc = getenv("CC"); + if (!cc || !*cc) cc = "cc"; + const char *ldflags = getenv("LDFLAGS"); + if (!ldflags) ldflags = ""; + char cmd[4096]; +# if defined(OBJFORMAT_ELF) + int n = snprintf(cmd, sizeof(cmd), + "%s %s -nostdlib -Wl,-r -Wl,--whole-archive '%s' -Wl,--no-whole-archive -o '%s' >'%s' 2>&1", + cc, ldflags, (const char *)archivePath, (const char *)tmp_out, log_path); +# elif defined(OBJFORMAT_MACHO) + int n = snprintf(cmd, sizeof(cmd), + "%s %s -nostdlib -Wl,-r -Wl,-all_load '%s' -o '%s' >'%s' 2>&1", + cc, ldflags, (const char *)archivePath, (const char *)tmp_out, log_path); +# endif + if (n < 0 || (size_t)n >= sizeof(cmd)) { + errorBelch("prelink: command too long while building cache for %s", base); + stgFree(tmp_out); + rmdir(lock_path); + stgFree(target); + return NULL; + } + + IF_DEBUG(linker, debugBelch("prelinking large archive: %" PATH_FMT " -> %s (cc=%s)\n", archivePath, (char *)tmp_out, cc)); + int rc = system(cmd); + if (rc != 0) { + char buf[1024]; + buf[0] = '\0'; + if (ghci_read_file_prefix(log_path, buf, sizeof(buf))) { + errorBelch("prelink failed (rc=%d) for %s; command: %s\n%s", rc, base, cmd, buf); + } else { + errorBelch("prelink failed (rc=%d) for %s; command: %s", rc, base, cmd); + } + unlink((const char *)tmp_out); + unlink(log_path); + rmdir(lock_path); + stgFree(tmp_out); + stgFree(target); + return NULL; + } + + // Atomically move into place + if (rename((const char *)tmp_out, (const char *)target) != 0) { + errorBelch("prelink: failed to rename '%s' to '%s' (errno=%d)", (char *)tmp_out, (char *)target, errno); + unlink((const char *)tmp_out); + unlink(log_path); + rmdir(lock_path); + stgFree(tmp_out); + stgFree(target); + return NULL; + } + unlink(log_path); + rmdir(lock_path); + stgFree(tmp_out); + return target; + } else { + // Someone else is building; wait for target to appear + const int max_ms = 30000; // 30 seconds + int waited = 0; + while (waited < max_ms) { + if (stat((const char *)target, &sb) == 0 && sb.st_size > 0) { + return target; + } + struct timespec ts = {0, 100 * 1000 * 1000}; // 100ms + nanosleep(&ts, NULL); + waited += 100; + } + // Give up + stgFree(target); + return NULL; + } +} +#endif + void initLinker_ (int retain_cafs) { @@ -1468,9 +1639,55 @@ static HsInt loadObj_ (pathchar *path) return 1; // success } + // Optionally pre-link large archives into a temporary .o to speed up loading. + // Runtime configuration precedence: + // 1) +RTS --linker-prelink-archive-threshold= + // 2) env GHCI_PRELINK_ARCHIVE_THRESHOLD= + // 3) default 100M + static size_t ghci_prelink_threshold = (size_t)-1; + if (ghci_prelink_threshold == (size_t)-1) { + // Env var takes precedence over the RTS field + size_t val; + const char *env = getenv("GHCI_PRELINK_ARCHIVE_THRESHOLD"); + if (env && *env) { + char *endp = NULL; + unsigned long long n = strtoull(env, &endp, 10); + if (endp && (*endp == 'K' || *endp == 'k')) { + val = (size_t)n * 1024ULL; + } else if (endp && (*endp == 'M' || *endp == 'm')) { + val = (size_t)n * 1024ULL * 1024ULL; + } else if (endp && (*endp == 'G' || *endp == 'g')) { + val = (size_t)n * 1024ULL * 1024ULL * 1024ULL; + } else { + val = (size_t)n; + } + } else { + // use the RTS field (set by +RTS or defaulted) + val = (size_t)RtsFlags.MiscFlags.linkerPrelinkArchiveThreshold; + } + ghci_prelink_threshold = val; + } + // Things that look like object files (e.g. end in `.o`) may nevertheless be // archives, as noted in Note [Object merging] in GHC.Driver.Pipeline.Execute. if (isArchive(path)) { + // Try pre-linking if the archive is large enough + pathchar *tmp_o = NULL; +#if defined(OBJFORMAT_ELF) || defined(OBJFORMAT_MACHO) + tmp_o = ghci_prelink_archive_to_tmp(path, ghci_prelink_threshold); +#endif + if (tmp_o != NULL) { + HsInt ok = loadObj_(tmp_o); + if (!ok) { + IF_DEBUG(linker, debugBelch("prelinked loading failed for %" PATH_FMT ", falling back to archive loader\n", path)); + // Fall back to archive loader + stgFree(tmp_o); + } else { + // Keep the temp file on disk to allow later unload/debug. Do not free oc->fileName here. + stgFree(tmp_o); + return 1; + } + } if (loadArchive_(path)) { return 1; // success } else { diff --git a/rts/RtsFlags.c b/rts/RtsFlags.c index 214255c590d..048addef099 100644 --- a/rts/RtsFlags.c +++ b/rts/RtsFlags.c @@ -272,6 +272,8 @@ void initRtsFlagsDefaults(void) RtsFlags.MiscFlags.linkerOptimistic = false; RtsFlags.MiscFlags.linkerMemBase = 0; RtsFlags.MiscFlags.ioManager = IO_MNGR_FLAG_AUTO; + /* Default to 100 MiB; can be overridden by env or +RTS */ + RtsFlags.MiscFlags.linkerPrelinkArchiveThreshold = (int64_t)(100ULL * 1024ULL * 1024ULL); #if defined(THREADED_RTS) && defined(mingw32_HOST_OS) RtsFlags.MiscFlags.numIoWorkerThreads = getNumberOfProcessors(); #else @@ -555,6 +557,9 @@ usage_text[] = { " -xm Base address to mmap memory in the GHCi linker", " (hex; must be <80000000)", #endif +" --linker-prelink-archive-threshold=", +" Pre-link large .a archives to a temporary .o before loading.", +" Units: K, M, G. 0 disables. Default: 100M (if not set via env)", " -xq The allocation limit given to a thread after it receives", " an AllocationLimitExceeded exception. (default: 100k)", "", @@ -1005,6 +1010,24 @@ error = true; OPTION_UNSAFE; RtsFlags.MiscFlags.linkerOptimistic = true; } + else if (!strncmp("linker-prelink-archive-threshold=", + &rts_argv[arg][2], 33)) { + OPTION_UNSAFE; + /* rts_argv[arg] is like "--linker-prelink-archive-threshold=" */ + /* The value begins after the '=' which is at index 36 of the string */ + /* We can't easily compute the offset robustly from here; instead find '=' */ + const char* full = rts_argv[arg]; + const char* eq = strchr(full, '='); + if (eq == NULL) { + errorBelch("%s: missing value", rts_argv[arg]); + error = true; + } else { + /* decodeSize expects the full flag and an offset to the value within it */ + uint32_t off = (uint32_t)(eq - full + 1); + StgWord64 bytes = decodeSize(full, off, 0, HS_WORD64_MAX); + RtsFlags.MiscFlags.linkerPrelinkArchiveThreshold = (int64_t)bytes; + } + } else if (strequal("null-eventlog-writer", &rts_argv[arg][2])) { OPTION_UNSAFE; diff --git a/rts/include/rts/Flags.h b/rts/include/rts/Flags.h index 3d5b3aa22af..cf36f81689b 100644 --- a/rts/include/rts/Flags.h +++ b/rts/include/rts/Flags.h @@ -272,6 +272,8 @@ typedef struct _MISC_FLAGS { * for the linker, NULL ==> off */ IO_MANAGER_FLAG ioManager; /* The I/O manager to use. */ uint32_t numIoWorkerThreads; /* Number of I/O worker threads to use. */ + /* Pre-link large archives before loading into the RTS linker. */ + int64_t linkerPrelinkArchiveThreshold; /* bytes; default set in RtsFlags; 0 = disable */ } MISC_FLAGS; /* See Note [Synchronization of flags and base APIs] */ From 81b1580e92c12c4cdefd02c6bce87bca0316c8ca Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 15 Sep 2025 16:03:50 +0900 Subject: [PATCH 07/23] Revert "compiler/rts: prelink archive threshold" This reverts commit d82107487ee83f1b27cf77a7e9398c4fd624457b. --- compiler/GHC/Driver/DynFlags.hs | 4 - compiler/GHC/Driver/Session.hs | 11 +- compiler/cbits/cutils.c | 12 -- rts/Linker.c | 217 -------------------------------- rts/RtsFlags.c | 23 ---- rts/include/rts/Flags.h | 2 - 6 files changed, 1 insertion(+), 268 deletions(-) diff --git a/compiler/GHC/Driver/DynFlags.hs b/compiler/GHC/Driver/DynFlags.hs index 13786cc03e7..4377094a98c 100644 --- a/compiler/GHC/Driver/DynFlags.hs +++ b/compiler/GHC/Driver/DynFlags.hs @@ -196,9 +196,6 @@ data DynFlags = DynFlags { enableTimeStats :: Bool, -- ^ Enable RTS timing statistics? ghcHeapSize :: Maybe Int, -- ^ The heap size to set. - -- | Threshold (in bytes) above which GHCi will pre-link archives (ld -r) - -- before loading. A value of 0 disables pre-linking. - ghciPrelinkArchiveThreshold :: Maybe Integer, maxRelevantBinds :: Maybe Int, -- ^ Maximum number of bindings from the type envt -- to show in type error messages @@ -586,7 +583,6 @@ defaultDynFlags mySettings = enableTimeStats = False, ghcHeapSize = Nothing, - ghciPrelinkArchiveThreshold = Nothing, importPaths = ["."], mainModuleNameIs = mAIN_NAME, diff --git a/compiler/GHC/Driver/Session.hs b/compiler/GHC/Driver/Session.hs index 38e9a8c5e3a..0ca58c33866 100644 --- a/compiler/GHC/Driver/Session.hs +++ b/compiler/GHC/Driver/Session.hs @@ -295,8 +295,6 @@ import qualified Data.Map as Map import qualified Data.Set as Set import Data.Word import System.FilePath --- no env tweaks needed here -import Data.Int (Int64) import Text.ParserCombinators.ReadP hiding (char) import Text.ParserCombinators.ReadP as R @@ -886,7 +884,7 @@ parseDynamicFlagsFull activeFlags cmdline logger dflags0 args = do map ((rdr . ppr . getLoc &&& unLoc) . errMsg) $ errs -- check for disabled flags in safe haskell - let (dflags2, sh_warns) = safeFlagCheck cmdline dflags1 + let (dflags2, sh_warns) = safeFlagCheck cmdline dflags1 theWays = ways dflags2 unless (allowed_combination theWays) $ liftIO $ @@ -901,10 +899,6 @@ parseDynamicFlagsFull activeFlags cmdline logger dflags0 args = do Just x -> liftIO (setHeapSize x) _ -> return () - case ghciPrelinkArchiveThreshold dflags3 of - Just n -> liftIO (setGhciPrelinkArchiveThreshold (fromIntegral (n :: Integer))) - _ -> return () - liftIO $ setUnsafeGlobalDynFlags dflags3 -- create message envelopes using final DynFlags: #23402 @@ -1081,8 +1075,6 @@ dynamic_flags_deps = [ -- RTS options ------------------------------------------------------------- , make_ord_flag defFlag "H" (HasArg (\s -> upd (\d -> d { ghcHeapSize = Just $ fromIntegral (decodeSize s)}))) - , make_ord_flag defGhcFlag "ghci-prelink-archive-threshold" - (HasArg (\s -> upd (\d -> d { ghciPrelinkArchiveThreshold = Just (decodeSize s) }))) , make_ord_flag defFlag "Rghc-timing" (NoArg (upd (\d -> d { enableTimeStats = True }))) @@ -3816,7 +3808,6 @@ decodeSize str foreign import ccall unsafe "setHeapSize" setHeapSize :: Int -> IO () foreign import ccall unsafe "enableTimingStats" enableTimingStats :: IO () -foreign import ccall unsafe "setGhciPrelinkArchiveThreshold" setGhciPrelinkArchiveThreshold :: Int64 -> IO () outputFile :: DynFlags -> Maybe String outputFile dflags diff --git a/compiler/cbits/cutils.c b/compiler/cbits/cutils.c index 0091dfa767e..fb7e3f7335b 100644 --- a/compiler/cbits/cutils.c +++ b/compiler/cbits/cutils.c @@ -7,11 +7,6 @@ places in the GHC library. #include -/* Prototype for FFI-callable helper */ -void enableTimingStats( void ); -void setHeapSize( HsInt size ); -void setGhciPrelinkArchiveThreshold( HsInt64 bytes ); - void enableTimingStats( void ) /* called from the driver */ { @@ -27,10 +22,3 @@ setHeapSize( HsInt size ) RtsFlags.GcFlags.maxHeapSize = RtsFlags.GcFlags.heapSizeSuggestion; } } - -/* Configure GHCi pre-link archive threshold (in bytes). 0 disables. */ -void -setGhciPrelinkArchiveThreshold( HsInt64 bytes ) -{ - RtsFlags.MiscFlags.linkerPrelinkArchiveThreshold = (StgInt64) bytes; -} diff --git a/rts/Linker.c b/rts/Linker.c index 3ee3545f865..dc9a3a336c6 100644 --- a/rts/Linker.c +++ b/rts/Linker.c @@ -35,7 +35,6 @@ #include "PathUtils.h" #include "CheckUnload.h" // createOCSectionIndices #include "ReportMemoryMap.h" -#include "xxhash.h" #if !defined(mingw32_HOST_OS) && defined(HAVE_SIGNAL_H) #include "posix/Signals.h" @@ -445,176 +444,6 @@ void initLinker (void) initLinker_(1); } -// Helper to pre-link big archives into a temporary object file so the -// internal linker can load a single .o instead of many members. -#if defined(OBJFORMAT_ELF) || defined(OBJFORMAT_MACHO) -#include -#include -#include -#include -#include - -static const char *ghci_basename_posix(const char *p) -{ - const char *last = p; - if (!p) return p; - for (const char *s = p; *s; ++s) { - if (*s == '/') last = s + 1; - } - return last; -} - -static const char *ghci_tmpdir(void) -{ - const char *t = getenv("TMPDIR"); - return t && *t ? t : "/tmp"; -} - -static bool ghci_read_file_prefix(const char *path, char *buf, size_t bufsz) -{ - FILE *f = fopen(path, "rb"); - if (!f) return false; - size_t n = fread(buf, 1, bufsz - 1, f); - buf[n] = '\0'; - fclose(f); - return true; -} - -static bool ghci_compute_cache_key(const char *path, char out_hex[65]) -{ - FILE *f = fopen(path, "rb"); - if (!f) return false; - XXH64_state_t* st = XXH64_createState(); - if (!st) { fclose(f); return false; } - if (XXH64_reset(st, 0) != XXH_OK) { XXH64_freeState(st); fclose(f); return false; } - unsigned char buf[256 * 1024]; - size_t n; - while ((n = fread(buf, 1, sizeof(buf), f)) > 0) { - if (XXH64_update(st, buf, n) != XXH_OK) { - XXH64_freeState(st); fclose(f); return false; - } - } - fclose(f); - XXH64_hash_t hv = XXH64_digest(st); - XXH64_freeState(st); - // render as 16-char hex (zero-padded) - // use unsigned long long to satisfy printf on most platforms - unsigned long long v = (unsigned long long)hv; - // ensure buffer large enough; we reserved 65 earlier - snprintf(out_hex, 65, "%016llx", v); - return true; -} - -static pathchar *ghci_prelink_archive_to_tmp(pathchar *archivePath, size_t threshold) -{ - struct_stat st; - if (pathstat(archivePath, &st) != 0) return NULL; - if (threshold == 0 || (size_t)st.st_size < threshold) return NULL; - - // Build cache target name using SHA-256 and include basename for readability - char hex[65]; - if (!ghci_compute_cache_key((const char *)archivePath, hex)) { - return NULL; - } - - const char *tmpdir = ghci_tmpdir(); - const char *base = ghci_basename_posix((const char *)archivePath); - int target_needed = snprintf(NULL, 0, "%s/ghc-prelink-%s-%s.o", tmpdir, base, hex); - pathchar *target = stgMallocBytes((target_needed + 1) * pathsize, "ghci_prelink(target)"); - snprintf((char *)target, target_needed + 1, "%s/ghc-prelink-%s-%s.o", tmpdir, base, hex); - - // If cached object exists, use it - struct stat sb; - if (stat((const char *)target, &sb) == 0 && sb.st_size > 0) { - return target; - } - - // Cross-process lock using a directory - char lock_path[PATH_MAX]; - snprintf(lock_path, sizeof(lock_path), "%s/ghc-prelink-%s-%s.building", tmpdir, base, hex); - if (mkdir(lock_path, 0700) == 0) { - // We are the builder - int tmp_needed = snprintf(NULL, 0, "%s/ghc-prelink-%d-%s.tmp.o", tmpdir, (int)getpid(), base); - pathchar *tmp_out = stgMallocBytes((tmp_needed + 1) * pathsize, "ghci_prelink(tmp)"); - snprintf((char *)tmp_out, tmp_needed + 1, "%s/ghc-prelink-%d-%s.tmp.o", tmpdir, (int)getpid(), base); - - // Log for diagnostics - char log_path[PATH_MAX]; - snprintf(log_path, sizeof(log_path), "%s/ghc-prelink-%s.log", tmpdir, hex); - - const char *cc = getenv("CC"); - if (!cc || !*cc) cc = "cc"; - const char *ldflags = getenv("LDFLAGS"); - if (!ldflags) ldflags = ""; - char cmd[4096]; -# if defined(OBJFORMAT_ELF) - int n = snprintf(cmd, sizeof(cmd), - "%s %s -nostdlib -Wl,-r -Wl,--whole-archive '%s' -Wl,--no-whole-archive -o '%s' >'%s' 2>&1", - cc, ldflags, (const char *)archivePath, (const char *)tmp_out, log_path); -# elif defined(OBJFORMAT_MACHO) - int n = snprintf(cmd, sizeof(cmd), - "%s %s -nostdlib -Wl,-r -Wl,-all_load '%s' -o '%s' >'%s' 2>&1", - cc, ldflags, (const char *)archivePath, (const char *)tmp_out, log_path); -# endif - if (n < 0 || (size_t)n >= sizeof(cmd)) { - errorBelch("prelink: command too long while building cache for %s", base); - stgFree(tmp_out); - rmdir(lock_path); - stgFree(target); - return NULL; - } - - IF_DEBUG(linker, debugBelch("prelinking large archive: %" PATH_FMT " -> %s (cc=%s)\n", archivePath, (char *)tmp_out, cc)); - int rc = system(cmd); - if (rc != 0) { - char buf[1024]; - buf[0] = '\0'; - if (ghci_read_file_prefix(log_path, buf, sizeof(buf))) { - errorBelch("prelink failed (rc=%d) for %s; command: %s\n%s", rc, base, cmd, buf); - } else { - errorBelch("prelink failed (rc=%d) for %s; command: %s", rc, base, cmd); - } - unlink((const char *)tmp_out); - unlink(log_path); - rmdir(lock_path); - stgFree(tmp_out); - stgFree(target); - return NULL; - } - - // Atomically move into place - if (rename((const char *)tmp_out, (const char *)target) != 0) { - errorBelch("prelink: failed to rename '%s' to '%s' (errno=%d)", (char *)tmp_out, (char *)target, errno); - unlink((const char *)tmp_out); - unlink(log_path); - rmdir(lock_path); - stgFree(tmp_out); - stgFree(target); - return NULL; - } - unlink(log_path); - rmdir(lock_path); - stgFree(tmp_out); - return target; - } else { - // Someone else is building; wait for target to appear - const int max_ms = 30000; // 30 seconds - int waited = 0; - while (waited < max_ms) { - if (stat((const char *)target, &sb) == 0 && sb.st_size > 0) { - return target; - } - struct timespec ts = {0, 100 * 1000 * 1000}; // 100ms - nanosleep(&ts, NULL); - waited += 100; - } - // Give up - stgFree(target); - return NULL; - } -} -#endif - void initLinker_ (int retain_cafs) { @@ -1639,55 +1468,9 @@ static HsInt loadObj_ (pathchar *path) return 1; // success } - // Optionally pre-link large archives into a temporary .o to speed up loading. - // Runtime configuration precedence: - // 1) +RTS --linker-prelink-archive-threshold= - // 2) env GHCI_PRELINK_ARCHIVE_THRESHOLD= - // 3) default 100M - static size_t ghci_prelink_threshold = (size_t)-1; - if (ghci_prelink_threshold == (size_t)-1) { - // Env var takes precedence over the RTS field - size_t val; - const char *env = getenv("GHCI_PRELINK_ARCHIVE_THRESHOLD"); - if (env && *env) { - char *endp = NULL; - unsigned long long n = strtoull(env, &endp, 10); - if (endp && (*endp == 'K' || *endp == 'k')) { - val = (size_t)n * 1024ULL; - } else if (endp && (*endp == 'M' || *endp == 'm')) { - val = (size_t)n * 1024ULL * 1024ULL; - } else if (endp && (*endp == 'G' || *endp == 'g')) { - val = (size_t)n * 1024ULL * 1024ULL * 1024ULL; - } else { - val = (size_t)n; - } - } else { - // use the RTS field (set by +RTS or defaulted) - val = (size_t)RtsFlags.MiscFlags.linkerPrelinkArchiveThreshold; - } - ghci_prelink_threshold = val; - } - // Things that look like object files (e.g. end in `.o`) may nevertheless be // archives, as noted in Note [Object merging] in GHC.Driver.Pipeline.Execute. if (isArchive(path)) { - // Try pre-linking if the archive is large enough - pathchar *tmp_o = NULL; -#if defined(OBJFORMAT_ELF) || defined(OBJFORMAT_MACHO) - tmp_o = ghci_prelink_archive_to_tmp(path, ghci_prelink_threshold); -#endif - if (tmp_o != NULL) { - HsInt ok = loadObj_(tmp_o); - if (!ok) { - IF_DEBUG(linker, debugBelch("prelinked loading failed for %" PATH_FMT ", falling back to archive loader\n", path)); - // Fall back to archive loader - stgFree(tmp_o); - } else { - // Keep the temp file on disk to allow later unload/debug. Do not free oc->fileName here. - stgFree(tmp_o); - return 1; - } - } if (loadArchive_(path)) { return 1; // success } else { diff --git a/rts/RtsFlags.c b/rts/RtsFlags.c index 048addef099..214255c590d 100644 --- a/rts/RtsFlags.c +++ b/rts/RtsFlags.c @@ -272,8 +272,6 @@ void initRtsFlagsDefaults(void) RtsFlags.MiscFlags.linkerOptimistic = false; RtsFlags.MiscFlags.linkerMemBase = 0; RtsFlags.MiscFlags.ioManager = IO_MNGR_FLAG_AUTO; - /* Default to 100 MiB; can be overridden by env or +RTS */ - RtsFlags.MiscFlags.linkerPrelinkArchiveThreshold = (int64_t)(100ULL * 1024ULL * 1024ULL); #if defined(THREADED_RTS) && defined(mingw32_HOST_OS) RtsFlags.MiscFlags.numIoWorkerThreads = getNumberOfProcessors(); #else @@ -557,9 +555,6 @@ usage_text[] = { " -xm Base address to mmap memory in the GHCi linker", " (hex; must be <80000000)", #endif -" --linker-prelink-archive-threshold=", -" Pre-link large .a archives to a temporary .o before loading.", -" Units: K, M, G. 0 disables. Default: 100M (if not set via env)", " -xq The allocation limit given to a thread after it receives", " an AllocationLimitExceeded exception. (default: 100k)", "", @@ -1010,24 +1005,6 @@ error = true; OPTION_UNSAFE; RtsFlags.MiscFlags.linkerOptimistic = true; } - else if (!strncmp("linker-prelink-archive-threshold=", - &rts_argv[arg][2], 33)) { - OPTION_UNSAFE; - /* rts_argv[arg] is like "--linker-prelink-archive-threshold=" */ - /* The value begins after the '=' which is at index 36 of the string */ - /* We can't easily compute the offset robustly from here; instead find '=' */ - const char* full = rts_argv[arg]; - const char* eq = strchr(full, '='); - if (eq == NULL) { - errorBelch("%s: missing value", rts_argv[arg]); - error = true; - } else { - /* decodeSize expects the full flag and an offset to the value within it */ - uint32_t off = (uint32_t)(eq - full + 1); - StgWord64 bytes = decodeSize(full, off, 0, HS_WORD64_MAX); - RtsFlags.MiscFlags.linkerPrelinkArchiveThreshold = (int64_t)bytes; - } - } else if (strequal("null-eventlog-writer", &rts_argv[arg][2])) { OPTION_UNSAFE; diff --git a/rts/include/rts/Flags.h b/rts/include/rts/Flags.h index cf36f81689b..3d5b3aa22af 100644 --- a/rts/include/rts/Flags.h +++ b/rts/include/rts/Flags.h @@ -272,8 +272,6 @@ typedef struct _MISC_FLAGS { * for the linker, NULL ==> off */ IO_MANAGER_FLAG ioManager; /* The I/O manager to use. */ uint32_t numIoWorkerThreads; /* Number of I/O worker threads to use. */ - /* Pre-link large archives before loading into the RTS linker. */ - int64_t linkerPrelinkArchiveThreshold; /* bytes; default set in RtsFlags; 0 = disable */ } MISC_FLAGS; /* See Note [Synchronization of flags and base APIs] */ From 20f14f1276b020cc73b98298b545d2d422f4ef1f Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 15 Sep 2025 16:26:25 +0900 Subject: [PATCH 08/23] rts: move cmm sources into common section, and move them into sublibs --- rts/rts.cabal | 63 ++++++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/rts/rts.cabal b/rts/rts.cabal index ad0674d5f10..5eeaf939f37 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -331,6 +331,34 @@ common rts-base-config rts-fs, rts +common rts-cmm-sources-base + if !arch(javascript) + -- FIXME: by virtue of being part of the rts main library, these do not get + -- the flags (debug, threaded, ...) as the sub libraries. Thus we are + -- likely missing -DDEBUG, -DTHREADED_RTS, etc. + -- One solution to this would be to turn all of these into `.h` files, and + -- then have the `AutoApply.cmm` in `rts-c-sources-base` include them. This + -- would mean they are included in the sublibraries which will in turn apply + -- the sublibrary specific (c)flags. + autogen-cmm-sources: + AutoApply.cmm + AutoApply_V16.cmm + + if arch(x86_64) + cmm-sources: + Jumps_V32.cmm (-mavx2) + Jumps_V64.cmm (-mavx512f) + autogen-cmm-sources: + AutoApply_V32.cmm (-mavx2) + AutoApply_V64.cmm (-mavx512f) + else + cmm-sources: + Jumps_V32.cmm + Jumps_V64.cmm + autogen-cmm-sources: + AutoApply_V32.cmm + AutoApply_V64.cmm + common rts-c-sources-base if !arch(javascript) cmm-sources: @@ -694,33 +722,6 @@ library rts-headers, rts-fs - if !arch(javascript) - -- FIXME: by virtue of being part of the rts main library, these do not get - -- the flags (debug, threaded, ...) as the sub libraries. Thus we are - -- likely missing -DDEBUG, -DTHREADED_RTS, etc. - -- One solution to this would be to turn all of these into `.h` files, and - -- then have the `AutoApply.cmm` in `rts-c-sources-base` include them. This - -- would mean they are included in the sublibraries which will in turn apply - -- the sublibrary specific (c)flags. - autogen-cmm-sources: - AutoApply.cmm - AutoApply_V16.cmm - - if arch(x86_64) - cmm-sources: - Jumps_V32.cmm (-mavx2) - Jumps_V64.cmm (-mavx512f) - autogen-cmm-sources: - AutoApply_V32.cmm (-mavx2) - AutoApply_V64.cmm (-mavx512f) - else - cmm-sources: - Jumps_V32.cmm - Jumps_V64.cmm - autogen-cmm-sources: - AutoApply_V32.cmm - AutoApply_V64.cmm - common ghcjs import: rts-base-config @@ -767,7 +768,7 @@ library nonthreaded-nodebug if arch(javascript) import: ghcjs else - import: rts-base-config, rts-c-sources-base, rts-link-options, rts-global-build-flags + import: rts-base-config, rts-cmm-sources-base, rts-c-sources-base, rts-link-options, rts-global-build-flags visibility: public @@ -775,21 +776,21 @@ library nonthreaded-nodebug library threaded-nodebug - import: rts-base-config, rts-c-sources-base, rts-link-options, rts-global-build-flags, rts-threaded-flags + import: rts-base-config, rts-cmm-sources-base, rts-c-sources-base, rts-link-options, rts-global-build-flags, rts-threaded-flags visibility: public build-depends: rts if arch(javascript) buildable: False library nonthreaded-debug - import: rts-base-config, rts-c-sources-base, rts-link-options, rts-global-build-flags, rts-debug-flags + import: rts-base-config, rts-cmm-sources-base, rts-c-sources-base, rts-link-options, rts-global-build-flags, rts-debug-flags visibility: public build-depends: rts if arch(javascript) buildable: False library threaded-debug - import: rts-base-config, rts-c-sources-base, rts-link-options, rts-global-build-flags, rts-threaded-flags, rts-debug-flags + import: rts-base-config, rts-cmm-sources-base, rts-c-sources-base, rts-link-options, rts-global-build-flags, rts-threaded-flags, rts-debug-flags visibility: public build-depends: rts if arch(javascript) From 26a1007546672e31363657504ec5cf92c291d16a Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 15 Sep 2025 20:20:25 +0900 Subject: [PATCH 09/23] rts: Add no-ghc-internal flag --- compiler/GHC/Driver/DynFlags.hs | 3 ++- compiler/GHC/Driver/Flags.hs | 1 + compiler/GHC/Driver/Session.hs | 2 ++ compiler/GHC/Unit/State.hs | 11 +++++++++-- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/compiler/GHC/Driver/DynFlags.hs b/compiler/GHC/Driver/DynFlags.hs index 4377094a98c..9a3fa58603c 100644 --- a/compiler/GHC/Driver/DynFlags.hs +++ b/compiler/GHC/Driver/DynFlags.hs @@ -878,7 +878,8 @@ packageFlagsChanged idflags1 idflags0 = [ Opt_HideAllPackages , Opt_HideAllPluginPackages , Opt_AutoLinkPackages - , Opt_NoRts ] + , Opt_NoRts + , Opt_NoGhcInternal ] instance Outputable PackageFlag where ppr (ExposePackage n arg rn) = text n <> braces (ppr arg <+> ppr rn) diff --git a/compiler/GHC/Driver/Flags.hs b/compiler/GHC/Driver/Flags.hs index 58e80e41b40..a10670b9eff 100644 --- a/compiler/GHC/Driver/Flags.hs +++ b/compiler/GHC/Driver/Flags.hs @@ -863,6 +863,7 @@ data GeneralFlag -- temporary flags | Opt_AutoLinkPackages | Opt_NoRts + | Opt_NoGhcInternal | Opt_ImplicitImportQualified -- keeping stuff diff --git a/compiler/GHC/Driver/Session.hs b/compiler/GHC/Driver/Session.hs index 0ca58c33866..5f791ecc219 100644 --- a/compiler/GHC/Driver/Session.hs +++ b/compiler/GHC/Driver/Session.hs @@ -1314,6 +1314,8 @@ dynamic_flags_deps = [ (NoArg (setGeneralFlag Opt_NoHsMain)) , make_ord_flag defGhcFlag "no-rts" (NoArg (setGeneralFlag Opt_NoRts)) + , make_ord_flag defGhcFlag "no-ghc-internal" + (NoArg (setGeneralFlag Opt_NoGhcInternal)) , make_ord_flag defGhcFlag "fno-state-hack" (NoArg (setGeneralFlag Opt_G_NoStateHack)) , make_ord_flag defGhcFlag "fno-opt-coercion" diff --git a/compiler/GHC/Unit/State.hs b/compiler/GHC/Unit/State.hs index ed8ea36db6d..43107587073 100644 --- a/compiler/GHC/Unit/State.hs +++ b/compiler/GHC/Unit/State.hs @@ -1636,7 +1636,11 @@ mkUnitState logger dflags cfg = do -- it modifies the unit ids of wired in packages, but when we process -- package arguments we need to key against the old versions. -- - (pkgs2, wired_map) <- findWiredInUnits logger (rtsWayUnitId dflags:wiredInUnitIds) prec_map pkgs1 vis_map2 + let wired_ids_all = rtsWayUnitId dflags : wiredInUnitIds + wired_ids + | gopt Opt_NoGhcInternal dflags = filter (/= ghcInternalUnitId) wired_ids_all + | otherwise = wired_ids_all + (pkgs2, wired_map) <- findWiredInUnits logger wired_ids prec_map pkgs1 vis_map2 -- -- Sanity check. If the rtsWayUnitId is not in the database, then we have a @@ -1648,6 +1652,7 @@ mkUnitState logger dflags cfg = do , nest 2 $ vcat [ text "pkgs1_count =" <+> ppr (length pkgs1) , text "Opt_NoRts =" <+> ppr (gopt Opt_NoRts dflags) + , text "Opt_NoGhcInternal =" <+> ppr (gopt Opt_NoGhcInternal dflags) , text "ghcLink =" <+> text (show (ghcLink dflags)) , text "platform =" <+> text (show (targetPlatform dflags)) , text "rtsWayUnitId=" <+> ppr (rtsWayUnitId dflags) @@ -1664,7 +1669,9 @@ mkUnitState logger dflags cfg = do <> text " Please check your installation." <> text " If this target doesn't need the RTS (e.g. building a shared library), you can add -no-rts to the relevant package's ghc-options in cabal.project to bypass this check." - let pkgs3 = if gopt Opt_NoRts dflags && not (anyUniqMap (== ghcInternalUnitId) wired_map) + let pkgs3 = if gopt Opt_NoGhcInternal dflags + then pkgs2 + else if gopt Opt_NoRts dflags && not (anyUniqMap (== ghcInternalUnitId) wired_map) then pkgs2 else -- At this point we should have `ghcInternalUnitId`, and the `rtsWiredUnitId dflags`. From b60cbda26faca97bb4152ea096152ec2ce9a21b7 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 15 Sep 2025 20:20:43 +0900 Subject: [PATCH 10/23] rts: Better AutoApply logic --- rts/AutoApply.cmm | 1 + rts/AutoApply_V16.cmm | 1 + rts/AutoApply_V32.cmm | 1 + rts/AutoApply_V64.cmm | 1 + rts/configure.ac | 24 ++++++++++++------------ rts/rts.cabal | 34 +++++++++++++++++++++++++++++----- 6 files changed, 45 insertions(+), 17 deletions(-) create mode 100644 rts/AutoApply.cmm create mode 100644 rts/AutoApply_V16.cmm create mode 100644 rts/AutoApply_V32.cmm create mode 100644 rts/AutoApply_V64.cmm diff --git a/rts/AutoApply.cmm b/rts/AutoApply.cmm new file mode 100644 index 00000000000..22ed09ee02c --- /dev/null +++ b/rts/AutoApply.cmm @@ -0,0 +1 @@ +#include diff --git a/rts/AutoApply_V16.cmm b/rts/AutoApply_V16.cmm new file mode 100644 index 00000000000..5cc5142317b --- /dev/null +++ b/rts/AutoApply_V16.cmm @@ -0,0 +1 @@ +#include diff --git a/rts/AutoApply_V32.cmm b/rts/AutoApply_V32.cmm new file mode 100644 index 00000000000..9a4427459b5 --- /dev/null +++ b/rts/AutoApply_V32.cmm @@ -0,0 +1 @@ +#include diff --git a/rts/AutoApply_V64.cmm b/rts/AutoApply_V64.cmm new file mode 100644 index 00000000000..343853d6e16 --- /dev/null +++ b/rts/AutoApply_V64.cmm @@ -0,0 +1 @@ +#include diff --git a/rts/configure.ac b/rts/configure.ac index ae110d76dce..7ee47bd4941 100644 --- a/rts/configure.ac +++ b/rts/configure.ac @@ -583,36 +583,36 @@ else AC_MSG_ERROR([Failed to run $DERIVE_CONSTANTS --gen-header ...]) fi -AC_MSG_CHECKING([for AutoApply.cmm]) -if $GENAPPLY include/DerivedConstants.h > AutoApply.cmm; then +AC_MSG_CHECKING([for include/rts/AutoApply.cmm.h]) +if mkdir -p include/rts && $GENAPPLY include/DerivedConstants.h > include/rts/AutoApply.cmm.h; then AC_MSG_RESULT([created]) else AC_MSG_RESULT([failed to create]) - AC_MSG_ERROR([Failed to run $GENAPPLY include/DerivedConstants.h > AutoApply.cmm]) + AC_MSG_ERROR([Failed to run $GENAPPLY include/DerivedConstants.h > include/rts/AutoApply.cmm.h]) fi -AC_MSG_CHECKING([for AutoApply_V16.cmm]) -if $GENAPPLY include/DerivedConstants.h -V16 > AutoApply_V16.cmm; then +AC_MSG_CHECKING([for include/rts/AutoApply_V16.cmm.h]) +if mkdir -p include/rts && $GENAPPLY include/DerivedConstants.h -V16 > include/rts/AutoApply_V16.cmm.h; then AC_MSG_RESULT([created]) else AC_MSG_RESULT([failed to create]) - AC_MSG_ERROR([Failed to run $GENAPPLY include/DerivedConstants.h -V16 > AutoApply_V16.cmm]) + AC_MSG_ERROR([Failed to run $GENAPPLY include/DerivedConstants.h -V16 > include/rts/AutoApply_V16.cmm.h]) fi -AC_MSG_CHECKING([for AutoApply_V32.cmm]) -if $GENAPPLY include/DerivedConstants.h -V32 > AutoApply_V32.cmm; then +AC_MSG_CHECKING([for include/rts/AutoApply_V32.cmm.h]) +if mkdir -p include/rts && $GENAPPLY include/DerivedConstants.h -V32 > include/rts/AutoApply_V32.cmm.h; then AC_MSG_RESULT([created]) else AC_MSG_RESULT([failed to create]) - AC_MSG_ERROR([Failed to run $GENAPPLY include/DerivedConstants.h -V32 > AutoApply_V32.cmm]) + AC_MSG_ERROR([Failed to run $GENAPPLY include/DerivedConstants.h -V32 > include/rts/AutoApply_V32.cmm.h]) fi -AC_MSG_CHECKING([for AutoApply_V64.cmm]) -if $GENAPPLY include/DerivedConstants.h -V64 > AutoApply_V64.cmm; then +AC_MSG_CHECKING([for include/rts/AutoApply_V64.cmm.h]) +if mkdir -p include/rts && $GENAPPLY include/DerivedConstants.h -V64 > include/rts/AutoApply_V64.cmm.h; then AC_MSG_RESULT([created]) else AC_MSG_RESULT([failed to create]) - AC_MSG_ERROR([Failed to run $GENAPPLY include/DerivedConstants.h -V64 > AutoApply_V64.cmm]) + AC_MSG_ERROR([Failed to run $GENAPPLY include/DerivedConstants.h -V64 > include/rts/AutoApply_V64.cmm.h]) fi AC_MSG_CHECKING([for include/rts/EventLogConstants.h]) diff --git a/rts/rts.cabal b/rts/rts.cabal index 5eeaf939f37..46acc0ba0b4 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -340,7 +340,7 @@ common rts-cmm-sources-base -- then have the `AutoApply.cmm` in `rts-c-sources-base` include them. This -- would mean they are included in the sublibraries which will in turn apply -- the sublibrary specific (c)flags. - autogen-cmm-sources: + cmm-sources: AutoApply.cmm AutoApply_V16.cmm @@ -348,17 +348,20 @@ common rts-cmm-sources-base cmm-sources: Jumps_V32.cmm (-mavx2) Jumps_V64.cmm (-mavx512f) - autogen-cmm-sources: AutoApply_V32.cmm (-mavx2) AutoApply_V64.cmm (-mavx512f) else cmm-sources: Jumps_V32.cmm Jumps_V64.cmm - autogen-cmm-sources: AutoApply_V32.cmm AutoApply_V64.cmm + -- this is required so we don't have ghc inject ghc-internal (which depends on the rts) + -- during the build phase of this library. + ghc-options: -no-ghc-internal + + common rts-c-sources-base if !arch(javascript) cmm-sources: @@ -637,12 +640,31 @@ library ghc-options: -this-unit-id rts -ghcversion-file=include/ghcversion.h -optc-DFS_NAMESPACE=rts cmm-options: -this-unit-id rts + -- [The AutoApply story] + -- + -- We use GenApply to generate the AutoApply[_V{16,32,64}].cmm files. + -- However cabal will run the ./configure script only for the main library. + -- To work around this shortcoming, we'll generate .cmm.h files (same + -- content as .cmm), and create AutoApply*.cmm files that just + -- + -- #include + -- + -- This way each sublib has it's own properly parameterized .cmm file, while + -- we only generate them once and stick them into the rts library. + -- + -- This is a hack, and it would be great if sublibs had access to their + -- parent libraries auto-gen folders, however as sublibs are supposed to be + -- separate components, this is a non-trivial (impossible?) task to resolve. autogen-includes: ghcautoconf.h ghcplatform.h DerivedConstants.h rts/EventLogConstants.h rts/EventTypes.h + AutoApply.cmm.h + AutoApply_V16.cmm.h + AutoApply_V32.cmm.h + AutoApply_V64.cmm.h install-includes: ghcautoconf.h @@ -650,6 +672,10 @@ library DerivedConstants.h rts/EventLogConstants.h rts/EventTypes.h + AutoApply.cmm.h + AutoApply_V16.cmm.h + AutoApply_V32.cmm.h + AutoApply_V64.cmm.h install-includes: -- Common headers for non-JS builds @@ -771,10 +797,8 @@ library nonthreaded-nodebug import: rts-base-config, rts-cmm-sources-base, rts-c-sources-base, rts-link-options, rts-global-build-flags visibility: public - ghc-options: -optc-DRtsWay="v" - library threaded-nodebug import: rts-base-config, rts-cmm-sources-base, rts-c-sources-base, rts-link-options, rts-global-build-flags, rts-threaded-flags visibility: public From 38af39d1982e6a064fb90929e241ce334578034e Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Tue, 16 Sep 2025 10:05:10 +0900 Subject: [PATCH 11/23] Makefile: symlink dylibs --- Makefile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Makefile b/Makefile index 454f3e18e11..f06ceb600d8 100644 --- a/Makefile +++ b/Makefile @@ -879,6 +879,13 @@ _build/bindist: stage2 driver/ghc-usage.txt driver/ghci-usage.txt $@/bin/ghc-pkg recache # Copy headers @$(call copy_all_stage2_h,$@/bin/ghc-pkg) + # Add basename symlinks for nested shared libs (.dylib, .so) in lib/$(HOST_PLATFORM) + @if [ -d "$@/lib/$(HOST_PLATFORM)" ]; then \ + cd "$@/lib/$(HOST_PLATFORM)" ; \ + for lib in $$(find . -mindepth 2 \( -name "*.dylib" -o -name "*.so" \) -type f) ; do \ + ln -sf "$$lib" "$$(basename "$$lib")" ; \ + done ; \ + fi @echo "::endgroup::" _build/bindist/ghc.tar.gz: _build/bindist @@ -903,6 +910,13 @@ _build/bindist/lib/targets/%: _build/bindist driver/ghc-usage.txt driver/ghci-us # Copy libraries and settings @if [ -e $(CURDIR)/_build/bindist/lib/targets/$(@F)/lib/$(@F) ] ; then find $(CURDIR)/_build/bindist/lib/targets/$(@F)/lib/$(@F)/ -mindepth 1 -type f -name "*.so" -execdir mv '{}' $(CURDIR)/_build/bindist/lib/targets/$(@F)/lib/$(@F)/'{}' \; ; fi $(call copycrosslib,$(@F)) + # Add basename symlinks for nested shared libs (.dylib, .so) in lib/$(@F) + @if [ -d $(CURDIR)/_build/bindist/lib/targets/$(@F)/lib/$(@F) ] ; then \ + cd $(CURDIR)/_build/bindist/lib/targets/$(@F)/lib/$(@F) ; \ + for lib in $$(find . -mindepth 2 \( -name "*.dylib" -o -name "*.so" \) -type f) ; do \ + ln -sf "$$lib" "$$(basename "$$lib")" ; \ + done ; \ + fi # --help @cp -rfp driver/ghc-usage.txt _build/bindist/lib/targets/$(@F)/lib/ @cp -rfp driver/ghci-usage.txt _build/bindist/lib/targets/$(@F)/lib/ From aed1df397727d688085122326f8c7ac8ae34cd67 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Tue, 16 Sep 2025 15:07:54 +0900 Subject: [PATCH 12/23] rts.cabal: double all the flags to ensure we definetly get them set This is kidna stupid. Julian hit something similar with the THREADED one :-/ --- rts/rts.cabal | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/rts/rts.cabal b/rts/rts.cabal index 46acc0ba0b4..aa2760dc4ae 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -615,25 +615,35 @@ common rts-link-options ld-options: -read_only_relocs warning common rts-global-build-flags - ghc-options: -DCOMPILING_RTS + ghc-options: -DCOMPILING_RTS -optc-DCOMPILING_RTS cpp-options: -DCOMPILING_RTS + cmm-options: -DCOMPILING_RTS + cc-options: -DCOMPILING_RTS if !flag(smp) - ghc-options: -DNOSMP + ghc-options: -DNOSMP -optc-DNOSMP cpp-options: -DNOSMP + cmm-options: -DNOSMP + cc-options: -DNOSMP if flag(dynamic) - ghc-options: -DDYNAMIC + ghc-options: -DDYNAMIC -optc-DDYNAMIC cpp-options: -DDYNAMIC + cmm-options: -DDYNAMIC + cc-options: -DDYNAMIC if flag(thread-sanitizer) cc-options: -fsanitize=thread ld-options: -fsanitize=thread common rts-debug-flags - ghc-options: -optc-DDEBUG + ghc-options: -DDEBUG -optc-DDEBUG cpp-options: -DDEBUG -fno-omit-frame-pointer -g3 -O0 + cmm-options: -DDEBUG -fno-omit-frame-pointer -g3 -O0 + cc-options: -DDEBUG -fno-omit-frame-pointer -g3 -O0 common rts-threaded-flags - ghc-options: -DTHREADED_RTS + ghc-options: -DTHREADED_RTS -optc-DTHREADED_RTS cpp-options: -DTHREADED_RTS + cmm-options: -DTHREADED_RTS + cc-options: -DTHREADED_RTS -- the _main_ library needs to deal with all the _configure_ time stuff. library From f270d6eb8e78ab64e5851d481e68fbe35774af73 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Tue, 16 Sep 2025 17:58:08 +0900 Subject: [PATCH 13/23] testsuite: Make sure we prefix the ghcconfig with a hash Ideally, ghc should just be able to produce this data outright, without having to resort to a separate tool. This seems highly questionable. It would be much better if ghc should emitted this information in a structured way. --- testsuite/.gitignore | 4 ++++ testsuite/mk/boilerplate.mk | 13 ++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/testsuite/.gitignore b/testsuite/.gitignore index 2c699cf0462..65095f05566 100644 --- a/testsuite/.gitignore +++ b/testsuite/.gitignore @@ -60,6 +60,10 @@ tmp.d *.so *bindisttest_install___dir_bin_ghc.mk *bindisttest_install___dir_bin_ghc.exe.mk +mk/*_ghcconfig*_bin_ghc*.mk +mk/*_ghcconfig*_bin_ghc*.exe.mk +mk/*_ghcconfig*_test___spaces_ghc*.mk +mk/*_ghcconfig*_test___spaces_ghc*.exe.mk mk/ghcconfig*_bin_ghc*.mk mk/ghcconfig*_bin_ghc*.exe.mk mk/ghcconfig*_test___spaces_ghc*.mk diff --git a/testsuite/mk/boilerplate.mk b/testsuite/mk/boilerplate.mk index 9ad8b3308e3..333285d2bc3 100644 --- a/testsuite/mk/boilerplate.mk +++ b/testsuite/mk/boilerplate.mk @@ -260,7 +260,18 @@ $(TOP)/ghc-config/ghc-config : $(TOP)/ghc-config/ghc-config.hs empty= space=$(empty) $(empty) ifeq "$(ghc_config_mk)" "" -ghc_config_mk = $(TOP)/mk/ghcconfig$(subst $(space),_,$(subst :,_,$(subst /,_,$(subst \,_,$(TEST_HC))))).mk +sanitized_hc := $(subst $(space),_,$(subst :,_,$(subst /,_,$(subst \,_,$(TEST_HC))))) +test_hc_hash := $(shell \ + if command -v openssl >/dev/null 2>&1; then \ + openssl dgst -sha256 $(TEST_HC) | awk '{print substr($$2, 1, 8)}'; \ + elif command -v sha256sum >/dev/null 2>&1; then \ + sha256sum $(TEST_HC) | awk '{print substr($$1, 1, 8)}'; \ + elif command -v shasum >/dev/null 2>&1; then \ + shasum -a 256 $(TEST_HC) | awk '{print substr($$1, 1, 8)}'; \ + else \ + echo "no_hash"; \ + fi) +ghc_config_mk = $(TOP)/mk/$(test_hc_hash)_ghcconfig$(sanitized_hc).mk $(ghc_config_mk) : $(TOP)/ghc-config/ghc-config $(TOP)/ghc-config/ghc-config "$(TEST_HC)" >"$@"; if [ "$$?" != "0" ]; then $(RM) "$@"; exit 1; fi From 1967f0806c272a318fa45adb27720e0ff1a88b7f Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Tue, 16 Sep 2025 18:26:14 +0900 Subject: [PATCH 14/23] testsuite: Add explicit flushs [WRONG!] This seems conceptually wrong. I don't fully understand why we need this? --- testsuite/tests/annotations/should_compile/th/annth.hs | 1 + testsuite/tests/simplStg/should_compile/T22840.hs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/testsuite/tests/annotations/should_compile/th/annth.hs b/testsuite/tests/annotations/should_compile/th/annth.hs index 8cc3a242ff9..87768e1a4ba 100644 --- a/testsuite/tests/annotations/should_compile/th/annth.hs +++ b/testsuite/tests/annotations/should_compile/th/annth.hs @@ -24,5 +24,6 @@ main = do runIO $ print (anns :: [String]) anns <- reifyAnnotations (AnnLookupName 'TestTypeTH) runIO $ print (anns :: [String]) + runIO $ hFlush stdout [| return () |] ) hFlush stdout diff --git a/testsuite/tests/simplStg/should_compile/T22840.hs b/testsuite/tests/simplStg/should_compile/T22840.hs index f7e4e59bc07..86ffbf9c310 100644 --- a/testsuite/tests/simplStg/should_compile/T22840.hs +++ b/testsuite/tests/simplStg/should_compile/T22840.hs @@ -6,9 +6,11 @@ module C where import T22840A import T22840B import Control.Monad.IO.Class +import System.IO (hFlush, stdout) $(liftIO $ do putStrLn "start" putStrLn (disp theT) putStrLn "end" + hFlush stdout return []) From 41b298d273080b38f7a73914d82aef5cf80e45d9 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Wed, 17 Sep 2025 13:18:22 +0900 Subject: [PATCH 15/23] fixup! rts.cabal: double all the flags to ensure we definetly get them set --- rts/rts.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/rts.cabal b/rts/rts.cabal index aa2760dc4ae..bb58a1614e5 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -636,7 +636,7 @@ common rts-global-build-flags common rts-debug-flags ghc-options: -DDEBUG -optc-DDEBUG cpp-options: -DDEBUG -fno-omit-frame-pointer -g3 -O0 - cmm-options: -DDEBUG -fno-omit-frame-pointer -g3 -O0 + cmm-options: -DDEBUG cc-options: -DDEBUG -fno-omit-frame-pointer -g3 -O0 common rts-threaded-flags From 49ce37f5f4ad2cfdfa492103dc764596b456e743 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Wed, 17 Sep 2025 13:18:49 +0900 Subject: [PATCH 16/23] makefile: symlink ghc-iserv-dyn <- ghc-iserv this is stupid, but it will have to do for now for dyanmic. --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index f06ceb600d8..ab31f45a622 100644 --- a/Makefile +++ b/Makefile @@ -886,6 +886,9 @@ _build/bindist: stage2 driver/ghc-usage.txt driver/ghci-usage.txt ln -sf "$$lib" "$$(basename "$$lib")" ; \ done ; \ fi + # Create -dyn iserv executable. #FIXME: THIS IS IDIOTIC! + @cd "$@/bin" + @ln -sf ghc-iserv-dyn ghc-iserv @echo "::endgroup::" _build/bindist/ghc.tar.gz: _build/bindist From 9e4cf800b95b8b40f67b34f58081f89d35ccf44f Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Wed, 17 Sep 2025 14:03:58 +0900 Subject: [PATCH 17/23] fixup! makefile: symlink ghc-iserv-dyn <- ghc-iserv --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ab31f45a622..4fa39da95af 100644 --- a/Makefile +++ b/Makefile @@ -888,7 +888,7 @@ _build/bindist: stage2 driver/ghc-usage.txt driver/ghci-usage.txt fi # Create -dyn iserv executable. #FIXME: THIS IS IDIOTIC! @cd "$@/bin" - @ln -sf ghc-iserv-dyn ghc-iserv + @ln -sf ghc-iserv ghc-iserv-dyn @echo "::endgroup::" _build/bindist/ghc.tar.gz: _build/bindist From 0c7e99d0427538d4b1ea05b721d13882c1738ad5 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Wed, 17 Sep 2025 14:42:48 +0900 Subject: [PATCH 18/23] testsuite: improve logging; also check non-inplace libraries. --- testsuite/driver/testlib.py | 69 ++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py index 609561ceac6..579b2f602b7 100644 --- a/testsuite/driver/testlib.py +++ b/testsuite/driver/testlib.py @@ -661,7 +661,19 @@ def collect_size ( deviation, path ): return collect_size_func(deviation, lambda: path) def collect_size_func ( deviation, path_func ): - return collect_generic_stat ( 'size', deviation, lambda way: os.path.getsize(in_testdir(path_func())) ) + # Wrap path resolution to avoid passing None/invalid paths to Path APIs. + def current(_way): + p = path_func() + if p is None: + raise StatsException("No path returned for size collection") + # If p looks absolute, use it directly; else resolve relative to testdir + pth = Path(p) + if not pth.is_absolute(): + pth = in_testdir(p) + if not pth.exists(): + raise StatsException(f"Path not found for size collection: {pth}") + return os.path.getsize(pth) + return collect_generic_stat ( 'size', deviation, current ) def get_dir_size(path): total = 0 @@ -674,7 +686,7 @@ def get_dir_size(path): total += get_dir_size(entry.path) return total except FileNotFoundError: - print("Exception: Could not find: " + path) + raise StatsException(f"Directory not found for size collection: {path}") def collect_size_dir ( deviation, path ): return collect_size_dir_func ( deviation, lambda: path ) @@ -706,7 +718,12 @@ def collect_size_ghc_pkg (deviation, library): # same for collect_size and find_so def collect_object_size (deviation, library, use_non_inplace=False): if use_non_inplace: - return collect_size_func(deviation, lambda: find_non_inplace_so(library)) + try: + return collect_size_func(deviation, lambda: find_non_inplace_so(library)) + except Exception as _: + # should we fail to find inplace, let's try to find non-inplace. + # FIXME: remove the whole inplace nonsense outright. + return collect_size_func(deviation, lambda: find_so(library)) else: return collect_size_func(deviation, lambda: find_so(library)) @@ -723,21 +740,20 @@ def path_from_ghcPkg (library, field): try: result = subprocess.run(ghcPkgCmd, capture_output=True, shell=True) - # check_returncode throws an exception if the return code is not 0. result.check_returncode() - - # if we get here then the call worked and we have the path we split by - # whitespace and then return the path which becomes the second element - # in the array - return re.split(r'\s+', result.stdout.decode("utf-8"))[1] + out = result.stdout.decode("utf-8").strip() + # Expected format: ": " possibly spanning lines; grab text after first colon. + m = re.split(r"^\s*[^:]+:\s*", out, maxsplit=1, flags=re.MULTILINE) + if len(m) == 2: + val = m[1].strip().splitlines()[0].strip() + if val: + return val + raise StatsException(f"ghc-pkg returned no {field} for {library}. Output: {out}") + except subprocess.CalledProcessError as e: + raise StatsException(f"ghc-pkg failed for {library} {field}: {e}") except Exception as e: - message = f""" - Attempt to find {field} of {library} using ghc-pkg failed. - ghc-pkg path: {config.ghc_pkg} - error" {e} - """ - print(message) + raise StatsException(f"Error parsing ghc-pkg output for {library} {field}: {e}") def _find_so(lib, directory, in_place): @@ -772,21 +788,34 @@ def _find_so(lib, directory, in_place): to_match = r'libHS{}-\d+(\.\d+)+-ghc\S+\.' + suffix matches = [] - # wrap this in some exception handling, hadrian test will error out because - # these files don't exist yet, so we pass when this occurs + # Robust error handling: raise a stats exception for missing directory or no match + if directory is None: + raise StatsException(f"No directory provided to find shared object for {lib}") try: + dsos = [] for f in os.listdir(directory): if f.endswith(suffix): + dsos.append(f) pattern = re.compile(to_match.format(re.escape(lib))) match = re.match(pattern, f) if match: matches.append(match.group()) + if not matches: + raise StatsException(f"Could not find shared object file for {lib} in {directory}") return os.path.join(directory, matches[0]) - except: - failBecause('Could not find shared object file: ' + lib) + except FileNotFoundError: + raise StatsException(f"Directory not found while searching shared object for {lib}: {directory}") + except Exception as e: + raise StatsException(f"Error while searching shared object for {lib} in {directory}: {e}") def find_so(lib): - return _find_so(lib,path_from_ghcPkg(lib, "dynamic-library-dirs"),True) + try: + return _find_so(lib,path_from_ghcPkg(lib, "dynamic-library-dirs"),True) + except Exception as _: + # if we fail to find the inplace so, fallback to trying to find the + # non-inplace so indead; + # FIXME: This whole inplace logic needs to be ripped out! + return _find_so(lib,path_from_ghcPkg(lib, "dynamic-library-dirs"),False) def find_non_inplace_so(lib): return _find_so(lib,path_from_ghcPkg(lib, "dynamic-library-dirs"),False) From 6a9fa8a312031a678948946cbd27d1fce684cda6 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Wed, 17 Sep 2025 17:33:59 +0900 Subject: [PATCH 19/23] fixup! testsuite: improve logging; also check non-inplace libraries. --- testsuite/driver/testlib.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py index 579b2f602b7..3356d4d2f85 100644 --- a/testsuite/driver/testlib.py +++ b/testsuite/driver/testlib.py @@ -792,10 +792,8 @@ def _find_so(lib, directory, in_place): if directory is None: raise StatsException(f"No directory provided to find shared object for {lib}") try: - dsos = [] for f in os.listdir(directory): if f.endswith(suffix): - dsos.append(f) pattern = re.compile(to_match.format(re.escape(lib))) match = re.match(pattern, f) if match: From b54cf122aeacbdba2f08ca665226baae22c205c6 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Fri, 19 Sep 2025 11:18:30 +0900 Subject: [PATCH 20/23] testsuite: disable rts test, which is invalid since the rts split. --- testsuite/tests/perf/size/all.T | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testsuite/tests/perf/size/all.T b/testsuite/tests/perf/size/all.T index 4c45cb4d11f..e5fd683fce7 100644 --- a/testsuite/tests/perf/size/all.T +++ b/testsuite/tests/perf/size/all.T @@ -84,7 +84,9 @@ test('mtl_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_obje test('os_string_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(size_acceptance_threshold, "os-string")] , static_stats, [] ) test('parsec_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(size_acceptance_threshold, "parsec")] , static_stats, [] ) test('process_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(size_acceptance_threshold, "process")] , static_stats, [] ) -test('rts_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(size_acceptance_threshold, "rts", True)] , static_stats, [] ) +# after the rts-split, there is not signle rts anymore. The single rts package is just headers, and thus empty. We now have one rts per threaded/debug combination. +# they are also sublibs, which means the regex in the test-driver doesn't work for this. Thus for now we disable this test. +# test('rts_so' ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(size_acceptance_threshold, "rts", True)] , static_stats, [] ) test('template_haskell_so',[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(size_acceptance_threshold, "template-haskell")] , static_stats, [] ) # terminfo is not built in cross ghc so skip it test('terminfo_so' ,[req_dynamic_ghc, when(config.cross, skip), windows_skip, collect_object_size(size_acceptance_threshold, "terminfo")], static_stats, [] ) From cf05f6e33856b20290a53f9956f40da0b78612d6 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Fri, 19 Sep 2025 11:19:05 +0900 Subject: [PATCH 21/23] testsuite: T2228 does not appear broken. --- testsuite/tests/ghc-e/should_run/all.T | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/testsuite/tests/ghc-e/should_run/all.T b/testsuite/tests/ghc-e/should_run/all.T index 03ee37cb9ff..7362b190741 100644 --- a/testsuite/tests/ghc-e/should_run/all.T +++ b/testsuite/tests/ghc-e/should_run/all.T @@ -6,9 +6,7 @@ test('ghc-e004', req_interp, makefile_test, ['ghc-e004']) test('ghc-e005', req_interp, makefile_test, ['ghc-e005']) test('ghc-e006', req_interp, makefile_test, ['ghc-e006']) -test('T2228', - [req_interp, when(ghc_dynamic(), expect_broken(7298))], - makefile_test, ['T2228']) +test('T2228', req_interp, makefile_test, ['T2228']) test('T2636', req_interp, makefile_test, ['T2636']) test('T3890', req_interp, makefile_test, ['T3890']) test('T7299', req_interp, makefile_test, ['T7299']) From a03584d8770522ab40606dddb77585226c21b5aa Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Fri, 19 Sep 2025 19:32:37 +0900 Subject: [PATCH 22/23] compiler: inject rpath for the rts as needed. This is needed until https://github.com/haskell/cabal/issues/11221 is properly addressed to ensure that a linked dynamic binary will have the appropriate path for the rts (sublib). --- compiler/GHC/Linker/Static.hs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/compiler/GHC/Linker/Static.hs b/compiler/GHC/Linker/Static.hs index 8a634258e7f..2d25c46ce56 100644 --- a/compiler/GHC/Linker/Static.hs +++ b/compiler/GHC/Linker/Static.hs @@ -100,11 +100,22 @@ linkBinary' staticLink logger tmpfs dflags unit_env o_files dep_units = do -- explicit packages with the auto packages and all of their -- dependencies, and eliminating duplicates. pkgs <- mayThrowUnitErr (preloadUnitsInfo' unit_env dep_units) + + -- Collect per-package library dirs (deduplicated, non-empty) let pkg_lib_paths = collectLibraryDirs ways_ pkgs - let pkg_lib_path_opts = concatMap get_pkg_lib_path_opts pkg_lib_paths - get_pkg_lib_path_opts l + -- Until: https://github.com/haskell/cabal/issues/11221 is in cabal, + -- we have to deal with cabal passing -dyload deploy, and manually + -- inject rpaths for the rts. + -- Build linker options per (pkg, libdir) + let pkg_lib_path_opts = + concat + [ get_pkg_lib_path_opts pkg l + | pkg <- pkgs + , l <- collectLibraryDirs ways_ [pkg] + ] + get_pkg_lib_path_opts pkg l | osElfTarget (platformOS platform) && - dynLibLoader dflags == SystemDependent && + (dynLibLoader dflags == SystemDependent || unitPackageNameString pkg == rts) && ways_ `hasWay` WayDyn = let libpath = if gopt Opt_RelativeDynlibPaths dflags then "$ORIGIN" @@ -125,7 +136,7 @@ linkBinary' staticLink logger tmpfs dflags unit_env o_files dep_units = do else ["-Xlinker", "-rpath-link", "-Xlinker", l] in ["-L" ++ l] ++ rpathlink ++ rpath | osMachOTarget (platformOS platform) && - dynLibLoader dflags == SystemDependent && + (dynLibLoader dflags == SystemDependent || unitPackageNameString pkg == rts) && ways_ `hasWay` WayDyn && useXLinkerRPath dflags (platformOS platform) = let libpath = if gopt Opt_RelativeDynlibPaths dflags From 8c7a84cafb319e96ef0e698c31d574a20a5ccb9d Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Fri, 19 Sep 2025 20:26:08 +0900 Subject: [PATCH 23/23] fixup! compiler: inject rpath for the rts as needed. --- compiler/GHC/Linker/Static.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/GHC/Linker/Static.hs b/compiler/GHC/Linker/Static.hs index 2d25c46ce56..c60e4cc594c 100644 --- a/compiler/GHC/Linker/Static.hs +++ b/compiler/GHC/Linker/Static.hs @@ -115,7 +115,7 @@ linkBinary' staticLink logger tmpfs dflags unit_env o_files dep_units = do ] get_pkg_lib_path_opts pkg l | osElfTarget (platformOS platform) && - (dynLibLoader dflags == SystemDependent || unitPackageNameString pkg == rts) && + (dynLibLoader dflags == SystemDependent || unitPackageNameString pkg == "rts") && ways_ `hasWay` WayDyn = let libpath = if gopt Opt_RelativeDynlibPaths dflags then "$ORIGIN" @@ -136,7 +136,7 @@ linkBinary' staticLink logger tmpfs dflags unit_env o_files dep_units = do else ["-Xlinker", "-rpath-link", "-Xlinker", l] in ["-L" ++ l] ++ rpathlink ++ rpath | osMachOTarget (platformOS platform) && - (dynLibLoader dflags == SystemDependent || unitPackageNameString pkg == rts) && + (dynLibLoader dflags == SystemDependent || unitPackageNameString pkg == "rts") && ways_ `hasWay` WayDyn && useXLinkerRPath dflags (platformOS platform) = let libpath = if gopt Opt_RelativeDynlibPaths dflags