From 44ecbc8ff8d10396dc67d5c27faa63806e9461ff Mon Sep 17 00:00:00 2001 From: Hamish Mackenzie Date: Fri, 19 Mar 2021 10:25:32 +1300 Subject: [PATCH 1/3] Improve cleaning of stack projects (#1074) Should help with #1013 --- lib/call-stack-to-nix.nix | 37 ++++++++++++++++++++++++++----------- lib/fetch-resolver.nix | 8 ++++++-- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/lib/call-stack-to-nix.nix b/lib/call-stack-to-nix.nix index 009aaf2e57..d755d8c9c7 100644 --- a/lib/call-stack-to-nix.nix +++ b/lib/call-stack-to-nix.nix @@ -23,9 +23,20 @@ let }) resolver fetchedResolver; subDir' = src.origSubDir or ""; + subDir = pkgs.lib.strings.removePrefix "/" subDir'; + maybeCleanedSource = + if haskellLib.canCleanSource src + then (haskellLib.cleanSourceWith { + name = if name != null then "${name}-root-cabal-files" else "source-root-cabal-files"; + src = src.origSrc or src; + filter = path: type: (!(src ? filter) || src.filter path type) && ( + type == "directory" || + pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ stackYaml ".cabal" "package.yaml" ]); }) + else src.origSrc or src; + stackToNixArgs = builtins.concatStringsSep " " [ "--full" - "--stack-yaml=$SRC/${stackYaml}" + "--stack-yaml=$SRC${subDir'}/${stackYaml}" (if ignorePackageYaml then "--ignore-package-yaml" else "") "-o ." ]; @@ -46,18 +57,22 @@ let preferLocalBuild = false; } ('' mkdir -p $out${subDir'} + SRC=$(mktemp -d) + cd $SRC + # if maybeCleanedSource is empty, this means it's a new + # project where the files haven't been added to the git + # repo yet. We fail early and provide a useful error + # message to prevent headaches (#290). + if [ -z "$(ls -A ${maybeCleanedSource})" ]; then + echo "cleaned source is empty. Did you forget to 'git add -A'?"; exit 1; + fi + lndir -silent "${maybeCleanedSource}/." $SRC + ${pkgs.lib.optionalString (subDir != "") "cd ${subDir}"} ${ - # If no resolver was fetched use the original stack.yaml - if fetchedResolver == null - then '' - SRC=${src} - '' - else + # If a resolver was fetched use the it instead of the original stack.yaml + pkgs.lib.optionalString (fetchedResolver != null) # Replace the resolver path in the stack.yaml with the fetched version '' - SRC=$(mktemp -d) - cd $SRC - lndir -silent "${src}/." $SRC rm ${stackYaml} cp ${src}/${stackYaml} . chmod +w ${stackYaml} @@ -71,7 +86,7 @@ let # We need to strip out any references to $src, as those won't # be accessable in restricted mode. for nixf in $(find $out -name "*.nix" -type f); do - substituteInPlace $nixf --replace "$SRC" "." + substituteInPlace $nixf --replace "$SRC${subDir'}" "." done # move pkgs.nix to default.nix ensure we can just nix `import` the result. diff --git a/lib/fetch-resolver.nix b/lib/fetch-resolver.nix index 31a58a7d5d..c9b55e35a5 100644 --- a/lib/fetch-resolver.nix +++ b/lib/fetch-resolver.nix @@ -5,10 +5,12 @@ , resolverSha256 ? null }: let + srcDir = src.origSrcSubDir or src; + # Using origSrcSubDir bypasses any cleanSourceWith so that it will work when # access to the store is restricted. If origSrc was already in the store # you can pass the project in as a string. - rawStackYaml = builtins.readFile ((src.origSrcSubDir or src) + ("/" + stackYaml)); + rawStackYaml = builtins.readFile (srcDir + "/${stackYaml}"); # Determine the resolver as it may point to another file we need # to look at. @@ -28,6 +30,8 @@ let url = resolver; sha256 = resolverSha256; } - else null; + else if resolver != null && __pathExists (srcDir + "/${resolver}") + then pkgs.copyPathToStore (srcDir + "/${resolver}") + else null; in { inherit resolver fetchedResolver; } From 96d769343a1aeb9fcabc92aee55876800294ff98 Mon Sep 17 00:00:00 2001 From: IOHK Date: Fri, 19 Mar 2021 01:16:46 +0000 Subject: [PATCH 2/3] Update Hackage and Stackage --- hackage-src.json | 8 ++++---- stackage-src.json | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hackage-src.json b/hackage-src.json index db981fb9fc..478a6fd23e 100644 --- a/hackage-src.json +++ b/hackage-src.json @@ -1,9 +1,9 @@ { "url": "https://github.com/input-output-hk/hackage.nix", - "rev": "48960ebaabc5f7f9433b2ed6d5dea66550f5e99e", - "date": "2021-03-18T01:14:50+00:00", - "path": "/nix/store/1d449n54vzgyw7sljy1f97yk1nign6ih-hackage.nix-48960eb", - "sha256": "024nxylwxikmlbwlniigzp3rcxyksjran1xpig2z7ynixpzwb0k7", + "rev": "abd1caa9acdab27fdad18b01f9a99d462e68b20b", + "date": "2021-03-19T01:15:05+00:00", + "path": "/nix/store/6yk9rl00fq53iwb132891iz5hl9wsjr7-hackage.nix-abd1caa", + "sha256": "0sxbhvqx2j1rdza739m5agmq1c2byd7hqzbpg6pksd1iy194ysv0", "fetchSubmodules": false, "deepClone": false, "leaveDotGit": false diff --git a/stackage-src.json b/stackage-src.json index a7404d9337..2b5a44e121 100644 --- a/stackage-src.json +++ b/stackage-src.json @@ -1,9 +1,9 @@ { "url": "https://github.com/input-output-hk/stackage.nix", - "rev": "41d84a92f0cd6c5b0d3759ce545e970cba78a44e", - "date": "2021-03-18T01:06:22+00:00", - "path": "/nix/store/0fphb4na5ff2cdrkjm2vjlrx352lhjqq-stackage.nix-41d84a9", - "sha256": "1mda55si7h2cc4nv10nkrqghm80mm7j7x9s3z593abixa9cr33lm", + "rev": "736c6784ddc081688c40fd9c4be1fad22360fe42", + "date": "2021-03-19T01:05:52+00:00", + "path": "/nix/store/aqr73x5kbx2b87p5x6hcag0kzrqv9b3l-stackage.nix-736c678", + "sha256": "17x0aakrcpv98abkyrr1drfhjb1q0hscg2aqksy1kw4nq411551q", "fetchSubmodules": false, "deepClone": false, "leaveDotGit": false From 1abbd1656db9feef08d7ba3a27360d24148d142d Mon Sep 17 00:00:00 2001 From: Hamish Mackenzie Date: Fri, 19 Mar 2021 17:01:45 +1300 Subject: [PATCH 3/3] Add GHC 8.10 Make markLiveObject thread-safe (#1076) See https://github.com/input-output-hk/haskell.nix/issues/1062 --- overlays/bootstrap.nix | 2 + ...-rts-make-markLiveObject-thread-safe.patch | 63 +++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 overlays/patches/ghc/ghc-8.10.3-rts-make-markLiveObject-thread-safe.patch diff --git a/overlays/bootstrap.nix b/overlays/bootstrap.nix index c08b8023d1..df80eb32da 100644 --- a/overlays/bootstrap.nix +++ b/overlays/bootstrap.nix @@ -155,6 +155,8 @@ in { ++ fromUntil "8.10.1" "8.10.3" ./patches/ghc/ghc-8.10-ubxt.patch ++ fromUntil "8.10.3" "8.11" ./patches/ghc/ghc-8.10.3-ubxt.patch ++ final.lib.optional (versionAtLeast "8.6.4") ./patches/ghc/Cabal-3886.patch + + ++ fromUntil "8.10.3" "8.10.5" ./patches/ghc/ghc-8.10.3-rts-make-markLiveObject-thread-safe.patch ; in ({ ghc844 = final.callPackage ../compiler/ghc { diff --git a/overlays/patches/ghc/ghc-8.10.3-rts-make-markLiveObject-thread-safe.patch b/overlays/patches/ghc/ghc-8.10.3-rts-make-markLiveObject-thread-safe.patch new file mode 100644 index 0000000000..1b3bb0e29a --- /dev/null +++ b/overlays/patches/ghc/ghc-8.10.3-rts-make-markLiveObject-thread-safe.patch @@ -0,0 +1,63 @@ +From 96f8e2a47c5e53ae5fb86739aecd27c502e7f121 Mon Sep 17 00:00:00 2001 +From: Ben Gamari +Date: Tue, 23 Feb 2021 18:30:48 +0000 +Subject: [PATCH] rts: Make markLiveObject thread-safe + +markLiveObject is called by GC worker threads and therefore must be +thread-safe. This was a rather egregious oversight which the testsuite +missed. + + +(cherry picked from commit fe28a062e47bd914a6879f2d01ff268983c075ad) +--- + rts/CheckUnload.c | 10 ++++++++-- + rts/LinkerInternals.h | 2 +- + 2 files changed, 9 insertions(+), 3 deletions(-) + +diff --git a/rts/CheckUnload.c b/rts/CheckUnload.c +index 8f834d13fa..345a17cfec 100644 +--- a/rts/CheckUnload.c ++++ b/rts/CheckUnload.c +@@ -381,11 +381,16 @@ static ObjectCode *findOC(OCSectionIndices *s_indices, const void *addr) { + + static bool markObjectLive(void *data STG_UNUSED, StgWord key, const void *value STG_UNUSED) { + ObjectCode *oc = (ObjectCode*)key; +- if (oc->mark == object_code_mark_bit) { ++ ++ // N.B. we may be called by the parallel GC and therefore this must be ++ // thread-safe. To avoid taking the linker_mutex in the fast path ++ // (when the object is already marked) we do an atomic exchange here and ++ // only take the lock in the case that the object is unmarked. ++ if (xchg(&oc->mark, object_code_mark_bit) == object_code_mark_bit) { + return true; // for hash table iteration + } + +- oc->mark = object_code_mark_bit; ++ ACQUIRE_LOCK(&linker_mutex); + // Remove from 'old_objects' list + if (oc->prev != NULL) { + // TODO(osa): Maybe 'prev' should be a pointer to the referencing +@@ -405,6 +410,7 @@ static bool markObjectLive(void *data STG_UNUSED, StgWord key, const void *value + objects->prev = oc; + } + objects = oc; ++ RELEASE_LOCK(&linker_mutex); + + // Mark its dependencies + iterHashTable(oc->dependencies, NULL, markObjectLive); +diff --git a/rts/LinkerInternals.h b/rts/LinkerInternals.h +index 44fe337802..444849fbac 100644 +--- a/rts/LinkerInternals.h ++++ b/rts/LinkerInternals.h +@@ -262,7 +262,7 @@ struct _ObjectCode { + struct _ObjectCode *next_loaded_object; + + // Mark bit +- uint8_t mark; ++ StgWord mark; + + // Set of dependencies (ObjectCode*) of the object file. Traverse + // dependencies using `iterHashTable`. +-- +GitLab +