[SRE-98] Migrate to haskell.nix

Fix failing tests - We need to cd into the correct directory before running the cardano-launcher tests. - Add required .yaml files as "extra-source-files" to the cardano-launcher cabal project. This allows Nix to find the data files required for the tests (when paired with keepSource = true). Co-authored-by: Samuel Evans-Powell <mail@sevanspowell.net> Co-authored-by: Rodney Lorrimar <rodney.lorrimar@iohk.io> Co-authored-by: Samuel Leathers <samuel.leathers@iohk.io>
input-output-hk · Jul 3, 2020 · eed2a02 · eed2a02
1 parent 601bb43
commit eed2a02
Show file tree

Hide file tree

Showing 16 changed files with 603 additions and 209 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -1,16 +1,25 @@
 steps:
+  # TODO: @craige fix this
   - label: 'stack coveralls coverage'
     command:
       - 'nix-shell -A runCoveralls'
+    soft_fail:
+      - exit_status: '*'
     agents:
       system: x86_64-linux
 
-  - label: 'check-hydra'
-    command: 'ci/check-hydra.sh'
+  - label: 'check-cabal-project'
+    command: 'nix-build ./nix -A iohkNix.checkCabalProject -o check-cabal-project.sh && ./check-cabal-project.sh'
+    agents:
+      system: x86_64-linux
+
+  - label: 'stack-cabal-sync'
+    command: 'nix-shell ./nix -A iohkNix.stack-cabal-sync-shell --run scripts/buildkite/stack-cabal-sync.sh'
     agents:
       system: x86_64-linux
 
   - label: 'stack2nix'
     command: 'ci/check-regenerate-nix.sh'
     agents:
       system: x86_64-linux
+
diff --git a/bors.toml b/bors.toml
@@ -1,5 +1,6 @@
 status = [
   "buildkite/cardano-shell",
+  "ci/hydra-eval",
   "ci/hydra:Cardano:cardano-shell:required",
 ]
 timeout_sec = 7200

diff --git a/cabal.project b/cabal.project
@@ -1,37 +1,59 @@
+index-state: 2020-04-01T00:00:00Z
+
 packages:
-  ./cardano-shell
-  ./cardano-launcher
+  cardano-shell
+  cardano-launcher
 
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/cardano-sl-x509
   tag: 43a036c5bbe68ca2e9cbe611eab7982e2348fe49
+  --sha256: 1aw7ns8lv51fjf45k8yzils2j7a6bqzy8hn65kb3z0dn1qsm8x88
 
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/cardano-prelude
   tag: 3c40edcf5bdba8721d3430d0aaaeea8770ce9bec
+  --sha256: 1z77nwjxj0v9gxhs3mlmqfq705mkkcpnwgr0d8shykjvf0iqdkcn
 
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/cardano-prelude
   tag: 3c40edcf5bdba8721d3430d0aaaeea8770ce9bec
+  --sha256: 1z77nwjxj0v9gxhs3mlmqfq705mkkcpnwgr0d8shykjvf0iqdkcn
   subdir: test
 
 source-repository-package
   type: git
   location: http://github.com/well-typed/canonical-json
   tag: ddfe3593b80b5ceb88842bb7a6f2268df75d2c2f
+  --sha256: 02fzn1xskis1lc1pkz0j92v6ipd89ww0k2p3dvwpm3yap5dpnm7k
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/ouroboros-network
+  tag: 7971422035c0c9e3e35358559b0d441baa24d1d2
+  --sha256: 1f1g8bnqv0j8vpy0cc8r18bp5xglk21n141309l218hgqlh12a9i
+  subdir: Win32-network
 
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/iohk-monitoring-framework
-  subdir: iohk-monitoring
   tag: 4956b32f039579a0e7e4fd10793f65b4c77d9044
+  --sha256: 03lyb2m4i6p7rpjqarnhsx21nx48fwk6rzsrx15k6274a4bv0pix
+  subdir: iohk-monitoring
 
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/iohk-monitoring-framework
+  tag: 4956b32f039579a0e7e4fd10793f65b4c77d9044
+  --sha256: 03lyb2m4i6p7rpjqarnhsx21nx48fwk6rzsrx15k6274a4bv0pix
   subdir:   contra-tracer
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/iohk-monitoring-framework
   tag: 4956b32f039579a0e7e4fd10793f65b4c77d9044
+  --sha256: 03lyb2m4i6p7rpjqarnhsx21nx48fwk6rzsrx15k6274a4bv0pix
+  subdir:   tracer-transformers
 
diff --git a/cardano-launcher/cardano-launcher.cabal b/cardano-launcher/cardano-launcher.cabal
@@ -11,6 +11,9 @@ copyright:      2018 IOHK
 license:        Apache-2.0
 license-files:  LICENSE, NOTICE
 build-type:     Simple
+extra-source-files:
+    configuration/launcher/jormungandr/launcher-config-qa.linux.yaml
+    configuration/launcher/jormungandr/launcher-config-qa.windows.yaml
 
 source-repository head
   type: git

diff --git a/default.nix b/default.nix
@@ -1,85 +1,61 @@
-#
-# The defaul.nix file. This will generate targets for all
-# buildables (see release.nix for nomenclature, excluding
-# the "build machine" last part, specific to release.nix), eg.:
-#
-# - nix build -f default.nix nix-tools.tests.cardano-shell # All `cardano-shell` tests
-# - nix build -f default.nix nix-tools.tests.cardano-shell.tests
-# - nix build -f default.nix nix-tools.exes.cardano-shell # All `cardano-shell` executables
-# - nix build -f default.nix nix-tools.cexes.cardano-launcher.cardano-launcher
-#
-# Generated targets include anything from stack.yaml (via
-# nix-tools:stack-to-nix and the nix/regenerate.sh script)
-# or cabal.project (via nix-tools:plan-to-nix), including all
-# version overrides specified there.
-#
-# Nix-tools stack-to-nix will generate the `nix/.stack-pkgs.nix`
-# file which is imported from the `nix/pkgs.nix` where further
-# customizations outside of the ones in stack.yaml/cabal.project
-# can be specified as needed for nix/ci.
-#
-# Please run `nix/regenerate.sh` after modifying stack.yaml
-# or relevant part of cabal configuration files.
-# When switching to recent stackage or hackage package version,
-# you might also need to update the iohk-nix common lib. You
-# can do so by running the `nix/update-iohk-nix.sh` script.
-#
-# More information about iohk-nix and nix-tools is available at:
-# https://github.com/input-output-hk/iohk-nix/blob/master/docs/nix-toolification.org#for-a-stackage-project
-#
-
-
-# We will need to import the iohk-nix common lib, which includes
-# the nix-tools tooling.
-let
-  commonLib = import ./nix/iohk-common.nix;
-in
-# This file needs to export a function that takes
-# the arguments it is passed and forwards them to
-# the default-nix template from iohk-nix. This is
-# important so that the release.nix file can properly
-# parameterize this file when targetting different
-# hosts.
 { system ? builtins.currentSystem
 , crossSystem ? null
+# allows to cutomize haskellNix (ghc and profiling, see ./nix/haskell.nix)
 , config ? {}
-, pkgs ? commonLib.getPkgs { inherit config crossSystem system; }
-, withHoogle ? true
+# override scripts with custom configuration
+, customConfig ? {}
+# allows to override dependencies of the project without modifications,
+# eg. to test build against local checkout of nixpkgs and iohk-nix:
+# nix build -f default.nix cardano-shell '{
+#   iohk-nix = ../iohk-nix;
+# }'
+, sourcesOverride ? {}
+# pinned version of nixpkgs augmented with overlays (iohk-nix and our packages).
+, pkgs ? import ./nix { inherit system crossSystem config sourcesOverride; }
+, gitrev ? pkgs.iohkNix.commitIdFromGitRepoOrZero ./.git
 }:
+with pkgs; with commonLib;
 let
-  # We will instantiate the default-nix template with the
-  # nix/pkgs.nix file...
-  defaultNix = commonLib.nix-tools.default-nix ./nix/pkgs.nix {
-    inherit system crossSystem config pkgs;
-  };
-in defaultNix // {
-  # ... and add additional packages we want to build on CI:
 
-  shell = defaultNix.nix-tools.shellFor {
-    inherit withHoogle;
-    # env will provide the dependencies of cardano-shell
-    packages = ps: with ps; [ cardano-shell ];
-    # This adds git to the shell, which is used by stack.
-    buildInputs = with pkgs; [
-      defaultNix.nix-tools._raw.cabal-install.components.exes.cabal
-      defaultNix.nix-tools._raw.ghcid.components.exes.ghcid
-      git
-      pkgconfig
-      stack
-      commonLib.stack-hpc-coveralls
-      systemd
-    ];
-  };
+  haskellPackages = recRecurseIntoAttrs
+    # the Haskell.nix package set, reduced to local packages.
+    (selectProjectPackages cardanoNodeHaskellPackages);
+
+  self = {
+    inherit haskellPackages hydraEvalErrors;
+
+    inherit (haskellPackages.cardano-shell.identifier) version;
+    # Grab the executable component of our package.
+    inherit (haskellPackages.cardano-shell.components.exes) node-ipc;
+    inherit (haskellPackages.cardano-launcher.components.exes) cardano-launcher;
+
+    inherit (pkgs.iohkNix) checkCabalProject;
+
+    # `tests` are the test suites which have been built.
+    tests = collectComponents' "tests" haskellPackages;
+    # `benchmarks` (only built, not run).
+    benchmarks = collectComponents' "benchmarks" haskellPackages;
+
+    checks = recurseIntoAttrs {
+      # `checks.tests` collect results of executing the tests:
+      tests = collectChecks haskellPackages;
+    };
+
+    runCoveralls = pkgs.stdenv.mkDerivation {
+      name = "run-coveralls";
+      buildInputs = with pkgs; [ commonLib.stack-hpc-coveralls stack ];
+      shellHook = ''
+        echo '~~~ stack nix test'
+        stack test --nix --coverage
+        echo '~~~ shc'
+        shc --repo-token=$COVERALLS_REPO_TOKEN combined all
+        exit
+      '';
+    };
 
-  runCoveralls = pkgs.stdenv.mkDerivation {
-    name = "run-coveralls";
-    buildInputs = with pkgs; [ commonLib.stack-hpc-coveralls stack ];
-    shellHook = ''
-      echo '~~~ stack nix test'
-      stack test --nix --coverage
-      echo '~~~ shc'
-      shc --repo-token=$COVERALLS_REPO_TOKEN combined all
-      exit
-    '';
-  };
-}
+    shell = import ./shell.nix {
+      inherit pkgs;
+      withHoogle = true;
+    };
+};
+in self
diff --git a/nix/default.nix b/nix/default.nix
@@ -0,0 +1,48 @@
+{ system ? builtins.currentSystem
+, crossSystem ? null
+, config ? {}
+, sourcesOverride ? {}
+}:
+let
+  sources = import ./sources.nix { inherit pkgs; }
+    // sourcesOverride;
+  iohkNix = import sources.iohk-nix {};
+  haskellNix = import sources."haskell.nix";
+  # use our own nixpkgs if it exists in our sources,
+  # otherwise use iohkNix default nixpkgs.
+  nixpkgs = if (sources ? nixpkgs)
+    then (builtins.trace "Not using IOHK default nixpkgs (use 'niv drop nixpkgs' to use default for better sharing)"
+      sources.nixpkgs)
+    else (builtins.trace "Using IOHK default nixpkgs"
+      iohkNix.nixpkgs);
+
+  # for inclusion in pkgs:
+  overlays =
+    # Haskell.nix (https://github.com/input-output-hk/haskell.nix)
+    haskellNix.overlays
+    # haskell-nix.haskellLib.extra: some useful extra utility functions for haskell.nix
+    ++ iohkNix.overlays.haskell-nix-extra
+    # iohkNix: nix utilities and niv:
+    ++ iohkNix.overlays.iohkNix
+    # our own overlays:
+    ++ [
+      (pkgs: _: with pkgs; {
+
+        # commonLib: mix pkgs.lib with iohk-nix utils and our own:
+        commonLib = lib // iohkNix // iohkNix.cardanoLib
+          // import ./util.nix { inherit haskell-nix; }
+          # also expose our sources and overlays
+          // { inherit overlays sources; };
+
+        svcLib = import ./svclib.nix { inherit pkgs; };
+      })
+      # And, of course, our haskell-nix-ified cabal project:
+      (import ./pkgs.nix)
+    ];
+
+  pkgs = import nixpkgs {
+    inherit system crossSystem overlays;
+    config = haskellNix.config // config;
+  };
+
+in pkgs
diff --git a/nix/haskell.nix b/nix/haskell.nix
@@ -0,0 +1,88 @@
+############################################################################
+# Builds Haskell packages with Haskell.nix
+############################################################################
+{ lib
+, stdenv
+, haskell-nix
+, buildPackages
+, config ? {}
+# GHC attribute name
+, compiler ? config.haskellNix.compiler or "ghc865"
+# Enable profiling
+, profiling ? config.haskellNix.profiling or false
+}:
+let
+  src = haskell-nix.haskellLib.cleanGit {
+    name = "cardano-shell";
+    src = ../.;
+  };
+
+  # This creates the Haskell package set.
+  # https://input-output-hk.github.io/haskell.nix/user-guide/projects/
+  pkgSet = haskell-nix.cabalProject (lib.optionalAttrs stdenv.hostPlatform.isWindows {
+    # FIXME: without this deprecated attribute, db-converter fails to compile directory with:
+    # Encountered missing dependencies: unix >=2.5.1 && <2.9
+    ghc = buildPackages.haskell-nix.compiler.${compiler};
+  } // {
+    inherit src;
+    modules = [
+      { compiler.nix-name = compiler; }
+
+      # Allow reinstallation of Win32
+      { nonReinstallablePkgs =
+        [ "rts" "ghc-heap" "ghc-prim" "integer-gmp" "integer-simple" "base"
+          "deepseq" "array" "ghc-boot-th" "pretty" "template-haskell"
+          # ghcjs custom packages
+          "ghcjs-prim" "ghcjs-th"
+          "ghc-boot"
+          "ghc" "array" "binary" "bytestring" "containers"
+          "filepath" "ghc-boot" "ghc-compact" "ghc-prim"
+          # "ghci" "haskeline"
+          "hpc"
+          "mtl" "parsec" "text" "transformers"
+          "xhtml"
+          # "stm" "terminfo"
+        ];
+
+      }
+      {
+        # These tests need files from the source tree.
+        packages.cardano-launcher.components.tests.cardano-launcher-test.keepSource = true;
+
+        # Packages we wish to ignore version bounds of.
+        # This is similar to jailbreakCabal, however it
+        # does not require any messing with cabal files.
+        packages.katip.doExactConfig = true;
+
+        # split data output for ekg to reduce closure size
+        packages.ekg.components.library.enableSeparateDataOutput = true;
+      }
+      (lib.optionalAttrs profiling {
+        enableLibraryProfiling = true;
+        packages.cardano-launcher.components.exes.cardano-launcher.enableExecutableProfiling = true;
+        profilingDetail = "default";
+      })
+      (lib.optionalAttrs stdenv.hostPlatform.isWindows {
+        # Make sure we use a buildPackages version of happy
+        packages.pretty-show.components.library.build-tools = [ buildPackages.haskell-nix.haskellPackages.happy ];
+
+        # Remove hsc2hs build-tool dependencies (suitable version will be available as part of the ghc derivation)
+        packages.Win32.components.library.build-tools = lib.mkForce [];
+        packages.terminal-size.components.library.build-tools = lib.mkForce [];
+        packages.network.components.library.build-tools = lib.mkForce [];
+
+
+        # Disable cabal-doctest tests by turning off custom setups
+        packages.comonad.package.buildType = lib.mkForce "Simple";
+        packages.distributive.package.buildType = lib.mkForce "Simple";
+        packages.lens.package.buildType = lib.mkForce "Simple";
+        packages.nonempty-vector.package.buildType = lib.mkForce "Simple";
+        packages.semigroupoids.package.buildType = lib.mkForce "Simple";
+      })
+    ];
+    # TODO add flags to packages (like cs-ledger) so we can turn off tests that will
+    # not build for windows on a per package bases (rather than using --disable-tests).
+    configureArgs = lib.optionalString stdenv.hostPlatform.isWindows "--disable-tests";
+  });
+in
+  pkgSet