Skip to content
Permalink
Browse files

Merge #714

714: SRE-35 haskell overhaul r=disassembler a=craigem

This is the overhaul of Haskell for `cardano-ledger` along the lines of `cardano-node` [PR #507](input-output-hk/cardano-node#507).

Co-authored-by: Craige McWhirter <craige.mcwhirter@iohk.io>
  • Loading branch information
iohk-bors and craigem committed Feb 14, 2020
2 parents 86f0cb4 + d05a8be commit f05953511db1b4770c209095ace624afbd471826
Showing with 658 additions and 1,696 deletions.
  1. +1 −1 .buildkite/nightly-tests.sh
  2. +9 −11 .buildkite/pipeline.yml
  3. +26 −39 README.md
  4. +1 −1 bors.toml
  5. +24 −2 cabal.project
  6. +3 −0 ci/check-hydra.sh
  7. +43 −43 default.nix
  8. +41 −0 lib.nix
  9. +0 −45 nix/.stack.nix/cardano-binary-test.nix
  10. +0 −66 nix/.stack.nix/cardano-binary.nix
  11. +0 −55 nix/.stack.nix/cardano-crypto-class.nix
  12. +0 −34 nix/.stack.nix/cardano-crypto-test.nix
  13. +0 −59 nix/.stack.nix/cardano-crypto-wrapper.nix
  14. +0 −88 nix/.stack.nix/cardano-crypto.nix
  15. +0 −55 nix/.stack.nix/cardano-ledger-test.nix
  16. +0 −119 nix/.stack.nix/cardano-ledger.nix
  17. +0 −49 nix/.stack.nix/cardano-prelude-test.nix
  18. +0 −80 nix/.stack.nix/cardano-prelude.nix
  19. +0 −125 nix/.stack.nix/cardano-shell.nix
  20. +0 −61 nix/.stack.nix/cardano-sl-x509.nix
  21. +0 −31 nix/.stack.nix/contra-tracer.nix
  22. +0 −57 nix/.stack.nix/cs-blockchain.nix
  23. +0 −76 nix/.stack.nix/cs-ledger.nix
  24. +0 −41 nix/.stack.nix/default.nix
  25. +0 −56 nix/.stack.nix/goblins.nix
  26. +0 −163 nix/.stack.nix/iohk-monitoring.nix
  27. +0 −82 nix/.stack.nix/small-steps.nix
  28. +0 −35 nix/.stack.nix/validate-mainnet.nix
  29. +46 −0 nix/default.nix
  30. +91 −0 nix/haskell.nix
  31. +0 −7 nix/iohk-nix-src.json
  32. +0 −18 nix/lib.nix
  33. +11 −77 nix/pkgs.nix
  34. +1 −1 nix/regenerate.sh
  35. +38 −0 nix/sources.json
  36. +134 −0 nix/sources.nix
  37. +5 −9 nix/stack-shell.nix
  38. +2 −8 nix/update-iohk-nix.sh
  39. +14 −0 nix/util.nix
  40. +65 −59 release.nix
  41. +3 −7 scripts/buildkite/default.nix
  42. +26 −0 scripts/buildkite/stack-cabal-sync.sh
  43. +0 −31 scripts/buildkite/stack-hpc-coveralls.nix
  44. +12 −0 scripts/nix/stack-shell.nix
  45. +62 −5 shell.nix
@@ -1,3 +1,3 @@
#!/usr/bin/env bash
nix build -f `dirname $0`/../default.nix nix-tools.tests.cardano-ledger.cardano-ledger-test -o cardano-ledger-test
nix build -f `dirname $0`/.. haskellPackages.cardano-ledger.components.tests.cardano-ledger-test -o cardano-ledger-test
cd cardano-ledger && ../cardano-ledger-test/bin/cardano-ledger-test --scenario=QualityAssurance
@@ -1,10 +1,13 @@
steps:
- label: 'hydra-eval-errors'
command: 'nix-build ./nix -A iohkNix.hydraEvalErrors && ./result/bin/hydra-eval-errors.py'
agents:
system: x86_64-linux

- label: 'stack rebuild'
env:
STACK_ROOT: "/build/cardano-ledger.stack"
command:
# cache-s3 needs a build directory that is the same across all buildkite agents.
# so copy the source into /build/cardano-ledger
- "rm -rf /build/cardano-ledger"
- "cp -R . /build/cardano-ledger"
- "cd /build/cardano-ledger"
@@ -13,17 +16,12 @@ steps:
agents:
system: x86_64-linux

- label: Check Hydra evaluation of release.nix
command: 'nix-build -A _lib.check-hydra -o check-hydra.sh && ./check-hydra.sh'
agents:
system: x86_64-linux

- label: Check auto-generated Nix
command: 'nix-build -A _lib.check-nix-tools -o check-nix-tools.sh && ./check-nix-tools.sh'
- label: 'stack-cabal-sync'
command: 'nix-shell ./nix -A iohkNix.stack-cabal-sync-shell --run scripts/buildkite/stack-cabal-sync.sh'
agents:
system: x86_64-linux

- label: Check that cabal.project and stack.yaml have consistent git hashes for dependencies
command: 'nix-shell --run scripts/buildkite/stack-cabal_config_check.sh'
- label: 'check-cabal-project'
command: 'nix-build ./nix -A iohkNix.checkCabalProject -o check-cabal-project.sh && ./check-cabal-project.sh'
agents:
system: x86_64-linux
@@ -45,12 +45,12 @@ build` or `cabal new-build`.
Alternatively you can use `nix` to install the external dependencies. For
`stack` simply add the `--nix` flag to your invocation of `stack build`. If
you're on `NixOS` this will happen automatically. For `cabal`, you can run
`nix-shell nix/stack-shell.nix` to enter a shell with the dependencies,
`nix-shell` to enter a shell with the dependencies,
and use `cabal new-build` as normal from there.

You can build directly with `nix`, by running `nix-build -A
nix-tools.libs.cardano-ledger`. To run the test executable you must first build
it with `nix-build -A nix-tools.tests.cardano-ledger.cardano-ledger-test -o
libs.cardano-ledger`. To run the test executable you must first build
it with `nix-build -A tests.cardano-ledger.cardano-ledger-test -o
cardano-ledger-test` and then run it with `./cardano-ledger-test`.


@@ -116,7 +116,7 @@ particular needs of the work being done. Such file would be a copy of the

## Updating GHC and Package Dependencies

`nix` building is handled by `nix-tools`, which generates `nix` infrastructure
`nix` building is handled by `haskell.nix`, which generates `nix` infrastructure
from `stack` and `cabal` files. To generate the infrastructure, run
`nix/regenerate.sh`.

@@ -186,55 +186,42 @@ The `nix` directory contains files related to `iohk-nix`, the `nix`-based
infrastructure were using to manage our dependencies and build on hydra. The
important files are:

- `nix/iohk-nix-src.json`, which contains JSON pointing to the `iohk-nix` source
- `nix/sources.json`, which contains JSON pointing to external nix
dependencies (eg. `iohk-nix`), designed to be updated with `niv`
(available in `nix-shell`).

- `nix/lib.nix`, which imports `iohk-nix` and creates and attribute set
containing its `lib`, `pkgs`, and `nix-tools` attributes
- `nix/default.nix`, which imports `nixpkgs` with overlays
including `iohkNix`, `commonLib` and `cardanoLedgerHaskellPackages`.

- `nix/regenerate.sh`, which generates `nix` expressions for all the Haskell
dependencies in the Stack project
- `nix/regenerate.sh`, which update `--sha256` in `cabal.project`.

- `nix/update-iohk-nix.sh`, which updates the `iohk-nix-src.json` according to
the latest version of [`iohk-nix`](https://github.com/input-output-hk/iohk-nix/)

- `nix/pkgs.nix`, which creates a package set from the generated `nix`
expressions
- `nix/haskell.nix`, which creates a package set from the cabal.project
using the [haskell.nix](https://github.com/input-output-hk/haskell.nix#quickstart)
`cabalProject` function.

- `default.nix`, the top-level `nix` expression for the project, based on the
generated package set

- `release.nix`, the specification for which packages to build on hydra

There are a couple of common issues that developers run into while working with
`iohk-nix`:

- `error: attribute 'ghc864' missing, at (string):1:43` is an error that you
will usually get if you're trying to evaluate some `nix` expression using your
system-wide `nixpkgs` and the version of `ghc` there doesn't match the one
that you're trying to build with. This is usually solved by getting the
offending `.nix` file to `import ./nix/lib.nix`. Sometime this occurs while
running `stack` and this means `stack` is running with the `--nix` flag. In
this case, you need to make sure you have a `shell.nix` file for `stack` that
points to `nix/lib.nix`. Then add:
```
nix:
shell-file: nix/stack-shell.nix
```
to your `stack.yaml`.
`haskell.nix`:

- While evaluating `release.nix`, e.g. using the
`scripts/buildkite/check-hydra.sh` script, you might see
- While evaluating `release.nix` or `default.nix`, you might see
```
"error": "attribute '1.0.0.0' missing, at /path/to/nix/.stack.nix/default.nix:6:30"
"error": "attribute '1.0.0.0' missing, at /nix/store/.../default.nix:6:30"
```
which indicates that a package version is missing from the `iohk-nix` package
set. This is usually because the version is new to Hackage, and so it has only
recently made it into
which indicates that a package version is missing from the `hackage.nix`
package set. This is usually because the version is new to Hackage, and so
it has only recently made it into
[`hackage.nix`](https://github.com/input-output-hk/hackage.nix/), a set of
`nix` expressions for all the packages on Hackage. This is pinned in
`iohk-nix`, so to solve this, submit a PR to `iohk-nix` updating the revision
of `pins/haskell-nix.json`, or get IOHK DevOps to submit this PR for you if
you're unsure how to do that.
`nix` expressions for all the packages on Hackage.
`haskell.nix` is regularly updated with latest version of `hackage.nix`, so
to solve this error use the following command:
```
nix-shell -A devops --run 'niv update haskell.nix'
```



<hr/>
@@ -5,7 +5,7 @@ status = [
# Hydra: build for required jobset
"ci/hydra:Cardano:cardano-ledger:required",
]
timeout_sec = 5400
timeout_sec = 10800
required_approvals = 1
delete_merged_branches = true
block_labels = [ "DO NOT MERGE", "wip" ]
@@ -1,3 +1,5 @@
index-state: 2020-01-31T00:00:00Z

packages:
cardano-ledger cardano-ledger/test
crypto crypto/test
@@ -6,67 +8,87 @@ package cardano-ledger
tests: True

constraints:
hashable == 1.2.*, network == 3.0.*, primitive == 0.6.*, cborg == 0.2.2.*
ip < 1.5,
hedgehog >= 1.0,
bimap >= 0.4.0,
brick >= 0.47,
hashable == 1.2.*,
network == 3.0.*,
primitive == 0.6.*,
cborg == 0.2.2.*

allow-newer: katip:Win32

source-repository-package
type: git
location: https://github.com/input-output-hk/cardano-prelude
tag: 00487726c4bc21b4744e59d913334ebfeac7d68e
--sha256: 0v4fcq5kdd2r5dgwys8kv46ff33qp756n26ycxrca10wq14zkwm5

source-repository-package
type: git
location: https://github.com/input-output-hk/cardano-prelude
tag: 00487726c4bc21b4744e59d913334ebfeac7d68e
--sha256: 0v4fcq5kdd2r5dgwys8kv46ff33qp756n26ycxrca10wq14zkwm5
subdir: test

source-repository-package
type: git
location: https://github.com/input-output-hk/cardano-base
tag: 4f24cf8d6686ac764c6ddbc188c88d9296497a50
--sha256: 1ydv47r7b6vhmjxhr71111sbrpbn3mz0fp5266l4ngr8ca1f8i06
subdir: binary

source-repository-package
type: git
location: https://github.com/input-output-hk/cardano-base
tag: 4f24cf8d6686ac764c6ddbc188c88d9296497a50
--sha256: 1ydv47r7b6vhmjxhr71111sbrpbn3mz0fp5266l4ngr8ca1f8i06
subdir: cardano-crypto-class

source-repository-package
type: git
location: https://github.com/input-output-hk/cardano-base
tag: 4f24cf8d6686ac764c6ddbc188c88d9296497a50
--sha256: 1ydv47r7b6vhmjxhr71111sbrpbn3mz0fp5266l4ngr8ca1f8i06
subdir: binary/test

source-repository-package
type: git
location: https://github.com/input-output-hk/cardano-crypto
tag: 4590efa638397e952a51a8994b5543e4ea3c1ecd
--sha256: 0hl2n3bba5v2j0lmxhs7hs01z3aznh2bwf9cb434icq2g0bl8ms3

source-repository-package
type: git
location: https://github.com/input-output-hk/cardano-ledger-specs
tag: 2aa807f4a4b6fd0ae0a511289586a2c2d18b56fb
--sha256: 0mb6r0sw1526m5cd4xjml8lc8vrjdql8plvfxhf9v6ifcv2218qz
subdir: byron/semantics/executable-spec

source-repository-package
type: git
location: https://github.com/input-output-hk/cardano-ledger-specs
tag: 2aa807f4a4b6fd0ae0a511289586a2c2d18b56fb
--sha256: 0mb6r0sw1526m5cd4xjml8lc8vrjdql8plvfxhf9v6ifcv2218qz
subdir: byron/ledger/executable-spec

source-repository-package
type: git
location: https://github.com/input-output-hk/cardano-ledger-specs
tag: 2aa807f4a4b6fd0ae0a511289586a2c2d18b56fb
--sha256: 0mb6r0sw1526m5cd4xjml8lc8vrjdql8plvfxhf9v6ifcv2218qz
subdir: byron/chain/executable-spec

source-repository-package
type: git
location: https://github.com/input-output-hk/iohk-monitoring-framework
subdir: contra-tracer
tag: dd30455144e11efb435619383ba84ce02aee720d
--sha256: 1g08bg99fvss99kg27l7pmxm7lh60573xln8l8x2rzvwfvfgk2i5
subdir: contra-tracer

source-repository-package
type: git
location: https://github.com/input-output-hk/goblins
tag: 26d35ad52fe9ade3391532dbfeb2f416f07650bc
--sha256: 17p5x0hj6c67jkdqx0cysqlwq2zs2l87azihn1alzajy9ak6ii0b
@@ -0,0 +1,3 @@
#!/usr/bin/env bash

eval "$(nix-build -A check-hydra lib.nix --no-out-link)"
@@ -1,44 +1,44 @@
#
# The default.nix file. This will generate targets for all
# buildables. These include anything from stack.yaml
# (via nix-tools:stack-to-nix) or cabal.project (via
# nix-tools:plan-to-nix). As well as custom definitions
# on top.
#
# nix-tools stack-to-nix or plan-to-nix will generate
# the `nix/plan.nix` file. Where further customizations
# outside of the ones in stack.yaml/cabal.project can
# be specified as needed for nix/ci.
#

# We will need to import the local lib, which will
# give us the iohk-nix tooling, which also includes
# the nix-tools tooling.
let
localLib = import ./nix/lib.nix;
in
# This file needs to export a function that takes
# the arguments it is passed and forwards them to
# the default-nix template from iohk-nix. This is
# important so that the release.nix file can properly
# parameterize this file when targetting different
# hosts.
{ withHoogle ? true
, ... }@args:
# We will instantiate the default-nix template with the
# nix/pkgs.nix file...
{ system ? builtins.currentSystem
, crossSystem ? null
# allows to cutomize haskellNix (ghc and profiling, see ./nix/haskell.nix)
, config ? {}
# allows to override dependencies of the project without modifications,
# eg. to test build against local checkout of nixpkgs and iohk-nix:
# nix build -f default.nix cardano-ledger --arg sourcesOverride '{
# iohk-nix = ../iohk-nix;
# }'
, sourcesOverride ? {}
# pinned version of nixpkgs augmented with overlays (iohk-nix and our packages).
, pkgs ? import ./nix { inherit system crossSystem config sourcesOverride; }
, gitrev ? pkgs.iohkNix.commitIdFromGitRepoOrZero ./.git
}:
with pkgs; with commonLib;
let
defaultNix = localLib.nix-tools.default-nix ./nix/pkgs.nix args;
in defaultNix //
{
shell = defaultNix.nix-tools.shellFor {
inherit withHoogle;
# env will provide the dependencies of cardano-shell
packages = ps: with ps; [ cardano-ledger ];
# This adds git to the shell, which is used by stack.
buildInputs = [
defaultNix.nix-tools._raw.cabal-install.components.exes.cabal
localLib.iohkNix.cardano-repo-tool
];
};
}

haskellPackages = recRecurseIntoAttrs
# the Haskell.nix package set, reduced to local packages.
(selectProjectPackages cardanoLedgerHaskellPackages);

self = {
inherit haskellPackages;

inherit (haskellPackages.cardano-ledger.identifier) version;

# `tests` are the test suites which have been built.
tests = collectComponents' "tests" haskellPackages;
# `benchmarks` (only built, not run).
benchmarks = collectComponents' "benchmarks" haskellPackages;

libs = collectComponents' "library" haskellPackages;

checks = recurseIntoAttrs {
# `checks.tests` collect results of executing the tests:
tests = collectChecks haskellPackages;
};

shell = import ./shell.nix {
inherit pkgs;
withHoogle = true;
};
};
in self
41 lib.nix
@@ -0,0 +1,41 @@
{ system ? builtins.currentSystem
, crossSystem ? null
, config ? {}
, overlays ? []
}:

let
sources = import ./nix/sources.nix;
iohkNix = import sources.iohk-nix {
sourcesOverride = sources;
};
haskellNix = import sources."haskell.nix";
args = haskellNix // {
inherit system crossSystem;
overlays = (haskellNix.overlays or []) ++ overlays;
config = (haskellNix.config or {}) // config;
};
nixpkgs = import sources.nixpkgs;
pkgs = nixpkgs args;
haskellPackages = import ./nix/pkgs.nix {
inherit pkgs;
src = ./.;
};
lib = pkgs.lib;
niv = (import sources.niv {}).niv;
isCardanoLedger = with lib; package:
(package.isHaskell or false);
filterCardanoPackages = pkgs.lib.filterAttrs (_: package: isCardanoLedger package);
getPackageChecks = pkgs.lib.mapAttrs (_: package: package.checks);
in lib // {
inherit (pkgs.haskell-nix.haskellLib) collectComponents;
inherit
niv
sources
haskellPackages
pkgs
iohkNix
isCardanoLedger
getPackageChecks
filterCardanoPackages;
}

0 comments on commit f059535

Please sign in to comment.
You can’t perform that action at this time.