Skip to content

Commit

Permalink
Flakify nix build.
Browse files Browse the repository at this point in the history
  • Loading branch information
jbgi committed Jan 18, 2022
1 parent a5085ac commit a975490
Show file tree
Hide file tree
Showing 172 changed files with 2,753 additions and 2,828 deletions.
2 changes: 1 addition & 1 deletion .buildkite/bench-db.sh
Expand Up @@ -8,7 +8,7 @@ bench_name=bench-db
rm -rf $bench_name

echo "--- Build"
nix-build -A benchmarks.cardano-wallet-core.db -o $bench_name
nix build .#benchmarks.cardano-wallet-core.db -o $bench_name

echo "+++ Run benchmark"

Expand Down
2 changes: 1 addition & 1 deletion .buildkite/bench-latency.sh
Expand Up @@ -6,7 +6,7 @@ set -euo pipefail
cd `dirname $0`/..

echo "--- Build"
nix-build -A benchmarks.cardano-wallet.latency -o bench-latency-shelley
nix build .#benchmarks.cardano-wallet.latency -o bench-latency-shelley

# Note: the tracing will not work if program output is piped
# to another process (e.g. "tee").
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/bench-restore.sh
Expand Up @@ -24,7 +24,7 @@ fi

echo "--- Build"

nix-build -A benchmarks.cardano-wallet.restore -o bench-restore
nix build .#benchmarks.cardano-wallet.restore -o bench-restore

bench="./bench-restore/bin/restore $network --node-db $node_db"

Expand Down
12 changes: 6 additions & 6 deletions .buildkite/default.nix
@@ -1,4 +1,9 @@
{ pkgs ? import ../nix/default.nix {} }:
{ pkgs ? (import ../default.nix {}).legacyPackages.${builtins.currentSystem}.pkgs
# Include the stack nix-shell in closure of stackRebuild, so that it
# doesn't get garbage-collected whilst the build is running.
# https://github.com/commercialhaskell/stack/issues/3479
, stackShell ? import ../nix/stack-shell.nix {}
}:

with pkgs.lib;
with pkgs;
Expand All @@ -12,11 +17,6 @@ let
libs = ps: with ps; [turtle safe transformers extra async digest];
ghc' = haskellPackages.ghcWithPackages libs;

# Include the stack nix-shell in closure of stackRebuild, so that it
# doesn't get garbage-collected whilst the build is running.
# https://github.com/commercialhaskell/stack/issues/3479
stackShell = import ../nix/stack-shell.nix {};

stackRebuild = runCommand "stack-rebuild" {
buildInputs = [ ghc' makeWrapper ];
} ''
Expand Down
7 changes: 3 additions & 4 deletions .buildkite/docker-build-push.nix
Expand Up @@ -25,10 +25,9 @@
# recent VERSION tag build (shelley backend).
#

{ walletPackages ? import ../default.nix {}
, pkgs ? walletPackages.private.pkgs
# TODO: take the musl build instead because it's smaller
, dockerImage ? walletPackages.dockerImage
{ defaultNix ? import ../default.nix {}
, pkgs ? defaultNix.legacyPackages.pkgs
, dockerImage ? defaultNix.hydraJobs.linux.musl.dockerImage

# Build system's Nixpkgs. We use this so that we have the same docker
# version as the docker daemon.
Expand Down
4 changes: 3 additions & 1 deletion .buildkite/nightly.yml
Expand Up @@ -64,7 +64,9 @@ steps:
# if: 'build.env("step") == null || build.env("step") =~ /migration-tests/'

- label: 'Full cabal build'
command: 'nix-shell nix/cabal-shell.nix --arg withCabalCache true --run "scripts/buildkite/cabal-ci.sh build"'
command:
- "mkdir -p config && echo '{ outputs = _: { withCabalCache = true; }; }' > config/flake.nix"
- "nix develop --override-input customConfig path:./config .#cabal --command scripts/buildkite/cabal-ci.sh build"
env:
CABAL_CACHE_ARCHIVE: "/cache/cardano-wallet"
CABAL_STORE_DIR: "/build/cardano-wallet.store"
Expand Down
21 changes: 11 additions & 10 deletions .buildkite/pipeline.yml
Expand Up @@ -26,7 +26,7 @@ steps:
- label: 'Stack Rebuild'
command:
- "rm -rf $TESTS_LOGDIR && mkdir -p $TESTS_LOGDIR"
- "nix-build .buildkite/default.nix -o sr"
- "nix build .#hydraJobs.linux.native.buildkiteScript -o sr"
- "./sr/bin/rebuild --build-dir=$BUILD_DIR --cache-dir=$CACHE_DIR"
timeout_in_minutes: 120
artifact_paths:
Expand All @@ -36,46 +36,47 @@ steps:
system: x86_64-linux

- label: 'Check Cabal Configure'
command: 'nix-shell --pure nix/cabal-shell.nix --run "scripts/buildkite/cabal-ci.sh configure"'
command: 'nix develop .#cabal --command scripts/buildkite/cabal-ci.sh configure'
agents:
system: x86_64-linux

- label: 'Check auto-generated Nix'
key: nix
commands:
- './nix/regenerate.sh --cache /dev/null'
- 'nix-build ./nix -A iohkNix.checkStackProject -o check-stack-project.sh && ./check-stack-project.sh'
- './nix/regenerate.sh'
- 'nix build .#checkStackProject -o check-stack-project.sh && ./check-stack-project.sh'
agents:
system: x86_64-linux

- label: 'Check Cabal Configure (Haskell.nix shellFor)'
depends_on: nix
command: 'nix-shell --run scripts/buildkite/check-haskell-nix-cabal.sh'
command: 'nix develop --command scripts/buildkite/check-haskell-nix-cabal.sh'
agents:
system: x86_64-linux

- label: 'Check Stylish Haskell'
depends_on: nix
command: 'nix-shell --run .buildkite/check-stylish.sh'
command: 'nix develop --command .buildkite/check-stylish.sh'
agents:
system: x86_64-linux

- label: 'HLint'
depends_on: nix
command: 'nix-shell --run "echo +++ HLint ; hlint lib"'
command: 'nix develop --command bash -c "echo +++ HLint ; hlint lib"'
agents:
system: x86_64-linux

- label: 'Validate OpenAPI Specification'
depends_on: nix
command: 'nix-shell --run "echo +++ openapi-spec-validator ; openapi-spec-validator --schema 3.0.0 specifications/api/swagger.yaml"'
command: 'nix develop --command bash -c "echo +++ openapi-spec-validator ; openapi-spec-validator --schema 3.0.0 specifications/api/swagger.yaml"'
agents:
system: x86_64-linux

- label: 'Docker Image'
depends_on: nix
command:
- "nix-build .buildkite/docker-build-push.nix --argstr dockerHubRepoName inputoutput/cardano-wallet -o docker-build-push"
- "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"inputoutput/cardano-wallet\"; }; }' > config/flake.nix"
- "nix build .#pushDockerImage --override-input hostNixpkgs \"path:$(nix eval --impure -I $NIX_PATH --expr '(import <nixpkgs> {}).path')\" --override-input customConfig path:./config -o docker-build-push"
- "./docker-build-push"
agents:
system: x86_64-linux
Expand All @@ -84,7 +85,7 @@ steps:

- label: 'TODO list'
depends_on: nix
command: 'nix-shell --run scripts/todo-list.sh'
command: 'nix develop --command scripts/todo-list.sh'
agents:
system: x86_64-linux

Expand Down
7 changes: 3 additions & 4 deletions .gitattributes
Expand Up @@ -5,12 +5,11 @@
# https://github.com/github/linguist/blob/master/docs/overrides.md

# Hide stuff that is generated by nix-tools stack-to-nix
nix/.stack.nix/*.nix linguist-generated=true merge=binary
nix/materialized/stack-nix/*.cache* linguist-generated=true merge=binary
nix/materialized/*/*.nix linguist-generated=true merge=binary
nix/materialized/*/.plan.nix/*.nix linguist-generated=true merge=binary

# Hide niv boilerplate
nix/sources.nix linguist-generated=true
# Hide nix flake lock boilerplate
flake.lock linguist-generated=true

# Hide the Cabal freeze file
cabal.project.freeze linguist-generated=true
Expand Down
2 changes: 1 addition & 1 deletion .github/ISSUE_TEMPLATE/failing_test.yml
Expand Up @@ -57,6 +57,6 @@ body:
The URL of a build job in CI where the error occurs. Please
link to a specific build number. In Hydra, please link to the
job page rather than the logs.
placeholder: https://hydra.iohk.io/job/Cardano/cardano-wallet-bors-staging/cardano-wallet-linux64
placeholder: https://hydra.iohk.io/job/Cardano/cardano-wallet-bors-staging/linux.musl.cardano-wallet-linux64
validations:
required: true
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Expand Up @@ -17,7 +17,7 @@ jobs:
id: hydra
with:
hydra: 'https://hydra.iohk.io'
jobs: 'cardano-wallet-linux64 cardano-wallet-macos64 cardano-wallet-win64'
jobs: 'linux.musl.cardano-wallet-linux64 macos.cardano-wallet-macos64 linux.windows.cardano-wallet-win64'

- name: '🍒 Fetch release files'
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/windows.yml
Expand Up @@ -21,7 +21,7 @@ jobs:
id: hydra
with:
hydra: 'https://hydra.iohk.io'
jobs: 'cardano-wallet-tests-win64'
jobs: 'linux.windows.cardano-wallet-tests-win64'
- name: 'Fetch Windows testing bundle'
shell: powershell
run: |
Expand Down
1 change: 0 additions & 1 deletion .gitignore
Expand Up @@ -32,7 +32,6 @@ hie.yaml

### Nix ###
/result*
/.stack-to-nix.cache
/.nix-shell-cabal.project*

### Direnv ###
Expand Down
4 changes: 2 additions & 2 deletions cabal.project
Expand Up @@ -28,11 +28,11 @@
-- from stack.yaml.
--
-- To build, start "nix-shell", then run Cabal inside:
-- nix-shell
-- nix develop
-- cabal build all -O0
--
-- To build with profiling enabled, use:
-- nix-shell --arg profiling true
-- nix develop .#profiled
-- cabal configure --enable-tests --enable-benchmarks --enable-profiling
-- cabal build all
--
Expand Down
141 changes: 7 additions & 134 deletions default.nix
@@ -1,6 +1,7 @@
############################################################################
#
# Cardano Wallet Nix build
# Cardano Wallet Nix build - legacy flake compatibility layer.
# See flake.nix for native flake instructions.
#
# Derivation attributes of this file can be build with "nix-build -A ..."
# Discover attribute names using tab-completion in your shell.
Expand All @@ -20,143 +21,15 @@
# - cardano-wallet-core.db
# - cardano-wallet.latency
# - etc
# - migration-tests - tests db migrations from previous versions
# - dockerImage - tarball of the docker image
# - private
# - shell - import of shell.nix
# - project.hsPkgs - a Haskell.nix package set of all packages and their dependencies
# - cardano-wallet-core.components.library
# - etc (layout is PACKAGE-NAME.components.COMPONENT-TYPE.COMPONENT-NAME)
#
# The attributes of this file are imported by the Hydra jobset and
# mapped into the layout TARGET-SYSTEM.ATTR-PATH.BUILD-SYSTEM.
# See release.nix for more info about that.
#
# Other documentation:
# https://input-output-hk.github.io/cardano-wallet/dev/Building#nix-build
#
############################################################################

{ system ? builtins.currentSystem
, crossSystem ? null
, config ? {}
# Import pinned Nixpkgs with iohk-nix and Haskell.nix overlays
, pkgs ? import ./nix/default.nix { inherit system crossSystem config sourcesOverride; }
# Use this git revision for stamping executables
, gitrev ? pkgs.commonLib.commitIdFromGitRepoOrZero ./.git
# Use this to reference local sources rather than the niv pinned versions (see nix/default.nix)
, sourcesOverride ? {}
# GitHub PR number (as a string), set when building a Hydra PR jobset.
, pr ? null
# Bors job type (as a string), set when building a Hydra bors jobset.
, borsBuild ? null
}:

let
# The project sources. Sources are filtered by filename, and then
# further filtered by package subdirectory.
src = lib.cleanSourceWith {
src = pkgs.haskell-nix.cleanSourceHaskell { src = ./.; };
name = "cardano-wallet-src";
filter = commonLib.removeSocketFilesFilter;
};

# Builds the Haskell project with Haskell.nix.
# There are variants with profiling or test coverage enabled.
buildHaskellProject = args: import ./nix/haskell.nix ({
inherit config pkgs;
inherit (pkgs) buildPackages lib stdenv haskell-nix;
inherit src gitrev pr borsBuild;
} // args);
project = buildHaskellProject {};
profiledProject = buildHaskellProject { profiling = true; };
coveredProject = buildHaskellProject { coverage = true; };

# Functions used in this file
inherit (pkgs) lib commonLib;
inherit (pkgs.haskell-nix.haskellLib)
selectProjectPackages
isProjectPackage
collectComponents
collectChecks;
getPackageChecks = lib.mapAttrs (_: package: package.checks);

self = {
inherit (project.hsPkgs.cardano-wallet-core.identifier) version;
inherit src;

# Cardano wallet
cardano-wallet = import ./nix/release-build.nix {
inherit pkgs gitrev;
exe = project.hsPkgs.cardano-wallet.components.exes.cardano-wallet;
backend = self.cardano-node;
};
# Local test cluster and mock metadata server
inherit (project.hsPkgs.cardano-wallet.components.exes)
local-cluster
mock-token-metadata-server;

# Adrestia tool belt
inherit (project.hsPkgs.bech32.components.exes) bech32;
inherit (project.hsPkgs.cardano-addresses-cli.components.exes) cardano-address;

# Cardano
inherit (project.hsPkgs.cardano-cli.components.exes) cardano-cli;
cardano-node = project.hsPkgs.cardano-node.components.exes.cardano-node // {
deployments = pkgs.cardano-node-deployments;
};

# Provide db-converter, so daedalus can ship it without needing to
# pin an ouroborus-network rev.
inherit (project.hsPkgs.ouroboros-consensus-byron.components.exes)
db-converter;

# `tests` are the test suites which have been built.
tests = collectComponents "tests" isProjectPackage coveredProject.hsPkgs;
# `checks` are the result of executing the tests.
checks = pkgs.recurseIntoAttrs (getPackageChecks (selectProjectPackages coveredProject.hsPkgs));
# Combined project coverage report
testCoverageReport = coveredProject.projectCoverageReport;
# `benchmarks` are only built, not run.
benchmarks = collectComponents "benchmarks" isProjectPackage project.hsPkgs;

# See the imported file for how to use the docker build.
dockerImage = pkgs.callPackage ./nix/docker.nix {
exes = with self; [ cardano-wallet local-cluster ];
base = with self; [
bech32
cardano-address
cardano-cli
cardano-node
(pkgs.linkFarm "docker-config-layer" [ { name = "config"; path = pkgs.cardano-node-deployments; } ])
];
};

# These attributes are part of the project build, but are
# considered to be implementation details, not public API.
# It's possible to use them, but they may change at any time
# without warning.
private = pkgs.recurseIntoAttrs {
inherit pkgs project profiledProject coveredProject;

# Shells
shell = import ./shell.nix { inherit pkgs; walletPackages = self; };
shell-prof = import ./shell.nix { inherit pkgs; walletPackages = self; profiling = true; };
cabalShell = import ./nix/cabal-shell.nix { inherit pkgs; walletPackages = self; };
stackShell = import ./nix/stack-shell.nix { inherit pkgs; walletPackages = self; };

# This is the ./nix/regenerate.sh script. Put it here so that it's
# built and cached on CI.
inherit (pkgs) stackNixRegenerate;

# Ensure that the project's eval-time GC roots are built and
# cached by CI.
roots = pkgs.recurseIntoAttrs {
project = project.roots;
iohk-nix-utils = pkgs.iohk-nix-utils.roots;
};
};
};

in
self
{ ... }@args:
with (import ./nix/flake-compat.nix args);
defaultNix // defaultNix.packages.${builtins.currentSystem} // {
private.project = defaultNix.legacyPackages.${builtins.currentSystem};
}

0 comments on commit a975490

Please sign in to comment.