From b5818b517f5b13faf22fb2e360c0fc7a8c6b8211 Mon Sep 17 00:00:00 2001 From: Konstantinos Lambrou-Latreille Date: Tue, 8 Jun 2021 15:55:59 -0400 Subject: [PATCH] WIP --- default.nix | 49 +------ examples/src/Plutus/Contracts/Game.hs | 2 +- nix/default.nix | 11 ++ nix/lib/ci.nix | 201 ++++++++++++++++++++++++++ nix/pkgs/default.nix | 25 ++++ nix/pkgs/haskell/default.nix | 24 +++ nix/pkgs/haskell/haskell.nix | 51 +++++++ release.nix | 48 +++++- shell.nix | 65 ++++++++- 9 files changed, 424 insertions(+), 52 deletions(-) create mode 100644 nix/default.nix create mode 100644 nix/lib/ci.nix create mode 100644 nix/pkgs/default.nix create mode 100644 nix/pkgs/haskell/default.nix create mode 100644 nix/pkgs/haskell/haskell.nix diff --git a/default.nix b/default.nix index cb3a6d0..a820ea5 100644 --- a/default.nix +++ b/default.nix @@ -1,49 +1,10 @@ let - sources = import ./nix/sources.nix { inherit pkgs; }; - plutus-sources = import sources.plutus {}; - pkgs = plutus-sources.pkgs; + packages = import ./nix; + inherit (packages) pkgs plutus-starter sources; + project = plutus-starter.haskell.project; in { - inherit (plutus-sources) pkgs; - inherit (plutus-sources) plutus; + inherit pkgs sources plutus-starter; - haskellNixProject = pkgs.haskell-nix.project { - # 'cleanGit' cleans a source directory based on the files known by git - src = pkgs.haskell-nix.haskellLib.cleanGit { - name = "plutus-starter"; - src = ./.; - }; - # Specify the GHC version to use. - compiler-nix-name = "ghc810420210212"; # Should use the same version as in `https://github.com/input-output-hk/plutus/blob/master/nix/pkgs/haskell/default.nix - - sha256map = { - "https://github.com/Quid2/flat.git"."95e5d7488451e43062ca84d5376b3adcc465f1cd" = "06l31x3y93rjpryvlxnpsyq2zyxvb0z6lik6yq2fvh36i5zwvwa3"; - "https://github.com/input-output-hk/plutus.git"."514f059a7aabc4b03ca80343d29cf0736f9ea498" = "0l848y4y6yqva1xfcv05dyhb6hhvlmnjcign4dwmf1cgy0kyz4v5"; - "https://github.com/shmish111/purescript-bridge.git"."6a92d7853ea514be8b70bab5e72077bf5a510596" = "13j64vv116in3c204qsl1v0ajphac9fqvsjp7x3zzfr7n7g61drb"; - "https://github.com/shmish111/servant-purescript.git"."a76104490499aa72d40c2790d10e9383e0dbde63" = "11nxxmi5bw66va7psvrgrw7b7n85fvqgfp58yva99w3v9q3a50v9"; - "https://github.com/input-output-hk/cardano-crypto.git"."f73079303f663e028288f9f4a9e08bcca39a923e" = "1n87i15x54s0cjkh3nsxs4r1x016cdw1fypwmr68936n3xxsjn6q"; - "https://github.com/input-output-hk/cardano-base"."4251c0bb6e4f443f00231d28f5f70d42876da055" = "02a61ymvx054pcdcgvg5qj9kpybiajg993nr22iqiya196jmgciv"; - "https://github.com/input-output-hk/cardano-prelude"."ee4e7b547a991876e6b05ba542f4e62909f4a571" = "0dg6ihgrn5mgqp95c4f11l6kh9k3y75lwfqf47hdp554w7wyvaw6"; - "https://github.com/input-output-hk/ouroboros-network"."6cb9052bde39472a0555d19ade8a42da63d3e904" = "0rz4acz15wda6yfc7nls6g94gcwg2an5zibv0irkxk297n76gkmg"; - "https://github.com/input-output-hk/iohk-monitoring-framework"."34abfb7f4f5610cabb45396e0496472446a0b2ca" = "1fdc0a02ipa385dnwa6r6jyc8jlg537i12hflfglkhjs2b7i92gs"; - "https://github.com/input-output-hk/cardano-ledger-specs"."097890495cbb0e8b62106bcd090a5721c3f4b36f" = "0i3y9n0rsyarvhfqzzzjccqnjgwb9fbmbs6b7vj40afjhimf5hcj"; - "https://github.com/input-output-hk/goblins"."cde90a2b27f79187ca8310b6549331e59595e7ba" = "17c88rbva3iw82yg9srlxjv2ia5wjb9cyqw44hik565f5v9svnyg"; - }; - - modules = [ - { - packages = { - eventful-sql-common = { - # This is needed so evenful-sql-common will build with a newer version of persistent. - ghcOptions = [ "-XDerivingStrategies -XStandaloneDeriving -XUndecidableInstances -XDataKinds -XFlexibleInstances -XMultiParamTypeClasses" ]; - doHaddock = false; - }; - - # Broken due to haddock errors. Refer to https://github.com/input-output-hk/plutus/blob/master/nix/pkgs/haskell/haskell.nix - plutus-ledger.doHaddock = false; - plutus-use-cases.doHaddock = false; - }; - } - ]; - }; + inherit project; } diff --git a/examples/src/Plutus/Contracts/Game.hs b/examples/src/Plutus/Contracts/Game.hs index 8ecbb62..2a58e2c 100644 --- a/examples/src/Plutus/Contracts/Game.hs +++ b/examples/src/Plutus/Contracts/Game.hs @@ -1,4 +1,4 @@ -{-# LANGUAGE AllowAmbiguousTypes #-} +{-# LANGUAGE AllowAmbiguousTypes #- {-# LANGUAGE DataKinds #-} {-# LANGUAGE DeriveAnyClass #-} {-# LANGUAGE DeriveGeneric #-} diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 0000000..c0f84a7 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,11 @@ +let + sources = import (import ./sources.nix { inherit pkgs; }).plutus {}; + pkgs = sources.pkgs; + + plutus-starter = import ./pkgs { inherit pkgs sources; }; + +in +{ + inherit pkgs plutus-starter sources; +} + diff --git a/nix/lib/ci.nix b/nix/lib/ci.nix new file mode 100644 index 0000000..80794a7 --- /dev/null +++ b/nix/lib/ci.nix @@ -0,0 +1,201 @@ +{ pkgs }: + +let + # Generic nixpkgs, use *only* for lib functions that are stable across versions + lib = pkgs.lib; +in +rec { + # Borrowed from https://github.com/cachix/ghcide-nix/pull/4/files#diff-70bfff902f4dec33e545cac10ee5844d + # Tweaked to use builtins.mapAttrs instead of needing the one from nixpkgs lib + /* + dimension: name -> attrs -> function -> attrs + where + function: keyText -> value -> attrsOf package + + WARNING: Attribute names must not contain periods ("."). + See https://github.com/NixOS/nix/issues/3088 + + NOTE: The dimension name will be picked up by agent and web ui soon. + + Specifies a dimension of the build matrix. For example + + dimension "Example" { + withP = { p = true; } + withoutP = { p = false; } + } (key: # either "withP" or "withoutP" + { p }: # either p = true or p = false + myProject p + ) + + evaluates roughly to + + { + withP = myProject true; + withoutP = myProject false; + } + + Use nested calls for multiple dimensions. + + Example: + + dimension "System" { + "x86_64-linux" = {}; + # ... + }: (system: {}: + + dimension "Nixpkgs release" ( + { + "nixpkgs-19_03".nixpkgs = someSource + } // optionalAttrs (system != "...") { + "nixpkgs-unstable".nixpkgs = someOtherSource + } + ) (_key: { nixpkgs }: + + myProject system nixpkgs + + ) + ) + + evaluates roughly to + + { + x86_64-linux.nixpkgs-19_03 = myProject "x86_64-linux" someSource; + x86_64-linux.nixpkgs-unstable = myProject "x86_64-linux" someOtherSource; + ... + } + + If you need to make references across attributes, you can do so by binding + the result. Wherever you write + + dimension "My dimension" {} (key: value: f1 key value) + + You can also write + + let + myDimension = dimension "My dimension" {} (key: value: f2 key value myDimension) + in + myDimension + + This example builds a single test runner to reuse across releases: + + let + overlay = + testRunnerPkgs: self: super: { + # ... + }; + myProject = + { nixpkgs, + pkgs ? import nixpkgs { overlays = [ overlay ]; }, + testRunnerPkgs ? pkgs + }: pkgs; + in + + let + latest = "nixpkgs-19_03"; + releases = + dimension "Nixpkgs release" + { + nixpkgs-18_09.nixpkgs = someSource + nixpkgs-19_03.nixpkgs = someOtherSource + } + (_key: { nixpkgs }: + + myProject { + inherit nixpkgs; + testRunnerPkgs = releases."${latest}"; + } + + ); + in releases; + + */ + dimension = name: attrs: f: + builtins.mapAttrs + (k: v: + let o = f k v; + in o // { recurseForDerivations = o.recurseForDerivations or true; } + ) + attrs + // { meta.dimension.name = name; }; + + /* + Takes an attribute set and returns all the paths to derivations within it, i.e. + derivationPaths { a = { b = ; }; c = ; } == [ "a.b" "c" ] + This can be used with 'attrByPath' or the 'constitutents' of an aggregate Hydra job. + */ + derivationPaths = + let + names = x: lib.filter (n: n != "recurseForDerivations" && n != "meta") (builtins.attrNames x); + go = nameSections: attrs: + builtins.concatMap + (n: + let + v = builtins.getAttr n attrs; + newNameSections = nameSections ++ [ n ]; + in + if pkgs.lib.isDerivation v + then [ (builtins.concatStringsSep "." newNameSections) ] + else if builtins.isAttrs v + then go newNameSections v + else [ ] + ) + (names attrs); + in + go [ ]; + + # Creates an aggregate job with the given name from every derivation in the attribute set. + derivationAggregate = name: attrs: pkgs.releaseTools.aggregate { + inherit name; + constituents = derivationPaths attrs; + }; + + # A filter for removing packages that aren't supported on the current platform + # according to 'meta.platforms'. + platformFilterGeneric = pkgs: system: + # This needs to use the correct nixpkgs version so all the systems line up + let + lib = pkgs.lib; + platform = lib.systems.elaborate { inherit system; }; + # Can't just default to [] for platforms, since no meta.platforms + # means "all platforms" not "no platforms" + in + drv: + if drv ? meta && drv.meta ? platforms then + lib.any (lib.meta.platformMatch platform) drv.meta.platforms + else true; + + # Hydra doesn't like these attributes hanging around in "jobsets": it thinks they're jobs! + stripAttrsForHydra = filterAttrsOnlyRecursive (n: _: n != "recurseForDerivations" && n != "dimension"); + + # Keep derivations and attrsets with 'recurseForDerivations'. This ensures that we match the + # derivations that Hercules will see, and prevents Hydra from trying to pick up all sorts of bad stuff + # (like attrsets that contain themselves!). + filterDerivations = filterAttrsOnlyRecursive (n: attrs: lib.isDerivation attrs || attrs.recurseForDerivations or false); + + # A version of 'filterAttrsRecursive' that doesn't recurse into derivations. This prevents us from going into an infinite + # loop with the 'out' attribute on derivations. + # TODO: Surely this shouldn't be necessary. I think normal 'filterAttrsRecursive' will effectively cause infinite loops + # if you keep derivations and your predicate forces the value of the attribute, as this then triggers a loop on the + # 'out' attribute. Weird. + filterAttrsOnlyRecursive = pred: set: + lib.listToAttrs ( + lib.concatMap + (name: + let v = set.${name}; in + if pred name v then [ + (lib.nameValuePair name ( + if builtins.isAttrs v && !lib.isDerivation v then filterAttrsOnlyRecursive pred v + else v + )) + ] else [ ] + ) + (builtins.attrNames set) + ); + + # Takes an array of systems and returns a `name: system` AttrSet + # filterSystems :: [ string ] -> AttrSet + filterSystems = systems: lib.filterAttrs (_: v: builtins.elem v systems) { + linux = "x86_64-linux"; + darwin = "x86_64-darwin"; + }; +} diff --git a/nix/pkgs/default.nix b/nix/pkgs/default.nix new file mode 100644 index 0000000..48a1e2e --- /dev/null +++ b/nix/pkgs/default.nix @@ -0,0 +1,25 @@ +{ pkgs +, sources +}: +let + inherit (pkgs) stdenv; + + gitignore-nix = pkgs.callPackage sources."gitignore.nix" { }; + + haskell = pkgs.callPackage ./haskell { + inherit gitignore-nix; + }; + + hlint = sources.plutus.hlint; + + cabal-install = sources.plutus.cabal-install; + + stylish-haskell = sources.plutus.stylish-haskell; + + haskell-language-server = sources.plutus.haskell-language-server; + +in +{ + inherit haskell hlint cabal-install stylish-haskell haskell-language-server; +} + diff --git a/nix/pkgs/haskell/default.nix b/nix/pkgs/haskell/default.nix new file mode 100644 index 0000000..1984cef --- /dev/null +++ b/nix/pkgs/haskell/default.nix @@ -0,0 +1,24 @@ +{ haskell-nix +, gitignore-nix +}: +let + # The compiler that we are using. We are using a patched version so we need to specify it explicitly. + # This version has the experimental core interface files patch, and a fix for unboxed tuples in + # GHCi, which helps with HLS. + compiler-nix-name = "ghc810420210212"; # Should use the same version as in `https://github.com/input-output-hk/plutus/blob/master/nix/pkgs/haskell/default.nix + + # The haskell project created by haskell-nix.cabalProject' + project = import ./haskell.nix { + inherit haskell-nix compiler-nix-name gitignore-nix; + }; + + # All the packages defined by our project, including dependencies + packages = project.hsPkgs; + + # Just the packages in the project + projectPackages = haskell-nix.haskellLib.selectProjectPackages packages; + +in +rec { + inherit project projectPackages packages; +} diff --git a/nix/pkgs/haskell/haskell.nix b/nix/pkgs/haskell/haskell.nix new file mode 100644 index 0000000..047d454 --- /dev/null +++ b/nix/pkgs/haskell/haskell.nix @@ -0,0 +1,51 @@ +############################################################################ +# Builds Haskell packages with Haskell.nix +############################################################################ +{ haskell-nix +, gitignore-nix +, compiler-nix-name +}: + +let + project = haskell-nix.project { + # 'cleanGit' cleans a source directory based on the files known by git + src = haskell-nix.haskellLib.cleanGit { + name = "plutus-starter"; + src = ../../../.; + }; + + inherit compiler-nix-name; + + sha256map = { + "https://github.com/Quid2/flat.git"."95e5d7488451e43062ca84d5376b3adcc465f1cd" = "06l31x3y93rjpryvlxnpsyq2zyxvb0z6lik6yq2fvh36i5zwvwa3"; + "https://github.com/input-output-hk/plutus.git"."514f059a7aabc4b03ca80343d29cf0736f9ea498" = "0l848y4y6yqva1xfcv05dyhb6hhvlmnjcign4dwmf1cgy0kyz4v5"; + "https://github.com/shmish111/purescript-bridge.git"."6a92d7853ea514be8b70bab5e72077bf5a510596" = "13j64vv116in3c204qsl1v0ajphac9fqvsjp7x3zzfr7n7g61drb"; + "https://github.com/shmish111/servant-purescript.git"."a76104490499aa72d40c2790d10e9383e0dbde63" = "11nxxmi5bw66va7psvrgrw7b7n85fvqgfp58yva99w3v9q3a50v9"; + "https://github.com/input-output-hk/cardano-crypto.git"."f73079303f663e028288f9f4a9e08bcca39a923e" = "1n87i15x54s0cjkh3nsxs4r1x016cdw1fypwmr68936n3xxsjn6q"; + "https://github.com/input-output-hk/cardano-base"."4251c0bb6e4f443f00231d28f5f70d42876da055" = "02a61ymvx054pcdcgvg5qj9kpybiajg993nr22iqiya196jmgciv"; + "https://github.com/input-output-hk/cardano-prelude"."ee4e7b547a991876e6b05ba542f4e62909f4a571" = "0dg6ihgrn5mgqp95c4f11l6kh9k3y75lwfqf47hdp554w7wyvaw6"; + "https://github.com/input-output-hk/ouroboros-network"."6cb9052bde39472a0555d19ade8a42da63d3e904" = "0rz4acz15wda6yfc7nls6g94gcwg2an5zibv0irkxk297n76gkmg"; + "https://github.com/input-output-hk/iohk-monitoring-framework"."34abfb7f4f5610cabb45396e0496472446a0b2ca" = "1fdc0a02ipa385dnwa6r6jyc8jlg537i12hflfglkhjs2b7i92gs"; + "https://github.com/input-output-hk/cardano-ledger-specs"."097890495cbb0e8b62106bcd090a5721c3f4b36f" = "0i3y9n0rsyarvhfqzzzjccqnjgwb9fbmbs6b7vj40afjhimf5hcj"; + "https://github.com/input-output-hk/goblins"."cde90a2b27f79187ca8310b6549331e59595e7ba" = "17c88rbva3iw82yg9srlxjv2ia5wjb9cyqw44hik565f5v9svnyg"; + }; + + modules = [ + { + packages = { + eventful-sql-common = { + # This is needed so evenful-sql-common will build with a newer version of persistent. + ghcOptions = [ "-XDerivingStrategies -XStandaloneDeriving -XUndecidableInstances -XDataKinds -XFlexibleInstances -XMultiParamTypeClasses" ]; + doHaddock = false; + }; + + # Broken due to haddock errors. Refer to https://github.com/input-output-hk/plutus/blob/master/nix/pkgs/haskell/haskell.nix + plutus-ledger.doHaddock = false; + plutus-use-cases.doHaddock = false; + }; + } + ]; + }; +in + project + diff --git a/release.nix b/release.nix index 54be690..d23deb0 100644 --- a/release.nix +++ b/release.nix @@ -1,8 +1,50 @@ let - jobs = rec { + sources = import ./nix/sources.nix { inherit pkgs; }; + + plutus-sources = import sources.plutus {}; + pkgs = plutus-sources.pkgs; + plutus = plutus-sources.plutus; + haskellNix = pkgs.haskell-nix; + + # Just the packages in the project + packages = (import ./.).project.hsPkgs; + projectPackages = haskellNix.haskellLib.selectProjectPackages packages; + + inherit (import ./nix/lib/ci.nix { inherit pkgs; }) dimension filterAttrsOnlyRecursive filterDerivations stripAttrsForHydra; + + # Collects haskell derivations and builds an attrset: + # + # { library = { ... } + # , tests = { ... } + # , benchmarks = { ... } + # , exes = { ... } + # , checks = { ... } + # } + # Where each attribute contains an attribute set + # with all haskell components of that type + mkHaskellDimension = pkgs: haskellProjects: + let + # retrieve all checks from a Haskell package + collectChecks = _: ps: pkgs.haskell-nix.haskellLib.collectChecks' ps; + # retrieve all components of a Haskell package + collectComponents = type: ps: pkgs.haskell-nix.haskellLib.collectComponents' type ps; + # Given a component type and the retrieve function, retrieve components from haskell packages + select = type: selector: (selector type) haskellProjects; + # { component-type : retriever-fn } + attrs = { + "library" = collectComponents; + "tests" = collectComponents; + "benchmarks" = collectComponents; + "exes" = collectComponents; + "checks" = collectChecks; + }; + in + dimension "Haskell component" attrs select; + + ciJobsets = rec { shell = (import ./shell.nix); - build = (import ./.).haskellNixProject; + build = pkgs.recurseIntoAttrs (mkHaskellDimension pkgs projectPackages); }; in - jobs + stripAttrsForHydra (filterDerivations ciJobsets) diff --git a/shell.nix b/shell.nix index d4c3840..9f9775c 100644 --- a/shell.nix +++ b/shell.nix @@ -1,12 +1,13 @@ let - packages = import ./default.nix; + packages = import ./.; + inherit (packages) pkgs plutus-starter; + inherit (plutus-starter) haskell; - inherit (packages) pkgs; in - packages.haskellNixProject.shellFor { + haskell.project.shellFor { withHoogle = false; - nativeBuildInputs = with packages.plutus; [ + nativeBuildInputs = with plutus-starter; [ hlint cabal-install haskell-language-server @@ -14,3 +15,59 @@ in pkgs.niv ]; } + +## This file is used by nix-shell. +## It just takes the shell attribute from default.nix. +#{ config ? {} +#, sourcesOverride ? {} +#, withHoogle ? false +#, pkgs ? import ./nix { +# inherit config sourcesOverride; +# } +#}: +#with pkgs; +#let +# # This provides a development environment that can be used with nix-shell or +# # lorri. See https://input-output-hk.github.io/haskell.nix/user-guide/development/ +# shell = skeletonHaskellPackages.shellFor { +# name = "cabal-dev-shell"; + +# # If shellFor local packages selection is wrong, +# # then list all local packages then include source-repository-package that cabal complains about: +# #packages = ps: with ps; [ +# # iohk-skeleton +# # cardano-prelude +# #]; + +# # These programs will be available inside the nix-shell. +# buildInputs = +# with haskellPackages; [ hlint stylish-haskell weeder ghcid lentil ] +# # TODO: Add your own packages to the shell. +# ++ [ jormungandr ]; + +# # Prevents cabal from choosing alternate plans, so that +# # *all* dependencies are provided by Nix. +# exactDeps = true; + +# inherit withHoogle; +# }; + +# devops = pkgs.stdenv.mkDerivation { +# name = "devops-shell"; +# buildInputs = [ +# niv +# ]; +# shellHook = '' +# echo "DevOps Tools" \ +# | ${figlet}/bin/figlet -f banner -c \ +# | ${lolcat}/bin/lolcat +# echo "NOTE: you may need to export GITHUB_TOKEN if you hit rate limits with niv" +# echo "Commands: +# * niv update - update package +# " +# ''; +# }; + +#in + +# shell // { inherit devops; }