diff --git a/cabal.project b/cabal.project index 02b3a406e..eab612a4b 100644 --- a/cabal.project +++ b/cabal.project @@ -56,8 +56,18 @@ package cardano-node package ouroboros-consensus-cardano tests: False + +allow-newer: + monoidal-containers:aeson, + size-based:template-haskell + -- --------------------------------------------------------- +-- The two following one-liners will cut off / restore the remainder of this file (for nix-shell users): +-- when using the "cabal" wrapper script provided by nix-shell. +-- --------------------------- 8< -------------------------- +-- Please do not put any `source-repository-package` clause above this line. + source-repository-package type: git location: https://github.com/input-output-hk/Win32-network @@ -163,7 +173,3 @@ source-repository-package typed-protocols typed-protocols-examples network-mux - -allow-newer: - monoidal-containers:aeson, - size-based:template-haskell diff --git a/custom-config/default.nix b/custom-config/default.nix new file mode 100644 index 000000000..9010ea8cb --- /dev/null +++ b/custom-config/default.nix @@ -0,0 +1,3 @@ +self: { + withHoogle = true; +} diff --git a/custom-config/flake.nix b/custom-config/flake.nix new file mode 100644 index 000000000..208a20258 --- /dev/null +++ b/custom-config/flake.nix @@ -0,0 +1,8 @@ +{ + outputs = {...}: { + # Example scripts customization: + #nixosModules.cardano-db-sync = { + # service.cardano-db-sync.postgres.port = 5433; + #} + }; +} diff --git a/default.nix b/default.nix index 215d0c821..d35b598a3 100644 --- a/default.nix +++ b/default.nix @@ -11,70 +11,25 @@ # }' , sourcesOverride ? {} # pinned version of nixpkgs augmented with overlays (iohk-nix and our packages). -, pkgs ? import ./nix { inherit system crossSystem config sourcesOverride gitrev; } -, gitrev ? pkgs.iohkNix.commitIdFromGitRepoOrZero ./.git +, pkgs ? import ./nix { inherit system crossSystem config sourcesOverride gitrev customConfig; } +, gitrev ? null }: with pkgs; with commonLib; let - customConfig' = if customConfig ? services then customConfig else { - services.cardano-db-sync = customConfig; - }; haskellPackages = recRecurseIntoAttrs # the Haskell.nix package set, reduced to local packages. (selectProjectPackages cardanoDbSyncHaskellPackages); - scripts = callPackage ./nix/scripts.nix { - customConfig = customConfig'; - }; - - rewrite-static = _: p: if (pkgs.stdenv.hostPlatform.isDarwin) then - pkgs.runCommandCC p.name { - nativeBuildInputs = [ pkgs.haskellBuildUtils.package pkgs.buildPackages.binutils pkgs.buildPackages.nix ]; - } '' - cp -R ${p} $out - chmod -R +w $out - rewrite-libs $out/bin $out/bin/* - '' else if (pkgs.stdenv.hostPlatform.isMusl) then - pkgs.runCommandCC p.name { } '' - cp -R ${p} $out - chmod -R +w $out - $STRIP $out/bin/* - '' else p; - packages = { - inherit haskellPackages cardano-db-sync cardano-db-sync-extended cardano-node scripts; + inherit haskellPackages cardano-db-sync cardano-db-sync-extended cardano-node scripts dockerImage; # so that eval time gc roots are cached (nix-tools stuff) - inherit (cardanoDbSyncHaskellPackages) roots; - inherit (haskellPackages.cardano-db-sync.project) plan-nix; + inherit (cardanoDbSyncProject) roots plan-nix; inherit (haskellPackages.cardano-db-sync.identifier) version; - exes = mapAttrsRecursiveCond (as: !(isDerivation as)) rewrite-static (collectComponents' "exes" haskellPackages); - - dockerImage = let - defaultConfig = rec { - services.cardano-db-sync = { - socketPath = lib.mkDefault ("/node-ipc/node.socket"); - postgres.generatePGPASS = false; - }; - }; - customConfig'' = mkMerge [ defaultConfig customConfig' ]; - in callPackage ./nix/docker.nix { - scripts = scripts.override { - customConfig = customConfig''; - }; - extendedScripts = scripts.override { - customConfig = mkMerge [ - customConfig'' - { services.cardano-db-sync = { - extended = true; - }; - } - ]; - }; - }; + exes = mapAttrsRecursiveCond (as: !(isDerivation as)) rewriteStatic (collectComponents' "exes" haskellPackages); # `tests` are the test suites which have been built. tests = collectComponents' "tests" haskellPackages; @@ -85,12 +40,12 @@ let # `checks.tests` collect results of executing the tests: tests = collectChecks haskellPackages; - hlint = callPackage iohkNix.tests.hlint { - src = ./. ; + hlint = callPackage hlintCheck { + inherit (pkgs.cardanoDbSyncProject.projectModule) src; }; - stylish-haskell = callPackage iohkNix.tests.stylish-haskell { - src = ./. ; + stylish-haskell = callPackage stylishHaskellCheck { + inherit (pkgs.cardanoDbSyncProject.projectModule) src; }; }; diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..b3af022cf --- /dev/null +++ b/flake.lock @@ -0,0 +1,149 @@ +{ + "nodes": { + "customConfig": { + "locked": { + "narHash": "sha256-A+seqqKiiqwEJZHRX3bGLFYhOy2DLEZij6MoEAjgSfA=", + "path": "./custom-config", + "type": "path" + }, + "original": { + "path": "./custom-config", + "type": "path" + } + }, + "haskellNix": { + "inputs": { + "nixpkgs": "nixpkgs", + "nixpkgs-2003": "nixpkgs-2003", + "nixpkgs-2009": "nixpkgs-2009", + "nixpkgs-unstable": "nixpkgs-unstable" + }, + "locked": { + "lastModified": 1620609272, + "narHash": "sha256-uSiv5puymiwDB0Jm9+sR7z7/quAJV57Tno2042rKKFI=", + "owner": "input-output-hk", + "repo": "haskell.nix", + "rev": "ecfd77b39b00a863ef82812e6824a32072dd945e", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "repo": "haskell.nix", + "type": "github" + } + }, + "iohkNix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1620314199, + "narHash": "sha256-JCu5wnt3ja8UBQv5G3aRhQ+ZcAKB6+ivodv3yA6UnK8=", + "owner": "input-output-hk", + "repo": "iohk-nix", + "rev": "0ce3abcfed803ed77472488f30b299fda9d941b1", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "repo": "iohk-nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1608007629, + "narHash": "sha256-lipVFC/a2pzzA5X2ULj64je+fz1JIp2XRrB5qyoizpQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "f02bf8ffb9a5ec5e8f6f66f1e5544fd2aa1a0693", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "f02bf8ffb9a5ec5e8f6f66f1e5544fd2aa1a0693", + "type": "github" + } + }, + "nixpkgs-2003": { + "locked": { + "lastModified": 1607708579, + "narHash": "sha256-QyADEDydJJPa8n3xawnA82IJAcZHNNm6Pp5DU7exMr4=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "7f73e46625f508a793700f5110b86f1a53341d6e", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "7f73e46625f508a793700f5110b86f1a53341d6e", + "type": "github" + } + }, + "nixpkgs-2009": { + "locked": { + "lastModified": 1608007629, + "narHash": "sha256-lipVFC/a2pzzA5X2ULj64je+fz1JIp2XRrB5qyoizpQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "f02bf8ffb9a5ec5e8f6f66f1e5544fd2aa1a0693", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "f02bf8ffb9a5ec5e8f6f66f1e5544fd2aa1a0693", + "type": "github" + } + }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1612284693, + "narHash": "sha256-efzJNF1jvjK3BMl0gZ0ZaUWcFMv0nLb9AHN/++5+u0U=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "410bbd828cdc6156aecd5bc91772ad3a6b1099c7", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "410bbd828cdc6156aecd5bc91772ad3a6b1099c7", + "type": "github" + } + }, + "root": { + "inputs": { + "customConfig": "customConfig", + "haskellNix": "haskellNix", + "iohkNix": "iohkNix", + "nixpkgs": [ + "haskellNix", + "nixpkgs-unstable" + ], + "utils": "utils" + } + }, + "utils": { + "locked": { + "lastModified": 1619345332, + "narHash": "sha256-qHnQkEp1uklKTpx3MvKtY6xzgcqXDsz5nLilbbuL+3A=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "2ebf2558e5bf978c7fb8ea927dfaed8fefab2e28", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..698a440d1 --- /dev/null +++ b/flake.nix @@ -0,0 +1,131 @@ +{ + description = "cardano-db-sync"; + + inputs = { + haskellNix.url = "github:input-output-hk/haskell.nix"; + iohkNix = { + url = "github:input-output-hk/iohk-nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + nixpkgs.follows = "haskellNix/nixpkgs-unstable"; + utils.url = "github:numtide/flake-utils"; + customConfig = { + url = "path:./custom-config"; + }; + }; + + outputs = { self, iohkNix, haskellNix, nixpkgs, utils, customConfig, ... }: + let + inherit (haskellNix.internal) config; + inherit (nixpkgs) lib; + inherit (lib) systems mapAttrs recursiveUpdate mkDefault optionalAttrs nameValuePair + attrNames getAttrs head; + inherit (utils.lib) eachSystem mkApp flattenTree; + inherit (iohkNix.lib) prefixNamesWith collectExes; + + supportedSystems = import ./supported-systems.nix; + defaultSystem = head supportedSystems; + + overlays = [ + haskellNix.overlay + iohkNix.overlays.haskell-nix-extra + iohkNix.overlays.crypto + iohkNix.overlays.cardano-lib + iohkNix.overlays.utils + (final: prev: { + customConfig = recursiveUpdate + (import ./custom-config final.customConfig) + customConfig.outputs; + gitrev = self.rev or "dirty"; + commonLib = lib + // iohkNix.lib; + }) + (import ./nix/pkgs.nix) + ]; + + in eachSystem supportedSystems (system: + let + pkgs = import nixpkgs { inherit system overlays config; }; + + devShell = import ./shell.nix { inherit pkgs; }; + + flake = pkgs.cardanoDbSyncProject.flake {}; + + muslFlake = (import nixpkgs { inherit system overlays config; + crossSystem = systems.examples.musl64; + }).cardanoDbSyncProject.flake {}; + + scripts = flattenTree pkgs.scripts; + + checkNames = attrNames flake.checks; + + checks = + # checks run on default system only; + optionalAttrs (system == defaultSystem) { + hlint = pkgs.callPackage pkgs.hlintCheck { + inherit (pkgs.cardanoDbSyncProject.projectModule) src; + }; + stylish-haskell = pkgs.callPackage pkgs.stylishHaskellCheck { + inherit (pkgs.cardanoDbSyncProject.projectModule) src; + }; + }; + + exes = collectExes flake.packages; + exeNames = attrNames exes; + lazyCollectExe = p: getAttrs exeNames (collectExes p); + + packages = { + inherit (devShell) devops; + inherit (pkgs) cardano-db-sync cardano-db-sync-extended cardano-node + dockerImage; + } + // exes + // (prefixNamesWith "static/" + (mapAttrs pkgs.rewriteStatic (lazyCollectExe + (if system == "x86_64-darwin" then flake else muslFlake).packages))) + // scripts + # Add checks to be able to build them individually + // (prefixNamesWith "checks/" checks); + + in recursiveUpdate flake { + + inherit packages checks; + + legacyPackages = pkgs; + + # Built by `nix build .` + defaultPackage = flake.packages."cardano-db-sync:exe:cardano-db-sync"; + + # Run by `nix run .` + defaultApp = flake.apps."cardano-db-sync:exe:cardano-db-sync"; + + # This is used by `nix develop .` to open a devShell + inherit devShell; + + apps = { + repl = mkApp { + drv = pkgs.writeShellScriptBin "repl" '' + confnix=$(mktemp) + echo "builtins.getFlake (toString $(git rev-parse --show-toplevel))" >$confnix + trap "rm $confnix" EXIT + nix repl $confnix + ''; + }; + cardano-node = { type = "app"; program = pkgs.cardano-node.exePath; }; + } # nix run .# + // (collectExes flake.apps); + + } + ) // { + overlay = final: prev: with self.legacyPackages.${final.system}; { + inherit cardano-db-sync cardano-db-sync-extended cardano-node + dockerImage; + }; + nixosModules = { + cardano-db-sync = { pkgs, lib, ... }: { + imports = [ ./nix/nixos/cardano-db-sync-service.nix ]; + services.cardano-db-sync.dbSyncPkgs = lib.mkDefault self.legacyPackages.${pkgs.system}; + }; + }; + }; +} diff --git a/nix/default.nix b/nix/default.nix index 751109cdd..c09175e24 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -1,37 +1,51 @@ { system ? builtins.currentSystem , crossSystem ? null , config ? {} +, customConfig ? {} , sourcesOverride ? {} , gitrev ? null }: let - sources = import ./sources.nix { inherit pkgs; } - // sourcesOverride; - iohkNix = import sources.iohk-nix {}; - haskellNix = import sources."haskell.nix" {}; - # use our own nixpkgs if it exists in our sources, - # otherwise use iohkNix default nixpkgs. - nixpkgs = if (sources ? nixpkgs) - then (builtins.trace "Not using IOHK default nixpkgs (use 'niv drop nixpkgs' to use default for better sharing)" - sources.nixpkgs) - else iohkNix.nixpkgs; + flakeSources = let + flakeLock = (builtins.fromJSON (builtins.readFile ../flake.lock)).nodes; + compat = s: builtins.fetchGit { + url = "https://github.com/${s.locked.owner}/${s.locked.repo}.git"; + inherit (s.locked) rev; + ref = s.original.ref or "master"; + }; + in { + "haskell.nix" = compat flakeLock.haskellNix; + "iohk-nix" = compat flakeLock.iohkNix; + }; + sources = flakeSources // sourcesOverride; + iohkNix = import sources.iohk-nix { inherit system; }; + haskellNix = import sources."haskell.nix" { inherit system sourcesOverride; }; + nixpkgs = haskellNix.sources.nixpkgs-unstable; # for inclusion in pkgs: overlays = # Haskell.nix (https://github.com/input-output-hk/haskell.nix) - haskellNix.overlays + haskellNix.nixpkgsArgs.overlays # haskell-nix.haskellLib.extra: some useful extra utility functions for haskell.nix ++ iohkNix.overlays.haskell-nix-extra ++ iohkNix.overlays.crypto # iohkNix: nix utilities and niv: ++ iohkNix.overlays.iohkNix + ++ iohkNix.overlays.utils # our own overlays: ++ [ (pkgs: _: with pkgs; { - inherit gitrev; + gitrev = if gitrev == null + then iohkNix.commitIdFromGitRepoOrZero ../.git + else gitrev; + + customConfig = lib.recursiveUpdate + (import ../custom-config pkgs.customConfig) + customConfig; + inherit (pkgs.iohkNix) cardanoLib; # commonLib: mix pkgs.lib with iohk-nix utils and our own: - commonLib = lib // iohkNix // iohkNix.cardanoLib + commonLib = lib // cardanoLib // iohk-nix.lib // import ./util.nix { inherit haskell-nix; } # also expose our sources and overlays // { inherit overlays sources; }; @@ -42,7 +56,7 @@ let pkgs = import nixpkgs { inherit system crossSystem overlays; - config = haskellNix.config // config; + config = haskellNix.nixpkgsArgs.config // config; }; in pkgs diff --git a/nix/docker.nix b/nix/docker.nix index 255a2a914..7cd30b926 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -34,8 +34,7 @@ # ############################################################################ -{ iohkNix -, commonLib +{ cardanoLib , dockerTools # The main contents of the image. @@ -45,7 +44,7 @@ , extendedScripts # Get the current commit -, gitrev ? iohkNix.commitIdFromGitRepoOrZero ../.git +, gitrev # Other things to include in the image. , bashInteractive @@ -102,13 +101,12 @@ let }; dbSyncDockerImage = let - clusterStatements = lib.concatStringsSep "\n" (lib.mapAttrsToList (_: value: value) (commonLib.forEnvironmentsCustom (env: let - dbSyncScript = scripts.${env.name}.db-sync; - dbSyncExtendedScript = extendedScripts.${env.name}.db-sync; + clusterStatements = lib.concatStringsSep "\n" (lib.mapAttrsToList (env: script: let + dbSyncScript = script.db-sync; + dbSyncExtendedScript = extendedScripts.${env}.db-sync; in '' - elif [[ "$NETWORK" == "${env.name}" ]]; then - ${if (env ? nodeConfig) then '' - echo "Connecting to network: ${env.name}" + elif [[ "$NETWORK" == "${env}" ]]; then + echo "Connecting to network: ${env}" if [[ ! -z "''${EXTENDED}" ]] && [[ "''${EXTENDED}" == true ]] then exec ${dbSyncExtendedScript} @@ -117,8 +115,7 @@ let fi echo "Cleaning up" '' - else "echo db-sync not supported on ${env.name} ; exit 1"} - '') scripts.environments)); + ) scripts); genPgPass = writeScript "gen-pgpass" '' #!${runtimeShell} SECRET_DIR=$1 diff --git a/nix/haskell.nix b/nix/haskell.nix index 5c6cc9715..d7b20c350 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -12,15 +12,15 @@ # Enable profiling , profiling ? config.haskellNix.profiling or false # Version info, to be passed when not building from a git work tree -, gitrev ? null +, gitrev }: let projectPackages = lib.attrNames (haskell-nix.haskellLib.selectProjectPackages - (haskell-nix.cabalProject { + (haskell-nix.cabalProject' { inherit src; compiler-nix-name = compiler; - })); + }).hsPkgs); preCheck = '' echo pre-check @@ -63,7 +63,7 @@ let # This creates the Haskell package set. # https://input-output-hk.github.io/haskell.nix/user-guide/projects/ - pkgSet = haskell-nix.cabalProject { + pkgSet = haskell-nix.cabalProject' { inherit src; compiler-nix-name = compiler; modules = [ @@ -132,11 +132,6 @@ let # setGitRev is a postInstall script to stamp executables with # version info. It uses the "gitrev" argument, if set. Otherwise, # the revision is sourced from the local git work tree. - setGitRev = ''${haskellBuildUtils}/bin/set-git-rev "${gitrev'}" $out/bin/*''; - # package with libsodium: - gitrev' = if (gitrev == null) - then buildPackages.commonLib.commitIdFromGitRepoOrZero ../.git - else gitrev; - haskellBuildUtils = buildPackages.haskellBuildUtils.package; + setGitRev = ''${buildPackages.haskellBuildUtils}/bin/set-git-rev "${gitrev}" $out/bin/*''; in pkgSet diff --git a/nix/nixos/cardano-db-sync-service.nix b/nix/nixos/cardano-db-sync-service.nix index 25a531e38..88eaa93b6 100644 --- a/nix/nixos/cardano-db-sync-service.nix +++ b/nix/nixos/cardano-db-sync-service.nix @@ -2,17 +2,12 @@ let cfg = config.services.cardano-db-sync; - self = config.internal.syncPackages; + self = cfg.dbSyncPkgs; envConfig = cfg.environment; configFile = __toFile "db-sync-config.json" (__toJSON (cfg.explorerConfig // cfg.logConfig)); stateDirBase = "/var/lib/"; in { options = { - internal = lib.mkOption { - type = lib.types.attrs; - internal = true; - default = { syncPackages = import ../. {}; }; - }; services.cardano-db-sync = { enable = lib.mkEnableOption "enable the cardano-db-sync service"; script = lib.mkOption { @@ -31,7 +26,7 @@ in { }; environment = lib.mkOption { type = lib.types.nullOr lib.types.attrs; - default = self.iohkNix.cardanoLib.environments.${cfg.cluster}; + default = self.cardanoLib.environments.${cfg.cluster}; }; cluster = lib.mkOption { type = lib.types.nullOr lib.types.str; @@ -43,7 +38,7 @@ in { }; logConfig = lib.mkOption { type = lib.types.attrs; - default = self.iohkNix.cardanoLib.defaultExplorerLogConfig; + default = self.cardanoLib.defaultExplorerLogConfig; }; socketPath = lib.mkOption { type = lib.types.nullOr lib.types.path; @@ -53,6 +48,16 @@ in { type = lib.types.nullOr lib.types.str; default = "/var/lib/${cfg.user}"; }; + dbSyncPkgs = lib.mkOption { + type = lib.types.attrs; + default = import ../. {}; + defaultText = "cardano-db-sync pkgs"; + description = '' + The cardano-db-sync packages and library that should be used. + Main usage is sharing optimization: + reduce eval time when service is instantiated multiple times. + ''; + }; package = lib.mkOption { type = lib.types.package; default = if cfg.extended then self.cardano-db-sync-extended else self.cardano-db-sync; diff --git a/nix/pkgs.nix b/nix/pkgs.nix index 704f077d3..d49a8be55 100644 --- a/nix/pkgs.nix +++ b/nix/pkgs.nix @@ -1,17 +1,18 @@ # our packages overlay -pkgs: super: with pkgs; - let - compiler = config.haskellNix.compiler or "ghc8104"; - src = haskell-nix.haskellLib.cleanGit { - src = ../.; - name = "cardano-db-sync"; - }; - in { - inherit src; +final: prev: with final; +let + compiler = config.haskellNix.compiler or "ghc8104"; +in { + src = haskell-nix.haskellLib.cleanGit { + src = ../.; + name = "cardano-db-sync"; + }; - cardanoDbSyncHaskellPackages = callPackage ./haskell.nix { - inherit compiler gitrev src; - }; + cardanoDbSyncProject = callPackage ./haskell.nix { + inherit compiler; + }; + + cardanoDbSyncHaskellPackages = cardanoDbSyncProject.hsPkgs; # Grab the executable component of our package. inherit (cardanoDbSyncHaskellPackages.cardano-db-sync.components.exes) @@ -21,22 +22,46 @@ pkgs: super: with pkgs; inherit (cardanoDbSyncHaskellPackages.cardano-node.components.exes) cardano-node; - inherit ((haskell-nix.hackage-package { - name = "hlint"; - version = "3.1.6"; - compiler-nix-name = compiler; - inherit (cardanoDbSyncHaskellPackages) index-state; - }).components.exes) hlint; + cabal = haskell-nix.tool compiler "cabal" { + version = "latest"; + inherit (cardanoDbSyncProject) index-state; + }; - inherit ((haskell-nix.hackage-package { - name = "stylish-haskell"; - version = "0.12.2.0"; - compiler-nix-name = compiler; - inherit (cardanoDbSyncHaskellPackages) index-state; - }).components.exes) stylish-haskell; + hlint = haskell-nix.tool compiler "hlint" { + version = "3.2.7"; + inherit (cardanoDbSyncProject) index-state; + }; + + stylish-haskell = haskell-nix.tool compiler "stylish-haskell" { + version = "latest"; + inherit (cardanoDbSyncProject) index-state; + }; # systemd can't be statically linked: - postgresql = super.postgresql.override { + postgresql = prev.postgresql.override { enableSystemd = stdenv.hostPlatform.isLinux && !stdenv.hostPlatform.isMusl; }; + + scripts = import ./scripts.nix { inherit pkgs; }; + + dockerImage = let + defaultConfig = rec { + services.cardano-db-sync = { + socketPath = lib.mkDefault ("/node-ipc/node.socket"); + postgres.generatePGPASS = false; + }; + }; + in callPackage ./docker.nix { + scripts = import ./scripts.nix { + inherit pkgs; + customConfigs = [ defaultConfig customConfig ]; + }; + extendedScripts = import ./scripts.nix { + inherit pkgs; + customConfigs = [ + defaultConfig customConfig + { services.cardano-db-sync.extended = true; } + ]; + }; + }; } diff --git a/nix/scripts.nix b/nix/scripts.nix index a829b31f8..a2bd051bf 100644 --- a/nix/scripts.nix +++ b/nix/scripts.nix @@ -1,30 +1,31 @@ -{ pkgs, lib, iohkNix, customConfig }: +{ pkgs +, customConfigs ? [ pkgs.customConfig ] }: let + inherit (pkgs) lib cardanoLib; + inherit (pkgs.commonLib) evalService; blacklistedEnvs = [ "selfnode" "shelley_selfnode" "latency-tests" "mainnet-ci" ]; - environments = lib.filterAttrs (k: v: (!builtins.elem k blacklistedEnvs)) iohkNix.cardanoLib.environments; - mkStartScripts = envConfig: let - systemdCompat.options = { - systemd.services = lib.mkOption {}; - services.postgresql = lib.mkOption {}; - assertions = lib.mkOption {}; - users = lib.mkOption {}; + environments = lib.filterAttrs (k: v: (!builtins.elem k blacklistedEnvs)) cardanoLib.environments; + mkScript = envConfig: let + service = evalService { + inherit pkgs customConfigs; + serviceName = "cardano-db-sync"; + modules = [ + ./nixos/cardano-db-sync-service.nix + { + services.cardano-db-sync = { + postgres.user = lib.mkDefault "*"; + environment = lib.mkDefault envConfig; + cluster = lib.mkDefault envConfig.name; + dbSyncPkgs = lib.mkDefault pkgs; + }; + } + ]; }; - eval = let - extra = { - internal.syncPackages = pkgs; - services.cardano-db-sync = { - enable = true; - postgres.user = "*"; - environment = envConfig; - cluster = envConfig.name; - }; - }; - in lib.evalModules { - prefix = []; - modules = import nixos/module-list.nix ++ [ systemdCompat customConfig extra ]; - args = { inherit pkgs; }; - }; - in { - db-sync = eval.config.services.cardano-db-sync.script; + in lib.recurseIntoAttrs { + db-sync = pkgs.writeScriptBin "cardano-db-sync-${service.cluster}" '' + #!${pkgs.runtimeShell} + set -euo pipefail + ${service.script} $@ + ''; }; -in iohkNix.cardanoLib.forEnvironmentsCustom mkStartScripts environments // { inherit environments; } +in cardanoLib.forEnvironmentsCustom mkScript environments diff --git a/nix/sources.json b/nix/sources.json deleted file mode 100644 index c07314af9..000000000 --- a/nix/sources.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "haskell.nix": { - "branch": "master", - "description": "Alternative Haskell Infrastructure for Nixpkgs", - "homepage": "https://input-output-hk.github.io/haskell.nix", - "owner": "input-output-hk", - "repo": "haskell.nix", - "rev": "9b7c3e6bb0f95759a582e710dc41b2607a775b26", - "sha256": "0fz44qlpwz2f468qn30nb1pw8857vnhiafnabrzvpkvf4xzrm6n8", - "type": "tarball", - "url": "https://github.com/input-output-hk/haskell.nix/archive/9b7c3e6bb0f95759a582e710dc41b2607a775b26.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "iohk-nix": { - "branch": "master", - "description": "nix scripts shared across projects", - "homepage": null, - "owner": "input-output-hk", - "repo": "iohk-nix", - "rev": "5b86d63a0b59b7666d19901b654d8fbde27d9500", - "sha256": "1mfb93nnw0x4gyq93v6lh6h7imliw4j0wp5l9gpdafy3rw621xzb", - "type": "tarball", - "url": "https://github.com/input-output-hk/iohk-nix/archive/5b86d63a0b59b7666d19901b654d8fbde27d9500.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - } -} diff --git a/nix/sources.nix b/nix/sources.nix deleted file mode 100644 index 1938409dd..000000000 --- a/nix/sources.nix +++ /dev/null @@ -1,174 +0,0 @@ -# This file has been generated by Niv. - -let - - # - # The fetchers. fetch_ fetches specs of type . - # - - fetch_file = pkgs: name: spec: - let - name' = sanitizeName name + "-src"; - in - if spec.builtin or true then - builtins_fetchurl { inherit (spec) url sha256; name = name'; } - else - pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; - - fetch_tarball = pkgs: name: spec: - let - name' = sanitizeName name + "-src"; - in - if spec.builtin or true then - builtins_fetchTarball { name = name'; inherit (spec) url sha256; } - else - pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; - - fetch_git = name: spec: - let - ref = - if spec ? ref then spec.ref else - if spec ? branch then "refs/heads/${spec.branch}" else - if spec ? tag then "refs/tags/${spec.tag}" else - abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; - in - builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; }; - - fetch_local = spec: spec.path; - - fetch_builtin-tarball = name: throw - ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. - $ niv modify ${name} -a type=tarball -a builtin=true''; - - fetch_builtin-url = name: throw - ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. - $ niv modify ${name} -a type=file -a builtin=true''; - - # - # Various helpers - # - - # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 - sanitizeName = name: - ( - concatMapStrings (s: if builtins.isList s then "-" else s) - ( - builtins.split "[^[:alnum:]+._?=-]+" - ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) - ) - ); - - # The set of packages used when specs are fetched using non-builtins. - mkPkgs = sources: system: - let - sourcesNixpkgs = - import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; - hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; - hasThisAsNixpkgsPath = == ./.; - in - if builtins.hasAttr "nixpkgs" sources - then sourcesNixpkgs - else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then - import {} - else - abort - '' - Please specify either (through -I or NIX_PATH=nixpkgs=...) or - add a package called "nixpkgs" to your sources.json. - ''; - - # The actual fetching function. - fetch = pkgs: name: spec: - - if ! builtins.hasAttr "type" spec then - abort "ERROR: niv spec ${name} does not have a 'type' attribute" - else if spec.type == "file" then fetch_file pkgs name spec - else if spec.type == "tarball" then fetch_tarball pkgs name spec - else if spec.type == "git" then fetch_git name spec - else if spec.type == "local" then fetch_local spec - else if spec.type == "builtin-tarball" then fetch_builtin-tarball name - else if spec.type == "builtin-url" then fetch_builtin-url name - else - abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; - - # If the environment variable NIV_OVERRIDE_${name} is set, then use - # the path directly as opposed to the fetched source. - replace = name: drv: - let - saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name; - ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; - in - if ersatz == "" then drv else - # this turns the string into an actual Nix path (for both absolute and - # relative paths) - if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; - - # Ports of functions for older nix versions - - # a Nix version of mapAttrs if the built-in doesn't exist - mapAttrs = builtins.mapAttrs or ( - f: set: with builtins; - listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) - ); - - # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 - range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1); - - # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 - stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); - - # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 - stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); - concatMapStrings = f: list: concatStrings (map f list); - concatStrings = builtins.concatStringsSep ""; - - # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 - optionalAttrs = cond: as: if cond then as else {}; - - # fetchTarball version that is compatible between all the versions of Nix - builtins_fetchTarball = { url, name ? null, sha256 }@attrs: - let - inherit (builtins) lessThan nixVersion fetchTarball; - in - if lessThan nixVersion "1.12" then - fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) - else - fetchTarball attrs; - - # fetchurl version that is compatible between all the versions of Nix - builtins_fetchurl = { url, name ? null, sha256 }@attrs: - let - inherit (builtins) lessThan nixVersion fetchurl; - in - if lessThan nixVersion "1.12" then - fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) - else - fetchurl attrs; - - # Create the final "sources" from the config - mkSources = config: - mapAttrs ( - name: spec: - if builtins.hasAttr "outPath" spec - then abort - "The values in sources.json should not have an 'outPath' attribute" - else - spec // { outPath = replace name (fetch config.pkgs name spec); } - ) config.sources; - - # The "config" used by the fetchers - mkConfig = - { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null - , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile) - , system ? builtins.currentSystem - , pkgs ? mkPkgs sources system - }: rec { - # The sources, i.e. the attribute set of spec name to spec - inherit sources; - - # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers - inherit pkgs; - }; - -in -mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); } diff --git a/release.nix b/release.nix index 2ac685d9b..6be4c9e13 100644 --- a/release.nix +++ b/release.nix @@ -50,7 +50,7 @@ let pkgs.runCommand "${image.name}-hydra" {} '' mkdir -pv $out/nix-support/ cat < $out/nix-support/hydra-build-products - file dockerimage-${image.name} ${image} + file binary-dist ${image} EOF ''; diff --git a/shell.nix b/shell.nix index d6bc6099e..1856631a3 100644 --- a/shell.nix +++ b/shell.nix @@ -3,28 +3,39 @@ { config ? {} , sourcesOverride ? {} , withHoogle ? true +, customConfig ? { + inherit withHoogle; + } , pkgs ? import ./nix { - inherit config sourcesOverride; + inherit config sourcesOverride customConfig; } }: with pkgs; let + inherit (pkgs.customConfig) withHoogle; + commandHelp = + '' + echo " + Commands: + * nix flake update --update-input - update input + " + ''; + # This provides a development environment that can be used with nix-shell or # lorri. See https://input-output-hk.github.io/haskell.nix/user-guide/development/ - shell = cardanoDbSyncHaskellPackages.shellFor { + shell = cardanoDbSyncProject.shellFor { name = "cabal-dev-shell"; packages = ps: lib.attrValues (haskell-nix.haskellLib.selectProjectPackages ps); # These programs will be available inside the nix-shell. - buildInputs = with haskellPackages; [ - cabal-install + nativeBuildInputs = with haskellPackages; [ + cabalWrapped ghcid hlint stylish-haskell weeder - nix - niv + nixWrapped pkgconfig sqlite-interactive tmux @@ -40,19 +51,17 @@ let devops = pkgs.stdenv.mkDerivation { name = "devops-shell"; - buildInputs = [ - niv + nativeBuildInputs = [ + nixWrapped ]; shellHook = '' echo "DevOps Tools" \ | ${figlet}/bin/figlet -f banner -c \ | ${lolcat}/bin/lolcat - echo "NOTE: you may need to export GITHUB_TOKEN if you hit rate limits with niv" - echo "Commands: - * niv update - update package - - " + echo "NOTE: you may need to export GITHUB_TOKEN if you hit rate limits with niv flake update:" + echo ' edit ~/.config/nix/nix.conf and add line `access-tokens = "github.com=23ac...b289"`' + ${commandHelp} ''; }; diff --git a/supported-systems.nix b/supported-systems.nix new file mode 100644 index 000000000..4aaea2aef --- /dev/null +++ b/supported-systems.nix @@ -0,0 +1 @@ +[ "x86_64-linux" "x86_64-darwin" ]