From 6a43164850106b27d52795ff318cbffd98b54496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Sat, 16 Mar 2024 21:17:04 +0000 Subject: [PATCH 01/11] cleanup --- tests/cardano-node.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/cardano-node.nix b/tests/cardano-node.nix index f675268..e57b8bc 100644 --- a/tests/cardano-node.nix +++ b/tests/cardano-node.nix @@ -2,8 +2,6 @@ nixosTests.tests.cardano-node = { impure = true; module = { - name = "cardano-node-test"; - nodes = { machine = {pkgs, ...}: { virtualisation = { From 26f5271cb961e62a2a24c96b5bc92d3aa81a17a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Thu, 21 Mar 2024 12:24:46 +0000 Subject: [PATCH 02/11] add ogmios package --- flake.lock | 66 +++++++----------------------------------- flake.nix | 47 ++++++++++++++---------------- modules/default.nix | 16 +++++----- packages/cardano.nix | 13 +++++++++ packages/default.nix | 22 ++++++-------- packages/ogmios.nix | 34 ++++++++++++++++++++++ tests/cardano-cli.nix | 3 +- tests/cardano-node.nix | 2 ++ 8 files changed, 97 insertions(+), 106 deletions(-) create mode 100644 packages/cardano.nix create mode 100644 packages/ogmios.nix diff --git a/flake.lock b/flake.lock index 9c99974..49fe18c 100644 --- a/flake.lock +++ b/flake.lock @@ -41,7 +41,9 @@ "nixpkgs": [ "nixpkgs" ], - "nixpkgs-stable": "nixpkgs-stable" + "nixpkgs-stable": [ + "nixpkgs" + ] }, "locked": { "lastModified": 1704159259, @@ -642,26 +644,6 @@ "type": "github" } }, - "flake-parts_2": { - "inputs": { - "nixpkgs-lib": [ - "hercules-ci-effects", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1709336216, - "narHash": "sha256-Dt/wOWeW6Sqm11Yh+2+t0dfEWxoMxGBvv3JpIocFl9E=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "f7b3c975cf067e56e7cda6cb098ebe3fb4d74ca2", - "type": "github" - }, - "original": { - "id": "flake-parts", - "type": "indirect" - } - }, "flake-root": { "locked": { "lastModified": 1692742795, @@ -982,8 +964,12 @@ }, "hercules-ci-effects": { "inputs": { - "flake-parts": "flake-parts_2", - "nixpkgs": "nixpkgs_10" + "flake-parts": [ + "flake-parts" + ], + "nixpkgs": [ + "nixpkgs" + ] }, "locked": { "lastModified": 1710396488, @@ -1600,22 +1586,6 @@ "type": "github" } }, - "nixpkgs-stable": { - "locked": { - "lastModified": 1702780907, - "narHash": "sha256-blbrBBXjjZt6OKTcYX1jpe9SRof2P9ZYWPzq22tzXAA=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "1e2e384c5b7c50dbf8e9c441a9e58d85f408b01f", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-23.11", - "repo": "nixpkgs", - "type": "github" - } - }, "nixpkgs-unstable": { "locked": { "lastModified": 1695318763, @@ -1633,22 +1603,6 @@ } }, "nixpkgs_10": { - "locked": { - "lastModified": 1709961763, - "narHash": "sha256-6H95HGJHhEZtyYA3rIQpvamMKAGoa8Yh2rFV29QnuGw=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "3030f185ba6a4bf4f18b87f345f104e6a6961f34", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_11": { "locked": { "lastModified": 1697059129, "narHash": "sha256-9NJcFF9CEYPvHJ5ckE8kvINvI84SZZ87PvqMbH6pro0=", @@ -2055,7 +2009,7 @@ "flake-parts": "flake-parts", "flake-root": "flake-root", "hercules-ci-effects": "hercules-ci-effects", - "nixpkgs": "nixpkgs_11", + "nixpkgs": "nixpkgs_10", "pre-commit-hooks-nix": "pre-commit-hooks-nix", "treefmt-nix": "treefmt-nix" } diff --git a/flake.nix b/flake.nix index b108547..cc6f80a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,36 +1,30 @@ { inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - flake-parts.url = "github:hercules-ci/flake-parts"; - - # we use effects for CI, documentation and pushing to public cache - hercules-ci-effects.url = "github:mlabs-haskell/hercules-ci-effects/push-cache-effect"; + # cardano-node cardano-node.url = "github:intersectmbo/cardano-node?ref=8.7.3"; # Utilities - devshell = { - url = "github:numtide/devshell"; - inputs.nixpkgs.follows = "nixpkgs"; - }; + attic.url = "github:zhaofengli/attic"; + devour-flake.url = "github:srid/devour-flake"; + devour-flake.flake = false; + devshell.url = "github:numtide/devshell"; + flake-parts.url = "github:hercules-ci/flake-parts"; flake-root.url = "github:srid/flake-root"; - treefmt-nix = { - url = "github:numtide/treefmt-nix"; - inputs.nixpkgs.follows = "nixpkgs"; - }; - devour-flake = { - url = "github:srid/devour-flake"; - flake = false; - }; - attic = { - url = "github:zhaofengli/attic"; - inputs.nixpkgs.follows = "nixpkgs"; - }; - pre-commit-hooks-nix = { - url = "github:cachix/pre-commit-hooks.nix"; - inputs.nixpkgs.follows = "nixpkgs"; - inputs.nixpkgs-stable.follows = "nixpkgs"; # prevent unnecessary download - }; + hercules-ci-effects.url = "github:mlabs-haskell/hercules-ci-effects/push-cache-effect"; + pre-commit-hooks-nix.url = "github:cachix/pre-commit-hooks.nix"; + treefmt-nix.url = "github:numtide/treefmt-nix"; + + # Prevent unnecessary downloads + attic.inputs.nixpkgs.follows = "nixpkgs"; + attic.inputs.nixpkgs-stable.follows = "nixpkgs"; + devshell.inputs.nixpkgs.follows = "nixpkgs"; + hercules-ci-effects.inputs.nixpkgs.follows = "nixpkgs"; + hercules-ci-effects.inputs.flake-parts.follows = "flake-parts"; + pre-commit-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + pre-commit-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; + treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; }; outputs = inputs @ {flake-parts, ...}: flake-parts.lib.mkFlake { @@ -51,7 +45,8 @@ systems = [ "x86_64-linux" "aarch64-linux" - "x86_64-darwin" + # Ogmios doesn't support it + # "x86_64-darwin" "aarch64-darwin" ]; }; diff --git a/modules/default.nix b/modules/default.nix index f95cbe3..a8b385d 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -5,22 +5,20 @@ }: { flake.nixosModules = rec { globals = ./globals; + cardano-cli = { + imports = [./cardano-cli]; + nixpkgs.overlays = [config.flake.overlays.cardano-cli]; + }; cardano-node = { imports = [ inputs.cardano-node.nixosModules.cardano-node - cardano-overlay ./cardano-node ]; - }; - cardano-cli = { - imports = [ - cardano-overlay - ./cardano-cli + nixpkgs.overlays = [ + config.flake.overlays.cardano-cli + config.flake.overlays.cardano-node ]; }; - cardano-overlay = { - nixpkgs.overlays = [config.flake.overlays.default]; - }; # the default module imports all modules default = { imports = with builtins; attrValues (removeAttrs config.flake.nixosModules ["default"]); diff --git a/packages/cardano.nix b/packages/cardano.nix new file mode 100644 index 0000000..d7e91d7 --- /dev/null +++ b/packages/cardano.nix @@ -0,0 +1,13 @@ +{inputs, ...}: { + imports = [inputs.flake-parts.flakeModules.easyOverlay]; + perSystem = { + system, + lib, + ... + }: { + packages = let + packageNames = ["cardano-node" "cardano-cli"]; + in + lib.filterAttrs (n: _: builtins.elem n packageNames) (inputs.cardano-node.packages.${system} or {}); + }; +} diff --git a/packages/default.nix b/packages/default.nix index d2452b9..c4659e0 100644 --- a/packages/default.nix +++ b/packages/default.nix @@ -1,15 +1,11 @@ -{inputs, ...}: { - imports = [inputs.flake-parts.flakeModules.easyOverlay]; - perSystem = { - config, - system, - lib, - ... - }: { - packages = let - packageNames = ["cardano-node" "cardano-cli"]; - in - lib.filterAttrs (n: _: builtins.elem n packageNames) (inputs.cardano-node.packages.${system} or {}); - overlayAttrs = config.packages; +{config, ...}: { + imports = [ + ./cardano.nix + ./ogmios.nix + ]; + flake.overlays = { + cardano-cli = final: _prev: {inherit ((config.perSystem final.system).packages) cardano-cli;}; + cardano-node = final: _prev: {inherit ((config.perSystem final.system).packages) cardano-node;}; + ogmios = final: _prev: {inherit ((config.perSystem final.system).packages) ogmios;}; }; } diff --git a/packages/ogmios.nix b/packages/ogmios.nix new file mode 100644 index 0000000..f6a6b30 --- /dev/null +++ b/packages/ogmios.nix @@ -0,0 +1,34 @@ +{ + perSystem = { + system, + pkgs, + ... + }: { + packages.ogmios = pkgs.stdenv.mkDerivation rec { + pname = "ogmios"; + version = "6.1.0"; + src = pkgs.fetchurl { + url = "https://github.com/CardanoSolutions/ogmios/releases/download/v${version}/ogmios-v${version}-${system}.zip"; + hash = + if system == "x86_64-linux" + then "sha256-JQJTaws+gihwHpTBtUqcNhgbhFDUAdiiXrO2kMX4ZkY=" + else if system == "aarch64-linux" + then "sha256-gWNTzUZHdp5rvbU0aIA3/GJ+YZeFx66h05ugJN4kMco=" + else if system == "aarch64-darwin" + then "sha256-gWNTzUZHdp5rvbU0aIA3/GJ+YZeFx66h05ugJN4kMco=" + else abort "Ogmios release not available for system ${system}"; + }; + nativeBuildInputs = [pkgs.unzip]; + unpackPhase = '' + unzip $src + ''; + buildPhase = '' + chmod a+x bin/ogmios + ''; + installPhase = '' + mkdir -p $out + mv bin share $out/ + ''; + }; + }; +} diff --git a/tests/cardano-cli.nix b/tests/cardano-cli.nix index 10ee63d..f32bd8a 100644 --- a/tests/cardano-cli.nix +++ b/tests/cardano-cli.nix @@ -1,7 +1,6 @@ { nixosTests.tests.cardano-cli.module = { - name = "cardano-cli-test"; - + name = "cardano-cli"; nodes = { machine = { virtualisation = { diff --git a/tests/cardano-node.nix b/tests/cardano-node.nix index e57b8bc..a8c9750 100644 --- a/tests/cardano-node.nix +++ b/tests/cardano-node.nix @@ -2,6 +2,8 @@ nixosTests.tests.cardano-node = { impure = true; module = { + name = "cardano-node"; + nodes = { machine = {pkgs, ...}: { virtualisation = { From 8881ab1c0e576e8f9b70d0118c23e6f88df31143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Thu, 21 Mar 2024 13:10:24 +0000 Subject: [PATCH 03/11] ogmios module --- README.md | 37 ++++- checks/default.nix | 10 +- checks/nixosTests.nix | 145 ----------------- checks/{run-test.nix => run-vm-test.nix} | 21 +-- checks/vmTests.nix | 148 ++++++++++++++++++ ci/default.nix | 5 +- docs/default.nix | 6 +- flake.lock | 18 +++ flake.nix | 5 +- formatter/default.nix | 4 +- .../default.nix => cardano-cli.nix} | 0 modules/cardano-node.nix | 67 ++++++++ modules/cardano-node/default.nix | 24 --- modules/default.nix | 28 +++- modules/{globals/default.nix => globals.nix} | 16 +- modules/ogmios.nix | 25 +++ modules/services/ogmios.nix | 118 ++++++++++++++ packages/cardano.nix | 22 +-- packages/default.nix | 13 +- shell/default.nix | 4 +- tests/cardano-cli.nix | 3 +- tests/cardano-node.nix | 7 +- tests/default.nix | 11 +- tests/ogmios.nix | 33 ++++ 24 files changed, 525 insertions(+), 245 deletions(-) delete mode 100644 checks/nixosTests.nix rename checks/{run-test.nix => run-vm-test.nix} (69%) create mode 100644 checks/vmTests.nix rename modules/{cardano-cli/default.nix => cardano-cli.nix} (100%) create mode 100644 modules/cardano-node.nix delete mode 100644 modules/cardano-node/default.nix rename modules/{globals/default.nix => globals.nix} (71%) create mode 100644 modules/ogmios.nix create mode 100644 modules/services/ogmios.nix create mode 100644 tests/ogmios.nix diff --git a/README.md b/README.md index 2726451..21780e4 100644 --- a/README.md +++ b/README.md @@ -23,23 +23,46 @@ Don't edit `~/.config/nix/nix.conf` in your home directory. Don't add users to ` ### Development Shell -`cardano.nix` provides a devshell that includes some useful tools and aliases: +Development is supported on linux systems. Virtual machines are run with `qemu` so `kvm` is recommended. + +`cardano.nix` provides a devshell that includes various tools to build, test, run and update the project: ``` ❯ nix develop ... ❄️ Welcome to the cardano.nix devshell ❄️ -... -[Tools] - build-all - Build all the checks - check - Alias of `nix flake check` - fmt - Format the source tree -... +[documentation] + + docs-build - build documentation + docs-serve - serve documentation web page + +[general commands] + + menu - prints this menu + +[tests] + + build-all - build all packages and checks with `devour-flake` + check - run `nix flake check` + run-vm-test - list and run virtual machine integration tests + +[tools] + + fmt - format the source tree + update-pre-commit-hooks - update git pre-commit hooks ``` A `.envrc` is also provided, using [direnv]() and [nix-direnv](https://github.com/nix-community/nix-direnv) is suggested. +### Run Integration Test + +From the devshell, run an integration test that starts `cardano-node` and `ogmios` on the `preview` testnet and checks for synchronization progress. + +``` +run-vm-test ogmios +``` + ## License information `cardano.nix` released under terms of [Apache-2.0](LICENSES/Apache-2.0.txt) license. diff --git a/checks/default.nix b/checks/default.nix index bec963f..6e0445d 100644 --- a/checks/default.nix +++ b/checks/default.nix @@ -1,6 +1,6 @@ {inputs, ...}: { imports = [ - ./nixosTests.nix + ./vmTests.nix ./licenses.nix ]; perSystem = { @@ -26,15 +26,15 @@ devshells.default.commands = [ { - category = "Tools"; + category = "tests"; name = "build-all"; - help = "Build all the checks"; + help = "build all packages and checks with `devour-flake`"; command = config.apps.nix-build-all.program; } { - category = "Tools"; + category = "tests"; name = "check"; - help = "Alias of `nix flake check`"; + help = "run `nix flake check`"; command = "nix flake check"; } ]; diff --git a/checks/nixosTests.nix b/checks/nixosTests.nix deleted file mode 100644 index bbaa9b2..0000000 --- a/checks/nixosTests.nix +++ /dev/null @@ -1,145 +0,0 @@ -{ - lib, - inputs, - config, - withSystem, - ... -}: let - inherit (lib) mkOption types mapAttrs' nameValuePair; - inherit (config.flake) nixosModules; -in { - perSystem = { - config, - lib, - system, - pkgs, - ... - }: let - cfg = config.nixosTests; - in { - options.nixosTests = { - tests = mkOption { - description = "NixOS tests as modules."; - type = types.lazyAttrsOf (types.submodule ({config, ...}: { - options = { - name = mkOption { - description = "The name of the test."; - type = types.str; - default = config._module.args.name; - internal = true; - }; - systems = mkOption { - description = "The systems to run the tests on."; - type = types.listOf types.str; - default = ["x86_64-linux"]; - }; - module = mkOption { - description = "The test module. Required."; - type = types.deferredModule; - }; - documentation = mkOption { - description = "Wether to generate documentation for the testnixos configuraion. False by default to speed up builds."; - type = types.bool; - default = false; - }; - specialArgs = mkOption { - description = "The specialArgs to pass to the test node."; - type = types.attrsOf types.anything; - default = {}; - }; - impure = mkOption { - description = "Wether the test requires internet access and should be run as an effect instead of a nix build."; - type = types.bool; - default = false; - }; - check = mkOption { - description = "The test derivation composed with _mkCheckFromTest from the module."; - type = types.package; - default = cfg._mkCheckFromTest config; - }; - checkEffect = mkOption { - description = "The test hercules-ci-effect composed with _mkEffectFromTest from the module."; - type = types.package; - default = cfg._mkEffectFromTest config; - }; - }; - })); - }; - runTestScript = mkOption { - type = types.package; - default = pkgs.callPackage ./run-test.nix {inherit (cfg) tests;}; - description = "A convenience script to run tests"; - }; - _nixosLib = mkOption { - type = types.anything; - default = import (inputs.nixpkgs.outPath + "/nixos/lib") {}; - internal = true; - }; - _mkCheckFromTest = mkOption { - type = types.functionTo types.package; - internal = true; - default = test: - (cfg._nixosLib.runTest { - hostPkgs = pkgs; - - defaults.documentation.enable = test.documentation; - - node = { - inherit (test) specialArgs; - }; - - defaults.imports = [ - # import all of our flake nixos modules by default - nixosModules.default - # fix missing pkgs.system in tests - {nixpkgs.overlays = [(_: _: {inherit system;})];} - ]; - - # import the test module - imports = [test.module]; - }) - .config - .result; - }; - _mkEffectFromTest = mkOption { - type = types.functionTo types.package; - internal = true; - default = test: - withSystem system ({hci-effects, ...}: - hci-effects.modularEffect { - mounts."/dev/kvm" = "kvm"; - effectScript = '' - ${test.check.driver}/bin/nixos-test-driver - ''; - }); - }; - }; - - config = { - checks = - mapAttrs' - (name: test: nameValuePair "nixosTests-${test.name}" test.check) - (lib.filterAttrs - (_: v: lib.elem system v.systems && !v.impure) - cfg.tests); - - apps.run-test.program = lib.getExe cfg.runTestScript; - - devshells.default.commands = [ - { - name = "run-test"; - category = "testing"; - help = "Run tests"; - command = "${lib.getExe cfg.runTestScript} $@"; - } - ]; - }; - }; - - herculesCI.onPush.default.outputs.effects = - mapAttrs' - (name: test: nameValuePair "nixosTests-${test.name}" test.checkEffect) - (lib.filterAttrs - (_: v: lib.elem config.defaultEffectSystem v.systems && v.impure) - (config.perSystem config.defaultEffectSystem).nixosTests.tests); -} diff --git a/checks/run-test.nix b/checks/run-vm-test.nix similarity index 69% rename from checks/run-test.nix rename to checks/run-vm-test.nix index 9c8c82e..e18b510 100644 --- a/checks/run-test.nix +++ b/checks/run-vm-test.nix @@ -6,7 +6,7 @@ ... }: writeShellApplication { - name = "run-test"; + name = "run-vm-test"; runtimeInputs = []; @@ -14,21 +14,21 @@ writeShellApplication { cmd_name=$(basename "$0") help() { - echo " build and run a test" + echo " Build and run integration test on a network of virtual machines." echo - echo " usage:" + echo " Usage:" echo " $cmd_name " echo " $cmd_name --interactive" echo " $cmd_name -s " echo - echo " arguments:" + echo " Arguments:" echo " " echo - echo " options:" - echo " -h --help show this screen." - echo " -l --list show available tests." - echo " -s --system specify the target platform [default: ${stdenv.system}]." - echo " -i --interactive run the test interactively." + echo " Options:" + echo " -h --help Show this screen." + echo " -l --list Show available tests." + echo " -s --system Specify the target platform [default: ${stdenv.system}]." + echo " -i --interactive Run the test interactively." echo } @@ -65,6 +65,7 @@ writeShellApplication { shift # build/run the test driver, passing any remaining arguments - nix run ".#checks.$system.testing-$name.driver" "''${nix_args}" -- "''${driver_args[@]}" + # shellcheck disable=SC2068,SC2086 + nix run ".#apps.$system.vmTests-$name" ''${nix_args} -- ''${driver_args[@]} ''; } diff --git a/checks/vmTests.nix b/checks/vmTests.nix new file mode 100644 index 0000000..67f41c4 --- /dev/null +++ b/checks/vmTests.nix @@ -0,0 +1,148 @@ +{ + lib, + inputs, + config, + withSystem, + ... +}: let + inherit (lib) mkOption types mapAttrs' nameValuePair; + inherit (config.flake) nixosModules; +in { + perSystem = { + config, + lib, + system, + pkgs, + ... + }: let + cfg = config.vmTests; + in { + options.vmTests = { + tests = mkOption { + description = "Run integration tests in networks of virtual machines."; + type = types.lazyAttrsOf (types.submodule ({config, ...}: { + options = { + name = mkOption { + description = "The name of the test. Defaults to attribute name."; + internal = true; + type = types.str; + default = config._module.args.name; + }; + systems = mkOption { + description = "The systems to run the test on."; + type = types.listOf types.str; + default = ["x86_64-linux"]; + }; + module = mkOption { + description = "The NixOS test module. Required. See https://nixos.org/manual/nixos/stable/#sec-nixos-tests ."; + type = types.deferredModule; + }; + impure = mkOption { + description = "Wether the test requires internet access and should be run as an effect instead of a nix build."; + type = types.bool; + default = false; + }; + check = mkOption { + description = "The test derivation. Result of calling `_mkCheck` with this test."; + type = types.package; + default = cfg._mkCheck config; + }; + effect = mkOption { + description = "The test hercules-ci-effect. Result of calling `_mkEffect` with this test."; + type = types.package; + default = cfg._mkEffect config; + }; + }; + })); + }; + runVmTestScript = mkOption { + description = "Script that lists and runs integration tests on networks of virtual machines."; + type = types.package; + default = pkgs.callPackage ./run-vm-test.nix {inherit (cfg) tests;}; + }; + _nixosLib = mkOption { + description = "Convenience access to `nixpkgs/nixos/lib`."; + internal = true; + type = types.anything; + default = import (inputs.nixpkgs.outPath + "/nixos/lib") {}; + }; + _mkCheck = mkOption { + description = "Function that takes a test `module` and returns a derivation that runs the test when built."; + internal = true; + type = types.functionTo types.package; + default = test: + (cfg._nixosLib.runTest { + name = lib.mkDefault test.name; + imports = [test.module]; + hostPkgs = pkgs; + defaults = { + imports = [ + # Import all of our NixOS modules by default. + nixosModules.default + # Fix missing `pkgs.system` in tests. + {nixpkgs.overlays = [(_: _: {inherit system;})];} + ]; + documentation.enable = lib.mkDefault false; + }; + }) + .config + .result; + }; + _mkEffect = mkOption { + description = "Function that takes a test `module` and returns a Hercules CI effect that runs the test."; + internal = true; + type = types.functionTo types.package; + default = testModule: + withSystem system ({hci-effects, ...}: + hci-effects.modularEffect { + mounts."/dev/kvm" = "kvm"; + effectScript = '' + ${testModule.check.driver}/bin/nixos-test-driver + ''; + }); + }; + }; + + config = { + checks = + mapAttrs' + (name: test: + nameValuePair + "vmTests-${test.name}" + test.check) + (lib.filterAttrs + (_: v: lib.elem system v.systems && !v.impure) + cfg.tests); + + apps = + {run-vm-tests.program = lib.getExe cfg.runVmTestScript;} + // mapAttrs' + (name: test: + nameValuePair + "vmTests-${test.name}" + {program = "${test.check.driver}/bin/nixos-test-driver";}) + (lib.filterAttrs + (_: v: lib.elem system v.systems) + cfg.tests); + + devshells.default.commands = [ + { + name = "run-vm-test"; + category = "tests"; + help = "list and run virtual machine integration tests"; + command = "${lib.getExe cfg.runVmTestScript} $@"; + } + ]; + }; + }; + + herculesCI.onPush.default.outputs.effects = + mapAttrs' + (name: test: + nameValuePair + "vmTests-${test.name}" + test.effect) + (lib.filterAttrs + (_: v: lib.elem config.defaultEffectSystem v.systems && v.impure) + (config.perSystem config.defaultEffectSystem).vmTests.tests); +} diff --git a/ci/default.nix b/ci/default.nix index 95bb972..4097ea2 100644 --- a/ci/default.nix +++ b/ci/default.nix @@ -22,7 +22,6 @@ perSystem = {config, ...}: { hercules-ci.github-pages.settings.contents = config.packages.docs; }; - herculesCI.ciSystems = ["x86_64-linux" "x86_64-darwin"]; push-cache-effect = { enable = true; @@ -35,9 +34,9 @@ flatten [ (forEach ["apps" "devShells" "packages"] (attr: - forEach ["x86_64-linux" "x86_64-darwin" "aarch64-linux"] + forEach config.systems (system: - collect isDerivation config.flake.${attr}.${system}))) + collect isDerivation (config.flake.${attr}.${system} or {})))) (forEach (attrValues config.flake.nixosConfigurations) (os: os.config.system.build.toplevel)) diff --git a/docs/default.nix b/docs/default.nix index 529abeb..ca01d3e 100644 --- a/docs/default.nix +++ b/docs/default.nix @@ -147,18 +147,18 @@ devshells.default = { commands = let - category = "Docs"; + category = "documentation"; in [ { inherit category; name = "docs-serve"; - help = "Serve docs"; + help = "serve documentation web page"; command = "nix run .#docs.serve"; } { inherit category; name = "docs-build"; - help = "Build docs"; + help = "build documentation"; command = "nix build .#docs"; } ]; diff --git a/flake.lock b/flake.lock index 49fe18c..71e00a4 100644 --- a/flake.lock +++ b/flake.lock @@ -184,6 +184,23 @@ "type": "github" } }, + "cardano-configurations": { + "flake": false, + "locked": { + "lastModified": 1702085095, + "narHash": "sha256-IJChESftdO2tj2pRB+82xMaLP/RqyKHzttE7QMLqvBQ=", + "owner": "input-output-hk", + "repo": "cardano-configurations", + "rev": "21249e0d5c68b4e8f3661b250aa8272a8785d678", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "repo": "cardano-configurations", + "rev": "21249e0d5c68b4e8f3661b250aa8272a8785d678", + "type": "github" + } + }, "cardano-mainnet-mirror": { "inputs": { "nixpkgs": "nixpkgs_4" @@ -2003,6 +2020,7 @@ "root": { "inputs": { "attic": "attic", + "cardano-configurations": "cardano-configurations", "cardano-node": "cardano-node", "devour-flake": "devour-flake", "devshell": "devshell_3", diff --git a/flake.nix b/flake.nix index cc6f80a..12a00f3 100644 --- a/flake.nix +++ b/flake.nix @@ -4,6 +4,8 @@ # cardano-node cardano-node.url = "github:intersectmbo/cardano-node?ref=8.7.3"; + cardano-configurations.url = "github:input-output-hk/cardano-configurations/21249e0d5c68b4e8f3661b250aa8272a8785d678"; + cardano-configurations.flake = false; # Utilities attic.url = "github:zhaofengli/attic"; @@ -47,7 +49,8 @@ "aarch64-linux" # Ogmios doesn't support it # "x86_64-darwin" - "aarch64-darwin" + # We don't have a builder + # "aarch64-darwin" ]; }; } diff --git a/formatter/default.nix b/formatter/default.nix index c3aef65..a9f8786 100644 --- a/formatter/default.nix +++ b/formatter/default.nix @@ -25,9 +25,9 @@ devshells.default.commands = [ { - category = "Tools"; + category = "tools"; name = "fmt"; - help = "Format the source tree"; + help = "format the source tree"; command = lib.getExe config.treefmt.build.wrapper; } ]; diff --git a/modules/cardano-cli/default.nix b/modules/cardano-cli.nix similarity index 100% rename from modules/cardano-cli/default.nix rename to modules/cardano-cli.nix diff --git a/modules/cardano-node.nix b/modules/cardano-node.nix new file mode 100644 index 0000000..e76ea9f --- /dev/null +++ b/modules/cardano-node.nix @@ -0,0 +1,67 @@ +{ + config, + lib, + pkgs, + ... +}: let + cfg = config.cardanoNix.cardano-node; +in { + options.cardanoNix.cardano-node = { + enable = lib.mkEnableOption "cardano-node service"; + + socketPath = lib.mkOption { + description = "Path to cardano-node socket."; + type = lib.types.path; + default = "/run/cardano-node/node.socket"; + }; + + configPath = lib.mkOption { + description = "Path to cardano-node configuration."; + type = lib.types.path; + # TODO: remove "_p2p" after updating cardano-node to >= 8.9.0 + default = "${pkgs.cardano-configurations}/network/${config.cardanoNix.globals.network}_p2p/cardano-node/config.json"; + defaultText = lib.literalExpression "\${pkgs.cardano-configurations}/network/\${config.cardanoNix.globals.network}_p2p/cardano-node/config.json"; + }; + }; + + config = lib.mkIf cfg.enable { + environment.variables = { + CARDANO_NODE_SOCKET_PATH = cfg.socketPath; + }; + + services.cardano-node = { + enable = true; + + inherit (cfg) socketPath; + nodeConfigFile = cfg.configPath; + environment = config.cardanoNix.globals.network; + + # Listen on all interfaces. + hostAddr = lib.mkDefault "0.0.0.0"; + + # Reload unit if p2p and only topology changed. + useSystemdReload = true; + }; + + # Workaround: cardano-node service does not support systemdSocketActivation with p2p topology. + # Socket created by cardano-node is not writable by group . So we wait until it appears and set the permissions. + systemd.services.cardano-node-socket = { + description = "Wait for cardano-node socket to appear and set permissions to allow group read and write."; + after = ["cardano-node.service"]; + requires = ["cardano-node.service"]; + bindsTo = ["cardano-node.service"]; + requiredBy = ["cardano-node.service"]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + # Using a path unit doesn't allow dependencies to be declared correctly, so poll. + script = '' + echo 'Waiting for ${cfg.socketPath} to appear...' + /bin/sh -c 'until test -e ${cfg.socketPath}; do sleep 1; done' + echo 'Changing permissions for ${cfg.socketPath}.' + chmod g+rw ${cfg.socketPath} + ''; + }; + }; +} diff --git a/modules/cardano-node/default.nix b/modules/cardano-node/default.nix deleted file mode 100644 index f44e21a..0000000 --- a/modules/cardano-node/default.nix +++ /dev/null @@ -1,24 +0,0 @@ -{ - config, - lib, - ... -}: let - cfg = config.cardanoNix.cardano-node; -in { - options.cardanoNix.cardano-node = { - enable = lib.mkEnableOption "cardano-node service"; - }; - - config = lib.mkIf cfg.enable { - environment.variables = { - CARDANO_NODE_SOCKET_PATH = config.services.cardano-node.socketPath 0; - }; - - services.cardano-node = { - enable = true; - hostAddr = "0.0.0.0"; - - environment = config.cardanoNix.globals.network; - }; - }; -} diff --git a/modules/default.nix b/modules/default.nix index a8b385d..3194449 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -3,20 +3,38 @@ config, ... }: { - flake.nixosModules = rec { - globals = ./globals; + flake.nixosModules = { + globals = { + imports = [ + ./globals.nix + ]; + }; cardano-cli = { - imports = [./cardano-cli]; - nixpkgs.overlays = [config.flake.overlays.cardano-cli]; + imports = [ + ./cardano-cli.nix + ]; + nixpkgs.overlays = [ + config.flake.overlays.cardano-cli + ]; }; cardano-node = { imports = [ inputs.cardano-node.nixosModules.cardano-node - ./cardano-node + ./cardano-node.nix ]; nixpkgs.overlays = [ config.flake.overlays.cardano-cli config.flake.overlays.cardano-node + config.flake.overlays.cardano-configurations + ]; + }; + ogmios = { + imports = [ + ./services/ogmios.nix + ./ogmios.nix + ]; + nixpkgs.overlays = [ + config.flake.overlays.ogmios ]; }; # the default module imports all modules diff --git a/modules/globals/default.nix b/modules/globals.nix similarity index 71% rename from modules/globals/default.nix rename to modules/globals.nix index 3f411ec..11ae923 100644 --- a/modules/globals/default.nix +++ b/modules/globals.nix @@ -11,22 +11,19 @@ in { options.cardanoNix.globals = { network = lib.mkOption { - type = types.enum (builtins.attrNames cfg.networkNumbers); + description = "Cardano network to operate on."; + type = types.enum (lib.attrNames cfg.networkNumbers); default = "mainnet"; - description = '' - Cardano network to join/use - ''; }; networkNumber = lib.mkOption { + description = "Cardano network number to operate on. Defaults to the network number of the selected network."; + internal = true; type = types.int; default = cfg.networkNumbers.${cfg.network}; defaultText = lib.literalExpression "config.cardanoNix.globals.networkNumbers.\${config.cardanoNix.globals.net}"; - description = '' - Cardano network number to join/use (should match cardanoNix.globals,network) - ''; - internal = true; }; networkNumbers = lib.mkOption { + description = "Map from network names to network numbers. Selected network must be present in the map"; type = types.attrsOf types.int; default = { mainnet = 0; @@ -35,9 +32,6 @@ in sanchonet = 4; private = 42; }; - description = '' - Cardano network numbers - ''; internal = true; }; }; diff --git a/modules/ogmios.nix b/modules/ogmios.nix new file mode 100644 index 0000000..abd7f17 --- /dev/null +++ b/modules/ogmios.nix @@ -0,0 +1,25 @@ +{ + config, + lib, + ... +}: let + cfg = config.cardanoNix.ogmios; +in { + options.cardanoNix.ogmios = { + enable = lib.mkEnableOption "cardano-node service"; + }; + + config = lib.mkIf cfg.enable { + services.ogmios = { + enable = true; + nodeConfigPath = + lib.mkIf (config.cardanoNix.cardano-node.enable or false) + config.cardanoNix.cardano-node.configPath; + }; + + systemd.services.ogmios = lib.mkIf (config.cardanoNix.cardano-node.enable or false) { + after = ["cardano-node-socket.service"]; + requires = ["cardano-node-socket.service"]; + }; + }; +} diff --git a/modules/services/ogmios.nix b/modules/services/ogmios.nix new file mode 100644 index 0000000..a3fcccd --- /dev/null +++ b/modules/services/ogmios.nix @@ -0,0 +1,118 @@ +# NixOS module for configuring Ogmios service. +{ + config, + lib, + pkgs, + ... +}: let + cfg = config.services.ogmios; +in + with lib; { + options.services.ogmios = with types; { + enable = mkEnableOption "Ogmios lightweight bridge interface for cardano-node"; + + package = mkOption { + description = "Ogmios package"; + type = package; + default = pkgs.ogmios; + }; + + user = mkOption { + description = "User to run Ogmios service as."; + type = str; + default = "ogmios"; + }; + + group = mkOption { + description = "Group to run Ogmios service as."; + type = str; + default = "ogmios"; + }; + + nodeSocketPath = mkOption { + description = "Path to cardano-node IPC socket."; + type = path; + default = "/run/cardano-node/node.socket"; + }; + + nodeConfigPath = mkOption { + description = "Path to cardano-node config.json file. Required."; + type = path; + }; + + host = mkOption { + description = "Host address or name to listen on."; + type = str; + default = "localhost"; + }; + + port = mkOption { + description = "TCP port to listen on."; + type = port; + default = 1337; + }; + + extraArgs = mkOption { + description = "Extra arguments to ogmios command."; + type = listOf str; + default = []; + }; + }; + + config = mkIf cfg.enable { + users.users.ogmios = mkIf (cfg.user == "ogmios") { + isSystemUser = true; + inherit (cfg) group; + extraGroups = ["cardano-node"]; + }; + users.groups.ogmios = mkIf (cfg.group == "ogmios") {}; + + systemd.services.ogmios = { + enable = true; + after = ["cardano-node.service"]; + wantedBy = ["multi-user.target"]; + + script = escapeShellArgs (concatLists [ + ["${cfg.package}/bin/ogmios"] + ["--node-socket" cfg.nodeSocketPath] + ["--node-config" cfg.nodeConfigPath] + ["--host" cfg.host] + ["--port" cfg.port] + cfg.extraArgs + ]); + + serviceConfig = { + User = cfg.user; + Group = cfg.group; + # Security + UMask = "0077"; + CapabilityBoundingSet = ""; + ProcSubset = "pid"; + ProtectProc = "invisible"; + NoNewPrivileges = true; + DevicePolicy = "closed"; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + PrivateUsers = true; + ProtectHostname = true; + ProtectClock = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"]; + RestrictNamespaces = true; + LockPersonality = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + RemoveIPC = true; + PrivateMounts = true; + SystemCallArchitectures = "native"; + SystemCallFilter = ["@system-service" "~@privileged"]; + MemoryDenyWriteExecute = true; + }; + }; + }; + } diff --git a/packages/cardano.nix b/packages/cardano.nix index d7e91d7..9264955 100644 --- a/packages/cardano.nix +++ b/packages/cardano.nix @@ -1,13 +1,13 @@ -{inputs, ...}: { - imports = [inputs.flake-parts.flakeModules.easyOverlay]; - perSystem = { - system, - lib, - ... - }: { - packages = let - packageNames = ["cardano-node" "cardano-cli"]; - in - lib.filterAttrs (n: _: builtins.elem n packageNames) (inputs.cardano-node.packages.${system} or {}); +{ + inputs, + lib, + ... +}: { + perSystem = {system, ...}: { + packages = lib.filterAttrs (_: v: v != null) { + cardano-cli = inputs.cardano-node.packages.${system}.cardano-cli or null; + cardano-node = inputs.cardano-node.packages.${system}.cardano-node or null; + inherit (inputs) cardano-configurations; + }; }; } diff --git a/packages/default.nix b/packages/default.nix index c4659e0..7193728 100644 --- a/packages/default.nix +++ b/packages/default.nix @@ -1,11 +1,16 @@ -{config, ...}: { +{config, ...}: let + mkOverlay = name: ( + final: _prev: {${name} = (config.perSystem final.system).packages.${name};} + ); +in { imports = [ ./cardano.nix ./ogmios.nix ]; flake.overlays = { - cardano-cli = final: _prev: {inherit ((config.perSystem final.system).packages) cardano-cli;}; - cardano-node = final: _prev: {inherit ((config.perSystem final.system).packages) cardano-node;}; - ogmios = final: _prev: {inherit ((config.perSystem final.system).packages) ogmios;}; + cardano-cli = mkOverlay "cardano-cli"; + cardano-node = mkOverlay "cardano-node"; + cardano-configurations = mkOverlay "cardano-configurations"; + ogmios = mkOverlay "ogmios"; }; } diff --git a/shell/default.nix b/shell/default.nix index 67cd694..dfbb3ba 100644 --- a/shell/default.nix +++ b/shell/default.nix @@ -27,8 +27,8 @@ { name = "update-pre-commit-hooks"; command = config.pre-commit.installationScript; - category = "Tools"; - help = "Update pre-commit-hooks"; + category = "tools"; + help = "update git pre-commit hooks"; } ]; }; diff --git a/tests/cardano-cli.nix b/tests/cardano-cli.nix index f32bd8a..da34bf0 100644 --- a/tests/cardano-cli.nix +++ b/tests/cardano-cli.nix @@ -1,6 +1,5 @@ { - nixosTests.tests.cardano-cli.module = { - name = "cardano-cli"; + perSystem.vmTests.tests.cardano-cli.module = { nodes = { machine = { virtualisation = { diff --git a/tests/cardano-node.nix b/tests/cardano-node.nix index a8c9750..90a89cb 100644 --- a/tests/cardano-node.nix +++ b/tests/cardano-node.nix @@ -1,9 +1,7 @@ { - nixosTests.tests.cardano-node = { + perSystem.vmTests.tests.cardano-node = { impure = true; module = { - name = "cardano-node"; - nodes = { machine = {pkgs, ...}: { virtualisation = { @@ -24,7 +22,8 @@ testScript = '' machine.wait_for_unit("cardano-node") - machine.wait_until_succeeds("""[[ $(echo "$(cardano-cli query tip --testnet-magic 2 | jq '.syncProgress' --raw-output) > 0.01" | bc) == "1" ]]""") + machine.wait_until_succeeds("""[[ $(echo "$(cardano-cli query tip --testnet-magic 2 | jq '.syncProgress' --raw-output) > 0.001" | bc) == "1" ]]""") + print(machine.succeed("systemd-analyze security cardano-node")) ''; }; }; diff --git a/tests/default.nix b/tests/default.nix index b1e0354..57f6822 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -1,8 +1,7 @@ { - perSystem = _: { - imports = [ - ./cardano-cli.nix - ./cardano-node.nix - ]; - }; + imports = [ + ./cardano-cli.nix + ./cardano-node.nix + ./ogmios.nix + ]; } diff --git a/tests/ogmios.nix b/tests/ogmios.nix new file mode 100644 index 0000000..9ec6f5d --- /dev/null +++ b/tests/ogmios.nix @@ -0,0 +1,33 @@ +{ + perSystem.vmTests.tests.ogmios = { + impure = true; + module = { + nodes = { + machine = {pkgs, ...}: { + virtualisation = { + cores = 1; + memorySize = 1024; + }; + cardanoNix = { + globals.network = "preview"; + cardano-cli.enable = true; + cardano-node.enable = true; + ogmios.enable = true; + }; + + environment.systemPackages = with pkgs; [jq bc curl]; + }; + }; + + testScript = '' + machine.wait_for_unit("cardano-node") + machine.wait_for_unit("cardano-node-socket") + machine.wait_until_succeeds("""[[ $(echo "$(cardano-cli query tip --testnet-magic 2 | jq '.syncProgress' --raw-output) > 0.001" | bc) == "1" ]]""") + machine.wait_for_unit("ogmios") + machine.succeed("curl --fail http://localhost:1337") + machine.wait_until_succeeds(r"""journalctl --no-pager -r -n 1 -u ogmios.service -g networkSynchronization\":0\.[0-9][0-9][0-9][1-9]""") + print(machine.succeed("systemd-analyze security ogmios")) + ''; + }; + }; +} From b00ae152842872aabdcc3181d42d4de902997fde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Tue, 26 Mar 2024 16:53:19 +0000 Subject: [PATCH 04/11] changes requested in review --- flake.nix | 68 ++++++++----- modules/globals.nix | 2 +- modules/ogmios.nix | 2 +- modules/services/ogmios.nix | 192 ++++++++++++++++++------------------ packages/ogmios.nix | 1 + tests/cardano-node.nix | 6 +- tests/ogmios.nix | 6 +- 7 files changed, 151 insertions(+), 126 deletions(-) diff --git a/flake.nix b/flake.nix index 12a00f3..3e8305e 100644 --- a/flake.nix +++ b/flake.nix @@ -1,32 +1,52 @@ { inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + nixpkgs = { + url = "github:NixOS/nixpkgs/nixos-unstable"; + }; - # cardano-node - cardano-node.url = "github:intersectmbo/cardano-node?ref=8.7.3"; - cardano-configurations.url = "github:input-output-hk/cardano-configurations/21249e0d5c68b4e8f3661b250aa8272a8785d678"; - cardano-configurations.flake = false; + # Cardano-node + cardano-node = { + url = "github:intersectmbo/cardano-node?ref=8.7.3"; + }; + cardano-configurations = { + url = "github:input-output-hk/cardano-configurations/21249e0d5c68b4e8f3661b250aa8272a8785d678"; + flake = false; + }; # Utilities - attic.url = "github:zhaofengli/attic"; - devour-flake.url = "github:srid/devour-flake"; - devour-flake.flake = false; - devshell.url = "github:numtide/devshell"; - flake-parts.url = "github:hercules-ci/flake-parts"; - flake-root.url = "github:srid/flake-root"; - hercules-ci-effects.url = "github:mlabs-haskell/hercules-ci-effects/push-cache-effect"; - pre-commit-hooks-nix.url = "github:cachix/pre-commit-hooks.nix"; - treefmt-nix.url = "github:numtide/treefmt-nix"; - - # Prevent unnecessary downloads - attic.inputs.nixpkgs.follows = "nixpkgs"; - attic.inputs.nixpkgs-stable.follows = "nixpkgs"; - devshell.inputs.nixpkgs.follows = "nixpkgs"; - hercules-ci-effects.inputs.nixpkgs.follows = "nixpkgs"; - hercules-ci-effects.inputs.flake-parts.follows = "flake-parts"; - pre-commit-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; - pre-commit-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; - treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; + attic = { + url = "github:zhaofengli/attic"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.nixpkgs-stable.follows = "nixpkgs"; + }; + devour-flake = { + url = "github:srid/devour-flake"; + flake = false; + }; + devshell = { + url = "github:numtide/devshell"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + flake-parts = { + url = "github:hercules-ci/flake-parts"; + }; + flake-root = { + url = "github:srid/flake-root"; + }; + hercules-ci-effects = { + url = "github:mlabs-haskell/hercules-ci-effects/push-cache-effect"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.flake-parts.follows = "flake-parts"; + }; + pre-commit-hooks-nix = { + url = "github:cachix/pre-commit-hooks.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.nixpkgs-stable.follows = "nixpkgs"; + }; + treefmt-nix = { + url = "github:numtide/treefmt-nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = inputs @ {flake-parts, ...}: flake-parts.lib.mkFlake { diff --git a/modules/globals.nix b/modules/globals.nix index 11ae923..af2f6d0 100644 --- a/modules/globals.nix +++ b/modules/globals.nix @@ -26,7 +26,7 @@ in description = "Map from network names to network numbers. Selected network must be present in the map"; type = types.attrsOf types.int; default = { - mainnet = 0; + mainnet = 764824073; preprod = 1; preview = 2; sanchonet = 4; diff --git a/modules/ogmios.nix b/modules/ogmios.nix index abd7f17..407253c 100644 --- a/modules/ogmios.nix +++ b/modules/ogmios.nix @@ -6,7 +6,7 @@ cfg = config.cardanoNix.ogmios; in { options.cardanoNix.ogmios = { - enable = lib.mkEnableOption "cardano-node service"; + enable = lib.mkEnableOption "Ogmios bridge interface for cardano-node"; }; config = lib.mkIf cfg.enable { diff --git a/modules/services/ogmios.nix b/modules/services/ogmios.nix index a3fcccd..0da0040 100644 --- a/modules/services/ogmios.nix +++ b/modules/services/ogmios.nix @@ -4,115 +4,115 @@ lib, pkgs, ... -}: let +}: +with lib; let cfg = config.services.ogmios; -in - with lib; { - options.services.ogmios = with types; { - enable = mkEnableOption "Ogmios lightweight bridge interface for cardano-node"; +in { + options.services.ogmios = with types; { + enable = mkEnableOption "Ogmios bridge interface for cardano-node"; - package = mkOption { - description = "Ogmios package"; - type = package; - default = pkgs.ogmios; - }; + package = mkOption { + description = "Ogmios package"; + type = package; + default = pkgs.ogmios; + }; - user = mkOption { - description = "User to run Ogmios service as."; - type = str; - default = "ogmios"; - }; + user = mkOption { + description = "User to run Ogmios service as."; + type = str; + default = "ogmios"; + }; - group = mkOption { - description = "Group to run Ogmios service as."; - type = str; - default = "ogmios"; - }; + group = mkOption { + description = "Group to run Ogmios service as."; + type = str; + default = "ogmios"; + }; - nodeSocketPath = mkOption { - description = "Path to cardano-node IPC socket."; - type = path; - default = "/run/cardano-node/node.socket"; - }; + nodeSocketPath = mkOption { + description = "Path to cardano-node IPC socket."; + type = path; + default = "/run/cardano-node/node.socket"; + }; - nodeConfigPath = mkOption { - description = "Path to cardano-node config.json file. Required."; - type = path; - }; + nodeConfigPath = mkOption { + description = "Path to cardano-node config.json file. Required."; + type = path; + }; - host = mkOption { - description = "Host address or name to listen on."; - type = str; - default = "localhost"; - }; + host = mkOption { + description = "Host address or name to listen on."; + type = str; + default = "localhost"; + }; - port = mkOption { - description = "TCP port to listen on."; - type = port; - default = 1337; - }; + port = mkOption { + description = "TCP port to listen on."; + type = port; + default = 1337; + }; - extraArgs = mkOption { - description = "Extra arguments to ogmios command."; - type = listOf str; - default = []; - }; + extraArgs = mkOption { + description = "Extra arguments to ogmios command."; + type = listOf str; + default = []; }; + }; - config = mkIf cfg.enable { - users.users.ogmios = mkIf (cfg.user == "ogmios") { - isSystemUser = true; - inherit (cfg) group; - extraGroups = ["cardano-node"]; - }; - users.groups.ogmios = mkIf (cfg.group == "ogmios") {}; + config = mkIf cfg.enable { + users.users.ogmios = mkIf (cfg.user == "ogmios") { + isSystemUser = true; + inherit (cfg) group; + extraGroups = ["cardano-node"]; + }; + users.groups.ogmios = mkIf (cfg.group == "ogmios") {}; - systemd.services.ogmios = { - enable = true; - after = ["cardano-node.service"]; - wantedBy = ["multi-user.target"]; + systemd.services.ogmios = { + enable = true; + after = ["cardano-node.service"]; + wantedBy = ["multi-user.target"]; - script = escapeShellArgs (concatLists [ - ["${cfg.package}/bin/ogmios"] - ["--node-socket" cfg.nodeSocketPath] - ["--node-config" cfg.nodeConfigPath] - ["--host" cfg.host] - ["--port" cfg.port] - cfg.extraArgs - ]); + script = escapeShellArgs (concatLists [ + ["${cfg.package}/bin/ogmios"] + ["--node-socket" cfg.nodeSocketPath] + ["--node-config" cfg.nodeConfigPath] + ["--host" cfg.host] + ["--port" cfg.port] + cfg.extraArgs + ]); - serviceConfig = { - User = cfg.user; - Group = cfg.group; - # Security - UMask = "0077"; - CapabilityBoundingSet = ""; - ProcSubset = "pid"; - ProtectProc = "invisible"; - NoNewPrivileges = true; - DevicePolicy = "closed"; - ProtectSystem = "strict"; - ProtectHome = true; - PrivateTmp = true; - PrivateDevices = true; - PrivateUsers = true; - ProtectHostname = true; - ProtectClock = true; - ProtectKernelTunables = true; - ProtectKernelModules = true; - ProtectKernelLogs = true; - ProtectControlGroups = true; - RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"]; - RestrictNamespaces = true; - LockPersonality = true; - RestrictRealtime = true; - RestrictSUIDSGID = true; - RemoveIPC = true; - PrivateMounts = true; - SystemCallArchitectures = "native"; - SystemCallFilter = ["@system-service" "~@privileged"]; - MemoryDenyWriteExecute = true; - }; + serviceConfig = { + User = cfg.user; + Group = cfg.group; + # Security + UMask = "0077"; + CapabilityBoundingSet = ""; + ProcSubset = "pid"; + ProtectProc = "invisible"; + NoNewPrivileges = true; + DevicePolicy = "closed"; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + PrivateUsers = true; + ProtectHostname = true; + ProtectClock = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + RestrictAddressFamilies = ["AF_UNIX" "AF_INET" "AF_INET6"]; + RestrictNamespaces = true; + LockPersonality = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + RemoveIPC = true; + PrivateMounts = true; + SystemCallArchitectures = "native"; + SystemCallFilter = ["@system-service" "~@privileged"]; + MemoryDenyWriteExecute = true; }; }; - } + }; +} diff --git a/packages/ogmios.nix b/packages/ogmios.nix index f6a6b30..ec43165 100644 --- a/packages/ogmios.nix +++ b/packages/ogmios.nix @@ -4,6 +4,7 @@ pkgs, ... }: { + # Doesn't buid with haskell.nix but master contains a fix. Move to haskell.nix build on next release. packages.ogmios = pkgs.stdenv.mkDerivation rec { pname = "ogmios"; version = "6.1.0"; diff --git a/tests/cardano-node.nix b/tests/cardano-node.nix index 90a89cb..29cf7d1 100644 --- a/tests/cardano-node.nix +++ b/tests/cardano-node.nix @@ -20,9 +20,11 @@ }; }; - testScript = '' + testScript = {nodes, ...}: let + magic = toString nodes.machine.config.cardanoNix.globals.networkNumber; + in '' machine.wait_for_unit("cardano-node") - machine.wait_until_succeeds("""[[ $(echo "$(cardano-cli query tip --testnet-magic 2 | jq '.syncProgress' --raw-output) > 0.001" | bc) == "1" ]]""") + machine.wait_until_succeeds("""[[ $(echo "$(cardano-cli query tip --testnet-magic ${magic} | jq '.syncProgress' --raw-output) > 0.001" | bc) == "1" ]]""") print(machine.succeed("systemd-analyze security cardano-node")) ''; }; diff --git a/tests/ogmios.nix b/tests/ogmios.nix index 9ec6f5d..a641652 100644 --- a/tests/ogmios.nix +++ b/tests/ogmios.nix @@ -19,10 +19,12 @@ }; }; - testScript = '' + testScript = {nodes, ...}: let + magic = toString nodes.machine.config.cardanoNix.globals.networkNumber; + in '' machine.wait_for_unit("cardano-node") machine.wait_for_unit("cardano-node-socket") - machine.wait_until_succeeds("""[[ $(echo "$(cardano-cli query tip --testnet-magic 2 | jq '.syncProgress' --raw-output) > 0.001" | bc) == "1" ]]""") + machine.wait_until_succeeds("""[[ $(echo "$(cardano-cli query tip --testnet-magic ${magic} | jq '.syncProgress' --raw-output) > 0.001" | bc) == "1" ]]""") machine.wait_for_unit("ogmios") machine.succeed("curl --fail http://localhost:1337") machine.wait_until_succeeds(r"""journalctl --no-pager -r -n 1 -u ogmios.service -g networkSynchronization\":0\.[0-9][0-9][0-9][1-9]""") From 9c5edb91cbc3e3dd6e2df43d119238bed465074c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Tue, 26 Mar 2024 21:52:41 +0200 Subject: [PATCH 05/11] Update packages/cardano.nix Co-authored-by: Andrea Ciceri --- packages/cardano.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cardano.nix b/packages/cardano.nix index 9264955..667f0bd 100644 --- a/packages/cardano.nix +++ b/packages/cardano.nix @@ -3,11 +3,11 @@ lib, ... }: { - perSystem = {system, ...}: { + perSystem = {inputs', ...}: { packages = lib.filterAttrs (_: v: v != null) { - cardano-cli = inputs.cardano-node.packages.${system}.cardano-cli or null; - cardano-node = inputs.cardano-node.packages.${system}.cardano-node or null; - inherit (inputs) cardano-configurations; + cardano-cli = inputs'.cardano-node.packages.cardano-cli or null; + cardano-node = inputs'.cardano-node.packages.cardano-node or null; + inherit (inputs') cardano-configurations; }; }; } From 08c611c67508379df29e929d6ecfd54f8a445bdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Wed, 27 Mar 2024 19:51:21 +0200 Subject: [PATCH 06/11] Update README.md Co-authored-by: Andrea Ciceri --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 21780e4..732b848 100644 --- a/README.md +++ b/README.md @@ -55,9 +55,9 @@ Development is supported on linux systems. Virtual machines are run with `qemu` A `.envrc` is also provided, using [direnv]() and [nix-direnv](https://github.com/nix-community/nix-direnv) is suggested. -### Run Integration Test +### Running Integration Tests -From the devshell, run an integration test that starts `cardano-node` and `ogmios` on the `preview` testnet and checks for synchronization progress. +From the devshell you can run integration tests with `run-vm-test`, for example the following will start `cardano-node` and `ogmios` on the `preview` testnet and will check for synchronization progress. ``` run-vm-test ogmios From b285b2404c095911c7136dd2f5f3ae4ba75fa95e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Wed, 27 Mar 2024 17:59:11 +0000 Subject: [PATCH 07/11] fix typo --- packages/cardano.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cardano.nix b/packages/cardano.nix index 667f0bd..27746b3 100644 --- a/packages/cardano.nix +++ b/packages/cardano.nix @@ -7,7 +7,7 @@ packages = lib.filterAttrs (_: v: v != null) { cardano-cli = inputs'.cardano-node.packages.cardano-cli or null; cardano-node = inputs'.cardano-node.packages.cardano-node or null; - inherit (inputs') cardano-configurations; + inherit (inputs) cardano-configurations; }; }; } From ac380995442124e9e331ebc8caa6a30a383e0164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Wed, 27 Mar 2024 18:04:46 +0000 Subject: [PATCH 08/11] query ogmios health endpoint --- tests/ogmios.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ogmios.nix b/tests/ogmios.nix index a641652..c69b1a7 100644 --- a/tests/ogmios.nix +++ b/tests/ogmios.nix @@ -26,8 +26,8 @@ machine.wait_for_unit("cardano-node-socket") machine.wait_until_succeeds("""[[ $(echo "$(cardano-cli query tip --testnet-magic ${magic} | jq '.syncProgress' --raw-output) > 0.001" | bc) == "1" ]]""") machine.wait_for_unit("ogmios") - machine.succeed("curl --fail http://localhost:1337") - machine.wait_until_succeeds(r"""journalctl --no-pager -r -n 1 -u ogmios.service -g networkSynchronization\":0\.[0-9][0-9][0-9][1-9]""") + machine.succeed("curl --fail http://localhost:1337/health") + machine.wait_until_succeeds("""[[ $(echo "$(curl --fail http://localhost:1337/health | jq '.networkSynchronization' --raw-output) > 0.00001" | bc) == "1" ]]""") print(machine.succeed("systemd-analyze security ogmios")) ''; }; From 6e789cddfa4a8255c84cc425524d43449b6c707d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Wed, 27 Mar 2024 19:25:12 +0000 Subject: [PATCH 09/11] fix evaluation error --- packages/cardano.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cardano.nix b/packages/cardano.nix index 27746b3..9264955 100644 --- a/packages/cardano.nix +++ b/packages/cardano.nix @@ -3,10 +3,10 @@ lib, ... }: { - perSystem = {inputs', ...}: { + perSystem = {system, ...}: { packages = lib.filterAttrs (_: v: v != null) { - cardano-cli = inputs'.cardano-node.packages.cardano-cli or null; - cardano-node = inputs'.cardano-node.packages.cardano-node or null; + cardano-cli = inputs.cardano-node.packages.${system}.cardano-cli or null; + cardano-node = inputs.cardano-node.packages.${system}.cardano-node or null; inherit (inputs) cardano-configurations; }; }; From f37c5d285f1cfffe7cf77fd65ff0f62844066b62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Thu, 28 Mar 2024 18:07:52 +0000 Subject: [PATCH 10/11] cleanup --- docs/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/default.nix b/docs/default.nix index ca01d3e..926fcad 100644 --- a/docs/default.nix +++ b/docs/default.nix @@ -32,14 +32,14 @@ chmod +x $out/bin/mkdocs ''; - eachOptions = removeAttrs rootConfig.flake.nixosModules ["default" "cardano-overlay"]; + eachOptions = removeAttrs rootConfig.flake.nixosModules ["default"]; eachOptionsDoc = lib.mapAttrs' ( name: value: lib.nameValuePair - (builtins.head (lib.splitString "." name)) # take foo.options and turn it into just foo - + # take foo.options and turn it into just foo + (builtins.head (lib.splitString "." name)) (pkgs.nixosOptionsDoc { # By default `nixosOptionsDoc` will ignore internal options but we want to show them # This hack will make all the options not internal and visible and optionally append to the From 60c0368e4a0a4928f8083faa0060c84a9fcf62c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Thu, 28 Mar 2024 19:29:02 +0000 Subject: [PATCH 11/11] update to ogmios 6.2.0 --- packages/ogmios.nix | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/ogmios.nix b/packages/ogmios.nix index ec43165..44e77f4 100644 --- a/packages/ogmios.nix +++ b/packages/ogmios.nix @@ -7,16 +7,14 @@ # Doesn't buid with haskell.nix but master contains a fix. Move to haskell.nix build on next release. packages.ogmios = pkgs.stdenv.mkDerivation rec { pname = "ogmios"; - version = "6.1.0"; + version = "6.2.0"; src = pkgs.fetchurl { url = "https://github.com/CardanoSolutions/ogmios/releases/download/v${version}/ogmios-v${version}-${system}.zip"; hash = if system == "x86_64-linux" - then "sha256-JQJTaws+gihwHpTBtUqcNhgbhFDUAdiiXrO2kMX4ZkY=" + then "sha256-Ryfuzu7JzbR7ivh2Sl9xyOuWh4btjCVSPhXfDLmepHk=u" else if system == "aarch64-linux" - then "sha256-gWNTzUZHdp5rvbU0aIA3/GJ+YZeFx66h05ugJN4kMco=" - else if system == "aarch64-darwin" - then "sha256-gWNTzUZHdp5rvbU0aIA3/GJ+YZeFx66h05ugJN4kMco=" + then "sha256-SJQWbIkXF2e4jJtq2hQFqSPO/EhbmLYeZx3aEaXv/gI=" else abort "Ogmios release not available for system ${system}"; }; nativeBuildInputs = [pkgs.unzip];