diff --git a/README.md b/README.md index a87d1be..b31c582 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,13 @@ * If you don't already have Nix installed, please install it by following the directions at [Nix Installation](https://nixos.org/manual/nix/stable/#chap-installation). * A multi-user Nix installation is recommended over a single-user Nix installation as the multi-user install has advantages, such as being able to handle multiple builds in parallel. * To install Nix as a single-user install, the following command can be run as a regular non-root user with `sudo` access: -``` -$ sh <(curl -L https://nixos.org/nix/install) --no-daemon -``` + ``` + $ sh <(curl -L https://nixos.org/nix/install) --no-daemon + ``` * To install Nix as a multi-user install, the following command can be run as a regular non-root user with `sudo` access: -``` -$ sh <(curl -L https://nixos.org/nix/install) --daemon -``` + ``` + $ sh <(curl -L https://nixos.org/nix/install) --daemon + ``` * After performing a Nix single or multi-user install, any shells should be exited and re-entered to ensure the new Nix settings take effect in the environment. * Alternatively, the operating system [NixOS](https://nixos.org/manual/nixos/stable/#sec-installation) can be used which includes a Nix installation. @@ -30,78 +30,84 @@ $ sh <(curl -L https://nixos.org/nix/install) --daemon * After Nix is installed, Nix needs to have a few things customized to utilize new features and a project specific cache. * For a Nix single-user install, the Nix configuration file will be found at the following location. Create this diretory structure and file if it does not yet exist: -``` -$ mkdir -p ~/.config/nix -$ touch ~/.config/nix/nix.conf -``` + ``` + $ mkdir -p ~/.config/nix + $ touch ~/.config/nix/nix.conf + ``` * For a Nix multi-user install, the Nix configuration file will be found at the following location: -``` -$ ls -la /etc/nix/nix.conf -``` + ``` + $ ls -la /etc/nix/nix.conf + ``` * The following configuration lines need to be added to the Nix configuration: -``` -experimental-features = nix-command flakes ca-references -substituters = https://cache.nixos.org https://mantis-ops.cachix.org -trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= mantis-ops.cachix.org-1:SornDcX8/9rFrpTjU+mAAb26sF8mUpnxgXNjmKGcglQ= -``` + ``` + experimental-features = nix-command flakes ca-references + substituters = https://cache.nixos.org https://mantis-ops.cachix.org + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= mantis-ops.cachix.org-1:SornDcX8/9rFrpTjU+mAAb26sF8mUpnxgXNjmKGcglQ= + ``` * Additionally, for a Nix multi-user install: * The following line should also be added, where `` is substituted with your actual non-root username: -``` -trusted-users = -``` + ``` + trusted-users = + ``` * Once the new configuration lines have been added, the nix-daemon service needs to be restarted for the full Nix configuration changes to take effect: -``` -sudo systemctl restart nix-daemon.service -``` + * Linux (shouldn't be necessary on NixOS as it's supposed to be handled by activation): + ``` + sudo systemctl restart nix-daemon.service + ``` + * Darwin: + ``` + sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist + sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist + ``` * If lines for `experimental-features`, `substituters`, `trusted-public-keys` or `trusted-users` already exist in your Nix configuration file, then merge the lines above with the content that is already pre-existing in your Nix configuration. * For a NixOS installation, the following declarative code snippet in the machine NixOS configuration file, usually found at `/etc/nixos/configuration.nix`, followed by a `sudo nixos-rebuild switch` will add and activate the modified Nix configuration: -``` - nix.binaryCaches = [ - "https://cache.nixos.org/" - "https://mantis-ops.cachix.org" - ]; - nix.binaryCachePublicKeys = [ - "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" - "mantis-ops.cachix.org-1:SornDcX8/9rFrpTjU+mAAb26sF8mUpnxgXNjmKGcglQ=" - ]; - nix.extraOptions = '' - experimental-features = nix-command flakes ca-references - ''; - -``` + ``` + nix.binaryCaches = [ + "https://cache.nixos.org/" + "https://mantis-ops.cachix.org" + ]; + nix.binaryCachePublicKeys = [ + "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + "mantis-ops.cachix.org-1:SornDcX8/9rFrpTjU+mAAb26sF8mUpnxgXNjmKGcglQ=" + ]; + nix.extraOptions = '' + experimental-features = nix-command flakes ca-references + ''; + + ``` ### Nix Shell * Once Nix is installed and configured, clone the mantis-ops repo to your local machine and enter the local repository directory: -``` -# For cloning using a github registered key: -$ git clone git@github.com:input-output-hk/mantis-ops - -# or, for cloning without a github registered key: -$ git clone https://github.com/input-output-hk/mantis-ops - -$ cd mantis-ops -``` + ``` + # For cloning using a github registered key: + $ git clone git@github.com:input-output-hk/mantis-ops + + # or, for cloning without a github registered key: + $ git clone https://github.com/input-output-hk/mantis-ops + + $ cd mantis-ops + ``` * After entering the repository directory, the following command needs to be run ONLY ONCE. This will set up all the required dependencies for the nix-shell environment, including Nix flakes support and install them into the nix user profile: -``` -nix-shell --run ' -( nix build .#nixFlakes -o devShell - nix-env -i ./devShell - rm devShell -) || nix profile install github:input-output-hk/mantis-ops#nixFlakes -' -``` + ``` + nix-shell --run ' + ( nix build .#nixFlakes -o devShell + nix-env -i ./devShell + rm devShell + ) || nix profile install github:input-output-hk/mantis-ops#nixFlakes + ' + ``` * Finally, enter the development environment with the following command. This command should be executed whenever you have a new shell and you'd like to enter the development enviroment: -``` -nix develop -``` + ``` + nix develop + ``` ### Github Access Token @@ -120,42 +126,42 @@ nix develop ### Vault Authentication * From your nix development environment, obtain a vault token by supplying the following command with your github mantis-ops personal access token when prompted: -``` -$ vault login -method github -path github-employees -``` + ``` + $ vault login -method github -path github-employees + ``` * After logging into vault, if you need to see your vault token again or review information associated with your token, you can view it with the following command: -``` -$ vault token lookup -``` + ``` + $ vault token lookup + ``` * To see only your vault token, without additional information, use the following command: -``` -$ vault print token -``` + ``` + $ vault print token + ``` ### Nomad Authentication * After logging into vault, you can obtain a nomad token for the developer role with the following command: -``` -$ vault read -field secret_id nomad/creds/developer -``` + ``` + $ vault read -field secret_id nomad/creds/developer + ``` * Optionally, you can export this token locally to have access to the nomad cli which will enable additional cli debugging capabilities. For example: -``` -# Export the nomad developer token: -$ export NOMAD_TOKEN="$(vault read -field secret_id nomad/creds/developer)" - -# Now the nomad cli becomes available. -# The following are some example commands that may be useful: -$ nomad status -$ nomad status mantis -$ nomad alloc logs $ALLOC_ID > mantis-$ALLOC_ID.log -$ nomad job stop mantis - -# etc. -``` + ``` + # Export the nomad developer token: + $ export NOMAD_TOKEN="$(vault read -field secret_id nomad/creds/developer)" + + # Now the nomad cli becomes available. + # The following are some example commands that may be useful: + $ nomad status + $ nomad status testnet-mantis + $ nomad alloc logs $ALLOC_ID > mantis-$ALLOC_ID.log + $ nomad job stop mantis + + # etc. + ``` * The nomad token is also used to authenticate to the mantis-ops Nomad web UI at: https://nomad.mantis.ws/ * In the upper right hand corner of the Nomad web UI, click "ACL Tokens". @@ -166,9 +172,9 @@ $ nomad job stop mantis ### Consul Authentication * Optionally, a Consul token can be exported in order to use Consul templates, described below: -``` -export CONSUL_HTTP_TOKEN="$(vault read -field token consul/creds/developer)" -``` + ``` + export CONSUL_HTTP_TOKEN="$(vault read -field token consul/creds/developer)" + ``` ### Mantis Ops Web UI Resources @@ -193,28 +199,28 @@ export CONSUL_HTTP_TOKEN="$(vault read -field token consul/creds/developer)" * Querying logs by job identifier can be done through the "Explore" icon in the left vertical panel. * If you don't see this "Explore" icon which looks like a compass, request "Editor" access from DevOps. * Examples of log queries to the `Loki` log datasource in Grafana are: -``` -# In the "Log labels" field enter the following to search for all logs related to the `mantis-1` taskgroup: -{syslog_identifier="mantis-1"} - -# In the "Log labels" field enter the following to search for all logs related to the `mantis-1` taskgroup -# and filter for DAG events: -{syslog_identifier="mantis-1"} |~ "DAG" -``` + ``` + # In the "Log labels" field enter the following to search for all logs related to the `mantis-1` taskgroup: + {syslog_identifier="mantis-1"} + + # In the "Log labels" field enter the following to search for all logs related to the `mantis-1` taskgroup + # and filter for DAG events: + {syslog_identifier="mantis-1"} |~ "DAG" + ``` ### Updating the Mantis source used for Deployments * Near the top of the `./overlay.nix` file of the mantis-ops repository, the mantis commit ref is seen, where $COMMIT represents the actual commit revision and $BRANCH represents the branch of the commit, typically `develop`: -``` - mantis-source = builtins.fetchGit { - url = "https://github.com/input-output-hk/mantis"; - rev = "$COMMIT"; - ref = "$BRANCH"; - submodules = true; - }; - -``` + ``` + mantis-source = builtins.fetchGit { + url = "https://github.com/input-output-hk/mantis"; + rev = "$COMMIT"; + ref = "$BRANCH"; + submodules = true; + }; + + ``` * To update the commit that a mantis job will be run with, update the `rev` and `ref` fields with the appropriate git commit revision and git commit branch. * It is a good idea to commit mantis-source updates since a github action will automatically push the build product to cachix for faster job deployments for everyone on the team. @@ -224,9 +230,9 @@ export CONSUL_HTTP_TOKEN="$(vault read -field token consul/creds/developer)" * Presently, there is no requirement to commit changes from a mantis job definition to the repository in order to deploy the job. * To minimize confusion in the team about what job definition is running on the testnet, any changes to the mantis job made and deployed should be committed. * To run a mantis job by deploying it to the testnet, execute the following command: -``` -$ nix run .#nomadJobs.mantis.run -``` + ``` + $ nix run .#nomadJobs.mantis.run + ``` * Versioning information about the deployment, including changes from the last version deployed, can be viewed in the Nomad UI in the [Versions](https://nomad.mantis.ws/ui/jobs/mantis/versions) section. @@ -265,9 +271,9 @@ $ nix run .#nomadJobs.mantis.run * Additional bootstrap miner state only needs to be generated if the total number to be scaled to exceeds the number which currently exists (6 at the time of writing). * Pre-existing bootstrap node state can be viewed at the [testnet Vault kv](https://vault.mantis.ws/ui/vault/secrets/kv/list/nomad-cluster/testnet/) path. * This state pre-generation is done with the following command: -``` -nix run .#generate-mantis-keys $TOTAL_NUM_BOOTSTRAP_NODES -``` + ``` + nix run .#generate-mantis-keys $TOTAL_NUM_BOOTSTRAP_NODES + ``` ### Scaling Infrastructure Requirements @@ -285,18 +291,18 @@ nix run .#generate-mantis-keys $TOTAL_NUM_BOOTSTRAP_NODES * For debugging purposes, it can be helpful to generate a real time map of running taskgroups and the associated public IP and node hostname (containing internal IP). * To do this we can use a Consul template: -``` -$ consul-template -template templates/map.tmpl -once -dry -log-level err -``` + ``` + $ consul-template -template templates/map.tmpl -once -dry -log-level err + ``` ### Starting a Mantis Local Node Connected to Testnet * A local mantis node can be build and run connected to the testnet: -``` -# Build the mantis executables and configuration files -$ nix build .#mantis -o mantis-node - -# Run a local mantis node against the testnet bootstrap cluster -$ consul-template -template templates/mantis.tmpl:mantis-local.conf -exec './mantis-node/bin/mantis -Dconfig.file=./mantis-local.conf' -``` + ``` + # Build the mantis executables and configuration files + $ nix build .#mantis -o mantis-node + + # Run a local mantis node against the testnet bootstrap cluster + $ consul-template -template templates/mantis.tmpl:mantis-local.conf -exec './mantis-node/bin/mantis -Dconfig.file=./mantis-local.conf' + ``` diff --git a/flake.lock b/flake.lock index b529829..fb85506 100644 --- a/flake.lock +++ b/flake.lock @@ -3,24 +3,28 @@ "bitte": { "inputs": { "bitte-cli": "bitte-cli", + "crystal": [ + "bitte", + "bitte-cli", + "crystal" + ], "inclusive": "inclusive_2", - "nixpkgs": "nixpkgs_2", - "nixpkgs-crystal": [ + "nixpkgs": [ "bitte", "bitte-cli", - "nixpkgs-crystal" + "nixpkgs" ], "nixpkgs-terraform": "nixpkgs-terraform", "ops-lib": "ops-lib", "terranix": "terranix", - "utils": "utils_2" + "utils": "utils_3" }, "locked": { - "lastModified": 1603201032, - "narHash": "sha256-EnFv23GRPM3a75S3tqHF7RApLUuL0zmT67RlrvBEsQ0=", + "lastModified": 1603269871, + "narHash": "sha256-ep1k9BU5Ty3ON+C4QnWLsCPebL7bwP5Yrtu4+MIa0xU=", "owner": "input-output-hk", "repo": "bitte", - "rev": "1c1a0e7cbe3d0cc02d2b0b7cfa7a1a191d1c7f74", + "rev": "081e8cd5e5feeca3a819f86443316f97c01c5eea", "type": "github" }, "original": { @@ -31,17 +35,17 @@ }, "bitte-cli": { "inputs": { + "crystal": "crystal", "inclusive": "inclusive", - "nixpkgs": "nixpkgs", - "nixpkgs-crystal": "nixpkgs-crystal", - "utils": "utils" + "nixpkgs": "nixpkgs_2", + "utils": "utils_2" }, "locked": { - "lastModified": 1602489451, - "narHash": "sha256-8Ikcv3qc8/4KGGS2JGuJgox1B6JQJL4tZeJvaAsKB1E=", + "lastModified": 1603186486, + "narHash": "sha256-AbmNtD5/lECuj/tKGY2y8mvoKKSqPY43foAAdh/YtOE=", "owner": "input-output-hk", "repo": "bitte-cli", - "rev": "0c6945eea09c724c5e228f2d0660bbacc597bc2f", + "rev": "5eaa1c6bdf1bf0a3cc5e63d89efee8588977b203", "type": "github" }, "original": { @@ -50,6 +54,25 @@ "type": "github" } }, + "crystal": { + "inputs": { + "nixpkgs": "nixpkgs", + "utils": "utils" + }, + "locked": { + "lastModified": 1602755823, + "narHash": "sha256-6MYxvT5e1xT4Kb1TKzk4AO3EefQTJ+juj38KUHV+NPI=", + "owner": "kreisys", + "repo": "crystal", + "rev": "6718e4f851347ce34d8fbabf516c6a0da85966f0", + "type": "github" + }, + "original": { + "owner": "kreisys", + "repo": "crystal", + "type": "github" + } + }, "inclusive": { "inputs": { "stdlib": "stdlib" @@ -73,11 +96,11 @@ "stdlib": "stdlib_2" }, "locked": { - "lastModified": 1591376980, - "narHash": "sha256-pjSyg7JX6fJ/QvVhNhlo794YFW2E4wGVQV+YfuZq5Dk=", + "lastModified": 1592950149, + "narHash": "sha256-tZSz9/WkOrtNbW7U2re4WoFPQ2tQX0I0GpCeoF3fY1M=", "owner": "manveru", "repo": "nix-inclusive", - "rev": "bb435b7dce2b8a27d174543f0d768646d0d48fa3", + "rev": "dd80ddcc6850d130b591fbc9b0aacacfbb3cf47c", "type": "github" }, "original": { @@ -104,34 +127,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1595425804, - "narHash": "sha256-RM8fHq1PoN8HHVRPKIjYLrXq54Z18iVodQrwxmH/J3g=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "b8c367a7bd05e3a514c2b057c09223c74804a21b", - "type": "github" - }, - "original": { + "lastModified": 1602739928, + "narHash": "sha256-mpfoeHf9QaMPNurSYeSkxerDfs6Fj7HDElSMXDaTOPQ=", "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-crystal": { - "locked": { - "lastModified": 1591785879, - "narHash": "sha256-RfTjvYLCdXw3dyOwbeDU4pDK9ex+QxrRQtMxqesWQOE=", - "owner": "manveru", "repo": "nixpkgs", - "rev": "62267e49f4e13ed34d1ed328b3b5ff54be5efd18", + "rev": "2a4607f44222a92b8a44e6e1dac715e7eca04239", "type": "github" }, "original": { - "owner": "manveru", - "ref": "crystal-0.35", - "repo": "nixpkgs", - "type": "github" + "id": "nixpkgs", + "type": "indirect" } }, "nixpkgs-terraform": { @@ -152,11 +157,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1595353179, - "narHash": "sha256-V6keEpEx23NrdztG7WGiwIiQTobRBfIvLTNo86x5YvI=", + "lastModified": 1595425804, + "narHash": "sha256-RM8fHq1PoN8HHVRPKIjYLrXq54Z18iVodQrwxmH/J3g=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "225628ff02cb4391db583cd6a82fb3f9026aa9ac", + "rev": "b8c367a7bd05e3a514c2b057c09223c74804a21b", "type": "github" }, "original": { @@ -169,11 +174,11 @@ "ops-lib": { "flake": false, "locked": { - "lastModified": 1591802626, - "narHash": "sha256-p2+PTgA+1LmJ6sTJiCRMXDUl9kkJ8pVTjnsSqP9C+XA=", + "lastModified": 1602203038, + "narHash": "sha256-SIKLst2fs7Rool+WPJTxUdOHxdr9HZt9rgGe8Y+/+uE=", "owner": "input-output-hk", "repo": "ops-lib", - "rev": "b0a2ff6bef37109465b0ec9a8dba1f8923059491", + "rev": "1f8481bdc21cea9430b830c3fd1eeb057ad5c8ee", "type": "github" }, "original": { @@ -217,7 +222,7 @@ "bitte", "terranix" ], - "utils": "utils_3" + "utils": "utils_4" } }, "stdlib": { @@ -269,11 +274,11 @@ }, "utils": { "locked": { - "lastModified": 1590791977, - "narHash": "sha256-l26u9vRsdV9jkoBGlzFxTKLKtrZ38xCMZIG9h9ibFjQ=", + "lastModified": 1601282935, + "narHash": "sha256-WQAFV6sGGQxrRs3a+/Yj9xUYvhTpukQJIcMbIi7LCJ4=", "owner": "numtide", "repo": "flake-utils", - "rev": "4fbce19a7a9647154a35f0e856d52171c41e14a4", + "rev": "588973065fce51f4763287f0fda87a174d78bf48", "type": "github" }, "original": { @@ -299,11 +304,26 @@ }, "utils_3": { "locked": { - "lastModified": 1600209923, - "narHash": "sha256-zoOWauTliFEjI++esk6Jzk7QO5EKpddWXQm9yQK24iM=", + "lastModified": 1601282935, + "narHash": "sha256-WQAFV6sGGQxrRs3a+/Yj9xUYvhTpukQJIcMbIi7LCJ4=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "588973065fce51f4763287f0fda87a174d78bf48", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "utils_4": { + "locked": { + "lastModified": 1601282935, + "narHash": "sha256-WQAFV6sGGQxrRs3a+/Yj9xUYvhTpukQJIcMbIi7LCJ4=", "owner": "numtide", "repo": "flake-utils", - "rev": "3cd06d3c1df6879c9e41cb2c33113df10566c760", + "rev": "588973065fce51f4763287f0fda87a174d78bf48", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 12b99a1..e707935 100644 --- a/flake.nix +++ b/flake.nix @@ -17,7 +17,7 @@ }; outputs = { self, nixpkgs, utils, ops-lib, bitte, ... }: - (utils.lib.eachSystem [ "x86_64-linux" ] (system: rec { + (utils.lib.eachSystem [ "x86_64-linux" "x86_64-darwin" ] (system: rec { overlay = import ./overlay.nix { inherit system self; }; legacyPackages = import nixpkgs { @@ -32,11 +32,16 @@ inherit (legacyPackages) devShell; packages = { - inherit (legacyPackages) bitte nixFlakes sops generate-mantis-keys; - inherit (self.inputs.bitte.packages.${system}) + inherit (legacyPackages) bitte nixFlakes sops generate-mantis-keys terraform-with-plugins cfssl consul; }; + hydraJobs = packages // { + prebuilt-devshell = devShell.overrideAttrs (_: { + nobuildPhase = "touch $out"; + }); + }; + apps.bitte = utils.lib.mkApp { drv = legacyPackages.bitte; }; })) // (let pkgs = import nixpkgs { diff --git a/overlay.nix b/overlay.nix index 7f44ffe..72de385 100644 --- a/overlay.nix +++ b/overlay.nix @@ -232,19 +232,6 @@ in { name = "devShell"; }; - # inject vault-bin into bitte wrapper - bitte = let - bitte-nixpkgs = import self.inputs.nixpkgs { - inherit system; - overlays = [ - (final: prev: { - vault-bin = self.inputs.bitte.legacyPackages.${system}.vault-bin; - }) - self.inputs.bitte-cli.overlay.${system} - ]; - }; - in bitte-nixpkgs.bitte; - mantis-explorer = final.callPackage ./pkgs/mantis-explorer.nix { src = self.inputs.mantis-explorer; }; @@ -259,8 +246,8 @@ in { }; inherit (self.inputs.bitte.legacyPackages.${system}) - vault-bin mkNomadJob terraform-with-plugins systemdSandbox nixFlakes nomad - consul consul-template bitte-tokens; + bitte vault-bin mkNomadJob terraform-with-plugins + systemdSandbox nixFlakes nomad consul consul-template bitte-tokens; nomadJobs = let jobsDir = ./jobs;