Embed the Nix to-be-installed in the binary (#897)

* Embed the Nix tarball into the binary This improves built-in offline support, and improves install time reliability. * Stop linking to nix-upgrade * Rename the NIX_INSTALLER_TARBALL environment variable to NIX_INSTALLER_TARBALL_PATH * Document the NIX_INSTALLER_TARBALL_PATH requirement * Link to our current Nix recommendation * Note why we set the upgrade url. * make the x86 builds run on x86 * Indicate the origins of the tarball * derp * fixup: release-prs, release-tags * empty --------- Co-authored-by: Cole Helbling <cole.helbling@determinate.systems>
DeterminateSystems · May 2, 2024 · 3ec6c32 · 3ec6c32
1 parent 1b6195b
commit 3ec6c32
Show file tree

Hide file tree

Showing 9 changed files with 83 additions and 80 deletions.
diff --git a/.github/workflows/build-x86_64-darwin.yml b/.github/workflows/build-x86_64-darwin.yml
@@ -1,6 +1,6 @@
 name: Build x86_64 Darwin
 
-on: 
+on:
   workflow_call:
     inputs:
       cache-key:
@@ -11,7 +11,7 @@ on:
 jobs:
   build-x86_64-darwin:
     name: Build x86_64 Darwin
-    runs-on: macos-latest-large
+    runs-on: macos-13-large
     concurrency: ${{ inputs.cache-key }}
     permissions:
       id-token: "write"

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -269,7 +269,7 @@ jobs:
 
   run-x86_64-darwin:
     name: Run x86_64 Darwin
-    runs-on: macos-latest
+    runs-on: macos-13
     needs: [lints, build-x86_64-darwin]
     permissions:
       id-token: "write"

diff --git a/.github/workflows/release-prs.yml b/.github/workflows/release-prs.yml
@@ -11,6 +11,10 @@ on:
       - synchronize
       - labeled
 
+permissions:
+  id-token: "write"
+  contents: "read"
+
 jobs:
   build-x86_64-linux:
     # Only intra-repo PRs are allowed to have PR artifacts uploaded
@@ -89,8 +93,6 @@ jobs:
         || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'upload to s3'))
       )
     runs-on: ubuntu-latest
-    permissions:
-      id-token: write # In order to request a JWT for AWS auth
     needs:
       - build-x86_64-linux
       - build-i686-linux

diff --git a/.github/workflows/release-tags.yml b/.github/workflows/release-tags.yml
@@ -8,6 +8,10 @@ on:
     tags:
       - "v*.*.*"
 
+permissions:
+  contents: write # In order to upload artifacts to GitHub releases
+  id-token: write # In order to request a JWT for AWS auth
+
 jobs:
   build-x86_64-linux:
     uses: ./.github/workflows/build-x86_64-linux.yml
@@ -32,9 +36,6 @@ jobs:
 
   release:
     runs-on: ubuntu-latest
-    permissions:
-      contents: write # In order to upload artifacts to GitHub releases
-      id-token: write # In order to request a JWT for AWS auth
     needs:
       - build-x86_64-linux
       - build-i686-linux

diff --git a/README.md b/README.md
@@ -97,7 +97,7 @@ $ NIX_BUILD_GROUP_NAME=nixbuilder ./nix-installer install linux-multi --nix-buil
 
 ### Upgrading Nix
 
-You can upgrade Nix to the version described in [`DeterminateSystems/nix-upgrade`][nix-upgrade] by running:
+You can upgrade Nix to [our currently recommended version of Nix][recommended-nix] by running:
 
 ```
 sudo -i nix upgrade-nix
@@ -389,6 +389,9 @@ You'll also need to edit your `.cargo/config.toml` to use `tokio_unstable` as we
 rustflags=["--cfg", "tokio_unstable"]
 ```
 
+You'll also need to set the `NIX_INSTALLER_TARBALL_PATH` environment variable to point to a target-appropriate Nix installation tarball, like nix-2.21.2-aarch64-darwin.tar.xz.
+The contents are embedded in the resulting binary instead of downloaded at installation time.
+
 Then it's possible to review the [documentation](https://docs.rs/nix-installer/latest/nix_installer/):
 
 ```bash
@@ -440,7 +443,7 @@ Differing from the upstream [Nix](https://github.com/NixOS/nix) installer script
   + `auto-optimise-store` is set to `true` (On Linux only)
   * `extra-nix-path` is set to `nixpkgs=flake:nixpkgs`
   * `max-jobs` is set to `auto`
-  * `upgrade-nix-store-path-url` is set to `https://install.determinate.systems/nix-upgrade/stable/universal`
+  * `upgrade-nix-store-path-url` is set to `https://install.determinate.systems/nix-upgrade/stable/universal`, to prevent unintentional downgrades.
 * an installation receipt (for uninstalling) is stored at `/nix/receipt.json` as well as a copy of the install binary at `/nix/nix-installer`
 * `nix-channel --update` is not run, `~/.nix-channels` is not provisioned
 * `ssl-cert-file` is set in `/etc/nix/nix.conf` if the `ssl-cert-file` argument is used.
@@ -495,8 +498,8 @@ You can read the full privacy policy for [Determinate Systems][detsys], the crea
 [detsys]: https://determinate.systems/
 [diagnosticdata]: https://github.com/DeterminateSystems/nix-installer/blob/f9f927840d532b71f41670382a30cfcbea2d8a35/src/diagnostics.rs#L29-L43
 [privacy]: https://determinate.systems/policies/privacy
+[recommended-nix]: https://github.com/DeterminateSystems/nix/releases/latest
 [systemd]: https://systemd.io
 [wslg]: https://github.com/microsoft/wslg
 [nixgl]: https://github.com/guibou/nixGL
 [Nix]: https://nixos.org
-[nix-upgrade]: https://github.com/DeterminateSystems/nix-upgrade/blob/main/versions.nix
diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -15,7 +15,7 @@
     };
 
     nix = {
-      url = "https://flakehub.com/f/NixOS/nix/=2.21.2.tar.gz";
+      url = "https://flakehub.com/f/DeterminateSystems/nix/=2.21.2.tar.gz";
       # Omitting `inputs.nixpkgs.follows = "nixpkgs";` on purpose
     };
 
@@ -87,6 +87,8 @@
             RUSTFLAGS = "--cfg tokio_unstable";
             cargoTestOptions = f: f ++ [ "--all" ];
 
+            NIX_INSTALLER_TARBALL_PATH = inputs.nix.tarballs_direct.${final.stdenv.system};
+
             override = { preBuild ? "", ... }: {
               preBuild = preBuild + ''
                 # logRun "cargo clippy --all-targets --all-features -- -D warnings"
@@ -130,6 +132,7 @@
             name = "nix-install-shell";
 
             RUST_SRC_PATH = "${toolchain}/lib/rustlib/src/rust/library";
+            NIX_INSTALLER_TARBALL_PATH = inputs.nix.tarballs_direct.${system};
 
             nativeBuildInputs = with pkgs; [ ];
             buildInputs = with pkgs; [

diff --git a/src/action/base/fetch_and_unpack_nix.rs b/src/action/base/fetch_and_unpack_nix.rs
@@ -15,7 +15,7 @@ Fetch a URL to the given path
 */
 #[derive(Debug, serde::Deserialize, serde::Serialize, Clone)]
 pub struct FetchAndUnpackNix {
-    url_or_path: UrlOrPath,
+    url_or_path: Option<UrlOrPath>,
     dest: PathBuf,
     proxy: Option<Url>,
     ssl_cert_file: Option<PathBuf>,
@@ -24,15 +24,15 @@ pub struct FetchAndUnpackNix {
 impl FetchAndUnpackNix {
     #[tracing::instrument(level = "debug", skip_all)]
     pub async fn plan(
-        url_or_path: UrlOrPath,
+        url_or_path: Option<UrlOrPath>,
         dest: PathBuf,
         proxy: Option<Url>,
         ssl_cert_file: Option<PathBuf>,
     ) -> Result<StatefulAction<Self>, ActionError> {
         // TODO(@hoverbear): Check URL exists?
         // TODO(@hoverbear): Check tempdir exists
 
-        if let UrlOrPath::Url(url) = &url_or_path {
+        if let Some(UrlOrPath::Url(url)) = &url_or_path {
             match url.scheme() {
                 "https" | "http" | "file" => (),
                 _ => return Err(Self::error(ActionErrorKind::UnknownUrlScheme)),
@@ -67,14 +67,21 @@ impl Action for FetchAndUnpackNix {
         ActionTag("fetch_and_unpack_nix")
     }
     fn tracing_synopsis(&self) -> String {
-        format!("Fetch `{}` to `{}`", self.url_or_path, self.dest.display())
+        if let Some(ref url_or_path) = self.url_or_path {
+            format!("Fetch `{}` to `{}`", url_or_path, self.dest.display())
+        } else {
+            format!(
+                "Extract the bundled Nix (originally from {})",
+                crate::settings::NIX_TARBALL_PATH
+            )
+        }
     }
 
     fn tracing_span(&self) -> Span {
         let span = span!(
             tracing::Level::DEBUG,
             "fetch_and_unpack_nix",
-            url_or_path = tracing::field::display(&self.url_or_path),
+            url_or_path = self.url_or_path.as_ref().map(tracing::field::display),
             proxy = tracing::field::Empty,
             ssl_cert_file = tracing::field::Empty,
             dest = tracing::field::display(self.dest.display()),
@@ -98,7 +105,8 @@ impl Action for FetchAndUnpackNix {
     #[tracing::instrument(level = "debug", skip_all)]
     async fn execute(&mut self) -> Result<(), ActionError> {
         let bytes = match &self.url_or_path {
-            UrlOrPath::Url(url) => {
+            &None => Bytes::from(crate::settings::NIX_TARBALL),
+            Some(UrlOrPath::Url(url)) => {
                 let bytes = match url.scheme() {
                     "https" | "http" => {
                         let mut buildable_client = reqwest::Client::builder();
@@ -144,7 +152,7 @@ impl Action for FetchAndUnpackNix {
                 };
                 bytes
             },
-            UrlOrPath::Path(path) => {
+            Some(UrlOrPath::Path(path)) => {
                 let buf = tokio::fs::read(path)
                     .await
                     .map_err(|e| ActionErrorKind::Read(PathBuf::from(path), e))

diff --git a/src/settings.rs b/src/settings.rs
@@ -11,21 +11,11 @@ use url::Url;
 
 pub const SCRATCH_DIR: &str = "/nix/temp-install-dir";
 
-/// Default [`nix_package_url`](CommonSettings::nix_package_url) for Linux x86_64
-pub const NIX_X64_64_LINUX_URL: &str =
-    "https://releases.nixos.org/nix/nix-2.21.2/nix-2.21.2-x86_64-linux.tar.xz";
-/// Default [`nix_package_url`](CommonSettings::nix_package_url) for Linux x86 (32 bit)
-pub const NIX_I686_LINUX_URL: &str =
-    "https://releases.nixos.org/nix/nix-2.21.2/nix-2.21.2-i686-linux.tar.xz";
-/// Default [`nix_package_url`](CommonSettings::nix_package_url) for Linux aarch64
-pub const NIX_AARCH64_LINUX_URL: &str =
-    "https://releases.nixos.org/nix/nix-2.21.2/nix-2.21.2-aarch64-linux.tar.xz";
-/// Default [`nix_package_url`](CommonSettings::nix_package_url) for Darwin x86_64
-pub const NIX_X64_64_DARWIN_URL: &str =
-    "https://releases.nixos.org/nix/nix-2.21.2/nix-2.21.2-x86_64-darwin.tar.xz";
-/// Default [`nix_package_url`](CommonSettings::nix_package_url) for Darwin aarch64
-pub const NIX_AARCH64_DARWIN_URL: &str =
-    "https://releases.nixos.org/nix/nix-2.21.2/nix-2.21.2-aarch64-darwin.tar.xz";
+pub const NIX_TARBALL_PATH: &str = env!("NIX_INSTALLER_TARBALL_PATH");
+/// The NIX_INSTALLER_TARBALL_PATH environment variable should point to a target-appropriate
+/// Nix installation tarball, like nix-2.21.2-aarch64-darwin.tar.xz. The contents are embedded
+/// in the resulting binary.
+pub const NIX_TARBALL: &[u8] = include_bytes!(env!("NIX_INSTALLER_TARBALL_PATH"));
 
 #[derive(Debug, serde::Deserialize, serde::Serialize, Clone, Copy, PartialEq, Eq)]
 #[cfg_attr(feature = "cli", derive(clap::ValueEnum))]
@@ -145,39 +135,9 @@ pub struct CommonSettings {
     /// The Nix package URL
     #[cfg_attr(
         feature = "cli",
-        clap(long, env = "NIX_INSTALLER_NIX_PACKAGE_URL", global = true, value_parser = clap::value_parser!(UrlOrPath))
+        clap(long, env = "NIX_INSTALLER_NIX_PACKAGE_URL", global = true, value_parser = clap::value_parser!(UrlOrPath), default_value = None)
     )]
-    #[cfg_attr(
-        all(target_os = "macos", target_arch = "x86_64", feature = "cli"),
-        clap(
-            default_value = NIX_X64_64_DARWIN_URL,
-        )
-    )]
-    #[cfg_attr(
-        all(target_os = "macos", target_arch = "aarch64", feature = "cli"),
-        clap(
-            default_value = NIX_AARCH64_DARWIN_URL,
-        )
-    )]
-    #[cfg_attr(
-        all(target_os = "linux", target_arch = "x86_64", feature = "cli"),
-        clap(
-            default_value = NIX_X64_64_LINUX_URL,
-        )
-    )]
-    #[cfg_attr(
-        all(target_os = "linux", target_arch = "x86", feature = "cli"),
-        clap(
-            default_value = NIX_I686_LINUX_URL,
-        )
-    )]
-    #[cfg_attr(
-        all(target_os = "linux", target_arch = "aarch64", feature = "cli"),
-        clap(
-            default_value = NIX_AARCH64_LINUX_URL,
-        )
-    )]
-    pub nix_package_url: UrlOrPath,
+    pub nix_package_url: Option<UrlOrPath>,
 
     /// The proxy to use (if any), valid proxy bases are `https://$URL`, `http://$URL` and `socks5://$URL`
     #[cfg_attr(feature = "cli", clap(long, env = "NIX_INSTALLER_PROXY"))]
@@ -250,7 +210,6 @@ pub struct CommonSettings {
 impl CommonSettings {
     /// The default settings for the given Architecture & Operating System
     pub async fn default() -> Result<Self, InstallSettingsError> {
-        let url;
         let nix_build_user_prefix;
         let nix_build_user_id_base;
         let nix_build_user_count;
@@ -259,37 +218,32 @@ impl CommonSettings {
         match (Architecture::host(), OperatingSystem::host()) {
             #[cfg(target_os = "linux")]
             (Architecture::X86_64, OperatingSystem::Linux) => {
-                url = NIX_X64_64_LINUX_URL;
                 nix_build_user_prefix = "nixbld";
                 nix_build_user_id_base = 30000;
                 nix_build_user_count = 32;
             },
             #[cfg(target_os = "linux")]
             (Architecture::X86_32(_), OperatingSystem::Linux) => {
-                url = NIX_I686_LINUX_URL;
                 nix_build_user_prefix = "nixbld";
                 nix_build_user_id_base = 30000;
                 nix_build_user_count = 32;
             },
             #[cfg(target_os = "linux")]
             (Architecture::Aarch64(_), OperatingSystem::Linux) => {
-                url = NIX_AARCH64_LINUX_URL;
                 nix_build_user_prefix = "nixbld";
                 nix_build_user_id_base = 30000;
                 nix_build_user_count = 32;
             },
             #[cfg(target_os = "macos")]
             (Architecture::X86_64, OperatingSystem::MacOSX { .. })
             | (Architecture::X86_64, OperatingSystem::Darwin) => {
-                url = NIX_X64_64_DARWIN_URL;
                 nix_build_user_prefix = "_nixbld";
                 nix_build_user_id_base = 300;
                 nix_build_user_count = 32;
             },
             #[cfg(target_os = "macos")]
             (Architecture::Aarch64(_), OperatingSystem::MacOSX { .. })
             | (Architecture::Aarch64(_), OperatingSystem::Darwin) => {
-                url = NIX_AARCH64_DARWIN_URL;
                 nix_build_user_prefix = "_nixbld";
                 nix_build_user_id_base = 300;
                 nix_build_user_count = 32;
@@ -308,7 +262,7 @@ impl CommonSettings {
             nix_build_user_id_base,
             nix_build_user_count,
             nix_build_user_prefix: nix_build_user_prefix.to_string(),
-            nix_package_url: url.parse()?,
+            nix_package_url: None,
             proxy: Default::default(),
             extra_conf: Default::default(),
             force: false,