feat: store dependencies of extensions in dedicated directory

.github/workflows/release-with-github.yml

@@ -43,16 +43,8 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - name: Install dependencies
-        run: |
-          wget https://github.com/cargo-bins/cargo-binstall/releases/latest/download/cargo-binstall-x86_64-unknown-linux-musl.tgz
-          tar -xvzf cargo-binstall-x86_64-unknown-linux-musl.tgz
-          rm cargo-binstall-x86_64-unknown-linux-musl.tgz
-          mv cargo-binstall $HOME/.cargo/bin
-          cargo binstall cargo-release -y
-          cargo binstall ripgrep -y
+      - uses: cargo-bins/cargo-binstall@main
+      - run: cargo binstall cargo-release ripgrep -y
       - name: Determine new version number by dry-running `cargo-release`
         id: determine_version
         continue-on-error: true
@@ -64,8 +56,8 @@ jobs:
             cargo release version -p ${{ inputs.whichCrate }} ${{ inputs.semverBump }} &> cargo-release-output.txt
           fi
           cat cargo-release-output.txt
-          NEW_VERSION=$(cat cargo-release-output.txt | rg "Upgrading .* from .* to (.*)" -r '$1' | tr -d ' ')
-          echo $NEW_VERSION
+          NEW_VERSION=$(rg "Upgrading .* from .* to (.*)" -r '$1' cargo-release-output.txt | tr -d ' ')
+          echo "$NEW_VERSION"
           echo "NEW_VERSION=$NEW_VERSION" >> "$GITHUB_ENV"
           echo "NEW_VERSION=$NEW_VERSION" >> "$GITHUB_OUTPUT"
           rm cargo-release-output.txt

.github/workflows/release.yml

@@ -1,3 +1,6 @@
+# Copyright 2022-2023, axodotdev
+# SPDX-License-Identifier: MIT or Apache-2.0
+#
 # CI that:
 #
 # * checks for a Git Tag that looks like a release
@@ -72,27 +75,25 @@ jobs:
           echo "INFO: The release tag is: $TAG"
           echo "TAG=$TAG" >> "$GITHUB_ENV"
           echo "TAG=$TAG" >> "$GITHUB_OUTPUT"
+
       - uses: actions/checkout@v3
         with:
           ref: ${{ env.TAG }}
-      - name: Install Rust
-        run: rustup update 1.66.1 --no-self-update && rustup default 1.66.1
+          submodules: recursive
+
       - name: Install cargo-dist
-        run: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.5/cargo-dist-v0.0.5-installer.sh | sh
+        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.1.0/cargo-dist-installer.sh | sh"
+
       - id: create-release
         run: |
-          cargo dist manifest --tag=${{ env.TAG }} --artifacts=all --no-local-paths --output-format=json > dist-manifest.json
-          echo "dist manifest ran successfully"
-          # temp hack until it is possible to customize output archive format,
-          # see: https://github.com/axodotdev/cargo-dist/issues/17
-          file_contents=$(<dist-manifest.json)
-          echo "${file_contents//.tar.xz/.tar.gz}" > dist-manifest.json
+          cargo dist plan --tag=${{ env.TAG }} --output-format=json > dist-manifest.json
+          echo "dist plan ran successfully"
           cat dist-manifest.json
 
           # Create the Github Release™ based on what cargo-dist thinks it should be
-          ANNOUNCEMENT_TITLE=$(cat dist-manifest.json | jq --raw-output ".announcement_title")
-          IS_PRERELEASE=$(cat dist-manifest.json | jq --raw-output ".announcement_is_prerelease")
-          cat dist-manifest.json | jq --raw-output ".announcement_github_body" > new_dist_announcement.md
+          ANNOUNCEMENT_TITLE=$(jq --raw-output ".announcement_title" dist-manifest.json)
+          IS_PRERELEASE=$(jq --raw-output ".announcement_is_prerelease" dist-manifest.json)
+          jq --raw-output ".announcement_github_body" dist-manifest.json > new_dist_announcement.md
           gh release create ${{ env.TAG }} --draft --prerelease="$IS_PRERELEASE" --title="$ANNOUNCEMENT_TITLE" --notes-file=new_dist_announcement.md
           echo "created announcement!"
 
@@ -101,7 +102,7 @@ jobs:
           echo "uploaded manifest!"
 
           # Disable all the upload-artifacts tasks if we have no actual releases
-          HAS_RELEASES=$(cat dist-manifest.json | jq --raw-output ".releases != null")
+          HAS_RELEASES=$(jq --raw-output ".releases != null" dist-manifest.json)
           echo "has-releases=$HAS_RELEASES" >> "$GITHUB_OUTPUT"
 
   # Build and packages all the things
@@ -110,32 +111,38 @@ jobs:
     needs: create-release
     if: ${{ needs.create-release.outputs.has-releases == 'true' }}
     strategy:
+      fail-fast: false
       matrix:
         # For these target platforms
         include:
-        - os: macos-12
-          dist-args: --artifacts=local --target=aarch64-apple-darwin --target=x86_64-apple-darwin
-          install-dist: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.5/cargo-dist-v0.0.5-installer.sh | sh
-        - os: ubuntu-20.04
-          dist-args: --artifacts=local --target=x86_64-unknown-linux-gnu
-          install-dist: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.5/cargo-dist-v0.0.5-installer.sh | sh
+        - os: "macos-latest"
+          dist-args: "--artifacts=local --target=aarch64-apple-darwin"
+          install-dist: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.1.0/cargo-dist-installer.sh | sh"
+        - os: "macos-latest"
+          dist-args: "--artifacts=local --target=x86_64-apple-darwin"
+          install-dist: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.1.0/cargo-dist-installer.sh | sh"
+        - os: "ubuntu-latest"
+          dist-args: "--artifacts=local --target=x86_64-unknown-linux-gnu"
+          install-dist: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.1.0/cargo-dist-installer.sh | sh"
 
     runs-on: ${{ matrix.os }}
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Get correct tag
-        run: echo "TAG=${{ needs.create-release.outputs.release-tag }}" >> $GITHUB_ENV
+        run: echo "TAG=${{ needs.create-release.outputs.release-tag }}" >> "$GITHUB_ENV"
+
       - uses: actions/checkout@v3
         with:
           ref: ${{ env.TAG }}
-      - name: Install Rust
-        run: rustup update 1.66.1 --no-self-update && rustup default 1.66.1
+          submodules: recursive
+
       - name: Install cargo-dist
         run: ${{ matrix.install-dist }}
+
       - name: Run cargo-dist
         # This logic is a bit janky because it's trying to be a polyglot between
-        # powershell and bash since this will run on macos, and linux!
+        # powershell and bash since this will run on windows, macos, and linux!
         # The two platforms don't agree on how to talk about env vars but they
         # do agree on 'cat' and '$()' so we use that to marshal values between commands.
         run: |
@@ -145,25 +152,9 @@ jobs:
           cat dist-manifest.json
 
           # Parse out what we just built and upload it to the Github Release™
-          cat dist-manifest.json | jq --raw-output ".artifacts[]?.path | select( . != null )" > uploads.txt
-
-          # temp hack until it is possible to customize output archive format,
-          # see: https://github.com/axodotdev/cargo-dist/issues/17
-          echo "changing compression format from .tar.xz to .tar.gz ..."
-          FILES=$(cat uploads.txt)
-          rm uploads.txt
-          for i in $FILES; do
-            TAR_ARCHIVE_FILENAME="${i//.tar.xz/.tar}"
-            TAR_GZ_ARCHIVE_FILENAME="$TAR_ARCHIVE_FILENAME.gz"
-            CHECKSUM_FILENAME="$TAR_GZ_ARCHIVE_FILENAME.sha256"
-            xz -d -v $i
-            gzip --best --keep --force --no-name "$TAR_ARCHIVE_FILENAME"
-            shasum -a 256 "$TAR_GZ_ARCHIVE_FILENAME" > "$CHECKSUM_FILENAME"
-            echo "$TAR_GZ_ARCHIVE_FILENAME" >> uploads.txt
-            echo "$CHECKSUM_FILENAME" >> uploads.txt
-          done
-
+          jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json > uploads.txt
           echo "uploading..."
+          cat uploads.txt
           gh release upload ${{ env.TAG }} $(cat uploads.txt)
           echo "uploaded!"
 
@@ -177,8 +168,10 @@ jobs:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Get correct tag
-        run: echo "TAG=${{ needs.create-release.outputs.release-tag }}" >> $GITHUB_ENV
+        run: echo "TAG=${{ needs.create-release.outputs.release-tag }}" >> "$GITHUB_ENV"
       - name: mark release as non-draft
         run: |
           gh release edit ${{ env.TAG }} --draft=false

.gitignore

@@ -16,7 +16,4 @@ target/
 *.pdb
 
 ## don't include binaries that are downloaded in build.rs scripts
-extensions/sns/sns-cli
-extensions/nns/ic-admin
-extensions/nns/ic-nns-init
-extensions/nns/sns-cli
+**/binary-dependencies

Cargo.toml

@@ -3,9 +3,7 @@ members = ["extensions/sns", "extensions/nns", "extensions-utils"]
 
 [workspace.package]
 authors = ["DFINITY Team"]
-# Temp hack until https://github.com/axodotdev/cargo-dist/issues/187 is resovled.
-# Nothing will actually be published (unless `cargo publish` is executed (and it shouldn't)).
-publish = true
+publish = false
 edition = "2021"
 license = "Apache-2.0"
 repository = "https://github.com/dfinity/dfx-extensions"
@@ -37,9 +35,7 @@ url = "2.3.1"
 # Config for 'cargo dist'
 [workspace.metadata.dist]
 # The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax)
-cargo-dist-version = "0.0.5"
-# The preferred Rust toolchain to use in CI (rustup toolchain syntax)
-rust-toolchain-version = "1.66.1"
+cargo-dist-version = "0.1.0"
 # CI backends to support (see 'cargo dist generate-ci')
 ci = ["github"]
 # Target platforms to build apps for (Rust target-triple syntax)
@@ -48,16 +44,16 @@ targets = [
     "x86_64-apple-darwin",
     "aarch64-apple-darwin",
 ]
+unix-archive = ".tar.gz"
+checksum = "sha256"
+dist = true
+
 
 # The profile that 'cargo dist' will build with
 [profile.dist]
 inherits = "release"
 lto = "thin"
 
-[workspace.metadata.release]
-# Temp hack until https://github.com/axodotdev/cargo-dist/issues/187 is resovled.
-publish = false
-
 [patch.crates-io.ic-agent]
 version = "0.25.0"
 git = "https://github.com/dfinity/agent-rs.git"

extensions-utils/src/dependencies/download_ic_binaries.rs

@@ -29,15 +29,13 @@ pub fn download_ic_binary(replica_rev: &str, binary_name: &str, destination_path
 
     let bytes = Runtime::new().unwrap().block_on(download_bytes(&url));
     let mut d = GzDecoder::new(&*bytes);
-    let tempdir = tempfile::tempdir().expect("Failed to create temp dir");
-    let temp_file = tempdir.path().join(binary_name);
-    let mut temp = fs::File::create(&temp_file).expect("Failed to create the file");
-    copy(&mut d, &mut temp).expect("Failed to copy content");
+    let mut temp_file = tempfile::NamedTempFile::new().expect("Failed to create temp file");
+    copy(&mut d, &mut temp_file).expect("Failed to copy content");
 
     #[cfg(unix)]
     {
         use std::os::unix::fs::PermissionsExt;
-        dfx_core::fs::set_permissions(&temp_file, std::fs::Permissions::from_mode(0o500))
+        dfx_core::fs::set_permissions(&temp_file.path(), std::fs::Permissions::from_mode(0o500))
             .expect("Failed to set permissions");
     }
     fs::rename(temp_file, destination_path).expect("Failed to move extension");

extensions/nns/Cargo.toml

@@ -5,9 +5,7 @@ authors.workspace = true
 edition.workspace = true
 license.workspace = true
 repository.workspace = true
-# Temp hack until https://github.com/axodotdev/cargo-dist/issues/187 is resovled.
-# Nothing will actually be published (unless `cargo publish` is executed (and it shouldn't)).
-publish = true
+publish.workspace = true
 build = "build.rs"
 
 [build-dependencies]
@@ -37,8 +35,6 @@ tokio.workspace = true
 
 
 [package.metadata.release]
-# Temp hack until https://github.com/axodotdev/cargo-dist/issues/187 is resovled
-publish = false
 # list of replacements to be made after issuing `cargo release -p nns SEMVER`
 pre-release-replacements = [
   {file="CHANGELOG.md", search="Unreleased", replace="{{version}}"},
@@ -50,4 +46,4 @@ pre-release-replacements = [
 ]
 
 [package.metadata.dist]
-include = ["extension.json", "sns-cli", "ic-admin", "ic-nns-init"]
+include = ["extension.json", "binary-dependencies/"]

extensions/nns/build.rs

@@ -11,8 +11,10 @@ const BINARY_DEPENDENCIES: &[(&str, &str)] = &[
 ];
 
 fn main() {
-    // keep copy of the dependency in the root of the project, so that cargo-dist will be able to package it into a tarball
-    let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
+    // keep copy of the dependency in extensions/nns/binary-dependencies, so that cargo-dist will be able to package it into a tarball
+    let dependencies_dir =
+        PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()).join("binary-dependencies");
+    std::fs::create_dir_all(&dependencies_dir).expect("Failed to create dependencies dir");
     // and also in `target/debug` or `target/release` for development purposes (e.g. cargo run), this is a bit hacky: https://github.com/rust-lang/cargo/issues/9661
     let target_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap())
         .parent()
@@ -23,14 +25,21 @@ fn main() {
         .unwrap()
         .to_path_buf();
     for (binary_name, renamed_binary_name) in BINARY_DEPENDENCIES {
-        let bin_in_manifest_dir = manifest_dir.join(renamed_binary_name);
+        let bin_in_dependencies_dir = dependencies_dir.join(renamed_binary_name);
         let bin_in_target_dir = target_dir.join(renamed_binary_name);
-        dbg!(&bin_in_manifest_dir, &bin_in_target_dir);
-        dfx_extensions_utils::download_ic_binary(REPLICA_REV, binary_name, &bin_in_manifest_dir);
+        dbg!(&bin_in_dependencies_dir, &bin_in_target_dir);
+        if bin_in_dependencies_dir.exists() {
+            std::fs::remove_file(&bin_in_target_dir).expect("Failed to remove file");
+        }
+        dfx_extensions_utils::download_ic_binary(
+            REPLICA_REV,
+            binary_name,
+            &bin_in_dependencies_dir,
+        );
         if bin_in_target_dir.exists() {
             std::fs::remove_file(&bin_in_target_dir).unwrap();
         }
         std::fs::create_dir_all(&target_dir).unwrap();
-        std::fs::copy(&bin_in_manifest_dir, &bin_in_target_dir).unwrap();
+        std::fs::copy(&bin_in_dependencies_dir, &bin_in_target_dir).unwrap();
     }
 }

extensions/sns/Cargo.toml

@@ -5,9 +5,7 @@ authors.workspace = true
 edition.workspace = true
 license.workspace = true
 repository.workspace = true
-# Temp hack until https://github.com/axodotdev/cargo-dist/issues/187 is resovled.
-# Nothing will actually be published (unless `cargo publish` is executed (and it shouldn't)).
-publish = true
+publish.workspace = true
 build = "build.rs"
 
 [build-dependencies]
@@ -39,4 +37,4 @@ pre-release-replacements = [
 ]
 
 [package.metadata.dist]
-include = ["extension.json", "sns-cli"]
+include = ["extension.json", "binary-dependencies/"]

extensions/sns/build.rs

@@ -9,8 +9,10 @@ const BINARY_DEPENDENCIES: &[(&str, &str)] = &[
 ];
 
 fn main() {
-    // keep copy of the dependency in the root of the project, so that cargo-dist will be able to package it into a tarball
-    let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
+    // keep copy of the dependency in extensions/sns/binary-dependencies, so that cargo-dist will be able to package it into a tarball
+    let dependencies_dir =
+        PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()).join("binary-dependencies");
+    std::fs::create_dir_all(&dependencies_dir).expect("Failed to create dependencies dir");
     // and also in `target/debug` or `target/release` for development purposes (e.g. cargo run), this is a bit hacky: https://github.com/rust-lang/cargo/issues/9661
     let target_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap())
         .parent()
@@ -21,14 +23,21 @@ fn main() {
         .unwrap()
         .to_path_buf();
     for (binary_name, renamed_binary_name) in BINARY_DEPENDENCIES {
-        let bin_in_manifest_dir = manifest_dir.join(renamed_binary_name);
+        let bin_in_dependencies_dir = dependencies_dir.join(renamed_binary_name);
         let bin_in_target_dir = target_dir.join(renamed_binary_name);
-        dbg!(&bin_in_manifest_dir, &bin_in_target_dir);
-        dfx_extensions_utils::download_ic_binary(REPLICA_REV, binary_name, &bin_in_manifest_dir);
+        dbg!(&bin_in_dependencies_dir, &bin_in_target_dir);
+        if bin_in_dependencies_dir.exists() {
+            std::fs::remove_file(&bin_in_target_dir).expect("Failed to remove file");
+        }
+        dfx_extensions_utils::download_ic_binary(
+            REPLICA_REV,
+            binary_name,
+            &bin_in_dependencies_dir,
+        );
         if bin_in_target_dir.exists() {
             std::fs::remove_file(&bin_in_target_dir).unwrap();
         }
         std::fs::create_dir_all(&target_dir).unwrap();
-        std::fs::copy(&bin_in_manifest_dir, &bin_in_target_dir).unwrap();
+        std::fs::copy(&bin_in_dependencies_dir, &bin_in_target_dir).unwrap();
     }
 }

rust-toolchain.toml

@@ -0,0 +1,3 @@
+[toolchain]
+channel = "1.71.1"
+components = ["rustfmt", "clippy"]