From bd8eed1add5f63aa88b41456eefa00192b78aa88 Mon Sep 17 00:00:00 2001 From: Greg Curtis Date: Tue, 24 Oct 2023 23:04:27 -0400 Subject: [PATCH 1/2] impl: use flake for profile packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a proof-of-concept that changes devbox so that it uses the flake when installing packages to a profile. The idea is that this moves a lot of logic out of Devbox and into Nix, where things are more reproducible. It also makes the flake the single source of truth -- packages added to the flake automatically get added to the profile. When running this in a project where most packages are already in the Nix store (and therefore most time is spent syncing the profile), this runs about a 2.3x faster: ``` $ cd ~/src/axiom $ rm -rf .devbox $ time devbox install # with this PR Ensuring packages are installed. ✓ Computed the Devbox environment. Finished installing packages. devbox install 0.25s user 0.16s system 48% cpu 0.837 total $ rm -rf .devbox $ time devbox install # current main Ensuring packages are installed. Installing 12 packages: go@latest, buf@latest, python310@latest, sops@latest, kubernetes-helm@latest, postgresql@latest, nodejs@latest, yarn@latest, awscli2@latest, golangci-lint@latest, flyctl@latest, gotty@latest. [1/12] go@latest [1/12] go@latest: Success [2/12] buf@latest [2/12] buf@latest: Success [3/12] python310@latest [3/12] python310@latest: Success [4/12] sops@latest [4/12] sops@latest: Success [5/12] kubernetes-helm@latest [5/12] kubernetes-helm@latest: Success [6/12] postgresql@latest [6/12] postgresql@latest: Success [7/12] nodejs@latest [7/12] nodejs@latest: Success [8/12] yarn@latest [8/12] yarn@latest: Success [9/12] awscli2@latest [9/12] awscli2@latest: Success [10/12] golangci-lint@latest [10/12] golangci-lint@latest: Success [11/12] flyctl@latest [11/12] flyctl@latest: Success [12/12] gotty@latest [12/12] gotty@latest: Success ✓ Computed the Devbox environment. Finished installing packages. devbox install 0.69s user 0.53s system 62% cpu 1.958 total ``` So far it passes all tests except for two: 1. The php testscript fails because the packages installed by the plugin get removed from the PATH by devbox. I'm assuming this is fixable. 2. The unfree testscript fails because the new syncing logic doesn't output progress (because it's a poc). --- internal/impl/packages.go | 12 ++++-- internal/impl/poc.go | 88 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 internal/impl/poc.go diff --git a/internal/impl/packages.go b/internal/impl/packages.go index 2c204d85f83..ef9e16dea53 100644 --- a/internal/impl/packages.go +++ b/internal/impl/packages.go @@ -240,10 +240,6 @@ func (d *Devbox) ensurePackagesAreInstalled(ctx context.Context, mode installMod } } - if err := d.syncPackagesToProfile(ctx, mode); err != nil { - return err - } - if err := d.InstallRunXPackages(ctx); err != nil { return err } @@ -264,6 +260,14 @@ func (d *Devbox) ensurePackagesAreInstalled(ctx context.Context, mode installMod return err } + profile, err := d.profilePath() + if err != nil { + return err + } + if err := syncFlakeToProfile(ctx, d.flakeDir(), profile); err != nil { + return err + } + // Ensure we clean out packages that are no longer needed. d.lockfile.Tidy() diff --git a/internal/impl/poc.go b/internal/impl/poc.go new file mode 100644 index 00000000000..5b1f51bbd85 --- /dev/null +++ b/internal/impl/poc.go @@ -0,0 +1,88 @@ +package impl + +import ( + "context" + "encoding/json" + "fmt" + "os/exec" + "slices" + + "go.jetpack.io/devbox/internal/nix" +) + +func syncFlakeToProfile(ctx context.Context, flakePath, profilePath string) error { + cmd := exec.CommandContext(ctx, "nix", "eval", "--json", flakePath+"#devShells."+nix.System()+".default.buildInputs") + b, err := cmd.Output() + if err != nil { + return fmt.Errorf("nix eval devShells: %v", err) + } + storePaths := []string{} + if err := json.Unmarshal(b, &storePaths); err != nil { + return fmt.Errorf("unmarshal store paths: %s: %v", b, err) + } + + listCmd := exec.CommandContext(ctx, "nix", "profile", "list", "--json", "--profile", profilePath) + b, err = listCmd.Output() + if err != nil { + return err + } + var profile struct { + Elements []struct { + StorePaths []string + } + } + if err := json.Unmarshal(b, &profile); err != nil { + return fmt.Errorf("unmarshal profile: %v", err) + } + got := make([]string, 0, len(profile.Elements)) + for _, e := range profile.Elements { + got = append(got, e.StorePaths...) + } + + add, remove := diffStorePaths(got, storePaths) + if len(remove) > 0 { + removeCmd := exec.CommandContext(ctx, "nix", "profile", "remove", "--profile", profilePath) + removeCmd.Args = append(removeCmd.Args, remove...) + if err := removeCmd.Run(); err != nil { + return err + } + } + if len(add) > 0 { + addCmd := exec.CommandContext(ctx, "nix", "profile", "install", "--profile", profilePath) + addCmd.Args = append(addCmd.Args, add...) + if err := addCmd.Run(); err != nil { + return err + } + } + return nil +} + +func diffStorePaths(got, want []string) (add, remove []string) { + slices.Sort(got) + slices.Sort(want) + + var g, w int + for { + if g >= len(got) { + add = append(add, want[w:]...) + break + } + if w >= len(want) { + remove = append(remove, got[g:]...) + break + } + + switch { + case got[g] == want[w]: + g++ + w++ + case got[g] < want[w]: + remove = append(remove, got[g]) + g++ + case got[g] > want[w]: + add = append(add, want[w]) + w++ + } + } + return add, remove +} From 4d962dd84e4407026c5c2d8a79e89b37bd8b2839 Mon Sep 17 00:00:00 2001 From: Greg Curtis Date: Wed, 25 Oct 2023 12:18:39 -0400 Subject: [PATCH 2/2] Remove old code --- internal/impl/packages.go | 151 -------------------------------------- 1 file changed, 151 deletions(-) diff --git a/internal/impl/packages.go b/internal/impl/packages.go index ef9e16dea53..c56983dd8ce 100644 --- a/internal/impl/packages.go +++ b/internal/impl/packages.go @@ -17,7 +17,6 @@ import ( "github.com/samber/lo" "go.jetpack.io/devbox/internal/devpkg" "go.jetpack.io/devbox/internal/devpkg/pkgtype" - "go.jetpack.io/devbox/internal/nix/nixprofile" "go.jetpack.io/devbox/internal/shellgen" "go.jetpack.io/devbox/internal/boxcli/usererr" @@ -305,156 +304,6 @@ func (d *Devbox) profilePath() (string, error) { return absPath, errors.WithStack(os.MkdirAll(filepath.Dir(absPath), 0o755)) } -// syncPackagesToProfile can ensure that all packages in devbox.json exist in the nix profile, -// and no more. However, it may skip some steps depending on the `mode`. -func (d *Devbox) syncPackagesToProfile(ctx context.Context, mode installMode) error { - defer debug.FunctionTimer().End() - defer trace.StartRegion(ctx, "syncPackagesToProfile").End() - - // First, fetch the profile items from the nix-profile, - // and get the installable packages - profileDir, err := d.profilePath() - if err != nil { - return err - } - profileItems, err := nixprofile.ProfileListItems(d.stderr, profileDir) - if err != nil { - return err - } - packages, err := d.AllInstallablePackages() - if err != nil { - return err - } - - // Remove non-nix packages from the list - packages = lo.Filter(packages, devpkg.IsNix) - - if err := devpkg.FillNarInfoCache(ctx, packages...); err != nil { - return err - } - - // Second, remove any packages from the nix-profile that are not in the config - itemsToKeep := profileItems - if mode != install { - itemsToKeep, err = d.removeExtraItemsFromProfile(ctx, profileDir, profileItems, packages) - if err != nil { - return err - } - } - - // we are done if mode is uninstall - if mode == uninstall { - return nil - } - - // Last, find the pending packages, and ensure they are added to the nix-profile - // Important to maintain the order of packages as specified by - // Devbox.InstallablePackages() (higher priority first) - pending := []*devpkg.Package{} - for _, pkg := range packages { - _, err := nixprofile.ProfileListIndex(&nixprofile.ProfileListIndexArgs{ - Items: itemsToKeep, - Lockfile: d.lockfile, - Writer: d.stderr, - Package: pkg, - ProfileDir: profileDir, - }) - if err != nil { - if !errors.Is(err, nix.ErrPackageNotFound) { - return err - } - pending = append(pending, pkg) - } - } - - return d.addPackagesToProfile(ctx, pending) -} - -func (d *Devbox) removeExtraItemsFromProfile( - ctx context.Context, - profileDir string, - profileItems []*nixprofile.NixProfileListItem, - packages []*devpkg.Package, -) ([]*nixprofile.NixProfileListItem, error) { - defer debug.FunctionTimer().End() - defer trace.StartRegion(ctx, "removeExtraPackagesFromProfile").End() - - itemsToKeep := []*nixprofile.NixProfileListItem{} - extras := []*nixprofile.NixProfileListItem{} - // Note: because devpkg.Package uses memoization when normalizing attribute paths (slow operation), - // and since we're reusing the Package objects, this O(n*m) loop becomes O(n+m) wrt the slow operation. - for _, item := range profileItems { - found := false - for _, pkg := range packages { - if item.Matches(pkg, d.lockfile) { - itemsToKeep = append(itemsToKeep, item) - found = true - break - } - } - if !found { - extras = append(extras, item) - } - } - // Remove by index to avoid comparing nix.ProfileListItem <> nix.Inputs again. - if err := nixprofile.ProfileRemoveItems(profileDir, extras); err != nil { - return nil, err - } - return itemsToKeep, nil -} - -// addPackagesToProfile inspects the packages in devbox.json, checks which of them -// are missing from the nix profile, and then installs each package individually into the -// nix profile. -func (d *Devbox) addPackagesToProfile(ctx context.Context, pkgs []*devpkg.Package) error { - defer debug.FunctionTimer().End() - defer trace.StartRegion(ctx, "addPackagesToProfile").End() - - if len(pkgs) == 0 { - return nil - } - - // If packages are in profile but nixpkgs has been purged, the experience - // will be poor when we try to run print-dev-env. So we ensure nixpkgs is - // prefetched for all relevant packages (those not in binary cache). - if err := devpkg.EnsureNixpkgsPrefetched(ctx, d.stderr, pkgs); err != nil { - return err - } - - var msg string - if len(pkgs) == 1 { - msg = fmt.Sprintf("Installing package: %s.", pkgs[0]) - } else { - pkgNames := lo.Map(pkgs, func(p *devpkg.Package, _ int) string { return p.Raw }) - msg = fmt.Sprintf("Installing %d packages: %s.", len(pkgs), strings.Join(pkgNames, ", ")) - } - fmt.Fprintf(d.stderr, "\n%s\n\n", msg) - - profileDir, err := d.profilePath() - if err != nil { - return fmt.Errorf("error getting profile path: %w", err) - } - - total := len(pkgs) - for idx, pkg := range pkgs { - stepNum := idx + 1 - - stepMsg := fmt.Sprintf("[%d/%d] %s", stepNum, total, pkg) - - if err = nixprofile.ProfileInstall(ctx, &nixprofile.ProfileInstallArgs{ - CustomStepMessage: stepMsg, - Lockfile: d.lockfile, - Package: pkg.Raw, - ProfilePath: profileDir, - Writer: d.stderr, - }); err != nil { - return fmt.Errorf("error installing package %s: %w", pkg, err) - } - } - - return nil -} - var resetCheckDone = false // resetProfileDirForFlakes ensures the profileDir directory is cleared of old