diff --git a/.github/workflows/cli-tests.yaml b/.github/workflows/cli-tests.yaml index 21a0a2038ae..a4deb69eed5 100644 --- a/.github/workflows/cli-tests.yaml +++ b/.github/workflows/cli-tests.yaml @@ -129,9 +129,8 @@ jobs: run-project-tests: ["project-tests", "project-tests-off"] # Run tests on: # 1. the oldest supported nix version (which is 2.9.0? But determinate-systems installer has 2.12.0) - # 2. nix version 2.17.0 which introduces a new code path that minimizes nixpkgs downloads. - # 3. latest nix version - nix-version: ["2.12.0", "2.17.0", "2.19.2"] + # 2. latest nix version, must be > 2.17.0 which introduces a new code path that minimizes nixpkgs downloads. + nix-version: ["2.12.0", "2.19.2"] exclude: - is-main: "not-main" os: "${{ inputs.run-mac-tests && 'dummy' || 'macos-latest' }}" diff --git a/internal/devbox/nixprofile.go b/internal/devbox/nixprofile.go new file mode 100644 index 00000000000..5414051897e --- /dev/null +++ b/internal/devbox/nixprofile.go @@ -0,0 +1,74 @@ +package devbox + +import ( + "context" + "fmt" + "strings" + + "github.com/samber/lo" + "go.jetpack.io/devbox/internal/nix" + "go.jetpack.io/devbox/internal/nix/nixprofile" + "go.jetpack.io/devbox/internal/ux" +) + +// syncNixProfile ensures the nix profile has the packages specified in wantStorePaths. +// It also removes any packages from the nix profile that are not in wantStorePaths. +func (d *Devbox) syncNixProfile(ctx context.Context, wantStorePaths []string) error { + profilePath, err := d.profilePath() + if err != nil { + return err + } + + // Get the store-paths of the packages currently installed in the nix profile + items, err := nixprofile.ProfileListItems(ctx, d.stderr, profilePath) + if err != nil { + return fmt.Errorf("nix profile list: %v", err) + } + gotStorePaths := make([]string, 0, len(items)) + for _, item := range items { + gotStorePaths = append(gotStorePaths, item.StorePaths()...) + } + + // Diff the store paths and install/remove packages as needed + remove, add := lo.Difference(gotStorePaths, wantStorePaths) + if len(remove) > 0 { + packagesToRemove := make([]string, 0, len(remove)) + for _, p := range remove { + storePath := nix.NewStorePathParts(p) + packagesToRemove = append(packagesToRemove, fmt.Sprintf("%s@%s", storePath.Name, storePath.Version)) + } + if len(packagesToRemove) == 1 { + ux.Finfo(d.stderr, "Removing %s\n", strings.Join(packagesToRemove, ", ")) + } else { + ux.Finfo(d.stderr, "Removing packages: %s\n", strings.Join(packagesToRemove, ", ")) + } + + if err := nix.ProfileRemove(profilePath, remove...); err != nil { + return err + } + } + if len(add) > 0 { + total := len(add) + for idx, addPath := range add { + stepNum := idx + 1 + storePath := nix.NewStorePathParts(addPath) + nameAndVersion := fmt.Sprintf("%s@%s", storePath.Name, storePath.Version) + stepMsg := fmt.Sprintf("[%d/%d] %s", stepNum, total, nameAndVersion) + + if err = nixprofile.ProfileInstall(ctx, &nixprofile.ProfileInstallArgs{ + CustomStepMessage: stepMsg, + Installable: addPath, + // Install in offline mode for speed. We know we should have all the files + // locally in /nix/store since we have run `nix print-dev-env` prior to this. + // Also avoids some "substituter not found for store-path" errors. + Offline: true, + PackageName: storePath.Name, + ProfilePath: profilePath, + Writer: d.stderr, + }); err != nil { + return fmt.Errorf("error installing package %s: %w", addPath, err) + } + } + } + return nil +} diff --git a/internal/devbox/packages.go b/internal/devbox/packages.go index c750f4d0b89..9f581269333 100644 --- a/internal/devbox/packages.go +++ b/internal/devbox/packages.go @@ -248,10 +248,20 @@ const ( // The `mode` is used for: // 1. Skipping certain operations that may not apply. // 2. User messaging to explain what operations are happening, because this function may take time to execute. -func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) error { +// +// linter:revive is turned off due to complaining about function complexity +func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) error { //nolint:revive defer trace.StartRegion(ctx, "devboxEnsureStateIsUpToDate").End() defer debug.FunctionTimer().End() + if mode != ensure && !d.IsEnvEnabled() { + // if mode is install/uninstall/update and we are not in a devbox environment, + // then we skip some operations below for speed. + // Remove the local.lock file so that we re-compute the project state when + // we are in the devbox environment again. + defer func() { _ = d.lockfile.RemoveLocal() }() + } + // if mode is install or uninstall, then we need to update the nix-profile // and lockfile, so we must continue below. upToDate, err := d.lockfile.IsUpToDateAndInstalled() @@ -267,6 +277,13 @@ func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) er fmt.Fprintln(d.stderr, "Ensuring packages are installed.") } + // Validate packages. Must be run up-front and definitely prior to computeEnv + // and syncNixProfile below that will evaluate the flake and may give + // inscrutable errors if the package is uninstallable. + if err := d.validatePackagesToBeInstalled(ctx); err != nil { + return err + } + // Create plugin directories first because packages might need them for _, pkg := range d.InstallablePackages() { if err := d.PluginManager().Create(pkg); err != nil { @@ -274,10 +291,6 @@ func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) er } } - if err := d.syncPackagesToProfile(ctx, mode); err != nil { - return err - } - if err := d.InstallRunXPackages(ctx); err != nil { return err } @@ -290,11 +303,34 @@ func (d *Devbox) ensureStateIsUpToDate(ctx context.Context, mode installMode) er return err } - // Use the printDevEnvCache if we are adding or removing or updating any package, - // AND we are not in the shellenv-enabled environment of the current devbox-project. - usePrintDevEnvCache := mode != ensure && !d.IsEnvEnabled() - if _, err := d.computeEnv(ctx, usePrintDevEnvCache); err != nil { - return err + // The steps contained in this if-block of computeEnv and syncNixProfile are a tad + // slow. So, we only do it if we are in a devbox environment, or if mode is ensure. + if mode == ensure || d.IsEnvEnabled() { + // Re-compute print-dev-env to ensure all packages are installed, and + // the correct set of packages are reflected in the nix-profile below. + env, err := d.computeEnv(ctx, false /*usePrintDevEnvCache*/) + if err != nil { + return err + } + + // Ensure the nix profile has the packages from the flake. + buildInputs := []string{} + if env["buildInputs"] != "" { + // env["buildInputs"] can be empty string if there are no packages in the project + // if buildInputs is empty, then we don't want wantStorePaths to be an array with a single "" entry + buildInputs = strings.Split(env["buildInputs"], " ") + } + if err := d.syncNixProfile(ctx, buildInputs); err != nil { + return err + } + + } else if mode == install || mode == update { + // Else: if we are not in a devbox environment, and we are installing or updating + // then we must ensure the new nix packages are in the nix store. This way, the + // next time we enter a devbox environment, we will have the packages available locally. + if err := d.installNixPackagesToStore(ctx); err != nil { + return err + } } // Ensure we clean out packages that are no longer needed. @@ -331,162 +367,6 @@ func (d *Devbox) profilePath() (string, error) { return absPath, errors.WithStack(os.MkdirAll(filepath.Dir(absPath), 0o755)) } -// syncPackagesToProfile can ensure that all packages in devbox.json exist in the nix profile, -// and no more. However, it may skip some steps depending on the `mode`. -func (d *Devbox) syncPackagesToProfile(ctx context.Context, mode installMode) error { - defer debug.FunctionTimer().End() - defer trace.StartRegion(ctx, "syncPackagesToProfile").End() - - // First, fetch the profile items from the nix-profile, - // and get the installable packages - profileDir, err := d.profilePath() - if err != nil { - return err - } - profileItems, err := nixprofile.ProfileListItems(d.stderr, profileDir) - if err != nil { - return err - } - packages, err := d.AllInstallablePackages() - if err != nil { - return err - } - - // Remove non-nix packages from the list - packages = lo.Filter(packages, devpkg.IsNix) - - if err := devpkg.FillNarInfoCache(ctx, packages...); err != nil { - return err - } - - // Second, remove any packages from the nix-profile that are not in the config - itemsToKeep := profileItems - if mode != install { - itemsToKeep, err = d.removeExtraItemsFromProfile(ctx, profileDir, profileItems, packages) - if err != nil { - return err - } - } - - // we are done if mode is uninstall - if mode == uninstall { - return nil - } - - // Last, find the pending packages, and ensure they are added to the nix-profile - // Important to maintain the order of packages as specified by - // Devbox.InstallablePackages() (higher priority first). - // We also run nix profile upgrade on any virtenv flakes. This is a bit of a - // blunt approach, but ensured any plugin created flakes are up-to-date. - pending := []*devpkg.Package{} - for _, pkg := range packages { - idx, err := nixprofile.ProfileListIndex(&nixprofile.ProfileListIndexArgs{ - Items: itemsToKeep, - Lockfile: d.lockfile, - Writer: d.stderr, - Package: pkg, - ProfileDir: profileDir, - }) - if err != nil { - if !errors.Is(err, nix.ErrPackageNotFound) { - return err - } - pending = append(pending, pkg) - } else if f, err := pkg.FlakeInstallable(); err == nil && d.pluginManager.PathIsInVirtenv(f.Ref.Path) { - if err := nix.ProfileUpgrade(profileDir, idx); err != nil { - return err - } - } - } - - return d.addPackagesToProfile(ctx, pending) -} - -func (d *Devbox) removeExtraItemsFromProfile( - ctx context.Context, - profileDir string, - profileItems []*nixprofile.NixProfileListItem, - packages []*devpkg.Package, -) ([]*nixprofile.NixProfileListItem, error) { - defer debug.FunctionTimer().End() - defer trace.StartRegion(ctx, "removeExtraPackagesFromProfile").End() - - itemsToKeep := []*nixprofile.NixProfileListItem{} - extras := []*nixprofile.NixProfileListItem{} - // Note: because devpkg.Package uses memoization when normalizing attribute paths (slow operation), - // and since we're reusing the Package objects, this O(n*m) loop becomes O(n+m) wrt the slow operation. - for _, item := range profileItems { - found := false - for _, pkg := range packages { - if item.Matches(pkg, d.lockfile) { - itemsToKeep = append(itemsToKeep, item) - found = true - break - } - } - if !found { - extras = append(extras, item) - } - } - // Remove by index to avoid comparing nix.ProfileListItem <> nix.Inputs again. - if err := nixprofile.ProfileRemoveItems(profileDir, extras); err != nil { - return nil, err - } - return itemsToKeep, nil -} - -// addPackagesToProfile inspects the packages in devbox.json, checks which of them -// are missing from the nix profile, and then installs each package individually into the -// nix profile. -func (d *Devbox) addPackagesToProfile(ctx context.Context, pkgs []*devpkg.Package) error { - defer debug.FunctionTimer().End() - defer trace.StartRegion(ctx, "addPackagesToProfile").End() - - if len(pkgs) == 0 { - return nil - } - - // If packages are in profile but nixpkgs has been purged, the experience - // will be poor when we try to run print-dev-env. So we ensure nixpkgs is - // prefetched for all relevant packages (those not in binary cache). - if err := devpkg.EnsureNixpkgsPrefetched(ctx, d.stderr, pkgs); err != nil { - return err - } - - var msg string - if len(pkgs) == 1 { - msg = fmt.Sprintf("Installing package: %s.", pkgs[0]) - } else { - pkgNames := lo.Map(pkgs, func(p *devpkg.Package, _ int) string { return p.Raw }) - msg = fmt.Sprintf("Installing %d packages: %s.", len(pkgs), strings.Join(pkgNames, ", ")) - } - fmt.Fprintf(d.stderr, "\n%s\n\n", msg) - - profileDir, err := d.profilePath() - if err != nil { - return fmt.Errorf("error getting profile path: %w", err) - } - - total := len(pkgs) - for idx, pkg := range pkgs { - stepNum := idx + 1 - - stepMsg := fmt.Sprintf("[%d/%d] %s", stepNum, total, pkg) - - if err = nixprofile.ProfileInstall(ctx, &nixprofile.ProfileInstallArgs{ - CustomStepMessage: stepMsg, - Lockfile: d.lockfile, - Package: pkg.Raw, - ProfilePath: profileDir, - Writer: d.stderr, - }); err != nil { - return fmt.Errorf("error installing package %s: %w", pkg, err) - } - } - - return nil -} - var resetCheckDone = false // resetProfileDirForFlakes ensures the profileDir directory is cleared of old @@ -536,3 +416,116 @@ func (d *Devbox) InstallRunXPackages(ctx context.Context) error { } return nil } + +// installNixPackagesToStore will install all the packages in the nix store, if +// mode is install or update, and we're not in a devbox environment. +// This is done by running `nix build` on the flake. We do this so that the +// packages will be available in the nix store when computing the devbox environment +// and installing in the nix profile (even if offline). +func (d *Devbox) installNixPackagesToStore(ctx context.Context) error { + packages, err := d.packagesToInstallInProfile(ctx) + if err != nil { + return err + } + + names := []string{} + installables := []string{} + for _, pkg := range packages { + i, err := pkg.Installable() + if err != nil { + return err + } + installables = append(installables, i) + names = append(names, pkg.String()) + } + + if len(installables) == 0 { + return nil + } + + ux.Finfo(d.stderr, "Installing to the nix store: %s. This may take a brief while.\n", strings.Join(names, " ")) + + // --no-link to avoid generating the result objects + return nix.Build(ctx, []string{"--no-link"}, installables...) +} + +// validatePackagesToBeInstalled will ensure that packages are available to be installed +// in the user's current system. +func (d *Devbox) validatePackagesToBeInstalled(ctx context.Context) error { + // First, get the packages to install + packagesToInstall, err := d.packagesToInstallInProfile(ctx) + if err != nil { + return err + } + + // Then, validate that packages that need to be installed are in fact installable + // on the user's current system. + for _, pkg := range packagesToInstall { + inCache, err := pkg.IsInBinaryCache() + if err != nil { + return err + } + + if !inCache && nix.IsGithubNixpkgsURL(pkg.URLForFlakeInput()) { + if err := nix.EnsureNixpkgsPrefetched(d.stderr, pkg.HashFromNixPkgsURL()); err != nil { + return err + } + if exists, err := pkg.ValidateInstallsOnSystem(); err != nil { + return err + } else if !exists { + platform := nix.System() + return usererr.New( + "package %s cannot be installed on your platform %s.\n"+ + "If you know this package is incompatible with %[2]s, then "+ + "you could run `devbox add %[1]s --exclude-platform %[2]s` and re-try.\n"+ + "If you think this package should be compatible with %[2]s, then "+ + "it's possible this particular version is not available yet from the nix registry. "+ + "You could try `devbox add` with a different version for this package.\n", + pkg.Raw, + platform, + ) + } + } + } + return nil +} + +func (d *Devbox) packagesToInstallInProfile(ctx context.Context) ([]*devpkg.Package, error) { + // First, fetch the profile items from the nix-profile, + profileDir, err := d.profilePath() + if err != nil { + return nil, err + } + profileItems, err := nixprofile.ProfileListItems(ctx, d.stderr, profileDir) + if err != nil { + return nil, err + } + + // Second, get and prepare all the packages that must be installed in this project + packages, err := d.AllInstallablePackages() + if err != nil { + return nil, err + } + packages = lo.Filter(packages, devpkg.IsNix) // Remove non-nix packages from the list + if err := devpkg.FillNarInfoCache(ctx, packages...); err != nil { + return nil, err + } + + // Third, compute which packages need to be installed + packagesToInstall := []*devpkg.Package{} + // Note: because devpkg.Package uses memoization when normalizing attribute paths (slow operation), + // and since we're reusing the Package objects, this O(n*m) loop becomes O(n+m) wrt the slow operation. + for _, pkg := range packages { + found := false + for _, item := range profileItems { + if item.Matches(pkg, d.lockfile) { + found = true + break + } + } + if !found { + packagesToInstall = append(packagesToInstall, pkg) + } + } + return packagesToInstall, nil +} diff --git a/internal/devbox/util.go b/internal/devbox/util.go index 64c5168a7cb..a79e89abaef 100644 --- a/internal/devbox/util.go +++ b/internal/devbox/util.go @@ -10,8 +10,9 @@ import ( "path/filepath" "github.com/pkg/errors" - "go.jetpack.io/devbox/internal/nix/nixprofile" + "go.jetpack.io/devbox/internal/devpkg" + "go.jetpack.io/devbox/internal/nix/nixprofile" "go.jetpack.io/devbox/internal/xdg" ) @@ -19,15 +20,19 @@ import ( // It's used to install applications devbox might need, like process-compose // This is an alternative to a global install which would modify a user's // environment. -func (d *Devbox) addDevboxUtilityPackage(ctx context.Context, pkg string) error { +func (d *Devbox) addDevboxUtilityPackage(ctx context.Context, pkgName string) error { + pkg := devpkg.PackageFromStringWithDefaults(pkgName, d.lockfile) + installable, err := pkg.Installable() + if err != nil { + return err + } profilePath, err := utilityNixProfilePath() if err != nil { return err } - return nixprofile.ProfileInstall(ctx, &nixprofile.ProfileInstallArgs{ - Lockfile: d.lockfile, - Package: pkg, + Installable: installable, + PackageName: pkgName, ProfilePath: profilePath, Writer: d.stderr, }) diff --git a/internal/devpkg/narinfo_cache.go b/internal/devpkg/narinfo_cache.go index ba8475bf320..170ff7e00cc 100644 --- a/internal/devpkg/narinfo_cache.go +++ b/internal/devpkg/narinfo_cache.go @@ -4,10 +4,8 @@ import ( "context" "io" "net/http" - "strings" "sync" "time" - "unicode" "github.com/pkg/errors" "go.jetpack.io/devbox/internal/boxcli/featureflag" @@ -110,8 +108,8 @@ func (p *Package) fetchNarInfoStatus() (bool, error) { ) } - pathParts := newStorePathParts(sysInfo.StorePath) - reqURL := BinaryCache + "/" + pathParts.hash + ".narinfo" + pathParts := nix.NewStorePathParts(sysInfo.StorePath) + reqURL := BinaryCache + "/" + pathParts.Hash + ".narinfo" ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodHead, reqURL, nil) @@ -181,35 +179,3 @@ func (p *Package) sysInfoIfExists() (*lock.SystemInfo, error) { } return sysInfo, nil } - -// storePath are the constituent parts of -// /nix/store/-- -// -// This is a helper struct for analyzing the string representation -type storePathParts struct { - hash string - name string - version string -} - -// newStorePathParts splits a Nix store path into its hash, name and version -// components in the same way that Nix does. -// -// See https://nixos.org/manual/nix/stable/language/builtins.html#builtins-parseDrvName -func newStorePathParts(path string) storePathParts { - path = strings.TrimPrefix(path, "/nix/store/") - // path is now -- 0 { errString += fmt.Sprintf("Known vulnerabilities: %s \n\n", knownVulnerabilities) } @@ -40,36 +48,40 @@ func ProfileInstall(writer io.Writer, profilePath, installable string) error { return usererr.New(errString) } - cmd := command( + cmd := commandContext( + ctx, "profile", "install", - "--profile", profilePath, + "--profile", args.ProfilePath, "--impure", // for NIXPKGS_ALLOW_UNFREE // Using an arbitrary priority to avoid conflicts with other packages. // Note that this is not really the priority we care about, since we // use the flake.nix to specify the priority. - "--priority", nextPriority(profilePath), - installable, + "--priority", nextPriority(args.ProfilePath), ) + if args.Offline { + cmd.Args = append(cmd.Args, "--offline") + } + cmd.Args = append(cmd.Args, args.Installable) cmd.Env = allowUnfreeEnv(os.Environ()) // If nix profile install runs as tty, the output is much nicer. If we ever // need to change this to our own writers, consider that you may need // to implement your own nicer output. --print-build-logs flag may be useful. cmd.Stdin = os.Stdin - cmd.Stdout = writer - cmd.Stderr = writer + cmd.Stdout = args.Writer + cmd.Stderr = args.Writer debug.Log("running command: %s\n", cmd) return cmd.Run() } -func ProfileRemove(profilePath string, indexes []string) error { +func ProfileRemove(profilePath string, elements ...string) error { cmd := command( append([]string{ "profile", "remove", "--profile", profilePath, "--impure", // for NIXPKGS_ALLOW_UNFREE - }, indexes...)..., + }, elements...)..., ) cmd.Env = allowUnfreeEnv(allowInsecureEnv(os.Environ())) diff --git a/internal/nix/storepath.go b/internal/nix/storepath.go new file mode 100644 index 00000000000..2f8acb638dc --- /dev/null +++ b/internal/nix/storepath.go @@ -0,0 +1,40 @@ +package nix + +import ( + "strings" + "unicode" +) + +// storePath are the constituent parts of +// /nix/store/-- +// +// This is a helper struct for analyzing the string representation +type StorePathParts struct { + Hash string + Name string + Version string +} + +// NewStorePathParts splits a Nix store path into its hash, name and version +// components in the same way that Nix does. +// +// See https://nixos.org/manual/nix/stable/language/builtins.html#builtins-parseDrvName +// +// TODO: store paths can also have `-{output}` suffixes, which need to be handled below. +func NewStorePathParts(path string) StorePathParts { + path = strings.TrimPrefix(path, "/nix/store/") + // path is now --