diff --git a/devbox.json b/devbox.json index 7b263cb4a99..ba04e5eeaf4 100644 --- a/devbox.json +++ b/devbox.json @@ -1,7 +1,6 @@ { "packages": [ "go@1.20", - "actionlint@1.6.23", "golangci-lint@1.52.2" ], "env": { diff --git a/devbox.lock b/devbox.lock index cd586e3c2f7..a52f1eda5b6 100644 --- a/devbox.lock +++ b/devbox.lock @@ -1,11 +1,6 @@ { "lockfile_version": "1", "packages": { - "actionlint@1.6.23": { - "last_modified": "2023-03-31T22:52:29Z", - "resolved": "github:NixOS/nixpkgs/242246ee1e58f54d2322227fc5eef53b4a616a31#actionlint", - "version": "1.6.23" - }, "go@1.20": { "last_modified": "2023-05-25T03:54:59Z", "resolved": "github:NixOS/nixpkgs/8d4d822bc0efa9de6eddc79cb0d82897a9baa750#go", diff --git a/internal/devpkg/package.go b/internal/devpkg/package.go index 8529e8274ec..946964c4f31 100644 --- a/internal/devpkg/package.go +++ b/internal/devpkg/package.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net/url" + "os" "path/filepath" "regexp" "strings" @@ -19,6 +20,7 @@ import ( "go.jetpack.io/devbox/internal/cuecfg" "go.jetpack.io/devbox/internal/lock" "go.jetpack.io/devbox/internal/nix" + "go.jetpack.io/devbox/internal/ux" ) // Package represents a "package" added to the devbox.json config. @@ -405,8 +407,10 @@ func (p *Package) HashFromNixPkgsURL() string { // BinaryCacheStore is the store from which to fetch this package's binaries. // It is used as FromStore in builtins.fetchClosure. +// TODO savil: rename to BinaryCache const BinaryCacheStore = "https://cache.nixos.org" +// TODO savil: rename to IsInBinaryCache func (p *Package) IsInBinaryStore() (bool, error) { if !p.isVersioned() { return false, nil @@ -432,7 +436,8 @@ func (p *Package) IsInBinaryStore() (bool, error) { } // PathInBinaryStore is the key in the BinaryCacheStore for this package -// This is used as FromPath in builtins.fetchClosure +// This is used as StorePath in builtins.fetchClosure +// TODO savil: rename to PathInBinaryCache func (p *Package) PathInBinaryStore() (string, error) { if isInStore, err := p.IsInBinaryStore(); err != nil { return "", err @@ -452,10 +457,41 @@ func (p *Package) PathInBinaryStore() (string, error) { } sysInfo := entry.Systems[userSystem] - storeDirParts := []string{sysInfo.FromHash, sysInfo.StoreName} - if sysInfo.StoreVersion != "" { - storeDirParts = append(storeDirParts, sysInfo.StoreVersion) + return sysInfo.StorePath, nil +} + +func (p *Package) ContentAddressedStorePath() (string, error) { + + if isInStore, err := p.IsInBinaryStore(); err != nil { + return "", err + } else if !isInStore { + return "", + errors.Errorf("Package %q cannot be fetched from binary cache store", p.Raw) + } + + entry, err := p.lockfile.Resolve(p.Raw) + if err != nil { + return "", err + } + + userSystem, err := nix.System() + if err != nil { + return "", err + } + + sysInfo := entry.Systems[userSystem] + if sysInfo.CAStorePath != "" { + return sysInfo.CAStorePath, nil + } + + ux.Fwarning( + os.Stderr, + "calculating local_store_path. This may be slow.\n"+ + "Run `devbox update` to speed this up for next time.", + ) + localPath, err := nix.ContentAddressedStorePath(sysInfo.StorePath) + if err != nil { + return "", err } - storeDir := strings.Join(storeDirParts, "-") - return filepath.Join("/nix/store", storeDir), nil + return localPath, err } diff --git a/internal/impl/update.go b/internal/impl/update.go index ab945c85d61..cd9e08b0970 100644 --- a/internal/impl/update.go +++ b/internal/impl/update.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "go.jetpack.io/devbox/internal/boxcli/featureflag" "go.jetpack.io/devbox/internal/devpkg" "go.jetpack.io/devbox/internal/nix" "go.jetpack.io/devbox/internal/nix/nixprofile" @@ -90,7 +91,13 @@ func (d *Devbox) updateDevboxPackage( if err != nil { return err } - if existing != nil && existing.Version != newEntry.Version { + if existing == nil { + ux.Finfo(d.writer, "Resolved %s to %[1]s %[2]s\n", pkg, newEntry.Resolved) + d.lockfile.Packages[pkg.Raw] = newEntry + return nil + } + + if existing.Version != newEntry.Version { ux.Finfo(d.writer, "Updating %s %s -> %s\n", pkg, existing.Version, newEntry.Version) if err := d.removePackagesFromProfile(ctx, []string{pkg.Raw}); err != nil { // Warn but continue. TODO(landau): ensurePackagesAreInstalled should @@ -98,13 +105,36 @@ func (d *Devbox) updateDevboxPackage( ux.Fwarning(d.writer, "Failed to remove %s from profile: %s\n", pkg, err) } d.lockfile.Packages[pkg.Raw] = newEntry - } else if existing == nil { - ux.Finfo(d.writer, "Resolved %s to %[1]s %[2]s\n", pkg, newEntry.Resolved) - d.lockfile.Packages[pkg.Raw] = newEntry - } else { - ux.Finfo(d.writer, "Already up-to-date %s %s\n", pkg, existing.Version) + return nil + } + + // Check if the package's system info is missing, or not complete. + if featureflag.RemoveNixpkgs.Enabled() { + userSystem, err := nix.System() + if err != nil { + return err + } + + // Check if the system info is missing for the user's system. + sysInfo := d.lockfile.Packages[pkg.Raw].Systems[userSystem] + if sysInfo == nil { + d.lockfile.Packages[pkg.Raw] = newEntry + ux.Finfo(d.writer, "Updated system information for %s\n", pkg) + return nil + } + + // Check if the CAStorePath is missing for the user's system. + // Since any one user cannot add this field for all systems, + // we'll need to progressively add it to a project's lockfile. + if sysInfo.CAStorePath == "" { + // Update the CAStorePath for the user's system + d.lockfile.Packages[pkg.Raw].Systems[userSystem].CAStorePath = newEntry.Systems[userSystem].CAStorePath + ux.Finfo(d.writer, "Updated system information for %s\n", pkg) + return nil + } } + ux.Finfo(d.writer, "Already up-to-date %s %s\n", pkg, existing.Version) return nil } diff --git a/internal/lock/lockfile.go b/internal/lock/lockfile.go index 0fd7d8a43d1..3048c76491f 100644 --- a/internal/lock/lockfile.go +++ b/internal/lock/lockfile.go @@ -11,8 +11,6 @@ import ( "github.com/pkg/errors" "github.com/samber/lo" - "go.jetpack.io/devbox/internal/boxcli/featureflag" - "go.jetpack.io/devbox/internal/nix" "go.jetpack.io/devbox/internal/searcher" "go.jetpack.io/devbox/internal/cuecfg" @@ -40,12 +38,13 @@ type Package struct { } type SystemInfo struct { - System string // stored elsewhere in json: it's the key for the Package.Systems - FromHash string `json:"from_hash,omitempty"` - // StoreName may be different from the canonicalName so we store it separately - StoreName string `json:"store_name,omitempty"` - StoreVersion string `json:"store_version,omitempty"` - ToHash string `json:"to_hash,omitempty"` + // StorePath is the cache key in the Binary Cache Store (cache.nixos.org) + // It is of the form -- + // may be different from the canonicalName so we store the full store path. + StorePath string `json:"store_path,omitempty"` + // CAStorePath is the content-addressed path for the nix package in /nix/store + // It is of the form -- + CAStorePath string `json:"ca_store_path,omitempty"` } func GetFile(project devboxProject) (*File, error) { @@ -86,17 +85,7 @@ func (l *File) Remove(pkgs ...string) error { func (l *File) Resolve(pkg string) (*Package, error) { entry, hasEntry := l.Packages[pkg] - // If the package's system info is missing, we need to resolve it again. - needsSysInfo := false - if hasEntry && featureflag.RemoveNixpkgs.Enabled() { - userSystem, err := nix.System() - if err != nil { - return nil, err - } - needsSysInfo = entry.Systems[userSystem] == nil - } - - if !hasEntry || entry.Resolved == "" || needsSysInfo { + if !hasEntry || entry.Resolved == "" { locked := &Package{} var err error if _, _, versioned := searcher.ParseVersionedPackage(pkg); versioned { @@ -109,19 +98,6 @@ func (l *File) Resolve(pkg string) (*Package, error) { // whatever hash is in the devbox.json locked = &Package{Resolved: l.LegacyNixpkgsPath(pkg)} } - - // Merge the system info from the lockfile with the queried system info. - // This is necessary because a different system's info may previously have - // been added. For example: `aarch64-darwin` was already added, but - // current user is on `x86_64-linux`. - if hasEntry && featureflag.RemoveNixpkgs.Enabled() { - for _, sysInfo := range entry.Systems { - if _, ok := locked.Systems[sysInfo.System]; !ok { - locked.Systems[sysInfo.System] = sysInfo - } - } - } - l.Packages[pkg] = locked } diff --git a/internal/lock/resolve.go b/internal/lock/resolve.go index 3ec9ddeeb1d..955a608bffb 100644 --- a/internal/lock/resolve.go +++ b/internal/lock/resolve.go @@ -34,7 +34,10 @@ func (l *File) FetchResolvedPackage(pkg string) (*Package, error) { sysInfos := map[string]*SystemInfo{} if featureflag.RemoveNixpkgs.Enabled() { - sysInfos = buildLockSystemInfos(packageVersion) + sysInfos, err = buildLockSystemInfos(packageVersion) + if err != nil { + return nil, err + } } packageInfo, err := selectForSystem(packageVersion) if err != nil { @@ -76,15 +79,32 @@ func selectForSystem(pkg *searcher.PackageVersion) (searcher.PackageInfo, error) return maps.Values(pkg.Systems)[0], nil } -func buildLockSystemInfos(pkg *searcher.PackageVersion) map[string]*SystemInfo { +func buildLockSystemInfos(pkg *searcher.PackageVersion) (map[string]*SystemInfo, error) { + userSystem, err := nix.System() + if err != nil { + return nil, err + } + sysInfos := map[string]*SystemInfo{} for sysName, sysInfo := range pkg.Systems { + + // guard against missing search data + if sysInfo.StoreHash == "" || sysInfo.StoreName == "" { + continue + } + + storePath := nix.StorePath(sysInfo.StoreHash, sysInfo.StoreName, sysInfo.StoreVersion) + caStorePath := "" + if sysName == userSystem { + caStorePath, err = nix.ContentAddressedStorePath(storePath) + if err != nil { + return nil, err + } + } sysInfos[sysName] = &SystemInfo{ - System: sysName, - FromHash: sysInfo.StoreHash, - StoreName: sysInfo.StoreName, - StoreVersion: sysInfo.StoreVersion, + StorePath: storePath, + CAStorePath: caStorePath, } } - return sysInfos + return sysInfos, nil } diff --git a/internal/nix/nix.go b/internal/nix/nix.go index df3b46dfb53..31200516ea5 100644 --- a/internal/nix/nix.go +++ b/internal/nix/nix.go @@ -66,9 +66,6 @@ func (*Nix) PrintDevEnv(ctx context.Context, args *PrintDevEnvArgs) (*PrintDevEn args.FlakesFilePath, ) cmd.Args = append(cmd.Args, ExperimentalFlags()...) - if featureflag.RemoveNixpkgs.Enabled() { - cmd.Args = append(cmd.Args, "--impure") - } cmd.Args = append(cmd.Args, "--json") debug.Log("Running print-dev-env cmd: %s\n", cmd) data, err = cmd.Output() diff --git a/internal/nix/store.go b/internal/nix/store.go new file mode 100644 index 00000000000..66912fa1800 --- /dev/null +++ b/internal/nix/store.go @@ -0,0 +1,44 @@ +package nix + +import ( + "encoding/json" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +func StorePath(hash, name, version string) string { + storeDirParts := []string{hash, name} + if version != "" { + storeDirParts = append(storeDirParts, version) + } + storeDir := strings.Join(storeDirParts, "-") + return filepath.Join("/nix/store", storeDir) +} + +// ContentAddressedStorePath takes a store path and returns the content-addressed store path. +func ContentAddressedStorePath(storePath string) (string, error) { + cmd := command("store", "make-content-addressed", storePath, "--json") + out, err := cmd.Output() + if err != nil { + return "", errors.WithStack(err) + } + // Example Output: + // > nix store make-content-addressed /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 --json + // {"rewrites":{"/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1":"/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1"}} + + type ContentAddressed struct { + Rewrites map[string]string `json:"rewrites"` + } + caOutput := ContentAddressed{} + if err := json.Unmarshal(out, &caOutput); err != nil { + return "", errors.WithStack(err) + } + + caStorePath, ok := caOutput.Rewrites[storePath] + if !ok { + return "", errors.Errorf("could not find content-addressed store path for %s", storePath) + } + return caStorePath, nil +} diff --git a/internal/nix/store_test.go b/internal/nix/store_test.go new file mode 100644 index 00000000000..3d6cba6e04c --- /dev/null +++ b/internal/nix/store_test.go @@ -0,0 +1,32 @@ +package nix + +import ( + "fmt" + "testing" +) + +func TestContentAddressedPath(t *testing.T) { + + testCases := []struct { + storePath string + expected string + }{ + { + "/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1", + "/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1", + }, + } + + for index, testCase := range testCases { + t.Run(fmt.Sprintf("%d", index), func(t *testing.T) { + out, err := ContentAddressedStorePath(testCase.storePath) + if err != nil { + t.Errorf("got error: %v", err) + } + if out != testCase.expected { + t.Errorf("got %s, want %s", out, testCase.expected) + } + }) + + } +} diff --git a/internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl b/internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl index ebbddc4e2cf..c382ff77af7 100644 --- a/internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl +++ b/internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl @@ -26,6 +26,7 @@ (builtins.fetchClosure{ fromStore = "{{ $.BinaryCacheStore }}"; fromPath = "{{ .PathInBinaryStore }}"; + toPath = "{{ .ContentAddressedStorePath }}"; }) {{- end }} {{- end }}