From 85b2c95734e663eda3dbfe2ef331208539163e27 Mon Sep 17 00:00:00 2001 From: Savil Srivastava <676452+savil@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:55:39 -0700 Subject: [PATCH 1/6] [remove nixpkgs] add toPath, and edit devbox.lock only on update command --- devbox.json | 1 - devbox.lock | 5 --- internal/devpkg/package.go | 44 ++++++++++++++++--- internal/impl/update.go | 42 +++++++++++++++--- internal/lock/lockfile.go | 40 ++++------------- internal/lock/resolve.go | 34 +++++++++++--- internal/nix/nix.go | 3 -- internal/nix/store.go | 42 ++++++++++++++++++ internal/nix/store_test.go | 32 ++++++++++++++ .../tmpl/flake_remove_nixpkgs.nix.tmpl | 1 + 10 files changed, 185 insertions(+), 59 deletions(-) create mode 100644 internal/nix/store.go create mode 100644 internal/nix/store_test.go diff --git a/devbox.json b/devbox.json index 7b263cb4a99..ba04e5eeaf4 100644 --- a/devbox.json +++ b/devbox.json @@ -1,7 +1,6 @@ { "packages": [ "go@1.20", - "actionlint@1.6.23", "golangci-lint@1.52.2" ], "env": { diff --git a/devbox.lock b/devbox.lock index cd586e3c2f7..a52f1eda5b6 100644 --- a/devbox.lock +++ b/devbox.lock @@ -1,11 +1,6 @@ { "lockfile_version": "1", "packages": { - "actionlint@1.6.23": { - "last_modified": "2023-03-31T22:52:29Z", - "resolved": "github:NixOS/nixpkgs/242246ee1e58f54d2322227fc5eef53b4a616a31#actionlint", - "version": "1.6.23" - }, "go@1.20": { "last_modified": "2023-05-25T03:54:59Z", "resolved": "github:NixOS/nixpkgs/8d4d822bc0efa9de6eddc79cb0d82897a9baa750#go", diff --git a/internal/devpkg/package.go b/internal/devpkg/package.go index 8529e8274ec..300b76a77df 100644 --- a/internal/devpkg/package.go +++ b/internal/devpkg/package.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net/url" + "os" "path/filepath" "regexp" "strings" @@ -19,6 +20,7 @@ import ( "go.jetpack.io/devbox/internal/cuecfg" "go.jetpack.io/devbox/internal/lock" "go.jetpack.io/devbox/internal/nix" + "go.jetpack.io/devbox/internal/ux" ) // Package represents a "package" added to the devbox.json config. @@ -432,7 +434,7 @@ func (p *Package) IsInBinaryStore() (bool, error) { } // PathInBinaryStore is the key in the BinaryCacheStore for this package -// This is used as FromPath in builtins.fetchClosure +// This is used as BinaryStorePath in builtins.fetchClosure func (p *Package) PathInBinaryStore() (string, error) { if isInStore, err := p.IsInBinaryStore(); err != nil { return "", err @@ -452,10 +454,40 @@ func (p *Package) PathInBinaryStore() (string, error) { } sysInfo := entry.Systems[userSystem] - storeDirParts := []string{sysInfo.FromHash, sysInfo.StoreName} - if sysInfo.StoreVersion != "" { - storeDirParts = append(storeDirParts, sysInfo.StoreVersion) + return sysInfo.BinaryStorePath, nil +} + +func (p *Package) PathInLocalStore() (string, error) { + + if isInStore, err := p.IsInBinaryStore(); err != nil { + return "", err + } else if !isInStore { + return "", + errors.Errorf("Package %q cannot be fetched from binary cache store", p.Raw) + } + + entry, err := p.lockfile.Resolve(p.Raw) + if err != nil { + return "", err + } + + userSystem, err := nix.System() + if err != nil { + return "", err + } + + sysInfo := entry.Systems[userSystem] + if sysInfo.LocalStorePath != "" { + return sysInfo.LocalStorePath, nil + } + + ux.Fwarning( + os.Stderr, + "calculating local_store_path. This may be slow.\n" + + "Run `devbox update` to speed this up for next time.") + localPath, err := nix.ContentAddressedStorePath(sysInfo.BinaryStorePath) + if err != nil { + return "", err } - storeDir := strings.Join(storeDirParts, "-") - return filepath.Join("/nix/store", storeDir), nil + return localPath, err } diff --git a/internal/impl/update.go b/internal/impl/update.go index ab945c85d61..50e0fb47b0b 100644 --- a/internal/impl/update.go +++ b/internal/impl/update.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "go.jetpack.io/devbox/internal/boxcli/featureflag" "go.jetpack.io/devbox/internal/devpkg" "go.jetpack.io/devbox/internal/nix" "go.jetpack.io/devbox/internal/nix/nixprofile" @@ -90,7 +91,13 @@ func (d *Devbox) updateDevboxPackage( if err != nil { return err } - if existing != nil && existing.Version != newEntry.Version { + if existing == nil { + ux.Finfo(d.writer, "Resolved %s to %[1]s %[2]s\n", pkg, newEntry.Resolved) + d.lockfile.Packages[pkg.Raw] = newEntry + return nil + } + + if existing.Version != newEntry.Version { ux.Finfo(d.writer, "Updating %s %s -> %s\n", pkg, existing.Version, newEntry.Version) if err := d.removePackagesFromProfile(ctx, []string{pkg.Raw}); err != nil { // Warn but continue. TODO(landau): ensurePackagesAreInstalled should @@ -98,13 +105,38 @@ func (d *Devbox) updateDevboxPackage( ux.Fwarning(d.writer, "Failed to remove %s from profile: %s\n", pkg, err) } d.lockfile.Packages[pkg.Raw] = newEntry - } else if existing == nil { - ux.Finfo(d.writer, "Resolved %s to %[1]s %[2]s\n", pkg, newEntry.Resolved) + return nil + } + + // Check if the package's system info is missing, or not complete. + userSystem, err := nix.System() + if err != nil { + return err + } + var needsSysInfo bool + var needsLocalStorePath bool + if featureflag.RemoveNixpkgs.Enabled() { + userSysInfo := d.lockfile.Packages[pkg.Raw].Systems[userSystem] + needsSysInfo = userSysInfo == nil + if !needsSysInfo { + // Check if the LocalStorePath is missing for the user's system. + // Since any one user cannot add this field for all systems, + // we'll need to progressively add it to a project's lockfile. + needsLocalStorePath = userSysInfo.LocalStorePath == "" + } + } + if needsSysInfo { d.lockfile.Packages[pkg.Raw] = newEntry - } else { - ux.Finfo(d.writer, "Already up-to-date %s %s\n", pkg, existing.Version) + ux.Finfo(d.writer, "Updated system information for %s\n", pkg) + return nil + } else if needsLocalStorePath { + // Update the LocalStorePath for the user's system + d.lockfile.Packages[pkg.Raw].Systems[userSystem].LocalStorePath = newEntry.Systems[userSystem].LocalStorePath + ux.Finfo(d.writer, "Updated system information for %s\n", pkg) + return nil } + ux.Finfo(d.writer, "Already up-to-date %s %s\n", pkg, existing.Version) return nil } diff --git a/internal/lock/lockfile.go b/internal/lock/lockfile.go index 0fd7d8a43d1..b95a50d7b9c 100644 --- a/internal/lock/lockfile.go +++ b/internal/lock/lockfile.go @@ -11,8 +11,6 @@ import ( "github.com/pkg/errors" "github.com/samber/lo" - "go.jetpack.io/devbox/internal/boxcli/featureflag" - "go.jetpack.io/devbox/internal/nix" "go.jetpack.io/devbox/internal/searcher" "go.jetpack.io/devbox/internal/cuecfg" @@ -40,12 +38,13 @@ type Package struct { } type SystemInfo struct { - System string // stored elsewhere in json: it's the key for the Package.Systems - FromHash string `json:"from_hash,omitempty"` - // StoreName may be different from the canonicalName so we store it separately - StoreName string `json:"store_name,omitempty"` - StoreVersion string `json:"store_version,omitempty"` - ToHash string `json:"to_hash,omitempty"` + // BinaryStorePath is the cache key in the Binary Cache Store (cache.nixos.org) + // It is of the form -- + // may be different from the canonicalName so we store the full store path. + BinaryStorePath string `json:"bin_store_path,omitempty"` + // LocalStorePath is the content-addressed path for the nix package in /nix/store + // It is of the form -- + LocalStorePath string `json:"local_store_path,omitempty"` } func GetFile(project devboxProject) (*File, error) { @@ -86,17 +85,7 @@ func (l *File) Remove(pkgs ...string) error { func (l *File) Resolve(pkg string) (*Package, error) { entry, hasEntry := l.Packages[pkg] - // If the package's system info is missing, we need to resolve it again. - needsSysInfo := false - if hasEntry && featureflag.RemoveNixpkgs.Enabled() { - userSystem, err := nix.System() - if err != nil { - return nil, err - } - needsSysInfo = entry.Systems[userSystem] == nil - } - - if !hasEntry || entry.Resolved == "" || needsSysInfo { + if !hasEntry || entry.Resolved == "" { locked := &Package{} var err error if _, _, versioned := searcher.ParseVersionedPackage(pkg); versioned { @@ -109,19 +98,6 @@ func (l *File) Resolve(pkg string) (*Package, error) { // whatever hash is in the devbox.json locked = &Package{Resolved: l.LegacyNixpkgsPath(pkg)} } - - // Merge the system info from the lockfile with the queried system info. - // This is necessary because a different system's info may previously have - // been added. For example: `aarch64-darwin` was already added, but - // current user is on `x86_64-linux`. - if hasEntry && featureflag.RemoveNixpkgs.Enabled() { - for _, sysInfo := range entry.Systems { - if _, ok := locked.Systems[sysInfo.System]; !ok { - locked.Systems[sysInfo.System] = sysInfo - } - } - } - l.Packages[pkg] = locked } diff --git a/internal/lock/resolve.go b/internal/lock/resolve.go index 3ec9ddeeb1d..c86f06f5015 100644 --- a/internal/lock/resolve.go +++ b/internal/lock/resolve.go @@ -34,7 +34,10 @@ func (l *File) FetchResolvedPackage(pkg string) (*Package, error) { sysInfos := map[string]*SystemInfo{} if featureflag.RemoveNixpkgs.Enabled() { - sysInfos = buildLockSystemInfos(packageVersion) + sysInfos, err = buildLockSystemInfos(packageVersion) + if err != nil { + return nil, err + } } packageInfo, err := selectForSystem(packageVersion) if err != nil { @@ -76,15 +79,32 @@ func selectForSystem(pkg *searcher.PackageVersion) (searcher.PackageInfo, error) return maps.Values(pkg.Systems)[0], nil } -func buildLockSystemInfos(pkg *searcher.PackageVersion) map[string]*SystemInfo { +func buildLockSystemInfos(pkg *searcher.PackageVersion) (map[string]*SystemInfo, error) { + userSystem, err := nix.System() + if err != nil { + return nil, err + } + sysInfos := map[string]*SystemInfo{} for sysName, sysInfo := range pkg.Systems { + + // guard against missing search data + if sysInfo.StoreHash == "" || sysInfo.StoreName == "" { + continue + } + + localStorePath := "" + if sysName == userSystem { + binaryStorePath := nix.StorePath(sysInfo.StoreHash, sysInfo.StoreName, sysInfo.StoreVersion) + localStorePath, err = nix.ContentAddressedStorePath(binaryStorePath) + if err != nil { + return nil, err + } + } sysInfos[sysName] = &SystemInfo{ - System: sysName, - FromHash: sysInfo.StoreHash, - StoreName: sysInfo.StoreName, - StoreVersion: sysInfo.StoreVersion, + BinaryStorePath: nix.StorePath(sysInfo.StoreHash, sysInfo.StoreName, sysInfo.StoreVersion), + LocalStorePath: localStorePath, } } - return sysInfos + return sysInfos, nil } diff --git a/internal/nix/nix.go b/internal/nix/nix.go index df3b46dfb53..31200516ea5 100644 --- a/internal/nix/nix.go +++ b/internal/nix/nix.go @@ -66,9 +66,6 @@ func (*Nix) PrintDevEnv(ctx context.Context, args *PrintDevEnvArgs) (*PrintDevEn args.FlakesFilePath, ) cmd.Args = append(cmd.Args, ExperimentalFlags()...) - if featureflag.RemoveNixpkgs.Enabled() { - cmd.Args = append(cmd.Args, "--impure") - } cmd.Args = append(cmd.Args, "--json") debug.Log("Running print-dev-env cmd: %s\n", cmd) data, err = cmd.Output() diff --git a/internal/nix/store.go b/internal/nix/store.go new file mode 100644 index 00000000000..1ecda8b95f1 --- /dev/null +++ b/internal/nix/store.go @@ -0,0 +1,42 @@ +package nix + +import ( + "path/filepath" + "regexp" + "strings" + + "github.com/pkg/errors" +) + +func StorePath(hash, name, version string) string { + storeDirParts := []string{hash, name} + if version != "" { + storeDirParts = append(storeDirParts, version) + } + storeDir := strings.Join(storeDirParts, "-") + return filepath.Join("/nix/store", storeDir) +} + +// contentAddressedRegex matches the output of `nix store make-content-addressed`. +// It is used to select the content-addressed store path (the second one in the example below). +// +// Example: +// > nix store make-content-addressed /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 +// rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1' +var contentAddressedRegex = regexp.MustCompile(`rewrote\s'[\/a-z0-9-\.]+'\sto\s'([a-z0-9-\/\.]+)'`) + +// ContentAddressedStorePath takes a store path and returns the content-addressed store path. +func ContentAddressedStorePath(storePath string) (string, error) { + cmd := command("store", "make-content-addressed", storePath) + out, err := cmd.CombinedOutput() + if err != nil { + return "", errors.WithStack(err) + } + + matches := contentAddressedRegex.FindStringSubmatch(string(out)) + if len(matches) < 2 { + return "", errors.Errorf("could not parse output of nix store make-content-addressed: %s", string(out)) + } + + return matches[1], nil +} diff --git a/internal/nix/store_test.go b/internal/nix/store_test.go new file mode 100644 index 00000000000..3d6cba6e04c --- /dev/null +++ b/internal/nix/store_test.go @@ -0,0 +1,32 @@ +package nix + +import ( + "fmt" + "testing" +) + +func TestContentAddressedPath(t *testing.T) { + + testCases := []struct { + storePath string + expected string + }{ + { + "/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1", + "/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1", + }, + } + + for index, testCase := range testCases { + t.Run(fmt.Sprintf("%d", index), func(t *testing.T) { + out, err := ContentAddressedStorePath(testCase.storePath) + if err != nil { + t.Errorf("got error: %v", err) + } + if out != testCase.expected { + t.Errorf("got %s, want %s", out, testCase.expected) + } + }) + + } +} diff --git a/internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl b/internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl index ebbddc4e2cf..1121512e4eb 100644 --- a/internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl +++ b/internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl @@ -26,6 +26,7 @@ (builtins.fetchClosure{ fromStore = "{{ $.BinaryCacheStore }}"; fromPath = "{{ .PathInBinaryStore }}"; + toPath = "{{ .PathInLocalStore }}"; }) {{- end }} {{- end }} From a735e64fc08e560db32e4f202e2b686ef6868f9f Mon Sep 17 00:00:00 2001 From: savil <676452+savil@users.noreply.github.com> Date: Fri, 7 Jul 2023 04:50:05 +0000 Subject: [PATCH 2/6] lint fix --- internal/devpkg/package.go | 7 ++++--- internal/lock/resolve.go | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/devpkg/package.go b/internal/devpkg/package.go index 300b76a77df..bd9e378ffa7 100644 --- a/internal/devpkg/package.go +++ b/internal/devpkg/package.go @@ -482,9 +482,10 @@ func (p *Package) PathInLocalStore() (string, error) { } ux.Fwarning( - os.Stderr, - "calculating local_store_path. This may be slow.\n" + - "Run `devbox update` to speed this up for next time.") + os.Stderr, + "calculating local_store_path. This may be slow.\n"+ + "Run `devbox update` to speed this up for next time.", + ) localPath, err := nix.ContentAddressedStorePath(sysInfo.BinaryStorePath) if err != nil { return "", err diff --git a/internal/lock/resolve.go b/internal/lock/resolve.go index c86f06f5015..91813a7762c 100644 --- a/internal/lock/resolve.go +++ b/internal/lock/resolve.go @@ -87,7 +87,7 @@ func buildLockSystemInfos(pkg *searcher.PackageVersion) (map[string]*SystemInfo, sysInfos := map[string]*SystemInfo{} for sysName, sysInfo := range pkg.Systems { - + // guard against missing search data if sysInfo.StoreHash == "" || sysInfo.StoreName == "" { continue From aeaa38e20902c7c6ab9ff6619b04d2b10a87475e Mon Sep 17 00:00:00 2001 From: savil <676452+savil@users.noreply.github.com> Date: Fri, 7 Jul 2023 04:57:20 +0000 Subject: [PATCH 3/6] lucille: rewrite for clarity --- internal/impl/update.go | 44 ++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/internal/impl/update.go b/internal/impl/update.go index 50e0fb47b0b..810694294bd 100644 --- a/internal/impl/update.go +++ b/internal/impl/update.go @@ -109,31 +109,29 @@ func (d *Devbox) updateDevboxPackage( } // Check if the package's system info is missing, or not complete. - userSystem, err := nix.System() - if err != nil { - return err - } - var needsSysInfo bool - var needsLocalStorePath bool if featureflag.RemoveNixpkgs.Enabled() { - userSysInfo := d.lockfile.Packages[pkg.Raw].Systems[userSystem] - needsSysInfo = userSysInfo == nil - if !needsSysInfo { - // Check if the LocalStorePath is missing for the user's system. - // Since any one user cannot add this field for all systems, - // we'll need to progressively add it to a project's lockfile. - needsLocalStorePath = userSysInfo.LocalStorePath == "" + userSystem, err := nix.System() + if err != nil { + return err + } + + // Check if the system info is missing for the user's system. + sysInfo := d.lockfile.Packages[pkg.Raw].Systems[userSystem] + if sysInfo == nil { + d.lockfile.Packages[pkg.Raw] = newEntry + ux.Finfo(d.writer, "Updated system information for %s\n", pkg) + return nil + } + + // Check if the LocalStorePath is missing for the user's system. + // Since any one user cannot add this field for all systems, + // we'll need to progressively add it to a project's lockfile. + if sysInfo.LocalStorePath == "" { + // Update the LocalStorePath for the user's system + d.lockfile.Packages[pkg.Raw].Systems[userSystem].LocalStorePath = newEntry.Systems[userSystem].LocalStorePath + ux.Finfo(d.writer, "Updated system information for %s\n", pkg) + return nil } - } - if needsSysInfo { - d.lockfile.Packages[pkg.Raw] = newEntry - ux.Finfo(d.writer, "Updated system information for %s\n", pkg) - return nil - } else if needsLocalStorePath { - // Update the LocalStorePath for the user's system - d.lockfile.Packages[pkg.Raw].Systems[userSystem].LocalStorePath = newEntry.Systems[userSystem].LocalStorePath - ux.Finfo(d.writer, "Updated system information for %s\n", pkg) - return nil } ux.Finfo(d.writer, "Already up-to-date %s %s\n", pkg, existing.Version) From e00b5c62ad53657dd47f42b8240f2a70225be76f Mon Sep 17 00:00:00 2001 From: Savil Srivastava <676452+savil@users.noreply.github.com> Date: Thu, 6 Jul 2023 22:21:47 -0700 Subject: [PATCH 4/6] gcurtis: renames, as requested. Some TODOs added to maintain scope --- internal/devpkg/package.go | 15 +++++++++------ internal/impl/update.go | 8 ++++---- internal/lock/lockfile.go | 8 ++++---- internal/lock/resolve.go | 10 +++++----- .../shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl | 2 +- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/internal/devpkg/package.go b/internal/devpkg/package.go index bd9e378ffa7..946964c4f31 100644 --- a/internal/devpkg/package.go +++ b/internal/devpkg/package.go @@ -407,8 +407,10 @@ func (p *Package) HashFromNixPkgsURL() string { // BinaryCacheStore is the store from which to fetch this package's binaries. // It is used as FromStore in builtins.fetchClosure. +// TODO savil: rename to BinaryCache const BinaryCacheStore = "https://cache.nixos.org" +// TODO savil: rename to IsInBinaryCache func (p *Package) IsInBinaryStore() (bool, error) { if !p.isVersioned() { return false, nil @@ -434,7 +436,8 @@ func (p *Package) IsInBinaryStore() (bool, error) { } // PathInBinaryStore is the key in the BinaryCacheStore for this package -// This is used as BinaryStorePath in builtins.fetchClosure +// This is used as StorePath in builtins.fetchClosure +// TODO savil: rename to PathInBinaryCache func (p *Package) PathInBinaryStore() (string, error) { if isInStore, err := p.IsInBinaryStore(); err != nil { return "", err @@ -454,10 +457,10 @@ func (p *Package) PathInBinaryStore() (string, error) { } sysInfo := entry.Systems[userSystem] - return sysInfo.BinaryStorePath, nil + return sysInfo.StorePath, nil } -func (p *Package) PathInLocalStore() (string, error) { +func (p *Package) ContentAddressedStorePath() (string, error) { if isInStore, err := p.IsInBinaryStore(); err != nil { return "", err @@ -477,8 +480,8 @@ func (p *Package) PathInLocalStore() (string, error) { } sysInfo := entry.Systems[userSystem] - if sysInfo.LocalStorePath != "" { - return sysInfo.LocalStorePath, nil + if sysInfo.CAStorePath != "" { + return sysInfo.CAStorePath, nil } ux.Fwarning( @@ -486,7 +489,7 @@ func (p *Package) PathInLocalStore() (string, error) { "calculating local_store_path. This may be slow.\n"+ "Run `devbox update` to speed this up for next time.", ) - localPath, err := nix.ContentAddressedStorePath(sysInfo.BinaryStorePath) + localPath, err := nix.ContentAddressedStorePath(sysInfo.StorePath) if err != nil { return "", err } diff --git a/internal/impl/update.go b/internal/impl/update.go index 810694294bd..cd9e08b0970 100644 --- a/internal/impl/update.go +++ b/internal/impl/update.go @@ -123,12 +123,12 @@ func (d *Devbox) updateDevboxPackage( return nil } - // Check if the LocalStorePath is missing for the user's system. + // Check if the CAStorePath is missing for the user's system. // Since any one user cannot add this field for all systems, // we'll need to progressively add it to a project's lockfile. - if sysInfo.LocalStorePath == "" { - // Update the LocalStorePath for the user's system - d.lockfile.Packages[pkg.Raw].Systems[userSystem].LocalStorePath = newEntry.Systems[userSystem].LocalStorePath + if sysInfo.CAStorePath == "" { + // Update the CAStorePath for the user's system + d.lockfile.Packages[pkg.Raw].Systems[userSystem].CAStorePath = newEntry.Systems[userSystem].CAStorePath ux.Finfo(d.writer, "Updated system information for %s\n", pkg) return nil } diff --git a/internal/lock/lockfile.go b/internal/lock/lockfile.go index b95a50d7b9c..11ebde51715 100644 --- a/internal/lock/lockfile.go +++ b/internal/lock/lockfile.go @@ -38,13 +38,13 @@ type Package struct { } type SystemInfo struct { - // BinaryStorePath is the cache key in the Binary Cache Store (cache.nixos.org) + // StorePath is the cache key in the Binary Cache Store (cache.nixos.org) // It is of the form -- // may be different from the canonicalName so we store the full store path. - BinaryStorePath string `json:"bin_store_path,omitempty"` - // LocalStorePath is the content-addressed path for the nix package in /nix/store + StorePath string `json:"store_path,omitempty"` + // CAStorePath is the content-addressed path for the nix package in /nix/store // It is of the form -- - LocalStorePath string `json:"local_store_path,omitempty"` + CAStorePath string `json:"local_store_path,omitempty"` } func GetFile(project devboxProject) (*File, error) { diff --git a/internal/lock/resolve.go b/internal/lock/resolve.go index 91813a7762c..955a608bffb 100644 --- a/internal/lock/resolve.go +++ b/internal/lock/resolve.go @@ -93,17 +93,17 @@ func buildLockSystemInfos(pkg *searcher.PackageVersion) (map[string]*SystemInfo, continue } - localStorePath := "" + storePath := nix.StorePath(sysInfo.StoreHash, sysInfo.StoreName, sysInfo.StoreVersion) + caStorePath := "" if sysName == userSystem { - binaryStorePath := nix.StorePath(sysInfo.StoreHash, sysInfo.StoreName, sysInfo.StoreVersion) - localStorePath, err = nix.ContentAddressedStorePath(binaryStorePath) + caStorePath, err = nix.ContentAddressedStorePath(storePath) if err != nil { return nil, err } } sysInfos[sysName] = &SystemInfo{ - BinaryStorePath: nix.StorePath(sysInfo.StoreHash, sysInfo.StoreName, sysInfo.StoreVersion), - LocalStorePath: localStorePath, + StorePath: storePath, + CAStorePath: caStorePath, } } return sysInfos, nil diff --git a/internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl b/internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl index 1121512e4eb..c382ff77af7 100644 --- a/internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl +++ b/internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl @@ -26,7 +26,7 @@ (builtins.fetchClosure{ fromStore = "{{ $.BinaryCacheStore }}"; fromPath = "{{ .PathInBinaryStore }}"; - toPath = "{{ .PathInLocalStore }}"; + toPath = "{{ .ContentAddressedStorePath }}"; }) {{- end }} {{- end }} From bfd8e5905c0394587133f6aa82393d114fed0f16 Mon Sep 17 00:00:00 2001 From: Savil Srivastava <676452+savil@users.noreply.github.com> Date: Thu, 6 Jul 2023 22:28:41 -0700 Subject: [PATCH 5/6] use --json for nix store make-content-addressed --- internal/nix/store.go | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/internal/nix/store.go b/internal/nix/store.go index 1ecda8b95f1..66912fa1800 100644 --- a/internal/nix/store.go +++ b/internal/nix/store.go @@ -1,8 +1,8 @@ package nix import ( + "encoding/json" "path/filepath" - "regexp" "strings" "github.com/pkg/errors" @@ -17,26 +17,28 @@ func StorePath(hash, name, version string) string { return filepath.Join("/nix/store", storeDir) } -// contentAddressedRegex matches the output of `nix store make-content-addressed`. -// It is used to select the content-addressed store path (the second one in the example below). -// -// Example: -// > nix store make-content-addressed /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 -// rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1' -var contentAddressedRegex = regexp.MustCompile(`rewrote\s'[\/a-z0-9-\.]+'\sto\s'([a-z0-9-\/\.]+)'`) - // ContentAddressedStorePath takes a store path and returns the content-addressed store path. func ContentAddressedStorePath(storePath string) (string, error) { - cmd := command("store", "make-content-addressed", storePath) - out, err := cmd.CombinedOutput() + cmd := command("store", "make-content-addressed", storePath, "--json") + out, err := cmd.Output() if err != nil { return "", errors.WithStack(err) } + // Example Output: + // > nix store make-content-addressed /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 --json + // {"rewrites":{"/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1":"/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1"}} - matches := contentAddressedRegex.FindStringSubmatch(string(out)) - if len(matches) < 2 { - return "", errors.Errorf("could not parse output of nix store make-content-addressed: %s", string(out)) + type ContentAddressed struct { + Rewrites map[string]string `json:"rewrites"` + } + caOutput := ContentAddressed{} + if err := json.Unmarshal(out, &caOutput); err != nil { + return "", errors.WithStack(err) } - return matches[1], nil + caStorePath, ok := caOutput.Rewrites[storePath] + if !ok { + return "", errors.Errorf("could not find content-addressed store path for %s", storePath) + } + return caStorePath, nil } From 39d01b368c91abe3c0a10e27ac54564673111b0d Mon Sep 17 00:00:00 2001 From: Savil Srivastava <676452+savil@users.noreply.github.com> Date: Thu, 6 Jul 2023 23:03:25 -0700 Subject: [PATCH 6/6] rename lockfile key to ca_store_path --- internal/lock/lockfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/lock/lockfile.go b/internal/lock/lockfile.go index 11ebde51715..3048c76491f 100644 --- a/internal/lock/lockfile.go +++ b/internal/lock/lockfile.go @@ -44,7 +44,7 @@ type SystemInfo struct { StorePath string `json:"store_path,omitempty"` // CAStorePath is the content-addressed path for the nix package in /nix/store // It is of the form -- - CAStorePath string `json:"local_store_path,omitempty"` + CAStorePath string `json:"ca_store_path,omitempty"` } func GetFile(project devboxProject) (*File, error) {