diff --git a/examples/flakes/remote/devbox.json b/examples/flakes/remote/devbox.json new file mode 100644 index 00000000000..ad908e947f2 --- /dev/null +++ b/examples/flakes/remote/devbox.json @@ -0,0 +1,13 @@ +{ + "packages": [ + "github:nixos/nixpkgs/5233fd2ba76a3accb5aaa999c00509a11fd0793c#hello", + "github:nix-community/fenix#stable.toolchain", + "github:F1bonacc1/process-compose" + ], + "shell": { + "init_hook": null + }, + "nixpkgs": { + "commit": "f80ac848e3d6f0c12c52758c0f25c10c97ca3b62" + } +} \ No newline at end of file diff --git a/internal/impl/devbox.go b/internal/impl/devbox.go index a1f29e94771..82c3a712c31 100644 --- a/internal/impl/devbox.go +++ b/internal/impl/devbox.go @@ -295,8 +295,8 @@ func (d *Devbox) ShellEnvHash(ctx context.Context) (string, error) { } func (d *Devbox) Info(pkg string, markdown bool) error { - info, hasInfo := nix.PkgInfo(d.cfg.Nixpkgs.Commit, pkg) - if !hasInfo { + info := nix.PkgInfo(d.cfg.Nixpkgs.Commit, pkg) + if info == nil { _, err := fmt.Fprintf(d.writer, "Package %s not found\n", pkg) return errors.WithStack(err) } diff --git a/internal/impl/devbox_test.go b/internal/impl/devbox_test.go index d4f2b2df1fb..8c8ef6b97da 100644 --- a/internal/impl/devbox_test.go +++ b/internal/impl/devbox_test.go @@ -10,7 +10,6 @@ import ( "github.com/bmatcuk/doublestar/v4" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "go.jetpack.io/devbox/internal/planner/plansdk" ) @@ -29,9 +28,6 @@ func TestDevbox(t *testing.T) { } } func testShellPlan(t *testing.T, testPath string) { - currentDir, err := os.Getwd() - require.New(t).NoError(err) - baseDir := filepath.Dir(testPath) testName := fmt.Sprintf("%s_shell_plan", filepath.Base(baseDir)) t.Run(testName, func(t *testing.T) { @@ -43,12 +39,6 @@ func testShellPlan(t *testing.T, testPath string) { box, err := Open(baseDir, os.Stdout) assert.NoErrorf(err, "%s should be a valid devbox project", baseDir) - // Just for tests, we make projectDir be a relative path so that the paths in plan.json - // of various test cases have relative paths. Absolute paths are a no-go because they'd - // be of the form `/Users/savil/...`, which are not generalized and cannot be checked in. - box.projectDir, err = filepath.Rel(currentDir, box.projectDir) - assert.NoErrorf(err, "expect to construct relative path from %s relative to base %s", box.projectDir, currentDir) - shellPlan, err := box.ShellPlan() assert.NoError(err, "devbox shell plan should not fail") diff --git a/internal/impl/flakes.go b/internal/impl/flakes.go index 7fd7263850f..3e68da27e13 100644 --- a/internal/impl/flakes.go +++ b/internal/impl/flakes.go @@ -6,20 +6,24 @@ import ( "go.jetpack.io/devbox/internal/planner/plansdk" ) -func (d *Devbox) flakeInputs() []plansdk.FlakeInput { - inputs := map[string]plansdk.FlakeInput{} +func (d *Devbox) flakeInputs() []*plansdk.FlakeInput { + inputs := map[string]*plansdk.FlakeInput{} for _, p := range d.cfg.MergedPackages(d.writer) { pkg := nix.InputFromString(p, d.projectDir) if pkg.IsFlake() { - if input, ok := inputs[pkg.Name()]; !ok { - inputs[pkg.Name()] = plansdk.FlakeInput{ + AttributePath, err := pkg.PackageAttributePath() + if err != nil { + panic(err) + } + if input, ok := inputs[pkg.URLWithoutFragment()]; !ok { + inputs[pkg.URLWithoutFragment()] = &plansdk.FlakeInput{ Name: pkg.Name(), URL: pkg.URLWithoutFragment(), - Packages: pkg.Packages(), + Packages: []string{AttributePath}, } } else { input.Packages = lo.Uniq( - append(inputs[pkg.Name()].Packages, pkg.Packages()...), + append(inputs[pkg.URLWithoutFragment()].Packages, AttributePath), ) } } diff --git a/internal/impl/tmpl/flake.nix.tmpl b/internal/impl/tmpl/flake.nix.tmpl index 6714f77e5b8..442858ab491 100644 --- a/internal/impl/tmpl/flake.nix.tmpl +++ b/internal/impl/tmpl/flake.nix.tmpl @@ -49,8 +49,8 @@ global-pkgs.{{.}} {{- end}} {{- range $_, $flake := .FlakeInputs }} - {{- range $flake.Packages}} - {{ $flake.Name }}.packages.${system}.{{.}} + {{- range $flake.Packages }} + {{ $flake.Name }}.{{.}} {{- end }} {{- end }} ]; diff --git a/internal/nix/input.go b/internal/nix/input.go index 1ee682005ed..4cbc1d6e107 100644 --- a/internal/nix/input.go +++ b/internal/nix/input.go @@ -3,58 +3,116 @@ package nix import ( "crypto/md5" "encoding/hex" - "encoding/json" + "fmt" "net/url" - "os/exec" "path/filepath" + "regexp" "strings" "github.com/samber/lo" "go.jetpack.io/devbox/internal/boxcli/usererr" ) -type Input url.URL +type Input struct { + url.URL +} func InputFromString(s, projectDir string) *Input { u, _ := url.Parse(s) - if u.Path == "" && u.Opaque != "" { + if u.Path == "" && u.Opaque != "" && u.Scheme == "path" { u.Path = filepath.Join(projectDir, u.Opaque) u.Opaque = "" } - return lo.ToPtr(Input(*u)) -} - -func (i *Input) String() string { - return (*url.URL)(i).String() + return &Input{*u} } // isFlake returns true if the package descriptor has a scheme. For now // we only support the "path" scheme. func (i *Input) IsFlake() bool { - // Technically flakes allows omitting the scheme for absolute paths, but + return i.IsLocal() || i.IsGithub() +} + +func (i *Input) IsLocal() bool { + // Technically flakes allows omitting the scheme for local absolute paths, but // we don't support that (yet). return i.Scheme == "path" } +func (i *Input) IsGithub() bool { + return i.Scheme == "github" +} + +var inputNameRegex = regexp.MustCompile("[^a-zA-Z0-9-]+") + func (i *Input) Name() string { - return filepath.Base(i.Path) + "-" + i.hash() + result := "" + if i.IsLocal() { + result = filepath.Base(i.Path) + "-" + i.hash() + } else if i.IsGithub() { + result = "gh-" + strings.Join(strings.Split(i.Opaque, "/"), "-") + } else { + result = i.String() + "-" + i.hash() + } + return inputNameRegex.ReplaceAllString(result, "-") } func (i *Input) URLWithoutFragment() string { - u := *(*url.URL)(i) // get copy + u := i.URL // get copy u.Fragment = "" // This will produce urls with extra slashes after the scheme, but that's ok return u.String() } -func (i *Input) Packages() []string { +func (i *Input) NormalizedName() (string, error) { + attrPath, err := i.PackageAttributePath() + if err != nil { + return "", err + } + return i.URLWithoutFragment() + "#" + attrPath, nil +} + +// PackageAttributePath returns just the name for non-flakes. For flake +// references is returns the full path to the package in the flake. e.g. +// packages.x86_64-linux.hello +func (i *Input) PackageAttributePath() (string, error) { if !i.IsFlake() { - return []string{i.String()} + return i.String(), nil + } + infos := search(i.String()) + + if len(infos) == 1 { + return lo.Keys(infos)[0], nil + } + + // If ambiguous, try to find a default output + if len(infos) > 1 && i.Fragment == "" { + for key := range infos { + if strings.HasSuffix(key, ".default") { + return key, nil + } + } + for key := range infos { + if strings.HasPrefix(key, "defaultPackage.") { + return key, nil + } + } } - if i.Fragment == "" { - return []string{"default"} + + // Still ambiguous, return error + if len(infos) > 1 { + outputs := fmt.Sprintf("It has %d possible outputs", len(infos)) + if len(infos) < 10 { + outputs = "It has the following possible outputs: \n" + + strings.Join(lo.Keys(infos), ", ") + } + return "", usererr.New( + "Flake \"%s\" is ambiguous. %s", + i.String(), + outputs, + ) } - return strings.Split(i.Fragment, ",") + + return "", usererr.New("Flake \"%s\" was not found", i.String()) } func (i *Input) hash() string { @@ -66,94 +124,27 @@ func (i *Input) hash() string { } func (i *Input) validateExists() (bool, error) { - system, err := currentSystem() - if err != nil { - return false, err - } - - outputs, err := outputs(i.Path) - if err != nil { - return false, err - } - - fragment := i.Fragment - if fragment == "" { - fragment = "default" // if no package is specified, check for default. - } - - if _, exists := outputs.Packages[system][fragment]; exists { - return true, nil - } - - if _, exists := outputs.LegacyPackages[system][fragment]; exists { - return true, nil - } - - // Another way to specify a default package is to output it as - // defaultPackage.${system} - if _, exists := outputs.DefaultPackage[system]; exists && i.Fragment == "" { - return true, nil - } - - return false, usererr.New( - "Flake \"%s\" was found but package \"%s\" was not found in flake. "+ - "Ensure the flake has a packages output", - i.Path, - fragment, - ) + info, err := i.PackageAttributePath() + return info != "", err } -func (i *Input) equals(o *Input) bool { - if i.String() == o.String() { +func (i *Input) equals(other *Input) bool { + if i.String() == other.String() { return true } - return i.Scheme == o.Scheme && - i.Path == o.Path && - i.Opaque == o.Opaque && - i.normalizedFragment() == o.normalizedFragment() -} -// normalizedFragment attempts to return the closest thing to a package name -// from a fragment. A fragment could be: -// * empty string -> default -// * a single package -> package -// * a qualified output (e.g. packages.aarch64-darwin.hello) -> hello -func (i *Input) normalizedFragment() string { - if i.Fragment == "" { - return "default" + // check inputs without fragments as optimization. Next step is expensive + if i.URLWithoutFragment() != other.URLWithoutFragment() { + return false } - parts := strings.Split(i.Fragment, ".") - return parts[len(parts)-1] -} - -func currentSystem() (string, error) { - cmd := exec.Command( - "nix", "eval", - "--impure", "--raw", "--expr", - "builtins.currentSystem", - ) - cmd.Args = append(cmd.Args, ExperimentalFlags()...) - o, err := cmd.Output() - return string(o), err -} - -type output struct { - LegacyPackages map[string]map[string]any `json:"legacyPackages"` - Packages map[string]map[string]any `json:"packages"` - DefaultPackage map[string]map[string]any `json:"defaultPackage"` -} -func outputs(path string) (*output, error) { - cmd := exec.Command( - "nix", "flake", "show", - path, - "--json", "--legacy", - ) - cmd.Args = append(cmd.Args, ExperimentalFlags()...) - commandOut, err := cmd.Output() + name, err := i.PackageAttributePath() + if err != nil { + return false + } + otherName, err := other.PackageAttributePath() if err != nil { - return nil, err + return false } - out := &output{} - return out, json.Unmarshal(commandOut, out) + return name == otherName } diff --git a/internal/nix/input_test.go b/internal/nix/input_test.go index ab0acda6ac4..c554303fcc7 100644 --- a/internal/nix/input_test.go +++ b/internal/nix/input_test.go @@ -1,9 +1,12 @@ package nix import ( + "fmt" "path/filepath" "reflect" "testing" + + "github.com/samber/lo" ) type inputTestCase struct { @@ -11,7 +14,7 @@ type inputTestCase struct { isFlake bool name string urlWithoutFragment string - packages []string + packageName string } func TestInput(t *testing.T) { @@ -22,40 +25,54 @@ func TestInput(t *testing.T) { isFlake: true, name: "my-flake-c7758d", urlWithoutFragment: "path://" + filepath.Join(projectDir, "path/to/my-flake"), - packages: []string{"my-package"}, + packageName: "packages.x86_64-darwin.my-package", }, { pkg: "path:.#my-package", isFlake: true, name: "my-project-744eaa", urlWithoutFragment: "path://" + projectDir, - packages: []string{"my-package"}, + packageName: "packages.x86_64-darwin.my-package", }, { pkg: "path:/tmp/my-project/path/to/my-flake#my-package", isFlake: true, name: "my-flake-773986", urlWithoutFragment: "path:" + filepath.Join(projectDir, "path/to/my-flake"), - packages: []string{"my-package"}, + packageName: "packages.x86_64-darwin.my-package", }, { pkg: "path:/tmp/my-project/path/to/my-flake", isFlake: true, name: "my-flake-eaedce", urlWithoutFragment: "path:" + filepath.Join(projectDir, "path/to/my-flake"), - packages: []string{"default"}, + packageName: "packages.x86_64-darwin.default", }, { pkg: "hello", isFlake: false, name: "hello-5d4140", urlWithoutFragment: "hello", - packages: []string{"hello"}, + packageName: "hello", + }, + { + pkg: "github:nixos/nixpkgs/5233fd2ba76a3accb5aaa999c00509a11fd0793c#hello", + isFlake: true, + name: "gh-nixos-nixpkgs-5233fd2ba76a3accb5aaa999c00509a11fd0793c", + urlWithoutFragment: "github:nixos/nixpkgs/5233fd2ba76a3accb5aaa999c00509a11fd0793c", + packageName: "packages.x86_64-darwin.hello", + }, + { + pkg: "github:F1bonacc1/process-compose", + isFlake: true, + name: "gh-F1bonacc1-process-compose", + urlWithoutFragment: "github:F1bonacc1/process-compose", + packageName: "packages.x86_64-darwin.default", }, } for _, testCase := range cases { - i := InputFromString(testCase.pkg, projectDir) + i := testInputFromString(testCase.pkg, projectDir) if isFLake := i.IsFlake(); testCase.isFlake != isFLake { t.Errorf("IsFlake() = %v, want %v", isFLake, testCase.isFlake) } @@ -65,8 +82,26 @@ func TestInput(t *testing.T) { if urlWithoutFragment := i.URLWithoutFragment(); testCase.urlWithoutFragment != urlWithoutFragment { t.Errorf("URLWithoutFragment() = %v, want %v", urlWithoutFragment, testCase.urlWithoutFragment) } - if packages := i.Packages(); !reflect.DeepEqual(testCase.packages, packages) { - t.Errorf("Packages() = %v, want %v", packages, testCase.packages) + if packages := i.Package(); !reflect.DeepEqual(testCase.packageName, packages) { + t.Errorf("Package() = %v, want %v", packages, testCase.packageName) } } } + +type testInput struct { + Input +} + +func testInputFromString(s, projectDir string) *testInput { + return lo.ToPtr(testInput{Input: *InputFromString(s, projectDir)}) +} + +func (i *testInput) Package() string { + if i.IsFlake() { + return fmt.Sprintf( + "packages.x86_64-darwin.%s", + lo.Ternary(i.Fragment != "", i.Fragment, "default"), + ) + } + return i.String() +} diff --git a/internal/nix/nix.go b/internal/nix/nix.go index bf4e1cd453b..ea916757065 100644 --- a/internal/nix/nix.go +++ b/internal/nix/nix.go @@ -13,6 +13,7 @@ import ( "runtime/trace" "github.com/pkg/errors" + "github.com/samber/lo" "go.jetpack.io/devbox/internal/debug" ) @@ -29,62 +30,64 @@ func PkgExists(nixpkgsCommit, pkg, projectDir string) (bool, error) { if input.IsFlake() { return input.validateExists() } - _, found := PkgInfo(nixpkgsCommit, pkg) - return found, nil + return PkgInfo(nixpkgsCommit, pkg) != nil, nil } type Info struct { // attribute key is different in flakes vs legacy so we should only use it // if we know exactly which version we are using attributeKey string - NixName string - Name string + PName string Version string } func (i *Info) String() string { - return fmt.Sprintf("%s-%s", i.Name, i.Version) + return fmt.Sprintf("%s-%s", i.PName, i.Version) } -func PkgInfo(nixpkgsCommit, pkg string) (*Info, bool) { +func PkgInfo(nixpkgsCommit, pkg string) *Info { exactPackage := fmt.Sprintf("%s#%s", FlakeNixpkgs(nixpkgsCommit), pkg) if nixpkgsCommit == "" { exactPackage = fmt.Sprintf("nixpkgs#%s", pkg) } - cmd := exec.Command("nix", "search", "--json", exactPackage) + results := search(exactPackage) + if len(results) == 0 { + return nil + } + // we should only have one result + return lo.Values(results)[0] +} + +func search(url string) map[string]*Info { + cmd := exec.Command("nix", "search", "--json", url) cmd.Args = append(cmd.Args, ExperimentalFlags()...) cmd.Stderr = os.Stderr debug.Log("running command: %s\n", cmd) out, err := cmd.Output() if err != nil { // for now, assume all errors are invalid packages. - return nil, false /* not found */ + return nil } - pkgInfo := parseInfo(pkg, out) - if pkgInfo == nil { - return nil, false /* not found */ - } - return pkgInfo, true /* found */ + return parseSearchResults(out) } -func parseInfo(pkg string, data []byte) *Info { +func parseSearchResults(data []byte) map[string]*Info { var results map[string]map[string]any err := json.Unmarshal(data, &results) if err != nil { panic(err) } + infos := map[string]*Info{} for key, result := range results { - pkgInfo := &Info{ + infos[key] = &Info{ attributeKey: key, - NixName: pkg, - Name: result["pname"].(string), + PName: result["pname"].(string), Version: result["version"].(string), } - return pkgInfo } - return nil + return infos } type printDevEnvOut struct { diff --git a/internal/nix/profile.go b/internal/nix/profile.go index ddb4b34b977..1ab2b621cec 100644 --- a/internal/nix/profile.go +++ b/internal/nix/profile.go @@ -250,6 +250,11 @@ func ProfileInstall(args *ProfileInstallArgs) error { fmt.Fprintf(args.Writer, "%s\n", stepMsg) } + path, err := flakePath(args) + if err != nil { + return err + } + cmd := exec.Command( "nix", "profile", "install", "--profile", args.ProfilePath, @@ -258,7 +263,7 @@ func ProfileInstall(args *ProfileInstallArgs) error { // Note that this is not really the priority we care about, since we // use the flake.nix to specify the priority. "--priority", nextPriority(args.ProfilePath), - flakePath(args), + path, ) cmd.Env = AllowUnfreeEnv() cmd.Args = append(cmd.Args, ExperimentalFlags()...) @@ -283,8 +288,8 @@ func ProfileInstall(args *ProfileInstallArgs) error { } func ProfileRemove(profilePath, nixpkgsCommit, pkg string) error { - info, found := PkgInfo(nixpkgsCommit, pkg) - if !found { + info := PkgInfo(nixpkgsCommit, pkg) + if info == nil { return ErrPackageNotFound } cmd := exec.Command("nix", "profile", "remove", @@ -340,11 +345,11 @@ func nextPriority(profilePath string) string { return fmt.Sprintf("%d", max+1) } -func flakePath(args *ProfileInstallArgs) string { +func flakePath(args *ProfileInstallArgs) (string, error) { input := InputFromString(args.Package, args.ProjectDir) if input.IsFlake() { - return input.String() + return input.NormalizedName() } - return FlakeNixpkgs(args.NixpkgsCommit) + "#" + args.Package + return FlakeNixpkgs(args.NixpkgsCommit) + "#" + args.Package, nil } diff --git a/internal/nix/profile_test.go b/internal/nix/profile_test.go index 888dcc673f5..c2b6da4fa6d 100644 --- a/internal/nix/profile_test.go +++ b/internal/nix/profile_test.go @@ -20,14 +20,14 @@ func TestNixProfileListItem(t *testing.T) { line: fmt.Sprintf( "%d %s %s %s", 0, - "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.go_1_19", + "flake:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.go_1_19", "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.go_1_19", "/nix/store/w0lyimyyxxfl3gw40n46rpn1yjrl3q85-go-1.19.3", ), expected: expectedTestData{ item: &NixProfileListItem{ index: 0, - unlockedReference: "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.go_1_19", + unlockedReference: "flake:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.go_1_19", lockedReference: "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.go_1_19", nixStorePath: "/nix/store/w0lyimyyxxfl3gw40n46rpn1yjrl3q85-go-1.19.3", }, diff --git a/internal/planner/plansdk/plansdk.go b/internal/planner/plansdk/plansdk.go index 31e77a24406..726cebff849 100644 --- a/internal/planner/plansdk/plansdk.go +++ b/internal/planner/plansdk/plansdk.go @@ -47,7 +47,7 @@ type ShellPlan struct { // in the .devbox/gen directory. (Use string to make it marshalled version nicer.) GeneratedFiles map[string]string `json:"generated_files,omitempty"` - FlakeInputs []FlakeInput + FlakeInputs []*FlakeInput } type Planner interface {