[nix] Allow github flakes in devbox.json

examples/flakes/remote/devbox.json

@@ -0,0 +1,13 @@
+{
+  "packages": [
+    "github:nixos/nixpkgs/5233fd2ba76a3accb5aaa999c00509a11fd0793c#hello",
+    "github:nix-community/fenix#stable.toolchain",
+    "github:F1bonacc1/process-compose"
+  ],
+  "shell": {
+    "init_hook": null
+  },
+  "nixpkgs": {
+    "commit": "f80ac848e3d6f0c12c52758c0f25c10c97ca3b62"
+  }
+}

internal/impl/devbox.go

@@ -295,8 +295,8 @@ func (d *Devbox) ShellEnvHash(ctx context.Context) (string, error) {
 }
 
 func (d *Devbox) Info(pkg string, markdown bool) error {
-	info, hasInfo := nix.PkgInfo(d.cfg.Nixpkgs.Commit, pkg)
-	if !hasInfo {
+	info := nix.PkgInfo(d.cfg.Nixpkgs.Commit, pkg)
+	if info == nil {
 		_, err := fmt.Fprintf(d.writer, "Package %s not found\n", pkg)
 		return errors.WithStack(err)
 	}

internal/impl/devbox_test.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/bmatcuk/doublestar/v4"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 	"go.jetpack.io/devbox/internal/planner/plansdk"
 )
 
@@ -29,9 +28,6 @@ func TestDevbox(t *testing.T) {
 	}
 }
 func testShellPlan(t *testing.T, testPath string) {
-	currentDir, err := os.Getwd()
-	require.New(t).NoError(err)
-
 	baseDir := filepath.Dir(testPath)
 	testName := fmt.Sprintf("%s_shell_plan", filepath.Base(baseDir))
 	t.Run(testName, func(t *testing.T) {
@@ -43,12 +39,6 @@ func testShellPlan(t *testing.T, testPath string) {
 		box, err := Open(baseDir, os.Stdout)
 		assert.NoErrorf(err, "%s should be a valid devbox project", baseDir)
 
-		// Just for tests, we make projectDir be a relative path so that the paths in plan.json
-		// of various test cases have relative paths. Absolute paths are a no-go because they'd
-		// be of the form `/Users/savil/...`, which are not generalized and cannot be checked in.
-		box.projectDir, err = filepath.Rel(currentDir, box.projectDir)
-		assert.NoErrorf(err, "expect to construct relative path from %s relative to base %s", box.projectDir, currentDir)
-
 		shellPlan, err := box.ShellPlan()
 		assert.NoError(err, "devbox shell plan should not fail")
 

internal/impl/flakes.go

@@ -6,20 +6,24 @@ import (
 	"go.jetpack.io/devbox/internal/planner/plansdk"
 )
 
-func (d *Devbox) flakeInputs() []plansdk.FlakeInput {
-	inputs := map[string]plansdk.FlakeInput{}
+func (d *Devbox) flakeInputs() []*plansdk.FlakeInput {
+	inputs := map[string]*plansdk.FlakeInput{}
 	for _, p := range d.cfg.MergedPackages(d.writer) {
 		pkg := nix.InputFromString(p, d.projectDir)
 		if pkg.IsFlake() {
-			if input, ok := inputs[pkg.Name()]; !ok {
-				inputs[pkg.Name()] = plansdk.FlakeInput{
+			AttributePath, err := pkg.PackageAttributePath()
+			if err != nil {
+				panic(err)
+			}
+			if input, ok := inputs[pkg.URLWithoutFragment()]; !ok {
+				inputs[pkg.URLWithoutFragment()] = &plansdk.FlakeInput{
 					Name:     pkg.Name(),
 					URL:      pkg.URLWithoutFragment(),
-					Packages: pkg.Packages(),
+					Packages: []string{AttributePath},
 				}
 			} else {
 				input.Packages = lo.Uniq(
-					append(inputs[pkg.Name()].Packages, pkg.Packages()...),
+					append(inputs[pkg.URLWithoutFragment()].Packages, AttributePath),
 				)
 			}
 		}

internal/impl/tmpl/flake.nix.tmpl

@@ -49,8 +49,8 @@
             global-pkgs.{{.}}
             {{- end}}
             {{- range $_, $flake := .FlakeInputs }}
-            {{- range $flake.Packages}}
-            {{ $flake.Name }}.packages.${system}.{{.}}
+            {{- range $flake.Packages }}
+            {{ $flake.Name }}.{{.}}
             {{- end }}
             {{- end }}
           ];

internal/nix/input.go

@@ -3,58 +3,116 @@ package nix
 import (
 	"crypto/md5"
 	"encoding/hex"
-	"encoding/json"
+	"fmt"
 	"net/url"
-	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/samber/lo"
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 )
 
-type Input url.URL
+type Input struct {
+	url.URL
+}
 
 func InputFromString(s, projectDir string) *Input {
 	u, _ := url.Parse(s)
-	if u.Path == "" && u.Opaque != "" {
+	if u.Path == "" && u.Opaque != "" && u.Scheme == "path" {
 		u.Path = filepath.Join(projectDir, u.Opaque)
 		u.Opaque = ""
 	}
-	return lo.ToPtr(Input(*u))
-}
-
-func (i *Input) String() string {
-	return (*url.URL)(i).String()
+	return &Input{*u}
 }
 
 // isFlake returns true if the package descriptor has a scheme. For now
 // we only support the "path" scheme.
 func (i *Input) IsFlake() bool {
-	// Technically flakes allows omitting the scheme for absolute paths, but
+	return i.IsLocal() || i.IsGithub()
+}
+
+func (i *Input) IsLocal() bool {
+	// Technically flakes allows omitting the scheme for local absolute paths, but
 	// we don't support that (yet).
 	return i.Scheme == "path"
 }
 
+func (i *Input) IsGithub() bool {
+	return i.Scheme == "github"
+}
+
+var inputNameRegex = regexp.MustCompile("[^a-zA-Z0-9-]+")
+
 func (i *Input) Name() string {
-	return filepath.Base(i.Path) + "-" + i.hash()
+	result := ""
+	if i.IsLocal() {
+		result = filepath.Base(i.Path) + "-" + i.hash()
+	} else if i.IsGithub() {
+		result = "gh-" + strings.Join(strings.Split(i.Opaque, "/"), "-")
+	} else {
+		result = i.String() + "-" + i.hash()
+	}
+	return inputNameRegex.ReplaceAllString(result, "-")
 }
 
 func (i *Input) URLWithoutFragment() string {
-	u := *(*url.URL)(i) // get copy
+	u := i.URL // get copy
 	u.Fragment = ""
 	// This will produce urls with extra slashes after the scheme, but that's ok
 	return u.String()
 }
 
-func (i *Input) Packages() []string {
+func (i *Input) NormalizedName() (string, error) {
+	attrPath, err := i.PackageAttributePath()
+	if err != nil {
+		return "", err
+	}
+	return i.URLWithoutFragment() + "#" + attrPath, nil
+}
+
+// PackageAttributePath returns just the name for non-flakes. For flake
+// references is returns the full path to the package in the flake. e.g.
+// packages.x86_64-linux.hello
+func (i *Input) PackageAttributePath() (string, error) {
 	if !i.IsFlake() {
-		return []string{i.String()}
+		return i.String(), nil
+	}
+	infos := search(i.String())
+
+	if len(infos) == 1 {
+		return lo.Keys(infos)[0], nil
+	}
+
+	// If ambiguous, try to find a default output
+	if len(infos) > 1 && i.Fragment == "" {
+		for key := range infos {
+			if strings.HasSuffix(key, ".default") {
+				return key, nil
+			}
+		}
+		for key := range infos {
+			if strings.HasPrefix(key, "defaultPackage.") {
+				return key, nil
+			}
+		}
 	}
-	if i.Fragment == "" {
-		return []string{"default"}
+
+	// Still ambiguous, return error
+	if len(infos) > 1 {
+		outputs := fmt.Sprintf("It has %d possible outputs", len(infos))
+		if len(infos) < 10 {
+			outputs = "It has the following possible outputs: \n" +
+				strings.Join(lo.Keys(infos), ", ")
+		}
+		return "", usererr.New(
+			"Flake \"%s\" is ambiguous. %s",
+			i.String(),
+			outputs,
+		)
 	}
-	return strings.Split(i.Fragment, ",")
+
+	return "", usererr.New("Flake \"%s\" was not found", i.String())
 }
 
 func (i *Input) hash() string {
@@ -66,94 +124,27 @@ func (i *Input) hash() string {
 }
 
 func (i *Input) validateExists() (bool, error) {
-	system, err := currentSystem()
-	if err != nil {
-		return false, err
-	}
-
-	outputs, err := outputs(i.Path)
-	if err != nil {
-		return false, err
-	}
-
-	fragment := i.Fragment
-	if fragment == "" {
-		fragment = "default" // if no package is specified, check for default.
-	}
-
-	if _, exists := outputs.Packages[system][fragment]; exists {
-		return true, nil
-	}
-
-	if _, exists := outputs.LegacyPackages[system][fragment]; exists {
-		return true, nil
-	}
-
-	// Another way to specify a default package is to output it as
-	// defaultPackage.${system}
-	if _, exists := outputs.DefaultPackage[system]; exists && i.Fragment == "" {
-		return true, nil
-	}
-
-	return false, usererr.New(
-		"Flake \"%s\" was found but package \"%s\" was not found in flake. "+
-			"Ensure the flake has a packages output",
-		i.Path,
-		fragment,
-	)
+	info, err := i.PackageAttributePath()
+	return info != "", err
 }
 
-func (i *Input) equals(o *Input) bool {
-	if i.String() == o.String() {
+func (i *Input) equals(other *Input) bool {
+	if i.String() == other.String() {
 		return true
 	}
-	return i.Scheme == o.Scheme &&
-		i.Path == o.Path &&
-		i.Opaque == o.Opaque &&
-		i.normalizedFragment() == o.normalizedFragment()
-}
 
-// normalizedFragment attempts to return the closest thing to a package name
-// from a fragment. A fragment could be:
-// * empty string -> default
-// * a single package -> package
-// * a qualified output (e.g. packages.aarch64-darwin.hello) -> hello
-func (i *Input) normalizedFragment() string {
-	if i.Fragment == "" {
-		return "default"
+	// check inputs without fragments as optimization. Next step is expensive
+	if i.URLWithoutFragment() != other.URLWithoutFragment() {
+		return false
 	}
-	parts := strings.Split(i.Fragment, ".")
-	return parts[len(parts)-1]
-}
-
-func currentSystem() (string, error) {
-	cmd := exec.Command(
-		"nix", "eval",
-		"--impure", "--raw", "--expr",
-		"builtins.currentSystem",
-	)
-	cmd.Args = append(cmd.Args, ExperimentalFlags()...)
-	o, err := cmd.Output()
-	return string(o), err
-}
-
-type output struct {
-	LegacyPackages map[string]map[string]any `json:"legacyPackages"`
-	Packages       map[string]map[string]any `json:"packages"`
-	DefaultPackage map[string]map[string]any `json:"defaultPackage"`
-}
 
-func outputs(path string) (*output, error) {
-	cmd := exec.Command(
-		"nix", "flake", "show",
-		path,
-		"--json", "--legacy",
-	)
-	cmd.Args = append(cmd.Args, ExperimentalFlags()...)
-	commandOut, err := cmd.Output()
+	name, err := i.PackageAttributePath()
+	if err != nil {
+		return false
+	}
+	otherName, err := other.PackageAttributePath()
 	if err != nil {
-		return nil, err
+		return false
 	}
-	out := &output{}
-	return out, json.Unmarshal(commandOut, out)
+	return name == otherName
 }