From 41e6cb6bb4083b2bfd2f8264651447ea767ee140 Mon Sep 17 00:00:00 2001 From: Mike Landau Date: Sun, 8 Oct 2023 15:42:31 -0700 Subject: [PATCH] [runx] Add runx validate, lockfile resolve --- .github/workflows/cli-tests.yaml | 1 + devbox.lock | 3 ++- go.mod | 2 +- go.sum | 2 ++ internal/devconfig/env.go | 9 +++++-- internal/devpkg/pkgtype/runx.go | 33 +++++++++++++++++++++++--- internal/devpkg/validation.go | 7 +++--- internal/impl/devbox.go | 23 +++++++++++------- internal/impl/packages.go | 29 +++++++++++----------- internal/impl/util.go | 7 ++++-- internal/integrations/envsec/envsec.go | 24 +++++++++++-------- internal/lock/lockfile.go | 23 ++++++++++++++++-- 12 files changed, 116 insertions(+), 47 deletions(-) diff --git a/.github/workflows/cli-tests.yaml b/.github/workflows/cli-tests.yaml index 0563ac3fe2e..76c07540569 100644 --- a/.github/workflows/cli-tests.yaml +++ b/.github/workflows/cli-tests.yaml @@ -37,6 +37,7 @@ defaults: env: HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEVBOX_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} HOMEBREW_NO_ANALYTICS: 1 HOMEBREW_NO_AUTO_UPDATE: 1 HOMEBREW_NO_EMOJI: 1 diff --git a/devbox.lock b/devbox.lock index ce75aa74924..c5a83a096b5 100644 --- a/devbox.lock +++ b/devbox.lock @@ -22,7 +22,8 @@ } }, "runx:golangci/golangci-lint@latest": { - "resolved": "runx:golangci/golangci-lint@latest" + "resolved": "golangci/golangci-lint@v1.54.2", + "version": "v1.54.2" } } } diff --git a/go.mod b/go.mod index 6e008e8dff8..c7b5d662cf1 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/wk8/go-ordered-map/v2 v2.1.8 github.com/zealic/go2node v0.1.0 - go.jetpack.io/pkg v0.0.0-20231002215645-9afeb0623fd3 + go.jetpack.io/pkg v0.0.0-20231006204718-f59feb213022 golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 golang.org/x/mod v0.12.0 golang.org/x/sync v0.3.0 diff --git a/go.sum b/go.sum index e095acb9d80..e1013a12907 100644 --- a/go.sum +++ b/go.sum @@ -337,6 +337,8 @@ github.com/zealic/go2node v0.1.0 h1:ofxpve08cmLJBwFdI0lPCk9jfwGWOSD+s6216x0oAaA= github.com/zealic/go2node v0.1.0/go.mod h1:GrkFr+HctXwP7vzcU9RsgtAeJjTQ6Ud0IPCQAqpTfBg= go.jetpack.io/pkg v0.0.0-20231002215645-9afeb0623fd3 h1:aMydtVCHn7dfotOyV41VAxX5b5OOsCc4TxOXwDt38Yw= go.jetpack.io/pkg v0.0.0-20231002215645-9afeb0623fd3/go.mod h1:iaf3e/aENp5luwYFlfCxj+GsiwqHagbvRAY3bIdEgGA= +go.jetpack.io/pkg v0.0.0-20231006204718-f59feb213022 h1:8TRpo0lYh1Y6zec8Px0yXbnVRXXs+yUPU+TnHNsREdA= +go.jetpack.io/pkg v0.0.0-20231006204718-f59feb213022/go.mod h1:iaf3e/aENp5luwYFlfCxj+GsiwqHagbvRAY3bIdEgGA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/internal/devconfig/env.go b/internal/devconfig/env.go index 065bf3fc172..e4c4f35cf07 100644 --- a/internal/devconfig/env.go +++ b/internal/devconfig/env.go @@ -1,15 +1,20 @@ package devconfig import ( + "context" + "go.jetpack.io/devbox/internal/boxcli/usererr" "go.jetpack.io/devbox/internal/integrations/envsec" ) -func (c *Config) ComputedEnv(projectDir string) (map[string]string, error) { +func (c *Config) ComputedEnv( + ctx context.Context, + projectDir string, +) (map[string]string, error) { env := map[string]string{} var err error if c.IsEnvsecEnabled() { - env, err = envsec.Env(projectDir) + env, err = envsec.Env(ctx, projectDir) if err != nil { return nil, err } diff --git a/internal/devpkg/pkgtype/runx.go b/internal/devpkg/pkgtype/runx.go index 7268fbd8edc..bab3d1d74e4 100644 --- a/internal/devpkg/pkgtype/runx.go +++ b/internal/devpkg/pkgtype/runx.go @@ -1,12 +1,39 @@ package pkgtype -import "strings" +import ( + "context" + "os" + "strings" + + "go.jetpack.io/pkg/sandbox/runx/impl/registry" + "go.jetpack.io/pkg/sandbox/runx/impl/runx" +) const ( - RunXScheme = "runx" - RunXPrefix = RunXScheme + ":" + RunXScheme = "runx" + RunXPrefix = RunXScheme + ":" + githubAPITokenVarName = "DEVBOX_GITHUB_API_TOKEN" ) +var cachedRegistry *registry.Registry + func IsRunX(s string) bool { return strings.HasPrefix(s, RunXPrefix) } + +func RunXClient() *runx.RunX { + return &runx.RunX{ + GithubAPIToken: os.Getenv(githubAPITokenVarName), + } +} + +func RunXRegistry(ctx context.Context) (*registry.Registry, error) { + if cachedRegistry == nil { + var err error + cachedRegistry, err = registry.NewLocalRegistry(ctx, os.Getenv(githubAPITokenVarName)) + if err != nil { + return nil, err + } + } + return cachedRegistry, nil +} diff --git a/internal/devpkg/validation.go b/internal/devpkg/validation.go index 067db758de4..f90ec3090e9 100644 --- a/internal/devpkg/validation.go +++ b/internal/devpkg/validation.go @@ -1,16 +1,17 @@ package devpkg import ( + "context" "strings" "go.jetpack.io/devbox/internal/boxcli/usererr" "go.jetpack.io/devbox/internal/nix" ) -func (p *Package) ValidateExists() (bool, error) { +func (p *Package) ValidateExists(ctx context.Context) (bool, error) { if p.IsRunX() { - // TODO implement runx validation - return true, nil + _, err := p.lockfile.Resolve(p.Raw) + return err == nil, err } if p.isVersioned() && p.version() == "" { return false, usererr.New("No version specified for %q.", p.Path) diff --git a/internal/impl/devbox.go b/internal/impl/devbox.go index 97f1979e1d7..9c6bb6c6f10 100644 --- a/internal/impl/devbox.go +++ b/internal/impl/devbox.go @@ -22,12 +22,12 @@ import ( "github.com/pkg/errors" "github.com/samber/lo" "go.jetpack.io/devbox/internal/devpkg" + "go.jetpack.io/devbox/internal/devpkg/pkgtype" "go.jetpack.io/devbox/internal/impl/envpath" "go.jetpack.io/devbox/internal/impl/generate" "go.jetpack.io/devbox/internal/searcher" "go.jetpack.io/devbox/internal/shellgen" "go.jetpack.io/devbox/internal/telemetry" - "go.jetpack.io/pkg/sandbox/runx" "go.jetpack.io/devbox/internal/boxcli/usererr" "go.jetpack.io/devbox/internal/cmdutil" @@ -842,13 +842,13 @@ func (d *Devbox) computeNixEnv(ctx context.Context, usePrintDevEnvCache bool) (m ) } - env["PATH"], err = d.addUtilitiesToPath(env["PATH"]) + env["PATH"], err = d.addUtilitiesToPath(ctx, env["PATH"]) if err != nil { return nil, err } // Include env variables in devbox.json - configEnv, err := d.configEnvs(env) + configEnv, err := d.configEnvs(ctx, env) if err != nil { return nil, err } @@ -876,7 +876,7 @@ func (d *Devbox) computeNixEnv(ctx context.Context, usePrintDevEnvCache bool) (m }) debug.Log("PATH after filtering with buildInputs (%v) is: %s", buildInputs, nixEnvPath) - runXPaths, err := d.RunXPaths() + runXPaths, err := d.RunXPaths(ctx) if err != nil { return nil, err } @@ -1044,8 +1044,11 @@ func (d *Devbox) checkOldEnvrc() error { // their value in the existing env variables. Note, this doesn't // allow env variables from outside the shell to be referenced so // no leaked variables are caused by this function. -func (d *Devbox) configEnvs(existingEnv map[string]string) (map[string]string, error) { - env, err := d.cfg.ComputedEnv(d.ProjectDir()) +func (d *Devbox) configEnvs( + ctx context.Context, + existingEnv map[string]string, +) (map[string]string, error) { + env, err := d.cfg.ComputedEnv(ctx, d.ProjectDir()) if err != nil { return nil, err } @@ -1206,11 +1209,15 @@ func (d *Devbox) Lockfile() *lock.File { return d.lockfile } -func (d *Devbox) RunXPaths() (string, error) { +func (d *Devbox) RunXPaths(ctx context.Context) (string, error) { packages := lo.Filter(d.InstallablePackages(), devpkg.IsRunX) paths := []string{} for _, pkg := range packages { - p, err := runx.Install(pkg.RunXPath()) + lockedPkg, err := d.lockfile.Resolve(pkg.Raw) + if err != nil { + return "", err + } + p, err := pkgtype.RunXClient().Install(ctx, lockedPkg.Resolved) if err != nil { return "", err } diff --git a/internal/impl/packages.go b/internal/impl/packages.go index ae6ef490aed..571eb2f0481 100644 --- a/internal/impl/packages.go +++ b/internal/impl/packages.go @@ -16,9 +16,9 @@ import ( "github.com/pkg/errors" "github.com/samber/lo" "go.jetpack.io/devbox/internal/devpkg" + "go.jetpack.io/devbox/internal/devpkg/pkgtype" "go.jetpack.io/devbox/internal/nix/nixprofile" "go.jetpack.io/devbox/internal/shellgen" - "go.jetpack.io/pkg/sandbox/runx" "go.jetpack.io/devbox/internal/boxcli/usererr" "go.jetpack.io/devbox/internal/debug" @@ -78,7 +78,7 @@ func (d *Devbox) Add(ctx context.Context, platforms, excludePlatforms []string, versionedPkg := devpkg.PackageFromString(pkg.Versioned(), d.lockfile) packageNameForConfig := pkg.Raw - ok, err := versionedPkg.ValidateExists() + ok, err := versionedPkg.ValidateExists(ctx) if (err == nil && ok) || errors.Is(err, devpkg.ErrCannotBuildPackageOnSystem) { // Only use versioned if it exists in search. We can disregard the error // about not building on the current system, since user's can continue @@ -236,7 +236,7 @@ func (d *Devbox) ensurePackagesAreInstalled(ctx context.Context, mode installMod return err } - if err := d.InstallRunXPackages(); err != nil { + if err := d.InstallRunXPackages(ctx); err != nil { return err } @@ -473,18 +473,17 @@ func resetProfileDirForFlakes(profileDir string) (err error) { return errors.WithStack(os.Remove(profileDir)) } -func (d *Devbox) InstallRunXPackages() error { - for _, pkg := range d.InstallablePackages() { - if pkg.IsRunX() { - // TODO: Once resolve is implemented, we use whatever version is in the lockfile. - if _, err := d.lockfile.Resolve(pkg.Raw); err != nil { - return err - } - _, err := runx.Install(pkg.RunXPath()) - if err != nil { - return fmt.Errorf("error installing runx package %s: %w", pkg, err) - } - +func (d *Devbox) InstallRunXPackages(ctx context.Context) error { + for _, pkg := range lo.Filter(d.InstallablePackages(), devpkg.IsRunX) { + lockedPkg, err := d.lockfile.Resolve(pkg.Raw) + if err != nil { + return err + } + if _, err := pkgtype.RunXClient().Install( + ctx, + lockedPkg.Resolved, + ); err != nil { + return fmt.Errorf("error installing runx package %s: %w", pkg, err) } } return nil diff --git a/internal/impl/util.go b/internal/impl/util.go index bcb01812337..17f688d5957 100644 --- a/internal/impl/util.go +++ b/internal/impl/util.go @@ -38,9 +38,12 @@ func (d *Devbox) addDevboxUtilityPackage(ctx context.Context, pkg string) error // to (e.g. envsec). // Question: Should we add utilityBinPath here? That would allow user to use // process-compose, etc -func (d *Devbox) addUtilitiesToPath(path string) (string, error) { +func (d *Devbox) addUtilitiesToPath( + ctx context.Context, + path string, +) (string, error) { if d.cfg.IsEnvsecEnabled() { - envsecPath, err := envsec.EnsureInstalled() + envsecPath, err := envsec.EnsureInstalled(ctx) if err != nil { return "", err } diff --git a/internal/integrations/envsec/envsec.go b/internal/integrations/envsec/envsec.go index 5adf817f075..baeab1d0d33 100644 --- a/internal/integrations/envsec/envsec.go +++ b/internal/integrations/envsec/envsec.go @@ -2,6 +2,7 @@ package envsec import ( "bytes" + "context" "encoding/json" "os/exec" "path/filepath" @@ -9,7 +10,7 @@ import ( "github.com/pkg/errors" "go.jetpack.io/devbox/internal/boxcli/usererr" "go.jetpack.io/devbox/internal/debug" - "go.jetpack.io/pkg/sandbox/runx" + "go.jetpack.io/devbox/internal/devpkg/pkgtype" ) var ( @@ -17,24 +18,24 @@ var ( binPathCache string ) -func Env(projectDir string) (map[string]string, error) { +func Env(ctx context.Context, projectDir string) (map[string]string, error) { defer debug.FunctionTimer().End() if envCache != nil { return envCache, nil } - if err := ensureInitialized(projectDir); err != nil { + if err := ensureInitialized(ctx, projectDir); err != nil { return nil, err } var err error - envCache, err = envsecList(projectDir) + envCache, err = envsecList(ctx, projectDir) return envCache, err } -func EnsureInstalled() (string, error) { +func EnsureInstalled(ctx context.Context) (string, error) { if binPathCache != "" { return binPathCache, nil } @@ -44,7 +45,7 @@ func EnsureInstalled() (string, error) { return binPathCache, nil } - paths, err := runx.Install("jetpack-io/envsec") + paths, err := pkgtype.RunXClient().Install(ctx, "jetpack-io/envsec") if err != nil { return "", errors.Wrap(err, "failed to install envsec") } @@ -57,8 +58,8 @@ func EnsureInstalled() (string, error) { return binPathCache, nil } -func ensureInitialized(projectDir string) error { - binPath, err := EnsureInstalled() +func ensureInitialized(ctx context.Context, projectDir string) error { + binPath, err := EnsureInstalled(ctx) if err != nil { return err } @@ -73,8 +74,11 @@ func ensureInitialized(projectDir string) error { return nil } -func envsecList(projectDir string) (map[string]string, error) { - binPath, err := EnsureInstalled() +func envsecList( + ctx context.Context, + projectDir string, +) (map[string]string, error) { + binPath, err := EnsureInstalled(ctx) if err != nil { return nil, err } diff --git a/internal/lock/lockfile.go b/internal/lock/lockfile.go index 226b811f01b..e55cae0e077 100644 --- a/internal/lock/lockfile.go +++ b/internal/lock/lockfile.go @@ -4,6 +4,7 @@ package lock import ( + "context" "fmt" "io/fs" "path/filepath" @@ -13,6 +14,7 @@ import ( "github.com/samber/lo" "go.jetpack.io/devbox/internal/devpkg/pkgtype" "go.jetpack.io/devbox/internal/searcher" + "go.jetpack.io/pkg/sandbox/runx/impl/types" "go.jetpack.io/devbox/internal/cuecfg" ) @@ -71,9 +73,13 @@ func (f *File) Resolve(pkg string) (*Package, error) { locked := &Package{} var err error if pkgtype.IsRunX(pkg) { - // TODO implement runx resolution. This can be done by reading the releases.json file + ref, err := ResolveRunXPackage(context.TODO(), pkg) + if err != nil { + return nil, err + } locked = &Package{ - Resolved: pkg, + Resolved: ref.String(), + Version: ref.Version, } } else if _, _, versioned := searcher.ParseVersionedPackage(pkg); versioned { locked, err = f.FetchResolvedPackage(pkg) @@ -185,3 +191,16 @@ func lockFilePath(project devboxProject) string { func getLockfileHash(project devboxProject) (string, error) { return cuecfg.FileHash(lockFilePath(project)) } + +func ResolveRunXPackage(ctx context.Context, pkg string) (types.PkgRef, error) { + ref, err := types.NewPkgRef(strings.TrimPrefix(pkg, pkgtype.RunXPrefix)) + if err != nil { + return types.PkgRef{}, err + } + + registry, err := pkgtype.RunXRegistry(ctx) + if err != nil { + return types.PkgRef{}, err + } + return registry.ResolveVersion(ref) +}