Merge pull request #1662 from aquaproj/feat/generate-registry-deep

feat(generate-registry): support generating version_overrides and testdata

pkg/checksum/checksum.go

@@ -72,7 +72,7 @@ func convertChecksumFileName(filename, version string) string {
 
 func GetChecksumConfigFromFilename(filename, version string) *registry.Checksum {
 	s := strings.ToLower(filename)
-	for _, suffix := range []string{"sig", "asc"} {
+	for _, suffix := range []string{"sig", "asc", "pem"} {
 		if strings.HasSuffix(s, "."+suffix) {
 			return nil
 		}

pkg/cli/generate_registry.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"fmt"
 	"net/http"
+	"os"
 
 	"github.com/aquaproj/aqua/pkg/config"
 	"github.com/aquaproj/aqua/pkg/controller"
@@ -35,6 +36,29 @@ packages:
       - linux
       - amd64
     rosetta2: true
+
+By default, aqua gets the information from the latest GitHub Releases.
+You can specify a specific package version.
+
+e.g.
+
+$ aqua gr cli/cli@v2.0.0
+
+By default, aqua gr doesn't generate version_overrides.
+If --deep is set, aqua generates version_overrides.
+
+e.g.
+
+$ aqua gr --deep suzuki-shunsuke/tfcmt
+
+Note that if --deep is set, GitHub API is called per GitHub Release.
+This may cause GitHub API rate limiting.
+
+If --out-testdata is set, aqua inserts testdata into the specified file.
+
+e.g.
+
+$ aqua gr --out-testdata testdata.yaml suzuki-shunsuke/tfcmt
 `
 
 func (runner *Runner) newGenerateRegistryCommand() *cli.Command {
@@ -46,12 +70,20 @@ func (runner *Runner) newGenerateRegistryCommand() *cli.Command {
 		Description: generateRegistryDescription,
 		Action:      runner.generateRegistryAction,
 		// TODO support "i" option
-		// Flags: []cli.Flag{
-		// 	&cli.StringFlag{
-		// 		Name:  "i",
-		// 		Usage: "Insert a registry to configuration file",
-		// 	},
-		// },
+		Flags: []cli.Flag{
+			// 	&cli.StringFlag{
+			// 		Name:  "i",
+			// 		Usage: "Insert a registry to configuration file",
+			// 	},
+			&cli.StringFlag{
+				Name:  "out-testdata",
+				Usage: "A file path where the testdata is outputted",
+			},
+			&cli.BoolFlag{
+				Name:  "deep",
+				Usage: "Resolve version_overrides",
+			},
+		},
 	}
 }
 
@@ -72,6 +104,6 @@ func (runner *Runner) generateRegistryAction(c *cli.Context) error {
 	if err := runner.setParam(c, "generate-registry", param); err != nil {
 		return fmt.Errorf("parse the command line arguments: %w", err)
 	}
-	ctrl := controller.InitializeGenerateRegistryCommandController(c.Context, param, http.DefaultClient)
+	ctrl := controller.InitializeGenerateRegistryCommandController(c.Context, param, http.DefaultClient, os.Stdout)
 	return ctrl.GenerateRegistry(c.Context, param, runner.LogE, c.Args().Slice()...) //nolint:wrapcheck
 }

pkg/cli/runner.go

@@ -40,6 +40,7 @@ func (runner *Runner) setParam(c *cli.Context, commandName string, param *config
 	}
 	param.ConfigFilePath = c.String("config")
 	param.Dest = c.String("o")
+	param.OutTestData = c.String("out-testdata")
 	param.OnlyLink = c.Bool("only-link")
 	param.IsTest = c.Bool("test")
 	if commandName == "generate-registry" {

pkg/config/package.go

@@ -175,6 +175,7 @@ type Param struct {
 	LogColor              string
 	Dest                  string
 	HomeDir               string
+	OutTestData           string
 	MaxParallelism        int
 	Args                  []string
 	Tags                  map[string]struct{}

pkg/config/registry/checksum.go

@@ -4,7 +4,7 @@ type Checksum struct {
 	Type         string           `json:"type,omitempty" jsonschema:"enum=github_release,enum=http"`
 	Asset        string           `json:"asset,omitempty"`
 	URL          string           `json:"url,omitempty"`
-	FileFormat   string           `yaml:"file_format" json:"file_format,omitempty"`
+	FileFormat   string           `yaml:"file_format,omitempty" json:"file_format,omitempty"`
 	Algorithm    string           `json:"algorithm,omitempty" jsonschema:"enum=md5,enum=sha1,enum=sha256,enum=sha512"`
 	Pattern      *ChecksumPattern `json:"pattern,omitempty"`
 	Enabled      *bool            `json:"enabled,omitempty"`

pkg/config/registry/package_info.go

@@ -20,35 +20,35 @@ const (
 
 type PackageInfo struct {
 	Name               string             `json:"name,omitempty" yaml:",omitempty"`
+	Aliases            []*Alias           `yaml:",omitempty" json:"aliases,omitempty"`
+	SearchWords        []string           `json:"search_words,omitempty" yaml:"search_words,omitempty"`
 	Type               string             `validate:"required" json:"type" jsonschema:"enum=github_release,enum=github_content,enum=github_archive,enum=http,enum=go,enum=go_install"`
 	RepoOwner          string             `yaml:"repo_owner,omitempty" json:"repo_owner,omitempty"`
 	RepoName           string             `yaml:"repo_name,omitempty" json:"repo_name,omitempty"`
+	Description        string             `json:"description,omitempty" yaml:",omitempty"`
+	Link               string             `json:"link,omitempty" yaml:",omitempty"`
 	Asset              *string            `json:"asset,omitempty" yaml:",omitempty"`
+	URL                *string            `json:"url,omitempty" yaml:",omitempty"`
 	Path               *string            `json:"path,omitempty" yaml:",omitempty"`
 	Format             string             `json:"format,omitempty" jsonschema:"example=tar.gz,example=raw,example=zip" yaml:",omitempty"`
-	Files              []*File            `json:"files,omitempty" yaml:",omitempty"`
-	URL                *string            `json:"url,omitempty" yaml:",omitempty"`
-	Description        string             `json:"description,omitempty" yaml:",omitempty"`
-	Link               string             `json:"link,omitempty" yaml:",omitempty"`
-	Replacements       Replacements       `json:"replacements,omitempty" yaml:",omitempty"`
 	Overrides          []*Override        `json:"overrides,omitempty" yaml:",omitempty"`
 	FormatOverrides    []*FormatOverride  `yaml:"format_overrides,omitempty" json:"format_overrides,omitempty"`
-	VersionConstraints string             `yaml:"version_constraint,omitempty" json:"version_constraint,omitempty"`
-	VersionOverrides   []*VersionOverride `yaml:"version_overrides,omitempty" json:"version_overrides,omitempty"`
+	Files              []*File            `json:"files,omitempty" yaml:",omitempty"`
+	Replacements       Replacements       `json:"replacements,omitempty" yaml:",omitempty"`
 	SupportedIf        *string            `yaml:"supported_if,omitempty" json:"supported_if,omitempty"`
 	SupportedEnvs      SupportedEnvs      `yaml:"supported_envs,omitempty" json:"supported_envs,omitempty"`
 	VersionFilter      *string            `yaml:"version_filter,omitempty" json:"version_filter,omitempty"`
 	VersionPrefix      *string            `yaml:"version_prefix,omitempty" json:"version_prefix,omitempty"`
 	Rosetta2           *bool              `yaml:",omitempty" json:"rosetta2,omitempty"`
-	Aliases            []*Alias           `yaml:",omitempty" json:"aliases,omitempty"`
 	VersionSource      string             `json:"version_source,omitempty" yaml:"version_source,omitempty" jsonschema:"enum=github_tag"`
 	CompleteWindowsExt *bool              `json:"complete_windows_ext,omitempty" yaml:"complete_windows_ext,omitempty"`
 	WindowsExt         string             `json:"windows_ext,omitempty" yaml:"windows_ext,omitempty"`
-	SearchWords        []string           `json:"search_words,omitempty" yaml:"search_words,omitempty"`
 	Checksum           *Checksum          `json:"checksum,omitempty"`
 	Cosign             *Cosign            `json:"cosign,omitempty"`
 	SLSAProvenance     *SLSAProvenance    `json:"slsa_provenance,omitempty" yaml:"slsa_provenance,omitempty"`
 	Private            bool               `json:"private,omitempty"`
+	VersionConstraints string             `yaml:"version_constraint,omitempty" json:"version_constraint,omitempty"`
+	VersionOverrides   []*VersionOverride `yaml:"version_overrides,omitempty" json:"version_overrides,omitempty"`
 }
 
 func (pkgInfo *PackageInfo) Copy() *PackageInfo {
@@ -221,20 +221,20 @@ func (pkgInfo *PackageInfo) OverrideByRuntime(rt *runtime.Runtime) { //nolint:cy
 }
 
 type VersionOverride struct {
+	VersionConstraints string            `yaml:"version_constraint,omitempty" json:"version_constraint,omitempty"`
 	Type               string            `yaml:",omitempty" json:"type,omitempty" jsonschema:"enum=github_release,enum=github_content,enum=github_archive,enum=http,enum=go,enum=go_install"`
 	RepoOwner          string            `yaml:"repo_owner,omitempty" json:"repo_owner,omitempty"`
 	RepoName           string            `yaml:"repo_name,omitempty" json:"repo_name,omitempty"`
 	Asset              *string           `yaml:",omitempty" json:"asset,omitempty"`
 	Path               *string           `yaml:",omitempty" json:"path,omitempty"`
-	Format             string            `yaml:",omitempty" json:"format,omitempty" jsonschema:"example=tar.gz,example=raw,example=zip"`
-	Files              []*File           `yaml:",omitempty" json:"files,omitempty"`
 	URL                *string           `yaml:",omitempty" json:"url,omitempty"`
-	Replacements       Replacements      `yaml:",omitempty" json:"replacements,omitempty"`
-	Overrides          []*Override       `yaml:",omitempty" json:"overrides,omitempty"`
+	Files              []*File           `yaml:",omitempty" json:"files,omitempty"`
+	Format             string            `yaml:",omitempty" json:"format,omitempty" jsonschema:"example=tar.gz,example=raw,example=zip"`
 	FormatOverrides    []*FormatOverride `yaml:"format_overrides,omitempty" json:"format_overrides,omitempty"`
+	Overrides          []*Override       `yaml:",omitempty" json:"overrides,omitempty"`
+	Replacements       Replacements      `yaml:",omitempty" json:"replacements,omitempty"`
 	SupportedIf        *string           `yaml:"supported_if,omitempty" json:"supported_if,omitempty"`
 	SupportedEnvs      SupportedEnvs     `yaml:"supported_envs,omitempty" json:"supported_envs,omitempty"`
-	VersionConstraints string            `yaml:"version_constraint,omitempty" json:"version_constraint,omitempty"`
 	VersionFilter      *string           `yaml:"version_filter,omitempty" json:"version_filter,omitempty"`
 	VersionPrefix      *string           `yaml:"version_prefix,omitempty" json:"version_prefix,omitempty"`
 	VersionSource      string            `json:"version_source,omitempty" yaml:"version_source,omitempty"`

pkg/config/registry/replacements.go

@@ -7,15 +7,15 @@ import (
 type Override struct {
 	GOOS               string          `yaml:",omitempty" json:"goos,omitempty" jsonschema:"enum=aix,enum=android,enum=darwin,enum=dragonfly,enum=freebsd,enum=illumos,enum=ios,enum=linux,enum=netbsd,enum=openbsd,enum=plan9,enum=solaris,enum=windows"`
 	GOArch             string          `yaml:",omitempty" json:"goarch,omitempty" jsonschema:"enum=386,enum=amd64,enum=arm,enum=arm64,enum=mips,enum=mips64,enum=mips64le,enum=mipsle,enum=ppc64,enum=ppc64le,enum=riscv64,enum=s390x"`
-	Replacements       Replacements    `yaml:",omitempty" json:"replacements,omitempty"`
+	Type               string          `json:"type,omitempty" jsonschema:"enum=github_release,enum=github_content,enum=github_archive,enum=http,enum=go,enum=go_install"`
 	Format             string          `yaml:",omitempty" json:"format,omitempty" jsonschema:"example=tar.gz,example=raw,example=zip"`
 	Asset              *string         `yaml:",omitempty" json:"asset,omitempty"`
 	Files              []*File         `yaml:",omitempty" json:"files,omitempty"`
 	URL                *string         `yaml:",omitempty" json:"url,omitempty"`
 	CompleteWindowsExt *bool           `json:"complete_windows_ext,omitempty" yaml:"complete_windows_ext,omitempty"`
 	WindowsExt         string          `json:"windows_ext,omitempty" yaml:"windows_ext,omitempty"`
+	Replacements       Replacements    `yaml:",omitempty" json:"replacements,omitempty"`
 	Checksum           *Checksum       `json:"checksum,omitempty"`
-	Type               string          `json:"type,omitempty" jsonschema:"enum=github_release,enum=github_content,enum=github_archive,enum=http,enum=go,enum=go_install"`
 	Cosign             *Cosign         `json:"cosign,omitempty"`
 	SLSAProvenance     *SLSAProvenance `json:"slsa_provenance,omitempty" yaml:"slsa_provenance,omitempty"`
 }

pkg/controller/generate-registry/generate.go

@@ -11,23 +11,30 @@ import (
 	"github.com/aquaproj/aqua/pkg/checksum"
 	"github.com/aquaproj/aqua/pkg/config"
 	"github.com/aquaproj/aqua/pkg/config/registry"
+	"github.com/aquaproj/aqua/pkg/controller/generate/output"
 	"github.com/aquaproj/aqua/pkg/github"
 	yaml "github.com/goccy/go-yaml"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
 )
 
 type Controller struct {
-	stdout io.Writer
-	fs     afero.Fs
-	github RepositoriesService
+	stdout            io.Writer
+	fs                afero.Fs
+	github            RepositoriesService
+	testdataOutputter TestdataOutputter
 }
 
-func NewController(fs afero.Fs, gh RepositoriesService) *Controller {
+type TestdataOutputter interface {
+	Output(param *output.Param) error
+}
+
+func NewController(fs afero.Fs, gh RepositoriesService, testdataOutputter TestdataOutputter) *Controller {
 	return &Controller{
-		stdout: os.Stdout,
-		fs:     fs,
-		github: gh,
+		stdout:            os.Stdout,
+		fs:                fs,
+		github:            gh,
+		testdataOutputter: testdataOutputter,
 	}
 }
 
@@ -44,7 +51,15 @@ func (ctrl *Controller) GenerateRegistry(ctx context.Context, param *config.Para
 }
 
 func (ctrl *Controller) genRegistry(ctx context.Context, param *config.Param, logE *logrus.Entry, pkgName string) error {
-	pkgInfo := ctrl.getPackageInfo(ctx, logE, pkgName)
+	pkgInfo, versions := ctrl.getPackageInfo(ctx, logE, pkgName, param.Deep)
+	if param.OutTestData != "" {
+		if err := ctrl.testdataOutputter.Output(&output.Param{
+			List: listPkgsFromVersions(pkgName, versions),
+			Dest: param.OutTestData,
+		}); err != nil {
+			return fmt.Errorf("output testdata to a file: %w", err)
+		}
+	}
 	if param.InsertFile == "" {
 		cfg := &registry.Config{
 			PackageInfos: registry.PackageInfos{
@@ -72,15 +87,15 @@ func (ctrl *Controller) getRelease(ctx context.Context, repoOwner, repoName, ver
 	return release, err //nolint:wrapcheck
 }
 
-func (ctrl *Controller) getPackageInfo(ctx context.Context, logE *logrus.Entry, arg string) *registry.PackageInfo {
+func (ctrl *Controller) getPackageInfo(ctx context.Context, logE *logrus.Entry, arg string, deep bool) (*registry.PackageInfo, []string) {
 	pkgName, version, _ := strings.Cut(arg, "@")
 	splitPkgNames := strings.Split(pkgName, "/")
 	pkgInfo := &registry.PackageInfo{
 		Type: "github_release",
 	}
 	if len(splitPkgNames) == 1 {
 		pkgInfo.Name = pkgName
-		return pkgInfo
+		return pkgInfo, nil
 	}
 	if len(splitPkgNames) != 2 { //nolint:gomnd
 		pkgInfo.Name = pkgName
@@ -96,28 +111,30 @@ func (ctrl *Controller) getPackageInfo(ctx context.Context, logE *logrus.Entry,
 	} else {
 		pkgInfo.Description = strings.TrimRight(strings.TrimSpace(repo.GetDescription()), ".!?")
 	}
+	if deep && version == "" {
+		return ctrl.getPackageInfoWithVersionOverrides(ctx, logE, pkgName, pkgInfo)
+	}
 	release, err := ctrl.getRelease(ctx, pkgInfo.RepoOwner, pkgInfo.RepoName, version)
 	if err != nil {
 		logE.WithFields(logrus.Fields{
 			"repo_owner": pkgInfo.RepoOwner,
 			"repo_name":  pkgInfo.RepoName,
 		}).WithError(err).Warn("get the release")
 	} else {
-		ctrl.patchRelease(ctx, logE, pkgInfo, pkgName, release)
+		logE.WithField("version", release.GetTagName()).Debug("got the release")
+		assets := ctrl.listReleaseAssets(ctx, logE, pkgInfo, release.GetID())
+		logE.WithField("num_of_assets", len(assets)).Debug("got assets")
+		ctrl.patchRelease(logE, pkgInfo, pkgName, release.GetTagName(), assets)
 	}
-	return pkgInfo
+	return pkgInfo, []string{version}
 }
 
-func (ctrl *Controller) patchRelease(ctx context.Context, logE *logrus.Entry, pkgInfo *registry.PackageInfo, pkgName string, release *github.RepositoryRelease) { //nolint:funlen,cyclop
-	logE.WithField("version", release.GetTagName()).Debug("got the release")
-	assets := ctrl.listReleaseAssets(ctx, logE, pkgInfo, release.GetID())
-	logE.WithField("num_of_assets", len(assets)).Debug("got assets")
+func (ctrl *Controller) patchRelease(logE *logrus.Entry, pkgInfo *registry.PackageInfo, pkgName string, tagName string, assets []*github.ReleaseAsset) { //nolint:funlen,cyclop
 	if len(assets) == 0 {
 		return
 	}
 	assetInfos := make([]*asset.AssetInfo, 0, len(assets))
 	pkgNameContainChecksum := strings.Contains(strings.ToLower(pkgName), "checksum")
-	tagName := release.GetTagName()
 	assetNames := map[string]struct{}{}
 	checksumNames := map[string]struct{}{}
 	for _, aset := range assets {
@@ -129,7 +146,7 @@ func (ctrl *Controller) patchRelease(ctx context.Context, logE *logrus.Entry, pk
 				continue
 			}
 		}
-		if asset.Exclude(pkgName, assetName, release.GetTagName()) {
+		if asset.Exclude(pkgName, assetName, tagName) {
 			logE.WithField("asset_name", assetName).Debug("exclude an asset")
 			continue
 		}

pkg/controller/generate-registry/generate_internal_test.go

@@ -111,8 +111,8 @@ func TestController_getPackageInfo(t *testing.T) { //nolint:funlen
 				Assets:   d.assets,
 				Repo:     d.repo,
 			}
-			ctrl := NewController(nil, gh)
-			pkgInfo := ctrl.getPackageInfo(ctx, logE, d.pkgName)
+			ctrl := NewController(nil, gh, nil)
+			pkgInfo, _ := ctrl.getPackageInfo(ctx, logE, d.pkgName, true)
 			if diff := cmp.Diff(d.exp, pkgInfo); diff != "" {
 				t.Fatal(diff)
 			}

pkg/controller/generate-registry/interface.go

@@ -11,4 +11,5 @@ type RepositoriesService interface {
 	GetLatestRelease(ctx context.Context, repoOwner, repoName string) (*github.RepositoryRelease, *github.Response, error)
 	GetReleaseByTag(ctx context.Context, owner, repo, tag string) (*github.RepositoryRelease, *github.Response, error)
 	ListReleaseAssets(ctx context.Context, owner, repo string, id int64, opts *github.ListOptions) ([]*github.ReleaseAsset, *github.Response, error)
+	ListReleases(ctx context.Context, owner, repo string, opts *github.ListOptions) ([]*github.RepositoryRelease, *github.Response, error)
 }