Automatically generate image tags

.image.env

@@ -0,0 +1,7 @@
+# Generated file, do not modify.  This file is generated from a text file containing a list of images. The
+# most recent tag is interpolated from the source repository and used to generate a fully qualified image
+# name.
+MINIO_TAG='RELEASE.2021-09-15T04-54-25Z'
+POSTGRES_ALPINE_TAG='10.18-alpine'
+POSTGRES_DEBIAN_TAG='10.18'
+DEX_TAG='v2.30.0'

cmd/imagedeps/README.md

@@ -0,0 +1,55 @@
+# Image Deps
+
+This is a utility designed to take a list of images as input, and to generate FQN's of each input image including the 
+latest tag and output this information into an environment file, and a Go source file containing constant declarations. It
+is designed to be run as a script, for example:
+```shell
+go run github.com/replicatedhq/kots/cmd/imagedeps 
+2021/09/15 10:33:46 started tagged image file generator
+2021/09/15 10:33:48 successfully generated constant file "pkg/image/constants.go"
+2021/09/15 10:33:48 successfully generated dot env file ".image.env"
+```
+If successful it will generate two files, a file of constant declarations *pkg/image/constants.go* and *.image.env*.  The 
+Go file contains constant declarations of image references with the latest version tags.  The .env file contains environment
+variables defining the latest tags for images. 
+
+## Input 
+Latest tags will be found for images that are defined in a text file *cmd/imagedeps/image-spec*. Each line contains space delimited
+information about an image and an optional filter. If the filter is present, only tags that match will be included.  This 
+is useful to restrict release tags to a major version, or to filter out garbage tags. 
+
+| Name | Image URI | Matcher Regexp (Optional) |
+|------|--------------------|----------|
+| Name of the image for example **minio** | Untagged image reference **ghcr.io/dexidp/dex**| An optional regular expression, only matching tags will be included.  |
+
+### Sample image-spec
+```text
+minio minio/minio
+postgres-alpine postgres ^10.\d+-alpine$
+postgres-debian postgres ^10.\d+$
+dex ghcr.io/dexipd/dex
+```
+The preceding image spec will produce the following environment and Go files. Note how the override tags are applied 
+to the Postgres definitions. 
+```shell
+MINIO_TAG='RELEASE.2021-09-15T04-54-25Z'
+POSTGRES_ALPINE_TAG='10.17-alpine'
+POSTGRES_DEBIAN_TAG='10.17'
+DEX_TAG='v2.30.0'
+```
+```go
+package image
+
+const (
+	Minio = "minio/minio:RELEASE.2021-09-15T04-54-25Z"
+	PostgresAlpine = "postgres:10.17-alpine"
+	PostgresDebian = "postgres:10.17"
+	Dex = "ghcr.io/dexipd/dex:v2.30.0"
+)
+```
+
+## Github 
+Some of the image tags are resolved by looking at the Github release history of their associated projects.  This involves 
+interacting with the Github API.  The program uses an optional environment variable `GITHUB_AUTH_TOKEN` which is a Github API token 
+with **public_repo** scope for the purpose of avoiding rate limiting.  The program will work without `GITHUB_AUTH_TOKEN`
+but it is not recommended. 

cmd/imagedeps/image-spec

@@ -0,0 +1,4 @@
+minio minio/minio
+postgres-alpine postgres ^10.\d+-alpine$
+postgres-debian postgres ^10.\d+$
+dex ghcr.io/dexipd/dex

cmd/imagedeps/main.go

@@ -0,0 +1,127 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"text/template"
+)
+
+const (
+	minioReference                    = "minio"
+	dexReference                      = "dex"
+	postgresAlpineReference           = "postgres-alpine"
+	postgresDebianReference           = "postgres-debian"
+	inputFilename                     = "cmd/imagedeps/image-spec"
+	outputConstantFilename            = "pkg/image/constants.go"
+	outputEnvFilename                 = ".image.env"
+	dockerRegistryUrl                 = "https://index.docker.io"
+	githubPageSize                    = 100
+	githubAuthTokenEnvironmentVarName = "GITHUB_AUTH_TOKEN"
+	constantFileTemplate              = `package image
+
+// Generated file, do not modify.  This file is generated from a text file containing a list of images. The
+// most recent tag is interpolated from the source repository and used to generate a fully qualified
+// image name.
+
+const (
+{{- range .}}
+{{.GetDeclarationLine}}
+{{- end}}
+)`
+	environmentFileTemplate = `# Generated file, do not modify.  This file is generated from a text file containing a list of images. The
+# most recent tag is interpolated from the source repository and used to generate a fully qualified image
+# name.
+{{- range .}}
+{{.GetEnvironmentLine}}
+{{- end}}`
+)
+
+type generationContext struct {
+	inputFilename          string
+	outputConstantFilename string
+	outputEnvFilename      string
+	tagFinderFn            tagFinderFn
+}
+
+func main() {
+	log.Println("started tagged image file generator")
+	ctx := generationContext{
+		inputFilename:          inputFilename,
+		outputConstantFilename: outputConstantFilename,
+		outputEnvFilename:      outputEnvFilename,
+		tagFinderFn:            getTagFinder(),
+	}
+	if err := generateTaggedImageFiles(ctx); err != nil {
+		log.Fatalf("generator failed: %s", err)
+	}
+	log.Printf("successfully generated constant file %q", outputConstantFilename)
+	log.Printf("successfully generated dot env file %q", outputEnvFilename)
+}
+
+func generateTaggedImageFiles(ctx generationContext) error {
+	f, err := os.Open(ctx.inputFilename)
+	if err != nil {
+		return fmt.Errorf("could not read image file %q %w", ctx.inputFilename, err)
+	}
+	defer f.Close()
+
+	var references []*ImageRef
+	scan := bufio.NewScanner(f)
+	for scan.Scan() {
+		line := scan.Text()
+		ref, err := ctx.tagFinderFn(line)
+		if err != nil {
+			return fmt.Errorf("could not process image line %q %w", line, err)
+		}
+		references = append(references, ref)
+	}
+
+	if len(references) == 0 {
+		return fmt.Errorf("no references to images found")
+	}
+
+	if err := generateOutput(ctx.outputConstantFilename, constantFileTemplate, references); err != nil {
+		return fmt.Errorf("failed to generate output file %q %w", ctx.outputConstantFilename, err)
+	}
+
+	if err := generateOutput(ctx.outputEnvFilename, environmentFileTemplate, references); err != nil {
+		return fmt.Errorf("failed to generate file %q %w", ctx.outputEnvFilename, err)
+	}
+
+	return nil
+}
+
+func generateOutput(filename, fileTemplate string, refs []*ImageRef) error {
+	var out bytes.Buffer
+	if err := template.Must(template.New("constants").Parse(fileTemplate)).Execute(&out, refs); err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(filename, out.Bytes(), 0644); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// converts a name from the input file into a package public constant name
+// for example: foo-bar-baz -> FooBarBaz
+func getConstantName(s string) string {
+	parts := strings.Split(s, "-")
+	result := ""
+	for _, part := range parts {
+		result += strings.Title(part)
+	}
+	return result
+}
+
+// converts a name from the input file into an environment variable name
+// for example: foo_bar_baz -> FOO_BAR_BAZ
+func getEnvironmentName(s string) string {
+	return strings.ToUpper(strings.ReplaceAll(s, "-", "_"))
+}

cmd/imagedeps/main_test.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"io/ioutil"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/google/go-github/v39/github"
+	"github.com/stretchr/testify/require"
+)
+
+func makeReleases() []*github.RepositoryRelease {
+	var releases []*github.RepositoryRelease
+	tags := []string{
+		"RELEASE.2021-09-09T21-37-07Z.fips",
+		"RELEASE.2021-09-09T21-37-06Z.xxx",
+		"RELEASE.2021-09-09T21-37-05Z",
+		"RELEASE.2021-09-09T21-37-04Z",
+	}
+	tm := time.Now()
+	for _, t := range tags {
+		s := t
+		r := github.RepositoryRelease{
+			TagName:     &s,
+			PublishedAt: &github.Timestamp{Time: tm},
+		}
+		releases = append(releases, &r)
+		tm = tm.Add(time.Second * -1)
+	}
+	return releases
+}
+
+func TestFunctional(t *testing.T) {
+	tt := []struct {
+		name        string
+		fn          tagFinderFn
+		expectError bool
+	}{
+		{
+			name: "basic",
+			fn: getTagFinder(
+				withGithubReleaseTagFinder(
+					func(_ string, _ string) ([]*github.RepositoryRelease, error) {
+						return makeReleases(), nil
+					},
+				),
+			),
+		},
+		{
+			name: "with-overrides",
+			fn: getTagFinder(
+				withRepoGetTags(
+					func(_ string) ([]string, error) {
+						return []string{
+							"10.16", "10.17", "10.18",
+							"10.19-zippy", "10.18-alpine", "10.16-alpine",
+						}, nil
+					},
+				),
+				withGithubReleaseTagFinder(
+					func(_ string, _ string) ([]*github.RepositoryRelease, error) {
+						return makeReleases(), nil
+					},
+				),
+			),
+		},
+		{
+			name: "postgres",
+			fn: getTagFinder(
+				withRepoGetTags(
+					func(_ string) ([]string, error) {
+						return []string{
+							"10.16", "10.17", "10.18",
+							"10.19-zippy", "10.18-alpine", "10.16-alpine",
+						}, nil
+					},
+				),
+			),
+		},
+		{
+			name: "filter-github",
+			fn: getTagFinder(
+				withGithubReleaseTagFinder(
+					func(_ string, _ string) ([]*github.RepositoryRelease, error) {
+						return makeReleases(), nil
+					},
+				),
+			),
+		},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+
+			rootDir := path.Join("testdata", tc.name)
+			expectedConstants, err := ioutil.ReadFile(path.Join(rootDir, "constants.go"))
+			require.Nil(t, err)
+			expectedEnvs, err := ioutil.ReadFile(path.Join(rootDir, ".image.env"))
+			require.Nil(t, err)
+			tempDir := t.TempDir()
+			constantFile := path.Join(tempDir, "constants.go")
+			envFile := path.Join(tempDir, ".image.env")
+			inputSpec := path.Join(rootDir, "input-spec")
+			ctx := generationContext{
+				inputFilename:          inputSpec,
+				outputConstantFilename: constantFile,
+				outputEnvFilename:      envFile,
+				tagFinderFn:            tc.fn,
+			}
+
+			err = generateTaggedImageFiles(ctx)
+			if tc.expectError {
+				require.NotNil(t, err)
+				return
+			}
+
+			require.Nil(t, err)
+
+			actualConstants, err := ioutil.ReadFile(constantFile)
+			require.Nil(t, err)
+
+			actualEnv, err := ioutil.ReadFile(envFile)
+			require.Nil(t, err)
+
+			require.Equal(t, string(expectedConstants), string(actualConstants))
+			require.Equal(t, string(expectedEnvs), string(actualEnv))
+
+		})
+	}
+}