From a8222a819ba040900e7df5f14fb5c346037ece6f Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Wed, 28 Aug 2024 20:47:41 -0400 Subject: [PATCH 1/3] feat: adds forge CLI --- blueprint/schema/_embed/schema.cue | 13 + blueprint/schema/schema.go | 9 + blueprint/schema/schema_go_gen.cue | 9 + forge/README.md | 4 + forge/cli/Earthfile | 58 +++ forge/cli/cmd/cmds/config_dump.go | 27 ++ forge/cli/cmd/cmds/config_validate.go | 25 ++ forge/cli/cmd/cmds/run.go | 57 +++ forge/cli/cmd/cmds/scan.go | 93 +++++ forge/cli/cmd/cmds/secret.go | 93 +++++ forge/cli/cmd/cmds/util.go | 103 +++++ forge/cli/cmd/main.go | 78 ++++ forge/cli/cmd/main_test.go | 57 +++ forge/cli/cmd/testdata/config_dump/1.txt | 27 ++ forge/cli/cmd/testdata/config_dump/2.txt | 38 ++ forge/cli/cmd/testdata/config_dump/3.txt | 39 ++ forge/cli/cmd/testdata/config_dump/4.txt | 39 ++ forge/cli/cmd/testdata/config_validate/1.txt | 9 + forge/cli/cmd/testdata/config_validate/2.txt | 11 + forge/cli/cmd/testdata/run/1.txt | 16 + forge/cli/cmd/testdata/run/2.txt | 19 + forge/cli/cmd/testdata/run/3.txt | 19 + forge/cli/cmd/testdata/run/4.txt | 34 ++ forge/cli/cmd/testdata/scan/1.txt | 28 ++ forge/cli/cmd/testdata/scan/2.txt | 42 ++ forge/cli/cmd/testdata/scan/3.txt | 55 +++ forge/cli/go.mod | 58 +++ forge/cli/go.sum | 136 ++++++ forge/cli/internal/testutils/helpers.go | 32 ++ .../internal/testutils/mocks/fileseeker.go | 33 ++ forge/cli/pkg/earthfile/earthfile.go | 78 ++++ forge/cli/pkg/earthfile/earthfile_test.go | 88 ++++ forge/cli/pkg/earthfile/scan.go | 43 ++ forge/cli/pkg/earthfile/scan_test.go | 142 +++++++ forge/cli/pkg/earthly/earthly.go | 256 ++++++++++++ forge/cli/pkg/earthly/earthly_test.go | 394 ++++++++++++++++++ forge/cli/pkg/executor/executor.go | 9 + forge/cli/pkg/executor/executor_mock.go | 80 ++++ forge/cli/pkg/executor/local.go | 89 ++++ forge/cli/pkg/secrets/client.go | 41 ++ forge/cli/pkg/secrets/interface.go | 9 + forge/cli/pkg/secrets/interface_mock.go | 124 ++++++ forge/cli/pkg/secrets/providers.go | 8 + forge/cli/pkg/secrets/providers/aws.go | 155 +++++++ .../pkg/secrets/providers/aws_mock_test.go | 200 +++++++++ forge/cli/pkg/secrets/providers/aws_test.go | 218 ++++++++++ forge/cli/pkg/secrets/providers/local.go | 32 ++ forge/cli/pkg/secrets/providers/local_test.go | 69 +++ forge/cli/pkg/walker/fs.go | 54 +++ forge/cli/pkg/walker/fs_test.go | 153 +++++++ forge/cli/pkg/walker/walker.go | 33 ++ forge/cli/pkg/walker/walker_mock.go | 80 ++++ 52 files changed, 3616 insertions(+) create mode 100644 forge/README.md create mode 100644 forge/cli/Earthfile create mode 100644 forge/cli/cmd/cmds/config_dump.go create mode 100644 forge/cli/cmd/cmds/config_validate.go create mode 100644 forge/cli/cmd/cmds/run.go create mode 100644 forge/cli/cmd/cmds/scan.go create mode 100644 forge/cli/cmd/cmds/secret.go create mode 100644 forge/cli/cmd/cmds/util.go create mode 100644 forge/cli/cmd/main.go create mode 100644 forge/cli/cmd/main_test.go create mode 100644 forge/cli/cmd/testdata/config_dump/1.txt create mode 100644 forge/cli/cmd/testdata/config_dump/2.txt create mode 100644 forge/cli/cmd/testdata/config_dump/3.txt create mode 100644 forge/cli/cmd/testdata/config_dump/4.txt create mode 100644 forge/cli/cmd/testdata/config_validate/1.txt create mode 100644 forge/cli/cmd/testdata/config_validate/2.txt create mode 100644 forge/cli/cmd/testdata/run/1.txt create mode 100644 forge/cli/cmd/testdata/run/2.txt create mode 100644 forge/cli/cmd/testdata/run/3.txt create mode 100644 forge/cli/cmd/testdata/run/4.txt create mode 100644 forge/cli/cmd/testdata/scan/1.txt create mode 100644 forge/cli/cmd/testdata/scan/2.txt create mode 100644 forge/cli/cmd/testdata/scan/3.txt create mode 100644 forge/cli/go.mod create mode 100644 forge/cli/go.sum create mode 100644 forge/cli/internal/testutils/helpers.go create mode 100644 forge/cli/internal/testutils/mocks/fileseeker.go create mode 100644 forge/cli/pkg/earthfile/earthfile.go create mode 100644 forge/cli/pkg/earthfile/earthfile_test.go create mode 100644 forge/cli/pkg/earthfile/scan.go create mode 100644 forge/cli/pkg/earthfile/scan_test.go create mode 100644 forge/cli/pkg/earthly/earthly.go create mode 100644 forge/cli/pkg/earthly/earthly_test.go create mode 100644 forge/cli/pkg/executor/executor.go create mode 100644 forge/cli/pkg/executor/executor_mock.go create mode 100644 forge/cli/pkg/executor/local.go create mode 100644 forge/cli/pkg/secrets/client.go create mode 100644 forge/cli/pkg/secrets/interface.go create mode 100644 forge/cli/pkg/secrets/interface_mock.go create mode 100644 forge/cli/pkg/secrets/providers.go create mode 100644 forge/cli/pkg/secrets/providers/aws.go create mode 100644 forge/cli/pkg/secrets/providers/aws_mock_test.go create mode 100644 forge/cli/pkg/secrets/providers/aws_test.go create mode 100644 forge/cli/pkg/secrets/providers/local.go create mode 100644 forge/cli/pkg/secrets/providers/local_test.go create mode 100644 forge/cli/pkg/walker/fs.go create mode 100644 forge/cli/pkg/walker/fs_test.go create mode 100644 forge/cli/pkg/walker/walker.go create mode 100644 forge/cli/pkg/walker/walker_mock.go diff --git a/blueprint/schema/_embed/schema.cue b/blueprint/schema/_embed/schema.cue index 2a79a64b..40f7257d 100644 --- a/blueprint/schema/_embed/schema.cue +++ b/blueprint/schema/_embed/schema.cue @@ -7,6 +7,9 @@ package schema registry: (_ | *"") & { string } @go(Registry) + secrets: { + [string]: #Secret + } @go(Secrets,map[string]Secret) targets: { [string]: #Target } @go(Targets,map[string]Target) @@ -18,6 +21,15 @@ package schema string } @go(Satellite) } + +// Secret contains the secret provider and a list of mappings +#Secret: { + path: string @go(Path) + provider: string @go(Provider) + maps: { + [string]: string + } @go(Maps,map[string]string) +} version: "1.0" // Target contains the configuration for a single target. @@ -33,4 +45,5 @@ version: "1.0" retries: (_ | *0) & { int } @go(Retries) + secrets: [...#Secret] @go(Secrets,[]Secret) } diff --git a/blueprint/schema/schema.go b/blueprint/schema/schema.go index 789a05ca..ad9e5c09 100644 --- a/blueprint/schema/schema.go +++ b/blueprint/schema/schema.go @@ -15,6 +15,7 @@ type Blueprint struct { Version string `json:"version"` Global Global `json:"global"` Registry string `json:"registry"` + Secrets map[string]Secret `json:"secrets"` Targets map[string]Target `json:"targets"` } @@ -23,9 +24,17 @@ type Global struct { Satellite string `json:"satellite"` } +// Secret contains the secret provider and a list of mappings +type Secret struct { + Path string `json:"path"` + Provider string `json:"provider"` + Maps map[string]string `json:"maps"` +} + // Target contains the configuration for a single target. type Target struct { Args map[string]string `json:"args"` Privileged bool `json:"privileged"` Retries int `json:"retries"` + Secrets []Secret `json:"secrets"` } diff --git a/blueprint/schema/schema_go_gen.cue b/blueprint/schema/schema_go_gen.cue index d0a562b7..369cba2d 100644 --- a/blueprint/schema/schema_go_gen.cue +++ b/blueprint/schema/schema_go_gen.cue @@ -9,6 +9,7 @@ package schema version: string @go(Version) global: #Global @go(Global) registry: string @go(Registry) + secrets: {[string]: #Secret} @go(Secrets,map[string]Secret) targets: {[string]: #Target} @go(Targets,map[string]Target) } @@ -17,9 +18,17 @@ package schema satellite: string @go(Satellite) } +// Secret contains the secret provider and a list of mappings +#Secret: { + path: string @go(Path) + provider: string @go(Provider) + maps: {[string]: string} @go(Maps,map[string]string) +} + // Target contains the configuration for a single target. #Target: { args: {[string]: string} @go(Args,map[string]string) privileged: bool @go(Privileged) retries: int @go(Retries) + secrets: [...#Secret] @go(Secrets,[]Secret) } diff --git a/forge/README.md b/forge/README.md new file mode 100644 index 00000000..24dbea3a --- /dev/null +++ b/forge/README.md @@ -0,0 +1,4 @@ +# Forge + +Forge is the subsystem within Catalyst Forge that powers the CI process. +It ships with both a CLI and a set of Github Actions that can be used to create CI pipelines. \ No newline at end of file diff --git a/forge/cli/Earthfile b/forge/cli/Earthfile new file mode 100644 index 00000000..b04ce8c4 --- /dev/null +++ b/forge/cli/Earthfile @@ -0,0 +1,58 @@ +VERSION 0.8 + +deps: + FROM golang:1.22.4-alpine3.19 + + WORKDIR /work + + RUN mkdir -p /go/cache && mkdir -p /go/modcache + ENV GOCACHE=/go/cache + ENV GOMODCACHE=/go/modcache + CACHE --persist --sharing shared /go + + COPY go.mod go.sum . + RUN go mod download + +src: + FROM +deps + + CACHE --persist --sharing shared /go + + COPY --dir cmd cue internal pkg . + RUN go generate ./... + +check: + FROM +src + + RUN gofmt -l . | grep . && exit 1 || exit 0 + RUN go vet ./... + +build: + FROM +src + + ARG version="0.0.0" + + ENV CGO_ENABLED=0 + RUN go build -ldflags="-extldflags=-static -X main.version=$version" -o bin/forge cmd/main.go + + SAVE ARTIFACT bin/forge forge + +test: + FROM +build + + RUN go test ./... + +release: + FROM +build + + SAVE ARTIFACT bin/forge forge + +publish: + FROM debian:bookworm-slim + WORKDIR /workspace + ARG tag=latest + + COPY +build/forge /usr/local/bin/forge + + ENTRYPOINT ["/usr/local/bin/forge"] + SAVE IMAGE --push forge:${tag} \ No newline at end of file diff --git a/forge/cli/cmd/cmds/config_dump.go b/forge/cli/cmd/cmds/config_dump.go new file mode 100644 index 00000000..ab53cb2e --- /dev/null +++ b/forge/cli/cmd/cmds/config_dump.go @@ -0,0 +1,27 @@ +package cmds + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" +) + +type DumpCmd struct { + Config string `arg:"" help:"Path to the configuration file."` + Pretty bool `help:"Pretty print JSON output."` +} + +func (c *DumpCmd) Run(logger *slog.Logger) error { + if _, err := os.Stat(c.Config); os.IsNotExist(err) { + return fmt.Errorf("configuration file does not exist: %s", c.Config) + } + + config, err := loadBlueprint(filepath.Dir(c.Config), logger) + if err != nil { + return err + } + + printJson(config, c.Pretty) + return nil +} diff --git a/forge/cli/cmd/cmds/config_validate.go b/forge/cli/cmd/cmds/config_validate.go new file mode 100644 index 00000000..c606fa67 --- /dev/null +++ b/forge/cli/cmd/cmds/config_validate.go @@ -0,0 +1,25 @@ +package cmds + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" +) + +type ValidateCmd struct { + Config string `arg:"" help:"Path to the configuration file."` +} + +func (c *ValidateCmd) Run(logger *slog.Logger) error { + if _, err := os.Stat(c.Config); os.IsNotExist(err) { + return fmt.Errorf("configuration file does not exist: %s", c.Config) + } + + _, err := loadBlueprint(filepath.Dir(c.Config), logger) + if err != nil { + return err + } + + return nil +} diff --git a/forge/cli/cmd/cmds/run.go b/forge/cli/cmd/cmds/run.go new file mode 100644 index 00000000..3c63b5f5 --- /dev/null +++ b/forge/cli/cmd/cmds/run.go @@ -0,0 +1,57 @@ +package cmds + +import ( + "fmt" + "log/slog" + "strings" + + "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/earthly" + "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/executor" + "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/secrets" +) + +type RunCmd struct { + Artifact bool `short:"a" help:"Enable artifact collection."` + Local bool `short:"l" help:"Forces the target to run locally (ignores satellite)."` + Path string `arg:"" help:"The path to the target to execute (i.e., ./dir1+test)."` + Pretty bool `help:"Pretty print JSON output."` +} + +func (c *RunCmd) Run(logger *slog.Logger) error { + if !strings.Contains(c.Path, "+") { + return fmt.Errorf("invalid Earthfile+Target pair: %s", c.Path) + } + + earthfileDir := strings.Split(c.Path, "+")[0] + target := strings.Split(c.Path, "+")[1] + + config, err := loadBlueprint(earthfileDir, logger) + if err != nil { + return err + } + + localExec := executor.NewLocalExecutor( + logger, + executor.WithRedirect(), + ) + + opts := generateOpts(target, c, &config) + earthlyExec := earthly.NewEarthlyExecutor( + earthfileDir, + target, + localExec, + secrets.NewDefaultSecretStore(), + logger, + opts..., + ) + + logger.Info("Executing Earthly target", "earthfile", earthfileDir, "target", target) + result, err := earthlyExec.Run() + if err != nil { + return err + } + + printJson(result, c.Pretty) + + return nil +} diff --git a/forge/cli/cmd/cmds/scan.go b/forge/cli/cmd/cmds/scan.go new file mode 100644 index 00000000..048ccd1c --- /dev/null +++ b/forge/cli/cmd/cmds/scan.go @@ -0,0 +1,93 @@ +package cmds + +import ( + "fmt" + "log/slog" + "path/filepath" + "regexp" + "sort" + + "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/earthfile" + "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/walker" +) + +type ScanCmd struct { + Absolute bool `short:"a" help:"Output absolute paths."` + Enumerate bool `short:"e" help:"Enumerate results into Earthfile+Target pairs."` + Filter []string `short:"f" help:"Filter discovered Earthfiles by target name using a regular expression."` + Pretty bool `help:"Pretty print JSON output."` + RootPath string `arg:"" help:"Root path to scan for Earthfiles and their respective targets."` +} + +func (c *ScanCmd) Run(logger *slog.Logger) error { + walker := walker.NewFilesystemWalker(logger) + + var rootPath string + if c.Absolute { + var err error + rootPath, err = filepath.Abs(c.RootPath) + if err != nil { + return fmt.Errorf("failed to get absolute path: %w", err) + } + } else { + rootPath = c.RootPath + } + + earthfiles, err := earthfile.ScanEarthfiles(rootPath, &walker, logger) + if err != nil { + return err + } + + if len(c.Filter) > 0 { + result := make(map[string]map[string][]string) + for _, filter := range c.Filter { + filterExpr, err := regexp.Compile(filter) + if err != nil { + return err + } + + for path, earthfile := range earthfiles { + targets := earthfile.FilterTargets(func(target string) bool { + return filterExpr.MatchString(target) + }) + + if len(targets) > 0 { + if _, ok := result[filter]; !ok { + result[filter] = make(map[string][]string) + } + + result[filter][path] = targets + } + + logger.Debug("Filtered Earthfile", "path", path, "targets", targets) + } + } + + if c.Enumerate { + enumerated := make(map[string][]string) + for filter, targetMap := range result { + enumerated[filter] = enumerate(targetMap) + sort.Strings(enumerated[filter]) // Sort to provide deterministic output + } + + printJson(enumerated, c.Pretty) + } else { + printJson(result, c.Pretty) + } + } else { + targetMap := make(map[string][]string) + for path, earthfile := range earthfiles { + targetMap[path] = earthfile.Targets() + } + + if c.Enumerate { + enumerated := enumerate(targetMap) + sort.Strings(enumerated) // Sort to provide deterministic output + printJson(enumerated, c.Pretty) + } else { + printJson(targetMap, c.Pretty) + } + } + + return nil +} diff --git a/forge/cli/cmd/cmds/secret.go b/forge/cli/cmd/cmds/secret.go new file mode 100644 index 00000000..6f746105 --- /dev/null +++ b/forge/cli/cmd/cmds/secret.go @@ -0,0 +1,93 @@ +package cmds + +import ( + "encoding/json" + "fmt" + "log/slog" + "strings" + + "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/secrets" +) + +const ( + ConfigFileName string = "chronos.cue" + SecretNamePrefix string = "services/bentest" +) + +type Get struct { + Provider string `short:"p" help:"The provider of the secret store." default:"aws"` + Path string `arg:"" help:"The path of the secret."` + Key string `arg:"" help:"The key inside of the secret to get."` +} + +type Set struct { + Field []string `short:"f" help:"A secret field to set."` + Provider string `short:"p" help:"The provider of the secret store." default:"aws"` + Path string `arg:"" help:"The path of the secret."` +} + +type SecretCmd struct { + Get *Get `cmd:"" help:"Get a secret."` + Set *Set `cmd:"" help:"Set a secret."` +} + +func (c *Get) Run(logger *slog.Logger) error { + store := secrets.NewDefaultSecretStore() + client, err := store.NewClient(logger, secrets.Provider(c.Provider)) + if err != nil { + logger.Error("Unable to create secret client.", "err", err) + return fmt.Errorf("unable to create secret client: %w", err) + } + + s, err := client.Get(c.Path) + if err != nil { + return fmt.Errorf("could not get secret: %w", err) + } + + m := make(map[string]string) + + if err := json.Unmarshal([]byte(s), &m); err != nil { + return err + } + + if _, ok := m[c.Key]; !ok { + return fmt.Errorf("key %s not found in secret at %s", c.Key, c.Path) + } + + fmt.Println(m[c.Key]) + return nil +} + +func (c *Set) Run(logger *slog.Logger) error { + store := secrets.NewDefaultSecretStore() + client, err := store.NewClient(logger, secrets.Provider(secrets.ProviderAWS)) + if err != nil { + logger.Error("Unable to create secret client.", "err", err) + return fmt.Errorf("unable to create secret client: %w", err) + } + + fields := make(map[string]string) + for _, f := range c.Field { + kv := strings.Split(f, "=") + if len(kv) != 2 { + return fmt.Errorf("invalid field format: %s: must be in the format of key=value", f) + } + + fields[kv[0]] = kv[1] + } + + b, err := json.Marshal(&fields) + if err != nil { + return err + } + + id, err := client.Set(c.Path, string(b)) + if err != nil { + logger.Error("could not set secret", "err", err) + return err + } + + logger.Info("Successfully set secret in AWS Secretsmanager.", "id", id) + + return nil +} diff --git a/forge/cli/cmd/cmds/util.go b/forge/cli/cmd/cmds/util.go new file mode 100644 index 00000000..2b04841c --- /dev/null +++ b/forge/cli/cmd/cmds/util.go @@ -0,0 +1,103 @@ +package cmds + +import ( + "encoding/json" + "fmt" + "log/slog" + + blueprint "github.com/input-output-hk/catalyst-forge/blueprint/pkg/loader" + "github.com/input-output-hk/catalyst-forge/blueprint/schema" + "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/earthly" +) + +// enumerate enumerates the Earthfile+Target pairs from the target map. +func enumerate(data map[string][]string) []string { + var result []string + for path, targets := range data { + for _, target := range targets { + result = append(result, fmt.Sprintf("%s+%s", path, target)) + } + } + + return result +} + +// generateOpts generates the options for the Earthly executor based on the configuration file and flags. +func generateOpts(target string, flags *RunCmd, config *schema.Blueprint) []earthly.EarthlyExecutorOption { + var opts []earthly.EarthlyExecutorOption + + if config != nil { + if _, ok := config.Targets[target]; ok { + targetConfig := config.Targets[target] + + if len(targetConfig.Args) > 0 { + var args []string + for k, v := range targetConfig.Args { + args = append(args, fmt.Sprintf("--%s", k), v) + } + + opts = append(opts, earthly.WithTargetArgs(args...)) + } + + if targetConfig.Privileged { + opts = append(opts, earthly.WithPrivileged()) + } + + if targetConfig.Retries > 0 { + opts = append(opts, earthly.WithRetries(targetConfig.Retries)) + } + + if len(targetConfig.Secrets) > 0 { + opts = append(opts, earthly.WithSecrets(targetConfig.Secrets)) + } + } + + if config.Global.Satellite != "" && !flags.Local { + opts = append(opts, earthly.WithSatellite(config.Global.Satellite)) + } + } + + if flags != nil { + if flags.Artifact { + opts = append(opts, earthly.WithArtifact()) + } + } + + return opts +} + +// loadBlueprint loads the blueprint file from the given root path. +func loadBlueprint(rootPath string, logger *slog.Logger) (schema.Blueprint, error) { + loader := blueprint.NewDefaultBlueprintLoader(rootPath, logger) + + err := loader.Load() + if err != nil { + return schema.Blueprint{}, fmt.Errorf("failed loading blueprint: %w", err) + } + + config, err := loader.Decode() + if err != nil { + return schema.Blueprint{}, fmt.Errorf("failed decoding blueprint: %w", err) + } + + return config, nil +} + +// printJson prints the given data as a JSON string. +func printJson(data interface{}, pretty bool) { + var out []byte + var err error + + if pretty { + out, err = json.MarshalIndent(data, "", " ") + } else { + out, err = json.Marshal(data) + } + + if err != nil { + fmt.Println(err) + return + } + + fmt.Println(string(out)) +} diff --git a/forge/cli/cmd/main.go b/forge/cli/cmd/main.go new file mode 100644 index 00000000..5895d2c0 --- /dev/null +++ b/forge/cli/cmd/main.go @@ -0,0 +1,78 @@ +package main + +import ( + "fmt" + "log/slog" + "os" + "runtime" + + "cuelang.org/go/cue/cuecontext" + "github.com/alecthomas/kong" + "github.com/charmbracelet/log" + "github.com/input-output-hk/catalyst-forge/blueprint/schema" + "github.com/input-output-hk/catalyst-forge/forge/cli/cmd/cmds" +) + +var version = "dev" + +var cli struct { + Config configCmd `cmd:"" help:"Configuration commands."` + Run cmds.RunCmd `cmd:"" help:"Run an Earthly target."` + Scan cmds.ScanCmd `cmd:"" help:"Scan for Earthfiles."` + Secret cmds.SecretCmd `cmd:"" help:"Manage secrets."` + Version VersionCmd `cmd:"" help:"Print the version."` + Verbose int `short:"v" type:"counter" help:"Enable verbose logging."` +} + +type configCmd struct { + Validate cmds.ValidateCmd `cmd:"" help:"Validates a blueprint file."` + Dump cmds.DumpCmd `cmd:"" help:"Dumps a blueprint file to JSON."` +} + +type VersionCmd struct{} + +func (c *VersionCmd) Run() error { + ctx := cuecontext.New() + schema, err := schema.LoadSchema(ctx) + if err != nil { + return err + } + + fmt.Printf("forge version %s %s/%s\n", version, runtime.GOOS, runtime.GOARCH) + fmt.Printf("config schema version %s\n", schema.Version) + return nil +} + +// Run is the entrypoint for the CLI tool. +func Run() int { + ctx := kong.Parse(&cli, + kong.Name("forge"), + kong.Description("The CLI tool powering Catalyst Forge")) + + handler := log.New(os.Stderr) + switch cli.Verbose { + case 0: + handler.SetLevel(log.FatalLevel) + case 1: + handler.SetLevel(log.WarnLevel) + case 2: + handler.SetLevel(log.InfoLevel) + case 3: + handler.SetLevel(log.DebugLevel) + } + + logger := slog.New(handler) + ctx.Bind(logger) + + err := ctx.Run() + if err != nil { + fmt.Printf("forge: %v", err) + return 1 + } + + return 0 +} + +func main() { + Run() +} diff --git a/forge/cli/cmd/main_test.go b/forge/cli/cmd/main_test.go new file mode 100644 index 00000000..8e0e6dba --- /dev/null +++ b/forge/cli/cmd/main_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/rogpeppe/go-internal/testscript" +) + +func TestMain(m *testing.M) { + os.Exit(testscript.RunMain(m, map[string]func() int{ + "forge": Run, + "earthly": mockEarthly, + })) +} + +func TestConfigDump(t *testing.T) { + testscript.Run(t, testscript.Params{ + Dir: "testdata/config_dump", + }) +} + +func TestConfigValidate(t *testing.T) { + testscript.Run(t, testscript.Params{ + Dir: "testdata/config_validate", + }) +} +func TestRun(t *testing.T) { + testscript.Run(t, testscript.Params{ + Dir: "testdata/run", + }) +} + +func TestScan(t *testing.T) { + testscript.Run(t, testscript.Params{ + Dir: "testdata/scan", + }) +} + +func mockEarthly() int { + for _, arg := range os.Args { + fmt.Println(arg) + } + + secrets := os.Getenv("EARTHLY_SECRETS") + if secrets != "" { + fmt.Println("EARTHLY_SECRETS=" + secrets) + } + + stdout, err := os.ReadFile("earthly_stdout.txt") + if err == nil { + fmt.Println(string(stdout)) + } + + return 0 +} diff --git a/forge/cli/cmd/testdata/config_dump/1.txt b/forge/cli/cmd/testdata/config_dump/1.txt new file mode 100644 index 00000000..ebd9a25a --- /dev/null +++ b/forge/cli/cmd/testdata/config_dump/1.txt @@ -0,0 +1,27 @@ +exec forge config dump --pretty ./blueprint.cue +cmp stdout golden.txt + +-- golden.txt -- +{ + "version": "1.0.0", + "global": { + "satellite": "" + }, + "registry": "", + "secrets": {}, + "targets": { + "test": { + "args": {}, + "privileged": true, + "retries": 0, + "secrets": [] + } + } +} +-- blueprint.cue -- +version: "1.0" +targets: { + test: { + privileged: true + } +} diff --git a/forge/cli/cmd/testdata/config_dump/2.txt b/forge/cli/cmd/testdata/config_dump/2.txt new file mode 100644 index 00000000..7a077634 --- /dev/null +++ b/forge/cli/cmd/testdata/config_dump/2.txt @@ -0,0 +1,38 @@ +exec mkdir -p .git +exec forge config dump --pretty ./dir1/dir2/blueprint.cue +cmpenv stdout golden.txt + +-- golden.txt -- +{ + "version": "1.0.0", + "global": { + "satellite": "" + }, + "registry": "test", + "secrets": {}, + "targets": { + "test": { + "args": {}, + "privileged": true, + "retries": 3, + "secrets": [] + } + } +} +-- dir1/dir2/blueprint.cue -- +version: "1.0" +targets: { + test: { + privileged: true + } +} +-- dir1/blueprint.cue -- +version: "1.0" +targets: { + test: { + retries: 3 + } +} +-- blueprint.cue -- +version: "1.0" +registry: "test" diff --git a/forge/cli/cmd/testdata/config_dump/3.txt b/forge/cli/cmd/testdata/config_dump/3.txt new file mode 100644 index 00000000..6356f91a --- /dev/null +++ b/forge/cli/cmd/testdata/config_dump/3.txt @@ -0,0 +1,39 @@ +exec mkdir -p .git +env RETRIES=3 +exec forge config dump --pretty ./dir1/dir2/blueprint.cue +cmpenv stdout golden.txt + +-- golden.txt -- +{ + "version": "1.2.0", + "global": { + "satellite": "" + }, + "registry": "test", + "secrets": {}, + "targets": { + "test": { + "args": {}, + "privileged": true, + "retries": 3, + "secrets": [] + } + } +} +-- dir1/dir2/blueprint.cue -- +version: "1.0" +targets: { + test: { + privileged: true + } +} +-- dir1/blueprint.cue -- +version: "1.2" +targets: { + test: { + retries: int @env(name=RETRIES,type=int) + } +} +-- blueprint.cue -- +version: "1.1" +registry: "test" diff --git a/forge/cli/cmd/testdata/config_dump/4.txt b/forge/cli/cmd/testdata/config_dump/4.txt new file mode 100644 index 00000000..6356f91a --- /dev/null +++ b/forge/cli/cmd/testdata/config_dump/4.txt @@ -0,0 +1,39 @@ +exec mkdir -p .git +env RETRIES=3 +exec forge config dump --pretty ./dir1/dir2/blueprint.cue +cmpenv stdout golden.txt + +-- golden.txt -- +{ + "version": "1.2.0", + "global": { + "satellite": "" + }, + "registry": "test", + "secrets": {}, + "targets": { + "test": { + "args": {}, + "privileged": true, + "retries": 3, + "secrets": [] + } + } +} +-- dir1/dir2/blueprint.cue -- +version: "1.0" +targets: { + test: { + privileged: true + } +} +-- dir1/blueprint.cue -- +version: "1.2" +targets: { + test: { + retries: int @env(name=RETRIES,type=int) + } +} +-- blueprint.cue -- +version: "1.1" +registry: "test" diff --git a/forge/cli/cmd/testdata/config_validate/1.txt b/forge/cli/cmd/testdata/config_validate/1.txt new file mode 100644 index 00000000..1afba624 --- /dev/null +++ b/forge/cli/cmd/testdata/config_validate/1.txt @@ -0,0 +1,9 @@ +exec forge config validate ./blueprint.cue + +-- blueprint.cue -- +version: "1.0" +targets: { + test: { + privileged: true + } +} \ No newline at end of file diff --git a/forge/cli/cmd/testdata/config_validate/2.txt b/forge/cli/cmd/testdata/config_validate/2.txt new file mode 100644 index 00000000..2113cf0f --- /dev/null +++ b/forge/cli/cmd/testdata/config_validate/2.txt @@ -0,0 +1,11 @@ +! exec forge config validate ./blueprint.cue + +-- golden.txt -- +forge: failed loading configuration: failed to validate configuration file: #Config.targets.test.privileged: conflicting values "true" and bool (mismatched types string and bool) +-- blueprint.cue -- +version: "1.0" +targets: { + test: { + privileged: "true" + } +} \ No newline at end of file diff --git a/forge/cli/cmd/testdata/run/1.txt b/forge/cli/cmd/testdata/run/1.txt new file mode 100644 index 00000000..c2dced7b --- /dev/null +++ b/forge/cli/cmd/testdata/run/1.txt @@ -0,0 +1,16 @@ +exec forge run ./dir1+test +cmp stdout golden.txt + +-- golden.txt -- +earthly +./dir1+test +Image ./dir1+test output as test + +{"artifacts":{},"images":{"./dir1+test":"test"}} +-- earthly_stdout.txt -- +Image ./dir1+test output as test +-- dir1/Earthfile -- +VERSION 0.8 + +foo: + RUN echo "foobar" \ No newline at end of file diff --git a/forge/cli/cmd/testdata/run/2.txt b/forge/cli/cmd/testdata/run/2.txt new file mode 100644 index 00000000..69b6e258 --- /dev/null +++ b/forge/cli/cmd/testdata/run/2.txt @@ -0,0 +1,19 @@ +exec forge run --artifact ./dir1+test +cmp stdout golden.txt + +-- golden.txt -- +earthly +--artifact +./dir1+test/* +Image ./dir1+test output as test +Artifact ./dir1+test output as test + +{"artifacts":{"./dir1+test":"test"},"images":{"./dir1+test":"test"}} +-- earthly_stdout.txt -- +Image ./dir1+test output as test +Artifact ./dir1+test output as test +-- dir1/Earthfile -- +VERSION 0.8 + +foo: + RUN echo "foobar" \ No newline at end of file diff --git a/forge/cli/cmd/testdata/run/3.txt b/forge/cli/cmd/testdata/run/3.txt new file mode 100644 index 00000000..3bcdd0b2 --- /dev/null +++ b/forge/cli/cmd/testdata/run/3.txt @@ -0,0 +1,19 @@ +exec forge run ./dir1+test +cmp stdout golden.txt + +-- golden.txt -- +earthly +--allow-privileged +./dir1+test +Image ./dir1+test output as test + +{"artifacts":{},"images":{"./dir1+test":"test"}} +-- earthly_stdout.txt -- +Image ./dir1+test output as test +-- dir1/blueprint.cue -- +version: "1.0" +targets: { + test: { + privileged: true + } +} \ No newline at end of file diff --git a/forge/cli/cmd/testdata/run/4.txt b/forge/cli/cmd/testdata/run/4.txt new file mode 100644 index 00000000..a1353c26 --- /dev/null +++ b/forge/cli/cmd/testdata/run/4.txt @@ -0,0 +1,34 @@ +exec forge run ./dir1+test +cmp stdout golden.txt + +-- dir1/Secretfile -- +{"secret_key": "secret_value"} +-- golden.txt -- +earthly +--allow-privileged +./dir1+test +EARTHLY_SECRETS=secret_id=secret_value +Image ./dir1+test output as test + +{"artifacts":{},"images":{"./dir1+test":"test"}} +-- earthly_stdout.txt -- +Image ./dir1+test output as test +-- dir1/blueprint.cue -- +version: "1.0" +S=secrets: { + my_secret_1: { + path: "./dir1/Secretfile" + provider: "local" + maps: { + "secret_key": "secret_id" + } + } +} +targets: { + test: { + privileged: true + secrets: [ + S.my_secret_1 + ] + } +} diff --git a/forge/cli/cmd/testdata/scan/1.txt b/forge/cli/cmd/testdata/scan/1.txt new file mode 100644 index 00000000..f28a3130 --- /dev/null +++ b/forge/cli/cmd/testdata/scan/1.txt @@ -0,0 +1,28 @@ +exec forge scan . +cmp stdout golden_1.txt + +exec forge scan --enumerate . +cmp stdout golden_1_enum.txt + +exec forge scan --absolute . +cmpenv stdout golden_2.txt + +exec forge scan --absolute --enumerate . +cmpenv stdout golden_2_enum.txt + +-- golden_1.txt -- +{"Earthfile":["foo","bar"]} +-- golden_1_enum.txt -- +["Earthfile+bar","Earthfile+foo"] +-- golden_2.txt -- +{"$WORK/Earthfile":["foo","bar"]} +-- golden_2_enum.txt -- +["$WORK/Earthfile+bar","$WORK/Earthfile+foo"] +-- Earthfile -- +VERSION 0.7 + +foo: + LET bar = baz + +bar: + LET bar = baz \ No newline at end of file diff --git a/forge/cli/cmd/testdata/scan/2.txt b/forge/cli/cmd/testdata/scan/2.txt new file mode 100644 index 00000000..6fdeda68 --- /dev/null +++ b/forge/cli/cmd/testdata/scan/2.txt @@ -0,0 +1,42 @@ +exec forge scan . +cmp stdout golden.txt + +exec forge scan --enumerate . +cmp stdout golden_enum.txt + +-- golden.txt -- +{"Earthfile":["foo","bar"],"dir1/Earthfile":["foo","bar"],"dir1/dir2/Earthfile":["foo","bar"],"dir3/dir4/dir5/Earthfile":["foo"]} +-- golden_enum.txt -- +["Earthfile+bar","Earthfile+foo","dir1/Earthfile+bar","dir1/Earthfile+foo","dir1/dir2/Earthfile+bar","dir1/dir2/Earthfile+foo","dir3/dir4/dir5/Earthfile+foo"] +-- Earthfile -- +VERSION 0.7 + +foo: + LET bar = baz + +bar: + LET bar = baz + +-- dir1/Earthfile -- +VERSION 0.7 + +foo: + LET bar = baz + +bar: + LET bar = baz + +-- dir1/dir2/Earthfile -- +VERSION 0.7 + +foo: + LET bar = baz + +bar: + LET bar = baz + +-- dir3/dir4/dir5/Earthfile -- +VERSION 0.7 + +foo: + LET bar = baz \ No newline at end of file diff --git a/forge/cli/cmd/testdata/scan/3.txt b/forge/cli/cmd/testdata/scan/3.txt new file mode 100644 index 00000000..9eabac85 --- /dev/null +++ b/forge/cli/cmd/testdata/scan/3.txt @@ -0,0 +1,55 @@ +exec forge scan --filter check . +cmp stdout golden_1.txt + +exec forge scan --filter check --filter build . +cmp stdout golden_2.txt + +exec forge scan --filter check --filter build-\w+ --filter test$ . +cmp stdout golden_3.txt + +exec forge scan --enumerate --filter check --filter build-\w+ --filter test$ . +cmp stdout golden_3_enum.txt + +-- golden_1.txt -- +{"check":{"Earthfile":["check"],"dir3/dir4/dir5/Earthfile":["check-foo"]}} +-- golden_2.txt -- +{"build":{"Earthfile":["build"],"dir1/Earthfile":["build"],"dir1/dir2/Earthfile":["build-thing"]},"check":{"Earthfile":["check"],"dir3/dir4/dir5/Earthfile":["check-foo"]}} +-- golden_3.txt -- +{"build-\\w+":{"dir1/dir2/Earthfile":["build-thing"]},"check":{"Earthfile":["check"],"dir3/dir4/dir5/Earthfile":["check-foo"]},"test$":{"dir3/dir4/dir5/Earthfile":["test"]}} +-- golden_3_enum.txt -- +{"build-\\w+":["dir1/dir2/Earthfile+build-thing"],"check":["Earthfile+check","dir3/dir4/dir5/Earthfile+check-foo"],"test$":["dir3/dir4/dir5/Earthfile+test"]} +-- Earthfile -- +VERSION 0.7 + +check: + LET bar = baz + +build: + LET bar = baz + +-- dir1/Earthfile -- +VERSION 0.7 + +build: + LET bar = baz + +publish: + LET bar = baz + +-- dir1/dir2/Earthfile -- +VERSION 0.7 + +build-thing: + LET bar = baz + +release: + LET bar = baz + +-- dir3/dir4/dir5/Earthfile -- +VERSION 0.7 + +check-foo: + LET bar = baz + +test: + LET bar = baz \ No newline at end of file diff --git a/forge/cli/go.mod b/forge/cli/go.mod new file mode 100644 index 00000000..18c64915 --- /dev/null +++ b/forge/cli/go.mod @@ -0,0 +1,58 @@ +module github.com/input-output-hk/catalyst-forge/forge/cli + +go 1.22.3 + +require ( + cuelang.org/go v0.10.0 + github.com/alecthomas/kong v0.9.0 + github.com/aws/aws-sdk-go v1.55.5 + github.com/aws/aws-sdk-go-v2/config v1.27.27 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.32.4 + github.com/charmbracelet/log v0.4.0 + github.com/earthly/earthly/ast v0.0.2-0.20240228223838-42e8ca204e8a + github.com/input-output-hk/catalyst-forge/blueprint v0.0.0 + github.com/rogpeppe/go-internal v1.12.1-0.20240709150035-ccf4b4329d21 + github.com/spf13/afero v1.11.0 +) + +require ( + github.com/Masterminds/semver/v3 v3.2.1 // indirect + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230219212500-1f9a474cc2dc // indirect + github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect + github.com/aws/smithy-go v1.20.3 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/charmbracelet/lipgloss v0.10.0 // indirect + github.com/cockroachdb/apd/v3 v3.2.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/input-output-hk/catalyst-forge/cuetools v0.0.0 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.18 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.15.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/mod v0.20.0 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/sys v0.23.0 // indirect + golang.org/x/text v0.17.0 // indirect + golang.org/x/tools v0.24.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/input-output-hk/catalyst-forge/blueprint => ../../blueprint + +replace github.com/input-output-hk/catalyst-forge/cuetools => ../../cuetools diff --git a/forge/cli/go.sum b/forge/cli/go.sum new file mode 100644 index 00000000..bf186d7d --- /dev/null +++ b/forge/cli/go.sum @@ -0,0 +1,136 @@ +cuelabs.dev/go/oci/ociregistry v0.0.0-20240807094312-a32ad29eed79 h1:EceZITBGET3qHneD5xowSTY/YHbNybvMWGh62K2fG/M= +cuelabs.dev/go/oci/ociregistry v0.0.0-20240807094312-a32ad29eed79/go.mod h1:5A4xfTzHTXfeVJBU6RAUf+QrlfTCW+017q/QiW+sMLg= +cuelang.org/go v0.10.0 h1:Y1Pu4wwga5HkXfLFK1sWAYaSWIBdcsr5Cb5AWj2pOuE= +cuelang.org/go v0.10.0/go.mod h1:HzlaqqqInHNiqE6slTP6+UtxT9hN6DAzgJgdbNxXvX8= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU= +github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/kong v0.9.0 h1:G5diXxc85KvoV2f0ZRVuMsi45IrBgx9zDNGNj165aPA= +github.com/alecthomas/kong v0.9.0/go.mod h1:Y47y5gKfHp1hDc7CH7OeXgLIpp+Q2m1Ni0L5s3bI8Os= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230219212500-1f9a474cc2dc h1:ikxgKfnYm4kXCOohe1uCkVFwZcABDZbVsqginko+GY8= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230219212500-1f9a474cc2dc/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= +github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= +github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90= +github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.32.4 h1:NgRFYyFpiMD62y4VPXh4DosPFbZd4vdMVBWKk0VmWXc= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.32.4/go.mod h1:TKKN7IQoM7uTnyuFm9bm9cw5P//ZYTl4m3htBWQ1G/c= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ= +github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= +github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/charmbracelet/lipgloss v0.10.0 h1:KWeXFSexGcfahHX+54URiZGkBFazf70JNMtwg/AFW3s= +github.com/charmbracelet/lipgloss v0.10.0/go.mod h1:Wig9DSfvANsxqkRsqj6x87irdy123SR4dOXlKa91ciE= +github.com/charmbracelet/log v0.4.0 h1:G9bQAcx8rWA2T3pWvx7YtPTPwgqpk7D68BX21IRW8ZM= +github.com/charmbracelet/log v0.4.0/go.mod h1:63bXt/djrizTec0l11H20t8FDSvA4CRZJ1KH22MdptM= +github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg= +github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/earthly/earthly/ast v0.0.2-0.20240228223838-42e8ca204e8a h1:Z4oatniIQ8EG0JL26cX9cc7IeJUe9Zs7wFetMJpbEhY= +github.com/earthly/earthly/ast v0.0.2-0.20240228223838-42e8ca204e8a/go.mod h1:74/Fa5yMVQdnD/a32pXf8CrzH6MfAaXNIFt15MoHuv0= +github.com/emicklei/proto v1.13.2 h1:z/etSFO3uyXeuEsVPzfl56WNgzcvIr42aQazXaQmFZY= +github.com/emicklei/proto v1.13.2/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= +github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= +github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0 h1:sadMIsgmHpEOGbUs6VtHBXRR1OHevnj7hLx9ZcdNGW4= +github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0/go.mod h1:jgxiZysxFPM+iWKwQwPR+y+Jvo54ARd4EisXxKYpB5c= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.12.1-0.20240709150035-ccf4b4329d21 h1:igWZJluD8KtEtAgRyF4x6lqcxDry1ULztksMJh2mnQE= +github.com/rogpeppe/go-internal v1.12.1-0.20240709150035-ccf4b4329d21/go.mod h1:RMRJLmBOqWacUkmJHRMiPKh1S1m3PA7Zh4W80/kWPpg= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= +golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/forge/cli/internal/testutils/helpers.go b/forge/cli/internal/testutils/helpers.go new file mode 100644 index 00000000..ea876881 --- /dev/null +++ b/forge/cli/internal/testutils/helpers.go @@ -0,0 +1,32 @@ +package testutils + +import ( + "fmt" + "io" + "log/slog" + "testing" +) + +// CheckError checks if the error is expected or not. If the error is +// unexpected, it returns an error. +// If the function returns true, the test should return immediately. +func CheckError(t *testing.T, err error, expected bool, expectedErr error) (bool, error) { + if expected && err != nil { + fmt.Println(expectedErr) + if expectedErr != nil && err.Error() != expectedErr.Error() { + return true, fmt.Errorf("got error %v, want error %v", err, expectedErr) + } + return true, nil + } else if !expected && err != nil { + return true, fmt.Errorf("unexpected error: %v", err) + } else if expected && err == nil { + return true, fmt.Errorf("expected error %v, got nil", expectedErr) + } + + return false, nil +} + +// NewNoopLogger creates a new logger that discards all logs. +func NewNoopLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} diff --git a/forge/cli/internal/testutils/mocks/fileseeker.go b/forge/cli/internal/testutils/mocks/fileseeker.go new file mode 100644 index 00000000..0754f291 --- /dev/null +++ b/forge/cli/internal/testutils/mocks/fileseeker.go @@ -0,0 +1,33 @@ +package mocks + +import ( + "io/fs" + "strings" +) + +// MockFileSeeker is a mock implementation of the FileSeeker interface. +type MockFileSeeker struct { + *strings.Reader +} + +func (MockFileSeeker) Stat() (fs.FileInfo, error) { + return MockFileInfo{}, nil +} + +func (MockFileSeeker) Close() error { + return nil +} + +// MockFileInfo is a mock implementation of the fs.FileInfo interface. +type MockFileInfo struct { + fs.FileInfo +} + +func (MockFileInfo) Name() string { + return "Earthfile" +} + +// NewMockFileSeeker creates a new MockFileSeeker with the given content. +func NewMockFileSeeker(s string) MockFileSeeker { + return MockFileSeeker{strings.NewReader(s)} +} diff --git a/forge/cli/pkg/earthfile/earthfile.go b/forge/cli/pkg/earthfile/earthfile.go new file mode 100644 index 00000000..d0f0cb80 --- /dev/null +++ b/forge/cli/pkg/earthfile/earthfile.go @@ -0,0 +1,78 @@ +package earthfile + +import ( + "context" + + "github.com/earthly/earthly/ast" + "github.com/earthly/earthly/ast/spec" + "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/walker" +) + +// Earthfile represents a parsed Earthfile. +type Earthfile struct { + spec spec.Earthfile +} + +// Targets returns the names of the targets in the Earthfile. +func (e Earthfile) Targets() []string { + var targetNames []string + for _, target := range e.spec.Targets { + targetNames = append(targetNames, target.Name) + } + + return targetNames +} + +// FilterTargets returns the names of the targets in the Earthfile that pass the given filter. +func (e Earthfile) FilterTargets(filter func(string) bool) []string { + var targetNames []string + for _, target := range e.spec.Targets { + if filter(target.Name) { + targetNames = append(targetNames, target.Name) + } + } + + return targetNames +} + +// ParseEarthfile parses an Earthfile from the given FileSeeker. +func ParseEarthfile(ctx context.Context, earthfile walker.FileSeeker) (Earthfile, error) { + nr, err := newNamedReader(earthfile) + if err != nil { + return Earthfile{}, err + } + + ef, err := ast.ParseOpts(ctx, ast.FromReader(nr)) + if err != nil { + return Earthfile{}, err + } + + return Earthfile{ + spec: ef, + }, nil +} + +// namedReader is a FileSeeker that also provides the name of the file. +// This is used to interopt with the Earthly AST parser. +type namedReader struct { + walker.FileSeeker + name string +} + +// Name returns the name of the file. +func (n namedReader) Name() string { + return n.name +} + +// newNamedReader wraps a FileSeeker in a namedReader. +func newNamedReader(reader walker.FileSeeker) (namedReader, error) { + stat, err := reader.Stat() + if err != nil { + return namedReader{}, err + } + + return namedReader{ + FileSeeker: reader, + name: stat.Name(), + }, nil +} diff --git a/forge/cli/pkg/earthfile/earthfile_test.go b/forge/cli/pkg/earthfile/earthfile_test.go new file mode 100644 index 00000000..6e309d59 --- /dev/null +++ b/forge/cli/pkg/earthfile/earthfile_test.go @@ -0,0 +1,88 @@ +package earthfile + +import ( + "context" + "testing" + + "github.com/earthly/earthly/ast/spec" + "github.com/input-output-hk/catalyst-forge/forge/cli/internal/testutils/mocks" +) + +func TestEarthfileTargets(t *testing.T) { + earthfile := Earthfile{ + spec: spec.Earthfile{ + Targets: []spec.Target{ + {Name: "target1"}, + {Name: "target2"}, + }, + }, + } + + targets := earthfile.Targets() + if len(targets) != 2 { + t.Errorf("expected 2 targets, got %d", len(targets)) + } + + if targets[0] != "target1" { + t.Errorf("expected target1, got %s", targets[0]) + } + + if targets[1] != "target2" { + t.Errorf("expected target2, got %s", targets[1]) + } +} + +func TestEarthfileFilterTargets(t *testing.T) { + earthfile := Earthfile{ + spec: spec.Earthfile{ + Targets: []spec.Target{ + {Name: "target1"}, + {Name: "target2"}, + }, + }, + } + + targets := earthfile.FilterTargets(func(target string) bool { + return target == "target1" + }) + + if len(targets) != 1 { + t.Errorf("expected 1 target, got %d", len(targets)) + } + + if targets[0] != "target1" { + t.Errorf("expected target1, got %s", targets[0]) + } +} + +func TestParseEarthfile(t *testing.T) { + tests := []struct { + name string + content string + hasError bool + }{ + { + name: "valid earthfile", + hasError: false, + content: ` +VERSION 0.7 + +foo: + LET foo = bar +`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := ParseEarthfile(context.Background(), mocks.NewMockFileSeeker(test.content)) + if test.hasError && err == nil { + t.Error("expected error, got nil") + } + + if !test.hasError && err != nil { + t.Errorf("expected no error, got %v", err) + } + }) + } +} diff --git a/forge/cli/pkg/earthfile/scan.go b/forge/cli/pkg/earthfile/scan.go new file mode 100644 index 00000000..4bf10094 --- /dev/null +++ b/forge/cli/pkg/earthfile/scan.go @@ -0,0 +1,43 @@ +package earthfile + +import ( + "context" + "fmt" + "log/slog" + "path/filepath" + + w "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/walker" +) + +// ScanEarthfiles scans the given root path for Earthfiles and returns a map +// that maps the path of the Earthfile to the targets defined in the Earthfile. +func ScanEarthfiles(rootPath string, walker w.Walker, logger *slog.Logger) (map[string]Earthfile, error) { + earthfiles := make(map[string]Earthfile) + + err := walker.Walk(rootPath, func(path string, fileType w.FileType, openFile func() (w.FileSeeker, error)) error { + if fileType != w.FileTypeFile { + return nil + } else if filepath.Base(path) != "Earthfile" { + return nil + } + + file, err := openFile() + if err != nil { + return err + } + defer file.Close() + + logger.Info("parsing Earthfile", "path", path) + earthfile, err := ParseEarthfile(context.Background(), file) + if err != nil { + logger.Error("error parsing Earthfile", "path", path, "error", err) + return fmt.Errorf("error parsing %s: %w", path, err) + } + + earthfiles[path] = earthfile + + return nil + }) + + return earthfiles, err +} diff --git a/forge/cli/pkg/earthfile/scan_test.go b/forge/cli/pkg/earthfile/scan_test.go new file mode 100644 index 00000000..92097c8f --- /dev/null +++ b/forge/cli/pkg/earthfile/scan_test.go @@ -0,0 +1,142 @@ +package earthfile + +import ( + "fmt" + "io" + "testing" + + "log/slog" + + "github.com/input-output-hk/catalyst-forge/forge/cli/internal/testutils/mocks" + "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/walker" +) + +func TestScanEarthfiles(t *testing.T) { + tests := []struct { + callbackErr error + walkErr error + files map[string]string + expectedResult map[string][]string + name string + }{ + { + name: "one earthfile", + files: map[string]string{ + "/tmp1/Earthfile": ` +VERSION 0.7 + +foo1: + LET foo = bar + +foo2: + LET foo = bar +`, + }, + expectedResult: map[string][]string{ + "/tmp1/Earthfile": {"foo1", "foo2"}, + }, + callbackErr: nil, + walkErr: nil, + }, + { + name: "multiple earthfiles", + files: map[string]string{ + "/tmp1/Earthfile": ` +VERSION 0.7 + +foo1: + LET foo = bar +`, + "/tmp2/Earthfile": ` +VERSION 0.7 + +foo2: + LET foo = bar +`, + }, + expectedResult: map[string][]string{ + "/tmp1/Earthfile": {"foo1"}, + "/tmp2/Earthfile": {"foo2"}, + }, + callbackErr: nil, + walkErr: nil, + }, + { + name: "callback error", + files: map[string]string{ + "/tmp1/Earthfile": ` +VERSION 0.7 + +foo1: + LET foo = bar +`, + }, + expectedResult: map[string][]string{}, + callbackErr: fmt.Errorf("callback error"), + walkErr: nil, + }, + { + name: "walk error", + files: map[string]string{ + "/tmp1/Earthfile": ` +VERSION 0.7 + +foo1: + LET foo = bar +`, + }, + expectedResult: map[string][]string{}, + callbackErr: nil, + walkErr: fmt.Errorf("walk error"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + walker := &walker.WalkerMock{ + WalkFunc: func(rootPath string, callback walker.WalkerCallback) error { + for path, content := range tt.files { + err := callback(path, walker.FileTypeFile, func() (walker.FileSeeker, error) { + return mocks.NewMockFileSeeker(content), tt.callbackErr + }) + + if err != nil { + return err + } + } + + return tt.walkErr + }, + } + result, err := ScanEarthfiles("", walker, slog.New(slog.NewTextHandler(io.Discard, nil))) + + if tt.callbackErr != nil && err == nil { + t.Error("expected error, got nil") + } else if tt.walkErr != nil && err == nil { + t.Error("expected error, got nil") + } else if tt.callbackErr == nil && tt.walkErr == nil && err != nil { + t.Errorf("expected no error, got %v", err) + } else { + if err != nil { + return + } + } + + if len(result) != len(tt.expectedResult) { + t.Errorf("expected %d earthfiles, got %d", len(tt.expectedResult), len(result)) + } + + for path, targets := range tt.expectedResult { + if len(result[path].Targets()) != len(targets) { + t.Errorf("expected %d targets for %s, got %d", len(targets), path, len(result[path].Targets())) + } + + for i, target := range targets { + if result[path].Targets()[i] != target { + t.Errorf("expected target %s at index %d, got %s", target, i, result[path].Targets()[i]) + } + } + } + }) + } +} diff --git a/forge/cli/pkg/earthly/earthly.go b/forge/cli/pkg/earthly/earthly.go new file mode 100644 index 00000000..573b4a8f --- /dev/null +++ b/forge/cli/pkg/earthly/earthly.go @@ -0,0 +1,256 @@ +package earthly + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "regexp" + "strconv" + "strings" + + "github.com/input-output-hk/catalyst-forge/blueprint/schema" + "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/executor" + secretstore "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/secrets" +) + +// EarthlyExecutorOption is an option for configuring an EarthlyExecutor. +type EarthlyExecutorOption func(e *EarthlyExecutor) + +// EarthlySecret represents a secret to be passed to Earthly. +type EarthlySecret struct { + Id string + Value string +} + +// earthlyExecutorOptions contains the configuration options for an +// EarthlyExecutor. +type earthlyExecutorOptions struct { + artifact bool + retries int +} + +// EarthlyExecutor is an Executor that runs Earthly targets. +type EarthlyExecutor struct { + logger *slog.Logger + opts earthlyExecutorOptions + executor executor.Executor + earthfile string + earthlyArgs []string + secrets []schema.Secret + secretsStore secretstore.SecretStore + target string + targetArgs []string +} + +// EarthlyExecutionResult contains the results of an Earthly execution. +type EarthlyExecutionResult struct { + Artifacts map[string]string `json:"artifacts"` + Images map[string]string `json:"images"` +} + +// Run executes the Earthly target and returns the resulting images and +// artifacts. +func (e EarthlyExecutor) Run() (EarthlyExecutionResult, error) { + var ( + err error + secrets []EarthlySecret + ) + + if e.secrets != nil { + secrets, err = e.buildSecrets() + if err != nil { + return EarthlyExecutionResult{}, err + } + + var secretString []string + for _, secret := range secrets { + e.logger.Info("Adding Earthly secret", "earthly_id", secret.Id, "value", secret.Value) + secretString = append(secretString, fmt.Sprintf("%s=%s", secret.Id, secret.Value)) + } + + if err := os.Setenv("EARTHLY_SECRETS", strings.Join(secretString, ",")); err != nil { + e.logger.Error("Failed to set secret environment varibles", "envvar", "EARTHLY_SECRETS") + } + } + + var output []byte + arguments := e.buildArguments() + for i := 0; i < e.opts.retries+1; i++ { + e.logger.Info("Executing Earthly", "attempt", i, "retries", e.opts.retries, "arguments", arguments) + output, err = e.executor.Execute("earthly", arguments) + if err == nil { + break + } + + e.logger.Error("Failed to run Earthly", "error", err) + } + + if err != nil { + e.logger.Error("Failed to run Earthly", "error", err) + return EarthlyExecutionResult{}, fmt.Errorf("failed to run Earthly: %w", err) + } + + return parseResult(string(output)), nil +} + +// buildArguments constructs the arguments to pass to the Earthly target. +func (e *EarthlyExecutor) buildArguments() []string { + var earthlyArgs []string + + earthlyArgs = append(earthlyArgs, e.earthlyArgs...) + + if e.opts.artifact { + earthlyArgs = append(earthlyArgs, "--artifact", fmt.Sprintf("%s+%s/*", e.earthfile, e.target)) + } else { + earthlyArgs = append(earthlyArgs, fmt.Sprintf("%s+%s", e.earthfile, e.target)) + } + + earthlyArgs = append(earthlyArgs, e.targetArgs...) + + return earthlyArgs +} + +// buildSecrets constructs the secrets to pass to Earthly. +func (e *EarthlyExecutor) buildSecrets() ([]EarthlySecret, error) { + var secrets []EarthlySecret + + for _, secret := range e.secrets { + secretClient, err := e.secretsStore.NewClient(e.logger, secretstore.Provider(secret.Provider)) + if err != nil { + e.logger.Error("Unable to create new secret client", "provider", secret.Provider, "error", err) + return secrets, fmt.Errorf("unable to create new secret client: %w", err) + } + + s, err := secretClient.Get(secret.Path) + if err != nil { + e.logger.Error("Unable to get secret", "provider", secret.Provider, "path", secret.Path, "error", err) + return secrets, fmt.Errorf("unable to get secret %s from provider: %s", secret.Path, secret.Provider) + } + + var secretValues map[string]interface{} + + if err := json.Unmarshal([]byte(s), &secretValues); err != nil { + e.logger.Error("Unable to unmarshal secret value", "provider", secret.Provider, "path", secret.Path, "error", err) + return secrets, fmt.Errorf("unable to unmarshal secret value: %w", err) + } + + for sk, eid := range secret.Maps { + if _, ok := secretValues[sk]; !ok { + e.logger.Error("Secret key not found in secret values", "key", sk, "provider", secret.Provider, "path", secret.Path) + return nil, fmt.Errorf("secret key not found in secret values: %s", sk) + } + + s := EarthlySecret{ + Id: eid, + } + + switch t := secretValues[sk].(type) { + case bool: + s.Value = strconv.FormatBool(t) + case int: + s.Value = strconv.FormatInt(int64(t), 10) + default: + s.Value = t.(string) + } + + secrets = append(secrets, s) + } + } + + return secrets, nil +} + +// NewEarthlyExecutor creates a new EarthlyExecutor. +func NewEarthlyExecutor( + earthfile, target string, + executor executor.Executor, + store secretstore.SecretStore, + logger *slog.Logger, + opts ...EarthlyExecutorOption, +) EarthlyExecutor { + e := EarthlyExecutor{ + earthfile: earthfile, + executor: executor, + logger: logger, + secretsStore: store, + target: target, + opts: earthlyExecutorOptions{}, + } + + for _, opt := range opts { + opt(&e) + } + + return e +} + +// WithArtifact is an option for configuring an EarthlyExecutor to output all +// artifacts contained within the given target into the local working directory. +func WithArtifact() EarthlyExecutorOption { + return func(e *EarthlyExecutor) { + e.opts.artifact = true + } +} + +// WithPrivileged is an option for configuring an EarthlyExecutor to run the +// Earthly target with elevated privileges. +func WithPrivileged() EarthlyExecutorOption { + return func(e *EarthlyExecutor) { + e.earthlyArgs = append(e.earthlyArgs, "--allow-privileged") + } +} + +// WithRetries is an option for configuring an EarthlyExecutor with the number +// of retries to attempt if the Earthly target fails. +func WithRetries(retries int) EarthlyExecutorOption { + return func(e *EarthlyExecutor) { + e.opts.retries = retries + } +} + +// WithSatellite is an option for configuring an EarthlyExecutor with the +// remote satellite to use. +func WithSatellite(s string) EarthlyExecutorOption { + return func(e *EarthlyExecutor) { + e.earthlyArgs = append(e.earthlyArgs, "--sat", s) + } +} + +// WithEarthlyArgs is an option for configuring an EarthlyExecutor with +// additional arguments that will be passed to the Earthly target. +func WithTargetArgs(args ...string) EarthlyExecutorOption { + return func(e *EarthlyExecutor) { + e.targetArgs = args + } +} + +func WithSecrets(secrets []schema.Secret) EarthlyExecutorOption { + return func(e *EarthlyExecutor) { + e.secrets = secrets + } +} + +// parseResult parses the output of an Earthly execution and returns the +// resulting images and artifacts. +func parseResult(output string) EarthlyExecutionResult { + images := make(map[string]string) + artifacts := make(map[string]string) + imageExpr := regexp.MustCompile(`^Image (.*?) output as (.*?)$`) + artifactExpr := regexp.MustCompile(`Artifact (.*?) output as (.*?)$`) + + for _, line := range strings.Split(string(output), "\n") { + if matches := imageExpr.FindStringSubmatch(line); matches != nil { + images[matches[1]] = matches[2] + } + + if matches := artifactExpr.FindStringSubmatch(line); matches != nil { + artifacts[matches[1]] = matches[2] + } + } + + return EarthlyExecutionResult{ + Artifacts: artifacts, + Images: images, + } +} diff --git a/forge/cli/pkg/earthly/earthly_test.go b/forge/cli/pkg/earthly/earthly_test.go new file mode 100644 index 00000000..67beeb4f --- /dev/null +++ b/forge/cli/pkg/earthly/earthly_test.go @@ -0,0 +1,394 @@ +package earthly + +import ( + "fmt" + "log/slog" + "maps" + "slices" + "testing" + + "github.com/input-output-hk/catalyst-forge/blueprint/schema" + "github.com/input-output-hk/catalyst-forge/forge/cli/internal/testutils" + "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/executor" + "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/secrets" +) + +func TestEarthlyExecutorRun(t *testing.T) { + tests := []struct { + expect EarthlyExecutionResult + name string + output string + earthlyExec EarthlyExecutor + mockExec executor.ExecutorMock + expectCalls int + expectErr bool + }{ + { + name: "simple", + earthlyExec: NewEarthlyExecutor("/test/dir", "foo", nil, secrets.SecretStore{}, + testutils.NewNoopLogger(), + ), + mockExec: executor.ExecutorMock{ + ExecuteFunc: func(command string, args []string) ([]byte, error) { + return []byte(`foobarbaz +Image foo output as bar +Artifact foo output as bar`), nil + }, + }, + expect: EarthlyExecutionResult{ + Images: map[string]string{ + "foo": "bar", + }, + Artifacts: map[string]string{ + "foo": "bar", + }, + }, + expectErr: false, + expectCalls: 1, + }, + { + name: "with retries", + earthlyExec: NewEarthlyExecutor("/test/dir", "foo", nil, secrets.SecretStore{}, + testutils.NewNoopLogger(), + WithRetries(3), + ), + mockExec: executor.ExecutorMock{ + ExecuteFunc: func(command string, args []string) ([]byte, error) { + return []byte{}, fmt.Errorf("error") + }, + }, + expect: EarthlyExecutionResult{ + Images: map[string]string{}, + Artifacts: map[string]string{}, + }, + expectErr: true, + expectCalls: 4, + }, + } + + for i := range tests { + tt := &tests[i] // Required to avoid copying the generaetd RWMutex + t.Run(tt.name, func(t *testing.T) { + tt.earthlyExec.executor = &tt.mockExec + got, err := tt.earthlyExec.Run() + + if tt.expectErr && err == nil { + t.Errorf("expected error, got nil") + } else if !tt.expectErr && err != nil { + t.Errorf("unexpected error: %v", err) + } + + if len(tt.mockExec.ExecuteCalls()) != tt.expectCalls { + t.Errorf("expected %d calls to Execute, got %d", tt.expectCalls, len(tt.mockExec.ExecuteCalls())) + } + + if !maps.Equal(got.Artifacts, tt.expect.Artifacts) { + t.Errorf("expected %v, got %v", tt.expect.Artifacts, got.Artifacts) + } + + if !maps.Equal(got.Images, tt.expect.Images) { + t.Errorf("expected %v, got %v", tt.expect.Images, got.Images) + } + }) + } +} + +func TestEarthlyExecutor_buildArguments(t *testing.T) { + tests := []struct { + name string + e EarthlyExecutor + expect []string + }{ + { + name: "simple", + e: NewEarthlyExecutor("/test/dir", "foo", nil, secrets.SecretStore{}, + testutils.NewNoopLogger(), + ), + expect: []string{"/test/dir+foo"}, + }, + { + name: "with target args", + e: NewEarthlyExecutor("/test/dir", "foo", nil, secrets.SecretStore{}, + testutils.NewNoopLogger(), + WithTargetArgs("--arg1", "foo", "--arg2", "bar"), + ), + expect: []string{"/test/dir+foo", "--arg1", "foo", "--arg2", "bar"}, + }, + { + name: "with artifact", + e: NewEarthlyExecutor("/test/dir", "foo", nil, secrets.SecretStore{}, + testutils.NewNoopLogger(), + WithArtifact(), + ), + expect: []string{"--artifact", "/test/dir+foo/*"}, + }, + { + name: "with privileged", + e: NewEarthlyExecutor("/test/dir", "foo", nil, secrets.SecretStore{}, + testutils.NewNoopLogger(), + WithPrivileged(), + ), + expect: []string{"--allow-privileged", "/test/dir+foo"}, + }, + { + name: "with satellite", + e: NewEarthlyExecutor("/test/dir", "foo", nil, secrets.SecretStore{}, + testutils.NewNoopLogger(), + WithSatellite("satellite"), + ), + expect: []string{"--sat", "satellite", "/test/dir+foo"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.e.buildArguments() + if !slices.Equal(got, tt.expect) { + t.Errorf("expected %v, got %v", tt.expect, got) + } + }) + } +} + +func TestEarthlyExecutor_buildSecrets(t *testing.T) { + tests := []struct { + name string + provider secrets.SecretProvider + secrets []schema.Secret + expect []EarthlySecret + expectErr bool + expectedErr error + }{ + { + name: "simple", + provider: &secrets.SecretProviderMock{ + GetFunc: func(path string) (string, error) { + return `{"key": "value"}`, nil + }, + }, + secrets: []schema.Secret{ + { + Path: "path", + Provider: "mock", + Maps: map[string]string{ + "key": "id", + }, + }, + }, + expect: []EarthlySecret{ + { + Id: "id", + Value: "value", + }, + }, + expectErr: false, + expectedErr: nil, + }, + { + name: "key does not exist", + provider: &secrets.SecretProviderMock{ + GetFunc: func(path string) (string, error) { + return `{"key": "value"}`, nil + }, + }, + secrets: []schema.Secret{ + { + Path: "path", + Provider: "mock", + Maps: map[string]string{ + "key1": "id1", + }, + }, + }, + expect: nil, + expectErr: true, + expectedErr: fmt.Errorf("secret key not found in secret values: key1"), + }, + { + name: "invalid JSON", + provider: &secrets.SecretProviderMock{ + GetFunc: func(path string) (string, error) { + return `invalid`, nil + }, + }, + secrets: []schema.Secret{ + { + Path: "path", + Provider: "mock", + Maps: map[string]string{}, + }, + }, + expect: nil, + expectErr: true, + expectedErr: fmt.Errorf("unable to unmarshal secret value: invalid character 'i' looking for beginning of value"), + }, + { + name: "secret provider does not exist", + provider: &secrets.SecretProviderMock{ + GetFunc: func(path string) (string, error) { + return "", nil + }, + }, + secrets: []schema.Secret{ + { + Path: "path", + Provider: "bad", + Maps: map[string]string{}, + }, + }, + expect: nil, + expectErr: true, + expectedErr: fmt.Errorf("unable to create new secret client: unknown secret provider: bad"), + }, + { + name: "secret provider error", + provider: &secrets.SecretProviderMock{ + GetFunc: func(path string) (string, error) { + return "", fmt.Errorf("error") + }, + }, + secrets: []schema.Secret{ + { + Path: "path", + Provider: "mock", + Maps: map[string]string{}, + }, + }, + expect: nil, + expectErr: true, + expectedErr: fmt.Errorf("unable to get secret path from provider: mock"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store := secrets.NewSecretStore(map[secrets.Provider]func(*slog.Logger) (secrets.SecretProvider, error){ + secrets.Provider("mock"): func(logger *slog.Logger) (secrets.SecretProvider, error) { + return tt.provider, nil + }, + }) + + executor := NewEarthlyExecutor("", "", nil, store, testutils.NewNoopLogger()) + executor.secrets = tt.secrets + got, err := executor.buildSecrets() + + ret, err := testutils.CheckError(t, err, tt.expectErr, tt.expectedErr) + if err != nil { + t.Error(err) + return + } else if ret { + return + } + + if !slices.Equal(got, tt.expect) { + t.Errorf("expected %v, got %v", tt.expect, got) + } + }) + } +} + +func Test_parseOutput(t *testing.T) { + tests := []struct { + expect EarthlyExecutionResult + name string + output string + }{ + { + name: "simple", + output: `foobarbaz +Image foo output as bar +Artifact foo output as bar`, + expect: EarthlyExecutionResult{ + Images: map[string]string{ + "foo": "bar", + }, + Artifacts: map[string]string{ + "foo": "bar", + }, + }, + }, + { + name: "no output", + output: "", + expect: EarthlyExecutionResult{ + Images: map[string]string{}, + Artifacts: map[string]string{}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseResult(tt.output) + if !maps.Equal(got.Images, tt.expect.Images) { + t.Errorf("expected %v, got %v", tt.expect.Images, got.Images) + } + if !maps.Equal(got.Artifacts, tt.expect.Artifacts) { + t.Errorf("expected %v, got %v", tt.expect.Artifacts, got.Artifacts) + } + }) + } +} + +// var data string = "{\"secret_key\":\"secret_value\"}" + +// func setup(t *testing.T) (*slog.Logger, string) { +// t.Helper() + +// handler := log.New(os.Stderr) +// handler.SetLevel(log.DebugLevel) +// logger := slog.New(handler) + +// f, err := os.CreateTemp("", "tmpfile-") +// if err != nil { +// t.Fatal(err) +// } +// defer f.Close() + +// b := []byte(data) +// if _, err := f.Write(b); err != nil { +// log.Fatal(err) +// } + +// return logger, f.Name() +// } + +// func TestCreateMapFromSecrets(t *testing.T) { +// logger, tmpFile := setup(t) + +// t.Cleanup(func() { +// os.Remove(tmpFile) +// }) + +// prefix := "path/to/earthfile" + +// c := &cue.Config{ +// Secrets: map[string]*cue.Secret{ +// "my_secret": { +// Provider: "local", +// Path: tmpFile, +// Maps: map[string]string{ +// "secret_key": "secret_id_for_earthly", +// }, +// }, +// }, +// } + +// var secrets []*cue.Secret + +// for _, v := range c.Secrets { +// secrets = append(secrets, v) +// } + +// got, err := newSecrets(logger, secrets, prefix) + +// want := []*Secret{ +// { +// Value: "secret_value", +// }, +// } + +// assert.Nil(t, err) +// assert.NotNil(t, got) +// assert.Equal(t, want[0].Value, got[0].Value) +// } diff --git a/forge/cli/pkg/executor/executor.go b/forge/cli/pkg/executor/executor.go new file mode 100644 index 00000000..750dfdf8 --- /dev/null +++ b/forge/cli/pkg/executor/executor.go @@ -0,0 +1,9 @@ +package executor + +//go:generate go run github.com/matryer/moq@latest -out executor_mock.go . Executor + +// Executor is an interface for executing commands. +type Executor interface { + // Execute executes the given command + Execute(command string, args []string) ([]byte, error) +} diff --git a/forge/cli/pkg/executor/executor_mock.go b/forge/cli/pkg/executor/executor_mock.go new file mode 100644 index 00000000..4e24f5b5 --- /dev/null +++ b/forge/cli/pkg/executor/executor_mock.go @@ -0,0 +1,80 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package executor + +import ( + "sync" +) + +// Ensure, that ExecutorMock does implement Executor. +// If this is not the case, regenerate this file with moq. +var _ Executor = &ExecutorMock{} + +// ExecutorMock is a mock implementation of Executor. +// +// func TestSomethingThatUsesExecutor(t *testing.T) { +// +// // make and configure a mocked Executor +// mockedExecutor := &ExecutorMock{ +// ExecuteFunc: func(command string, args []string) ([]byte, error) { +// panic("mock out the Execute method") +// }, +// } +// +// // use mockedExecutor in code that requires Executor +// // and then make assertions. +// +// } +type ExecutorMock struct { + // ExecuteFunc mocks the Execute method. + ExecuteFunc func(command string, args []string) ([]byte, error) + + // calls tracks calls to the methods. + calls struct { + // Execute holds details about calls to the Execute method. + Execute []struct { + // Command is the command argument value. + Command string + // Args is the args argument value. + Args []string + } + } + lockExecute sync.RWMutex +} + +// Execute calls ExecuteFunc. +func (mock *ExecutorMock) Execute(command string, args []string) ([]byte, error) { + if mock.ExecuteFunc == nil { + panic("ExecutorMock.ExecuteFunc: method is nil but Executor.Execute was just called") + } + callInfo := struct { + Command string + Args []string + }{ + Command: command, + Args: args, + } + mock.lockExecute.Lock() + mock.calls.Execute = append(mock.calls.Execute, callInfo) + mock.lockExecute.Unlock() + return mock.ExecuteFunc(command, args) +} + +// ExecuteCalls gets all the calls that were made to Execute. +// Check the length with: +// +// len(mockedExecutor.ExecuteCalls()) +func (mock *ExecutorMock) ExecuteCalls() []struct { + Command string + Args []string +} { + var calls []struct { + Command string + Args []string + } + mock.lockExecute.RLock() + calls = mock.calls.Execute + mock.lockExecute.RUnlock() + return calls +} diff --git a/forge/cli/pkg/executor/local.go b/forge/cli/pkg/executor/local.go new file mode 100644 index 00000000..843aaf05 --- /dev/null +++ b/forge/cli/pkg/executor/local.go @@ -0,0 +1,89 @@ +package executor + +import ( + "bytes" + "io" + "log/slog" + "os" + "os/exec" +) + +// LocalExecutorOption is an option for configuring a LocalExecutor. +type LocalExecutorOption func(e *LocalExecutor) + +// LocalExecutor is an Executor that runs commands locally. +type LocalExecutor struct { + logger *slog.Logger + redirect bool +} + +func (e *LocalExecutor) Execute(command string, args []string) ([]byte, error) { + cmd := exec.Command(command, args...) + e.logger.Debug("Executing local command", "command", cmd.String()) + + if e.redirect { + var buffer bytes.Buffer + errChan := make(chan error, 2) + + stdoutPipe, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + + stderrPipe, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + + stdoutWriter := io.MultiWriter(os.Stdout, &buffer) + stderrWriter := io.MultiWriter(os.Stderr, &buffer) + + if err := cmd.Start(); err != nil { + return nil, err + } + + go func() { + _, err := io.Copy(stdoutWriter, stdoutPipe) + errChan <- err + }() + + go func() { + _, err := io.Copy(stderrWriter, stderrPipe) + errChan <- err + }() + + if err := cmd.Wait(); err != nil { + return nil, err + } + + if err := <-errChan; err != nil { + return nil, err + } + + return buffer.Bytes(), nil + } + + return cmd.CombinedOutput() +} + +// WithRedirect is an option that configures the LocalExecutor to redirect the +// stdout and stderr of the commands to the stdout and stderr of the local +// process. +func WithRedirect() LocalExecutorOption { + return func(e *LocalExecutor) { + e.redirect = true + } +} + +// NewLocalExecutor creates a new LocalExecutor with the given options. +func NewLocalExecutor(logger *slog.Logger, options ...LocalExecutorOption) *LocalExecutor { + e := &LocalExecutor{ + logger: logger, + } + + for _, option := range options { + option(e) + } + + return e +} diff --git a/forge/cli/pkg/secrets/client.go b/forge/cli/pkg/secrets/client.go new file mode 100644 index 00000000..88563639 --- /dev/null +++ b/forge/cli/pkg/secrets/client.go @@ -0,0 +1,41 @@ +package secrets + +import ( + "fmt" + "log/slog" + + "github.com/input-output-hk/catalyst-forge/forge/cli/pkg/secrets/providers" +) + +// SecretStore is a store of secret providers. +type SecretStore struct { + store map[Provider]func(*slog.Logger) (SecretProvider, error) +} + +// NewDefaultSecretStore returns a new SecretStore with the default providers. +func NewDefaultSecretStore() SecretStore { + return SecretStore{ + store: map[Provider]func(*slog.Logger) (SecretProvider, error){ + ProviderLocal: func(logger *slog.Logger) (SecretProvider, error) { + return providers.NewLocalClient(logger) + }, + ProviderAWS: func(logger *slog.Logger) (SecretProvider, error) { + return providers.NewDefaultAWSClient(logger) + }, + }, + } +} + +// NewSecretStore returns a new SecretStore with the given providers. +func NewSecretStore(store map[Provider]func(*slog.Logger) (SecretProvider, error)) SecretStore { + return SecretStore{store: store} +} + +// NewClient returns a new SecretProvider client for the given provider. +func (s SecretStore) NewClient(logger *slog.Logger, p Provider) (SecretProvider, error) { + if f, ok := s.store[p]; ok { + return f(logger) + } + + return nil, fmt.Errorf("unknown secret provider: %s", p) +} diff --git a/forge/cli/pkg/secrets/interface.go b/forge/cli/pkg/secrets/interface.go new file mode 100644 index 00000000..9f80b968 --- /dev/null +++ b/forge/cli/pkg/secrets/interface.go @@ -0,0 +1,9 @@ +package secrets + +//go:generate go run github.com/matryer/moq@latest -out interface_mock.go . SecretProvider + +// SecretProvider is an interface for getting and setting secrets. +type SecretProvider interface { + Get(key string) (string, error) + Set(key, value string) (string, error) +} diff --git a/forge/cli/pkg/secrets/interface_mock.go b/forge/cli/pkg/secrets/interface_mock.go new file mode 100644 index 00000000..1b02028b --- /dev/null +++ b/forge/cli/pkg/secrets/interface_mock.go @@ -0,0 +1,124 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package secrets + +import ( + "sync" +) + +// Ensure, that SecretProviderMock does implement SecretProvider. +// If this is not the case, regenerate this file with moq. +var _ SecretProvider = &SecretProviderMock{} + +// SecretProviderMock is a mock implementation of SecretProvider. +// +// func TestSomethingThatUsesSecretProvider(t *testing.T) { +// +// // make and configure a mocked SecretProvider +// mockedSecretProvider := &SecretProviderMock{ +// GetFunc: func(key string) (string, error) { +// panic("mock out the Get method") +// }, +// SetFunc: func(key string, value string) (string, error) { +// panic("mock out the Set method") +// }, +// } +// +// // use mockedSecretProvider in code that requires SecretProvider +// // and then make assertions. +// +// } +type SecretProviderMock struct { + // GetFunc mocks the Get method. + GetFunc func(key string) (string, error) + + // SetFunc mocks the Set method. + SetFunc func(key string, value string) (string, error) + + // calls tracks calls to the methods. + calls struct { + // Get holds details about calls to the Get method. + Get []struct { + // Key is the key argument value. + Key string + } + // Set holds details about calls to the Set method. + Set []struct { + // Key is the key argument value. + Key string + // Value is the value argument value. + Value string + } + } + lockGet sync.RWMutex + lockSet sync.RWMutex +} + +// Get calls GetFunc. +func (mock *SecretProviderMock) Get(key string) (string, error) { + if mock.GetFunc == nil { + panic("SecretProviderMock.GetFunc: method is nil but SecretProvider.Get was just called") + } + callInfo := struct { + Key string + }{ + Key: key, + } + mock.lockGet.Lock() + mock.calls.Get = append(mock.calls.Get, callInfo) + mock.lockGet.Unlock() + return mock.GetFunc(key) +} + +// GetCalls gets all the calls that were made to Get. +// Check the length with: +// +// len(mockedSecretProvider.GetCalls()) +func (mock *SecretProviderMock) GetCalls() []struct { + Key string +} { + var calls []struct { + Key string + } + mock.lockGet.RLock() + calls = mock.calls.Get + mock.lockGet.RUnlock() + return calls +} + +// Set calls SetFunc. +func (mock *SecretProviderMock) Set(key string, value string) (string, error) { + if mock.SetFunc == nil { + panic("SecretProviderMock.SetFunc: method is nil but SecretProvider.Set was just called") + } + callInfo := struct { + Key string + Value string + }{ + Key: key, + Value: value, + } + mock.lockSet.Lock() + mock.calls.Set = append(mock.calls.Set, callInfo) + mock.lockSet.Unlock() + return mock.SetFunc(key, value) +} + +// SetCalls gets all the calls that were made to Set. +// Check the length with: +// +// len(mockedSecretProvider.SetCalls()) +func (mock *SecretProviderMock) SetCalls() []struct { + Key string + Value string +} { + var calls []struct { + Key string + Value string + } + mock.lockSet.RLock() + calls = mock.calls.Set + mock.lockSet.RUnlock() + return calls +} diff --git a/forge/cli/pkg/secrets/providers.go b/forge/cli/pkg/secrets/providers.go new file mode 100644 index 00000000..420e3dcd --- /dev/null +++ b/forge/cli/pkg/secrets/providers.go @@ -0,0 +1,8 @@ +package secrets + +type Provider string + +const ( + ProviderLocal Provider = "local" + ProviderAWS Provider = "aws" +) diff --git a/forge/cli/pkg/secrets/providers/aws.go b/forge/cli/pkg/secrets/providers/aws.go new file mode 100644 index 00000000..2920d69c --- /dev/null +++ b/forge/cli/pkg/secrets/providers/aws.go @@ -0,0 +1,155 @@ +package providers + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + smtypes "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" + "github.com/aws/aws-sdk-go/aws" +) + +//go:generate go run github.com/matryer/moq@latest -out aws_mock_test.go . SecretsManagerClient + +var ( + AWSSecretsManagerResourceExistsException *smtypes.ResourceExistsException + AWSSecretsManagerInvalidRequestException *smtypes.InvalidRequestException +) + +// SecretsManagerClient is an interface for the AWS Secrets Manager client. +type SecretsManagerClient interface { + CreateSecret(ctx context.Context, params *secretsmanager.CreateSecretInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.CreateSecretOutput, error) + GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) + PutSecretValue(ctx context.Context, params *secretsmanager.PutSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.PutSecretValueOutput, error) +} + +// AWSClient is a client for interacting with AWS Secrets Manager. +type AWSClient struct { + logger *slog.Logger + client SecretsManagerClient +} + +// Get retrieves a secret from AWS Secrets Manager. +func (c *AWSClient) Get(key string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + c.logger.Debug("Getting secret from AWS Secretsmanager", "key", key) + + resp, err := c.client.GetSecretValue( + ctx, + &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(key), + }, + ) + if err != nil { + return "", fmt.Errorf("unable to get secret: %w", err) + } + + return *resp.SecretString, nil +} + +// Set sets a secret in AWS Secrets Manager. +func (c *AWSClient) Set(key, value string) (string, error) { + var ( + err error + versionId string + ) + + resp, err := c.createSecret(key, value) + if err != nil { + if errors.As(err, &AWSSecretsManagerResourceExistsException) { + c.logger.Warn("Secret already exists. Creating new secret version.") + resp, err := c.putSecretValue(key, value) + if err != nil { + return "", fmt.Errorf("unable to set secret: %w", err) + } + + versionId = *resp.VersionId + } + if errors.As(err, &AWSSecretsManagerInvalidRequestException) { + return "", fmt.Errorf("invalid request: %w", err) + } + + if versionId == "" { + return "", fmt.Errorf("unable to set secret: %w", err) + } + } + + if versionId == "" { + versionId = *resp.VersionId + } + + c.logger.Info("Successfully set secret using AWS Secretsmanager provider", "versionId", versionId) + + return versionId, nil +} + +// createSecret creates a new secret in AWS Secrets Manager. +func (c *AWSClient) createSecret(key, value string) (*secretsmanager.CreateSecretOutput, error) { + params := &secretsmanager.CreateSecretInput{ + Name: aws.String(key), + SecretString: aws.String(value), + Tags: []smtypes.Tag{ + { + Key: aws.String("CreatedBy"), + Value: aws.String("Forge"), + }, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + resp, err := c.client.CreateSecret(ctx, params) + if err != nil { + return nil, err + } + + return resp, nil +} + +// putSecretValue creates a new version of a secret in AWS Secrets Manager. +func (c *AWSClient) putSecretValue(key, value string) (*secretsmanager.PutSecretValueOutput, error) { + params := &secretsmanager.PutSecretValueInput{ + SecretId: aws.String(key), + SecretString: aws.String(value), + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + resp, err := c.client.PutSecretValue(ctx, params) + if err != nil { + return nil, err + } + + return resp, nil +} + +// NewAWSClient creates a new AWSClient with the default configuration. +func NewDefaultAWSClient(logger *slog.Logger) (*AWSClient, error) { + cfg, err := config.LoadDefaultConfig(context.Background(), + config.WithRegion("eu-central-1"), + ) + if err != nil { + return nil, fmt.Errorf("unable to load SDK config: %w", err) + } + + return &AWSClient{ + logger: logger, + client: secretsmanager.NewFromConfig(cfg), + }, nil +} + +// NewAWSClient creates a new AWSClient. +func NewAWSClient(client SecretsManagerClient, logger *slog.Logger) *AWSClient { + return &AWSClient{ + logger: logger, + client: client, + } +} diff --git a/forge/cli/pkg/secrets/providers/aws_mock_test.go b/forge/cli/pkg/secrets/providers/aws_mock_test.go new file mode 100644 index 00000000..f632c2bf --- /dev/null +++ b/forge/cli/pkg/secrets/providers/aws_mock_test.go @@ -0,0 +1,200 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package providers + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "sync" +) + +// Ensure, that SecretsManagerClientMock does implement SecretsManagerClient. +// If this is not the case, regenerate this file with moq. +var _ SecretsManagerClient = &SecretsManagerClientMock{} + +// SecretsManagerClientMock is a mock implementation of SecretsManagerClient. +// +// func TestSomethingThatUsesSecretsManagerClient(t *testing.T) { +// +// // make and configure a mocked SecretsManagerClient +// mockedSecretsManagerClient := &SecretsManagerClientMock{ +// CreateSecretFunc: func(ctx context.Context, params *secretsmanager.CreateSecretInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.CreateSecretOutput, error) { +// panic("mock out the CreateSecret method") +// }, +// GetSecretValueFunc: func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { +// panic("mock out the GetSecretValue method") +// }, +// PutSecretValueFunc: func(ctx context.Context, params *secretsmanager.PutSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.PutSecretValueOutput, error) { +// panic("mock out the PutSecretValue method") +// }, +// } +// +// // use mockedSecretsManagerClient in code that requires SecretsManagerClient +// // and then make assertions. +// +// } +type SecretsManagerClientMock struct { + // CreateSecretFunc mocks the CreateSecret method. + CreateSecretFunc func(ctx context.Context, params *secretsmanager.CreateSecretInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.CreateSecretOutput, error) + + // GetSecretValueFunc mocks the GetSecretValue method. + GetSecretValueFunc func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) + + // PutSecretValueFunc mocks the PutSecretValue method. + PutSecretValueFunc func(ctx context.Context, params *secretsmanager.PutSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.PutSecretValueOutput, error) + + // calls tracks calls to the methods. + calls struct { + // CreateSecret holds details about calls to the CreateSecret method. + CreateSecret []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Params is the params argument value. + Params *secretsmanager.CreateSecretInput + // OptFns is the optFns argument value. + OptFns []func(*secretsmanager.Options) + } + // GetSecretValue holds details about calls to the GetSecretValue method. + GetSecretValue []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Params is the params argument value. + Params *secretsmanager.GetSecretValueInput + // OptFns is the optFns argument value. + OptFns []func(*secretsmanager.Options) + } + // PutSecretValue holds details about calls to the PutSecretValue method. + PutSecretValue []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Params is the params argument value. + Params *secretsmanager.PutSecretValueInput + // OptFns is the optFns argument value. + OptFns []func(*secretsmanager.Options) + } + } + lockCreateSecret sync.RWMutex + lockGetSecretValue sync.RWMutex + lockPutSecretValue sync.RWMutex +} + +// CreateSecret calls CreateSecretFunc. +func (mock *SecretsManagerClientMock) CreateSecret(ctx context.Context, params *secretsmanager.CreateSecretInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.CreateSecretOutput, error) { + if mock.CreateSecretFunc == nil { + panic("SecretsManagerClientMock.CreateSecretFunc: method is nil but SecretsManagerClient.CreateSecret was just called") + } + callInfo := struct { + Ctx context.Context + Params *secretsmanager.CreateSecretInput + OptFns []func(*secretsmanager.Options) + }{ + Ctx: ctx, + Params: params, + OptFns: optFns, + } + mock.lockCreateSecret.Lock() + mock.calls.CreateSecret = append(mock.calls.CreateSecret, callInfo) + mock.lockCreateSecret.Unlock() + return mock.CreateSecretFunc(ctx, params, optFns...) +} + +// CreateSecretCalls gets all the calls that were made to CreateSecret. +// Check the length with: +// +// len(mockedSecretsManagerClient.CreateSecretCalls()) +func (mock *SecretsManagerClientMock) CreateSecretCalls() []struct { + Ctx context.Context + Params *secretsmanager.CreateSecretInput + OptFns []func(*secretsmanager.Options) +} { + var calls []struct { + Ctx context.Context + Params *secretsmanager.CreateSecretInput + OptFns []func(*secretsmanager.Options) + } + mock.lockCreateSecret.RLock() + calls = mock.calls.CreateSecret + mock.lockCreateSecret.RUnlock() + return calls +} + +// GetSecretValue calls GetSecretValueFunc. +func (mock *SecretsManagerClientMock) GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + if mock.GetSecretValueFunc == nil { + panic("SecretsManagerClientMock.GetSecretValueFunc: method is nil but SecretsManagerClient.GetSecretValue was just called") + } + callInfo := struct { + Ctx context.Context + Params *secretsmanager.GetSecretValueInput + OptFns []func(*secretsmanager.Options) + }{ + Ctx: ctx, + Params: params, + OptFns: optFns, + } + mock.lockGetSecretValue.Lock() + mock.calls.GetSecretValue = append(mock.calls.GetSecretValue, callInfo) + mock.lockGetSecretValue.Unlock() + return mock.GetSecretValueFunc(ctx, params, optFns...) +} + +// GetSecretValueCalls gets all the calls that were made to GetSecretValue. +// Check the length with: +// +// len(mockedSecretsManagerClient.GetSecretValueCalls()) +func (mock *SecretsManagerClientMock) GetSecretValueCalls() []struct { + Ctx context.Context + Params *secretsmanager.GetSecretValueInput + OptFns []func(*secretsmanager.Options) +} { + var calls []struct { + Ctx context.Context + Params *secretsmanager.GetSecretValueInput + OptFns []func(*secretsmanager.Options) + } + mock.lockGetSecretValue.RLock() + calls = mock.calls.GetSecretValue + mock.lockGetSecretValue.RUnlock() + return calls +} + +// PutSecretValue calls PutSecretValueFunc. +func (mock *SecretsManagerClientMock) PutSecretValue(ctx context.Context, params *secretsmanager.PutSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.PutSecretValueOutput, error) { + if mock.PutSecretValueFunc == nil { + panic("SecretsManagerClientMock.PutSecretValueFunc: method is nil but SecretsManagerClient.PutSecretValue was just called") + } + callInfo := struct { + Ctx context.Context + Params *secretsmanager.PutSecretValueInput + OptFns []func(*secretsmanager.Options) + }{ + Ctx: ctx, + Params: params, + OptFns: optFns, + } + mock.lockPutSecretValue.Lock() + mock.calls.PutSecretValue = append(mock.calls.PutSecretValue, callInfo) + mock.lockPutSecretValue.Unlock() + return mock.PutSecretValueFunc(ctx, params, optFns...) +} + +// PutSecretValueCalls gets all the calls that were made to PutSecretValue. +// Check the length with: +// +// len(mockedSecretsManagerClient.PutSecretValueCalls()) +func (mock *SecretsManagerClientMock) PutSecretValueCalls() []struct { + Ctx context.Context + Params *secretsmanager.PutSecretValueInput + OptFns []func(*secretsmanager.Options) +} { + var calls []struct { + Ctx context.Context + Params *secretsmanager.PutSecretValueInput + OptFns []func(*secretsmanager.Options) + } + mock.lockPutSecretValue.RLock() + calls = mock.calls.PutSecretValue + mock.lockPutSecretValue.RUnlock() + return calls +} diff --git a/forge/cli/pkg/secrets/providers/aws_test.go b/forge/cli/pkg/secrets/providers/aws_test.go new file mode 100644 index 00000000..f13b6869 --- /dev/null +++ b/forge/cli/pkg/secrets/providers/aws_test.go @@ -0,0 +1,218 @@ +package providers + +import ( + "context" + "fmt" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go/aws" + "github.com/input-output-hk/catalyst-forge/forge/cli/internal/testutils" +) + +func TestAWSClientGet(t *testing.T) { + tests := []struct { + name string + path string + mock SecretsManagerClientMock + expect string + expectErr bool + expectedErr error + cond func(*SecretsManagerClientMock) error + }{ + { + name: "simple", + path: "path", + mock: SecretsManagerClientMock{ + GetSecretValueFunc: func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + return &secretsmanager.GetSecretValueOutput{ + SecretString: aws.String("secret"), + }, nil + }, + }, + expect: "secret", + expectErr: false, + expectedErr: nil, + cond: func(m *SecretsManagerClientMock) error { + if len(m.calls.GetSecretValue) != 1 { + return fmt.Errorf("expected GetSecretValue to be called once, got %d", len(m.calls.GetSecretValue)) + } + + if *m.calls.GetSecretValue[0].Params.SecretId != "path" { + return fmt.Errorf("expected GetSecretValue to be called with path, got %s", *m.calls.GetSecretValue[0].Params.SecretId) + } + + return nil + }, + }, + { + name: "error", + mock: SecretsManagerClientMock{ + GetSecretValueFunc: func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + return nil, fmt.Errorf("error") + }, + }, + expect: "", + expectErr: true, + expectedErr: fmt.Errorf("unable to get secret: error"), + }, + } + + for i := range tests { + tt := &tests[i] // Required to avoid copying the generaetd RWMutex + t.Run(tt.name, func(t *testing.T) { + client := &AWSClient{ + client: &tt.mock, + logger: testutils.NewNoopLogger(), + } + + got, err := client.Get(tt.path) + + ret, err := testutils.CheckError(t, err, tt.expectErr, tt.expectedErr) + if err != nil { + t.Error(err) + return + } else if ret { + return + } + + if tt.cond != nil { + if err := tt.cond(&tt.mock); err != nil { + t.Error(err) + return + } + } + + if got != tt.expect { + t.Errorf("expected: %s, got: %s", tt.expect, got) + } + }) + } +} + +func TestAWSClientSet(t *testing.T) { + tests := []struct { + name string + mock SecretsManagerClientMock + expect string + expectErr bool + expectedErr error + cond func(*SecretsManagerClientMock) error + }{ + { + name: "simple", + mock: SecretsManagerClientMock{ + CreateSecretFunc: func(ctx context.Context, params *secretsmanager.CreateSecretInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.CreateSecretOutput, error) { + return &secretsmanager.CreateSecretOutput{ + VersionId: aws.String("version"), + }, nil + }, + }, + expect: "version", + expectErr: false, + expectedErr: nil, + cond: func(m *SecretsManagerClientMock) error { + if len(m.calls.CreateSecret) != 1 { + return fmt.Errorf("expected CreateSecret to be called once, got %d", len(m.calls.CreateSecret)) + } + + if *m.calls.CreateSecret[0].Params.Name != "path" { + return fmt.Errorf("expected CreateSecret to be called with path, got %s", *m.calls.CreateSecret[0].Params.Name) + } + + return nil + }, + }, + { + name: "secret already exists", + mock: SecretsManagerClientMock{ + CreateSecretFunc: func(ctx context.Context, params *secretsmanager.CreateSecretInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.CreateSecretOutput, error) { + return nil, AWSSecretsManagerResourceExistsException + }, + PutSecretValueFunc: func(ctx context.Context, params *secretsmanager.PutSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.PutSecretValueOutput, error) { + return &secretsmanager.PutSecretValueOutput{ + VersionId: aws.String("version"), + }, nil + }, + }, + expect: "version", + expectErr: false, + expectedErr: nil, + cond: func(m *SecretsManagerClientMock) error { + if len(m.calls.CreateSecret) != 1 { + return fmt.Errorf("expected CreateSecret to be called once, got %d", len(m.calls.CreateSecret)) + } + + if *m.calls.CreateSecret[0].Params.Name != "path" { + return fmt.Errorf("expected CreateSecret to be called with path, got %s", *m.calls.CreateSecret[0].Params.Name) + } + + if len(m.calls.PutSecretValue) != 1 { + return fmt.Errorf("expected PutSecretValue to be called once, got %d", len(m.calls.PutSecretValue)) + } + + if *m.calls.PutSecretValue[0].Params.SecretId != "path" { + return fmt.Errorf("expected PutSecretValue to be called with path, got %s", *m.calls.PutSecretValue[0].Params.SecretId) + } + + return nil + }, + }, + { + name: "error creating secret", + mock: SecretsManagerClientMock{ + CreateSecretFunc: func(ctx context.Context, params *secretsmanager.CreateSecretInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.CreateSecretOutput, error) { + return nil, fmt.Errorf("error") + }, + }, + expect: "", + expectErr: true, + expectedErr: fmt.Errorf("unable to set secret: error"), + }, + { + name: "error putting secret value", + mock: SecretsManagerClientMock{ + CreateSecretFunc: func(ctx context.Context, params *secretsmanager.CreateSecretInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.CreateSecretOutput, error) { + return nil, AWSSecretsManagerResourceExistsException + }, + PutSecretValueFunc: func(ctx context.Context, params *secretsmanager.PutSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.PutSecretValueOutput, error) { + return nil, fmt.Errorf("error") + }, + }, + expect: "", + expectErr: true, + expectedErr: fmt.Errorf("unable to set secret: error"), + }, + } + + for i := range tests { + tt := &tests[i] + t.Run(tt.name, func(t *testing.T) { + client := &AWSClient{ + client: &tt.mock, + logger: testutils.NewNoopLogger(), + } + + got, err := client.Set("path", "value") + + ret, err := testutils.CheckError(t, err, tt.expectErr, tt.expectedErr) + if err != nil { + t.Error(err) + return + } else if ret { + return + } + + if tt.cond != nil { + if err := tt.cond(&tt.mock); err != nil { + t.Error(err) + return + } + } + + if got != tt.expect { + t.Errorf("expected: %s, got: %s", tt.expect, got) + } + }) + } +} diff --git a/forge/cli/pkg/secrets/providers/local.go b/forge/cli/pkg/secrets/providers/local.go new file mode 100644 index 00000000..bedb6742 --- /dev/null +++ b/forge/cli/pkg/secrets/providers/local.go @@ -0,0 +1,32 @@ +package providers + +import ( + "log/slog" + + "github.com/spf13/afero" +) + +type LocalClient struct { + fs afero.Fs + logger *slog.Logger +} + +func NewLocalClient(logger *slog.Logger) (*LocalClient, error) { + return &LocalClient{ + fs: afero.NewOsFs(), + logger: logger, + }, nil +} + +func (c *LocalClient) Get(key string) (string, error) { + b, err := afero.ReadFile(c.fs, key) + if err != nil { + return string(b), err + } + + return string(b), nil +} + +func (c *LocalClient) Set(key, value string) (string, error) { + panic("not implemented") +} diff --git a/forge/cli/pkg/secrets/providers/local_test.go b/forge/cli/pkg/secrets/providers/local_test.go new file mode 100644 index 00000000..fb1b0cf1 --- /dev/null +++ b/forge/cli/pkg/secrets/providers/local_test.go @@ -0,0 +1,69 @@ +package providers + +import ( + "fmt" + "testing" + + "github.com/input-output-hk/catalyst-forge/forge/cli/internal/testutils" + "github.com/spf13/afero" +) + +func TestLocalClientGet(t *testing.T) { + tests := []struct { + name string + key string + files map[string]string + expect string + expectErr bool + expectedErr error + }{ + { + name: "simple", + key: ".secrets", + files: map[string]string{ + ".secrets": "secret", + }, + expect: "secret", + expectErr: false, + expectedErr: nil, + }, + { + name: "file not found", + key: "foo", + files: map[string]string{ + ".secrets": "secret", + }, + expect: "", + expectErr: true, + expectedErr: fmt.Errorf("open foo: file does not exist"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := afero.NewMemMapFs() + for k, v := range tt.files { + afero.WriteFile(fs, k, []byte(v), 0644) + } + + client := &LocalClient{ + fs: fs, + logger: testutils.NewNoopLogger(), + } + + got, err := client.Get(tt.key) + + ret, err := testutils.CheckError(t, err, tt.expectErr, tt.expectedErr) + if err != nil { + t.Error(err) + return + } else if ret { + return + } + + if got != tt.expect { + t.Errorf("expected: %s, got: %s", tt.expect, got) + } + }) + } +} diff --git a/forge/cli/pkg/walker/fs.go b/forge/cli/pkg/walker/fs.go new file mode 100644 index 00000000..40554bd1 --- /dev/null +++ b/forge/cli/pkg/walker/fs.go @@ -0,0 +1,54 @@ +package walker + +import ( + "log/slog" + "os" + + "github.com/spf13/afero" +) + +// FilesystemWalker is a walker that walks over the local filesystem. +type FilesystemWalker struct { + fs afero.Fs + logger *slog.Logger +} + +// Walk walks over the files and directories in the given root path and calls +// the given function for each entry. +// The reader passed to the function is closed after the function returns. +func (w *FilesystemWalker) Walk(rootPath string, callback WalkerCallback) error { + return afero.Walk(w.fs, rootPath, func(path string, info os.FileInfo, err error) error { + w.logger.Debug("walking path", "path", path) + if err != nil { + w.logger.Error("error walking path", "path", path, "error", err) + return err + } + + if info.Mode().IsRegular() { + return callback(path, FileTypeFile, func() (FileSeeker, error) { + w.logger.Debug("opening file", "path", path) + reader, err := w.fs.Open(path) + if err != nil { + w.logger.Error("error opening file", "path", path, "error", err) + return nil, err + } + + return reader, nil + }) + } else if info.Mode().IsDir() { + return callback(path, FileTypeDir, func() (FileSeeker, error) { + return nil, nil + }) + } else { + return nil + } + }) +} + +// NewFilesystemWalker creates a new FilesystemWalker. +func NewFilesystemWalker(logger *slog.Logger) FilesystemWalker { + return FilesystemWalker{ + fs: afero.NewOsFs(), + logger: logger, + } +} diff --git a/forge/cli/pkg/walker/fs_test.go b/forge/cli/pkg/walker/fs_test.go new file mode 100644 index 00000000..e28ee7c0 --- /dev/null +++ b/forge/cli/pkg/walker/fs_test.go @@ -0,0 +1,153 @@ +package walker + +import ( + "fmt" + "io" + "log/slog" + "maps" + "path/filepath" + "testing" + + "github.com/input-output-hk/catalyst-forge/forge/cli/internal/testutils" + "github.com/spf13/afero" +) + +type wrapfs struct { + afero.Fs + trigger error +} + +func (w *wrapfs) Open(name string) (afero.File, error) { + return nil, w.trigger +} + +func TestFileSystemWalkerWalk(t *testing.T) { + tests := []struct { + name string + fs afero.Fs + callbackErr error + path string + files map[string]string + expectedFiles map[string]string + expectErr bool + expectedErr error + }{ + { + name: "single directory", + fs: afero.NewMemMapFs(), + callbackErr: nil, + path: "/test1", + files: map[string]string{ + "/test1/file1": "file1", + "/test1/file2": "file2", + }, + expectedFiles: map[string]string{ + "/test1/file1": "file1", + "/test1/file2": "file2", + }, + expectErr: false, + expectedErr: nil, + }, + { + name: "nested directories", + fs: afero.NewMemMapFs(), + callbackErr: nil, + path: "/test1", + files: map[string]string{ + "/test1/file1": "file1", + "/test1/dir1/file2": "file2", + "/test1/dir1/dir2/file3": "file3", + }, + expectedFiles: map[string]string{ + "/test1/file1": "file1", + "/test1/dir1/file2": "file2", + "/test1/dir1/dir2/file3": "file3", + }, + expectErr: false, + expectedErr: nil, + }, + { + name: "error opening file", + fs: &wrapfs{ + Fs: afero.NewMemMapFs(), + trigger: fmt.Errorf("fail"), + }, + callbackErr: nil, + path: "/test1", + files: map[string]string{ + "/test1/file1": "file1", + }, + expectedFiles: map[string]string{}, + expectErr: true, + expectedErr: fmt.Errorf("fail"), + }, + { + name: "callback error", + fs: afero.NewMemMapFs(), + callbackErr: fmt.Errorf("callback error"), + path: "/test1", + files: map[string]string{ + "/test1/file1": "file1", + }, + expectedFiles: map[string]string{}, + expectErr: true, + expectedErr: fmt.Errorf("callback error"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + walker := FilesystemWalker{ + fs: tt.fs, + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + + for path, content := range tt.files { + dir := filepath.Dir(path) + if err := tt.fs.MkdirAll(dir, 0755); err != nil { + t.Fatalf("failed to create directory %s: %v", dir, err) + } + + if err := afero.WriteFile(tt.fs, path, []byte(content), 0644); err != nil { + t.Fatalf("failed to write file %s: %v", path, err) + } + } + + callbackFiles := make(map[string]string) + err := walker.Walk(tt.path, func(path string, fileType FileType, openFile func() (FileSeeker, error)) error { + if tt.callbackErr != nil { + return tt.callbackErr + } + + if fileType == FileTypeDir { + return nil + } + + file, err := openFile() + if err != nil { + return err + } + defer file.Close() + + content, err := io.ReadAll(file) + if err != nil { + return err + } + + callbackFiles[path] = string(content) + return nil + }) + + ret, err := testutils.CheckError(t, err, tt.expectErr, tt.expectedErr) + if err != nil { + t.Fatal(err) + } else if ret { + return + } + + if !maps.Equal(callbackFiles, tt.expectedFiles) { + t.Fatalf("expected: %v, got: %v", tt.expectedFiles, callbackFiles) + } + }) + } +} diff --git a/forge/cli/pkg/walker/walker.go b/forge/cli/pkg/walker/walker.go new file mode 100644 index 00000000..4a915178 --- /dev/null +++ b/forge/cli/pkg/walker/walker.go @@ -0,0 +1,33 @@ +package walker + +import ( + "io" + "io/fs" +) + +//go:generate go run github.com/matryer/moq@latest -out walker_mock.go . Walker + +// FileType is an enum that represents the type of a file. +type FileType int + +const ( + FileTypeFile FileType = iota + FileTypeDir +) + +// FileSeeker is an interface that combines the fs.File and io.Seeker interfaces. +type FileSeeker interface { + fs.File + io.Seeker +} + +// WalkerCallback is a callback function that is called for each file in the +// Walk function. +type WalkerCallback func(string, FileType, func() (FileSeeker, error)) error + +// Walker is an interface that allows walking over a set of files. +type Walker interface { + // Walk walks over the files in the given root path and calls the given + // function for each file. + Walk(rootPath string, callback WalkerCallback) error +} diff --git a/forge/cli/pkg/walker/walker_mock.go b/forge/cli/pkg/walker/walker_mock.go new file mode 100644 index 00000000..905a44b1 --- /dev/null +++ b/forge/cli/pkg/walker/walker_mock.go @@ -0,0 +1,80 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package walker + +import ( + "sync" +) + +// Ensure, that WalkerMock does implement Walker. +// If this is not the case, regenerate this file with moq. +var _ Walker = &WalkerMock{} + +// WalkerMock is a mock implementation of Walker. +// +// func TestSomethingThatUsesWalker(t *testing.T) { +// +// // make and configure a mocked Walker +// mockedWalker := &WalkerMock{ +// WalkFunc: func(rootPath string, callback WalkerCallback) error { +// panic("mock out the Walk method") +// }, +// } +// +// // use mockedWalker in code that requires Walker +// // and then make assertions. +// +// } +type WalkerMock struct { + // WalkFunc mocks the Walk method. + WalkFunc func(rootPath string, callback WalkerCallback) error + + // calls tracks calls to the methods. + calls struct { + // Walk holds details about calls to the Walk method. + Walk []struct { + // RootPath is the rootPath argument value. + RootPath string + // Callback is the callback argument value. + Callback WalkerCallback + } + } + lockWalk sync.RWMutex +} + +// Walk calls WalkFunc. +func (mock *WalkerMock) Walk(rootPath string, callback WalkerCallback) error { + if mock.WalkFunc == nil { + panic("WalkerMock.WalkFunc: method is nil but Walker.Walk was just called") + } + callInfo := struct { + RootPath string + Callback WalkerCallback + }{ + RootPath: rootPath, + Callback: callback, + } + mock.lockWalk.Lock() + mock.calls.Walk = append(mock.calls.Walk, callInfo) + mock.lockWalk.Unlock() + return mock.WalkFunc(rootPath, callback) +} + +// WalkCalls gets all the calls that were made to Walk. +// Check the length with: +// +// len(mockedWalker.WalkCalls()) +func (mock *WalkerMock) WalkCalls() []struct { + RootPath string + Callback WalkerCallback +} { + var calls []struct { + RootPath string + Callback WalkerCallback + } + mock.lockWalk.RLock() + calls = mock.calls.Walk + mock.lockWalk.RUnlock() + return calls +} From b50f79a9d6941906efb9d97079fac0b536f8777f Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Wed, 28 Aug 2024 20:48:57 -0400 Subject: [PATCH 2/3] chore: removes unused code --- forge/cli/pkg/earthly/earthly_test.go | 63 --------------------------- 1 file changed, 63 deletions(-) diff --git a/forge/cli/pkg/earthly/earthly_test.go b/forge/cli/pkg/earthly/earthly_test.go index 67beeb4f..d84dac48 100644 --- a/forge/cli/pkg/earthly/earthly_test.go +++ b/forge/cli/pkg/earthly/earthly_test.go @@ -329,66 +329,3 @@ Artifact foo output as bar`, }) } } - -// var data string = "{\"secret_key\":\"secret_value\"}" - -// func setup(t *testing.T) (*slog.Logger, string) { -// t.Helper() - -// handler := log.New(os.Stderr) -// handler.SetLevel(log.DebugLevel) -// logger := slog.New(handler) - -// f, err := os.CreateTemp("", "tmpfile-") -// if err != nil { -// t.Fatal(err) -// } -// defer f.Close() - -// b := []byte(data) -// if _, err := f.Write(b); err != nil { -// log.Fatal(err) -// } - -// return logger, f.Name() -// } - -// func TestCreateMapFromSecrets(t *testing.T) { -// logger, tmpFile := setup(t) - -// t.Cleanup(func() { -// os.Remove(tmpFile) -// }) - -// prefix := "path/to/earthfile" - -// c := &cue.Config{ -// Secrets: map[string]*cue.Secret{ -// "my_secret": { -// Provider: "local", -// Path: tmpFile, -// Maps: map[string]string{ -// "secret_key": "secret_id_for_earthly", -// }, -// }, -// }, -// } - -// var secrets []*cue.Secret - -// for _, v := range c.Secrets { -// secrets = append(secrets, v) -// } - -// got, err := newSecrets(logger, secrets, prefix) - -// want := []*Secret{ -// { -// Value: "secret_value", -// }, -// } - -// assert.Nil(t, err) -// assert.NotNil(t, got) -// assert.Equal(t, want[0].Value, got[0].Value) -// } From 20f248d78f6d5289d3bf9ead7adbfdddad56ad2e Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Wed, 28 Aug 2024 21:00:41 -0400 Subject: [PATCH 3/3] chore: adds Earthfiles --- blueprint/Earthfile | 38 ++++++++++++++++++++++++++++++++++++++ cuetools/Earthfile | 35 +++++++++++++++++++++++++++++++++++ forge/cli/Earthfile | 9 ++++++--- 3 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 blueprint/Earthfile create mode 100644 cuetools/Earthfile diff --git a/blueprint/Earthfile b/blueprint/Earthfile new file mode 100644 index 00000000..4fc5067f --- /dev/null +++ b/blueprint/Earthfile @@ -0,0 +1,38 @@ +VERSION 0.8 + +deps: + FROM golang:1.23.0-alpine3.19 + + WORKDIR /work + + RUN mkdir -p /go/cache && mkdir -p /go/modcache + ENV GOCACHE=/go/cache + ENV GOMODCACHE=/go/modcache + CACHE --persist --sharing shared /go + + COPY ../cuetools+src/src /cuetools + + COPY go.mod go.sum . + RUN go mod download + +src: + FROM +deps + + CACHE --persist --sharing shared /go + + COPY --dir internal pkg schema . + + RUN go generate ./... + + SAVE ARTIFACT . src + +check: + FROM +src + + RUN gofmt -l . | grep . && exit 1 || exit 0 + RUN go vet ./... + +test: + FROM +src + + RUN go test ./... \ No newline at end of file diff --git a/cuetools/Earthfile b/cuetools/Earthfile new file mode 100644 index 00000000..e3f3bdc8 --- /dev/null +++ b/cuetools/Earthfile @@ -0,0 +1,35 @@ +VERSION 0.8 + +deps: + FROM golang:1.23.0-alpine3.19 + + WORKDIR /work + + RUN mkdir -p /go/cache && mkdir -p /go/modcache + ENV GOCACHE=/go/cache + ENV GOMODCACHE=/go/modcache + CACHE --persist --sharing shared /go + + COPY go.mod go.sum . + RUN go mod download + +src: + FROM +deps + + CACHE --persist --sharing shared /go + + COPY --dir pkg . + RUN go generate ./... + + SAVE ARTIFACT . src + +check: + FROM +src + + RUN gofmt -l . | grep . && exit 1 || exit 0 + RUN go vet ./... + +test: + FROM +src + + RUN go test ./... \ No newline at end of file diff --git a/forge/cli/Earthfile b/forge/cli/Earthfile index b04ce8c4..f2501bd8 100644 --- a/forge/cli/Earthfile +++ b/forge/cli/Earthfile @@ -1,15 +1,18 @@ VERSION 0.8 deps: - FROM golang:1.22.4-alpine3.19 + FROM golang:1.23.0-alpine3.19 - WORKDIR /work + WORKDIR /work/cli RUN mkdir -p /go/cache && mkdir -p /go/modcache ENV GOCACHE=/go/cache ENV GOMODCACHE=/go/modcache CACHE --persist --sharing shared /go + COPY ../../blueprint+src/src /blueprint + COPY ../../cuetools+src/src /cuetools + COPY go.mod go.sum . RUN go mod download @@ -18,7 +21,7 @@ src: CACHE --persist --sharing shared /go - COPY --dir cmd cue internal pkg . + COPY --dir cmd internal pkg . RUN go generate ./... check: