From 83469ca998b8756673cc9ff06c8225bd3cc62e61 Mon Sep 17 00:00:00 2001 From: Louis Garman <75728+leg100@users.noreply.github.com> Date: Fri, 13 Oct 2023 10:03:58 +0100 Subject: [PATCH] feat: always use latest terraform version (#616) Fixes #608 --- internal/agent/agent.go | 24 ++-- internal/agent/client.go | 11 +- internal/agent/environment.go | 27 ++-- internal/agent/executor.go | 7 - internal/agent/steps.go | 16 ++- internal/agent/terraform_downloader.go | 93 ------------- internal/agent/terraform_path_finder.go | 27 ---- internal/daemon/config.go | 2 + internal/daemon/daemon.go | 20 ++- .../templates/content/workspace_edit.tmpl | 4 +- internal/integration/daemon_helpers_test.go | 39 +++++- internal/integration/helpers_test.go | 14 -- internal/integration/main_test.go | 8 -- internal/integration/run_cancel_test.go | 7 +- internal/integration/tag_e2e_test.go | 2 +- .../integration/terraform_cli_cancel_test.go | 2 +- .../integration/terraform_cli_discard_test.go | 2 +- internal/integration/terraform_login_test.go | 2 +- internal/organization/tfe.go | 2 + internal/releases/db.go | 46 +++++++ .../download.go} | 13 +- internal/releases/downloader.go | 88 ++++++++++++ .../downloader_test.go} | 5 +- internal/releases/latest_checker.go | 39 ++++++ internal/releases/latest_checker_test.go | 46 +++++++ internal/releases/releases.go | 128 ++++++++++++++++++ internal/releases/testdata/latest.json | 95 +++++++++++++ .../1.2.3/terraform_1.2.3_linux_amd64.zip | Bin .../1.2.3/terraform_1.2.3_linux_arm64.zip | Bin internal/run/factory.go | 8 ++ internal/run/factory_test.go | 34 ++++- internal/run/jsonapi_unmarshal.go | 1 + internal/run/service.go | 3 + ...1010191539_create_table_latest_version.sql | 8 ++ internal/sql/pggen/agent_token.sql.go | 30 ++++ internal/sql/pggen/releases.sql.go | 128 ++++++++++++++++++ internal/sql/queries/releases.sql | 17 +++ internal/tfeapi/types/organization.go | 2 + internal/workspace/workspace.go | 10 +- internal/workspace/workspace_test.go | 8 ++ 40 files changed, 797 insertions(+), 221 deletions(-) delete mode 100644 internal/agent/terraform_downloader.go delete mode 100644 internal/agent/terraform_path_finder.go create mode 100644 internal/releases/db.go rename internal/{agent/terraform_download.go => releases/download.go} (86%) create mode 100644 internal/releases/downloader.go rename internal/{agent/terraform_downloader_test.go => releases/downloader_test.go} (90%) create mode 100644 internal/releases/latest_checker.go create mode 100644 internal/releases/latest_checker_test.go create mode 100644 internal/releases/releases.go create mode 100644 internal/releases/testdata/latest.json rename internal/{agent => releases}/testdata/releases/terraform/1.2.3/terraform_1.2.3_linux_amd64.zip (100%) rename internal/{agent => releases}/testdata/releases/terraform/1.2.3/terraform_1.2.3_linux_arm64.zip (100%) create mode 100644 internal/sql/migrations/20231010191539_create_table_latest_version.sql create mode 100644 internal/sql/pggen/releases.sql.go create mode 100644 internal/sql/queries/releases.sql diff --git a/internal/agent/agent.go b/internal/agent/agent.go index cca12fe86..ef7b7f4ce 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -35,10 +35,8 @@ type agent struct { logr.Logger client - spooler // spools new run events - *terminator // terminates runs - Downloader // terraform cli downloader - *TerraformPathFinder // determines destination dir for terraform bins + spooler // spools new run events + *terminator // terminates runs envs []string // terraform environment variables } @@ -61,17 +59,13 @@ func NewAgent(logger logr.Logger, app client, cfg Config) (*agent, error) { logger.V(0).Info("enabled debug mode") } - pathFinder := newTerraformPathFinder(cfg.TerraformBinDir) - agent := &agent{ - client: app, - Config: cfg, - Logger: logger, - envs: DefaultEnvs, - spooler: newSpooler(app, logger, cfg), - terminator: newTerminator(), - Downloader: NewDownloader(pathFinder), - TerraformPathFinder: pathFinder, + client: app, + Config: cfg, + Logger: logger, + envs: DefaultEnvs, + spooler: newSpooler(app, logger, cfg), + terminator: newTerminator(), } if cfg.PluginCache { @@ -89,7 +83,7 @@ func NewAgent(logger logr.Logger, app client, cfg Config) (*agent, error) { // via http func NewExternalAgent(ctx context.Context, logger logr.Logger, cfg ExternalConfig) (*agent, error) { // Sends unauthenticated ping to server - app, err := newClient(cfg.HTTPConfig) + app, err := newClient(cfg) if err != nil { return nil, err } diff --git a/internal/agent/client.go b/internal/agent/client.go index 4e7946d1b..537259a5f 100644 --- a/internal/agent/client.go +++ b/internal/agent/client.go @@ -8,6 +8,7 @@ import ( "github.com/leg100/otf/internal/http" "github.com/leg100/otf/internal/logs" "github.com/leg100/otf/internal/pubsub" + "github.com/leg100/otf/internal/releases" "github.com/leg100/otf/internal/resource" "github.com/leg100/otf/internal/run" "github.com/leg100/otf/internal/state" @@ -43,6 +44,7 @@ type ( tokens.RunTokenService internal.PutChunkService + releases.Downloader } // LocalClient is the client for an internal agent. @@ -53,6 +55,7 @@ type ( workspace.WorkspaceService internal.HostnameService configversion.ConfigurationVersionService + releases.Downloader run.RunService logs.LogsService } @@ -69,6 +72,7 @@ type ( *workspaceClient *runClient *logsClient + releases.Downloader } stateClient = state.Client @@ -82,20 +86,21 @@ type ( // New constructs a client that uses http to remotely invoke OTF // services. -func newClient(config http.Config) (*remoteClient, error) { - httpClient, err := http.NewClient(config) +func newClient(config ExternalConfig) (*remoteClient, error) { + httpClient, err := http.NewClient(config.HTTPConfig) if err != nil { return nil, err } return &remoteClient{ Client: httpClient, + Downloader: releases.NewDownloader(config.TerraformBinDir), stateClient: &stateClient{JSONAPIClient: httpClient}, configClient: &configClient{JSONAPIClient: httpClient}, variableClient: &variableClient{JSONAPIClient: httpClient}, tokensClient: &tokensClient{JSONAPIClient: httpClient}, workspaceClient: &workspaceClient{JSONAPIClient: httpClient}, - runClient: &runClient{JSONAPIClient: httpClient, Config: config}, + runClient: &runClient{JSONAPIClient: httpClient, Config: config.HTTPConfig}, logsClient: &logsClient{JSONAPIClient: httpClient}, }, nil } diff --git a/internal/agent/environment.go b/internal/agent/environment.go index 8b848c676..a2ab85df3 100644 --- a/internal/agent/environment.go +++ b/internal/agent/environment.go @@ -24,7 +24,6 @@ import ( type environment struct { client logr.Logger - Downloader // Downloader for workers to download terraform cli on demand steps []step // sequence of steps to execute @@ -86,21 +85,19 @@ func newEnvironment( }) env := &environment{ - Logger: logger, - client: agent, - Downloader: agent, - out: writer, - workdir: wd, - variables: variables, - ctx: ctx, - runner: &runner{out: writer}, + Logger: logger, + client: agent, + out: writer, + workdir: wd, + variables: variables, + ctx: ctx, + runner: &runner{out: writer}, executor: &executor{ - Config: agent.Config, - TerraformPathFinder: agent.TerraformPathFinder, - version: ws.TerraformVersion, - out: writer, - envs: envs, - workdir: wd, + Config: agent.Config, + version: run.TerraformVersion, + out: writer, + envs: envs, + workdir: wd, }, } diff --git a/internal/agent/executor.go b/internal/agent/executor.go index fd56d2753..354be9c64 100644 --- a/internal/agent/executor.go +++ b/internal/agent/executor.go @@ -19,7 +19,6 @@ type ( // executor executes processes. executor struct { Config - *TerraformPathFinder version string // terraform cli version out io.Writer @@ -79,12 +78,6 @@ func (e *executor) execute(args []string, opts ...executionOption) error { return nil } -// executeTerraform executes a terraform process -func (e *executor) executeTerraform(args []string, opts ...executionOption) error { - args = append([]string{e.TerraformPath(e.version)}, args...) - return e.execute(args, opts...) -} - func (e *execution) execute(args []string) error { if len(args) == 0 { return fmt.Errorf("missing command name") diff --git a/internal/agent/steps.go b/internal/agent/steps.go index 86edf8f20..7cf658b1d 100644 --- a/internal/agent/steps.go +++ b/internal/agent/steps.go @@ -31,6 +31,8 @@ type ( stepsBuilder struct { *run.Run *environment + + terraformPath string } runner struct { @@ -106,7 +108,8 @@ func (r *runner) cancel(force bool) { } func (b *stepsBuilder) downloadTerraform(ctx context.Context) error { - _, err := b.Download(ctx, b.version, b.out) + var err error + b.terraformPath, err = b.Download(ctx, b.version, b.out) return err } @@ -163,7 +166,7 @@ func (b *stepsBuilder) writeTerraformVars(ctx context.Context) error { } func (b *stepsBuilder) terraformInit(ctx context.Context) error { - return b.executeTerraform([]string{"init"}) + return b.executor.execute([]string{b.terraformPath, "init"}) } func (b *stepsBuilder) terraformPlan(ctx context.Context) error { @@ -172,7 +175,7 @@ func (b *stepsBuilder) terraformPlan(ctx context.Context) error { args = append(args, "-destroy") } args = append(args, "-out="+planFilename) - return b.executeTerraform(args) + return b.executor.execute(append([]string{b.terraformPath}, args...)) } func (b *stepsBuilder) terraformApply(ctx context.Context) (err error) { @@ -208,12 +211,15 @@ func (b *stepsBuilder) terraformApply(ctx context.Context) (err error) { args = append(args, "-destroy") } args = append(args, planFilename) - return b.executeTerraform(args) + return b.executor.execute(append([]string{b.terraformPath}, args...)) } func (b *stepsBuilder) convertPlanToJSON(ctx context.Context) error { args := []string{"show", "-json", planFilename} - return b.executeTerraform(args, redirectStdout(jsonPlanFilename)) + return b.executor.execute( + append([]string{b.terraformPath}, args...), + redirectStdout(jsonPlanFilename), + ) } func (b *stepsBuilder) uploadPlan(ctx context.Context) error { diff --git a/internal/agent/terraform_downloader.go b/internal/agent/terraform_downloader.go deleted file mode 100644 index 76fe40841..000000000 --- a/internal/agent/terraform_downloader.go +++ /dev/null @@ -1,93 +0,0 @@ -package agent - -import ( - "context" - "fmt" - "io" - "net/http" - "net/url" - "path" - "runtime" - - "github.com/leg100/otf/internal" -) - -const HashicorpReleasesHost = "releases.hashicorp.com" - -type ( - // terraformDownloader downloads terraform binaries - terraformDownloader struct { - *TerraformPathFinder // used to lookup destination path for saving download - - host string // server hosting binaries - client *http.Client // client for downloading from server via http - mu chan struct{} // ensures only one download at a time - } - - // Downloader downloads a specific version of a binary and returns its path - Downloader interface { - Download(ctx context.Context, version string, w io.Writer) (string, error) - } -) - -// NewDownloader constructs a terraform downloader. Pass a path finder to -// customise the location to which the bins are persisted, or pass nil to use -// the default. -func NewDownloader(pathFinder *TerraformPathFinder) *terraformDownloader { - if pathFinder == nil { - pathFinder = newTerraformPathFinder(defaultTerraformBinDir) - } - - mu := make(chan struct{}, 1) - mu <- struct{}{} - - return &terraformDownloader{ - host: HashicorpReleasesHost, - TerraformPathFinder: pathFinder, - client: &http.Client{}, - mu: mu, - } -} - -// Download ensures the given version of terraform is available on the local -// filesystem and returns its path. Thread-safe: if a Download is in-flight and -// another Download is requested then it'll be made to wait until the -// former has finished. -func (d *terraformDownloader) Download(ctx context.Context, version string, w io.Writer) (string, error) { - if internal.Exists(d.dest(version)) { - return d.dest(version), nil - } - - select { - case <-d.mu: - case <-ctx.Done(): - return "", ctx.Err() - } - - err := (&download{ - Writer: w, - version: version, - src: d.src(version), - dest: d.dest(version), - client: d.client, - }).download() - - d.mu <- struct{}{} - - return d.dest(version), err -} - -func (d *terraformDownloader) src(version string) string { - return (&url.URL{ - Scheme: "https", - Host: d.host, - Path: path.Join( - "terraform", - version, - fmt.Sprintf("terraform_%s_%s_%s.zip", version, runtime.GOOS, runtime.GOARCH)), - }).String() -} - -func (d *terraformDownloader) dest(version string) string { - return d.TerraformPath(version) -} diff --git a/internal/agent/terraform_path_finder.go b/internal/agent/terraform_path_finder.go deleted file mode 100644 index eb1e7912e..000000000 --- a/internal/agent/terraform_path_finder.go +++ /dev/null @@ -1,27 +0,0 @@ -package agent - -import ( - "os" - "path" -) - -var defaultTerraformBinDir = path.Join(os.TempDir(), "otf-terraform-bins") - -type ( - TerraformPathFinder struct { - dest string - } -) - -func newTerraformPathFinder(dest string) *TerraformPathFinder { - if dest == "" { - dest = defaultTerraformBinDir - } - return &TerraformPathFinder{ - dest: dest, - } -} - -func (t *TerraformPathFinder) TerraformPath(version string) string { - return path.Join(t.dest, version, "terraform") -} diff --git a/internal/daemon/config.go b/internal/daemon/config.go index 9adc0f887..ad6cbafbb 100644 --- a/internal/daemon/config.go +++ b/internal/daemon/config.go @@ -37,6 +37,8 @@ type Config struct { DisableScheduler bool RestrictOrganizationCreation bool SiteAdmins []string + // skip checks for latest terraform version + DisableLatestChecker *bool tokens.GoogleIAPConfig } diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index ffda85763..3d9e1e213 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -25,6 +25,7 @@ import ( "github.com/leg100/otf/internal/notifications" "github.com/leg100/otf/internal/organization" "github.com/leg100/otf/internal/pubsub" + "github.com/leg100/otf/internal/releases" "github.com/leg100/otf/internal/repo" "github.com/leg100/otf/internal/run" "github.com/leg100/otf/internal/scheduler" @@ -166,7 +167,13 @@ func New(ctx context.Context, logger logr.Logger, cfg Config) (*Daemon, error) { OrganizationService: orgService, VCSProviderService: vcsProviderService, }) - + releasesService := releases.NewService(releases.Options{ + Logger: logger, + DB: db, + }) + if cfg.DisableLatestChecker == nil || !*cfg.DisableLatestChecker { + releasesService.StartLatestChecker(ctx) + } workspaceService := workspace.NewService(workspace.Options{ Logger: logger, DB: db, @@ -201,6 +208,7 @@ func New(ctx context.Context, logger logr.Logger, cfg Config) (*Daemon, error) { Cache: cache, Subscriber: repoService, Signer: signer, + ReleasesService: releasesService, }) logsService := logs.NewService(logs.Options{ Logger: logger, @@ -249,6 +257,7 @@ func New(ctx context.Context, logger logr.Logger, cfg Config) (*Daemon, error) { ConfigurationVersionService: configService, RunService: runService, LogsService: logsService, + Downloader: releasesService, }, *cfg.AgentConfig, ) @@ -326,10 +335,11 @@ func New(ctx context.Context, logger logr.Logger, cfg Config) (*Daemon, error) { LogsService: logsService, RepoService: repoService, NotificationService: notificationService, - Broker: broker, - DB: db, - agent: agent, - cloudService: cloudService, + //ReleasesService: releasesService, + Broker: broker, + DB: db, + agent: agent, + cloudService: cloudService, }, nil } diff --git a/internal/http/html/static/templates/content/workspace_edit.tmpl b/internal/http/html/static/templates/content/workspace_edit.tmpl index dc71877ef..26e93a599 100644 --- a/internal/http/html/static/templates/content/workspace_edit.tmpl +++ b/internal/http/html/static/templates/content/workspace_edit.tmpl @@ -72,9 +72,9 @@
- + - The version of Terraform to use for this workspace. Upon creating this workspace, the default version was selected and will be used until it is changed manually. It will not upgrade automatically. + The version of Terraform to use for this workspace. Upon creating this workspace, the default version was selected and will be used until it is changed manually. It will not upgrade automatically unless you specify latest, in which case the latest version of terraform is used.
diff --git a/internal/integration/daemon_helpers_test.go b/internal/integration/daemon_helpers_test.go index 889d3ba13..bd5f5c387 100644 --- a/internal/integration/daemon_helpers_test.go +++ b/internal/integration/daemon_helpers_test.go @@ -3,6 +3,7 @@ package integration import ( "bytes" "context" + "io" "os" "os/exec" "testing" @@ -21,6 +22,7 @@ import ( "github.com/leg100/otf/internal/notifications" "github.com/leg100/otf/internal/organization" "github.com/leg100/otf/internal/pubsub" + "github.com/leg100/otf/internal/releases" "github.com/leg100/otf/internal/run" "github.com/leg100/otf/internal/sql" "github.com/leg100/otf/internal/state" @@ -39,6 +41,8 @@ type ( *github.TestServer // event subscription for test to use. sub <-chan pubsub.Event + // releases service to allow tests to download terraform + releases.ReleasesService } // configures the daemon for integration tests @@ -46,6 +50,8 @@ type ( daemon.Config // skip creation of default organization skipDefaultOrganization bool + // customise path in which terraform bins are saved + terraformBinDir string } ) @@ -65,6 +71,11 @@ func setup(t *testing.T, cfg *config, gopts ...github.TestServerOption) (*testDa if cfg.Secret == nil { cfg.Secret = sharedSecret } + // Unless test has specified otherwise, disable checking for latest + // terraform version + if cfg.DisableLatestChecker == nil || !*cfg.DisableLatestChecker { + cfg.DisableLatestChecker = internal.Bool(true) + } daemon.ApplyDefaults(&cfg.Config) cfg.SSL = true cfg.CertFile = "./fixtures/cert.pem" @@ -78,7 +89,7 @@ func setup(t *testing.T, cfg *config, gopts ...github.TestServerOption) (*testDa var logger logr.Logger if _, ok := os.LookupEnv("OTF_INTEGRATION_TEST_ENABLE_LOGGER"); ok { var err error - logger, err = logr.New(&logr.Config{Verbosity: 1, Format: "default"}) + logger, err = logr.New(&logr.Config{Verbosity: 9, Format: "default"}) require.NoError(t, err) } else { logger = logr.Discard() @@ -115,10 +126,17 @@ func setup(t *testing.T, cfg *config, gopts ...github.TestServerOption) (*testDa sub, err := d.Broker.Subscribe(ctx, "") require.NoError(t, err) + releasesService := releases.NewService(releases.Options{ + Logger: logger, + DB: d.DB, + TerraformBinDir: cfg.terraformBinDir, + }) + daemon := &testDaemon{ - Daemon: d, - TestServer: githubServer, - sub: sub, + Daemon: d, + TestServer: githubServer, + ReleasesService: releasesService, + sub: sub, } // create a dedicated user account and context for test to use. @@ -438,7 +456,7 @@ func (s *testDaemon) tfcli(t *testing.T, ctx context.Context, command, configPat func (s *testDaemon) tfcliWithError(t *testing.T, ctx context.Context, command, configPath string, args ...string) (string, error) { t.Helper() - tfpath := downloadTerraform(t, ctx, nil) + tfpath := s.downloadTerraform(t, ctx, nil) // Create user token expressly for the terraform cli user := userFromContext(t, ctx) @@ -476,3 +494,14 @@ func (s *testDaemon) otfcli(t *testing.T, ctx context.Context, args ...string) s require.NoError(t, err, "otf cli failed: %s", buf.String()) return buf.String() } + +func (s *testDaemon) downloadTerraform(t *testing.T, ctx context.Context, version *string) string { + t.Helper() + + if version == nil { + version = internal.String(releases.DefaultTerraformVersion) + } + tfpath, err := s.Download(ctx, *version, io.Discard) + require.NoError(t, err) + return tfpath +} diff --git a/internal/integration/helpers_test.go b/internal/integration/helpers_test.go index 888c393e4..95c16962e 100644 --- a/internal/integration/helpers_test.go +++ b/internal/integration/helpers_test.go @@ -3,14 +3,11 @@ package integration import ( "context" "fmt" - "io" "os" "path/filepath" "testing" - "github.com/leg100/otf/internal" "github.com/leg100/otf/internal/auth" - "github.com/leg100/otf/internal/workspace" "github.com/stretchr/testify/require" ) @@ -83,14 +80,3 @@ func userFromContext(t *testing.T, ctx context.Context) *auth.User { require.NoError(t, err) return user } - -func downloadTerraform(t *testing.T, ctx context.Context, version *string) string { - t.Helper() - - if version == nil { - version = internal.String(workspace.DefaultTerraformVersion) - } - tfpath, err := tfDownloader.Download(ctx, *version, io.Discard) - require.NoError(t, err) - return tfpath -} diff --git a/internal/integration/main_test.go b/internal/integration/main_test.go index 8c5dc14bb..cdd36542c 100644 --- a/internal/integration/main_test.go +++ b/internal/integration/main_test.go @@ -11,7 +11,6 @@ import ( "testing" "github.com/leg100/otf/internal" - "github.com/leg100/otf/internal/agent" "github.com/leg100/otf/internal/auth" "github.com/leg100/otf/internal/testbrowser" "github.com/leg100/otf/internal/testcompose" @@ -29,9 +28,6 @@ var ( // pool of web browsers browser *testbrowser.Pool - - // downloader for specific versions of terraform for tests to use - tfDownloader agent.Downloader ) func TestMain(m *testing.M) { @@ -148,10 +144,6 @@ func doMain(m *testing.M) (int, error) { defer cleanup() browser = pool - // Setup terraform downloader. The default (nil) saves the terraform bins to - // the system temp directory so they can be persisted between tests. - tfDownloader = agent.NewDownloader(nil) - return m.Run(), nil } diff --git a/internal/integration/run_cancel_test.go b/internal/integration/run_cancel_test.go index d8a85849a..4d5934254 100644 --- a/internal/integration/run_cancel_test.go +++ b/internal/integration/run_cancel_test.go @@ -9,6 +9,7 @@ import ( "github.com/leg100/otf/internal" "github.com/leg100/otf/internal/agent" + "github.com/leg100/otf/internal/releases" "github.com/leg100/otf/internal/variable" "github.com/leg100/otf/internal/workspace" "github.com/stretchr/testify/require" @@ -18,12 +19,10 @@ import ( func TestIntegration_RunCancel(t *testing.T) { integrationTest(t) - daemon, org, ctx := setup(t, nil) - // stage a fake terraform bin that sleeps until it receives an interrupt // signal bins := filepath.Join(t.TempDir(), "bins") - dst := filepath.Join(bins, workspace.DefaultTerraformVersion, "terraform") + dst := filepath.Join(bins, releases.DefaultTerraformVersion, "terraform") err := os.MkdirAll(filepath.Dir(dst), 0o755) require.NoError(t, err) wd, err := os.Getwd() @@ -31,6 +30,8 @@ func TestIntegration_RunCancel(t *testing.T) { err = os.Symlink(filepath.Join(wd, "testdata/cancelme"), dst) require.NoError(t, err) + daemon, org, ctx := setup(t, &config{terraformBinDir: dst}) + // run a temporary http server as a means of communicating with the fake // bin got := make(chan string) diff --git a/internal/integration/tag_e2e_test.go b/internal/integration/tag_e2e_test.go index 34ddc3851..dc715be8f 100644 --- a/internal/integration/tag_e2e_test.go +++ b/internal/integration/tag_e2e_test.go @@ -36,7 +36,7 @@ terraform { resource "null_resource" "tags_e2e" {} `, daemon.Hostname(), org.Name)) - tfpath := downloadTerraform(t, ctx, nil) + tfpath := daemon.downloadTerraform(t, ctx, nil) // run terraform init _, token := daemon.createToken(t, ctx, nil) diff --git a/internal/integration/terraform_cli_cancel_test.go b/internal/integration/terraform_cli_cancel_test.go index 57f96f85e..887432e0c 100644 --- a/internal/integration/terraform_cli_cancel_test.go +++ b/internal/integration/terraform_cli_cancel_test.go @@ -41,7 +41,7 @@ data "http" "wait" { `, srv.URL)) svc.tfcli(t, ctx, "init", config) - tfpath := downloadTerraform(t, ctx, nil) + tfpath := svc.downloadTerraform(t, ctx, nil) // Invoke terraform plan _, token := svc.createToken(t, ctx, nil) diff --git a/internal/integration/terraform_cli_discard_test.go b/internal/integration/terraform_cli_discard_test.go index 77410d49b..0c5d286e1 100644 --- a/internal/integration/terraform_cli_discard_test.go +++ b/internal/integration/terraform_cli_discard_test.go @@ -27,7 +27,7 @@ func TestIntegration_TerraformCLIDiscard(t *testing.T) { // Create user token expressly for terraform apply _, token := svc.createToken(t, ctx, nil) - tfpath := downloadTerraform(t, ctx, nil) + tfpath := svc.downloadTerraform(t, ctx, nil) // Invoke terraform apply e, tferr, err := expect.SpawnWithArgs( diff --git a/internal/integration/terraform_login_test.go b/internal/integration/terraform_login_test.go index 52f4364cd..644750388 100644 --- a/internal/integration/terraform_login_test.go +++ b/internal/integration/terraform_login_test.go @@ -31,7 +31,7 @@ func TestTerraformLogin(t *testing.T) { require.NoError(t, err) killBrowserPath := path.Join(wd, "./fixtures/kill-browser") - tfpath := downloadTerraform(t, ctx, nil) + tfpath := svc.downloadTerraform(t, ctx, nil) e, tferr, err := expect.SpawnWithArgs( []string{tfpath, "login", svc.Hostname()}, diff --git a/internal/organization/tfe.go b/internal/organization/tfe.go index 343e0ff23..310057556 100644 --- a/internal/organization/tfe.go +++ b/internal/organization/tfe.go @@ -184,6 +184,8 @@ func (a *tfe) toOrganization(from *Organization) *types.Organization { SessionTimeout: from.SessionTimeout, AllowForceDeleteWorkspaces: from.AllowForceDeleteWorkspaces, CostEstimationEnabled: from.CostEstimationEnabled, + // go-tfe tests expect this attribute to be equal to 5 + RemainingTestableCount: 5, } if from.Email != nil { to.Email = *from.Email diff --git a/internal/releases/db.go b/internal/releases/db.go new file mode 100644 index 000000000..a2dd4babd --- /dev/null +++ b/internal/releases/db.go @@ -0,0 +1,46 @@ +package releases + +import ( + "context" + "time" + + "github.com/leg100/otf/internal" + "github.com/leg100/otf/internal/sql" + "github.com/leg100/otf/internal/sql/pggen" +) + +type db struct { + *sql.DB +} + +func (db *db) updateLatestVersion(ctx context.Context, v string) error { + return db.Lock(ctx, "latest_terraform_version", func(ctx context.Context, q pggen.Querier) error { + rows, err := q.FindLatestTerraformVersion(ctx) + if err != nil { + return err + } + if len(rows) == 0 { + _, err = q.InsertLatestTerraformVersion(ctx, sql.String(v)) + if err != nil { + return err + } + } else { + _, err = q.UpdateLatestTerraformVersion(ctx, sql.String(v)) + if err != nil { + return err + } + } + return nil + }) +} + +func (db *db) getLatest(ctx context.Context) (string, time.Time, error) { + rows, err := db.Conn(ctx).FindLatestTerraformVersion(ctx) + if err != nil { + return "", time.Time{}, err + } + if len(rows) == 0 { + return "", time.Time{}, internal.ErrResourceNotFound + } + return rows[0].Version.String, rows[0].Checkpoint.Time, nil +} diff --git a/internal/agent/terraform_download.go b/internal/releases/download.go similarity index 86% rename from internal/agent/terraform_download.go rename to internal/releases/download.go index 290019f01..9a0f21793 100644 --- a/internal/agent/terraform_download.go +++ b/internal/releases/download.go @@ -1,4 +1,4 @@ -package agent +package releases import ( "archive/zip" @@ -23,14 +23,14 @@ type download struct { client *http.Client } -func (d *download) download() error { +func (d *download) download(ctx context.Context) error { if internal.Exists(d.dest) { return nil } - zipfile, err := d.getZipfile() + zipfile, err := d.getZipfile(ctx) if err != nil { - return fmt.Errorf("downloading zipfile: %w", err) + return fmt.Errorf("downloading zipfile from %s: %w", d.src, err) } defer os.Remove(zipfile) @@ -45,9 +45,8 @@ func (d *download) download() error { return nil } -func (d *download) getZipfile() (string, error) { - // TODO: why no context? - req, err := http.NewRequestWithContext(context.Background(), "GET", d.src, nil) +func (d *download) getZipfile(ctx context.Context) (string, error) { + req, err := http.NewRequestWithContext(ctx, "GET", d.src, nil) if err != nil { return "", fmt.Errorf("building request: %w", err) } diff --git a/internal/releases/downloader.go b/internal/releases/downloader.go new file mode 100644 index 000000000..459e91433 --- /dev/null +++ b/internal/releases/downloader.go @@ -0,0 +1,88 @@ +package releases + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "runtime" + + "github.com/leg100/otf/internal" +) + +const hashicorpReleasesHost = "releases.hashicorp.com" + +var defaultTerraformBinDir = path.Join(os.TempDir(), "otf-terraform-bins") + +// downloader downloads terraform binaries +type downloader struct { + destdir string // destination directory for binaries + host string // server hosting binaries + client *http.Client // client for downloading from server via http + mu chan struct{} // ensures only one download at a time +} + +// NewDownloader constructs a terraform downloader. Pass a path finder to +// customise the location to which the bins are persisted, or pass nil to use +// the default. +func NewDownloader(destdir string) *downloader { + if destdir == "" { + destdir = defaultTerraformBinDir + } + + mu := make(chan struct{}, 1) + mu <- struct{}{} + + return &downloader{ + host: hashicorpReleasesHost, + destdir: destdir, + client: &http.Client{}, + mu: mu, + } +} + +// Download ensures the given version of terraform is available on the local +// filesystem and returns its path. Thread-safe: if a Download is in-flight and +// another Download is requested then it'll be made to wait until the +// former has finished. +func (d *downloader) Download(ctx context.Context, version string, w io.Writer) (string, error) { + if internal.Exists(d.dest(version)) { + return d.dest(version), nil + } + + select { + case <-d.mu: + case <-ctx.Done(): + return "", ctx.Err() + } + + err := (&download{ + Writer: w, + version: version, + src: d.src(version), + dest: d.dest(version), + client: d.client, + }).download(ctx) + + d.mu <- struct{}{} + + return d.dest(version), err +} + +func (d *downloader) src(version string) string { + return (&url.URL{ + Scheme: "https", + Host: d.host, + Path: path.Join( + "terraform", + version, + fmt.Sprintf("terraform_%s_%s_%s.zip", version, runtime.GOOS, runtime.GOARCH)), + }).String() +} + +func (d *downloader) dest(version string) string { + return path.Join(d.destdir, version, "terraform") +} diff --git a/internal/agent/terraform_downloader_test.go b/internal/releases/downloader_test.go similarity index 90% rename from internal/agent/terraform_downloader_test.go rename to internal/releases/downloader_test.go index 813aa6828..5c6ccae2c 100644 --- a/internal/agent/terraform_downloader_test.go +++ b/internal/releases/downloader_test.go @@ -1,4 +1,4 @@ -package agent +package releases import ( "bytes" @@ -24,8 +24,7 @@ func TestDownloader(t *testing.T) { u, err := url.Parse(srv.URL) require.NoError(t, err) - pathFinder := newTerraformPathFinder(t.TempDir()) - dl := NewDownloader(pathFinder) + dl := NewDownloader(t.TempDir()) dl.host = u.Host dl.client = &http.Client{ Transport: otfhttp.DefaultTransport(true), diff --git a/internal/releases/latest_checker.go b/internal/releases/latest_checker.go new file mode 100644 index 000000000..bdf998b52 --- /dev/null +++ b/internal/releases/latest_checker.go @@ -0,0 +1,39 @@ +package releases + +import ( + "encoding/json" + "fmt" + "net/http" + "time" +) + +const latestEndpoint = "https://api.releases.hashicorp.com/v1/releases/terraform/latest" + +// latestChecker checks for a new latest release of terraform. +type latestChecker struct { + endpoint string +} + +func (c latestChecker) check(last time.Time) (string, error) { + // skip check if already checked within last 24 hours + if last.After(time.Now().Add(-24 * time.Hour)) { + return "", nil + } + // check releases endpoint + resp, err := http.Get(c.endpoint) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return "", fmt.Errorf("%s return non-200 status code: %s", c.endpoint, resp.Status) + } + // decode endpoint response + var release struct { + Version string `json:"version"` + } + if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + return "", err + } + return release.Version, nil +} diff --git a/internal/releases/latest_checker_test.go b/internal/releases/latest_checker_test.go new file mode 100644 index 000000000..801658ff9 --- /dev/null +++ b/internal/releases/latest_checker_test.go @@ -0,0 +1,46 @@ +package releases + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/leg100/otf/internal/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_latestChecker(t *testing.T) { + tests := []struct { + name string + last time.Time // last time checked + got string // version returned + }{ + {"skip check", time.Now(), ""}, + {"perform check", time.Time{}, "1.6.1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // endpoint is a stub endpoint that always returns 1.6.1 as latest + // version + endpoint := func() string { + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.Write(testutils.ReadFile(t, "./testdata/latest.json")) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + u, err := url.Parse(srv.URL) + require.NoError(t, err) + return u.String() + }() + + v, err := latestChecker{endpoint}.check(tt.last) + require.NoError(t, err) + assert.Equal(t, tt.got, v) + }) + } +} diff --git a/internal/releases/releases.go b/internal/releases/releases.go new file mode 100644 index 000000000..9d757e621 --- /dev/null +++ b/internal/releases/releases.go @@ -0,0 +1,128 @@ +// Package releases manages terraform releases. +package releases + +import ( + "context" + "errors" + "fmt" + "io" + "time" + + "github.com/leg100/otf/internal" + "github.com/leg100/otf/internal/logr" + "github.com/leg100/otf/internal/semver" + "github.com/leg100/otf/internal/sql" +) + +const ( + DefaultTerraformVersion = "1.5.2" + LatestVersionString = "latest" +) + +type ( + ReleasesService = Service + + Service interface { + // GetLatest returns the latest version of terraform along with the + // time when the latest version was last determined. + GetLatest(ctx context.Context) (string, time.Time, error) + + Downloader + } + + Downloader interface { + // Download a terraform release with the given version and log progress + // updates to logger. Once complete, the path to the release executable + // is returned. + Download(ctx context.Context, version string, w io.Writer) (string, error) + } + + service struct { + logr.Logger + *downloader + latestChecker + + db *db + } + Options struct { + logr.Logger + *sql.DB + + TerraformBinDir string // destination directory for terraform binaries + } +) + +func NewService(opts Options) *service { + svc := &service{ + Logger: opts.Logger, + db: &db{opts.DB}, + latestChecker: latestChecker{latestEndpoint}, + downloader: NewDownloader(opts.TerraformBinDir), + } + return svc +} + +// StartLatestChecker starts the latest checker go routine, checking the Hashicorp +// API endpoint for a new latest version. +func (s *service) StartLatestChecker(ctx context.Context) { + check := func() { + err := func() error { + before, checkpoint, err := s.GetLatest(ctx) + if err != nil { + return err + } + after, err := s.latestChecker.check(checkpoint) + if err != nil { + return err + } + if after == "" { + // check was skipped (too early) + return nil + } + // perform sanity check + if n := semver.Compare(after, before); n <= 0 { + return fmt.Errorf("endpoint returned older version: before: %s; after: %s", before, after) + } + // update db (even if version hasn't changed we need to update the + // checkpoint) + if err := s.db.updateLatestVersion(ctx, after); err != nil { + return err + } + s.V(1).Info("checked latest terraform version", "before", before, "after", after) + return nil + }() + if err != nil { + s.Error(err, "checking latest terraform version") + } + } + // check once at startup + check() + // ...and check every 5 mins thereafter + go func() { + ticker := time.NewTicker(5 * time.Minute) + for { + select { + case <-ticker.C: + check() + case <-ctx.Done(): + ticker.Stop() + return + } + } + }() +} + +// GetLatest returns the latest terraform version and the time when it was +// fetched; if it has not yet been fetched then the default version is returned +// instead along with zero time. +func (s *service) GetLatest(ctx context.Context) (string, time.Time, error) { + latest, checkpoint, err := s.db.getLatest(ctx) + if errors.Is(err, internal.ErrResourceNotFound) { + // no latest version has yet been persisted to the database so return + // the default version instead + return DefaultTerraformVersion, time.Time{}, nil + } else if err != nil { + return "", time.Time{}, err + } + return latest, checkpoint, nil +} diff --git a/internal/releases/testdata/latest.json b/internal/releases/testdata/latest.json new file mode 100644 index 000000000..e106026d1 --- /dev/null +++ b/internal/releases/testdata/latest.json @@ -0,0 +1,95 @@ +{ + "builds": [ + { + "arch": "amd64", + "os": "darwin", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_darwin_amd64.zip" + }, + { + "arch": "arm64", + "os": "darwin", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_darwin_arm64.zip" + }, + { + "arch": "386", + "os": "freebsd", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_freebsd_386.zip" + }, + { + "arch": "amd64", + "os": "freebsd", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_freebsd_amd64.zip" + }, + { + "arch": "arm", + "os": "freebsd", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_freebsd_arm.zip" + }, + { + "arch": "386", + "os": "linux", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_linux_386.zip" + }, + { + "arch": "amd64", + "os": "linux", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_linux_amd64.zip" + }, + { + "arch": "arm", + "os": "linux", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_linux_arm.zip" + }, + { + "arch": "arm64", + "os": "linux", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_linux_arm64.zip" + }, + { + "arch": "386", + "os": "openbsd", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_openbsd_386.zip" + }, + { + "arch": "amd64", + "os": "openbsd", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_openbsd_amd64.zip" + }, + { + "arch": "amd64", + "os": "solaris", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_solaris_amd64.zip" + }, + { + "arch": "386", + "os": "windows", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_windows_386.zip" + }, + { + "arch": "amd64", + "os": "windows", + "url": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_windows_amd64.zip" + } + ], + "is_prerelease": false, + "license_class": "oss", + "name": "terraform", + "status": { + "state": "supported", + "timestamp_updated": "2023-10-10T17:43:14.551Z" + }, + "timestamp_created": "2023-10-10T17:43:14.551Z", + "timestamp_updated": "2023-10-10T17:43:14.551Z", + "url_changelog": "https://github.com/hashicorp/terraform/blob/v1.6/CHANGELOG.md", + "url_docker_registry_dockerhub": "https://hub.docker.com/r/hashicorp/terraform", + "url_docker_registry_ecr": "https://gallery.ecr.aws/hashicorp/terraform", + "url_license": "https://github.com/hashicorp/terraform/blob/main/LICENSE", + "url_project_website": "https://www.terraform.io", + "url_shasums": "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_SHA256SUMS", + "url_shasums_signatures": [ + "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_SHA256SUMS.sig", + "https://releases.hashicorp.com/terraform/1.6.1/terraform_1.6.1_SHA256SUMS.72D7468F.sig" + ], + "url_source_repository": "https://github.com/hashicorp/terraform", + "version": "1.6.1" +} diff --git a/internal/agent/testdata/releases/terraform/1.2.3/terraform_1.2.3_linux_amd64.zip b/internal/releases/testdata/releases/terraform/1.2.3/terraform_1.2.3_linux_amd64.zip similarity index 100% rename from internal/agent/testdata/releases/terraform/1.2.3/terraform_1.2.3_linux_amd64.zip rename to internal/releases/testdata/releases/terraform/1.2.3/terraform_1.2.3_linux_amd64.zip diff --git a/internal/agent/testdata/releases/terraform/1.2.3/terraform_1.2.3_linux_arm64.zip b/internal/releases/testdata/releases/terraform/1.2.3/terraform_1.2.3_linux_arm64.zip similarity index 100% rename from internal/agent/testdata/releases/terraform/1.2.3/terraform_1.2.3_linux_arm64.zip rename to internal/releases/testdata/releases/terraform/1.2.3/terraform_1.2.3_linux_arm64.zip diff --git a/internal/run/factory.go b/internal/run/factory.go index 8ceabda70..5865b394f 100644 --- a/internal/run/factory.go +++ b/internal/run/factory.go @@ -6,6 +6,7 @@ import ( "github.com/leg100/otf/internal/cloud" "github.com/leg100/otf/internal/configversion" + "github.com/leg100/otf/internal/releases" "github.com/leg100/otf/internal/workspace" ) @@ -15,6 +16,7 @@ type factory struct { WorkspaceService ConfigurationVersionService VCSProviderService + releases.ReleasesService } // NewRun constructs a new run using the provided options. @@ -27,6 +29,12 @@ func (f *factory) NewRun(ctx context.Context, workspaceID string, opts CreateOpt if err != nil { return nil, err } + if ws.TerraformVersion == releases.LatestVersionString { + ws.TerraformVersion, _, err = f.GetLatest(ctx) + if err != nil { + return nil, err + } + } // There are two possibilities for the ConfigurationVersionID value: // (a) non-nil, in which case it is deemed to be a configuration version id diff --git a/internal/run/factory_test.go b/internal/run/factory_test.go index 6c5719aa9..a78fcfdbd 100644 --- a/internal/run/factory_test.go +++ b/internal/run/factory_test.go @@ -3,11 +3,13 @@ package run import ( "context" "testing" + "time" "github.com/leg100/otf/internal" "github.com/leg100/otf/internal/cloud" "github.com/leg100/otf/internal/configversion" "github.com/leg100/otf/internal/organization" + "github.com/leg100/otf/internal/releases" "github.com/leg100/otf/internal/vcsprovider" "github.com/leg100/otf/internal/workspace" "github.com/stretchr/testify/assert" @@ -22,6 +24,7 @@ func TestFactory(t *testing.T) { &organization.Organization{}, &workspace.Workspace{}, &configversion.ConfigurationVersion{}, + "", ) got, err := f.NewRun(ctx, "", CreateOptions{}) @@ -39,6 +42,7 @@ func TestFactory(t *testing.T) { &organization.Organization{}, &workspace.Workspace{}, &configversion.ConfigurationVersion{Speculative: true}, + "", ) got, err := f.NewRun(ctx, "", CreateOptions{}) @@ -52,6 +56,7 @@ func TestFactory(t *testing.T) { &organization.Organization{}, &workspace.Workspace{}, &configversion.ConfigurationVersion{}, + "", ) got, err := f.NewRun(ctx, "", CreateOptions{PlanOnly: internal.Bool(true)}) @@ -65,6 +70,7 @@ func TestFactory(t *testing.T) { &organization.Organization{}, &workspace.Workspace{AutoApply: true}, &configversion.ConfigurationVersion{}, + "", ) got, err := f.NewRun(ctx, "", CreateOptions{}) @@ -78,6 +84,7 @@ func TestFactory(t *testing.T) { &organization.Organization{}, &workspace.Workspace{}, &configversion.ConfigurationVersion{}, + "", ) got, err := f.NewRun(ctx, "", CreateOptions{ @@ -93,6 +100,7 @@ func TestFactory(t *testing.T) { &organization.Organization{CostEstimationEnabled: true}, &workspace.Workspace{}, &configversion.ConfigurationVersion{}, + "", ) got, err := f.NewRun(ctx, "", CreateOptions{}) @@ -108,6 +116,7 @@ func TestFactory(t *testing.T) { Connection: &workspace.Connection{}, }, &configversion.ConfigurationVersion{}, + "", ) got, err := f.NewRun(ctx, "", CreateOptions{}) @@ -117,6 +126,20 @@ func TestFactory(t *testing.T) { // if it was newly created assert.Equal(t, "created", got.ConfigurationVersionID) }) + + t.Run("get latest version", func(t *testing.T) { + f := newTestFactory( + &organization.Organization{}, + &workspace.Workspace{TerraformVersion: releases.LatestVersionString}, + &configversion.ConfigurationVersion{}, + "1.2.3", + ) + + got, err := f.NewRun(ctx, "", CreateOptions{}) + require.NoError(t, err) + + assert.Equal(t, "1.2.3", got.TerraformVersion) + }) } type ( @@ -138,14 +161,19 @@ type ( fakeFactoryCloudClient struct { cloud.Client } + fakeReleasesService struct { + latestVersion string + releases.ReleasesService + } ) -func newTestFactory(org *organization.Organization, ws *workspace.Workspace, cv *configversion.ConfigurationVersion) *factory { +func newTestFactory(org *organization.Organization, ws *workspace.Workspace, cv *configversion.ConfigurationVersion, latestVersion string) *factory { return &factory{ OrganizationService: &fakeFactoryOrganizationService{org: org}, WorkspaceService: &fakeFactoryWorkspaceService{ws: ws}, ConfigurationVersionService: &fakeFactoryConfigurationVersionService{cv: cv}, VCSProviderService: &fakeFactoryVCSProviderService{}, + ReleasesService: &fakeReleasesService{latestVersion: latestVersion}, } } @@ -188,3 +216,7 @@ func (f *fakeFactoryCloudClient) GetRepository(context.Context, string) (cloud.R func (f *fakeFactoryCloudClient) GetCommit(context.Context, string, string) (cloud.Commit, error) { return cloud.Commit{}, nil } + +func (f *fakeReleasesService) GetLatest(context.Context) (string, time.Time, error) { + return f.latestVersion, time.Time{}, nil +} diff --git a/internal/run/jsonapi_unmarshal.go b/internal/run/jsonapi_unmarshal.go index e7013d573..1a23f8c49 100644 --- a/internal/run/jsonapi_unmarshal.go +++ b/internal/run/jsonapi_unmarshal.go @@ -35,6 +35,7 @@ func newFromJSONAPI(from *types.Run) *Run { TargetAddrs: from.TargetAddrs, WorkspaceID: from.Workspace.ID, ConfigurationVersionID: from.ConfigurationVersion.ID, + TerraformVersion: from.TerraformVersion, // TODO: unmarshal plan and apply relations } } diff --git a/internal/run/service.go b/internal/run/service.go index cd5f84733..76ab3570f 100644 --- a/internal/run/service.go +++ b/internal/run/service.go @@ -13,6 +13,7 @@ import ( "github.com/leg100/otf/internal/organization" "github.com/leg100/otf/internal/pubsub" "github.com/leg100/otf/internal/rbac" + "github.com/leg100/otf/internal/releases" "github.com/leg100/otf/internal/repo" "github.com/leg100/otf/internal/resource" "github.com/leg100/otf/internal/sql" @@ -98,6 +99,7 @@ type ( WorkspaceService ConfigurationVersionService VCSProviderService + releases.ReleasesService logr.Logger internal.Cache @@ -130,6 +132,7 @@ func NewService(opts Options) *service { opts.WorkspaceService, opts.ConfigurationVersionService, opts.VCSProviderService, + opts.ReleasesService, } svc.web = &webHandlers{ diff --git a/internal/sql/migrations/20231010191539_create_table_latest_version.sql b/internal/sql/migrations/20231010191539_create_table_latest_version.sql new file mode 100644 index 000000000..be323806b --- /dev/null +++ b/internal/sql/migrations/20231010191539_create_table_latest_version.sql @@ -0,0 +1,8 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS latest_terraform_version ( + version TEXT NOT NULL, + checkpoint TIMESTAMPTZ NOT NULL +); + +-- +goose Down +DROP TABLE IF EXISTS latest_terraform_version; diff --git a/internal/sql/pggen/agent_token.sql.go b/internal/sql/pggen/agent_token.sql.go index ba3f3f976..2247fbad4 100644 --- a/internal/sql/pggen/agent_token.sql.go +++ b/internal/sql/pggen/agent_token.sql.go @@ -468,6 +468,27 @@ type Querier interface { // UpdatePlanJSONByIDScan scans the result of an executed UpdatePlanJSONByIDBatch query. UpdatePlanJSONByIDScan(results pgx.BatchResults) (pgtype.Text, error) + InsertLatestTerraformVersion(ctx context.Context, version pgtype.Text) (pgconn.CommandTag, error) + // InsertLatestTerraformVersionBatch enqueues a InsertLatestTerraformVersion query into batch to be executed + // later by the batch. + InsertLatestTerraformVersionBatch(batch genericBatch, version pgtype.Text) + // InsertLatestTerraformVersionScan scans the result of an executed InsertLatestTerraformVersionBatch query. + InsertLatestTerraformVersionScan(results pgx.BatchResults) (pgconn.CommandTag, error) + + UpdateLatestTerraformVersion(ctx context.Context, version pgtype.Text) (pgconn.CommandTag, error) + // UpdateLatestTerraformVersionBatch enqueues a UpdateLatestTerraformVersion query into batch to be executed + // later by the batch. + UpdateLatestTerraformVersionBatch(batch genericBatch, version pgtype.Text) + // UpdateLatestTerraformVersionScan scans the result of an executed UpdateLatestTerraformVersionBatch query. + UpdateLatestTerraformVersionScan(results pgx.BatchResults) (pgconn.CommandTag, error) + + FindLatestTerraformVersion(ctx context.Context) ([]FindLatestTerraformVersionRow, error) + // FindLatestTerraformVersionBatch enqueues a FindLatestTerraformVersion query into batch to be executed + // later by the batch. + FindLatestTerraformVersionBatch(batch genericBatch) + // FindLatestTerraformVersionScan scans the result of an executed FindLatestTerraformVersionBatch query. + FindLatestTerraformVersionScan(results pgx.BatchResults) ([]FindLatestTerraformVersionRow, error) + InsertRepoConnection(ctx context.Context, params InsertRepoConnectionParams) (pgconn.CommandTag, error) // InsertRepoConnectionBatch enqueues a InsertRepoConnection query into batch to be executed // later by the batch. @@ -1510,6 +1531,15 @@ func PrepareAllQueries(ctx context.Context, p preparer) error { if _, err := p.Prepare(ctx, updatePlanJSONByIDSQL, updatePlanJSONByIDSQL); err != nil { return fmt.Errorf("prepare query 'UpdatePlanJSONByID': %w", err) } + if _, err := p.Prepare(ctx, insertLatestTerraformVersionSQL, insertLatestTerraformVersionSQL); err != nil { + return fmt.Errorf("prepare query 'InsertLatestTerraformVersion': %w", err) + } + if _, err := p.Prepare(ctx, updateLatestTerraformVersionSQL, updateLatestTerraformVersionSQL); err != nil { + return fmt.Errorf("prepare query 'UpdateLatestTerraformVersion': %w", err) + } + if _, err := p.Prepare(ctx, findLatestTerraformVersionSQL, findLatestTerraformVersionSQL); err != nil { + return fmt.Errorf("prepare query 'FindLatestTerraformVersion': %w", err) + } if _, err := p.Prepare(ctx, insertRepoConnectionSQL, insertRepoConnectionSQL); err != nil { return fmt.Errorf("prepare query 'InsertRepoConnection': %w", err) } diff --git a/internal/sql/pggen/releases.sql.go b/internal/sql/pggen/releases.sql.go new file mode 100644 index 000000000..4ecaf53af --- /dev/null +++ b/internal/sql/pggen/releases.sql.go @@ -0,0 +1,128 @@ +// Code generated by pggen. DO NOT EDIT. + +package pggen + +import ( + "context" + "fmt" + + "github.com/jackc/pgconn" + "github.com/jackc/pgtype" + "github.com/jackc/pgx/v4" +) + +const insertLatestTerraformVersionSQL = `INSERT INTO latest_terraform_version ( + version, + checkpoint +) VALUES ( + $1, + current_timestamp +);` + +// InsertLatestTerraformVersion implements Querier.InsertLatestTerraformVersion. +func (q *DBQuerier) InsertLatestTerraformVersion(ctx context.Context, version pgtype.Text) (pgconn.CommandTag, error) { + ctx = context.WithValue(ctx, "pggen_query_name", "InsertLatestTerraformVersion") + cmdTag, err := q.conn.Exec(ctx, insertLatestTerraformVersionSQL, version) + if err != nil { + return cmdTag, fmt.Errorf("exec query InsertLatestTerraformVersion: %w", err) + } + return cmdTag, err +} + +// InsertLatestTerraformVersionBatch implements Querier.InsertLatestTerraformVersionBatch. +func (q *DBQuerier) InsertLatestTerraformVersionBatch(batch genericBatch, version pgtype.Text) { + batch.Queue(insertLatestTerraformVersionSQL, version) +} + +// InsertLatestTerraformVersionScan implements Querier.InsertLatestTerraformVersionScan. +func (q *DBQuerier) InsertLatestTerraformVersionScan(results pgx.BatchResults) (pgconn.CommandTag, error) { + cmdTag, err := results.Exec() + if err != nil { + return cmdTag, fmt.Errorf("exec InsertLatestTerraformVersionBatch: %w", err) + } + return cmdTag, err +} + +const updateLatestTerraformVersionSQL = `UPDATE latest_terraform_version +SET version = $1, + checkpoint = current_timestamp;` + +// UpdateLatestTerraformVersion implements Querier.UpdateLatestTerraformVersion. +func (q *DBQuerier) UpdateLatestTerraformVersion(ctx context.Context, version pgtype.Text) (pgconn.CommandTag, error) { + ctx = context.WithValue(ctx, "pggen_query_name", "UpdateLatestTerraformVersion") + cmdTag, err := q.conn.Exec(ctx, updateLatestTerraformVersionSQL, version) + if err != nil { + return cmdTag, fmt.Errorf("exec query UpdateLatestTerraformVersion: %w", err) + } + return cmdTag, err +} + +// UpdateLatestTerraformVersionBatch implements Querier.UpdateLatestTerraformVersionBatch. +func (q *DBQuerier) UpdateLatestTerraformVersionBatch(batch genericBatch, version pgtype.Text) { + batch.Queue(updateLatestTerraformVersionSQL, version) +} + +// UpdateLatestTerraformVersionScan implements Querier.UpdateLatestTerraformVersionScan. +func (q *DBQuerier) UpdateLatestTerraformVersionScan(results pgx.BatchResults) (pgconn.CommandTag, error) { + cmdTag, err := results.Exec() + if err != nil { + return cmdTag, fmt.Errorf("exec UpdateLatestTerraformVersionBatch: %w", err) + } + return cmdTag, err +} + +const findLatestTerraformVersionSQL = `SELECT * +FROM latest_terraform_version;` + +type FindLatestTerraformVersionRow struct { + Version pgtype.Text `json:"version"` + Checkpoint pgtype.Timestamptz `json:"checkpoint"` +} + +// FindLatestTerraformVersion implements Querier.FindLatestTerraformVersion. +func (q *DBQuerier) FindLatestTerraformVersion(ctx context.Context) ([]FindLatestTerraformVersionRow, error) { + ctx = context.WithValue(ctx, "pggen_query_name", "FindLatestTerraformVersion") + rows, err := q.conn.Query(ctx, findLatestTerraformVersionSQL) + if err != nil { + return nil, fmt.Errorf("query FindLatestTerraformVersion: %w", err) + } + defer rows.Close() + items := []FindLatestTerraformVersionRow{} + for rows.Next() { + var item FindLatestTerraformVersionRow + if err := rows.Scan(&item.Version, &item.Checkpoint); err != nil { + return nil, fmt.Errorf("scan FindLatestTerraformVersion row: %w", err) + } + items = append(items, item) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("close FindLatestTerraformVersion rows: %w", err) + } + return items, err +} + +// FindLatestTerraformVersionBatch implements Querier.FindLatestTerraformVersionBatch. +func (q *DBQuerier) FindLatestTerraformVersionBatch(batch genericBatch) { + batch.Queue(findLatestTerraformVersionSQL) +} + +// FindLatestTerraformVersionScan implements Querier.FindLatestTerraformVersionScan. +func (q *DBQuerier) FindLatestTerraformVersionScan(results pgx.BatchResults) ([]FindLatestTerraformVersionRow, error) { + rows, err := results.Query() + if err != nil { + return nil, fmt.Errorf("query FindLatestTerraformVersionBatch: %w", err) + } + defer rows.Close() + items := []FindLatestTerraformVersionRow{} + for rows.Next() { + var item FindLatestTerraformVersionRow + if err := rows.Scan(&item.Version, &item.Checkpoint); err != nil { + return nil, fmt.Errorf("scan FindLatestTerraformVersionBatch row: %w", err) + } + items = append(items, item) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("close FindLatestTerraformVersionBatch rows: %w", err) + } + return items, err +} diff --git a/internal/sql/queries/releases.sql b/internal/sql/queries/releases.sql new file mode 100644 index 000000000..744894a47 --- /dev/null +++ b/internal/sql/queries/releases.sql @@ -0,0 +1,17 @@ +-- name: InsertLatestTerraformVersion :exec +INSERT INTO latest_terraform_version ( + version, + checkpoint +) VALUES ( + pggen.arg('version'), + current_timestamp +); + +-- name: UpdateLatestTerraformVersion :exec +UPDATE latest_terraform_version +SET version = pggen.arg('version'), + checkpoint = current_timestamp; + +-- name: FindLatestTerraformVersion :many +SELECT * +FROM latest_terraform_version; diff --git a/internal/tfeapi/types/organization.go b/internal/tfeapi/types/organization.go index 44bc8d69c..44e72d211 100644 --- a/internal/tfeapi/types/organization.go +++ b/internal/tfeapi/types/organization.go @@ -28,6 +28,8 @@ type Organization struct { TrialExpiresAt time.Time `jsonapi:"attribute" json:"trial-expires-at"` TwoFactorConformant bool `jsonapi:"attribute" json:"two-factor-conformant"` SendPassingStatusesForUntriggeredSpeculativePlans bool `jsonapi:"attribute" json:"send-passing-statuses-for-untriggered-speculative-plans"` + RemainingTestableCount int `jsonapi:"attribute" json:"remaining-testable-count"` + // Note: This will be false for TFE versions older than v202211, where the setting was introduced. // On those TFE versions, safe delete does not exist, so ALL deletes will be force deletes. AllowForceDeleteWorkspaces bool `jsonapi:"attribute" json:"allow-force-delete-workspaces"` diff --git a/internal/workspace/workspace.go b/internal/workspace/workspace.go index 6ada8d541..77b262ae8 100644 --- a/internal/workspace/workspace.go +++ b/internal/workspace/workspace.go @@ -13,6 +13,7 @@ import ( "github.com/gobwas/glob" "github.com/leg100/otf/internal" + "github.com/leg100/otf/internal/releases" "github.com/leg100/otf/internal/resource" "github.com/leg100/otf/internal/semver" ) @@ -23,9 +24,7 @@ const ( AgentExecutionMode ExecutionMode = "agent" DefaultAllowDestroyPlan = true - MinTerraformVersion = "1.2.0" - DefaultTerraformVersion = "1.5.2" ) var ( @@ -202,7 +201,7 @@ func NewWorkspace(opts CreateOptions) (*Workspace, error) { UpdatedAt: internal.CurrentTimestamp(), AllowDestroyPlan: DefaultAllowDestroyPlan, ExecutionMode: RemoteExecutionMode, - TerraformVersion: DefaultTerraformVersion, + TerraformVersion: releases.DefaultTerraformVersion, SpeculativeEnabled: true, Organization: *opts.Organization, } @@ -490,10 +489,13 @@ func (ws *Workspace) setExecutionMode(m ExecutionMode) error { } func (ws *Workspace) setTerraformVersion(v string) error { + if v == releases.LatestVersionString { + ws.TerraformVersion = v + return nil + } if !semver.IsValid(v) { return internal.ErrInvalidTerraformVersion } - // only accept terraform versions above the minimum requirement. // // NOTE: we make an exception for the specific versions posted by the go-tfe diff --git a/internal/workspace/workspace_test.go b/internal/workspace/workspace_test.go index 50a58c615..bfe013077 100644 --- a/internal/workspace/workspace_test.go +++ b/internal/workspace/workspace_test.go @@ -43,6 +43,14 @@ func TestNewWorkspace(t *testing.T) { }, want: internal.ErrInvalidName, }, + { + name: "specifying latest for terraform version", + opts: CreateOptions{ + Name: internal.String("my-workspace"), + Organization: internal.String("my-org"), + TerraformVersion: internal.String("latest"), + }, + }, { name: "bad terraform version", opts: CreateOptions{