Skip to content

Commit

Permalink
refactor: workspace builds (#7541)
Browse files Browse the repository at this point in the history
* refactor workspace builds

Signed-off-by: Spike Curtis <spike@coder.com>

* make gen

Signed-off-by: Spike Curtis <spike@coder.com>

* Remove ParameterResolver from typescript

Signed-off-by: Spike Curtis <spike@coder.com>

* rename conversion -> database/db2sdk

Signed-off-by: Spike Curtis <spike@coder.com>

* tests for db2sdk

Signed-off-by: Spike Curtis <spike@coder.com>

* Tests for ParameterResolver

Signed-off-by: Spike Curtis <spike@coder.com>

* wsbuilder tests

Signed-off-by: Spike Curtis <spike@coder.com>

* Move parameter validation tests to richparameters_test.go

Signed-off-by: Spike Curtis <spike@coder.com>

* Fix CI generation; rename mock->dbmock

Signed-off-by: Spike Curtis <spike@coder.com>

* Fix test imports

Signed-off-by: Spike Curtis <spike@coder.com>

---------

Signed-off-by: Spike Curtis <spike@coder.com>
  • Loading branch information
spikecurtis committed May 23, 2023
1 parent 622456f commit cd416c8
Show file tree
Hide file tree
Showing 31 changed files with 5,678 additions and 1,235 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,8 @@ jobs:
run: go install golang.org/x/tools/cmd/goimports@latest
- name: Install yq
run: go run github.com/mikefarah/yq/v4@v4.30.6
- name: Install mockgen
run: go install github.com/golang/mock/mockgen@v1.6.0

- name: Install Protoc
run: |
Expand Down
6 changes: 6 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,7 @@ lint/shellcheck: $(SHELL_SRC_FILES)
gen: \
coderd/database/dump.sql \
coderd/database/querier.go \
coderd/database/dbmock/store.go \
provisionersdk/proto/provisioner.pb.go \
provisionerd/proto/provisionerd.pb.go \
site/src/api/typesGenerated.ts \
Expand All @@ -441,6 +442,7 @@ gen/mark-fresh:
files="\
coderd/database/dump.sql \
coderd/database/querier.go \
coderd/database/dbmock/store.go \
provisionersdk/proto/provisioner.pb.go \
provisionerd/proto/provisionerd.pb.go \
site/src/api/typesGenerated.ts \
Expand Down Expand Up @@ -476,6 +478,10 @@ coderd/database/dump.sql: coderd/database/gen/dump/main.go $(wildcard coderd/dat
coderd/database/querier.go: coderd/database/sqlc.yaml coderd/database/dump.sql $(wildcard coderd/database/queries/*.sql) coderd/database/gen/enum/main.go
./coderd/database/generate.sh


coderd/database/dbmock/store.go: coderd/database/db.go coderd/database/querier.go
go generate ./coderd/database/dbmock/

provisionersdk/proto/provisioner.pb.go: provisionersdk/proto/provisioner.proto
protoc \
--go_out=. \
Expand Down
28 changes: 12 additions & 16 deletions cli/templatedelete_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,15 +51,13 @@ func TestTemplateDelete(t *testing.T) {

client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})

This comment has been minimized.

Copy link
@Pearlminima-prog

Pearlminima-prog May 23, 2023

cli/templatedelete_test.go

This comment has been minimized.

Copy link
@Pearlminima-prog

Pearlminima-prog May 23, 2023

coderd/apidoc/docs.go

user := coderdtest.CreateFirstUser(t, client)

This comment has been minimized.

Copy link
@Pearlminima-prog

Pearlminima-prog May 23, 2023

coderd/apidoc/docs.go

version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
templates := []codersdk.Template{
coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID),
coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID),
coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID),
}
templates := []codersdk.Template{}
templateNames := []string{}
for _, template := range templates {
for i := 0; i < 3; i++ {
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
templates = append(templates, template)
templateNames = append(templateNames, template.Name)
}

Expand All @@ -78,15 +76,13 @@ func TestTemplateDelete(t *testing.T) {

client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
user := coderdtest.CreateFirstUser(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
templates := []codersdk.Template{
coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID),
coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID),
coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID),
}
templates := []codersdk.Template{}
templateNames := []string{}
for _, template := range templates {
for i := 0; i < 3; i++ {
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
templates = append(templates, template)
templateNames = append(templateNames, template.Name)
}

Expand Down
3 changes: 3 additions & 0 deletions coderd/apidoc/docs.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions coderd/apidoc/swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

120 changes: 23 additions & 97 deletions coderd/autobuild/executor/lifecycle_executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package executor

import (
"context"
"encoding/json"
"database/sql"
"sync/atomic"
"time"

Expand All @@ -13,8 +13,8 @@ import (
"cdr.dev/slog"
"github.com/coder/coder/coderd/database"
"github.com/coder/coder/coderd/database/dbauthz"
"github.com/coder/coder/coderd/provisionerdserver"
"github.com/coder/coder/coderd/schedule"
"github.com/coder/coder/coderd/wsbuilder"
)

// Executor automatically starts or stops workspaces.
Expand Down Expand Up @@ -168,20 +168,35 @@ func (e *Executor) runOnce(t time.Time) Stats {
)
return nil
}

log.Info(e.ctx, "scheduling workspace transition", slog.F("transition", validTransition))

stats.Transitions[ws.ID] = validTransition
if err := build(e.ctx, db, ws, validTransition, priorHistory, priorJob); err != nil {
builder := wsbuilder.New(ws, validTransition).
SetLastWorkspaceBuildInTx(&priorHistory).
SetLastWorkspaceBuildJobInTx(&priorJob)

switch validTransition {
case database.WorkspaceTransitionStart:
builder = builder.Reason(database.BuildReasonAutostart)
case database.WorkspaceTransitionStop:
builder = builder.Reason(database.BuildReasonAutostop)
default:
log.Error(e.ctx, "unsupported transition", slog.F("transition", validTransition))
return nil
}
if _, _, err := builder.Build(e.ctx, db, nil); err != nil {
log.Error(e.ctx, "unable to transition workspace",
slog.F("transition", validTransition),
slog.Error(err),
)
return nil
}
stats.Transitions[ws.ID] = validTransition

log.Info(e.ctx, "scheduling workspace transition", slog.F("transition", validTransition))

return nil
}, nil)

// Run with RepeatableRead isolation so that the build process sees the same data
// as our calculation that determines whether an autobuild is necessary.
}, &sql.TxOptions{Isolation: sql.LevelRepeatableRead})
if err != nil {
log.Error(e.ctx, "workspace scheduling failed", slog.Error(err))
}
Expand Down Expand Up @@ -248,92 +263,3 @@ func getNextTransition(
return "", time.Time{}, xerrors.Errorf("last transition not valid for autostart or autostop")
}
}

// TODO(cian): this function duplicates most of api.postWorkspaceBuilds. Refactor.
// See: https://github.com/coder/coder/issues/1401
func build(ctx context.Context, store database.Store, workspace database.Workspace, trans database.WorkspaceTransition, priorHistory database.WorkspaceBuild, priorJob database.ProvisionerJob) error {
template, err := store.GetTemplateByID(ctx, workspace.TemplateID)
if err != nil {
return xerrors.Errorf("get workspace template: %w", err)
}

priorBuildNumber := priorHistory.BuildNumber

// This must happen in a transaction to ensure history can be inserted, and
// the prior history can update it's "after" column to point at the new.
workspaceBuildID := uuid.New()
input, err := json.Marshal(provisionerdserver.WorkspaceProvisionJob{
WorkspaceBuildID: workspaceBuildID,
})
if err != nil {
return xerrors.Errorf("marshal provision job: %w", err)
}
provisionerJobID := uuid.New()
now := database.Now()

var buildReason database.BuildReason
switch trans {
case database.WorkspaceTransitionStart:
buildReason = database.BuildReasonAutostart
case database.WorkspaceTransitionStop:
buildReason = database.BuildReasonAutostop
default:
return xerrors.Errorf("Unsupported transition: %q", trans)
}

lastBuildParameters, err := store.GetWorkspaceBuildParameters(ctx, priorHistory.ID)
if err != nil {
return xerrors.Errorf("fetch prior workspace build parameters: %w", err)
}

return store.InTx(func(db database.Store) error {
newProvisionerJob, err := store.InsertProvisionerJob(ctx, database.InsertProvisionerJobParams{
ID: provisionerJobID,
CreatedAt: now,
UpdatedAt: now,
InitiatorID: workspace.OwnerID,
OrganizationID: template.OrganizationID,
Provisioner: template.Provisioner,
Type: database.ProvisionerJobTypeWorkspaceBuild,
StorageMethod: priorJob.StorageMethod,
FileID: priorJob.FileID,
Tags: priorJob.Tags,
Input: input,
})
if err != nil {
return xerrors.Errorf("insert provisioner job: %w", err)
}
workspaceBuild, err := store.InsertWorkspaceBuild(ctx, database.InsertWorkspaceBuildParams{
ID: workspaceBuildID,
CreatedAt: now,
UpdatedAt: now,
WorkspaceID: workspace.ID,
TemplateVersionID: priorHistory.TemplateVersionID,
BuildNumber: priorBuildNumber + 1,
ProvisionerState: priorHistory.ProvisionerState,
InitiatorID: workspace.OwnerID,
Transition: trans,
JobID: newProvisionerJob.ID,
Reason: buildReason,
})
if err != nil {
return xerrors.Errorf("insert workspace build: %w", err)
}

names := make([]string, 0, len(lastBuildParameters))
values := make([]string, 0, len(lastBuildParameters))
for _, param := range lastBuildParameters {
names = append(names, param.Name)
values = append(values, param.Value)
}
err = db.InsertWorkspaceBuildParameters(ctx, database.InsertWorkspaceBuildParametersParams{
WorkspaceBuildID: workspaceBuild.ID,
Name: names,
Value: values,
})
if err != nil {
return xerrors.Errorf("insert workspace build parameters: %w", err)
}
return nil
}, nil)
}
113 changes: 113 additions & 0 deletions coderd/database/db2sdk/db2sdk.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// Package db2sdk provides common conversion routines from database types to codersdk types
package db2sdk

import (
"encoding/json"
"time"

"github.com/coder/coder/coderd/database"
"github.com/coder/coder/coderd/parameter"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/provisionersdk/proto"
)

func WorkspaceBuildParameters(params []database.WorkspaceBuildParameter) []codersdk.WorkspaceBuildParameter {
out := make([]codersdk.WorkspaceBuildParameter, len(params))
for i, p := range params {
out[i] = WorkspaceBuildParameter(p)
}
return out
}

func WorkspaceBuildParameter(p database.WorkspaceBuildParameter) codersdk.WorkspaceBuildParameter {
return codersdk.WorkspaceBuildParameter{
Name: p.Name,
Value: p.Value,
}
}

func TemplateVersionParameter(param database.TemplateVersionParameter) (codersdk.TemplateVersionParameter, error) {
var protoOptions []*proto.RichParameterOption
err := json.Unmarshal(param.Options, &protoOptions)
if err != nil {
return codersdk.TemplateVersionParameter{}, err
}
options := make([]codersdk.TemplateVersionParameterOption, 0)
for _, option := range protoOptions {
options = append(options, codersdk.TemplateVersionParameterOption{
Name: option.Name,
Description: option.Description,
Value: option.Value,
Icon: option.Icon,
})
}

descriptionPlaintext, err := parameter.Plaintext(param.Description)
if err != nil {
return codersdk.TemplateVersionParameter{}, err
}
return codersdk.TemplateVersionParameter{
Name: param.Name,
DisplayName: param.DisplayName,
Description: param.Description,
DescriptionPlaintext: descriptionPlaintext,
Type: param.Type,
Mutable: param.Mutable,
DefaultValue: param.DefaultValue,
Icon: param.Icon,
Options: options,
ValidationRegex: param.ValidationRegex,
ValidationMin: param.ValidationMin,
ValidationMax: param.ValidationMax,
ValidationError: param.ValidationError,
ValidationMonotonic: codersdk.ValidationMonotonicOrder(param.ValidationMonotonic),
Required: param.Required,
LegacyVariableName: param.LegacyVariableName,
}, nil
}

func Parameters(params []database.ParameterValue) []codersdk.Parameter {
out := make([]codersdk.Parameter, len(params))
for i, p := range params {
out[i] = Parameter(p)
}
return out
}

func Parameter(parameterValue database.ParameterValue) codersdk.Parameter {
return codersdk.Parameter{
ID: parameterValue.ID,
CreatedAt: parameterValue.CreatedAt,
UpdatedAt: parameterValue.UpdatedAt,
Scope: codersdk.ParameterScope(parameterValue.Scope),
ScopeID: parameterValue.ScopeID,
Name: parameterValue.Name,
SourceScheme: codersdk.ParameterSourceScheme(parameterValue.SourceScheme),
DestinationScheme: codersdk.ParameterDestinationScheme(parameterValue.DestinationScheme),
SourceValue: parameterValue.SourceValue,
}
}

func ProvisionerJobStatus(provisionerJob database.ProvisionerJob) codersdk.ProvisionerJobStatus {
switch {
case provisionerJob.CanceledAt.Valid:
if !provisionerJob.CompletedAt.Valid {
return codersdk.ProvisionerJobCanceling
}
if provisionerJob.Error.String == "" {
return codersdk.ProvisionerJobCanceled
}
return codersdk.ProvisionerJobFailed
case !provisionerJob.StartedAt.Valid:
return codersdk.ProvisionerJobPending
case provisionerJob.CompletedAt.Valid:
if provisionerJob.Error.String == "" {
return codersdk.ProvisionerJobSucceeded
}
return codersdk.ProvisionerJobFailed
case database.Now().Sub(provisionerJob.UpdatedAt) > 30*time.Second:
return codersdk.ProvisionerJobFailed
default:
return codersdk.ProvisionerJobRunning
}
}

0 comments on commit cd416c8

Please sign in to comment.