fix: Convert all jobs to use a common resource and agent type (#369)

* ci: Update DataDog GitHub branch to fallback to GITHUB_REF This was detecting branches, but not our "main" branch before. Hopefully this fixes it! * Add basic Terraform Provider * Rename post files to upload * Add tests for resources * Skip instance identity test * Add tests for ensuring agent get's passed through properly * Fix linting errors * Add echo path * Fix agent authentication * fix: Convert all jobs to use a common resource and agent type This enables a consistent API for project import and provisioned resources.
coder · Feb 28, 2022 · bd0293a · bd0293a
1 parent 2d6804c
commit bd0293a
Show file tree

Hide file tree

Showing 18 changed files with 629 additions and 635 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -50,6 +50,7 @@
     "ntqry",
     "oneof",
     "parameterscopeid",
+    "pqtype",
     "promptui",
     "protobuf",
     "provisionerd",

diff --git a/cli/projects.go b/cli/projects.go
@@ -40,7 +40,7 @@ func projects() *cobra.Command {
 	return cmd
 }
 
-func displayProjectImportInfo(cmd *cobra.Command, parameterSchemas []coderd.ParameterSchema, parameterValues []coderd.ComputedParameterValue, resources []coderd.ProjectImportJobResource) error {
+func displayProjectImportInfo(cmd *cobra.Command, parameterSchemas []coderd.ParameterSchema, parameterValues []coderd.ComputedParameterValue, resources []coderd.ProvisionerJobResource) error {
 	schemaByID := map[string]coderd.ParameterSchema{}
 	for _, schema := range parameterSchemas {
 		schemaByID[schema.ID.String()] = schema

diff --git a/coderd/projectimport.go b/coderd/projectimport.go
@@ -21,9 +21,6 @@ type ParameterSchema database.ParameterSchema
 // ComputedParameterValue represents a computed parameter value.
 type ComputedParameterValue parameter.ComputedValue
 
-// ProjectImportJobResource is a resource created by a project import job.
-type ProjectImportJobResource database.ProjectImportJobResource
-
 // CreateProjectImportJobRequest provides options to create a project import job.
 type CreateProjectImportJobRequest struct {
 	StorageMethod database.ProvisionerStorageMethod `json:"storage_method" validate:"oneof=file,required"`
@@ -167,7 +164,7 @@ func (api *api) projectImportJobResourcesByID(rw http.ResponseWriter, r *http.Re
 		})
 		return
 	}
-	resources, err := api.Database.GetProjectImportJobResourcesByJobID(r.Context(), job.ID)
+	resources, err := api.Database.GetProvisionerJobResourcesByJobID(r.Context(), job.ID)
 	if errors.Is(err, sql.ErrNoRows) {
 		err = nil
 	}
@@ -178,7 +175,7 @@ func (api *api) projectImportJobResourcesByID(rw http.ResponseWriter, r *http.Re
 		return
 	}
 	if resources == nil {
-		resources = []database.ProjectImportJobResource{}
+		resources = []database.ProvisionerJobResource{}
 	}
 	render.Status(r, http.StatusOK)
 	render.JSON(rw, r, resources)

diff --git a/coderd/provisionerdaemons.go b/coderd/provisionerdaemons.go
@@ -15,6 +15,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/hashicorp/yamux"
 	"github.com/moby/moby/pkg/namesgenerator"
+	"github.com/tabbed/pqtype"
 	"golang.org/x/xerrors"
 	"nhooyr.io/websocket"
 	"storj.io/drpc/drpcmux"
@@ -453,14 +454,8 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr
 					slog.F("resource_name", resource.Name),
 					slog.F("resource_type", resource.Type),
 					slog.F("transition", transition))
-				_, err = server.Database.InsertProjectImportJobResource(ctx, database.InsertProjectImportJobResourceParams{
-					ID:         uuid.New(),
-					CreatedAt:  database.Now(),
-					JobID:      jobID,
-					Transition: transition,
-					Type:       resource.Type,
-					Name:       resource.Name,
-				})
+
+				err = insertProvisionerJobResource(ctx, server.Database, jobID, transition, resource)
 				if err != nil {
 					return nil, xerrors.Errorf("insert resource: %w", err)
 				}
@@ -516,26 +511,9 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr
 			}
 			// This could be a bulk insert to improve performance.
 			for _, protoResource := range jobType.WorkspaceProvision.Resources {
-				var instanceID sql.NullString
-				if protoResource.Agent != nil && protoResource.Agent.GetGoogleInstanceIdentity() != nil {
-					instanceID = sql.NullString{
-						String: protoResource.Agent.GetGoogleInstanceIdentity().InstanceId,
-						Valid:  true,
-					}
-				}
-				_, err = db.InsertWorkspaceResource(ctx, database.InsertWorkspaceResourceParams{
-					ID:                 uuid.New(),
-					CreatedAt:          database.Now(),
-					WorkspaceHistoryID: input.WorkspaceHistoryID,
-					Type:               protoResource.Type,
-					Name:               protoResource.Name,
-					InstanceID:         instanceID,
-					// TODO: Generate this at the variable validation phase.
-					// Set the value in `default_source`, and disallow overwrite.
-					WorkspaceAgentToken: uuid.NewString(),
-				})
+				err = insertProvisionerJobResource(ctx, db, job.ID, workspaceHistory.Transition, protoResource)
 				if err != nil {
-					return xerrors.Errorf("insert workspace resource %q: %w", protoResource.Name, err)
+					return xerrors.Errorf("insert provisioner job: %w", err)
 				}
 			}
 			return nil
@@ -551,6 +529,61 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr
 	return &proto.Empty{}, nil
 }
 
+func insertProvisionerJobResource(ctx context.Context, db database.Store, jobID uuid.UUID, transition database.WorkspaceTransition, protoResource *sdkproto.Resource) error {
+	resource, err := db.InsertProvisionerJobResource(ctx, database.InsertProvisionerJobResourceParams{
+		ID:         uuid.New(),
+		CreatedAt:  database.Now(),
+		JobID:      jobID,
+		Transition: transition,
+		Type:       protoResource.Type,
+		Name:       protoResource.Name,
+		AgentID: uuid.NullUUID{
+			UUID:  uuid.New(),
+			Valid: protoResource.Agent != nil,
+		},
+	})
+	if err != nil {
+		return xerrors.Errorf("insert provisioner job resource %q: %w", protoResource.Name, err)
+	}
+	if resource.AgentID.Valid {
+		var instanceID sql.NullString
+		if protoResource.Agent.GetGoogleInstanceIdentity() != nil {
+			instanceID = sql.NullString{
+				String: protoResource.Agent.GetGoogleInstanceIdentity().InstanceId,
+				Valid:  true,
+			}
+		}
+		var env pqtype.NullRawMessage
+		if protoResource.Agent.Env != nil {
+			data, err := json.Marshal(protoResource.Agent.Env)
+			if err != nil {
+				return xerrors.Errorf("marshal env: %w", err)
+			}
+			env = pqtype.NullRawMessage{
+				RawMessage: data,
+				Valid:      true,
+			}
+		}
+
+		_, err := db.InsertProvisionerJobAgent(ctx, database.InsertProvisionerJobAgentParams{
+			ID:                   resource.AgentID.UUID,
+			CreatedAt:            database.Now(),
+			ResourceID:           resource.ID,
+			AuthToken:            uuid.New(),
+			AuthInstanceID:       instanceID,
+			EnvironmentVariables: env,
+			StartupScript: sql.NullString{
+				String: protoResource.Agent.StartupScript,
+				Valid:  protoResource.Agent.StartupScript != "",
+			},
+		})
+		if err != nil {
+			return xerrors.Errorf("insert agent: %w", err)
+		}
+	}
+	return nil
+}
+
 func convertValidationTypeSystem(typeSystem sdkproto.ParameterSchema_TypeSystem) (database.ParameterTypeSystem, error) {
 	switch typeSystem {
 	case sdkproto.ParameterSchema_None:

diff --git a/coderd/provisionerjobs.go b/coderd/provisionerjobs.go
@@ -50,13 +50,32 @@ type ProvisionerJob struct {
 
 // ProvisionerJobLog represents a single log from a provisioner job.
 type ProvisionerJobLog struct {
-	ID        uuid.UUID
+	ID        uuid.UUID          `json:"id"`
 	CreatedAt time.Time          `json:"created_at"`
 	Source    database.LogSource `json:"log_source"`
 	Level     database.LogLevel  `json:"log_level"`
 	Output    string             `json:"output"`
 }
 
+type ProvisionerJobResource struct {
+	ID         uuid.UUID                    `json:"id"`
+	CreatedAt  time.Time                    `json:"created_at"`
+	JobID      uuid.UUID                    `json:"job_id"`
+	Transition database.WorkspaceTransition `json:"workspace_transition"`
+	Type       string                       `json:"type"`
+	Name       string                       `json:"name"`
+}
+
+type ProvisionerJobAgent struct {
+	ID                   uuid.UUID         `json:"id"`
+	CreatedAt            time.Time         `json:"created_at"`
+	UpdatedAt            time.Time         `json:"updated_at"`
+	ResourceID           uuid.UUID         `json:"resource_id"`
+	InstanceID           string            `json:"instance_id,omitempty"`
+	EnvironmentVariables map[string]string `json:"environment_variables"`
+	StartupScript        string            `json:"startup_script,omitempty"`
+}
+
 func (*api) provisionerJobByID(rw http.ResponseWriter, r *http.Request) {
 	job := httpmw.ProvisionerJobParam(r)
 	render.Status(r, http.StatusOK)

diff --git a/coderd/workspaceagent.go b/coderd/workspaceagent.go
@@ -2,12 +2,14 @@ package coderd
 
 import (
 	"database/sql"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
 
 	"github.com/go-chi/render"
 
+	"github.com/coder/coder/database"
 	"github.com/coder/coder/httpapi"
 
 	"github.com/mitchellh/mapstructure"
@@ -54,14 +56,48 @@ func (api *api) postAuthenticateWorkspaceAgentUsingGoogleInstanceIdentity(rw htt
 		})
 		return
 	}
-	resource, err := api.Database.GetWorkspaceResourceByInstanceID(r.Context(), claims.Google.ComputeEngine.InstanceID)
+	agent, err := api.Database.GetProvisionerJobAgentByInstanceID(r.Context(), claims.Google.ComputeEngine.InstanceID)
 	if errors.Is(err, sql.ErrNoRows) {
 		httpapi.Write(rw, http.StatusNotFound, httpapi.Response{
 			Message: fmt.Sprintf("instance with id %q not found", claims.Google.ComputeEngine.InstanceID),
 		})
 		return
 	}
-	resourceHistory, err := api.Database.GetWorkspaceHistoryByID(r.Context(), resource.WorkspaceHistoryID)
+	if err != nil {
+		httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
+			Message: fmt.Sprintf("get provisioner job agent: %s", err),
+		})
+		return
+	}
+	resource, err := api.Database.GetProvisionerJobResourceByID(r.Context(), agent.ResourceID)
+	if err != nil {
+		httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
+			Message: fmt.Sprintf("get provisioner job resource: %s", err),
+		})
+		return
+	}
+	job, err := api.Database.GetProvisionerJobByID(r.Context(), resource.JobID)
+	if err != nil {
+		httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
+			Message: fmt.Sprintf("get provisioner job: %s", err),
+		})
+		return
+	}
+	if job.Type != database.ProvisionerJobTypeWorkspaceProvision {
+		httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{
+			Message: fmt.Sprintf("%q jobs cannot be authenticated", job.Type),
+		})
+		return
+	}
+	var jobData workspaceProvisionJob
+	err = json.Unmarshal(job.Input, &jobData)
+	if err != nil {
+		httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
+			Message: fmt.Sprintf("extract job data: %s", err),
+		})
+		return
+	}
+	resourceHistory, err := api.Database.GetWorkspaceHistoryByID(r.Context(), jobData.WorkspaceHistoryID)
 	if err != nil {
 		httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
 			Message: fmt.Sprintf("get workspace history: %s", err),
@@ -86,6 +122,6 @@ func (api *api) postAuthenticateWorkspaceAgentUsingGoogleInstanceIdentity(rw htt
 	}
 	render.Status(r, http.StatusOK)
 	render.JSON(rw, r, WorkspaceAgentAuthenticateResponse{
-		SessionToken: resource.WorkspaceAgentToken,
+		SessionToken: agent.AuthToken.String(),
 	})
 }
diff --git a/codersdk/files.go b/codersdk/files.go
@@ -3,9 +3,7 @@ package codersdk
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"net/http"
-	"net/url"
 
 	"github.com/coder/coder/coderd"
 )
@@ -28,8 +26,3 @@ func (c *Client) UploadFile(ctx context.Context, contentType string, content []b
 	var resp coderd.UploadFileResponse
 	return resp, json.NewDecoder(res.Body).Decode(&resp)
 }
-
-// DownloadURL returns the download URL for the specified asset
-func (c *Client) DownloadURL(asset string) (*url.URL, error) {
-	return c.URL.Parse(fmt.Sprintf("/api/v2/downloads/%s", asset))
-}
diff --git a/codersdk/projectimport.go b/codersdk/projectimport.go
@@ -81,7 +81,7 @@ func (c *Client) ProjectImportJobParameters(ctx context.Context, organization st
 }
 
 // ProjectImportJobResources returns resources for a project import job.
-func (c *Client) ProjectImportJobResources(ctx context.Context, organization string, job uuid.UUID) ([]coderd.ProjectImportJobResource, error) {
+func (c *Client) ProjectImportJobResources(ctx context.Context, organization string, job uuid.UUID) ([]coderd.ProvisionerJobResource, error) {
 	res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/projectimport/%s/%s/resources", organization, job), nil)
 	if err != nil {
 		return nil, err
@@ -90,6 +90,6 @@ func (c *Client) ProjectImportJobResources(ctx context.Context, organization str
 	if res.StatusCode != http.StatusOK {
 		return nil, readBodyAsError(res)
 	}
-	var resources []coderd.ProjectImportJobResource
+	var resources []coderd.ProvisionerJobResource
 	return resources, json.NewDecoder(res.Body).Decode(&resources)
 }