feat: Add streaming endpoint for workspace history (#157)

* feat: Add parameter querying to the API * feat: Add streaming endpoint for workspace history Enables a buildlog-like flow for reading job output. * Fix empty parameter source and destination * Add comment for usage of workspace history logs endpoint
coder · Feb 4, 2022 · 65de6ee · 65de6ee
1 parent 430341b
commit 65de6ee
Show file tree

Hide file tree

Showing 22 changed files with 688 additions and 115 deletions.
diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -72,6 +72,7 @@ func New(options *Options) http.Handler {
 						r.Route("/{projecthistory}", func(r chi.Router) {
 							r.Use(httpmw.ExtractProjectHistoryParam(api.Database))
 							r.Get("/", api.projectHistoryByOrganizationAndName)
+							r.Get("/parameters", api.projectHistoryParametersByOrganizationAndName)
 						})
 					})
 				})
@@ -96,6 +97,7 @@ func New(options *Options) http.Handler {
 						r.Route("/{workspacehistory}", func(r chi.Router) {
 							r.Use(httpmw.ExtractWorkspaceHistoryParam(options.Database))
 							r.Get("/", api.workspaceHistoryByName)
+							r.Get("/logs", api.workspaceHistoryLogsByName)
 						})
 					})
 				})

diff --git a/coderd/parameters.go b/coderd/parameters.go
@@ -16,11 +16,11 @@ import (
 
 // CreateParameterValueRequest is used to create a new parameter value for a scope.
 type CreateParameterValueRequest struct {
-	Name              string                              `json:"name"`
-	SourceValue       string                              `json:"source_value"`
-	SourceScheme      database.ParameterSourceScheme      `json:"source_scheme"`
-	DestinationScheme database.ParameterDestinationScheme `json:"destination_scheme"`
-	DestinationValue  string                              `json:"destination_value"`
+	Name              string                              `json:"name" validate:"required"`
+	SourceValue       string                              `json:"source_value" validate:"required"`
+	SourceScheme      database.ParameterSourceScheme      `json:"source_scheme" validate:"oneof=data,required"`
+	DestinationScheme database.ParameterDestinationScheme `json:"destination_scheme" validate:"oneof=environment_variable provisioner_variable,required"`
+	DestinationValue  string                              `json:"destination_value" validate:"required"`
 }
 
 // ParameterValue represents a set value for the scope.

diff --git a/coderd/projecthistory.go b/coderd/projecthistory.go
@@ -31,6 +31,27 @@ type ProjectHistory struct {
 	Import        ProvisionerJob                `json:"import"`
 }
 
+// ProjectParameter represents a parameter parsed from project history source on creation.
+type ProjectParameter struct {
+	ID                       uuid.UUID                           `json:"id"`
+	CreatedAt                time.Time                           `json:"created_at"`
+	ProjectHistoryID         uuid.UUID                           `json:"project_history_id"`
+	Name                     string                              `json:"name"`
+	Description              string                              `json:"description,omitempty"`
+	DefaultSourceScheme      database.ParameterSourceScheme      `json:"default_source_scheme,omitempty"`
+	DefaultSourceValue       string                              `json:"default_source_value,omitempty"`
+	AllowOverrideSource      bool                                `json:"allow_override_source"`
+	DefaultDestinationScheme database.ParameterDestinationScheme `json:"default_destination_scheme,omitempty"`
+	DefaultDestinationValue  string                              `json:"default_destination_value,omitempty"`
+	AllowOverrideDestination bool                                `json:"allow_override_destination"`
+	DefaultRefresh           string                              `json:"default_refresh"`
+	RedisplayValue           bool                                `json:"redisplay_value"`
+	ValidationError          string                              `json:"validation_error,omitempty"`
+	ValidationCondition      string                              `json:"validation_condition,omitempty"`
+	ValidationTypeSystem     database.ParameterTypeSystem        `json:"validation_type_system,omitempty"`
+	ValidationValueType      string                              `json:"validation_value_type,omitempty"`
+}
+
 // CreateProjectHistoryRequest enables callers to create a new Project Version.
 type CreateProjectHistoryRequest struct {
 	StorageMethod database.ProjectStorageMethod `json:"storage_method" validate:"oneof=inline-archive,required"`
@@ -160,14 +181,81 @@ func (api *api) postProjectHistoryByOrganization(rw http.ResponseWriter, r *http
 	render.JSON(rw, r, convertProjectHistory(projectHistory, provisionerJob))
 }
 
+func (api *api) projectHistoryParametersByOrganizationAndName(rw http.ResponseWriter, r *http.Request) {
+	projectHistory := httpmw.ProjectHistoryParam(r)
+	job, err := api.Database.GetProvisionerJobByID(r.Context(), projectHistory.ImportJobID)
+	if err != nil {
+		httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
+			Message: fmt.Sprintf("get provisioner job: %s", err),
+		})
+		return
+	}
+	apiJob := convertProvisionerJob(job)
+	if !apiJob.Status.Completed() {
+		httpapi.Write(rw, http.StatusPreconditionRequired, httpapi.Response{
+			Message: fmt.Sprintf("import job hasn't completed: %s", apiJob.Status),
+		})
+		return
+	}
+	if apiJob.Status != ProvisionerJobStatusSucceeded {
+		httpapi.Write(rw, http.StatusPreconditionFailed, httpapi.Response{
+			Message: "import job wasn't successful. no parameters were parsed",
+		})
+		return
+	}
+
+	parameters, err := api.Database.GetProjectParametersByHistoryID(r.Context(), projectHistory.ID)
+	if errors.Is(err, sql.ErrNoRows) {
+		err = nil
+		parameters = []database.ProjectParameter{}
+	}
+	if err != nil {
+		httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
+			Message: fmt.Sprintf("get project parameters: %s", err),
+		})
+		return
+	}
+
+	apiParameters := make([]ProjectParameter, 0, len(parameters))
+	for _, parameter := range parameters {
+		apiParameters = append(apiParameters, convertProjectParameter(parameter))
+	}
+
+	render.Status(r, http.StatusOK)
+	render.JSON(rw, r, apiParameters)
+}
+
 func convertProjectHistory(history database.ProjectHistory, job database.ProvisionerJob) ProjectHistory {
 	return ProjectHistory{
-		ID:        history.ID,
-		ProjectID: history.ProjectID,
-		CreatedAt: history.CreatedAt,
-		UpdatedAt: history.UpdatedAt,
-		Name:      history.Name,
-		Import:    convertProvisionerJob(job),
+		ID:            history.ID,
+		ProjectID:     history.ProjectID,
+		CreatedAt:     history.CreatedAt,
+		UpdatedAt:     history.UpdatedAt,
+		Name:          history.Name,
+		StorageMethod: history.StorageMethod,
+		Import:        convertProvisionerJob(job),
+	}
+}
+
+func convertProjectParameter(parameter database.ProjectParameter) ProjectParameter {
+	return ProjectParameter{
+		ID:                       parameter.ID,
+		CreatedAt:                parameter.CreatedAt,
+		ProjectHistoryID:         parameter.ProjectHistoryID,
+		Name:                     parameter.Name,
+		Description:              parameter.Description,
+		DefaultSourceScheme:      parameter.DefaultSourceScheme,
+		DefaultSourceValue:       parameter.DefaultSourceValue.String,
+		AllowOverrideSource:      parameter.AllowOverrideSource,
+		DefaultDestinationScheme: parameter.DefaultDestinationScheme,
+		DefaultDestinationValue:  parameter.DefaultDestinationValue.String,
+		AllowOverrideDestination: parameter.AllowOverrideDestination,
+		DefaultRefresh:           parameter.DefaultRefresh,
+		RedisplayValue:           parameter.RedisplayValue,
+		ValidationError:          parameter.ValidationError,
+		ValidationCondition:      parameter.ValidationCondition,
+		ValidationTypeSystem:     parameter.ValidationTypeSystem,
+		ValidationValueType:      parameter.ValidationValueType,
 	}
 }
 

diff --git a/coderd/projects_test.go b/coderd/projects_test.go
@@ -1,8 +1,11 @@
 package coderd_test
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
@@ -124,4 +127,42 @@ func TestProjects(t *testing.T) {
 		})
 		require.NoError(t, err)
 	})
+
+	t.Run("Import", func(t *testing.T) {
+		t.Parallel()
+		server := coderdtest.New(t)
+		user := server.RandomInitialUser(t)
+		_ = server.AddProvisionerd(t)
+		project, err := server.Client.CreateProject(context.Background(), user.Organization, coderd.CreateProjectRequest{
+			Name:        "someproject",
+			Provisioner: database.ProvisionerTypeTerraform,
+		})
+		require.NoError(t, err)
+		var buffer bytes.Buffer
+		writer := tar.NewWriter(&buffer)
+		content := `variable "example" {
+	default = "hi"
+}`
+		err = writer.WriteHeader(&tar.Header{
+			Name: "main.tf",
+			Size: int64(len(content)),
+		})
+		require.NoError(t, err)
+		_, err = writer.Write([]byte(content))
+		require.NoError(t, err)
+		history, err := server.Client.CreateProjectHistory(context.Background(), user.Organization, project.Name, coderd.CreateProjectHistoryRequest{
+			StorageMethod: database.ProjectStorageMethodInlineArchive,
+			StorageSource: buffer.Bytes(),
+		})
+		require.NoError(t, err)
+		require.Eventually(t, func() bool {
+			projectHistory, err := server.Client.ProjectHistory(context.Background(), user.Organization, project.Name, history.Name)
+			require.NoError(t, err)
+			return projectHistory.Import.Status.Completed()
+		}, 15*time.Second, 10*time.Millisecond)
+		params, err := server.Client.ProjectHistoryParameters(context.Background(), user.Organization, project.Name, history.Name)
+		require.NoError(t, err)
+		require.Len(t, params, 1)
+		require.Equal(t, "example", params[0].Name)
+	})
 }
diff --git a/coderd/provisionerdaemons.go b/coderd/provisionerdaemons.go
@@ -451,6 +451,9 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr
 				ValidationValueType:  protoParameter.ValidationValueType,
 				ValidationTypeSystem: validationTypeSystem,
 
+				DefaultSourceScheme:      database.ParameterSourceSchemeNone,
+				DefaultDestinationScheme: database.ParameterDestinationSchemeNone,
+
 				AllowOverrideDestination: protoParameter.AllowOverrideDestination,
 				AllowOverrideSource:      protoParameter.AllowOverrideSource,
 			}
@@ -574,6 +577,8 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr
 
 func convertValidationTypeSystem(typeSystem sdkproto.ParameterSchema_TypeSystem) (database.ParameterTypeSystem, error) {
 	switch typeSystem {
+	case sdkproto.ParameterSchema_None:
+		return database.ParameterTypeSystemNone, nil
 	case sdkproto.ParameterSchema_HCL:
 		return database.ParameterTypeSystemHCL, nil
 	default:

diff --git a/coderd/workspacehistory.go b/coderd/workspacehistory.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/go-chi/render"
 	"github.com/google/uuid"
+	"github.com/moby/moby/pkg/namesgenerator"
 	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/database"
@@ -155,6 +156,7 @@ func (api *api) postWorkspaceHistoryByUser(rw http.ResponseWriter, r *http.Reque
 			WorkspaceID:      workspace.ID,
 			ProjectHistoryID: projectHistory.ID,
 			BeforeID:         priorHistoryID,
+			Name:             namesgenerator.GetRandomName(1),
 			Initiator:        user.ID,
 			Transition:       createBuild.Transition,
 			ProvisionJobID:   provisionerJob.ID,
@@ -253,7 +255,3 @@ func convertWorkspaceHistory(workspaceHistory database.WorkspaceHistory, provisi
 		Provision:        convertProvisionerJob(provisionerJob),
 	})
 }
-
-func workspaceHistoryLogsChannel(workspaceHistoryID uuid.UUID) string {
-	return fmt.Sprintf("workspace-history-logs:%s", workspaceHistoryID)
-}