Skip to content

Commit

Permalink
refactor: Allow provisioner jobs to be disconnected from projects (#194)
Browse files Browse the repository at this point in the history
* Nest jobs under an organization

* Rename project parameter to parameter schema

* Update references when computing project parameters

* Add files endpoint

* Allow one-off project import jobs

* Allow variables to be injected that are not defined by the schema

* Update API to use jobs first

* Fix CLI tests

* Fix linting

* Fix hex length for files table

* Reduce memory allocation for windows
  • Loading branch information
kylecarbs committed Feb 8, 2022
1 parent 4c5e443 commit 7364933
Show file tree
Hide file tree
Showing 37 changed files with 1,373 additions and 988 deletions.
19 changes: 16 additions & 3 deletions coderd/coderd.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,14 +102,27 @@ func New(options *Options) http.Handler {
})
})

r.Route("/files", func(r chi.Router) {
r.Use(httpmw.ExtractAPIKey(options.Database, nil))
r.Post("/", api.postFiles)
})

r.Route("/provisioners", func(r chi.Router) {
r.Route("/daemons", func(r chi.Router) {
r.Get("/", api.provisionerDaemons)
r.Get("/serve", api.provisionerDaemonsServe)
})
r.Route("/jobs/{provisionerjob}", func(r chi.Router) {
r.Use(httpmw.ExtractProvisionerJobParam(options.Database))
r.Get("/logs", api.provisionerJobLogsByID)
r.Route("/jobs/{organization}", func(r chi.Router) {
r.Use(
httpmw.ExtractAPIKey(options.Database, nil),
httpmw.ExtractOrganizationParam(options.Database),
)
r.Post("/import", api.postProvisionerImportJobByOrganization)
r.Route("/{provisionerjob}", func(r chi.Router) {
r.Use(httpmw.ExtractProvisionerJobParam(options.Database))
r.Get("/", api.provisionerJobByOrganization)
r.Get("/logs", api.provisionerJobLogsByID)
})
})
})
})
Expand Down
60 changes: 26 additions & 34 deletions coderd/coderdtest/coderdtest.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,40 +122,44 @@ func CreateInitialUser(t *testing.T, client *codersdk.Client) coderd.CreateIniti
return req
}

// CreateProject creates a project with the "echo" provisioner for
// compatibility with testing. The name assigned is randomly generated.
func CreateProject(t *testing.T, client *codersdk.Client, organization string) coderd.Project {
project, err := client.CreateProject(context.Background(), organization, coderd.CreateProjectRequest{
Name: randomUsername(),
Provisioner: database.ProvisionerTypeEcho,
// CreateProjectImportProvisionerJob creates a project import provisioner job
// with the responses provided. It uses the "echo" provisioner for compatibility
// with testing.
func CreateProjectImportProvisionerJob(t *testing.T, client *codersdk.Client, organization string, res *echo.Responses) coderd.ProvisionerJob {
data, err := echo.Tar(res)
require.NoError(t, err)
file, err := client.UploadFile(context.Background(), codersdk.ContentTypeTar, data)
require.NoError(t, err)
job, err := client.CreateProjectVersionImportProvisionerJob(context.Background(), organization, coderd.CreateProjectImportJobRequest{
StorageSource: file.Hash,
StorageMethod: database.ProvisionerStorageMethodFile,
Provisioner: database.ProvisionerTypeEcho,
})
require.NoError(t, err)
return project
return job
}

// CreateProjectVersion creates a project version for the "echo" provisioner
// for compatibility with testing.
func CreateProjectVersion(t *testing.T, client *codersdk.Client, organization, project string, responses *echo.Responses) coderd.ProjectVersion {
data, err := echo.Tar(responses)
require.NoError(t, err)
version, err := client.CreateProjectVersion(context.Background(), organization, project, coderd.CreateProjectVersionRequest{
StorageMethod: database.ProjectStorageMethodInlineArchive,
StorageSource: data,
// CreateProject creates a project with the "echo" provisioner for
// compatibility with testing. The name assigned is randomly generated.
func CreateProject(t *testing.T, client *codersdk.Client, organization string, job uuid.UUID) coderd.Project {
project, err := client.CreateProject(context.Background(), organization, coderd.CreateProjectRequest{
Name: randomUsername(),
VersionImportJobID: job,
})
require.NoError(t, err)
return version
return project
}

// AwaitProjectVersionImported awaits for the project import job to reach completed status.
func AwaitProjectVersionImported(t *testing.T, client *codersdk.Client, organization, project, version string) coderd.ProjectVersion {
var projectVersion coderd.ProjectVersion
// AwaitProvisionerJob awaits for a job to reach completed status.
func AwaitProvisionerJob(t *testing.T, client *codersdk.Client, organization string, job uuid.UUID) coderd.ProvisionerJob {
var provisionerJob coderd.ProvisionerJob
require.Eventually(t, func() bool {
var err error
projectVersion, err = client.ProjectVersion(context.Background(), organization, project, version)
provisionerJob, err = client.ProvisionerJob(context.Background(), organization, job)
require.NoError(t, err)
return projectVersion.Import.Status.Completed()
return provisionerJob.Status.Completed()
}, 3*time.Second, 25*time.Millisecond)
return projectVersion
return provisionerJob
}

// CreateWorkspace creates a workspace for the user and project provided.
Expand All @@ -169,18 +173,6 @@ func CreateWorkspace(t *testing.T, client *codersdk.Client, user string, project
return workspace
}

// AwaitWorkspaceHistoryProvisioned awaits for the workspace provision job to reach completed status.
func AwaitWorkspaceHistoryProvisioned(t *testing.T, client *codersdk.Client, user, workspace, history string) coderd.WorkspaceHistory {
var workspaceHistory coderd.WorkspaceHistory
require.Eventually(t, func() bool {
var err error
workspaceHistory, err = client.WorkspaceHistory(context.Background(), user, workspace, history)
require.NoError(t, err)
return workspaceHistory.Provision.Status.Completed()
}, 3*time.Second, 25*time.Millisecond)
return workspaceHistory
}

func randomUsername() string {
return strings.ReplaceAll(namesgenerator.GetRandomName(0), "_", "-")
}
10 changes: 5 additions & 5 deletions coderd/coderdtest/coderdtest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,15 @@ func TestNew(t *testing.T) {
client := coderdtest.New(t)
user := coderdtest.CreateInitialUser(t, client)
closer := coderdtest.NewProvisionerDaemon(t, client)
project := coderdtest.CreateProject(t, client, user.Organization)
version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil)
coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
coderdtest.AwaitProvisionerJob(t, client, user.Organization, job.ID)
project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
workspace := coderdtest.CreateWorkspace(t, client, "me", project.ID)
history, err := client.CreateWorkspaceHistory(context.Background(), "me", workspace.Name, coderd.CreateWorkspaceHistoryRequest{
ProjectVersionID: version.ID,
ProjectVersionID: project.ActiveVersionID,
Transition: database.WorkspaceTransitionStart,
})
require.NoError(t, err)
coderdtest.AwaitWorkspaceHistoryProvisioned(t, client, "me", workspace.Name, history.Name)
coderdtest.AwaitProvisionerJob(t, client, user.Organization, history.ProvisionJobID)
closer.Close()
}
60 changes: 60 additions & 0 deletions coderd/files.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package coderd

import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"

"github.com/go-chi/render"

"github.com/coder/coder/database"
"github.com/coder/coder/httpapi"
"github.com/coder/coder/httpmw"
)

type UploadFileResponse struct {
Hash string `json:"hash"`
}

func (api *api) postFiles(rw http.ResponseWriter, r *http.Request) {
apiKey := httpmw.APIKey(r)
contentType := r.Header.Get("Content-Type")

switch contentType {
case "application/x-tar":
default:
httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{
Message: fmt.Sprintf("unsupported content type: %s", contentType),
})
return
}

r.Body = http.MaxBytesReader(rw, r.Body, 10*(10<<20))
data, err := io.ReadAll(r.Body)
if err != nil {
httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{
Message: fmt.Sprintf("read file: %s", err),
})
return
}
hashBytes := sha256.Sum256(data)
file, err := api.Database.InsertFile(r.Context(), database.InsertFileParams{
Hash: hex.EncodeToString(hashBytes[:]),
CreatedBy: apiKey.UserID,
CreatedAt: database.Now(),
Mimetype: contentType,
Data: data,
})
if err != nil {
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
Message: fmt.Sprintf("insert file: %s", err),
})
return
}
render.Status(r, http.StatusCreated)
render.JSON(rw, r, UploadFileResponse{
Hash: file.Hash,
})
}
30 changes: 30 additions & 0 deletions coderd/files_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package coderd_test

import (
"context"
"testing"

"github.com/stretchr/testify/require"

"github.com/coder/coder/coderd/coderdtest"
"github.com/coder/coder/codersdk"
)

func TestPostFiles(t *testing.T) {
t.Parallel()
t.Run("BadContentType", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t)
_ = coderdtest.CreateInitialUser(t, client)
_, err := client.UploadFile(context.Background(), "bad", []byte{'a'})
require.Error(t, err)
})

t.Run("Insert", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t)
_ = coderdtest.CreateInitialUser(t, client)
_, err := client.UploadFile(context.Background(), codersdk.ContentTypeTar, make([]byte, 1024))
require.NoError(t, err)
})
}

0 comments on commit 7364933

Please sign in to comment.