feat: Refactor API routes to use UUIDs instead of friendly names (#401)

* Add client for agent

* Cleanup code

* Fix linting error

* Rename routes to be simpler

* Rename workspace history to workspace build

* Refactor HTTP middlewares to use UUIDs

* Cleanup routes

* Compiles!

* Fix files and organizations

* Fix querying

* Fix agent lock

* Cleanup database abstraction

* Add parameters

* Fix linting errors

* Fix log race

* Lock on close wait

* Fix log cleanup

* Fix e2e tests

* Fix upstream version of opencensus-go

* Update coderdtest.go

* Fix coverpkg

* Fix codecov ignore

.github/workflows/coder.yaml

@@ -157,6 +157,7 @@ jobs:
           GOMAXPROCS: ${{ runner.os == 'Windows' && 1 || 2 }}
         run: gotestsum --junitfile="gotests.xml" --packages="./..." --
           -covermode=atomic -coverprofile="gotests.coverage"
+          -coverpkg=./...,github.com/coder/coder/codersdk
           -timeout=3m -count=$GOCOUNT -race -short -failfast
 
       - name: Upload DataDog Trace
@@ -172,6 +173,7 @@ jobs:
         if: runner.os == 'Linux'
         run: DB=true gotestsum --junitfile="gotests.xml" --packages="./..." --
           -covermode=atomic -coverprofile="gotests.coverage" -timeout=3m
+          -coverpkg=./...,github.com/coder/coder/codersdk
           -count=1 -race -parallel=2 -failfast
 
       - name: Upload DataDog Trace

.vscode/settings.json

@@ -6,6 +6,9 @@
   "go.lintFlags": ["--fast"],
   "go.lintOnSave": "package",
   "go.coverOnSave": true,
+  // The codersdk is used by coderd another other packages extensively.
+  // To reduce redundancy in tests, it's covered by other packages.
+  "go.testFlags": ["-coverpkg=./.,github.com/coder/coder/codersdk"],
   "go.coverageDecorator": {
     "type": "gutter",
     "coveredHighlightColor": "rgba(64,128,128,0.5)",

agent/agent.go

@@ -59,9 +59,9 @@ type Options struct {
 	Logger slog.Logger
 }
 
-type Dialer func(ctx context.Context) (*peerbroker.Listener, error)
+type Dialer func(ctx context.Context, options *peer.ConnOptions) (*peerbroker.Listener, error)
 
-func New(dialer Dialer, options *Options) io.Closer {
+func New(dialer Dialer, options *peer.ConnOptions) io.Closer {
 	ctx, cancelFunc := context.WithCancel(context.Background())
 	server := &server{
 		clientDialer: dialer,
@@ -75,11 +75,12 @@ func New(dialer Dialer, options *Options) io.Closer {
 
 type server struct {
 	clientDialer Dialer
-	options      *Options
+	options      *peer.ConnOptions
 
-	closeCancel context.CancelFunc
-	closeMutex  sync.Mutex
-	closed      chan struct{}
+	connCloseWait sync.WaitGroup
+	closeCancel   context.CancelFunc
+	closeMutex    sync.Mutex
+	closed        chan struct{}
 
 	sshServer *ssh.Server
 }
@@ -249,7 +250,7 @@ func (s *server) run(ctx context.Context) {
 	// An exponential back-off occurs when the connection is failing to dial.
 	// This is to prevent server spam in case of a coderd outage.
 	for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); {
-		peerListener, err = s.clientDialer(ctx)
+		peerListener, err = s.clientDialer(ctx, s.options)
 		if err != nil {
 			if errors.Is(err, context.Canceled) {
 				return
@@ -279,11 +280,18 @@ func (s *server) run(ctx context.Context) {
 			s.run(ctx)
 			return
 		}
+		s.closeMutex.Lock()
+		s.connCloseWait.Add(1)
+		s.closeMutex.Unlock()
 		go s.handlePeerConn(ctx, conn)
 	}
 }
 
 func (s *server) handlePeerConn(ctx context.Context, conn *peer.Conn) {
+	go func() {
+		<-conn.Closed()
+		s.connCloseWait.Done()
+	}()
 	for {
 		channel, err := conn.Accept(ctx)
 		if err != nil {
@@ -325,5 +333,6 @@ func (s *server) Close() error {
 	close(s.closed)
 	s.closeCancel()
 	_ = s.sshServer.Close()
+	s.connCloseWait.Wait()
 	return nil
 }

agent/agent_test.go

@@ -94,11 +94,9 @@ func TestAgent(t *testing.T) {
 
 func setup(t *testing.T) proto.DRPCPeerBrokerClient {
 	client, server := provisionersdk.TransportPipe()
-	closer := agent.New(func(ctx context.Context) (*peerbroker.Listener, error) {
-		return peerbroker.Listen(server, nil, &peer.ConnOptions{
-			Logger: slogtest.Make(t, nil),
-		})
-	}, &agent.Options{
+	closer := agent.New(func(ctx context.Context, opts *peer.ConnOptions) (*peerbroker.Listener, error) {
+		return peerbroker.Listen(server, nil, opts)
+	}, &peer.ConnOptions{
 		Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug),
 	})
 	t.Cleanup(func() {

cli/login.go

@@ -59,7 +59,7 @@ func login() *cobra.Command {
 			}
 
 			client := codersdk.New(serverURL)
-			hasInitialUser, err := client.HasInitialUser(cmd.Context())
+			hasInitialUser, err := client.HasFirstUser(cmd.Context())
 			if err != nil {
 				return xerrors.Errorf("has initial user: %w", err)
 			}
@@ -119,7 +119,7 @@ func login() *cobra.Command {
 					return xerrors.Errorf("specify password prompt: %w", err)
 				}
 
-				_, err = client.CreateInitialUser(cmd.Context(), coderd.CreateInitialUserRequest{
+				_, err = client.CreateFirstUser(cmd.Context(), coderd.CreateFirstUserRequest{
 					Email:        email,
 					Username:     username,
 					Password:     password,

cli/login_test.go

@@ -1,13 +1,11 @@
 package cli_test
 
 import (
-	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/coder/coder/cli/clitest"
-	"github.com/coder/coder/coderd"
 	"github.com/coder/coder/coderd/coderdtest"
 	"github.com/coder/coder/pty/ptytest"
 )
@@ -56,18 +54,7 @@ func TestLogin(t *testing.T) {
 	t.Run("ExistingUserValidTokenTTY", func(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t, nil)
-		_, err := client.CreateInitialUser(context.Background(), coderd.CreateInitialUserRequest{
-			Username:     "test-user",
-			Email:        "test-user@coder.com",
-			Organization: "acme-corp",
-			Password:     "password",
-		})
-		require.NoError(t, err)
-		token, err := client.LoginWithPassword(context.Background(), coderd.LoginWithPasswordRequest{
-			Email:    "test-user@coder.com",
-			Password: "password",
-		})
-		require.NoError(t, err)
+		coderdtest.CreateFirstUser(t, client)
 
 		root, _ := clitest.New(t, "login", client.URL.String(), "--force-tty", "--no-open")
 		pty := ptytest.New(t)
@@ -79,20 +66,14 @@ func TestLogin(t *testing.T) {
 		}()
 
 		pty.ExpectMatch("Paste your token here:")
-		pty.WriteLine(token.SessionToken)
+		pty.WriteLine(client.SessionToken)
 		pty.ExpectMatch("Welcome to Coder")
 	})
 
 	t.Run("ExistingUserInvalidTokenTTY", func(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t, nil)
-		_, err := client.CreateInitialUser(context.Background(), coderd.CreateInitialUserRequest{
-			Username:     "test-user",
-			Email:        "test-user@coder.com",
-			Organization: "acme-corp",
-			Password:     "password",
-		})
-		require.NoError(t, err)
+		coderdtest.CreateFirstUser(t, client)
 
 		root, _ := clitest.New(t, "login", client.URL.String(), "--force-tty", "--no-open")
 		pty := ptytest.New(t)

cli/projectcreate.go

@@ -56,7 +56,7 @@ func projectCreate() *cobra.Command {
 				Default: filepath.Base(directory),
 				Label:   "What's your project's name?",
 				Validate: func(s string) error {
-					project, _ := client.Project(cmd.Context(), organization.Name, s)
+					project, _ := client.ProjectByName(cmd.Context(), organization.ID, s)
 					if project.ID.String() != uuid.Nil.String() {
 						return xerrors.New("A project already exists with that name!")
 					}
@@ -71,9 +71,9 @@ func projectCreate() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			project, err := client.CreateProject(cmd.Context(), organization.Name, coderd.CreateProjectRequest{
-				Name:               name,
-				VersionImportJobID: job.ID,
+			project, err := client.CreateProject(cmd.Context(), organization.ID, coderd.CreateProjectRequest{
+				Name:      name,
+				VersionID: job.ID,
 			})
 			if err != nil {
 				return err
@@ -118,7 +118,7 @@ func projectCreate() *cobra.Command {
 	return cmd
 }
 
-func validateProjectVersionSource(cmd *cobra.Command, client *codersdk.Client, organization coderd.Organization, provisioner database.ProvisionerType, directory string, parameters ...coderd.CreateParameterValueRequest) (*coderd.ProvisionerJob, error) {
+func validateProjectVersionSource(cmd *cobra.Command, client *codersdk.Client, organization coderd.Organization, provisioner database.ProvisionerType, directory string, parameters ...coderd.CreateParameterRequest) (*coderd.ProjectVersion, error) {
 	spin := spinner.New(spinner.CharSets[5], 100*time.Millisecond)
 	spin.Writer = cmd.OutOrStdout()
 	spin.Suffix = " Uploading current directory..."
@@ -133,13 +133,13 @@ func validateProjectVersionSource(cmd *cobra.Command, client *codersdk.Client, o
 	if err != nil {
 		return nil, err
 	}
-	resp, err := client.UploadFile(cmd.Context(), codersdk.ContentTypeTar, tarData)
+	resp, err := client.Upload(cmd.Context(), codersdk.ContentTypeTar, tarData)
 	if err != nil {
 		return nil, err
 	}
 
 	before := time.Now()
-	job, err := client.CreateProjectImportJob(cmd.Context(), organization.Name, coderd.CreateProjectImportJobRequest{
+	version, err := client.CreateProjectVersion(cmd.Context(), organization.ID, coderd.CreateProjectVersionRequest{
 		StorageMethod:   database.ProvisionerStorageMethodFile,
 		StorageSource:   resp.Hash,
 		Provisioner:     provisioner,
@@ -149,7 +149,7 @@ func validateProjectVersionSource(cmd *cobra.Command, client *codersdk.Client, o
 		return nil, err
 	}
 	spin.Suffix = " Waiting for the import to complete..."
-	logs, err := client.ProjectImportJobLogsAfter(cmd.Context(), organization.Name, job.ID, before)
+	logs, err := client.ProjectVersionLogsAfter(cmd.Context(), version.ID, before)
 	if err != nil {
 		return nil, err
 	}
@@ -162,22 +162,22 @@ func validateProjectVersionSource(cmd *cobra.Command, client *codersdk.Client, o
 		logBuffer = append(logBuffer, log)
 	}
 
-	job, err = client.ProjectImportJob(cmd.Context(), organization.Name, job.ID)
+	version, err = client.ProjectVersion(cmd.Context(), version.ID)
 	if err != nil {
 		return nil, err
 	}
-	parameterSchemas, err := client.ProjectImportJobSchemas(cmd.Context(), organization.Name, job.ID)
+	parameterSchemas, err := client.ProjectVersionSchema(cmd.Context(), version.ID)
 	if err != nil {
 		return nil, err
 	}
-	parameterValues, err := client.ProjectImportJobParameters(cmd.Context(), organization.Name, job.ID)
+	parameterValues, err := client.ProjectVersionParameters(cmd.Context(), version.ID)
 	if err != nil {
 		return nil, err
 	}
 	spin.Stop()
 
-	if provisionerd.IsMissingParameterError(job.Error) {
-		valuesBySchemaID := map[string]coderd.ComputedParameterValue{}
+	if provisionerd.IsMissingParameterError(version.Job.Error) {
+		valuesBySchemaID := map[string]coderd.ProjectVersionParameter{}
 		for _, parameterValue := range parameterValues {
 			valuesBySchemaID[parameterValue.SchemaID.String()] = parameterValue
 		}
@@ -192,7 +192,7 @@ func validateProjectVersionSource(cmd *cobra.Command, client *codersdk.Client, o
 			if err != nil {
 				return nil, err
 			}
-			parameters = append(parameters, coderd.CreateParameterValueRequest{
+			parameters = append(parameters, coderd.CreateParameterRequest{
 				Name:              parameterSchema.Name,
 				SourceValue:       value,
 				SourceScheme:      database.ParameterSourceSchemeData,
@@ -202,21 +202,20 @@ func validateProjectVersionSource(cmd *cobra.Command, client *codersdk.Client, o
 		return validateProjectVersionSource(cmd, client, organization, provisioner, directory, parameters...)
 	}
 
-	if job.Status != coderd.ProvisionerJobStatusSucceeded {
+	if version.Job.Status != coderd.ProvisionerJobSucceeded {
 		for _, log := range logBuffer {
 			_, _ = fmt.Fprintf(cmd.OutOrStdout(), "%s %s\n", color.HiGreenString("[tf]"), log.Output)
 		}
-
-		return nil, xerrors.New(job.Error)
+		return nil, xerrors.New(version.Job.Error)
 	}
 
 	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "%s Successfully imported project source!\n", color.HiGreenString("✓"))
 
-	resources, err := client.ProjectImportJobResources(cmd.Context(), organization.Name, job.ID)
+	resources, err := client.ProjectVersionResources(cmd.Context(), version.ID)
 	if err != nil {
 		return nil, err
 	}
-	return &job, displayProjectImportInfo(cmd, parameterSchemas, parameterValues, resources)
+	return &version, displayProjectImportInfo(cmd, parameterSchemas, parameterValues, resources)
 }
 
 func tarDirectory(directory string) ([]byte, error) {

cli/projectcreate_test.go

@@ -18,7 +18,7 @@ func TestProjectCreate(t *testing.T) {
 	t.Run("NoParameters", func(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t, nil)
-		coderdtest.CreateInitialUser(t, client)
+		coderdtest.CreateFirstUser(t, client)
 		source := clitest.CreateProjectVersionSource(t, &echo.Responses{
 			Parse:     echo.ParseComplete,
 			Provision: echo.ProvisionComplete,
@@ -54,7 +54,7 @@ func TestProjectCreate(t *testing.T) {
 	t.Run("Parameter", func(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t, nil)
-		coderdtest.CreateInitialUser(t, client)
+		coderdtest.CreateFirstUser(t, client)
 		source := clitest.CreateProjectVersionSource(t, &echo.Responses{
 			Parse: []*proto.Parse_Response{{
 				Type: &proto.Parse_Response_Complete{

cli/projectlist.go

@@ -23,7 +23,7 @@ func projectList() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			projects, err := client.Projects(cmd.Context(), organization.Name)
+			projects, err := client.ProjectsByOrganization(cmd.Context(), organization.ID)
 			if err != nil {
 				return err
 			}

cli/projectlist_test.go

@@ -15,7 +15,7 @@ func TestProjectList(t *testing.T) {
 	t.Run("None", func(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t, nil)
-		coderdtest.CreateInitialUser(t, client)
+		coderdtest.CreateFirstUser(t, client)
 		cmd, root := clitest.New(t, "projects", "list")
 		clitest.SetupConfig(t, client, root)
 		pty := ptytest.New(t)
@@ -33,12 +33,12 @@ func TestProjectList(t *testing.T) {
 	t.Run("List", func(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t, nil)
-		user := coderdtest.CreateInitialUser(t, client)
+		user := coderdtest.CreateFirstUser(t, client)
 		daemon := coderdtest.NewProvisionerDaemon(t, client)
-		job := coderdtest.CreateProjectImportJob(t, client, user.Organization, nil)
-		coderdtest.AwaitProjectImportJob(t, client, user.Organization, job.ID)
+		version := coderdtest.CreateProjectVersion(t, client, user.OrganizationID, nil)
+		coderdtest.AwaitProjectVersionJob(t, client, version.ID)
 		_ = daemon.Close()
-		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
+		project := coderdtest.CreateProject(t, client, user.OrganizationID, version.ID)
 		cmd, root := clitest.New(t, "projects", "list")
 		clitest.SetupConfig(t, client, root)
 		pty := ptytest.New(t)

cli/projects.go

@@ -40,8 +40,8 @@ func projects() *cobra.Command {
 	return cmd
 }
 
-func displayProjectImportInfo(cmd *cobra.Command, parameterSchemas []coderd.ParameterSchema, parameterValues []coderd.ComputedParameterValue, resources []coderd.ProvisionerJobResource) error {
-	schemaByID := map[string]coderd.ParameterSchema{}
+func displayProjectImportInfo(cmd *cobra.Command, parameterSchemas []coderd.ProjectVersionParameterSchema, parameterValues []coderd.ProjectVersionParameter, resources []coderd.WorkspaceResource) error {
+	schemaByID := map[string]coderd.ProjectVersionParameterSchema{}
 	for _, schema := range parameterSchemas {
 		schemaByID[schema.ID.String()] = schema
 	}

cli/root.go

@@ -108,7 +108,7 @@ func createClient(cmd *cobra.Command) (*codersdk.Client, error) {
 
 // currentOrganization returns the currently active organization for the authenticated user.
 func currentOrganization(cmd *cobra.Command, client *codersdk.Client) (coderd.Organization, error) {
-	orgs, err := client.UserOrganizations(cmd.Context(), "me")
+	orgs, err := client.OrganizationsByUser(cmd.Context(), "me")
 	if err != nil {
 		return coderd.Organization{}, nil
 	}