feat!: agent pools

Makefile

@@ -3,7 +3,7 @@ GIT_COMMIT = $(shell git rev-parse HEAD)
 RANDOM_SUFFIX := $(shell cat /dev/urandom | tr -dc 'a-z0-9' | head -c5)
 IMAGE_NAME = leg100/otfd
 IMAGE_TAG ?= $(VERSION)-$(RANDOM_SUFFIX)
-GOOSE_DBSTRING=postgres:///otf
+DBSTRING=postgres://postgres:password@localhost/otf
 LD_FLAGS = " \
     -s -w \
 	-X 'github.com/leg100/otf/internal.Version=$(VERSION)' \
@@ -64,7 +64,7 @@ install-latest-release:
 # Run docker compose stack
 .PHONY: compose-up
 compose-up: image
-	docker compose up -d
+	docker compose up -d --wait --wait-timeout 60
 
 # Remove docker compose stack
 .PHONY: compose-rm
@@ -116,13 +116,13 @@ install-pggen:
 .PHONY: sql
 sql: install-pggen
 	pggen gen go \
-		--postgres-connection "dbname=otf" \
+		--postgres-connection $(DBSTRING) \
 		--query-glob 'internal/sql/queries/*.sql' \
 		--output-dir ./internal/sql/pggen \
 		--go-type 'text=github.com/jackc/pgtype.Text' \
 		--go-type 'int4=github.com/jackc/pgtype.Int4' \
 		--go-type 'int8=github.com/jackc/pgtype.Int8' \
-		--go-type 'bool=bool' \
+		--go-type 'bool=github.com/jackc/pgtype.Bool' \
 		--go-type 'bytea=[]byte' \
 		--acronym url \
 		--acronym cli \
@@ -133,7 +133,8 @@ sql: install-pggen
 		--acronym http \
 		--acronym tls \
 		--acronym sso \
-		--acronym hcl
+		--acronym hcl \
+		--acronym ip
 	goimports -w ./internal/sql/pggen
 	go fmt ./internal/sql/pggen
 
@@ -145,22 +146,22 @@ install-goose:
 # Migrate SQL schema to latest version
 .PHONY: migrate
 migrate: install-goose
-	GOOSE_DBSTRING=$(GOOSE_DBSTRING) GOOSE_DRIVER=postgres goose -dir ./internal/sql/migrations up
+	GOOSE_DBSTRING=$(DBSTRING) GOOSE_DRIVER=postgres goose -dir ./internal/sql/migrations up
 
 # Redo SQL schema migration
 .PHONY: migrate-redo
 migrate-redo: install-goose
-	GOOSE_DBSTRING=$(GOOSE_DBSTRING) GOOSE_DRIVER=postgres goose -dir ./internal/sql/migrations redo
+	GOOSE_DBSTRING=$(DBSTRING) GOOSE_DRIVER=postgres goose -dir ./internal/sql/migrations redo
 
 # Rollback SQL schema by one version
 .PHONY: migrate-rollback
 migrate-rollback: install-goose
-	GOOSE_DBSTRING=$(GOOSE_DBSTRING) GOOSE_DRIVER=postgres goose -dir ./internal/sql/migrations down
+	GOOSE_DBSTRING=$(DBSTRING) GOOSE_DRIVER=postgres goose -dir ./internal/sql/migrations down
 
 # Get SQL schema migration status
 .PHONY: migrate-status
 migrate-status: install-goose
-	GOOSE_DBSTRING=$(GOOSE_DBSTRING) GOOSE_DRIVER=postgres goose -dir ./internal/sql/migrations status
+	GOOSE_DBSTRING=$(DBSTRING) GOOSE_DRIVER=postgres goose -dir ./internal/sql/migrations status
 
 # Run docs server with live reload
 .PHONY: serve-docs

cmd/otf-agent/main.go

@@ -4,19 +4,26 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/signal"
+	"syscall"
 
 	cmdutil "github.com/leg100/otf/cmd"
 	"github.com/leg100/otf/internal"
+	"github.com/leg100/otf/internal/agent"
+	otfapi "github.com/leg100/otf/internal/api"
 	"github.com/leg100/otf/internal/logr"
-	"github.com/leg100/otf/internal/remoteops"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
 
 func main() {
 	// Configure ^C to terminate program
-	ctx, cancel := context.WithCancel(context.Background())
-	cmdutil.CatchCtrlC(cancel)
+	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
+	go func() {
+		<-ctx.Done()
+		// Stop handling ^C; another ^C will exit the program.
+		cancel()
+	}()
 
 	if err := run(ctx, os.Args[1:]); err != nil {
 		cmdutil.PrintError(err)
@@ -26,8 +33,9 @@ func main() {
 
 func run(ctx context.Context, args []string) error {
 	var (
-		loggerCfg *logr.Config
-		cfg       *remoteops.AgentConfig
+		loggerConfig *logr.Config
+		clientConfig otfapi.Config
+		agentConfig  *agent.Config
 	)
 
 	cmd := &cobra.Command{
@@ -36,25 +44,26 @@ func run(ctx context.Context, args []string) error {
 		SilenceErrors: true,
 		Version:       internal.Version,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			logger, err := logr.New(loggerCfg)
+			logger, err := logr.New(loggerConfig)
 			if err != nil {
 				return err
 			}
-
-			agent, err := remoteops.NewAgent(cmd.Context(), logger, *cfg)
+			agent, err := agent.NewRPC(logger, *agentConfig, clientConfig)
 			if err != nil {
-				return fmt.Errorf("unable to start agent: %w", err)
+				return fmt.Errorf("initializing agent: %w", err)
 			}
 			// blocks
-			return agent.Start(ctx)
+			return agent.Start(cmd.Context())
 		},
 	}
 
+	cmd.Flags().StringVar(&clientConfig.Address, "address", otfapi.DefaultAddress, "Address of OTF server")
+	cmd.Flags().StringVar(&clientConfig.Token, "token", "", "Agent token for authentication")
 	cmd.MarkFlagRequired("token")
 	cmd.SetArgs(args)
 
-	loggerCfg = logr.NewConfigFromFlags(cmd.Flags())
-	cfg = remoteops.NewAgentConfigFromFlags(cmd.Flags())
+	loggerConfig = logr.NewConfigFromFlags(cmd.Flags())
+	agentConfig = agent.NewConfigFromFlags(cmd.Flags())
 
 	if err := cmdutil.SetFlagsFromEnvVariables(cmd.Flags()); err != nil {
 		return errors.Wrap(err, "failed to populate config from environment vars")

cmd/otf/main.go

@@ -3,15 +3,21 @@ package main
 import (
 	"context"
 	"os"
+	"os/signal"
+	"syscall"
 
 	cmdutil "github.com/leg100/otf/cmd"
 	"github.com/leg100/otf/internal/cli"
 )
 
 func main() {
 	// Configure ^C to terminate program
-	ctx, cancel := context.WithCancel(context.Background())
-	cmdutil.CatchCtrlC(cancel)
+	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
+	go func() {
+		<-ctx.Done()
+		// Stop handling ^C; another ^C will exit the program.
+		cancel()
+	}()
 
 	if err := cli.NewCLI().Run(ctx, os.Args[1:], os.Stdout); err != nil {
 		cmdutil.PrintError(err)

cmd/otfd/main.go

@@ -4,15 +4,17 @@ import (
 	"context"
 	"io"
 	"os"
+	"os/signal"
+	"syscall"
 
 	cmdutil "github.com/leg100/otf/cmd"
 	"github.com/leg100/otf/internal"
+	"github.com/leg100/otf/internal/agent"
 	"github.com/leg100/otf/internal/authenticator"
 	"github.com/leg100/otf/internal/daemon"
 	"github.com/leg100/otf/internal/github"
 	"github.com/leg100/otf/internal/gitlab"
 	"github.com/leg100/otf/internal/logr"
-	"github.com/leg100/otf/internal/remoteops"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
@@ -24,8 +26,12 @@ const (
 
 func main() {
 	// Configure ^C to terminate program
-	ctx, cancel := context.WithCancel(context.Background())
-	cmdutil.CatchCtrlC(cancel)
+	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
+	go func() {
+		<-ctx.Done()
+		// Stop handling ^C; another ^C will exit the program.
+		cancel()
+	}()
 
 	if err := parseFlags(ctx, os.Args[1:], os.Stdout); err != nil {
 		cmdutil.PrintError(err)
@@ -103,7 +109,7 @@ func parseFlags(ctx context.Context, args []string, out io.Writer) error {
 	cmd.Flags().StringVar(&cfg.GoogleIAPConfig.Audience, "google-jwt-audience", "", "The Google JWT audience claim for validation. If unspecified then validation is skipped")
 
 	loggerConfig = logr.NewConfigFromFlags(cmd.Flags())
-	cfg.RemoteOpsConfig = remoteops.NewConfigFromFlags(cmd.Flags())
+	cfg.AgentConfig = agent.NewConfigFromFlags(cmd.Flags())
 
 	if err := cmdutil.SetFlagsFromEnvVariables(cmd.Flags()); err != nil {
 		return errors.Wrap(err, "failed to populate config from environment vars")

docs/agents.md

@@ -1,44 +1,81 @@
 # Agents
 
-OTF agents are dedicated processes for executing runs. They are functionally equivalent to [Terraform Cloud Agents](https://developer.hashicorp.com/terraform/cloud-docs/agents).
+An agent handles the execution of runs. There are two types of agents:
 
+* The agent built into `otfd`, referred to as a *server agent*.
+* The separate agent process, `otf-agent`, referred to as a *pool agent*.
 
-The `otf-agent` process maintains an outbound connection to the otf server; no inbound connectivity is required. This makes it suited to deployment in parts of your network that are segregated. For example, you may have a kubernetes cluster for which connectivity is only possible within a local subnet. By deploying an agent to the subnet, terraform can connect to the cluster and provision kubernetes resources.
+## Server agents
 
-!!! Note
-    An agent only handles runs for a single organization.
+A server agent handle runs for workspaces that are configured with the *remote* execution mode. It is built into the `otfd` process, so whenever you run `otfd` you are automatically running a server agent.
 
-### Setup agent
+## Pool agents
 
-* Log into the web app.
-* Select an organization. This will be the organization that the agent handles runs on behalf of.
-* Ensure you are on the main menu for the organization.
-* Select `agent tokens`.
-* Click `New Agent Token`.
-* Provide a description for the token.
-* Click the `Create token`.
-* Copy the token to your clipboard (clicking on the token should do this).
-* Start the agent in your terminal:
+A pool agent handles runs for workspaces that are configured with the *agent* execution mode. It is invoked as a dedicated process, `otf-agent`.
+
+A pool agent belongs to an *agent pool*. An agent pool is a group of `otf-agent` processes that can be used to communicate with isolated, private, or on-premises infrastructure. Each agent pool has its own set of tokens which are not shared across pools. When a workspace is configured to execute runs using the *agent* execution mode, any available agent in that workspace's assigned agent pool is eligible to execute the run.
+
+!!! note
+    Pool agents are functionally equivalent to [Terraform Cloud Agents](https://developer.hashicorp.com/terraform/cloud-docs/agents).
+
+### Walkthrough: pool agents
+
+First, create an agent pool in your organization. Go to the organization main menu and select **agent pools**.
+
+![organization main menu](images/organization_main_menu.png){.screenshot}
+
+Select **New agent pool** to reveal the form.
+
+![new agent pool](./images/new_agent_pool.png){.screenshot}
+
+Give the pool a name and click **Create agent pool**.
+
+![created agent pool](./images/created_agent_pool.png){.screenshot}
+
+By default you can assign *any* workspace to the agent pool. To grant access only to specific workspaces, select **Grant access to specific workspaces**.
+
+![grant access to specific workspace form](./images/agent_pool_grant_workspace_form.png){.screenshot}
+
+Select a workspace from the dropdown menu and it should be added to the list of granted workspaces.
+
+![granted access to specific workspace](./images/agent_pool_granted_workspace.png){.screenshot}
+
+Click **Save changes** to persist the change.
+
+You then need to *assign* a workspace to the pool. Go to the settings of a workspace and change the execution mode to **agent**:
+
+![setting execution mode on workspace](./images/workspace_select_agent_execution_mode.png){.screenshot}
+
+Click **Save changes** and return to the agent pool. You should see that the workspace is both *granted* and *assigned*.
+
+![granted and assigned workspace](./images/agent_pool_workspace_granted_and_assigned.png){.screenshot}
+
+Now create an agent token. A pool agent needs to authenticate with a token in order to join a pool. Click **New token** to reveal the form.
+
+![new agent token form](./images/agent_pool_open_new_token_form.png){.screenshot}
+
+Give the token a description and click **Create token**.
+
+![created agent token](./images/agent_pool_token_created.png){.screenshot}
+
+Copy the token to your system clipboard. Now you can run the agent:
 
 ```bash
-otf-agent --token <the-token-string> --address <otf-server-hostname>
+otf-agent --token <token> --address <otfd-hostname>
 ```
 
-* The agent will confirm it has successfully authenticated:
+The agent should confirm it has registered successfully:
 
 ```bash
-2022-10-30T09:15:30Z INF successfully authenticated organization=automatize
+2023/12/04 21:52:06 INFO starting agent version=unknown
+2023/12/04 21:52:06 INFO registered successfully agent.id=agent-NGB0H1QskahiN9xR agent
+.server=false agent.status=idle agent.ip_address=192.168.1.155 agent.pool_id=apool-d68
+ab60a67ccf4fc
+2023/12/04 21:52:06 INFO waiting for next job
 ```
 
-### Configure workspace
+Go to the agent pool and you should see the agent listed:
 
-* Login into the web app
-* Select the organization in which you created an agent
-* Ensure you are on the main menu for the organization.
-* Select `workspaces`.
-* Select a workspace.
-* Click `settings` in the top right menu.
-* Set `execution mode` to `agent`
-* Click `save changes`.
+![agent pool with agent idle](./images/agent_pool_with_idle_agent.png){.screenshot}
 
-Now runs for that workspace will be handled by an agent.
+You've successfully reached the end of this walkthrough. Any runs triggered on the workspace above will now be executed on the agent. You can create more agent pools and agents and assign workspaces to specific pools, giving you control over where runs are executed.

docs/auth/providers/github.md

@@ -12,7 +12,7 @@ Create an OAuth application in Github by following their [step-by-step instructi
     `https://<otf_hostname>/oauth/github/callback`
 
 !!! note
-    It is recommended that you first set the [`--hostname` flag](../../../config/flags/#-hostname) to a hostname that is accessible by Github, and that you use this hostname in the authorization callback URL above.
+    It is recommended that you first set the [`--hostname` flag](../../config/flags.md#-hostname) to a hostname that is accessible by Github, and that you use this hostname in the authorization callback URL above.
 
 Once you've registered the application, note the client ID and secret.
 

docs/auth/providers/gitlab.md

@@ -12,7 +12,7 @@ Create an OAuth application for your Gitlab group by following their [step-by-st
     `https://<otfd_install_hostname>/oauth/gitlab/callback`
 
 !!! note
-    It is recommended that you first set the [`--hostname` flag](../../../config/flags/#-hostname) to a hostname that is accessible by Gitlab, and that you use this hostname in the redirect URI above.
+    It is recommended that you first set the [`--hostname` flag](../../config/flags.md#-hostname) to a hostname that is accessible by Gitlab, and that you use this hostname in the redirect URI above.
 
 Once you've created the application, note the Application ID and Secret.