fix: flaky browser tests (#484)

internal/integration/agent_token_ui_test.go

@@ -23,17 +23,17 @@ func TestAgentTokenUI(t *testing.T) {
 			chromedp.Navigate(organizationURL(svc.Hostname(), org.Name)),
 			screenshot(t),
 			// go to list of agent tokens
-			chromedp.Click("#agent_tokens > a", chromedp.NodeVisible, chromedp.ByQuery),
+			chromedp.Click("#agent_tokens > a", chromedp.ByQuery),
 			screenshot(t),
 			// go to new agent token page
-			chromedp.Click(`//button[text()='New Agent Token']`, chromedp.NodeVisible),
+			chromedp.Click(`//button[text()='New Agent Token']`),
 			screenshot(t),
 			// enter description for new agent token
 			chromedp.Focus("input#description", chromedp.NodeVisible, chromedp.ByQuery),
 			input.InsertText("my-new-agent-token"),
 			screenshot(t),
 			// submit form
-			chromedp.Click(`//button[text()='Create token']`, chromedp.NodeVisible),
+			chromedp.Click(`//button[text()='Create token']`),
 			screenshot(t),
 			matchRegex(t, ".flash-success", `Created token:\s+[\w-]+\.[\w-]+\.[\w-]+`),
 			// click clipboard icon to copy token into clipboard
@@ -42,9 +42,9 @@ func TestAgentTokenUI(t *testing.T) {
 				return p.WithAwaitPromise(true)
 			}),
 			// delete the token
-			chromedp.Click(`//button[text()='delete']`, chromedp.NodeVisible),
+			chromedp.Click(`//button[text()='delete']`),
 			screenshot(t),
-			matchText(t, ".flash-success", "Deleted token: my-new-agent-token"),
+			matchText(t, `.flash-success`, `Deleted token: my-new-agent-token`, chromedp.ByQuery),
 		},
 	})
 

internal/integration/auto_apply_ui_test.go

@@ -22,16 +22,16 @@ func TestAutoApply(t *testing.T) {
 			chromedp.Navigate(workspaceURL(svc.Hostname(), org.Name, t.Name())),
 			screenshot(t),
 			// go to workspace settings
-			chromedp.Click(`//a[text()='settings']`, chromedp.NodeVisible),
+			chromedp.Click(`//a[text()='settings']`),
 			screenshot(t),
 			// enable auto-apply
-			chromedp.Click("input#auto_apply", chromedp.NodeVisible, chromedp.ByQuery),
+			chromedp.Click("input#auto_apply", chromedp.ByQuery),
 			screenshot(t),
 			// submit form
-			chromedp.Click(`//button[text()='Save changes']`, chromedp.NodeVisible),
+			chromedp.Click(`//button[text()='Save changes']`),
 			screenshot(t),
 			// confirm workspace updated
-			matchText(t, ".flash-success", "updated workspace"),
+			matchText(t, ".flash-success", "updated workspace", chromedp.ByQuery),
 		},
 	})
 

internal/integration/connect_repo_e2e_test.go

@@ -69,33 +69,33 @@ func TestConnectRepoE2E(t *testing.T) {
 		chromedp.Navigate(workspaceURL(daemon.Hostname(), org.Name, "my-test-workspace")),
 		screenshot(t),
 		// go to workspace settings
-		chromedp.Click(`//a[text()='settings']`, chromedp.NodeVisible),
+		chromedp.Click(`//a[text()='settings']`),
 		screenshot(t),
 		// click disconnect button
-		chromedp.Click(`//button[@id='disconnect-workspace-repo-button']`, chromedp.NodeVisible),
+		chromedp.Click(`//button[@id='disconnect-workspace-repo-button']`),
 		screenshot(t),
 		// confirm disconnected
-		matchText(t, ".flash-success", "disconnected workspace from repo"),
+		matchText(t, ".flash-success", "disconnected workspace from repo", chromedp.ByQuery),
 		// go to workspace settings
-		chromedp.Click(`//a[text()='settings']`, chromedp.NodeVisible),
+		chromedp.Click(`//a[text()='settings']`),
 		screenshot(t),
 		// delete workspace
-		chromedp.Click(`//button[@id='delete-workspace-button']`, chromedp.NodeVisible),
+		chromedp.Click(`//button[@id='delete-workspace-button']`),
 		screenshot(t),
 		// confirm deletion
-		matchText(t, ".flash-success", "deleted workspace: my-test-workspace"),
+		matchText(t, ".flash-success", "deleted workspace: my-test-workspace", chromedp.ByQuery),
 		//
 		// delete vcs provider
 		//
 		// go to org
 		chromedp.Navigate(organizationURL(daemon.Hostname(), org.Name)),
 		screenshot(t),
 		// go to vcs providers
-		chromedp.Click("#vcs_providers > a", chromedp.NodeVisible),
+		chromedp.Click("#vcs_providers > a"),
 		screenshot(t),
 		// click delete button for one and only vcs provider
-		chromedp.Click(`//button[text()='delete']`, chromedp.NodeVisible),
+		chromedp.Click(`//button[text()='delete']`),
 		screenshot(t),
-		matchText(t, ".flash-success", "deleted provider: github"),
+		matchText(t, ".flash-success", "deleted provider: github", chromedp.ByQuery),
 	})
 }

internal/integration/daemon_helpers_test.go

@@ -448,9 +448,9 @@ func (s *testDaemon) tfcliWithError(t *testing.T, ctx context.Context, command,
 
 	cmd := exec.Command("terraform", cmdargs...)
 	cmd.Dir = configPath
-	cmd.Env = append(envs,
-		internal.CredentialEnv(s.Hostname(), token),
-	)
+
+	cmd.Env = appendSharedEnvs(internal.CredentialEnv(s.Hostname(), token))
+
 	out, err := cmd.CombinedOutput()
 	return string(out), err
 }

internal/integration/github_login_test.go

@@ -46,9 +46,9 @@ func TestGithubLogin(t *testing.T) {
 		chromedp.Navigate("https://" + svc.Hostname() + "/login"),
 		screenshot(t, "github_login_button"),
 		// login
-		chromedp.Click("a.login-button-github", chromedp.NodeVisible, chromedp.ByQuery),
+		chromedp.Click("a.login-button-github", chromedp.ByQuery),
 		screenshot(t),
 		// check login confirmation message
-		matchText(t, ".content > p", "You are logged in as bobby"),
+		matchText(t, `.content > p`, `You are logged in as bobby`, chromedp.ByQuery),
 	})
 }

internal/integration/main_test.go

@@ -21,7 +21,7 @@ var (
 	sharedSecret []byte
 
 	// shared environment variables for individual tests to use
-	envs []string
+	sharedEnvs []string
 
 	// Context conferring site admin privileges
 	adminCtx = internal.AddSubjectToContext(context.Background(), &auth.SiteAdmin)
@@ -157,8 +157,16 @@ func setenv(name, value string) (func(), error) {
 	if err != nil {
 		return nil, fmt.Errorf("setting %s=%s: %w", name, value, err)
 	}
-	envs = append(envs, name+"="+value)
+	sharedEnvs = append(sharedEnvs, name+"="+value)
 	return func() {
 		os.Unsetenv(name)
 	}, nil
 }
+
+// appendSharedEnvs appends environment variables to the shared environment
+// variables in a thread-safe manner.
+func appendSharedEnvs(envs ...string) []string {
+	dst := make([]string, len(sharedEnvs)+len(envs))
+	copy(dst, sharedEnvs)
+	return append(dst, envs...)
+}

internal/integration/module_e2e_test.go

@@ -36,22 +36,22 @@ func TestModuleE2E(t *testing.T) {
 			chromedp.Navigate(organizationURL(svc.Hostname(), org.Name)),
 			screenshot(t),
 			// go to modules
-			chromedp.Click("#modules > a", chromedp.NodeVisible, chromedp.ByQuery),
+			chromedp.Click("#modules > a", chromedp.ByQuery),
 			screenshot(t, "modules_list"),
 			// click publish button
-			chromedp.Click(`//button[text()='Publish']`, chromedp.NodeVisible),
+			chromedp.Click(`//button[text()='Publish']`),
 			screenshot(t, "modules_select_provider"),
 			// select provider
-			chromedp.Click(`//button[text()='connect']`, chromedp.NodeVisible),
+			chromedp.Click(`//button[text()='connect']`),
 			screenshot(t, "modules_select_repo"),
 			// connect to first repo in list (there should only be one)
-			chromedp.Click(`//div[@class='content-list']//button[text()='connect']`, chromedp.NodeVisible),
+			chromedp.Click(`//div[@class='content-list']//button[text()='connect']`),
 			screenshot(t, "modules_confirm"),
 			// confirm module details
-			chromedp.Click(`//button[text()='connect']`, chromedp.NodeVisible),
+			chromedp.Click(`//button[text()='connect']`),
 			screenshot(t, "newly_created_module_page"),
 			// flash message indicates success
-			matchText(t, ".flash-success", "published module: mod"),
+			matchText(t, `.flash-success`, `published module: mod`, chromedp.ByQuery),
 			// TODO: confirm versions are populated
 			// capture module url so we can visit it later
 			chromedp.Location(&moduleURL),

internal/integration/oidc_test.go

@@ -33,9 +33,9 @@ func TestIntegration_OIDC(t *testing.T) {
 		chromedp.Navigate("https://" + svc.Hostname() + "/login"),
 		screenshot(t, "oidc_login_button"),
 		// login
-		chromedp.Click("a.login-button-google", chromedp.NodeVisible),
+		chromedp.Click("a.login-button-google"),
 		screenshot(t),
 		// check login confirmation message
-		matchText(t, ".content > p", "You are logged in as bobby"),
+		matchText(t, ".content > p", "You are logged in as bobby", chromedp.ByQuery),
 	})
 }

internal/integration/plan_permission_e2e_test.go

@@ -63,10 +63,9 @@ func TestIntegration_PlanPermission(t *testing.T) {
 		chromedp.SetValue(`//select[@id="start-run-operation"]`, "plan-only"),
 		screenshot(t),
 		// confirm plan begins and ends
-		chromedp.WaitReady(`body`),
 		chromedp.WaitReady(`//*[@id='tailed-plan-logs']//text()[contains(.,'Initializing the backend')]`),
 		screenshot(t),
-		chromedp.WaitReady(`#plan-status.phase-status-finished`),
+		chromedp.WaitReady(`#plan-status.phase-status-finished`, chromedp.ByQuery),
 		screenshot(t),
 		// wait for run to enter planned-and-finished state
 		chromedp.WaitReady(`//*[@class='status status-planned_and_finished']`),

internal/integration/retry_run_ui_test.go

@@ -35,18 +35,17 @@ func TestIntegration_RetryRunUI(t *testing.T) {
 	browser.Run(t, ctx, chromedp.Tasks{
 		chromedp.Navigate(runURL(daemon.Hostname(), r.ID)),
 		// run should be in planned and finished state
-		chromedp.WaitReady(`//*[@class='status status-planned_and_finished']`),
+		chromedp.WaitVisible(`//*[@class='status status-planned_and_finished']`),
 		screenshot(t, "run_page_planned_and_finished_state"),
 		// click retry button
-		chromedp.Click(`//button[text()='retry run']`, chromedp.NodeVisible),
+		chromedp.Click(`//button[text()='retry run']`),
 		screenshot(t),
 		// confirm plan begins and ends
-		chromedp.WaitReady(`body`),
 		chromedp.WaitReady(`//*[@id='tailed-plan-logs']//text()[contains(.,'Initializing the backend')]`),
 		screenshot(t),
-		chromedp.WaitReady(`#plan-status.phase-status-finished`, chromedp.ByQuery),
+		chromedp.WaitVisible(`#plan-status.phase-status-finished`, chromedp.ByQuery),
 		// confirm retry button re-appears
-		chromedp.WaitReady(`//button[text()='retry run']`),
+		chromedp.WaitVisible(`//button[text()='retry run']`),
 		screenshot(t),
 	})
 }

internal/integration/run_list_ui_test.go

@@ -22,7 +22,7 @@ func TestIntegration_RunListUI(t *testing.T) {
 		// navigate to workspace page
 		chromedp.Navigate(workspaceURL(daemon.Hostname(), ws.Organization, ws.Name)),
 		// navigate to runs page
-		chromedp.Click(`//a[text()='runs']`, chromedp.NodeVisible),
+		chromedp.Click(`//a[text()='runs']`),
 		// should be no runs listed
 		matchText(t, `//div[@id='content-list']`, `No items currently exist.`),
 		chromedp.ActionFunc(func(context.Context) error {

internal/integration/site_admin_ui_test.go

@@ -20,49 +20,48 @@ func TestSiteAdminUI(t *testing.T) {
 
 	var orgLocation string
 
-	// Click OK on any browser javascript dialog boxes that pop up
 	browser.Run(t, nil, chromedp.Tasks{
 		// login as site admin
 		chromedp.Navigate("https://" + daemon.Hostname() + "/login"),
 		screenshot(t, "no_authenticators_site_admin_login"),
 		// use the link in the bottom right corner
-		matchText(t, ".footer-site-login", "site admin"),
-		chromedp.Click(".footer-site-login > a", chromedp.NodeVisible, chromedp.ByQuery),
+		matchText(t, ".footer-site-login", "site admin", chromedp.ByQuery),
+		chromedp.Click(".footer-site-login > a", chromedp.ByQuery),
 		screenshot(t),
 		// enter token
 		chromedp.Focus("input#token", chromedp.NodeVisible, chromedp.ByQuery),
 		input.InsertText("abc123"),
 		screenshot(t, "site_admin_login_enter_token"),
 		chromedp.Submit("input#token", chromedp.ByQuery),
 		screenshot(t, "site_admin_profile"),
-		matchText(t, ".content > p", "You are logged in as site-admin"),
+		matchText(t, ".content > p", "You are logged in as site-admin", chromedp.ByQuery),
 		// now go to the list of organizations
 		chromedp.Navigate("https://" + daemon.Hostname() + "/app/organizations"),
 		// add an org
-		chromedp.Click("#new-organization-button", chromedp.NodeVisible, chromedp.ByQuery),
+		chromedp.Click("#new-organization-button", chromedp.ByQuery),
 		screenshot(t),
 		chromedp.Focus("input#name", chromedp.NodeVisible, chromedp.ByQuery),
 		input.InsertText("my-new-org"),
 		screenshot(t, "new_org_enter_name"),
 		chromedp.Submit("input#name", chromedp.ByQuery),
 		screenshot(t, "new_org_created"),
 		chromedp.Location(&orgLocation),
-		matchText(t, ".flash-success", "created organization: my-new-org"),
+		matchText(t, ".flash-success", "created organization: my-new-org", chromedp.ByQuery),
 		// go to organization settings
-		chromedp.Click("#settings > a", chromedp.NodeVisible, chromedp.ByQuery),
+		chromedp.Click("#settings > a", chromedp.ByQuery),
 		screenshot(t),
 		// change organization name
 		chromedp.Focus("input#name", chromedp.NodeVisible, chromedp.ByQuery),
 		chromedp.Clear("input#name", chromedp.ByQuery),
 		input.InsertText("newly-named-org"),
 		screenshot(t),
-		chromedp.Click(`//button[text()='Update organization name']`, chromedp.NodeVisible),
+		chromedp.Click(`//button[text()='Update organization name']`),
 		screenshot(t),
-		matchText(t, ".flash-success", "updated organization"),
+		matchText(t, ".flash-success", "updated organization", chromedp.ByQuery),
 		// delete the organization
-		chromedp.Click(`//button[@id='delete-organization-button']`, chromedp.NodeVisible),
+		chromedp.Click(`//button[@id='delete-organization-button']`),
 		screenshot(t),
-		matchText(t, ".flash-success", "deleted organization: newly-named-org"),
+		matchText(t, ".flash-success", "deleted organization: newly-named-org", chromedp.ByQuery),
 	})
 
 	assert.Equal(t, organizationURL(daemon.Hostname(), "my-new-org"), orgLocation)

internal/integration/start_run_ui_test.go

@@ -27,13 +27,12 @@ func TestStartRunUI(t *testing.T) {
 		chromedp.Navigate(workspaceURL(svc.Hostname(), ws.Organization, ws.Name)),
 		screenshot(t, "workspace_page"),
 		// navigate to workspace settings
-		chromedp.Click(`//a[text()='settings']`, chromedp.NodeVisible),
+		chromedp.Click(`//a[text()='settings']`),
 		screenshot(t),
 		// click 'queue destroy plan' button
 		chromedp.Click(`//button[@id='queue-destroy-plan-button']`),
 		screenshot(t),
 		// confirm plan begins and ends
-		chromedp.WaitReady(`body`),
 		chromedp.WaitReady(`//*[@id='tailed-plan-logs']//text()[contains(.,'Initializing the backend')]`),
 		screenshot(t),
 		chromedp.WaitReady(`#plan-status.phase-status-finished`, chromedp.ByQuery),
@@ -45,10 +44,10 @@ func TestStartRunUI(t *testing.T) {
 		matchRegex(t, `//div[@class='item']//div[@class='resource-summary']`, `\+[0-9]+ \~[0-9]+ \-[0-9]+`),
 		screenshot(t),
 		// run widget should show discard button
-		chromedp.WaitReady(`//button[@id='run-discard-button']`),
+		chromedp.WaitVisible(`//button[@id='run-discard-button']`),
 		screenshot(t),
 		// click 'confirm & apply' button once it becomes visible
-		chromedp.Click(`//button[text()='apply']`, chromedp.NodeVisible),
+		chromedp.Click(`//button[text()='apply']`),
 		screenshot(t),
 		// confirm apply begins and ends
 		chromedp.WaitReady(`//*[@id='tailed-apply-logs']//text()[contains(.,'Initializing the backend')]`),

internal/integration/tag_e2e_test.go

@@ -42,9 +42,7 @@ resource "null_resource" "tags_e2e" {}
 		[]string{"terraform", "-chdir=" + root, "init", "-no-color"},
 		time.Minute,
 		expect.PartialMatch(true),
-		expect.SetEnv(
-			append(envs, internal.CredentialEnv(daemon.Hostname(), token)),
-		),
+		expect.SetEnv(appendSharedEnvs(internal.CredentialEnv(daemon.Hostname(), token))),
 	)
 	require.NoError(t, err)
 	defer e.Close()
@@ -77,28 +75,28 @@ resource "null_resource" "tags_e2e" {}
 		chromedp.WaitVisible(`//*[@class='workspace-tag'][contains(text(),'foo')]`),
 		chromedp.WaitVisible(`//*[@class='workspace-tag'][contains(text(),'bar')]`),
 		// go to tag settings
-		chromedp.Click(`//a[@id='tags-add-remove-link']`, chromedp.NodeVisible),
+		chromedp.Click(`//a[@id='tags-add-remove-link']`),
 		screenshot(t),
 		// remove bar tag
-		chromedp.Click(`//button[@id='button-remove-tag-bar']`, chromedp.NodeVisible),
+		chromedp.Click(`//button[@id='button-remove-tag-bar']`),
 		screenshot(t),
-		matchText(t, ".flash-success", "removed tag: bar"),
+		matchText(t, ".flash-success", "removed tag: bar", chromedp.ByQuery),
 		// add new tag
-		chromedp.Focus("input#new-tag-name", chromedp.NodeVisible, chromedp.ByQuery),
+		chromedp.Focus("input#new-tag-name", chromedp.ByQuery),
 		input.InsertText("baz"),
-		chromedp.Click(`//button[text()='Add new tag']`, chromedp.NodeVisible),
+		chromedp.Click(`//button[text()='Add new tag']`),
 		screenshot(t),
-		matchText(t, ".flash-success", "created tag: baz"),
+		matchText(t, ".flash-success", "created tag: baz", chromedp.ByQuery),
 		// go to workspace listing
-		chromedp.Click(`//div[@class='content-header-title']//a[text()='workspaces']`, chromedp.NodeVisible),
+		chromedp.Click(`//div[@class='content-header-title']//a[text()='workspaces']`),
 		screenshot(t),
 		// filter by tag foo
-		chromedp.Click(`//label[@for='workspace-tag-filter-foo']`, chromedp.NodeVisible),
+		chromedp.Click(`//label[@for='workspace-tag-filter-foo']`),
 		screenshot(t),
 		// filter by tag bar
-		chromedp.Click(`//label[@for='workspace-tag-filter-baz']`, chromedp.NodeVisible),
+		chromedp.Click(`//label[@for='workspace-tag-filter-baz']`),
 		screenshot(t),
 		// confirm workspace listing contains tagged workspace
-		chromedp.WaitVisible(`//div[@id='content-list']//a[text()='tagged']`, chromedp.NodeVisible),
+		chromedp.WaitVisible(`//div[@id='content-list']//a[text()='tagged']`),
 	})
 }