feat: add a flag to silence policy checks comments when there's no er…

…ror (#2405)

* Add a flag to silence policy checks comments when there's no error

* Add e2e test cases for the quiet policy checks flag

* add documentation for quiet-policy-checks flag

* Update user_config.go

Fix indentation

* Update user_config.go

* fmt

runatlantis.io/docs/server-configuration.md

@@ -468,6 +468,12 @@ Values are chosen in this order:
   ```
   Port to bind to. Defaults to `4141`.
 
+### `--quiet-policy-checks`
+  ```bash
+  atlantis server --quiet-policy-checks
+  ```
+  Exclude policy check comments from pull requests unless there's an actual error from conftest. This also excludes warnings. Defaults to `false`.
+
 ### `--redis-host`
   ```bash
   atlantis server --redis-host="localhost"

server/controllers/events/events_controller_e2e_test.go

@@ -653,6 +653,12 @@ func TestGitHubWorkflowWithPolicyCheck(t *testing.T) {
 		ExpAutomerge bool
 		// ExpAutoplan is true if we expect Atlantis to autoplan.
 		ExpAutoplan bool
+		// ExpQuietPolicyChecks is true if we expect Atlantis to exclude policy check output
+		// when there's no error
+		ExpQuietPolicyChecks bool
+		// ExpQuietPolicyCheckFailure is true when we expect Atlantis to post back policy check failures
+		// even when QuietPolicyChecks is enabled
+		ExpQuietPolicyCheckFailure bool
 		// ExpParallel is true if we expect Atlantis to run parallel plans or applies.
 		ExpParallel bool
 		// ExpReplies is a list of files containing the expected replies that
@@ -737,6 +743,38 @@ func TestGitHubWorkflowWithPolicyCheck(t *testing.T) {
 				{"exp-output-merge.txt"},
 			},
 		},
+		{
+			Description:          "successful policy checks with quiet flag enabled",
+			RepoDir:              "policy-checks-success-silent",
+			ModifiedFiles:        []string{"main.tf"},
+			ExpAutoplan:          true,
+			ExpQuietPolicyChecks: true,
+			Comments: []string{
+				"atlantis apply",
+			},
+			ExpReplies: [][]string{
+				{"exp-output-autoplan.txt"},
+				{"exp-output-apply.txt"},
+				{"exp-output-merge.txt"},
+			},
+		},
+		{
+			Description:                "failing policy checks with quiet flag enabled",
+			RepoDir:                    "policy-checks",
+			ModifiedFiles:              []string{"main.tf"},
+			ExpAutoplan:                true,
+			ExpQuietPolicyChecks:       true,
+			ExpQuietPolicyCheckFailure: true,
+			Comments: []string{
+				"atlantis apply",
+			},
+			ExpReplies: [][]string{
+				{"exp-output-autoplan.txt"},
+				{"exp-output-auto-policy-check.txt"},
+				{"exp-output-apply-failed.txt"},
+				{"exp-output-merge.txt"},
+			},
+		},
 	}
 
 	for _, c := range cases {
@@ -746,6 +784,7 @@ func TestGitHubWorkflowWithPolicyCheck(t *testing.T) {
 			// reset userConfig
 			userConfig = server.UserConfig{}
 			userConfig.EnablePolicyChecksFlag = true
+			userConfig.QuietPolicyChecks = c.ExpQuietPolicyChecks
 
 			ctrl, vcsClient, githubGetter, atlantisWorkspace := setupE2E(t, c.RepoDir)
 
@@ -805,6 +844,10 @@ func TestGitHubWorkflowWithPolicyCheck(t *testing.T) {
 				expNumReplies++
 			}
 
+			if c.ExpQuietPolicyChecks && !c.ExpQuietPolicyCheckFailure {
+				expNumReplies--
+			}
+
 			_, _, actReplies, _ := vcsClient.VerifyWasCalled(Times(expNumReplies)).CreateComment(AnyRepo(), AnyInt(), AnyString(), AnyString()).GetAllCapturedArguments()
 			Assert(t, len(c.ExpReplies) == len(actReplies), "missing expected replies, got %d but expected %d", len(actReplies), len(c.ExpReplies))
 			for i, expReply := range c.ExpReplies {
@@ -1008,6 +1051,7 @@ func setupE2E(t *testing.T, repoDir string) (events_controllers.VCSEventsControl
 		projectCommandRunner,
 		parallelPoolSize,
 		false,
+		userConfig.QuietPolicyChecks,
 	)
 
 	planCommandRunner := events.NewPlanCommandRunner(

server/controllers/events/testfixtures/test-repos/policy-checks-success-silent/atlantis.yaml

@@ -0,0 +1,4 @@
+version: 3
+projects:
+- dir: .
+  workspace: default

server/controllers/events/testfixtures/test-repos/policy-checks-success-silent/exp-output-apply.txt

@@ -0,0 +1,12 @@
+Ran Apply for dir: `.` workspace: `default`
+
+```diff
+
+Apply complete! Resources: 0 added, 0 changed, 0 destroyed.
+
+Outputs:
+
+workspace = "default"
+
+```
+

server/controllers/events/testfixtures/test-repos/policy-checks-success-silent/exp-output-autoplan.txt

@@ -0,0 +1,23 @@
+Ran Plan for dir: `.` workspace: `default`
+
+```diff
+
+Changes to Outputs:
++ workspace = "default"
+
+You can apply this plan to save these new output values to the Terraform
+state, without changing any real infrastructure.
+
+```
+
+* :arrow_forward: To **apply** this plan, comment:
+    * `atlantis apply -d .`
+* :put_litter_in_its_place: To **delete** this plan click [here](lock-url)
+* :repeat: To **plan** this project again, comment:
+    * `atlantis plan -d .`
+
+---
+* :fast_forward: To **apply** all unapplied plans from this pull request, comment:
+    * `atlantis apply`
+* :put_litter_in_its_place: To delete all plans and locks for the PR, comment:
+    * `atlantis unlock`

server/controllers/events/testfixtures/test-repos/policy-checks-success-silent/exp-output-merge.txt

@@ -0,0 +1,3 @@
+Locks and plans deleted for the projects and workspaces modified in this pull request:
+
+- dir: `.` workspace: `default`

server/controllers/events/testfixtures/test-repos/policy-checks-success-silent/main.tf

@@ -0,0 +1,3 @@
+output "workspace" {
+  value = terraform.workspace
+}

server/controllers/events/testfixtures/test-repos/policy-checks-success-silent/policies/policy.rego

@@ -0,0 +1,28 @@
+package main
+
+import input as tfplan
+
+deny[reason] {
+	num_deletes.null_resource > 0
+	reason := "WARNING: Null Resource creation is prohibited."
+}
+
+resource_types = {"null_resource"}
+
+resources[resource_type] = all {
+	some resource_type
+	resource_types[resource_type]
+	all := [name |
+		name := tfplan.resource_changes[_]
+		name.type == resource_type
+	]
+}
+
+# number of deletions of resources of a given type
+num_deletes[resource_type] = num {
+	some resource_type
+	resource_types[resource_type]
+	all := resources[resource_type]
+	deletions := [res | res := all[_]; res.change.actions[_] == "create"]
+	num := count(deletions)
+}

server/controllers/events/testfixtures/test-repos/policy-checks-success-silent/repos.yaml

@@ -0,0 +1,12 @@
+repos:
+- id: /.*/
+  apply_requirements: [approved]
+policies:
+  owners:
+    users:
+      - runatlantis
+  policy_sets:
+    - name: test_policy
+      path: policies/policy.rego
+      source: local
+

server/events/command_runner_test.go

@@ -117,6 +117,7 @@ func setup(t *testing.T) *vcsmocks.MockClient {
 		projectCommandRunner,
 		parallelPoolSize,
 		false,
+		false,
 	)
 
 	planCommandRunner = events.NewPlanCommandRunner(

server/events/policy_check_command_runner.go

@@ -12,6 +12,7 @@ func NewPolicyCheckCommandRunner(
 	projectCommandRunner ProjectPolicyCheckCommandRunner,
 	parallelPoolSize int,
 	silenceVCSStatusNoProjects bool,
+	quietPolicyChecks bool,
 ) *PolicyCheckCommandRunner {
 	return &PolicyCheckCommandRunner{
 		dbUpdater:                  dbUpdater,
@@ -20,6 +21,7 @@ func NewPolicyCheckCommandRunner(
 		prjCmdRunner:               projectCommandRunner,
 		parallelPoolSize:           parallelPoolSize,
 		silenceVCSStatusNoProjects: silenceVCSStatusNoProjects,
+		quietPolicyChecks:          quietPolicyChecks,
 	}
 }
 
@@ -32,6 +34,7 @@ type PolicyCheckCommandRunner struct {
 	// SilenceVCSStatusNoProjects is whether any plan should set commit status if no projects
 	// are found
 	silenceVCSStatusNoProjects bool
+	quietPolicyChecks          bool
 }
 
 func (p *PolicyCheckCommandRunner) Run(ctx *command.Context, cmds []command.ProjectContext) {
@@ -62,7 +65,10 @@ func (p *PolicyCheckCommandRunner) Run(ctx *command.Context, cmds []command.Proj
 		result = runProjectCmds(cmds, p.prjCmdRunner.PolicyCheck)
 	}
 
-	p.pullUpdater.updatePull(ctx, PolicyCheckCommand{}, result)
+	// Quiet policy checks unless there's an error
+	if result.HasErrors() || !p.quietPolicyChecks {
+		p.pullUpdater.updatePull(ctx, PolicyCheckCommand{}, result)
+	}
 
 	pullStatus, err := p.dbUpdater.updateDB(ctx, ctx.Pull, result.ProjectResults)
 	if err != nil {

server/server.go

@@ -621,6 +621,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) {
 		instrumentedProjectCmdRunner,
 		userConfig.ParallelPoolSize,
 		userConfig.SilenceVCSStatusNoProjects,
+		userConfig.QuietPolicyChecks,
 	)
 
 	planCommandRunner := events.NewPlanCommandRunner(

server/user_config.go

@@ -55,6 +55,7 @@ type UserConfig struct {
 	StatsNamespace                  string `mapstructure:"stats-namespace"`
 	PlanDrafts                      bool   `mapstructure:"allow-draft-prs"`
 	Port                            int    `mapstructure:"port"`
+	QuietPolicyChecks               bool   `mapstructure:"quiet-policy-checks"`
 	RedisDB                         int    `mapstructure:"redis-db"`
 	RedisHost                       string `mapstructure:"redis-host"`
 	RedisPassword                   string `mapstructure:"redis-password"`