modify exit codes related to scan errors

pkg/cli/run.go

@@ -207,9 +207,12 @@ func (s *ScanOptions) Run() error {
 		return err
 	}
 
-	if !s.configOnly && results.Violations.ViolationStore.Summary.ViolatedPolicies != 0 && flag.Lookup("test.v") == nil {
+	if !s.configOnly && flag.Lookup("test.v") == nil {
 		os.RemoveAll(tempDir)
-		os.Exit(3)
+		exitCode := getExitCode(results)
+		if exitCode != 0 {
+			os.Exit(exitCode)
+		}
 	}
 	return nil
 }
@@ -247,3 +250,16 @@ func (s ScanOptions) writeResults(results runtime.Output) error {
 
 	return writer.Write(s.outputType, results.Violations, outputWriter)
 }
+
+// getExitCode returns appropriate exit code for terrascan based on scan output
+func getExitCode(o runtime.Output) int {
+	if len(o.Violations.ViolationStore.DirScanErrors) > 0 {
+		if o.Violations.ViolationStore.Summary.ViolatedPolicies > 0 {
+			return 5
+		}
+		return 4
+	} else if o.Violations.ViolationStore.Summary.ViolatedPolicies > 0 {
+		return 3
+	}
+	return 0
+}

pkg/cli/run_test.go

@@ -567,3 +567,91 @@ func TestScanOptionsScan(t *testing.T) {
 		})
 	}
 }
+
+func Test_getExitCode(t *testing.T) {
+	testDirScanErrors := []results.DirScanErr{
+		{
+			IacType:    "all",
+			Directory:  "test",
+			ErrMessage: "error occurred",
+		},
+	}
+
+	testScanSummary := results.ScanSummary{
+		ViolatedPolicies: 1,
+	}
+
+	scanOutputWithDirErrorsOnly := runtime.Output{
+		Violations: policy.EngineOutput{
+			ViolationStore: &results.ViolationStore{
+				DirScanErrors: testDirScanErrors,
+			},
+		},
+	}
+
+	scanOutputWithDirErrorsAndViolatedPolicies := runtime.Output{
+		Violations: policy.EngineOutput{
+			ViolationStore: &results.ViolationStore{
+				DirScanErrors: testDirScanErrors,
+				Summary:       testScanSummary,
+			},
+		},
+	}
+
+	scanOutputWithViolatedPoliciesOnly := runtime.Output{
+		Violations: policy.EngineOutput{
+			ViolationStore: &results.ViolationStore{
+				Summary: testScanSummary,
+			},
+		},
+	}
+
+	type args struct {
+		o runtime.Output
+	}
+	tests := []struct {
+		name string
+		args args
+		want int
+	}{
+		{
+			name: "has directory scan errors without violated policies",
+			args: args{
+				o: scanOutputWithDirErrorsOnly,
+			},
+			want: 4,
+		},
+		{
+			name: "has directory scan errors with violated policies",
+			args: args{
+				o: scanOutputWithDirErrorsAndViolatedPolicies,
+			},
+			want: 5,
+		},
+		{
+			name: "has violated policies but no directory scan errors",
+			args: args{
+				o: scanOutputWithViolatedPoliciesOnly,
+			},
+			want: 3,
+		},
+		{
+			name: "neither violations nor directory scan errors",
+			args: args{
+				o: runtime.Output{
+					Violations: policy.EngineOutput{
+						ViolationStore: &results.ViolationStore{},
+					},
+				},
+			},
+			want: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getExitCode(tt.args.o); got != tt.want {
+				t.Errorf("getExitCode() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

test/e2e/scan/scan_docker_file_test.go

@@ -53,8 +53,9 @@ var _ = Describe("Scan is run for dockerfile directories and files", func() {
 				It("should scan all iac and display violations", func() {
 					scanArgs := []string{scanUtils.ScanCommand, "-d", iacDir}
 					session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...)
-					// exit code is 3 because iac files in directory has violations
-					helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeThree)
+					// exit code is 5 because iac files in directory has violations
+					// and directory scan errors
+					helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeFive)
 				})
 			})
 		})

test/e2e/scan/scan_k8s_files_test.go

@@ -53,8 +53,9 @@ var _ = Describe("Scan is run for k8s directories and files", func() {
 				It("should scan will all iac and display violations", func() {
 					scanArgs := []string{scanUtils.ScanCommand, "-d", iacDir}
 					session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...)
-					// exit code is 3 because iac files in directory has violations
-					helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeThree)
+					// exit code is 5 because iac files in directory has violations
+					// and directory scan errors
+					helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeFive)
 				})
 			})
 		})

test/e2e/scan/scan_remote_test.go

@@ -139,14 +139,18 @@ var _ = Describe("Scan Command using remote types", func() {
 			It("should download the resource and generate scan results", func() {
 				scanArgs := []string{scanUtils.ScanCommand, "-r", "git", "--remote-url", remoteURL}
 				session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...)
-				Eventually(session, scanUtils.RemoteScanTimeout).Should(gexec.Exit(helper.ExitCodeThree))
+				// exit code is 5 because iac files in directory has violations
+				// and directory scan errors
+				Eventually(session, scanUtils.RemoteScanTimeout).Should(gexec.Exit(helper.ExitCodeFive))
 			})
 
 			It("should download the resource and generate scan results", func() {
 				remoteURL := "https://github.com/accurics/KaiMonkey.git//terraform/aws"
 				scanArgs := []string{scanUtils.ScanCommand, "-r", "git", "--remote-url", remoteURL}
 				session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...)
-				Eventually(session, scanUtils.RemoteScanTimeout).Should(gexec.Exit(helper.ExitCodeThree))
+				// exit code is 5 because iac files in directory has violations
+				// and directory scan errors
+				Eventually(session, scanUtils.RemoteScanTimeout).Should(gexec.Exit(helper.ExitCodeFive))
 			})
 		})
 
@@ -175,7 +179,8 @@ var _ = Describe("Scan Command using remote types", func() {
 					scanArgs := []string{scanUtils.ScanCommand, "-r", "terraform-registry", "--remote-url", remoteURL}
 					session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...)
 					// has a OR condition because we don't know if there would be violations or not
-					Eventually(session, scanUtils.RemoteScanTimeout).Should(Or(gexec.Exit(helper.ExitCodeThree), gexec.Exit(helper.ExitCodeZero)))
+					// there would be directory scan errors due to all iac type
+					Eventually(session, scanUtils.RemoteScanTimeout).Should(Or(gexec.Exit(helper.ExitCodeFive), gexec.Exit(helper.ExitCodeFour)))
 				})
 			})
 
@@ -185,15 +190,16 @@ var _ = Describe("Scan Command using remote types", func() {
 					scanArgs := []string{scanUtils.ScanCommand, "-r", "terraform-registry", "--remote-url", remoteURL}
 					session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...)
 					// has a OR condition because we don't know if there would be violations or not
-					Eventually(session, scanUtils.RemoteScanTimeout).Should(Or(gexec.Exit(helper.ExitCodeThree), gexec.Exit(helper.ExitCodeZero)))
+					// there would be directory scan errors due to all iac type
+					Eventually(session, scanUtils.RemoteScanTimeout).Should(Or(gexec.Exit(helper.ExitCodeFive), gexec.Exit(helper.ExitCodeFour)))
 				})
 			})
 
 			Context("remote modules has reference to its local modules", func() {
 				When("remote type is terraform registry and remote url has a subdirectory", func() {
 					remoteURL := "terraform-aws-modules/security-group/aws//modules/http-80"
 					It("should download the remote registry and generate scan results", func() {
-						scanArgs := []string{scanUtils.ScanCommand, "-r", "terraform-registry", "--remote-url", remoteURL}
+						scanArgs := []string{scanUtils.ScanCommand, "-r", "terraform-registry", "--remote-url", remoteURL, "-i", "terraform", "--non-recursive"}
 						session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...)
 						// has a OR condition because we don't know if there would be violations or not
 						Eventually(session, scanUtils.RemoteScanTimeout).Should(Or(gexec.Exit(helper.ExitCodeThree), gexec.Exit(helper.ExitCodeZero)))
@@ -203,7 +209,7 @@ var _ = Describe("Scan Command using remote types", func() {
 				When("remote type is git and remote url has a subdirectory", func() {
 					remoteURL := "github.com/terraform-aws-modules/terraform-aws-security-group//modules/http-80"
 					It("should download the remote registry and generate scan results", func() {
-						scanArgs := []string{scanUtils.ScanCommand, "-r", "git", "--remote-url", remoteURL}
+						scanArgs := []string{scanUtils.ScanCommand, "-r", "git", "--remote-url", remoteURL, "-i", "terraform", "--non-recursive"}
 						session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...)
 						// has a OR condition because we don't know if there would be violations or not
 						Eventually(session, scanUtils.RemoteScanTimeout).Should(Or(gexec.Exit(helper.ExitCodeThree), gexec.Exit(helper.ExitCodeZero)))

test/e2e/scan/scan_rules_filtering_test.go

@@ -129,18 +129,19 @@ var _ = Describe("Scan command with rule filtering options", func() {
 				Context("severity leve specified is 'low'", func() {
 					Context("iac file has only medium severity violations", func() {
 						It("should report the violations and exit with status code 3", func() {
-							scanArgs := []string{scanUtils.ScanCommand, "-p", policyDir, "-d", iacDir, "-o", "json", "--severity", "low"}
+							scanArgs := []string{scanUtils.ScanCommand, "-p", policyDir, "-d", iacDir, "-o", "json", "--severity", "low", "-i", "terraform"}
 							session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...)
 							Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeThree))
 						})
 					})
 				})
-				Context("severity leve specified is 'high'", func() {
+				Context("severity level specified is 'high'", func() {
 					Context("iac files has only medium severity violations", func() {
-						It("should not report any violation and exit with status code 0", func() {
+						// there would not no violations but directory scan errors would be present due to all iac scan
+						It("should not report any violation and exit with status code 4", func() {
 							scanArgs := []string{scanUtils.ScanCommand, "-p", policyDir, "-d", iacDir, "-o", "json", "--severity", "high"}
 							session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...)
-							Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeZero))
+							Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeFour))
 						})
 					})
 				})
@@ -171,7 +172,8 @@ var _ = Describe("Scan command with rule filtering options", func() {
 						It("should not report any violation and exit with status code 0", func() {
 							scanArgs := []string{scanUtils.ScanCommand, "-p", policyDir, "-d", iacDir, "-o", "json", "--categories", "COMPLIANCE VALIDATION"}
 							session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...)
-							Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeZero))
+							// summary would contain directory scan errors due to all iac scan
+							Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeFour))
 						})
 					})
 				})

test/e2e/scan/scan_test.go

@@ -81,10 +81,10 @@ var _ = Describe("Scan", func() {
 	Describe("scan command is run", func() {
 		Context("when no iac type is provided, terrascan scans with all iac providers", func() {
 			Context("no tf files are present in the working directory", func() {
-				It("scan the directory and display results", func() {
+				It("scans the directory with all iac and display results", func() {
 					scanArgs := []string{scanUtils.ScanCommand}
 					session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...)
-					helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeZero)
+					helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeFour)
 				})
 				Context("iac loading errors would be displayed in the output, output type is json", func() {
 					It("scan the directory and display results", func() {
@@ -106,17 +106,17 @@ var _ = Describe("Scan", func() {
 				})
 			})
 			Context("tf files are present in the working directory", func() {
-				It("should scan the directory, return results and exit with status code 3", func() {
+				It("should scan the directory, return results and exit with status code 3 as there would no directory scan errors", func() {
 					workDir, err := filepath.Abs(filepath.Join(awsIacRelPath, "aws_ami_violation"))
 					Expect(err).NotTo(HaveOccurred())
 
-					scanArgs := []string{scanUtils.ScanCommand}
+					scanArgs := []string{scanUtils.ScanCommand, "-i", "terraform", "--non-recursive"}
 					session = helper.RunCommandDir(terrascanBinaryPath, workDir, outWriter, errWriter, scanArgs...)
 					Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeThree))
 				})
 
 				When("tf file present in the dir has no violations", func() {
-					Context("when there are no violations, terrascan exits with status code 0", func() {
+					Context("when there are no violations, but has dir scan erros, terrascan exits with status code 4", func() {
 						It("should scan the directory and exit with status code 0", func() {
 							workDir, err := filepath.Abs(filepath.Join(awsIacRelPath, "aws_db_instance_violation"))
 							Expect(err).NotTo(HaveOccurred())
@@ -127,7 +127,7 @@ var _ = Describe("Scan", func() {
 
 							scanArgs := []string{scanUtils.ScanCommand, "-p", policyDir}
 							session = helper.RunCommandDir(terrascanBinaryPath, workDir, outWriter, errWriter, scanArgs...)
-							Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeZero))
+							Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeFour))
 						})
 					})
 				})
@@ -194,10 +194,10 @@ var _ = Describe("Scan", func() {
 
 		When("--iac-version flag is supplied invalid version", func() {
 			Context("default iac type is all and --iac-version would be ignored", func() {
-				It("should error out and exit with status code 1", func() {
+				It("should error out and exit with status code 4", func() {
 					scanArgs := []string{scanUtils.ScanCommand, "--iac-version", "test"}
 					session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanArgs...)
-					helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeZero)
+					helper.ValidateExitCode(session, scanUtils.ScanTimeout, helper.ExitCodeFour)
 				})
 			})
 		})
@@ -276,9 +276,9 @@ var _ = Describe("Scan", func() {
 						It("should scan with the policies and exit with status code 0", func() {
 							scanArgs := []string{scanUtils.ScanCommand, "-p", validPolicyPath1, "-p", validPolicyPath2}
 							session = helper.RunCommandDir(terrascanBinaryPath, workDirPath, outWriter, errWriter, scanArgs...)
-							// exits with status code 0, because all iac scan should display results
-							// and the directory doesn't have iac files for violations
-							Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeZero))
+							// exits with status code 4, because there are no iac files for violations but
+							// would contain directory scan errors
+							Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeFour))
 						})
 					})
 

test/e2e/scan/scan_tf_files_test.go

@@ -128,7 +128,7 @@ var _ = Describe("Scan is run for terraform files", func() {
 				Context("when iac type is not specified and a directory is specified, it will be scanned will all iac providers", func() {
 					It("should display violations in json format, and should have iac type as 'all'", func() {
 						scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "json"}
-						scanUtils.RunScanAndAssertGoldenOutputRegex(terrascanBinaryPath, filepath.Join(tfAwsAmiGoldenRelPath, "aws_ami_violation_json_all.txt"), helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...)
+						scanUtils.RunScanAndAssertGoldenOutputRegex(terrascanBinaryPath, filepath.Join(tfAwsAmiGoldenRelPath, "aws_ami_violation_json_all.txt"), helper.ExitCodeFive, false, true, outWriter, errWriter, scanArgs...)
 					})
 				})
 			})
@@ -217,7 +217,7 @@ var _ = Describe("Scan is run for terraform files", func() {
 				It("should display violations in json format", func() {
 					iacDir := filepath.Join(iacRootRelPath, "terraform_recursive")
 					scanArgs := []string{"-i", "terraform", "-p", policyDir, "-d", iacDir, "-o", "json"}
-					scanUtils.RunScanAndAssertGoldenOutputRegex(terrascanBinaryPath, filepath.Join(tfAwsAmiGoldenRelPath, "aws_ami_violation_json_recursive.txt"), helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...)
+					scanUtils.RunScanAndAssertGoldenOutputRegex(terrascanBinaryPath, filepath.Join(tfAwsAmiGoldenRelPath, "aws_ami_violation_json_recursive.txt"), helper.ExitCodeFive, false, true, outWriter, errWriter, scanArgs...)
 				})
 			})
 		})

test/helper/helper.go

@@ -43,11 +43,15 @@ import (
 
 const (
 	// ExitCodeZero represents command exit code 0
-	ExitCodeZero = 0
-	// ExitCodeOne represents command exit code 0
-	ExitCodeOne = 1
-	// ExitCodeThree represents command exit code 0
-	ExitCodeThree = 3
+	ExitCodeZero = iota
+	// ExitCodeOne represents command exit code 1
+	ExitCodeOne
+	// ExitCodeThree represents command exit code 3
+	ExitCodeThree = iota + 1
+	// ExitCodeThree represents command exit code 4
+	ExitCodeFour
+	// ExitCodeThree represents command exit code 5
+	ExitCodeFive
 )
 
 var (