diff --git a/checks/branch_protection_test.go b/checks/branch_protection_test.go index 81f5ddcdb0b..c1e46fca0fe 100644 --- a/checks/branch_protection_test.go +++ b/checks/branch_protection_test.go @@ -90,7 +90,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) { { name: "Only development branch", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 1, NumberOfWarn: 6, NumberOfInfo: 2, @@ -138,7 +138,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) { { name: "Take worst of release and development", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 5, NumberOfWarn: 8, NumberOfInfo: 9, @@ -219,7 +219,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) { { name: "Both release and development are OK", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 9, NumberOfWarn: 4, NumberOfInfo: 14, @@ -300,7 +300,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) { { name: "Ignore a non-branch targetcommitish", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 1, NumberOfWarn: 6, NumberOfInfo: 2, @@ -348,7 +348,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) { { name: "TargetCommittish nil", expected: scut.TestReturn{ - Errors: []error{sce.ErrScorecardInternal}, + Error: sce.ErrScorecardInternal, Score: checker.InconclusiveResultScore, NumberOfWarn: 0, NumberOfInfo: 0, @@ -396,7 +396,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) { { name: "Non-admin check with protected release and development", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 1, NumberOfWarn: 2, NumberOfInfo: 0, @@ -442,7 +442,9 @@ func TestReleaseAndDevBranchProtected(t *testing.T) { dl := scut.TestDetailLogger{} r := checkReleaseAndDevBranchProtection(context.Background(), mockRepoClient, m, &dl, "testowner", "testrepo") - scut.ValidateTestReturn(t, tt.name, &tt.expected, &r, &dl) + if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &r, &dl) { + t.Fail() + } ctrl.Finish() }) } @@ -459,7 +461,7 @@ func TestIsBranchProtected(t *testing.T) { { name: "Nothing is enabled", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 1, NumberOfWarn: 6, NumberOfInfo: 2, @@ -502,7 +504,7 @@ func TestIsBranchProtected(t *testing.T) { { name: "Nothing is enabled and values in github.Protection are nil", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 1, NumberOfWarn: 4, NumberOfInfo: 2, @@ -513,7 +515,7 @@ func TestIsBranchProtected(t *testing.T) { { name: "Required status check enabled", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 2, NumberOfWarn: 6, NumberOfInfo: 3, @@ -556,7 +558,7 @@ func TestIsBranchProtected(t *testing.T) { { name: "Required status check enabled without checking for status string", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 2, NumberOfWarn: 6, NumberOfInfo: 3, @@ -599,7 +601,7 @@ func TestIsBranchProtected(t *testing.T) { { name: "Required pull request enabled", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 2, NumberOfWarn: 5, NumberOfInfo: 3, @@ -642,7 +644,7 @@ func TestIsBranchProtected(t *testing.T) { { name: "Required admin enforcement enabled", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 3, NumberOfWarn: 5, NumberOfInfo: 3, @@ -685,7 +687,7 @@ func TestIsBranchProtected(t *testing.T) { { name: "Required linear history enabled", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 2, NumberOfWarn: 5, NumberOfInfo: 3, @@ -728,7 +730,7 @@ func TestIsBranchProtected(t *testing.T) { { name: "Allow force push enabled", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 0, NumberOfWarn: 7, NumberOfInfo: 1, @@ -771,7 +773,7 @@ func TestIsBranchProtected(t *testing.T) { { name: "Allow deletions enabled", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 0, NumberOfWarn: 7, NumberOfInfo: 1, @@ -814,7 +816,7 @@ func TestIsBranchProtected(t *testing.T) { { name: "Branches are protected", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: 9, NumberOfWarn: 2, NumberOfInfo: 7, @@ -860,8 +862,12 @@ func TestIsBranchProtected(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} - score := IsBranchProtected(tt.protection, "test", &dl) - scut.ValidateTestValues(t, tt.name, &tt.expected, score, nil, &dl) + actual := &checker.CheckResult{ + Score: IsBranchProtected(tt.protection, "test", &dl), + } + if !scut.ValidateTestReturn(t, tt.name, &tt.expected, actual, &dl) { + t.Fail() + } }) } } diff --git a/checks/permissions_test.go b/checks/permissions_test.go index 2e28635c57f..f20c2b0f065 100644 --- a/checks/permissions_test.go +++ b/checks/permissions_test.go @@ -36,7 +36,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "run workflow codeql write test", filename: "./testdata/github-workflow-permissions-run-codeql-write.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -47,7 +47,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "run workflow no codeql write test", filename: "./testdata/github-workflow-permissions-run-no-codeql-write.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore - 1, NumberOfWarn: 1, NumberOfInfo: 1, @@ -58,7 +58,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "run workflow write test", filename: "./testdata/github-workflow-permissions-run-writes-2.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 3, NumberOfInfo: 2, @@ -69,7 +69,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "run package workflow write test", filename: "./testdata/github-workflow-permissions-run-package-workflow-write.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 2, @@ -80,7 +80,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "run package write test", filename: "./testdata/github-workflow-permissions-run-package-write.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 1, @@ -91,7 +91,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "run writes test", filename: "./testdata/github-workflow-permissions-run-writes.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -102,7 +102,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "write all test", filename: "./testdata/github-workflow-permissions-writeall.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 0, @@ -113,7 +113,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "read all test", filename: "./testdata/github-workflow-permissions-readall.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -124,7 +124,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "no permission test", filename: "./testdata/github-workflow-permissions-absent.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 0, @@ -135,7 +135,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "writes test", filename: "./testdata/github-workflow-permissions-writes.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -146,7 +146,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "reads test", filename: "./testdata/github-workflow-permissions-reads.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 10, @@ -157,7 +157,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "nones test", filename: "./testdata/github-workflow-permissions-nones.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 10, @@ -168,7 +168,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "none test", filename: "./testdata/github-workflow-permissions-none.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -179,7 +179,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "status/checks write", filename: "./testdata/github-workflow-permissions-status-checks.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore - 1, NumberOfWarn: 2, NumberOfInfo: 2, @@ -190,7 +190,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "sec-events/deployments write", filename: "./testdata/github-workflow-permissions-secevent-deployments.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore - 2, NumberOfWarn: 2, NumberOfInfo: 3, @@ -201,7 +201,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "contents write", filename: "./testdata/github-workflow-permissions-contents.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 2, @@ -212,7 +212,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "actions write", filename: "./testdata/github-workflow-permissions-actions.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 2, @@ -223,7 +223,7 @@ func TestGithubTokenPermissions(t *testing.T) { name: "packages write", filename: "./testdata/github-workflow-permissions-packages.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 1, diff --git a/checks/pinned_dependencies_test.go b/checks/pinned_dependencies_test.go index 1c7cae0d31e..4fe7145fec6 100644 --- a/checks/pinned_dependencies_test.go +++ b/checks/pinned_dependencies_test.go @@ -36,7 +36,7 @@ func TestGithubWorkflowPinning(t *testing.T) { name: "empty file", filename: "./testdata/github-workflow-empty", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -47,7 +47,7 @@ func TestGithubWorkflowPinning(t *testing.T) { name: "comments only", filename: "./testdata/github-workflow-comments", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -58,7 +58,7 @@ func TestGithubWorkflowPinning(t *testing.T) { name: "Pinned workflow", filename: "./testdata/workflow-pinned.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -69,7 +69,7 @@ func TestGithubWorkflowPinning(t *testing.T) { name: "Non-pinned workflow", filename: "./testdata/workflow-not-pinned.yaml", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 0, @@ -93,7 +93,13 @@ func TestGithubWorkflowPinning(t *testing.T) { } dl := scut.TestDetailLogger{} s, e := testIsGitHubActionsWorkflowPinned(tt.filename, content, &dl) - scut.ValidateTestValues(t, tt.name, &tt.expected, s, e, &dl) + actual := checker.CheckResult{ + Score: s, + Error2: e, + } + if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl) { + t.Fail() + } }) } } @@ -109,7 +115,7 @@ func TestDockerfilePinning(t *testing.T) { name: "invalid dockerfile", filename: "./testdata/Dockerfile-invalid", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -120,7 +126,7 @@ func TestDockerfilePinning(t *testing.T) { name: "invalid dockerfile sh", filename: "./testdata/script-sh", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -131,7 +137,7 @@ func TestDockerfilePinning(t *testing.T) { name: "empty file", filename: "./testdata/Dockerfile-empty", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -142,7 +148,7 @@ func TestDockerfilePinning(t *testing.T) { name: "comments only", filename: "./testdata/Dockerfile-comments", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -153,7 +159,7 @@ func TestDockerfilePinning(t *testing.T) { name: "Pinned dockerfile", filename: "./testdata/Dockerfile-pinned", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -164,7 +170,7 @@ func TestDockerfilePinning(t *testing.T) { name: "Pinned dockerfile as", filename: "./testdata/Dockerfile-pinned-as", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -175,7 +181,7 @@ func TestDockerfilePinning(t *testing.T) { name: "Non-pinned dockerfile as", filename: "./testdata/Dockerfile-not-pinned-as", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 3, // TODO: should be 2, https://github.com/ossf/scorecard/issues/701. NumberOfInfo: 0, @@ -186,7 +192,7 @@ func TestDockerfilePinning(t *testing.T) { name: "Non-pinned dockerfile", filename: "./testdata/Dockerfile-not-pinned", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 0, @@ -210,7 +216,13 @@ func TestDockerfilePinning(t *testing.T) { } dl := scut.TestDetailLogger{} s, e := testValidateDockerfileIsPinned(tt.filename, content, &dl) - scut.ValidateTestValues(t, tt.name, &tt.expected, s, e, &dl) + actual := checker.CheckResult{ + Score: s, + Error2: e, + } + if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl) { + t.Fail() + } }) } } @@ -226,7 +238,7 @@ func TestDockerfileScriptDownload(t *testing.T) { name: "curl | sh", filename: "testdata/Dockerfile-curl-sh", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 4, NumberOfInfo: 0, @@ -237,7 +249,7 @@ func TestDockerfileScriptDownload(t *testing.T) { name: "empty file", filename: "./testdata/Dockerfile-empty", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -248,7 +260,7 @@ func TestDockerfileScriptDownload(t *testing.T) { name: "invalid file sh", filename: "./testdata/script.sh", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -259,7 +271,7 @@ func TestDockerfileScriptDownload(t *testing.T) { name: "comments only", filename: "./testdata/Dockerfile-comments", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -270,7 +282,7 @@ func TestDockerfileScriptDownload(t *testing.T) { name: "wget | /bin/sh", filename: "testdata/Dockerfile-wget-bin-sh", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 3, NumberOfInfo: 0, @@ -281,7 +293,7 @@ func TestDockerfileScriptDownload(t *testing.T) { name: "wget no exec", filename: "testdata/Dockerfile-script-ok", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -292,7 +304,7 @@ func TestDockerfileScriptDownload(t *testing.T) { name: "curl file sh", filename: "testdata/Dockerfile-curl-file-sh", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 12, NumberOfInfo: 0, @@ -303,7 +315,7 @@ func TestDockerfileScriptDownload(t *testing.T) { name: "proc substitution", filename: "testdata/Dockerfile-proc-subs", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 6, NumberOfInfo: 0, @@ -314,7 +326,7 @@ func TestDockerfileScriptDownload(t *testing.T) { name: "wget file", filename: "testdata/Dockerfile-wget-file", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 10, NumberOfInfo: 0, @@ -325,7 +337,7 @@ func TestDockerfileScriptDownload(t *testing.T) { name: "gsutil file", filename: "testdata/Dockerfile-gsutil-file", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 17, NumberOfInfo: 0, @@ -336,7 +348,7 @@ func TestDockerfileScriptDownload(t *testing.T) { name: "aws file", filename: "testdata/Dockerfile-aws-file", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 15, NumberOfInfo: 0, @@ -347,7 +359,7 @@ func TestDockerfileScriptDownload(t *testing.T) { name: "pkg managers", filename: "testdata/Dockerfile-pkg-managers", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 27, NumberOfInfo: 0, @@ -358,7 +370,7 @@ func TestDockerfileScriptDownload(t *testing.T) { name: "download with some python", filename: "testdata/Dockerfile-some-python", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 0, @@ -382,7 +394,13 @@ func TestDockerfileScriptDownload(t *testing.T) { } dl := scut.TestDetailLogger{} s, e := testValidateDockerfileIsFreeOfInsecureDownloads(tt.filename, content, &dl) - scut.ValidateTestValues(t, tt.name, &tt.expected, s, e, &dl) + actual := checker.CheckResult{ + Score: s, + Error2: e, + } + if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl) { + t.Fail() + } }) } } @@ -398,7 +416,7 @@ func TestShellScriptDownload(t *testing.T) { name: "sh script", filename: "testdata/script-sh", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 7, NumberOfInfo: 0, @@ -409,7 +427,7 @@ func TestShellScriptDownload(t *testing.T) { name: "empty file", filename: "./testdata/script-empty.sh", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -420,7 +438,7 @@ func TestShellScriptDownload(t *testing.T) { name: "comments", filename: "./testdata/script-comments.sh", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -431,7 +449,7 @@ func TestShellScriptDownload(t *testing.T) { name: "bash script", filename: "testdata/script-bash", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 7, NumberOfInfo: 0, @@ -442,7 +460,7 @@ func TestShellScriptDownload(t *testing.T) { name: "sh script 2", filename: "testdata/script.sh", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 7, NumberOfInfo: 0, @@ -453,7 +471,7 @@ func TestShellScriptDownload(t *testing.T) { name: "pkg managers", filename: "testdata/script-pkg-managers", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 24, NumberOfInfo: 0, @@ -477,7 +495,13 @@ func TestShellScriptDownload(t *testing.T) { } dl := scut.TestDetailLogger{} s, e := testValidateShellScriptIsFreeOfInsecureDownloads(tt.filename, content, &dl) - scut.ValidateTestValues(t, tt.name, &tt.expected, s, e, &dl) + actual := checker.CheckResult{ + Score: s, + Error2: e, + } + if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl) { + t.Fail() + } }) } } @@ -493,7 +517,7 @@ func TestGitHubWorflowRunDownload(t *testing.T) { name: "workflow curl default", filename: "testdata/github-workflow-curl-default", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 0, @@ -504,7 +528,7 @@ func TestGitHubWorflowRunDownload(t *testing.T) { name: "workflow curl no default", filename: "testdata/github-workflow-curl-no-default", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 0, @@ -515,7 +539,7 @@ func TestGitHubWorflowRunDownload(t *testing.T) { name: "wget across steps", filename: "testdata/github-workflow-wget-across-steps", expected: scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 2, NumberOfInfo: 0, @@ -539,7 +563,13 @@ func TestGitHubWorflowRunDownload(t *testing.T) { } dl := scut.TestDetailLogger{} s, e := testValidateGitHubWorkflowScriptFreeOfInsecureDownloads(tt.filename, content, &dl) - scut.ValidateTestValues(t, tt.name, &tt.expected, s, e, &dl) + actual := checker.CheckResult{ + Score: s, + Error2: e, + } + if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl) { + t.Fail() + } }) } } diff --git a/e2e/binary_artifacts_test.go b/e2e/binary_artifacts_test.go index ab525b5ab22..dd191387ada 100644 --- a/e2e/binary_artifacts_test.go +++ b/e2e/binary_artifacts_test.go @@ -47,7 +47,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 0, @@ -79,7 +79,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { } // TODO: upload real binaries to the repo as well. expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 35, NumberOfInfo: 0, diff --git a/e2e/branch_protection_test.go b/e2e/branch_protection_test.go index b6a5ddce35c..c39b10402c3 100644 --- a/e2e/branch_protection_test.go +++ b/e2e/branch_protection_test.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// nolint: dupl package e2e import ( @@ -22,6 +23,7 @@ import ( "github.com/ossf/scorecard/v2/checker" "github.com/ossf/scorecard/v2/checks" + "github.com/ossf/scorecard/v2/clients/githubrepo" scut "github.com/ossf/scorecard/v2/utests" ) @@ -29,18 +31,21 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() { Context("E2E TEST:Validating branch protection", func() { It("Should fail to return branch protection on other repositories", func() { dl := scut.TestDetailLogger{} + repoClient := githubrepo.CreateGithubRepoClient(context.Background(), ghClient, graphClient) + err := repoClient.InitRepo("apache", "airflow") + Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), Client: ghClient, HTTPClient: httpClient, - RepoClient: nil, + RepoClient: repoClient, Owner: "apache", Repo: "airflow", GraphClient: graphClient, Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: 1, NumberOfWarn: 3, NumberOfInfo: 0, @@ -57,18 +62,21 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() { Context("E2E TEST:Validating branch protection", func() { It("Should fail to return branch protection on other repositories", func() { dl := scut.TestDetailLogger{} + repoClient := githubrepo.CreateGithubRepoClient(context.Background(), ghClient, graphClient) + err := repoClient.InitRepo("ossf-tests", "scorecard-check-branch-protection-e2e") + Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), Client: ghClient, HTTPClient: httpClient, - RepoClient: nil, + RepoClient: repoClient, Owner: "ossf-tests", Repo: "scorecard-check-branch-protection-e2e", GraphClient: graphClient, Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: 9, NumberOfWarn: 1, NumberOfInfo: 8, diff --git a/e2e/ci_tests_test.go b/e2e/ci_tests_test.go index 15bb2a28525..14b29de9902 100644 --- a/e2e/ci_tests_test.go +++ b/e2e/ci_tests_test.go @@ -45,11 +45,11 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 0, - NumberOfDebug: 24, + NumberOfDebug: 30, } result := checks.CITests(&req) // UPGRADEv2: to remove. diff --git a/e2e/cii_best_practices_test.go b/e2e/cii_best_practices_test.go index 0b9a384dcc3..1100497814c 100644 --- a/e2e/cii_best_practices_test.go +++ b/e2e/cii_best_practices_test.go @@ -40,7 +40,7 @@ var _ = Describe("E2E TEST:CIIBestPractices", func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: 5, NumberOfWarn: 0, NumberOfInfo: 0, diff --git a/e2e/code_review_test.go b/e2e/code_review_test.go index 2bab9fe1dbe..4d82a4b259c 100644 --- a/e2e/code_review_test.go +++ b/e2e/code_review_test.go @@ -48,7 +48,7 @@ var _ = Describe("E2E TEST:CodeReview", func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 2, diff --git a/e2e/contributors_test.go b/e2e/contributors_test.go index ecf12cba047..3b52dd7abfd 100644 --- a/e2e/contributors_test.go +++ b/e2e/contributors_test.go @@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckContributors, func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -75,7 +75,7 @@ var _ = Describe("E2E TEST:"+checks.CheckContributors, func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, diff --git a/e2e/dependency_update_tool_test.go b/e2e/dependency_update_tool_test.go index 229b59dafa6..6745dcd63b9 100644 --- a/e2e/dependency_update_tool_test.go +++ b/e2e/dependency_update_tool_test.go @@ -47,7 +47,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -78,7 +78,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, diff --git a/e2e/fuzzing_test.go b/e2e/fuzzing_test.go index 7d25a47aa54..25983723f82 100644 --- a/e2e/fuzzing_test.go +++ b/e2e/fuzzing_test.go @@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 0, diff --git a/e2e/maintained_test.go b/e2e/maintained_test.go index b76343d1e33..a7a270eb17a 100644 --- a/e2e/maintained_test.go +++ b/e2e/maintained_test.go @@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckMaintained, func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 0, diff --git a/e2e/packaging_test.go b/e2e/packaging_test.go index c5c4223b7b8..d7383dd6d06 100644 --- a/e2e/packaging_test.go +++ b/e2e/packaging_test.go @@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPackaging, func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 2, diff --git a/e2e/permissions_test.go b/e2e/permissions_test.go index 40af7a00d1b..fe23bbf146b 100644 --- a/e2e/permissions_test.go +++ b/e2e/permissions_test.go @@ -44,7 +44,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 2, diff --git a/e2e/pinned_dependencies_test.go b/e2e/pinned_dependencies_test.go index 829d0c8a542..9cd4d79fb3a 100644 --- a/e2e/pinned_dependencies_test.go +++ b/e2e/pinned_dependencies_test.go @@ -47,7 +47,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 154, NumberOfInfo: 0, diff --git a/e2e/sast_test.go b/e2e/sast_test.go index 9eac8c9fa63..c75623c45c7 100644 --- a/e2e/sast_test.go +++ b/e2e/sast_test.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// nolint: dupl package e2e import ( @@ -44,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSAST, func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: 7, NumberOfWarn: 1, NumberOfInfo: 1, diff --git a/e2e/security_policy_test.go b/e2e/security_policy_test.go index 2e9479ed085..d2e73b395a6 100644 --- a/e2e/security_policy_test.go +++ b/e2e/security_policy_test.go @@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:SecurityPolicy", func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, @@ -76,7 +76,7 @@ var _ = Describe("E2E TEST:SecurityPolicy", func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 1, diff --git a/e2e/signedreleases_test.go b/e2e/signedreleases_test.go index 7fc0c208e63..cd840456cbf 100644 --- a/e2e/signedreleases_test.go +++ b/e2e/signedreleases_test.go @@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSignedReleases, func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 5, diff --git a/e2e/vulnerabilities_test.go b/e2e/vulnerabilities_test.go index ab4bd154cb9..71b69dbf69b 100644 --- a/e2e/vulnerabilities_test.go +++ b/e2e/vulnerabilities_test.go @@ -46,7 +46,7 @@ var _ = Describe("E2E TEST:Vulnerabilities", func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, NumberOfInfo: 0, @@ -78,7 +78,7 @@ var _ = Describe("E2E TEST:Vulnerabilities", func() { Dlogger: &dl, } expected := scut.TestReturn{ - Errors: nil, + Error: nil, Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 0, diff --git a/utests/utlib.go b/utests/utlib.go index 6d882548d70..e131d12925e 100644 --- a/utests/utlib.go +++ b/utests/utlib.go @@ -18,46 +18,28 @@ package utests import ( "errors" "fmt" + "log" "testing" + "github.com/google/go-cmp/cmp" + "github.com/ossf/scorecard/v2/checker" ) -func validateDetailTypes(messages []checker.CheckDetail, nw, ni, nd int) bool { - enw := 0 - eni := 0 - end := 0 - for _, v := range messages { - switch v.Type { - default: - panic(fmt.Sprintf("invalid type %v", v.Type)) - case checker.DetailInfo: - eni++ - case checker.DetailDebug: - end++ - case checker.DetailWarn: - enw++ - } - } - return enw == nw && - eni == ni && - end == nd -} - -// TestDetailLogger implements `checker.DetailLogger`. -type TestDetailLogger struct { - messages []checker.CheckDetail -} - // TestReturn encapsulates expected CheckResult return values. type TestReturn struct { - Errors []error + Error error Score int NumberOfWarn int NumberOfInfo int NumberOfDebug int } +// TestDetailLogger implements `checker.DetailLogger`. +type TestDetailLogger struct { + messages []checker.CheckDetail +} + // Info implements DetailLogger.Info. func (l *TestDetailLogger) Info(desc string, args ...interface{}) { cd := checker.CheckDetail{Type: checker.DetailInfo, Msg: checker.LogMessage{Text: fmt.Sprintf(desc, args...)}} @@ -77,7 +59,8 @@ func (l *TestDetailLogger) Debug(desc string, args ...interface{}) { } // UPGRADEv3: to rename. -//nolint:revive + +// Info3 implements DetailLogger.Info3. func (l *TestDetailLogger) Info3(msg *checker.LogMessage) { cd := checker.CheckDetail{ Type: checker.DetailInfo, @@ -87,7 +70,7 @@ func (l *TestDetailLogger) Info3(msg *checker.LogMessage) { l.messages = append(l.messages, cd) } -//nolint:revive +// Warn3 implements DetailLogger.Warn3. func (l *TestDetailLogger) Warn3(msg *checker.LogMessage) { cd := checker.CheckDetail{ Type: checker.DetailWarn, @@ -97,7 +80,7 @@ func (l *TestDetailLogger) Warn3(msg *checker.LogMessage) { l.messages = append(l.messages, cd) } -//nolint:revive +// Debug3 implements DetailLogger.Debug3. func (l *TestDetailLogger) Debug3(msg *checker.LogMessage) { cd := checker.CheckDetail{ Type: checker.DetailDebug, @@ -107,38 +90,43 @@ func (l *TestDetailLogger) Debug3(msg *checker.LogMessage) { l.messages = append(l.messages, cd) } -// ValidateTestValues validates returned score and log values. -// nolint: thelper -func ValidateTestValues(t *testing.T, name string, te *TestReturn, - score int, err error, dl *TestDetailLogger) bool { - for _, we := range te.Errors { - if !errors.Is(err, we) { - if t != nil { - t.Errorf("%v: invalid error returned: %v is not of type %v", - name, err, we) - } - fmt.Printf("%v: invalid error returned: %v is not of type %v", - name, err, we) - return false - } - } - if score != te.Score || - !validateDetailTypes(dl.messages, te.NumberOfWarn, - te.NumberOfInfo, te.NumberOfDebug) { - if t != nil { - t.Errorf("%v: Got (score=%v) expected (%v)\n%v", - name, score, te.Score, dl.messages) +func getTestReturn(cr *checker.CheckResult, logger *TestDetailLogger) (*TestReturn, error) { + ret := new(TestReturn) + for _, v := range logger.messages { + switch v.Type { + default: + // nolint: goerr113 + return nil, fmt.Errorf("invalid type %v", v.Type) + case checker.DetailInfo: + ret.NumberOfInfo++ + case checker.DetailDebug: + ret.NumberOfDebug++ + case checker.DetailWarn: + ret.NumberOfWarn++ } - return false } - return true + ret.Score = cr.Score + ret.Error = cr.Error + return ret, nil +} + +func errCmp(e1, e2 error) bool { + return errors.Is(e1, e2) || errors.Is(e2, e1) } // ValidateTestReturn validates expected TestReturn with actual checker.CheckResult values. // nolint: thelper -func ValidateTestReturn(t *testing.T, name string, te *TestReturn, - tr *checker.CheckResult, dl *TestDetailLogger) bool { - return ValidateTestValues(t, name, te, tr.Score, tr.Error2, dl) +func ValidateTestReturn(t *testing.T, name string, expected *TestReturn, + actual *checker.CheckResult, logger *TestDetailLogger) bool { + actualTestReturn, err := getTestReturn(actual, logger) + if err != nil { + panic(err) + } + if !cmp.Equal(*actualTestReturn, *expected, cmp.Comparer(errCmp)) { + log.Println(cmp.Diff(*actualTestReturn, *expected)) + return false + } + return true } // ValidateLogMessage tests that at least one log message returns true for isExpectedMessage.