Skip to content

Commit

Permalink
probe name alignment, updated evaluation tests
Browse files Browse the repository at this point in the history 
Signed-off-by: Allen Shearin <allen.p.shearin@gmail.com>
  • Loading branch information
@ashearin
ashearin committed Apr 4, 2024
1 parent ed58add commit 037eb1a
Show file tree
Hide file tree
Showing 10 changed files with 244 additions and 28 deletions.
8 changes: 4 additions & 4 deletions checks/evaluation/sbom.go
View file
Expand Up @@ -22,7 +22,7 @@ import (
"github.com/ossf/scorecard/v4/finding"
"github.com/ossf/scorecard/v4/probes/sbomCICDArtifactExists"
"github.com/ossf/scorecard/v4/probes/sbomExists"
"github.com/ossf/scorecard/v4/probes/sbomReleaseArtifactExists"
"github.com/ossf/scorecard/v4/probes/sbomReleaseAssetExists"
"github.com/ossf/scorecard/v4/probes/sbomStandardsFileUsed"
)

Expand All @@ -34,7 +34,7 @@ func Sbom(name string,
// We have 4 unique probes, each should have a finding.
expectedProbes := []string{
sbomExists.Probe,
sbomReleaseArtifactExists.Probe,
sbomReleaseAssetExists.Probe,
sbomStandardsFileUsed.Probe,
sbomCICDArtifactExists.Probe,
}
Expand Down Expand Up @@ -76,7 +76,7 @@ func Sbom(name string,
Text: cicdMsg,
})
score += scoreProbeOnce(f.Probe, m, 3)
case sbomReleaseArtifactExists.Probe:
case sbomReleaseAssetExists.Probe:
dl.Info(&checker.LogMessage{
Type: finding.FileTypeURL,
Path: f.Message,
Expand Down Expand Up @@ -110,7 +110,7 @@ func Sbom(name string,
Text: "Sbom file not generated in project CICD",
})
cicdMsg = f.Message
case sbomReleaseArtifactExists.Probe:
case sbomReleaseAssetExists.Probe:
dl.Warn(&checker.LogMessage{
Type: finding.FileTypeURL,
Path: f.Message,
Expand Down
218 changes: 203 additions & 15 deletions checks/evaluation/sbom_test.go
View file
Expand Up @@ -29,59 +29,247 @@ func TestSbom(t *testing.T) {
result scut.TestReturn
}{
{
name: "Positive outcome = Max Score",
name: "Negative outcome = Min Score",
findings: []finding.Finding{
{
Probe: "sbomExists",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sbomReleaseAssetExists",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sbomStandardsFileUsed",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sbomCICDArtifactExists",
Outcome: finding.OutcomeNegative,
},
},
result: scut.TestReturn{
Score: checker.MinResultScore,
NumberOfInfo: 0,
NumberOfWarn: 4,
},
},
{
name: "Exists in Source: Positive outcome.",
findings: []finding.Finding{
{
Probe: "sbomExists",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomReleaseAssetExists",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sbomStandardsFileUsed",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sbomCICDArtifactExists",
Outcome: finding.OutcomeNegative,
},
},
result: scut.TestReturn{
Score: 3,
NumberOfInfo: 1,
NumberOfWarn: 3,
},
},
{
name: "Exists in Release Assets: Positive outcome.",
findings: []finding.Finding{
{
Probe: "hasSbomFile",
Probe: "sbomExists",
Outcome: finding.OutcomePositive,
},
{
Probe: "hasSbomReleaseArtifact",
Probe: "sbomReleaseAssetExists",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomStandardsFileUsed",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sbomCICDArtifactExists",
Outcome: finding.OutcomeNegative,
},
},
result: scut.TestReturn{
Score: checker.MaxResultScore,
Score: 6,
NumberOfInfo: 2,
NumberOfWarn: 2,
},
}, {
name: "Negative outcomes from all probes = Min score",
},
{
name: "Exists in Standards File: Positive outcome.",
findings: []finding.Finding{
{
Probe: "hasLicenseFile",
Probe: "sbomExists",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomReleaseAssetExists",
Outcome: finding.OutcomeNegative,
},
{
Probe: "hasSbomReleaseArtifact",
Probe: "sbomStandardsFileUsed",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomCICDArtifactExists",
Outcome: finding.OutcomeNegative,
},
},
result: scut.TestReturn{
Score: checker.MinResultScore,
Score: 4,
NumberOfInfo: 2,
NumberOfWarn: 2,
},
}, {
name: "Has license file but not a top level or in OSI/FSF format",
},
{
name: "Exists in CICD Artifacts: Positive outcome.",
findings: []finding.Finding{
{
Probe: "hasLicenseFile",
Probe: "sbomExists",
Outcome: finding.OutcomePositive,
},
{
Probe: "hasSbomReleaseArtifact",
Probe: "sbomReleaseAssetExists",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sbomStandardsFileUsed",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sbomCICDArtifactExists",
Outcome: finding.OutcomePositive,
},
},
result: scut.TestReturn{
Score: 3,
Score: 6,
NumberOfInfo: 2,
NumberOfWarn: 2,
},
},
{
name: "Exists in Release Assets and Standards File: Positive outcome.",
findings: []finding.Finding{
{
Probe: "sbomExists",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomReleaseAssetExists",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomStandardsFileUsed",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomCICDArtifactExists",
Outcome: finding.OutcomeNegative,
},
},
result: scut.TestReturn{
Score: 7,
NumberOfInfo: 3,
NumberOfWarn: 1,
},
},
{
name: "Exists in Release Assets and CICD Artifacts: Positive outcome.",
findings: []finding.Finding{
{
Probe: "sbomExists",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomReleaseAssetExists",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomStandardsFileUsed",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sbomCICDArtifactExists",
Outcome: finding.OutcomePositive,
},
},
result: scut.TestReturn{
Score: 9,
NumberOfInfo: 3,
NumberOfWarn: 1,
},
},
{
name: "Exists in CICD Artifacts and Standards File: Positive outcome.",
findings: []finding.Finding{
{
Probe: "sbomExists",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomReleaseAssetExists",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sbomStandardsFileUsed",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomCICDArtifactExists",
Outcome: finding.OutcomePositive,
},
},
result: scut.TestReturn{
Score: 7,
NumberOfInfo: 3,
NumberOfWarn: 1,
},
},
{
name: "Positive outcome = Max Score",
findings: []finding.Finding{
{
Probe: "sbomExists",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomReleaseAssetExists",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomStandardsFileUsed",
Outcome: finding.OutcomePositive,
},
{
Probe: "sbomCICDArtifactExists",
Outcome: finding.OutcomePositive,
},
},
result: scut.TestReturn{
Score: checker.MaxResultScore,
NumberOfInfo: 4,
NumberOfWarn: 0,
},
},
}
for _, tt := range tests {
tt := tt // Parallel testing scoping hack.
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
dl := scut.TestDetailLogger{}
got := Sbom(tt.name, tt.findings, &dl)
scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
})
}
}
6 changes: 3 additions & 3 deletions checks/webhook_test.go
View file
Expand Up @@ -38,7 +38,7 @@ func TestWebhooks(t *testing.T) {
name: "No Webhooks",
uri: "github.com/owner/repo",
expected: checker.CheckResult{
Score: 10,
Score: checker.MaxResultScore,
},
err: nil,
webhooks: []clients.Webhook{},
Expand All @@ -47,7 +47,7 @@ func TestWebhooks(t *testing.T) {
name: "With Webhooks and secret set",
uri: "github.com/owner/repo",
expected: checker.CheckResult{
Score: 10,
Score: checker.MaxResultScore,
},
err: nil,
webhooks: []clients.Webhook{
Expand All @@ -61,7 +61,7 @@ func TestWebhooks(t *testing.T) {
name: "With Webhooks and no secret set",
uri: "github.com/owner/repo",
expected: checker.CheckResult{
Score: 0,
Score: checker.MinResultScore,
},
err: nil,
webhooks: []clients.Webhook{
Expand Down
4 changes: 2 additions & 2 deletions probes/entries.go
View file
Expand Up @@ -55,7 +55,7 @@ import (
"github.com/ossf/scorecard/v4/probes/sastToolRunsOnAllCommits"
"github.com/ossf/scorecard/v4/probes/sbomCICDArtifactExists"
"github.com/ossf/scorecard/v4/probes/sbomExists"
"github.com/ossf/scorecard/v4/probes/sbomReleaseArtifactExists"
"github.com/ossf/scorecard/v4/probes/sbomReleaseAssetExists"
"github.com/ossf/scorecard/v4/probes/sbomStandardsFileUsed"
"github.com/ossf/scorecard/v4/probes/securityPolicyContainsLinks"
"github.com/ossf/scorecard/v4/probes/securityPolicyContainsText"
Expand Down Expand Up @@ -133,7 +133,7 @@ var (
}
Sbom = []ProbeImpl{
sbomExists.Run,
sbomReleaseArtifactExists.Run,
sbomReleaseAssetExists.Run,
sbomStandardsFileUsed.Run,
sbomCICDArtifactExists.Run,
}
Expand Down
7 changes: 7 additions & 0 deletions probes/sbomCICDArtifactExists/impl_test.go
View file
Expand Up @@ -23,6 +23,7 @@ import (

"github.com/ossf/scorecard/v4/checker"
"github.com/ossf/scorecard/v4/finding"
"github.com/ossf/scorecard/v4/probes/internal/utils/uerror"
)

func Test_Run(t *testing.T) {
Expand Down Expand Up @@ -96,6 +97,12 @@ func Test_Run(t *testing.T) {
finding.OutcomeNegative,
},
},
{
name: "no raw data",
raw: nil,
err: uerror.ErrNil,
outcomes: nil,
},
}
for _, tt := range tests {
tt := tt // Re-initializing variable so it is not changed while executing the closure below
Expand Down
7 changes: 7 additions & 0 deletions probes/sbomExists/impl_test.go
View file
Expand Up @@ -23,6 +23,7 @@ import (

"github.com/ossf/scorecard/v4/checker"
"github.com/ossf/scorecard/v4/finding"
"github.com/ossf/scorecard/v4/probes/internal/utils/uerror"
)

func Test_Run(t *testing.T) {
Expand Down Expand Up @@ -73,6 +74,12 @@ func Test_Run(t *testing.T) {
finding.OutcomeNegative,
},
},
{
name: "no raw data",
raw: nil,
err: uerror.ErrNil,
outcomes: nil,
},
}
for _, tt := range tests {
tt := tt // Re-initializing variable so it is not changed while executing the closure below
Expand Down
2 changes: 1 addition & 1 deletion probes/sbomReleaseArtifactExists/def.yml → probes/sbomReleaseAssetExists/def.yml
View file
Expand Up @@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

id: sbomReleaseArtifactExists
id: sbomReleaseAssetExists
short: Check that the project publishes an sbom as part of its release artifact.
motivation: >
blurb about importance of sboms.
Expand Down

0 comments on commit 037eb1a

Please sign in to comment.