diff --git a/pkg/policies/engine/engine.go b/pkg/policies/engine/engine.go index dd61453ee..294ef051b 100644 --- a/pkg/policies/engine/engine.go +++ b/pkg/policies/engine/engine.go @@ -29,6 +29,7 @@ type EvaluationResult struct { Violations []*PolicyViolation Skipped bool SkipReason string + Ignore bool } // PolicyViolation represents a policy failure diff --git a/pkg/policies/engine/rego/rego.go b/pkg/policies/engine/rego/rego.go index a8395b592..7845b229f 100644 --- a/pkg/policies/engine/rego/rego.go +++ b/pkg/policies/engine/rego/rego.go @@ -166,6 +166,7 @@ func parseViolationsRule(res rego.ResultSet, policy *engine.Policy) (*engine.Eva Violations: violations, Skipped: false, // best effort SkipReason: "", + Ignore: false, // Assume old rules should not be ignored }, nil } @@ -189,6 +190,11 @@ func parseResultRule(res rego.ResultSet, policy *engine.Policy) (*engine.Evaluat reason = val } + var ignore bool + if val, ok := ruleResult["ignore"].(bool); ok { + ignore = val + } + violations, ok := ruleResult["violations"].([]any) if !ok { return nil, engine.ResultFormatError{Field: "violations"} @@ -196,6 +202,7 @@ func parseResultRule(res rego.ResultSet, policy *engine.Policy) (*engine.Evaluat result.Skipped = skipped result.SkipReason = reason + result.Ignore = ignore for _, violation := range violations { vs, ok := violation.(string) diff --git a/pkg/policies/engine/rego/rego_test.go b/pkg/policies/engine/rego/rego_test.go index 2ee748de8..948db9037 100644 --- a/pkg/policies/engine/rego/rego_test.go +++ b/pkg/policies/engine/rego/rego_test.go @@ -193,6 +193,7 @@ func TestRego_ResultFormat(t *testing.T) { require.NoError(t, err) assert.True(t, result.Skipped) assert.Equal(t, "invalid input", result.SkipReason) + assert.False(t, result.Ignore) }) t.Run("valid input, no violations", func(t *testing.T) { @@ -200,6 +201,7 @@ func TestRego_ResultFormat(t *testing.T) { require.NoError(t, err) assert.False(t, result.Skipped) assert.Len(t, result.Violations, 0) + assert.False(t, result.Ignore) }) t.Run("valid input, violations", func(t *testing.T) { @@ -208,6 +210,33 @@ func TestRego_ResultFormat(t *testing.T) { assert.False(t, result.Skipped) assert.Len(t, result.Violations, 1) assert.Equal(t, "wrong CycloneDX version. Expected 1.5, but it was 1.4", result.Violations[0].Violation) + assert.False(t, result.Ignore) + }) + + t.Run("valid input, not ignore", func(t *testing.T) { + result, err := r.Verify(context.TODO(), policy, []byte("{\"specVersion\": \"1.0\"}"), nil) + require.NoError(t, err) + assert.False(t, result.Skipped) + assert.Len(t, result.Violations, 1) + assert.Equal(t, "wrong CycloneDX version. Expected 1.5, but it was 1.0", result.Violations[0].Violation) + assert.True(t, result.Ignore) + }) +} + +func TestRego_ResultFormatWithoutIgnoreValue(t *testing.T) { + regoContent, err := os.ReadFile("testfiles/result_format_without_ignore.rego") + require.NoError(t, err) + + r := &Rego{} + policy := &engine.Policy{ + Name: "result-output", + Source: regoContent, + } + + t.Run("by default return always ignore to false", func(t *testing.T) { + result, err := r.Verify(context.TODO(), policy, []byte("{\"foo\": \"bar\"}"), nil) + require.NoError(t, err) + assert.False(t, result.Ignore) }) } diff --git a/pkg/policies/engine/rego/testfiles/result_format.rego b/pkg/policies/engine/rego/testfiles/result_format.rego index 89dd3cbd7..2cb5484fb 100644 --- a/pkg/policies/engine/rego/testfiles/result_format.rego +++ b/pkg/policies/engine/rego/testfiles/result_format.rego @@ -6,6 +6,7 @@ result := { "skipped": skipped, "violations": violations, "skip_reason": skip_reason, + "ignore": ignore, } default skip_reason := "" @@ -17,8 +18,14 @@ skip_reason := m if { default skipped := true +default ignore := false + skipped := false if valid_input +ignore := true if { + input.specVersion == "1.0" +} + violations contains msg if { valid_input input.specVersion != "1.5" diff --git a/pkg/policies/engine/rego/testfiles/result_format_without_ignore.rego b/pkg/policies/engine/rego/testfiles/result_format_without_ignore.rego new file mode 100644 index 000000000..89dd3cbd7 --- /dev/null +++ b/pkg/policies/engine/rego/testfiles/result_format_without_ignore.rego @@ -0,0 +1,30 @@ +package main + +import rego.v1 + +result := { + "skipped": skipped, + "violations": violations, + "skip_reason": skip_reason, +} + +default skip_reason := "" + +skip_reason := m if { + not valid_input + m := "invalid input" +} + +default skipped := true + +skipped := false if valid_input + +violations contains msg if { + valid_input + input.specVersion != "1.5" + msg := sprintf("wrong CycloneDX version. Expected 1.5, but it was %s", [input.specVersion]) +} + +valid_input if { + input.specVersion +} diff --git a/pkg/policies/policies.go b/pkg/policies/policies.go index 6296e093b..22f091277 100644 --- a/pkg/policies/policies.go +++ b/pkg/policies/policies.go @@ -95,7 +95,9 @@ func (pv *PolicyVerifier) VerifyMaterial(ctx context.Context, material *v12.Atte return nil, NewPolicyError(err) } - result = append(result, ev) + if ev != nil { + result = append(result, ev) + } } return result, nil @@ -149,6 +151,11 @@ func (pv *PolicyVerifier) evaluatePolicyAttachment(ctx context.Context, attachme return nil, NewPolicyError(err) } + // Skip if the script explicitly instructs us to ignore it, effectively preventing it from being added to the evaluation results + if r.Ignore { + continue + } + // Gather merged results evalResults = append(evalResults, r) @@ -161,6 +168,11 @@ func (pv *PolicyVerifier) evaluatePolicyAttachment(ctx context.Context, attachme sources = append(sources, base64.StdEncoding.EncodeToString(script.Source)) } + if len(sources) == 0 { + pv.logger.Debug().Msgf("policy %s explicitly ignored by definition", policy.Metadata.Name) + return nil, nil + } + var evaluationSources []string if ref != nil && !IsProviderScheme(ref.URI) { evaluationSources = sources @@ -263,7 +275,9 @@ func (pv *PolicyVerifier) VerifyStatement(ctx context.Context, statement *intoto return nil, NewPolicyError(err) } - result = append(result, ev) + if ev != nil { + result = append(result, ev) + } } return result, nil diff --git a/pkg/policies/policies_test.go b/pkg/policies/policies_test.go index 3025f7be5..19ce899b0 100644 --- a/pkg/policies/policies_test.go +++ b/pkg/policies/policies_test.go @@ -956,6 +956,7 @@ func (s *testSuite) TestNewResultFormat() { expectErr bool expectViolations int expectSkipped bool + expectIgnore bool expectReasons []string }{ { @@ -963,6 +964,7 @@ func (s *testSuite) TestNewResultFormat() { policy: "file://testdata/policy_result_format.yaml", material: "{\"specVersion\": \"1.4\"}", expectViolations: 1, + expectIgnore: false, }, { name: "skip", @@ -970,6 +972,7 @@ func (s *testSuite) TestNewResultFormat() { material: "{\"invalid\": \"1.4\"}", expectSkipped: true, expectReasons: []string{"invalid input"}, + expectIgnore: false, }, { name: "skip multiple", @@ -977,6 +980,13 @@ func (s *testSuite) TestNewResultFormat() { material: "{}", expectSkipped: true, expectReasons: []string{"this one is skipped", "this is also skipped"}, + expectIgnore: false, + }, + { + name: "ignore", + policy: "file://testdata/policy_with_ignore.yaml", + material: "{\"specVersion\": \"1.0\"}", + expectIgnore: true, }, } @@ -1014,6 +1024,12 @@ func (s *testSuite) TestNewResultFormat() { return } + if tc.expectIgnore { + s.Nil(err) + s.Equal([]*v1.PolicyEvaluation{}, res) + return + } + s.Require().NoError(err) s.Len(res, 1) s.Len(res[0].Violations, tc.expectViolations) @@ -1089,6 +1105,99 @@ func (s *testSuite) TestContainerMaterial() { } } +func (s *testSuite) TestMultiKindAWithIgnore() { + cases := []struct { + name string + policy string + material string + expectErr bool + expectedEvaluations int + expectSkipped bool + expectReasons []string + expecteIgnore bool + }{ + { + name: "scripts should not be ignored and skipped", + policy: "file://testdata/policy_multi_kind_with_ignore.yaml", + material: "{\"specVersion\": \"1.4\"}", + expectedEvaluations: 1, + expectSkipped: true, + expectReasons: []string{"this on is skipped"}, + }, + { + name: "scripts should be ignored", + policy: "file://testdata/policy_multi_kind_with_ignore.yaml", + material: "{\"specVersion\": \"1.0\"}", + expectedEvaluations: 0, + expecteIgnore: true, + }, + { + name: "all scripts should not be ignored", + policy: "file://testdata/policy_multi_kind_with_ignore.yaml", + material: "{\"specVersion\": \"1.4\"}", + expectedEvaluations: 1, + expectSkipped: false, + }, + } + + for _, tc := range cases { + s.Run(tc.name, func() { + schema := &v12.CraftingSchema{ + Materials: []*v12.CraftingSchema_Material{ + { + Name: "sbom", + Type: v12.CraftingSchema_Material_SBOM_CYCLONEDX_JSON, + }, + }, + Policies: &v12.Policies{ + Materials: []*v12.PolicyAttachment{ + { + Policy: &v12.PolicyAttachment_Ref{Ref: tc.policy}, + }, + }, + }, + } + + material := &v1.Attestation_Material{ + M: &v1.Attestation_Material_Artifact_{Artifact: &v1.Attestation_Material_Artifact{ + Content: []byte(tc.material), + }}, + MaterialType: v12.CraftingSchema_Material_SBOM_CYCLONEDX_JSON, + InlineCas: true, + } + + if tc.expecteIgnore { + material.MaterialType = v12.CraftingSchema_Material_SARIF + } + + if tc.name == "all scripts should not be ignored" { + material.MaterialType = v12.CraftingSchema_Material_OPENVEX + } + + verifier := NewPolicyVerifier(schema, nil, &s.logger) + res, err := verifier.VerifyMaterial(context.TODO(), material, "") + + if tc.expectErr { + s.Error(err) + return + } + + if tc.expecteIgnore { + s.Nil(err) + s.Len(res, tc.expectedEvaluations) + return + } + + s.Require().NoError(err) + s.Len(res, tc.expectedEvaluations) + s.Equal(tc.expectSkipped, res[0].Skipped) + if len(res[0].SkipReasons) > 0 { + s.Equal(tc.expectReasons, res[0].SkipReasons) + } + }) + } +} + type testSuite struct { suite.Suite diff --git a/pkg/policies/policy_groups.go b/pkg/policies/policy_groups.go index 9a75ea421..38e733d9f 100644 --- a/pkg/policies/policy_groups.go +++ b/pkg/policies/policy_groups.go @@ -87,6 +87,11 @@ func (pgv *PolicyGroupVerifier) VerifyMaterial(ctx context.Context, material *ap return nil, NewPolicyError(err) } + if ev == nil { + // no evaluation, skip + continue + } + // Assign group reference to this evaluation ev.GroupReference = &api.PolicyEvaluation_Reference{ Name: group.GetMetadata().GetName(), @@ -133,6 +138,11 @@ func (pgv *PolicyGroupVerifier) VerifyStatement(ctx context.Context, statement * return nil, NewPolicyError(err) } + if ev == nil { + // no evaluation, skip + continue + } + // Assign group reference to this evaluation ev.GroupReference = &api.PolicyEvaluation_Reference{ Name: group.GetMetadata().GetName(), diff --git a/pkg/policies/policy_groups_test.go b/pkg/policies/policy_groups_test.go index c58b976df..17aaffcde 100644 --- a/pkg/policies/policy_groups_test.go +++ b/pkg/policies/policy_groups_test.go @@ -270,6 +270,85 @@ func (s *groupsTestSuite) TestVerifyStatement() { } } +func (s *groupsTestSuite) TestVerifyMaterialMultiKind() { + cases := []struct { + name string + policyGroup string + material string + expectErr bool + expectedEvaluations int + expectSkipped bool + expectReasons []string + expectIgnore bool + }{ + { + name: "not evaluation results, ignore", + policyGroup: "file://testdata/policy_group_multikind.yaml", + material: "{\"specVersion\": \"1.0\"}", + expectedEvaluations: 0, + expectIgnore: true, + }, + { + name: "evaluation results, no ignore", + policyGroup: "file://testdata/policy_group_multikind.yaml", + material: "{\"specVersion\": \"1.4\"}", + expectedEvaluations: 1, + expectIgnore: false, + }, + } + + for _, tc := range cases { + s.Run(tc.name, func() { + schema := &v1.CraftingSchema{ + Materials: []*v1.CraftingSchema_Material{ + { + Name: "sbom", + Type: v1.CraftingSchema_Material_SBOM_CYCLONEDX_JSON, + }, + }, + PolicyGroups: []*v1.PolicyGroupAttachment{ + { + Ref: tc.policyGroup, + }, + }, + } + + material := &api.Attestation_Material{ + M: &api.Attestation_Material_Artifact_{Artifact: &api.Attestation_Material_Artifact{ + Content: []byte(tc.material), + }}, + MaterialType: v1.CraftingSchema_Material_SBOM_CYCLONEDX_JSON, + InlineCas: true, + } + + if !tc.expectIgnore { + material.MaterialType = v1.CraftingSchema_Material_OPENVEX + } + + verifier := NewPolicyGroupVerifier(schema, nil, &s.logger) + res, err := verifier.VerifyMaterial(context.TODO(), material, "") + + if tc.expectErr { + s.Error(err) + return + } + + if tc.expectIgnore { + s.Nil(err) + s.Len(res, tc.expectedEvaluations) + return + } + + s.Require().NoError(err) + s.Len(res, tc.expectedEvaluations) + s.Equal(tc.expectSkipped, res[0].Skipped) + if len(res[0].SkipReasons) > 0 { + s.Equal(tc.expectReasons, res[0].SkipReasons) + } + }) + } +} + func (s *groupsTestSuite) TestGroupInputs() { cases := []struct { name string diff --git a/pkg/policies/testdata/policy_group_multikind.yaml b/pkg/policies/testdata/policy_group_multikind.yaml new file mode 100644 index 000000000..ea828e1ac --- /dev/null +++ b/pkg/policies/testdata/policy_group_multikind.yaml @@ -0,0 +1,16 @@ +apiVersion: workflowcontract.chainloop.dev/v1 +kind: PolicyGroup +metadata: + name: sbom-quality + description: This policy group applies a number of SBOM-related policies + annotations: + category: SBOM +spec: + policies: + materials: + - type: SBOM_CYCLONEDX_JSON + policies: + - ref: file://testdata/policy_with_ignore.yaml + - type: OPENVEX + policies: + - ref: file://testdata/policy_openvex_no_ignore.yaml diff --git a/pkg/policies/testdata/policy_multi_kind_with_ignore.yaml b/pkg/policies/testdata/policy_multi_kind_with_ignore.yaml new file mode 100644 index 000000000..9bdc95b28 --- /dev/null +++ b/pkg/policies/testdata/policy_multi_kind_with_ignore.yaml @@ -0,0 +1,69 @@ +apiVersion: workflowcontract.chainloop.dev/v1 +kind: Policy +metadata: + name: multikindignore + description: multikind policy + annotations: + category: SBOM +spec: + policies: + - kind: SARIF + embedded: | + package main + + import rego.v1 + + result := { + "skipped": true, + "violations": [], + "skip_reason": "this one should be ignored", + "ignore": true, + } + - kind: SBOM_CYCLONEDX_JSON + embedded: | + package main + + import rego.v1 + + result := { + "skipped": true, + "violations": [], + "skip_reason": "this one should be ignored", + "ignore": true, + } + - kind: SBOM_CYCLONEDX_JSON + embedded: | + package main + + import rego.v1 + + result := { + "skipped": true, + "violations": [], + "skip_reason": "this on is skipped", + "ignore": false, + } + - kind: OPENVEX + embedded: | + package main + + import rego.v1 + + result := { + "skipped": false, + "violations": [], + "skip_reason": "", + "ignore": false, + } + - kind: OPENVEX + embedded: | + package main + + import rego.v1 + + result := { + "skipped": false, + "violations": [], + "skip_reason": "", + "ignore": false, + } diff --git a/pkg/policies/testdata/policy_openvex_no_ignore.yaml b/pkg/policies/testdata/policy_openvex_no_ignore.yaml new file mode 100644 index 000000000..c5eb0faf4 --- /dev/null +++ b/pkg/policies/testdata/policy_openvex_no_ignore.yaml @@ -0,0 +1,33 @@ +apiVersion: workflowcontract.chainloop.dev/v1 +kind: Policy +metadata: + name: multikindignore + description: multikind policy + annotations: + category: SBOM +spec: + policies: + - kind: OPENVEX + embedded: | + package main + + import rego.v1 + + result := { + "skipped": false, + "violations": [], + "skip_reason": "", + "ignore": false, + } + - kind: OPENVEX + embedded: | + package main + + import rego.v1 + + result := { + "skipped": false, + "violations": [], + "skip_reason": "", + "ignore": false, + } diff --git a/pkg/policies/testdata/policy_with_ignore.yaml b/pkg/policies/testdata/policy_with_ignore.yaml new file mode 100644 index 000000000..9942fa993 --- /dev/null +++ b/pkg/policies/testdata/policy_with_ignore.yaml @@ -0,0 +1,47 @@ +apiVersion: workflowcontract.chainloop.dev/v1 +kind: Policy +metadata: + name: policy-result-format + description: Policy with new result format + annotations: + category: SBOM +spec: + policies: + - kind: SBOM_CYCLONEDX_JSON + embedded: | + package main + + import rego.v1 + + result := { + "skipped": skipped, + "violations": violations, + "skip_reason": skip_reason, + "ignore": ignore, + } + + default skip_reason := "" + + skip_reason := m if { + not valid_input + m := "invalid input" + } + + default skipped := true + default ignore := false + + skipped := false if valid_input + + ignore := true if { + input.specVersion == "1.0" + } + + violations contains msg if { + valid_input + input.specVersion != "1.5" + msg := sprintf("wrong CycloneDX version. Expected 1.5, but it was %s", [input.specVersion]) + } + + valid_input if { + input.specVersion + }