From 60e7b3b9925c1d90b4df9eb02f56304de6197f9a Mon Sep 17 00:00:00 2001 From: Sylwester Piskozub Date: Tue, 5 Aug 2025 12:20:57 +0200 Subject: [PATCH 1/3] feat(policy): change devel commands return to stdout Signed-off-by: Sylwester Piskozub --- app/cli/cmd/output.go | 3 +- app/cli/cmd/policy_develop_eval.go | 22 +-------- app/cli/cmd/policy_develop_lint.go | 21 ++++++-- .../internal/action/policy_develop_eval.go | 29 ++++++----- app/cli/internal/policydevel/eval.go | 48 +++++++++++-------- app/cli/internal/policydevel/eval_test.go | 21 ++++---- 6 files changed, 75 insertions(+), 69 deletions(-) diff --git a/app/cli/cmd/output.go b/app/cli/cmd/output.go index 9310a119e..622e55eac 100644 --- a/app/cli/cmd/output.go +++ b/app/cli/cmd/output.go @@ -54,7 +54,8 @@ type tabulatedData interface { *action.APITokenItem | []*action.APITokenItem | *action.AttestationStatusMaterial | - *action.ListMembershipResult + *action.ListMembershipResult | + *action.PolicyEvalResult } var ErrOutputFormatNotImplemented = errors.New("format not implemented") diff --git a/app/cli/cmd/policy_develop_eval.go b/app/cli/cmd/policy_develop_eval.go index cdc3767cf..b5e80967b 100644 --- a/app/cli/cmd/policy_develop_eval.go +++ b/app/cli/cmd/policy_develop_eval.go @@ -61,27 +61,7 @@ evaluates the policy against the provided material or attestation.`, return newGracefulError(err) } - switch { - case result.Skipped: - logger.Info().Msg("policy evaluation skipped") - for _, reason := range result.SkipReasons { - logger.Info().Msgf("%s", reason) - } - - case len(result.Violations) > 0: - for _, violation := range result.Violations { - logger.Info().Msgf("- %s", violation) - } - logger.Info().Msg("policy evaluation failed") - - default: - if result.Ignored { - logger.Info().Msg("policy was ignored") - } - logger.Info().Msg("policy evaluation passed") - } - - return nil + return encodeJSON(result) }, } diff --git a/app/cli/cmd/policy_develop_lint.go b/app/cli/cmd/policy_develop_lint.go index 439becd7d..3126d2e52 100644 --- a/app/cli/cmd/policy_develop_lint.go +++ b/app/cli/cmd/policy_develop_lint.go @@ -17,6 +17,7 @@ package cmd import ( "fmt" + "os" "github.com/chainloop-dev/chainloop/app/cli/internal/action" "github.com/spf13/cobra" @@ -55,11 +56,7 @@ func newPolicyDevelopLintCmd() *cobra.Command { return nil } - // Print all validation errors - for _, err := range result.Errors { - logger.Error().Msg(err) - } - return fmt.Errorf("policy validation failed with %d issues", len(result.Errors)) + return encodeResults(result) }, } @@ -68,3 +65,17 @@ func newPolicyDevelopLintCmd() *cobra.Command { cmd.Flags().StringVar(®alConfig, "regal-config", "", "Path to custom regal config (Default: https://github.com/chainloop-dev/chainloop/tree/main/app/cli/internal/policydevel/.regal.yaml)") return cmd } + +func encodeResults(result *action.PolicyLintResult) error { + if result == nil { + return nil + } + + fmt.Fprintf(os.Stdout, "Found %d issues:\n", len(result.Errors)) + + for i, err := range result.Errors { + fmt.Fprintf(os.Stdout, " %d. %s\n", i+1, err) + } + + return fmt.Errorf("policy validation failed with %d issues", len(result.Errors)) +} diff --git a/app/cli/internal/action/policy_develop_eval.go b/app/cli/internal/action/policy_develop_eval.go index 24b7efde5..ba3f736d4 100644 --- a/app/cli/internal/action/policy_develop_eval.go +++ b/app/cli/internal/action/policy_develop_eval.go @@ -30,10 +30,10 @@ type PolicyEvalOpts struct { } type PolicyEvalResult struct { - Skipped bool - SkipReasons []string - Violations []string - Ignored bool + Violations []string `json:"violations"` + SkipReasons []string `json:"skip_reasons"` + Skipped bool `json:"skipped"` + Ignored bool `json:"ignored,omitempty"` } type PolicyEval struct { @@ -48,7 +48,7 @@ func NewPolicyEval(opts *PolicyEvalOpts, actionOpts *ActionsOpts) (*PolicyEval, }, nil } -func (action *PolicyEval) Run() (*PolicyEvalResult, error) { +func (action *PolicyEval) Run() ([]*PolicyEvalResult, error) { evalOpts := &policydevel.EvalOptions{ PolicyPath: action.opts.PolicyPath, MaterialKind: action.opts.Kind, @@ -58,15 +58,20 @@ func (action *PolicyEval) Run() (*PolicyEvalResult, error) { } // Evaluate policy - result, err := policydevel.Evaluate(evalOpts, action.Logger) + resp, err := policydevel.Evaluate(evalOpts, action.Logger) if err != nil { return nil, fmt.Errorf("evaluating policy: %w", err) } - return &PolicyEvalResult{ - Ignored: result.Ignored, - Skipped: result.Skipped, - SkipReasons: result.SkipReasons, - Violations: result.Violations, - }, nil + results := make([]*PolicyEvalResult, 0, len(resp)) + for _, r := range resp { + results = append(results, &PolicyEvalResult{ + Violations: r.Violations, + SkipReasons: r.SkipReasons, + Skipped: r.Skipped, + Ignored: r.Ignored, + }) + } + + return results, nil } diff --git a/app/cli/internal/policydevel/eval.go b/app/cli/internal/policydevel/eval.go index 4ac326a06..561f7a69b 100644 --- a/app/cli/internal/policydevel/eval.go +++ b/app/cli/internal/policydevel/eval.go @@ -43,7 +43,7 @@ type EvalResult struct { Ignored bool } -func Evaluate(opts *EvalOptions, logger zerolog.Logger) (*EvalResult, error) { +func Evaluate(opts *EvalOptions, logger zerolog.Logger) ([]*EvalResult, error) { // 1. Create crafting schema schema, err := createCraftingSchema(opts.PolicyPath, opts.Inputs) if err != nil { @@ -81,36 +81,44 @@ func createCraftingSchema(policyPath string, inputs map[string]string) (*v1.Craf }, nil } -func verifyMaterial(schema *v1.CraftingSchema, material *v12.Attestation_Material, materialPath string, logger *zerolog.Logger) (*EvalResult, error) { +func verifyMaterial(schema *v1.CraftingSchema, material *v12.Attestation_Material, materialPath string, logger *zerolog.Logger) ([]*EvalResult, error) { v := policies.NewPolicyVerifier(schema, nil, logger) - evs, err := v.VerifyMaterial(context.Background(), material, materialPath) + policyEvs, err := v.VerifyMaterial(context.Background(), material, materialPath) if err != nil { return nil, err } - result := &EvalResult{ - Skipped: false, - SkipReasons: []string{}, - Violations: []string{}, - Ignored: true, - } - - if len(evs) == 0 { - return result, nil + // no evaluations were returned + if len(policyEvs) == 0 { + return []*EvalResult{ + { + Ignored: true, + Skipped: false, + SkipReasons: []string{}, + Violations: []string{}, + }, + }, nil } - result.Ignored = false - result.Skipped = evs[0].GetSkipped() - result.SkipReasons = evs[0].SkipReasons - result.Violations = make([]string, 0, len(evs[0].Violations)) + results := make([]*EvalResult, 0, len(policyEvs)) + for _, policyEv := range policyEvs { + result := &EvalResult{ + Skipped: policyEv.GetSkipped(), + SkipReasons: policyEv.SkipReasons, + Ignored: false, + } - for _, e := range evs { - for _, v := range e.Violations { - result.Violations = append(result.Violations, fmt.Sprintf("%s: %s", v.Subject, v.Message)) + // Collect all violation messages + violations := make([]string, 0, len(policyEv.Violations)) + for _, v := range policyEv.Violations { + violations = append(violations, v.Message) } + result.Violations = violations + + results = append(results, result) } - return result, nil + return results, nil } func craftMaterial(materialPath, materialKind string, logger *zerolog.Logger) (*v12.Attestation_Material, error) { diff --git a/app/cli/internal/policydevel/eval_test.go b/app/cli/internal/policydevel/eval_test.go index aa6034152..140144374 100644 --- a/app/cli/internal/policydevel/eval_test.go +++ b/app/cli/internal/policydevel/eval_test.go @@ -40,9 +40,10 @@ func TestEvaluate(t *testing.T) { Annotations: map[string]string{"key": "value"}, } - result, err := Evaluate(opts, logger) + results, err := Evaluate(opts, logger) require.NoError(t, err) - assert.NotNil(t, result) + require.NotEmpty(t, results) + assert.NotNil(t, results[0]) }) t.Run("evaluation with auto-detected SBOM CYCLONEDX kind", func(t *testing.T) { @@ -55,14 +56,14 @@ func TestEvaluate(t *testing.T) { Annotations: map[string]string{"key": "value"}, } - result, err := Evaluate(opts, logger) + results, err := Evaluate(opts, logger) require.NoError(t, err) - assert.NotNil(t, result) + require.NotEmpty(t, results) - if len(result.Violations) == 0 { + if len(results[0].Violations) == 0 { t.Log("Policy evaluation passed (no violations)") } else { - for _, violation := range result.Violations { + for _, violation := range results[0].Violations { t.Logf("Violation: %s", violation) } } @@ -78,14 +79,14 @@ func TestEvaluate(t *testing.T) { Annotations: map[string]string{"key": "value"}, } - result, err := Evaluate(opts, logger) + results, err := Evaluate(opts, logger) require.NoError(t, err) - assert.NotNil(t, result) + require.NotEmpty(t, results) - if len(result.Violations) == 0 { + if len(results[0].Violations) == 0 { t.Log("Policy evaluation passed (no violations)") } else { - for _, violation := range result.Violations { + for _, violation := range results[0].Violations { t.Logf("Violation: %s", violation) } } From 6f2aed761d18889acace8362ee1c40839402cf5a Mon Sep 17 00:00:00 2001 From: Sylwester Piskozub Date: Tue, 5 Aug 2025 14:19:02 +0200 Subject: [PATCH 2/3] change fprintf to sprintf Signed-off-by: Sylwester Piskozub --- app/cli/cmd/policy_develop_lint.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/cli/cmd/policy_develop_lint.go b/app/cli/cmd/policy_develop_lint.go index 3126d2e52..1ddf27dfd 100644 --- a/app/cli/cmd/policy_develop_lint.go +++ b/app/cli/cmd/policy_develop_lint.go @@ -56,7 +56,7 @@ func newPolicyDevelopLintCmd() *cobra.Command { return nil } - return encodeResults(result) + return encodeResult(result) }, } @@ -66,16 +66,18 @@ func newPolicyDevelopLintCmd() *cobra.Command { return cmd } -func encodeResults(result *action.PolicyLintResult) error { +func encodeResult(result *action.PolicyLintResult) error { if result == nil { return nil } - fmt.Fprintf(os.Stdout, "Found %d issues:\n", len(result.Errors)) + output := fmt.Sprintf("Found %d issues:\n", len(result.Errors)) for i, err := range result.Errors { - fmt.Fprintf(os.Stdout, " %d. %s\n", i+1, err) + output += fmt.Sprintf(" %d. %s\n", i+1, err) } + fmt.Print(output) + return fmt.Errorf("policy validation failed with %d issues", len(result.Errors)) } From 5519d1a035eacbaaf348754aa13d427a7a19bf70 Mon Sep 17 00:00:00 2001 From: Sylwester Piskozub Date: Tue, 5 Aug 2025 14:35:21 +0200 Subject: [PATCH 3/3] fix linter error Signed-off-by: Sylwester Piskozub --- app/cli/cmd/policy_develop_lint.go | 1 - 1 file changed, 1 deletion(-) diff --git a/app/cli/cmd/policy_develop_lint.go b/app/cli/cmd/policy_develop_lint.go index 1ddf27dfd..954ece92e 100644 --- a/app/cli/cmd/policy_develop_lint.go +++ b/app/cli/cmd/policy_develop_lint.go @@ -17,7 +17,6 @@ package cmd import ( "fmt" - "os" "github.com/chainloop-dev/chainloop/app/cli/internal/action" "github.com/spf13/cobra"