feat(policy): change devel commands return to stdout (#2308)

Signed-off-by: Sylwester Piskozub <sylwesterpiskozub@gmail.com>

Author: Piskoo

app/cli/cmd/output.go

@@ -54,7 +54,8 @@ type tabulatedData interface {
 		*action.APITokenItem |
 		[]*action.APITokenItem |
 		*action.AttestationStatusMaterial |
-		*action.ListMembershipResult
+		*action.ListMembershipResult |
+		*action.PolicyEvalResult
 }
 
 var ErrOutputFormatNotImplemented = errors.New("format not implemented")

app/cli/cmd/policy_develop_eval.go

@@ -61,27 +61,7 @@ evaluates the policy against the provided material or attestation.`,
 				return newGracefulError(err)
 			}
 
-			switch {
-			case result.Skipped:
-				logger.Info().Msg("policy evaluation skipped")
-				for _, reason := range result.SkipReasons {
-					logger.Info().Msgf("%s", reason)
-				}
-
-			case len(result.Violations) > 0:
-				for _, violation := range result.Violations {
-					logger.Info().Msgf("- %s", violation)
-				}
-				logger.Info().Msg("policy evaluation failed")
-
-			default:
-				if result.Ignored {
-					logger.Info().Msg("policy was ignored")
-				}
-				logger.Info().Msg("policy evaluation passed")
-			}
-
-			return nil
+			return encodeJSON(result)
 		},
 	}
 

app/cli/cmd/policy_develop_lint.go

@@ -55,11 +55,7 @@ func newPolicyDevelopLintCmd() *cobra.Command {
 				return nil
 			}
 
-			// Print all validation errors
-			for _, err := range result.Errors {
-				logger.Error().Msg(err)
-			}
-			return fmt.Errorf("policy validation failed with %d issues", len(result.Errors))
+			return encodeResult(result)
 		},
 	}
 
@@ -68,3 +64,19 @@ func newPolicyDevelopLintCmd() *cobra.Command {
 	cmd.Flags().StringVar(&regalConfig, "regal-config", "", "Path to custom regal config (Default: https://github.com/chainloop-dev/chainloop/tree/main/app/cli/internal/policydevel/.regal.yaml)")
 	return cmd
 }
+
+func encodeResult(result *action.PolicyLintResult) error {
+	if result == nil {
+		return nil
+	}
+
+	output := fmt.Sprintf("Found %d issues:\n", len(result.Errors))
+
+	for i, err := range result.Errors {
+		output += fmt.Sprintf("  %d. %s\n", i+1, err)
+	}
+
+	fmt.Print(output)
+
+	return fmt.Errorf("policy validation failed with %d issues", len(result.Errors))
+}

app/cli/internal/action/policy_develop_eval.go

@@ -30,10 +30,10 @@ type PolicyEvalOpts struct {
 }
 
 type PolicyEvalResult struct {
-	Skipped     bool
-	SkipReasons []string
-	Violations  []string
-	Ignored     bool
+	Violations  []string `json:"violations"`
+	SkipReasons []string `json:"skip_reasons"`
+	Skipped     bool     `json:"skipped"`
+	Ignored     bool     `json:"ignored,omitempty"`
 }
 
 type PolicyEval struct {
@@ -48,7 +48,7 @@ func NewPolicyEval(opts *PolicyEvalOpts, actionOpts *ActionsOpts) (*PolicyEval,
 	}, nil
 }
 
-func (action *PolicyEval) Run() (*PolicyEvalResult, error) {
+func (action *PolicyEval) Run() ([]*PolicyEvalResult, error) {
 	evalOpts := &policydevel.EvalOptions{
 		PolicyPath:   action.opts.PolicyPath,
 		MaterialKind: action.opts.Kind,
@@ -58,15 +58,20 @@ func (action *PolicyEval) Run() (*PolicyEvalResult, error) {
 	}
 
 	// Evaluate policy
-	result, err := policydevel.Evaluate(evalOpts, action.Logger)
+	resp, err := policydevel.Evaluate(evalOpts, action.Logger)
 	if err != nil {
 		return nil, fmt.Errorf("evaluating policy: %w", err)
 	}
 
-	return &PolicyEvalResult{
-		Ignored:     result.Ignored,
-		Skipped:     result.Skipped,
-		SkipReasons: result.SkipReasons,
-		Violations:  result.Violations,
-	}, nil
+	results := make([]*PolicyEvalResult, 0, len(resp))
+	for _, r := range resp {
+		results = append(results, &PolicyEvalResult{
+			Violations:  r.Violations,
+			SkipReasons: r.SkipReasons,
+			Skipped:     r.Skipped,
+			Ignored:     r.Ignored,
+		})
+	}
+
+	return results, nil
 }

app/cli/internal/policydevel/eval.go

@@ -43,7 +43,7 @@ type EvalResult struct {
 	Ignored     bool
 }
 
-func Evaluate(opts *EvalOptions, logger zerolog.Logger) (*EvalResult, error) {
+func Evaluate(opts *EvalOptions, logger zerolog.Logger) ([]*EvalResult, error) {
 	// 1. Create crafting schema
 	schema, err := createCraftingSchema(opts.PolicyPath, opts.Inputs)
 	if err != nil {
@@ -81,36 +81,44 @@ func createCraftingSchema(policyPath string, inputs map[string]string) (*v1.Craf
 	}, nil
 }
 
-func verifyMaterial(schema *v1.CraftingSchema, material *v12.Attestation_Material, materialPath string, logger *zerolog.Logger) (*EvalResult, error) {
+func verifyMaterial(schema *v1.CraftingSchema, material *v12.Attestation_Material, materialPath string, logger *zerolog.Logger) ([]*EvalResult, error) {
 	v := policies.NewPolicyVerifier(schema, nil, logger)
-	evs, err := v.VerifyMaterial(context.Background(), material, materialPath)
+	policyEvs, err := v.VerifyMaterial(context.Background(), material, materialPath)
 	if err != nil {
 		return nil, err
 	}
 
-	result := &EvalResult{
-		Skipped:     false,
-		SkipReasons: []string{},
-		Violations:  []string{},
-		Ignored:     true,
-	}
-
-	if len(evs) == 0 {
-		return result, nil
+	// no evaluations were returned
+	if len(policyEvs) == 0 {
+		return []*EvalResult{
+			{
+				Ignored:     true,
+				Skipped:     false,
+				SkipReasons: []string{},
+				Violations:  []string{},
+			},
+		}, nil
 	}
 
-	result.Ignored = false
-	result.Skipped = evs[0].GetSkipped()
-	result.SkipReasons = evs[0].SkipReasons
-	result.Violations = make([]string, 0, len(evs[0].Violations))
+	results := make([]*EvalResult, 0, len(policyEvs))
+	for _, policyEv := range policyEvs {
+		result := &EvalResult{
+			Skipped:     policyEv.GetSkipped(),
+			SkipReasons: policyEv.SkipReasons,
+			Ignored:     false,
+		}
 
-	for _, e := range evs {
-		for _, v := range e.Violations {
-			result.Violations = append(result.Violations, fmt.Sprintf("%s: %s", v.Subject, v.Message))
+		// Collect all violation messages
+		violations := make([]string, 0, len(policyEv.Violations))
+		for _, v := range policyEv.Violations {
+			violations = append(violations, v.Message)
 		}
+		result.Violations = violations
+
+		results = append(results, result)
 	}
 
-	return result, nil
+	return results, nil
 }
 
 func craftMaterial(materialPath, materialKind string, logger *zerolog.Logger) (*v12.Attestation_Material, error) {

app/cli/internal/policydevel/eval_test.go

@@ -40,9 +40,10 @@ func TestEvaluate(t *testing.T) {
 			Annotations:  map[string]string{"key": "value"},
 		}
 
-		result, err := Evaluate(opts, logger)
+		results, err := Evaluate(opts, logger)
 		require.NoError(t, err)
-		assert.NotNil(t, result)
+		require.NotEmpty(t, results)
+		assert.NotNil(t, results[0])
 	})
 
 	t.Run("evaluation with auto-detected SBOM CYCLONEDX kind", func(t *testing.T) {
@@ -55,14 +56,14 @@ func TestEvaluate(t *testing.T) {
 			Annotations:  map[string]string{"key": "value"},
 		}
 
-		result, err := Evaluate(opts, logger)
+		results, err := Evaluate(opts, logger)
 		require.NoError(t, err)
-		assert.NotNil(t, result)
+		require.NotEmpty(t, results)
 
-		if len(result.Violations) == 0 {
+		if len(results[0].Violations) == 0 {
 			t.Log("Policy evaluation passed (no violations)")
 		} else {
-			for _, violation := range result.Violations {
+			for _, violation := range results[0].Violations {
 				t.Logf("Violation: %s", violation)
 			}
 		}
@@ -78,14 +79,14 @@ func TestEvaluate(t *testing.T) {
 			Annotations:  map[string]string{"key": "value"},
 		}
 
-		result, err := Evaluate(opts, logger)
+		results, err := Evaluate(opts, logger)
 		require.NoError(t, err)
-		assert.NotNil(t, result)
+		require.NotEmpty(t, results)
 
-		if len(result.Violations) == 0 {
+		if len(results[0].Violations) == 0 {
 			t.Log("Policy evaluation passed (no violations)")
 		} else {
-			for _, violation := range result.Violations {
+			for _, violation := range results[0].Violations {
 				t.Logf("Violation: %s", violation)
 			}
 		}