Skip to content

feat(policy): change devel commands return to stdout #2308

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 5, 2025
Merged

feat(policy): change devel commands return to stdout #2308

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
60e7b3b
feat(policy): change devel commands return to stdout
Piskoo Aug 5, 2025
6f2aed7
change fprintf to sprintf
Piskoo Aug 5, 2025
5519d1a
fix linter error
Piskoo Aug 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion app/cli/cmd/output.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ type tabulatedData interface {
*action.APITokenItem |
[]*action.APITokenItem |
*action.AttestationStatusMaterial |
*action.ListMembershipResult
*action.ListMembershipResult |
*action.PolicyEvalResult
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does this mean that now we support json output?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Correct, the text version didn't follow any structure so it would cause problems later on

}

var ErrOutputFormatNotImplemented = errors.New("format not implemented")
Expand Down
22 changes: 1 addition & 21 deletions app/cli/cmd/policy_develop_eval.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,27 +61,7 @@ evaluates the policy against the provided material or attestation.`,
return newGracefulError(err)
}

switch {
case result.Skipped:
logger.Info().Msg("policy evaluation skipped")
for _, reason := range result.SkipReasons {
logger.Info().Msgf("%s", reason)
}

case len(result.Violations) > 0:
for _, violation := range result.Violations {
logger.Info().Msgf("- %s", violation)
}
logger.Info().Msg("policy evaluation failed")

default:
if result.Ignored {
logger.Info().Msg("policy was ignored")
}
logger.Info().Msg("policy evaluation passed")
}

return nil
return encodeJSON(result)
},
}

Expand Down
22 changes: 17 additions & 5 deletions app/cli/cmd/policy_develop_lint.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,7 @@ func newPolicyDevelopLintCmd() *cobra.Command {
return nil
}

// Print all validation errors
for _, err := range result.Errors {
logger.Error().Msg(err)
}
return fmt.Errorf("policy validation failed with %d issues", len(result.Errors))
return encodeResult(result)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what's the output when we do not have any violations? Do we return an empty violations array?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, here's the output

[
   {
      "violations": [],
      "skip_reasons": [],
      "skipped": false
   }
]

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

awesome, please merge :)

},
}

Expand All @@ -68,3 +64,19 @@ func newPolicyDevelopLintCmd() *cobra.Command {
cmd.Flags().StringVar(&regalConfig, "regal-config", "", "Path to custom regal config (Default: https://github.com/chainloop-dev/chainloop/tree/main/app/cli/internal/policydevel/.regal.yaml)")
return cmd
}

func encodeResult(result *action.PolicyLintResult) error {
if result == nil {
return nil
}

output := fmt.Sprintf("Found %d issues:\n", len(result.Errors))

for i, err := range result.Errors {
output += fmt.Sprintf(" %d. %s\n", i+1, err)
}

fmt.Print(output)

return fmt.Errorf("policy validation failed with %d issues", len(result.Errors))
}
29 changes: 17 additions & 12 deletions app/cli/internal/action/policy_develop_eval.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@ type PolicyEvalOpts struct {
}

type PolicyEvalResult struct {
Skipped bool
SkipReasons []string
Violations []string
Ignored bool
Violations []string `json:"violations"`
SkipReasons []string `json:"skip_reasons"`
Skipped bool `json:"skipped"`
Ignored bool `json:"ignored,omitempty"`
}

type PolicyEval struct {
Expand All @@ -48,7 +48,7 @@ func NewPolicyEval(opts *PolicyEvalOpts, actionOpts *ActionsOpts) (*PolicyEval,
}, nil
}

func (action *PolicyEval) Run() (*PolicyEvalResult, error) {
func (action *PolicyEval) Run() ([]*PolicyEvalResult, error) {
evalOpts := &policydevel.EvalOptions{
PolicyPath: action.opts.PolicyPath,
MaterialKind: action.opts.Kind,
Expand All @@ -58,15 +58,20 @@ func (action *PolicyEval) Run() (*PolicyEvalResult, error) {
}

// Evaluate policy
result, err := policydevel.Evaluate(evalOpts, action.Logger)
resp, err := policydevel.Evaluate(evalOpts, action.Logger)
if err != nil {
return nil, fmt.Errorf("evaluating policy: %w", err)
}

return &PolicyEvalResult{
Ignored: result.Ignored,
Skipped: result.Skipped,
SkipReasons: result.SkipReasons,
Violations: result.Violations,
}, nil
results := make([]*PolicyEvalResult, 0, len(resp))
for _, r := range resp {
results = append(results, &PolicyEvalResult{
Violations: r.Violations,
SkipReasons: r.SkipReasons,
Skipped: r.Skipped,
Ignored: r.Ignored,
})
}

return results, nil
}
48 changes: 28 additions & 20 deletions app/cli/internal/policydevel/eval.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ type EvalResult struct {
Ignored bool
}

func Evaluate(opts *EvalOptions, logger zerolog.Logger) (*EvalResult, error) {
func Evaluate(opts *EvalOptions, logger zerolog.Logger) ([]*EvalResult, error) {
// 1. Create crafting schema
schema, err := createCraftingSchema(opts.PolicyPath, opts.Inputs)
if err != nil {
Expand Down Expand Up @@ -81,36 +81,44 @@ func createCraftingSchema(policyPath string, inputs map[string]string) (*v1.Craf
}, nil
}

func verifyMaterial(schema *v1.CraftingSchema, material *v12.Attestation_Material, materialPath string, logger *zerolog.Logger) (*EvalResult, error) {
func verifyMaterial(schema *v1.CraftingSchema, material *v12.Attestation_Material, materialPath string, logger *zerolog.Logger) ([]*EvalResult, error) {
v := policies.NewPolicyVerifier(schema, nil, logger)
evs, err := v.VerifyMaterial(context.Background(), material, materialPath)
policyEvs, err := v.VerifyMaterial(context.Background(), material, materialPath)
if err != nil {
return nil, err
}

result := &EvalResult{
Skipped: false,
SkipReasons: []string{},
Violations: []string{},
Ignored: true,
}

if len(evs) == 0 {
return result, nil
// no evaluations were returned
if len(policyEvs) == 0 {
return []*EvalResult{
{
Ignored: true,
Skipped: false,
SkipReasons: []string{},
Violations: []string{},
},
}, nil
}

result.Ignored = false
result.Skipped = evs[0].GetSkipped()
result.SkipReasons = evs[0].SkipReasons
result.Violations = make([]string, 0, len(evs[0].Violations))
results := make([]*EvalResult, 0, len(policyEvs))
for _, policyEv := range policyEvs {
result := &EvalResult{
Skipped: policyEv.GetSkipped(),
SkipReasons: policyEv.SkipReasons,
Ignored: false,
}

for _, e := range evs {
for _, v := range e.Violations {
result.Violations = append(result.Violations, fmt.Sprintf("%s: %s", v.Subject, v.Message))
// Collect all violation messages
violations := make([]string, 0, len(policyEv.Violations))
for _, v := range policyEv.Violations {
violations = append(violations, v.Message)
}
result.Violations = violations

results = append(results, result)
}

return result, nil
return results, nil
}

func craftMaterial(materialPath, materialKind string, logger *zerolog.Logger) (*v12.Attestation_Material, error) {
Expand Down
21 changes: 11 additions & 10 deletions app/cli/internal/policydevel/eval_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,10 @@ func TestEvaluate(t *testing.T) {
Annotations: map[string]string{"key": "value"},
}

result, err := Evaluate(opts, logger)
results, err := Evaluate(opts, logger)
require.NoError(t, err)
assert.NotNil(t, result)
require.NotEmpty(t, results)
assert.NotNil(t, results[0])
})

t.Run("evaluation with auto-detected SBOM CYCLONEDX kind", func(t *testing.T) {
Expand All @@ -55,14 +56,14 @@ func TestEvaluate(t *testing.T) {
Annotations: map[string]string{"key": "value"},
}

result, err := Evaluate(opts, logger)
results, err := Evaluate(opts, logger)
require.NoError(t, err)
assert.NotNil(t, result)
require.NotEmpty(t, results)

if len(result.Violations) == 0 {
if len(results[0].Violations) == 0 {
t.Log("Policy evaluation passed (no violations)")
} else {
for _, violation := range result.Violations {
for _, violation := range results[0].Violations {
t.Logf("Violation: %s", violation)
}
}
Expand All @@ -78,14 +79,14 @@ func TestEvaluate(t *testing.T) {
Annotations: map[string]string{"key": "value"},
}

result, err := Evaluate(opts, logger)
results, err := Evaluate(opts, logger)
require.NoError(t, err)
assert.NotNil(t, result)
require.NotEmpty(t, results)

if len(result.Violations) == 0 {
if len(results[0].Violations) == 0 {
t.Log("Policy evaluation passed (no violations)")
} else {
for _, violation := range result.Violations {
for _, violation := range results[0].Violations {
t.Logf("Violation: %s", violation)
}
}
Expand Down
Loading