Make command to fail if missing Rego definition

cmd/check.go

@@ -20,6 +20,7 @@ import (
 	"github.com/jetstack/preflight/pkg/output/gcs"
 	"github.com/jetstack/preflight/pkg/packagesources/local"
 	"github.com/jetstack/preflight/pkg/packaging"
+	"github.com/jetstack/preflight/pkg/reports"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -307,6 +308,7 @@ func check() {
 		log.Fatal("No packages were enabled. Use 'enables-packages' option in configuration to enable the packages you want to use.")
 	}
 
+	missingRules := false
 	for _, pkgID := range enabledPackages {
 		// Make sure we loaded the package for this.
 		pkg := packages[pkgID]
@@ -330,7 +332,12 @@ func check() {
 
 		rc, err := packaging.EvalPackage(ctx, pkg, input)
 		if err != nil {
-			log.Fatalf("Cannot evaluate package %q: %v", manifest.ID, err)
+			if _, ok := err.(*reports.MissingRegoDefinitionError); ok {
+				missingRules = true
+				log.Printf("%+v", err)
+			} else {
+				log.Fatalf("Cannot evaluate package %q: %v", manifest.ID, err)
+			}
 		}
 
 		intermediateBytes, err := json.Marshal(input)
@@ -346,7 +353,11 @@ func check() {
 		}
 	}
 
-	log.Printf("Done.")
+	if missingRules {
+		log.Fatalf("Some of the rules are missing their corresponding Rego definition. See the rest of the log or the reports to see more details.")
+	} else {
+		log.Printf("Done.")
+	}
 }
 
 func homeDir() string {

pkg/exporter/json.go

@@ -23,15 +23,18 @@ func NewJSONExporter() *JSONExporter {
 func (e *JSONExporter) Export(ctx context.Context, policyManifest *packaging.PolicyManifest, intermediateJSON []byte, rc *results.ResultCollection) (*bytes.Buffer, error) {
 	report, err := reports.NewReport(policyManifest, rc)
 	if err != nil {
-		return nil, err
+		if _, ok := err.(*reports.MissingRegoDefinitionError); !ok {
+			return nil, err
+		}
 	}
+	missingRuleError := err
 
 	b, err := json.Marshal(report)
 	if err != nil {
 		return nil, err
 	}
 
-	return bytes.NewBuffer(b), nil
+	return bytes.NewBuffer(b), missingRuleError
 }
 
 // FileExtension returns the file extension for this exporter's format

pkg/exporter/json_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/yudai/gojsondiff/formatter"
 
 	"github.com/jetstack/preflight/pkg/packaging"
+	"github.com/jetstack/preflight/pkg/reports"
 	"github.com/jetstack/preflight/pkg/results"
 	"github.com/jetstack/preflight/pkg/rules"
 )
@@ -178,7 +179,7 @@ func TestJSONExport(t *testing.T) {
 }`
 
 	buf, err := jsonExporter.Export(context.Background(), pm, nil, rc)
-	if err != nil {
+	if _, ok := err.(*reports.MissingRegoDefinitionError); !ok {
 		t.Fatalf("unexpected err: %+v", err)
 	}
 

pkg/packaging/eval.go

@@ -27,10 +27,5 @@ func EvalPackage(ctx context.Context, pkg Package, input interface{}) (*results.
 		allResults = append(allResults, rs...)
 	}
 
-	rc, err := results.NewResultCollectionFromRegoResultSet(&allResults)
-	if err != nil {
-		return nil, fmt.Errorf("cannot read results from rego: %s", err)
-	}
-
-	return rc, nil
+	return results.NewResultCollectionFromRegoResultSet(&allResults)
 }

pkg/reports/reports.go

@@ -2,6 +2,7 @@ package reports
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/jetstack/preflight/api"
 	"github.com/jetstack/preflight/pkg/packaging"
@@ -91,6 +92,7 @@ func NewReport(pm *packaging.PolicyManifest, rc *results.ResultCollection) (api.
 		Sections:    make([]api.ReportSection, len(pm.Sections)),
 	}
 
+	missingRules := []string{}
 	for idxSection, section := range pm.Sections {
 		report.Sections[idxSection] = api.ReportSection{
 			ID:          section.ID,
@@ -125,6 +127,7 @@ func NewReport(pm *packaging.PolicyManifest, rc *results.ResultCollection) (api.
 			switch {
 			case result == nil:
 				missing = true
+				missingRules = append(missingRules, rule.ID)
 			case result.IsFailureState():
 				success = false
 				violations = result.Violations
@@ -149,5 +152,21 @@ func NewReport(pm *packaging.PolicyManifest, rc *results.ResultCollection) (api.
 		}
 	}
 
+	if len(missingRules) > 0 {
+		return report, &MissingRegoDefinitionError{
+			pkg: report.Package,
+			ids: missingRules,
+		}
+	}
 	return report, nil
 }
+
+// MissingRegoDefinitionError error to be returned when a rule from the PolicyManifest was not found in Rego.
+type MissingRegoDefinitionError struct {
+	pkg string
+	ids []string
+}
+
+func (e *MissingRegoDefinitionError) Error() string {
+	return fmt.Sprintf("the following rules from the package %q are missing their Rego definitions: %s", e.pkg, strings.Join(e.ids, ", "))
+}

pkg/reports/reports_test.go

@@ -245,74 +245,117 @@ func TestNewReport(t *testing.T) {
 						ID:   "b_rule",
 						Name: "My Rule B",
 					},
-					{
-						ID:   "c_rule",
-						Name: "My Rule C (missing)",
-					},
 				},
 			},
 		},
 	}
 
-	resultCollection := &results.ResultCollection{
-		&results.Result{ID: rules.RuleToResult("a_rule"), Violations: []string{}},
-		&results.Result{ID: rules.RuleToResult("b_rule"), Violations: []string{"violation"}},
-	}
+	t.Run("returns a report", func(t *testing.T) {
+		resultCollection := &results.ResultCollection{
+			&results.Result{ID: rules.RuleToResult("a_rule"), Violations: []string{}},
+			&results.Result{ID: rules.RuleToResult("b_rule"), Violations: []string{"violation"}},
+		}
 
-	got, err := NewReport(examplePackage, resultCollection)
-	if err != nil {
-		t.Fatalf("NewReport returned error: %v", err)
-	}
+		got, err := NewReport(examplePackage, resultCollection)
+		if err != nil {
+			t.Fatalf("NewReport returned error: %v", err)
+		}
 
-	want := api.Report{
-		PreflightVersion: version.PreflightVersion,
-		Package:          examplePackage.ID,
-		PackageInformation: api.PackageInformation{
-			Namespace:     examplePackage.Namespace,
-			ID:            examplePackage.ID,
-			Version:       examplePackage.PackageVersion,
-			SchemaVersion: examplePackage.SchemaVersion,
-		},
-		Name:        examplePackage.Name,
-		Description: examplePackage.Description,
-		Sections: []api.ReportSection{
-			api.ReportSection{
-				ID:   "a_section",
-				Name: "My section",
-				Rules: []api.ReportRule{
-					api.ReportRule{
-						ID:         "a_rule",
-						Name:       "My Rule A",
-						Manual:     false,
-						Success:    true,
-						Missing:    false,
-						Links:      []string{},
-						Violations: []string{},
-					},
-					api.ReportRule{
-						ID:         "b_rule",
-						Name:       "My Rule B",
-						Manual:     false,
-						Success:    false,
-						Missing:    false,
-						Links:      []string{},
-						Violations: []string{"violation"},
+		want := api.Report{
+			PreflightVersion: version.PreflightVersion,
+			Package:          examplePackage.ID,
+			PackageInformation: api.PackageInformation{
+				Namespace:     examplePackage.Namespace,
+				ID:            examplePackage.ID,
+				Version:       examplePackage.PackageVersion,
+				SchemaVersion: examplePackage.SchemaVersion,
+			},
+			Name:        examplePackage.Name,
+			Description: examplePackage.Description,
+			Sections: []api.ReportSection{
+				api.ReportSection{
+					ID:   "a_section",
+					Name: "My section",
+					Rules: []api.ReportRule{
+						api.ReportRule{
+							ID:         "a_rule",
+							Name:       "My Rule A",
+							Manual:     false,
+							Success:    true,
+							Missing:    false,
+							Links:      []string{},
+							Violations: []string{},
+						},
+						api.ReportRule{
+							ID:         "b_rule",
+							Name:       "My Rule B",
+							Manual:     false,
+							Success:    false,
+							Missing:    false,
+							Links:      []string{},
+							Violations: []string{"violation"},
+						},
 					},
-					api.ReportRule{
-						ID:         "c_rule",
-						Name:       "My Rule C (missing)",
-						Manual:     false,
-						Success:    false,
-						Missing:    true,
-						Links:      []string{},
-						Violations: []string{},
+				},
+			},
+		}
+
+		if !reflect.DeepEqual(got, want) {
+			t.Fatalf("got != want; got=%+v, want=%+v", got, want)
+		}
+	})
+
+	t.Run("returns error MissingRegoDefinitionError if definition missing", func(t *testing.T) {
+		resultCollection := &results.ResultCollection{
+			&results.Result{ID: rules.RuleToResult("a_rule"), Violations: []string{}},
+		}
+
+		got, err := NewReport(examplePackage, resultCollection)
+		if _, ok := err.(*MissingRegoDefinitionError); !ok {
+			t.Errorf("expected MissingRegoDefinitionError but got: %v", err)
+		}
+
+		want := api.Report{
+			PreflightVersion: version.PreflightVersion,
+			Package:          examplePackage.ID,
+			PackageInformation: api.PackageInformation{
+				Namespace:     examplePackage.Namespace,
+				ID:            examplePackage.ID,
+				Version:       examplePackage.PackageVersion,
+				SchemaVersion: examplePackage.SchemaVersion,
+			},
+			Name:        examplePackage.Name,
+			Description: examplePackage.Description,
+			Sections: []api.ReportSection{
+				api.ReportSection{
+					ID:   "a_section",
+					Name: "My section",
+					Rules: []api.ReportRule{
+						api.ReportRule{
+							ID:         "a_rule",
+							Name:       "My Rule A",
+							Manual:     false,
+							Success:    true,
+							Missing:    false,
+							Links:      []string{},
+							Violations: []string{},
+						},
+						api.ReportRule{
+							ID:         "b_rule",
+							Name:       "My Rule B",
+							Manual:     false,
+							Success:    false,
+							Missing:    true,
+							Links:      []string{},
+							Violations: []string{},
+						},
 					},
 				},
 			},
-		},
-	}
+		}
 
-	if !reflect.DeepEqual(got, want) {
-		t.Fatalf("got != want; got=%+v, want=%+v", got, want)
-	}
+		if !reflect.DeepEqual(got, want) {
+			t.Fatalf("got != want; got=%+v, want=%+v", got, want)
+		}
+	})
 }