From b9f8d952cf33b560657281012869e291bbd0591e Mon Sep 17 00:00:00 2001 From: lhitchon Date: Sat, 29 Sep 2018 08:48:38 -0700 Subject: [PATCH] add CSV linter --- assertion/types.go | 6 ++++ linter/common.go | 10 ++++++ linter/csv_resource_loader.go | 52 +++++++++++++++++++++++++++ linter/csv_resource_loader_test.go | 32 +++++++++++++++++ linter/json_resource_loader.go | 2 +- linter/json_resource_loader_test.go | 5 +-- linter/linter.go | 2 ++ linter/linter_test.go | 1 + linter/testdata/resources/users.csv | 3 ++ linter/testdata/rules/generic-csv.yml | 17 +++++++++ linter/yaml_resource_loader_test.go | 22 ++++-------- 11 files changed, 131 insertions(+), 21 deletions(-) create mode 100644 linter/csv_resource_loader.go create mode 100644 linter/csv_resource_loader_test.go create mode 100644 linter/testdata/resources/users.csv create mode 100644 linter/testdata/rules/generic-csv.yml diff --git a/assertion/types.go b/assertion/types.go index 98e83c1..55ce3e2 100644 --- a/assertion/types.go +++ b/assertion/types.go @@ -20,6 +20,7 @@ type ( Rules []Rule Version string Resources []ResourceConfig + Columns []ColumnConfig } // Rule is part of a RuleSet @@ -78,6 +79,11 @@ type ( Key string } + // ColumnConfig describes how to discover resources in a CSV file + ColumnConfig struct { + Name string + } + // ValidationReport summarizes validation for resources using rules ValidationReport struct { FilesScanned []string diff --git a/linter/common.go b/linter/common.go index f108e02..ad0aff6 100644 --- a/linter/common.go +++ b/linter/common.go @@ -1,12 +1,14 @@ package linter import ( + "encoding/csv" "encoding/json" "github.com/ghodss/yaml" "github.com/stelligent/config-lint/assertion" "io/ioutil" "os" "path/filepath" + "strings" ) func readContent(filename string) ([]byte, error) { @@ -48,6 +50,14 @@ func loadJSON(filename string) ([]interface{}, error) { return []interface{}{m}, nil } +func loadCSV(filename string) ([][]string, error) { + content, err := readContent(filename) + if err != nil { + return [][]string{}, err + } + return csv.NewReader(strings.NewReader(string(content))).ReadAll() +} + func getResourceIDFromFilename(filename string) string { _, resourceID := filepath.Split(filename) return resourceID diff --git a/linter/csv_resource_loader.go b/linter/csv_resource_loader.go new file mode 100644 index 0000000..477f428 --- /dev/null +++ b/linter/csv_resource_loader.go @@ -0,0 +1,52 @@ +package linter + +import ( + "github.com/stelligent/config-lint/assertion" + "path/filepath" +) + +// CSVResourceLoader loads a list of Resource objects based on the list of ResourceConfig objects +type CSVResourceLoader struct { + Columns []assertion.ColumnConfig +} + +func extractCSVResourceID(expression string, properties interface{}) string { + resourceID := "None" + result, err := assertion.SearchData(expression, properties) + if err == nil { + resourceID, _ = result.(string) + } + return resourceID +} + +// Load converts a text file into a collection of Resource objects +func (l CSVResourceLoader) Load(filename string) (FileResources, error) { + loaded := FileResources{ + Resources: make([]assertion.Resource, 0), + } + csvRows, err := loadCSV(filename) + if err != nil { + return loaded, err + } + for rowNumber, row := range csvRows { + properties := map[string]interface{}{} + properties["__file__"] = filename + properties["__dir__"] = filepath.Dir(filename) + for columnNumber, columnConfig := range l.Columns { + properties[columnConfig.Name] = row[columnNumber] + } + resource := assertion.Resource{ + ID: string(rowNumber), + Type: "row", + Properties: properties, + Filename: filename, + } + loaded.Resources = append(loaded.Resources, resource) + } + return loaded, nil +} + +// PostLoad does no additional processing fro a CSVResourceLoader +func (l CSVResourceLoader) PostLoad(r FileResources) ([]assertion.Resource, error) { + return r.Resources, nil +} diff --git a/linter/csv_resource_loader_test.go b/linter/csv_resource_loader_test.go new file mode 100644 index 0000000..c8618ca --- /dev/null +++ b/linter/csv_resource_loader_test.go @@ -0,0 +1,32 @@ +package linter + +import ( + "bytes" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestCSVLinterValidate(t *testing.T) { + options := Options{ + Tags: []string{}, + RuleIDs: []string{}, + } + ruleSet := loadRulesForTest("./testdata/rules/generic-csv.yml", t) + filenames := []string{"./testdata/resources/users.csv"} + loader := CSVResourceLoader{Columns: ruleSet.Columns} + linter := FileLinter{Filenames: filenames, ValueSource: TestingValueSource{}, Loader: loader} + report, err := linter.Validate(ruleSet, options) + assert.Nil(t, err, "Expecting Validate to run without error") + assert.Equal(t, 3, len(report.ResourcesScanned), "Expecting Validate to scan 3 resources") + assert.Equal(t, 1, len(report.Violations), "Expecting Validate to find 1 violation") +} + +func TestCSVLinterSearch(t *testing.T) { + ruleSet := loadRulesForTest("./testdata/rules/generic-csv.yml", t) + filenames := []string{"./testdata/resources/users.csv"} + loader := CSVResourceLoader{Columns: ruleSet.Columns} + linter := FileLinter{Filenames: filenames, ValueSource: TestingValueSource{}, Loader: loader} + var b bytes.Buffer + linter.Search(ruleSet, "Department", &b) + assert.Contains(t, b.String(), "Audit", "Expecting TestCSVLinterSearch to find string in output") +} diff --git a/linter/json_resource_loader.go b/linter/json_resource_loader.go index 03b2331..54695b1 100644 --- a/linter/json_resource_loader.go +++ b/linter/json_resource_loader.go @@ -54,7 +54,7 @@ func (l JSONResourceLoader) Load(filename string) (FileResources, error) { return loaded, nil } -// PostLoad does no additional processing fro a YAMLResourceLoader +// PostLoad does no additional processing fro a JSONResourceLoader func (l JSONResourceLoader) PostLoad(r FileResources) ([]assertion.Resource, error) { return r.Resources, nil } diff --git a/linter/json_resource_loader_test.go b/linter/json_resource_loader_test.go index 26b592b..0371c18 100644 --- a/linter/json_resource_loader_test.go +++ b/linter/json_resource_loader_test.go @@ -3,7 +3,6 @@ package linter import ( "bytes" "github.com/stretchr/testify/assert" - "strings" "testing" ) @@ -30,7 +29,5 @@ func TestJSONLinterSearch(t *testing.T) { linter := FileLinter{Filenames: filenames, ValueSource: TestingValueSource{}, Loader: loader} var b bytes.Buffer linter.Search(ruleSet, "Department", &b) - if !strings.Contains(b.String(), "Audit") { - t.Error("Expecting TestJSONLinterSearch to find string in output") - } + assert.Contains(t, b.String(), "Audit", "Expecting TestJSONLinterSearch to find string in output") } diff --git a/linter/linter.go b/linter/linter.go index 425ac97..dcd1447 100644 --- a/linter/linter.go +++ b/linter/linter.go @@ -39,6 +39,8 @@ func NewLinter(ruleSet assertion.RuleSet, vs assertion.ValueSource, filenames [] return FileLinter{Filenames: filenames, ValueSource: vs, Loader: YAMLResourceLoader{Resources: ruleSet.Resources}}, nil case "JSON": return FileLinter{Filenames: filenames, ValueSource: vs, Loader: JSONResourceLoader{Resources: ruleSet.Resources}}, nil + case "CSV": + return FileLinter{Filenames: filenames, ValueSource: vs, Loader: CSVResourceLoader{Columns: ruleSet.Columns}}, nil default: return nil, fmt.Errorf("Type not supported: %s", ruleSet.Type) } diff --git a/linter/linter_test.go b/linter/linter_test.go index 305ac75..4047413 100644 --- a/linter/linter_test.go +++ b/linter/linter_test.go @@ -17,6 +17,7 @@ func TestNewLinter(t *testing.T) { {"./testdata/rules/terraform_instance.yml", "FileLinter"}, {"./testdata/rules/generic-yaml.yml", "FileLinter"}, {"./testdata/rules/generic-json.yml", "FileLinter"}, + {"./testdata/rules/generic-csv.yml", "FileLinter"}, {"./testdata/rules/aws_sg_resource.yml", "AWSResourceLinter"}, {"./testdata/rules/aws_iam_resource.yml", "AWSResourceLinter"}, {"./testdata/rules/kubernetes.yml", "FileLinter"}, diff --git a/linter/testdata/resources/users.csv b/linter/testdata/resources/users.csv new file mode 100644 index 0000000..65435e1 --- /dev/null +++ b/linter/testdata/resources/users.csv @@ -0,0 +1,3 @@ +admin,Admin +readonly,Audit +user1, diff --git a/linter/testdata/rules/generic-csv.yml b/linter/testdata/rules/generic-csv.yml new file mode 100644 index 0000000..08954e3 --- /dev/null +++ b/linter/testdata/rules/generic-csv.yml @@ -0,0 +1,17 @@ +version: 1 +description: Rules for users in CSV file +type: CSV +files: + - "*.csv" + +columns: + - name: User + - name: Department + +rules: + - id: DEPARTMENT_REQUIRED + message: User must have a department + resource: row + assertions: + - key: Department + op: not-empty diff --git a/linter/yaml_resource_loader_test.go b/linter/yaml_resource_loader_test.go index 661cbe5..035b609 100644 --- a/linter/yaml_resource_loader_test.go +++ b/linter/yaml_resource_loader_test.go @@ -2,7 +2,7 @@ package linter import ( "bytes" - "strings" + "github.com/stretchr/testify/assert" "testing" ) @@ -16,18 +16,10 @@ func TestYAMLLinterValidate(t *testing.T) { loader := YAMLResourceLoader{Resources: ruleSet.Resources} linter := FileLinter{Filenames: filenames, ValueSource: TestingValueSource{}, Loader: loader} report, err := linter.Validate(ruleSet, options) - if err != nil { - t.Error("Expecting TestYAMLLinter to not return an error") - } - if len(report.ResourcesScanned) != 17 { - t.Errorf("TestYAMLLinter scanned %d resources, expecting 17", len(report.ResourcesScanned)) - } - if len(report.FilesScanned) != 1 { - t.Errorf("TestYAMLLinter scanned %d files, expecting 1", len(report.FilesScanned)) - } - if len(report.Violations) != 3 { - t.Errorf("TestYAMLLinter returned %d violations, expecting 3", len(report.Violations)) - } + assert.Nil(t, err, "Expecting Validate to run without error") + assert.Equal(t, 17, len(report.ResourcesScanned), "Expecting Validate to scan 17 resources") + assert.Equal(t, 1, len(report.FilesScanned), "Expecting Validate to scan 1 file") + assert.Equal(t, 3, len(report.Violations), "Expecting Validate to find 3 violations") } func TestYAMLLinterSearch(t *testing.T) { @@ -37,7 +29,5 @@ func TestYAMLLinterSearch(t *testing.T) { linter := FileLinter{Filenames: filenames, ValueSource: TestingValueSource{}, Loader: loader} var b bytes.Buffer linter.Search(ruleSet, "name", &b) - if !strings.Contains(b.String(), "gadget") { - t.Error("Expecting TestYAMLLinterSearch to find string in output") - } + assert.Contains(t, b.String(), "gadget", "Expecting TestYAMLLinterSearch to find string in output") }