add CSV linter

assertion/types.go

@@ -20,6 +20,7 @@ type (
 		Rules       []Rule
 		Version     string
 		Resources   []ResourceConfig
+		Columns     []ColumnConfig
 	}
 
 	// Rule is part of a RuleSet
@@ -78,6 +79,11 @@ type (
 		Key  string
 	}
 
+	// ColumnConfig describes how to discover resources in a CSV file
+	ColumnConfig struct {
+		Name string
+	}
+
 	// ValidationReport summarizes validation for resources using rules
 	ValidationReport struct {
 		FilesScanned     []string

linter/common.go

@@ -1,12 +1,14 @@
 package linter
 
 import (
+	"encoding/csv"
 	"encoding/json"
 	"github.com/ghodss/yaml"
 	"github.com/stelligent/config-lint/assertion"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 func readContent(filename string) ([]byte, error) {
@@ -48,6 +50,14 @@ func loadJSON(filename string) ([]interface{}, error) {
 	return []interface{}{m}, nil
 }
 
+func loadCSV(filename string) ([][]string, error) {
+	content, err := readContent(filename)
+	if err != nil {
+		return [][]string{}, err
+	}
+	return csv.NewReader(strings.NewReader(string(content))).ReadAll()
+}
+
 func getResourceIDFromFilename(filename string) string {
 	_, resourceID := filepath.Split(filename)
 	return resourceID

linter/csv_resource_loader.go

@@ -0,0 +1,52 @@
+package linter
+
+import (
+	"github.com/stelligent/config-lint/assertion"
+	"path/filepath"
+)
+
+// CSVResourceLoader loads a list of Resource objects based on the list of ResourceConfig objects
+type CSVResourceLoader struct {
+	Columns []assertion.ColumnConfig
+}
+
+func extractCSVResourceID(expression string, properties interface{}) string {
+	resourceID := "None"
+	result, err := assertion.SearchData(expression, properties)
+	if err == nil {
+		resourceID, _ = result.(string)
+	}
+	return resourceID
+}
+
+// Load converts a text file into a collection of Resource objects
+func (l CSVResourceLoader) Load(filename string) (FileResources, error) {
+	loaded := FileResources{
+		Resources: make([]assertion.Resource, 0),
+	}
+	csvRows, err := loadCSV(filename)
+	if err != nil {
+		return loaded, err
+	}
+	for rowNumber, row := range csvRows {
+		properties := map[string]interface{}{}
+		properties["__file__"] = filename
+		properties["__dir__"] = filepath.Dir(filename)
+		for columnNumber, columnConfig := range l.Columns {
+			properties[columnConfig.Name] = row[columnNumber]
+		}
+		resource := assertion.Resource{
+			ID:         string(rowNumber),
+			Type:       "row",
+			Properties: properties,
+			Filename:   filename,
+		}
+		loaded.Resources = append(loaded.Resources, resource)
+	}
+	return loaded, nil
+}
+
+// PostLoad does no additional processing fro a CSVResourceLoader
+func (l CSVResourceLoader) PostLoad(r FileResources) ([]assertion.Resource, error) {
+	return r.Resources, nil
+}

linter/csv_resource_loader_test.go

@@ -0,0 +1,32 @@
+package linter
+
+import (
+	"bytes"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestCSVLinterValidate(t *testing.T) {
+	options := Options{
+		Tags:    []string{},
+		RuleIDs: []string{},
+	}
+	ruleSet := loadRulesForTest("./testdata/rules/generic-csv.yml", t)
+	filenames := []string{"./testdata/resources/users.csv"}
+	loader := CSVResourceLoader{Columns: ruleSet.Columns}
+	linter := FileLinter{Filenames: filenames, ValueSource: TestingValueSource{}, Loader: loader}
+	report, err := linter.Validate(ruleSet, options)
+	assert.Nil(t, err, "Expecting Validate to run without error")
+	assert.Equal(t, 3, len(report.ResourcesScanned), "Expecting Validate to scan 3 resources")
+	assert.Equal(t, 1, len(report.Violations), "Expecting Validate to find 1 violation")
+}
+
+func TestCSVLinterSearch(t *testing.T) {
+	ruleSet := loadRulesForTest("./testdata/rules/generic-csv.yml", t)
+	filenames := []string{"./testdata/resources/users.csv"}
+	loader := CSVResourceLoader{Columns: ruleSet.Columns}
+	linter := FileLinter{Filenames: filenames, ValueSource: TestingValueSource{}, Loader: loader}
+	var b bytes.Buffer
+	linter.Search(ruleSet, "Department", &b)
+	assert.Contains(t, b.String(), "Audit", "Expecting TestCSVLinterSearch to find string in output")
+}

linter/json_resource_loader.go

@@ -54,7 +54,7 @@ func (l JSONResourceLoader) Load(filename string) (FileResources, error) {
 	return loaded, nil
 }
 
-// PostLoad does no additional processing fro a YAMLResourceLoader
+// PostLoad does no additional processing fro a JSONResourceLoader
 func (l JSONResourceLoader) PostLoad(r FileResources) ([]assertion.Resource, error) {
 	return r.Resources, nil
 }

linter/json_resource_loader_test.go

@@ -3,7 +3,6 @@ package linter
 import (
 	"bytes"
 	"github.com/stretchr/testify/assert"
-	"strings"
 	"testing"
 )
 
@@ -30,7 +29,5 @@ func TestJSONLinterSearch(t *testing.T) {
 	linter := FileLinter{Filenames: filenames, ValueSource: TestingValueSource{}, Loader: loader}
 	var b bytes.Buffer
 	linter.Search(ruleSet, "Department", &b)
-	if !strings.Contains(b.String(), "Audit") {
-		t.Error("Expecting TestJSONLinterSearch to find string in output")
-	}
+	assert.Contains(t, b.String(), "Audit", "Expecting TestJSONLinterSearch to find string in output")
 }

linter/linter.go

@@ -39,6 +39,8 @@ func NewLinter(ruleSet assertion.RuleSet, vs assertion.ValueSource, filenames []
 		return FileLinter{Filenames: filenames, ValueSource: vs, Loader: YAMLResourceLoader{Resources: ruleSet.Resources}}, nil
 	case "JSON":
 		return FileLinter{Filenames: filenames, ValueSource: vs, Loader: JSONResourceLoader{Resources: ruleSet.Resources}}, nil
+	case "CSV":
+		return FileLinter{Filenames: filenames, ValueSource: vs, Loader: CSVResourceLoader{Columns: ruleSet.Columns}}, nil
 	default:
 		return nil, fmt.Errorf("Type not supported: %s", ruleSet.Type)
 	}

linter/linter_test.go

@@ -17,6 +17,7 @@ func TestNewLinter(t *testing.T) {
 		{"./testdata/rules/terraform_instance.yml", "FileLinter"},
 		{"./testdata/rules/generic-yaml.yml", "FileLinter"},
 		{"./testdata/rules/generic-json.yml", "FileLinter"},
+		{"./testdata/rules/generic-csv.yml", "FileLinter"},
 		{"./testdata/rules/aws_sg_resource.yml", "AWSResourceLinter"},
 		{"./testdata/rules/aws_iam_resource.yml", "AWSResourceLinter"},
 		{"./testdata/rules/kubernetes.yml", "FileLinter"},

linter/testdata/resources/users.csv

@@ -0,0 +1,3 @@
+admin,Admin
+readonly,Audit
+user1,

linter/testdata/rules/generic-csv.yml

@@ -0,0 +1,17 @@
+version: 1
+description: Rules for users in CSV file
+type: CSV
+files:
+  - "*.csv"
+
+columns:
+  - name: User
+  - name: Department
+
+rules:
+  - id: DEPARTMENT_REQUIRED
+    message: User must have a department
+    resource: row
+    assertions:
+      - key: Department
+        op: not-empty

linter/yaml_resource_loader_test.go

@@ -2,7 +2,7 @@ package linter
 
 import (
 	"bytes"
-	"strings"
+	"github.com/stretchr/testify/assert"
 	"testing"
 )
 
@@ -16,18 +16,10 @@ func TestYAMLLinterValidate(t *testing.T) {
 	loader := YAMLResourceLoader{Resources: ruleSet.Resources}
 	linter := FileLinter{Filenames: filenames, ValueSource: TestingValueSource{}, Loader: loader}
 	report, err := linter.Validate(ruleSet, options)
-	if err != nil {
-		t.Error("Expecting TestYAMLLinter to not return an error")
-	}
-	if len(report.ResourcesScanned) != 17 {
-		t.Errorf("TestYAMLLinter scanned %d resources, expecting 17", len(report.ResourcesScanned))
-	}
-	if len(report.FilesScanned) != 1 {
-		t.Errorf("TestYAMLLinter scanned %d files, expecting 1", len(report.FilesScanned))
-	}
-	if len(report.Violations) != 3 {
-		t.Errorf("TestYAMLLinter returned %d violations, expecting 3", len(report.Violations))
-	}
+	assert.Nil(t, err, "Expecting Validate to run without error")
+	assert.Equal(t, 17, len(report.ResourcesScanned), "Expecting Validate to scan 17 resources")
+	assert.Equal(t, 1, len(report.FilesScanned), "Expecting Validate to scan 1 file")
+	assert.Equal(t, 3, len(report.Violations), "Expecting Validate to find 3 violations")
 }
 
 func TestYAMLLinterSearch(t *testing.T) {
@@ -37,7 +29,5 @@ func TestYAMLLinterSearch(t *testing.T) {
 	linter := FileLinter{Filenames: filenames, ValueSource: TestingValueSource{}, Loader: loader}
 	var b bytes.Buffer
 	linter.Search(ruleSet, "name", &b)
-	if !strings.Contains(b.String(), "gadget") {
-		t.Error("Expecting TestYAMLLinterSearch to find string in output")
-	}
+	assert.Contains(t, b.String(), "gadget", "Expecting TestYAMLLinterSearch to find string in output")
 }