Skip to content

Commit

Permalink
adds support to scan directory with all iac providers in cli mode (#674)
Browse files Browse the repository at this point in the history 
* initial changes
* fix unit tests
* run dir scan and engine evaluation concurrently
* e2e test fix
  • Loading branch information
@patilpankaj212
patilpankaj212 committed Apr 30, 2021
1 parent d8fd9c4 commit 762c561
Show file tree
Hide file tree
Showing 28 changed files with 562 additions and 221 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
Expand Up @@ -13,6 +13,7 @@ require (
github.com/gorilla/mux v1.8.0
github.com/hashicorp/go-cleanhttp v0.5.1
github.com/hashicorp/go-getter v1.5.1
github.com/hashicorp/go-multierror v1.0.0
github.com/hashicorp/go-retryablehttp v0.6.6
github.com/hashicorp/go-version v1.2.1
github.com/hashicorp/hcl v1.0.0
Expand Down
21 changes: 15 additions & 6 deletions pkg/iac-providers/helm/v3/load-dir.go
View file
Expand Up @@ -25,7 +25,9 @@ import (

k8sv1 "github.com/accurics/terrascan/pkg/iac-providers/kubernetes/v1"
"github.com/accurics/terrascan/pkg/iac-providers/output"
"github.com/accurics/terrascan/pkg/results"
"github.com/accurics/terrascan/pkg/utils"
"github.com/hashicorp/go-multierror"
"go.uber.org/zap"
"gopkg.in/yaml.v3"
"helm.sh/helm/v3/pkg/chart"
Expand All @@ -34,9 +36,10 @@ import (
)

var (
errSkipTestDir = fmt.Errorf("skipping test directory")
errBadChartName = fmt.Errorf("invalid chart name in Chart.yaml")
errBadChartVersion = fmt.Errorf("invalid chart version in Chart.yaml")
errSkipTestDir = fmt.Errorf("skipping test directory")
errBadChartName = fmt.Errorf("invalid chart name in Chart.yaml")
errBadChartVersion = fmt.Errorf("invalid chart version in Chart.yaml")
errIacLoadDirs *multierror.Error = nil
)

// LoadIacDir loads all helm charts under the specified directory
Expand All @@ -48,12 +51,13 @@ func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error
fileMap, err := utils.FindFilesBySuffix(absRootDir, h.getHelmChartFilenames())
if err != nil {
zap.S().Debug("error while searching for helm charts", zap.String("root dir", absRootDir), zap.Error(err))
return allResourcesConfig, err
return allResourcesConfig, multierror.Append(errIacLoadDirs, results.DirScanErr{IacType: "helm", Directory: absRootDir, ErrMessage: err.Error()})
}

if len(fileMap) == 0 {
errMsg := fmt.Sprintf("no helm charts found in directory %s", absRootDir)
zap.S().Debug(zap.String("root dir", absRootDir), zap.Error(err))
return allResourcesConfig, fmt.Errorf("no helm charts found in directory %s", absRootDir)
return allResourcesConfig, multierror.Append(errIacLoadDirs, results.DirScanErr{IacType: "helm", Directory: absRootDir, ErrMessage: errMsg})
}

// fileDir now contains the chart path
Expand All @@ -68,7 +72,9 @@ func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error
var chartMap helmChartData
iacDocuments, chartMap, err = h.loadChart(chartPath)
if err != nil && err != errSkipTestDir {
errMsg := fmt.Sprintf("error occurred while loading chart. err: %v", err)
logger.Debug("error occurred while loading chart", zap.Error(err))
errIacLoadDirs = multierror.Append(errIacLoadDirs, results.DirScanErr{IacType: "helm", Directory: fileDir, ErrMessage: errMsg})
continue
}

Expand All @@ -78,7 +84,9 @@ func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error
var config *output.ResourceConfig
config, err = h.createHelmChartResource(chartPath, chartMap)
if err != nil {
errMsg := fmt.Sprintf("failed to create helm chart resource. err: %v", err)
logger.Error("failed to create helm chart resource", zap.Any("config", config), zap.Error(err))
errIacLoadDirs = multierror.Append(errIacLoadDirs, results.DirScanErr{IacType: "helm", Directory: fileDir, ErrMessage: errMsg})
continue
}

Expand All @@ -97,6 +105,7 @@ func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error
// in that case, we should not output an error as it was the user's intention to prevent rendering the resource
if err != k8sv1.ErrNoKind {
zap.S().Error("unable to normalize data", zap.Error(err), zap.String("file", doc.FilePath))
errIacLoadDirs = multierror.Append(errIacLoadDirs, err)
}
continue
}
Expand All @@ -108,7 +117,7 @@ func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error
}
}

return allResourcesConfig, nil
return allResourcesConfig, errIacLoadDirs
}

// createHelmChartResource returns normalized Helm Chart resource data
Expand Down
15 changes: 12 additions & 3 deletions pkg/iac-providers/helm/v3/load-dir_test.go
View file
Expand Up @@ -29,6 +29,7 @@ import (

"github.com/accurics/terrascan/pkg/iac-providers/output"
"github.com/accurics/terrascan/pkg/utils"
"github.com/hashicorp/go-multierror"
)

var testDataDir = "testdata"
Expand Down Expand Up @@ -64,22 +65,30 @@ func TestLoadIacDir(t *testing.T) {
name: "bad directory",
dirPath: filepath.Join(testDataDir, "bad-dir"),
helmv3: HelmV3{},
wantErr: invalidDirErr,
wantErr: multierror.Append(invalidDirErr),
resourceCount: 0,
},
{
name: "no helm charts in directory",
dirPath: filepath.Join(testDataDir, "no-helm-charts"),
helmv3: HelmV3{},
wantErr: fmt.Errorf("no helm charts found in directory %s", filepath.Join(testDataDir, "no-helm-charts")),
wantErr: multierror.Append(fmt.Errorf("no helm charts found in directory %s", filepath.Join(testDataDir, "no-helm-charts"))),
resourceCount: 0,
},
}

for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
resources, gotErr := tt.helmv3.LoadIacDir(tt.dirPath)
if !reflect.DeepEqual(gotErr, tt.wantErr) {
me, ok := gotErr.(*multierror.Error)
if !ok {
t.Errorf("expected multierror.Error, got %T", gotErr)
}
if tt.wantErr == nil {
if err := me.ErrorOrNil(); err != nil {
t.Errorf("unexpected error; gotErr: '%v', wantErr: '%v'", gotErr, tt.wantErr)
}
} else if me.Error() != tt.wantErr.Error() {
t.Errorf("unexpected error; gotErr: '%v', wantErr: '%v'", gotErr, tt.wantErr)
}

Expand Down
13 changes: 11 additions & 2 deletions pkg/iac-providers/kubernetes/v1/load-dir.go
View file
@@ -1,13 +1,20 @@
package k8sv1

import (
"fmt"
"path/filepath"
"strings"

"go.uber.org/zap"

"github.com/accurics/terrascan/pkg/iac-providers/output"
"github.com/accurics/terrascan/pkg/results"
"github.com/accurics/terrascan/pkg/utils"
"github.com/hashicorp/go-multierror"
)

var (
errIacLoadDirs *multierror.Error = nil
)

func (*K8sV1) getFileType(file string) string {
Expand All @@ -29,7 +36,7 @@ func (k *K8sV1) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error)
fileMap, err := utils.FindFilesBySuffix(absRootDir, K8sFileExtensions())
if err != nil {
zap.S().Debug("error while searching for iac files", zap.String("root dir", absRootDir), zap.Error(err))
return allResourcesConfig, err
return allResourcesConfig, multierror.Append(errIacLoadDirs, results.DirScanErr{IacType: "k8s", Directory: absRootDir, ErrMessage: err.Error()})
}

for fileDir, files := range fileMap {
Expand All @@ -38,7 +45,9 @@ func (k *K8sV1) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error)

var configData output.AllResourceConfigs
if configData, err = k.LoadIacFile(file); err != nil {
errMsg := fmt.Sprintf("error while loading iac file '%s'. err: %v", file, err)
zap.S().Debug("error while loading iac files", zap.String("IAC file", file), zap.Error(err))
errIacLoadDirs = multierror.Append(errIacLoadDirs, results.DirScanErr{IacType: "k8s", Directory: fileDir, ErrMessage: errMsg})
continue
}

Expand All @@ -52,7 +61,7 @@ func (k *K8sV1) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error)
}
}

return allResourcesConfig, nil
return allResourcesConfig, errIacLoadDirs
}

// makeSourcePathRelative modifies the source path of each resource from absolute to relative path
Expand Down
16 changes: 12 additions & 4 deletions pkg/iac-providers/kubernetes/v1/load-dir_test.go
View file
Expand Up @@ -20,12 +20,12 @@ import (
"fmt"
"os"
"path/filepath"
"reflect"
"syscall"
"testing"

"github.com/accurics/terrascan/pkg/iac-providers/output"
"github.com/accurics/terrascan/pkg/utils"
"github.com/hashicorp/go-multierror"
)

func TestLoadIacDir(t *testing.T) {
Expand All @@ -46,7 +46,7 @@ func TestLoadIacDir(t *testing.T) {
name: "empty config",
dirPath: filepath.Join(testDataDir, "testfile"),
k8sV1: K8sV1{},
wantErr: fmt.Errorf("no directories found for path %s", filepath.Join(testDataDir, "testfile")),
wantErr: multierror.Append(fmt.Errorf("no directories found for path %s", filepath.Join(testDataDir, "testfile"))),
},
{
name: "load invalid config dir",
Expand All @@ -58,7 +58,7 @@ func TestLoadIacDir(t *testing.T) {
name: "invalid dirPath",
dirPath: "not-there",
k8sV1: K8sV1{},
wantErr: invalidDirErr,
wantErr: multierror.Append(invalidDirErr),
},
{
name: "yaml with multiple documents",
Expand Down Expand Up @@ -89,7 +89,15 @@ func TestLoadIacDir(t *testing.T) {
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
_, gotErr := tt.k8sV1.LoadIacDir(tt.dirPath)
if !reflect.DeepEqual(gotErr, tt.wantErr) {
me, ok := gotErr.(*multierror.Error)
if !ok {
t.Errorf("expected multierror.Error, got %T", gotErr)
}
if tt.wantErr == nil {
if err := me.ErrorOrNil(); err != nil {
t.Errorf("unexpected error; gotErr: '%v', wantErr: '%v'", gotErr, tt.wantErr)
}
} else if me.Error() != tt.wantErr.Error() {
t.Errorf("unexpected error; gotErr: '%v', wantErr: '%v'", gotErr, tt.wantErr)
}
})
Expand Down
20 changes: 13 additions & 7 deletions pkg/iac-providers/kustomize/v3/load-dir.go
View file
Expand Up @@ -6,7 +6,9 @@ import (

k8sv1 "github.com/accurics/terrascan/pkg/iac-providers/kubernetes/v1"
"github.com/accurics/terrascan/pkg/iac-providers/output"
"github.com/accurics/terrascan/pkg/results"
"github.com/accurics/terrascan/pkg/utils"
"github.com/hashicorp/go-multierror"
"go.uber.org/zap"
"sigs.k8s.io/kustomize/api/filesys"
"sigs.k8s.io/kustomize/api/krusty"
Expand All @@ -17,7 +19,8 @@ const (
)

var (
kustomizeErrMessage = "error from kustomization. error : %v"
kustomizeErrMessage = "error from kustomization. error : %v"
errIacLoadDirs *multierror.Error = nil
)

// LoadIacDir loads the kustomize directory and returns the ResourceConfig mapping which is evaluated by the policy engine
Expand All @@ -28,17 +31,19 @@ func (k *KustomizeV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs,
files, err := utils.FindFilesBySuffixInDir(absRootDir, KustomizeFileNames())
if err != nil {
zap.S().Debug("error while searching for iac files", zap.String("root dir", absRootDir), zap.Error(err))
return allResourcesConfig, err
return allResourcesConfig, multierror.Append(errIacLoadDirs, results.DirScanErr{IacType: "kustomize", Directory: absRootDir, ErrMessage: err.Error()})
}

if len(files) == 0 {
errMsg := fmt.Sprintf("kustomization.y(a)ml file not found in the directory %s", absRootDir)
zap.S().Debug("error while searching for iac files", zap.String("root dir", absRootDir), zap.Error(err))
return allResourcesConfig, fmt.Errorf("kustomization.y(a)ml file not found in the directory %s", absRootDir)
return allResourcesConfig, multierror.Append(errIacLoadDirs, results.DirScanErr{IacType: "kustomize", Directory: absRootDir, ErrMessage: errMsg})
}

if len(files) > 1 {
errMsg := fmt.Sprintf("multiple kustomization.y(a)ml found in the directory %s", absRootDir)
zap.S().Debug("error while searching for iac files", zap.String("root dir", absRootDir), zap.Error(err))
return allResourcesConfig, fmt.Errorf("multiple kustomization.y(a)ml found in the directory %s", absRootDir)
return allResourcesConfig, multierror.Append(errIacLoadDirs, results.DirScanErr{IacType: "kustomize", Directory: absRootDir, ErrMessage: errMsg})
}

kustomizeFileName := *files[0]
Expand All @@ -47,7 +52,7 @@ func (k *KustomizeV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs,
if err != nil {
err = fmt.Errorf("unable to read the kustomization file in the directory %s, error: %v", absRootDir, err)
zap.S().Error("error while reading the file", kustomizeFileName, zap.Error(err))
return allResourcesConfig, err
return allResourcesConfig, multierror.Append(errIacLoadDirs, results.DirScanErr{IacType: "kustomize", Directory: absRootDir, ErrMessage: err.Error()})
}

// ResourceConfig representing the kustomization.y(a)ml file
Expand All @@ -65,8 +70,9 @@ func (k *KustomizeV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs,
// obtaining list of IacDocuments from the target working directory
iacDocuments, err := LoadKustomize(absRootDir, kustomizeFileName)
if err != nil {
errMsg := fmt.Sprintf("error occurred while loading kustomize directory '%s'. err: %v", absRootDir, err)
zap.S().Error("error occurred while loading kustomize directory", zap.String("kustomize directory", absRootDir), zap.Error(err))
return nil, err
return nil, multierror.Append(errIacLoadDirs, results.DirScanErr{IacType: "kustomize", Directory: absRootDir, ErrMessage: errMsg})
}

for _, doc := range iacDocuments {
Expand All @@ -85,7 +91,7 @@ func (k *KustomizeV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs,
allResourcesConfig[config.Type] = append(allResourcesConfig[config.Type], *config)
}

return allResourcesConfig, nil
return allResourcesConfig, errIacLoadDirs
}

// LoadKustomize loads up a 'kustomized' directory and returns a returns a list of IacDocuments
Expand Down
20 changes: 14 additions & 6 deletions pkg/iac-providers/kustomize/v3/load-dir_test.go
View file
Expand Up @@ -10,7 +10,7 @@ import (
"testing"

"github.com/accurics/terrascan/pkg/iac-providers/output"
"github.com/accurics/terrascan/pkg/utils"
"github.com/hashicorp/go-multierror"
)

const kustomizeErrPrefix = "error from kustomization."
Expand All @@ -32,7 +32,7 @@ func TestLoadIacDir(t *testing.T) {
name: "invalid dirPath",
dirPath: "not-there",
kustomize: KustomizeV3{},
wantErr: &os.PathError{Err: syscall.ENOENT, Op: "open", Path: "not-there"},
wantErr: multierror.Append(&os.PathError{Err: syscall.ENOENT, Op: "open", Path: "not-there"}),
resourceCount: 0,
},
{
Expand Down Expand Up @@ -76,26 +76,34 @@ func TestLoadIacDir(t *testing.T) {
name: "no-kustomize-directory",
dirPath: filepath.Join(testDataDir, "no-kustomizefile"),
kustomize: KustomizeV3{},
wantErr: fmt.Errorf("kustomization.y(a)ml file not found in the directory %s", filepath.Join(testDataDir, "no-kustomizefile")),
wantErr: multierror.Append(fmt.Errorf("kustomization.y(a)ml file not found in the directory %s", filepath.Join(testDataDir, "no-kustomizefile"))),
resourceCount: 0,
},
{
name: "kustomize-file-empty",
dirPath: filepath.Join(testDataDir, "kustomize-file-empty"),
kustomize: KustomizeV3{},
wantErr: fmt.Errorf("unable to read the kustomization file in the directory %s, error: yaml file is empty", filepath.Join(testDataDir, "kustomize-file-empty")),
wantErr: multierror.Append(fmt.Errorf("unable to read the kustomization file in the directory %s, error: yaml file is empty", filepath.Join(testDataDir, "kustomize-file-empty"))),
resourceCount: 0,
},
}

for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
resourceMap, gotErr := tt.kustomize.LoadIacDir(tt.dirPath)
if !reflect.DeepEqual(gotErr, tt.wantErr) {
me, ok := gotErr.(*multierror.Error)
if !ok {
t.Errorf("expected multierror.Error, got %T", gotErr)
}
if tt.wantErr == nil {
if err := me.ErrorOrNil(); err != nil {
t.Errorf("unexpected error; gotErr: '%v', wantErr: '%v'", gotErr, tt.wantErr)
}
} else if me.Error() != tt.wantErr.Error() {
t.Errorf("unexpected error; gotErr: '%v', wantErr: '%v'", gotErr, tt.wantErr)
}

resCount := utils.GetResourceCount(resourceMap)
resCount := resourceMap.GetResourceCount()
if resCount != tt.resourceCount {
t.Errorf("resource count (%d) does not match expected (%d)", resCount, tt.resourceCount)
}
Expand Down

0 comments on commit 762c561

Please sign in to comment.