Skip to content

Commit

Permalink
enhancement: scan terraform registry modules as remote type (#513)
Browse files Browse the repository at this point in the history 
* support tf registry as remote type
* support version with remote terraform registry
* load-dir refactor and tests
* moved remote module utils to downloader pkg
* added wiki link in code for punycode
  • Loading branch information
@patilpankaj212
patilpankaj212 committed Feb 10, 2021
1 parent 9f3569b commit 26bae02
Show file tree
Hide file tree
Showing 16 changed files with 1,354 additions and 739 deletions.
7 changes: 7 additions & 0 deletions pkg/cli/register.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ package cli

import (
"flag"
"io/ioutil"
"log"
"os"

"github.com/accurics/terrascan/pkg/config"
Expand Down Expand Up @@ -59,6 +61,11 @@ func Execute() {
}
}

// disable terraform logs when TF_LOG env variable is not set
if os.Getenv("TF_LOG") == "" {
log.SetOutput(ioutil.Discard)
}

if err := rootCmd.Execute(); err != nil {
os.Exit(1)
}
Expand Down
29 changes: 28 additions & 1 deletion pkg/cli/run_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
package cli

import (
"io/ioutil"
"log"
"os"
"path/filepath"
"testing"
Expand Down Expand Up @@ -47,6 +49,11 @@ func shutdown() {
}

func TestRun(t *testing.T) {
// disable terraform logs when TF_LOG env variable is not set
if os.Getenv("TF_LOG") == "" {
log.SetOutput(ioutil.Discard)
}

testDirPath := "testdata/run-test"
kustomizeTestDirPath := testDirPath + "/kustomize-test"
testTerraformFilePath := testDirPath + "/config-only.tf"
Expand Down Expand Up @@ -174,14 +181,34 @@ func TestRun(t *testing.T) {
},
},
{
name: "config file with remote module",
name: "scan file with remote module",
scanOptions: &ScanOptions{
policyType: []string{"all"},
iacFilePath: testRemoteModuleFilePath,
outputType: "human",
configFile: "testdata/configFile.toml",
},
},
{
name: "invalid remote type",
scanOptions: &ScanOptions{
policyType: []string{"all"},
remoteType: "test",
remoteURL: "test",
outputType: "human",
},
wantErr: true,
},
{
name: "valid remote type with invalid remote url",
scanOptions: &ScanOptions{
policyType: []string{"all"},
remoteType: "terraform-registry",
remoteURL: "terraform-aws-modules/eks",
outputType: "human",
},
wantErr: true,
},
}

for _, tt := range table {
Expand Down
2 changes: 1 addition & 1 deletion pkg/cli/scan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func init() {
scanCmd.Flags().StringVarP(&scanOptions.iacFilePath, "iac-file", "f", "", "path to a single IaC file")
scanCmd.Flags().StringVarP(&scanOptions.iacDirPath, "iac-dir", "d", ".", "path to a directory containing one or more IaC files")
scanCmd.Flags().StringArrayVarP(&scanOptions.policyPath, "policy-path", "p", []string{}, "policy path directory")
scanCmd.Flags().StringVarP(&scanOptions.remoteType, "remote-type", "r", "", "type of remote backend (git, s3, gcs, http)")
scanCmd.Flags().StringVarP(&scanOptions.remoteType, "remote-type", "r", "", "type of remote backend (git, s3, gcs, http, terraform-registry)")
scanCmd.Flags().StringVarP(&scanOptions.remoteURL, "remote-url", "u", "", "url pointing to remote IaC repository")
scanCmd.Flags().BoolVarP(&scanOptions.configOnly, "config-only", "", false, "will output resource config (should only be used for debugging purposes)")
// flag passes a string, but we normalize to bool in PreRun
Expand Down
46 changes: 36 additions & 10 deletions pkg/downloader/getter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,24 +21,28 @@ import (
"path/filepath"

getter "github.com/hashicorp/go-getter"
"github.com/hashicorp/go-version"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/registry/regsrc"
"go.uber.org/zap"
)

// list of errors
var (
ErrEmptyURLType = fmt.Errorf("empty remote url and type")
ErrEmptyURLDest = fmt.Errorf("remote url or destination dir path cannot be empty")
ErrEmptyURLTypeDest = fmt.Errorf("empty remote url or type or desitnation dir path")
ErrEmptyURLType = fmt.Errorf("empty remote url and type")
ErrEmptyURLDest = fmt.Errorf("remote url or destination dir path cannot be empty")
ErrEmptyURLTypeDest = fmt.Errorf("empty remote url or type or desitnation dir path")
ErrInvalidRemoteType = fmt.Errorf("supplied remote type is not supported")
)

// NewGoGetter returns a new GoGetter struct
func NewGoGetter() *GoGetter {
return &GoGetter{}
// newGoGetter returns a new GoGetter struct
func newGoGetter() *goGetter {
return &goGetter{}
}

// GetURLSubDir returns the download URL with it's respective type prefix
// along with subDir path, if present.
func (g *GoGetter) GetURLSubDir(remoteURL, destPath string) (string, string, error) {
func (g *goGetter) GetURLSubDir(remoteURL, destPath string) (string, string, error) {

// get subDir, if present
repoURL, subDir := SplitAddrSubdir(remoteURL)
Expand Down Expand Up @@ -69,7 +73,7 @@ func (g *GoGetter) GetURLSubDir(remoteURL, destPath string) (string, string, err
// Download retrieves the remote repository referenced in the given remoteURL
// into the destination path and then returns the full path to any subdir
// indicated in the URL
func (g *GoGetter) Download(remoteURL, destPath string) (string, error) {
func (g *goGetter) Download(remoteURL, destPath string) (string, error) {

zap.S().Debugf("download with remote url: %q, destination dir: %q",
remoteURL, destPath)
Expand Down Expand Up @@ -127,7 +131,7 @@ func (g *GoGetter) Download(remoteURL, destPath string) (string, error) {
//
// DownloadWithType enforces download type on go-getter to get rid of any
// ambiguities in remoteURL
func (g *GoGetter) DownloadWithType(remoteType, remoteURL, destPath string) (string, error) {
func (g *goGetter) DownloadWithType(remoteType, remoteURL, destPath string) (string, error) {

zap.S().Debugf("download with remote type: %q, remote URL: %q, destination dir: %q",
remoteType, remoteURL, destPath)
Expand All @@ -144,14 +148,36 @@ func (g *GoGetter) DownloadWithType(remoteType, remoteURL, destPath string) (str
zap.S().Error(ErrEmptyURLTypeDest)
return "", ErrEmptyURLDest
}

if !IsValidRemoteType(remoteType) {
return "", ErrInvalidRemoteType
}

if IsRemoteTypeTerraformRegistry(remoteType) {
sourceAddr, ver := GetSourceAddrAndVersion(remoteURL)
if IsRegistrySourceAddr(sourceAddr) {
module, _ := regsrc.ParseModuleSource(sourceAddr)
versionConstraints := configs.VersionConstraint{}
if ver != "" {
versionConstraint, err := version.NewConstraint(ver)
if err != nil {
return "", err
}
versionConstraints.Required = versionConstraint
}
return NewRemoteDownloader().DownloadRemoteModule(versionConstraints, destPath, module)
}
return "", fmt.Errorf("%s, is not a valid terraform registry", remoteURL)
}

URLWithType := fmt.Sprintf("%s::%s", remoteType, remoteURL)

// Download
return g.Download(URLWithType, destPath)
}

// SubDirGlob returns the actual subdir with globbing processed
func (g *GoGetter) SubDirGlob(destPath, subDir string) (string, error) {
func (g *goGetter) SubDirGlob(destPath, subDir string) (string, error) {
return getter.SubdirGlob(destPath, subDir)
}

Expand Down
81 changes: 70 additions & 11 deletions pkg/downloader/getter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ var (
func TestNewGoGetter(t *testing.T) {
t.Run("new GoGetter", func(t *testing.T) {
var (
want = &GoGetter{}
got = NewGoGetter()
want = &goGetter{}
got = newGoGetter()
)
if !reflect.DeepEqual(got, want) {
t.Errorf("got: '%v', want: '%v'", got, want)
Expand Down Expand Up @@ -126,7 +126,7 @@ func TestGetURLSubDir(t *testing.T) {
}

for _, tt := range table {
g := NewGoGetter()
g := newGoGetter()
gotURL, gotSubDir, gotErr := g.GetURLSubDir(tt.URL, tt.dest)
if !reflect.DeepEqual(gotURL, tt.wantURL) {
t.Errorf("url got: '%v', want: '%v'", gotURL, tt.wantURL)
Expand All @@ -148,6 +148,8 @@ func TestDownload(t *testing.T) {
dest string
wantDest string
wantErr error
// when error is expected, but assertion is not required
skipErrAssert bool
}{
{
name: "empty URL",
Expand All @@ -170,14 +172,27 @@ func TestDownload(t *testing.T) {
wantDest: "",
wantErr: fmt.Errorf("GitHub URLs should be github.com/username/repo"),
},
{
name: "valid url, non existing repo",
URL: "github.com/testuser/testrepo",
dest: someDest,
wantDest: "",
skipErrAssert: true,
},
}

for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
g := NewGoGetter()
g := newGoGetter()
gotDest, gotErr := g.Download(tt.URL, tt.dest)
if !reflect.DeepEqual(gotErr, tt.wantErr) {
t.Errorf("error got: '%v', want: '%v'", gotErr, tt.wantErr)
if !tt.skipErrAssert {
if !reflect.DeepEqual(gotErr, tt.wantErr) {
t.Errorf("error got: '%v', want: '%v'", gotErr, tt.wantErr)
}
} else {
if gotErr == nil {
t.Error("error expected")
}
}
if !reflect.DeepEqual(gotDest, tt.wantDest) {
t.Errorf("dest got: '%v', want: '%v'", gotDest, tt.wantDest)
Expand All @@ -188,13 +203,19 @@ func TestDownload(t *testing.T) {

func TestDownloadWithType(t *testing.T) {

remoteTypeTerraformRegistry := "terraform-registry"
testInvalidRegistrySource := "test/some-url"
testValidNonExistentRegistrySource := "terraform-aws-modules/xyz/aws:1.0.0"

table := []struct {
name string
Type string
URL string
dest string
wantDest string
wantErr error
// when error is expected, but assertion is not required
skipErrAssert bool
}{
{
name: "empty URL and Type",
Expand Down Expand Up @@ -229,21 +250,59 @@ func TestDownloadWithType(t *testing.T) {
wantErr: ErrEmptyURLDest,
},
{
name: "invalid url",
name: "invalid remote type",
Type: someType,
URL: "github.com/some-url",
dest: someDest,
wantDest: "",
wantErr: fmt.Errorf("download not supported for scheme 'some-type'"),
wantErr: ErrInvalidRemoteType,
},
{
name: "valid remote type with invalid url",
Type: "git",
URL: "github.com/some-url",
dest: someDest,
wantDest: "",
wantErr: fmt.Errorf("GitHub URLs should be github.com/username/repo"),
},
{
name: "terraform-registry remote type with invalid source addr",
Type: remoteTypeTerraformRegistry,
URL: testInvalidRegistrySource,
dest: someDest,
wantDest: "",
wantErr: fmt.Errorf("%s, is not a valid terraform registry", testInvalidRegistrySource),
},
{
name: "terraform-registry remote type with valid non-existent source addr",
Type: remoteTypeTerraformRegistry,
URL: testValidNonExistentRegistrySource,
dest: someDest,
wantDest: "",
skipErrAssert: true,
},
{
name: "terraform-registry remote type with invalid version",
Type: remoteTypeTerraformRegistry,
URL: "terraform-aws-modules/xyz/aws:x.y",
dest: someDest,
wantDest: "",
skipErrAssert: true,
},
}

for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
g := NewGoGetter()
g := newGoGetter()
gotDest, gotErr := g.DownloadWithType(tt.Type, tt.URL, tt.dest)
if !reflect.DeepEqual(gotErr, tt.wantErr) {
t.Errorf("error got: '%v', want: '%v'", gotErr, tt.wantErr)
if !tt.skipErrAssert {
if !reflect.DeepEqual(gotErr, tt.wantErr) {
t.Errorf("error got: '%v', want: '%v'", gotErr, tt.wantErr)
}
} else {
if gotErr == nil {
t.Error("error expected")
}
}
if !reflect.DeepEqual(gotDest, tt.wantDest) {
t.Errorf("dest got: '%v', want: '%v'", gotDest, tt.wantDest)
Expand Down
31 changes: 30 additions & 1 deletion pkg/downloader/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@

package downloader

import (
hclConfigs "github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/registry/regsrc"
"github.com/hashicorp/terraform/registry/response"
)

// Downloader helps in downloading different kinds of modules from
// different types of sources
type Downloader interface {
Expand All @@ -25,7 +31,30 @@ type Downloader interface {
SubDirGlob(string, string) (string, error)
}

// ModuleDownloader helps in downloading the remote modules
type ModuleDownloader interface {
DownloadModule(addr, destPath string) (string, error)
DownloadRemoteModule(requiredVersion hclConfigs.VersionConstraint, destPath string, module *regsrc.Module) (string, error)
CleanUp()
}

// terraformRegistryClient will help interact with terraform registries
type terraformRegistryClient interface {
ModuleVersions(module *regsrc.Module) (*response.ModuleVersions, error)
ModuleLocation(module *regsrc.Module, version string) (string, error)
}

// NewDownloader returns a new downloader
func NewDownloader() Downloader {
return NewGoGetter()
return newGoGetter()
}

// NewRemoteDownloader returns a new ModuleDownloader
func NewRemoteDownloader() ModuleDownloader {
return newRemoteModuleInstaller()
}

// newClientRegistry returns a terraformClientRegistry to query terraform registries
func newClientRegistry() terraformRegistryClient {
return newTerraformRegistryClient()
}

0 comments on commit 26bae02

Please sign in to comment.