Add ListFiles, Tabify and TruncateString. Closes #7. Closes #9. Closes …

…#10

files/list.go

@@ -0,0 +1,152 @@
+package files
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/danwakefield/fnmatch"
+)
+
+type ListFlag uint
+
+const (
+	Files ListFlag = 1 << iota
+	Directories
+	Recursive
+	AllFlat              = Files | Directories
+	AllRecursive         = Files | Directories | Recursive
+	FilesRecursive       = Files | Recursive
+	DirectoriesRecursive = Directories | Recursive
+	FilesFlat            = Files
+	DirectoriesFlat      = Directories
+)
+
+type ListOptions struct {
+	Flags ListFlag
+	// .gitignore (fnmatch) format patterns for file inclusions and exclusions
+	Include []string
+	Exclude []string
+}
+
+// ListFiles :: list files and or folders under list path, based on options
+func ListFiles(listPath string, opts *ListOptions) ([]string, error) {
+	// check folder exists
+	if _, err := os.Stat(listPath); os.IsNotExist(err) {
+		return nil, nil
+	}
+	if opts == nil {
+		opts = &ListOptions{Flags: Files & Directories & Recursive}
+	}
+	// if no include list provided, default to including everything
+	if len(opts.Include) == 0 {
+		opts.Include = []string{"*"}
+	}
+
+	if opts.Flags&Recursive != 0 {
+		return listFilesRecursive(listPath, opts)
+	}
+	return listFilesFlat(listPath, opts)
+}
+
+// InclusionsFromExtensions :: take a list of file extensions and convert into a .gitgnore format inclusions list
+func InclusionsFromExtensions(extensions []string) []string {
+	// build include string from extensions
+	var includeStrings []string
+	for _, extension := range extensions {
+		includeStrings = append(includeStrings, fmt.Sprintf("**/*%s", extension))
+	}
+	return includeStrings
+}
+
+// InclusionsFromFiles :: take a list of file names convert into a .gitgnore format inclusions list
+func InclusionsFromFiles(filenames []string) []string {
+	// build include string from extensions
+	var includeStrings []string
+	for _, extension := range filenames {
+		includeStrings = append(includeStrings, fmt.Sprintf("**/%s", extension))
+	}
+	return includeStrings
+}
+
+func listFilesRecursive(listPath string, opts *ListOptions) ([]string, error) {
+	var res []string
+	err := filepath.Walk(listPath,
+		func(path string, entry os.FileInfo, err error) error {
+			if err != nil {
+				if _, ok := err.(*os.PathError); ok {
+					// ignore path errors - this may be for a file which has been removed during the walk
+					return nil
+				}
+				return err
+			}
+			// ignore list path itself
+			if path == listPath {
+				return nil
+			}
+
+			// should we include this file?
+			if shouldIncludeEntry(path, entry, opts) {
+				res = append(res, path)
+			}
+
+			return nil
+		})
+	return res, err
+}
+
+func listFilesFlat(path string, opts *ListOptions) ([]string, error) {
+	entries, err := ioutil.ReadDir(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read folder %s: %v", path, err)
+	}
+
+	matches := []string{}
+	for _, entry := range entries {
+		path := filepath.Join(path, entry.Name())
+		if shouldIncludeEntry(path, entry, opts) {
+			matches = append(matches, path)
+		}
+	}
+	return matches, nil
+}
+
+// should the list results include this entry, based on the list options
+func shouldIncludeEntry(path string, entry os.FileInfo, opts *ListOptions) bool {
+	if entry.IsDir() {
+		// if this is a directory and we are not including directories, exclude
+		if opts.Flags&Directories == 0 {
+			return false
+		}
+	} else {
+		// if this is a file and we are not including files, exclude
+		if opts.Flags&Files == 0 {
+			return false
+		}
+	}
+
+	return ShouldIncludePath(path, opts.Include, opts.Exclude)
+}
+
+// ShouldIncludePath :: does the specified file path satisfy the inclusion and exclusion options (in .gitignore format)
+func ShouldIncludePath(path string, include, exclude []string) bool {
+	// if no include list provided, default to including everything
+	if len(include) == 0 {
+		include = []string{"*"}
+	}
+	// if the entry matches any of the exclude patterns, exclude
+	for _, excludePattern := range exclude {
+		if fnmatch.Match(excludePattern, path, 0) {
+			return false
+		}
+	}
+	// if the entry matches ANY of the include patterns, include
+	shouldInclude := false
+	for _, includePattern := range include {
+		if fnmatch.Match(includePattern, path, 0) {
+			shouldInclude = true
+		}
+	}
+	return shouldInclude
+}

files/list_test.go

@@ -0,0 +1,224 @@
+package files
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+)
+
+type ListFilesTest struct {
+	source   string
+	options  *ListOptions
+	expected interface{}
+}
+
+var testCasesListFiles = map[string]ListFilesTest{
+	"AllRecursive, exclude **/a*, **/*.swp, **/.steampipe*": {
+		source: "test_data/list_test1",
+		options: &ListOptions{
+			Flags:   AllRecursive,
+			Exclude: []string{"**/a*", "**/*.swp", "**/.steampipe*"},
+		},
+		expected: []string{
+			"test_data/list_test1/b",
+			"test_data/list_test1/b/mod.sp",
+			"test_data/list_test1/b/q1.sp",
+			"test_data/list_test1/b/q2.sp",
+			"test_data/list_test1/config",
+			"test_data/list_test1/config/default.spc",
+		},
+	},
+	"AllRecursive": {
+		source: "test_data/list_test1",
+		options: &ListOptions{
+			Flags: AllRecursive,
+		},
+		expected: []string{
+			"test_data/list_test1/.steampipe",
+			"test_data/list_test1/.steampipe/mods",
+			"test_data/list_test1/.steampipe/mods/github.com",
+			"test_data/list_test1/.steampipe/mods/github.com/turbot",
+			"test_data/list_test1/.steampipe/mods/github.com/turbot/m1",
+			"test_data/list_test1/.steampipe/mods/github.com/turbot/m1/mod.sp",
+			"test_data/list_test1/.steampipe/mods/github.com/turbot/m1/q1.sp",
+			"test_data/list_test1/.steampipe/mods/github.com/turbot/m2",
+			"test_data/list_test1/.steampipe/mods/github.com/turbot/m2/mod.sp",
+			"test_data/list_test1/.steampipe/mods/github.com/turbot/m2/q1.sp",
+			"test_data/list_test1/a",
+			"test_data/list_test1/a/mod.sp",
+			"test_data/list_test1/a/q1.sp",
+			"test_data/list_test1/a/q2.sp",
+			"test_data/list_test1/a.swp",
+			"test_data/list_test1/b",
+			"test_data/list_test1/b/mod.sp",
+			"test_data/list_test1/b/q1.sp",
+			"test_data/list_test1/b/q2.sp",
+			"test_data/list_test1/config",
+			"test_data/list_test1/config/aws.spc",
+			"test_data/list_test1/config/default.spc",
+		},
+	},
+	"AllFlat": {
+		source: "test_data/list_test1",
+		options: &ListOptions{
+			Flags: AllFlat,
+		},
+		expected: []string{
+			"test_data/list_test1/.steampipe",
+			"test_data/list_test1/a",
+			"test_data/list_test1/a.swp",
+			"test_data/list_test1/b",
+			"test_data/list_test1/config",
+		},
+	},
+	"FilesFlat": {
+		source: "test_data/list_test1",
+		options: &ListOptions{
+			Flags: FilesFlat,
+		},
+		expected: []string{
+			"test_data/list_test1/a.swp",
+		},
+	},
+	"DirectoriesFlat": {
+		source: "test_data/list_test1",
+		options: &ListOptions{
+			Flags: DirectoriesFlat,
+		},
+		expected: []string{
+			"test_data/list_test1/.steampipe",
+			"test_data/list_test1/a",
+			"test_data/list_test1/b",
+			"test_data/list_test1/config",
+		},
+	},
+	"DirectoriesRecursive": {
+		source: "test_data/list_test1",
+		options: &ListOptions{
+			Flags: DirectoriesRecursive,
+		},
+		expected: []string{
+			"test_data/list_test1/.steampipe",
+			"test_data/list_test1/.steampipe/mods",
+			"test_data/list_test1/.steampipe/mods/github.com",
+			"test_data/list_test1/.steampipe/mods/github.com/turbot",
+			"test_data/list_test1/.steampipe/mods/github.com/turbot/m1",
+			"test_data/list_test1/.steampipe/mods/github.com/turbot/m2",
+			"test_data/list_test1/a",
+			"test_data/list_test1/b",
+			"test_data/list_test1/config",
+		},
+	},
+	"DirectoriesRecursive, exclude  **/.steampipe*": {
+		source: "test_data/list_test1",
+		options: &ListOptions{
+			Flags:   DirectoriesRecursive,
+			Exclude: []string{"**/.steampipe*"},
+		},
+		expected: []string{
+			"test_data/list_test1/a",
+			"test_data/list_test1/b",
+			"test_data/list_test1/config",
+		},
+	},
+	"FilesRecursive": {
+		source: "test_data/list_test1",
+		options: &ListOptions{
+			Flags: FilesRecursive,
+		},
+		expected: []string{
+			"test_data/list_test1/.steampipe/mods/github.com/turbot/m1/mod.sp",
+			"test_data/list_test1/.steampipe/mods/github.com/turbot/m1/q1.sp",
+			"test_data/list_test1/.steampipe/mods/github.com/turbot/m2/mod.sp",
+			"test_data/list_test1/.steampipe/mods/github.com/turbot/m2/q1.sp",
+			"test_data/list_test1/a/mod.sp",
+			"test_data/list_test1/a/q1.sp",
+			"test_data/list_test1/a/q2.sp",
+			"test_data/list_test1/a.swp",
+			"test_data/list_test1/b/mod.sp",
+			"test_data/list_test1/b/q1.sp",
+			"test_data/list_test1/b/q2.sp",
+			"test_data/list_test1/config/aws.spc",
+			"test_data/list_test1/config/default.spc",
+		},
+	},
+	"FilesRecursive, exclude  **/.steampipe*": {
+		source: "test_data/list_test1",
+		options: &ListOptions{
+			Flags:   FilesRecursive,
+			Exclude: []string{"**/.steampipe*"},
+		},
+		expected: []string{
+			"test_data/list_test1/a/mod.sp",
+			"test_data/list_test1/a/q1.sp",
+			"test_data/list_test1/a/q2.sp",
+			"test_data/list_test1/a.swp",
+			"test_data/list_test1/b/mod.sp",
+			"test_data/list_test1/b/q1.sp",
+			"test_data/list_test1/b/q2.sp",
+			"test_data/list_test1/config/aws.spc",
+			"test_data/list_test1/config/default.spc",
+		},
+	},
+	"FilesRecursive, include exclude  **/.steampipe* **/*.sp": {
+		source: "test_data/list_test1",
+		options: &ListOptions{
+			Flags:   FilesRecursive,
+			Exclude: []string{"**/.steampipe*"},
+			Include: []string{"**/*.sp"},
+		},
+		expected: []string{
+			"test_data/list_test1/a/mod.sp",
+			"test_data/list_test1/a/q1.sp",
+			"test_data/list_test1/a/q2.sp",
+			"test_data/list_test1/b/mod.sp",
+			"test_data/list_test1/b/q1.sp",
+			"test_data/list_test1/b/q2.sp",
+		},
+	},
+}
+
+func TestListFiles(t *testing.T) {
+	for name, test := range testCasesListFiles {
+		listPath, err := filepath.Abs(test.source)
+		if err != nil {
+			t.Errorf("failed to build absolute list filepath from %s", test.source)
+		}
+
+		files, err := ListFiles(listPath, test.options)
+
+		if err != nil {
+			if test.expected != "ERROR" {
+				t.Errorf("Test: '%s'' FAILED with unexpected error: %v", name, err)
+			}
+			continue
+		}
+
+		if test.expected == "ERROR" {
+			t.Errorf("Test: '%s'' FAILED - expected error", name)
+			continue
+		}
+
+		// now remove loacl path from files for expectation testing (as expectations are relative)
+		localDirectory, err := os.Getwd()
+		if err != nil {
+			t.Errorf("failed to get working directory %v", err)
+			continue
+		}
+
+		for i, f := range files {
+			rel, err := filepath.Rel(localDirectory, f)
+			if err != nil {
+				t.Errorf("failed to convert %s to a relatyive path for verification: %v", f, err)
+			}
+			files[i] = rel
+		}
+
+		if !reflect.DeepEqual(test.expected, files) {
+			fmt.Printf("")
+			t.Errorf("Test: '%s'' FAILED : expected:\n\n%s\n\ngot:\n\n%s", name, test.expected, files)
+		}
+	}
+}

go.mod

@@ -4,6 +4,7 @@ go 1.15
 
 require (
 	github.com/btubbs/datetime v0.1.1
+	github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964
 	github.com/mitchellh/mapstructure v1.3.3
 	github.com/stretchr/testify v1.6.1
 	github.com/tkrajina/go-reflector v0.5.4

go.sum

@@ -1,5 +1,7 @@
 github.com/btubbs/datetime v0.1.1 h1:KuV+F9tyq/hEnezmKZNGk8dzqMVsId6EpFVrQCfA3To=
 github.com/btubbs/datetime v0.1.1/go.mod h1:n2BZ/2ltnRzNiz27aE3wUb2onNttQdC+WFxAoks5jJM=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8=