From 359d163fd8cf23babab4e4bd73485f07d2735e66 Mon Sep 17 00:00:00 2001 From: Inhere Date: Thu, 25 May 2023 11:18:02 +0800 Subject: [PATCH] :recycle: feat: fsutil/finder - refactoring the find and filter logic --- fsutil/finder/README.md | 31 +++ fsutil/finder/config.go | 75 ++++++ fsutil/finder/elem.go | 42 +++ fsutil/finder/filter.go | 269 ++++++++----------- fsutil/finder/filter_test.go | 5 +- fsutil/finder/filters.go | 280 ++++++++++++++++++++ fsutil/finder/filters_test.go | 30 +++ fsutil/finder/finder.go | 476 +++++++++++++++++++--------------- fsutil/finder/finder_test.go | 72 ++--- fsutil/finder/result.go | 45 ---- 10 files changed, 876 insertions(+), 449 deletions(-) create mode 100644 fsutil/finder/README.md create mode 100644 fsutil/finder/config.go create mode 100644 fsutil/finder/elem.go create mode 100644 fsutil/finder/filters.go create mode 100644 fsutil/finder/filters_test.go delete mode 100644 fsutil/finder/result.go diff --git a/fsutil/finder/README.md b/fsutil/finder/README.md new file mode 100644 index 000000000..a22a29d1a --- /dev/null +++ b/fsutil/finder/README.md @@ -0,0 +1,31 @@ +# finder + +[![GoDoc](https://godoc.org/github.com/goutil/fsutil/finder?status.svg)](https://godoc.org/github.com/goutil/fsutil/finder) + +`finder` provide a finder tool for find files, dirs. + +## Usage + +```go +package main + +import ( + "github.com/gookit/goutil/dump" + "github.com/goutil/fsutil/finder" +) + +func main() { + ff := finder.NewFinder() + ff.AddPath("/tmp") + ff.AddPath("/usr/local") + ff.AddPath("/usr/local/bin") + ff.AddPath("/usr/local/lib") + ff.AddPath("/usr/local/libexec") + ff.AddPath("/usr/local/sbin") + ff.AddPath("/usr/local/share") + + ss := ff.FindPaths() + dump.P(ss) +} +``` + diff --git a/fsutil/finder/config.go b/fsutil/finder/config.go new file mode 100644 index 000000000..a5665607b --- /dev/null +++ b/fsutil/finder/config.go @@ -0,0 +1,75 @@ +package finder + +// commonly dot file and dirs +var ( + CommonlyDotDir = []string{".git", ".idea", ".vscode", ".svn", ".hg"} + CommonlyDotFile = []string{".gitignore", ".dockerignore", ".npmignore", ".DS_Store"} +) + +// FindFlag type for find result. +type FindFlag uint8 + +// flags for find result. +const ( + FlagFile FindFlag = iota + 1 // only find files(default) + FlagDir +) + +// Config for finder +type Config struct { + curDepth int + + // DirPaths src paths for find file. + DirPaths []string + // FindFlags for find result. default is FlagFile + FindFlags FindFlag + // MaxDepth for find result. default is 0 + MaxDepth int + // CacheResult cache result for find result. default is false + CacheResult bool + + // IncludeDirs name list. eg: {"model"} + IncludeDirs []string + // IncludeExts name list. eg: {".go", ".md"} + IncludeExts []string + // IncludeFiles name list. eg: {"go.mod"} + IncludeFiles []string + // IncludePaths list. eg: {"path/to"} + IncludePaths []string + + // ExcludeDirs name list. eg: {"test"} + ExcludeDirs []string + // ExcludeExts name list. eg: {".go", ".md"} + ExcludeExts []string + // ExcludeFiles name list. eg: {"go.mod"} + ExcludeFiles []string + // ExcludePaths list. eg: {"path/to"} + ExcludePaths []string + // ExcludeNames file/dir name list. eg: {"test", "some.go"} + ExcludeNames []string + + ExcludeDotDir bool + ExcludeDotFile bool + + // Filters generic filters for filter file/dir paths + Filters []Filter + + DirFilters []Filter // filters for filter dir paths + FileFilters []Filter // filters for filter file paths + BodyFilters []BodyFilter // filters for filter file body +} + +// NewConfig create a new Config +func NewConfig(dirs ...string) *Config { + return &Config{ + DirPaths: dirs, + FindFlags: FlagFile, + // with default setting. + ExcludeDotDir: true, + } +} + +// NewFinder create a new FileFinder by config +func (c *Config) NewFinder() *FileFinder { + return NewWithConfig(c) +} diff --git a/fsutil/finder/elem.go b/fsutil/finder/elem.go new file mode 100644 index 000000000..f07006a14 --- /dev/null +++ b/fsutil/finder/elem.go @@ -0,0 +1,42 @@ +package finder + +import ( + "io/fs" +) + +// Elem of find file/dir result +type Elem interface { + fs.DirEntry + // Path get file/dir path. eg: "/path/to/file.go" + Path() string + // Info get file info. like fs.DirEntry.Info(), but will cache result. + Info() (fs.FileInfo, error) +} + +type elem struct { + fs.DirEntry + path string + stat fs.FileInfo + sErr error +} + +// NewElem create a new Elem instance +func NewElem(fPath string, ent fs.DirEntry) Elem { + return &elem{ + path: fPath, + DirEntry: ent, + } +} + +// Path get full file/dir path. eg: "/path/to/file.go" +func (e *elem) Path() string { + return e.path +} + +// Info get file info, will cache result +func (e *elem) Info() (fs.FileInfo, error) { + if e.stat == nil { + e.stat, e.sErr = e.DirEntry.Info() + } + return e.stat, e.sErr +} diff --git a/fsutil/finder/filter.go b/fsutil/finder/filter.go index 4dbbb2a9f..b254245ec 100644 --- a/fsutil/finder/filter.go +++ b/fsutil/finder/filter.go @@ -1,201 +1,158 @@ package finder import ( - "os" - "path" - "regexp" - "strings" - "time" + "bytes" + "io/fs" + + "github.com/gookit/goutil/fsutil" ) -// FileFilter for filter file path. -type FileFilter interface { - FilterFile(filePath, filename string) bool +// Filter for filter file path. +type Filter interface { + // Apply check find elem. return False will filter this file. + Apply(elem Elem) bool } -// FileFilterFunc for filter file path. -type FileFilterFunc func(filePath, filename string) bool +// FilterFunc for filter file info, return False will filter this file +type FilterFunc func(elem Elem) bool -// FilterFile Filter for filter file path. -func (fn FileFilterFunc) FilterFile(filePath, filename string) bool { - return fn(filePath, filename) +// Apply check file path. return False will filter this file. +func (fn FilterFunc) Apply(elem Elem) bool { + return fn(elem) } -// DirFilter for filter dir path. -type DirFilter interface { - FilterDir(dirPath, dirName string) bool +// ------------------ raw filter wrapper ------------------ + +// RawFilter for filter file path. +type RawFilter interface { + // Apply check file path. return False will filter this file. + Apply(fPath string, ent fs.DirEntry) bool } -// DirFilterFunc for filter file path. -type DirFilterFunc func(dirPath, dirName string) bool +// RawFilterFunc for filter file info, return False will filter this file +type RawFilterFunc func(fPath string, ent fs.DirEntry) bool -// FilterDir Filter for filter file path. -func (fn DirFilterFunc) FilterDir(dirPath, dirName string) bool { - return fn(dirPath, dirName) +// Apply check file path. return False will filter this file. +func (fn RawFilterFunc) Apply(fPath string, ent fs.DirEntry) bool { + return fn(fPath, ent) } -// // BodyFilter for filter file contents. -// type BodyFilter interface { -// FilterBody(contents, filePath string) bool -// } -// -// // BodyFilterFunc for filter file contents. -// type BodyFilterFunc func(contents, filePath string) bool -// -// // Filter for filter file path. -// func (fn BodyFilterFunc) FilterBody(contents, filePath string) bool { -// return fn(contents, filePath) -// } - -// FilterFunc for filter file path. -type FilterFunc func(filePath, filename string) bool +// WrapRawFilter wrap a RawFilter to Filter +func WrapRawFilter(rf RawFilter) Filter { + return FilterFunc(func(elem Elem) bool { + return rf.Apply(elem.Path(), elem) + }) +} -// Filter for filter file path. -func (fn FilterFunc) Filter(filePath, filename string) bool { - return fn(filePath, filename) -} - -// -// ------------------ built in file path filters ------------------ -// - -// ExtFilterFunc filter filepath by given file ext. -// -// Usage: -// -// f := EmptyFinder() -// f.AddFilter(ExtFilterFunc([]string{".go", ".md"}, true)) -// f.AddFilter(ExtFilterFunc([]string{".log", ".tmp"}, false)) -func ExtFilterFunc(exts []string, include bool) FileFilterFunc { - return func(filePath, _ string) bool { - fExt := path.Ext(filePath) - for _, ext := range exts { - if fExt == ext { - return include - } - } - return !include +// WrapRawFilters wrap RawFilter list to Filter list +func WrapRawFilters(rfs ...RawFilter) []Filter { + fls := make([]Filter, len(rfs)) + for i, rf := range rfs { + fls[i] = WrapRawFilter(rf) } + return fls } -// SuffixFilterFunc filter filepath by given file ext. -// -// Usage: -// -// f := EmptyFinder() -// f.AddFilter(SuffixFilterFunc([]string{"util.go", ".md"}, true)) -// f.AddFilter(SuffixFilterFunc([]string{"_test.go", ".log"}, false)) -func SuffixFilterFunc(suffixes []string, include bool) FileFilterFunc { - return func(filePath, _ string) bool { - for _, sfx := range suffixes { - if strings.HasSuffix(filePath, sfx) { - return include - } +// ------------------ Multi filter wrapper ------------------ + +// MultiFilter wrapper for multi filters +type MultiFilter struct { + Before Filter + Filters []Filter +} + +// AddFilter add filters +func (mf *MultiFilter) AddFilter(fls ...Filter) { + mf.Filters = append(mf.Filters, fls...) +} + +// Apply check file path. return False will filter this file. +func (mf *MultiFilter) Apply(el Elem) bool { + if mf.Before != nil && !mf.Before.Apply(el) { + return false + } + + for _, fl := range mf.Filters { + if !fl.Apply(el) { + return false } - return !include } + return true } -// PathNameFilterFunc filter filepath by given path names. -func PathNameFilterFunc(names []string, include bool) FileFilterFunc { - return func(filePath, _ string) bool { - for _, name := range names { - if strings.Contains(filePath, name) { - return include - } - } - return !include +// NewDirFilters create a new dir filters +func NewDirFilters(fls ...Filter) *MultiFilter { + return &MultiFilter{ + Before: OnlyDirFilter, + Filters: fls, } } -// DotFileFilterFunc filter dot filename. eg: ".gitignore" -func DotFileFilterFunc(include bool) FileFilterFunc { - return func(filePath, filename string) bool { - // filename := path.Base(filePath) - if filename[0] == '.' { - return include - } - return !include +// NewFileFilters create a new dir filters +func NewFileFilters(fls ...Filter) *MultiFilter { + return &MultiFilter{ + Before: OnlyFileFilter, + Filters: fls, } } -// ModTimeFilterFunc filter file by modify time. -func ModTimeFilterFunc(limitSec int, op rune, include bool) FileFilterFunc { - return func(filePath, filename string) bool { - fi, err := os.Stat(filePath) - if err != nil { - return !include - } +// ------------------ Body Filter ------------------ - now := time.Now().Second() - if op == '>' { - if now-fi.ModTime().Second() > limitSec { - return include - } - return !include - } +// BodyFilter for filter file contents. +type BodyFilter interface { + Apply(filePath string, buf *bytes.Buffer) bool +} - // '<' - if now-fi.ModTime().Second() < limitSec { - return include - } - return !include - } +// BodyFilterFunc for filter file contents. +type BodyFilterFunc func(filePath string, buf *bytes.Buffer) bool + +// Apply for filter file contents. +func (fn BodyFilterFunc) Apply(filePath string, buf *bytes.Buffer) bool { + return fn(filePath, buf) } -// GlobFilterFunc filter filepath by given patterns. -// -// Usage: -// -// f := EmptyFiler() -// f.AddFilter(GlobFilterFunc([]string{"*_test.go"}, true)) -func GlobFilterFunc(patterns []string, include bool) FileFilterFunc { - return func(_, filename string) bool { - for _, pattern := range patterns { - if ok, _ := path.Match(pattern, filename); ok { - return include - } - } - return !include +// BodyFilters multi body filters as Filter +type BodyFilters struct { + Filters []BodyFilter +} + +// NewBodyFilters create a new body filters +func NewBodyFilters(fls ...BodyFilter) *BodyFilters { + return &BodyFilters{ + Filters: fls, } } -// RegexFilterFunc filter filepath by given regex pattern -// -// Usage: -// -// f := EmptyFiler() -// f.AddFilter(RegexFilterFunc(`[A-Z]\w+`, true)) -func RegexFilterFunc(pattern string, _ bool) FileFilterFunc { - reg := regexp.MustCompile(pattern) +// AddFilter add filters +func (mf *BodyFilters) AddFilter(fls ...BodyFilter) { + mf.Filters = append(mf.Filters, fls...) +} - return func(_, filename string) bool { - return reg.MatchString(filename) +// Apply check file path. return False will filter this file. +func (mf *BodyFilters) Apply(fPath string, ent fs.DirEntry) bool { + if ent.IsDir() { + return false } -} -// -// ----------------- built in dir path filters ----------------- -// + // read file contents + buf := bytes.NewBuffer(nil) + file, err := fsutil.OpenReadFile(fPath) + if err != nil { + return false + } -// DotDirFilterFunc filter dot dirname. eg: ".idea" -func DotDirFilterFunc(include bool) DirFilterFunc { - return func(_, dirname string) bool { - if dirname[0] == '.' { - return include - } - return !include + _, err = buf.ReadFrom(file) + if err != nil { + file.Close() + return false } -} + file.Close() -// DirNameFilterFunc filter filepath by given dir names. -func DirNameFilterFunc(names []string, include bool) DirFilterFunc { - return func(_, dirName string) bool { - for _, name := range names { - if dirName == name { - return include - } + // apply filters + for _, fl := range mf.Filters { + if !fl.Apply(fPath, buf) { + return false } - return !include } + return true } diff --git a/fsutil/finder/filter_test.go b/fsutil/finder/filter_test.go index aabbd5dcd..7082d20d6 100644 --- a/fsutil/finder/filter_test.go +++ b/fsutil/finder/filter_test.go @@ -4,13 +4,14 @@ import ( "testing" "github.com/gookit/goutil/fsutil/finder" + "github.com/gookit/goutil/testutil" "github.com/gookit/goutil/testutil/assert" ) func TestFilterFunc(t *testing.T) { - fn := finder.FilterFunc(func(filePath, filename string) bool { + fn := finder.FilterFunc(func(el finder.Elem) bool { return false }) - assert.False(t, fn("path/some.txt", "some.txt")) + assert.False(t, fn(finder.NewElem("path/some.txt", &testutil.DirEnt{}))) } diff --git a/fsutil/finder/filters.go b/fsutil/finder/filters.go new file mode 100644 index 000000000..06a7001ee --- /dev/null +++ b/fsutil/finder/filters.go @@ -0,0 +1,280 @@ +package finder + +import ( + "io/fs" + "path" + "regexp" + "strings" + + "github.com/gookit/goutil/strutil" + "github.com/gookit/goutil/timex" +) + +// ------------------ built in filters ------------------ + +// OnlyFileFilter only allow file path. +var OnlyFileFilter = FilterFunc(func(el Elem) bool { + return !el.IsDir() +}) + +// OnlyDirFilter only allow dir path. +var OnlyDirFilter = FilterFunc(func(el Elem) bool { + return el.IsDir() +}) + +// DotDirFilter filter dot dirname. eg: ".idea" +func DotDirFilter(include bool) FilterFunc { + return func(el Elem) bool { + if el.IsDir() && el.Path()[0] == '.' { + return include + } + return !include + } +} + +// OnlyFileFilter2 filter only file path. +func OnlyFileFilter2(exts ...string) FilterFunc { + return func(el Elem) bool { + if el.IsDir() { + return false + } + + if len(exts) == 0 { + return true + } + return isContains(path.Ext(el.Name()), exts, true) + } +} + +func isContains(sub string, list []string, include bool) bool { + for _, s := range list { + if s == sub { + return include + } + } + return !include +} + +// ExtFilter filter filepath by given file ext. +// +// Usage: +// +// f := NewEmpty() +// f.AddFilter(ExtFilter(".go")) +// f.AddFilter(ExtFilter(".go", ".php")) +func ExtFilter(include bool, exts ...string) FilterFunc { + return func(el Elem) bool { + if len(exts) == 0 { + return true + } + return isContains(path.Ext(el.Path()), exts, include) + } +} + +// NameFilter filter filepath by given names. +func NameFilter(include bool, names ...string) FilterFunc { + return func(el Elem) bool { + return isContains(el.Name(), names, include) + } +} + +// SuffixFilter filter filepath by check given suffixes. +// +// Usage: +// +// f := EmptyFinder() +// f.AddFilter(finder.SuffixFilter(true, "util.go", "en.md")) +// f.AddFilter(finder.SuffixFilter(false, "_test.go", ".log")) +func SuffixFilter(include bool, suffixes ...string) FilterFunc { + return func(el Elem) bool { + for _, sfx := range suffixes { + if strings.HasSuffix(el.Path(), sfx) { + return include + } + } + return !include + } +} + +// PathFilter filter filepath by given sub paths. +// +// Usage: +// +// f.AddFilter(PathFilter(true, "need/path")) +func PathFilter(include bool, subPaths ...string) FilterFunc { + return func(el Elem) bool { + for _, subPath := range subPaths { + if strings.Contains(el.Path(), subPath) { + return include + } + } + return !include + } +} + +// DotFileFilter filter dot filename. eg: ".gitignore" +func DotFileFilter(include bool) FilterFunc { + return func(el Elem) bool { + name := el.Name() + if len(name) > 0 && name[0] == '.' { + return include + } + return !include + } +} + +// GlobFilterFunc filter filepath by given patterns. +// +// Usage: +// +// f := EmptyFiler() +// f.AddFilter(GlobFilterFunc(true, "*_test.go")) +func GlobFilterFunc(include bool, patterns ...string) FilterFunc { + return func(el Elem) bool { + for _, pattern := range patterns { + if ok, _ := path.Match(pattern, el.Path()); ok { + return include + } + } + return !include + } +} + +// RegexFilterFunc filter filepath by given regex pattern +// +// Usage: +// +// f := EmptyFiler() +// f.AddFilter(RegexFilterFunc(`[A-Z]\w+`, true)) +func RegexFilterFunc(pattern string, include bool) FilterFunc { + reg := regexp.MustCompile(pattern) + + return func(el Elem) bool { + if reg.MatchString(el.Path()) { + return include + } + return !include + } +} + +// +// ----------------- built in file info filters ----------------- +// + +// ModTimeFilter filter file by modify time. +// +// Usage: +// +// f := EmptyFinder() +// f.AddFilter(ModTimeFilter(600, '>', true)) // 600 seconds to Now(last 10 minutes +// f.AddFilter(ModTimeFilter(600, '<', false)) // before 600 seconds(before 10 minutes) +func ModTimeFilter(limitSec int, op rune, include bool) FilterFunc { + return func(el Elem) bool { + fi, err := el.Info() + if err != nil { + return !include + } + + lt := timex.Now().AddSeconds(-limitSec) + if op == '>' { + if lt.After(fi.ModTime()) { + return include + } + return !include + } + + // '<' + if lt.Before(fi.ModTime()) { + return include + } + return !include + } +} + +// HumanModTimeFilter filter file by modify time string. +// +// Usage: +// +// f := EmptyFinder() +// f.AddFilter(HumanModTimeFilter("10m", '>', true)) // 10 minutes to Now +// f.AddFilter(HumanModTimeFilter("10m", '<', false)) // before 10 minutes +func HumanModTimeFilter(limit string, op rune, include bool) FilterFunc { + return func(elem Elem) bool { + fi, err := elem.Info() + if err != nil { + return !include + } + + lt, err := strutil.ToDuration(limit) + if err != nil { + return !include + } + + if op == '>' { + if timex.Now().Add(-lt).After(fi.ModTime()) { + return include + } + return !include + } + + // '<' + if timex.Now().Add(-lt).Before(fi.ModTime()) { + return include + } + return !include + } +} + +// FileSizeFilter filter file by file size. +func FileSizeFilter(min, max int64, include bool) FilterFunc { + return func(el Elem) bool { + if el.IsDir() { + return false + } + + fi, err := el.Info() + if err != nil { + return false + } + + return ByteSizeCheck(fi, min, max, include) + } +} + +// HumanSizeFilter filter file by file size string. eg: 1KB, 2MB, 3GB +func HumanSizeFilter(min, max string, include bool) FilterFunc { + minSize, err := strutil.ToByteSize(min) + if err != nil { + panic(err) + } + + maxSize, err := strutil.ToByteSize(max) + if err != nil { + panic(err) + } + + return func(el Elem) bool { + if el.IsDir() { + return false + } + + fi, err := el.Info() + if err != nil { + return false + } + + return ByteSizeCheck(fi, int64(minSize), int64(maxSize), include) + } +} + +// ByteSizeCheck filter file by file size. +func ByteSizeCheck(fi fs.FileInfo, min, max int64, include bool) bool { + if min > 0 && fi.Size() < min { + return !include + } + + if max > 0 && fi.Size() > max { + return !include + } + return include +} diff --git a/fsutil/finder/filters_test.go b/fsutil/finder/filters_test.go new file mode 100644 index 000000000..e7b850253 --- /dev/null +++ b/fsutil/finder/filters_test.go @@ -0,0 +1,30 @@ +package finder_test + +import ( + "testing" + + "github.com/gookit/goutil/fsutil/finder" + "github.com/gookit/goutil/testutil" + "github.com/gookit/goutil/testutil/assert" +) + +func TestRegexFilterFunc(t *testing.T) { + tests := []struct { + filePath string + pattern string + include bool + match bool + }{ + {"path/to/util.go", `\.go$`, true, true}, + {"path/to/util.go", `\.go$`, false, false}, + {"path/to/util.go", `\.py$`, true, false}, + {"path/to/util.go", `\.py$`, false, true}, + } + + ent := &testutil.DirEnt{} + + for _, tt := range tests { + fn := finder.RegexFilterFunc(tt.pattern, tt.include) + assert.Eq(t, tt.match, fn(finder.NewElem(tt.filePath, ent))) + } +} diff --git a/fsutil/finder/finder.go b/fsutil/finder/finder.go index bd266a45c..19b725613 100644 --- a/fsutil/finder/finder.go +++ b/fsutil/finder/finder.go @@ -2,98 +2,104 @@ package finder import ( - "io/ioutil" "os" "path/filepath" "strings" ) -// TODO use excludeDotFlag 1 file 2 dir 1|2 both -type exDotFlag uint8 - -const ( - ExDotFile exDotFlag = 1 - ExDotDir exDotFlag = 2 -) - // FileFinder struct type FileFinder struct { - // r *FindResults - - // dir paths for find file. - dirPaths []string - // file paths for handle. - srcFiles []string - - // builtin include filters - includeDirs []string // include dir names. eg: {"model"} - includeExts []string // include ext names. eg: {".go", ".md"} - - // builtin exclude filters - excludeDirs []string // exclude dir names. eg: {"test"} - excludeExts []string // exclude ext names. eg: {".go", ".md"} - excludeNames []string // exclude file names. eg: {"go.mod"} + // config for finder + c *Config + // last error + err error + // ch - founded file elem chan + ch chan Elem + // caches - cache found file elem. if config.CacheResult is true + caches []Elem +} - // builtin dot filters. - // TODO use excludeDotFlag 1 file 2 dir 1|2 both - // excludeDotFlag exDotFlag - excludeDotDir bool - excludeDotFile bool +// NewEmpty new empty FileFinder instance +func NewEmpty() *FileFinder { return New([]string{}) } - // fileFlags int +// EmptyFinder new empty FileFinder instance +func EmptyFinder() *FileFinder { return NewEmpty() } - dirFilters []DirFilter // filters for filter dir paths - fileFilters []FileFilter // filters for filter file paths +// New instance with source dir paths. +func New(dirs []string, fls ...Filter) *FileFinder { + c := NewConfig(dirs...) + c.Filters = fls - // mark has been run find() - founded bool - // founded file paths. - filePaths []string + return NewWithConfig(c) +} - // the founded file instances - // osFiles map[string]*os.File - osInfos map[string]os.FileInfo +// NewFinder new instance with source dir paths. +func NewFinder(dirPaths ...string) *FileFinder { + return New(dirPaths) +} - // handlers on found - pathHandler func(filePath string) - statHandler func(fi os.FileInfo, filePath string) +// NewWithConfig new instance with config. +func NewWithConfig(c *Config) *FileFinder { + return &FileFinder{ + c: c, + } } -// EmptyFinder new empty FileFinder instance -func EmptyFinder() *FileFinder { - return &FileFinder{} +// WithConfig on the finder +func (f *FileFinder) WithConfig(c *Config) *FileFinder { + f.c = c + return f } -// NewFinder new instance with source dir paths. -func NewFinder(dirPaths []string, filePaths ...string) *FileFinder { - return &FileFinder{ - dirPaths: dirPaths, - filePaths: filePaths, +// ConfigFn the finder +func (f *FileFinder) ConfigFn(fns ...func(c *Config)) *FileFinder { + if f.c == nil { + f.c = &Config{} + } + + for _, fn := range fns { + fn(f.c) } + return f } // AddDirPath add source dir for find func (f *FileFinder) AddDirPath(dirPaths ...string) *FileFinder { - f.dirPaths = append(f.dirPaths, dirPaths...) + f.c.DirPaths = append(f.c.DirPaths, dirPaths...) return f } // AddDir add source dir for find. alias of AddDirPath() func (f *FileFinder) AddDir(dirPaths ...string) *FileFinder { - f.dirPaths = append(f.dirPaths, dirPaths...) + f.c.DirPaths = append(f.c.DirPaths, dirPaths...) + return f +} + +// CacheResult cache result for find result. +func (f *FileFinder) CacheResult(enable ...bool) *FileFinder { + if len(enable) > 0 { + f.c.CacheResult = enable[0] + } else { + f.c.CacheResult = true + } return f } // ExcludeDotDir exclude dot dir names. eg: ".idea" func (f *FileFinder) ExcludeDotDir(exclude ...bool) *FileFinder { if len(exclude) > 0 { - f.excludeDotDir = exclude[0] + f.c.ExcludeDotDir = exclude[0] } else { - f.excludeDotDir = true + f.c.ExcludeDotDir = true } return f } +// WithoutDotDir exclude dot dir names. alias of ExcludeDotDir(). +func (f *FileFinder) WithoutDotDir(exclude ...bool) *FileFinder { + return f.ExcludeDotDir(exclude...) +} + // NoDotDir exclude dot dir names. alias of ExcludeDotDir(). func (f *FileFinder) NoDotDir(exclude ...bool) *FileFinder { return f.ExcludeDotDir(exclude...) @@ -102,13 +108,18 @@ func (f *FileFinder) NoDotDir(exclude ...bool) *FileFinder { // ExcludeDotFile exclude dot dir names. eg: ".gitignore" func (f *FileFinder) ExcludeDotFile(exclude ...bool) *FileFinder { if len(exclude) > 0 { - f.excludeDotFile = exclude[0] + f.c.ExcludeDotFile = exclude[0] } else { - f.excludeDotFile = true + f.c.ExcludeDotFile = true } return f } +// WithoutDotFile exclude dot dir names. alias of ExcludeDotFile(). +func (f *FileFinder) WithoutDotFile(exclude ...bool) *FileFinder { + return f.ExcludeDotFile(exclude...) +} + // NoDotFile exclude dot dir names. alias of ExcludeDotFile(). func (f *FileFinder) NoDotFile(exclude ...bool) *FileFinder { return f.ExcludeDotFile(exclude...) @@ -116,132 +127,182 @@ func (f *FileFinder) NoDotFile(exclude ...bool) *FileFinder { // ExcludeDir exclude dir names. func (f *FileFinder) ExcludeDir(dirs ...string) *FileFinder { - f.excludeDirs = append(f.excludeDirs, dirs...) + f.c.ExcludeDirs = append(f.c.ExcludeDirs, dirs...) return f } // ExcludeName exclude file names. func (f *FileFinder) ExcludeName(files ...string) *FileFinder { - f.excludeNames = append(f.excludeNames, files...) + f.c.ExcludeNames = append(f.c.ExcludeNames, files...) return f } -// AddFilter for filter filepath or dirpath -func (f *FileFinder) AddFilter(filterFuncs ...any) *FileFinder { - return f.WithFilter(filterFuncs...) +// AddFilter for filter filepath or dir path +func (f *FileFinder) AddFilter(filters ...Filter) *FileFinder { + return f.WithFilter(filters...) } -// WithFilter add filter func for filtering filepath or dirpath -func (f *FileFinder) WithFilter(filterFuncs ...any) *FileFinder { - for _, filterFunc := range filterFuncs { - if fileFilter, ok := filterFunc.(FileFilter); ok { - f.fileFilters = append(f.fileFilters, fileFilter) - } else if dirFilter, ok := filterFunc.(DirFilter); ok { - f.dirFilters = append(f.dirFilters, dirFilter) - } - } +// WithFilter add filter func for filtering filepath or dir path +func (f *FileFinder) WithFilter(filters ...Filter) *FileFinder { + f.c.Filters = append(f.c.Filters, filters...) return f } // AddFileFilter for filter filepath -func (f *FileFinder) AddFileFilter(filterFuncs ...FileFilter) *FileFinder { - f.fileFilters = append(f.fileFilters, filterFuncs...) - return f +func (f *FileFinder) AddFileFilter(filters ...Filter) *FileFinder { + return f.WithFileFilter(filters...) } // WithFileFilter for filter func for filtering filepath -func (f *FileFinder) WithFileFilter(filterFuncs ...FileFilter) *FileFinder { - f.fileFilters = append(f.fileFilters, filterFuncs...) +func (f *FileFinder) WithFileFilter(filters ...Filter) *FileFinder { + f.c.FileFilters = append(f.c.FileFilters, filters...) return f } // AddDirFilter for filter file contents -func (f *FileFinder) AddDirFilter(filterFuncs ...DirFilter) *FileFinder { - f.dirFilters = append(f.dirFilters, filterFuncs...) - return f +func (f *FileFinder) AddDirFilter(fls ...Filter) *FileFinder { + return f.WithDirFilter(fls...) } // WithDirFilter for filter func for filtering file contents -func (f *FileFinder) WithDirFilter(filterFuncs ...DirFilter) *FileFinder { - f.dirFilters = append(f.dirFilters, filterFuncs...) +func (f *FileFinder) WithDirFilter(filters ...Filter) *FileFinder { + f.c.DirFilters = append(f.c.DirFilters, filters...) return f } -// // AddBodyFilter for filter file contents -// func (f *FileFinder) AddBodyFilter(filterFuncs ...BodyFilter) *FileFinder { -// f.bodyFilters = append(f.bodyFilters, filterFuncs...) -// return f -// } -// -// // WithBodyFilter for filter func for filtering file contents -// func (f *FileFinder) WithBodyFilter(filterFuncs ...BodyFilter) *FileFinder { -// f.bodyFilters = append(f.bodyFilters, filterFuncs...) -// return f -// } - -// AddFilePaths set founded files -func (f *FileFinder) AddFilePaths(filePaths []string) { - f.filePaths = append(f.filePaths, filePaths...) +// AddBodyFilter for filter file contents +func (f *FileFinder) AddBodyFilter(fls ...BodyFilter) *FileFinder { + return f.WithBodyFilter(fls...) } -// AddFilePath add source file -func (f *FileFinder) AddFilePath(filePaths ...string) *FileFinder { - f.filePaths = append(f.filePaths, filePaths...) +// WithBodyFilter for filter func for filtering file contents +func (f *FileFinder) WithBodyFilter(fls ...BodyFilter) *FileFinder { + f.c.BodyFilters = append(f.c.BodyFilters, fls...) return f } -// AddFile add source file. alias of AddFilePath() -func (f *FileFinder) AddFile(filePaths ...string) *FileFinder { - f.filePaths = append(f.filePaths, filePaths...) - return f +// Find files in given dir paths. will return a channel, you can use it to get the result. +// +// Usage: +// +// f := NewFinder("/path/to/dir").Find() +// for el := range f { +// fmt.Println(el.Path()) +// } +func (f *FileFinder) Find() <-chan Elem { + f.find() + return f.ch } -// FindAll find and return founded file paths. -func (f *FileFinder) FindAll() []string { +// Results find and return founded file Elem. alias of Find() +// +// Usage: +// +// rs := NewFinder("/path/to/dir").Results() +// for el := range rs { +// fmt.Println(el.Path()) +// } +func (f *FileFinder) Results() <-chan Elem { f.find() - return f.filePaths + return f.ch } -// Find files in given dir paths. -func (f *FileFinder) Find() *FileFinder { +// FindPaths find and return founded file paths. +func (f *FileFinder) FindPaths() []string { f.find() - return f + + paths := make([]string, 0, 8*len(f.c.DirPaths)) + for el := range f.ch { + paths = append(paths, el.Path()) + } + return paths } -// do finding -func (f *FileFinder) find() { - if f.founded { - return +// Each file or dir Elem. +func (f *FileFinder) Each(fn func(el Elem)) { + f.EachElem(fn) +} + +// EachElem file or dir Elem. +func (f *FileFinder) EachElem(fn func(el Elem)) { + f.find() + for el := range f.ch { + fn(el) } +} - // mark found - f.founded = true - for _, filePath := range f.filePaths { - fi, err := os.Stat(filePath) - if err != nil { - continue // ignore I/O error - } - if fi.IsDir() { - continue // ignore I/O error +// EachPath file paths. +func (f *FileFinder) EachPath(fn func(filePath string)) { + f.EachElem(func(el Elem) { + fn(el.Path()) + }) +} + +// EachFile each file os.File +func (f *FileFinder) EachFile(fn func(file *os.File)) { + f.EachElem(func(el Elem) { + file, err := os.Open(el.Path()) + if err == nil { + fn(file) + } else { + f.err = err } + }) +} - // call handler - if f.pathHandler != nil { - f.pathHandler(filePath) +// EachStat each file os.FileInfo +func (f *FileFinder) EachStat(fn func(fi os.FileInfo, filePath string)) { + f.EachElem(func(el Elem) { + fi, err := el.Info() + if err == nil { + fn(fi, el.Path()) + } else { + f.err = err } - if f.statHandler != nil { - f.statHandler(fi, filePath) + }) +} + +// EachContents handle each found file contents +func (f *FileFinder) EachContents(fn func(contents, filePath string)) { + f.EachElem(func(el Elem) { + bs, err := os.ReadFile(el.Path()) + if err == nil { + fn(string(bs), el.Path()) + } else { + f.err = err } - } + }) +} - // do finding - for _, dirPath := range f.dirPaths { - f.findInDir(dirPath) +// do finding +func (f *FileFinder) find() { + f.err = nil + f.ch = make(chan Elem, 8) + + if f.c == nil { + f.c = NewConfig() } + + go func() { + defer close(f.ch) + + // read from caches + if f.c.CacheResult && len(f.caches) > 0 { + for _, el := range f.caches { + f.ch <- el + } + return + } + + // do finding + for _, dirPath := range f.c.DirPaths { + f.findDir(dirPath, f.c) + } + }() } // code refer filepath.glob() -func (f *FileFinder) findInDir(dirPath string) { +func (f *FileFinder) findDir(dirPath string, c *Config) { dfi, err := os.Stat(dirPath) if err != nil { return // ignore I/O error @@ -250,124 +311,119 @@ func (f *FileFinder) findInDir(dirPath string) { return // ignore I/O error } - // sort.Strings(names) - // names, _ := d.Readdirnames(-1) - stats, err := ioutil.ReadDir(dirPath) + des, err := os.ReadDir(dirPath) if err != nil { return // ignore I/O error } - for _, fi := range stats { - baseName := fi.Name() + c.curDepth++ + for _, ent := range des { + baseName := ent.Name() fullPath := filepath.Join(dirPath, baseName) + ok := false + el := NewElem(fullPath, ent) + + // apply generic filters + for _, filter := range c.Filters { + if filter.Apply(el) { // 有一个满足即可 + ok = true + break + } + } + if !ok { + continue + } + // --- dir - if fi.IsDir() { - if f.excludeDotDir && baseName[0] == '.' { + if ent.IsDir() { + if c.ExcludeDotDir && baseName[0] == '.' { continue } - ok := true - for _, df := range f.dirFilters { - ok = df.FilterDir(fullPath, baseName) - if ok { // 有一个满足即可 + // apply dir filters + ok = false + for _, df := range c.DirFilters { + if df.Apply(el) { // 有一个满足即可 + ok = true break } } - // find in sub dir. if ok { - f.findInDir(fullPath) + if c.FindFlags&FlagDir > 0 { + if c.CacheResult { + f.caches = append(f.caches, el) + } + f.ch <- el + } + + // find in sub dir. + if c.curDepth < c.MaxDepth { + f.findDir(fullPath, c) + } } continue } - // --- file - if f.excludeDotFile && baseName[0] == '.' { + if c.FindFlags&FlagDir > 0 { + continue + } + + // --- type: file + if c.ExcludeDotFile && baseName[0] == '.' { continue } // use custom filter functions - ok := true - for _, ff := range f.fileFilters { - if ok = ff.FilterFile(fullPath, baseName); ok { // 有一个满足即可 + ok = false + for _, ff := range c.FileFilters { + if ff.Apply(el) { // 有一个满足即可 + ok = true break } } - // append - if ok { - f.filePaths = append(f.filePaths, fullPath) - - // call handler - if f.pathHandler != nil { - f.pathHandler(fullPath) + // write to consumer + if ok && c.FindFlags&FlagFile > 0 { + if c.CacheResult { + f.caches = append(f.caches, el) } - if f.statHandler != nil { - f.statHandler(fi, fullPath) - } - } - } -} - -// EachFile each file os.File -func (f *FileFinder) EachFile(fn func(file *os.File)) { - f.Each(func(filePath string) { - file, err := os.Open(filePath) - if err != nil { - return - } - fn(file) - }) -} - -// Each file paths. -func (f *FileFinder) Each(fn func(filePath string)) { - f.pathHandler = fn - if f.founded { - for _, filePath := range f.filePaths { - fn(filePath) + f.ch <- el } - return } - - f.find() } -// EachStat each file os.FileInfo -func (f *FileFinder) EachStat(fn func(fi os.FileInfo, filePath string)) { - f.statHandler = fn - f.find() +// Reset filters config setting and results info. +func (f *FileFinder) Reset() { + c := NewConfig(f.c.DirPaths...) + c.ExcludeDotDir = f.c.ExcludeDotDir + c.FindFlags = f.c.FindFlags + c.MaxDepth = f.c.MaxDepth + c.curDepth = 0 + + f.c = c + f.err = nil + f.ch = make(chan Elem, 8) + f.caches = []Elem{} } -// EachContents handle each found file contents -func (f *FileFinder) EachContents(fn func(contents, filePath string)) { - f.Each(func(filePath string) { - bts, err := os.ReadFile(filePath) - if err != nil { - return - } - - fn(string(bts), filePath) - }) +// Err get last error +func (f *FileFinder) Err() error { + return f.err } -// Reset data setting. -func (f *FileFinder) Reset() { - f.founded = false - f.filePaths = f.filePaths[:0] - - f.excludeNames = make([]string, 0) - f.excludeExts = make([]string, 0) - f.excludeDirs = make([]string, 0) +// CacheNum get +func (f *FileFinder) CacheNum() int { + return len(f.caches) } -// FilePaths get -func (f *FileFinder) FilePaths() []string { - return f.filePaths +// Config get +func (f *FileFinder) Config() Config { + return *f.c } -// String all file paths +// String all dir paths func (f *FileFinder) String() string { - return strings.Join(f.filePaths, "\n") + return strings.Join(f.c.DirPaths, ",") } diff --git a/fsutil/finder/finder_test.go b/fsutil/finder/finder_test.go index 1e09b342f..252020160 100644 --- a/fsutil/finder/finder_test.go +++ b/fsutil/finder/finder_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/gookit/goutil/fsutil/finder" + "github.com/gookit/goutil/testutil" "github.com/gookit/goutil/testutil/assert" ) @@ -14,23 +15,21 @@ func TestEmptyFinder(t *testing.T) { f. AddDir("./testdata"). - AddFile("finder.go"). NoDotFile(). + CacheResult(). // NoDotDir(). - Find(). - Each(func(filePath string) { + EachPath(func(filePath string) { fmt.Println(filePath) }) - assert.NotEmpty(t, f.FilePaths()) + assert.NotEmpty(t, f.FindPaths()) f.Reset() - assert.Empty(t, f.FilePaths()) + assert.Empty(t, f.FindPaths()) } func TestNewFinder(t *testing.T) { - finder.NewFinder([]string{"./testdata"}). - AddFile("finder.go"). + finder.NewFinder("./testdata"). NoDotDir(). EachStat(func(fi os.FileInfo, filePath string) { fmt.Println(filePath, "=>", fi.ModTime()) @@ -38,59 +37,58 @@ func TestNewFinder(t *testing.T) { } func TestDotFileFilterFunc(t *testing.T) { - f := finder.EmptyFinder(). - AddDir("./testdata"). - Find() + f := finder.NewEmpty(). + AddDir("./testdata") + assert.NotEmpty(t, f.String()) + fmt.Println("no limits:") fmt.Println(f) fileName := ".env" - assert.Contains(t, f.String(), fileName) + assert.Contains(t, f.FindPaths(), fileName) f = finder.EmptyFinder(). AddDir("./testdata"). - NoDotFile(). - Find() + NoDotFile() + fmt.Println("NoDotFile limits:") fmt.Println(f) - assert.NotContains(t, f.String(), fileName) + assert.NotContains(t, f.FindPaths(), fileName) f = finder.EmptyFinder(). AddDir("./testdata"). - WithFilter(finder.DotFileFilterFunc(false)). - Find() + WithFilter(finder.DotFileFilter(false)) - fmt.Println("DotFileFilterFunc limits:") + fmt.Println("DotFileFilter limits:") fmt.Println(f) - assert.NotContains(t, f.String(), fileName) + assert.NotContains(t, f.FindPaths(), fileName) } func TestDotDirFilterFunc(t *testing.T) { f := finder.EmptyFinder(). - AddDir("./testdata"). - Find() + AddDir("./testdata") + fmt.Println("no limits:") fmt.Println(f) dirName := ".dotdir" - assert.Contains(t, f.String(), dirName) + assert.Contains(t, f.FindPaths(), dirName) f = finder.EmptyFinder(). AddDir("./testdata"). - NoDotDir(). - Find() + NoDotDir() + fmt.Println("NoDotDir limits:") - fmt.Println(f) - assert.NotContains(t, f.String(), dirName) + fmt.Println(f.Config()) + assert.NotContains(t, f.FindPaths(), dirName) - f = finder.EmptyFinder(). + f = finder.NewEmpty(). AddDir("./testdata"). - WithDirFilter(finder.DotDirFilterFunc(false)). - Find() + WithDirFilter(finder.DotDirFilter(false)) - fmt.Println("DotDirFilterFunc limits:") + fmt.Println("DotDirFilter limits:") fmt.Println(f) - assert.NotContains(t, f.String(), dirName) + assert.NotContains(t, f.FindPaths(), dirName) } var testFiles = []string{ @@ -102,12 +100,14 @@ var testFiles = []string{ } func TestExtFilterFunc(t *testing.T) { - fn := finder.ExtFilterFunc([]string{".log"}, true) - assert.True(t, fn("info.log", "")) - assert.False(t, fn("info.tmp", "")) + ent := &testutil.DirEnt{} + + fn := finder.ExtFilter(true, ".log") + assert.True(t, fn(finder.NewElem("info.log", ent))) + assert.False(t, fn(finder.NewElem("info.tmp", ent))) - fn = finder.ExtFilterFunc([]string{".log"}, false) - assert.False(t, fn("info.log", "")) - assert.True(t, fn("info.tmp", "")) + fn = finder.ExtFilter(false, ".log") + assert.False(t, fn(finder.NewElem("info.log", ent))) + assert.True(t, fn(finder.NewElem("info.tmp", ent))) } diff --git a/fsutil/finder/result.go b/fsutil/finder/result.go deleted file mode 100644 index 9b4b8bb9d..000000000 --- a/fsutil/finder/result.go +++ /dev/null @@ -1,45 +0,0 @@ -package finder - -// FileMeta struct -// type FileMeta struct { -// filePath string -// filename string -// } - -// FindResults struct -type FindResults struct { - f *FileFilter - - // founded file paths. - filePaths []string - - // filters - dirFilters []DirFilter // filters for filter dir paths - fileFilters []FileFilter // filters for filter file paths - // bodyFilters []BodyFilter // filters for filter file contents -} - -func (r *FindResults) append(filePath ...string) { - r.filePaths = append(r.filePaths, filePath...) -} - -// AddFilters Result get find paths -func (r *FindResults) AddFilters(filterFuncs ...FileFilter) *FindResults { - // TODO - return r -} - -// Filter Result get find paths -func (r *FindResults) Filter() *FindResults { - return r -} - -// Each Result get find paths -func (r *FindResults) Each() *FindResults { - return r -} - -// Result get find paths -func (r *FindResults) Result() []string { - return r.filePaths -}