diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..cc1aafb --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,35 @@ +version: 2.1 +executors: + base: + docker: + - image: circleci/golang:1.13 + working_directory: /go/src/github.com/spatialcurrent/go-walker +jobs: + pre_deps_golang: + executor: base + steps: + - checkout + - run: make deps_go + - run: sudo chown -R circleci /go/src + - save_cache: + key: v1-go-src-{{ .Branch }}-{{ .Revision }} + paths: + - /go/src + test_go: + executor: base + steps: + - run: sudo chown -R circleci /go/src + - restore_cache: + keys: + - v1-go-src-{{ .Branch }}-{{ .Revision }} + - run: make deps_go_test + - run: make test_go + - run: make imports + - run: git diff --exit-code +workflows: + main: + jobs: + - pre_deps_golang + - test_go: + requires: + - pre_deps_golang diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5950002 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +bin +dist +vendor +Gopkg.lock +*.so +*.h +.DS_Store +.npm +node_modules +npm-debug.* +package-lock.json diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..678d7fe --- /dev/null +++ b/AUTHORS @@ -0,0 +1,5 @@ +go-walker is maintained by Spatial Current, Inc. + +Authors: + +* Patrick Dufour (pjdufour) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..df86e1e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,15 @@ +# Contributing to go-walker + +## Contributor License Agreement + +Thank you for your interest in contributing. You will first need to agree to the license. Simply post the following paragraph, with "your name" and "your github account" substituted as needed, to the first issue [#1](https://github.com/spatialcurrent/go-walker/issues/1). Otherwise, you can email us [here](mailto:opensource@spatialcurrent.io?subject=CLA). + +I, **< YOUR NAME > (< YOUR GITHUB ACCOUNT >)**, agree to the license terms. My contributions to this repo are granted to **Spatial Current, Inc.** under a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable license and/or copyright is transferred. + +## Versioning + +This library is still in alpha (0.x.x), so versioning is not semantic yet. + +## Authors + +See [AUTHORS](https://github.com/spatialcurrent/go-walker/blob/master/AUTHORS) for a list of contributors. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ecfa533 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Spatial Current, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3d46b2e --- /dev/null +++ b/Makefile @@ -0,0 +1,59 @@ +# ================================================================= +# +# Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +# Released as open source under the MIT License. See LICENSE file. +# +# ================================================================= + +.PHONY: help +help: ## Print the help documentation + @grep -E '^[a-zA-Z_-\]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +# +# Dependencies +# + +.PHONY: deps_go +deps_go: ## Install Go dependencies + go get -d -t ./... + +.PHONY: deps_go_test +deps_go_test: ## Download Go dependencies for tests + go get golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow # download shadow + go install golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow # install shadow + go get -u github.com/kisielk/errcheck # download and install errcheck + go get -u github.com/client9/misspell/cmd/misspell # download and install misspell + go get -u github.com/gordonklaus/ineffassign # download and install ineffassign + go get -u honnef.co/go/tools/cmd/staticcheck # download and instal staticcheck + go get -u golang.org/x/tools/cmd/goimports # download and install goimports + +deps_gopherjs: ## Install GopherJS + go get -u github.com/gopherjs/gopherjs + +deps_javascript: ## Install dependencies for JavaScript tests + npm install . + +# +# Go building, formatting, testing, and installing +# + +.PHONY: fmt +fmt: ## Format Go source code + go fmt $$(go list ./... ) + +.PHONY: imports +imports: ## Update imports in Go source code + goimports -w -local github.com/spatialcurrent/go-walker,github.com/spatialcurrent/ $$(find . -iname '*.go') + +.PHONY: vet +vet: ## Vet Go source code + go vet $$(go list ./... ) + +.PHONY: test_go +test_go: ## Run Go tests + bash scripts/test.sh + +## Clean + +clean: ## Clean artifacts + rm -fr bin diff --git a/pkg/iterator/Iterator.go b/pkg/iterator/Iterator.go new file mode 100644 index 0000000..5b9e679 --- /dev/null +++ b/pkg/iterator/Iterator.go @@ -0,0 +1,84 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package iterator + +import ( + "io" + "os" + "syscall" +) + +type Iterator struct { + fd int + data []byte + buf []byte + names []string + eof bool + err error +} + +func New(fd int) *Iterator { + return &Iterator{ + fd: fd, + data: make([]byte, 0), + buf: make([]byte, os.Getpagesize()), + names: make([]string, 0), + eof: false, + err: nil, + } +} + +func (it *Iterator) Reset(fd int) { + it.fd = fd + it.data = make([]byte, 0) + it.buf = make([]byte, os.Getpagesize()) + it.names = make([]string, 0) + it.eof = false + it.err = nil +} + +func (it *Iterator) Next() (string, error) { + if it.err != nil { + return "", it.err + } + if it.eof { + return "", io.EOF + } + if len(it.names) > 0 { + name := it.names[0] + it.names = it.names[1:] + return name, nil + } + for { + consumed, count, names := syscall.ParseDirent(it.data, 1, it.names[:0]) + it.data = it.data[consumed:] + if count == 0 { + for { + n, err := syscall.ReadDirent(it.fd, it.buf) + if err != nil { + it.err = err + return "", err + } + if n == 0 { + it.eof = true + return "", io.EOF + } + it.data = append(it.data, it.buf[0:n]...) + break + } + continue + } + if count == 1 { + return names[0], nil + } + it.names = names[1:] + return names[0], nil + } + it.eof = true + return "", io.EOF +} diff --git a/pkg/iterator/doc.go b/pkg/iterator/doc.go new file mode 100644 index 0000000..9e2dd6b --- /dev/null +++ b/pkg/iterator/doc.go @@ -0,0 +1,9 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +// Package iterator is used for iterator through a list of directory entries. +package iterator diff --git a/pkg/modeutil/IsIrregular.go b/pkg/modeutil/IsIrregular.go new file mode 100644 index 0000000..5aba22d --- /dev/null +++ b/pkg/modeutil/IsIrregular.go @@ -0,0 +1,16 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package modeutil + +import ( + "os" +) + +func IsIrregular(fi os.FileInfo) bool { + return fi.Mode()&os.ModeIrregular != 0 +} diff --git a/pkg/modeutil/IsLink.go b/pkg/modeutil/IsLink.go new file mode 100644 index 0000000..5750671 --- /dev/null +++ b/pkg/modeutil/IsLink.go @@ -0,0 +1,16 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package modeutil + +import ( + "os" +) + +func IsLink(fi os.FileInfo) bool { + return fi.Mode()&os.ModeSymlink != 0 +} diff --git a/pkg/modeutil/IsLink_test.go b/pkg/modeutil/IsLink_test.go new file mode 100644 index 0000000..ad33d6c --- /dev/null +++ b/pkg/modeutil/IsLink_test.go @@ -0,0 +1,44 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package modeutil + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsLinkLstatRegular(t *testing.T) { + fi, err := os.Lstat("testdata/doc.1.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.False(t, IsLink(fi)) +} + +func TestIsLinkLstatLink(t *testing.T) { + fi, err := os.Lstat("testdata/doc.2.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.True(t, IsLink(fi)) +} + +func TestIsLinkLstatNamedPipe(t *testing.T) { + createNamedPipeIfNotExist("testdata/doc.3.txt") + fi, err := os.Lstat("testdata/doc.3.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.False(t, IsLink(fi)) +} + +func TestIsLinkStatLink(t *testing.T) { + fi, err := os.Stat("testdata/doc.2.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.False(t, IsLink(fi)) +} diff --git a/pkg/modeutil/IsNamedPipe.go b/pkg/modeutil/IsNamedPipe.go new file mode 100644 index 0000000..5113279 --- /dev/null +++ b/pkg/modeutil/IsNamedPipe.go @@ -0,0 +1,16 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package modeutil + +import ( + "os" +) + +func IsNamedPipe(fi os.FileInfo) bool { + return fi.Mode()&os.ModeNamedPipe != 0 +} diff --git a/pkg/modeutil/IsNamedPipe_test.go b/pkg/modeutil/IsNamedPipe_test.go new file mode 100644 index 0000000..43e8626 --- /dev/null +++ b/pkg/modeutil/IsNamedPipe_test.go @@ -0,0 +1,44 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package modeutil + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsNamedPipeLstatRegular(t *testing.T) { + fi, err := os.Lstat("testdata/doc.1.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.False(t, IsNamedPipe(fi)) +} + +func TestIsNamedPipeLstatLink(t *testing.T) { + fi, err := os.Lstat("testdata/doc.2.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.False(t, IsNamedPipe(fi)) +} + +func TestIsNamedPipeLstatNamedPipe(t *testing.T) { + createNamedPipeIfNotExist("testdata/doc.3.txt") + fi, err := os.Lstat("testdata/doc.3.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.True(t, IsNamedPipe(fi)) +} + +func TestIsNamedPipeStatLink(t *testing.T) { + fi, err := os.Stat("testdata/doc.2.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.False(t, IsNamedPipe(fi)) +} diff --git a/pkg/modeutil/IsRegular.go b/pkg/modeutil/IsRegular.go new file mode 100644 index 0000000..0042bd3 --- /dev/null +++ b/pkg/modeutil/IsRegular.go @@ -0,0 +1,16 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package modeutil + +import ( + "os" +) + +func IsRegular(fi os.FileInfo) bool { + return fi.Mode()&os.ModeType == 0 +} diff --git a/pkg/modeutil/IsRegular_test.go b/pkg/modeutil/IsRegular_test.go new file mode 100644 index 0000000..6f797e0 --- /dev/null +++ b/pkg/modeutil/IsRegular_test.go @@ -0,0 +1,44 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package modeutil + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsRegularLstatRegular(t *testing.T) { + fi, err := os.Lstat("testdata/doc.1.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.True(t, IsRegular(fi)) +} + +func TestIsRegularLstatLink(t *testing.T) { + fi, err := os.Lstat("testdata/doc.2.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.False(t, IsRegular(fi)) +} + +func TestIsRegularLstatNamedPipe(t *testing.T) { + createNamedPipeIfNotExist("testdata/doc.3.txt") + fi, err := os.Lstat("testdata/doc.3.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.False(t, IsRegular(fi)) +} + +func TestIsRegularStatLink(t *testing.T) { + fi, err := os.Stat("testdata/doc.2.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.True(t, IsRegular(fi)) +} diff --git a/pkg/modeutil/IsSocket.go b/pkg/modeutil/IsSocket.go new file mode 100644 index 0000000..b41f3ce --- /dev/null +++ b/pkg/modeutil/IsSocket.go @@ -0,0 +1,16 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package modeutil + +import ( + "os" +) + +func IsSocket(fi os.FileInfo) bool { + return fi.Mode()&os.ModeSocket != 0 +} diff --git a/pkg/modeutil/IsSocket_test.go b/pkg/modeutil/IsSocket_test.go new file mode 100644 index 0000000..59a3fe8 --- /dev/null +++ b/pkg/modeutil/IsSocket_test.go @@ -0,0 +1,44 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package modeutil + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsSocketLstatRegular(t *testing.T) { + fi, err := os.Lstat("testdata/doc.1.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.False(t, IsSocket(fi)) +} + +func TestIsSocketLstatLink(t *testing.T) { + fi, err := os.Lstat("testdata/doc.2.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.False(t, IsSocket(fi)) +} + +func TestIsSocketLstatNamedPipe(t *testing.T) { + createNamedPipeIfNotExist("testdata/doc.3.txt") + fi, err := os.Lstat("testdata/doc.3.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.False(t, IsSocket(fi)) +} + +func TestIsSocketStatLink(t *testing.T) { + fi, err := os.Stat("testdata/doc.2.txt") + assert.NoError(t, err) + assert.NotNil(t, fi) + assert.False(t, IsSocket(fi)) +} diff --git a/pkg/modeutil/doc.go b/pkg/modeutil/doc.go new file mode 100644 index 0000000..b8e43ae --- /dev/null +++ b/pkg/modeutil/doc.go @@ -0,0 +1,9 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +// Package modeutil contains convenience functions for checking the mode of a file. +package modeutil diff --git a/pkg/modeutil/modeutil_test.go b/pkg/modeutil/modeutil_test.go new file mode 100644 index 0000000..94f3b77 --- /dev/null +++ b/pkg/modeutil/modeutil_test.go @@ -0,0 +1,20 @@ +package modeutil + +import ( + "os" + "syscall" +) + +func createNamedPipeIfNotExist(p string) { + _, err := os.Lstat(p) + if err == nil { + return + } + if !os.IsNotExist(err) { + panic(err) + } + err = syscall.Mkfifo(p, 0660) + if err != nil { + panic(err) + } +} diff --git a/pkg/modeutil/testdata/doc.1.txt b/pkg/modeutil/testdata/doc.1.txt new file mode 100644 index 0000000..3b18e51 --- /dev/null +++ b/pkg/modeutil/testdata/doc.1.txt @@ -0,0 +1 @@ +hello world diff --git a/pkg/modeutil/testdata/doc.2.txt b/pkg/modeutil/testdata/doc.2.txt new file mode 120000 index 0000000..5d801be --- /dev/null +++ b/pkg/modeutil/testdata/doc.2.txt @@ -0,0 +1 @@ +doc.1.txt \ No newline at end of file diff --git a/pkg/oserror/IsDeniedPermission.go b/pkg/oserror/IsDeniedPermission.go new file mode 100644 index 0000000..0a587c7 --- /dev/null +++ b/pkg/oserror/IsDeniedPermission.go @@ -0,0 +1,21 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package oserror + +import ( + "errors" + "os" +) + +func IsDeniedPermission(err error) bool { + var pathError *os.PathError + if errors.As(err, &pathError) && os.IsPermission(pathError) { + return true + } + return false +} diff --git a/pkg/oserror/IsNotExist.go b/pkg/oserror/IsNotExist.go new file mode 100644 index 0000000..c7370c2 --- /dev/null +++ b/pkg/oserror/IsNotExist.go @@ -0,0 +1,21 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package oserror + +import ( + "errors" + "os" +) + +func IsNotExist(err error) bool { + var pathError *os.PathError + if errors.As(err, &pathError) && os.IsNotExist(pathError) { + return true + } + return false +} diff --git a/pkg/oserror/doc.go b/pkg/oserror/doc.go new file mode 100644 index 0000000..a4b1200 --- /dev/null +++ b/pkg/oserror/doc.go @@ -0,0 +1,9 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +// Package oserror contains convenience functions for checking the error returned by the operating system. +package oserror diff --git a/pkg/pathutil/IsCycle.go b/pkg/pathutil/IsCycle.go new file mode 100644 index 0000000..cadd7ed --- /dev/null +++ b/pkg/pathutil/IsCycle.go @@ -0,0 +1,33 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package pathutil + +import ( + "path/filepath" + "strings" +) + +// IsCycle returns true if a link targets itself or a parent directory, +// therefore creating a infinite loop or cycle. +func IsCycle(path string, target string) bool { + targetCleaned := filepath.Clean(target) + if target == "." || target == ".." { + //fmt.Println("Is Cycle:", target) + return true + } + pathCleaned := filepath.Clean(path) + if strings.HasPrefix(pathCleaned, targetCleaned) { + //fmt.Println("Is Cycle:", path, target) + return true + } + if strings.HasPrefix(target, "../") { + //fmt.Println("Is Cycle:", target) + return true + } + return false +} diff --git a/pkg/pathutil/doc.go b/pkg/pathutil/doc.go new file mode 100644 index 0000000..0617a33 --- /dev/null +++ b/pkg/pathutil/doc.go @@ -0,0 +1,9 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +// Package pathutil contains convenience functions for checking a path. +package pathutil diff --git a/pkg/walker/CachedFile.go b/pkg/walker/CachedFile.go new file mode 100644 index 0000000..2fb431d --- /dev/null +++ b/pkg/walker/CachedFile.go @@ -0,0 +1,44 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package walker + +import ( + "os" + + "github.com/spatialcurrent/go-lazy/pkg/lazy" +) + +type CachedFile struct { + *lazy.LazyFile + fileInfo os.FileInfo + err error +} + +func NewCachedFile(lazyFile *lazy.LazyFile) *CachedFile { + return &CachedFile{ + LazyFile: lazyFile, + fileInfo: nil, + err: nil, + } +} + +func (f *CachedFile) Stat() (os.FileInfo, error) { + if f.err != nil { + return nil, f.err + } + if f.fileInfo != nil { + return f.fileInfo, nil + } + fileInfo, err := f.LazyFile.Stat() + if err != nil { + f.err = err + return nil, err + } + f.fileInfo = fileInfo + return fileInfo, nil +} diff --git a/pkg/walker/File.go b/pkg/walker/File.go new file mode 100644 index 0000000..3953110 --- /dev/null +++ b/pkg/walker/File.go @@ -0,0 +1,18 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package walker + +import ( + "os" +) + +type File interface { + Read(b []byte) (n int, err error) + Stat() (os.FileInfo, error) + Close() error +} diff --git a/pkg/walker/Walker.go b/pkg/walker/Walker.go new file mode 100644 index 0000000..c910b4f --- /dev/null +++ b/pkg/walker/Walker.go @@ -0,0 +1,425 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package walker + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/spatialcurrent/go-lazy/pkg/lazy" + "github.com/spatialcurrent/go-walker/pkg/iterator" + "github.com/spatialcurrent/go-walker/pkg/modeutil" + "github.com/spatialcurrent/go-walker/pkg/oserror" + "github.com/spatialcurrent/go-walker/pkg/pathutil" +) + +const ( + NoLimit = -1 +) + +type cacheFileInfo struct { + *os.File + fileInfo os.FileInfo + err error +} + +func (f *cacheFileInfo) Stat() (os.FileInfo, error) { + if f.err != nil { + return nil, f.err + } + if f.fileInfo != nil { + return f.fileInfo, nil + } + fi, err := f.File.Stat() + if err != nil { + f.err = err + return nil, err + } + f.fileInfo = fi + return fi, nil +} + +type Walker struct { + skipPathFn func(path string) (bool, error) + skipFileFn func(path string, file File) (bool, error) + skipLinkFn func(path string, target string) (bool, error) + errorLinkStatFn func(path string, err error) (bool, bool, error) + errorStatFn func(path string, file File, err error) (bool, error) + errorWalkFn func(path string, file File, err error) (bool, error) + limit int +} + +type NewWalkerInput struct { + SkipPath func(path string) (bool, error) + SkipFile func(path string, file File) (bool, error) + SkipLink func(path string, target string) (bool, error) + ErrorLinkStat func(path string, err error) (bool, bool, error) + ErrorStat func(path string, file File, err error) (bool, error) + ErrorWalk func(path string, file File, err error) (bool, error) + Limit int +} + +func NewWalker(input *NewWalkerInput) (*Walker, error) { + w := &Walker{ + skipPathFn: input.SkipPath, + skipFileFn: input.SkipFile, + skipLinkFn: input.SkipLink, + errorLinkStatFn: input.ErrorLinkStat, + errorStatFn: input.ErrorStat, + errorWalkFn: input.ErrorWalk, + limit: input.Limit, + } + return w, nil +} + +func (w *Walker) skipPath(path string) (bool, error) { + if w.skipPathFn == nil { + return false, nil + } + return w.skipPathFn(path) +} + +func (w *Walker) skipFile(path string, file File) (bool, error) { + if w.skipFileFn == nil { + return false, nil + } + return w.skipFileFn(path, file) +} + +func (w *Walker) skipLink(path string) (bool, error) { + if w.skipLinkFn == nil { + return false, nil + } + target, err := os.Readlink(path) + if err != nil { + if oserror.IsNotExist(err) { + return w.skipLinkFn(path, "") // if not exist then pass with target as a blank string. + } + if oserror.IsDeniedPermission(err) { + return w.skipLinkFn(path, "") // if denied permission then pass with target as a blank string. + } + return false, fmt.Errorf("error reading link %q: %w", path, err) + } + return w.skipLinkFn(path, target) +} + +func (w *Walker) handleErrorLinkStat(path string, err error) (bool, bool, error) { + if w.errorLinkStatFn == nil { + return false, true, err // skip but do not abort + } + return w.errorLinkStatFn(path, err) +} + +func (w *Walker) handleErrorStat(path string, file File, err error) (bool, error) { + if w.errorStatFn == nil { + return true, err + } + return w.errorStatFn(path, file, err) +} + +func (w *Walker) handleErrorWalk(path string, file File, err error) (bool, error) { + if w.errorWalkFn == nil { + return true, err + } + return w.errorWalkFn(path, file, err) +} + +func (w *Walker) WalkBucket(ctx context.Context, bucket string, prefix string, f func(ctx context.Context, path string, file File) error) (int, error) { + return 0, nil +} + +func (w *Walker) walkDirectory(ctx context.Context, dir string, d File, fd int, fn func(ctx context.Context, path string, file File) error, limit int) (int, error) { + + if limit == 0 { + return 0, nil + } + + it := iterator.New(fd) + + count := 0 + for { + + if limit == 0 { + break + } + + name, err := it.Next() + if err != nil { + if err == io.EOF { + break + } + return count, fmt.Errorf("error reading next directory entry for %q: %w", dir, err) + } + + path := filepath.Join(dir, name) + + skip, err := w.skipPath(path) + if err != nil { + return count, err + } + if skip { + continue + } + + linkFileInfo, err := os.Lstat(path) + if err != nil { + abort, skip, err := w.handleErrorLinkStat(path, err) + if abort { + return 0, fmt.Errorf("error link stating file %q: %w", path, err) + } + if skip { + continue + } + linkFileInfo = nil + } + + if modeutil.IsNamedPipe(linkFileInfo) || modeutil.IsIrregular(linkFileInfo) || modeutil.IsSocket(linkFileInfo) { + continue + } + + file := NewCachedFile(lazy.NewLazyFile(path, os.O_RDONLY, 0)) + + skip, err = w.skipFile(path, file) + if err != nil { + _ = file.Close() + return count, err + } + if skip { + continue + } + + if w.skipLinkFn != nil { + if linkFileInfo == nil { + panic(path) + } + if modeutil.IsLink(linkFileInfo) { + skip, err := w.skipLink(path) + if err != nil { + _ = file.Close() + if !oserror.IsNotExist(err) { + return count, err + } + } + if skip { + continue + } + } + } + + // increment counter + count += 1 + + // decrement limit + if limit > 0 { + limit -= 1 + } + + err = fn(ctx, path, file) + if err != nil { + abort, err := w.handleErrorWalk(path, file, err) + if abort { + _ = file.Close() + return count, fmt.Errorf("error calling walk function for file %q: %w", name, err) + } + } + + if limit == 0 { + break + } + + /* if we received an error when stating before, then do not try again + if fileInfoError != nil { + _ = file.Close() + continue + } + */ + + fileInfo, fileInfoError := file.Stat() + if fileInfoError != nil { + _ = file.Close() + abort, err := w.handleErrorStat(path, file, fileInfoError) + if abort { + return count, fmt.Errorf("error stating file %q: %w", name, err) + } else { + continue + } + } + + if fileInfo.IsDir() { + if linkFileInfo == nil { + panic(path) + } + if modeutil.IsLink(linkFileInfo) { + target, err := os.Readlink(path) + if err != nil { + _ = file.Close() + return count, fmt.Errorf("error reading link to directory %q: %w", path, err) + } + if !pathutil.IsCycle(path, target) { + fd, err := file.Fd() + if err != nil { + return count, fmt.Errorf("error opening directory %q: %w", dir, err) + } + n, err := w.walkDirectory(ctx, target, file, int(fd), fn, limit) + if err != nil { + _ = file.Close() + return count, fmt.Errorf("error walking directory %q: %w", path, err) + } + count += n + } + } else { + fd, err := file.Fd() + if err != nil { + return count, fmt.Errorf("error opening directory %q: %w", dir, err) + } + n, err := w.walkDirectory(ctx, path, file, int(fd), fn, limit) + if err != nil { + _ = file.Close() + return count, fmt.Errorf("error walking directory %q: %w", path, err) + } + count += n + } + } + + _ = file.Close() + + } + + return count, nil +} + +func (w *Walker) WalkFileSystem(ctx context.Context, root string, fn func(ctx context.Context, path string, file File) error, limit int) (int, error) { + + if limit == 0 { + return 0, nil + } + + if skip, err := w.skipPath(root); err != nil || skip { + return 0, err + } + + linkFileInfo, err := os.Lstat(root) + if err != nil { + abort, skip, err := w.handleErrorLinkStat(root, err) + if abort || skip { + return 0, fmt.Errorf("error link stating root %q: %w", root, err) + } + } + + r, err := os.Open(root) + if err != nil { + return 0, fmt.Errorf("error opening root %q: %w", root, err) + } + + cacheFileInfo := &cacheFileInfo{File: r, fileInfo: nil} + + if skip, err := w.skipFile(root, cacheFileInfo); err != nil || skip { + _ = r.Close() + return 0, err + } + + if w.skipLinkFn != nil { + if modeutil.IsLink(linkFileInfo) { + if skip, err := w.skipLink(root); err != nil || skip { + return 0, err + } + } + } + + err = fn(ctx, root, r) + if err != nil { + abort, err := w.handleErrorWalk(root, r, err) + if abort { + _ = r.Close() + return 1, fmt.Errorf("error calling walk function for root %q: %w", root, err) + } + } + + // decrement limit + if limit > 0 { + limit -= 1 + if limit == 0 { + return 1, nil + } + } + + fi, err := cacheFileInfo.Stat() + if err != nil { + _, err := w.handleErrorStat(root, r, err) + if err != nil { + _ = r.Close() + return 1, fmt.Errorf("error stating root %q: %w", root, err) + } + } + + if fi.IsDir() { + n, err := w.walkDirectory(ctx, root, cacheFileInfo, int(r.Fd()), fn, limit) + if err != nil { + return 1 + n, fmt.Errorf("error walking root directory %q: %w", root, err) + } + _ = r.Close() + return 1 + n, nil + } + + _ = r.Close() + return 1, nil +} + +func (w *Walker) splitUri(uri string) (string, string) { + if i := strings.Index(uri, "://"); i != -1 { + return uri[0:i], uri[i+3:] + } + return "", uri +} + +func (w *Walker) Walk(ctx context.Context, uris []string, f func(ctx context.Context, path string, file File) error) (int, error) { + if w.limit == 0 { + return 0, nil + } + count := 0 + for i, uri := range uris { + scheme, path := w.splitUri(uri) + if len(scheme) > 0 { + switch scheme { + case "s3": + if i := strings.Index(uri, "/"); i != -1 { + n, err := w.WalkBucket(ctx, uri[0:i], uri[i+1:], f) + if err != nil { + return n + count, fmt.Errorf("error walking bucket %q: %w", uri, err) + } + count += n + } else { + n, err := w.WalkBucket(ctx, path, "", f) + if err != nil { + return n + count, fmt.Errorf("error walking bucket %q: %w", uri, err) + } + count += n + } + case "file": + n, err := w.WalkFileSystem(ctx, path, f, w.limit) + if err != nil { + return n + count, fmt.Errorf("error walking file system %q: %w", uri, err) + } + count += n + default: + return 0, fmt.Errorf("error walking uri %q (%d): unknown scheme %q", uri, i, scheme) + } + } else { + n, err := w.WalkFileSystem(ctx, uri, f, w.limit) + if err != nil { + return n + count, fmt.Errorf("error walking file system %q: %w", uri, err) + } + count += n + } + } + return count, nil +} diff --git a/pkg/walker/Walker_test.go b/pkg/walker/Walker_test.go new file mode 100644 index 0000000..e9efcad --- /dev/null +++ b/pkg/walker/Walker_test.go @@ -0,0 +1,34 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package walker + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWalker(t *testing.T) { + w, err := NewWalker(&NewWalkerInput{ + SkipPath: nil, + SkipFile: nil, + SkipLink: nil, + ErrorLinkStat: nil, + ErrorStat: nil, + ErrorWalk: nil, + Limit: NoLimit, + }) + require.NoError(t, err) + n, err := w.Walk(context.Background(), []string{"testdata"}, func(ctx context.Context, p string, f File) error { + return nil + }) + assert.Equal(t, 11, n) + assert.NoError(t, err) +} diff --git a/pkg/walker/doc.go b/pkg/walker/doc.go new file mode 100644 index 0000000..a0d5563 --- /dev/null +++ b/pkg/walker/doc.go @@ -0,0 +1,9 @@ +// ================================================================= +// +// Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +// Package walker is used for walking local and remote filesystems. +package walker diff --git a/pkg/walker/testdata/a/b/doc.1.txt b/pkg/walker/testdata/a/b/doc.1.txt new file mode 100644 index 0000000..b6fc4c6 --- /dev/null +++ b/pkg/walker/testdata/a/b/doc.1.txt @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/pkg/walker/testdata/a/b/doc.2.txt b/pkg/walker/testdata/a/b/doc.2.txt new file mode 100644 index 0000000..04fea06 --- /dev/null +++ b/pkg/walker/testdata/a/b/doc.2.txt @@ -0,0 +1 @@ +world \ No newline at end of file diff --git a/pkg/walker/testdata/a/doc.1.txt b/pkg/walker/testdata/a/doc.1.txt new file mode 100644 index 0000000..b6fc4c6 --- /dev/null +++ b/pkg/walker/testdata/a/doc.1.txt @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/pkg/walker/testdata/a/doc.2.txt b/pkg/walker/testdata/a/doc.2.txt new file mode 100644 index 0000000..04fea06 --- /dev/null +++ b/pkg/walker/testdata/a/doc.2.txt @@ -0,0 +1 @@ +world \ No newline at end of file diff --git a/pkg/walker/testdata/b/doc.1.txt b/pkg/walker/testdata/b/doc.1.txt new file mode 100644 index 0000000..b6fc4c6 --- /dev/null +++ b/pkg/walker/testdata/b/doc.1.txt @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/pkg/walker/testdata/b/doc.2.txt b/pkg/walker/testdata/b/doc.2.txt new file mode 100644 index 0000000..04fea06 --- /dev/null +++ b/pkg/walker/testdata/b/doc.2.txt @@ -0,0 +1 @@ +world \ No newline at end of file diff --git a/pkg/walker/testdata/doc.1.txt b/pkg/walker/testdata/doc.1.txt new file mode 100644 index 0000000..b6fc4c6 --- /dev/null +++ b/pkg/walker/testdata/doc.1.txt @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100644 index 0000000..f4b996f --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# ================================================================= +# +# Copyright (C) 2020 Spatial Current, Inc. - All Rights Reserved +# Released as open source under the MIT License. See LICENSE file. +# +# ================================================================= + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +set -eu +cd $DIR/.. +pkgs=$(go list ./... | grep -v /vendor/ | tr "\n" " ") +echo "******************" +echo "Running unit tests" +go test -p 1 -count 1 -short $pkgs +echo "******************" +echo "Running go vet" +go vet $pkgs +echo "******************" +echo "Running go vet with shadow" +go vet -vettool=$(which shadow) $pkgs +echo "******************" +echo "Running errcheck" +errcheck ${pkgs} +echo "******************" +echo "Running ineffassign" +find . -name '*.go' | xargs ineffassign +echo "******************" +echo "Running staticcheck" +staticcheck -checks all ${pkgs} +echo "******************" +echo "Running misspell" +misspell -locale US -error *.md *.go +echo "******************"