diff --git a/downloader/downloader.go b/downloader/downloader.go index 63035c4c..f5f14ad1 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -16,7 +16,7 @@ import ( "github.com/slack-go/slack" "golang.org/x/time/rate" - "github.com/rusq/slackdump/v2/fsadapter" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2/internal/network" "github.com/rusq/slackdump/v2/logger" ) @@ -238,6 +238,10 @@ func (c *Client) saveFile(ctx context.Context, dir string, sf *slack.File) (int6 if c.fs == nil { return 0, ErrNoFS } + if mode := sf.Mode; mode == "hidden_by_limit" || mode == "external" || sf.IsExternal { + trace.Logf(ctx, "info", "file %q is not downloadable", sf.Name) + return 0, nil + } filePath := filepath.Join(dir, c.nameFn(sf)) tf, err := os.CreateTemp("", "") diff --git a/downloader/downloader_example_test.go b/downloader/downloader_example_test.go index 914c790c..06cd06d5 100644 --- a/downloader/downloader_example_test.go +++ b/downloader/downloader_example_test.go @@ -4,8 +4,8 @@ import ( "context" "fmt" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2/downloader" - "github.com/rusq/slackdump/v2/fsadapter" "github.com/slack-go/slack" "golang.org/x/time/rate" ) @@ -37,6 +37,7 @@ func ExampleNew_advanced() { fmt.Println("failed to initialise the file system") return } + defer fs.Close() dl := downloader.New( client, diff --git a/downloader/downloader_test.go b/downloader/downloader_test.go index 6636b549..991534b9 100644 --- a/downloader/downloader_test.go +++ b/downloader/downloader_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/time/rate" - "github.com/rusq/slackdump/v2/fsadapter" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2/internal/fixtures" "github.com/rusq/slackdump/v2/internal/mocks/mock_downloader" ) diff --git a/export/expfmt.go b/export/expfmt.go index 3aefe5ca..8e7e5fab 100644 --- a/export/expfmt.go +++ b/export/expfmt.go @@ -3,7 +3,7 @@ package export import ( "github.com/slack-go/slack" - "github.com/rusq/slackdump/v2/fsadapter" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2/internal/structures/files/dl" "github.com/rusq/slackdump/v2/logger" ) diff --git a/export/expfmt_test.go b/export/expfmt_test.go index a676beb4..12e92f91 100644 --- a/export/expfmt_test.go +++ b/export/expfmt_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/rusq/slackdump/v2/fsadapter" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2/logger" "github.com/slack-go/slack" ) diff --git a/export/export.go b/export/export.go index 2a4688da..51a1afcf 100644 --- a/export/export.go +++ b/export/export.go @@ -9,10 +9,10 @@ import ( "path/filepath" "runtime/trace" + "github.com/rusq/fsadapter" "github.com/slack-go/slack" "github.com/rusq/slackdump/v2" - "github.com/rusq/slackdump/v2/fsadapter" "github.com/rusq/slackdump/v2/internal/network" "github.com/rusq/slackdump/v2/internal/structures" "github.com/rusq/slackdump/v2/internal/structures/files/dl" diff --git a/export/export_test.go b/export/export_test.go index e5808e32..14c0b392 100644 --- a/export/export_test.go +++ b/export/export_test.go @@ -14,8 +14,8 @@ import ( "time" gomock "github.com/golang/mock/gomock" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2" - "github.com/rusq/slackdump/v2/fsadapter" "github.com/rusq/slackdump/v2/internal/fixtures" "github.com/rusq/slackdump/v2/internal/mocks/mock_dl" "github.com/rusq/slackdump/v2/internal/mocks/mock_fsadapter" diff --git a/export/index.go b/export/index.go index b607a7c8..4fc2ca30 100644 --- a/export/index.go +++ b/export/index.go @@ -6,7 +6,7 @@ import ( "reflect" "strings" - "github.com/rusq/slackdump/v2/fsadapter" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2/internal/structures" "github.com/rusq/slackdump/v2/types" diff --git a/export/index_test.go b/export/index_test.go index 46657ed6..1faa1f5b 100644 --- a/export/index_test.go +++ b/export/index_test.go @@ -3,7 +3,7 @@ package export import ( "testing" - "github.com/rusq/slackdump/v2/fsadapter" + "github.com/rusq/fsadapter" "github.com/slack-go/slack" ) diff --git a/fsadapter/directory.go b/fsadapter/directory.go deleted file mode 100644 index 68838a82..00000000 --- a/fsadapter/directory.go +++ /dev/null @@ -1,80 +0,0 @@ -package fsadapter - -import ( - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" -) - -var _ FS = Directory{} - -type Directory struct { - dir string -} - -func NewDirectory(dir string) Directory { - return Directory{dir: dir} -} - -func (d Directory) String() string { - return "" -} - -func (fs Directory) Create(fpath string) (io.WriteCloser, error) { - node := filepath.Join(fs.dir, fpath) - if err := fs.ensureSubdir(node); err != nil { - return nil, fmt.Errorf("failed to create %s: %w", node, err) - } - nodeDir := filepath.Dir(node) - if err := mkdirAll(nodeDir); err != nil { - return nil, err - } - return os.Create(node) -} - -var ErrIllegalDir = errors.New("illegal file path reference outside of working directory") - -// ensureSubdir ensures that the node is a subdirectory of -// the fs.dir, and returns ErrIllegalDir, if not. This ensures -// that caller won't be able to do anything naughty. -func (fs Directory) ensureSubdir(node string) error { - if rel, err := filepath.Rel(fs.dir, node); err != nil { - return err - } else if strings.HasPrefix(rel, "..") { - return ErrIllegalDir - } - - return nil -} - -// mkdirAll creates a directory "name", if the directory exists, it does nothing. -func mkdirAll(name string) error { - if name == "" { - return errors.New("empty directory") - } - - fi, err := os.Stat(name) - if err == nil && fi.IsDir() { - // exists and is a directory - return nil - } - - if err := os.MkdirAll(name, 0755); err != nil { - return err - } - return nil -} - -func (fs Directory) WriteFile(name string, data []byte, perm os.FileMode) error { - node := filepath.Join(fs.dir, name) - if err := fs.ensureSubdir(node); err != nil { - return fmt.Errorf("WriteFile: %w", err) - } - if err := mkdirAll(filepath.Dir(node)); err != nil { - return err - } - return os.WriteFile(node, data, perm) -} diff --git a/fsadapter/directory_test.go b/fsadapter/directory_test.go deleted file mode 100644 index d116ebd9..00000000 --- a/fsadapter/directory_test.go +++ /dev/null @@ -1,235 +0,0 @@ -package fsadapter - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestDirectory_Create(t *testing.T) { - tmpdir := t.TempDir() - type fields struct { - dir string - } - type args struct { - fpath string - } - tests := []struct { - name string - fields fields - args args - testData []byte - wantErr bool - }{ - { - "ensure file is created and data is written (root dir)", - fields{dir: tmpdir}, - args{"testfile.txt"}, - []byte("123"), - false, - }, - { - "ensure file is created and data is written (subdir)", - fields{dir: tmpdir}, - args{filepath.Join("ooooh", "testfile.txt")}, - []byte("123"), - false, - }, - { - "directory (error)", - fields{dir: tmpdir}, - args{""}, - []byte("123"), - true, - }, - { - "invalid filename", - fields{dir: tmpdir}, - args{".."}, - []byte("123"), - true, - }, - { - "outside of root directory", - fields{dir: tmpdir}, - args{strings.Repeat(".."+string(filepath.Separator), 20) + filepath.Join("tmp", "hello_rootfs.txt")}, - []byte("123"), - true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := Directory{ - dir: tt.fields.dir, - } - f, err := fs.Create(tt.args.fpath) - if (err != nil) != tt.wantErr { - t.Errorf("Directory.Create() error = %v, wantErr %v", err, tt.wantErr) - return - } - if err != nil { - return - } - - assert, require := assert.New(t), require.New(t) - - n, err := f.Write(tt.testData) - require.NoError(err) - assert.Equal(3, n) - - assert.NoError(f.Close()) - - testFile := filepath.Join(tt.fields.dir, tt.args.fpath) - assert.FileExists(testFile) - - fileData, err := os.ReadFile(testFile) - assert.NoError(err) - - assert.Equal(tt.testData, fileData) - }) - } -} - -func TestDirectory_ensureSubdir(t *testing.T) { - type fields struct { - dir string - } - type args struct { - node string - } - tests := []struct { - name string - fields fields - args args - wantErr bool - }{ - { - "is a subdir", - fields{dir: filepath.Join("a", "b")}, - args{filepath.Join("a", "b", "c", "d", "e")}, - false, - }, - { - "is not a subdir", - fields{dir: filepath.Join("a", "b", "d")}, - args{filepath.Join("a", "b", "c")}, - true, - }, - { - "path hack", - fields{dir: filepath.Join("a", "b", "d")}, - args{filepath.Join("..", "..", "..", "tmp")}, - true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := Directory{ - dir: tt.fields.dir, - } - if err := fs.ensureSubdir(tt.args.node); (err != nil) != tt.wantErr { - t.Errorf("Directory.ensureSubdir() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func Test_mkdirAll(t *testing.T) { - tmpdir := t.TempDir() - existingFile := filepath.Join(tmpdir, "existing.txt") - if err := os.WriteFile(existingFile, []byte("123"), 0640); err != nil { - t.Fatal(err) - } - - type args struct { - name string - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - "ok", - args{filepath.Join(tmpdir, "abc")}, - false, - }, - { - "empty dir - error", - args{""}, - true, - }, - { - "already exists", - args{tmpdir}, - false, - }, - { - "is a file", - args{existingFile}, - true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := mkdirAll(tt.args.name); (err != nil) != tt.wantErr { - t.Errorf("mkdirAll() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestDirectory_WriteFile(t *testing.T) { - tmpdir := t.TempDir() - type fields struct { - dir string - } - type args struct { - name string - data []byte - perm os.FileMode - } - tests := []struct { - name string - fields fields - args args - wantErr bool - }{ - { - "all ok", - fields{dir: tmpdir}, - args{"blah.txt", []byte("blah"), 0640}, - false, - }, - { - "outside of base path is an error", - fields{dir: tmpdir}, - args{filepath.Join("..", "blah.txt"), []byte("blah"), 0640}, - true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := Directory{ - dir: tt.fields.dir, - } - err := fs.WriteFile(tt.args.name, tt.args.data, tt.args.perm) - if (err != nil) != tt.wantErr { - t.Errorf("Directory.WriteFile() error = %v, wantErr %v", err, tt.wantErr) - } - if err != nil { - return - } - // verify file contents - data, err := os.ReadFile(filepath.Join(fs.dir, tt.args.name)) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, tt.args.data, data) - }) - } -} diff --git a/fsadapter/fsadapter.go b/fsadapter/fsadapter.go deleted file mode 100644 index 6cea9d96..00000000 --- a/fsadapter/fsadapter.go +++ /dev/null @@ -1,39 +0,0 @@ -package fsadapter - -import ( - "io" - "os" - "path/filepath" - "strings" -) - -// FS is interface for operating on the files of the underlying filesystem. -// -//go:generate mockgen -destination ../internal/mocks/mock_fsadapter/mock_fs.go github.com/rusq/slackdump/v2/fsadapter FS -type FS interface { - Create(string) (io.WriteCloser, error) - WriteFile(name string, data []byte, perm os.FileMode) error -} - -// ForFilename returns appropriate filesystem based on the name of the file or -// directory given. -// Logic is simple: -// - if file has a known extension, the appropriate adapter will be returned. -// - else: it's a directory. -func ForFilename(name string) (FS, error) { - switch strings.ToUpper(filepath.Ext(name)) { - case ".ZIP": - return NewZipFile(name) - default: - return NewDirectory(name), nil - } -} - -// Close closes the filesystem, if it implements the io.Closer interface. -func Close(fs FS) error { - closer, ok := fs.(io.Closer) - if !ok { - return nil - } - return closer.Close() -} diff --git a/fsadapter/fsadapter_test.go b/fsadapter/fsadapter_test.go deleted file mode 100644 index bb58bc84..00000000 --- a/fsadapter/fsadapter_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package fsadapter - -import ( - "fmt" - "io" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" -) - -type fakeCloser struct { - fakeFS -} - -type fakeFS struct { - isClosed bool -} - -func (c *fakeCloser) Close() error { - c.fakeFS.isClosed = true - return nil -} - -func (c *fakeFS) Create(_ string) (io.WriteCloser, error) { - return nil, nil -} - -func (c *fakeFS) WriteFile(name string, data []byte, perm os.FileMode) error { - return nil -} - -func TestClose(t *testing.T) { - clfs := new(fakeCloser) - if err := Close(clfs); err != nil { - t.Error("that is completely unexpected") - } - assert.True(t, clfs.isClosed) - - ffs := new(fakeFS) - if err := Close(ffs); err != nil { - t.Errorf("that is even more unexpected") - } - assert.False(t, ffs.isClosed) -} - -func TestForFilename(t *testing.T) { - tmp := t.TempDir() - type args struct { - name string - } - tests := []struct { - name string - args args - wantString string - wantErr bool - }{ - { - "directory", - args{filepath.Join(tmp, "blah")}, - "", - false, - }, - { - "zip file", - args{filepath.Join(tmp, "bloop.zip")}, - "", - false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := ForFilename(tt.args.name) - if (err != nil) != tt.wantErr { - t.Errorf("ForFilename() error = %v, wantErr %v", err, tt.wantErr) - return - } - defer Close(got) - - assert.Equal(t, tt.wantString, fmt.Sprint(got)) - }) - } -} diff --git a/fsadapter/zipfs.go b/fsadapter/zipfs.go deleted file mode 100644 index 7cbd93eb..00000000 --- a/fsadapter/zipfs.go +++ /dev/null @@ -1,168 +0,0 @@ -package fsadapter - -import ( - "archive/zip" - "bytes" - "fmt" - "io" - "os" - "path" - "path/filepath" - "strings" - "sync" - "time" -) - -var _ FS = &ZIP{} - -type ZIP struct { - zw *zip.Writer - mu sync.Mutex - f *os.File - seen map[string]bool // seen holds the list of seen directories. -} - -func (z *ZIP) String() string { - return fmt.Sprintf("", z.f.Name()) -} - -func NewZIP(zw *zip.Writer) *ZIP { - return &ZIP{zw: zw, seen: make(map[string]bool)} -} - -func NewZipFile(filename string) (*ZIP, error) { - f, err := os.Create(filename) - if err != nil { - return nil, err - } - zw := zip.NewWriter(f) - return &ZIP{zw: zw, f: f, seen: make(map[string]bool)}, nil -} - -func (*ZIP) normalizePath(p string) string { - split := strings.Split(filepath.Clean(p), string(os.PathSeparator)) - return path.Join(split...) -} - -func (z *ZIP) Create(filename string) (io.WriteCloser, error) { - // reassemble path in correct format for ZIP file - // in case it uses OS specific path. - filename = z.normalizePath(filename) - - z.mu.Lock() // mutex will be unlocked, when the user calls Close. - w, err := z.create(filename) - if err != nil { - return nil, err - } - return &syncWriter{w: w, mu: &z.mu}, nil -} - -func (z *ZIP) create(filename string) (io.Writer, error) { - if err := z.ensureDir(filename); err != nil { - return nil, err - } - header := &zip.FileHeader{ - Name: filename, - Method: zip.Deflate, - Modified: time.Now(), - } - return z.zw.CreateHeader(header) -} - -func (z *ZIP) ensureDir(filename string) error { - if z.seen == nil { - z.seen = make(map[string]bool, 0) - } - var ensureFn = func(dir string) error { - if _, seen := z.seen[dir]; seen { - return nil - } - // not seen, create an empty directory. - if _, err := z.zw.Create(dir); err != nil { - return err - } - z.seen[dir] = true - return nil - } - dir, _ := path.Split(filename) - for _, d := range z.dirpath(dir) { - if err := ensureFn(d); err != nil { - return err - } - } - return nil -} - -func (*ZIP) dirpath(dir string) []string { - const sep = "/" - if len(dir) == 0 { - return nil - } - var ret []string - d := strings.TrimRight(dir, sep) - for len(d) > 0 { - ret = append([]string{strings.TrimRight(d, sep) + sep}, ret...) - d, _ = path.Split(strings.TrimRight(d, sep)) - } - return ret -} - -func (z *ZIP) WriteFile(filename string, data []byte, _ os.FileMode) error { - z.mu.Lock() - defer z.mu.Unlock() - zf, err := z.create(filename) - if err != nil { - return err - } - - _, err = io.Copy(zf, bytes.NewReader(data)) - return err - -} - -// Close closes the underlying zip writer and the file handle. It is only necessary if -// ZIP was initialised using NewZipFile -func (z *ZIP) Close() error { - if !z.ourHandles() { - // we don't own the handles, so just bail out. - return nil - } - z.mu.Lock() - defer z.mu.Unlock() - - return z.closeHandles() -} - -func (z *ZIP) closeHandles() error { - if err := z.zw.Close(); err != nil { - return err - } - if z.f == nil { - return nil - } - return z.f.Close() -} - -func (z *ZIP) ourHandles() bool { - return z.f != nil -} - -type syncWriter struct { - w io.Writer // underlying writer - - // zip writer can only process one file at a time, so any process that wants - // to Create the file will have to wait until Close is called: - // - // From zip.Create doc: The file's contents must be written to the - // io.Writer before the next call to Create, CreateHeader, or Close. - mu *sync.Mutex -} - -func (sw *syncWriter) Write(p []byte) (int, error) { - return sw.w.Write(p) -} - -func (sw *syncWriter) Close() error { - sw.mu.Unlock() - return nil -} diff --git a/fsadapter/zipfs_test.go b/fsadapter/zipfs_test.go deleted file mode 100644 index ab59dbbd..00000000 --- a/fsadapter/zipfs_test.go +++ /dev/null @@ -1,266 +0,0 @@ -package fsadapter - -import ( - "archive/zip" - "bytes" - "crypto/rand" - "fmt" - "io" - "os" - "path/filepath" - "reflect" - "strings" - "sync" - "testing" - - "github.com/rusq/slackdump/v2/internal/fixtures" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNewZipFile(t *testing.T) { - tmp := t.TempDir() - t.Run("zip file is created", func(t *testing.T) { - zipPath, zf := gimmeZIP(t, tmp) - - assert.NotNil(t, zf.zw) - assert.NotNil(t, zf.f) - - require.NoError(t, zf.Close()) - require.FileExists(t, zipPath) - }) - t.Run("error creating a file", func(t *testing.T) { - zipPath := tmp // should fail on directory - _, err := NewZipFile(zipPath) - require.Error(t, err) - }) -} - -func TestZIP_Create(t *testing.T) { - tmp := t.TempDir() - t.Run("file is created", func(t *testing.T) { - testsuiteZipFile(t, filepath.Join(tmp, "test.zip"), "abc/def.txt", "abcdef") - }) - t.Run("backslashes", func(t *testing.T) { - testsuiteZipFile(t, filepath.Join(tmp, "test.zip"), "abc\\backslash.txt", "abcdef") - }) -} - -func testsuiteZipFile(t *testing.T, zipFile, filename, content string) { - hArc, err := os.Create(zipFile) - require.NoError(t, err) - defer hArc.Close() - zw := zip.NewWriter(hArc) - z := ZIP{zw: zw} - - // create test file - hTest, err := z.Create(filename) - require.NoError(t, err) - _, err = io.Copy(hTest, strings.NewReader(content)) - assert.NoError(t, err) - assert.NoError(t, z.Close()) - // test file closed - - assert.NoError(t, zw.Close()) - assert.NoError(t, hArc.Close()) - - assertZippedFileSize(t, zipFile, filename, uint64(len(content))) -} - -func assertZippedFileSize(t *testing.T, zipfile string, fullpath string, size uint64) { - zr, err := zip.OpenReader(zipfile) - if err != nil { - t.Fatal(err) - } - defer zr.Close() - - zzz := ZIP{} - - found := false - for _, f := range zr.File { - t.Log(f.Name) - if f.Name == zzz.normalizePath(fullpath) { - if f.UncompressedSize64 != size { - t.Errorf("file: %s, size mismatch: want: %d, got %d", fullpath, size, f.UncompressedSize64) - return - } - t.Log("size OK") - return - } - } - if !found { - t.Errorf("file not found: %s", fullpath) - } -} - -func TestZIP_WriteFile(t *testing.T) { - tmp := t.TempDir() - - t.Run("write file creates the file in the zip archive", func(t *testing.T) { - zipfile, hZF := gimmeZIP(t, tmp) - if err := hZF.WriteFile("test1.txt", []byte("0123456789abcdef"), 0750); err != nil { - hZF.Close() - t.Fatalf("ZIP.WriteFile err=%s", err) - } - assert.NoError(t, hZF.Close()) - - assertZippedFileSize(t, zipfile, "test1.txt", 16) - }) -} - -// gimmeZIP creates a zip file, returns it's name and initialised *ZIP instance. -// Don't forget to close it before running assertions. -func gimmeZIP(t *testing.T, tmpdir string) (filename string, hZF *ZIP) { - zipfile := filepath.Join(tmpdir, fixtures.RandString(8)+".zip") - hZF, err := NewZipFile(zipfile) - if err != nil { - t.Fatal(err) - } - return zipfile, hZF -} - -func Test_syncWriter_Close(t *testing.T) { - t.Run("should unlock the mutex", func(t *testing.T) { - sw := syncWriter{mu: &sync.Mutex{}} - - sw.mu.Lock() - - sw.Close() - assert.True(t, sw.mu.TryLock()) - }) -} - -func TestNewZIP(t *testing.T) { - tmpdir := t.TempDir() - t.Run("ensure it's the same zw", func(t *testing.T) { - hFile, err := os.Create(filepath.Join(tmpdir, "x.zip")) - assert.NoError(t, err) - defer hFile.Close() - - zw := zip.NewWriter(hFile) - zf := NewZIP(zw) - - assert.Equal(t, zw, zf.zw) - }) -} - -func TestCreateConcurrency(t *testing.T) { - // test for GH issue#90 - race condition in ZIP.Create - const ( - numRoutines = 16 - testContentsSz = 1 * (1 << 20) - ) - - var buf bytes.Buffer - var wg sync.WaitGroup - - zw := zip.NewWriter(&buf) - defer zw.Close() - - fsa := NewZIP(zw) - defer fsa.Close() - - // prepare workers - readySteadyGo := make(chan struct{}) - panicAttacks := make(chan any, numRoutines) - - for i := 0; i < numRoutines; i++ { - wg.Add(1) - go func(n int) { - defer func() { - if r := recover(); r != nil { - panicAttacks <- fmt.Sprintf("ZIP.Create race condition in gr %d: %v", n, r) - } - }() - - defer wg.Done() - var contents bytes.Buffer - if _, err := io.CopyN(&contents, rand.Reader, testContentsSz); err != nil { - panic(err) - } - - <-readySteadyGo - fw, err := fsa.Create(fmt.Sprintf("file%d", n)) - if err != nil { - panic(err) - } - defer fw.Close() - - if _, err := io.Copy(fw, &contents); err != nil { - panic(err) - } - }(i) - } - close(readySteadyGo) - wg.Wait() - close(panicAttacks) - for r := range panicAttacks { - if r != nil { - t.Error(r) - } - } -} - -func TestZIP_normalizePath(t *testing.T) { - type args struct { - p string - } - tests := []struct { - name string - z *ZIP - args args - want string - }{ - { - "windows", - &ZIP{}, - args{filepath.Join("sample", "directory", "and", "file.txt")}, - "sample/directory/and/file.txt", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.z.normalizePath(tt.args.p); got != tt.want { - t.Errorf("ZIP.normalizePath() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestZIP_dirpath(t *testing.T) { - type args struct { - dir string - } - tests := []struct { - name string - z *ZIP - args args - want []string - }{ - { - "single", - &ZIP{}, - args{"foo/"}, - []string{"foo/"}, - }, - { - "single", - &ZIP{}, - args{"foo"}, - []string{"foo/"}, - }, - { - "two", - &ZIP{}, - args{"foo/bar"}, - []string{"foo/", "foo/bar/"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.z.dirpath(tt.args.dir); !reflect.DeepEqual(got, tt.want) { - t.Errorf("ZIP.dirpath() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/go.mod b/go.mod index 3eb27eee..062ab322 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/playwright-community/playwright-go v0.2000.1 github.com/rusq/chttp v1.0.1 github.com/rusq/dlog v1.4.0 + github.com/rusq/fsadapter v1.0.0 github.com/rusq/osenv/v2 v2.0.1 github.com/rusq/secure v0.0.4 github.com/rusq/tracer v1.0.1 @@ -31,7 +32,6 @@ require ( github.com/mattn/go-isatty v0.0.18 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect - github.com/microsoft/playwright v1.32.1 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect diff --git a/go.sum b/go.sum index 4a47cb66..f96499ad 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= -github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= -github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= @@ -41,7 +39,6 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -50,8 +47,6 @@ github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/microsoft/playwright v1.32.1 h1:ihlRNF/3aEOIyfZ7Pq2S1NuTf9nfeL8OiHAl/SUp9Zg= -github.com/microsoft/playwright v1.32.1/go.mod h1:QnVBXukiHsBkJ8Qg+f61SQd3gSgiWgwQj1QAX0QxBC4= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/playwright-community/playwright-go v0.2000.1 h1:2JViSHpJQ/UL/PO1Gg6gXV5IcXAAsoBJ3KG9L3wKXto= @@ -64,10 +59,10 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rusq/chttp v1.0.1 h1:j3WE7+jQE9Rgw0E6mGMje8HxMCv09QRkKvR0oZ1R2vY= github.com/rusq/chttp v1.0.1/go.mod h1:9H/mMp/iUc4xDkSOY0rL6ecxd/YyaW7zE9GhR+mZfRg= -github.com/rusq/dlog v1.3.3 h1:Q9fZW1H/YEnlDg3Ph1k/BRSBfi/q5ezI+8Metws9tTI= -github.com/rusq/dlog v1.3.3/go.mod h1:kjZAEvBu7m3+mnJQKoIeLul1YB3kJq/6lZBdDTZmpzA= github.com/rusq/dlog v1.4.0 h1:64oHTSzHjzG6TXKvMbPKQzvqADCZRn6XgAWnp7ASr5k= github.com/rusq/dlog v1.4.0/go.mod h1:kjZAEvBu7m3+mnJQKoIeLul1YB3kJq/6lZBdDTZmpzA= +github.com/rusq/fsadapter v1.0.0 h1:6IZFkpX/MgAKFMyKPDWAnBz2RxYg8MH4EqV9Xy3m4lQ= +github.com/rusq/fsadapter v1.0.0/go.mod h1:56enDqgnY1Mu+brFsVoA3WW4VbKdtjflR7qz/7Rab1E= github.com/rusq/osenv/v2 v2.0.1 h1:1LtNt8VNV/W86wb38Hyu5W3Rwqt/F1JNRGE+8GRu09o= github.com/rusq/osenv/v2 v2.0.1/go.mod h1:+wJBSisjNZpfoD961JzqjaM+PtaqSusO3b4oVJi7TFY= github.com/rusq/secure v0.0.4 h1:svpiZHfHnx89eEDCCFI9OXG1Y8hL9kUWUG6fJbrWUOI= diff --git a/internal/app/dump.go b/internal/app/dump.go index 5fbf7d67..fc8709d0 100644 --- a/internal/app/dump.go +++ b/internal/app/dump.go @@ -12,9 +12,9 @@ import ( "strings" "time" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2" "github.com/rusq/slackdump/v2/auth" - "github.com/rusq/slackdump/v2/fsadapter" "github.com/rusq/slackdump/v2/internal/app/config" "github.com/rusq/slackdump/v2/internal/structures" "github.com/rusq/slackdump/v2/logger" @@ -75,11 +75,11 @@ func (app *dump) Dump(ctx context.Context) (int, error) { return 0, errors.New("no valid input") } - fs, err := fsadapter.ForFilename(app.cfg.Output.Base) + fs, err := fsadapter.New(app.cfg.Output.Base) if err != nil { return 0, err } - defer fsadapter.Close(fs) + defer fs.Close() app.sess.SetFS(fs) tmpl, err := app.cfg.CompileTemplates() diff --git a/internal/app/emoji/emoji.go b/internal/app/emoji/emoji.go index f87541ca..04ec6668 100644 --- a/internal/app/emoji/emoji.go +++ b/internal/app/emoji/emoji.go @@ -27,9 +27,9 @@ import ( "sync" "github.com/rusq/dlog" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2" "github.com/rusq/slackdump/v2/auth" - "github.com/rusq/slackdump/v2/fsadapter" "github.com/rusq/slackdump/v2/internal/app/config" ) @@ -56,11 +56,11 @@ type emojidumper interface { } func download(ctx context.Context, sess emojidumper, base string, ignoreErrors bool) error { - fsa, err := fsadapter.ForFilename(base) + fsa, err := fsadapter.New(base) if err != nil { return fmt.Errorf("unable to initialise adapter for %s: %w", base, err) } - defer fsadapter.Close(fsa) + defer fsa.Close() emojis, err := sess.DumpEmojis(ctx) if err != nil { diff --git a/internal/app/emoji/emoji_test.go b/internal/app/emoji/emoji_test.go index 0bb42563..37daf28e 100644 --- a/internal/app/emoji/emoji_test.go +++ b/internal/app/emoji/emoji_test.go @@ -16,7 +16,7 @@ import ( "time" "github.com/golang/mock/gomock" - "github.com/rusq/slackdump/v2/fsadapter" + "github.com/rusq/fsadapter" ) type fetchFunc func(ctx context.Context, fsa fsadapter.FS, dir string, name string, uri string) error @@ -86,7 +86,7 @@ func Test_fetchEmoji(t *testing.T) { defer server.Close() dir := t.TempDir() - fsa, err := fsadapter.ForFilename(dir) + fsa, err := fsadapter.New(dir) if err != nil { t.Fatalf("failed to create test dir: %s", err) } @@ -182,7 +182,7 @@ func Test_worker(t *testing.T) { t.Run(tt.name, func(t *testing.T) { setGlobalFetchFn(tt.fetchFn) - fsa, _ := fsadapter.ForFilename(t.TempDir()) + fsa, _ := fsadapter.New(t.TempDir()) resultC := make(chan result) var wg sync.WaitGroup @@ -208,7 +208,7 @@ func Test_worker(t *testing.T) { func Test_fetch(t *testing.T) { emojis := generateEmojis(50) - fsa, _ := fsadapter.ForFilename(t.TempDir()) + fsa, _ := fsadapter.New(t.TempDir()) got := make(map[string]string, len(emojis)) var mu sync.Mutex diff --git a/internal/app/export.go b/internal/app/export.go index e3963c97..19d17e2e 100644 --- a/internal/app/export.go +++ b/internal/app/export.go @@ -7,10 +7,10 @@ import ( "runtime/trace" "time" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2" "github.com/rusq/slackdump/v2/auth" "github.com/rusq/slackdump/v2/export" - "github.com/rusq/slackdump/v2/fsadapter" "github.com/rusq/slackdump/v2/internal/app/config" ) @@ -33,14 +33,14 @@ func Export(ctx context.Context, cfg config.Params, prov auth.Provider) error { return err } - fs, err := fsadapter.ForFilename(cfg.ExportName) + fs, err := fsadapter.New(cfg.ExportName) if err != nil { cfg.Logger().Debugf("Export: filesystem error: %s", err) return fmt.Errorf("failed to initialise the filesystem: %w", err) } defer func() { cfg.Logger().Debugf("Export: closing file system") - if err := fsadapter.Close(fs); err != nil { + if err := fs.Close(); err != nil { cfg.Logger().Printf("Export: error closing filesystem") } }() diff --git a/internal/mocks/mock_fsadapter/mock_fs.go b/internal/mocks/mock_fsadapter/mock_fs.go index 1c5889c9..59aadb34 100644 --- a/internal/mocks/mock_fsadapter/mock_fs.go +++ b/internal/mocks/mock_fsadapter/mock_fs.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/rusq/slackdump/v2/fsadapter (interfaces: FS) +// Source: github.com/rusq/fsadapter (interfaces: FS) // Package mock_fsadapter is a generated GoMock package. package mock_fsadapter diff --git a/internal/structures/files/dl/mattermost.go b/internal/structures/files/dl/mattermost.go index 0caa71df..066b8926 100644 --- a/internal/structures/files/dl/mattermost.go +++ b/internal/structures/files/dl/mattermost.go @@ -8,9 +8,9 @@ import ( "github.com/slack-go/slack" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2" "github.com/rusq/slackdump/v2/downloader" - "github.com/rusq/slackdump/v2/fsadapter" "github.com/rusq/slackdump/v2/internal/structures/files" "github.com/rusq/slackdump/v2/logger" "github.com/rusq/slackdump/v2/types" diff --git a/internal/structures/files/dl/standard.go b/internal/structures/files/dl/standard.go index cb8a6f2b..b784b6da 100644 --- a/internal/structures/files/dl/standard.go +++ b/internal/structures/files/dl/standard.go @@ -9,9 +9,9 @@ import ( "github.com/slack-go/slack" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2" "github.com/rusq/slackdump/v2/downloader" - "github.com/rusq/slackdump/v2/fsadapter" "github.com/rusq/slackdump/v2/internal/structures/files" "github.com/rusq/slackdump/v2/logger" "github.com/rusq/slackdump/v2/types" diff --git a/slackdump.go b/slackdump.go index dadccf4b..b9f54b7d 100644 --- a/slackdump.go +++ b/slackdump.go @@ -15,8 +15,8 @@ import ( "golang.org/x/time/rate" "github.com/rusq/chttp" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2/auth" - "github.com/rusq/slackdump/v2/fsadapter" "github.com/rusq/slackdump/v2/internal/network" "github.com/rusq/slackdump/v2/internal/structures" "github.com/rusq/slackdump/v2/logger" diff --git a/slackdump_test.go b/slackdump_test.go index 6a875d54..6e38c7e8 100644 --- a/slackdump_test.go +++ b/slackdump_test.go @@ -11,8 +11,8 @@ import ( "github.com/golang/mock/gomock" "github.com/rusq/dlog" + "github.com/rusq/fsadapter" "github.com/rusq/slackdump/v2/auth" - "github.com/rusq/slackdump/v2/fsadapter" "github.com/rusq/slackdump/v2/internal/fixtures" "github.com/rusq/slackdump/v2/internal/mocks/mock_os" "github.com/rusq/slackdump/v2/internal/network"