179 changes: 179 additions & 0 deletions internal/ufs/file.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
// SPDX-License-Identifier: MIT
// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner

package ufs

import (
"io"
iofs "io/fs"

"golang.org/x/sys/unix"
)

// DirEntry is an entry read from a directory.
type DirEntry = iofs.DirEntry

// File describes readable and/or writable file from a Filesystem.
type File interface {
// Name returns the base name of the file.
Name() string

// Stat returns the FileInfo structure describing the file.
// If there is an error, it will be of type *PathError.
Stat() (FileInfo, error)

// ReadDir reads the contents of the directory associated with the file f
// and returns a slice of DirEntry values in directory order.
// Subsequent calls on the same file will yield later DirEntry records in the directory.
//
// If n > 0, ReadDir returns at most n DirEntry records.
// In this case, if ReadDir returns an empty slice, it will return an error explaining why.
// At the end of a directory, the error is io.EOF.
//
// If n <= 0, ReadDir returns all the DirEntry records remaining in the directory.
// When it succeeds, it returns a nil error (not io.EOF).
ReadDir(n int) ([]DirEntry, error)

// Readdirnames reads the contents of the directory associated with file
// and returns a slice of up to n names of files in the directory,
// in directory order. Subsequent calls on the same file will yield
// further names.
//
// If n > 0, Readdirnames returns at most n names. In this case, if
// Readdirnames returns an empty slice, it will return a non-nil error
// explaining why. At the end of a directory, the error is io.EOF.
//
// If n <= 0, Readdirnames returns all the names from the directory in
// a single slice. In this case, if Readdirnames succeeds (reads all
// the way to the end of the directory), it returns the slice and a
// nil error. If it encounters an error before the end of the
// directory, Readdirnames returns the names read until that point and
// a non-nil error.
Readdirnames(n int) (names []string, err error)

// Fd returns the integer Unix file descriptor referencing the open file.
// If f is closed, the file descriptor becomes invalid.
// If f is garbage collected, a finalizer may close the file descriptor,
// making it invalid; see runtime.SetFinalizer for more information on when
// a finalizer might be run. On Unix systems this will cause the SetDeadline
// methods to stop working.
// Because file descriptors can be reused, the returned file descriptor may
// only be closed through the Close method of f, or by its finalizer during
// garbage collection. Otherwise, during garbage collection the finalizer
// may close an unrelated file descriptor with the same (reused) number.
//
// As an alternative, see the f.SyscallConn method.
Fd() uintptr

// Truncate changes the size of the file.
// It does not change the I/O offset.
// If there is an error, it will be of type *PathError.
Truncate(size int64) error

io.Closer

io.Reader
io.ReaderAt
io.ReaderFrom

io.Writer
io.WriterAt

io.Seeker
}

// FileInfo describes a file and is returned by Stat and Lstat.
type FileInfo = iofs.FileInfo

// FileMode represents a file's mode and permission bits.
// The bits have the same definition on all systems, so that
// information about files can be moved from one system
// to another portably. Not all bits apply to all systems.
// The only required bit is ModeDir for directories.
type FileMode = iofs.FileMode

// The defined file mode bits are the most significant bits of the FileMode.
// The nine least-significant bits are the standard Unix rwxrwxrwx permissions.
// The values of these bits should be considered part of the public API and
// may be used in wire protocols or disk representations: they must not be
// changed, although new bits might be added.
const (
// ModeDir represents a directory.
// d: is a directory
ModeDir = iofs.ModeDir
// ModeAppend represents an append-only file.
// a: append-only
ModeAppend = iofs.ModeAppend
// ModeExclusive represents an exclusive file.
// l: exclusive use
ModeExclusive = iofs.ModeExclusive
// ModeTemporary .
// T: temporary file; Plan 9 only.
ModeTemporary = iofs.ModeTemporary
// ModeSymlink .
// L: symbolic link.
ModeSymlink = iofs.ModeSymlink
// ModeDevice .
// D: device file.
ModeDevice = iofs.ModeDevice
// ModeNamedPipe .
// p: named pipe (FIFO)
ModeNamedPipe = iofs.ModeNamedPipe
// ModeSocket .
// S: Unix domain socket.
ModeSocket = iofs.ModeSocket
// ModeSetuid .
// u: setuid
ModeSetuid = iofs.ModeSetuid
// ModeSetgid .
// g: setgid
ModeSetgid = iofs.ModeSetgid
// ModeCharDevice .
// c: Unix character device, when ModeDevice is set
ModeCharDevice = iofs.ModeCharDevice
// ModeSticky .
// t: sticky
ModeSticky = iofs.ModeSticky
// ModeIrregular .
// ?: non-regular file; nothing else is known about this file.
ModeIrregular = iofs.ModeIrregular

// ModeType .
ModeType = iofs.ModeType

// ModePerm .
// Unix permission bits, 0o777.
ModePerm = iofs.ModePerm
)

const (
// O_RDONLY opens the file read-only.
O_RDONLY = unix.O_RDONLY
// O_WRONLY opens the file write-only.
O_WRONLY = unix.O_WRONLY
// O_RDWR opens the file read-write.
O_RDWR = unix.O_RDWR
// O_APPEND appends data to the file when writing.
O_APPEND = unix.O_APPEND
// O_CREATE creates a new file if it doesn't exist.
O_CREATE = unix.O_CREAT
// O_EXCL is used with O_CREATE, file must not exist.
O_EXCL = unix.O_EXCL
// O_SYNC open for synchronous I/O.
O_SYNC = unix.O_SYNC
// O_TRUNC truncates regular writable file when opened.
O_TRUNC = unix.O_TRUNC
// O_DIRECTORY opens a directory only. If the entry is not a directory an
// error will be returned.
O_DIRECTORY = unix.O_DIRECTORY
// O_NOFOLLOW opens the exact path given without following symlinks.
O_NOFOLLOW = unix.O_NOFOLLOW
O_CLOEXEC = unix.O_CLOEXEC
O_LARGEFILE = unix.O_LARGEFILE
)

const (
AT_SYMLINK_NOFOLLOW = unix.AT_SYMLINK_NOFOLLOW
AT_REMOVEDIR = unix.AT_REMOVEDIR
AT_EMPTY_PATH = unix.AT_EMPTY_PATH
)
49 changes: 49 additions & 0 deletions internal/ufs/file_posix.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// SPDX-License-Identifier: BSD-3-Clause

// Code in this file was copied from `go/src/os/file_posix.go`.

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the `go.LICENSE` file.

//go:build unix || (js && wasm) || wasip1 || windows

package ufs

import (
"golang.org/x/sys/unix"
)

// ignoringEINTR makes a function call and repeats it if it returns an
// EINTR error. This appears to be required even though we install all
// signal handlers with SA_RESTART: see https://go.dev/issue/22838,
// https://go.dev/issue/38033, https://go.dev/issue/38836,
// https://go.dev/issue/40846. Also, https://go.dev/issue/20400 and
// https://go.dev/issue/36644 are issues in which a signal handler is
// installed without setting SA_RESTART. None of these are the common case,
// but there are enough of them that it seems that we can't avoid
// an EINTR loop.
func ignoringEINTR(fn func() error) error {
for {
err := fn()
if err != unix.EINTR {
return err
}
}
}

// syscallMode returns the syscall-specific mode bits from Go's portable mode bits.
func syscallMode(i FileMode) (o FileMode) {
o |= i.Perm()
if i&ModeSetuid != 0 {
o |= unix.S_ISUID
}
if i&ModeSetgid != 0 {
o |= unix.S_ISGID
}
if i&ModeSticky != 0 {
o |= unix.S_ISVTX
}
// No mapping for Go's ModeTemporary (plan9 only).
return
}
168 changes: 168 additions & 0 deletions internal/ufs/filesystem.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
// SPDX-License-Identifier: MIT
// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner

package ufs

import (
"time"
)

// Filesystem represents a filesystem capable of performing I/O operations.
type Filesystem interface {
// Chmod changes the mode of the named file to mode.
//
// If the file is a symbolic link, it changes the mode of the link's target.
// If there is an error, it will be of type *PathError.
//
// A different subset of the mode bits are used, depending on the
// operating system.
//
// On Unix, the mode's permission bits, ModeSetuid, ModeSetgid, and
// ModeSticky are used.
//
// On Windows, only the 0200 bit (owner writable) of mode is used; it
// controls whether the file's read-only attribute is set or cleared.
// The other bits are currently unused. For compatibility with Go 1.12
// and earlier, use a non-zero mode. Use mode 0400 for a read-only
// file and 0600 for a readable+writable file.
//
// On Plan 9, the mode's permission bits, ModeAppend, ModeExclusive,
// and ModeTemporary are used.
Chmod(name string, mode FileMode) error

// Chown changes the numeric uid and gid of the named file.
//
// If the file is a symbolic link, it changes the uid and gid of the link's target.
// A uid or gid of -1 means to not change that value.
// If there is an error, it will be of type *PathError.
//
// On Windows or Plan 9, Chown always returns the syscall.EWINDOWS or
// EPLAN9 error, wrapped in *PathError.
Chown(name string, uid, gid int) error

// Lchown changes the numeric uid and gid of the named file.
//
// If the file is a symbolic link, it changes the uid and gid of the link itself.
// If there is an error, it will be of type *PathError.
//
// On Windows, it always returns the syscall.EWINDOWS error, wrapped
// in *PathError.
Lchown(name string, uid, gid int) error

// Chtimes changes the access and modification times of the named
// file, similar to the Unix utime() or utimes() functions.
//
// The underlying filesystem may truncate or round the values to a
// less precise time unit.
//
// If there is an error, it will be of type *PathError.
Chtimes(name string, atime, mtime time.Time) error

// Create creates or truncates the named file. If the file already exists,
// it is truncated.
//
// If the file does not exist, it is created with mode 0666
// (before umask). If successful, methods on the returned File can
// be used for I/O; the associated file descriptor has mode O_RDWR.
// If there is an error, it will be of type *PathError.
Create(name string) (File, error)

// Mkdir creates a new directory with the specified name and permission
// bits (before umask).
//
// If there is an error, it will be of type *PathError.
Mkdir(name string, perm FileMode) error

// MkdirAll creates a directory named path, along with any necessary
// parents, and returns nil, or else returns an error.
//
// The permission bits perm (before umask) are used for all
// directories that MkdirAll creates.
// If path is already a directory, MkdirAll does nothing
// and returns nil.
MkdirAll(path string, perm FileMode) error

// Open opens the named file for reading.
//
// If successful, methods on the returned file can be used for reading; the
// associated file descriptor has mode O_RDONLY.
//
// If there is an error, it will be of type *PathError.
Open(name string) (File, error)

// OpenFile is the generalized open call; most users will use Open
// or Create instead. It opens the named file with specified flag
// (O_RDONLY etc.).
//
// If the file does not exist, and the O_CREATE flag
// is passed, it is created with mode perm (before umask). If successful,
// methods on the returned File can be used for I/O.
//
// If there is an error, it will be of type *PathError.
OpenFile(name string, flag int, perm FileMode) (File, error)

// ReadDir reads the named directory,
//
// returning all its directory entries sorted by filename.
// If an error occurs reading the directory, ReadDir returns the entries it
// was able to read before the error, along with the error.
ReadDir(name string) ([]DirEntry, error)

// Remove removes the named file or (empty) directory.
//
// If there is an error, it will be of type *PathError.
Remove(name string) error

// RemoveAll removes path and any children it contains.
//
// It removes everything it can but returns the first error
// it encounters. If the path does not exist, RemoveAll
// returns nil (no error).
//
// If there is an error, it will be of type *PathError.
RemoveAll(path string) error

// Rename renames (moves) oldpath to newpath.
//
// If newpath already exists and is not a directory, Rename replaces it.
// OS-specific restrictions may apply when oldpath and newpath are in different directories.
// Even within the same directory, on non-Unix platforms Rename is not an atomic operation.
//
// If there is an error, it will be of type *LinkError.
Rename(oldname, newname string) error

// Stat returns a FileInfo describing the named file.
//
// If there is an error, it will be of type *PathError.
Stat(name string) (FileInfo, error)

// Lstat returns a FileInfo describing the named file.
//
// If the file is a symbolic link, the returned FileInfo
// describes the symbolic link. Lstat makes no attempt to follow the link.
//
// If there is an error, it will be of type *PathError.
Lstat(name string) (FileInfo, error)

// Symlink creates newname as a symbolic link to oldname.
//
// On Windows, a symlink to a non-existent oldname creates a file symlink;
// if oldname is later created as a directory the symlink will not work.
//
// If there is an error, it will be of type *LinkError.
Symlink(oldname, newname string) error

// WalkDir walks the file tree rooted at root, calling fn for each file or
// directory in the tree, including root.
//
// All errors that arise visiting files and directories are filtered by fn:
// see the [WalkDirFunc] documentation for details.
//
// The files are walked in lexical order, which makes the output deterministic
// but requires WalkDir to read an entire directory into memory before proceeding
// to walk that directory.
//
// WalkDir does not follow symbolic links found in directories,
// but if root itself is a symbolic link, its target will be walked.
WalkDir(root string, fn WalkDirFunc) error
}
159 changes: 159 additions & 0 deletions internal/ufs/fs_quota.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
// SPDX-License-Identifier: MIT
// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner

package ufs

import (
"sync/atomic"
)

type Quota struct {
// fs is the underlying filesystem that runs the actual I/O operations.
*UnixFS

// limit is the size limit of the filesystem.
//
// limit is atomic to allow the limit to be safely changed after the
// filesystem was created.
//
// A limit of `-1` disables any write operation from being performed.
// A limit of `0` disables any limit checking.
limit atomic.Int64

// usage is the current usage of the filesystem.
//
// If usage is set to `-1`, it hasn't been calculated yet.
usage atomic.Int64
}

func NewQuota(fs *UnixFS, limit int64) *Quota {
qfs := Quota{UnixFS: fs}
qfs.limit.Store(limit)
return &qfs
}

// Close closes the filesystem.
func (fs *Quota) Close() (err error) {
err = fs.UnixFS.Close()
return
}

// Limit returns the limit of the filesystem.
func (fs *Quota) Limit() int64 {
return fs.limit.Load()
}

// SetLimit returns the limit of the filesystem.
func (fs *Quota) SetLimit(newLimit int64) int64 {
return fs.limit.Swap(newLimit)
}

// Usage returns the current usage of the filesystem.
func (fs *Quota) Usage() int64 {
return fs.usage.Load()
}

// SetUsage updates the total usage of the filesystem.
func (fs *Quota) SetUsage(newUsage int64) int64 {
return fs.usage.Swap(newUsage)
}

// Add adds `i` to the tracked usage total.
func (fs *Quota) Add(i int64) int64 {
usage := fs.Usage()

// If adding `i` to the usage will put us below 0, cap it. (`i` can be negative)
if usage+i < 0 {
fs.usage.Store(0)
return 0
}
return fs.usage.Add(i)
}

// CanFit checks if the given size can fit in the filesystem without exceeding
// the limit of the filesystem.
func (fs *Quota) CanFit(size int64) bool {
// Get the size limit of the filesystem.
limit := fs.Limit()
switch limit {
case -1:
// A limit of -1 means no write operations are allowed.
return false
case 0:
// A limit of 0 means unlimited.
return true
}

// Any other limit is a value we need to check.
usage := fs.Usage()
if usage == -1 {
// We don't know what the current usage is yet.
return true
}

// If the current usage + the requested size are under the limit of the
// filesystem, allow it.
if usage+size <= limit {
return true
}

// Welp, the size would exceed the limit of the filesystem, deny it.
return false
}

func (fs *Quota) Remove(name string) error {
// For information on why this interface is used here, check its
// documentation.
s, err := fs.RemoveStat(name)
if err != nil {
return err
}

// Don't reduce the quota's usage as `name` is not a regular file.
if !s.Mode().IsRegular() {
return nil
}

// Remove the size of the deleted file from the quota usage.
fs.Add(-s.Size())
return nil
}

// RemoveAll removes path and any children it contains.
//
// It removes everything it can but returns the first error
// it encounters. If the path does not exist, RemoveAll
// returns nil (no error).
//
// If there is an error, it will be of type *PathError.
func (fs *Quota) RemoveAll(name string) error {
name, err := fs.unsafePath(name)
if err != nil {
return err
}
// While removeAll internally checks this, I want to make sure we check it
// and return the proper error so our tests can ensure that this will never
// be a possibility.
if name == "." {
return &PathError{
Op: "removeall",
Path: name,
Err: ErrBadPathResolution,
}
}
return fs.removeAll(name)
}

func (fs *Quota) removeAll(path string) error {
return removeAll(fs, path)
}

func (fs *Quota) unlinkat(dirfd int, name string, flags int) error {
if flags == 0 {
s, err := fs.Lstatat(dirfd, name)
if err == nil && s.Mode().IsRegular() {
fs.Add(-s.Size())
}
}
return fs.UnixFS.unlinkat(dirfd, name, flags)
}
825 changes: 825 additions & 0 deletions internal/ufs/fs_unix.go
View file

Large diffs are not rendered by default.

255 changes: 255 additions & 0 deletions internal/ufs/fs_unix_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,255 @@
// SPDX-License-Identifier: MIT
// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner

//go:build unix

package ufs_test

import (
"errors"
"os"
"path/filepath"
"testing"

"github.com/pterodactyl/wings/internal/ufs"
)

type testUnixFS struct {
*ufs.UnixFS

TmpDir string
Root string
}

func (fs *testUnixFS) Cleanup() {
_ = fs.Close()
_ = os.RemoveAll(fs.TmpDir)
}

func newTestUnixFS() (*testUnixFS, error) {
tmpDir, err := os.MkdirTemp(os.TempDir(), "ufs")
if err != nil {
return nil, err
}
root := filepath.Join(tmpDir, "root")
if err := os.Mkdir(root, 0o755); err != nil {
return nil, err
}
// TODO: test both disabled and enabled.
fs, err := ufs.NewUnixFS(root, false)
if err != nil {
return nil, err
}
tfs := &testUnixFS{
UnixFS: fs,
TmpDir: tmpDir,
Root: root,
}
return tfs, nil
}

func TestUnixFS_Remove(t *testing.T) {
t.Parallel()
fs, err := newTestUnixFS()
if err != nil {
t.Fatal(err)
return
}
defer fs.Cleanup()

t.Run("base directory", func(t *testing.T) {
// Try to remove the base directory.
if err := fs.Remove(""); !errors.Is(err, ufs.ErrBadPathResolution) {
t.Errorf("expected an a bad path resolution error, but got: %v", err)
return
}
})

t.Run("path traversal", func(t *testing.T) {
// Try to remove the base directory.
if err := fs.RemoveAll("../root"); !errors.Is(err, ufs.ErrBadPathResolution) {
t.Errorf("expected an a bad path resolution error, but got: %v", err)
return
}
})
}

func TestUnixFS_RemoveAll(t *testing.T) {
t.Parallel()
fs, err := newTestUnixFS()
if err != nil {
t.Fatal(err)
return
}
defer fs.Cleanup()

t.Run("base directory", func(t *testing.T) {
// Try to remove the base directory.
if err := fs.RemoveAll(""); !errors.Is(err, ufs.ErrBadPathResolution) {
t.Errorf("expected an a bad path resolution error, but got: %v", err)
return
}
})

t.Run("path traversal", func(t *testing.T) {
// Try to remove the base directory.
if err := fs.RemoveAll("../root"); !errors.Is(err, ufs.ErrBadPathResolution) {
t.Errorf("expected an a bad path resolution error, but got: %v", err)
return
}
})
}

func TestUnixFS_Rename(t *testing.T) {
t.Parallel()
fs, err := newTestUnixFS()
if err != nil {
t.Fatal(err)
return
}
defer fs.Cleanup()

t.Run("rename base directory", func(t *testing.T) {
// Try to rename the base directory.
if err := fs.Rename("", "yeet"); !errors.Is(err, ufs.ErrBadPathResolution) {
t.Errorf("expected an a bad path resolution error, but got: %v", err)
return
}
})

t.Run("rename over base directory", func(t *testing.T) {
// Create a directory that we are going to try and move over top of the
// existing base directory.
if err := fs.Mkdir("overwrite_dir", 0o755); err != nil {
t.Error(err)
return
}

// Try to rename over the base directory.
if err := fs.Rename("overwrite_dir", ""); !errors.Is(err, ufs.ErrBadPathResolution) {
t.Errorf("expected an a bad path resolution error, but got: %v", err)
return
}
})

t.Run("directory rename", func(t *testing.T) {
// Create a directory to rename to something else.
if err := fs.Mkdir("test_directory", 0o755); err != nil {
t.Error(err)
return
}

// Try to rename "test_directory" to "directory".
if err := fs.Rename("test_directory", "directory"); err != nil {
t.Errorf("expected no error, but got: %v", err)
return
}

// Sanity check
if _, err := os.Lstat(filepath.Join(fs.Root, "directory")); err != nil {
t.Errorf("Lstat errored when performing sanity check: %v", err)
return
}
})

t.Run("file rename", func(t *testing.T) {
// Create a directory to rename to something else.
if f, err := fs.Create("test_file"); err != nil {
t.Error(err)
return
} else {
_ = f.Close()
}

// Try to rename "test_file" to "file".
if err := fs.Rename("test_file", "file"); err != nil {
t.Errorf("expected no error, but got: %v", err)
return
}

// Sanity check
if _, err := os.Lstat(filepath.Join(fs.Root, "file")); err != nil {
t.Errorf("Lstat errored when performing sanity check: %v", err)
return
}
})
}

func TestUnixFS_Touch(t *testing.T) {
t.Parallel()
fs, err := newTestUnixFS()
if err != nil {
t.Fatal(err)
return
}
defer fs.Cleanup()

t.Run("base directory", func(t *testing.T) {
path := "i_touched_a_file"
f, err := fs.Touch(path, ufs.O_RDWR, 0o644)
if err != nil {
t.Error(err)
return
}
_ = f.Close()

// Sanity check
if _, err := os.Lstat(filepath.Join(fs.Root, path)); err != nil {
t.Errorf("Lstat errored when performing sanity check: %v", err)
return
}
})

t.Run("existing parent directory", func(t *testing.T) {
dir := "some_parent_directory"
if err := fs.Mkdir(dir, 0o755); err != nil {
t.Errorf("error creating parent directory: %v", err)
return
}
path := filepath.Join(dir, "i_touched_a_file")
f, err := fs.Touch(path, ufs.O_RDWR, 0o644)
if err != nil {
t.Errorf("error touching file: %v", err)
return
}
_ = f.Close()

// Sanity check
if _, err := os.Lstat(filepath.Join(fs.Root, path)); err != nil {
t.Errorf("Lstat errored when performing sanity check: %v", err)
return
}
})

t.Run("non-existent parent directory", func(t *testing.T) {
path := "some_other_directory/i_touched_a_file"
f, err := fs.Touch(path, ufs.O_RDWR, 0o644)
if err != nil {
t.Errorf("error touching file: %v", err)
return
}
_ = f.Close()

// Sanity check
if _, err := os.Lstat(filepath.Join(fs.Root, path)); err != nil {
t.Errorf("Lstat errored when performing sanity check: %v", err)
return
}
})

t.Run("non-existent parent directories", func(t *testing.T) {
path := "some_other_directory/some_directory/i_touched_a_file"
f, err := fs.Touch(path, ufs.O_RDWR, 0o644)
if err != nil {
t.Errorf("error touching file: %v", err)
return
}
_ = f.Close()

// Sanity check
if _, err := os.Lstat(filepath.Join(fs.Root, path)); err != nil {
t.Errorf("Lstat errored when performing sanity check: %v", err)
return
}
})
}
27 changes: 27 additions & 0 deletions internal/ufs/go.LICENSE
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
Copyright (c) 2009 The Go Authors. All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
67 changes: 67 additions & 0 deletions internal/ufs/mkdir_unix.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// SPDX-License-Identifier: BSD-3-Clause

// Code in this file was derived from `go/src/os/path.go`.

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the `go.LICENSE` file.

//go:build unix

package ufs

import (
"golang.org/x/sys/unix"
)

// mkdirAll is a recursive Mkdir implementation that properly handles symlinks.
func (fs *UnixFS) mkdirAll(name string, mode FileMode) error {
// Fast path: if we can tell whether path is a directory or file, stop with success or error.
dir, err := fs.Lstat(name)
if err == nil {
if dir.Mode()&ModeSymlink != 0 {
// If the final path is a symlink, resolve its target and use that
// to check instead.
dir, err = fs.Stat(name)
if err != nil {
return err
}
}
if dir.IsDir() {
return nil
}
return convertErrorType(&PathError{Op: "mkdir", Path: name, Err: unix.ENOTDIR})
}

// Slow path: make sure parent exists and then call Mkdir for path.
i := len(name)
for i > 0 && name[i-1] == '/' { // Skip trailing path separator.
i--
}

j := i
for j > 0 && name[j-1] != '/' { // Scan backward over element.
j--
}

if j > 1 {
// Create parent.
err = fs.mkdirAll(name[:j-1], mode)
if err != nil {
return err
}
}

// Parent now exists; invoke Mkdir and use its result.
err = fs.Mkdir(name, mode)
if err != nil {
// Handle arguments like "foo/." by
// double-checking that directory doesn't exist.
dir, err1 := fs.Lstat(name)
if err1 == nil && dir.IsDir() {
return nil
}
return err
}
return nil
}
80 changes: 80 additions & 0 deletions internal/ufs/path_unix.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// SPDX-License-Identifier: BSD-3-Clause

// Code in this file was copied from `go/src/os/path.go`
// and `go/src/os/path_unix.go`.

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the `go.LICENSE` file.

//go:build unix

package ufs

import (
"os"
)

// basename removes trailing slashes and the leading directory name from path name.
func basename(name string) string {
i := len(name) - 1
// Remove trailing slashes
for ; i > 0 && name[i] == '/'; i-- {
name = name[:i]
}
// Remove leading directory name
for i--; i >= 0; i-- {
if name[i] == '/' {
name = name[i+1:]
break
}
}
return name
}

// endsWithDot reports whether the final component of path is ".".
func endsWithDot(path string) bool {
if path == "." {
return true
}
if len(path) >= 2 && path[len(path)-1] == '.' && os.IsPathSeparator(path[len(path)-2]) {
return true
}
return false
}

// splitPath returns the base name and parent directory.
func splitPath(path string) (string, string) {
// if no better parent is found, the path is relative from "here"
dirname := "."

// Remove all but one leading slash.
for len(path) > 1 && path[0] == '/' && path[1] == '/' {
path = path[1:]
}

i := len(path) - 1

// Remove trailing slashes.
for ; i > 0 && path[i] == '/'; i-- {
path = path[:i]
}

// if no slashes in path, base is path
basename := path

// Remove leading directory path
for i--; i >= 0; i-- {
if path[i] == '/' {
if i == 0 {
dirname = path[:1]
} else {
dirname = path[:i]
}
basename = path[i+1:]
break
}
}

return dirname, basename
}
117 changes: 117 additions & 0 deletions internal/ufs/quota_writer.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// SPDX-License-Identifier: MIT
// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner

package ufs

import (
"errors"
"io"
"sync/atomic"
)

// CountedWriter is a writer that counts the amount of data written to the
// underlying writer.
type CountedWriter struct {
File

counter atomic.Int64
err error
}

// NewCountedWriter returns a new countedWriter that counts the amount of bytes
// written to the underlying writer.
func NewCountedWriter(f File) *CountedWriter {
return &CountedWriter{File: f}
}

// BytesWritten returns the amount of bytes that have been written to the
// underlying writer.
func (w *CountedWriter) BytesWritten() int64 {
return w.counter.Load()
}

// Error returns the error from the writer if any. If the error is an EOF, nil
// will be returned.
func (w *CountedWriter) Error() error {
if errors.Is(w.err, io.EOF) {
return nil
}
return w.err
}

// Write writes bytes to the underlying writer while tracking the total amount
// of bytes written.
func (w *CountedWriter) Write(p []byte) (int, error) {
if w.err != nil {
return 0, io.EOF
}

// Write is a very simple operation for us to handle.
n, err := w.File.Write(p)
w.counter.Add(int64(n))
w.err = err

// TODO: is this how we actually want to handle errors with this?
if err == io.EOF {
return n, io.EOF
} else {
return n, nil
}
}

func (w *CountedWriter) ReadFrom(r io.Reader) (n int64, err error) {
cr := NewCountedReader(r)
n, err = w.File.ReadFrom(cr)
w.counter.Add(n)
return
}

// CountedReader is a reader that counts the amount of data read from the
// underlying reader.
type CountedReader struct {
reader io.Reader

counter atomic.Int64
err error
}

var _ io.Reader = (*CountedReader)(nil)

// NewCountedReader returns a new countedReader that counts the amount of bytes
// read from the underlying reader.
func NewCountedReader(r io.Reader) *CountedReader {
return &CountedReader{reader: r}
}

// BytesRead returns the amount of bytes that have been read from the underlying
// reader.
func (r *CountedReader) BytesRead() int64 {
return r.counter.Load()
}

// Error returns the error from the reader if any. If the error is an EOF, nil
// will be returned.
func (r *CountedReader) Error() error {
if errors.Is(r.err, io.EOF) {
return nil
}
return r.err
}

// Read reads bytes from the underlying reader while tracking the total amount
// of bytes read.
func (r *CountedReader) Read(p []byte) (int, error) {
if r.err != nil {
return 0, io.EOF
}

n, err := r.reader.Read(p)
r.counter.Add(int64(n))
r.err = err

if err == io.EOF {
return n, io.EOF
} else {
return n, nil
}
}
207 changes: 207 additions & 0 deletions internal/ufs/removeall_unix.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,207 @@
// SPDX-License-Identifier: BSD-3-Clause

// Code in this file was derived from `go/src/os/removeall_at.go`.

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the `go.LICENSE` file.

//go:build unix

package ufs

import (
"errors"
"io"
"os"

"golang.org/x/sys/unix"
)

type unixFS interface {
Open(name string) (File, error)
Remove(name string) error
unlinkat(dirfd int, path string, flags int) error
}

func (fs *UnixFS) removeAll(path string) error {
return removeAll(fs, path)
}

func removeAll(fs unixFS, path string) error {
if path == "" {
// fail silently to retain compatibility with previous behavior
// of RemoveAll. See issue https://go.dev/issue/28830.
return nil
}

// The rmdir system call does not permit removing ".",
// so we don't permit it either.
if endsWithDot(path) {
return &PathError{Op: "removeall", Path: path, Err: unix.EINVAL}
}

// Simple case: if Remove works, we're done.
err := fs.Remove(path)
if err == nil || errors.Is(err, ErrNotExist) {
return nil
}

// RemoveAll recurses by deleting the path base from
// its parent directory
parentDir, base := splitPath(path)

parent, err := fs.Open(parentDir)
if errors.Is(err, ErrNotExist) {
// If parent does not exist, base cannot exist. Fail silently
return nil
}
if err != nil {
return err
}
defer parent.Close()

if err := removeAllFrom(fs, parent, base); err != nil {
if pathErr, ok := err.(*PathError); ok {
pathErr.Path = parentDir + string(os.PathSeparator) + pathErr.Path
err = pathErr
}
return convertErrorType(err)
}
return nil
}

func removeAllFrom(fs unixFS, parent File, base string) error {
parentFd := int(parent.Fd())
// Simple case: if Unlink (aka remove) works, we're done.
err := fs.unlinkat(parentFd, base, 0)
if err == nil || errors.Is(err, ErrNotExist) {
return nil
}

// EISDIR means that we have a directory, and we need to
// remove its contents.
// EPERM or EACCES means that we don't have write permission on
// the parent directory, but this entry might still be a directory
// whose contents need to be removed.
// Otherwise, just return the error.
if err != unix.EISDIR && err != unix.EPERM && err != unix.EACCES {
return &PathError{Op: "unlinkat", Path: base, Err: err}
}

// Is this a directory we need to recurse into?
var statInfo unix.Stat_t
statErr := ignoringEINTR(func() error {
return unix.Fstatat(parentFd, base, &statInfo, AT_SYMLINK_NOFOLLOW)
})
if statErr != nil {
if errors.Is(statErr, ErrNotExist) {
return nil
}
return &PathError{Op: "fstatat", Path: base, Err: statErr}
}
if statInfo.Mode&unix.S_IFMT != unix.S_IFDIR {
// Not a directory; return the error from the unix.Unlinkat.
return &PathError{Op: "unlinkat", Path: base, Err: err}
}

// Remove the directory's entries.
var recurseErr error
for {
const reqSize = 1024
var respSize int

// Open the directory to recurse into
file, err := openFdAt(parentFd, base)
if err != nil {
if errors.Is(err, ErrNotExist) {
return nil
}
recurseErr = &PathError{Op: "openfdat", Path: base, Err: err}
break
}

for {
numErr := 0

names, readErr := file.Readdirnames(reqSize)
// Errors other than EOF should stop us from continuing.
if readErr != nil && readErr != io.EOF {
_ = file.Close()
if errors.Is(readErr, ErrNotExist) {
return nil
}
return &PathError{Op: "readdirnames", Path: base, Err: readErr}
}

respSize = len(names)
for _, name := range names {
err := removeAllFrom(fs, file, name)
if err != nil {
if pathErr, ok := err.(*PathError); ok {
pathErr.Path = base + string(os.PathSeparator) + pathErr.Path
}
numErr++
if recurseErr == nil {
recurseErr = err
}
}
}

// If we can delete any entry, break to start new iteration.
// Otherwise, we discard current names, get next entries and try deleting them.
if numErr != reqSize {
break
}
}

// Removing files from the directory may have caused
// the OS to reshuffle it. Simply calling Readdirnames
// again may skip some entries. The only reliable way
// to avoid this is to close and re-open the
// directory. See issue https://go.dev/issue/20841.
_ = file.Close()

// Finish when the end of the directory is reached
if respSize < reqSize {
break
}
}

// Remove the directory itself.
unlinkErr := fs.unlinkat(parentFd, base, AT_REMOVEDIR)
if unlinkErr == nil || errors.Is(unlinkErr, ErrNotExist) {
return nil
}

if recurseErr != nil {
return recurseErr
}
return &PathError{Op: "unlinkat", Path: base, Err: unlinkErr}
}

// openFdAt opens path relative to the directory in fd.
// Other than that this should act like openFileNolog.
// This acts like openFileNolog rather than OpenFile because
// we are going to (try to) remove the file.
// The contents of this file are not relevant for test caching.
func openFdAt(dirfd int, name string) (File, error) {
var fd int
for {
var err error
fd, err = unix.Openat(dirfd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW, 0)
if err == nil {
break
}

// See comment in openFileNolog.
if err == unix.EINTR {
continue
}

return nil, err
}
// This is stupid, os.NewFile immediately casts `fd` to an `int`, but wants
// it to be passed as a `uintptr`.
return os.NewFile(uintptr(fd), name), nil
}
67 changes: 67 additions & 0 deletions internal/ufs/stat_unix.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// SPDX-License-Identifier: BSD-3-Clause

// Code in this file was copied from `go/src/os/stat_linux.go`
// and `go/src/os/types_unix.go`.

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the `go.LICENSE` file.

//go:build unix

package ufs

import (
"time"

"golang.org/x/sys/unix"
)

type fileStat struct {
name string
size int64
mode FileMode
modTime time.Time
sys unix.Stat_t
}

var _ FileInfo = (*fileStat)(nil)

func (fs *fileStat) Size() int64 { return fs.size }
func (fs *fileStat) Mode() FileMode { return fs.mode }
func (fs *fileStat) ModTime() time.Time { return fs.modTime }
func (fs *fileStat) Sys() any { return &fs.sys }
func (fs *fileStat) Name() string { return fs.name }
func (fs *fileStat) IsDir() bool { return fs.Mode().IsDir() }

func fillFileStatFromSys(fs *fileStat, name string) {
fs.name = basename(name)
fs.size = fs.sys.Size
fs.modTime = time.Unix(fs.sys.Mtim.Unix())
fs.mode = FileMode(fs.sys.Mode & 0o777)
switch fs.sys.Mode & unix.S_IFMT {
case unix.S_IFBLK:
fs.mode |= ModeDevice
case unix.S_IFCHR:
fs.mode |= ModeDevice | ModeCharDevice
case unix.S_IFDIR:
fs.mode |= ModeDir
case unix.S_IFIFO:
fs.mode |= ModeNamedPipe
case unix.S_IFLNK:
fs.mode |= ModeSymlink
case unix.S_IFREG:
// nothing to do
case unix.S_IFSOCK:
fs.mode |= ModeSocket
}
if fs.sys.Mode&unix.S_ISGID != 0 {
fs.mode |= ModeSetgid
}
if fs.sys.Mode&unix.S_ISUID != 0 {
fs.mode |= ModeSetuid
}
if fs.sys.Mode&unix.S_ISVTX != 0 {
fs.mode |= ModeSticky
}
}
123 changes: 123 additions & 0 deletions internal/ufs/walk.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
// SPDX-License-Identifier: BSD-3-Clause

// Code in this file was derived from `go/src/io/fs/walk.go`.

// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the `go.LICENSE` file.

package ufs

import (
iofs "io/fs"
"path"
)

// SkipDir is used as a return value from [WalkDirFunc] to indicate that
// the directory named in the call is to be skipped. It is not returned
// as an error by any function.
var SkipDir = iofs.SkipDir

// SkipAll is used as a return value from [WalkDirFunc] to indicate that
// all remaining files and directories are to be skipped. It is not returned
// as an error by any function.
var SkipAll = iofs.SkipAll

// WalkDirFunc is the type of the function called by [WalkDir] to visit
// each file or directory.
//
// The path argument contains the argument to [WalkDir] as a prefix.
// That is, if WalkDir is called with root argument "dir" and finds a file
// named "a" in that directory, the walk function will be called with
// argument "dir/a".
//
// The d argument is the [DirEntry] for the named path.
//
// The error result returned by the function controls how [WalkDir]
// continues. If the function returns the special value [SkipDir], WalkDir
// skips the current directory (path if d.IsDir() is true, otherwise
// path's parent directory). If the function returns the special value
// [SkipAll], WalkDir skips all remaining files and directories. Otherwise,
// if the function returns a non-nil error, WalkDir stops entirely and
// returns that error.
//
// The err argument reports an error related to path, signaling that
// [WalkDir] will not walk into that directory. The function can decide how
// to handle that error; as described earlier, returning the error will
// cause WalkDir to stop walking the entire tree.
//
// [WalkDir] calls the function with a non-nil err argument in two cases.
//
// First, if the initial [Stat] on the root directory fails, WalkDir
// calls the function with path set to root, d set to nil, and err set to
// the error from [fs.Stat].
//
// Second, if a directory's ReadDir method (see [ReadDirFile]) fails, WalkDir calls the
// function with path set to the directory's path, d set to an
// [DirEntry] describing the directory, and err set to the error from
// ReadDir. In this second case, the function is called twice with the
// path of the directory: the first call is before the directory read is
// attempted and has err set to nil, giving the function a chance to
// return [SkipDir] or [SkipAll] and avoid the ReadDir entirely. The second call
// is after a failed ReadDir and reports the error from ReadDir.
// (If ReadDir succeeds, there is no second call.)
type WalkDirFunc func(path string, d DirEntry, err error) error

// WalkDir walks the file tree rooted at root, calling fn for each file or
// directory in the tree, including root.
//
// All errors that arise visiting files and directories are filtered by fn:
// see the [WalkDirFunc] documentation for details.
//
// The files are walked in lexical order, which makes the output deterministic
// but requires WalkDir to read an entire directory into memory before proceeding
// to walk that directory.
//
// WalkDir does not follow symbolic links found in directories,
// but if root itself is a symbolic link, its target will be walked.
func WalkDir(fs Filesystem, root string, fn WalkDirFunc) error {
info, err := fs.Stat(root)
if err != nil {
err = fn(root, nil, err)
} else {
err = walkDir(fs, root, iofs.FileInfoToDirEntry(info), fn)
}
if err == SkipDir || err == SkipAll {
return nil
}
return err
}

// walkDir recursively descends path, calling walkDirFn.
func walkDir(fs Filesystem, name string, d DirEntry, walkDirFn WalkDirFunc) error {
if err := walkDirFn(name, d, nil); err != nil || !d.IsDir() {
if err == SkipDir && d.IsDir() {
// Successfully skipped directory.
err = nil
}
return err
}

dirs, err := fs.ReadDir(name)
if err != nil {
// Second call, to report ReadDir error.
err = walkDirFn(name, d, err)
if err != nil {
if err == SkipDir && d.IsDir() {
err = nil
}
return err
}
}

for _, d1 := range dirs {
name1 := path.Join(name, d1.Name())
if err := walkDir(fs, name1, d1, walkDirFn); err != nil {
if err == SkipDir {
break
}
return err
}
}
return nil
}
291 changes: 291 additions & 0 deletions internal/ufs/walk_unix.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,291 @@
// SPDX-License-Identifier: BSD-2-Clause

// Some code in this file was derived from https://github.com/karrick/godirwalk.

//go:build unix

package ufs

import (
"bytes"
iofs "io/fs"
"os"
"path"
"path/filepath"
"reflect"
"unsafe"

"golang.org/x/sys/unix"
)

type WalkDiratFunc func(dirfd int, name, relative string, d DirEntry, err error) error

func (fs *UnixFS) WalkDirat(dirfd int, name string, fn WalkDiratFunc) error {
if dirfd == 0 {
// TODO: proper validation, ideally a dedicated function.
dirfd = int(fs.dirfd.Load())
}
info, err := fs.Lstatat(dirfd, name)
if err != nil {
err = fn(dirfd, name, name, nil, err)
} else {
b := newScratchBuffer()
err = fs.walkDir(b, dirfd, name, name, iofs.FileInfoToDirEntry(info), fn)
}
if err == SkipDir || err == SkipAll {
return nil
}
return err
}

func (fs *UnixFS) walkDir(b []byte, parentfd int, name, relative string, d DirEntry, walkDirFn WalkDiratFunc) error {
if err := walkDirFn(parentfd, name, relative, d, nil); err != nil || !d.IsDir() {
if err == SkipDir && d.IsDir() {
// Successfully skipped directory.
err = nil
}
return err
}

dirfd, err := fs.openat(parentfd, name, O_DIRECTORY|O_RDONLY, 0)
if err != nil {
return err
}
defer unix.Close(dirfd)

dirs, err := fs.readDir(dirfd, name, b)
if err != nil {
// Second call, to report ReadDir error.
err = walkDirFn(dirfd, name, relative, d, err)
if err != nil {
if err == SkipDir && d.IsDir() {
err = nil
}
return err
}
}

for _, d1 := range dirs {
if err := fs.walkDir(b, dirfd, d1.Name(), path.Join(relative, d1.Name()), d1, walkDirFn); err != nil {
if err == SkipDir {
break
}
return err
}
}
return nil
}

// ReadDirMap .
// TODO: document
func ReadDirMap[T any](fs *UnixFS, path string, fn func(DirEntry) (T, error)) ([]T, error) {
dirfd, name, closeFd, err := fs.safePath(path)
defer closeFd()
if err != nil {
return nil, err
}
fd, err := fs.openat(dirfd, name, O_DIRECTORY|O_RDONLY, 0)
if err != nil {
return nil, err
}
defer unix.Close(fd)

entries, err := fs.readDir(fd, ".", nil)
if err != nil {
return nil, err
}

out := make([]T, len(entries))
for i, e := range entries {
idx := i
e := e
v, err := fn(e)
if err != nil {
return nil, err
}
out[idx] = v
}
return out, nil
}

// nameOffset is a compile time constant
const nameOffset = int(unsafe.Offsetof(unix.Dirent{}.Name))

func nameFromDirent(de *unix.Dirent) (name []byte) {
// Because this GOOS' syscall.Dirent does not provide a field that specifies
// the name length, this function must first calculate the max possible name
// length, and then search for the NULL byte.
ml := int(de.Reclen) - nameOffset

// Convert syscall.Dirent.Name, which is array of int8, to []byte, by
// overwriting Cap, Len, and Data slice header fields to the max possible
// name length computed above, and finding the terminating NULL byte.
//
// TODO: is there an alternative to the deprecated SliceHeader?
// SliceHeader was mainly deprecated due to it being misused for avoiding
// allocations when converting a byte slice to a string, ref;
// https://go.dev/issue/53003
sh := (*reflect.SliceHeader)(unsafe.Pointer(&name))
sh.Cap = ml
sh.Len = ml
sh.Data = uintptr(unsafe.Pointer(&de.Name[0]))

if index := bytes.IndexByte(name, 0); index >= 0 {
// Found NULL byte; set slice's cap and len accordingly.
sh.Cap = index
sh.Len = index
return
}

// NOTE: This branch is not expected, but included for defensive
// programming, and provides a hard stop on the name based on the structure
// field array size.
sh.Cap = len(de.Name)
sh.Len = sh.Cap
return
}

// modeTypeFromDirent converts a syscall defined constant, which is in purview
// of OS, to a constant defined by Go, assumed by this project to be stable.
//
// When the syscall constant is not recognized, this function falls back to a
// Stat on the file system.
func (fs *UnixFS) modeTypeFromDirent(fd int, de *unix.Dirent, osDirname, osBasename string) (FileMode, error) {
switch de.Type {
case unix.DT_REG:
return 0, nil
case unix.DT_DIR:
return ModeDir, nil
case unix.DT_LNK:
return ModeSymlink, nil
case unix.DT_CHR:
return ModeDevice | ModeCharDevice, nil
case unix.DT_BLK:
return ModeDevice, nil
case unix.DT_FIFO:
return ModeNamedPipe, nil
case unix.DT_SOCK:
return ModeSocket, nil
default:
// If syscall returned unknown type (e.g., DT_UNKNOWN, DT_WHT), then
// resolve actual mode by reading file information.
return fs.modeType(fd, filepath.Join(osDirname, osBasename))
}
}

// modeType returns the mode type of the file system entry identified by
// osPathname by calling os.LStat function, to intentionally not follow symbolic
// links.
//
// Even though os.LStat provides all file mode bits, we want to ensure same
// values returned to caller regardless of whether we obtained file mode bits
// from syscall or stat call. Therefore, mask out the additional file mode bits
// that are provided by stat but not by the syscall, so users can rely on their
// values.
func (fs *UnixFS) modeType(dirfd int, name string) (os.FileMode, error) {
fi, err := fs.Lstatat(dirfd, name)
if err == nil {
return fi.Mode() & ModeType, nil
}
return 0, err
}

var minimumScratchBufferSize = os.Getpagesize()

func newScratchBuffer() []byte {
return make([]byte, minimumScratchBufferSize)
}

func (fs *UnixFS) readDir(fd int, name string, b []byte) ([]DirEntry, error) {
scratchBuffer := b
if scratchBuffer == nil || len(scratchBuffer) < minimumScratchBufferSize {
scratchBuffer = newScratchBuffer()
}

var entries []DirEntry
var workBuffer []byte

var sde unix.Dirent
for {
if len(workBuffer) == 0 {
n, err := unix.Getdents(fd, scratchBuffer)
if err != nil {
if err == unix.EINTR {
continue
}
return nil, convertErrorType(err)
}
if n <= 0 {
// end of directory: normal exit
return entries, nil
}
workBuffer = scratchBuffer[:n] // trim work buffer to number of bytes read
}

// "Go is like C, except that you just put `unsafe` all over the place".
copy((*[unsafe.Sizeof(unix.Dirent{})]byte)(unsafe.Pointer(&sde))[:], workBuffer)
workBuffer = workBuffer[sde.Reclen:] // advance buffer for next iteration through loop

if sde.Ino == 0 {
continue // inode set to 0 indicates an entry that was marked as deleted
}

nameSlice := nameFromDirent(&sde)
nameLength := len(nameSlice)

if nameLength == 0 || (nameSlice[0] == '.' && (nameLength == 1 || (nameLength == 2 && nameSlice[1] == '.'))) {
continue
}

childName := string(nameSlice)
mt, err := fs.modeTypeFromDirent(fd, &sde, name, childName)
if err != nil {
return nil, convertErrorType(err)
}
entries = append(entries, &dirent{name: childName, path: name, modeType: mt, dirfd: fd, fs: fs})
}
}

// dirent stores the name and file system mode type of discovered file system
// entries.
type dirent struct {
name string
path string
modeType FileMode

dirfd int
fs *UnixFS
}

func (de dirent) Name() string {
return de.name
}

func (de dirent) IsDir() bool {
return de.modeType&ModeDir != 0
}

func (de dirent) Type() FileMode {
return de.modeType
}

func (de dirent) Info() (FileInfo, error) {
if de.fs == nil {
return nil, nil
}
return de.fs.Lstatat(de.dirfd, de.name)
}

func (de dirent) Open() (File, error) {
if de.fs == nil {
return nil, nil
}
return de.fs.OpenFileat(de.dirfd, de.name, O_RDONLY, 0)
}

// reset releases memory held by entry err and name, and resets mode type to 0.
func (de *dirent) reset() {
de.name = ""
de.path = ""
de.modeType = 0
}
18 changes: 3 additions & 15 deletions parser/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package parser

import (
"bytes"
"io"
"os"
"regexp"
"strconv"
"strings"
Expand All @@ -29,24 +27,14 @@ var configMatchRegex = regexp.MustCompile(`{{\s?config\.([\w.-]+)\s?}}`)
// matching:
//
// <Root>
// <Property value="testing"/>
//
// <Property value="testing"/>
//
// </Root>
//
// noinspection RegExpRedundantEscape
var xmlValueMatchRegex = regexp.MustCompile(`^\[([\w]+)='(.*)'\]$`)

// Gets the []byte representation of a configuration file to be passed through to other
// handler functions. If the file does not currently exist, it will be created.
func readFileBytes(path string) ([]byte, error) {
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644)
if err != nil {
return nil, err
}
defer file.Close()

return io.ReadAll(file)
}

// Gets the value of a key based on the value type defined.
func (cfr *ConfigurationFileReplacement) getKeyValue(value string) interface{} {
if cfr.ReplaceWith.Type() == jsonparser.Boolean {
Expand Down
246 changes: 130 additions & 116 deletions parser/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ package parser

import (
"bufio"
"os"
"path/filepath"
"bytes"
"io"
"strconv"
"strings"

Expand All @@ -18,6 +18,7 @@ import (
"gopkg.in/yaml.v3"

"github.com/pterodactyl/wings/config"
"github.com/pterodactyl/wings/internal/ufs"
)

// The file parsing options that are available for a server configuration file.
Expand Down Expand Up @@ -74,6 +75,26 @@ func (cv *ReplaceValue) String() string {
}
}

func (cv *ReplaceValue) Bytes() []byte {
switch cv.Type() {
case jsonparser.String:
var stackbuf [64]byte
bU, err := jsonparser.Unescape(cv.value, stackbuf[:])
if err != nil {
panic(errors.Wrap(err, "parser: could not parse value"))
}
return bU
case jsonparser.Null:
return []byte("<nil>")
case jsonparser.Boolean:
return cv.value
case jsonparser.Number:
return cv.value
default:
return []byte("<invalid>")
}
}

type ConfigurationParser string

func (cp ConfigurationParser) String() string {
Expand Down Expand Up @@ -167,11 +188,12 @@ func (cfr *ConfigurationFileReplacement) UnmarshalJSON(data []byte) error {
return nil
}

// Parses a given configuration file and updates all of the values within as defined
// in the API response from the Panel.
func (f *ConfigurationFile) Parse(path string, internal bool) error {
log.WithField("path", path).WithField("parser", f.Parser.String()).Debug("parsing server configuration file")
// Parse parses a given configuration file and updates all the values within
// as defined in the API response from the Panel.
func (f *ConfigurationFile) Parse(file ufs.File) error {
//log.WithField("path", path).WithField("parser", f.Parser.String()).Debug("parsing server configuration file")

// What the fuck is going on here?
if mb, err := json.Marshal(config.Get()); err != nil {
return err
} else {
Expand All @@ -182,56 +204,24 @@ func (f *ConfigurationFile) Parse(path string, internal bool) error {

switch f.Parser {
case Properties:
err = f.parsePropertiesFile(path)
break
err = f.parsePropertiesFile(file)
case File:
err = f.parseTextFile(path)
break
err = f.parseTextFile(file)
case Yaml, "yml":
err = f.parseYamlFile(path)
break
err = f.parseYamlFile(file)
case Json:
err = f.parseJsonFile(path)
break
err = f.parseJsonFile(file)
case Ini:
err = f.parseIniFile(path)
break
err = f.parseIniFile(file)
case Xml:
err = f.parseXmlFile(path)
break
err = f.parseXmlFile(file)
}

if errors.Is(err, os.ErrNotExist) {
// File doesn't exist, we tried creating it, and same error is returned? Pretty
// sure this pathway is impossible, but if not, abort here.
if internal {
return nil
}

b := strings.TrimSuffix(path, filepath.Base(path))
if err := os.MkdirAll(b, 0o755); err != nil {
return errors.WithMessage(err, "failed to create base directory for missing configuration file")
} else {
if _, err := os.Create(path); err != nil {
return errors.WithMessage(err, "failed to create missing configuration file")
}
}

return f.Parse(path, true)
}

return err
}

// Parses an xml file.
func (f *ConfigurationFile) parseXmlFile(path string) error {
func (f *ConfigurationFile) parseXmlFile(file ufs.File) error {
doc := etree.NewDocument()
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644)
if err != nil {
return err
}
defer file.Close()

if _, err := doc.ReadFrom(file); err != nil {
return err
}
Expand Down Expand Up @@ -291,41 +281,27 @@ func (f *ConfigurationFile) parseXmlFile(path string) error {
}
}

// If you don't truncate the file you'll end up duplicating the data in there (or just appending
// to the end of the file. We don't want to do that.
if _, err := file.Seek(0, io.SeekStart); err != nil {
return err
}
if err := file.Truncate(0); err != nil {
return err
}

// Move the cursor to the start of the file to avoid weird spacing issues.
file.Seek(0, 0)

// Ensure the XML is indented properly.
doc.Indent(2)

// Truncate the file before attempting to write the changes.
if err := os.Truncate(path, 0); err != nil {
// Write the XML to the file.
if _, err := doc.WriteTo(file); err != nil {
return err
}

// Write the XML to the file.
_, err = doc.WriteTo(file)

return err
return nil
}

// Parses an ini file.
func (f *ConfigurationFile) parseIniFile(path string) error {
// Ini package can't handle a non-existent file, so handle that automatically here
// by creating it if not exists. Then, immediately close the file since we will use
// other methods to write the new contents.
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644)
if err != nil {
return err
}
file.Close()

cfg, err := ini.Load(path)
func (f *ConfigurationFile) parseIniFile(file ufs.File) error {
// Wrap the file in a NopCloser so the ini package doesn't close the file.
cfg, err := ini.Load(io.NopCloser(file))
if err != nil {
return err
}
Expand Down Expand Up @@ -388,14 +364,24 @@ func (f *ConfigurationFile) parseIniFile(path string) error {
}
}

return cfg.SaveTo(path)
if _, err := file.Seek(0, io.SeekStart); err != nil {
return err
}
if err := file.Truncate(0); err != nil {
return err
}

if _, err := cfg.WriteTo(file); err != nil {
return err
}
return nil
}

// Parses a json file updating any matching key/value pairs. If a match is not found, the
// value is set regardless in the file. See the commentary in parseYamlFile for more details
// about what is happening during this process.
func (f *ConfigurationFile) parseJsonFile(path string) error {
b, err := readFileBytes(path)
func (f *ConfigurationFile) parseJsonFile(file ufs.File) error {
b, err := io.ReadAll(file)
if err != nil {
return err
}
Expand All @@ -405,14 +391,24 @@ func (f *ConfigurationFile) parseJsonFile(path string) error {
return err
}

output := []byte(data.StringIndent("", " "))
return os.WriteFile(path, output, 0o644)
if _, err := file.Seek(0, io.SeekStart); err != nil {
return err
}
if err := file.Truncate(0); err != nil {
return err
}

// Write the data to the file.
if _, err := io.Copy(file, bytes.NewReader(data.BytesIndent("", " "))); err != nil {
return errors.Wrap(err, "parser: failed to write properties file to disk")
}
return nil
}

// Parses a yaml file and updates any matching key/value pairs before persisting
// it back to the disk.
func (f *ConfigurationFile) parseYamlFile(path string) error {
b, err := readFileBytes(path)
func (f *ConfigurationFile) parseYamlFile(file ufs.File) error {
b, err := io.ReadAll(file)
if err != nil {
return err
}
Expand Down Expand Up @@ -443,35 +439,56 @@ func (f *ConfigurationFile) parseYamlFile(path string) error {
return err
}

return os.WriteFile(path, marshaled, 0o644)
if _, err := file.Seek(0, io.SeekStart); err != nil {
return err
}
if err := file.Truncate(0); err != nil {
return err
}

// Write the data to the file.
if _, err := io.Copy(file, bytes.NewReader(marshaled)); err != nil {
return errors.Wrap(err, "parser: failed to write properties file to disk")
}
return nil
}

// Parses a text file using basic find and replace. This is a highly inefficient method of
// scanning a file and performing a replacement. You should attempt to use anything other
// than this function where possible.
func (f *ConfigurationFile) parseTextFile(path string) error {
input, err := os.ReadFile(path)
if err != nil {
return err
}

lines := strings.Split(string(input), "\n")
for i, line := range lines {
func (f *ConfigurationFile) parseTextFile(file ufs.File) error {
b := bytes.NewBuffer(nil)
s := bufio.NewScanner(file)
var replaced bool
for s.Scan() {
line := s.Bytes()
replaced = false
for _, replace := range f.Replace {
// If this line doesn't match what we expect for the replacement, move on to the next
// line. Otherwise, update the line to have the replacement value.
if !strings.HasPrefix(line, replace.Match) {
if !bytes.HasPrefix(line, []byte(replace.Match)) {
continue
}

lines[i] = replace.ReplaceWith.String()
b.Write(replace.ReplaceWith.Bytes())
replaced = true
}
if !replaced {
b.Write(line)
}
b.WriteByte('\n')
}

if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")), 0o644); err != nil {
if _, err := file.Seek(0, io.SeekStart); err != nil {
return err
}
if err := file.Truncate(0); err != nil {
return err
}

// Write the data to the file.
if _, err := io.Copy(file, b); err != nil {
return errors.Wrap(err, "parser: failed to write properties file to disk")
}
return nil
}

Expand Down Expand Up @@ -501,31 +518,29 @@ func (f *ConfigurationFile) parseTextFile(path string) error {
//
// @see https://github.com/pterodactyl/panel/issues/2308 (original)
// @see https://github.com/pterodactyl/panel/issues/3009 ("bug" introduced as result)
func (f *ConfigurationFile) parsePropertiesFile(path string) error {
var s strings.Builder
// Open the file and attempt to load any comments that currenty exist at the start
// of the file. This is kind of a hack, but should work for a majority of users for
// the time being.
if fd, err := os.Open(path); err != nil {
return errors.Wrap(err, "parser: could not open file for reading")
} else {
scanner := bufio.NewScanner(fd)
// Scan until we hit a line that is not a comment that actually has content
// on it. Keep appending the comments until that time.
for scanner.Scan() {
text := scanner.Text()
if len(text) > 0 && text[0] != '#' {
break
}
s.WriteString(text + "\n")
}
_ = fd.Close()
if err := scanner.Err(); err != nil {
return errors.WithStackIf(err)
func (f *ConfigurationFile) parsePropertiesFile(file ufs.File) error {
b, err := io.ReadAll(file)
if err != nil {
return err
}

s := bytes.NewBuffer(nil)
scanner := bufio.NewScanner(bytes.NewReader(b))
// Scan until we hit a line that is not a comment that actually has content
// on it. Keep appending the comments until that time.
for scanner.Scan() {
text := scanner.Bytes()
if len(text) > 0 && text[0] != '#' {
break
}
s.Write(text)
s.WriteByte('\n')
}
if err := scanner.Err(); err != nil {
return errors.WithStackIf(err)
}

p, err := properties.LoadFile(path, properties.UTF8)
p, err := properties.Load(b, properties.UTF8)
if err != nil {
return errors.Wrap(err, "parser: could not load properties file for configuration update")
}
Expand Down Expand Up @@ -563,17 +578,16 @@ func (f *ConfigurationFile) parsePropertiesFile(path string) error {
s.WriteString(key + "=" + strings.Trim(strconv.QuoteToASCII(value), "\"") + "\n")
}

// Open the file for writing.
w, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
if err != nil {
if _, err := file.Seek(0, io.SeekStart); err != nil {
return err
}
if err := file.Truncate(0); err != nil {
return err
}
defer w.Close()

// Write the data to the file.
if _, err := w.Write([]byte(s.String())); err != nil {
if _, err := io.Copy(file, s); err != nil {
return errors.Wrap(err, "parser: failed to write properties file to disk")
}

return nil
}
13 changes: 5 additions & 8 deletions router/downloader/downloader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,13 +167,8 @@ func (dl *Download) Execute() error {
return errors.New("downloader: got bad response status from endpoint: " + res.Status)
}

// If there is a Content-Length header on this request go ahead and check that we can
// even write the whole file before beginning this process. If there is no header present
// we'll just have to give it a spin and see how it goes.
if res.ContentLength > 0 {
if err := dl.server.Filesystem().HasSpaceFor(res.ContentLength); err != nil {
return errors.WrapIf(err, "downloader: failed to write file: not enough space")
}
if res.ContentLength < 1 {
return errors.New("downloader: request is missing ContentLength")
}

if dl.req.UseHeader {
Expand All @@ -200,8 +195,10 @@ func (dl *Download) Execute() error {
p := dl.Path()
dl.server.Log().WithField("path", p).Debug("writing remote file to disk")

// Write the file while tracking the progress, Write will check that the
// size of the file won't exceed the disk limit.
r := io.TeeReader(res.Body, dl.counter(res.ContentLength))
if err := dl.server.Filesystem().Writefile(p, r); err != nil {
if err := dl.server.Filesystem().Write(p, r, res.ContentLength, 0o644); err != nil {
return errors.WrapIf(err, "downloader: failed to write file to server directory")
}
return nil
Expand Down
18 changes: 6 additions & 12 deletions router/router_download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ func getDownloadBackup(c *gin.Context) {
return
}

// The use of `os` here is safe as backups are not stored within server
// accessible directories.
f, err := os.Open(b.Path())
if err != nil {
middleware.CaptureAndAbort(c, err)
Expand Down Expand Up @@ -76,27 +78,19 @@ func getDownloadFile(c *gin.Context) {
return
}

p, _ := s.Filesystem().SafePath(token.FilePath)
st, err := os.Stat(p)
// If there is an error or we're somehow trying to download a directory, just
// respond with the appropriate error.
f, st, err := s.Filesystem().File(token.FilePath)
if err != nil {
middleware.CaptureAndAbort(c, err)
return
} else if st.IsDir() {
}
defer f.Close()
if st.IsDir() {
c.AbortWithStatusJSON(http.StatusNotFound, gin.H{
"error": "The requested resource was not found on this server.",
})
return
}

f, err := os.Open(p)
if err != nil {
middleware.CaptureAndAbort(c, err)
return
}
defer f.Close()

c.Header("Content-Length", strconv.Itoa(int(st.Size())))
c.Header("Content-Disposition", "attachment; filename="+strconv.Quote(st.Name()))
c.Header("Content-Type", "application/octet-stream")
Expand Down
1 change: 1 addition & 0 deletions router/router_server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ func deleteServer(c *gin.Context) {
// In addition, servers with large amounts of files can take some time to finish deleting,
// so we don't want to block the HTTP call while waiting on this.
go func(p string) {
_ = s.Filesystem().UnixFS().Close()
if err := os.RemoveAll(p); err != nil {
log.WithFields(log.Fields{"path": p, "error": err}).Warn("failed to remove server files during deletion process")
}
Expand Down
24 changes: 13 additions & 11 deletions router/router_server_files.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
// getServerFileContents returns the contents of a file on the server.
func getServerFileContents(c *gin.Context) {
s := middleware.ExtractServer(c)
p := "/" + strings.TrimLeft(c.Query("file"), "/")
p := strings.TrimLeft(c.Query("file"), "/")
f, st, err := s.Filesystem().File(p)
if err != nil {
middleware.CaptureAndAbort(c, err)
Expand Down Expand Up @@ -129,7 +129,6 @@ func putServerRenameFiles(c *gin.Context) {
}
if err := fs.Rename(pf, pt); err != nil {
// Return nil if the error is an is not exists.
// NOTE: os.IsNotExist() does not work if the error is wrapped.
if errors.Is(err, os.ErrNotExist) {
s.Log().WithField("error", err).
WithField("from_path", pf).
Expand Down Expand Up @@ -239,7 +238,15 @@ func postServerWriteFile(c *gin.Context) {
middleware.CaptureAndAbort(c, err)
return
}
if err := s.Filesystem().Writefile(f, c.Request.Body); err != nil {

if c.Request.ContentLength < 1 {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
"error": "Missing Content-Length",
})
return
}

if err := s.Filesystem().Write(f, c.Request.Body, c.Request.ContentLength, 0o644); err != nil {
if filesystem.IsErrorCode(err, filesystem.ErrCodeIsDirectory) {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
"error": "Cannot write file, name conflicts with an existing directory by the same name.",
Expand Down Expand Up @@ -589,15 +596,9 @@ func postServerUploadFiles(c *gin.Context) {
}

for _, header := range headers {
p, err := s.Filesystem().SafePath(filepath.Join(directory, header.Filename))
if err != nil {
middleware.CaptureAndAbort(c, err)
return
}

// We run this in a different method so I can use defer without any of
// the consequences caused by calling it in a loop.
if err := handleFileUpload(p, s, header); err != nil {
if err := handleFileUpload(filepath.Join(directory, header.Filename), s, header); err != nil {
middleware.CaptureAndAbort(c, err)
return
} else {
Expand All @@ -619,7 +620,8 @@ func handleFileUpload(p string, s *server.Server, header *multipart.FileHeader)
if err := s.Filesystem().IsIgnored(p); err != nil {
return err
}
if err := s.Filesystem().Writefile(p, file); err != nil {

if err := s.Filesystem().Write(p, file, header.Size, 0o644); err != nil {
return err
}
return nil
Expand Down
1 change: 1 addition & 0 deletions router/router_transfer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ func postTransfers(c *gin.Context) {
if !successful && err != nil {
// Delete all extracted files.
go func(trnsfr *transfer.Transfer) {
_ = trnsfr.Server.Filesystem().UnixFS().Close()
if err := os.RemoveAll(trnsfr.Server.Filesystem().Path()); err != nil && !os.IsNotExist(err) {
trnsfr.Log().WithError(err).Warn("failed to delete local server files")
}
Expand Down
15 changes: 6 additions & 9 deletions server/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func (s *Server) Backup(b backup.BackupInterface) error {
}
}

ad, err := b.Generate(s.Context(), s.Filesystem().Path(), ignored)
ad, err := b.Generate(s.Context(), s.Filesystem(), ignored)
if err != nil {
if err := s.notifyPanelOfBackup(b.Identifier(), &backup.ArchiveDetails{}, false); err != nil {
s.Log().WithFields(log.Fields{
Expand Down Expand Up @@ -154,17 +154,14 @@ func (s *Server) RestoreBackup(b backup.BackupInterface, reader io.ReadCloser) (
err = b.Restore(s.Context(), reader, func(file string, info fs.FileInfo, r io.ReadCloser) error {
defer r.Close()
s.Events().Publish(DaemonMessageEvent, "(restoring): "+file)

if err := s.Filesystem().Writefile(file, r); err != nil {
return err
}
if err := s.Filesystem().Chmod(file, info.Mode()); err != nil {
// TODO: since this will be called a lot, it may be worth adding an optimized
// Write with Chtimes method to the UnixFS that is able to re-use the
// same dirfd and file name.
if err := s.Filesystem().Write(file, r, info.Size(), info.Mode()); err != nil {
return err
}

atime := info.ModTime()
mtime := atime
return s.Filesystem().Chtimes(file, atime, mtime)
return s.Filesystem().Chtimes(file, atime, atime)
})

return errors.WithStackIf(err)
Expand Down
3 changes: 2 additions & 1 deletion server/backup/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (

"github.com/pterodactyl/wings/config"
"github.com/pterodactyl/wings/remote"
"github.com/pterodactyl/wings/server/filesystem"
)

var format = archiver.CompressedArchive{
Expand Down Expand Up @@ -46,7 +47,7 @@ type BackupInterface interface {
WithLogContext(map[string]interface{})
// Generate creates a backup in whatever the configured source for the
// specific implementation is.
Generate(context.Context, string, string) (*ArchiveDetails, error)
Generate(context.Context, *filesystem.Filesystem, string) (*ArchiveDetails, error)
// Ignored returns the ignored files for this backup instance.
Ignored() string
// Checksum returns a SHA1 checksum for the generated backup.
Expand Down
6 changes: 3 additions & 3 deletions server/backup/backup_local.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,10 @@ func (b *LocalBackup) WithLogContext(c map[string]interface{}) {

// Generate generates a backup of the selected files and pushes it to the
// defined location for this instance.
func (b *LocalBackup) Generate(ctx context.Context, basePath, ignore string) (*ArchiveDetails, error) {
func (b *LocalBackup) Generate(ctx context.Context, fsys *filesystem.Filesystem, ignore string) (*ArchiveDetails, error) {
a := &filesystem.Archive{
BasePath: basePath,
Ignore: ignore,
Filesystem: fsys,
Ignore: ignore,
}

b.log().WithField("path", b.Path()).Info("creating backup for server")
Expand Down
6 changes: 3 additions & 3 deletions server/backup/backup_s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,12 @@ func (s *S3Backup) WithLogContext(c map[string]interface{}) {

// Generate creates a new backup on the disk, moves it into the S3 bucket via
// the provided presigned URL, and then deletes the backup from the disk.
func (s *S3Backup) Generate(ctx context.Context, basePath, ignore string) (*ArchiveDetails, error) {
func (s *S3Backup) Generate(ctx context.Context, fsys *filesystem.Filesystem, ignore string) (*ArchiveDetails, error) {
defer s.Remove()

a := &filesystem.Archive{
BasePath: basePath,
Ignore: ignore,
Filesystem: fsys,
Ignore: ignore,
}

s.log().WithField("path", s.Path()).Info("creating backup for server")
Expand Down
14 changes: 8 additions & 6 deletions server/config_parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ import (
"runtime"

"github.com/gammazero/workerpool"

"github.com/pterodactyl/wings/internal/ufs"
)

// UpdateConfigurationFiles updates all of the defined configuration files for
// UpdateConfigurationFiles updates all the defined configuration files for
// a server automatically to ensure that they always use the specified values.
func (s *Server) UpdateConfigurationFiles() {
pool := workerpool.New(runtime.NumCPU())
Expand All @@ -18,18 +20,18 @@ func (s *Server) UpdateConfigurationFiles() {
f := cf

pool.Submit(func() {
p, err := s.Filesystem().SafePath(f.FileName)
file, err := s.Filesystem().UnixFS().Touch(f.FileName, ufs.O_RDWR|ufs.O_CREATE, 0o644)
if err != nil {
s.Log().WithField("error", err).Error("failed to generate safe path for configuration file")

s.Log().WithField("file_name", f.FileName).WithField("error", err).Error("failed to open file for configuration")
return
}
defer file.Close()

if err := f.Parse(p, false); err != nil {
if err := f.Parse(file); err != nil {
s.Log().WithField("error", err).Error("failed to parse and update server configuration file")
}

s.Log().WithField("path", f.FileName).Debug("finished processing server configuration file")
s.Log().WithField("file_name", f.FileName).Debug("finished processing server configuration file")
})
}

Expand Down
121 changes: 63 additions & 58 deletions server/filesystem/archive.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package filesystem
import (
"archive/tar"
"context"
"fmt"
"io"
"io/fs"
"os"
Expand All @@ -14,12 +13,12 @@ import (
"emperror.dev/errors"
"github.com/apex/log"
"github.com/juju/ratelimit"
"github.com/karrick/godirwalk"
"github.com/klauspost/pgzip"
ignore "github.com/sabhiram/go-gitignore"

"github.com/pterodactyl/wings/config"
"github.com/pterodactyl/wings/internal/progress"
"github.com/pterodactyl/wings/internal/ufs"
)

const memory = 4 * 1024
Expand Down Expand Up @@ -57,14 +56,16 @@ func (p *TarProgress) Write(v []byte) (int, error) {
}

type Archive struct {
// BasePath is the absolute path to create the archive from where Files and Ignore are
// relative to.
BasePath string
// Filesystem to create the archive with.
Filesystem *Filesystem

// Ignore is a gitignore string (most likely read from a file) of files to ignore
// from the archive.
Ignore string

// BaseDirectory .
BaseDirectory string

// Files specifies the files to archive, this takes priority over the Ignore option, if
// unspecified, all files in the BasePath will be archived unless Ignore is set.
//
Expand All @@ -73,11 +74,18 @@ type Archive struct {

// Progress wraps the writer of the archive to pass through the progress tracker.
Progress *progress.Progress

w *TarProgress
}

// Create creates an archive at dst with all the files defined in the
// included Files array.
//
// THIS IS UNSAFE TO USE IF `dst` IS PROVIDED BY A USER! ONLY USE THIS WITH
// CONTROLLED PATHS!
func (a *Archive) Create(ctx context.Context, dst string) error {
// Using os.OpenFile here is expected, as long as `dst` is not a user
// provided path.
f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
if err != nil {
return err
Expand All @@ -98,14 +106,19 @@ func (a *Archive) Create(ctx context.Context, dst string) error {
return a.Stream(ctx, writer)
}

// Stream .
type walkFunc func(dirfd int, name, relative string, d ufs.DirEntry) error

// Stream streams the creation of the archive to the given writer.
func (a *Archive) Stream(ctx context.Context, w io.Writer) error {
for _, f := range a.Files {
if strings.HasPrefix(f, a.BasePath) {
if a.Filesystem == nil {
return errors.New("filesystem: archive.Filesystem is unset")
}

for i, f := range a.Files {
if !strings.HasPrefix(f, a.Filesystem.Path()) {
continue
}

return fmt.Errorf("archive: all entries in Files must be absolute and within BasePath: %s\n", f)
a.Files[i] = strings.TrimPrefix(strings.TrimPrefix(f, a.Filesystem.Path()), "/")
}

// Choose which compression level to use based on the compression_level configuration option
Expand All @@ -130,107 +143,100 @@ func (a *Archive) Stream(ctx context.Context, w io.Writer) error {
tw := tar.NewWriter(gw)
defer tw.Close()

pw := NewTarProgress(tw, a.Progress)
a.w = NewTarProgress(tw, a.Progress)

// Configure godirwalk.
options := &godirwalk.Options{
FollowSymbolicLinks: false,
Unsorted: true,
}
fs := a.Filesystem.unixFS

// If we're specifically looking for only certain files, or have requested
// that certain files be ignored we'll update the callback function to reflect
// that request.
var callback godirwalk.WalkFunc
var callback walkFunc
if len(a.Files) == 0 && len(a.Ignore) > 0 {
i := ignore.CompileIgnoreLines(strings.Split(a.Ignore, "\n")...)

callback = a.callback(pw, func(_ string, rp string) error {
if i.MatchesPath(rp) {
return godirwalk.SkipThis
callback = a.callback(func(_ int, _, relative string, _ ufs.DirEntry) error {
if i.MatchesPath(relative) {
return ufs.SkipDir
}

return nil
})
} else if len(a.Files) > 0 {
callback = a.withFilesCallback(pw)
callback = a.withFilesCallback()
} else {
callback = a.callback(pw)
callback = a.callback()
}

// Set the callback function, wrapped with support for context cancellation.
options.Callback = func(path string, de *godirwalk.Dirent) error {
dirfd, name, closeFd, err := fs.SafePath(a.BaseDirectory)
defer closeFd()
if err != nil {
return err
}
return fs.WalkDirat(dirfd, name, func(dirfd int, name, relative string, d ufs.DirEntry, err error) error {
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
return callback(path, de)
return callback(dirfd, name, relative, d)
}
}

// Recursively walk the path we are archiving.
return godirwalk.Walk(a.BasePath, options)
})
}

// Callback function used to determine if a given file should be included in the archive
// being generated.
func (a *Archive) callback(tw *TarProgress, opts ...func(path string, relative string) error) func(path string, de *godirwalk.Dirent) error {
return func(path string, de *godirwalk.Dirent) error {
func (a *Archive) callback(opts ...walkFunc) walkFunc {
return func(dirfd int, name, relative string, d ufs.DirEntry) error {
// Skip directories because we are walking them recursively.
if de.IsDir() {
if d.IsDir() {
return nil
}

relative := filepath.ToSlash(strings.TrimPrefix(path, a.BasePath+string(filepath.Separator)))

// Call the additional options passed to this callback function. If any of them return
// a non-nil error we will exit immediately.
for _, opt := range opts {
if err := opt(path, relative); err != nil {
if err := opt(dirfd, name, relative, d); err != nil {
return err
}
}

// Add the file to the archive, if it is nested in a directory,
// the directory will be automatically "created" in the archive.
return a.addToArchive(path, relative, tw)
return a.addToArchive(dirfd, name, relative, d)
}
}

// Pushes only files defined in the Files key to the final archive.
func (a *Archive) withFilesCallback(tw *TarProgress) func(path string, de *godirwalk.Dirent) error {
return a.callback(tw, func(p string, rp string) error {
func (a *Archive) withFilesCallback() walkFunc {
return a.callback(func(_ int, _, relative string, _ ufs.DirEntry) error {
for _, f := range a.Files {
// Allow exact file matches, otherwise check if file is within a parent directory.
//
// The slashes are added in the prefix checks to prevent partial name matches from being
// included in the archive.
if f != p && !strings.HasPrefix(strings.TrimSuffix(p, "/")+"/", strings.TrimSuffix(f, "/")+"/") {
if f != relative && !strings.HasPrefix(strings.TrimSuffix(relative, "/")+"/", strings.TrimSuffix(f, "/")+"/") {
continue
}

// Once we have a match return a nil value here so that the loop stops and the
// call to this function will correctly include the file in the archive. If there
// are no matches we'll never make it to this line, and the final error returned
// will be the godirwalk.SkipThis error.
// will be the ufs.SkipDir error.
return nil
}

return godirwalk.SkipThis
return ufs.SkipDir
})
}

// Adds a given file path to the final archive being created.
func (a *Archive) addToArchive(p string, rp string, w *TarProgress) error {
// Lstat the file, this will give us the same information as Stat except that it will not
// follow a symlink to its target automatically. This is important to avoid including
// files that exist outside the server root unintentionally in the backup.
s, err := os.Lstat(p)
func (a *Archive) addToArchive(dirfd int, name, relative string, entry ufs.DirEntry) error {
s, err := entry.Info()
if err != nil {
if os.IsNotExist(err) {
if errors.Is(err, ufs.ErrNotExist) {
return nil
}
return errors.WrapIff(err, "failed executing os.Lstat on '%s'", rp)
return errors.WrapIff(err, "failed executing os.Lstat on '%s'", name)
}

// Skip socket files as they are unsupported by archive/tar.
Expand All @@ -250,7 +256,7 @@ func (a *Archive) addToArchive(p string, rp string, w *TarProgress) error {
if err != nil {
// Ignore the not exist errors specifically, since there is nothing important about that.
if !os.IsNotExist(err) {
log.WithField("path", rp).WithField("readlink_err", err.Error()).Warn("failed reading symlink for target path; skipping...")
log.WithField("name", name).WithField("readlink_err", err.Error()).Warn("failed reading symlink for target path; skipping...")
}
return nil
}
Expand All @@ -259,17 +265,17 @@ func (a *Archive) addToArchive(p string, rp string, w *TarProgress) error {
// Get the tar FileInfoHeader in order to add the file to the archive.
header, err := tar.FileInfoHeader(s, filepath.ToSlash(target))
if err != nil {
return errors.WrapIff(err, "failed to get tar#FileInfoHeader for '%s'", rp)
return errors.WrapIff(err, "failed to get tar#FileInfoHeader for '%s'", name)
}

// Fix the header name if the file is not a symlink.
if s.Mode()&fs.ModeSymlink == 0 {
header.Name = rp
header.Name = relative
}

// Write the tar FileInfoHeader to the archive.
if err := w.WriteHeader(header); err != nil {
return errors.WrapIff(err, "failed to write tar#FileInfoHeader for '%s'", rp)
if err := a.w.WriteHeader(header); err != nil {
return errors.WrapIff(err, "failed to write tar#FileInfoHeader for '%s'", name)
}

// If the size of the file is less than 1 (most likely for symlinks), skip writing the file.
Expand All @@ -291,7 +297,7 @@ func (a *Archive) addToArchive(p string, rp string, w *TarProgress) error {
}

// Open the file.
f, err := os.Open(p)
f, err := a.Filesystem.unixFS.OpenFileat(dirfd, name, ufs.O_RDONLY, 0)
if err != nil {
if os.IsNotExist(err) {
return nil
Expand All @@ -301,9 +307,8 @@ func (a *Archive) addToArchive(p string, rp string, w *TarProgress) error {
defer f.Close()

// Copy the file's contents to the archive using our buffer.
if _, err := io.CopyBuffer(w, io.LimitReader(f, header.Size), buf); err != nil {
if _, err := io.CopyBuffer(a.w, io.LimitReader(f, header.Size), buf); err != nil {
return errors.WrapIff(err, "failed to copy '%s' to archive", header.Name)
}

return nil
}
35 changes: 13 additions & 22 deletions server/filesystem/archive_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,43 +20,34 @@ func TestArchive_Stream(t *testing.T) {
g.Describe("Archive", func() {
g.AfterEach(func() {
// Reset the filesystem after each run.
rfs.reset()
})

g.It("throws an error when passed invalid file paths", func() {
a := &Archive{
BasePath: fs.Path(),
Files: []string{
// To use the archiver properly, this needs to be filepath.Join(BasePath, "yeet")
// However, this test tests that we actually validate that behavior.
"yeet",
},
}

g.Assert(a.Create(context.Background(), "")).IsNotNil()
_ = fs.TruncateRootDirectory()
})

g.It("creates archive with intended files", func() {
g.Assert(fs.CreateDirectory("test", "/")).IsNil()
g.Assert(fs.CreateDirectory("test2", "/")).IsNil()

err := fs.Writefile("test/file.txt", strings.NewReader("hello, world!\n"))
r := strings.NewReader("hello, world!\n")
err := fs.Write("test/file.txt", r, r.Size(), 0o644)
g.Assert(err).IsNil()

err = fs.Writefile("test2/file.txt", strings.NewReader("hello, world!\n"))
r = strings.NewReader("hello, world!\n")
err = fs.Write("test2/file.txt", r, r.Size(), 0o644)
g.Assert(err).IsNil()

err = fs.Writefile("test_file.txt", strings.NewReader("hello, world!\n"))
r = strings.NewReader("hello, world!\n")
err = fs.Write("test_file.txt", r, r.Size(), 0o644)
g.Assert(err).IsNil()

err = fs.Writefile("test_file.txt.old", strings.NewReader("hello, world!\n"))
r = strings.NewReader("hello, world!\n")
err = fs.Write("test_file.txt.old", r, r.Size(), 0o644)
g.Assert(err).IsNil()

a := &Archive{
BasePath: fs.Path(),
Filesystem: fs,
Files: []string{
filepath.Join(fs.Path(), "test"),
filepath.Join(fs.Path(), "test_file.txt"),
"test",
"test_file.txt",
},
}

Expand Down Expand Up @@ -119,7 +110,7 @@ func getFiles(f iofs.ReadDirFS, name string) ([]string, error) {
if files == nil {
return nil, nil
}

v = append(v, files...)
continue
}
Expand Down
Loading