From d4e49d7e78624aae0ecd0d93f7de1c909386dbd0 Mon Sep 17 00:00:00 2001 From: Charlie Vieth Date: Thu, 24 Feb 2022 13:19:48 -0500 Subject: [PATCH] internal/fastwalk: improve Darwin performance by ~3x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This changes fastwalk to use readdir_r() instead of syscall.ReadDirent() on Darwin. This improves performance by ~3x and matches the behavior of os.readDir(). The ReadDirent function is slow on Darwin because it uses syscall.Getdirentries(), which due to golang/go#30933 is now simulated on Darwin. To support using readdir_r we this change adds the following syscalls: closedir, readdir_r, and opendir. goos: darwin goarch: arm64 name old time/op new time/op delta FastWalk-10 29.1ms ± 1% 11.2ms ± 1% -61.53% (p=0.000 n=20+17) name old alloc/op new alloc/op delta FastWalk-10 827kB ± 0% 823kB ± 0% -0.48% (p=0.000 n=20+19) name old allocs/op new allocs/op delta FastWalk-10 19.1k ± 0% 17.6k ± 0% -8.10% (p=0.000 n=18+19) --- internal/fastwalk/fastwalk_darwin.go | 96 +++++++++++++++++++ internal/fastwalk/fastwalk_dirent_ino.go | 5 +- .../fastwalk/fastwalk_dirent_namlen_bsd.go | 4 +- internal/fastwalk/fastwalk_unix.go | 4 +- internal/fastwalk/zsyscall_darwin.go | 56 +++++++++++ .../fastwalk/zsyscall_darwin_amd64.1_13.s | 28 ++++++ .../fastwalk/zsyscall_darwin_arm64.1_13.s | 28 ++++++ 7 files changed, 214 insertions(+), 7 deletions(-) create mode 100644 internal/fastwalk/fastwalk_darwin.go create mode 100644 internal/fastwalk/zsyscall_darwin.go create mode 100644 internal/fastwalk/zsyscall_darwin_amd64.1_13.s create mode 100644 internal/fastwalk/zsyscall_darwin_arm64.1_13.s diff --git a/internal/fastwalk/fastwalk_darwin.go b/internal/fastwalk/fastwalk_darwin.go new file mode 100644 index 00000000000..582037588e9 --- /dev/null +++ b/internal/fastwalk/fastwalk_darwin.go @@ -0,0 +1,96 @@ +//go:build darwin +// +build darwin + +package fastwalk + +import ( + "os" + "syscall" + "unsafe" +) + +func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error { + fd, err := openDir(dirName) + if err != nil { + return &os.PathError{Op: "opendir", Path: dirName, Err: err} + } + defer closedir(fd) + + skipFiles := false + var dirent syscall.Dirent + var entptr *syscall.Dirent + for { + if errno := readdir_r(uintptr(fd), &dirent, &entptr); errno != 0 { + if errno == syscall.EINTR { + continue + } + return &os.PathError{Op: "readdir", Path: dirName, Err: errno} + } + if entptr == nil { // EOF + break + } + if dirent.Ino == 0 { + continue + } + typ := dtToType(dirent.Type) + if skipFiles && typ.IsRegular() { + continue + } + name := (*[len(syscall.Dirent{}.Name)]byte)(unsafe.Pointer(&dirent.Name))[:] + for i, c := range name { + if c == 0 { + name = name[:i] + break + } + } + // Check for useless names before allocating a string. + if string(name) == "." || string(name) == ".." { + continue + } + nm := string(name) + if err := fn(dirName, nm, typ); err != nil { + if err != ErrSkipFiles { + return err + } + skipFiles = true + } + } + + return nil +} + +func dtToType(typ uint8) os.FileMode { + switch typ { + case syscall.DT_BLK: + return os.ModeDevice + case syscall.DT_CHR: + return os.ModeDevice | os.ModeCharDevice + case syscall.DT_DIR: + return os.ModeDir + case syscall.DT_FIFO: + return os.ModeNamedPipe + case syscall.DT_LNK: + return os.ModeSymlink + case syscall.DT_REG: + return 0 + case syscall.DT_SOCK: + return os.ModeSocket + } + return ^os.FileMode(0) +} + +// According to https://golang.org/doc/go1.14#runtime +// A consequence of the implementation of preemption is that on Unix systems, +// including Linux and macOS systems, programs built with Go 1.14 will receive +// more signals than programs built with earlier releases. +// +// This causes opendir to sometimes fail with EINTR errors. +// We need to retry in this case. +func openDir(path string) (fd uintptr, err error) { + for { + fd, err := opendir(path) + if err != syscall.EINTR { + return fd, err + } + } +} diff --git a/internal/fastwalk/fastwalk_dirent_ino.go b/internal/fastwalk/fastwalk_dirent_ino.go index ea02b9ebfe8..61b159a8ab7 100644 --- a/internal/fastwalk/fastwalk_dirent_ino.go +++ b/internal/fastwalk/fastwalk_dirent_ino.go @@ -2,9 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (linux || darwin) && !appengine -// +build linux darwin -// +build !appengine +//go:build linux && !appengine +// +build linux,!appengine package fastwalk diff --git a/internal/fastwalk/fastwalk_dirent_namlen_bsd.go b/internal/fastwalk/fastwalk_dirent_namlen_bsd.go index d5c9c321ed2..08184a17d8f 100644 --- a/internal/fastwalk/fastwalk_dirent_namlen_bsd.go +++ b/internal/fastwalk/fastwalk_dirent_namlen_bsd.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build darwin || freebsd || openbsd || netbsd -// +build darwin freebsd openbsd netbsd +//go:build freebsd || openbsd || netbsd +// +build freebsd openbsd netbsd package fastwalk diff --git a/internal/fastwalk/fastwalk_unix.go b/internal/fastwalk/fastwalk_unix.go index 58bd87841e1..7629448c8e4 100644 --- a/internal/fastwalk/fastwalk_unix.go +++ b/internal/fastwalk/fastwalk_unix.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (linux || darwin || freebsd || openbsd || netbsd) && !appengine -// +build linux darwin freebsd openbsd netbsd +//go:build (linux || freebsd || openbsd || netbsd) && !appengine +// +build linux freebsd openbsd netbsd // +build !appengine package fastwalk diff --git a/internal/fastwalk/zsyscall_darwin.go b/internal/fastwalk/zsyscall_darwin.go new file mode 100644 index 00000000000..ea8562a207e --- /dev/null +++ b/internal/fastwalk/zsyscall_darwin.go @@ -0,0 +1,56 @@ +//go:build darwin && go1.13 +// +build darwin,go1.13 + +package fastwalk + +import ( + "syscall" + "unsafe" +) + +// Implemented in the runtime package (runtime/sys_darwin.go) +func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) +func syscall_syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) + +//go:linkname syscall_syscall syscall.syscall +//go:linkname syscall_syscallPtr syscall.syscallPtr + +func closedir(dir uintptr) (err error) { + _, _, e1 := syscall_syscall(libc_closedir_trampoline_addr, uintptr(dir), 0, 0) + if e1 != 0 { + err = e1 + } + return +} + +var libc_closedir_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_closedir closedir "/usr/lib/libSystem.B.dylib" + +func readdir_r(dir uintptr, entry *syscall.Dirent, result **syscall.Dirent) syscall.Errno { + res, _, _ := syscall_syscall(libc_readdir_r_trampoline_addr, uintptr(dir), uintptr(unsafe.Pointer(entry)), uintptr(unsafe.Pointer(result))) + return syscall.Errno(res) +} + +var libc_readdir_r_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_readdir_r readdir_r "/usr/lib/libSystem.B.dylib" + +func opendir(path string) (dir uintptr, err error) { + // We implent opendir so that we don't have to open a file, duplicate + // it's FD, then call fdopendir with it. + p, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + r0, _, e1 := syscall_syscallPtr(libc_opendir_trampoline_addr, uintptr(unsafe.Pointer(p)), 0, 0) + dir = uintptr(r0) + if e1 != 0 { + err = e1 + } + return dir, err +} + +var libc_opendir_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_opendir opendir "/usr/lib/libSystem.B.dylib" diff --git a/internal/fastwalk/zsyscall_darwin_amd64.1_13.s b/internal/fastwalk/zsyscall_darwin_amd64.1_13.s new file mode 100644 index 00000000000..ccf2ff6f8c5 --- /dev/null +++ b/internal/fastwalk/zsyscall_darwin_amd64.1_13.s @@ -0,0 +1,28 @@ +//go:build go1.13 +// +build go1.13 + +#include "textflag.h" + +TEXT libc_closedir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_closedir(SB) + +GLOBL ·libc_closedir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_closedir_trampoline_addr(SB)/8, $libc_closedir_trampoline<>(SB) + +TEXT libc_readdir_r_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readdir_r(SB) + +GLOBL ·libc_readdir_r_trampoline_addr(SB), RODATA, $8 +DATA ·libc_readdir_r_trampoline_addr(SB)/8, $libc_readdir_r_trampoline<>(SB) + +TEXT libc_opendir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_opendir(SB) + +GLOBL ·libc_opendir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_opendir_trampoline_addr(SB)/8, $libc_opendir_trampoline<>(SB) + +TEXT libc___getdirentries64_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc___getdirentries64(SB) + +GLOBL ·libc___getdirentries64_trampoline_addr(SB), RODATA, $8 +DATA ·libc___getdirentries64_trampoline_addr(SB)/8, $libc___getdirentries64_trampoline<>(SB) diff --git a/internal/fastwalk/zsyscall_darwin_arm64.1_13.s b/internal/fastwalk/zsyscall_darwin_arm64.1_13.s new file mode 100644 index 00000000000..ccf2ff6f8c5 --- /dev/null +++ b/internal/fastwalk/zsyscall_darwin_arm64.1_13.s @@ -0,0 +1,28 @@ +//go:build go1.13 +// +build go1.13 + +#include "textflag.h" + +TEXT libc_closedir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_closedir(SB) + +GLOBL ·libc_closedir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_closedir_trampoline_addr(SB)/8, $libc_closedir_trampoline<>(SB) + +TEXT libc_readdir_r_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readdir_r(SB) + +GLOBL ·libc_readdir_r_trampoline_addr(SB), RODATA, $8 +DATA ·libc_readdir_r_trampoline_addr(SB)/8, $libc_readdir_r_trampoline<>(SB) + +TEXT libc_opendir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_opendir(SB) + +GLOBL ·libc_opendir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_opendir_trampoline_addr(SB)/8, $libc_opendir_trampoline<>(SB) + +TEXT libc___getdirentries64_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc___getdirentries64(SB) + +GLOBL ·libc___getdirentries64_trampoline_addr(SB), RODATA, $8 +DATA ·libc___getdirentries64_trampoline_addr(SB)/8, $libc___getdirentries64_trampoline<>(SB)