From 61caf7fcdc987a87c2e11f65a2df490c4b18b639 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 31 May 2021 23:19:07 +0200 Subject: [PATCH] More detailed StatStruct show (#39463) Co-authored-by: Jameson Nash --- base/filesystem.jl | 16 ++++ base/libc.jl | 73 +++++++++++++++ base/reflection.jl | 27 +++--- base/stat.jl | 154 +++++++++++++++++++++++++++++- src/sys.c | 226 +++++++++++++++++++++++++++++++++++++++++++++ test/file.jl | 31 ++++++- 6 files changed, 508 insertions(+), 19 deletions(-) diff --git a/base/filesystem.jl b/base/filesystem.jl index 191f6537d8041..569b71995688d 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -4,6 +4,22 @@ module Filesystem +const S_IFDIR = 0o040000 # directory +const S_IFCHR = 0o020000 # character device +const S_IFBLK = 0o060000 # block device +const S_IFREG = 0o100000 # regular file +const S_IFIFO = 0o010000 # fifo (named pipe) +const S_IFLNK = 0o120000 # symbolic link +const S_IFSOCK = 0o140000 # socket file +const S_IFMT = 0o170000 + +const S_ISUID = 0o4000 # set UID bit +const S_ISGID = 0o2000 # set GID bit +const S_ENFMT = S_ISGID # file locking enforcement +const S_ISVTX = 0o1000 # sticky bit +const S_IRWXU = 0o0700 # mask for owner permissions +const S_IRUSR = 0o0400 # read by owner + const S_IRUSR = 0o400 const S_IWUSR = 0o200 const S_IXUSR = 0o100 diff --git a/base/libc.jl b/base/libc.jl index cb0f05f1dec7b..98d2910917ee4 100644 --- a/base/libc.jl +++ b/base/libc.jl @@ -402,6 +402,79 @@ Interface to the C `srand(seed)` function. """ srand(seed=floor(Int, time()) % Cuint) = ccall(:srand, Cvoid, (Cuint,), seed) +struct Cpasswd + username::Cstring + uid::Clong + gid::Clong + shell::Cstring + homedir::Cstring + gecos::Cstring + Cpasswd() = new(C_NULL, -1, -1, C_NULL, C_NULL, C_NULL) +end +mutable struct Cgroup + groupname::Cstring # group name + gid::Clong # group ID + mem::Ptr{Cstring} # group members + Cgroup() = new(C_NULL, -1, C_NULL) +end +struct Passwd + username::String + uid::Int + gid::Int + shell::String + homedir::String + gecos::String +end +struct Group + groupname::String + gid::Int + mem::Vector{String} +end + +function getpwuid(uid::Unsigned, throw_error::Bool=true) + ref_pd = Ref(Cpasswd()) + ret = ccall(:jl_os_get_passwd, Cint, (Ref{Cpasswd}, UInt), ref_pd, uid) + if ret != 0 + throw_error && Base.uv_error("getpwuid", ret) + return + end + pd = ref_pd[] + pd = Passwd( + pd.username == C_NULL ? "" : unsafe_string(pd.username), + pd.uid, + pd.gid, + pd.shell == C_NULL ? "" : unsafe_string(pd.shell), + pd.homedir == C_NULL ? "" : unsafe_string(pd.homedir), + pd.gecos == C_NULL ? "" : unsafe_string(pd.gecos), + ) + ccall(:uv_os_free_passwd, Cvoid, (Ref{Cpasswd},), ref_pd) + return pd +end +function getgrgid(gid::Unsigned, throw_error::Bool=true) + ref_gp = Ref(Cgroup()) + ret = ccall(:jl_os_get_group, Cint, (Ref{Cgroup}, UInt), ref_gp, gid) + if ret != 0 + throw_error && Base.uv_error("getgrgid", ret) + return + end + gp = ref_gp[] + members = String[] + if gp.mem != C_NULL + while true + mem = unsafe_load(gp.mem, length(members) + 1) + mem == C_NULL && break + push!(members, unsafe_string(mem)) + end + end + gp = Group( + gp.groupname == C_NULL ? "" : unsafe_string(gp.groupname), + gp.gid, + members, + ) + ccall(:jl_os_free_group, Cvoid, (Ref{Cgroup},), ref_gp) + return gp +end + # Include dlopen()/dlpath() code include("libdl.jl") using .Libdl diff --git a/base/reflection.jl b/base/reflection.jl index 33327b64a6a2a..8417efcb7171c 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -654,19 +654,20 @@ use it in the following manner to summarize information about a struct: julia> structinfo(T) = [(fieldoffset(T,i), fieldname(T,i), fieldtype(T,i)) for i = 1:fieldcount(T)]; julia> structinfo(Base.Filesystem.StatStruct) -12-element Vector{Tuple{UInt64, Symbol, DataType}}: - (0x0000000000000000, :device, UInt64) - (0x0000000000000008, :inode, UInt64) - (0x0000000000000010, :mode, UInt64) - (0x0000000000000018, :nlink, Int64) - (0x0000000000000020, :uid, UInt64) - (0x0000000000000028, :gid, UInt64) - (0x0000000000000030, :rdev, UInt64) - (0x0000000000000038, :size, Int64) - (0x0000000000000040, :blksize, Int64) - (0x0000000000000048, :blocks, Int64) - (0x0000000000000050, :mtime, Float64) - (0x0000000000000058, :ctime, Float64) +13-element Vector{Tuple{UInt64, Symbol, Type}}: + (0x0000000000000000, :desc, Union{RawFD, String}) + (0x0000000000000008, :device, UInt64) + (0x0000000000000010, :inode, UInt64) + (0x0000000000000018, :mode, UInt64) + (0x0000000000000020, :nlink, Int64) + (0x0000000000000028, :uid, UInt64) + (0x0000000000000030, :gid, UInt64) + (0x0000000000000038, :rdev, UInt64) + (0x0000000000000040, :size, Int64) + (0x0000000000000048, :blksize, Int64) + (0x0000000000000050, :blocks, Int64) + (0x0000000000000058, :mtime, Float64) + (0x0000000000000060, :ctime, Float64) ``` """ fieldoffset(x::DataType, idx::Integer) = (@_pure_meta; ccall(:jl_get_field_offset, Csize_t, (Any, Cint), x, idx)) diff --git a/base/stat.jl b/base/stat.jl index e6adda11fef6f..dc7ab9243d173 100644 --- a/base/stat.jl +++ b/base/stat.jl @@ -26,6 +26,7 @@ export uperm struct StatStruct + desc :: Union{String, OS_HANDLE} # for show method, not included in equality or hash device :: UInt inode :: UInt mode :: UInt @@ -39,10 +40,26 @@ struct StatStruct mtime :: Float64 ctime :: Float64 end +const StatFieldTypes = Union{UInt,Int,Int64,Float64} -StatStruct() = StatStruct(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) +function Base.:(==)(x::StatStruct, y::StatStruct) # do not include `desc` in equality or hash + for i = 2:nfields(x) + xi = getfield(x, i)::StatFieldTypes + xi === getfield(y, i) || return false + end + return true +end +function Base.hash(obj::StatStruct, h::UInt) + for i = 2:nfields(obj) + h = hash(getfield(obj, i)::StatFieldTypes, h) + end + return h +end -StatStruct(buf::Union{Vector{UInt8},Ptr{UInt8}}) = StatStruct( +StatStruct() = StatStruct("", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) +StatStruct(buf::Union{Vector{UInt8},Ptr{UInt8}}) = StatStruct("", buf) +StatStruct(desc::Union{AbstractString, OS_HANDLE}, buf::Union{Vector{UInt8},Ptr{UInt8}}) = StatStruct( + desc isa OS_HANDLE ? desc : String(desc), ccall(:jl_stat_dev, UInt32, (Ptr{UInt8},), buf), ccall(:jl_stat_ino, UInt32, (Ptr{UInt8},), buf), ccall(:jl_stat_mode, UInt32, (Ptr{UInt8},), buf), @@ -57,7 +74,72 @@ StatStruct(buf::Union{Vector{UInt8},Ptr{UInt8}}) = StatStruct( ccall(:jl_stat_ctime, Float64, (Ptr{UInt8},), buf), ) -show(io::IO, st::StatStruct) = print(io, "StatStruct(mode=0o$(string(filemode(st), base = 8, pad = 6)), size=$(filesize(st)))") +function iso_datetime_with_relative(t, tnow) + str = Libc.strftime("%FT%T%z", t) + secdiff = t - tnow + for (d, name) in ((24*60*60, "day"), (60*60, "hour"), (60, "minute"), (1, "second")) + tdiff = round(Int, div(abs(secdiff), d)) + if tdiff != 0 # find first unit difference + plural = tdiff == 1 ? "" : "s" + when = secdiff < 0 ? "ago" : "in the future" + return "$str ($tdiff $name$plural $when)" + end + end + return "$str (just now)" +end + + +function getusername(uid::Unsigned) + pwd = Libc.getpwuid(uid, false) + pwd === nothing && return + isempty(pwd.username) && return + return pwd.username +end + +function getgroupname(gid::Unsigned) + gp = Libc.getgrgid(gid, false) + gp === nothing && return + isempty(gp.groupname) && return + return gp.groupname +end + +function show_statstruct(io::IO, st::StatStruct, oneline::Bool) + print(io, oneline ? "StatStruct(" : "StatStruct for ") + show(io, st.desc) + oneline || print("\n ") + print(io, " size: ", st.size, " bytes") + oneline || print("\n") + print(io, " device: ", st.device) + oneline || print("\n ") + print(io, " inode: ", st.inode) + oneline || print("\n ") + print(io, " mode: 0o", string(filemode(st), base = 8, pad = 6), " (", filemode_string(st), ")") + oneline || print("\n ") + print(io, " nlink: ", st.nlink) + oneline || print("\n ") + print(io, " uid: $(st.uid)") + username = getusername(st.uid) + username === nothing || print(io, " (", username, ")") + oneline || print("\n ") + print(io, " gid: ", st.gid) + groupname = getgroupname(st.gid) + groupname === nothing || print(io, " (", groupname, ")") + oneline || print("\n ") + print(io, " rdev: ", st.rdev) + oneline || print("\n ") + print(io, " blksz: ", st.blksize) + oneline || print("\n") + print(io, " blocks: ", st.blocks) + tnow = round(UInt, time()) + oneline || print("\n ") + print(io, " mtime: ", iso_datetime_with_relative(st.mtime, tnow)) + oneline || print("\n ") + print(io, " ctime: ", iso_datetime_with_relative(st.ctime, tnow)) + print(io, oneline ? ")" : "\n") +end + +show(io::IO, st::StatStruct) = show_statstruct(io, st, true) +show(io::IO, ::MIME"text/plain", st::StatStruct) = show_statstruct(io, st, false) # stat & lstat functions @@ -66,9 +148,9 @@ macro stat_call(sym, arg1type, arg) stat_buf = zeros(UInt8, ccall(:jl_sizeof_stat, Int32, ())) r = ccall($(Expr(:quote, sym)), Int32, ($(esc(arg1type)), Ptr{UInt8}), $(esc(arg)), stat_buf) if !(r in (0, Base.UV_ENOENT, Base.UV_ENOTDIR, Base.UV_EINVAL)) - uv_error(string("stat(",repr($(esc(arg))),")"), r) + uv_error(string("stat(", repr($(esc(arg))), ")"), r) end - st = StatStruct(stat_buf) + st = StatStruct($(esc(arg)), stat_buf) if ispath(st) != (r == 0) error("stat returned zero type for a valid path") end @@ -92,6 +174,7 @@ The fields of the structure are: | Name | Description | |:--------|:-------------------------------------------------------------------| +| desc | The path or OS file descriptor | | size | The size (in bytes) of the file | | device | ID of the device that contains the file | | inode | The inode number of the file | @@ -120,12 +203,73 @@ lstat(path...) = lstat(joinpath(path...)) # some convenience functions +const filemode_table = ( + [ + (S_IFLNK, "l"), + (S_IFSOCK, "s"), # Must appear before IFREG and IFDIR as IFSOCK == IFREG | IFDIR + (S_IFREG, "-"), + (S_IFBLK, "b"), + (S_IFDIR, "d"), + (S_IFCHR, "c"), + (S_IFIFO, "p") + ], + [ + (S_IRUSR, "r"), + ], + [ + (S_IWUSR, "w"), + ], + [ + (S_IXUSR|S_ISUID, "s"), + (S_ISUID, "S"), + (S_IXUSR, "x") + ], + [ + (S_IRGRP, "r"), + ], + [ + (S_IWGRP, "w"), + ], + [ + (S_IXGRP|S_ISGID, "s"), + (S_ISGID, "S"), + (S_IXGRP, "x") + ], + [ + (S_IROTH, "r"), + ], + [ + (S_IWOTH, "w"), + ], + [ + (S_IXOTH|S_ISVTX, "t"), + (S_ISVTX, "T"), + (S_IXOTH, "x") + ] +) + """ filemode(file) Equivalent to `stat(file).mode`. """ filemode(st::StatStruct) = st.mode +filemode_string(st::StatStruct) = filemode_string(st.mode) +function filemode_string(mode) + str = IOBuffer() + for table in filemode_table + complete = true + for (bit, char) in table + if mode & bit == bit + write(str, char) + complete = false + break + end + end + complete && write(str, "-") + end + return String(take!(str)) +end """ filesize(path...) diff --git a/src/sys.c b/src/sys.c index 664489397977b..da5b7dabbc7f6 100644 --- a/src/sys.c +++ b/src/sys.c @@ -27,6 +27,7 @@ #include #include #include +#include #endif #ifndef _OS_WINDOWS_ @@ -231,6 +232,231 @@ JL_DLLEXPORT double jl_stat_ctime(char *statbuf) return (double)s->st_ctim.tv_sec + (double)s->st_ctim.tv_nsec * 1e-9; } +JL_DLLEXPORT int jl_os_get_passwd(uv_passwd_t *pwd, size_t uid) +{ +#ifdef _OS_WINDOWS_ + return UV_ENOTSUP; +#else + // taken directly from libuv + struct passwd pw; + struct passwd* result; + char* buf; + size_t bufsize; + size_t name_size; + size_t homedir_size; + size_t shell_size; + size_t gecos_size; + long initsize; + int r; + + if (pwd == NULL) + return UV_EINVAL; + + initsize = sysconf(_SC_GETPW_R_SIZE_MAX); + + if (initsize <= 0) + bufsize = 4096; + else + bufsize = (size_t) initsize; + + buf = NULL; + + for (;;) { + free(buf); + buf = (char*)malloc(bufsize); + + if (buf == NULL) + return UV_ENOMEM; + + r = getpwuid_r(uid, &pw, buf, bufsize, &result); + + if (r != ERANGE) + break; + + bufsize *= 2; + } + + if (r != 0) { + free(buf); + return -r; + } + + if (result == NULL) { + free(buf); + return UV_ENOENT; + } + + /* Allocate memory for the username, gecos, shell, and home directory. */ + name_size = strlen(pw.pw_name) + 1; + homedir_size = strlen(pw.pw_dir) + 1; + shell_size = strlen(pw.pw_shell) + 1; + +#ifdef __MVS__ + gecos_size = 0; /* pw_gecos does not exist on zOS. */ +#else + if (pw.pw_gecos != NULL) + gecos_size = strlen(pw.pw_gecos) + 1; + else + gecos_size = 0; +#endif + + pwd->username = (char*)malloc(name_size + + homedir_size + + shell_size + + gecos_size); + + if (pwd->username == NULL) { + free(buf); + return UV_ENOMEM; + } + + /* Copy the username */ + memcpy(pwd->username, pw.pw_name, name_size); + + /* Copy the home directory */ + pwd->homedir = pwd->username + name_size; + memcpy(pwd->homedir, pw.pw_dir, homedir_size); + + /* Copy the shell */ + pwd->shell = pwd->homedir + homedir_size; + memcpy(pwd->shell, pw.pw_shell, shell_size); + + /* Copy the gecos field */ +#ifdef __MVS__ + pwd->gecos = NULL; /* pw_gecos does not exist on zOS. */ +#else + if (pw.pw_gecos == NULL) { + pwd->gecos = NULL; + } else { + pwd->gecos = pwd->shell + shell_size; + memcpy(pwd->gecos, pw.pw_gecos, gecos_size); + } +#endif + + /* Copy the uid and gid */ + pwd->uid = pw.pw_uid; + pwd->gid = pw.pw_gid; + + free(buf); + + return 0; +#endif +} + +typedef struct jl_group_s { + char* groupname; + long gid; + char** members; +} jl_group_t; + +JL_DLLEXPORT int jl_os_get_group(jl_group_t *grp, size_t gid) +{ +#ifdef _OS_WINDOWS_ + return UV_ENOTSUP; +#else + // modified directly from uv_os_get_password + struct group gp; + struct group* result; + char* buf; + char* gr_mem; + size_t bufsize; + size_t name_size; + long members; + size_t mem_size; + long initsize; + int r; + + if (grp == NULL) + return UV_EINVAL; + + initsize = sysconf(_SC_GETGR_R_SIZE_MAX); + + if (initsize <= 0) + bufsize = 4096; + else + bufsize = (size_t) initsize; + + buf = NULL; + + for (;;) { + free(buf); + buf = (char*)malloc(bufsize); + + if (buf == NULL) + return UV_ENOMEM; + + r = getgrgid_r(gid, &gp, buf, bufsize, &result); + + if (r != ERANGE) + break; + + bufsize *= 2; + } + + if (r != 0) { + free(buf); + return -r; + } + + if (result == NULL) { + free(buf); + return UV_ENOENT; + } + + /* Allocate memory for the groupname and members. */ + name_size = strlen(gp.gr_name) + 1; + members = 0; + mem_size = sizeof(char*); + for (r = 0; gp.gr_mem[r] != NULL; r++) { + mem_size += strlen(gp.gr_mem[r]) + 1 + sizeof(char*); + members++; + } + + gr_mem = (char*)malloc(name_size + mem_size); + if (gr_mem == NULL) { + free(buf); + return UV_ENOMEM; + } + + /* Copy the members */ + grp->members = (char**) gr_mem; + grp->members[members] = NULL; + gr_mem = (char*) ((char**) gr_mem + members + 1); + for (r = 0; r < members; r++) { + grp->members[r] = gr_mem; + gr_mem = stpcpy(gr_mem, gp.gr_mem[r]) + 1; + } + assert(gr_mem == (char*)grp->members + mem_size); + + /* Copy the groupname */ + grp->groupname = gr_mem; + memcpy(grp->groupname, gp.gr_name, name_size); + gr_mem += name_size; + + /* Copy the gid */ + grp->gid = gp.gr_gid; + + free(buf); + + return 0; +#endif +} + +JL_DLLEXPORT void jl_os_free_group(jl_group_t *grp) +{ + if (grp == NULL) + return; + + /* + The memory for is allocated in a single uv__malloc() call. The base of the + pointer is stored in grp->members, so that is the only field that needs + to be freed. + */ + free(grp->members); + grp->members = NULL; + grp->groupname = NULL; +} + // --- buffer manipulation --- JL_DLLEXPORT jl_array_t *jl_take_buffer(ios_t *s) diff --git a/test/file.jl b/test/file.jl index b732134ec51e6..bd88e1aedefa3 100644 --- a/test/file.jl +++ b/test/file.jl @@ -702,7 +702,7 @@ let @test a_stat.size == b_stat.size @test a_stat.size == c_stat.size - @test parse(Int, match(r"mode=(.*),", sprint(show, a_stat)).captures[1]) == a_stat.mode + @test parse(Int, split(sprint(show, a_stat),"mode: ")[2][1:8]) == a_stat.mode close(af) rm(afile) @@ -1605,3 +1605,32 @@ if Sys.iswindows() @test rm(tmp) === nothing end end + +@testset "StatStruct show's extended details" begin + f, io = mktemp() + s = stat(f) + stat_show_str = sprint(show, s) + @test occursin(f, stat_show_str) + if Sys.iswindows() + @test occursin("mode: 0o100666 (-rw-rw-rw-)", stat_show_str) + else + @test occursin("mode: 0o100600 (-rw-------)", stat_show_str) + end + if Sys.iswindows() == false + @test !isnothing(Base.Filesystem.getusername(s.uid)) + @test !isnothing(Base.Filesystem.getgroupname(s.gid)) + end + d = mktempdir() + s = stat(d) + stat_show_str = sprint(show, s) + @test occursin(d, stat_show_str) + if Sys.iswindows() + @test occursin("mode: 0o040666 (drw-rw-rw-)", stat_show_str) + else + @test occursin("mode: 0o040700 (drwx------)", stat_show_str) + end + if Sys.iswindows() == false + @test !isnothing(Base.Filesystem.getusername(s.uid)) + @test !isnothing(Base.Filesystem.getgroupname(s.gid)) + end +end