add tests for undocumented symbols (#52723)

Following #52413 by @jariji and the discussion in #51174, this adds
tests to ensure that there are no undocumented *(= lacking docstrings)*
public symbols in any documented module.

Many of the tests are broken — we have a lot of undocumented exports. In
such cases I added both a `@test_broken` and a `@test` to ensure that
there are no additional regressions added in the future.

~~(This PR may have to be updated when
#52413 (comment) is
fixed, i.e. when ~~#52727~~ #52743 is merged.)~~ Has been updated.

---------

Co-authored-by: Lilith Orion Hafner <lilithhafner@gmail.com>

base/docs/Docs.jl

@@ -650,11 +650,37 @@ function loaddocs(docs::Vector{Core.SimpleVector})
     nothing
 end
 
+# FIXME: formatdoc, parsedoc, apropos, and doc are defined here (but only doc is exported)
+# for historical reasons (#25738), but are *implemented* in REPL/src/docview.jl, while
+# apropos is *exported* by InteractiveUtils and doc is exported by Docs.  Seems
+# like a more sensible refactoring should be possible.
+
 function formatdoc end
 function parsedoc end
+
+"""
+    apropos([io::IO=stdout], pattern::Union{AbstractString,Regex})
+
+Search available docstrings for entries containing `pattern`.
+
+When `pattern` is a string, case is ignored. Results are printed to `io`.
+
+`apropos` can be called from the help mode in the REPL by wrapping the query in double quotes:
+```
+help?> "pattern"
+```
+"""
 function apropos end
-function doc end
 
+"""
+    Docs.doc(binding, sig)
+
+Return all documentation that matches both `binding` and `sig`.
+
+If `getdoc` returns a non-`nothing` result on the value of the binding, then a
+dynamic docstring is returned instead of one based on the binding itself.
+"""
+function doc end
 
 """
     Docs.hasdoc(mod::Module, sym::Symbol)::Bool

base/libc.jl

@@ -45,6 +45,11 @@ show(io::IO, fd::RawFD) = print(io, "RawFD(", bitcast(UInt32, fd), ')')  # avoid
 
 # Wrapper for an OS file descriptor (for Windows)
 if Sys.iswindows()
+    @doc """
+        WindowsRawSocket
+
+    Primitive type which wraps the native Windows file `HANDLE`.
+    """
     primitive type WindowsRawSocket sizeof(Ptr) * 8 end # On Windows file descriptors are HANDLE's and 64-bit on 64-bit Windows
     WindowsRawSocket(handle::Ptr{Cvoid}) = bitcast(WindowsRawSocket, handle)
     WindowsRawSocket(handle::WindowsRawSocket) = handle

stdlib/Artifacts/test/runtests.jl

@@ -261,3 +261,9 @@ end
     @test length(Base.manifest_names) == 2n # there are two manifest names per project name
     @test length(Base.preferences_names) == n
 end
+
+@testset "Docstrings" begin
+    undoc = Docs.undocumented_names(Artifacts)
+    @test_broken isempty(undoc)
+    @test undoc == [:Artifacts]
+end

stdlib/Base64/test/runtests.jl

@@ -1,7 +1,8 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
 using Test, Random
-import Base64:
+using Base64:
+    Base64,
     Base64EncodePipe,
     base64encode,
     Base64DecodePipe,
@@ -142,3 +143,7 @@ end
         @test String(base64decode(splace(longEncodedText))) == longDecodedText
     end
 end
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(Base64))
+end

stdlib/CRC32c/test/runtests.jl

@@ -77,3 +77,7 @@ end
 crc32c_sw(io::IO, crc::UInt32=0x00000000) = crc32c_sw(io, typemax(Int64), crc)
 test_crc32c(crc32c)
 test_crc32c(crc32c_sw)
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(CRC32c))
+end

stdlib/Dates/test/runtests.jl

@@ -2,8 +2,16 @@
 
 module DateTests
 
+using Test, Dates
+
 for file in readlines(joinpath(@__DIR__, "testgroups"))
     include(file * ".jl")
 end
 
+@testset "Docstrings" begin
+    undoc = Docs.undocumented_names(Dates)
+    @test_broken isempty(undoc)
+    @test undoc == [:adjust]
+end
+
 end

stdlib/FileWatching/test/runtests.jl

@@ -447,4 +447,10 @@ rm(dir)
     include("pidfile.jl")
 end
 
+@testset "Docstrings" begin
+    undoc = Docs.undocumented_names(FileWatching)
+    @test_broken isempty(undoc)
+    @test undoc == [:FDWatcher, :FileMonitor, :FolderMonitor, :PollingFileWatcher]
+end
+
 end # testset

stdlib/Future/test/runtests.jl

@@ -2,3 +2,7 @@
 
 using Test
 using Future
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(Future))
+end

stdlib/InteractiveUtils/test/runtests.jl

@@ -724,3 +724,9 @@ end
 end
 
 @test Base.infer_effects(sin, (Int,)) == InteractiveUtils.@infer_effects sin(42)
+
+@testset "Docstrings" begin
+    undoc = Docs.undocumented_names(InteractiveUtils)
+    @test_broken isempty(undoc)
+    @test undoc == [:InteractiveUtils]
+end

stdlib/LibGit2/test/runtests.jl

@@ -1,6 +1,11 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-using Test
+using Test, LibGit2
+
 @testset verbose=true "LibGit2 $test" for test in eachline(joinpath(@__DIR__, "testgroups"))
     include("$test.jl")
 end
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(LibGit2))
+end

stdlib/Libdl/test/runtests.jl

@@ -329,3 +329,9 @@ end
     lazy_name_lazy_lib = LazyLibrary(libname)
     @test dlpath(lazy_name_lazy_lib) == realpath(string(libname))
 end; end
+
+@testset "Docstrings" begin
+    undoc = Docs.undocumented_names(Libdl)
+    @test_broken isempty(undoc)
+    @test undoc == [:Libdl]
+end

stdlib/LinearAlgebra/test/runtests.jl

@@ -1,5 +1,12 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
+using Test, LinearAlgebra
 
 for file in readlines(joinpath(@__DIR__, "testgroups"))
     include(file * ".jl")
 end
+
+@testset "Docstrings" begin
+    undoc = Docs.undocumented_names(LinearAlgebra)
+    @test_broken isempty(undoc)
+    @test undoc == [:ColumnNorm, :LAPACKException, :NoPivot, :RankDeficientException, :RowMaximum, :RowNonZero, :copy_transpose!]
+end

stdlib/Logging/test/runtests.jl

@@ -300,4 +300,10 @@ end
     @test occursin("CustomLog2: hello", String(take!(buf)))
 end
 
+@testset "Docstrings" begin
+    undoc = Docs.undocumented_names(Logging)
+    @test_broken isempty(undoc)
+    @test undoc == [:AboveMaxLevel, :BelowMinLevel]
+end
+
 end

stdlib/Markdown/test/runtests.jl

@@ -1293,3 +1293,7 @@ end
     # see issue #42139
     @test md"<一轮红日初升>" |> html == """<p>&lt;一轮红日初升&gt;</p>\n"""
 end
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(Markdown))
+end

stdlib/Mmap/test/runtests.jl

@@ -339,3 +339,7 @@ open(file, "r+") do s
     finalize(A); A = nothing; GC.gc()
 end
 rm(file)
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(Mmap))
+end

stdlib/Printf/test/runtests.jl

@@ -1145,6 +1145,12 @@ end
     @test_throws Printf.InvalidFormatStringError Printf.Format("%z")
 end
 
+@testset "Docstrings" begin
+    undoc = Docs.undocumented_names(Printf)
+    @test_broken isempty(undoc)
+    @test undoc == [:Printf]
+end
+
 # issue #52749
 @test @sprintf("%.160g", 1.38e-23) == "1.380000000000000060010582465734078799297660966782642624395399644741944111814291318296454846858978271484375e-23"
 

stdlib/Profile/test/runtests.jl

@@ -300,3 +300,9 @@ end
 end
 
 include("allocs.jl")
+
+@testset "Docstrings" begin
+    undoc = Docs.undocumented_names(Profile)
+    @test_broken isempty(undoc)
+    @test undoc == [:Allocs]
+end

stdlib/REPL/src/docview.jl

@@ -198,14 +198,6 @@ function insert_internal_warning(other, internal_access::Set{Pair{Module,Symbol}
     other
 end
 
-"""
-    Docs.doc(binding, sig)
-
-Return all documentation that matches both `binding` and `sig`.
-
-If `getdoc` returns a non-`nothing` result on the value of the binding, then a
-dynamic docstring is returned instead of one based on the binding itself.
-"""
 function doc(binding::Binding, sig::Type = Union{})
     if defined(binding)
         result = getdoc(resolve(binding), sig)
@@ -920,18 +912,6 @@ stripmd(x::Markdown.Footnote) = "$(stripmd(x.id)) $(stripmd(x.text))"
 stripmd(x::Markdown.Table) =
     join([join(map(stripmd, r), " ") for r in x.rows], " ")
 
-"""
-    apropos([io::IO=stdout], pattern::Union{AbstractString,Regex})
-
-Search available docstrings for entries containing `pattern`.
-
-When `pattern` is a string, case is ignored. Results are printed to `io`.
-
-`apropos` can be called from the help mode in the REPL by wrapping the query in double quotes:
-```
-help?> "pattern"
-```
-"""
 apropos(string) = apropos(stdout, string)
 apropos(io::IO, string) = apropos(io, Regex("\\Q$string", "i"))
 

stdlib/REPL/test/repl.jl

@@ -1749,3 +1749,9 @@ let io = IOBuffer()
     seek(io, 0)
     @test countlines(io) == 2
 end
+
+@testset "Docstrings" begin
+    undoc = Docs.undocumented_names(REPL)
+    @test_broken isempty(undoc)
+    @test undoc == [:AbstractREPL, :BasicREPL, :LineEditREPL, :StreamREPL]
+end

stdlib/Random/test/runtests.jl

@@ -1232,3 +1232,7 @@ end
     @test xs isa Vector{Pair{Bool, Char}}
     @test length(xs) == 3
 end
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(Random))
+end

stdlib/Serialization/test/runtests.jl

@@ -655,3 +655,9 @@ end
     @test l2 == l1
     @test l2.parts === ()
 end
+
+@testset "Docstrings" begin
+    undoc = Docs.undocumented_names(Serialization)
+    @test_broken isempty(undoc)
+    @test undoc == [:AbstractSerializer, :Serializer]
+end

stdlib/SharedArrays/test/runtests.jl

@@ -324,3 +324,7 @@ end
 @test SharedMatrix([0.1 0.2; 0.3 0.4]) == [0.1 0.2; 0.3 0.4]
 @test_throws MethodError SharedVector(rand(4,4))
 @test_throws MethodError SharedMatrix(rand(4))
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(SharedArrays))
+end

stdlib/Sockets/test/runtests.jl

@@ -682,3 +682,7 @@ end
 
 
 close(sockets_watchdog_timer)
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(Sockets))
+end

stdlib/TOML/test/runtests.jl

@@ -25,3 +25,9 @@ include("print.jl")
 include("parse.jl")
 
 @inferred TOML.parse("foo = 3")
+
+@testset "Docstrings" begin
+    undoc = Docs.undocumented_names(TOML)
+    @test_broken isempty(undoc)
+    @test undoc == [:TOML]
+end

stdlib/Test/test/runtests.jl

@@ -1582,3 +1582,7 @@ let
         end
     end
 end
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(Test))
+end

stdlib/UUIDs/test/runtests.jl

@@ -73,3 +73,7 @@ str = "22b4a8a1-e548-4eeb-9270-60426d66a48e"
 for r in rand(UInt128, 10^3)
     @test UUID(r) == UUID(string(UUID(r)))
 end
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(UUIDs))
+end

stdlib/Unicode/test/runtests.jl

@@ -534,3 +534,7 @@ isequal_normalized_naive(s1, s2; kws...) = normalize(s1; kws...) == normalize(s2
         @test !isequal_normalized("x\u0334\u0335", "x\u0335\u0334")
     end
 end
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(Unicode))
+end

test/broadcast.jl

@@ -1176,6 +1176,12 @@ import Base.Broadcast: BroadcastStyle, DefaultArrayStyle
 f51129(v, x) = (1 .- (v ./ x) .^ 2)
 @test @inferred(f51129([13.0], 6.5)) == [-3.0]
 
+@testset "Docstrings" begin
+    undoc = Docs.undocumented_names(Broadcast)
+    @test_broken isempty(undoc)
+    @test undoc == [:dotview]
+end
+
 @testset "broadcast for `AbstractArray` without `CartesianIndex` support" begin
     struct BVec52775 <: AbstractVector{Int}
         a::Vector{Int}

test/checked.jl

@@ -358,3 +358,7 @@ end
     @test checked_mul(1, 2, 3, 4, 5, 6, 7) === 5040
     @test checked_mul(1, 2, 3, 4, 5, 6, 7, 8) === 40320
 end
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(Base.Checked))
+end

test/docs.jl

@@ -1556,3 +1556,9 @@ Base.@ccallable c51586_long()::Int = 3
 
 @test docstrings_equal(@doc(c51586_short()), doc"ensure we can document ccallable functions")
 @test docstrings_equal(@doc(c51586_long()), doc"ensure we can document ccallable functions")
+
+@testset "Docs docstrings" begin
+    undoc = Docs.undocumented_names(Docs)
+    @test_broken isempty(undoc)
+    @test undoc == [Symbol("@var")]
+end

test/filesystem.jl

@@ -40,3 +40,9 @@ import Base.Filesystem: S_IRUSR, S_IRGRP, S_IROTH
   @test S_IRUSR & ~S_IRGRP == S_IRUSR
   @test typeof(S_IRUSR) == typeof(S_IRGRP) == typeof(S_IROTH)
 end
+
+@testset "Base.Filesystem docstrings" begin
+  undoc = Docs.undocumented_names(Base.Filesystem)
+  @test_broken isempty(undoc)
+  @test undoc == [:File, :Filesystem, :JL_O_APPEND, :JL_O_ASYNC, :JL_O_CLOEXEC, :JL_O_CREAT, :JL_O_DIRECT, :JL_O_DIRECTORY, :JL_O_DSYNC, :JL_O_EXCL, :JL_O_FSYNC, :JL_O_LARGEFILE, :JL_O_NDELAY, :JL_O_NOATIME, :JL_O_NOCTTY, :JL_O_NOFOLLOW, :JL_O_NONBLOCK, :JL_O_PATH, :JL_O_RANDOM, :JL_O_RDONLY, :JL_O_RDWR, :JL_O_RSYNC, :JL_O_SEQUENTIAL, :JL_O_SHORT_LIVED, :JL_O_SYNC, :JL_O_TEMPORARY, :JL_O_TMPFILE, :JL_O_TRUNC, :JL_O_WRONLY, :S_IRGRP, :S_IROTH, :S_IRUSR, :S_IRWXG, :S_IRWXO, :S_IRWXU, :S_IWGRP, :S_IWOTH, :S_IWUSR, :S_IXGRP, :S_IXOTH, :S_IXUSR, :cptree, :futime, :rename, :sendfile, :unlink]
+end

test/gc.jl

@@ -33,3 +33,7 @@ end
     run_gctest("gc/chunks.jl")
     run_nonzero_page_utilization_test()
 end
+
+@testset "Base.GC docstrings" begin
+    @test isempty(Docs.undocumented_names(GC))
+end

test/iterators.jl

@@ -1009,3 +1009,7 @@ end
 @testset "collect partition substring" begin
     @test collect(Iterators.partition(lstrip("01111", '0'), 2)) == ["11", "11"]
 end
+
+@testset "Iterators docstrings" begin
+    @test isempty(Docs.undocumented_names(Iterators))
+end

test/math.jl

@@ -1599,3 +1599,7 @@ end === Float64
     @test tanpi(big(1//1)) == big(0.0)
     @test cospi(big(1//1)) == big(-1.0)
 end
+
+@testset "Docstrings" begin
+    @test isempty(Docs.undocumented_names(MathConstants))
+end