Break out doctesting / refactor

CHANGELOG.md

@@ -4,7 +4,19 @@
 
 * Documenter v0.23 requires Julia v1.0. ([#1015][github-1015])
 
-* ![Feature][badge-feature] `makedocs` now accepts the `expandfirst` argument, which allows specifying a set of pages that should be evaluated before the other. ([#1027][github-1027], [#1029][github-1029])
+* ![BREAKING][badge-breaking] `DocTestSetup`s that are defined in `@meta` blocks no longer apply to doctests that are in docstrings. ([#774][github-774])
+
+  - Specifically, the pattern where `@docs` or `@autodocs` blocks were surrounded by `@meta` blocks, setting up a shared `DocTestSetup` for many docstrings, no longer works.
+
+  - Documenter now exports the `DocMeta` module, which provides an alternative way to add `DocTestSetup` to docstrings.
+
+  **For upgrading:** Use `DocMeta.setdocmeta!` in `make.jl` to set up a `DocTestSetup` that applies to all the docstrings in a particular module instead and, if applicable, remove the now redundant `@meta` blocks. See the ["Setup code" section under "Doctesting"](https://juliadocs.github.io/Documenter.jl/v0.23.0/man/doctests/#Setup-Code-1) in the manual for more information.
+
+* ![Feature][badge-feature] `makedocs` now accepts the `doctest = :only` keyword, which allows doctests to be run while most other build steps, such as rendering, are skipped. This makes it more feasible to run doctests as part of the test suite (see the manual for more information). ([#198][github-198], [#535][github-535], [#756][github-756], [#774][github-774])
+
+* ![Feature][badge-feature] Documenter now exports the `doctest` function, which verifies the doctests in all the docstrings of a given module. This can be used to verify docstring doctests as part of test suite. ([#198][github-198], [#535][github-535], [#756][github-756], [#774][github-774])
+
+* ![Feature][badge-feature] `makedocs` now accepts the `expandfirst` argument, which allows specifying a set of pages that should be evaluated before others. ([#1027][github-1027], [#1029][github-1029])
 
 * ![Enhancement][badge-enhancement] The evaluation order of pages is now fixed (unless customized with `expandfirst`). The pages are evaluated in the alphabetical order of their file paths. ([#1027][github-1027], [#1029][github-1029])
 
@@ -269,10 +281,14 @@
 
 * ![Bugfix][badge-bugfix] At-docs blocks no longer give an error when containing empty lines. ([#823][github-823], [#824][github-824])
 
+[github-198]: https://github.com/JuliaDocs/Documenter.jl/issues/198
 [github-511]: https://github.com/JuliaDocs/Documenter.jl/pull/511
+[github-535]: https://github.com/JuliaDocs/Documenter.jl/issues/535
 [github-697]: https://github.com/JuliaDocs/Documenter.jl/pull/697
 [github-706]: https://github.com/JuliaDocs/Documenter.jl/pull/706
+[github-756]: https://github.com/JuliaDocs/Documenter.jl/issues/756
 [github-764]: https://github.com/JuliaDocs/Documenter.jl/pull/764
+[github-774]: https://github.com/JuliaDocs/Documenter.jl/pull/774
 [github-789]: https://github.com/JuliaDocs/Documenter.jl/pull/789
 [github-792]: https://github.com/JuliaDocs/Documenter.jl/pull/792
 [github-793]: https://github.com/JuliaDocs/Documenter.jl/pull/793

Project.toml

@@ -14,14 +14,17 @@ REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 
 [compat]
+Compat = "2.1"
 DocStringExtensions = "0.4, 0.5, 0.6, 0.7, 0.8"
 JSON = "0.19, 0.20"
 julia = "1"
 
 [extras]
+Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
 DocumenterMarkdown = "997ab1e6-3595-5248-9280-8efb232c3433"
+DocumenterTools = "35a29f4d-8980-5a13-9543-d66fff28ecb8"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test", "DocumenterMarkdown", "Random"]
+test = ["Test", "Random", "Compat", "DocumenterMarkdown", "DocumenterTools"]

docs/make.jl

@@ -33,13 +33,15 @@ makedocs(
                 "lib/internals/builder.md",
                 "lib/internals/cross-references.md",
                 "lib/internals/docchecks.md",
+                "lib/internals/docmeta.md",
                 "lib/internals/docsystem.md",
                 "lib/internals/doctests.md",
                 "lib/internals/documenter.md",
                 "lib/internals/documentertools.md",
                 "lib/internals/documents.md",
                 "lib/internals/dom.md",
                 "lib/internals/expanders.md",
+                "lib/internals/markdown2.md",
                 "lib/internals/mdflatten.md",
                 "lib/internals/selectors.md",
                 "lib/internals/textdiff.md",
@@ -50,6 +52,7 @@ makedocs(
         "contributing.md",
     ],
     strict = true,
+    doctest = ("doctest-only" in ARGS) ? :only : true,
 )
 
 deploydocs(

docs/src/lib/internals/docmeta.md

@@ -0,0 +1,9 @@
+# DocMeta
+
+```@docs
+DocMeta.initdocmeta!
+DocMeta.META
+DocMeta.METAMODULES
+DocMeta.METATYPE
+DocMeta.VALIDMETA
+```

docs/src/lib/internals/markdown2.md

@@ -0,0 +1,15 @@
+# Markdown2
+
+Documentation for the private [`Markdown2`](@ref Documenter.Utilities.Markdown2) module.
+
+## Index
+
+```@index
+Pages = ["markdown2.md"]
+```
+
+## Docstrings
+
+```@autodocs
+Modules = [Documenter.Utilities.Markdown2]
+```

docs/src/lib/public.md

@@ -25,6 +25,10 @@ hide
 deploydocs
 Deps
 Deps.pip
+doctest
+DocMeta
+DocMeta.getdocmeta
+DocMeta.setdocmeta!
 ```
 
 ## DocumenterTools

docs/src/man/doctests.md

@@ -1,9 +1,9 @@
 # Doctests
 
-Documenter will, by default, try to run `jldoctest` code blocks that it finds in the generated
-documentation. This can help to avoid documentation examples from becoming outdated,
-incorrect, or misleading. It's recommended that as many of a package's examples be runnable
-by Documenter's doctest.
+Documenter will, by default, run `jldoctest` code blocks that it finds and makes sure that
+the actual output matches what's in the doctest. This can help to avoid documentation
+examples from becoming outdated, incorrect, or misleading. It's recommended that as many of
+a package's examples be runnable by Documenter's doctest.
 
 This section of the manual outlines how to go about enabling doctests for code blocks in
 your package's documentation.
@@ -178,10 +178,14 @@ julia> println(foo)
 ## Setup Code
 
 Doctests may require some setup code that must be evaluated prior to that of the actual
-example, but that should not be displayed in the final documentation. For this purpose a
-`@meta` block containing a `DocTestSetup = ...` value can be used. In the example below,
-the function `foo` is defined inside a `@meta` block. This block will be evaluated at
-the start of the following doctest blocks:
+example, but that should not be displayed in the final documentation. There are three ways
+to specify the setup code, each appropriate in a different situation.
+
+### `DocTestSetup` in `@meta` blocks
+
+For doctests in the Markdown source files, an `@meta` block containing a `DocTestSetup =
+...` value can be used. In the example below, the function `foo` is defined inside a `@meta`
+block. This block will be evaluated at the start of the following doctest blocks:
 
 ````markdown
 ```@meta
@@ -205,8 +209,32 @@ DocTestSetup = nothing
 The `DocTestSetup = nothing` is not strictly necessary, but good practice nonetheless to
 help avoid unintentional definitions in following doctest blocks.
 
-Another option is to use the `setup` keyword argument, which is convenient for short definitions,
-and for setups needed in inline docstrings.
+While technically the `@meta` blocks also work within docstrings, their use there is
+discouraged since the `@meta` blocks will show up when querying docstrings in the REPL.
+
+!!! note "Historic note"
+    It used to be that `DocTestSetup`s in `@meta` blocks in Markdown files that included
+    docstrings also affected the doctests in the docstrings. Since Documenter 0.23 that is
+    no longer the case. You should use [Module-level metadata](@ref) or [Block-level setup
+    code](@ref) instead.
+
+### Module-level metadata
+
+For doctests that are in docstrings, the exported [`DocMeta`](@ref) module provides an API
+to attach metadata that applies to all the docstrings in a particular module. Setting up the
+`DocTestSetup` metadata should be done before the [`makedocs`](@ref) or [`doctest`](@ref)
+call:
+
+```julia
+using MyPackage, Documenter
+DocMeta.setdocmeta!(MyPackage, :DocTestSetup, :(using MyPackage); recursive=true)
+makedocs(modules=[MyPackage], ...)
+```
+
+### Block-level setup code
+
+Yet another option is to use the `setup` keyword argument to the `jldoctest` block, which is
+convenient for short definitions, and for setups needed in inline docstrings.
 
 ````markdown
 ```jldoctest; setup = :(foo(x) = x^2)
@@ -221,10 +249,6 @@ julia> foo(2)
     and no state is shared between any code blocks.
     To preserve definitions see [Preserving Definitions Between Blocks](@ref).
 
-!!! note
-
-    If you rely on setup-code for doctests inside docstrings, included in the document with `@docs` or `@autodocs`, the `@meta` block must be in the markdown file that calls these macros and not within the docstrings themselves, otherwise they will be ignored.
-
 ## Filtering Doctests
 
 A part of the output of a doctest might be non-deterministic, e.g. pointer addresses and timings.
@@ -286,6 +310,63 @@ julia> @time [1,2,3,4]
     The global filters, filters defined in `@meta` blocks, and filters defined with the `filter`
     keyword argument are all applied to each doctest.
 
+
+## Doctesting Without Building the Docs
+
+Documenter has a few ways to verify the doctests without having to run a potentially
+expensive full documentation build.
+
+### Doctesting docstrings only
+
+An option for doctesting just the docstrings of a particular module (and all its submodules)
+is to use the [`doctest`](@ref) function. It takes a list of modules as an argument and runs
+all the doctests in all the docstrings it finds. This can be handy for quick tests when
+writing docstrings of a package.
+
+[`doctest`](@ref) will return `true` or `false`, depending on whether the doctests pass or
+not, making it easy to include a doctest of all the docstrings in the package test suite:
+
+```julia
+using MyPackage, Documenter, Test
+@test doctest([MyPackage])
+```
+
+Note that you still need to make sure that all the necessary [Module-level metadata](@ref)
+for the doctests is set up before [`doctest`](@ref) is called.
+
+### Doctesting without a full build
+
+An alternative, which also runs doctests on the manual pages, but still skips most other
+build steps, is to pass `doctest = :only` to [`makedocs`](@ref).
+
+This also makes it more practical to include doctests as part of the normal test suite of a
+package. One option to set it up is to make the `doctest` keyword depend on command line
+arguments passed to the `make.jl` script:
+
+```julia
+makedocs(...,
+    doctest = ("doctest-only" in ARGS) ? :only : true
+)
+```
+
+Now, the `make.jl` script can be run on the command line as `julia docs/make.jl
+doctest-only` and it will only run the doctests. On doctest failure, the `makedocs` throws
+an error and `julia` exits with a non-zero exit code.
+
+For running the doctests as part of the standard test suite, the  `docs/make.jl` can simply
+be `include`d in the `test/runtest.jl` file:
+
+```julia
+push!(ARGS, "doctest-only")
+include(joinpath(@__DIR__, "..", "docs", "make.jl"))
+```
+
+The `push!` to `ARGS` emulates the passing of the `doctest-only` command line argument.
+
+Note that, for this to work, you need to add Documenter and all the other packages that get
+loaded in `make.jl`, or in the doctest, as test dependencies.
+
+
 ## Fixing Outdated Doctests
 
 To fix outdated doctests, the `doctest` flag to [`makedocs`](@ref) can be set to

src/Builder.jl

@@ -9,6 +9,7 @@ module Builder
 
 import ..Documenter:
     Anchors,
+    DocTests,
     Documents,
     Documenter,
     Utilities
@@ -24,6 +25,7 @@ using DocStringExtensions
 The default document processing "pipeline", which consists of the following actions:
 
 - [`SetupBuildDirectory`](@ref)
+- [`Doctest`](@ref)
 - [`ExpandTemplates`](@ref)
 - [`CrossReferences`](@ref)
 - [`CheckDocument`](@ref)
@@ -38,6 +40,11 @@ Creates the correct directory layout within the `build` folder and parses markdo
 """
 abstract type SetupBuildDirectory <: DocumentPipeline end
 
+"""
+Runs all the doctests in all docstrings and Markdown files.
+"""
+abstract type Doctest <: DocumentPipeline end
+
 """
 Executes a sequence of actions on each node of the parsed markdown files in turn.
 """
@@ -65,6 +72,7 @@ Writes the document tree to the `build` directory.
 abstract type RenderDocument <: DocumentPipeline end
 
 Selectors.order(::Type{SetupBuildDirectory})   = 1.0
+Selectors.order(::Type{Doctest})               = 1.1
 Selectors.order(::Type{ExpandTemplates})       = 2.0
 Selectors.order(::Type{CrossReferences})       = 3.0
 Selectors.order(::Type{CheckDocument})         = 4.0
@@ -165,7 +173,7 @@ function walk_navpages(visible, title, src, children, parent, doc)
     parent_visible = (parent === nothing) || parent.visible
     if src !== nothing
         src = normpath(src)
-        src in keys(doc.internal.pages) || error("'$src' is not an existing page!")
+        src in keys(doc.blueprint.pages) || error("'$src' is not an existing page!")
     end
     nn = Documents.NavNode(src, title, parent)
     (src === nothing) || push!(doc.internal.navlist, nn)
@@ -187,31 +195,48 @@ walk_navpages(ps::Vector, parent, doc) = [walk_navpages(p, parent, doc)::Documen
 walk_navpages(src::String, parent, doc) = walk_navpages(true, nothing, src, [], parent, doc)
 
 
+function Selectors.runner(::Type{Doctest}, doc::Documents.Document)
+    if doc.user.doctest in [:fix, :only, true]
+        @info "Doctest: running doctests."
+        DocTests.doctest(doc.blueprint, doc)
+        num_errors = length(doc.internal.errors)
+        if (doc.user.doctest === :only || doc.user.strict) && num_errors > 0
+            error("`makedocs` encountered $(num_errors > 1 ? "$(num_errors) doctest errors" : "a doctest error"). Terminating build")
+        end
+    else
+        @info "Doctest: skipped."
+    end
+end
+
 function Selectors.runner(::Type{ExpandTemplates}, doc::Documents.Document)
+    is_doctest_only(doc, "ExpandTemplates") && return
     @info "ExpandTemplates: expanding markdown templates."
     Documenter.Expanders.expand(doc)
 end
 
 function Selectors.runner(::Type{CrossReferences}, doc::Documents.Document)
+    is_doctest_only(doc, "CrossReferences") && return
     @info "CrossReferences: building cross-references."
     Documenter.CrossReferences.crossref(doc)
 end
 
 function Selectors.runner(::Type{CheckDocument}, doc::Documents.Document)
+    is_doctest_only(doc, "CheckDocument") && return
     @info "CheckDocument: running document checks."
     Documenter.DocChecks.missingdocs(doc)
-    Documenter.DocTests.doctest(doc)
     Documenter.DocChecks.footnotes(doc)
     Documenter.DocChecks.linkcheck(doc)
 end
 
 function Selectors.runner(::Type{Populate}, doc::Documents.Document)
+    is_doctest_only(doc, "Populate") && return
     @info "Populate: populating indices."
     Documents.doctest_replace!(doc)
     Documents.populate!(doc)
 end
 
 function Selectors.runner(::Type{RenderDocument}, doc::Documents.Document)
+    is_doctest_only(doc, "RenderDocument") && return
     count = length(doc.internal.errors)
     if doc.user.strict && count > 0
         error("`makedocs` encountered $(count > 1 ? "errors" : "an error"). Terminating build")
@@ -223,4 +248,12 @@ end
 
 Selectors.runner(::Type{DocumentPipeline}, doc::Documents.Document) = nothing
 
+function is_doctest_only(doc, stepname)
+    if doc.user.doctest in [:fix, :only]
+        @info "Skipped $stepname step (doctest only)."
+        return true
+    end
+    return false
+end
+
 end

src/CrossReferences.jl

@@ -21,7 +21,7 @@ Traverses a [`Documents.Document`](@ref) and replaces links containg `@ref` URLs
 their real URLs.
 """
 function crossref(doc::Documents.Document)
-    for (src, page) in doc.internal.pages
+    for (src, page) in doc.blueprint.pages
         empty!(page.globals.meta)
         for element in page.elements
             crossref(page.mapping[element], page, doc)