Move template expansion to "format-time"

This moves the expansion of templates from macro expansion time to later
on. This allows us to avoid having to search through expressions looking
for the actual expression type since those may be hidden by "decorator"
macros. It also handles cases where `@doc` is used on a string macro
docstring, such as

    @doc raw"..." f(x) = x

Implementation is based on a new internal abbreviation type called
`Template` which gets prepended and appended to docstrings in modules
where templates are available. Then, during formatting, we expand these
`Template` abbreviations into there definitions.

Fixes JuliaDocs#73.

.gitignore

@@ -4,3 +4,4 @@
 docs/build
 docs/site
 docs/Manifest.toml
+Manifest.toml

src/abbreviations.jl

@@ -611,3 +611,63 @@ of the docstring body that should be spliced into a template.
 const DOCSTRING = DocStringTemplate()
 
 # NOTE: no `format` needed for this 'mock' abbreviation.
+
+is_docstr_template(::DocStringTemplate) = true
+is_docstr_template(other) = false
+
+"""
+Internal abbreviation type used to wrap templated docstrings.
+
+`Location` is a `Symbol`, either `:before` or `:after`. `dict` stores a
+reference to a module's templates.
+"""
+struct Template{Location} <: Abbreviation
+    dict::Dict{Symbol,Vector{Any}}
+end
+
+function format(abbr::Template, buf, doc)
+    # Find the applicable template based on the kind of docstr.
+    parts = get_template(abbr.dict, template_key(doc))
+    # Replace the abbreviation with either the parts of the template found
+    # before the `DOCSTRING` abbreviation, or after it. When no `DOCSTRING`
+    # exists in the template, which shouldn't really happen then nothing will
+    # get included here.
+    for index in included_range(abbr, parts)
+        # We don't call `DocStringExtensions.format` here since we need to be
+        # able to format any content in docstrings, rather than just
+        # abbreviations.
+        Docs.formatdoc(buf, doc, parts[index])
+    end
+end
+
+function included_range(abbr::Template, parts::Vector)
+    # Select the correct indexing depending on what we find.
+    build_range(::Template, ::Nothing) = 0:-1
+    build_range(::Template{:before}, index) = 1:(index - 1)
+    build_range(::Template{:after}, index) = (index + 1):lastindex(parts)
+    # Search for index from either the front or back.
+    find_index(::Template{:before}) = findfirst(is_docstr_template, parts)
+    find_index(::Template{:after}) = findlast(is_docstr_template, parts)
+    # Find and return the correct indices.
+    return build_range(abbr, find_index(abbr))
+end
+
+function template_key(doc::Docs.DocStr)
+    # Local helper methods for extracting the template key from a docstring.
+    ismacro(b::Docs.Binding) = startswith(string(b.var), '@')
+    objname(obj::Union{Function,Module,DataType,UnionAll,Core.IntrinsicFunction}, b::Docs.Binding) = nameof(obj)
+    objname(obj, b::Docs.Binding) = Symbol("") # Empty to force resolving to `:CONSTANTS` below.
+    # Select the key returned based on input argument types.
+    _key(::Module, sig, binding)                 = :MODULES
+    _key(::Function, ::typeof(Union{}), binding) = ismacro(binding) ? :MACROS : :FUNCTIONS
+    _key(::Function, sig, binding)               = ismacro(binding) ? :MACROS : :METHODS
+    _key(::DataType, ::typeof(Union{}), binding) = :TYPES
+    _key(::DataType, sig, binding)               = :METHODS
+    _key(other, sig, binding)                    = :DEFAULT
+
+    binding = doc.data[:binding]
+    obj = Docs.resolve(binding)
+    name = objname(obj, binding)
+    key = name === binding.var ? _key(obj, doc.data[:typesig], binding) : :CONSTANTS
+    return key
+end

src/templates.jl

@@ -102,18 +102,21 @@ end
 # On v0.6 and below it seems it was assumed to be (docstr::String, expr::Expr), but on v0.7
 # it is (source::LineNumberNode, mod::Module, docstr::String, expr::Expr)
 function template_hook(source::LineNumberNode, mod::Module, docstr, expr::Expr)
-    local docex = interp_string(docstr)
-    if isdefined(mod, TEMP_SYM) && Meta.isexpr(docex, :string)
-        local templates = getfield(mod, TEMP_SYM)
-        local template = get_template(templates, expression_type(expr))
-        local out = Expr(:string)
-        for t in template
-            t == DOCSTRING ? append!(out.args, docex.args) : push!(out.args, t)
-        end
-        return (source, mod, out, expr)
-    else
-        return (source, mod, docstr, expr)
+    # During macro expansion we only need to wrap docstrings in special
+    # abbreviations that later print out what was before and after the
+    # docstring in it's specific template. This is only done when the module
+    # actually defines templates.
+    if isdefined(mod, TEMP_SYM)
+        dict = getfield(mod, TEMP_SYM)
+        # We unwrap interpolated strings so that we can add the `:before` and
+        # `:after` abbreviations. Otherwise they're just left as is.
+        unwrapped = Meta.isexpr(docstr, :string) ? docstr.args : [docstr]
+        before, after = Template{:before}(dict), Template{:after}(dict)
+        # Rebuild the original docstring, but with the template abbreviations
+        # surrounding it.
+        docstr = Expr(:string, before, unwrapped..., after)
     end
+    return (source, mod, docstr, expr)
 end
 
 function template_hook(docstr, expr::Expr)
@@ -123,29 +126,4 @@ end
 
 template_hook(args...) = args
 
-interp_string(str::AbstractString) = Expr(:string, str)
-interp_string(other) = other
-
 get_template(t::Dict, k::Symbol) = haskey(t, k) ? t[k] : get(t, :DEFAULT, Any[DOCSTRING])
-
-function expression_type(ex::Expr)
-    # Expression heads changed in JuliaLang/julia/pull/23157 to match the new keyword syntax.
-    if VERSION < v"0.7.0-DEV.1263" && Meta.isexpr(ex, [:type, :bitstype])
-        :TYPES
-    elseif Meta.isexpr(ex, :module)
-        :MODULES
-    elseif Meta.isexpr(ex, [:struct, :abstract, :typealias, :primitive])
-        :TYPES
-    elseif Meta.isexpr(ex, :macro)
-        :MACROS
-    elseif Meta.isexpr(ex, [:function, :(=)]) && Meta.isexpr(ex.args[1], :call) || (Meta.isexpr(ex.args[1], :where) && Meta.isexpr(ex.args[1].args[1], :call))
-        :METHODS
-    elseif Meta.isexpr(ex, :function)
-        :FUNCTIONS
-    elseif Meta.isexpr(ex, [:const, :(=)])
-        :CONSTANTS
-    else
-        :DEFAULT
-    end
-end
-expression_type(other) = :DEFAULT

test/templates.jl

@@ -35,12 +35,18 @@ const K = 1
 "mutable struct `T`"
 mutable struct T end
 
+"`@kwdef` struct `S`"
+Base.@kwdef struct S end
+
 "method `f`"
 f(x) = x
 
 "method `g`"
 g(::Type{T}) where {T} = T # Issue 32
 
+"inlined method `h`"
+@inline h(x) = x
+
 "macro `@m`"
 macro m(x) end
 
@@ -66,8 +72,15 @@ module InnerModule
     "constant `K`"
     const K = 1
 
-    "mutable struct `T`"
-    mutable struct T end
+    """
+    mutable struct `T`
+
+    $(FIELDS)
+    """
+    mutable struct T
+        "field docs for x"
+        x
+    end
 
     "method `f`"
     f(x) = x

test/tests.jl

@@ -423,12 +423,15 @@ end
         let fmt = expr -> Markdown.plain(eval(:(@doc $expr)))
             @test occursin("(DEFAULT)", fmt(:(TemplateTests.K)))
             @test occursin("(TYPES)", fmt(:(TemplateTests.T)))
+            @test occursin("(TYPES)", fmt(:(TemplateTests.S)))
             @test occursin("(METHODS, MACROS)", fmt(:(TemplateTests.f)))
             @test occursin("(METHODS, MACROS)", fmt(:(TemplateTests.g)))
+            @test occursin("(METHODS, MACROS)", fmt(:(TemplateTests.h)))
             @test occursin("(METHODS, MACROS)", fmt(:(TemplateTests.@m)))
 
             @test occursin("(DEFAULT)", fmt(:(TemplateTests.InnerModule.K)))
             @test occursin("(DEFAULT)", fmt(:(TemplateTests.InnerModule.T)))
+            @test occursin("field docs for x", fmt(:(TemplateTests.InnerModule.T)))
             @test occursin("(METHODS, MACROS)", fmt(:(TemplateTests.InnerModule.f)))
             @test occursin("(MACROS)", fmt(:(TemplateTests.InnerModule.@m)))