diff --git a/docs/src/fileformat.md b/docs/src/fileformat.md index 611e26cf..8ec47230 100644 --- a/docs/src/fileformat.md +++ b/docs/src/fileformat.md @@ -1,4 +1,4 @@ -# **2.** File Format +# [**2.** File Format](@id File-Format) The source file format for Literate is a regular, commented, julia (`.jl`) scripts. The idea is that the scripts also serve as documentation on their own and it is also diff --git a/docs/src/index.md b/docs/src/index.md index 3c312bc3..7cac91d0 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,4 +1,4 @@ -# **1.** Introduction +# [**1.** Introduction](@id Introduction) Welcome to the documentation for Literate -- a simplistic package for [Literate Programming](https://en.wikipedia.org/wiki/Literate_programming). @@ -13,11 +13,13 @@ an option to "clean" the source from all metadata, and produce a pure Julia scri The main design goal is simplicity. It should be simple to use, and the syntax should be simple. In short, all you have to do is to write a commented julia script! -The public interface consists mainly of three functions, all of which take the same script file +The public interface consists of three functions, all of which take the same script file as input, but generate different output: -- [`Literate.markdown`](@ref): generates a markdown file -- [`Literate.notebook`](@ref): generates an (optionally executed) notebook -- [`Literate.script`](@ref): generates a plain script file, removing all metadata +- [`Literate.markdown`](@ref) generates a markdown file. Code snippets can be executed and + the results included in the output. +- [`Literate.notebook`](@ref) generates a notebook. Code snippets can be executed and + the results included in the output. +- [`Literate.script`](@ref) generates a plain script file scrubbed from all metadata and special syntax. ### Why? diff --git a/docs/src/outputformats.md b/docs/src/outputformats.md index 7cef1c64..0f1a6169 100644 --- a/docs/src/outputformats.md +++ b/docs/src/outputformats.md @@ -29,8 +29,32 @@ an `@meta` block have been added, that sets the `EditURL` variable. This is used by Documenter to redirect the "Edit on GitHub" link for the page, see [Interaction with Documenter](@ref). -See the section about [Configuration](@ref) for how to configure the behavior and resulting -output of [`Literate.markdown`](@ref). +It possible to configure `Literate.markdown` to also evaluate code snippets, capture the +result and include it in the output, by passing `execute=true` as a keyword argument. +The result of the first code-block in the example above would then become +````markdown +```julia +x = 1//3 +``` +``` +1//3 +``` +```` + +In this example the output is just plain text. However, if the resulting value of the code +block can be displayed as an image (png or jpeg) Literate will include the image +representation of the output. + +!!! note + Since Documenter executes and captures results of `@example` block it is not necessary + to use `execute=true` for markdown output that is meant to be used as input to + Documenter. + +!!! compat "Literate 2.3" + Code execution of markdown output requires at least Literate version 2.3. + +See the section about [Configuration](@ref) for more information about how to configure the +behavior and resulting output of [`Literate.markdown`](@ref). ```@docs Literate.markdown diff --git a/docs/src/pipeline.md b/docs/src/pipeline.md index 9c710610..2428e25a 100644 --- a/docs/src/pipeline.md +++ b/docs/src/pipeline.md @@ -1,4 +1,4 @@ -# **3.** Processing pipeline +# [**3.** Processing pipeline](@id Processing-pipeline) The generation of output follows the same pipeline for all output formats: 1. [Pre-processing](@ref) diff --git a/src/Literate.jl b/src/Literate.jl index 3f7ddf1a..5484d82c 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -202,7 +202,7 @@ end filename(str) = first(splitext(last(splitdir(str)))) -function create_configuration(inputfile; user_config, user_kwargs) +function create_configuration(inputfile; user_config, user_kwargs, type=nothing) # Combine user config with user kwargs user_config = Dict{String,Any}(string(k) => v for (k, v) in user_config) user_kwargs = Dict{String,Any}(string(k) => v for (k, v) in user_kwargs) @@ -216,9 +216,9 @@ function create_configuration(inputfile; user_config, user_kwargs) cfg["documenter"] = true cfg["credit"] = true cfg["keep_comments"] = false - cfg["codefence"] = get(user_config, "documenter", true) ? + cfg["execute"] = type === :md ? false : true + cfg["codefence"] = get(user_config, "documenter", true) && !get(user_config, "execute", cfg["execute"]) ? ("```@example $(get(user_config, "name", cfg["name"]))" => "```") : ("```julia" => "```") - cfg["execute"] = true # Guess the package (or repository) root url edit_commit = "master" # TODO: Make this configurable like Documenter? deploy_branch = "gh-pages" # TODO: Make this configurable like Documenter? @@ -288,8 +288,8 @@ See the manual section about [Configuration](@ref) for more information. | `documenter` | Boolean signaling that the source contains Documenter.jl elements. | `true` | See [Interaction with Documenter](@ref Interaction-with-Documenter). | | `credit` | Boolean for controlling the addition of `This file was generated with Literate.jl ...` to the bottom of the page. If you find Literate.jl useful then feel free to keep this. | `true` | | | `keep_comments` | When `true`, keeps markdown lines as comments in the output script. | `false` | Only applicable for `Literate.script`. | +| `execute` | Whether to execute and capture the output. | `true` (notebook), `false` (markdown) | Only applicable for `Literate.notebook` and `Literate.markdown`. For markdown this requires at least Literate 2.3. | | `codefence` | Pair containing opening and closing fence for wrapping code blocks. | `````"```julia" => "```"````` | If `documenter` is `true` the default is `````"```@example"=>"```"`````. | -| `execute` | Whether to execute and capture the output. | `true` | Only applicable for `Literate.notebook`. | | `devurl` | URL for "in-development" docs. | `"dev"` | See [Documenter docs](https://juliadocs.github.io/Documenter.jl/). Unused if `repo_root_url`/`nbviewer_root_url`/`binder_root_url` are set. | | `repo_root_url` | URL to the root of the repository. | - | Determined automatically on Travis CI, GitHub Actions and GitLab CI. Used for `@__REPO_ROOT_URL__`. | | `nbviewer_root_url` | URL to the root of the repository as seen on nbviewer. | - | Determined automatically on Travis CI, GitHub Actions and GitLab CI. Used for `@__NBVIEWER_ROOT_URL__`. | @@ -368,7 +368,7 @@ of possible configuration with `config` and other keyword arguments. """ function markdown(inputfile, outputdir; config::Dict=Dict(), kwargs...) # Create configuration by merging default and userdefined - config = create_configuration(inputfile; user_config=config, user_kwargs=kwargs) + config = create_configuration(inputfile; user_config=config, user_kwargs=kwargs, type=:md) # normalize paths inputfile = normpath(inputfile) @@ -401,6 +401,7 @@ function markdown(inputfile, outputdir; config::Dict=Dict(), kwargs...) content = replace_default(content, :md; config=config) # create the markdown file + sb = sandbox() chunks = parse(content) iomd = IOBuffer() continued = false @@ -417,15 +418,17 @@ function markdown(inputfile, outputdir; config::Dict=Dict(), kwargs...) write(iomd, "; continued = true") end write(iomd, '\n') - last_line = "" for line in chunk.lines write(iomd, line, '\n') - last_line = line end - if config["documenter"]::Bool && REPL.ends_with_semicolon(last_line) + if config["documenter"]::Bool && REPL.ends_with_semicolon(chunk.lines[end]) write(iomd, "nothing #hide\n") end write(iomd, codefence.second, '\n') + if config["execute"]::Bool + res = execute_markdown(sb, join(chunk.lines, '\n'), outputdir) + write(iomd, res, '\n') + end end write(iomd, '\n') # add a newline between each chunk end @@ -443,6 +446,31 @@ function markdown(inputfile, outputdir; config::Dict=Dict(), kwargs...) return outputfile end +function execute_markdown(sb::Module, block::String, outputdir) + r, str = execute_block(sb, block) + if r !== nothing && !REPL.ends_with_semicolon(block) + for (mime, ext) in [(MIME("image/png"), ".png"), (MIME("image/jpeg"), ".jpeg")] + if showable(mime, r) + file = string(hash(block) % UInt32) * ext + open(joinpath(outputdir, file), "w") do io + Base.invokelatest(show, io, mime, r) + end + return "![]($(file))" + end + end + io = IOBuffer() + write(io, "```\n") + Base.invokelatest(show, io, "text/plain", r) + write(io, "\n```\n") + return String(take!(io)) + elseif !isempty(str) + return "```\n" * str * "\n```\n" + else + return "" + end +end + + const JUPYTER_VERSION = v"4.3.0" parse_nbmeta(line::Pair) = parse_nbmeta(line.second) @@ -568,38 +596,14 @@ function notebook(inputfile, outputdir; config::Dict=Dict(), kwargs...) end function execute_notebook(nb) - m = Module(gensym()) - # eval(expr) is available in the REPL (i.e. Main) so we emulate that for the sandbox - Core.eval(m, :(eval(x) = Core.eval($m, x))) - # modules created with Module() does not have include defined - # abspath is needed since this will call `include_relative` - Core.eval(m, :(include(x) = Base.include($m, abspath(x)))) - - io = IOBuffer() - + sb = sandbox() execution_count = 0 for cell in nb["cells"] cell["cell_type"] == "code" || continue execution_count += 1 cell["execution_count"] = execution_count block = join(cell["source"]) - # r is the result - # status = (true|false) - # _: backtrace - # str combined stdout, stderr output - r, status, _, str = Documenter.withoutput() do - include_string(m, block) - end - if !status - error(""" - $(sprint(showerror, r)) - when executing the following code block - - ```julia - $block - ``` - """) - end + r, str = execute_block(sb, block) # str should go into stream if !isempty(str) @@ -635,4 +639,37 @@ function execute_notebook(nb) return nb end +# Create a sandbox module for evaluation +function sandbox() + m = Module(gensym()) + # eval(expr) is available in the REPL (i.e. Main) so we emulate that for the sandbox + Core.eval(m, :(eval(x) = Core.eval($m, x))) + # modules created with Module() does not have include defined + # abspath is needed since this will call `include_relative` + Core.eval(m, :(include(x) = Base.include($m, abspath(x)))) + return m +end + +# Execute a code-block in a module and capture stdout/stderr and the result +function execute_block(sb::Module, block::String) + # r is the result + # status = (true|false) + # _: backtrace + # str combined stdout, stderr output + r, status, _, str = Documenter.withoutput() do + include_string(sb, block) + end + if !status + error(""" + $(sprint(showerror, r)) + when executing the following code block + + ```julia + $block + ``` + """) + end + return r, str +end + end # module diff --git a/test/runtests.jl b/test/runtests.jl index 924cdf83..dec04f73 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -644,6 +644,44 @@ end end @test !occursin("name: inputfile", markdown) @test !occursin("name: @__NAME__", markdown) + # execute + write(inputfile, """ + 1+1 + #- + [1 2; 3 4] + #- + struct PNG end + Base.show(io::IO, mime::MIME"image/png", ::PNG) = print(io, "PNG") + PNG() + #- + struct JPEG end + Base.show(io::IO, mime::MIME"image/jpeg", ::JPEG) = print(io, "JPEG") + JPEG() + #- + print("hello"); print(stdout, ", "); print(stderr, "world") + #- + print("hej, världen") + 42 + #- + 123+123; + #- + nothing + #- + print("hello there") + nothing + """) + Literate.markdown(inputfile, outdir; execute=true) + markdown = read(joinpath(outdir, "inputfile.md"), String) + @test occursin("```\n2\n```", markdown) # text/plain + @test occursin("```\n2×2 Array{$(Int),2}:\n 1 2\n 3 4\n```", markdown) # text/plain + @test occursin(r"!\[\]\(\d+\.png\)", markdown) # image/png + @test occursin(r"!\[\]\(\d+\.jpeg\)", markdown) # image/jpeg + @test occursin("```\nhello, world\n```", markdown) # stdout/stderr + @test occursin("```\n42\n```", markdown) # result over stdout/stderr + @test !occursin("246", markdown) # empty output because trailing ; + @test !occursin("```\nnothing\n```", markdown) # empty output because nothing as return value + @test occursin("```\nhello there\n```", markdown) # nothing as return value, non-empty stdout + # verify that inputfile exists @test_throws ArgumentError Literate.markdown("nonexistent.jl", outdir) end @@ -1029,6 +1067,15 @@ end end @test occursin("Link to repo root: www.example1.com/file.jl", script) @test occursin("Link to nbviewer: www.example2.com/file.jl", script) @test occursin("Link to binder: www.example3.com/file.jl", script) + + # Misc default configs + create(; kw...) = Literate.create_configuration(inputfile; user_config=Dict(), user_kwargs=kw) + cfg = create(; type=:md, execute=true) + @test cfg["execute"] + @test cfg["codefence"] == ("```julia" => "```") + cfg = create(; type=:md, execute=false) + @test !cfg["execute"] + @test cfg["codefence"] == ("```@example inputfile" => "```") end end end end