Permalink
Browse files

truncate excess output per execution request (#463)

* truncate excess output per execution request

fixes timholy/Images.jl#558, fixes #455

* apply to both stderr and stdout; rename accessor function
  • Loading branch information...
1 parent 4cd1a63 commit e9699c4d1194559df147fe069467fa9a56b4ad71 @tlnagy tlnagy committed with stevengj Sep 21, 2016
Showing with 48 additions and 5 deletions.
  1. +10 −0 README.md
  2. +13 −0 src/IJulia.jl
  3. +4 −0 src/execute_request.jl
  4. +21 −5 src/stdio.jl
View
@@ -250,3 +250,13 @@ your `.julia` package directory) to change the line `verbose = false`
at the top to `verbose = true` and `const capture_stderr = true` to
`const capture_stderr = false`. Then restart the kernel or open a new
notebook and look for the error message when IJulia dies.
+
+### Preventing truncation of output
+
+The new default behavior of IJulia is to truncate stdout (via `show` or `println`)
+after 512kb. This to prevent browsers from getting bogged down when displaying the
+results. This limit can be increased to a custom value, like 1MB, as follows
+
+```
+IJulia.set_max_stdio(1 << 20)
+```
View
@@ -284,6 +284,19 @@ function clear_output(wait=false)
Dict("wait" => wait)))
end
+
+"""
+ set_max_stdio(max_output::Integer)
+
+Sets the maximum number of bytes, `max_output`, that can be written to stdout and
+stderr before getting truncated. A large value here allows a lot of output to be
+displayed in the notebook, potentially bogging down the browser.
+"""
+function set_max_stdio(max_output::Integer)
+ max_output_per_request[] = max_output
+end
+
+
#######################################################################
include("init.jl")
@@ -111,6 +111,9 @@ end
# global variable so that display can be done in the correct Msg context
execute_msg = Msg(["julia"], Dict("username"=>"julia", "session"=>"????"), Dict())
+# global variable tracking the number of bytes written in the current execution
+# request
+const stdio_bytes = Ref(0)
function helpcode(code::AbstractString)
code_ = strip(code)
@@ -132,6 +135,7 @@ function execute_request(socket, msg)
@vprintln("EXECUTING ", code)
global execute_msg = msg
global n, In, Out, ans
+ stdio_bytes[] = 0
silent = msg.content["silent"]
store_history = get(msg.content, "store_history", !silent)
empty!(execute_payloads)
View
@@ -31,12 +31,18 @@ end
#name=>iobuffer for each stream ("stdout","stderr") so they can be sent in flush
const bufs = Dict{String,IOBuffer}()
const stream_interval = 0.1
+# maximum number of bytes in libuv/os buffer before emptying
const max_bytes = 10*1024
+# max output per code cell is 512 kb by default
+const max_output_per_request = Ref(1 << 19)
-"""Continually read from (size limited) Libuv/OS buffer into an (effectively unlimited) `IObuffer`
-to avoid problems when the Libuv/OS buffer gets full (https://github.com/JuliaLang/julia/issues/8789).
-Send data immediately when buffer contains more than `max_bytes` bytes. Otherwise, if data is available
-it will be sent every `stream_interval` seconds (see the Timers set up in watch_stdio)"""
+"""
+Continually read from (size limited) Libuv/OS buffer into an `IObuffer` to avoid problems when
+the Libuv/OS buffer gets full (https://github.com/JuliaLang/julia/issues/8789). Send data immediately
+when buffer contains more than `max_bytes` bytes. Otherwise, if data is available it will be sent every
+`stream_interval` seconds (see the Timers set up in watch_stdio). Truncate the output to `max_output_per_request`
+bytes per execution request since excessive output can bring browsers to a grinding halt.
+"""
function watch_stream(rd::IO, name::AbstractString)
task_local_storage(:IJulia_task, "read $name task")
try
@@ -45,7 +51,17 @@ function watch_stream(rd::IO, name::AbstractString)
while !eof(rd) # blocks until something is available
nb = nb_available(rd)
if nb > 0
- write(buf, read(rd, nb))
+ stdio_bytes[] += nb
+ # if this stream has surpassed the maximum output limit then ignore future bytes
+ if stdio_bytes[] >= max_output_per_request[]
+ read(rd, nb) # read from libuv/os buffer and discard
+ if stdio_bytes[] - nb < max_output_per_request[]
+ send_ipython(publish[], msg_pub(execute_msg, "stream",
+ @compat Dict("name" => "stderr", "text" => "Excessive output truncated after $(stdio_bytes[]) bytes.")))
+ end
+ else
+ write(buf, read(rd, nb))
+ end
end
if buf.size > 0
if buf.size >= max_bytes

0 comments on commit e9699c4

Please sign in to comment.