Skip to content
Permalink
Browse files

py3: ensure python output is not lost

Unbuffer python's stdio to avoid output getting stuck in buffer when
stdout is not a TTY. Normally buffers are flushed by `Py_Finalize` but
Boost has a long-standing issue preventing the proper shutdown of the
interpreter with `Py_Finalize` when embedded [1].

This applies the same fix as 139beba but to any ledger usage rather than
only the test suite. I removed `PYTHONUNBUFFERED=1` from tests as there
is no expectation that users should need to have this set for ledger to
function.

For example without this fix piping ledger into cat usually loses any
output (unless the output is large enough to cause the buffer to flush):

    $ ./ledger -f "test/baseline/feat-value_py3.test" reg
    <class 'bool'> True
    [...]
    $ ./ledger -f "test/baseline/feat-value_py3.test" reg | cat
    $

Interestingly `--verify` causes `python_interpreter_t` to be destroyed
-- it doesn't appear to be otherwise -- which does call `Py_Finalize`.
As expected this fixes the issue but can also crash due to the boost
issue mentioned above:

    $ ./ledger -f "test/baseline/feat-value_py3.test" --verify reg
    <class 'bool'> True
    [...]
    Segmentation fault (core dumped)
    $ ./ledger -f "test/baseline/feat-value_py3.test" --verify reg | cat
    <class 'bool'> True
    [...]
    $

1. https://www.boost.org/doc/libs/1_62_0/libs/python/doc/html/tutorial/tutorial/embedding.html
  • Loading branch information
q3aiml authored and tbm committed Mar 26, 2020
1 parent 2c77b82 commit 6d20807d1ac95feef1dc24712e16f63007311c94
Showing with 9 additions and 4 deletions.
  1. +5 −0 src/pyinterp.cc
  2. +3 −3 test/CMakeLists.txt
  3. +1 −1 test/unit/CMakeLists.txt
@@ -151,6 +151,11 @@ void python_interpreter_t::initialize()
DEBUG("python.interp", "Initializing Python");

#if PY_MAJOR_VERSION >= 3
// Unbuffer stdio to avoid python output getting stuck in buffer when
// stdout is not a TTY. Normally buffers are flushed by Py_Finalize but
// Boost has a long-standing issue preventing proper shutdown of the
// interpreter with Py_Finalize when embedded.
Py_UnbufferedStdioFlag = 1;
// PyImport_AppendInittab docs: "This should be called before Py_Initialize()".
PyImport_AppendInittab((char*)"ledger", PyInit_ledger);
#endif
@@ -34,7 +34,7 @@ macro(add_ledger_harness_tests _class)
$<TARGET_FILE:ledger> ${PROJECT_SOURCE_DIR}
${TestFile} ${TEST_PYTHON_FLAGS})
set_tests_properties(${_class}Test_${TestFile_Name}
PROPERTIES ENVIRONMENT "PYTHONUNBUFFERED=1;TZ=${Ledger_TEST_TIMEZONE}")
PROPERTIES ENVIRONMENT "TZ=${Ledger_TEST_TIMEZONE}")
endif()
endforeach()
endif()
@@ -53,7 +53,7 @@ if (Python_EXECUTABLE)
COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/${_class}.py
--ledger $<TARGET_FILE:ledger> --file ${TestFile})
set_tests_properties(${_class}Test_${TestFile_Name}
PROPERTIES ENVIRONMENT "PYTHONUNBUFFERED=1;TZ=${Ledger_TEST_TIMEZONE}")
PROPERTIES ENVIRONMENT "TZ=${Ledger_TEST_TIMEZONE}")
endforeach()

# CheckManpage and CheckTexinfo are disabled, since they do not work
@@ -64,7 +64,7 @@ if (Python_EXECUTABLE)
COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/${_class}.py
--ledger $<TARGET_FILE:ledger> --source ${PROJECT_SOURCE_DIR})
set_tests_properties(${_class}
PROPERTIES ENVIRONMENT "PYTHONUNBUFFERED=1;TZ=${Ledger_TEST_TIMEZONE}")
PROPERTIES ENVIRONMENT "TZ=${Ledger_TEST_TIMEZONE}")
endforeach()
endif()

@@ -2,7 +2,7 @@ macro(add_ledger_test _name)
target_link_libraries(${_name} libledger)
add_test(Ledger${_name} ${PROJECT_BINARY_DIR}/${_name})
set_tests_properties(Ledger${_name}
PROPERTIES ENVIRONMENT "PYTHONUNBUFFERED=1;TZ=${Ledger_TEST_TIMEZONE}")
PROPERTIES ENVIRONMENT "TZ=${Ledger_TEST_TIMEZONE}")
endmacro(add_ledger_test _name)

include_directories(${PROJECT_SOURCE_DIR}/src)

0 comments on commit 6d20807

Please sign in to comment.
You can’t perform that action at this time.