Skip to content
Permalink
Browse files
fix: provide temporary IPYTHONDIR for notebook execution in order to …
…avoid race conditions in https://github.com/ipython/ipython/blob/master/IPython/paths.py#L20 upon execution of multiple notebooks at the same time. (#1280)

* fix: provide temporary IPYTHONDIR for notebook execution in order to avoid race conditions in https://github.com/ipython/ipython/blob/master/IPython/paths.py#L20 upon execution of multiple notebooks at the same time.

* fix tmpdir usage
  • Loading branch information
johanneskoester committed Nov 26, 2021
1 parent 2883718 commit 4d70da11f810224ddce192ae1472a6380898865f
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 24 deletions.
@@ -71,37 +71,43 @@ def execute_script(self, fname, edit=None):
fname_out = os.path.join(os.getcwd(), fname_out)
output_parameter = "--output {fname_out:q}"

if edit is not None:
logger.info("Opening notebook for editing.")
cmd = (
"jupyter notebook --browser ':' --no-browser --log-level ERROR --ip {edit.ip} --port {edit.port} "
"--NotebookApp.quit_button=True {{fname:q}}".format(edit=edit)
)
else:
cmd = (
"jupyter-nbconvert --log-level ERROR --execute {output_parameter} "
"--to notebook --ExecutePreprocessor.timeout=-1 {{fname:q}}".format(
output_parameter=output_parameter
with tempfile.TemporaryDirectory() as tmp:
if edit is not None:
logger.info("Opening notebook for editing.")
cmd = (
"jupyter notebook --browser ':' --no-browser --log-level ERROR --ip {edit.ip} --port {edit.port} "
"--NotebookApp.quit_button=True {{fname:q}}".format(edit=edit)
)
else:
cmd = (
"jupyter-nbconvert --log-level ERROR --execute {output_parameter} "
"--to notebook --ExecutePreprocessor.timeout=-1 {{fname:q}}".format(
output_parameter=output_parameter,
)
)
)

if ON_WINDOWS:
fname = fname.replace("\\", "/")
fname_out = fname_out.replace("\\", "/") if fname_out else fname_out
if ON_WINDOWS:
fname = fname.replace("\\", "/")
fname_out = fname_out.replace("\\", "/") if fname_out else fname_out

self._execute_cmd(cmd, fname_out=fname_out, fname=fname)
self._execute_cmd(
cmd,
fname_out=fname_out,
fname=fname,
additional_envvars={"IPYTHONDIR": tmp},
)

if edit:
logger.info("Saving modified notebook.")
nb = nbformat.read(fname, as_version=4)
if edit:
logger.info("Saving modified notebook.")
nb = nbformat.read(fname, as_version=4)

self.remove_preamble_cell(nb)
self.remove_preamble_cell(nb)

# clean up all outputs
for cell in nb["cells"]:
cell["outputs"] = []
# clean up all outputs
for cell in nb["cells"]:
cell["outputs"] = []

nbformat.write(nb, self.local_path)
nbformat.write(nb, self.local_path)

def insert_preamble_cell(self, preamble, notebook):
import nbformat
@@ -219,6 +219,17 @@ def __new__(
envvars["TEMPDIR"] = tmpdir_resource
envvars["TEMP"] = tmpdir_resource

if "additional_envvars" in kwargs:
env = kwargs["additional_envvars"]
if not isinstance(env, dict) or not all(
isinstance(v, str) for v in env.values()
):
raise WorkflowError(
"Given environment variables for shell command have to be a dict of strings, "
"but the following was provided instead:\n{}".format(env)
)
envvars.update(env)

if conda_env and cls.conda_block_conflicting_envvars:
# remove envvars that conflict with conda
for var in ["R_LIBS", "PYTHONPATH", "PERLLIB", "PERL5LIB"]:

0 comments on commit 4d70da1

Please sign in to comment.