forked from ucbds-infra/otter-grader
/
execute_notebook.py
184 lines (152 loc) · 7.24 KB
/
execute_notebook.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
"""
Execution of an IPython notebook
"""
import os
import re
import ast
import copy
from unittest import mock
from contextlib import redirect_stdout, redirect_stderr
from IPython.display import display
from IPython.core.inputsplitter import IPythonInputSplitter
from .check_wrapper import CheckCallWrapper
from ..utils import hide_outputs
IGNORE_CELL_TAG = "otter_ignore"
CELL_METADATA_KEY = "otter"
def filter_ignored_cells(nb):
"""
Filters out all cells in the notebook ``nb`` that are tagged with ``otter_ignore``. Returns a copy
of the original notebook.
Args:
nb (``dict``): JSON representation of a notebook
Returns:
``dict``: the notebook with cells removed
"""
nb = copy.deepcopy(nb)
to_delete = []
for i, cell in enumerate(nb['cells']):
metadata = cell.get("metadata", {})
tags = metadata.get("tags", [])
if IGNORE_CELL_TAG in tags or metadata.get(CELL_METADATA_KEY, {}).get("ignore", False):
# del nb['cells'][i]
to_delete.append(i)
to_delete.reverse()
for i in to_delete:
del nb['cells'][i]
return nb
def execute_notebook(nb, secret='secret', initial_env=None, ignore_errors=False, cwd=None, test_dir=None, seed=None):
"""
Executes a notebook and returns the global environment that results from execution
Execute notebook & return the global environment that results from execution. If ``ignore_errors``
is ``True``, exceptions are swallowed. ``secret`` contains random digits so ``check_results`` and
``check`` are not easily modifiable. ``nb`` is passed in as a dictionary that's a parsed notebook
Args:
nb (``dict``): JSON representation of a notebook
secret (``str``, optional): randomly generated integer used to rebind check function
initial_env (``str``, optional): name of initial environment
ignore_errors (``bool``, optional): whether exceptions should be ignored
cwd (``str``, optional): working directory of execution to be appended to ``sys.path`` in
grading environment
test_dir (``str``, optional): path to directory of tests in grading environment
seed (``int``, optional): random seed for intercell seeding
Results:
``dict``: global environment resulting from executing all code of the input notebook
"""
# with hide_outputs(): # causing issues with image/svg+xml KeyError in IPython
if initial_env:
global_env = initial_env.copy()
else:
global_env = {}
# add display from IPython
global_env["display"] = display
# add dummy Notebook class so that we can collect results w/out altering how the CheckCallWrapper
# needs to function
from ..check.notebook import Notebook
notebook_class_name = f"Notebook_{secret}"
global_env[notebook_class_name] = Notebook
source = ""
if cwd:
source = f"import sys\nsys.path.append(r\"{cwd}\")\n"
exec(source, global_env)
if seed is not None:
# source += "import numpy as np\nimport random\n"
import numpy as np
import random
global_env["np"] = np
global_env["random"] = random
if test_dir is None:
test_dir = "/home/tests"
# Before rewriting AST, find cells of code that generate errors.
# One round of execution is done beforehand to mimic the Jupyter notebook style of running
# (e.g. code runs up to the point of execution).
# The reason this is workaround is introduced is because once the
# source code is parsed into an AST, there is no sense of local cells
for cell in nb['cells']:
if cell['cell_type'] == 'code':
# transform the input to executable Python
# FIXME: use appropriate IPython functions here
isp = IPythonInputSplitter(line_input_checker=False)
try:
code_lines = []
cell_source_lines = cell['source']
source_is_str_bool = False
if isinstance(cell_source_lines, str):
source_is_str_bool = True
cell_source_lines = cell_source_lines.split('\n')
for line in cell_source_lines:
# Filter out ipython magic commands
# Filter out interact widget
if not line.startswith('%'):
if "interact(" not in line and not re.search(r"otter\.Notebook\(.*?\)", line):
code_lines.append(line)
if source_is_str_bool:
code_lines.append('\n')
elif re.search(r"otter\.Notebook\(.*?\)", line):
# TODO: move this check into CheckCallWrapper
# if gradescope:
# line = re.sub(r"otter\.Notebook\(.*?\)", "otter.Notebook(\"/autograder/submission/tests\")", line)
# el
line = re.sub(r"otter\.Notebook\(.*?\)", f"otter.Notebook(\"{test_dir}\")", line)
code_lines.append(line)
if source_is_str_bool:
code_lines.append('\n')
if seed is not None:
cell_source = "np.random.seed({})\nrandom.seed({})\n".format(seed, seed) + isp.transform_cell(''.join(code_lines))
else:
cell_source = isp.transform_cell(''.join(code_lines))
# patch otter.Notebook.export so that we don't create PDFs in notebooks
# TODO: move this patch into CheckCallWrapper
m = mock.mock_open()
with mock.patch('otter.Notebook.export', m), mock.patch("otter.Notebook._log_event", m):
exec(cell_source, global_env)
source += cell_source
except:
if not ignore_errors:
raise
# add checks from metadata
otter_config = cell.get("metadata", {}).get(CELL_METADATA_KEY, {})
check_results_list_name = f"check_results_{secret}"
if otter_config.get("tests", []):
tests = otter_config.get("tests", [])
for test in tests:
source += f"\n{check_results_list_name}.append({notebook_class_name}('{test_dir}').check('{test}'))\n"
tree = ast.parse(source)
# # CODE BELOW COMMENTED OUT BECAUSE the only check function is within the Notebook class
# if find_check_assignment(tree) or find_check_definition(tree):
# # an empty global_env will fail all the tests
# return global_env
# wrap check(..) calls into a check_results_X.append(check(..))
transformer = CheckCallWrapper(secret)
tree = transformer.visit(tree)
ast.fix_missing_locations(tree)
try:
cleaned_source = compile(tree, filename="nb-ast", mode="exec")
with open(os.devnull, 'w') as f, redirect_stdout(f), redirect_stderr(f):
# patch otter.Notebook.export so that we don't create PDFs in notebooks
m = mock.mock_open()
with mock.patch('otter.Notebook.export', m), mock.patch("otter.Notebook._log_event", m):
exec(cleaned_source, global_env)
except:
if not ignore_errors:
raise
return global_env