forked from ucbds-infra/otter-grader
-
Notifications
You must be signed in to change notification settings - Fork 0
/
test_assign.py
159 lines (130 loc) · 5.45 KB
/
test_assign.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
##################################
##### Tests for otter assign #####
##################################
import unittest
import sys
import os
import shutil
import subprocess
from subprocess import PIPE
from glob import glob
from unittest.mock import patch
# from otter.argparser import get_parser
from otter.assign import main as assign
from otter.generate.token import APIClient
from otter.runner import run_otter
from . import TestCase
TEST_FILES_PATH = "test/test-assign/"
class TestAssign(TestCase):
def check_gradescope_zipfile(self, path, correct_dir_path):
# unzip the zipfile
unzip_command = ["unzip", "-o", path, "-d", TEST_FILES_PATH + "autograder"]
unzip = subprocess.run(unzip_command, stdout=PIPE, stderr=PIPE)
self.assertEqual(len(unzip.stderr), 0, unzip.stderr.decode("utf-8"))
self.assertDirsEqual(TEST_FILES_PATH + "autograder", correct_dir_path, ignore_ext=[])
# cleanup
if os.path.exists(TEST_FILES_PATH + "autograder"):
shutil.rmtree(TEST_FILES_PATH + "autograder")
def test_convert_example(self):
"""
Checks that otter assign filters and outputs correctly
"""
# run otter assign
run_assign_args = [
"assign", "--no-run-tests", TEST_FILES_PATH + "example.ipynb", TEST_FILES_PATH + "output",
]
# args = parser.parse_args(run_assign_args)
# args.func = assign
# args.func(args)
run_otter(run_assign_args)
self.assertDirsEqual(TEST_FILES_PATH + "output", TEST_FILES_PATH + "example-correct")
def test_otter_example(self):
"""
Checks that otter assign filters and outputs correctly, as well as creates a correct .otter file
"""
run_assign_args = [
"assign", TEST_FILES_PATH + "generate-otter.ipynb",
TEST_FILES_PATH + "output",
]
# args = parser.parse_args(run_assign_args)
# args.func = assign
# args.func(args)
run_otter(run_assign_args)
self.assertDirsEqual(TEST_FILES_PATH + "output", TEST_FILES_PATH + "otter-correct")
def test_pdf_example(self):
"""
Checks that otter assign filters and outputs correctly, as well as creates a correct .zip file along with PDFs
"""
run_assign_args = [
"assign", "--no-run-tests", TEST_FILES_PATH + "generate-pdf.ipynb", TEST_FILES_PATH + "output",
]
# args = parser.parse_args(run_assign_args)
# args.func = assign
# args.func(args)
run_otter(run_assign_args)
self.assertDirsEqual(TEST_FILES_PATH + "output", TEST_FILES_PATH + "pdf-correct", ignore_ext=[".pdf",".zip"])
# # check gradescope zip file
# self.check_gradescope_zipfile(
# TEST_FILES_PATH + "output/autograder/autograder.zip", TEST_FILES_PATH + "pdf-autograder-correct",
# )
@patch.object(APIClient,"get_token")
def test_gradescope_example(self, mocked_client):
"""
Checks that otter assign filters and outputs correctly, as well as creates a correct .zip file along with PDFs.
Additionally, includes testing Gradescope integration.
"""
mocked_client.return_value = 'token'
run_gradescope_args = [
"assign", "--no-run-tests", TEST_FILES_PATH + "generate-gradescope.ipynb",
TEST_FILES_PATH + "output",
]
# args = parser.parse_args(run_gradescope_args)
# args.func = assign
# args.func(args)
run_otter(run_gradescope_args)
self.assertDirsEqual(TEST_FILES_PATH + "output", TEST_FILES_PATH + "gs-correct", ignore_ext=[".pdf",".zip"])
# check gradescope zip file
self.check_gradescope_zipfile(
TEST_FILES_PATH + "output/autograder/autograder.zip", TEST_FILES_PATH + "gs-autograder-correct",
)
def test_r_example(self):
"""
Checks that otter assign works for R notebooks correctly
"""
run_assign_args = [
"assign", TEST_FILES_PATH + "r-example.ipynb",
TEST_FILES_PATH + "output",
]
# args = parser.parse_args(run_assign_args)
# args.func = assign
# args.func(args)
run_otter(run_assign_args)
self.assertDirsEqual(TEST_FILES_PATH + "output", TEST_FILES_PATH + "r-correct", ignore_ext=[".pdf",".zip"])
# # check gradescope zip file
# self.check_gradescope_zipfile(
# TEST_FILES_PATH + "output/autograder/autograder.zip", TEST_FILES_PATH + "r-autograder-correct",
# )
def test_rmd_example(self):
"""
Checks that otter assign works for Rmd files
"""
run_assign_args = [
"assign", TEST_FILES_PATH + "rmd-example.Rmd",
TEST_FILES_PATH + "output",
]
# args = parser.parse_args(run_assign_args)
# args.func = assign
# args.func(args)
run_otter(run_assign_args)
self.assertDirsEqual(TEST_FILES_PATH + "output", TEST_FILES_PATH + "rmd-correct", ignore_ext=[".zip"])
# check gradescope zip file
self.check_gradescope_zipfile(
TEST_FILES_PATH + "output/autograder/autograder.zip", TEST_FILES_PATH + "rmd-autograder-correct",
)
def tearDown(self):
"""
Removes assign output
"""
# cleanup
if os.path.exists(TEST_FILES_PATH + "output"):
shutil.rmtree(TEST_FILES_PATH + "output")