-
Notifications
You must be signed in to change notification settings - Fork 0
/
checker.py
344 lines (303 loc) · 14.8 KB
/
checker.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
import os
import ast
import inspect
from os import listdir
from os.path import isfile, join
import Julian_To_HW4
IMPORT_NAME = Julian_To_HW4
"""
Checker Program
Checker.py is designed to run unit tests on the functions in another .py file by using
unit test data that is stored in .txt files. Checker is designed for computer science
teachers to grade student work.
In order to use Checker, you must change the import name to be the name of the .py file
that you want to grade (see lines 6 and 7)
All functions that will be tested must have 'foo_' in the function name, and must be in same
order as the unit_test_#assignmentname_#.txt files
The Python equivalent of the Java Checker / UnitTester.java programs, for checking student
code files and running unit tests for student-written functions / programs
@author Peter Olson
@version 12.1.22
"""
# ---------------------------------------------------------------------------------------------
# Convert .py to .txt. Collect function names in a list.
# --> Run functions using getattr + parameters. Probably want to decorate correct functions
# --> to run using some identifiable token, e.g. foo_<original_function_name>, search for 'foo'
# ---------------------------------------------------------------------------------------------
# Globals for functions
LIST_DELIMITER = ","
DATA_SEPARATOR = "|"
ROW_SEPARATOR = "$"
EXPECTED_IDENTIFIER = "EXPECTED"
FUNCTION_NAME_IDENTIFIER = "foo_"
UNIT_TEST_TEXT_FILE_ROOT = "unit_tests_"
# Classes
class BColors:
HEADER = '\033[95m'
OK_BLUE = '\033[94m'
OK_CYAN = '\033[96m'
OK_GREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
END_C = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
# Functions
# Given a string with elements separated by a delimiter, fill a list with the elements
def fill_list(list_1d, arg_line):
str_list = arg_line.split(LIST_DELIMITER)
for token in str_list:
list_1d.append(token)
# Given a string with rows separated by a delimiter (ROW_SEPARATOR, and elements separated
# by a delimiter (LIST_DELIMITER), fill a 2D list with the elements
def fill_2d_list(list_2d, arg_line):
row_text_list = arg_line.split(ROW_SEPARATOR)
for row in row_text_list:
row_list = []
fill_list(row_list, row)
list_2d.append(row_list)
# From a text file, get unit test input data as a list, as well as the expected value after running
# the given function
#
# @return [] --> list of unit test inputs
# @return _ --> expected output... data type varies
def get_unit_tests(text_file_name):
# Delimiters / Tokens for scraping data
function_args = []
expected_list = []
text_file = open(text_file_name, "r")
text_lines = text_file.readlines()
for text_line in text_lines:
text_line = text_line.replace("\n", "")
args_obj_list = []
args_list = text_line.split(DATA_SEPARATOR)
for arg_num in range(0, len(args_list)):
line_data = args_list[arg_num]
if ROW_SEPARATOR in line_data and EXPECTED_IDENTIFIER not in line_data:
grid_list = []
fill_2d_list(grid_list, line_data)
args_obj_list.append(grid_list)
elif LIST_DELIMITER in line_data and EXPECTED_IDENTIFIER not in line_data:
row_list = []
fill_list(row_list, line_data)
args_obj_list.append(row_list)
elif EXPECTED_IDENTIFIER in line_data:
expected_data = line_data.replace(EXPECTED_IDENTIFIER + " ", "")
if ROW_SEPARATOR in expected_data:
grid_list = []
fill_2d_list(grid_list, expected_data)
expected_list.append(grid_list)
elif LIST_DELIMITER in expected_data:
row_list = []
fill_list(row_list, expected_data)
expected_list.append(row_list)
else:
expected_list.append(expected_data)
else:
args_obj_list.append(line_data)
if EXPECTED_IDENTIFIER not in text_line:
function_args.append(args_obj_list)
return function_args, expected_list
# Round any float values or list of float values to only two decimal places. Leaves any other
# data unchanged
def round_any(value):
if isinstance(value, float):
value = round(value, 2)
elif isinstance(value, list):
if isinstance(value[0], list): # 2D list
if isinstance(value[0][0], float): # Round decimals
for row in range(0, len(value)):
for col in range(0, value[row]):
value[row][col] = round(value[row][col], 2)
else: # 1D list
if isinstance(value[0], float): # Round decimals
for value_index in range(0, len(value)):
value[index] = round(value[value_index], 2)
return value
# Convert a list of string literals to their proper data types using ast.literal_eval()
def convert_str_list_to_type(str_list):
if not isinstance(str_list[0], list):
for list_index in range(0, len(str_list)):
str_list[list_index] = ast.literal_eval(str_list[list_index])
else:
convert_str_2d_list_to_type(str_list)
def convert_str_2d_list_to_type(str_list):
for row_index in range(0, len(str_list)):
for col_index in range(0, len(str_list[row_index])):
str_list[row_index][col_index] = ast.literal_eval(str_list[row_index][col_index])
# Returns whether a function contains an explicit return statement or not
def contains_explicit_return(f):
return any(isinstance(node, ast.Return) for node in ast.walk(ast.parse(inspect.getsource(f))))
# Get files in current directory "C:\\Users\\Teacher Loaner\\PycharmProjects\\Checker"
mypath = os.getcwd()
only_files = [f for f in listdir(mypath) if isfile(join(mypath, f))]
# Get correct unit test files
assignment_name = input("Enter the assignment being tested:\n"
"(p# for project, h# for homework) - Eg. p2\n")
UNIT_TEST_TEXT_FILE_ROOT += assignment_name.lower() + "_"
# Find correct file (assuming only this file and one other .py file are in the directory),
# along with the unit test files and any other directories, such as venv
py_file_name = None
for file in only_files:
file_name = os.path.splitext(file)
if ".py" not in file_name:
continue
if "main" not in file_name and ".txt" not in file_name:
py_file_name = file_name[0] + file_name[1]
# debug: print(file_name[0] + file_name[1])
# Read file and grab correct function names with the specified token
has_correct_char_decode = False
file_reader = None
lines = []
encoding_list = ['mbcs', 'utf8', 'utf_8', 'utf_16', 'cp437', 'utf16',
'ascii', 'big5', 'big5hkscs', 'cp037', 'cp273',
'cp424', 'cp500', 'cp720', 'cp737', 'cp775', 'cp850',
'cp852', 'cp855', 'cp856', 'cp857', 'cp858', 'cp860',
'cp861', 'cp862', 'cp863', 'cp864', 'cp865', 'cp866',
'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950',
'cp1006', 'cp1026', 'cp1125', 'cp1140', 'cp1250',
'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255',
'cp1256', 'cp1257', 'cp1258', 'euc_jp', 'euc_jis_2004',
'euc_jisx0213', 'euc_kr', 'gb2312', 'gbk', 'gb18030',
'hz', 'iso2022_jp', 'iso2022_jp_1', 'iso2022_jp_2',
'iso2022_jp_2004', 'iso2022_jp_3', 'iso2022_jp_ext',
'iso2022_kr', 'latin_1', 'iso8859_2', 'iso8859_3',
'iso8859_4', 'iso8859_5', 'iso8859_6', 'iso8859_7',
'iso8859_8', 'iso8859_9', 'iso8859_10', 'iso8859_11',
'iso8859_13', 'iso8859_14', 'iso8859_15', 'iso8859_16',
'johab', 'koi8_r', 'koi8_t', 'koi8_u', 'kz1048',
'mac_cyrillic', 'mac_greek', 'mac_iceland',
'mac_latin2', 'mac_roman', 'mac_turkish',
'ptcp154', 'shift_jis', 'shift_jis_2004',
'shift_jisx0213', 'utf_32', 'utf_32_be', 'utf_32_le',
'utf_16_be', 'utf_16_le', 'utf_7', 'utf_8_sig']
encoding_list_index = 0
# Find the correct encoding
try:
file_reader = open(py_file_name, "r")
lines = file_reader.readlines()
except UnicodeDecodeError:
while not has_correct_char_decode:
try:
file_reader = open(py_file_name, "r", encoding=encoding_list[encoding_list_index])
lines = file_reader.readlines()
except UnicodeDecodeError:
encoding_list_index += 1
else:
has_correct_char_decode = True
# print("debug: encoding is", encoding_list[encoding_list_index])
else:
# print("debug: no encoding")
pass
finally:
if file_reader is None:
print("No suitable encoding found for this file.")
exit()
function_list = []
for line in lines:
function_header_identifier = "def "
if function_header_identifier in line:
start_pos = line.find("def ") + len(function_header_identifier)
end_pos = line.find("(", start_pos)
function_name = line[start_pos:end_pos].strip()
if FUNCTION_NAME_IDENTIFIER in function_name:
function_list.append(function_name)
# Test functions using unit tests that are read from the unit test text files
# Save results of expected return values vs actual result values found when running functions
# debug: print(function_list)
index = 0
has_return_values = []
shallow_copy_problem_set = [] # Holds the final values of the inputs after the function is run
expected_return_values = []
actual_return_values = []
for x in range(0, len(function_list)):
no_return = False # Keeps track of whether the function has a return
function_name_instance = function_list[x]
unit_test_text_file_name = UNIT_TEST_TEXT_FILE_ROOT + str(index) + ".txt"
# list_args is a 2D list, expected_value_list is a 1D list
list_args, expected_value_list = get_unit_tests(unit_test_text_file_name)
expected_return_values.append(expected_value_list)
function_obj = getattr(IMPORT_NAME, function_name_instance)
# If function does not have a return statement, make copy of original arguments and compare to
# arguments after the function has been run
if contains_explicit_return(function_obj):
has_return_values.append(True)
else:
has_return_values.append(False)
# Convert arguments to their proper data type
# Convert list arguments
for unit_test_data_list_index in range(0, len(list_args)):
for unit_test_value_index in range(0, len(list_args[unit_test_data_list_index])):
unit_test_value = list_args[unit_test_data_list_index][unit_test_value_index]
if not isinstance(unit_test_value, list):
unit_test_value = ast.literal_eval(unit_test_value)
else: # Convert elements inside list to their proper type
convert_str_list_to_type(unit_test_value)
list_args[unit_test_data_list_index][unit_test_value_index] = unit_test_value
# Convert expected values
for unit_test_value_index in range(0, len(expected_value_list)):
unit_test_value = expected_value_list[unit_test_value_index]
if isinstance(unit_test_value, str):
unit_test_value = unit_test_value.lower()
if not isinstance(unit_test_value, list):
if unit_test_value.lstrip('-').replace('.', '', 1).isdigit():
unit_test_value = ast.literal_eval(unit_test_value)
# else, if we do have a string, do not call ast.literal_eval, which cannot convert
# strings to strings apparently, which is stupid
else: # Convert elements inside list to their proper type
convert_str_list_to_type(unit_test_value)
expected_value_correct_data_type = round_any(unit_test_value)
expected_value_list[unit_test_value_index] = expected_value_correct_data_type
# Run all unit tests
actual_value_list = []
shallow_copy = []
original_input_values = []
for args in list_args:
if has_return_values[x]: # Grab list of non-lists
shallow_copy.append(args)
else: # Grab the first list in the list of arguments
for arg_list in args:
if isinstance(arg_list, list):
shallow_copy.append(args[0])
break
actual_value = function_obj(*args) # Run the function with the arguments; * expands into args
actual_value = round_any(actual_value) # Round to 2 decimal places, if it is a decimal
if isinstance(actual_value, str):
actual_value = actual_value.lower()
actual_value_list.append(actual_value)
actual_return_values.append(actual_value_list)
shallow_copy_problem_set.append(shallow_copy)
index += 1
# Check expected unit test results verses actual unit test results
score = 100
max_score_per_function = 100 / len(expected_return_values)
# Missing function
missing_problems_margin = 0
if len(expected_return_values) != len(actual_return_values):
score -= max_score_per_function
missing_problems_margin += len(expected_return_values) - len(actual_return_values)
for problem_number in range(0, len(expected_return_values) - missing_problems_margin):
total_unit_tests = len(expected_return_values[problem_number])
points_per_unit_test = max_score_per_function / total_unit_tests
points_per_problem = 0
for unit_test_number in range(0, total_unit_tests):
expected_result = expected_return_values[problem_number][unit_test_number]
actual_result = actual_return_values[problem_number][unit_test_number]
function_name_sans_foo = function_list[problem_number].replace("foo_", "")
if not has_return_values[problem_number]: # Compare expected input and input state instead
actual_result = shallow_copy_problem_set[problem_number][unit_test_number]
if actual_result == expected_result:
print("Unit test #", problem_number, " for ", function_name_sans_foo, " function:\n",
f"{BColors.OK_GREEN}Passed!{BColors.END_C}", " Expected: ", expected_result,
", Found: ", actual_result, sep='')
points_per_problem += points_per_unit_test
else:
print("Unit test #", problem_number, " for ", function_name_sans_foo, " function:\n",
f"{BColors.FAIL}Fail!{BColors.END_C}", " Expected: ", expected_result, ", Found: ", actual_result, sep='')
score -= points_per_unit_test
print("\n#", problem_number, f"{BColors.OK_BLUE} score: {BColors.END_C}", round(points_per_problem, 2),
"/", round(max_score_per_function, 2), "\n", sep='', end='\n')
# Print final score results
student_name = py_file_name.split(".")[0]
score = round(score, 2)
print(student_name, f"{BColors.OK_BLUE} score: {BColors.END_C}", score, sep='')