Convert example scenarios into full end-to-end tests and add GitHub workflows CI

.github/workflows/continuous_integration.yml

@@ -0,0 +1,25 @@
+name: Continuous Integration
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v2.3.1
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+          # cache: 'pip' # uncomment when https://github.com/actions/cache/issues/698 fixed
+
+      - run: pip install -r requirements.txt
+
+      - run: python -m pytest

.gitignore

@@ -136,4 +136,7 @@ scantron-reader
 *.exe
 
 # Manual
-manual.pdf
+manual.pdf
+
+# E2E Test Output
+/test/end-to-end/**/actual_output

code/data_exporting.py

@@ -25,8 +25,8 @@
 KEY_NOT_FOUND_MESSAGE = "NO KEY FOUND"
 
 
-def format_timestamp_for_file(timestamp: datetime) -> str:
-    return timestamp.isoformat(sep="_").replace(":", "-")
+def format_timestamp_for_file(timestamp: tp.Optional[datetime]) -> str:
+    return timestamp.isoformat(sep="_").replace(":", "-") + "__" if timestamp else ""
 
 
 def make_dir_if_not_exists(path: pathlib.Path):
@@ -72,10 +72,10 @@ def __init__(self, columns: tp.List[RealOrVirtualField], num_questions: int):
         self.row_count = 0
 
     def save(self, path: pathlib.PurePath, filebasename: str, sort: bool,
-             timestamp: datetime, transpose: bool = False) -> pathlib.PurePath:
+             timestamp: tp.Optional[datetime], transpose: bool = False) -> pathlib.PurePath:
         if sort:
             self.sortByName()
-        output_path = path / f"{format_timestamp_for_file(timestamp)}__{filebasename}.csv"
+        output_path = path / f"{format_timestamp_for_file(timestamp)}{filebasename}.csv"
         data = self.data
         if(transpose):
             data = list_utils.transpose(data)

code/file_handling.py

@@ -4,6 +4,7 @@
 import typing as tp
 
 from image_utils import SUPPORTED_IMAGE_EXTENSIONS
+from str_utils import strip_double_quotes
 
 
 def list_file_paths(directory: pathlib.Path) -> tp.List[pathlib.Path]:
@@ -39,3 +40,10 @@ def filter_by_extensions(files: tp.Sequence[pathlib.Path],
 def filter_images(files: tp.Sequence[pathlib.Path]) -> tp.List[pathlib.Path]:
     """Filter a list of Paths and return only the images."""
     return filter_by_extensions(files, SUPPORTED_IMAGE_EXTENSIONS)
+
+
+
+def parse_path_arg(path_arg: str) -> pathlib.Path:
+    """Parse a path argument into a Path object, stripping quotes if present.
+    """
+    return pathlib.Path(strip_double_quotes(path_arg))

code/main.py

@@ -1,8 +1,9 @@
 import argparse
 import sys
-from pathlib import Path
+from datetime import datetime
 
 import file_handling
+from file_handling import parse_path_arg
 import grid_info as grid_i
 from process_input import process_input
 
@@ -14,16 +15,16 @@
     parser.add_argument('input_folder',
                         help='Path to a folder containing scanned input sheets.\n'
                              'Sheets with student ID of "9999999999" treated as keys. Ignores subfolders.',
-                        type=Path)
+                        type=parse_path_arg)
     parser.add_argument('output_folder',
                         help='Path to a folder to save result to.',
-                        type=Path)
+                        type=parse_path_arg)
     parser.add_argument('--anskeys',
                         help='Answer Keys CSV file path. If given, will be used over other keys.',
-                        type=Path)
+                        type=parse_path_arg)
     parser.add_argument('--formmap',
                         help='Form Arrangement Map CSV file path. If given, only one answer key may be provided.',
-                        type=Path)
+                        type=parse_path_arg)
     parser.add_argument('--variant',
                         default='75',
                         choices=['75', '150'],
@@ -43,6 +44,9 @@
     parser.add_argument('--mcta',
                         action='store_true',
                         help='Output additional files for Multiple Choice Test Analysis.')
+    parser.add_argument('--disable-timestamps',
+                        action='store_true',
+                        help='Disable timestamps in file names. Useful when consistent file names are required. Existing files will be overwritten without warning!')
 
     # prints help and exits when called w/o arguments
     if len(sys.argv) == 1:
@@ -61,6 +65,8 @@
     output_mcta = args.mcta
     debug_mode_on = args.debug
     form_variant = grid_i.form_150q if args.variant == '150' else grid_i.form_75q
+    files_timestamp = datetime.now().replace(microsecond=0) if not args.disable_timestamps else None
+    print(arrangement_file)
     process_input(image_paths,
                   output_folder,
                   multi_answers_as_f,
@@ -71,4 +77,5 @@
                   output_mcta,
                   debug_mode_on,
                   form_variant,
-                  None)
+                  None,
+                  files_timestamp)

code/main_gui.py

@@ -3,6 +3,7 @@
 import user_interface
 import sys
 from process_input import process_input
+from datetime import datetime
 
 user_input = user_interface.MainWindow()
 if (user_input.cancelled):
@@ -21,6 +22,7 @@
 debug_mode_on = user_input.debug_mode
 form_variant = grid_i.form_150q if user_input.form_variant == user_interface.FormVariantSelection.VARIANT_150_Q else grid_i.form_75q
 progress_tracker = user_input.create_and_pack_progress(maximum=len(image_paths))
+files_timestamp = datetime.now().replace(microsecond=0)
 
 process_input(image_paths,
               output_folder,
@@ -32,4 +34,5 @@
               output_mcta,
               debug_mode_on,
               form_variant,
-              progress_tracker)
+              progress_tracker,
+              files_timestamp)

code/mcta_processing.py

@@ -1,6 +1,6 @@
 import typing as tp
 import pathlib
-import datetime
+from datetime import datetime
 import itertools
 
 from data_exporting import format_timestamp_for_file, save_csv, OutputSheet
@@ -10,7 +10,7 @@
 
 def transform_and_save_mcta_output(answers_results: OutputSheet,
                                    keys_results: OutputSheet,
-                                   files_timestamp: datetime,
+                                   files_timestamp: tp.Optional[datetime],
                                    output_folder: pathlib.Path):
     """Generate and save files that are specific to a downstream Multiple Choice Test Analysis
     software. The format of these files is dependend on the downstream software, so they are not
@@ -19,7 +19,7 @@ def transform_and_save_mcta_output(answers_results: OutputSheet,
     create_answers_files(answers_results, output_folder, files_timestamp)
 
 
-def create_keys_files(keys_results: OutputSheet, output_folder: pathlib.Path, files_timestamp: datetime):
+def create_keys_files(keys_results: OutputSheet, output_folder: pathlib.Path, files_timestamp: tp.Optional[datetime]):
     """Create the key files for the Multiple Choice Test Analysis software.
 
     Params:
@@ -37,7 +37,7 @@ def create_keys_files(keys_results: OutputSheet, output_folder: pathlib.Path, fi
 
 def create_answers_files(answers_results: OutputSheet,
                          output_folder: pathlib.Path,
-                         files_timestamp: datetime):
+                         files_timestamp: tp.Optional[datetime]):
     """Create the answer files for the Multiple Choice Test Analysis software.
 
     Params:
@@ -93,6 +93,6 @@ def build_answers_csv(data: tp.List[tp.Tuple[int, tp.List[str]]]) -> tp.List[tp.
 def save_mcta_csv(data: tp.List[tp.List[str]],
                   path: pathlib.PurePath,
                   basefilename: str,
-                  timestamp: datetime):
-    filename = path / f"{format_timestamp_for_file(timestamp)}__mcta_{basefilename}.csv"
+                  timestamp: tp.Optional[datetime]):
+    filename = path / f"{format_timestamp_for_file(timestamp)}mcta_{basefilename}.csv"
     save_csv(data, filename)

code/process_input.py

@@ -24,7 +24,8 @@ def process_input(
         output_mcta: bool,
         debug_mode_on: bool,
         form_variant: grid_i.FormVariant,
-        progress_tracker: tp.Optional[ProgressTrackerWidget]):
+        progress_tracker: tp.Optional[ProgressTrackerWidget],
+        files_timestamp: tp.Optional[datetime]):
     """Takes input as parameters and process it for either gui or cli.
 
     Parameter progress_tracker determines whith interface in use.
@@ -39,10 +40,8 @@ def process_input(
 
     rejected_files = data_exporting.OutputSheet([grid_i.Field.IMAGE_FILE], 0)
 
-    files_timestamp = datetime.now().replace(microsecond=0)
-
     debug_dir = output_folder / (
-            data_exporting.format_timestamp_for_file(files_timestamp) + "__debug")
+            data_exporting.format_timestamp_for_file(files_timestamp) + "debug")
     if debug_mode_on:
         data_exporting.make_dir_if_not_exists(debug_dir)
 

code/str_utils.py

@@ -6,3 +6,9 @@ def trim_middle_to_len(string: str, length: int, start: int = 15):
     if difference > 0:
         return string[:start] + " ... " + string[start + difference + 5:]
     return string
+
+
+def strip_double_quotes(string: str) -> str:
+    if string[0] == '"' and string[-1] == '"':
+        return string[1:-1]
+    return string

requirements.txt

@@ -1,7 +1,7 @@
 PyInstaller==3.5
 typing==3.7.4.1
-numpy==1.17.2
-pathlib==1.0.1
-opencv-python==4.2.0.32
+numpy==1.21.4
+opencv-python==4.5.4.60
 flake8==3.7.8
 yapf==0.28.0
+pytest==6.2.5

test/end-to-end/150q-core/args.txt

@@ -0,0 +1 @@
+--variant 150

test/end-to-end/150q-core/output/keys.csv

@@ -0,0 +1,2 @@
+Test Form Code,Source File,Q1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q12,Q13,Q14,Q15,Q16,Q17,Q18,Q19,Q20,Q21,Q22,Q23,Q24,Q25,Q26,Q27,Q28,Q29,Q30,Q31,Q32,Q33,Q34,Q35,Q36,Q37,Q38,Q39,Q40,Q41,Q42,Q43,Q44,Q45,Q46,Q47,Q48,Q49,Q50,Q51,Q52,Q53,Q54,Q55,Q56,Q57,Q58,Q59,Q60,Q61,Q62,Q63,Q64,Q65,Q66,Q67,Q68,Q69,Q70,Q71,Q72,Q73,Q74,Q75,Q76,Q77,Q78,Q79,Q80,Q81,Q82,Q83,Q84,Q85,Q86,Q87,Q88,Q89,Q90,Q91,Q92,Q93,Q94,Q95,Q96,Q97,Q98,Q99,Q100,Q101,Q102,Q103,Q104,Q105,Q106,Q107,Q108,Q109,Q110,Q111,Q112,Q113,Q114,Q115,Q116,Q117,Q118,Q119,Q120,Q121,Q122,Q123,Q124,Q125,Q126,Q127,Q128,Q129,Q130,Q131,Q132,Q133,Q134,Q135,Q136,Q137,Q138,Q139,Q140,Q141,Q142,Q143,Q144,Q145,Q146,Q147,Q148,Q149,Q150
+,answer_key.jpg,A,B,C,D,E,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,

test/end-to-end/150q-core/output/results.csv

@@ -0,0 +1,3 @@
+Last Name,First Name,Middle Name,Test Form Code,Student ID,Course ID,Source File,Q1,Q2,Q3,Q4,Q5
+,,,,12345678,,a.jpg,A,B,B,D,E
+,,,,12334235,,b.jpg,A,,C,D,E

test/end-to-end/150q-core/output/scores.csv

@@ -0,0 +1,3 @@
+Last Name,First Name,Middle Name,Test Form Code,Student ID,Course ID,Source File,Total Score (%),Total Points,Q1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q12,Q13,Q14,Q15,Q16,Q17,Q18,Q19,Q20,Q21,Q22,Q23,Q24,Q25,Q26,Q27,Q28,Q29,Q30,Q31,Q32,Q33,Q34,Q35,Q36,Q37,Q38,Q39,Q40,Q41,Q42,Q43,Q44,Q45,Q46,Q47,Q48,Q49,Q50,Q51,Q52,Q53,Q54,Q55,Q56,Q57,Q58,Q59,Q60,Q61,Q62,Q63,Q64,Q65,Q66,Q67,Q68,Q69,Q70,Q71,Q72,Q73,Q74,Q75,Q76,Q77,Q78,Q79,Q80,Q81,Q82,Q83,Q84,Q85,Q86,Q87,Q88,Q89,Q90,Q91,Q92,Q93,Q94,Q95,Q96,Q97,Q98,Q99,Q100,Q101,Q102,Q103,Q104,Q105,Q106,Q107,Q108,Q109,Q110,Q111,Q112,Q113,Q114,Q115,Q116,Q117,Q118,Q119,Q120,Q121,Q122,Q123,Q124,Q125,Q126,Q127,Q128,Q129,Q130,Q131,Q132,Q133,Q134,Q135,Q136,Q137,Q138,Q139,Q140,Q141,Q142,Q143,Q144,Q145,Q146,Q147,Q148,Q149,Q150
+,,,,12345678,,a.jpg,80.0,4,1,1,0,1,1
+,,,,12334235,,b.jpg,80.0,4,1,0,1,1,1

test/end-to-end/75q-core-1/output/results.csv

@@ -0,0 +1,18 @@
+Last Name,First Name,Middle Name,Test Form Code,Student ID,Course ID,Source File,Q1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q12,Q13,Q14,Q15,Q16,Q17,Q18,Q19,Q20,Q21,Q22,Q23,Q24
+STUDENT,A,,,0123456789,02468,scanned_page (1).png,D,C,B,D,D,A,C,C,D,B,D,D,D,C,B,D,D,A,C,C,D,B,D,D
+STUDENT,B,,,0123456790,02468,scanned_page (17).png,D,D,B,D,C,D,C,C,A,A,A,A,D,D,B,D,C,D,C,C,A,A,A,A
+STUDENT,C,,,0123456791,0246,scanned_page (16).png,C,C,C,D,A,B,D,D,C,B,D,D,C,C,C,D,A,B,B,D,C,B,D,D
+STUDENT,D,,,0123456792,02468,scanned_page (15).png,C,B,B,D,C,B,B,C,B,A,C,B,C,B,B,D,C,B,B,C,B,A,C,B
+STUDENT,E,,,0123456793,,scanned_page (14).png,A,C,B,D,C,B,B,A,A,B,C,D,A,C,B,D,C,B,B,A,A,B,C,D
+STUDENT,F,,,0123456794,02468,scanned_page (13).png,B,C,D,D,C,B,A,C,B,B,C,D,B,C,D,D,C,B,A,C,B,B,C,D
+STUDENT,G,,,0123456795,02468,scanned_page (12).png,D,C,D,D,C,B,B,C,B,B,C,A,D,C,D,D,C,B,B,C,B,B,C,B
+STUDENT,H,,,0123456796,02468,scanned_page (11).png,D,D,C,D,B,C,B,A,A,B,A,B,D,D,C,D,B,C,B,A,A,B,A,B
+STUDENT,I,,,,,scanned_page (10).png,D,C,D,D,C,B,B,B,A,D,C,D,D,C,D,D,C,B,B,B,A,D,C,D
+STUDENT,J,,,,,scanned_page (9).png,D,C,D,B,C,B,C,C,A,A,C,D,D,C,D,B,C,B,C,C,A,A,C,D
+STUDENT,K,,,,,scanned_page (8).png,D,C,C,D,B,A,B,A,C,D,A,B,D,C,C,D,B,A,B,A,C,B,A,D
+STUDENT,L,,,,,scanned_page (7).png,C,D,B,B,C,D,D,B,C,A,C,B,C,D,B,B,C,D,D,B,C,A,C,B
+STUDENT,M,,,,,scanned_page (6).png,A,C,D,D,C,B,B,B,A,B,A,D,A,C,D,D,C,B,B,B,A,B,A,D
+STUDENT,N,,,,,scanned_page (5).png,C,C,D,D,C,B,B,D,B,D,A,D,C,C,B,D,C,B,B,D,B,D,A,D
+STUDENT,O,,,,,scanned_page (4).png,D,C,B,D,C,B,B,C,B,B,A,D,D,C,B,D,C,B,B,C,B,B,A,D
+STUDENT,P,,,,,scanned_page (3).png,C,C,A,B,A,B,B,D,C,B,D,B,C,C,A,B,A,B,B,D,C,B,D,B
+STUDENT,Q,,,,,scanned_page (2).png,B,B,B,D,C,B,B,C,A,B,A,B,B,B,B,D,C,B,B,C,A,B,A,B

test/end-to-end/75q-core-2/args.txt

@@ -0,0 +1 @@
+--empty --multiple

test/end-to-end/75q-core-2/output/results.csv

@@ -0,0 +1,19 @@
+Last Name,First Name,Middle Name,Test Form Code,Student ID,Course ID,Source File,Q1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q12,Q13,Q14,Q15,Q16,Q17,Q18,Q19,Q20,Q21,Q22,Q23,Q24
+STUDENT,A,,B,0123456789,02468,sample scanned sheets_Page_18.jpg,D,C,B,D,D,A,C,C,D,B,D,D,D,C,B,D,D,A,C,C,D,B,D,D
+STUDENT,B,,B,0123456790,02468,sample scanned sheets_Page_17.jpg,D,D,B,D,C,D,C,C,A,A,A,A,D,D,B,D,C,D,C,C,A,A,A,A
+STUDENT,C,,B,0123456791,0246,sample scanned sheets_Page_16.jpg,C,C,C,D,A,B,D,D,C,B,D,D,C,C,C,D,A,B,B,D,C,B,D,D
+STUDENT,D,,A,0123456792,02468,sample scanned sheets_Page_15.jpg,C,B,B,D,C,B,B,C,B,A,C,B,C,B,B,D,C,B,B,C,B,A,C,B
+STUDENT,E,,B,0123456793,,sample scanned sheets_Page_14.jpg,A,C,B,D,C,B,B,A,A,B,C,D,A,C,B,D,C,B,B,A,A,B,C,D
+STUDENT,F,,B,0123456794,02468,sample scanned sheets_Page_13.jpg,B,C,D,D,C,B,A,C,B,B,C,D,B,C,D,D,C,B,A,C,B,B,C,D
+STUDENT,G,,C,0123456795,02468,sample scanned sheets_Page_12.jpg,D,C,D,D,C,B,B,C,B,B,C,A,D,C,D,D,C,B,B,C,B,B,C,B
+STUDENT,H,,C,0123456796,02468,sample scanned sheets_Page_11.jpg,D,D,C,D,B,C,B,A,A,B,A,B,D,D,C,D,B,C,B,A,A,B,A,B
+STUDENT,I,,B,888,222,sample scanned sheets_Page_01.jpg,D,C,D,D,C,B,B,B,A,D,C,D,D,C,D,D,C,B,B,B,A,D,C,D
+STUDENT,J,,B,6666666666,33333,sample scanned sheets_Page_02.jpg,D,C,D,B,C,B,C,C,A,A,C,D,D,C,D,B,C,B,C,C,A,A,C,D
+STUDENT,K,,B,444,321,sample scanned sheets_Page_03.jpg,D,C,C,D,B,A,B,A,C,D,A,B,D,C,C,D,B,A,B,A,C,B,A,D
+STUDENT,L,,B,555,11,sample scanned sheets_Page_04.jpg,C,D,B,B,C,D,D,B,C,A,C,B,C,D,B,B,C,D,D,B,C,A,C,B
+STUDENT,M,,B,222,233,sample scanned sheets_Page_05.jpg,A,C,D,D,C,B,B,B,A,B,A,D,A,C,D,D,C,B,B,B,A,B,A,D
+STUDENT,N,,C,2345,1233,sample scanned sheets_Page_06.jpg,C,C,D,D,C,B,B,D,B,D,A,D,C,C,B,D,C,B,B,D,B,D,A,D
+STUDENT,O,,A,0122,1222,sample scanned sheets_Page_07.jpg,D,C,B,D,C,B,B,C,B,B,A,D,D,C,B,D,C,B,B,C,B,B,A,D
+STUDENT,P,,B,0122222222,1222,sample scanned sheets_Page_08.jpg,C,C,A,B,A,B,B,D,C,B,D,B,C,C,A,B,A,B,B,D,C,B,D,B
+STUDENT,Q,,B,1222222222,01234,sample scanned sheets_Page_09.jpg,B,B,B,D,C,B,B,C,A,B,A,B,B,B,B,D,C,B,B,C,A,B,A,B
+ZZZZZZZZZZZZ,,,B,,,sample scanned sheets_Page_10.jpg,B,C,D,D,C,B,A,C,B,B,C,D,B,C,D,D,C,B,A,C,B,B,C,D

test/end-to-end/75q-core-3/output/keys.csv

@@ -0,0 +1,2 @@
+Test Form Code,Source File,Q1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q12,Q13,Q14,Q15,Q16,Q17,Q18,Q19,Q20,Q21,Q22,Q23,Q24,Q25,Q26,Q27,Q28,Q29,Q30,Q31,Q32,Q33,Q34,Q35,Q36,Q37,Q38,Q39,Q40,Q41,Q42,Q43,Q44,Q45,Q46,Q47,Q48,Q49,Q50,Q51,Q52,Q53,Q54,Q55,Q56,Q57,Q58,Q59,Q60,Q61,Q62,Q63,Q64,Q65,Q66,Q67,Q68,Q69,Q70,Q71,Q72,Q73,Q74,Q75
+F,11.jpg,C,B,C,B,C,E,A,B,A,B,C,B,A,B,C,E,E,E,E,E,E,E,E,E,E,E,E,E,E,E,C,B,A,B,B,D,D,C,D,C,B,C,B,D,B,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,

test/end-to-end/75q-core-3/output/results.csv

@@ -0,0 +1,11 @@
+Last Name,First Name,Middle Name,Test Form Code,Student ID,Course ID,Source File,Q1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q12,Q13,Q14,Q15,Q16,Q17,Q18,Q19,Q20,Q21,Q22,Q23,Q24,Q25,Q26,Q27,Q28,Q29,Q30,Q31,Q32,Q33,Q34,Q35,Q36,Q37,Q38,Q39,Q40,Q41,Q42,Q43,Q44,Q45,Q46,Q47,Q48,Q49,Q50,Q51,Q52,Q53,Q54,Q55,Q56,Q57,Q58,Q59,Q60,Q61,Q62,Q63,Q64,Q65,Q66,Q67,Q68,Q69,Q70,Q71,Q72,Q73,Q74,Q75
+BLUE PEN,XX,ZZ,A,0000000000,99,6.jpg,B,A,C,D,E,C,B,D,E,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+DAVID,XHAFER,YN,F,0123456789,9876543210,7.jpg,C,B,D,B,C,E,A,B,A,B,C,B,A,B,C,A,B,C,B,C,B,C,B,A,B,C,D,C,B,C,C,B,A,B,B,D,D,C,D,C,B,C,B,D,B,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+ERASED,XX,ZZ,,0123456789,0102345655,5.jpg,A,B,A,C,C,D,A,C,D,B,E,C,E,B,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+MESSY,XX,ZZ,C,9897968 95,1235456666,2.jpg,A,C,,B,C,B,C,B,D,C,B,A,E,E,,,[A|B],,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,E
+MULTIPLE,XX,ZZ,F,6654989578,0123,3.jpg,A,[A|B|C|D|E],[A|C|E],[B|D],[A|B|E],A,B,C,D,E,[C|D],C,C,B,C,[D|E],[A|B],[D|E],C,B,B,C,B,A,B,A,C,D,D,C,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,C,[B|D],C,B,D,B,D,B,[A|E],B,E,C,B,C,B
+PARISI,MAHIR,SE,,013579753,001325,9.jpg,E,D,C,B,A,B,B,A,B,C,B,B,D,E,E,,E,C,D,C,C,B,C,B,C,B,A,A,A,E,A,B,C,D,E,D,C,B,A,B,A,B,D,C,B,A,B,C,D,E,C,B,C,C,C,B,B,B,B,B,A,B,C,D,E,D,E,D,C,C,C,B,B,C,B
+PATERNOSTER,AUGUST,SO,D,12153746,4512,10.jpg,A,C,B,D,C,B,C,D,E,D,[B|C],,C,C,B,A,,C,B,,C,C,B,C,A,A,E,C,B,D,A,C,B,A,A,B,C,B,C,D,E,C,B,B,[B|D],A,C,B,B,B,C,B,C,B,B,C,B,B,C,C,B,B,A,B,B,B,B,A,B,D,D,E,D,D,E
+PEN WHITEOUT,XX,ZZ,C,0246788765,002,1.jpg,A,B,B,D,C,[C|E],C,E,D,E,,B,C,B,D,D,D,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+SCRIBBLED,XX,ZZ,D,0102453212,,4.jpg,A,B,A,B,B,C,D,E,D,C,B,D,E,D,A,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+WASHINGTON,RAELYN,RY,[A|C],1357924686,0099999999,8.jpg,A,A,A,B,C,C,A,E,C,B,D,B,C,B,D,A,D,D,A,B,C,B,C,C,C,B,B,D,C,C,E,E,A,A,C,B,C,B,C,B,B,D,E,A,B,E,D,D,C,B,C,D,D,D,C,A,B,D,E,D,A,B,A,B,C,A,B,A,B,A,B,C,D,C,D

test/end-to-end/75q-core-3/output/scores.csv

@@ -0,0 +1,11 @@
+Last Name,First Name,Middle Name,Test Form Code,Student ID,Course ID,Source File,Total Score (%),Total Points,Q1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q12,Q13,Q14,Q15,Q16,Q17,Q18,Q19,Q20,Q21,Q22,Q23,Q24,Q25,Q26,Q27,Q28,Q29,Q30,Q31,Q32,Q33,Q34,Q35,Q36,Q37,Q38,Q39,Q40,Q41,Q42,Q43,Q44,Q45,Q46,Q47,Q48,Q49,Q50,Q51,Q52,Q53,Q54,Q55,Q56,Q57,Q58,Q59,Q60,Q61,Q62,Q63,Q64,Q65,Q66,Q67,Q68,Q69,Q70,Q71,Q72,Q73,Q74,Q75
+BLUE PEN,XX,ZZ,A,0000000000,99,6.jpg,NO KEY FOUND,NO KEY FOUND
+DAVID,XHAFER,YN,F,0123456789,9876543210,7.jpg,78.67,59,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
+ERASED,XX,ZZ,,0123456789,0102345655,5.jpg,NO KEY FOUND,NO KEY FOUND
+MESSY,XX,ZZ,C,9897968 95,1235456666,2.jpg,NO KEY FOUND,NO KEY FOUND
+MULTIPLE,XX,ZZ,F,6654989578,0123,3.jpg,22.67,17,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+PARISI,MAHIR,SE,,013579753,001325,9.jpg,NO KEY FOUND,NO KEY FOUND
+PATERNOSTER,AUGUST,SO,D,12153746,4512,10.jpg,NO KEY FOUND,NO KEY FOUND
+PEN WHITEOUT,XX,ZZ,C,0246788765,002,1.jpg,NO KEY FOUND,NO KEY FOUND
+SCRIBBLED,XX,ZZ,D,0102453212,,4.jpg,NO KEY FOUND,NO KEY FOUND
+WASHINGTON,RAELYN,RY,[A|C],1357924686,0099999999,8.jpg,NO KEY FOUND,NO KEY FOUND

test/end-to-end/low-resolution/output/results.csv

@@ -0,0 +1,2 @@
+Last Name,First Name,Middle Name,Test Form Code,Student ID,Course ID,Source File,Q1,Q2
+,A,,,1,,example.png,C,B

test/end-to-end/readme.md

@@ -0,0 +1,19 @@
+# End-To-End Tests
+
+The end-to-end tests run the CLI program against example datasets and compare the output CSV files
+to expected output. Unlike unit tests, the end-to-end tests run the full program from start to
+finish and test every part of execution.
+
+## Test Setup
+
+The `test_e2e.py` file runs end-to-end tests by locating all subdirectories in the `end_to_end`
+directory. Each test is defined by a set of files in its subdirectory:
+
+- `input/**` The input files including all images and keys.
+- `output/**` The expected output files to compare against. When functionality changes, these files
+  may need to be updated.
+- `args.txt` Define additional arguments to pass to the CLI. For example, putting `--variant 150q`
+  here will make the program run in 150-question mode. Leaving this file blank or not creating it
+  will result in running with the default arguments (input and out paths plus `--disable-timestamp`).
+  Any instance of `$$INPUT_DIR$$` in this file will be replaced with the full absolute input directory
+  path.

test/end-to-end/rearrangement/args.txt

@@ -0,0 +1 @@
+--formmap "$$INPUT_DIR$$arrangement.csv"