[Feature:Autograding] Automated output and input generation

.setup/data/courses/development.yml

@@ -169,7 +169,28 @@ gradeables:
   g_grade_released_date: +1010 days
   eg_regrade_request_date: +1015 days
   eg_max_random_submissions: 0
-
+
+- gradeable_config: cpp_random_input_output
+  components:
+  - gc_lower_clamp: 0
+    gc_default: 5
+    gc_max_value: 5
+    gc_upper_clamp: 5
+    gc_title: Question 1
+    marks:
+      - gcm_note: Full Credit
+        gcm_points : 0
+      - gcm_note: Question 1 Mark 1
+        gcm_points : -5
+  g_ta_view_start_date: -2 days
+  eg_submission_open_date: -1 days
+  eg_submission_due_date: +1000 days
+  g_grade_start_date: +1005 days
+  g_grade_due_date: +1010 days
+  g_grade_released_date: +1010 days
+  eg_regrade_request_date: +1015 days
+  eg_max_random_submissions: 0
+
 - gradeable_config: early_submission_incentive
   components:
   - gc_lower_clamp: 0
@@ -443,6 +464,27 @@ gradeables:
   eg_regrade_request_date: +1015 days
   eg_max_random_submissions: 0
 
+- gradeable_config: python_random_input_output
+  components:
+  - gc_lower_clamp: 0
+    gc_default: 5
+    gc_max_value: 5
+    gc_upper_clamp: 5
+    gc_title: Question 1
+    marks:
+      - gcm_note: Full Credit
+        gcm_points : 0
+      - gcm_note: Question 1 Mark 1
+        gcm_points : -5
+  g_ta_view_start_date: -2 days
+  eg_submission_open_date: -1 days
+  eg_submission_due_date: +1000 days
+  g_grade_start_date: +1005 days
+  g_grade_due_date: +1010 days
+  g_grade_released_date: +1010 days
+  eg_regrade_request_date: +1015 days
+  eg_max_random_submissions: 0
+
 - gradeable_config: test_notes_upload
   components:
   - gc_lower_clamp: 0

autograder/autograder/grade_item.py

@@ -122,15 +122,13 @@ def zip_my_directory(path,zipfilename):
             zipf.write(os.path.join(root,my_file),os.path.join(relpath,my_file))
     zipf.close()
 
-
 def unzip_this_file(zipfilename,path):
     if not os.path.exists(zipfilename):
         raise RuntimeError("ERROR: zip file does not exist '", zipfilename, "'")
     zip_ref = zipfile.ZipFile(zipfilename,'r')
     zip_ref.extractall(path)
     zip_ref.close()
 
-
 def allow_only_one_part(path, log_path=os.devnull):
     """
     Given a path to a directory, iterate through the directory and detect folders that start with
@@ -190,6 +188,14 @@ def remove_test_input_files(overall_log,test_input_path,testcase_folder):
                     print ("removing (likely) stale test_input file: ", my_file, file=f)
                     os.remove(my_file)
 
+def add_all_permissions(folder):
+    add_permissions_recursive(folder,
+                      stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
+                      stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
+                      stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH)
+
+def remove_read_permissions(top_dir):
+    os.chmod(top_dir,os.stat(top_dir).st_mode & ~stat.S_IRGRP & ~stat.S_IWGRP & ~stat.S_IXGRP & ~stat.S_IROTH & ~stat.S_IWOTH & ~stat.S_IXOTH)
 
 def grade_from_zip(my_autograding_zip_file,my_submission_zip_file,which_untrusted):
 
@@ -370,10 +376,7 @@ def grade_from_zip(my_autograding_zip_file,my_submission_zip_file,which_untruste
             add_permissions(os.path.join(testcase_folder,"my_compile.out"), stat.S_IXUSR | stat.S_IXGRP |stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH)
             #untrusted_grant_rwx_access(which_untrusted, tmp_compilation)          
             untrusted_grant_rwx_access(which_untrusted, testcase_folder)
-            add_permissions_recursive(testcase_folder,
-                      stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
-                      stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
-                      stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH)
+            add_all_permissions(testcase_folder)
 
             if USE_DOCKER:
                 try:
@@ -431,22 +434,17 @@ def grade_from_zip(my_autograding_zip_file,my_submission_zip_file,which_untruste
     else:
         print (which_machine,which_untrusted,"COMPILATION FAILURE")
         grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,message="COMPILATION FAILURE")
-    add_permissions_recursive(tmp_compilation,
-                      stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
-                      stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
-                      stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH)
-
+
+    add_all_permissions(tmp_compilation)
 
     # return to the main tmp directory
     os.chdir(tmp)
 
-
-    # --------------------------------------------------------------------
-    # make the runner directory
-
+    # --------------------------------------------------------------------    
+    # RUN INPUT GENERATION
     with open(os.path.join(tmp_logs,"overall.txt"),'a') as f:
-        print ("====================================\nRUNNER STARTS", file=f)
-        
+        print ("====================================\nINPUT GENERATION STARTS", file=f)
+
     tmp_work = os.path.join(tmp,"TMP_WORK")
     tmp_work_test_input = os.path.join(tmp_work, "test_input")
     tmp_work_test_output = os.path.join(tmp_work, "test_output")
@@ -466,6 +464,60 @@ def grade_from_zip(my_autograding_zip_file,my_submission_zip_file,which_untruste
 
     os.chdir(tmp_work)
 
+    #random_input_tmp_work = os.path.join(tmp_work,"random_input")
+    #os.mkdir(random_input_tmp_work)
+
+    with open(os.path.join(tmp_logs,"input_generator_log.txt"), 'w') as logfile:
+        for testcase_num in range(1, len(my_testcases)+1):
+            random_input_testcase_folder = os.path.join(tmp_work,"random_input", "test{:02}".format(testcase_num))
+
+            os.makedirs(random_input_testcase_folder)
+            os.chdir(random_input_testcase_folder)
+
+            # copy any instructor provided solution code files to testcase folder
+            copy_contents_into(job_id,instructor_solution_path,random_input_testcase_folder,tmp_logs)
+
+            # copy compile.out to the current directory
+            shutil.copy (os.path.join(bin_path,"solution_runner.out"),os.path.join(random_input_testcase_folder,"my_solution_runner.out"))
+            add_permissions(os.path.join(random_input_testcase_folder,"my_solution_runner.out"), stat.S_IXUSR | stat.S_IXGRP |stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH)          
+            untrusted_grant_rwx_access(which_untrusted, random_input_testcase_folder)
+            add_all_permissions(random_input_testcase_folder)
+
+            inout_generator_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"),
+                                                which_untrusted,
+                                                os.path.join(random_input_testcase_folder,"my_solution_runner.out"),
+                                                queue_obj["gradeable"],
+                                                queue_obj["who"],
+                                                str(queue_obj["version"]),
+                                                submission_string,
+                                                "input",
+                                                '--testcase', str(testcase_num)],
+                                                stdout=logfile, 
+                                                cwd=random_input_testcase_folder)
+            # remove the compilation program
+            untrusted_grant_rwx_access(which_untrusted, random_input_testcase_folder)
+            os.remove(os.path.join(random_input_testcase_folder,"my_solution_runner.out"))
+
+        if inout_generator_success == 0:
+            print (which_machine,which_untrusted,"INPUT GENERATOR OK")
+        else:
+            print (which_machine,which_untrusted,"INPUT GENERATOR FAILURE")
+            grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,message="INPUT GENERATOR FAILURE")
+
+    # return to the main tmp directory
+    os.chdir(tmp_work)
+    subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a'))
+
+    # --------------------------------------------------------------------
+
+    # --------------------------------------------------------------------
+    # make the runner directory
+
+    with open(os.path.join(tmp_logs,"overall.txt"),'a') as f:
+        print ("====================================\nRUNNER STARTS", file=f)
+
+    os.chdir(tmp_work)
+
     # move all executable files from the compilation directory to the main tmp directory
     # Note: Must preserve the directory structure of compiled files (esp for Java)
 
@@ -531,14 +583,58 @@ def grade_from_zip(my_autograding_zip_file,my_submission_zip_file,which_untruste
         print (which_machine,which_untrusted, "RUNNER FAILURE")
         grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="RUNNER FAILURE")
 
-    add_permissions_recursive(tmp_work,
-                          stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
-                          stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
-                          stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) 
-    add_permissions_recursive(tmp_compilation,
-                          stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
-                          stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
-                          stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) 
+    # RUN SOLUTION RUNNER
+    with open(os.path.join(tmp_logs,"overall.txt"),'a') as f:
+        print ("====================================\nRUNNER SOLUTION STARTS", file=f)
+    tmp_work_random_output = os.path.join(tmp_work,"random_output")
+    os.mkdir(tmp_work_random_output)
+
+    with open(os.path.join(tmp_logs,"output_generator_log.txt"), 'w') as logfile:
+        for testcase_num in range(1, len(my_testcases)+1):
+            testcase_folder = os.path.join(tmp_work_random_output, "test{:02}".format(testcase_num))
+            random_input_testcase_folder = os.path.join(tmp_work, "random_input", "test{:02}".format(testcase_num))
+            os.makedirs(testcase_folder)
+            os.chdir(testcase_folder)
+
+            # copy any instructor provided solution code files to testcase folder
+            copy_contents_into(job_id,instructor_solution_path,testcase_folder,tmp_logs)
+
+            # copy test input into testcase folder
+            copy_contents_into(job_id,test_input_path,testcase_folder,tmp_logs)
+            pattern_copy("random_input_to_runner",["*.txt"],random_input_testcase_folder,testcase_folder,tmp_logs)
+
+            # copy compile.out to the current directory
+            shutil.copy (os.path.join(bin_path,"solution_runner.out"),os.path.join(testcase_folder,"my_solution_runner.out"))
+            add_permissions(os.path.join(testcase_folder,"my_solution_runner.out"), stat.S_IXUSR | stat.S_IXGRP |stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH)          
+            untrusted_grant_rwx_access(which_untrusted, testcase_folder)
+            add_all_permissions(testcase_folder)
+
+            output_generator_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"),
+                                                which_untrusted,
+                                                os.path.join(testcase_folder,"my_solution_runner.out"),
+                                                queue_obj["gradeable"],
+                                                queue_obj["who"],
+                                                str(queue_obj["version"]),
+                                                submission_string,
+                                                "output",
+                                                '--testcase', str(testcase_num)],
+                                                stdout=logfile, 
+                                                cwd=testcase_folder)
+            # remove the compilation program
+            untrusted_grant_rwx_access(which_untrusted, testcase_folder)
+            os.remove(os.path.join(testcase_folder,"my_solution_runner.out"))
+
+        if output_generator_success == 0:
+            print (which_machine,which_untrusted,"OUTPUT GENERATION OK")
+        else:
+            print (which_machine,which_untrusted,"OUTPUT GENERATION FAILURE")
+            grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,message="OUTPUT GENERATION FAILURE")
+        add_all_permissions(tmp_work)
+        add_all_permissions(tmp_compilation)
+
+    # return to the main tmp directory
+    os.chdir(tmp_work)
+    subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a'))
 
     # --------------------------------------------------------------------
     # RUN VALIDATOR
@@ -646,14 +742,33 @@ def grade_from_zip(my_autograding_zip_file,my_submission_zip_file,which_untruste
     # remove the test_input directory, so we don't archive it!
     shutil.rmtree(os.path.join(tmp_work,"test_input"))
 
+    # remove the test_output directory, so we don't archive it!
+    shutil.rmtree(os.path.join(tmp_work,"test_output"))
+
     # loop over the test case directories, and remove any files that are also in the test_input folder
     for testcase_num in range(1, len(my_testcases)+1):
         testcase_folder = os.path.join(tmp_work, "test{:02}".format(testcase_num))
+        random_input_testcase_folder = os.path.join(tmp_work,"random_input","test{:02}".format(testcase_num))
+        remove_test_input_files(os.path.join(tmp_logs,"overall.txt"),test_input_path,testcase_folder)
+        remove_test_input_files(os.path.join(tmp_logs,"overall.txt"),random_input_testcase_folder,testcase_folder)
+
+    # loop over the random output test case directories, and remove any files that are also in the test_input folder
+    for testcase_num in range(1, len(my_testcases)+1):
+        testcase_folder = os.path.join(tmp_work,"random_output", "test{:02}".format(testcase_num))
+        random_input_testcase_folder = os.path.join(tmp_work,"random_input","test{:02}".format(testcase_num))
         remove_test_input_files(os.path.join(tmp_logs,"overall.txt"),test_input_path,testcase_folder)
+        remove_test_input_files(os.path.join(tmp_logs,"overall.txt"),random_input_testcase_folder,testcase_folder)
 
     patterns_work_to_details = complete_config_obj["autograding"]["work_to_details"]
     pattern_copy("work_to_details",patterns_work_to_details,tmp_work,os.path.join(tmp_results,"details"),tmp_logs)
 
+    try:
+        patterns_work_to_random_output = complete_config_obj["autograding"]["work_to_random_output"]
+        pattern_copy("work_to_random_output", patterns_work_to_random_output, tmp_results, tmp_logs)
+    except:
+        with open(os.path.join(tmp_logs,"overall.txt"),'a') as f:
+            print ("Sorry this is not working")
+
     if ("work_to_public" in complete_config_obj["autograding"] and
         len(complete_config_obj["autograding"]["work_to_public"]) > 0):
         # create the directory

autograder/autograder/grade_item_main_runner.py

@@ -83,7 +83,7 @@ def executeTestcases(complete_config_obj, tmp_logs, tmp_work, queue_obj, submiss
                     for name, info in container_info.items():
                         mounted_directory = info['mounted_directory']
                         #Copies the code needed to run into mounted_directory
-                        setup_folder_for_grading(mounted_directory, tmp_work, job_id, tmp_logs,testcases[testcase_num-1])
+                        setup_folder_for_grading(mounted_directory, tmp_work, job_id, tmp_logs,testcases[testcase_num-1],testcase_num)
 
 
                     # Start the docker containers.
@@ -168,8 +168,9 @@ def executeTestcases(complete_config_obj, tmp_logs, tmp_work, queue_obj, submiss
             else:
                 try:
                     # Move the files necessary for grading (runner, inputs, etc.) into the testcase folder.
-                    setup_folder_for_grading(testcase_folder, tmp_work, job_id, tmp_logs,testcases[testcase_num-1])
+                    setup_folder_for_grading(testcase_folder, tmp_work, job_id, tmp_logs,testcases[testcase_num-1],testcase_num)
                     my_testcase_runner = os.path.join(testcase_folder, 'my_runner.out')
+                    grade_item.remove_read_permissions(os.path.join(tmp_work,"random_input")) 
 
                     display_sys_variable = os.environ.get('DISPLAY', None)
                     display_line = [] if display_sys_variable is None else ['--display', str(display_sys_variable)]
@@ -184,6 +185,7 @@ def executeTestcases(complete_config_obj, tmp_logs, tmp_work, queue_obj, submiss
                                                       '--testcase', str(testcase_num)]
                                                       + display_line,
                                                       stdout=logfile)
+                    grade_item.add_all_permissions(os.path.join(tmp_work,"random_input"))
                 except Exception as e:
                     grade_items_logging.log_message(job_id, message="ERROR thrown by main runner. See traces entry for more details.")
                     print ("ERROR caught runner.out exception={0}".format(str(e.args[0])).encode("utf-8"),file=logfile)
@@ -294,12 +296,13 @@ def pre_command_copy_file(tmp_work, source_testcase, source_directory, destinati
 
 
 
-def setup_folder_for_grading(target_folder, tmp_work, job_id, tmp_logs, testcase):
+def setup_folder_for_grading(target_folder, tmp_work, job_id, tmp_logs, testcase, testcase_num):
     #The paths to the important folders.
     tmp_work_test_input = os.path.join(tmp_work, "test_input")
     tmp_work_submission = os.path.join(tmp_work, "submitted_files")
     tmp_work_compiled = os.path.join(tmp_work, "compiled_files")
     tmp_work_checkout = os.path.join(tmp_work, "checkout")
+    tmp_work_random_input = os.path.join(tmp_work, "random_input", "test{:02}".format(testcase_num))
     my_runner = os.path.join(tmp_work,"my_runner.out")
 
     #######################################################################################
@@ -332,6 +335,7 @@ def setup_folder_for_grading(target_folder, tmp_work, job_id, tmp_logs, testcase
     grade_item.copy_contents_into(job_id,tmp_work_compiled  ,target_folder,tmp_logs)
     grade_item.copy_contents_into(job_id,tmp_work_checkout  ,target_folder,tmp_logs)
     grade_item.copy_contents_into(job_id,tmp_work_test_input,target_folder,tmp_logs)
+    grade_item.pattern_copy("random_input_to_runner",["*.txt"],tmp_work_random_input,target_folder,tmp_logs)
     #copy the compiled runner to the test directory
     shutil.copy(my_runner,target_folder)