Simplify autograding configuration for plaintext diff (#1395)

* more c++ file types & start of compare script * prevent untrusted users from looking at other tmp folders * more print statements in compare * deal with exceptions in grading scheduler * change command name * small changes * revise myersdiff config.json interface * initial refactor * debug using test suite * refactor code * more refactor * update tests to use new diff syntax * update tests to use new diff syntax * more testing * cleanup code a bit
Submitty · Aug 25, 2017 · d4eee12 · d4eee12
1 parent f86d229
commit d4eee12
Show file tree

Hide file tree

Showing 32 changed files with 800 additions and 239 deletions.
diff --git a/.setup/install_system.sh b/.setup/install_system.sh
@@ -300,7 +300,7 @@ else
     git clone 'https://github.com/Submitty/Tutorial' ${SUBMITTY_INSTALL_DIR}/GIT_CHECKOUT_Tutorial
     pushd ${SUBMITTY_INSTALL_DIR}/GIT_CHECKOUT_Tutorial
     # remember to change this version in .setup/travis/autograder.sh too
-    git checkout v0.92
+    git checkout v0.93
     popd
 fi
 

diff --git a/.setup/travis/autograder.sh b/.setup/travis/autograder.sh
@@ -71,5 +71,5 @@ echo -e "Compile and install the tutorial repository"
 git clone 'https://github.com/Submitty/Tutorial' ${SUBMITTY_INSTALL_DIR}/GIT_CHECKOUT_Tutorial
 pushd ${SUBMITTY_INSTALL_DIR}/GIT_CHECKOUT_Tutorial
 # remember to change this version in .setup/install_system.sh too
-git checkout v0.92
+git checkout v0.93
 popd
diff --git a/grading/CMakeLists.txt b/grading/CMakeLists.txt
@@ -14,6 +14,7 @@ set (GRADINGCODE ${SUBMITTY_INSTALL_DIR}/src)
 # source files
 
 add_library(submitty_grading
+  ${GRADINGCODE}/grading/load_config_json.cpp
   ${GRADINGCODE}/grading/TestCase.cpp 
   ${GRADINGCODE}/grading/JUnitGrader.cpp
   ${GRADINGCODE}/grading/DrMemoryGrader.cpp

diff --git a/grading/TestCase.cpp b/grading/TestCase.cpp
@@ -221,11 +221,7 @@ TestResults* TestCase::dispatch(const nlohmann::json& grader, int autocheck_numb
   else if (method == "PacmanGrader")               { return PacmanGrader_doit(*this,grader);                }
   else if (method == "searchToken")                { return searchToken_doit(*this,grader);                 }
   else if (method == "intComparison")              { return intComparison_doit(*this,grader);               }
-  else if (method == "myersDiffbyLinebyChar")      { return myersDiffbyLinebyChar_doit(*this,grader);       }
-  else if (method == "myersDiffbyLinebyWord")      { return myersDiffbyLinebyWord_doit(*this,grader);       }
-  else if (method == "myersDiffbyLine")            { return myersDiffbyLine_doit(*this,grader);             }
-  else if (method == "myersDiffbyLineNoWhite")     { return myersDiffbyLineNoWhite_doit(*this,grader);      }
-  else if (method == "diffLineSwapOk")             { return diffLineSwapOk_doit(*this,grader);              }
+  else if (method == "diff")                       { return diff_doit(*this,grader);                        }
   else if (method == "fileExists")                 { return fileExists_doit(*this,grader);                  }
   else if (method == "warnIfNotEmpty")             { return warnIfNotEmpty_doit(*this,grader);              }
   else if (method == "warnIfEmpty")                { return warnIfEmpty_doit(*this,grader);                 }

diff --git a/grading/default_config.h b/grading/default_config.h
@@ -5,6 +5,9 @@
 
 extern const char *GLOBAL_config_json_string;  // defined in json_generated.cpp
 
+// defined in load_config_json.cpp
+nlohmann::json LoadAndProcessConfigJSON(const std::string &rcsid);
+
 // ========================================================================================
 // ========================================================================================
 

diff --git a/grading/load_config_json.cpp b/grading/load_config_json.cpp
@@ -0,0 +1,106 @@
+#include <unistd.h>
+#include <cstdlib>
+#include <string>
+#include <iostream>
+#include <cassert>
+
+#include "TestCase.h"
+#include "execute.h"
+
+extern const char *GLOBAL_config_json_string;  // defined in json_generated.cpp
+
+void AddAutogradingConfiguration(nlohmann::json &whole_config) {
+  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.cpp");
+  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.cxx");
+  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.c");
+  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.h");
+  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.hpp");
+  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.hxx");
+  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.java");
+
+  whole_config["autograding"]["submission_to_runner"].push_back("**/*.py");
+  whole_config["autograding"]["submission_to_runner"].push_back("**/*.pdf");
+
+  whole_config["autograding"]["compilation_to_runner"].push_back("**/*.out");
+  whole_config["autograding"]["compilation_to_runner"].push_back("**/*.class");
+
+  whole_config["autograding"]["compilation_to_validation"].push_back("test*.txt");
+
+  whole_config["autograding"]["submission_to_validation"].push_back("**/README.txt");
+  whole_config["autograding"]["submission_to_validation"].push_back("**/*.pdf");
+
+  whole_config["autograding"]["work_to_details"].push_back("test*.txt");
+  whole_config["autograding"]["work_to_details"].push_back("test*_diff.json");
+  whole_config["autograding"]["work_to_details"].push_back("**/README.txt");
+}
+
+
+void RewriteDeprecatedMyersDiff(nlohmann::json &whole_config) {
+
+  nlohmann::json::iterator tc = whole_config.find("testcases");
+  if (tc == whole_config.end()) { /* no testcases */ return; }
+
+  // loop over testcases
+  int which_testcase = 0;
+  for (nlohmann::json::iterator my_testcase = tc->begin();
+       my_testcase != tc->end(); my_testcase++,which_testcase++) {
+    nlohmann::json::iterator validators = my_testcase->find("validation");
+    if (validators == my_testcase->end()) { /* no autochecks */ continue; }
+
+    // loop over autochecks
+    for (int which_autocheck = 0; which_autocheck < validators->size(); which_autocheck++) {
+      nlohmann::json& autocheck = (*validators)[which_autocheck];
+      std::string method = autocheck.value("method","");
+
+      // if autocheck is old myersdiff format...  rewrite it!
+      if (method == "myersDiffbyLinebyChar") {
+        autocheck["method"] = "diff";
+        assert (autocheck.find("comparison") == autocheck.end());
+        autocheck["comparison"] = "byLinebyChar";
+      } else if (method == "myersDiffbyLinebyWord") {
+        autocheck["method"] = "diff";
+        assert (autocheck.find("comparison") == autocheck.end());
+        autocheck["comparison"] = "byLinebyWord";
+      } else if (method == "myersDiffbyLine") {
+        autocheck["method"] = "diff";
+        assert (autocheck.find("comparison") == autocheck.end());
+        autocheck["comparison"] = "byLine";
+      } else if (method == "myersDiffbyLineNoWhite") {
+        autocheck["method"] = "diff";
+        assert (autocheck.find("comparison") == autocheck.end());
+        autocheck["comparison"] = "byLine";
+        assert (autocheck.find("ignoreWhitespace") == autocheck.end());
+        autocheck["ignoreWhitespace"] = true;
+      } else if (method == "diffLineSwapOk") {
+        autocheck["method"] = "diff";
+        assert (autocheck.find("comparison") == autocheck.end());
+        autocheck["comparison"] = "byLine";
+        assert (autocheck.find("lineSwapOk") == autocheck.end());
+        autocheck["lineSwapOk"] = true;
+      }
+    }
+  }
+}
+
+
+// =====================================================================
+// =====================================================================
+
+nlohmann::json LoadAndProcessConfigJSON(const std::string &rcsid) {
+
+  nlohmann::json answer;
+  std::stringstream sstr(GLOBAL_config_json_string);
+  sstr >> answer;
+
+  AddSubmissionLimitTestCase(answer);
+  AddAutogradingConfiguration(answer);
+  if (rcsid != "") {
+    CustomizeAutoGrading(rcsid,answer);
+  }
+
+  RewriteDeprecatedMyersDiff(answer);
+
+  std::cout << "JSON PARSED" << std::endl;
+
+  return answer;
+}
diff --git a/grading/main_compile.cpp b/grading/main_compile.cpp
@@ -23,7 +23,6 @@
 // =====================================================================
 // =====================================================================
 
-
 void CleanUpMultipleParts() {
 
   std::cout << "Clean up multiple parts" << std::endl;
@@ -86,21 +85,6 @@ void CleanUpMultipleParts() {
 
 int main(int argc, char *argv[]) {
 
-  std::cout << "MAIN COMPILE" << std::endl;
-  std::vector<std::string> actions;
-  nlohmann::json config_json;
-  std::stringstream sstr(GLOBAL_config_json_string);
-  sstr >> config_json;
-  AddSubmissionLimitTestCase(config_json);
-
-  std::cout << "JSON PARSED" << std::endl;
-
-  nlohmann::json grading_parameters = config_json.value("grading_parameters",nlohmann::json::object());
-  int AUTO_POINTS         = grading_parameters.value("AUTO_POINTS",0);
-  int EXTRA_CREDIT_POINTS = grading_parameters.value("EXTRA_CREDIT_POINTS",0);
-  int TA_POINTS           = grading_parameters.value("TA_POINTS",0);
-  int TOTAL_POINTS        = grading_parameters.value("TOTAL_POINTS",AUTO_POINTS+TA_POINTS);
-
   std::string hw_id = "";
   std::string rcsid = "";
   int subnum = -1;
@@ -116,14 +100,23 @@ int main(int argc, char *argv[]) {
   else if (argc != 1) {
     std::cerr << "INCORRECT ARGUMENTS TO COMPILER" << std::endl;
     return 1;
-  } 
+  }
 
-  std::cout << "Compiling User Code..." << std::endl;
+  // LOAD HW CONFIGURATION JSON
+  nlohmann::json config_json = LoadAndProcessConfigJSON(rcsid);
 
-  system("find . -type f -exec ls -sh {} +");
+  std::cout << "MAIN COMPILE" << std::endl;
+  std::vector<std::string> actions;
+
+  nlohmann::json grading_parameters = config_json.value("grading_parameters",nlohmann::json::object());
+  int AUTO_POINTS         = grading_parameters.value("AUTO_POINTS",0);
+  int EXTRA_CREDIT_POINTS = grading_parameters.value("EXTRA_CREDIT_POINTS",0);
+  int TA_POINTS           = grading_parameters.value("TA_POINTS",0);
+  int TOTAL_POINTS        = grading_parameters.value("TOTAL_POINTS",AUTO_POINTS+TA_POINTS);
 
-  CustomizeAutoGrading(rcsid,config_json);
+  std::cout << "Compiling User Code..." << std::endl;
 
+  system("find . -type f -exec ls -sh {} +");
 
   // if it's a "one part only" assignment, check if student
   // submitted to multiple parts

diff --git a/grading/main_configure.cpp b/grading/main_configure.cpp
@@ -32,40 +32,12 @@ nlohmann::json printTestCase(TestCase test) {
   return j;
 }
 
-void AddAutogradingConfiguration(nlohmann::json &whole_config) {
-  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.cpp");
-  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.cxx");
-  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.c");
-  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.h");
-  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.hpp");
-  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.hxx");
-  whole_config["autograding"]["submission_to_compilation"].push_back("**/*.java");
-
-  whole_config["autograding"]["submission_to_runner"].push_back("**/*.py");
-  whole_config["autograding"]["submission_to_runner"].push_back("**/*.pdf");
-
-  whole_config["autograding"]["compilation_to_runner"].push_back("**/*.out");
-  whole_config["autograding"]["compilation_to_runner"].push_back("**/*.class");
-
-  whole_config["autograding"]["compilation_to_validation"].push_back("test*.txt");
-
-  whole_config["autograding"]["submission_to_validation"].push_back("**/README.txt");
-  whole_config["autograding"]["submission_to_validation"].push_back("**/*.pdf");
-
-  whole_config["autograding"]["work_to_details"].push_back("test*.txt");
-  whole_config["autograding"]["work_to_details"].push_back("test*_diff.json");
-  whole_config["autograding"]["work_to_details"].push_back("**/README.txt");
-}
 
 
 int main(int argc, char *argv[]) {
 
-  nlohmann::json config_json;
-  std::stringstream sstr(GLOBAL_config_json_string);
-  sstr >> config_json;
-  AddSubmissionLimitTestCase(config_json);
-
-  AddAutogradingConfiguration(config_json);
+  // LOAD HW CONFIGURATION JSON
+  nlohmann::json config_json = LoadAndProcessConfigJSON("");  // don't know the username yet
 
   nlohmann::json j;
 

diff --git a/grading/main_runner.cpp b/grading/main_runner.cpp
@@ -16,18 +16,6 @@
 int main(int argc, char *argv[]) {
   std::cout << "Running User Code..." << std::endl;
 
-  nlohmann::json config_json;
-  std::stringstream sstr(GLOBAL_config_json_string);
-  sstr >> config_json;
-  AddSubmissionLimitTestCase(config_json);
-
-  nlohmann::json grading_parameters = config_json.value("grading_parameters",nlohmann::json::object());
-  int AUTO_POINTS         = grading_parameters.value("AUTO_POINTS",0);
-  int EXTRA_CREDIT_POINTS = grading_parameters.value("EXTRA_CREDIT_POINTS",0);
-  int TA_POINTS           = grading_parameters.value("TA_POINTS",0);
-  int TOTAL_POINTS        = grading_parameters.value("TOTAL_POINTS",AUTO_POINTS+TA_POINTS);
-
-
   std::string hw_id = "";
   std::string rcsid = "";
   int subnum = -1;
@@ -45,11 +33,18 @@ int main(int argc, char *argv[]) {
     return 1;
   } 
 
+  // LOAD HW CONFIGURATION JSON
+  nlohmann::json config_json = LoadAndProcessConfigJSON(rcsid);
+
+  nlohmann::json grading_parameters = config_json.value("grading_parameters",nlohmann::json::object());
+  int AUTO_POINTS         = grading_parameters.value("AUTO_POINTS",0);
+  int EXTRA_CREDIT_POINTS = grading_parameters.value("EXTRA_CREDIT_POINTS",0);
+  int TA_POINTS           = grading_parameters.value("TA_POINTS",0);
+  int TOTAL_POINTS        = grading_parameters.value("TOTAL_POINTS",AUTO_POINTS+TA_POINTS);
+
   // necessary since the untrusted user does not have a home directory
   setenv("DYNAMORIO_CONFIGDIR", ".", 1);
 
-  CustomizeAutoGrading(rcsid,config_json);
-
   system("find . -type f -exec ls -sh {} +");
 
   // Run each test case and create output files

diff --git a/grading/main_validator.cpp b/grading/main_validator.cpp
@@ -347,16 +347,11 @@ void ValidateATestCase(nlohmann::json config_json, int which_testcase,
 }
 
 
-
 /* Runs through each test case, pulls in the correct files, validates, and outputs the results */
 int validateTestCases(const std::string &hw_id, const std::string &rcsid, int subnum, const std::string &subtime) {
 
-
   // LOAD HW CONFIGURATION JSON
-  nlohmann::json config_json;
-  std::stringstream sstr(GLOBAL_config_json_string);
-  sstr >> config_json;
-  AddSubmissionLimitTestCase(config_json);
+  nlohmann::json config_json = LoadAndProcessConfigJSON(rcsid);
 
   // PREPARE GRADE.TXT FILE
   std::string grade_path = "grade.txt";
@@ -370,7 +365,7 @@ int validateTestCases(const std::string &hw_id, const std::string &rcsid, int su
   std::stringstream testcase_json;
   nlohmann::json all_testcases;
 
-  CustomizeAutoGrading(rcsid,config_json);
+
 
   system("find . -type f -exec ls -sh {} +");