Improve error handling

Current implementation was ignoring errors encountered while executing
queries form the problem statement - this commit adds checks and
handling for such errors

sql_grader/problem.py

@@ -39,7 +39,7 @@ def __init__(
         self.answer_query = answer_query
         self.verify_query = verify_query
         self.modification_query = modification_query
-        self.answer_result, _ = SqlProblem.run_query(
+        self.answer_result, self.answer_error = SqlProblem.run_query(
             self.database,
             answer_query,
             verify_query,
@@ -50,6 +50,11 @@ def attempt(self, query):
         """
         Attempt to answer the problem with the provided query
         """
+        if self.answer_error:
+            return (None,
+                    None,
+                    'Problem setup incorrectly: {}'.format(self.answer_error),
+                    False)
         submission_result, error = SqlProblem.run_query(
             self.database,
             query,
@@ -125,20 +130,25 @@ def run(database, query, is_single_query):
 
         database = cls.clone_database(source)
 
-        if verify_query:
-            is_single_query = False
-        else:
-            is_single_query = True
-        result, error = run(database, query, is_single_query)
+        # `query` should be a single select statement if no verify_query is
+        # present in the problem
+        result, error = run(database, query, is_single_query=not verify_query)
+        if error:
+            return None, error
 
         if verify_query:
             if modification_query:
-                # TODO: Add error checking here too
-                result, _ = run(database, modification_query,
-                                is_single_query=False)
+                _, error = run(database, modification_query,
+                               is_single_query=False)
+                if error:
+                    return None, 'modification_query: {}'.format(error)
+
             result, error = run(database, verify_query,
                                 is_single_query=True)
-        return result, error
+            if error:
+                return None, 'verify_query: {}'.format(error)
+
+        return result, None
 
     @staticmethod
     def compare_rows(expected, actual, is_ordered=True):

sql_grader/tests/unit/test_problem.py

@@ -50,7 +50,7 @@ def test_select(self):
 
         # Verify that comparison is False if different rows are returned
         submission_result, answer_result, error, comparison = SqlProblem(
-            answer_query="answer_query",
+            answer_query=answer_query,
             database=self.database,
             verify_query=verify_query,
             is_ordered=False
@@ -60,7 +60,7 @@ def test_select(self):
 
         # Verify that comparison is False if different columns are returned
         submission_result, answer_result, error, comparison = SqlProblem(
-            answer_query="answer_query",
+            answer_query=answer_query,
             database=self.database,
             verify_query=verify_query,
             is_ordered=False
@@ -148,7 +148,7 @@ def test_pre_verification(self):
         for each row
         when (new.year is NULL)
           begin update Movie
-            set director=NULL where mID=new.mID end;
+            set director=NULL where mID=new.mID; end;
         """
         query = answer_query
         modification_query = """
@@ -179,16 +179,54 @@ def test_error(self):
         """
         Test that invalid queries produce error
         """
-        verify_query = None
-        answer_query = "SELECT * FROM Movie;"
-        query = "Not a SQL Query;"
+        # Verify that errors are displayed when problem itself contains errors
+        _, _, error, _ = SqlProblem(
+            answer_query="Not a query;",
+            database=self.database,
+            verify_query=None,
+            is_ordered=True
+        ).attempt("Not a SQL Query;")
+        self.assertIn("Problem setup incorrectly", error)
+        self.assertIn("syntax error", error)
 
-        # Verify that comparison returns False if is_ordered = True
         _, _, error, _ = SqlProblem(
-            answer_query=answer_query,
+            answer_query="",
             database=self.database,
-            verify_query=verify_query,
+            verify_query="Not a query;",
             is_ordered=True
-        ).attempt(query)
+        ).attempt("Not a SQL Query;")
+        self.assertIn("Problem setup incorrectly", error)
+        self.assertIn("verify_query", error)
+        self.assertIn("syntax error", error)
 
-        self.assertNotEqual(error, None)
+        _, _, error, _ = SqlProblem(
+            answer_query="",
+            database=self.database,
+            verify_query="SELECT;",
+            modification_query="Not a query;",
+            is_ordered=True
+        ).attempt("Not a SQL Query;")
+        self.assertIn("Problem setup incorrectly", error)
+        self.assertIn("modification_query", error)
+        self.assertIn("syntax error", error)
+
+        # Verify that errors are displayed when submitted query contains errors
+        _, _, error, _ = SqlProblem(
+            answer_query="SELECT * FROM Movie;",
+            database=self.database,
+            verify_query=None,
+            is_ordered=True
+        ).attempt("Not a SQL Query;")
+        self.assertNotIn("Problem setup incorrectly", error)
+        self.assertIn("syntax error", error)
+
+        # Verify that errors are displayed when verification queries error out
+        _, _, error, _ = SqlProblem(
+            answer_query="",
+            database=self.database,
+            verify_query="select * from Movie",
+            is_ordered=True
+        ).attempt("alter table Movie rename to Movie2")
+        self.assertNotIn("Problem setup incorrectly", error)
+        self.assertIn("no such table", error)
+        self.assertIn("verify_query", error)