Testrunning moved to model.Submission and made working again

This is step 1 towards resolving Issue #15.
The whole running tests and submitting testruns to the session is now
done in the submission model itself.
So the frontend controllers only have to get the output and show whats
relevant to the user.
Also this allows us to re-run testruns from the teachers point of view,
if needed.

sauce/lib/runner.py

@@ -19,7 +19,8 @@
 log = logging.getLogger(__name__)
 
 process = namedtuple('process', ['returncode', 'stdout', 'stderr'])
-testresult = namedtuple('testresult', ['result', 'test', 'returncode', 'output_data', 'error_data', 'runtime'])
+compileresult = namedtuple('compileresult', ['result', 'stdout', 'stderr'])
+testresult = namedtuple('testresult', ['result', 'partial', 'test', 'runtime', 'test_output', 'run_output', 'error_data', 'returncode'])
 
 # Timeout value for join between sending SIGTERM and SIGKILL to process
 THREADKILLTIMEOUT = 0.5
@@ -109,7 +110,7 @@ def compile(compiler, dir, srcfile, binfile):
     log.debug('Process stdout: %s' % stdoutdata.strip())
     log.debug('Process stderr: %s' % stderrdata.strip())
 
-    return process(returncode, stdoutdata, stderrdata)
+    return compileresult(returncode==0, stdoutdata, stderrdata)
 
 def execute(interpreter, timeout, dir, basename, binfile, stdin=None, argv=''):
     '''Execute or interpret a binfile
@@ -295,6 +296,7 @@ def test(self, only_visible=False):
                 tests = self.assignment.visible_tests
             else:
                 tests = self.assignment.tests
+
             for test in tests:
 
                 # Write test file, if needed
@@ -322,17 +324,20 @@ def test(self, only_visible=False):
                 process = execute(self.language.interpreter, test.timeout, 
                                   self.tempdir, self.basename, self.binfile, input, a)
                 end = time()
+                runtime = end - start
 
                 if test.output_type == 'file':
                     with open(os.path.join(self.tempdir, test.output_filename or 'outdata'), 'r') as outfd:
                         output = outfd.read()
                 else:
                     output = process.stdout
-
-                if process.returncode == 0 and compareTestOutput(test.output_data, output):
-                    yield testresult(True, test, process.returncode, output, process.stderr, end-start)
-                else: 
-                    yield testresult(False, test, process.returncode, output, process.stderr, end-start)
+
+                (result, partial, test_output, run_output) = test.validate(output)
+
+                if result or not test.ignore_returncode and process.returncode != 0:
+                    yield testresult(result, partial, test, runtime, test_output, run_output, process.stderr, process.returncode)
+                else:
+                    yield testresult(False, partial, test, runtime, test_output, run_output, process.stderr, process.returncode)
         else:
             raise CompileFirstException('Y U NO COMPILE FIRST?!')
 

sauce/model/submission.py

@@ -4,17 +4,25 @@
 @author: moschlar
 '''
 
+from time import time
 from datetime import datetime
+import logging
 
 from sqlalchemy import Column, ForeignKey
 from sqlalchemy.types import Integer, Unicode, DateTime, Boolean, PickleType
 from sqlalchemy.orm import relationship, backref, deferred
 from sqlalchemy.sql import desc
 
 from sauce.model import DeclarativeBase
+from sauce.model.test import Testrun
 
+from sauce.lib.runner import Runner
 from sauce.lib.helpers import link
 
+import transaction
+
+log = logging.getLogger(__name__)
+
 class Submission(DeclarativeBase):
     __tablename__ = 'submissions'
     __mapper_args__ = {'order_by': desc('date')}
@@ -44,6 +52,72 @@ class Submission(DeclarativeBase):
     def __unicode__(self):
         return u'Submission %s' % (self.id or '')
 
+    def run_tests(self, submit=False):
+        submitted=False
+        with Runner(self) as r:
+            start = time()
+            compilation = r.compile()
+            end = time()
+            compilation_time = end - start
+            log.debug(compilation)
+
+            if not compilation or compilation.returncode == 0:
+                start = time()
+                testruns = [testrun for testrun in r.test_visible()]
+                end = time()
+                run_time = end - start
+                log.debug(testruns)
+                log.debug(run_time)
+
+
+                if [testrun for testrun in testruns if not testrun.result]:
+                    #flash('Test run did not run successfully, you may not submit', 'error')
+
+                    log.debug('No submission')
+
+                else:
+
+                    if submit:
+                        self.complete = True
+
+                        testresults = [test for test in r.test()]
+
+                        test_time = sum(t.runtime for t in testresults)
+
+                        log.debug(testresults)
+                        log.debug(test_time)
+
+                        #if False in (t.result for t in testresults):
+                        #    self.submission.result = False
+                        #else:
+                        #    self.submission.result = True
+
+                        for t in testresults:
+                            self.testruns.append(Testrun(runtime=t.runtime, test=t.test, 
+                                                        result=t.result, partial=t.partial, 
+                                                        submission=self,
+                                                        output_data=t.run_output, error_data=t.error_data))
+
+                        #if self.result:
+                        #    flash('All tests completed. Runtime: %f' % test_time, 'ok')
+                        #else:
+                        #    flash('Tests failed. Runtime: %f' % test_time, 'error')
+                        transaction.commit()
+                        log.debug(self.result)
+                        submitted=True
+                        #transaction.commit()
+                        #self.submission = DBSession.merge(self.submission)
+                        #redirect(url('/submissions/%d' % self.submission.id))
+                    else:
+                        #flash('Tests successfully run in %f' % run_time, 'ok')
+                        log.debug('Tests sucessfully run')
+            elif compilation and not compilation.result:
+                 #flash('Compilation failed, see below', 'error')
+                 log.debug('Compilation failed')
+            else:
+                pass
+        return (compilation, testruns, submitted, self.result)
+
     @property
     def url(self):
         return '/submissions/%s' % self.id

sauce/model/test.py

@@ -147,12 +147,17 @@ def unconvert(self, data):
     def validate(self, output):
         ''''''
 
-        test_data = self.convert(self.output_data)
-        run_data = self.convert(output)
-
-
+        test_output = self.unconvert(self.convert(self.output_data))
+        run_output = self.unconvert(self.convert(output))
 
+        if test_output == run_output:
+            result, partial = True, False
+        elif self.show_partial_match and test_output.startswith(run_output):
+            result, partial = False, True
+        else:
+            result, partial = False, False
 
+        return (result, partial, test_output, run_output)
 
     @property
     def timeout(self):