[frontend] Moving the concept of evaluation submission into the Task …

…Dispenser (#854)

inginious/frontend/pages/course_admin/submission.py

@@ -42,12 +42,12 @@ def POST_AUTH(self, submissionid):  # pylint: disable=arguments-differ
 
         webinput = flask.request.form
         if "replay" in webinput and is_admin:
-            self.submission_manager.replay_job(task, submission)
+            self.submission_manager.replay_job(task, submission, course.get_task_dispenser())
         elif "replay-copy" in webinput:  # Authorized for tutors
-            self.submission_manager.replay_job(task, submission, True)
+            self.submission_manager.replay_job(task, submission, course.get_task_dispenser(), True)
             return redirect(self.app.get_homepath() + "/course/" + course.get_id() + "/" + task.get_id())
         elif "replay-debug" in webinput and is_admin:
-            self.submission_manager.replay_job(task, submission, True, "ssh")
+            self.submission_manager.replay_job(task, submission, course.get_task_dispenser(), True, "ssh")
             return redirect(self.app.get_homepath() + "/course/" + course.get_id() + "/" + task.get_id())
 
         return self.page(course, task, submission)

inginious/frontend/pages/course_admin/submissions.py

@@ -34,7 +34,7 @@ def POST_AUTH(self, courseid):  # pylint: disable=arguments-differ
             if submission is None:
                 raise NotFound(description=_("This submission doesn't exist."))
 
-            self.submission_manager.replay_job(course.get_task(submission["taskid"]), submission)
+            self.submission_manager.replay_job(course.get_task(submission["taskid"]), submission, course.get_task_dispenser())
             return Response(response=json.dumps({"status": "waiting"}), content_type='application/json')
 
         elif "csv" in user_input or "download" in user_input or "replay" in user_input:
@@ -68,7 +68,7 @@ def POST_AUTH(self, courseid):  # pylint: disable=arguments-differ
 
                 tasks = course.get_tasks()
                 for submission in data:
-                    self.submission_manager.replay_job(tasks[submission["taskid"]], submission)
+                    self.submission_manager.replay_job(tasks[submission["taskid"]], submission, course.get_task_dispenser())
                 msgs.append(_("{0} selected submissions were set for replay.").format(str(len(data))))
                 return self.page(course, params, msgs=msgs)
 

inginious/frontend/static/js/task_dispensers.js

@@ -372,6 +372,7 @@ function dispenser_util_get_sections_list(element) {
             }
 
             structure["no_stored_submissions"] = dispenser_util_get_no_stored_submissions(tasks_id);
+            structure["evaluation_mode"] = dispenser_util_get_evaluation_mode(tasks_id);
         } else if ($(this).hasClass("sections_list")) {
             structure["sections_list"] = dispenser_util_get_sections_list(content);
         }
@@ -408,11 +409,11 @@ function dispenser_util_get_weights(tasks_id) {
 function dispenser_util_get_no_stored_submissions(tasks_id){
     const no_stored_submissions = {};
     $(".no_stored_submissions").each(function(){
-        var taskid = this.name;
-        if(taskid in tasks_id && this.checked && this.id === "store_all"){
+        var taskid = this.id;
+        if(taskid in tasks_id && this.checked && this.value === "store_all"){
             no_stored_submissions[taskid] = 0;
-        }else if(taskid in tasks_id && this.checked && this.id === "store_not_all"){
-            $("#no_stored_submissions_value_"+taskid).each(function(index){
+        }else if(taskid in tasks_id && this.checked && this.value === "store_not_all"){
+            $("#no_stored_submissions_value_"+taskid).each(function(){
                 var value = parseInt(this.value);
                 if(!isNaN(value)){
                     no_stored_submissions[taskid] = value;
@@ -425,6 +426,19 @@ function dispenser_util_get_no_stored_submissions(tasks_id){
     return no_stored_submissions;
 }
 
+function dispenser_util_get_evaluation_mode(tasks_id){
+    const evaluation_mode = {};
+    $(".evaluation_submission").each(function(){
+        var taskid = this.id;
+        if(taskid in tasks_id && this.checked && this.value === "best"){
+            evaluation_mode[taskid] = "best";
+        }else if(taskid in tasks_id && this.checked && this.value === "last"){
+            evaluation_mode[taskid] = "last"
+        }
+    });
+    return evaluation_mode;
+}
+
 function dispenser_util_get_tasks_list(element) {
     const tasks_list = {};
     element.children(".task").each(function (index) {

inginious/frontend/submission_manager.py

@@ -45,7 +45,7 @@ def __init__(self, client, user_manager, database, gridfs, plugin_manager, lti_o
         self._lti_outcome_manager = lti_outcome_manager
 
     def _job_done_callback(self, submissionid, task, result, grade, problems, tests, custom, state, archive, stdout,
-                           stderr, newsub=True):
+                           stderr, task_dispenser,  newsub=True):
         """ Callback called by Client when a job is done. Updates the submission in the database with the data returned after the completion of the
         job """
         submission = self.get_submission(submissionid, False)
@@ -85,7 +85,7 @@ def _job_done_callback(self, submissionid, task, result, grade, problems, tests,
         self._plugin_manager.call_hook("submission_done", submission=submission, archive=archive, newsub=newsub)
 
         for username in submission["username"]:
-            self._user_manager.update_user_stats(username, task, submission, result[0], grade, state, newsub)
+            self._user_manager.update_user_stats(username, task, submission, result[0], grade, state, newsub, task_dispenser)
 
         if "outcome_service_url" in submission and "outcome_result_id" in submission and "outcome_consumer_key" in submission:
             for username in submission["username"]:
@@ -151,7 +151,7 @@ def _after_submission_insertion(self, task, inputdata, debug, submission, submis
 
         return self._delete_exceeding_submissions(self._user_manager.session_username(), task, task_dispenser)
 
-    def replay_job(self, task, submission, copy=False, debug=False):
+    def replay_job(self, task, submission, task_dispenser, copy=False, debug=False):
         """
         Replay a submission: add the same job in the queue, keeping submission id, submission date and input data
         :param submission: Submission to replay
@@ -203,7 +203,7 @@ def replay_job(self, task, submission, copy=False, debug=False):
         jobid = self._client.new_job(1, task, inputdata,
                                      (lambda result, grade, problems, tests, custom, state, archive, stdout, stderr:
                                       self._job_done_callback(submissionid, task, result, grade, problems, tests,
-                                                              custom, state, archive, stdout, stderr, copy)),
+                                                              custom, state, archive, stdout, stderr, task_dispenser, copy)),
                                      "Frontend - {}".format(submission["username"]), debug, ssh_callback)
 
 
@@ -307,7 +307,7 @@ def add_job(self, task, inputdata, task_dispenser, debug=False):
         jobid = self._client.new_job(0, task, inputdata,
                                      (lambda result, grade, problems, tests, custom, state, archive, stdout, stderr:
                                       self._job_done_callback(submissionid, task, result, grade, problems, tests,
-                                                              custom, state, archive, stdout, stderr, True)),
+                                                              custom, state, archive, stdout, stderr, task_dispenser, True)),
                                      "Frontend - {}".format(username), debug, ssh_callback)
 
         self._database.submissions.update_one(
@@ -342,7 +342,7 @@ def _delete_exceeding_submissions(self, username, task, task_dispenser):
         # List the entries to keep
         to_keep = set([])
 
-        if task.get_evaluate() == 'best':
+        if task_dispenser.get_evaluation_mode(task.get_id()) == 'best':
             # Find the best "status"="done" and "result"="success"
             idx_best = -1
             for idx, val in enumerate(tasks):

inginious/frontend/task_dispensers/__init__.py

@@ -17,6 +17,11 @@ def get_no_stored_submissions(self, taskid):
         """Returns the maximum stored submission specified by the administrator"""
         pass
 
+    @abstractmethod
+    def get_evaluation_mode(self, taskid):
+        """Returns the evaluation mode specified by the administrator"""
+        pass
+
     @abstractmethod
     def get_course_grade(self, username):
         """Returns the current grade of the course for a specific user"""

inginious/frontend/task_dispensers/combinatory_test.py

@@ -46,6 +46,18 @@ def get_no_stored_submissions(self,taskid):
         except:
             return 0
 
+    def get_evaluation_mode(self,taskid):
+        """Returns the evaluation mode specified by the administrator"""
+        try:
+            struct = self._data.to_structure()
+            for elem in struct:
+                evaluation_mode = self._data.get_value_rec(taskid,elem,"evaluation_mode")
+                if evaluation_mode is not None:
+                    return evaluation_mode
+            return "best"
+        except:
+            return "best"
+
     def get_course_grade(self, username):
         """ Returns the grade of a user for the current course"""
         task_list = self.get_user_task_list([username])[username]

inginious/frontend/task_dispensers/toc.py

@@ -52,6 +52,18 @@ def get_no_stored_submissions(self,taskid):
         except:
             return 0
 
+    def get_evaluation_mode(self,taskid):
+        """Returns the evaluation mode specified by the administrator"""
+        try:
+            struct = self._toc.to_structure()
+            for elem in struct:
+                evaluation_mode = self._toc.get_value_rec(taskid,elem,"evaluation_mode")
+                if evaluation_mode is not None:
+                    return evaluation_mode
+            return "best"
+        except:
+            return "best"
+
     def get_course_grade(self, username):
         """ Returns the grade of a user for the current course"""
         task_list = self.get_user_task_list([username])[username]

inginious/frontend/task_dispensers/util.py

@@ -223,6 +223,14 @@ def __init__(self, structure):
                 else:
                     raise InvalidTocException( ("The store submission must be an integer > 1 for the task: " + str(taskid)) )
 
+        self._evaluation_mode = {}
+        if "evaluation_mode" in structure:
+            for taskid,evaluation_mode in structure["evaluation_mode"].items():
+                if evaluation_mode != "best" and evaluation_mode != "last":
+                    raise InvalidTocException( ("The evaluation mode must be either best or last for the task: '" + str(taskid)) +"' but is " + str(evaluation_mode) )
+                else:
+                    self._evaluation_mode = structure["evaluation_mode"]
+
     def is_terminal(self):
         return True
 
@@ -265,7 +273,7 @@ def to_structure(self, rank):
         :return: The structure in YAML format
         """
         return {"id": self._id, "rank": rank, "title": self._title,
-                "tasks_list": {taskid: rank for rank, taskid in enumerate(self._task_list)}, "weights": self._weights, "no_stored_submissions": self._no_stored_submissions}
+                "tasks_list": {taskid: rank for rank, taskid in enumerate(self._task_list)}, "weights": self._weights, "no_stored_submissions": self._no_stored_submissions, "evaluation_mode": self._evaluation_mode}
 
 
 def check_toc(toc):

inginious/frontend/tasks.py

@@ -103,9 +103,6 @@ def __init__(self, course, taskid, content, filesystem, plugin_manager, task_pro
         else:
             self._contact_url = ""
 
-        # Default download
-        self._evaluate = self._data.get("evaluate", "best")
-
         # _accessible
         self._accessible = AccessibleTime(self._data.get("accessible", None))
 
@@ -243,10 +240,6 @@ def adapt_input_for_backend(self, input_data):
             input_data = problem.adapt_input_for_backend(input_data)
         return input_data
 
-    def get_evaluate(self):
-        """ Indicates the default download for the task """
-        return self._evaluate
-
     def get_categories(self):
         """ Returns the tags id associated to the task """
         return [category for category in self._categories if category in self._course.get_tags()]

inginious/frontend/templates/course_admin/edit_tabs/basic.html

@@ -104,21 +104,6 @@
         </div>
     </div>
 </div>
-<div class="form-group row">
-    <label for="groups" class="col-sm-2 control-label">{{_("Evaluation submission")}}</label>
-    <div class="col-sm-10">
-        <label>
-            <input type="radio" value="best" name="evaluate" id="evaluate"
-                {{ 'checked="checked"'|safe if task_data.get('evaluate', 'best') == 'best' }}
-            /> {{ _("Best submission") }}
-        </label><br/>
-        <label>
-            <input type="radio" value="last" name="evaluate"
-                {{ 'checked="checked"'|safe if task_data.get('evaluate', 'best') == 'last' }}
-            /> {{ _("Last submission") }}
-        </label>
-    </div>
-</div>
 <div class="form-group row">
     <label class="col-sm-2 control-label">{{_("Accessible")}}</label>
     <div class="col-sm-10">

inginious/frontend/templates/course_admin/task_dispensers/util_task_edit_modal.html

@@ -21,22 +21,39 @@ <h5 class="modal-title">{{_("Edit Task")}}</h5>
                 {% endif %}
             </div>
 
-            <div style="margin-top: 15px;margin-left: 5px;">{{_("Submission Storage")}}</div>
+            <!--no_stored_submissions-->
+            <div style="margin-top: 15px;">{{_("Submission Storage")}}</div>
             <div class="no_stored_submissions">
                 {% set no_stored_submissions = course.get_task_dispenser().get_no_stored_submissions(taskid) %}
                 <label>
-                    <input type="radio" value="true" class="no_stored_submissions" name="{{taskid}}" id="store_all" {{ 'checked'|safe if no_stored_submissions == 0 }} />
+                    <input type="radio" value="store_all" class="no_stored_submissions" name="no_stored_submissions_{{taskid}}" id="{{taskid}}" {{ 'checked'|safe if no_stored_submissions == 0 }} />
                     {{_("All submissions")}}
                 </label><br/>
                 <div class="row">
                     <div>
                         <label class="col-xs-12 col-lg-12">
-                            <input type="radio" value="false" class="no_stored_submissions" name="{{taskid}}" id="store_not_all" {{ 'checked'|safe if no_stored_submissions > 0 }} />
+                            <input type="radio" value="store_not_all" class="no_stored_submissions" name="no_stored_submissions_{{taskid}}" id="{{taskid}}" {{ 'checked'|safe if no_stored_submissions > 0 }} />
                             {{ _("Only the last {nbr_submissions} submissions").format(nbr_submissions='</label></div><div class="col-xs-offset-1 col-lg-offset-0 col-xs-11 col-lg-3"><input name="no_stored_submissions" class="form-control input-xs" id="no_stored_submissions_value_' + (taskid) + '" style="height:20px;padding: 0 10px;" value="' + (no_stored_submissions|string if no_stored_submissions > 0 else '') + '" placeholder="5" type="number"></div><div class="col-xs-offset-1 col-lg-offset-0 col-xs-11 col-lg-3"><label>') | safe }}
                         </label>
                     </div>
                 </div>
             </div>
+            <div style="margin-top: 15px;">{{_("Evaluation Submission")}}</div>
+            <div class="evaluation_submission">
+                {% set evaluation_value = course.get_task_dispenser().get_evaluation_mode(taskid) %}
+                <label>
+                    <input type="radio" value="best" class="evaluation_submission" name="evaluation_submission_{{taskid}}" id="{{taskid}}" {{ 'checked'|safe if evaluation_value == "best" }} />
+                    {{ _("Best submission") }}
+                </label><br/>
+                <div class="row">
+                    <div>
+                        <label class="col-xs-12 col-lg-12">
+                            <input type="radio" value="last" class="evaluation_submission" name="evaluation_submission_{{taskid}}" id="{{taskid}}" {{ 'checked'|safe if evaluation_value == "last" }} />
+                            {{ _("Last submission") }}
+                        </label>
+                    </div>
+                </div>
+            </div>
         </div>
         <div class="modal-footer">
             <button type="button" class="btn btn-primary" data-dismiss="modal">Close</button>

inginious/frontend/templates/task.html

@@ -172,7 +172,7 @@ <h3>{{ _("Submitting as") }}</h3>
         <h3> {{ _("For evaluation") }}</h3>
         <div class="list-group mb-3">
             <div class="list-group-item list-group-item-info"><i class="fa fa-info fa-fw"></i>
-                {% if task.get_evaluate() == "last" %}
+                {% if course.get_task_dispenser().get_evaluation_mode(task.get_id()) == "last" %}
                     {{ _("Last submission") }}
                 {% else %}
                     {{ _("Best submission") }}
@@ -465,7 +465,7 @@ <h4 class="modal-title" id="socialModalLabel">{{ _("Share my result on: ") }}</h
     var taskid = '{{ task.get_id() }}';
 
     $(document).ready(function() {
-        init_task_page("{{ task.get_evaluate() }}");
+        init_task_page("{{ course.get_task_dispenser().get_evaluation_mode(task.get_id()) }}");
     });
 
     var input = {

inginious/frontend/user_manager.py

@@ -670,7 +670,7 @@ def user_saw_task(self, username, courseid, taskid):
                                                                "submissionid": None, "state": ""}},
                                              upsert=True)
 
-    def update_user_stats(self, username, task, submission, result_str, grade, state, newsub):
+    def update_user_stats(self, username, task, submission, result_str, grade, state, newsub, task_dispenser):
         """ Update stats with a new submission """
         self.user_saw_task(username, submission["courseid"], submission["taskid"])
 
@@ -680,8 +680,8 @@ def update_user_stats(self, username, task, submission, result_str, grade, state
                 {"$inc": {"tried": 1, "tokens.amount": 1}})
 
             # Check if the submission is the default download
-            set_default = task.get_evaluate() == 'last' or \
-                          (task.get_evaluate() == 'best' and old_submission.get('grade', 0.0) <= grade)
+            set_default = task_dispenser.get_evaluation_mode(task.get_id()) == 'last' or \
+                          (task_dispenser.get_evaluation_mode(task.get_id()) == 'best' and old_submission.get('grade', 0.0) <= grade)
 
             if set_default:
                 self._database.user_tasks.find_one_and_update(
@@ -692,13 +692,13 @@ def update_user_stats(self, username, task, submission, result_str, grade, state
             old_submission = self._database.user_tasks.find_one(
                 {"username": username, "courseid": submission["courseid"], "taskid": submission["taskid"]})
             def_sub = []
-            if task.get_evaluate() == 'best':  # if best, update cache consequently (with best submission)
+            if task_dispenser.get_evaluation_mode(task.get_id()) == 'best':  # if best, update cache consequently (with best submission)
                 def_sub = list(self._database.submissions.find(
                     {"username": username, "courseid": task.get_course_id(), "taskid": task.get_id(),
                      "status": "done"}).sort(
                     [("grade", pymongo.DESCENDING), ("submitted_on", pymongo.DESCENDING)]).limit(1))
 
-            elif task.get_evaluate() == 'last':  # if last, update cache with last submission
+            elif task_dispenser.get_evaluation_mode(task.get_id()) == 'last':  # if last, update cache with last submission
                 def_sub = list(self._database.submissions.find(
                     {"username": username, "courseid": task.get_course_id(), "taskid": task.get_id()})
                                .sort([("submitted_on", pymongo.DESCENDING)]).limit(1))