SIO-1918 Implement priorities for scheduled tasks

Change-Id: I0e5efb58e84665bf8f89acfed4e26a1ebc966aec

README.rst

@@ -540,6 +540,20 @@ List of changes since the *CONFIG_VERSION* numbering was introduced:
         command=twistd -n -l- --pidfile={{ PROJECT_DIR }}/pidfiles/sioworkersd.pid sioworkersd --database={{ PROJECT_DIR }}/sioworkersd.db
         # (...)
 
+#. * Added commented out *OIOIOI_INSTANCE_PRIORITY_BONUS* and
+     *OIOIOI_INSTANCE_WEIGHT_BONUS* entries to *deployment/settings.py*.::
+
+        # Bonus to judging priority ang judging weight for each contest on this
+        # OIOIOI instance.
+        #OIOIOI_INSTANCE_PRIORITY_BONUS = 0
+        #OIOIOI_INSTANCE_WEIGHT_BONUS = 0
+
+   * Modified comment to *SITE_NAME* entry in *deployment/settings.py*.::
+
+        # Site name displayed in the title and used by sioworkersd
+        # to distinguish OIOIOI instances.
+        SITE_NAME = 'OIOIOI'
+
 Usage
 -----
 

oioioi/contests/admin.py

@@ -143,8 +143,6 @@ class ContestLinkInline(admin.TabularInline):
 
 class ContestAdmin(admin.ModelAdmin):
     inlines = [RoundInline, AttachmentInline, ContestLinkInline]
-    fields = ['name', 'id', 'controller_name', 'default_submissions_limit',
-              'contact_email']
     readonly_fields = ['creation_date']
     prepopulated_fields = {'id': ('name',)}
     list_display = ['name', 'id', 'creation_date']
@@ -162,6 +160,13 @@ def has_change_permission(self, request, obj=None):
     def has_delete_permission(self, request, obj=None):
         return self.has_change_permission(request, obj)
 
+    def get_fields(self, request, obj=None):
+        fields = ['name', 'id', 'controller_name', 'default_submissions_limit',
+            'contact_email']
+        if request.user.is_superuser:
+            fields += ['judging_priority', 'judging_weight']
+        return fields
+
     def get_fieldsets(self, request, obj=None):
         if obj and not request.GET.get('simple', False):
             return super(ContestAdmin, self).get_fieldsets(request, obj)

oioioi/contests/migrations/0008_auto_20170424_1623.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.core.validators
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contests', '0007_auto_20161214_1411'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='contest',
+            name='judging_priority',
+            field=models.IntegerField(default=10, help_text='Contest with higher judging priority is always judged before contest with lower judging priority.', verbose_name='judging priority'),
+        ),
+        migrations.AddField(
+            model_name='contest',
+            name='judging_weight',
+            field=models.IntegerField(default=1000, help_text='If some contests have the same judging priority, the judging resources are allocated proportionally to their weights.', verbose_name='judging weight', validators=[django.core.validators.MinValueValidator(1)]),
+        ),
+    ]

oioioi/contests/models.py

@@ -4,6 +4,7 @@
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.exceptions import ValidationError
+from django.core.validators import MinValueValidator
 from django.db import models
 from django.db.models import Max
 from django.db.models.signals import pre_save, post_save
@@ -59,6 +60,21 @@ class Contest(models.Model):
                 "to this contest (i.e. submission confirmations) "
                 "will have the return address set to this value. "
                 "Defaults to system admins address if left empty."))
+    judging_priority = models.IntegerField(
+            verbose_name=_("judging priority"),
+            default=settings.DEFAULT_CONTEST_PRIORITY,
+            help_text=_(
+                "Contest with higher judging priority is always judged "
+                "before contest with lower judging priority."))
+    judging_weight = models.IntegerField(
+            verbose_name=_("judging weight"),
+            default=settings.DEFAULT_CONTEST_WEIGHT,
+            validators=[MinValueValidator(1)],
+            help_text=_(
+                "If some contests have the same judging priority, the "
+                "judging resources are allocated proportionally to "
+                "their weights."
+            ))
 
     @property
     def controller(self):

oioioi/contests/tests/tests.py

@@ -14,6 +14,7 @@
 from django.utils.timezone import utc, LocalTimezone
 from django.contrib.auth.models import User, AnonymousUser
 from django.contrib.admin.utils import quote
+from django.conf import settings
 from nose.tools import nottest
 
 from oioioi.base.tests import TestCase, check_not_accessible, fake_time, \
@@ -38,10 +39,15 @@
 
 
 class TestModels(TestCase):
+
     def test_fields_autogeneration(self):
         contest = Contest()
         contest.save()
         self.assertEqual(contest.id, 'c1')
+        self.assertEqual(contest.judging_priority,
+            settings.DEFAULT_CONTEST_PRIORITY)
+        self.assertEqual(contest.judging_weight,
+            settings.DEFAULT_CONTEST_WEIGHT)
         round = Round(contest=contest)
         round.save()
         self.assertEqual(round.name, 'Round 1')
@@ -888,6 +894,9 @@ def test_simple_contest_create_and_change(self):
         url = reverse('oioioiadmin:contests_contest_add')
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
+        self.assertContains(response, "Start date")
+        self.assertNotContains(response, "Judging priority")
+        self.assertNotContains(response, "Judging weight")
         post_data = {
                 'name': 'cname',
                 'id': 'cid',
@@ -907,6 +916,10 @@ def test_simple_contest_create_and_change(self):
         contest = Contest.objects.get()
         self.assertEqual(contest.id, 'cid')
         self.assertEqual(contest.name, 'cname')
+        self.assertEqual(contest.judging_priority,
+            settings.DEFAULT_CONTEST_PRIORITY)
+        self.assertEqual(contest.judging_weight,
+            settings.DEFAULT_CONTEST_WEIGHT)
         self.assertEqual(contest.round_set.count(), 1)
         round = contest.round_set.get()
         self.assertEqual(round.start_date,
@@ -921,6 +934,8 @@ def test_simple_contest_create_and_change(self):
         response = self.client.get(url)
         self.assertIn('2012-02-05', response.content)
         self.assertIn('06:07:08', response.content)
+        self.assertNotContains(response, "Judging priority")
+        self.assertNotContains(response, "Judging weight")
 
         post_data = {
                 'name': 'cname1',
@@ -1773,6 +1788,8 @@ def test_modify_contest(self):
         url = reverse('oioioiadmin:contests_contest_add')
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
+        self.assertNotContains(response, "Judging priority")
+        self.assertNotContains(response, "Judging weight")
         post_data = {
                 'name': 'Yet Another Contest',
                 'id': 'yac',
@@ -1792,11 +1809,21 @@ def test_modify_contest(self):
         ContestPermission(user=User.objects.get(pk=1001), contest=contest,
             permission='contests.contest_admin').save()
 
+        url = reverse('oioioiadmin:contests_contest_change',
+            args=(quote('yac'),))
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertContains(response, "Judging priority")
+        self.assertContains(response, "Judging weight")
+
         self.client.login(username='test_user')
         url = reverse('oioioiadmin:contests_contest_change',
                 args=(quote('yac'),)) + '?simple=true'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
+        self.assertNotContains(response, "Default submissions limit")
+        self.assertNotContains(response, "Judging priority")
+        self.assertNotContains(response, "Judging weight")
         post_data = {
                 'name': 'New Name',
                 'start_date_0': '2013-02-03',
@@ -1812,6 +1839,14 @@ def test_modify_contest(self):
         self.assertEqual(contest.id, 'yac')
         self.assertEqual(controller_name, contest.controller_name)
 
+        url = reverse('oioioiadmin:contests_contest_change',
+            args=(quote('yac'),))
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertContains(response, "Default submissions limit")
+        self.assertNotContains(response, "Judging priority")
+        self.assertNotContains(response, "Judging weight")
+
 
 class TestRegistrationController(TestCase):
     fixtures = ['test_two_empty_contests', 'test_users']

oioioi/default_settings.py

@@ -13,13 +13,14 @@
 import oioioi
 from oioioi.contests.current_contest import ContestMode
 
-INSTALLATION_CONFIG_VERSION = 11
+INSTALLATION_CONFIG_VERSION = 12
 
 DEBUG = False
 TEMPLATE_DEBUG = DEBUG
 INTERNAL_IPS = ('127.0.0.1',)
 
-# Site name displayed in the title
+# Site name displayed in the title and used by sioworkersd
+# to distinguish OIOIOI instances.
 SITE_NAME = 'OIOIOI'
 
 # Run uwsgi daemon. Shall be True, False or 'auto'.
@@ -479,3 +480,11 @@
 # Domain to use for serving IP to hostname mappings
 # using ./manage.py ipauth-dnsserver
 IPAUTH_DNSSERVER_DOMAIN = None
+
+# Judging priority and weight settings
+DEFAULT_CONTEST_PRIORITY = 10
+DEFAULT_CONTEST_WEIGHT = 1000
+OIOIOI_INSTANCE_PRIORITY_BONUS = 0
+OIOIOI_INSTANCE_WEIGHT_BONUS = 0
+NON_CONTEST_PRIORITY = 0
+NON_CONTEST_WEIGHT = 1000

oioioi/deployment/settings.py.template

@@ -17,7 +17,8 @@ TEMPLATE_DEBUG = DEBUG
 if DEBUG:
     TEMPLATE_LOADERS = UNCACHED_TEMPLATE_LOADERS
 
-# Site name displayed in the title
+# Site name displayed in the title and used by sioworkersd
+# to distinguish OIOIOI instances.
 SITE_NAME = 'OIOIOI'
 
 # Email addresses to send error message reports.
@@ -372,3 +373,8 @@ RAVEN_CONFIG = {
     'release': raven.fetch_git_sha(
         os.path.join(os.path.dirname(oioioi.__file__), os.pardir)),
 }
+
+# Bonus to judging priority ang judging weight for each contest on this
+# OIOIOI instance.
+#OIOIOI_INSTANCE_PRIORITY_BONUS = 0
+#OIOIOI_INSTANCE_WEIGHT_BONUS = 0

oioioi/programs/controllers.py

@@ -63,6 +63,13 @@ def generate_initial_evaluation_environ(self, environ, submission,
             environ['contest_id'] = contest.id
         environ['submission_owner'] = submission.user.username \
                                       if submission.user else None
+        environ['oioioi_instance'] = settings.SITE_NAME
+        environ['contest_priority'] = contest.judging_priority \
+            if contest is not None else settings.NON_CONTEST_PRIORITY
+        environ['contest_priority'] += settings.OIOIOI_INSTANCE_PRIORITY_BONUS
+        environ['contest_weight'] = contest.judging_weight \
+            if contest is not None else settings.NON_CONTEST_WEIGHT
+        environ['contest_weight'] += settings.OIOIOI_INSTANCE_WEIGHT_BONUS
 
         environ.setdefault('report_kinds', ['INITIAL', 'NORMAL'])
         if 'hidden_judge' in environ['extra_args']:

oioioi/programs/handlers.py

@@ -23,8 +23,11 @@
 
 logger = logging.getLogger(__name__)
 
-DEFAULT_NORMAL_PRIORITY = 100
-DEFAULT_HIGH_PRIORITY = 50
+COMPILE_TASK_PRIORITY = 200
+EXAMPLE_TEST_TASK_PRIORITY = 300
+TESTRUN_TEST_TASK_PRIORITY = 300
+DEFAULT_TEST_TASK_PRIORITY = 100
+# There is also TASK_PRIORITY in oioioi/sinolpack/package.py.
 
 
 def _make_filename(env, base_name):
@@ -84,6 +87,7 @@ def compile(env, **kwargs):
 
     compilation_job = env.copy()
     compilation_job['job_type'] = 'compile'
+    compilation_job['task_priority'] = COMPILE_TASK_PRIORITY
     compilation_job['out_file'] = _make_filename(env, 'exe')
     if 'language' in env and 'compiler' not in env:
         compilation_job['compiler'] = 'default-' + env['language']
@@ -218,9 +222,6 @@ def run_tests(env, kind=None, **kwargs):
 
            If the dictionary already exists, new test results are appended.
     """
-    priority = DEFAULT_NORMAL_PRIORITY
-    if kind == 'INITIAL' or kind == 'EXAMPLE':
-        priority = DEFAULT_HIGH_PRIORITY
     jobs = dict()
     not_to_judge = []
     for test_name, test_env in env['tests'].iteritems():
@@ -231,9 +232,14 @@ def run_tests(env, kind=None, **kwargs):
             continue
         job = test_env.copy()
         job['job_type'] = (env.get('exec_mode', '') + '-exec').lstrip('-')
+        if kind == 'INITIAL' or kind == 'EXAMPLE':
+            job['task_priority'] = EXAMPLE_TEST_TASK_PRIORITY
+        elif env['submission_kind'] == 'TESTRUN':
+            job['task_priority'] = TESTRUN_TEST_TASK_PRIORITY
+        else:
+            job['task_priority'] = DEFAULT_TEST_TASK_PRIORITY
         job['exe_file'] = env['compiled_file']
         job['exec_info'] = env['exec_info']
-        job['priority'] = priority
         job['check_output'] = env.get('check_outputs', True)
         if env.get('checker'):
             job['chk_file'] = env['checker']

oioioi/sinolpack/package.py

@@ -34,6 +34,7 @@
 
 DEFAULT_TIME_LIMIT = 10000
 DEFAULT_MEMORY_LIMIT = 66000
+TASK_PRIORITY = 500
 C_EXTRA_ARGS = ['-Wall', '-Wno-unused-result', '-Werror']
 PAS_EXTRA_ARGS = ['-Ci', '-Cr', '-Co', '-gl']
 
@@ -325,6 +326,7 @@ def _compile(self, filename, prog_name, ext, out_name=None):
 
         compilation_job = self.env.copy()
         compilation_job['job_type'] = 'compile'
+        compilation_job['task_priority'] = TASK_PRIORITY
         compilation_job['source_file'] = ft_source_name
         compilation_job['out_file'] = out_name
         lang = ext
@@ -400,6 +402,7 @@ def _make_ins(self, re_string):
         env = self._find_and_compile('ingen')
         if env and not self.use_make:
             env['job_type'] = 'ingen'
+            env['task_priority'] = TASK_PRIORITY
             env['exe_file'] = env['compiled_file']
             env['re_string'] = re_string
             env['use_sandboxes'] = self.use_sandboxes
@@ -420,6 +423,7 @@ def _make_outs(self, outs_to_make):
         for outname, test in outs_to_make:
             job = env.copy()
             job['job_type'] = 'exec' if self.use_sandboxes else 'unsafe-exec'
+            job['task_priority'] = TASK_PRIORITY
             job['exe_file'] = env['compiled_file']
             job['upload_out'] = True
             job['in_file'] = django_to_filetracker_path(test.input_file)
@@ -438,6 +442,7 @@ def _verify_ins(self, tests):
             for test in tests:
                 job = env.copy()
                 job['job_type'] = 'inwer'
+                job['task_priority'] = TASK_PRIORITY
                 job['exe_file'] = env['compiled_file']
                 job['in_file'] = django_to_filetracker_path(test.input_file)
                 job['use_sandboxes'] = self.use_sandboxes

oioioi/sioworkers/backends.py

@@ -82,6 +82,11 @@ class SioworkersdBackend(object):
     def run_job(self, job, **kwargs):
         env = {'workers_jobs': {'dummy_name': job}}
         env['workers_jobs.extra_args'] = kwargs
+        env['oioioi_instance'] = settings.SITE_NAME
+        env['contest_priority'] = (settings.OIOIOI_INSTANCE_PRIORITY_BONUS +
+            settings.NON_CONTEST_PRIORITY)
+        env['contest_weight'] = (settings.OIOIOI_INSTANCE_WEIGHT_BONUS +
+            settings.NON_CONTEST_WEIGHT)
         ans = SioworkersdBackend.server.sync_run_group(json.dumps(env))
         if 'error' in ans:
             raise RuntimeError('Error from workers:\n%s\nTB:\n%s' %
@@ -91,6 +96,11 @@ def run_job(self, job, **kwargs):
     def run_jobs(self, dict_of_jobs, **kwargs):
         env = {'workers_jobs': dict_of_jobs,
                 'workers_jobs.extra_args': kwargs}
+        env['oioioi_instance'] = settings.SITE_NAME
+        env['contest_priority'] = (settings.OIOIOI_INSTANCE_PRIORITY_BONUS +
+            settings.NON_CONTEST_PRIORITY)
+        env['contest_weight'] = (settings.OIOIOI_INSTANCE_WEIGHT_BONUS +
+            settings.NON_CONTEST_WEIGHT)
         ans = SioworkersdBackend.server.sync_run_group(json.dumps(env))
         if 'error' in ans:
             raise RuntimeError('Error from workers:\n%s\nTB:\n%s' %