Add basic support for lower priority PR builds

readthedocs/core/utils/__init__.py

@@ -18,6 +18,7 @@
     EXTERNAL,
 )
 from readthedocs.doc_builder.constants import DOCKER_LIMITS
+from readthedocs.projects.constants import CELERY_LOW, CELERY_MEDIUM, CELERY_HIGH
 from readthedocs.doc_builder.exceptions import BuildMaxConcurrencyError
 
 
@@ -150,6 +151,14 @@ def prepare_build(
         # Send Webhook notification for build triggered.
         send_notifications.delay(version.pk, build_pk=build.pk, email=False)
 
+    options['priority'] = CELERY_HIGH
+    if project.main_language_project:
+        # Translations should be medium priority
+        options['priority'] = CELERY_MEDIUM
+    if version.type == EXTERNAL:
+        # External builds should be lower priority.
+        options['priority'] = CELERY_LOW
+
     # Start the build in X minutes and mark it as limited
     if project.has_feature(Feature.LIMIT_CONCURRENT_BUILDS):
         running_builds = (

readthedocs/projects/constants.py

@@ -374,3 +374,9 @@
 # Git provider names
 GITHUB_BRAND = 'GitHub'
 GITLAB_BRAND = 'GitLab'
+
+# Set 3 priorities, [low, medium, high] -- default is medium
+# Leave some space on each side of the set to expand if needed
+CELERY_LOW = 3
+CELERY_MEDIUM = 5
+CELERY_HIGH = 7

readthedocs/rtd_tests/tests/test_core_utils.py

@@ -13,14 +13,15 @@
 from readthedocs.core.utils import slugify, trigger_build, prepare_build
 from readthedocs.core.utils.general import wipe_version_via_slugs
 from readthedocs.doc_builder.exceptions import BuildMaxConcurrencyError
+from readthedocs.projects.constants import CELERY_LOW, CELERY_MEDIUM, CELERY_HIGH
 from readthedocs.projects.models import Project, Feature
 from readthedocs.projects.tasks import remove_dirs
 
 
 class CoreUtilTests(TestCase):
 
     def setUp(self):
-        self.project = get(Project, container_time_limit=None)
+        self.project = get(Project, container_time_limit=None, main_language_project=None)
         self.version = get(Version, project=self.project)
 
     @mock.patch('readthedocs.projects.tasks.update_docs_task')
@@ -101,6 +102,7 @@ def test_trigger_custom_queue(self, update_docs):
             'queue': 'build03',
             'time_limit': 720,
             'soft_time_limit': 600,
+            'priority': CELERY_HIGH,
         }
         update_docs.signature.assert_called_with(
             args=(self.version.pk,),
@@ -123,6 +125,7 @@ def test_trigger_build_time_limit(self, update_docs):
             'queue': mock.ANY,
             'time_limit': 720,
             'soft_time_limit': 600,
+            'priority': CELERY_HIGH,
         }
         update_docs.signature.assert_called_with(
             args=(self.version.pk,),
@@ -146,6 +149,7 @@ def test_trigger_build_invalid_time_limit(self, update_docs):
             'queue': mock.ANY,
             'time_limit': 720,
             'soft_time_limit': 600,
+            'priority': CELERY_HIGH,
         }
         update_docs.signature.assert_called_with(
             args=(self.version.pk,),
@@ -169,6 +173,7 @@ def test_trigger_build_rounded_time_limit(self, update_docs):
             'queue': mock.ANY,
             'time_limit': 3,
             'soft_time_limit': 3,
+            'priority': CELERY_HIGH,
         }
         update_docs.signature.assert_called_with(
             args=(self.version.pk,),
@@ -177,6 +182,7 @@ def test_trigger_build_rounded_time_limit(self, update_docs):
             immutable=True,
         )
 
+
     @mock.patch('readthedocs.projects.tasks.update_docs_task')
     def test_trigger_max_concurrency_reached(self, update_docs):
         get(
@@ -208,6 +214,7 @@ def test_trigger_max_concurrency_reached(self, update_docs):
             'soft_time_limit': 600,
             'countdown': 5 * 60,
             'max_retries': 25,
+            'priority': CELERY_HIGH,
         }
         update_docs.signature.assert_called_with(
             args=(self.version.pk,),
@@ -218,6 +225,54 @@ def test_trigger_max_concurrency_reached(self, update_docs):
         build = self.project.builds.first()
         self.assertEqual(build.error, BuildMaxConcurrencyError.message.format(limit=max_concurrent_builds))
 
+    @mock.patch('readthedocs.projects.tasks.update_docs_task')
+    def test_trigger_external_build_low_priority(self, update_docs):
+        """Time limit should round down."""
+        self.version.type = 'external'
+        trigger_build(project=self.project, version=self.version)
+        kwargs = {
+            'record': True,
+            'force': False,
+            'build_pk': mock.ANY,
+            'commit': None
+        }
+        options = {
+            'queue': mock.ANY,
+            'time_limit': mock.ANY,
+            'soft_time_limit': mock.ANY,
+            'priority': CELERY_LOW,
+        }
+        update_docs.signature.assert_called_with(
+            args=(self.version.pk,),
+            kwargs=kwargs,
+            options=options,
+            immutable=True,
+        )
+
+    @mock.patch('readthedocs.projects.tasks.update_docs_task')
+    def test_trigger_build_translation_medium_priority(self, update_docs):
+        """Time limit should round down."""
+        self.project.main_language_project = get(Project, slug='main')
+        trigger_build(project=self.project, version=self.version)
+        kwargs = {
+            'record': True,
+            'force': False,
+            'build_pk': mock.ANY,
+            'commit': None
+        }
+        options = {
+            'queue': mock.ANY,
+            'time_limit': mock.ANY,
+            'soft_time_limit': mock.ANY,
+            'priority': CELERY_MEDIUM,
+        }
+        update_docs.signature.assert_called_with(
+            args=(self.version.pk,),
+            kwargs=kwargs,
+            options=options,
+            immutable=True,
+        )
+
     def test_slugify(self):
         """Test additional slugify."""
         self.assertEqual(

readthedocs/settings/base.py

@@ -6,6 +6,7 @@
 from celery.schedules import crontab
 
 from readthedocs.core.settings import Settings
+from readthedocs.projects.constants import CELERY_LOW, CELERY_MEDIUM, CELERY_HIGH
 
 
 try:
@@ -332,6 +333,12 @@ def USE_PROMOS(self):  # noqa
     CELERYD_PREFETCH_MULTIPLIER = 1
     CELERY_CREATE_MISSING_QUEUES = True
 
+
+    BROKER_TRANSPORT_OPTIONS = {
+        'queue_order_strategy': 'priority',
+        'priority_steps': [CELERY_LOW, CELERY_MEDIUM, CELERY_HIGH],
+    }
+
     CELERY_DEFAULT_QUEUE = 'celery'
     CELERYBEAT_SCHEDULE = {
         # Ran every hour on minute 30