From 23d6756256a2602dbc17cc9c0af83d12b02bb0f5 Mon Sep 17 00:00:00 2001
From: Andrew Nicols <andrew@nicols.co.uk>
Date: Tue, 15 Jan 2019 07:55:15 +0800
Subject: [PATCH] MDL-64347 task: Add restrictions to scheduled task runner

---
 admin/settings/server.php | 19 ++++++++++++++++
 lang/en/admin.php         |  4 ++++
 lib/cronlib.php           | 47 ++++++++++++++++++++++++++++++++++-----
 version.php               |  2 +-
 4 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/admin/settings/server.php b/admin/settings/server.php
index 1616fc4bb9cd5..96c7871dd6eb1 100644
--- a/admin/settings/server.php
+++ b/admin/settings/server.php
@@ -213,6 +213,25 @@
 
 $ADMIN->add('server', new admin_category('taskconfig', new lang_string('taskadmintitle', 'admin')));
 $temp = new admin_settingpage('taskprocessing', new lang_string('taskprocessing','admin'));
+$temp->add(
+    new admin_setting_configtext(
+        'task_scheduled_concurrency_limit',
+        new lang_string('task_scheduled_concurrency_limit', 'admin'),
+        new lang_string('task_scheduled_concurrency_limit_desc', 'admin'),
+        3,
+        PARAM_INT
+    )
+);
+
+$temp->add(
+    new admin_setting_configduration(
+        'task_scheduled_max_runtime',
+        new lang_string('task_scheduled_max_runtime', 'admin'),
+        new lang_string('task_scheduled_max_runtime_desc', 'admin'),
+        30 * MINSECS
+    )
+);
+
 $temp->add(
     new admin_setting_configtext(
         'task_adhoc_concurrency_limit',
diff --git a/lang/en/admin.php b/lang/en/admin.php
index c3ecd5432d37e..dbb7577c9c7b2 100644
--- a/lang/en/admin.php
+++ b/lang/en/admin.php
@@ -1165,6 +1165,10 @@
 $string['tabselectedtofront'] = 'On tables with tabs, should the row with the currently selected tab be placed at the front';
 $string['tabselectedtofronttext'] = 'Bring selected tab row to front';
 $string['testsiteupgradewarning'] = 'You are currently using the {$a} test site, to upgrade it properly use the command line interface tool';
+$string['task_scheduled_concurrency_limit'] = 'Scheduled task concurrency limit';
+$string['task_scheduled_concurrency_limit_desc'] = 'The number of scheduled task runners allowed to run concurrently. If the limit is high then the server may experience high load which affects performance. A setting of 0 will disable processing of scheduled tasks completely.';
+$string['task_scheduled_max_runtime'] = 'Scheduled task runner lifetime';
+$string['task_scheduled_max_runtime_desc'] = 'The age of a scheduled task runner before it is freed.';
 $string['task_adhoc_concurrency_limit'] = 'Adhoc task concurrency limit';
 $string['task_adhoc_concurrency_limit_desc'] = 'The number of adhoc task runners allowed to run concurrently. If the limit is high then scheduled tasks may not run regularly when there are lots of adhoc tasks. A setting of 0 will disable processing of adhoc tasks completely.';
 $string['task_adhoc_max_runtime'] = 'Adhoc task runner lifetime';
diff --git a/lib/cronlib.php b/lib/cronlib.php
index 77d93abbc72ce..7341ad1e9552e 100644
--- a/lib/cronlib.php
+++ b/lib/cronlib.php
@@ -62,11 +62,7 @@ function cron_run() {
     mtrace("Server Time: ".date('r', $timenow)."\n\n");
 
     // Run all scheduled tasks.
-    while (!\core\task\manager::static_caches_cleared_since($timenow) &&
-           $task = \core\task\manager::get_next_scheduled_task($timenow)) {
-        cron_run_inner_scheduled_task($task);
-        unset($task);
-    }
+    cron_run_scheduled_tasks($timenow);
 
     // Run adhoc tasks.
     cron_run_adhoc_tasks($timenow);
@@ -79,6 +75,47 @@ function cron_run() {
     mtrace("Execution took ".$difftime." seconds");
 }
 
+/**
+ * Execute all queued scheduled tasks, applying necessary concurrency limits and time limits.
+ *
+ * @param   int     $timenow The time this process started.
+ */
+function cron_run_scheduled_tasks(int $timenow) {
+    // Allow a restriction on the number of scheduled task runners at once.
+    $cronlockfactory = \core\lock\lock_config::get_lock_factory('cron');
+    $maxruns = get_config('core', 'task_scheduled_concurrency_limit');
+    $maxruntime = get_config('core', 'task_scheduled_max_runtime');
+
+    $scheduledlock = null;
+    for ($run = 0; $run < $maxruns; $run++) {
+        if ($scheduledlock = $cronlockfactory->get_lock("scheduled_task_runner_{$run}", 1)) {
+            break;
+        }
+    }
+
+    if (!$scheduledlock) {
+        mtrace("Skipping processing of scheduled tasks. Concurrency limit reached.");
+        return;
+    }
+
+    $starttime = time();
+
+    // Run all scheduled tasks.
+    while (!\core\task\manager::static_caches_cleared_since($timenow) &&
+            $task = \core\task\manager::get_next_scheduled_task($timenow)) {
+        cron_run_inner_scheduled_task($task);
+        unset($task);
+
+        if ((time() - $starttime) > $maxruntime) {
+            mtrace("Stopping processing of scheduled tasks as time limit has been reached.");
+            break;
+        }
+    }
+
+    // Release the scheduled task runner lock.
+    $scheduledlock->release();
+}
+
 /**
  * Execute all queued adhoc tasks, applying necessary concurrency limits and time limits.
  *
diff --git a/version.php b/version.php
index cdf02207c319f..eb88f2080616f 100644
--- a/version.php
+++ b/version.php
@@ -29,7 +29,7 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$version  = 2019011502.00;              // YYYYMMDD      = weekly release date of this DEV branch.
+$version  = 2019011503.00;              // YYYYMMDD      = weekly release date of this DEV branch.
                                         //         RR    = release increments - 00 in DEV branches.
                                         //           .XX = incremental changes.