MDL-27251 Files API - added timeout re-calculation as an optional arg…

…ument. added setting for minimum Kbps for large files fetched from internet where the passed in timeout maybe too low.

	allowed turning off the http HEAD request timeout calculation with zero (or negative) bitrate

	This was added in to allow servers that have a problem with
	HEAD requests to carry on with the given timeout without re-calculations.
	See PULL-651 for the discussion.

	the optional argument to force recalculation of timeout has been forced within scorm/locallib.php

	timeout re-calculation only increments timeout.

admin/settings/server.php

@@ -241,6 +241,8 @@
 $temp->add(new admin_setting_configtext('curlcache', get_string('curlcache', 'admin'),
                                         get_string('configcurlcache', 'admin'), 120, PARAM_INT));
 
+$temp->add(new admin_setting_configtext('curltimeoutkbitrate', get_string('curltimeoutkbitrate', 'admin'),
+                                        get_string('configcurltimeoutkbitrate_help', 'admin'), 56, PARAM_INT));
 /* //TODO: we need to fix code instead of relying on slow rcache, enable this once we have some code that is actually using it
 $temp->add(new admin_setting_special_selectsetup('cachetype', get_string('cachetype', 'admin'),
                                           get_string('configcachetype', 'admin'), '',

lang/en/admin.php

@@ -398,6 +398,8 @@
 $string['curlcache'] = 'cURL cache TTL';
 $string['curlrecommended'] = 'Installing the optional cURL library is highly recommended in order to enable Moodle Networking functionality.';
 $string['curlrequired'] = 'The cURL PHP extension is now required by Moodle, in order to communicate with Moodle repositories.';
+$string['curltimeoutkbitrate'] = 'Minimum cURL timeout bitrate (Kbps)';
+$string['curltimeoutkbitrate_help'] = 'A slow enough bitrate to call timeout when downloading file contents from the internet. HTTP HEAD requests determine the file size to calculate timeout. 0 disables any HEAD request. ';
 $string['customcheck'] = 'Other checks';
 $string['custommenu'] = 'Custom menu';
 $string['custommenuitems'] = 'Custom menu items';

lib/filelib.php

@@ -918,9 +918,10 @@ function format_postdata_for_curlcall($postdata) {
  *   may not work when using proxy
  * @param bool $skipcertverify If true, the peer's SSL certificate will not be checked. Only use this when already in a trusted location.
  * @param string $tofile store the downloaded content to file instead of returning it
+ * @param bool $calctimeout false by default, true enables an extra head request to try and determine filesize and appropriately larger timeout based on $CFG->curltimeoutkbitrate
  * @return mixed false if request failed or content of the file as string if ok. true if file downloaded into $tofile successfully.
  */
-function download_file_content($url, $headers=null, $postdata=null, $fullresponse=false, $timeout=300, $connecttimeout=20, $skipcertverify=false, $tofile=NULL) {
+function download_file_content($url, $headers=null, $postdata=null, $fullresponse=false, $timeout=300, $connecttimeout=20, $skipcertverify=false, $tofile=NULL, $calctimeout=false) {
     global $CFG;
 
     // some extra security
@@ -962,7 +963,6 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons
         curl_setopt($ch, CURLOPT_HTTPHEADER, $headers2);
     }
 
-
     if ($skipcertverify) {
         curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
     }
@@ -977,7 +977,7 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons
     curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
     curl_setopt($ch, CURLOPT_HEADER, false);
     curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $connecttimeout);
-    curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
+
     if (!ini_get('open_basedir') and !ini_get('safe_mode')) {
         // TODO: add version test for '7.10.5'
         curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
@@ -1033,6 +1033,34 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons
         curl_setopt($ch, CURLOPT_WRITEFUNCTION, partial('download_file_content_write_handler', $received));
     }
 
+    if (!isset($CFG->curltimeoutkbitrate)) {
+        //use very slow rate of 56kbps as a timeout speed when not set
+        $bitrate = 56;
+    } else {
+        $bitrate = $CFG->curltimeoutkbitrate;
+    }
+
+    //try to calculate the proper amount for timeout from remote file size.
+    if ($calctimeout && $bitrate > 0) { // if disabled or zero, we won't do any checks nor head requests.
+        //setup header request only options
+        curl_setopt_array ($ch , array(
+            CURLOPT_RETURNTRANSFER => false,
+            CURLOPT_NOBODY => true ));
+
+        curl_exec($ch);
+        $info = curl_getinfo($ch);
+        $err = curl_error($ch);
+
+        if ($err === '' && $info['download_content_length'] > 0) { //no curl errors
+            $timeout = max($timeout,ceil($info['download_content_length']*8/($bitrate*1024))); //adjust for large files only - take max timeout.
+        }
+        //reinstate affected curl options
+        curl_setopt_array ($ch , array(
+            CURLOPT_RETURNTRANSFER => true,
+            CURLOPT_NOBODY => false ));
+    }
+
+    curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
     $result = curl_exec($ch);
 
     // try to detect encoding problems

lib/filestorage/file_storage.php

@@ -702,6 +702,7 @@ public function create_file_from_url($file_record, $url, array $options = NULL,
         $timeout        = isset($options['timeout'])        ? $options['timeout'] : 300;
         $connecttimeout = isset($options['connecttimeout']) ? $options['connecttimeout'] : 20;
         $skipcertverify = isset($options['skipcertverify']) ? $options['skipcertverify'] : false;
+        $calctimeout    = isset($options['calctimeout']) ? $options['calctimeout'] : false;
 
         if (!isset($file_record->filename)) {
             $parts = explode('/', $url);
@@ -714,7 +715,7 @@ public function create_file_from_url($file_record, $url, array $options = NULL,
         if ($usetempfile) {
             check_dir_exists($this->tempdir);
             $tmpfile = tempnam($this->tempdir, 'newfromurl');
-            $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify, $tmpfile);
+            $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify, $tmpfile, $calctimeout);
             if ($content === false) {
                 throw new file_exception('storedfileproblem', 'Can not fetch file form URL');
             }
@@ -728,7 +729,7 @@ public function create_file_from_url($file_record, $url, array $options = NULL,
             }
 
         } else {
-            $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify);
+            $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify, NULL, $calctimeout);
             if ($content === false) {
                 throw new file_exception('storedfileproblem', 'Can not fetch file form URL');
             }

mod/scorm/locallib.php

@@ -186,7 +186,7 @@ function scorm_parse($scorm, $full) {
             if ($scorm->reference !== '' and (!$full or $scorm->sha1hash !== sha1($scorm->reference))) {
                 $fs->delete_area_files($context->id, 'mod_scorm', 'package');
                 $file_record = array('contextid'=>$context->id, 'component'=>'mod_scorm', 'filearea'=>'package', 'itemid'=>0, 'filepath'=>'/');
-                if ($packagefile = $fs->create_file_from_url($file_record, $scorm->reference)) {
+                if ($packagefile = $fs->create_file_from_url($file_record, $scorm->reference, array('calctimeout' => true))) {
                     $newhash = sha1($scorm->reference);
                 } else {
                     $newhash = null;