Merged MDL-27251 Fixed up whitespace, Fixed broken string, reworded n…

…ew strings

admin/settings/server.php

@@ -241,6 +241,8 @@
 $temp->add(new admin_setting_configtext('curlcache', get_string('curlcache', 'admin'),
                                         get_string('configcurlcache', 'admin'), 120, PARAM_INT));
 
+$temp->add(new admin_setting_configtext('curltimeoutkbitrate', get_string('curltimeoutkbitrate', 'admin'),
+                                        get_string('curltimeoutkbitrate_help', 'admin'), 56, PARAM_INT));
 /* //TODO: we need to fix code instead of relying on slow rcache, enable this once we have some code that is actually using it
 $temp->add(new admin_setting_special_selectsetup('cachetype', get_string('cachetype', 'admin'),
                                           get_string('configcachetype', 'admin'), '',

lang/en/admin.php

@@ -398,6 +398,8 @@
 $string['curlcache'] = 'cURL cache TTL';
 $string['curlrecommended'] = 'Installing the optional cURL library is highly recommended in order to enable Moodle Networking functionality.';
 $string['curlrequired'] = 'The cURL PHP extension is now required by Moodle, in order to communicate with Moodle repositories.';
+$string['curltimeoutkbitrate'] = 'Bitrate to use when calculating cURL timeouts (Kbps)';
+$string['curltimeoutkbitrate_help'] = 'This setting is used to calculate an appropriate timeout during large cURL requests. As part of this calculation an HTTP HEAD request is made to determine the size of the content. Setting this to 0 disables this request from being made.';
 $string['customcheck'] = 'Other checks';
 $string['custommenu'] = 'Custom menu';
 $string['custommenuitems'] = 'Custom menu items';

lib/filelib.php

@@ -904,7 +904,6 @@ function format_postdata_for_curlcall($postdata) {
  * Fetches content of file from Internet (using proxy if defined). Uses cURL extension if present.
  * Due to security concerns only downloads from http(s) sources are supported.
  *
- * @global object
  * @param string $url file url starting with http(s)://
  * @param array $headers http headers, null if none. If set, should be an
  *   associative array of header name => value pairs.
@@ -916,11 +915,14 @@ function format_postdata_for_curlcall($postdata) {
  * @param int $connecttimeout timeout for connection to server; this is the timeout that
  *   usually happens if the remote server is completely down (default 20 seconds);
  *   may not work when using proxy
- * @param bool $skipcertverify If true, the peer's SSL certificate will not be checked. Only use this when already in a trusted location.
- * @param string $tofile store the downloaded content to file instead of returning it
- * @return mixed false if request failed or content of the file as string if ok. true if file downloaded into $tofile successfully.
+ * @param bool $skipcertverify If true, the peer's SSL certificate will not be checked. 
+ *   Only use this when already in a trusted location.
+ * @param string $tofile store the downloaded content to file instead of returning it.
+ * @param bool $calctimeout false by default, true enables an extra head request to try and determine 
+ *   filesize and appropriately larger timeout based on $CFG->curltimeoutkbitrate
+ * @return mixed false if request failed or content of the file as string if ok. True if file downloaded into $tofile successfully.
  */
-function download_file_content($url, $headers=null, $postdata=null, $fullresponse=false, $timeout=300, $connecttimeout=20, $skipcertverify=false, $tofile=NULL) {
+function download_file_content($url, $headers=null, $postdata=null, $fullresponse=false, $timeout=300, $connecttimeout=20, $skipcertverify=false, $tofile=NULL, $calctimeout=false) {
     global $CFG;
 
     // some extra security
@@ -962,7 +964,6 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons
         curl_setopt($ch, CURLOPT_HTTPHEADER, $headers2);
     }
 
-
     if ($skipcertverify) {
         curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
     }
@@ -977,7 +978,7 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons
     curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
     curl_setopt($ch, CURLOPT_HEADER, false);
     curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $connecttimeout);
-    curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
+
     if (!ini_get('open_basedir') and !ini_get('safe_mode')) {
         // TODO: add version test for '7.10.5'
         curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
@@ -1033,6 +1034,37 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons
         curl_setopt($ch, CURLOPT_WRITEFUNCTION, partial('download_file_content_write_handler', $received));
     }
 
+    if (!isset($CFG->curltimeoutkbitrate)) {
+        //use very slow rate of 56kbps as a timeout speed when not set
+        $bitrate = 56;
+    } else {
+        $bitrate = $CFG->curltimeoutkbitrate;
+    }
+
+    // try to calculate the proper amount for timeout from remote file size.
+    // if disabled or zero, we won't do any checks nor head requests.
+    if ($calctimeout && $bitrate > 0) {
+        //setup header request only options
+        curl_setopt_array ($ch, array(
+            CURLOPT_RETURNTRANSFER => false,
+            CURLOPT_NOBODY         => true)
+        );
+
+        curl_exec($ch);
+        $info = curl_getinfo($ch);
+        $err = curl_error($ch);
+
+        if ($err === '' && $info['download_content_length'] > 0) { //no curl errors
+            $timeout = max($timeout, ceil($info['download_content_length'] * 8 / ($bitrate * 1024))); //adjust for large files only - take max timeout.
+        }
+        //reinstate affected curl options
+        curl_setopt_array ($ch, array(
+            CURLOPT_RETURNTRANSFER => true,
+            CURLOPT_NOBODY         => false)
+        );
+    }
+
+    curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
     $result = curl_exec($ch);
 
     // try to detect encoding problems

lib/filestorage/file_storage.php

@@ -702,6 +702,7 @@ public function create_file_from_url($file_record, $url, array $options = NULL,
         $timeout        = isset($options['timeout'])        ? $options['timeout'] : 300;
         $connecttimeout = isset($options['connecttimeout']) ? $options['connecttimeout'] : 20;
         $skipcertverify = isset($options['skipcertverify']) ? $options['skipcertverify'] : false;
+        $calctimeout    = isset($options['calctimeout'])    ? $options['calctimeout'] : false;
 
         if (!isset($file_record->filename)) {
             $parts = explode('/', $url);
@@ -714,7 +715,7 @@ public function create_file_from_url($file_record, $url, array $options = NULL,
         if ($usetempfile) {
             check_dir_exists($this->tempdir);
             $tmpfile = tempnam($this->tempdir, 'newfromurl');
-            $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify, $tmpfile);
+            $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify, $tmpfile, $calctimeout);
             if ($content === false) {
                 throw new file_exception('storedfileproblem', 'Can not fetch file form URL');
             }
@@ -728,7 +729,7 @@ public function create_file_from_url($file_record, $url, array $options = NULL,
             }
 
         } else {
-            $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify);
+            $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify, NULL, $calctimeout);
             if ($content === false) {
                 throw new file_exception('storedfileproblem', 'Can not fetch file form URL');
             }

mod/scorm/locallib.php

@@ -186,7 +186,7 @@ function scorm_parse($scorm, $full) {
             if ($scorm->reference !== '' and (!$full or $scorm->sha1hash !== sha1($scorm->reference))) {
                 $fs->delete_area_files($context->id, 'mod_scorm', 'package');
                 $file_record = array('contextid'=>$context->id, 'component'=>'mod_scorm', 'filearea'=>'package', 'itemid'=>0, 'filepath'=>'/');
-                if ($packagefile = $fs->create_file_from_url($file_record, $scorm->reference)) {
+                if ($packagefile = $fs->create_file_from_url($file_record, $scorm->reference, array('calctimeout' => true))) {
                     $newhash = sha1($scorm->reference);
                 } else {
                     $newhash = null;