From 60b5a2fec77f5adf9656c2f825242d7bec485b6c Mon Sep 17 00:00:00 2001 From: Aparup Banerjee Date: Mon, 28 Mar 2011 16:36:42 +0800 Subject: [PATCH] MDL-27251 Files API - added timeout re-calculation as an optional argument. added setting for minimum Kbps for large files fetched from internet where the passed in timeout maybe too low. allowed turning off the http HEAD request timeout calculation with zero (or negative) bitrate This was added in to allow servers that have a problem with HEAD requests to carry on with the given timeout without re-calculations. See PULL-651 for the discussion. the optional argument to force recalculation of timeout has been forced within scorm/locallib.php timeout re-calculation only increments timeout. --- admin/settings/server.php | 2 ++ lang/en/admin.php | 2 ++ lib/filelib.php | 34 +++++++++++++++++++++++++++++--- lib/filestorage/file_storage.php | 5 +++-- mod/scorm/locallib.php | 2 +- 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/admin/settings/server.php b/admin/settings/server.php index 807bf75f8e9d7..57a0548d12c78 100644 --- a/admin/settings/server.php +++ b/admin/settings/server.php @@ -241,6 +241,8 @@ $temp->add(new admin_setting_configtext('curlcache', get_string('curlcache', 'admin'), get_string('configcurlcache', 'admin'), 120, PARAM_INT)); +$temp->add(new admin_setting_configtext('curltimeoutkbitrate', get_string('curltimeoutkbitrate', 'admin'), + get_string('configcurltimeoutkbitrate_help', 'admin'), 56, PARAM_INT)); /* //TODO: we need to fix code instead of relying on slow rcache, enable this once we have some code that is actually using it $temp->add(new admin_setting_special_selectsetup('cachetype', get_string('cachetype', 'admin'), get_string('configcachetype', 'admin'), '', diff --git a/lang/en/admin.php b/lang/en/admin.php index 427fa20f55bcf..18ad9c523afc8 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -398,6 +398,8 @@ $string['curlcache'] = 'cURL cache TTL'; $string['curlrecommended'] = 'Installing the optional cURL library is highly recommended in order to enable Moodle Networking functionality.'; $string['curlrequired'] = 'The cURL PHP extension is now required by Moodle, in order to communicate with Moodle repositories.'; +$string['curltimeoutkbitrate'] = 'Minimum cURL timeout bitrate (Kbps)'; +$string['curltimeoutkbitrate_help'] = 'A slow enough bitrate to call timeout when downloading file contents from the internet. HTTP HEAD requests determine the file size to calculate timeout. 0 disables any HEAD request. '; $string['customcheck'] = 'Other checks'; $string['custommenu'] = 'Custom menu'; $string['custommenuitems'] = 'Custom menu items'; diff --git a/lib/filelib.php b/lib/filelib.php index 3e86263bc1273..93aeac87e0228 100644 --- a/lib/filelib.php +++ b/lib/filelib.php @@ -918,9 +918,10 @@ function format_postdata_for_curlcall($postdata) { * may not work when using proxy * @param bool $skipcertverify If true, the peer's SSL certificate will not be checked. Only use this when already in a trusted location. * @param string $tofile store the downloaded content to file instead of returning it + * @param bool $calctimeout false by default, true enables an extra head request to try and determine filesize and appropriately larger timeout based on $CFG->curltimeoutkbitrate * @return mixed false if request failed or content of the file as string if ok. true if file downloaded into $tofile successfully. */ -function download_file_content($url, $headers=null, $postdata=null, $fullresponse=false, $timeout=300, $connecttimeout=20, $skipcertverify=false, $tofile=NULL) { +function download_file_content($url, $headers=null, $postdata=null, $fullresponse=false, $timeout=300, $connecttimeout=20, $skipcertverify=false, $tofile=NULL, $calctimeout=false) { global $CFG; // some extra security @@ -962,7 +963,6 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons curl_setopt($ch, CURLOPT_HTTPHEADER, $headers2); } - if ($skipcertverify) { curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); } @@ -977,7 +977,7 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_HEADER, false); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $connecttimeout); - curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); + if (!ini_get('open_basedir') and !ini_get('safe_mode')) { // TODO: add version test for '7.10.5' curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); @@ -1033,6 +1033,34 @@ function download_file_content($url, $headers=null, $postdata=null, $fullrespons curl_setopt($ch, CURLOPT_WRITEFUNCTION, partial('download_file_content_write_handler', $received)); } + if (!isset($CFG->curltimeoutkbitrate)) { + //use very slow rate of 56kbps as a timeout speed when not set + $bitrate = 56; + } else { + $bitrate = $CFG->curltimeoutkbitrate; + } + + //try to calculate the proper amount for timeout from remote file size. + if ($calctimeout && $bitrate > 0) { // if disabled or zero, we won't do any checks nor head requests. + //setup header request only options + curl_setopt_array ($ch , array( + CURLOPT_RETURNTRANSFER => false, + CURLOPT_NOBODY => true )); + + curl_exec($ch); + $info = curl_getinfo($ch); + $err = curl_error($ch); + + if ($err === '' && $info['download_content_length'] > 0) { //no curl errors + $timeout = max($timeout,ceil($info['download_content_length']*8/($bitrate*1024))); //adjust for large files only - take max timeout. + } + //reinstate affected curl options + curl_setopt_array ($ch , array( + CURLOPT_RETURNTRANSFER => true, + CURLOPT_NOBODY => false )); + } + + curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); $result = curl_exec($ch); // try to detect encoding problems diff --git a/lib/filestorage/file_storage.php b/lib/filestorage/file_storage.php index 79f9770fc8aff..ecdd6e74b82ad 100644 --- a/lib/filestorage/file_storage.php +++ b/lib/filestorage/file_storage.php @@ -702,6 +702,7 @@ public function create_file_from_url($file_record, $url, array $options = NULL, $timeout = isset($options['timeout']) ? $options['timeout'] : 300; $connecttimeout = isset($options['connecttimeout']) ? $options['connecttimeout'] : 20; $skipcertverify = isset($options['skipcertverify']) ? $options['skipcertverify'] : false; + $calctimeout = isset($options['calctimeout']) ? $options['calctimeout'] : false; if (!isset($file_record->filename)) { $parts = explode('/', $url); @@ -714,7 +715,7 @@ public function create_file_from_url($file_record, $url, array $options = NULL, if ($usetempfile) { check_dir_exists($this->tempdir); $tmpfile = tempnam($this->tempdir, 'newfromurl'); - $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify, $tmpfile); + $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify, $tmpfile, $calctimeout); if ($content === false) { throw new file_exception('storedfileproblem', 'Can not fetch file form URL'); } @@ -728,7 +729,7 @@ public function create_file_from_url($file_record, $url, array $options = NULL, } } else { - $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify); + $content = download_file_content($url, $headers, $postdata, $fullresponse, $timeout, $connecttimeout, $skipcertverify, NULL, $calctimeout); if ($content === false) { throw new file_exception('storedfileproblem', 'Can not fetch file form URL'); } diff --git a/mod/scorm/locallib.php b/mod/scorm/locallib.php index 627f17c4bad6c..67b8987308917 100644 --- a/mod/scorm/locallib.php +++ b/mod/scorm/locallib.php @@ -186,7 +186,7 @@ function scorm_parse($scorm, $full) { if ($scorm->reference !== '' and (!$full or $scorm->sha1hash !== sha1($scorm->reference))) { $fs->delete_area_files($context->id, 'mod_scorm', 'package'); $file_record = array('contextid'=>$context->id, 'component'=>'mod_scorm', 'filearea'=>'package', 'itemid'=>0, 'filepath'=>'/'); - if ($packagefile = $fs->create_file_from_url($file_record, $scorm->reference)) { + if ($packagefile = $fs->create_file_from_url($file_record, $scorm->reference, array('calctimeout' => true))) { $newhash = sha1($scorm->reference); } else { $newhash = null;