New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enabled configurable retry attempts for downloads #726

Open
wants to merge 5 commits into
from
@@ -13,7 +13,13 @@
function make_download_factory($name, $type, $download, $download_location) {
$function = 'make_download_' . $download['type'];
if (function_exists($function)) {
- return $function($name, $type, $download, $download_location);
+ $destination_path = drush_download_with_retries($name, $function, $name, $type, $download, $download_location);
+
+ if (!$destination_path) {
+ make_error('DOWNLOAD_ERROR', dt('Unable to download @project from @url.', array('@project' => $name, '@url' => $download['url'])));
+ }
+
+ return $destination_path;
}
else {
return FALSE;
@@ -51,11 +57,14 @@ function make_download_pm($name, $type, $download, $download_location) {
// Perform actual download with `drush pm-download`.
$return = drush_invoke_process('@none', 'pm-download', array($full_project_version), $options, $backend_options);
- if (empty($return['error_log'])) {
- // @todo Report the URL we used for download. See
- // http://drupal.org/node/1452672.
- drush_log(dt('@project downloaded.', array('@project' => $full_project_version)), 'ok');
+ if (!empty($return['error_log'])) {
+ return FALSE;
}
+
+ // @todo Report the URL we used for download. See
+ // http://drupal.org/node/1452672.
+ drush_log(dt('@project downloaded.', array('@project' => $full_project_version)), 'ok');
+ return $full_project_version;
}
/**
@@ -74,10 +83,24 @@ function make_download_file($name, $type, $download, $download_location, $cache_
$subtree = isset($download['subtree']) ? $download['subtree'] : NULL;
return make_download_file_unpack($filename, $download_location, $download_filename, $subtree);
}
- make_error('DOWNLOAD_ERROR', dt('Unable to download @project from @url.', array('@project' => $name, '@url' => $download['url'])));
+ drush_log(dt('Failed to download @project from @url.', array('@project' => $name, '@url' => $download['url'])), 'warning');
return FALSE;
}
+/**
+ * Makes the simple _make_download_file callable from make_download_factory.
+ *
+ * @return mixed
+ * The destination filename on success, FALSE on failure.
+ */
+function make_download_simplefile($name, $type, $download, $download_location = NULL, $cache_duration = DRUSH_CACHE_LIFETIME_DEFAULT) {
+ if (!$filename = _make_download_file($download['url'], $cache_duration)) {
+ drush_log(dt('Failed to download @project from @url.', array('@project' => $name, '@url' => $download['url'])), 'warning');
+ }
+
+ return $filename;
+}
+
/**
* Wrapper to drush_download_file().
*
@@ -259,7 +282,7 @@ function make_download_git($name, $type, $download, $download_location) {
// Before we can checkout anything, we need to clone the repository.
if (!drush_shell_exec($command, $url, $tmp_location)) {
- make_error('DOWNLOAD_ERROR', dt('Unable to clone @project from @url.', array('@project' => $name, '@url' => $url)));
+ drush_log(dt('Unable to clone @project from @url.', array('@project' => $name, '@url' => $url)), 'warning');
return FALSE;
}
@@ -382,7 +405,7 @@ function make_download_bzr($name, $type, $download, $download_location) {
else {
$download['url'] = dt("unspecified location");
}
- make_error('DOWNLOAD_ERROR', dt('Unable to download @project from @url.', array('@project' => $name, '@url' => $download['url'])));
+ drush_log(dt('Unable to download @project from @url.', array('@project' => $name, '@url' => $download['url'])), 'warning');
drush_delete_dir(dirname($tmp_location), TRUE);
return FALSE;
}
@@ -448,7 +471,7 @@ function make_download_svn($name, $type, $download, $download_location) {
}
}
else {
- make_error('DOWNLOAD_ERROR', dt('Unable to download @project from @url.', array('@project' => $name, '@url' => $download['url'])));
+ drush_log(dt('Failed to download @project from @url.', array('@project' => $name, '@url' => $download['url'])), 'warning');
return FALSE;
}
}
@@ -66,6 +66,9 @@ function make_drush_command() {
'translations' => 'Retrieve translations for the specified comma-separated list of language(s) if available for all projects.',
'working-copy' => 'Preserves VCS directories, like .git, for projects downloaded using such methods.',
'download-mechanism' => 'How to download files. Should be autodetected, but this is an override if it doesn\'t work. Options are "curl" and "make" (a native download method).',
+ 'download-attempts' => 'The number of attempts to download files.',
+ 'download-retry-delay-min-seconds' => 'The minimum number of seconds to wait before attempting to retry a download.',
+ 'download-retry-delay-max-seconds' => 'The maximum number of seconds to wait before attempting to retry a download.',
'projects' => array(
'description' => 'Restrict the make to this comma-separated list of projects. To specify all projects, pass *.',
'example' => 'views,ctools',
@@ -254,9 +254,13 @@ class DrushMakeProject {
$ignore_checksums = drush_get_option('ignore-checksums');
foreach ($this->patch as $info) {
$this->preprocessLocalFileUrl($info);
+ $download = array(
+ 'type' => 'simplefile',
+ 'url' => $info['url'],
+ );
// Download the patch.
- if ($filename = _make_download_file($info['url'])) {
+ if ($filename = make_download_factory(basename($download_factory['url']), 'Patch', $download, NULL)) {
$patched = FALSE;
$output = '';
// Test each patch style; -p1 is the default with git. See
@@ -135,7 +135,7 @@ function drush_pm_download() {
$request['base_project_path'] = drush_tempdir();
$request['full_project_path'] = $request['base_project_path'] .'/'. $request['project_dir'];
drush_log(dt('Downloading project !name to !dir ...', array('!name' => $request['name'], '!dir' => $request['base_project_path'])));
- if (!package_handler_download_project($request, $release)) {
+ if (!drush_download_with_retries($request['name'], 'package_handler_download_project', $request, $release)) {
// Delete the cached updatexml since it may be invalid.
drush_delete_dir(drush_download_file_name(updatexml_get_url($request)), TRUE);
drush_log(dt('Error downloading !name', array('!name' => $request['name']), 'error'));
View
@@ -344,6 +344,9 @@ function pm_drush_command() {
),
'skip' => 'Skip automatic downloading of libraries (c.f. devel).',
'pipe' => 'Returns a list of the names of the extensions (modules and themes) contained in the downloaded projects.',
+ 'download-attempts' => 'The number of attempts to download files.',
+ 'download-retry-delay-min-seconds' => 'The minimum number of seconds to wait before attempting to retry a download.',
+ 'download-retry-delay-max-seconds' => 'The maximum number of seconds to wait before attempting to retry a download.',
),
'bootstrap' => DRUSH_BOOTSTRAP_MAX,
'aliases' => array('dl'),
@@ -467,9 +467,11 @@ function updatexml_get_release_history_xml($request) {
$url = updatexml_get_url($request);
drush_log('Downloading release history from ' . $url);
// Some hosts have allow_url_fopen disabled.
- if ($path = drush_download_file($url, drush_tempnam($request['name']), drush_get_option('cache-duration-releasexml', 24*3600))) {
+ $xml = NULL;
+ if ($path = drush_download_with_retries($url, 'drush_download_file', $url, drush_tempnam($request['name']), drush_get_option('cache-duration-releasexml', 24*3600))) {
$xml = simplexml_load_file($path);
}
+
if (!$xml) {
// We are not getting here since drupal.org always serves an XML response.
return drush_set_error('DRUSH_PM_DOWNLOAD_FAILED', dt('Could not download project status information from !url', array('!url' => $url)));
View
@@ -735,7 +735,7 @@ function drush_download_file($url, $destination = FALSE, $cache_duration = 0) {
drush_log(dt('!name retrieved from cache.', array('!name' => $cache_file)));
}
else {
- if (_drush_download_file($url, $cache_file, TRUE)) {
+ if (drush_download_with_retries($url, '_drush_download_file', $url, $cache_file, TRUE)) {
// Cache was set just by downloading file to right location.
}
elseif (file_exists($cache_file)) {
@@ -1974,3 +1974,56 @@ function drush_complete_cache_clear($type = NULL, $command = NULL) {
// No type or command, so clear the entire complete cache.
drush_cache_clear_all('*', 'complete', TRUE);
}
+
+/**
+ * Enables multiple attempts at downloading a file.
+ *
+ * Designed to make downloading from drush more robust, this function retries
+ * each download multiple times on failure. Behaviour is controlled via three
+ * drush options:
+ * 'download-attempts': The number of attempts to download files
+ * 'download-retry-delay-min-seconds': The minimum number of seconds to wait before attempting to retry a download
+ * 'download-retry-delay-max-seconds': The maximum number of seconds to wait before attempting to retry a download
+ * (the actual time between download attempts is a random number between download-retry-delay-min-seconds and
+ * download-retry-delay-max-seconds.
+ *
+ * @param $title
+ * The title of this download, to display in log messages
+ * @param $function
+ * The name of the function to call to download the file
+ * @param $param1...$paramN
+ * Additional parameters to pass to $function.
+ */
+function drush_download_with_retries($title) {
+ if (func_num_args() < 2) {
+ return;
+ }
+
+ $attempts = 0;
+ $success = FALSE;
+ while (!$success && $attempts < (int) drush_get_option('download-attempts', 5)) {
+
+ // If this is a retry then inform the user.
+ if ($attempts > 0) {
+ drush_log(dt('Retry number @attempts for @title', array('@attempts' => $attempts, '@title' => $title)), 'warning');
+ }
+
+ drush_log(dt('running command @command', array('@command' => func_get_arg(1))), 'warning');
+
+ $success = ($ret = call_user_func_array(func_get_arg(1), array_slice(func_get_args(), 2)));
+ $attempts++;
+
+ // If we failed to download, and we will be trying again, wait for
+ // a random amount of time within a user specified period.
+ if (!$success && $attempts < (int) drush_get_option('download-attempts', 5)) {
+ $wait_time = rand(drush_get_option('download-retry-delay-min-seconds', 5), drush_get_option('download-retry-delay-max-seconds', 30));
+ if (drush_get_context('DRUSH_VERBOSE') || drush_get_context('DRUSH_SIMULATE')) {
+ drush_print('Download failed. Sleeping for ' . $wait_time . ' seconds.', 0, STDERR);
+ }
+ sleep($wait_time);
+ }
+
+ }
+
+ return $ret;
+}