Skip to content

Commit

Permalink
Merge pull request #120 from endlessm/T22555-retry-downloads
Browse files Browse the repository at this point in the history 
T22555 Retry downloads on flaky networks
  • Loading branch information
Roddy Shuler committed May 30, 2018
2 parents e555598 + 50a9cf2 commit b8d7b8f
Show file tree
Hide file tree
Showing 9 changed files with 461 additions and 135 deletions.
14 changes: 1 addition & 13 deletions src/libostree/ostree-fetcher-curl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,19 +337,7 @@ check_multi_info (OstreeFetcher *fetcher)
curl_easy_getinfo (easy, CURLINFO_RESPONSE_CODE, &response);
if (!is_file && !(response >= 200 && response < 300))
{
GIOErrorEnum giocode;

/* TODO - share with soup */
switch (response)
{
case 404:
case 403:
case 410:
giocode = G_IO_ERROR_NOT_FOUND;
break;
default:
giocode = G_IO_ERROR_FAILED;
}
GIOErrorEnum giocode = _ostree_fetcher_http_status_code_to_io_error (response);

if (req->idx + 1 == req->mirrorlist->len)
{
Expand Down
22 changes: 17 additions & 5 deletions src/libostree/ostree-fetcher-soup.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1070,15 +1070,27 @@ on_request_sent (GObject *object,
soup_uri_to_string (soup_request_get_uri (pending->request), FALSE);

GIOErrorEnum code;

switch (msg->status_code)
{
case 404:
case 403:
case 410:
code = G_IO_ERROR_NOT_FOUND;
/* These statuses are internal to libsoup, and not standard HTTP ones: */
case SOUP_STATUS_CANCELLED:
code = G_IO_ERROR_CANCELLED;
break;
case SOUP_STATUS_CANT_RESOLVE:
case SOUP_STATUS_CANT_CONNECT:
code = G_IO_ERROR_HOST_NOT_FOUND;
break;
case SOUP_STATUS_IO_ERROR:
#if !GLIB_CHECK_VERSION(2, 44, 0)
code = G_IO_ERROR_BROKEN_PIPE;
#else
code = G_IO_ERROR_CONNECTION_CLOSED;
#endif
break;
default:
code = G_IO_ERROR_FAILED;
code = _ostree_fetcher_http_status_code_to_io_error (msg->status_code);
break;
}

{
Expand Down
113 changes: 103 additions & 10 deletions src/libostree/ostree-fetcher-util.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,15 +52,15 @@ fetch_uri_sync_on_complete (GObject *object,
data->done = TRUE;
}

gboolean
_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
GBytes **out_contents,
guint64 max_size,
GCancellable *cancellable,
GError **error)
static gboolean
_ostree_fetcher_mirrored_request_to_membuf_once (OstreeFetcher *fetcher,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
GBytes **out_contents,
guint64 max_size,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
g_autoptr(GMainContext) mainctx = NULL;
Expand Down Expand Up @@ -108,11 +108,42 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
return ret;
}

gboolean
_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
guint n_network_retries,
GBytes **out_contents,
guint64 max_size,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GError) local_error = NULL;
guint n_retries_remaining = n_network_retries;

do
{
g_clear_error (&local_error);
if (_ostree_fetcher_mirrored_request_to_membuf_once (fetcher, mirrorlist,
filename, flags,
out_contents, max_size,
cancellable, &local_error))
return TRUE;
}
while (_ostree_fetcher_should_retry_request (local_error, n_retries_remaining--));

g_assert (local_error != NULL);
g_propagate_error (error, g_steal_pointer (&local_error));
return FALSE;
}

/* Helper for callers who just want to fetch single one-off URIs */
gboolean
_ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
OstreeFetcherURI *uri,
OstreeFetcherRequestFlags flags,
guint n_network_retries,
GBytes **out_contents,
guint64 max_size,
GCancellable *cancellable,
Expand All @@ -121,7 +152,7 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
g_ptr_array_add (mirrorlist, uri); /* no transfer */
return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL, flags,
out_contents, max_size,
n_network_retries, out_contents, max_size,
cancellable, error);
}

Expand All @@ -144,3 +175,65 @@ _ostree_fetcher_journal_failure (const char *remote_name,
NULL);
#endif
}

/* Check whether a particular operation should be retried. This is entirely
* based on how it failed (if at all) last time, and whether the operation has
* some retries left. The retry count is set when the operation is first
* created, and must be decremented by the caller. (@n_retries_remaining == 0)
* will always return %FALSE from this function.
*
* FIXME: In future, we may decide to use transient failures like this as a hint
* to prioritise other mirrors for a particular pull operation (for example). */
gboolean
_ostree_fetcher_should_retry_request (const GError *error,
guint n_retries_remaining)
{
if (error == NULL)
g_debug ("%s: error: unset, n_retries_remaining: %u",
G_STRFUNC, n_retries_remaining);
else
g_debug ("%s: error: %u:%u %s, n_retries_remaining: %u",
G_STRFUNC, error->domain, error->code, error->message,
n_retries_remaining);

if (error == NULL || n_retries_remaining == 0)
return FALSE;

/* Return TRUE for transient errors. */
if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_TIMED_OUT) ||
g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_NOT_FOUND) ||
g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_UNREACHABLE) ||
#if !GLIB_CHECK_VERSION(2, 44, 0)
g_error_matches (error, G_IO_ERROR, G_IO_ERROR_BROKEN_PIPE) ||
#else
g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CONNECTION_CLOSED) ||
#endif
g_error_matches (error, G_RESOLVER_ERROR, G_RESOLVER_ERROR_NOT_FOUND) ||
g_error_matches (error, G_RESOLVER_ERROR, G_RESOLVER_ERROR_TEMPORARY_FAILURE))
{
g_debug ("Should retry request (remaining: %u retries), due to transient error: %s",
n_retries_remaining, error->message);
return TRUE;
}

return FALSE;
}

/* Convert a HTTP status code representing an error from libsoup or libcurl to
* a #GIOError. This will return %G_IO_ERROR_FAILED if the status code is
* unknown or otherwise unhandled. */
GIOError
_ostree_fetcher_http_status_code_to_io_error (guint status_code)
{
switch (status_code)
{
case 403: /* SOUP_STATUS_FORBIDDEN */
case 404: /* SOUP_STATUS_NOT_FOUND */
case 410: /* SOUP_STATUS_GONE */
return G_IO_ERROR_NOT_FOUND;
case 408: /* SOUP_STATUS_REQUEST_TIMEOUT */
return G_IO_ERROR_TIMED_OUT;
default:
return G_IO_ERROR_FAILED;
}
}
6 changes: 6 additions & 0 deletions src/libostree/ostree-fetcher-util.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
guint n_network_retries,
GBytes **out_contents,
guint64 max_size,
GCancellable *cancellable,
Expand All @@ -64,6 +65,7 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
OstreeFetcherURI *uri,
OstreeFetcherRequestFlags flags,
guint n_network_retries,
GBytes **out_contents,
guint64 max_size,
GCancellable *cancellable,
Expand All @@ -73,6 +75,10 @@ void _ostree_fetcher_journal_failure (const char *remote_name,
const char *url,
const char *msg);

gboolean _ostree_fetcher_should_retry_request (const GError *error,
guint n_retries_remaining);

GIOError _ostree_fetcher_http_status_code_to_io_error (guint status_code);

G_END_DECLS

Expand Down
10 changes: 8 additions & 2 deletions src/libostree/ostree-metalink.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ struct OstreeMetalink
OstreeFetcher *fetcher;
char *requested_file;
guint64 max_size;
guint n_network_retries;
};

G_DEFINE_TYPE (OstreeMetalink, _ostree_metalink, G_TYPE_OBJECT)
Expand Down Expand Up @@ -401,14 +402,16 @@ OstreeMetalink *
_ostree_metalink_new (OstreeFetcher *fetcher,
const char *requested_file,
guint64 max_size,
OstreeFetcherURI *uri)
OstreeFetcherURI *uri,
guint n_network_retries)
{
OstreeMetalink *self = (OstreeMetalink*)g_object_new (OSTREE_TYPE_METALINK, NULL);

self->fetcher = g_object_ref (fetcher);
self->requested_file = g_strdup (requested_file);
self->max_size = max_size;
self->uri = _ostree_fetcher_uri_clone (uri);
self->n_network_retries = n_network_retries;

return self;
}
Expand All @@ -432,7 +435,9 @@ try_one_url (OstreeMetalinkRequest *self,
gssize n_bytes;

if (!_ostree_fetcher_request_uri_to_membuf (self->metalink->fetcher,
uri, 0, &bytes,
uri, 0,
self->metalink->n_network_retries,
&bytes,
self->metalink->max_size,
self->cancellable,
error))
Expand Down Expand Up @@ -613,6 +618,7 @@ _ostree_metalink_request_sync (OstreeMetalink *self,
request.parser = g_markup_parse_context_new (&metalink_parser, G_MARKUP_PREFIX_ERROR_POSITION, &request, NULL);

if (!_ostree_fetcher_request_uri_to_membuf (self->fetcher, self->uri, 0,
self->n_network_retries,
&contents, self->max_size,
cancellable, error))
goto out;
Expand Down
3 changes: 2 additions & 1 deletion src/libostree/ostree-metalink.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ GType _ostree_metalink_get_type (void) G_GNUC_CONST;
OstreeMetalink *_ostree_metalink_new (OstreeFetcher *fetcher,
const char *requested_file,
guint64 max_size,
OstreeFetcherURI *uri);
OstreeFetcherURI *uri,
guint n_network_retries);

gboolean _ostree_metalink_request_sync (OstreeMetalink *self,
OstreeFetcherURI **out_target_uri,
Expand Down
Loading

0 comments on commit b8d7b8f

Please sign in to comment.