fetcher: Send Accept-Encoding: gzip when downloading summary

The summary file can get large, but it compresses well (something
which is not true of other files in the ostree repo which are
already compressed). By sending Accept-Encoding: gzip (and
handling the compressed results) we send a lot less data.

I set up the flathub repo (http://flathub.org/repo) to enable
gzip for the summary file (only), and the result is that the
331514 byte large summary was transferred in 122889 bytes.
On my (fast) network this decreased the time i took to do
"flatpak remote-ls flathub" by about 100msec.

This fixes #802

Closes: #882
Approved by: cgwalters

src/libostree/ostree-fetcher-curl.c

@@ -762,6 +762,9 @@ initiate_next_curl_request (FetcherRequest *req,
       curl_easy_setopt (req->easy, CURLOPT_SSLKEY, self->tls_client_key_path);
     }
 
+  if ((self->config_flags & OSTREE_FETCHER_FLAGS_TRANSFER_GZIP) > 0)
+    curl_easy_setopt (req->easy, CURLOPT_ACCEPT_ENCODING, "");
+
   /* We should only speak HTTP; TODO: only enable file if specified */
   curl_easy_setopt (req->easy, CURLOPT_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FILE));
   /* Picked the current version in F25 as of 20170127, since

src/libostree/ostree-fetcher-soup.c

@@ -63,6 +63,7 @@ typedef struct {
 
   GVariant *extra_headers;
   int max_outstanding;
+  gboolean transfer_gzip;
 
   /* Our active HTTP requests */
   GHashTable *outstanding;
@@ -553,6 +554,9 @@ ostree_fetcher_session_thread (gpointer data)
                                                           SOUP_SESSION_IDLE_TIMEOUT, 60,
                                                           NULL);
 
+  if (closure->transfer_gzip)
+    soup_session_add_feature_by_type (closure->session, SOUP_TYPE_CONTENT_DECODER);
+
   /* XXX: Now that we have mirrorlist support, we could make this even smarter
    * by spreading requests across mirrors. */
   g_object_get (closure->session, "max-conns-per-host", &max_conns, NULL);
@@ -663,6 +667,7 @@ _ostree_fetcher_constructed (GObject *object)
   self->thread_closure->ref_count = 1;
   self->thread_closure->main_context = g_main_context_ref (main_context);
   self->thread_closure->running = 1;
+  self->thread_closure->transfer_gzip = (self->config_flags & OSTREE_FETCHER_FLAGS_TRANSFER_GZIP) != 0;
   self->thread_closure->tmpdir_dfd = -1;
   self->thread_closure->tmpdir_lock = empty_lockfile;
 

src/libostree/ostree-fetcher.h

@@ -48,7 +48,8 @@ struct OstreeFetcherClass
 
 typedef enum {
   OSTREE_FETCHER_FLAGS_NONE = 0,
-  OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE = (1 << 0)
+  OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE = (1 << 0),
+  OSTREE_FETCHER_FLAGS_TRANSFER_GZIP = (1 << 1)
 } OstreeFetcherConfigFlags;
 
 void

src/libostree/ostree-repo-pull.c

@@ -2161,6 +2161,7 @@ _ostree_repo_cache_summary (OstreeRepo        *self,
 static OstreeFetcher *
 _ostree_repo_remote_new_fetcher (OstreeRepo  *self,
                                  const char  *remote_name,
+                                 gboolean     gzip,
                                  GError     **error)
 {
   OstreeFetcher *fetcher = NULL;
@@ -2179,6 +2180,9 @@ _ostree_repo_remote_new_fetcher (OstreeRepo  *self,
   if (tls_permissive)
     fetcher_flags |= OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE;
 
+  if (gzip)
+    fetcher_flags |= OSTREE_FETCHER_FLAGS_TRANSFER_GZIP;
+
   fetcher = _ostree_fetcher_new (self->tmp_dir_fd, remote_name, fetcher_flags);
 
   {
@@ -2436,7 +2440,7 @@ repo_remote_fetch_summary (OstreeRepo    *self,
   mainctx = g_main_context_new ();
   g_main_context_push_thread_default (mainctx);
 
-  fetcher = _ostree_repo_remote_new_fetcher (self, name, error);
+  fetcher = _ostree_repo_remote_new_fetcher (self, name, TRUE, error);
   if (fetcher == NULL)
     goto out;
 
@@ -2544,7 +2548,7 @@ static gboolean
 reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, GError **error)
 {
   g_clear_object (&pull_data->fetcher);
-  pull_data->fetcher = _ostree_repo_remote_new_fetcher (pull_data->repo, remote_name, error);
+  pull_data->fetcher = _ostree_repo_remote_new_fetcher (pull_data->repo, remote_name, FALSE, error);
   if (pull_data->fetcher == NULL)
     return FALSE;