From fba14118508dac76847afd415288e2a5cddbb87a Mon Sep 17 00:00:00 2001
From: Mark Timms <mark.timms@capitalone.com>
Date: Thu, 9 May 2019 17:30:28 +0000
Subject: [PATCH 1/3] Issue #21: Retry GET using uncompressed files after 404

---
 R/async-http.R | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/R/async-http.R b/R/async-http.R
index 777c1560..ccfada58 100644
--- a/R/async-http.R
+++ b/R/async-http.R
@@ -327,6 +327,25 @@ download_files <- function(data, ...) {
         result
       })
 
+    if (!isTRUE(row$mayfail) && grepl("\\.gz$", row$url)) {
+      dx <- dx$catch(error = function(err) {
+        nogzip_url <- gsub("\\.gz$", "", row$url)
+        download_if_newer(nogzip_url, row$path, row$etag,
+                          options = list(timeout = row$timeout %||% 10),
+                          ...
+        )$
+          then(function(result) {
+            status_code <- result$response$status_code
+            if (status_code == 304) {
+              update_progress_bar_uptodate(bar, row$url)
+            } else {
+              update_progress_bar_done(bar, row$url)
+            }
+            result
+          })
+      })
+    }
+
     if (isTRUE(row$mayfail)) {
       dx$catch(error = function(err) {
         cat("", file = row$path, append = TRUE)

From 2553093b799d8c4d7e8812edebcdbaa54a950875 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= <csardi.gabor@gmail.com>
Date: Mon, 14 Oct 2019 13:38:03 +0100
Subject: [PATCH 2/3] Generic fallaback URL for metadata downloads

Falls back on PACKAGES now.
---
 R/async-http.R                       | 30 ++++++++++------------------
 R/metadata-cache.R                   |  3 +++
 tests/testthat/test-metadata-cache.R |  2 +-
 3 files changed, 15 insertions(+), 20 deletions(-)

diff --git a/R/async-http.R b/R/async-http.R
index ccfada58..104478f5 100644
--- a/R/async-http.R
+++ b/R/async-http.R
@@ -316,7 +316,17 @@ download_files <- function(data, ...) {
     row <- data[idx, ]
     dx <- download_if_newer(row$url, row$path, row$etag,
       on_progress = prog_cb, options = list(timeout = row$timeout %||% 100),
-      ...)$
+      ...)
+
+    if ("fallback_url" %in% names(row) && !is.na(row$fallback_url)) {
+      dx <- dx$catch(error = function(err) {
+        download_if_newer(row$fallback_url, row$path, row$etag,
+                          options = list(timeout = row$timeout %||% 10),
+                          ...)
+      })
+    }
+
+    dx <- dx$
       then(function(result) {
         status_code <- result$response$status_code
         if (status_code == 304) {
@@ -327,24 +337,6 @@ download_files <- function(data, ...) {
         result
       })
 
-    if (!isTRUE(row$mayfail) && grepl("\\.gz$", row$url)) {
-      dx <- dx$catch(error = function(err) {
-        nogzip_url <- gsub("\\.gz$", "", row$url)
-        download_if_newer(nogzip_url, row$path, row$etag,
-                          options = list(timeout = row$timeout %||% 10),
-                          ...
-        )$
-          then(function(result) {
-            status_code <- result$response$status_code
-            if (status_code == 304) {
-              update_progress_bar_uptodate(bar, row$url)
-            } else {
-              update_progress_bar_done(bar, row$url)
-            }
-            result
-          })
-      })
-    }
 
     if (isTRUE(row$mayfail)) {
       dx$catch(error = function(err) {
diff --git a/R/metadata-cache.R b/R/metadata-cache.R
index f0cdf63f..e8faebfd 100644
--- a/R/metadata-cache.R
+++ b/R/metadata-cache.R
@@ -420,6 +420,7 @@ cmc__get_cache_files <- function(self, private, which) {
   repo_enc <- rep(repo_encode(private$repos), each = nrow(private$dirs))
   pkgs_dirs <- rep(private$dirs$contriburl, nrow(private$repos))
   pkgs_files <- file.path(pkgs_dirs, "PACKAGES.gz")
+  pkgs_files2 <- file.path(pkgs_dirs, "PACKAGES")
   mirror <- rep(private$repos$url, each = nrow(private$dirs))
   type <- rep(private$repos$type, each = nrow(private$dirs))
   bioc_version <- rep(private$repos$bioc_version, each = nrow(private$dirs))
@@ -448,6 +449,7 @@ cmc__get_cache_files <- function(self, private, which) {
       base = pkgs_files,
       mirror = mirror,
       url = paste0(mirror, "/", pkgs_files),
+      fallback_url = paste0(mirror, "/", pkgs_files2),
       platform = rep(private$dirs$platform, nrow(private$repos)),
       type = type,
       bioc_version = bioc_version,
@@ -673,6 +675,7 @@ cmc__update_replica_pkgs <- function(self, private) {
   dls <- data.frame(
     stringsAsFactors = FALSE,
     url = c(pkgs$url, pkgs$meta_url[meta]),
+    fallback_url = c(pkgs$fallback_url, rep(NA_character_, sum(meta))),
     path = c(pkgs$path, pkgs$meta_path[meta]),
     etag = c(pkgs$etag, pkgs$meta_etag[meta]),
     timeout = rep(c(200, 100), c(nrow(pkgs), sum(meta))),
diff --git a/tests/testthat/test-metadata-cache.R b/tests/testthat/test-metadata-cache.R
index a9b81b40..a085bf8a 100644
--- a/tests/testthat/test-metadata-cache.R
+++ b/tests/testthat/test-metadata-cache.R
@@ -21,7 +21,7 @@ test_that("get_cache_files", {
     expect_true(tibble::is_tibble(files$pkgs))
     expect_equal(
       sort(names(files$pkgs)),
-      sort(c("path", "etag", "basedir", "base", "mirror", "url",
+      sort(c("path", "etag", "basedir", "base", "mirror", "url", "fallback_url",
              "platform", "type", "bioc_version", "meta_path", "meta_etag",
              "meta_url")))
     expect_equal(

From a598eabe107d1234e548104994490ee26fabc9a0 Mon Sep 17 00:00:00 2001
From: Mark Timms <mark.timms@capitalone.com>
Date: Thu, 9 May 2019 17:30:28 +0000
Subject: [PATCH 3/3] Issue #21: Retry GET using uncompressed files after 404

---
 R/async-http.R | 1 -
 1 file changed, 1 deletion(-)

diff --git a/R/async-http.R b/R/async-http.R
index 104478f5..08d80dbe 100644
--- a/R/async-http.R
+++ b/R/async-http.R
@@ -337,7 +337,6 @@ download_files <- function(data, ...) {
         result
       })
 
-
     if (isTRUE(row$mayfail)) {
       dx$catch(error = function(err) {
         cat("", file = row$path, append = TRUE)