Merge pull request #9956 from rouault/fix_9941

CPLHTTPFetch(): add a RETRY_CODES option, and C++'ify HTTP retry logic

autotest/gcore/vsiaz.py

@@ -751,15 +751,15 @@ def test_vsiaz_write_blockblob_retry():
 
     gdal.VSICurlClearCache()
 
-    # Test creation of BlockBob
-    f = gdal.VSIFOpenL("/vsiaz/test_copy/file.bin", "wb")
-    assert f is not None
-
     with gdaltest.config_options(
         {"GDAL_HTTP_MAX_RETRY": "2", "GDAL_HTTP_RETRY_DELAY": "0.01"},
         thread_local=False,
     ):
 
+        # Test creation of BlockBob
+        f = gdal.VSIFOpenL("/vsiaz/test_copy/file.bin", "wb")
+        assert f is not None
+
         handler = webserver.SequentialHandler()
 
         def method(request):

autotest/gcore/vsicurl.py

@@ -613,6 +613,97 @@ def test_vsicurl_test_retry(server):
 ###############################################################################
 
 
+def test_vsicurl_retry_codes_ALL(server):
+
+    gdal.VSICurlClearCache()
+
+    handler = webserver.SequentialHandler()
+    handler.add("GET", "/test_retry/", 404)
+    handler.add("HEAD", "/test_retry/test.txt", 200, {"Content-Length": "3"})
+    handler.add(
+        "GET", "/test_retry/test.txt", 400
+    )  #  non retriable by default, but here allowed because of retry_codes=ALL
+    handler.add("GET", "/test_retry/test.txt", 200, {}, "foo")
+    with webserver.install_http_handler(handler):
+        f = gdal.VSIFOpenL(
+            "/vsicurl?max_retry=1&retry_delay=0.01&retry_codes=ALL&url=http://localhost:%d/test_retry/test.txt"
+            % server.port,
+            "rb",
+        )
+        assert f is not None
+        gdal.ErrorReset()
+        with gdal.quiet_errors():
+            data = gdal.VSIFReadL(1, 3, f).decode("ascii")
+        error_msg = gdal.GetLastErrorMsg()
+        gdal.VSIFCloseL(f)
+        assert data == "foo"
+        assert "400" in error_msg
+
+
+###############################################################################
+
+
+def test_vsicurl_retry_codes_enumerated(server):
+
+    gdal.VSICurlClearCache()
+
+    handler = webserver.SequentialHandler()
+    handler.add("GET", "/test_retry/", 404)
+    handler.add("HEAD", "/test_retry/test.txt", 200, {"Content-Length": "3"})
+    handler.add(
+        "GET", "/test_retry/test.txt", 400
+    )  #  non retriable by default, but here allowed because of retry_codes=ALL
+    handler.add("GET", "/test_retry/test.txt", 200, {}, "foo")
+    with webserver.install_http_handler(handler), gdal.config_option(
+        "GDAL_HTTP_RETRY_CODES", "400"
+    ):
+        f = gdal.VSIFOpenL(
+            "/vsicurl?max_retry=1&retry_delay=0.01&url=http://localhost:%d/test_retry/test.txt"
+            % server.port,
+            "rb",
+        )
+        assert f is not None
+        gdal.ErrorReset()
+        with gdal.quiet_errors():
+            data = gdal.VSIFReadL(1, 3, f).decode("ascii")
+        error_msg = gdal.GetLastErrorMsg()
+        gdal.VSIFCloseL(f)
+        assert data == "foo"
+        assert "400" in error_msg
+
+
+###############################################################################
+
+
+def test_vsicurl_retry_codes_no_match(server):
+
+    gdal.VSICurlClearCache()
+
+    handler = webserver.SequentialHandler()
+    handler.add("GET", "/test_retry/", 404)
+    handler.add("HEAD", "/test_retry/test.txt", 200, {"Content-Length": "3"})
+    handler.add(
+        "GET", "/test_retry/test.txt", 400
+    )  #  non retriable by default, and not listed in GDAL_HTTP_RETRY_CODES
+    with webserver.install_http_handler(handler), gdal.config_option(
+        "GDAL_HTTP_RETRY_CODES", "409"
+    ):
+        f = gdal.VSIFOpenL(
+            "/vsicurl?max_retry=1&retry_delay=0.01&url=http://localhost:%d/test_retry/test.txt"
+            % server.port,
+            "rb",
+        )
+        assert f is not None
+        gdal.ErrorReset()
+        with gdal.quiet_errors():
+            data = gdal.VSIFReadL(1, 3, f).decode("ascii")
+        gdal.VSIFCloseL(f)
+        assert len(data) == 0
+
+
+###############################################################################
+
+
 def test_vsicurl_test_fallback_from_head_to_get(server):
 
     gdal.VSICurlClearCache()

doc/source/user/configoptions.rst

@@ -744,15 +744,28 @@ Networking options
 
 -  .. config:: GDAL_HTTP_MAX_RETRY
       :since: 2.3
+      :default: 0
 
-      Set the number of HTTP attempts in case of HTTP errors 429, 502, 503, or 504.
+      Set the number of HTTP attempts, when a retry is allowed.
+      (cf :config:`GDAL_HTTP_RETRY_CODES` for conditions where a retry is attempted.)
+      The default value is 0, meaning no retry.
 
 -  .. config:: GDAL_HTTP_RETRY_DELAY
       :choices: <seconds>
       :since: 2.3
+      :default: 30
 
       Set the delay between HTTP attempts.
 
+-  .. config:: GDAL_HTTP_RETRY_CODES
+      :choices: ALL or comma-separated list of codes
+      :since: 3.10
+
+      Specify which HTTP error codes should trigger a retry attempt.
+      Valid values are ``ALL`` or a comma-separated list of HTTP codes.
+      By default, 429, 500, 502, 503 or 504 HTTP errors are considered, as
+      well as other situations with a particular HTTP or Curl error message.
+
 -  .. config:: GDAL_HTTP_TCP_KEEPALIVE
       :choices: YES, NO
       :default: NO

doc/source/user/virtual_file_systems.rst

@@ -379,6 +379,7 @@ Starting with GDAL 2.3, options can be passed in the filename with the following
 - use_head=yes/no: whether the HTTP HEAD request can be emitted. Default to YES. Setting this option overrides the behavior of the :config:`CPL_VSIL_CURL_USE_HEAD` configuration option.
 - max_retry=number: default to 0. Setting this option overrides the behavior of the :config:`GDAL_HTTP_MAX_RETRY` configuration option.
 - retry_delay=number_in_seconds: default to 30. Setting this option overrides the behavior of the :config:`GDAL_HTTP_RETRY_DELAY` configuration option.
+- retry_codes=``ALL`` or comma-separated list of HTTP error codes. Setting this option overrides the behavior of the :config:`GDAL_HTTP_RETRY_CODES` configuration option. (GDAL >= 3.10)
 - list_dir=yes/no: whether an attempt to read the file list of the directory where the file is located should be done. Default to YES.
 - useragent=value: HTTP UserAgent header
 - referer=value: HTTP Referer header