Merge pull request #10050 from rouault/VSIMultipartUpload

Add VSIMultipartUploadXXXXX() API for multi-part upload

autotest/gcore/vsiadls.py

@@ -1161,3 +1161,56 @@ def test_vsiadls_fake_metadata():
         assert not gdal.SetFileMetadata(
             "/vsiadls/test/foo.bin", {"x-ms-properties": "foo=bar"}, "PROPERTIES"
         )
+
+
+###############################################################################
+# Test VSIMultipartUploadXXXX()
+
+
+def test_vsiadls_MultipartUpload():
+
+    if gdaltest.webserver_port == 0:
+        pytest.skip()
+
+    # Test MultipartUploadGetCapabilities()
+    info = gdal.MultipartUploadGetCapabilities("/vsiadls/")
+    assert info.non_sequential_upload_supported
+    assert info.parallel_upload_supported
+    assert not info.abort_supported
+    assert info.min_part_size == 0
+    assert info.max_part_size >= 1024
+    assert info.max_part_count > 0
+
+    # Test MultipartUploadAddPart()
+    handler = webserver.SequentialHandler()
+    handler.add(
+        "PATCH",
+        "/azure/blob/myaccount/test_multipartupload/test.bin?action=append&position=123456",
+        202,
+        expected_body=b"foo",
+    )
+    with webserver.install_http_handler(handler):
+        part_id = gdal.MultipartUploadAddPart(
+            "/vsiadls/test_multipartupload/test.bin", "my_upload_id", 1, 123456, b"foo"
+        )
+    assert part_id == "dummy"
+
+    # Test MultipartUploadEnd()
+    handler = webserver.SequentialHandler()
+    handler.add(
+        "PATCH",
+        "/azure/blob/myaccount/test_multipartupload/test.bin?action=flush&close=true&position=3",
+        200,
+    )
+    with webserver.install_http_handler(handler):
+        assert gdal.MultipartUploadEnd(
+            "/vsiadls/test_multipartupload/test.bin", "my_upload_id", ["dummy"], 3
+        )
+
+    # Test unsupported MultipartUploadAbort()
+    with gdal.ExceptionMgr(useExceptions=True):
+        with pytest.raises(
+            Exception,
+            match=r"MultipartUploadAbort\(\) not supported by this file system",
+        ):
+            gdal.MultipartUploadAbort("/vsiadls/foo/bar", "upload_id")

autotest/gcore/vsiaz.py

@@ -3059,3 +3059,30 @@ def method(request):
             )
             == 0
         )
+
+
+###############################################################################
+# Test VSIMultipartUploadXXXX()
+
+
+def test_vsiaz_MultipartUpload():
+
+    if gdaltest.webserver_port == 0:
+        pytest.skip()
+
+    # Test MultipartUploadGetCapabilities()
+    info = gdal.MultipartUploadGetCapabilities("/vsiaz/")
+    assert info.non_sequential_upload_supported
+    assert info.parallel_upload_supported
+    assert not info.abort_supported
+    assert info.min_part_size == 0
+    assert info.max_part_size >= 1024
+    assert info.max_part_count == 50000
+
+    # Test unsupported MultipartUploadAbort()
+    with gdal.ExceptionMgr(useExceptions=True):
+        with pytest.raises(
+            Exception,
+            match=r"MultipartUploadAbort\(\) not supported by this file system",
+        ):
+            gdal.MultipartUploadAbort("/vsiaz/foo/bar", "upload_id")

autotest/gcore/vsifile.py

@@ -1696,3 +1696,65 @@ def test_vsifile_stat_directory_trailing_slash():
     res = gdal.VSIStatL("data/")
     assert res
     assert res.IsDirectory()
+
+
+###############################################################################
+# Test VSIMultipartUploadXXXX(), unsupported on regular file systems
+
+
+def test_vsifile_MultipartUpload():
+
+    with gdal.ExceptionMgr(useExceptions=False):
+        with gdal.quiet_errors():
+            assert gdal.MultipartUploadGetCapabilities("foo") is None
+    with gdal.ExceptionMgr(useExceptions=True):
+        with pytest.raises(ValueError):
+            gdal.MultipartUploadGetCapabilities(None)
+
+        with pytest.raises(
+            Exception,
+            match=r"MultipartUploadGetCapabilities\(\) not supported by this file system",
+        ):
+            gdal.MultipartUploadGetCapabilities("foo")
+
+        with pytest.raises(ValueError):
+            gdal.MultipartUploadStart(None)
+
+        with pytest.raises(
+            Exception,
+            match=r"MultipartUploadStart\(\) not supported by this file system",
+        ):
+            gdal.MultipartUploadStart("foo")
+
+        with pytest.raises(ValueError):
+            gdal.MultipartUploadAddPart(None, "", 1, 0, b"")
+        with pytest.raises(ValueError):
+            gdal.MultipartUploadAddPart("", None, 1, 0, b"")
+
+        with pytest.raises(
+            Exception,
+            match=r"MultipartUploadAddPart\(\) not supported by this file system",
+        ):
+            gdal.MultipartUploadAddPart("", "", 1, 0, b"")
+
+        with pytest.raises(ValueError):
+            gdal.MultipartUploadEnd(None, "", [], 0)
+        with pytest.raises(ValueError):
+            gdal.MultipartUploadEnd("", None, [], 0)
+
+        with pytest.raises(
+            Exception,
+            match=r"MultipartUploadEnd\(\) not supported by this file system",
+        ):
+            gdal.MultipartUploadEnd("", "", [], 0)
+
+        with pytest.raises(ValueError):
+            gdal.MultipartUploadAbort(None, "")
+        with pytest.raises(ValueError):
+            gdal.MultipartUploadAbort("", None)
+
+        with pytest.raises(
+            Exception,
+            match=r"MultipartUploadAbort\(\) not supported by this file system",
+        ):
+            gdal.MultipartUploadAbort("", "")

autotest/gcore/vsigs.py

@@ -1614,6 +1614,22 @@ def my_error_handler(errclass, errno, errmsg):
     assert not error[0]
 
 
+###############################################################################
+# Test VSIMultipartUploadXXXX()
+
+
+def test_vsigs_MultipartUpload(gs_test_config, webserver_port):
+
+    # Test MultipartUploadGetCapabilities()
+    info = gdal.MultipartUploadGetCapabilities("/vsigs/")
+    assert info.non_sequential_upload_supported
+    assert info.parallel_upload_supported
+    assert info.abort_supported
+    assert info.min_part_size == 5
+    assert info.max_part_size >= 1024
+    assert info.max_part_count == 10000
+
+
 ###############################################################################
 # Nominal cases (require valid credentials)
 

autotest/gcore/vsioss.py

@@ -1118,6 +1118,22 @@ def method(request):
             assert gdal.GetLastErrorMsg() != "", filename
 
 
+###############################################################################
+# Test VSIMultipartUploadXXXX()
+
+
+def test_vsioss_MultipartUpload(server):
+
+    # Test MultipartUploadGetCapabilities()
+    info = gdal.MultipartUploadGetCapabilities("/vsioss/")
+    assert info.non_sequential_upload_supported
+    assert info.parallel_upload_supported
+    assert info.abort_supported
+    assert info.min_part_size == 5
+    assert info.max_part_size >= 1024
+    assert info.max_part_count == 10000
+
+
 ###############################################################################
 # Test Mkdir() / Rmdir()
 
@@ -1228,31 +1244,6 @@ def test_vsioss_8(server):
         assert stat.S_ISDIR(gdal.VSIStatL("/vsioss/vsioss_8/test/").mode)
 
 
-###############################################################################
-# Test gdal.CopyFileRestartable() with fallback to regular copy
-
-
-def test_vsioss_CopyFileRestartable_fallback_to_regular_copy(tmp_vsimem, server):
-
-    gdal.VSICurlClearCache()
-
-    srcfilename = str(tmp_vsimem / "foo")
-    gdal.FileFromMemBuffer(srcfilename, "foo\n")
-
-    dstfilename = "/vsioss/test_bucket/foo"
-
-    handler = webserver.SequentialHandler()
-    handler.add("PUT", "/test_bucket/foo", 200, expected_body=b"foo\n")
-
-    with webserver.install_http_handler(handler):
-        ret_code, restart_payload = gdal.CopyFileRestartable(
-            srcfilename,
-            dstfilename,
-            None,  # input payload
-        )
-    assert ret_code == 0
-
-
 ###############################################################################
 # Nominal cases (require valid credentials)