Updates to minio python client

.github/workflows/ci.yml

@@ -8,8 +8,7 @@ on:
 
 env:
   POETRY_VERSION: 1.6.1
-  MINIO_SERVER_DOWNLOAD_URL: https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20230907020502.0.0_amd64.deb
-  MINIO_CLIENT_DOWNLOAD_URL: https://dl.min.io/client/mc/release/linux-amd64/archive/mcli_20230907224855.0.0_amd64.deb
+  MINIO_SERVER_DOWNLOAD_URL: https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20231111081441.0.0_amd64.deb
 
 jobs:
   check_skip:
@@ -38,7 +37,6 @@ jobs:
           export HASH=$(sha256sum <<EOT | cut -c -16
           ${{ env.POETRY_VERSION }}
           ${{ env.MINIO_SERVER_DOWNLOAD_URL }}
-          ${{ env.MINIO_CLIENT_DOWNLOAD_URL }}
           EOT
           )
           echo "INSTALL_CACHE_HASH=$HASH" >> $GITHUB_ENV
@@ -63,23 +61,18 @@ jobs:
       - name: Test poetry run
         run: poetry --version
 
-      - name: Install Minio Server and Client
+      - name: Install Minio Server
         if: steps.cache-installs.outputs.cache-hit != 'true'
         run: |
           cd /tmp
           wget -q -c ${{ env.MINIO_SERVER_DOWNLOAD_URL }} -O minio.deb
-          wget -q -c ${{ env.MINIO_CLIENT_DOWNLOAD_URL }} -O mcli.deb
           dpkg-deb --extract minio.deb /tmp/minio
-          dpkg-deb --extract mcli.deb /tmp/mcli
           mkdir -p "${HOME}/.local/bin"
           echo "${HOME}/.local/bin" >> $GITHUB_PATH
           install /tmp/minio/usr/local/bin/minio "${HOME}/.local/bin/minio"
-          install /tmp/mcli/usr/local/bin/mcli "${HOME}/.local/bin/mc"
 
       - name: Test minio run
-        run: |
-          minio --help
-          mc --help
+        run: minio --help
 
       - name: Checkout source
         uses: actions/checkout@v2

daskms/conftest.py

@@ -235,7 +235,9 @@ def ant_table(tmp_path_factory, wsrt_antenna_positions):
     yield fn
 
 
-MINIO_URL = urlparse("http://127.0.0.1:9000")
+MINIO_ADMIN = "admin"
+MINIO_PASSWORD = "password"
+MINIO_URL = "http://127.0.0.1:9000"
 
 
 @pytest.fixture(scope="function")
@@ -278,11 +280,21 @@ def minio_server(tmp_path_factory):
         pytest.skip('Unable to find "minio" server binary')
 
     data_dir = tmp_path_factory.mktemp("data")
-    args = [str(server_path), "server", str(data_dir), f"--address={MINIO_URL.netloc}"]
+    args = [
+        str(server_path),
+        "server",
+        str(data_dir),
+        f"--address={urlparse(MINIO_URL).netloc}",
+    ]
+    env = {
+        "MINIO_ROOT_USER": MINIO_ADMIN,
+        "MINIO_ROOT_PASSWORD": MINIO_PASSWORD,
+        **os.environ,
+    }
 
     # Start the server process and read a line from stdout so that we know
     # it's started
-    server_process = Popen(args, shell=False, stdout=PIPE, stderr=PIPE)
+    server_process = Popen(args, env=env, shell=False, stdout=PIPE, stderr=PIPE)
 
     try:
         while line := server_process.stdout.readline():
@@ -304,43 +316,20 @@ def minio_alias():
     return "testcloud"
 
 
-@pytest.fixture
-def minio_client(minio_server, minio_alias):
-    client_path = find_executable("mc")
-
-    if not client_path:
-        pytest.skip('Unable to find "mc" binary')
-
-    # Set the server alias on the client
-    args = [
-        str(client_path),
-        "alias",
-        "set",
-        minio_alias,
-        MINIO_URL.geturl(),
-        "minioadmin",
-        "minioadmin",
-    ]
-
-    ctx = multiprocessing.get_context("spawn")  # noqa
-    with Popen(args, shell=False, stdout=PIPE, stderr=PIPE) as client_process:
-        retcode = client_process.wait()
-
-        if retcode != 0:
-            raise ValueError(f"mc set alias failed with return code {retcode}")
-
-    yield client_path
-
-
 @pytest.fixture
 def minio_user_key():
     return "abcdef1234567890"
 
 
 @pytest.fixture
-def minio_admin(minio_client, minio_alias, minio_user_key):
+def minio_admin(minio_server, minio_alias, minio_user_key):
     minio = pytest.importorskip("minio")
-    minio_admin = minio.MinioAdmin(minio_alias, binary_path=str(minio_client))
+    credentials = pytest.importorskip("minio.credentials")
+    minio_admin = minio.MinioAdmin(
+        urlparse(MINIO_URL).netloc,
+        credentials.StaticProvider(MINIO_ADMIN, MINIO_PASSWORD),
+        secure=False,
+    )
     # Add a user and give it readwrite access
     minio_admin.user_add(minio_user_key, minio_user_key)
     minio_admin.policy_set("readwrite", user=minio_user_key)
@@ -349,11 +338,12 @@ def minio_admin(minio_client, minio_alias, minio_user_key):
 
 
 @pytest.fixture
-def py_minio_client(minio_client, minio_admin, minio_alias, minio_user_key):
+def py_minio_client(minio_admin, minio_alias, minio_user_key):
     minio = pytest.importorskip("minio")
+    parsed_url = urlparse(MINIO_URL)
     yield minio.Minio(
-        MINIO_URL.netloc,
+        parsed_url.netloc,
         access_key=minio_user_key,
         secret_key=minio_user_key,
-        secure=MINIO_URL.scheme == "https",
+        secure=False,
     )

daskms/experimental/arrow/tests/test_parquet.py

@@ -173,7 +173,7 @@ def test_xds_to_parquet_s3(
         key=minio_user_key,
         secret=minio_user_key,
         client_kwargs={
-            "endpoint_url": minio_url.geturl(),
+            "endpoint_url": minio_url,
             "region_name": "af-cpt",
         },
     )

daskms/experimental/zarr/tests/test_zarr.py

@@ -193,7 +193,7 @@ def test_xds_to_zarr_s3(
         key=minio_user_key,
         secret=minio_user_key,
         client_kwargs={
-            "endpoint_url": minio_url.geturl(),
+            "endpoint_url": minio_url,
             "region_name": "af-cpt",
         },
     )

pyproject.toml

@@ -17,7 +17,7 @@ pyarrow = {version = "^13.0.0", optional=true}
 zarr = {version = "^2.12.0", optional=true}
 xarray = {version = "^2023.01.0", optional=true}
 s3fs = {version = "^2023.1.0", optional=true}
-minio = {version = "^7.1.11", optional=true}
+minio = {version = "^7.2.0", optional = true}
 pytest = {version = "^7.1.3", optional=true}
 pandas = {version = "^2.1.2", optional = true}