Use new minio.MinioAdmin client in test suite and update minio server…

… version (#298)

.github/workflows/ci.yml

@@ -8,8 +8,7 @@ on:
 
 env:
   POETRY_VERSION: 1.6.1
-  MINIO_SERVER_DOWNLOAD_URL: https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20230907020502.0.0_amd64.deb
-  MINIO_CLIENT_DOWNLOAD_URL: https://dl.min.io/client/mc/release/linux-amd64/archive/mcli_20230907224855.0.0_amd64.deb
+  MINIO_SERVER_DOWNLOAD_URL: https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20231111081441.0.0_amd64.deb
 
 jobs:
   check_skip:
@@ -38,7 +37,6 @@ jobs:
           export HASH=$(sha256sum <<EOT | cut -c -16
           ${{ env.POETRY_VERSION }}
           ${{ env.MINIO_SERVER_DOWNLOAD_URL }}
-          ${{ env.MINIO_CLIENT_DOWNLOAD_URL }}
           EOT
           )
           echo "INSTALL_CACHE_HASH=$HASH" >> $GITHUB_ENV
@@ -63,23 +61,18 @@ jobs:
       - name: Test poetry run
         run: poetry --version
 
-      - name: Install Minio Server and Client
+      - name: Install Minio Server
         if: steps.cache-installs.outputs.cache-hit != 'true'
         run: |
           cd /tmp
           wget -q -c ${{ env.MINIO_SERVER_DOWNLOAD_URL }} -O minio.deb
-          wget -q -c ${{ env.MINIO_CLIENT_DOWNLOAD_URL }} -O mcli.deb
           dpkg-deb --extract minio.deb /tmp/minio
-          dpkg-deb --extract mcli.deb /tmp/mcli
           mkdir -p "${HOME}/.local/bin"
           echo "${HOME}/.local/bin" >> $GITHUB_PATH
           install /tmp/minio/usr/local/bin/minio "${HOME}/.local/bin/minio"
-          install /tmp/mcli/usr/local/bin/mcli "${HOME}/.local/bin/mc"
 
       - name: Test minio run
-        run: |
-          minio --help
-          mc --help
+        run: minio --help
 
       - name: Checkout source
         uses: actions/checkout@v2

HISTORY.rst

@@ -4,6 +4,7 @@ History
 
 X.Y.Z (YYYY-MM-DD)
 ------------------
+* Use new minio.MinioAdmin client in test suite and update minio server version (:pr:`298`)
 * Replace black with ruff in pre-commit hooks (:pr:`297`)
 * Lazily load casacore tables module (:pr:`294`)
 * Deprecate Python 3.8 support (:pr:`296`)

daskms/conftest.py

@@ -235,7 +235,9 @@ def ant_table(tmp_path_factory, wsrt_antenna_positions):
     yield fn
 
 
-MINIO_URL = urlparse("http://127.0.0.1:9000")
+MINIO_ADMIN = "admin"
+MINIO_PASSWORD = "password"
+MINIO_URL = "http://127.0.0.1:9000"
 
 
 @pytest.fixture(scope="function")
@@ -278,11 +280,21 @@ def minio_server(tmp_path_factory):
         pytest.skip('Unable to find "minio" server binary')
 
     data_dir = tmp_path_factory.mktemp("data")
-    args = [str(server_path), "server", str(data_dir), f"--address={MINIO_URL.netloc}"]
+    args = [
+        str(server_path),
+        "server",
+        str(data_dir),
+        f"--address={urlparse(MINIO_URL).netloc}",
+    ]
+    env = {
+        "MINIO_ROOT_USER": MINIO_ADMIN,
+        "MINIO_ROOT_PASSWORD": MINIO_PASSWORD,
+        **os.environ,
+    }
 
     # Start the server process and read a line from stdout so that we know
     # it's started
-    server_process = Popen(args, shell=False, stdout=PIPE, stderr=PIPE)
+    server_process = Popen(args, env=env, shell=False, stdout=PIPE, stderr=PIPE)
 
     try:
         while line := server_process.stdout.readline():
@@ -299,48 +311,20 @@ def minio_server(tmp_path_factory):
         server_process.kill()
 
 
-@pytest.fixture
-def minio_alias():
-    return "testcloud"
-
-
-@pytest.fixture
-def minio_client(minio_server, minio_alias):
-    client_path = find_executable("mc")
-
-    if not client_path:
-        pytest.skip('Unable to find "mc" binary')
-
-    # Set the server alias on the client
-    args = [
-        str(client_path),
-        "alias",
-        "set",
-        minio_alias,
-        MINIO_URL.geturl(),
-        "minioadmin",
-        "minioadmin",
-    ]
-
-    ctx = multiprocessing.get_context("spawn")  # noqa
-    with Popen(args, shell=False, stdout=PIPE, stderr=PIPE) as client_process:
-        retcode = client_process.wait()
-
-        if retcode != 0:
-            raise ValueError(f"mc set alias failed with return code {retcode}")
-
-    yield client_path
-
-
 @pytest.fixture
 def minio_user_key():
     return "abcdef1234567890"
 
 
 @pytest.fixture
-def minio_admin(minio_client, minio_alias, minio_user_key):
+def minio_admin(minio_server, minio_user_key):
     minio = pytest.importorskip("minio")
-    minio_admin = minio.MinioAdmin(minio_alias, binary_path=str(minio_client))
+    credentials = pytest.importorskip("minio.credentials")
+    minio_admin = minio.MinioAdmin(
+        urlparse(MINIO_URL).netloc,
+        credentials.StaticProvider(MINIO_ADMIN, MINIO_PASSWORD),
+        secure=False,
+    )
     # Add a user and give it readwrite access
     minio_admin.user_add(minio_user_key, minio_user_key)
     minio_admin.policy_set("readwrite", user=minio_user_key)
@@ -349,11 +333,12 @@ def minio_admin(minio_client, minio_alias, minio_user_key):
 
 
 @pytest.fixture
-def py_minio_client(minio_client, minio_admin, minio_alias, minio_user_key):
+def py_minio_client(minio_admin, minio_user_key):
     minio = pytest.importorskip("minio")
+    parsed_url = urlparse(MINIO_URL)
     yield minio.Minio(
-        MINIO_URL.netloc,
+        parsed_url.netloc,
         access_key=minio_user_key,
         secret_key=minio_user_key,
-        secure=MINIO_URL.scheme == "https",
+        secure=False,
     )

daskms/experimental/arrow/tests/test_parquet.py

@@ -173,7 +173,7 @@ def test_xds_to_parquet_s3(
         key=minio_user_key,
         secret=minio_user_key,
         client_kwargs={
-            "endpoint_url": minio_url.geturl(),
+            "endpoint_url": minio_url,
             "region_name": "af-cpt",
         },
     )

daskms/experimental/zarr/tests/test_zarr.py

@@ -193,7 +193,7 @@ def test_xds_to_zarr_s3(
         key=minio_user_key,
         secret=minio_user_key,
         client_kwargs={
-            "endpoint_url": minio_url.geturl(),
+            "endpoint_url": minio_url,
             "region_name": "af-cpt",
         },
     )

daskms/tests/test_fsspec_store.py

@@ -95,11 +95,10 @@ def test_store_subtable_access(tmp_path_factory):
     assert (table_dir / "foo.txt").exists()
 
 
+@pytest.mark.skipif(s3fs is None, reason="s3fs not installed")
 def test_minio_server(
     tmp_path,
     py_minio_client,
-    minio_admin,
-    minio_alias,
     minio_user_key,
     minio_url,
     s3_bucket_name,
@@ -111,11 +110,10 @@ def test_minio_server(
     py_minio_client.make_bucket(s3_bucket_name)
     py_minio_client.fput_object(s3_bucket_name, "stuff.txt", str(stuff))
 
-    s3fs = pytest.importorskip("s3fs")
     s3 = s3fs.S3FileSystem(
         key=minio_user_key,
         secret=minio_user_key,
-        client_kwargs={"endpoint_url": minio_url.geturl(), "region_name": "af-cpt"},
+        client_kwargs={"endpoint_url": minio_url, "region_name": "af-cpt"},
     )
 
     with s3.open(f"{s3_bucket_name}/stuff.txt", "rb") as f:
@@ -126,8 +124,6 @@ def test_minio_server(
 def test_storage_options_from_config(
     tmp_path,
     py_minio_client,
-    minio_admin,
-    minio_alias,
     minio_user_key,
     minio_url,
     s3_bucket_name,
@@ -148,7 +144,7 @@ def test_storage_options_from_config(
         "key": minio_user_key,
         "secret": minio_user_key,
         "client_kwargs": {
-            "endpoint_url": minio_url.geturl(),
+            "endpoint_url": minio_url,
             "region_name": "af-south-1",
             "verify": False,
         },

pyproject.toml

@@ -17,7 +17,7 @@ pyarrow = {version = "^13.0.0", optional=true}
 zarr = {version = "^2.12.0", optional=true}
 xarray = {version = "^2023.01.0", optional=true}
 s3fs = {version = "^2023.1.0", optional=true}
-minio = {version = "^7.1.11", optional=true}
+minio = {version = "^7.2.0", optional = true}
 pytest = {version = "^7.1.3", optional=true}
 pandas = {version = "^2.1.2", optional = true}