fix: Fix cargo test hangs on macos.

crates/catalog/glue/testdata/glue_catalog/docker-compose.yaml

@@ -15,14 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-version: '3.8'
-
 services:
   minio:
     image: minio/minio:RELEASE.2024-03-07T00-43-48Z
-    expose:
-      - 9000
-      - 9001
+    ports:
+      - "9000"
+      - "9001"
     environment:
       - MINIO_ROOT_USER=admin
       - MINIO_ROOT_PASSWORD=password
@@ -42,5 +40,5 @@ services:
 
   moto:
     image: motoserver/moto:5.0.3
-    expose:
-      - 5000
+    ports:
+      - "5000"

crates/catalog/glue/tests/glue_catalog_test.rs

@@ -25,7 +25,7 @@ use iceberg::{Catalog, Namespace, NamespaceIdent, Result, TableCreation, TableId
 use iceberg_catalog_glue::{
     GlueCatalog, GlueCatalogConfig, AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY,
 };
-use iceberg_test_utils::docker::DockerCompose;
+use iceberg_test_utils::docker::{DockerCompose, LOCALHOST};
 use iceberg_test_utils::{normalize_test_name, set_up};
 use port_scanner::scan_port_addr;
 use tokio::time::sleep;
@@ -49,10 +49,15 @@ async fn set_test_fixture(func: &str) -> TestFixture {
 
     docker_compose.run();
 
-    let glue_catalog_ip = docker_compose.get_container_ip("moto");
-    let minio_ip = docker_compose.get_container_ip("minio");
+    log::info!(
+        "Docker compose started for {}",
+        docker_compose.project_name()
+    );
+
+    let glue_catalog_port = docker_compose.get_container_port("moto", GLUE_CATALOG_PORT);
+    let minio_port = docker_compose.get_container_port("minio", MINIO_PORT);
 
-    let read_port = format!("{}:{}", glue_catalog_ip, GLUE_CATALOG_PORT);
+    let read_port = format!("{}:{}", LOCALHOST, glue_catalog_port);
     loop {
         if !scan_port_addr(&read_port) {
             log::info!("Waiting for 1s glue catalog to ready...");
@@ -71,15 +76,15 @@ async fn set_test_fixture(func: &str) -> TestFixture {
         (AWS_REGION_NAME.to_string(), "us-east-1".to_string()),
         (
             S3_ENDPOINT.to_string(),
-            format!("http://{}:{}", minio_ip, MINIO_PORT),
+            format!("http://{}:{}", LOCALHOST, minio_port),
         ),
         (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()),
         (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()),
         (S3_REGION.to_string(), "us-east-1".to_string()),
     ]);
 
     let config = GlueCatalogConfig::builder()
-        .uri(format!("http://{}:{}", glue_catalog_ip, GLUE_CATALOG_PORT))
+        .uri(format!("http://{}:{}", LOCALHOST, glue_catalog_port))
         .warehouse("s3a://warehouse/hive".to_string())
         .props(props.clone())
         .build();

crates/catalog/hms/testdata/hms_catalog/docker-compose.yaml

@@ -15,14 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-version: '3.8'
-
 services:
   minio:
     image: minio/minio:RELEASE.2024-03-07T00-43-48Z
-    expose:
-      - 9000
-      - 9001
+    ports:
+      - "9000"
+      - "9001"
     environment:
       - MINIO_ROOT_USER=admin
       - MINIO_ROOT_PASSWORD=password
@@ -43,8 +41,8 @@ services:
   hive-metastore:
     image: iceberg-hive-metastore
     build: ./
-    expose:
-      - 9083
+    ports:
+      - "9083"
     environment:
       SERVICE_NAME: "metastore"
       SERVICE_OPTS: "-Dmetastore.warehouse.dir=s3a://warehouse/hive/"

crates/catalog/hms/tests/hms_catalog_test.rs

@@ -23,7 +23,7 @@ use iceberg::io::{S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY
 use iceberg::spec::{NestedField, PrimitiveType, Schema, Type};
 use iceberg::{Catalog, Namespace, NamespaceIdent, TableCreation, TableIdent};
 use iceberg_catalog_hms::{HmsCatalog, HmsCatalogConfig, HmsThriftTransport};
-use iceberg_test_utils::docker::DockerCompose;
+use iceberg_test_utils::docker::{DockerCompose, LOCALHOST};
 use iceberg_test_utils::{normalize_test_name, set_up};
 use port_scanner::scan_port_addr;
 use tokio::time::sleep;
@@ -47,10 +47,10 @@ async fn set_test_fixture(func: &str) -> TestFixture {
 
     docker_compose.run();
 
-    let hms_catalog_ip = docker_compose.get_container_ip("hive-metastore");
-    let minio_ip = docker_compose.get_container_ip("minio");
+    let hms_catalog_port = docker_compose.get_container_port("hive-metastore", HMS_CATALOG_PORT);
+    let minio_port = docker_compose.get_container_port("minio", MINIO_PORT);
 
-    let read_port = format!("{}:{}", hms_catalog_ip, HMS_CATALOG_PORT);
+    let read_port = format!("{}:{}", LOCALHOST, hms_catalog_port);
     loop {
         if !scan_port_addr(&read_port) {
             log::info!("Waiting for 1s hms catalog to ready...");
@@ -63,15 +63,15 @@ async fn set_test_fixture(func: &str) -> TestFixture {
     let props = HashMap::from([
         (
             S3_ENDPOINT.to_string(),
-            format!("http://{}:{}", minio_ip, MINIO_PORT),
+            format!("http://{}:{}", LOCALHOST, minio_port),
         ),
         (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()),
         (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()),
         (S3_REGION.to_string(), "us-east-1".to_string()),
     ]);
 
     let config = HmsCatalogConfig::builder()
-        .address(format!("{}:{}", hms_catalog_ip, HMS_CATALOG_PORT))
+        .address(format!("{}:{}", LOCALHOST, hms_catalog_port))
         .thrift_transport(HmsThriftTransport::Buffered)
         .warehouse("s3a://warehouse/hive".to_string())
         .props(props)

crates/catalog/rest/testdata/rest_catalog/docker-compose.yaml

@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-version: '3.8'
-
 services:
   rest:
     image: tabulario/iceberg-rest:0.10.0
@@ -33,18 +31,18 @@ services:
       - minio
     links:
       - minio:icebergdata.minio
-    expose:
-      - 8181
+    ports:
+      - "8181"
 
   minio:
     image: minio/minio:RELEASE.2024-03-07T00-43-48Z
     environment:
       - MINIO_ROOT_USER=admin
       - MINIO_ROOT_PASSWORD=password
       - MINIO_DOMAIN=minio
-    expose:
-      - 9001
-      - 9000
+    ports:
+      - "9001"
+      - "9000"
     command: [ "server", "/data", "--console-address", ":9001" ]
 
   mc:

crates/catalog/rest/tests/rest_catalog_test.rs

@@ -21,7 +21,7 @@ use iceberg::spec::{FormatVersion, NestedField, PrimitiveType, Schema, Type};
 use iceberg::transaction::Transaction;
 use iceberg::{Catalog, Namespace, NamespaceIdent, TableCreation, TableIdent};
 use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig};
-use iceberg_test_utils::docker::DockerCompose;
+use iceberg_test_utils::docker::{DockerCompose, LOCALHOST};
 use iceberg_test_utils::{normalize_test_name, set_up};
 use port_scanner::scan_port_addr;
 use std::collections::HashMap;
@@ -44,9 +44,9 @@ async fn set_test_fixture(func: &str) -> TestFixture {
     // Start docker compose
     docker_compose.run();
 
-    let rest_catalog_ip = docker_compose.get_container_ip("rest");
+    let rest_catalog_port = docker_compose.get_container_port("rest", REST_CATALOG_PORT);
 
-    let read_port = format!("{}:{}", rest_catalog_ip, REST_CATALOG_PORT);
+    let read_port = format!("{}:{}", LOCALHOST, rest_catalog_port);
     loop {
         if !scan_port_addr(&read_port) {
             log::info!("Waiting for 1s rest catalog to ready...");
@@ -57,7 +57,7 @@ async fn set_test_fixture(func: &str) -> TestFixture {
     }
 
     let config = RestCatalogConfig::builder()
-        .uri(format!("http://{}:{}", rest_catalog_ip, REST_CATALOG_PORT))
+        .uri(format!("http://{}:{}", LOCALHOST, rest_catalog_port))
         .build();
     let rest_catalog = RestCatalog::new(config).await.unwrap();
 

crates/iceberg/testdata/file_io_s3/docker-compose.yaml

@@ -15,13 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-version: '3'
 services:
   minio:
     image: minio/minio:RELEASE.2024-02-26T09-33-48Z
     ports:
-      - 9000
-      - 9001
+      - "9000"
+      - "9001"
     environment:
       MINIO_ROOT_USER: 'admin'
       MINIO_ROOT_PASSWORD: 'password'

crates/iceberg/tests/file_io_s3_test.rs

@@ -20,7 +20,7 @@
 use iceberg::io::{
     FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY,
 };
-use iceberg_test_utils::docker::DockerCompose;
+use iceberg_test_utils::docker::{DockerCompose, LOCALHOST};
 
 struct MinIOFixture {
     _docker_compose: DockerCompose,
@@ -34,8 +34,8 @@ impl MinIOFixture {
             format!("{}/testdata/file_io_s3", env!("CARGO_MANIFEST_DIR")),
         );
         docker.run();
-        let container_ip = docker.get_container_ip("minio");
-        let read_port = format!("{}:{}", container_ip, 9000);
+        let container_port = docker.get_container_port("minio", 9000);
+        let read_port = format!("{}:{}", LOCALHOST, container_port);
 
         MinIOFixture {
             _docker_compose: docker,

crates/integrations/datafusion/testdata/docker-compose.yaml

@@ -20,9 +20,9 @@ version: '3.8'
 services:
   minio:
     image: minio/minio:RELEASE.2024-03-07T00-43-48Z
-    expose:
-      - 9000
-      - 9001
+    ports:
+      - "9000"
+      - "9001"
     environment:
       - MINIO_ROOT_USER=admin
       - MINIO_ROOT_PASSWORD=password
@@ -43,8 +43,8 @@ services:
   hive-metastore:
     image: iceberg-hive-metastore
     build: ./hms_catalog/
-    expose:
-      - 9083
+    ports:
+      - "9083"
     environment:
       SERVICE_NAME: "metastore"
       SERVICE_OPTS: "-Dmetastore.warehouse.dir=s3a://warehouse/hive/"

crates/integrations/datafusion/tests/integration_datafusion_hms_test.rs

@@ -27,7 +27,7 @@ use iceberg::spec::{NestedField, PrimitiveType, Schema, Type};
 use iceberg::{Catalog, NamespaceIdent, Result, TableCreation};
 use iceberg_catalog_hms::{HmsCatalog, HmsCatalogConfig, HmsThriftTransport};
 use iceberg_datafusion::IcebergCatalogProvider;
-use iceberg_test_utils::docker::DockerCompose;
+use iceberg_test_utils::docker::{DockerCompose, LOCALHOST};
 use iceberg_test_utils::{normalize_test_name, set_up};
 use port_scanner::scan_port_addr;
 use tokio::time::sleep;
@@ -50,10 +50,10 @@ async fn set_test_fixture(func: &str) -> TestFixture {
 
     docker_compose.run();
 
-    let hms_catalog_ip = docker_compose.get_container_ip("hive-metastore");
-    let minio_ip = docker_compose.get_container_ip("minio");
+    let hms_catalog_port = docker_compose.get_container_port("hive-metastore", HMS_CATALOG_PORT);
+    let minio_port = docker_compose.get_container_port("minio", MINIO_PORT);
 
-    let read_port = format!("{}:{}", hms_catalog_ip, HMS_CATALOG_PORT);
+    let read_port = format!("{}:{}", LOCALHOST, hms_catalog_port);
     loop {
         if !scan_port_addr(&read_port) {
             log::info!("Waiting for 1s hms catalog to ready...");
@@ -66,15 +66,15 @@ async fn set_test_fixture(func: &str) -> TestFixture {
     let props = HashMap::from([
         (
             S3_ENDPOINT.to_string(),
-            format!("http://{}:{}", minio_ip, MINIO_PORT),
+            format!("http://{}:{}", LOCALHOST, minio_port),
         ),
         (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()),
         (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()),
         (S3_REGION.to_string(), "us-east-1".to_string()),
     ]);
 
     let config = HmsCatalogConfig::builder()
-        .address(format!("{}:{}", hms_catalog_ip, HMS_CATALOG_PORT))
+        .address(format!("{}:{}", LOCALHOST, hms_catalog_port))
         .thrift_transport(HmsThriftTransport::Buffered)
         .warehouse("s3a://warehouse/hive".to_string())
         .props(props)

crates/test_utils/src/docker.rs

@@ -17,6 +17,10 @@
 
 use crate::cmd::{get_cmd_output, run_command};
 use std::process::Command;
+use std::thread::sleep;
+use std::time::Duration;
+
+pub const LOCALHOST: &str = "localhost";
 
 /// A utility to manage the lifecycle of `docker compose`.
 ///
@@ -60,25 +64,37 @@ impl DockerCompose {
                 "Starting docker compose in {}, project name: {}",
                 self.docker_compose_dir, self.project_name
             ),
-        )
+        );
+
+        // It seems that too fast after starting the docker compose, the container is not ready yet,
+        // let's wait for a while.
+        sleep(Duration::from_secs(3));
     }
 
-    pub fn get_container_ip(&self, service_name: impl AsRef<str>) -> String {
+    /// Returns the mapped port of the container.
+    pub fn get_container_port(&self, service_name: impl AsRef<str>, inner_port: u16) -> u16 {
         let container_name = format!("{}-{}-1", self.project_name, service_name.as_ref());
         let mut cmd = Command::new("docker");
         cmd.arg("inspect")
             .arg("-f")
-            .arg("{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}")
+            .arg(format!(
+                "{{{{(index (index .NetworkSettings.Ports \"{inner_port}/tcp\") 0).HostPort}}}}"
+            ))
             .arg(&container_name);
 
-        get_cmd_output(cmd, format!("Get container ip of {container_name}"))
+        get_cmd_output(cmd, format!("Get container port of {container_name}"))
             .trim()
-            .to_string()
+            .parse()
+            .expect("Failed to parse container port to u16")
     }
 }
 
 impl Drop for DockerCompose {
     fn drop(&mut self) {
+        log::info!(
+            "Trying to stop docker compose project: {}",
+            self.project_name
+        );
         let mut cmd = Command::new("docker");
         cmd.current_dir(&self.docker_compose_dir);