From 097dc57553f3419f74445b89de469a3262225dd9 Mon Sep 17 00:00:00 2001 From: Fokko Date: Mon, 9 Dec 2024 14:52:01 +0100 Subject: [PATCH] Clean up docker Docker - Switch to the `apache/iceberg-rest-fixture` - That contains now a healthcheck: https://github.com/apache/iceberg/pull/11705 - Update docs to point to Orbstack, native Docker doesn't work (today) --- CONTRIBUTING.md | 8 +++---- .../testdata/rest_catalog/docker-compose.yaml | 6 ++---- crates/integration_tests/Cargo.toml | 2 -- crates/integration_tests/src/lib.rs | 21 +++++-------------- .../testdata/docker-compose.yaml | 6 ++---- 5 files changed, 12 insertions(+), 31 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f66d3248e1..71a222e14a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -114,13 +114,11 @@ cargo 1.69.0 (6e9a83356 2023-04-12) #### Install Docker or Podman -Currently, iceberg-rust uses Docker to set up environment for integration tests. Podman is also supported. +Currently, iceberg-rust uses Docker to set up environment for integration tests. Native Docker has some limitations, please check (https://github.com/apache/iceberg-rust/pull/748). Please use Orbstack or Podman. -You can learn how to install Docker from [here](https://docs.docker.com/get-docker/). +For MacOS users, you can install [OrbStack](https://orbstack.dev/) as a docker alternative. -For macos users, you can install [OrbStack](https://orbstack.dev/) as a docker alternative. - -For podman users, refer to [Using Podman instead of Docker](docs/contributing/podman.md) +For podman, refer to [Using Podman instead of Docker](docs/contributing/podman.md) ## Build diff --git a/crates/catalog/rest/testdata/rest_catalog/docker-compose.yaml b/crates/catalog/rest/testdata/rest_catalog/docker-compose.yaml index 34ba3c8743..245519bf82 100644 --- a/crates/catalog/rest/testdata/rest_catalog/docker-compose.yaml +++ b/crates/catalog/rest/testdata/rest_catalog/docker-compose.yaml @@ -20,12 +20,12 @@ networks: services: rest: - image: tabulario/iceberg-rest:0.10.0 + image: apache/iceberg-rest-fixture environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password - AWS_REGION=us-east-1 - - CATALOG_CATOLOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog + - CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory - CATALOG_WAREHOUSE=s3://icebergdata/demo - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO @@ -34,8 +34,6 @@ services: - minio networks: rest_bridge: - aliases: - - icebergdata.minio expose: - 8181 diff --git a/crates/integration_tests/Cargo.toml b/crates/integration_tests/Cargo.toml index f9ba9e4144..a047d75804 100644 --- a/crates/integration_tests/Cargo.toml +++ b/crates/integration_tests/Cargo.toml @@ -31,7 +31,5 @@ futures = { workspace = true } iceberg = { workspace = true } iceberg-catalog-rest = { workspace = true } iceberg_test_utils = { path = "../test_utils", features = ["tests"] } -log = { workspace = true } parquet = { workspace = true } -port_scanner = { workspace = true } tokio = { workspace = true } diff --git a/crates/integration_tests/src/lib.rs b/crates/integration_tests/src/lib.rs index 5777a4018a..c9311c45bd 100644 --- a/crates/integration_tests/src/lib.rs +++ b/crates/integration_tests/src/lib.rs @@ -21,8 +21,6 @@ use iceberg::io::{S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig}; use iceberg_test_utils::docker::DockerCompose; use iceberg_test_utils::{normalize_test_name, set_up}; -use port_scanner::scan_port_addr; -use tokio::time::sleep; const REST_CATALOG_PORT: u16 = 8181; @@ -42,24 +40,15 @@ pub async fn set_test_fixture(func: &str) -> TestFixture { docker_compose.run(); let rest_catalog_ip = docker_compose.get_container_ip("rest"); - - let read_port = format!("{}:{}", rest_catalog_ip, REST_CATALOG_PORT); - loop { - if !scan_port_addr(&read_port) { - log::info!("Waiting for 1s rest catalog to ready..."); - sleep(std::time::Duration::from_millis(1000)).await; - } else { - break; - } - } - - let container_ip = docker_compose.get_container_ip("minio"); - let read_port = format!("{}:{}", container_ip, 9000); + let minio_ip = docker_compose.get_container_ip("minio"); let config = RestCatalogConfig::builder() .uri(format!("http://{}:{}", rest_catalog_ip, REST_CATALOG_PORT)) .props(HashMap::from([ - (S3_ENDPOINT.to_string(), format!("http://{}", read_port)), + ( + S3_ENDPOINT.to_string(), + format!("http://{}:{}", minio_ip, 9000), + ), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), diff --git a/crates/integration_tests/testdata/docker-compose.yaml b/crates/integration_tests/testdata/docker-compose.yaml index 490f4eb944..034a0d9e21 100644 --- a/crates/integration_tests/testdata/docker-compose.yaml +++ b/crates/integration_tests/testdata/docker-compose.yaml @@ -20,12 +20,12 @@ networks: services: rest: - image: tabulario/iceberg-rest:0.10.0 + image: apache/iceberg-rest-fixture environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password - AWS_REGION=us-east-1 - - CATALOG_CATOLOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog + - CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory - CATALOG_WAREHOUSE=s3://icebergdata/demo - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO @@ -34,8 +34,6 @@ services: - minio networks: rest_bridge: - aliases: - - icebergdata.minio ports: - 8181:8181 expose: