From 10501dd6d9a1f2a07a1a414ddc855ac6d98d1336 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 11 Jun 2025 11:22:54 -0700 Subject: [PATCH 1/2] Add ability to modify AWSLOCAL_ENDPOINT --- .github/workflows/DockerCI.yml | 1 + aws_local/src/lib.rs | 6 ++++++ mobile_verifier/Cargo.toml | 3 +-- mobile_verifier/tests/integrations/boosting_oracles.rs | 6 ++++-- .../tests/integrations/subscriber_map_act_daemon.rs | 5 +++-- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/DockerCI.yml b/.github/workflows/DockerCI.yml index c71f1da43..72171764c 100644 --- a/.github/workflows/DockerCI.yml +++ b/.github/workflows/DockerCI.yml @@ -219,6 +219,7 @@ jobs: --rm \ --network=host \ -e DATABASE_URL="postgres://postgres:postgres@localhost:5432/postgres" \ + -e AWSLOCAL_ENDPOINT="http://localhost:4566" \ -v $PWD:/app \ -w /app \ ghcr.io/${{ github.repository }}/$TARGET:$IMAGE_TAG \ diff --git a/aws_local/src/lib.rs b/aws_local/src/lib.rs index 5d43f89cd..8cd7197ba 100644 --- a/aws_local/src/lib.rs +++ b/aws_local/src/lib.rs @@ -4,6 +4,7 @@ use aws_sdk_s3::{Client, Endpoint, Region}; use chrono::Utc; use file_store::traits::MsgBytes; use file_store::{file_sink, file_upload, FileStore, FileType, Settings}; +use std::env; use std::path::Path; use std::{str::FromStr, sync::Arc}; use tempfile::TempDir; @@ -11,8 +12,13 @@ use tokio::sync::Mutex; use tonic::transport::Uri; use uuid::Uuid; +pub const AWSLOCAL_ENDPOINT_ENV: &str = "AWSLOCAL_ENDPOINT"; pub const AWSLOCAL_DEFAULT_ENDPOINT: &str = "http://localstack:4566"; +pub fn aws_local_default_endpoint() -> String { + env::var(AWSLOCAL_ENDPOINT_ENV).unwrap_or_else(|_| AWSLOCAL_DEFAULT_ENDPOINT.to_string()) +} + pub fn gen_bucket_name() -> String { format!("mvr-{}-{}", Uuid::new_v4(), Utc::now().timestamp_millis()) } diff --git a/mobile_verifier/Cargo.toml b/mobile_verifier/Cargo.toml index 747de4738..051dc0ad9 100644 --- a/mobile_verifier/Cargo.toml +++ b/mobile_verifier/Cargo.toml @@ -64,7 +64,6 @@ solana = { path = "../solana" } task-manager = { path = "../task_manager" } [dev-dependencies] -proptest = "1.5.0" aws-local = { path = "../aws_local" } +proptest = "1.5.0" tempfile = "3" - diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index ee3ddf735..4c13ec16c 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -117,7 +117,8 @@ pub async fn create_data_set_downloader( ) -> (DataSetDownloaderDaemon, PathBuf, String) { let bucket_name = gen_bucket_name(); - let awsl = AwsLocal::new(AWSLOCAL_DEFAULT_ENDPOINT, &bucket_name).await; + let endpoint = aws_local_default_endpoint(); + let awsl = AwsLocal::new(endpoint.as_str(), &bucket_name).await; for file_path in file_paths { awsl.put_file_to_aws(&file_path).await.unwrap(); @@ -209,7 +210,8 @@ async fn test_dataset_downloader(pool: PgPool) { assert!(hex_assignment_file_exist(&pool, "landtype.1722895200000.gz").await); assert!(hex_assignment_file_exist(&pool, "service_provider_override.1739404800000.gz").await); - let awsl = AwsLocal::new(AWSLOCAL_DEFAULT_ENDPOINT, &bucket_name).await; + let endpoint = aws_local_default_endpoint(); + let awsl = AwsLocal::new(endpoint.as_str(), &bucket_name).await; awsl.put_file_to_aws( &PathBuf::from_str("./tests/integrations/fixtures/footfall.1732895200000.gz").unwrap(), ) diff --git a/mobile_verifier/tests/integrations/subscriber_map_act_daemon.rs b/mobile_verifier/tests/integrations/subscriber_map_act_daemon.rs index 1afaca247..09fcf7055 100644 --- a/mobile_verifier/tests/integrations/subscriber_map_act_daemon.rs +++ b/mobile_verifier/tests/integrations/subscriber_map_act_daemon.rs @@ -1,5 +1,5 @@ use async_trait::async_trait; -use aws_local::{gen_bucket_name, AwsLocal, AWSLOCAL_DEFAULT_ENDPOINT}; +use aws_local::{gen_bucket_name, AwsLocal}; use chrono::DateTime; use file_store::{ file_info_poller::LookbackBehavior, @@ -63,7 +63,8 @@ pub async fn put_subscriber_reports_to_aws( #[sqlx::test] async fn test_process_map_act_ingest_report_file(pool: PgPool) { let bucket_name = gen_bucket_name(); - let awsl = AwsLocal::new(AWSLOCAL_DEFAULT_ENDPOINT, &bucket_name).await; + let endpoint = aws_local_default_endpoint(); + let awsl = AwsLocal::new(endpoint.as_str(), &bucket_name).await; let (stream_receiver, stream_server) = file_source::Continuous::prost_source() .state(pool.clone()) From 8497f6588c605bd95c1644fbcfab4db97711284e Mon Sep 17 00:00:00 2001 From: Anatolii Kurotych Date: Fri, 13 Jun 2025 14:55:09 +0300 Subject: [PATCH 2/2] http://localhost:4566 as AWSLOCAL_DEFAULT_ENDPOINT --- .github/workflows/CI.yml | 1 + .github/workflows/DockerCI.yml | 1 - aws_local/src/lib.rs | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6b6fc8bc8..539942a66 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -153,6 +153,7 @@ jobs: - name: Run unit and integration tests env: DATABASE_URL: "postgres://postgres:postgres@postgres:5432/postgres" + AWSLOCAL_ENDPOINT: "http://localstack:4566" run: cargo test -p ${{ matrix.package }} -- --include-ignored build-release: diff --git a/.github/workflows/DockerCI.yml b/.github/workflows/DockerCI.yml index 72171764c..c71f1da43 100644 --- a/.github/workflows/DockerCI.yml +++ b/.github/workflows/DockerCI.yml @@ -219,7 +219,6 @@ jobs: --rm \ --network=host \ -e DATABASE_URL="postgres://postgres:postgres@localhost:5432/postgres" \ - -e AWSLOCAL_ENDPOINT="http://localhost:4566" \ -v $PWD:/app \ -w /app \ ghcr.io/${{ github.repository }}/$TARGET:$IMAGE_TAG \ diff --git a/aws_local/src/lib.rs b/aws_local/src/lib.rs index 8cd7197ba..30e9d4b43 100644 --- a/aws_local/src/lib.rs +++ b/aws_local/src/lib.rs @@ -13,7 +13,7 @@ use tonic::transport::Uri; use uuid::Uuid; pub const AWSLOCAL_ENDPOINT_ENV: &str = "AWSLOCAL_ENDPOINT"; -pub const AWSLOCAL_DEFAULT_ENDPOINT: &str = "http://localstack:4566"; +pub const AWSLOCAL_DEFAULT_ENDPOINT: &str = "http://localhost:4566"; pub fn aws_local_default_endpoint() -> String { env::var(AWSLOCAL_ENDPOINT_ENV).unwrap_or_else(|_| AWSLOCAL_DEFAULT_ENDPOINT.to_string())