From 372779f103288390912c0ad0f1882309f6c36b2a Mon Sep 17 00:00:00 2001 From: sanadhis Date: Sat, 17 Feb 2024 11:47:22 +0100 Subject: [PATCH 1/3] fix: gcs endpoint should be independently configured with credential --- ReadMe.md | 1 + pkg/storage/gcs.go | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ReadMe.md b/ReadMe.md index fbb88a59..94aa7fac 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -529,6 +529,7 @@ gcs: credentials_file: "" # GCS_CREDENTIALS_FILE credentials_json: "" # GCS_CREDENTIALS_JSON credentials_json_encoded: "" # GCS_CREDENTIALS_JSON_ENCODED + endpoint: "" # GCS_ENDPOINT, use it for custom GCS endpoint/compatible storage. For example, when using custom endpoint via private service connect bucket: "" # GCS_BUCKET path: "" # GCS_PATH, `system.macros` values can be applied as {macro_name} object_disk_path: "" # GCS_OBJECT_DISK_PATH, path for backup of part from `s3` object disk (clickhouse support only gcs over s3 protocol), if disk present, then shall not be zero and shall not be prefixed by `path` diff --git a/pkg/storage/gcs.go b/pkg/storage/gcs.go index ca28f080..d436eab9 100644 --- a/pkg/storage/gcs.go +++ b/pkg/storage/gcs.go @@ -90,15 +90,18 @@ func (gcs *GCS) Connect(ctx context.Context) error { if gcs.Config.Endpoint != "" { endpoint = gcs.Config.Endpoint - clientOptions = append([]option.ClientOption{option.WithoutAuthentication()}, clientOptions...) clientOptions = append(clientOptions, option.WithEndpoint(endpoint)) - } else if gcs.Config.CredentialsJSON != "" { + } + + if gcs.Config.CredentialsJSON != "" { clientOptions = append(clientOptions, option.WithCredentialsJSON([]byte(gcs.Config.CredentialsJSON))) } else if gcs.Config.CredentialsJSONEncoded != "" { d, _ := base64.StdEncoding.DecodeString(gcs.Config.CredentialsJSONEncoded) clientOptions = append(clientOptions, option.WithCredentialsJSON(d)) } else if gcs.Config.CredentialsFile != "" { clientOptions = append(clientOptions, option.WithCredentialsFile(gcs.Config.CredentialsFile)) + } else { + clientOptions = append(clientOptions, option.WithoutAuthentication()) } if gcs.Config.ForceHttp { From f2baf64042cfa565c25603b0df4ff802bc334c28 Mon Sep 17 00:00:00 2001 From: sanadhis Date: Tue, 20 Feb 2024 11:53:01 +0100 Subject: [PATCH 2/3] test(gcs): add test for custom endpoint --- .../config-gcs-custom-endpoint.yml | 29 ++++++++++++++++++ test/integration/docker-compose.yml | 30 ++++++++----------- test/integration/docker-compose_advanced.yml | 30 ++++++++----------- test/integration/dynamic_settings.sh | 10 ++++--- test/integration/integration_test.go | 13 ++++++-- 5 files changed, 71 insertions(+), 41 deletions(-) create mode 100644 test/integration/config-gcs-custom-endpoint.yml mode change 100644 => 100755 test/integration/dynamic_settings.sh diff --git a/test/integration/config-gcs-custom-endpoint.yml b/test/integration/config-gcs-custom-endpoint.yml new file mode 100644 index 00000000..6534184a --- /dev/null +++ b/test/integration/config-gcs-custom-endpoint.yml @@ -0,0 +1,29 @@ +general: + disable_progress_bar: true + remote_storage: gcs + upload_concurrency: 4 + download_concurrency: 4 + skip_tables: + - " system.*" + - "INFORMATION_SCHEMA.*" + - "information_schema.*" + - "_temporary_and_external_tables.*" + restore_schema_on_cluster: "{cluster}" +clickhouse: + host: clickhouse + port: 9440 + username: backup + password: meow=& 123?*%# МЯУ + secure: true + skip_verify: true + sync_replicated_tables: true + timeout: 5s + restart_command: "sql:SYSTEM RELOAD USERS; sql:SYSTEM RELOAD CONFIG; exec:ls -la /var/lib/clickhouse/access; sql:SYSTEM SHUTDOWN" + # restart_command: bash -c 'echo "FAKE RESTART"' + backup_mutations: true +gcs: + bucket: altinity-qa-test + path: backup/{cluster}/{shard} + object_disk_path: object_disks/{cluster}/{shard} + compression_format: tar + endpoint: http://gcs:8080/storage/v1/ diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml index 2652a0d2..a88700b9 100644 --- a/test/integration/docker-compose.yml +++ b/test/integration/docker-compose.yml @@ -43,17 +43,18 @@ services: - clickhouse-backup # todo need to reproduce download after upload -# gcs: -# image: fsouza/fake-gcs-server:latest -# hostname: gcs -# entrypoint: -# - /bin/sh -# command: -# - -c -# - "mkdir -p /data/clickhouse-backup-test-gcs && fake-gcs-server -data /data -scheme http -port 8080 -public-host gsc:8080" -# networks: -# - clickhouse-backup - + gcs: + image: fsouza/fake-gcs-server:latest + hostname: gcs + entrypoint: + - /bin/sh + command: + - -c + - "mkdir -p /data/altinity-qa-test && mkdir -p /data/${QA_GCS_OVER_S3_BUCKET} && fake-gcs-server -data /data -scheme http -port 8080 -public-host gcs:8080" + networks: + - clickhouse-backup + environment: + QA_GCS_OVER_S3_BUCKET: "${QA_GCS_OVER_S3_BUCKET}" azure: image: mcr.microsoft.com/azure-storage/azurite:latest @@ -126,9 +127,6 @@ services: AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}" CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}" GOCOVERDIR: "/tmp/_coverage_/" -# fake-gcs-server -# STORAGE_EMULATOR_HOST: "http://gsc:8080" -# GOOGLE_API_USE_CLIENT_CERTIFICATE: "false" # FIPS QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY} QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY} @@ -166,9 +164,6 @@ services: AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}" CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}" GOCOVERDIR: "/tmp/_coverage_/" -# fake-gcs-server -# STORAGE_EMULATOR_HOST: "http://gsc:8080" -# GOOGLE_API_USE_CLIENT_CERTIFICATE: "false" # FIPS QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY} QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY} @@ -196,6 +191,7 @@ services: - ./config-ftp.yaml:/etc/clickhouse-backup/config-ftp.yaml - ./config-ftp-old.yaml:/etc/clickhouse-backup/config-ftp-old.yaml - ./config-gcs.yml:/etc/clickhouse-backup/config-gcs.yml + - ./config-gcs-custom-endpoint.yml:/etc/clickhouse-backup/config-gcs-custom-endpoint.yml - ./config-s3.yml:/etc/clickhouse-backup/config-s3.yml - ./config-s3-embedded.yml:/etc/clickhouse-backup/config-s3-embedded.yml - ./config-s3-fips.yml:/etc/clickhouse-backup/config-s3-fips.yml.template diff --git a/test/integration/docker-compose_advanced.yml b/test/integration/docker-compose_advanced.yml index ade980ee..5a7baf08 100644 --- a/test/integration/docker-compose_advanced.yml +++ b/test/integration/docker-compose_advanced.yml @@ -57,17 +57,18 @@ services: - clickhouse-backup # todo need to reproduce download after upload -# gcs: -# image: fsouza/fake-gcs-server:latest -# hostname: gcs -# entrypoint: -# - /bin/sh -# command: -# - -c -# - "mkdir -p /data/clickhouse-backup-test-gcs && fake-gcs-server -data /data -scheme http -port 8080 -public-host gsc:8080" -# networks: -# - clickhouse-backup - + gcs: + image: fsouza/fake-gcs-server:latest + hostname: gcs + entrypoint: + - /bin/sh + command: + - -c + - "mkdir -p /data/altinity-qa-test && mkdir -p /data/${QA_GCS_OVER_S3_BUCKET} && fake-gcs-server -data /data -scheme http -port 8080 -public-host gcs:8080" + networks: + - clickhouse-backup + environment: + QA_GCS_OVER_S3_BUCKET: "${QA_GCS_OVER_S3_BUCKET}" azure: image: mcr.microsoft.com/azure-storage/azurite:latest @@ -177,9 +178,6 @@ services: AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}" CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}" GOCOVERDIR: "/tmp/_coverage_/" -# fake-gcs-server -# STORAGE_EMULATOR_HOST: "http://gsc:8080" -# GOOGLE_API_USE_CLIENT_CERTIFICATE: "false" # FIPS QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY} QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY} @@ -217,9 +215,6 @@ services: AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}" CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}" GOCOVERDIR: "/tmp/_coverage_/" -# fake-gcs-server -# STORAGE_EMULATOR_HOST: "http://gsc:8080" -# GOOGLE_API_USE_CLIENT_CERTIFICATE: "false" # FIPS QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY} QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY} @@ -254,6 +249,7 @@ services: - ./config-ftp.yaml:/etc/clickhouse-backup/config-ftp.yaml - ./config-ftp-old.yaml:/etc/clickhouse-backup/config-ftp-old.yaml - ./config-gcs.yml:/etc/clickhouse-backup/config-gcs.yml + - ./config-gcs-custom-endpoint.yml:/etc/clickhouse-backup/config-gcs-custom-endpoint.yml - ./config-s3.yml:/etc/clickhouse-backup/config-s3.yml - ./config-s3-embedded.yml:/etc/clickhouse-backup/config-s3-embedded.yml - ./config-s3-fips.yml:/etc/clickhouse-backup/config-s3-fips.yml.template diff --git a/test/integration/dynamic_settings.sh b/test/integration/dynamic_settings.sh old mode 100644 new mode 100755 index 303cf29f..87925796 --- a/test/integration/dynamic_settings.sh +++ b/test/integration/dynamic_settings.sh @@ -130,12 +130,14 @@ cat < /etc/clickhouse-server/config.d/storage_configuration_gcs.xml s3 - https://storage.googleapis.com/${QA_GCS_OVER_S3_BUCKET}/clickhouse_backup_disk_gcs_over_s3/${HOSTNAME}/{cluster}/{shard}/ - ${QA_GCS_OVER_S3_ACCESS_KEY} - ${QA_GCS_OVER_S3_SECRET_KEY} + http://gcs:8080/${QA_GCS_OVER_S3_BUCKET}/disk_gcs/{cluster}/{shard}/ + + 1 false - false diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 76a4b184..4126e9e7 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -1720,6 +1720,15 @@ func TestIntegrationGCS(t *testing.T) { runMainIntegrationScenario(t, "GCS", "config-gcs.yml") } +func TestIntegrationGCSWithCustomEndpoint(t *testing.T) { + if isTestShouldSkip("GCS_TESTS") { + t.Skip("Skipping GCS integration tests...") + return + } + //t.Parallel() + runMainIntegrationScenario(t, "GCS", "config-gcs-custom-endpoint.yml") +} + func TestIntegrationSFTPAuthPassword(t *testing.T) { //t.Parallel() runMainIntegrationScenario(t, "SFTP", "config-sftp-auth-password.yaml") @@ -2110,7 +2119,6 @@ func checkObjectStorageIsEmpty(t *testing.T, r *require.Assertions, remoteStorag } if remoteStorageType == "SFTP" { checkRemoteDir("total 0", "sshd", "bash", "-c", "ls -lh /root/") - } if remoteStorageType == "FTP" { if strings.Contains(os.Getenv("COMPOSE_FILE"), "advanced") { @@ -2119,9 +2127,8 @@ func checkObjectStorageIsEmpty(t *testing.T, r *require.Assertions, remoteStorag checkRemoteDir("total 0", "ftp", "bash", "-c", "ls -lh /home/vsftpd/test_backup/backup/") } } - //todo check gcs backup is empty if remoteStorageType == "GCS" { - + checkRemoteDir("total 0", "gcs", "bash", "-c", "ls -lh /data/clickhouse-backup-test-gcs") } } From b2f07e977389806f2750a7a47eae87ac3c3912b1 Mon Sep 17 00:00:00 2001 From: sanadhis Date: Tue, 20 Feb 2024 14:27:34 +0100 Subject: [PATCH 3/3] test(gcs): use GCS_SIMULATOR --- test/integration/docker-compose.yml | 1 + test/integration/docker-compose_advanced.yml | 1 + test/integration/dynamic_settings.sh | 10 ++++------ test/integration/integration_test.go | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml index a88700b9..ee0cd73b 100644 --- a/test/integration/docker-compose.yml +++ b/test/integration/docker-compose.yml @@ -46,6 +46,7 @@ services: gcs: image: fsouza/fake-gcs-server:latest hostname: gcs + container_name: gcs entrypoint: - /bin/sh command: diff --git a/test/integration/docker-compose_advanced.yml b/test/integration/docker-compose_advanced.yml index 5a7baf08..53671bcf 100644 --- a/test/integration/docker-compose_advanced.yml +++ b/test/integration/docker-compose_advanced.yml @@ -60,6 +60,7 @@ services: gcs: image: fsouza/fake-gcs-server:latest hostname: gcs + container_name: gcs entrypoint: - /bin/sh command: diff --git a/test/integration/dynamic_settings.sh b/test/integration/dynamic_settings.sh index 87925796..303cf29f 100755 --- a/test/integration/dynamic_settings.sh +++ b/test/integration/dynamic_settings.sh @@ -130,14 +130,12 @@ cat < /etc/clickhouse-server/config.d/storage_configuration_gcs.xml s3 - http://gcs:8080/${QA_GCS_OVER_S3_BUCKET}/disk_gcs/{cluster}/{shard}/ - - 1 + https://storage.googleapis.com/${QA_GCS_OVER_S3_BUCKET}/clickhouse_backup_disk_gcs_over_s3/${HOSTNAME}/{cluster}/{shard}/ + ${QA_GCS_OVER_S3_ACCESS_KEY} + ${QA_GCS_OVER_S3_SECRET_KEY} false + false diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 4126e9e7..3b505c5e 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -1726,7 +1726,7 @@ func TestIntegrationGCSWithCustomEndpoint(t *testing.T) { return } //t.Parallel() - runMainIntegrationScenario(t, "GCS", "config-gcs-custom-endpoint.yml") + runMainIntegrationScenario(t, "GCS_EMULATOR", "config-gcs-custom-endpoint.yml") } func TestIntegrationSFTPAuthPassword(t *testing.T) { @@ -2127,8 +2127,8 @@ func checkObjectStorageIsEmpty(t *testing.T, r *require.Assertions, remoteStorag checkRemoteDir("total 0", "ftp", "bash", "-c", "ls -lh /home/vsftpd/test_backup/backup/") } } - if remoteStorageType == "GCS" { - checkRemoteDir("total 0", "gcs", "bash", "-c", "ls -lh /data/clickhouse-backup-test-gcs") + if remoteStorageType == "GCS_EMULATOR" { + checkRemoteDir("total 0", "gcs", "sh", "-c", "ls -lh /data/altinity-qa-test/") } }