Merge 862abee into 5ab0892

ReadMe.md

@@ -529,6 +529,7 @@ gcs:
   credentials_file: ""         # GCS_CREDENTIALS_FILE
   credentials_json: ""         # GCS_CREDENTIALS_JSON
   credentials_json_encoded: "" # GCS_CREDENTIALS_JSON_ENCODED
+  endpoint: ""                 # GCS_ENDPOINT, use it for custom GCS endpoint/compatible storage. For example, when using custom endpoint via private service connect
   bucket: ""                   # GCS_BUCKET
   path: ""                     # GCS_PATH, `system.macros` values can be applied as {macro_name}
   object_disk_path: ""         # GCS_OBJECT_DISK_PATH, path for backup of part from `s3` object disk (clickhouse support only gcs over s3 protocol), if disk present, then shall not be zero and shall not be prefixed by `path`

pkg/storage/gcs.go

@@ -90,15 +90,18 @@ func (gcs *GCS) Connect(ctx context.Context) error {
 
 	if gcs.Config.Endpoint != "" {
 		endpoint = gcs.Config.Endpoint
-		clientOptions = append([]option.ClientOption{option.WithoutAuthentication()}, clientOptions...)
 		clientOptions = append(clientOptions, option.WithEndpoint(endpoint))
-	} else if gcs.Config.CredentialsJSON != "" {
+	}
+
+	if gcs.Config.CredentialsJSON != "" {
 		clientOptions = append(clientOptions, option.WithCredentialsJSON([]byte(gcs.Config.CredentialsJSON)))
 	} else if gcs.Config.CredentialsJSONEncoded != "" {
 		d, _ := base64.StdEncoding.DecodeString(gcs.Config.CredentialsJSONEncoded)
 		clientOptions = append(clientOptions, option.WithCredentialsJSON(d))
 	} else if gcs.Config.CredentialsFile != "" {
 		clientOptions = append(clientOptions, option.WithCredentialsFile(gcs.Config.CredentialsFile))
+	} else {
+		clientOptions = append(clientOptions, option.WithoutAuthentication())
 	}
 
 	if gcs.Config.ForceHttp {

test/integration/config-gcs.yml

@@ -3,15 +3,30 @@ general:
   remote_storage: gcs
   upload_concurrency: 4
   download_concurrency: 4
+  skip_tables:
+    - " system.*"
+    - "INFORMATION_SCHEMA.*"
+    - "information_schema.*"
+    - "_temporary_and_external_tables.*"
   restore_schema_on_cluster: "{cluster}"
 clickhouse:
+  # wrong disk name mapping for https://github.com/Altinity/clickhouse-backup/issues/676
+  disk_mapping:
+    default-gp3: /var/lib/clickhouse
   host: clickhouse
-  port: 9000
-  restart_command: bash -c 'echo "FAKE RESTART"'
+  port: 9440
+  username: backup
+  password: meow=& 123?*%# МЯУ
+  secure: true
+  skip_verify: true
+  sync_replicated_tables: true
+  timeout: 5s
+  restart_command: "sql:SYSTEM RELOAD USERS; sql:SYSTEM RELOAD CONFIG; exec:ls -la /var/lib/clickhouse/access; sql:SYSTEM SHUTDOWN"
+  # restart_command: bash -c 'echo "FAKE RESTART"'
+  backup_mutations: true
 gcs:
   bucket: altinity-qa-test
   path: backup/{cluster}/{shard}
   object_disk_path: object_disks/{cluster}/{shard}
-  credentials_file: /etc/clickhouse-backup/credentials.json
   compression_format: tar
-#  endpoint: http://gcs:8080/storage/v1/
+  endpoint: http://gcs:8080/storage/v1/

test/integration/docker-compose.yml

@@ -43,17 +43,16 @@ services:
       - clickhouse-backup
 
 # todo need to reproduce download after upload
-#  gcs:
-#    image: fsouza/fake-gcs-server:latest
-#    hostname: gcs
-#    entrypoint:
-#      - /bin/sh
-#    command:
-#      - -c
-#      - "mkdir -p /data/clickhouse-backup-test-gcs && fake-gcs-server -data /data -scheme http -port 8080 -public-host gsc:8080"
-#    networks:
-#      - clickhouse-backup
-
+  gcs:
+    image: fsouza/fake-gcs-server:latest
+    hostname: gcs
+    entrypoint:
+      - /bin/sh
+    command:
+      - -c
+      - "mkdir -p /data/altinity-qa-test && fake-gcs-server -data /data -scheme http -port 8080 -public-host gcs:8080"
+    networks:
+      - clickhouse-backup
 
   azure:
     image: mcr.microsoft.com/azure-storage/azurite:latest
@@ -126,9 +125,6 @@ services:
       AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}"
       CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}"
       GOCOVERDIR: "/tmp/_coverage_/"
-# fake-gcs-server
-#       STORAGE_EMULATOR_HOST: "http://gsc:8080"
-#       GOOGLE_API_USE_CLIENT_CERTIFICATE: "false"
 # FIPS
       QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY}
       QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY}
@@ -166,9 +162,6 @@ services:
       AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}"
       CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}"
       GOCOVERDIR: "/tmp/_coverage_/"
-# fake-gcs-server
-#       STORAGE_EMULATOR_HOST: "http://gsc:8080"
-#       GOOGLE_API_USE_CLIENT_CERTIFICATE: "false"
 # FIPS
       QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY}
       QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY}

test/integration/docker-compose_advanced.yml

@@ -57,17 +57,16 @@ services:
       - clickhouse-backup
 
 # todo need to reproduce download after upload
-#  gcs:
-#    image: fsouza/fake-gcs-server:latest
-#    hostname: gcs
-#    entrypoint:
-#      - /bin/sh
-#    command:
-#      - -c
-#      - "mkdir -p /data/clickhouse-backup-test-gcs && fake-gcs-server -data /data -scheme http -port 8080 -public-host gsc:8080"
-#    networks:
-#      - clickhouse-backup
-
+  gcs:
+    image: fsouza/fake-gcs-server:latest
+    hostname: gcs
+    entrypoint:
+      - /bin/sh
+    command:
+      - -c
+      - "mkdir -p /data/altinity-qa-test && fake-gcs-server -data /data -scheme http -port 8080 -public-host gcs:8080"
+    networks:
+      - clickhouse-backup
 
   azure:
     image: mcr.microsoft.com/azure-storage/azurite:latest
@@ -177,9 +176,6 @@ services:
       AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}"
       CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}"
       GOCOVERDIR: "/tmp/_coverage_/"
-# fake-gcs-server
-#       STORAGE_EMULATOR_HOST: "http://gsc:8080"
-#       GOOGLE_API_USE_CLIENT_CERTIFICATE: "false"
 # FIPS
       QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY}
       QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY}
@@ -217,9 +213,6 @@ services:
       AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}"
       CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}"
       GOCOVERDIR: "/tmp/_coverage_/"
-# fake-gcs-server
-#       STORAGE_EMULATOR_HOST: "http://gsc:8080"
-#       GOOGLE_API_USE_CLIENT_CERTIFICATE: "false"
 # FIPS
       QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY}
       QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY}

test/integration/integration_test.go

@@ -2110,7 +2110,6 @@ func checkObjectStorageIsEmpty(t *testing.T, r *require.Assertions, remoteStorag
 	}
 	if remoteStorageType == "SFTP" {
 		checkRemoteDir("total 0", "sshd", "bash", "-c", "ls -lh /root/")
-
 	}
 	if remoteStorageType == "FTP" {
 		if strings.Contains(os.Getenv("COMPOSE_FILE"), "advanced") {
@@ -2119,9 +2118,8 @@ func checkObjectStorageIsEmpty(t *testing.T, r *require.Assertions, remoteStorag
 			checkRemoteDir("total 0", "ftp", "bash", "-c", "ls -lh /home/vsftpd/test_backup/backup/")
 		}
 	}
-	//todo check gcs backup is empty
 	if remoteStorageType == "GCS" {
-
+		checkRemoteDir("total 0", "gcs", "bash", "-c", "ls -lh /data/clickhouse-backup-test-gcs")
 	}
 }
 

test/integration/run.sh

@@ -17,12 +17,6 @@ fi
 export CLICKHOUSE_BACKUP_BIN="$(pwd)/clickhouse-backup/clickhouse-backup-race"
 export LOG_LEVEL=${LOG_LEVEL:-info}
 
-if [[ -f "${CUR_DIR}/credentials.json" ]]; then
-  export GCS_TESTS=${GCS_TESTS:-1}
-else
-  export GCS_TESTS=${GCS_TESTS:-}
-fi
-
 export GLACIER_TESTS=${GLACIER_TESTS:-0}
 
 export AZURE_TESTS=${AZURE_TESTS:-1}