Enable cloud tests (#1202)

* Enable cloud tests

* fixes

* fixes

* ensure select_sequential_consistency for some selects

* The JSON data type is an obsolete feature. Skip tests on Cloud.

* Mark flaky tests

main.tf → .github/cloud/service.tf

@@ -1,8 +1,8 @@
 terraform {
   required_providers {
     clickhouse = {
-      source = "ClickHouse/clickhouse"
-      version = "~> 0.0.2"
+      source  = "ClickHouse/clickhouse"
+      version = "~> 0.0.5"
     }
   }
 }
@@ -21,31 +21,40 @@ variable "token_secret" {
 
 variable "service_name" {
   type = string
+  default = "clickhouse-go-tests"
 }
 
 variable "service_password" {
   type = string
 }
 
-provider clickhouse {
-  environment     = "production"
+variable "api_url" {
+  type = string
+}
+
+variable "allowed_cidr" {
+  type = string
+  default = "0.0.0.0/0"
+}
+
+provider "clickhouse" {
   organization_id = var.organization_id
   token_key       = var.token_key
   token_secret    = var.token_secret
+  api_url         = var.api_url
 }
 
 resource "clickhouse_service" "service" {
   name           = var.service_name
   cloud_provider = "aws"
   region         = "us-east-2"
   tier           = "development"
-  idle_scaling   = true
-  password  = var.service_password
+  password       = var.service_password
 
   ip_access = [
     {
-        source      = "0.0.0.0/0"
-        description = "Anywhere"
+      source      = var.allowed_cidr
+      description = "Allowed CIDR"
     }
   ]
 }

.github/workflows/run-tests.yml

@@ -5,11 +5,9 @@ permissions: write-all
 on:
   push:
     branches:
-      - v1
       - main
   pull_request:
     branches:
-      - v1
       - main
 
 jobs:
@@ -41,7 +39,6 @@ jobs:
           CLICKHOUSE_VERSION=${{ matrix.clickhouse }} make test
 
   integration-tests-cloud:
-    if: ${{ false }}  # disabled for now
     runs-on: [self-hosted, style-checker]
     defaults:
       run:
@@ -50,12 +47,15 @@ jobs:
       fail-fast: true
       matrix:
         go:
-          - "1.19"
-          - "1.20"
+          - "1.22"
+          - "1.21"
     steps:
       - name: Check Out Code
         uses: actions/checkout@v3
 
+      - name: Resolve runner IP
+        run: echo "TF_VAR_allowed_cidr=$(curl -s ifconfig.me)/32" >> $GITHUB_ENV
+
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v2.0.3
         with:
@@ -64,28 +64,41 @@ jobs:
 
       - name: Terraform Init
         id: init
+        working-directory: .github/cloud/
         run: terraform init
 
       - name: Terraform Validate
         id: validate
+        working-directory: .github/cloud/
         run: terraform validate -no-color
 
-      - name: Set Service Name
-        run: echo "TF_VAR_service_name=go_client_tests_$(date +'%Y_%m_%d_%H_%M_%S')" >> $GITHUB_ENV
+      - name: Set service name for pull request
+        if: github.event_name == 'pull_request'
+        working-directory: .github/cloud/
+        run: echo "TF_VAR_service_name=clickhouse-go ${{ matrix.go }} PR${{ github.event.pull_request.number }} $(date +'%Y%m%d%H%M%S')" >> $GITHUB_ENV
+
+      - name: Set service name for push
+        if: github.event_name == 'push'
+        working-directory: .github/cloud/
+        run: echo "TF_VAR_service_name=clickhouse-go ${{ matrix.go }} $(git rev-parse --short HEAD) $(date +'%Y%m%d%H%M%S')" >> $GITHUB_ENV
 
       - name: Terraform Apply
+        working-directory: .github/cloud/
         id: apply
-        run: terraform apply -no-color -auto-approve
+        run: terraform apply -no-color -auto-approve -input=false -lock=false
         env:
           TF_VAR_organization_id: ${{ secrets.INTEGRATIONS_TEAM_TESTS_ORGANIZATION_ID }}
           TF_VAR_token_key:  ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_KEY }}
           TF_VAR_token_secret:  ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_SECRET }}
           TF_VAR_service_password: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD }}
+          TF_VAR_api_url: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_API_URL }}
 
       - name: Set Host
+        working-directory: .github/cloud/
         run: echo "CLICKHOUSE_HOST=$(terraform output -raw CLICKHOUSE_HOST)" >> $GITHUB_ENV
 
-      - name: Service Id
+      - name: Service ID
+        working-directory: .github/cloud/
         run: terraform output -raw SERVICE_ID
 
       - name: Install Go ${{ matrix.go }}
@@ -104,9 +117,11 @@ jobs:
 
       - name: Cleanup
         if: always()
-        run: terraform destroy -no-color -auto-approve
+        working-directory: .github/cloud/
+        run: terraform destroy -no-color -auto-approve -input=false -lock=false
         env:
           TF_VAR_organization_id: ${{ secrets.INTEGRATIONS_TEAM_TESTS_ORGANIZATION_ID }}
           TF_VAR_token_key:  ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_KEY }}
           TF_VAR_token_secret:  ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_SECRET }}
           TF_VAR_service_password: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD }}
+          TF_VAR_api_url: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_API_URL }}

.gitignore

@@ -36,3 +36,6 @@ vendor
 .terraform.lock.hcl
 **/.terraform*
 pipeline.auto.tfvars
+*.tfvars
+
+.env

tests/batch_release_connection_test.go

@@ -3,13 +3,16 @@ package tests
 import (
 	"context"
 	"fmt"
+	"testing"
+
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 	"github.com/stretchr/testify/require"
-	"testing"
 )
 
 func TestBatchReleaseConnection(t *testing.T) {
+	SkipOnCloud(t, "This test is flaky on cloud ClickHouse")
+
 	conn, err := GetNativeConnection(nil, nil, &clickhouse.Compression{
 		Method: clickhouse.CompressionLZ4,
 	})

tests/conn_test.go

@@ -266,10 +266,7 @@ func TestConnCustomDialStrategy(t *testing.T) {
 }
 
 func TestEmptyDatabaseConfig(t *testing.T) {
-	runInDocker, _ := strconv.ParseBool(GetEnv("CLICKHOUSE_USE_DOCKER", "true"))
-	if !runInDocker {
-		t.Skip("Skip test in cloud environment.")
-	}
+	SkipOnCloud(t)
 
 	env, err := GetNativeTestEnvironment()
 	require.NoError(t, err)
@@ -309,10 +306,7 @@ func TestEmptyDatabaseConfig(t *testing.T) {
 }
 
 func TestCustomSettings(t *testing.T) {
-	runInDocker, _ := strconv.ParseBool(GetEnv("CLICKHOUSE_USE_DOCKER", "true"))
-	if !runInDocker {
-		t.Skip("Skip test in cloud environment.") // todo configure cloud instance with custom settings
-	}
+	SkipOnCloud(t, "Custom settings are not supported on ClickHouse Cloud")
 
 	conn, err := GetNativeConnection(clickhouse.Settings{
 		"custom_setting": clickhouse.CustomSetting{"custom_value"},
@@ -350,10 +344,7 @@ func TestCustomSettings(t *testing.T) {
 }
 
 func TestConnectionExpiresIdleConnection(t *testing.T) {
-	runInDocker, _ := strconv.ParseBool(GetEnv("CLICKHOUSE_USE_DOCKER", "true"))
-	if !runInDocker {
-		t.Skip("Skip test in cloud environment. This test is not stable in cloud environment, due to race conditions.")
-	}
+	SkipOnCloud(t)
 
 	// given
 	ctx := context.Background()
@@ -404,10 +395,8 @@ func getActiveConnections(t *testing.T, client clickhouse.Conn) (conns int64) {
 }
 
 func TestConnectionCloseIdle(t *testing.T) {
-	runInDocker, _ := strconv.ParseBool(GetEnv("CLICKHOUSE_USE_DOCKER", "true"))
-	if !runInDocker {
-		t.Skip("Skip test in cloud environment. This test is not stable in cloud environment, due to race conditions.")
-	}
+	SkipOnCloud(t)
+
 	testEnv, err := GetTestEnvironment(testSet)
 	require.NoError(t, err)
 	baseGoroutine := runtime.NumGoroutine()

tests/custom_dial_test.go

@@ -58,7 +58,7 @@ func TestCustomDialContext(t *testing.T) {
 			dialCount++
 			var d net.Dialer
 			if tlsConfig != nil {
-				return tls.DialWithDialer(&net.Dialer{Timeout: time.Duration(5) * time.Second}, "tcp", addr, tlsConfig)
+				return tls.DialWithDialer(&net.Dialer{Timeout: time.Duration(30) * time.Second}, "tcp", addr, tlsConfig)
 			}
 			return d.DialContext(ctx, "tcp", addr)
 		},

tests/json_test.go

@@ -21,16 +21,17 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"log"
+	"net"
+	"testing"
+	"time"
+
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 	"github.com/google/uuid"
 	"github.com/shopspring/decimal"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"log"
-	"net"
-	"testing"
-	"time"
 )
 
 type Releases struct {
@@ -67,6 +68,8 @@ type GithubEvent struct {
 var testDate, _ = time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", "2022-05-25 17:20:57 +0100 WEST")
 
 func setupConnection(t *testing.T) driver.Conn {
+	SkipOnCloud(t, "The JSON data type is an obsolete feature on Cloud.")
+
 	conn, err := GetNativeConnection(clickhouse.Settings{
 		"allow_experimental_object_type": 1,
 	}, nil, nil)

tests/package.go

@@ -0,0 +1,33 @@
+package tests
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+var LocalClickHouse = false
+var RemoteClickHouse = false
+var CloudClickHouse = false
+
+func init() {
+	if host, found := os.LookupEnv("CLICKHOUSE_HOST"); found {
+		if strings.HasSuffix(host, "clickhouse.cloud") ||
+			strings.HasSuffix(host, "clickhouse-staging.com") {
+			CloudClickHouse = true
+		} else {
+			RemoteClickHouse = true
+		}
+	} else {
+		LocalClickHouse = true
+	}
+}
+
+func SkipOnCloud(t *testing.T, reasons ...string) {
+	if CloudClickHouse {
+		t.Skip(append(
+			[]string{"Skipping test on cloud ClickHouse"},
+			reasons...,
+		))
+	}
+}

tests/std/conn_test.go

@@ -289,10 +289,7 @@ func TestMaxExecutionTime(t *testing.T) {
 }
 
 func TestHttpConnWithOptions(t *testing.T) {
-	runInDocker, _ := strconv.ParseBool(clickhouse_tests.GetEnv("CLICKHOUSE_USE_DOCKER", "true"))
-	if !runInDocker {
-		t.Skip("Skip test in cloud environment.")
-	}
+	clickhouse_tests.SkipOnCloud(t)
 
 	env, err := GetStdTestEnvironment()
 	require.NoError(t, err)
@@ -327,10 +324,7 @@ func TestHttpConnWithOptions(t *testing.T) {
 }
 
 func TestEmptyDatabaseConfig(t *testing.T) {
-	runInDocker, _ := strconv.ParseBool(clickhouse_tests.GetEnv("CLICKHOUSE_USE_DOCKER", "true"))
-	if !runInDocker {
-		t.Skip("Skip test in cloud environment.")
-	}
+	clickhouse_tests.SkipOnCloud(t)
 
 	env, err := GetStdTestEnvironment()
 	require.NoError(t, err)
@@ -371,11 +365,7 @@ func TestEmptyDatabaseConfig(t *testing.T) {
 func TestHTTPProxy(t *testing.T) {
 	t.Skip("test is flaky, tinyproxy container can't be started in CI")
 
-	// check if CLICKHOUSE_HOST env is postfixed with "clickhouse.cloud", skip if not
-	clickHouseHost := clickhouse_tests.GetEnv("CLICKHOUSE_HOST", "")
-	if !strings.HasSuffix(clickHouseHost, "clickhouse.cloud") {
-		t.Skip("Skip test in non cloud environment.")
-	}
+	clickhouse_tests.SkipOnCloud(t)
 
 	proxyEnv, err := clickhouse_tests.CreateTinyProxyTestEnvironment(t)
 	defer func() {
@@ -415,15 +405,12 @@ func TestHTTPProxy(t *testing.T) {
 
 		text := scanner.Text()
 		t.Log(text)
-		return strings.Contains(text, fmt.Sprintf("Established connection to host \"%s\"", clickHouseHost))
+		return strings.Contains(text, fmt.Sprintf("Established connection to host \"%s\"", ""))
 	}, 60*time.Second, time.Millisecond, "proxy logs should contain clickhouse.cloud instance host")
 }
 
 func TestCustomSettings(t *testing.T) {
-	runInDocker, _ := strconv.ParseBool(clickhouse_tests.GetEnv("CLICKHOUSE_USE_DOCKER", "true"))
-	if !runInDocker {
-		t.Skip("Skip test in cloud environment.")
-	}
+	clickhouse_tests.SkipOnCloud(t)
 
 	dsns := map[string]clickhouse.Protocol{"Native": clickhouse.Native, "Http": clickhouse.HTTP}
 	for name, protocol := range dsns {