diff --git a/Makefile b/Makefile index a150438614..8acf44fba2 100644 --- a/Makefile +++ b/Makefile @@ -144,6 +144,15 @@ integration: integration-short: VERBOSE_TEST=1 $(INTEGRATION) -short +dbr-integration: + DBR_ENABLED=true go test -v -timeout 4h -run TestDbrAcceptance$$ ./acceptance + +# DBR acceptance tests - run on Databricks Runtime using serverless compute +# These require deco env run for authentication +# Set DBR_TEST_VERBOSE=1 for detailed output (e.g., DBR_TEST_VERBOSE=1 make dbr-test) +dbr-test: + deco env run -i -n aws-prod-ucws -- make dbr-integration + generate-validation: go run ./bundle/internal/validation/. gofmt -w -s ./bundle/internal/validation/generated diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 0c5b43ed66..fcb3ea8e65 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -46,7 +46,6 @@ var ( SkipLocal bool UseVersion string WorkspaceTmpDir bool - TerraformDir string OnlyOutTestToml bool ) @@ -79,11 +78,6 @@ func init() { // DABs in the workspace runs on the workspace file system. This flags does the same for acceptance tests // to simulate an identical environment. flag.BoolVar(&WorkspaceTmpDir, "workspace-tmp-dir", false, "Run tests on the workspace file system (For DBR testing).") - - // Symlinks from workspace file system to local file mount are not supported on DBR. Terraform implicitly - // creates these symlinks when a file_mirror is used for a provider (in .terraformrc). This flag - // allows us to download the provider to the workspace file system on DBR enabling DBR integration testing. - flag.StringVar(&TerraformDir, "terraform-dir", "", "Directory to download the terraform provider to") flag.BoolVar(&OnlyOutTestToml, "only-out-test-toml", false, "Only regenerate out.test.toml files without running tests") } @@ -173,14 +167,11 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { buildDir := getBuildDir(t, cwd, runtime.GOOS, runtime.GOARCH) - terraformDir := TerraformDir - if terraformDir == "" { - terraformDir = buildDir + // Set up terraform for tests. Skip on DBR - tests with RunsOnDbr only use direct deployment. + if !WorkspaceTmpDir { + setupTerraform(t, cwd, buildDir, &repls) } - // Download terraform and provider and create config. - RunCommand(t, []string{"python3", filepath.Join(cwd, "install_terraform.py"), "--targetdir", terraformDir}, ".", []string{}) - wheelPath := buildDatabricksBundlesWheel(t, buildDir) if wheelPath != "" { t.Setenv("DATABRICKS_BUNDLES_WHEEL", wheelPath) @@ -210,7 +201,12 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { } } - BuildYamlfmt(t) + // Skip building yamlfmt when running on workspace filesystem (DBR). + // This fails today on DBR. Can be looked into and fixed as a follow-up + // as and when needed. + if !WorkspaceTmpDir { + BuildYamlfmt(t) + } t.Setenv("CLI", execPath) repls.SetPath(execPath, "[CLI]") @@ -255,16 +251,6 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { t.Setenv("CLI_RELEASES_DIR", releasesDir) } - terraformrcPath := filepath.Join(terraformDir, ".terraformrc") - t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath) - t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath) - repls.SetPath(terraformrcPath, "[DATABRICKS_TF_CLI_CONFIG_FILE]") - - terraformExecPath := filepath.Join(terraformDir, "terraform") + exeSuffix - t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath) - t.Setenv("TERRAFORM", terraformExecPath) - repls.SetPath(terraformExecPath, "[TERRAFORM]") - // do it last so that full paths match first: repls.SetPath(buildDir, "[BUILD_DIR]") @@ -429,8 +415,8 @@ func getSkipReason(config *internal.TestConfig, configPath string) string { return "" } - if isTruePtr(config.SkipOnDbr) && WorkspaceTmpDir { - return "Disabled via SkipOnDbr setting in " + configPath + if WorkspaceTmpDir && !isTruePtr(config.RunsOnDbr) { + return "Disabled because RunsOnDbr is not set in " + configPath } if isTruePtr(config.Slow) && testing.Short() { @@ -530,7 +516,7 @@ func runTest(t *testing.T, // If the test is being run on DBR, auth is already configured // by the dbr_runner notebook by reading a token from the notebook context and // setting DATABRICKS_TOKEN and DATABRICKS_HOST environment variables. - _, _, tmpDir = workspaceTmpDir(t.Context(), t) + tmpDir = workspaceTmpDir(t.Context(), t) // Run DBR tests on the workspace file system to mimic usage from // DABs in the workspace. @@ -1383,6 +1369,22 @@ func BuildYamlfmt(t *testing.T) { RunCommand(t, args, "..", []string{}) } +// setupTerraform installs terraform and configures environment variables for tests. +func setupTerraform(t *testing.T, cwd, buildDir string, repls *testdiff.ReplacementsContext) { + RunCommand(t, []string{"python3", filepath.Join(cwd, "install_terraform.py"), "--targetdir", buildDir}, ".", []string{}) + + terraformrcPath := filepath.Join(buildDir, ".terraformrc") + terraformExecPath := filepath.Join(buildDir, "terraform") + exeSuffix + + t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath) + t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath) + t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath) + t.Setenv("TERRAFORM", terraformExecPath) + + repls.SetPath(terraformrcPath, "[DATABRICKS_TF_CLI_CONFIG_FILE]") + repls.SetPath(terraformExecPath, "[TERRAFORM]") +} + func loadUserReplacements(t *testing.T, repls *testdiff.ReplacementsContext, tmpDir string) { b, err := os.ReadFile(filepath.Join(tmpDir, userReplacementsFilename)) if os.IsNotExist(err) { diff --git a/acceptance/bundle/artifacts/test.toml b/acceptance/bundle/artifacts/test.toml index 039796d1ee..61bf8345e7 100644 --- a/acceptance/bundle/artifacts/test.toml +++ b/acceptance/bundle/artifacts/test.toml @@ -11,7 +11,7 @@ RecordRequests = true # Failed to inspect Python interpreter from active virtual environment at `.venv/bin/python3` # Caused by: Failed to query Python interpreter # Caused by: failed to canonicalize path `/Workspace/abcd/.venv/bin/python3`: Invalid cross-device link (os error 18) -SkipOnDbr = true +# Thus we cannot run this test on DBR. Ignore = [ '.venv', diff --git a/acceptance/bundle/integration_whl/test.toml b/acceptance/bundle/integration_whl/test.toml index b1783f1d25..d7c6d5fa9a 100644 --- a/acceptance/bundle/integration_whl/test.toml +++ b/acceptance/bundle/integration_whl/test.toml @@ -12,7 +12,7 @@ CloudSlow = true # Failed to inspect Python interpreter from active virtual environment at `.venv/bin/python3` # Caused by: Failed to query Python interpreter # Caused by: failed to canonicalize path `/Workspace/abcd/.venv/bin/python3`: Invalid cross-device link (os error 18) -SkipOnDbr = true +# Thus we cannot run this test on DBR. Ignore = [ ".databricks", diff --git a/acceptance/bundle/resources/alerts/basic/out.test.toml b/acceptance/bundle/resources/alerts/basic/out.test.toml index d560f1de04..abf6bb2d5a 100644 --- a/acceptance/bundle/resources/alerts/basic/out.test.toml +++ b/acceptance/bundle/resources/alerts/basic/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false +RunsOnDbr = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/basic/test.toml b/acceptance/bundle/resources/alerts/basic/test.toml index 12380e0b69..7ed7df78d7 100644 --- a/acceptance/bundle/resources/alerts/basic/test.toml +++ b/acceptance/bundle/resources/alerts/basic/test.toml @@ -3,6 +3,10 @@ Cloud = false RecordRequests = false Ignore = [".databricks"] +# known_failures.txt in ciconfig is not respected yet. This test is broken due to +# workspace limits so we do not run it on DBR. +RunsOnDbr = false + # Alert tests timeout during bundle deploy (hang at file upload for 50+ minutes). # Use aggressive 5-minute timeout until the issue is resolved. # See: https://github.com/databricks/cli/issues/4221 diff --git a/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml b/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml index f474b1b917..0ebfd0a96b 100644 --- a/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml @@ -1,5 +1,6 @@ Local = false Cloud = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/data_security_mode/test.toml b/acceptance/bundle/resources/clusters/deploy/data_security_mode/test.toml index d075c79606..d0aefc3fcf 100644 --- a/acceptance/bundle/resources/clusters/deploy/data_security_mode/test.toml +++ b/acceptance/bundle/resources/clusters/deploy/data_security_mode/test.toml @@ -1,6 +1,7 @@ Local = false Cloud = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml", diff --git a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml index f474b1b917..0ebfd0a96b 100644 --- a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml @@ -1,5 +1,6 @@ Local = false Cloud = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/simple/test.toml b/acceptance/bundle/resources/clusters/deploy/simple/test.toml index d075c79606..d0aefc3fcf 100644 --- a/acceptance/bundle/resources/clusters/deploy/simple/test.toml +++ b/acceptance/bundle/resources/clusters/deploy/simple/test.toml @@ -1,6 +1,7 @@ Local = false Cloud = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml", diff --git a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml index e26b67058a..2f71d08ba8 100644 --- a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml +++ b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml @@ -1,6 +1,7 @@ Local = false Cloud = true CloudSlow = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/run/spark_python_task/test.toml b/acceptance/bundle/resources/clusters/run/spark_python_task/test.toml index 954fa0ee01..927221d715 100644 --- a/acceptance/bundle/resources/clusters/run/spark_python_task/test.toml +++ b/acceptance/bundle/resources/clusters/run/spark_python_task/test.toml @@ -1,6 +1,7 @@ Local = false CloudSlow = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml", diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml index 87248584bc..8b01f72900 100644 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresWarehouse = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/test.toml b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/test.toml index a86b47bf5f..819b410389 100644 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/test.toml +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/test.toml @@ -2,6 +2,7 @@ Local = true Cloud = true RequiresWarehouse = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml", diff --git a/acceptance/bundle/resources/dashboards/destroy/out.test.toml b/acceptance/bundle/resources/dashboards/destroy/out.test.toml index 87248584bc..8b01f72900 100644 --- a/acceptance/bundle/resources/dashboards/destroy/out.test.toml +++ b/acceptance/bundle/resources/dashboards/destroy/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresWarehouse = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/destroy/test.toml b/acceptance/bundle/resources/dashboards/destroy/test.toml index a86b47bf5f..819b410389 100644 --- a/acceptance/bundle/resources/dashboards/destroy/test.toml +++ b/acceptance/bundle/resources/dashboards/destroy/test.toml @@ -2,6 +2,7 @@ Local = true Cloud = true RequiresWarehouse = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml", diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml index a50e6a7eed..ed27be1295 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml +++ b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml @@ -1,6 +1,7 @@ Local = false Cloud = true RequiresWarehouse = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/test.toml b/acceptance/bundle/resources/dashboards/generate_inplace/test.toml index 61a6c889cd..731f8ff31b 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/test.toml +++ b/acceptance/bundle/resources/dashboards/generate_inplace/test.toml @@ -1,6 +1,7 @@ Cloud = true Local = false RecordRequests = false +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml b/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml index 87248584bc..8b01f72900 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml +++ b/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresWarehouse = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/nested-folders/test.toml b/acceptance/bundle/resources/dashboards/nested-folders/test.toml index 2e782e5fd8..3ca6ec1011 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/test.toml +++ b/acceptance/bundle/resources/dashboards/nested-folders/test.toml @@ -3,6 +3,7 @@ Local = true Cloud = true RequiresWarehouse = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml", diff --git a/acceptance/bundle/resources/dashboards/simple/out.test.toml b/acceptance/bundle/resources/dashboards/simple/out.test.toml index 87248584bc..8b01f72900 100644 --- a/acceptance/bundle/resources/dashboards/simple/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresWarehouse = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/simple/test.toml b/acceptance/bundle/resources/dashboards/simple/test.toml index a86b47bf5f..819b410389 100644 --- a/acceptance/bundle/resources/dashboards/simple/test.toml +++ b/acceptance/bundle/resources/dashboards/simple/test.toml @@ -2,6 +2,7 @@ Local = true Cloud = true RequiresWarehouse = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml", diff --git a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml index 87248584bc..8b01f72900 100644 --- a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresWarehouse = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/test.toml b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/test.toml index 4ff4567314..c1898ee881 100644 --- a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/test.toml +++ b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/test.toml @@ -2,6 +2,7 @@ Local = true Cloud = true RequiresWarehouse = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml", diff --git a/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml b/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml index 87248584bc..8b01f72900 100644 --- a/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresWarehouse = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/simple_syncroot/test.toml b/acceptance/bundle/resources/dashboards/simple_syncroot/test.toml index 4ff4567314..c1898ee881 100644 --- a/acceptance/bundle/resources/dashboards/simple_syncroot/test.toml +++ b/acceptance/bundle/resources/dashboards/simple_syncroot/test.toml @@ -2,6 +2,7 @@ Local = true Cloud = true RequiresWarehouse = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml", diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml index 87248584bc..8b01f72900 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresWarehouse = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/test.toml b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/test.toml index a86b47bf5f..819b410389 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/test.toml +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/test.toml @@ -2,6 +2,7 @@ Local = true Cloud = true RequiresWarehouse = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml", diff --git a/acceptance/bundle/resources/database_catalogs/basic/out.test.toml b/acceptance/bundle/resources/database_catalogs/basic/out.test.toml index 8d2e954f48..5b15f017db 100644 --- a/acceptance/bundle/resources/database_catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/database_catalogs/basic/out.test.toml @@ -2,6 +2,7 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +RunsOnDbr = true [CloudEnvs] gcp = false diff --git a/acceptance/bundle/resources/database_catalogs/basic/test.toml b/acceptance/bundle/resources/database_catalogs/basic/test.toml index 8540a3ae8a..8d789f1c2c 100644 --- a/acceptance/bundle/resources/database_catalogs/basic/test.toml +++ b/acceptance/bundle/resources/database_catalogs/basic/test.toml @@ -5,3 +5,4 @@ Cloud = true CloudSlow = true RecordRequests = false +RunsOnDbr = true diff --git a/acceptance/bundle/resources/experiments/basic/out.test.toml b/acceptance/bundle/resources/experiments/basic/out.test.toml index 01ed6822af..a9766d99c9 100644 --- a/acceptance/bundle/resources/experiments/basic/out.test.toml +++ b/acceptance/bundle/resources/experiments/basic/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/experiments/basic/test.toml b/acceptance/bundle/resources/experiments/basic/test.toml index a29b360f36..89f6a633e9 100644 --- a/acceptance/bundle/resources/experiments/basic/test.toml +++ b/acceptance/bundle/resources/experiments/basic/test.toml @@ -1,6 +1,7 @@ Cloud = true Local = true RecordRequests = false +RunsOnDbr = true [[Repls]] Old = '\d{3,}' diff --git a/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml b/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml index 01ed6822af..a9766d99c9 100644 --- a/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml +++ b/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/double-underscore-keys/test.toml b/acceptance/bundle/resources/jobs/double-underscore-keys/test.toml index 0c61b733d7..c469454658 100644 --- a/acceptance/bundle/resources/jobs/double-underscore-keys/test.toml +++ b/acceptance/bundle/resources/jobs/double-underscore-keys/test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml", diff --git a/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml b/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml index 01ed6822af..a9766d99c9 100644 --- a/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml +++ b/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/fail-on-active-runs/test.toml b/acceptance/bundle/resources/jobs/fail-on-active-runs/test.toml index 0c61b733d7..c469454658 100644 --- a/acceptance/bundle/resources/jobs/fail-on-active-runs/test.toml +++ b/acceptance/bundle/resources/jobs/fail-on-active-runs/test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml", diff --git a/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml b/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml index f474b1b917..0ebfd0a96b 100644 --- a/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml +++ b/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml @@ -1,5 +1,6 @@ Local = false Cloud = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/no-git-provider/test.toml b/acceptance/bundle/resources/jobs/no-git-provider/test.toml index b78de06d59..ca29b49668 100644 --- a/acceptance/bundle/resources/jobs/no-git-provider/test.toml +++ b/acceptance/bundle/resources/jobs/no-git-provider/test.toml @@ -2,3 +2,4 @@ Local = false Cloud = true RecordRequests = false +RunsOnDbr = true diff --git a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml index f474b1b917..0ebfd0a96b 100644 --- a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml +++ b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml @@ -1,5 +1,6 @@ Local = false Cloud = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/shared-root-path/test.toml b/acceptance/bundle/resources/jobs/shared-root-path/test.toml index bb36485bba..aec9368771 100644 --- a/acceptance/bundle/resources/jobs/shared-root-path/test.toml +++ b/acceptance/bundle/resources/jobs/shared-root-path/test.toml @@ -1,6 +1,7 @@ Local = false Cloud = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml", diff --git a/acceptance/bundle/resources/models/basic/out.test.toml b/acceptance/bundle/resources/models/basic/out.test.toml index 01ed6822af..a9766d99c9 100644 --- a/acceptance/bundle/resources/models/basic/out.test.toml +++ b/acceptance/bundle/resources/models/basic/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/models/basic/test.toml b/acceptance/bundle/resources/models/basic/test.toml index eaf15c1932..3ce090441c 100644 --- a/acceptance/bundle/resources/models/basic/test.toml +++ b/acceptance/bundle/resources/models/basic/test.toml @@ -1,3 +1,4 @@ Cloud = true Local = true RecordRequests = false +RunsOnDbr = true diff --git a/acceptance/bundle/resources/permissions/factcheck/out.test.toml b/acceptance/bundle/resources/permissions/factcheck/out.test.toml index 69e2e2028f..746cd40b8c 100644 --- a/acceptance/bundle/resources/permissions/factcheck/out.test.toml +++ b/acceptance/bundle/resources/permissions/factcheck/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true CloudSlow = true +RunsOnDbr = true [CloudEnvs] gcp = false diff --git a/acceptance/bundle/resources/permissions/factcheck/test.toml b/acceptance/bundle/resources/permissions/factcheck/test.toml index b85832beff..e28d280b8a 100644 --- a/acceptance/bundle/resources/permissions/factcheck/test.toml +++ b/acceptance/bundle/resources/permissions/factcheck/test.toml @@ -1,6 +1,7 @@ Local = true CloudSlow = true RecordRequests = false +RunsOnDbr = true # I get this error, not sure why: # -=== Since we take the most recent level, IS_OWNER is lost, which results in the error due to lack of owner defined diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml index 99957c7287..626b7427cf 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml @@ -1,5 +1,6 @@ Local = false Cloud = true +RunsOnDbr = false [CloudEnvs] azure = false diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/test.toml index 9d3d6e6798..b6d371ec60 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/test.toml @@ -3,5 +3,6 @@ Cloud = true RecordRequests = false CloudEnvs.gcp = false CloudEnvs.azure = false +RunsOnDbr = false # Requires TEST_SP_TOKEN to be set which is not available on DBR Ignore = ["take_ownership.json", "databricks.yml"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml index 99957c7287..626b7427cf 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml @@ -1,5 +1,6 @@ Local = false Cloud = true +RunsOnDbr = false [CloudEnvs] azure = false diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/test.toml index 9d3d6e6798..b6d371ec60 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/test.toml @@ -3,5 +3,6 @@ Cloud = true RecordRequests = false CloudEnvs.gcp = false CloudEnvs.azure = false +RunsOnDbr = false # Requires TEST_SP_TOKEN to be set which is not available on DBR Ignore = ["take_ownership.json", "databricks.yml"] diff --git a/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml b/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml index 01ed6822af..a9766d99c9 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/auto-approve/test.toml b/acceptance/bundle/resources/pipelines/auto-approve/test.toml index 42d8ebd57e..1cf3c3c72c 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/test.toml +++ b/acceptance/bundle/resources/pipelines/auto-approve/test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RecordRequests = false +RunsOnDbr = true Ignore = [ "databricks.yml" diff --git a/acceptance/bundle/resources/pipelines/recreate/out.test.toml b/acceptance/bundle/resources/pipelines/recreate/out.test.toml index d61c11e25c..8d6b9baeb5 100644 --- a/acceptance/bundle/resources/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/recreate/test.toml b/acceptance/bundle/resources/pipelines/recreate/test.toml index a50176b151..b8d5d703d3 100644 --- a/acceptance/bundle/resources/pipelines/recreate/test.toml +++ b/acceptance/bundle/resources/pipelines/recreate/test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RecordRequests = false +RunsOnDbr = true RequiresUnityCatalog = true Ignore = [ diff --git a/acceptance/bundle/resources/registered_models/basic/out.test.toml b/acceptance/bundle/resources/registered_models/basic/out.test.toml index d61c11e25c..8d6b9baeb5 100644 --- a/acceptance/bundle/resources/registered_models/basic/out.test.toml +++ b/acceptance/bundle/resources/registered_models/basic/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/registered_models/basic/test.toml b/acceptance/bundle/resources/registered_models/basic/test.toml index fe8dde32bd..33508ef327 100644 --- a/acceptance/bundle/resources/registered_models/basic/test.toml +++ b/acceptance/bundle/resources/registered_models/basic/test.toml @@ -1,4 +1,5 @@ Cloud = true Local = true RecordRequests = false +RunsOnDbr = true RequiresUnityCatalog = true diff --git a/acceptance/bundle/resources/schemas/auto-approve/out.test.toml b/acceptance/bundle/resources/schemas/auto-approve/out.test.toml index d61c11e25c..8d6b9baeb5 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/schemas/auto-approve/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/schemas/auto-approve/test.toml b/acceptance/bundle/resources/schemas/auto-approve/test.toml index 9f38ea0e49..150da453b7 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/test.toml +++ b/acceptance/bundle/resources/schemas/auto-approve/test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RecordRequests = false +RunsOnDbr = true RequiresUnityCatalog = true Ignore = [ diff --git a/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml b/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml index 86511f4e0a..b9c4b0e467 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +RunsOnDbr = true [CloudEnvs] gcp = false diff --git a/acceptance/bundle/resources/secret_scopes/permissions-collapse/test.toml b/acceptance/bundle/resources/secret_scopes/permissions-collapse/test.toml index fc70446234..1574268725 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions-collapse/test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions-collapse/test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RecordRequests = false +RunsOnDbr = true IsServicePrincipal = true # The current user on GCP is not a service principal. This causes diverging output since we set permissions for the current user. Thus we skip this test on GCP. diff --git a/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml b/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml index a888431266..b426ff341a 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +RunsOnDbr = true [CloudEnvs] gcp = false diff --git a/acceptance/bundle/resources/secret_scopes/permissions/test.toml b/acceptance/bundle/resources/secret_scopes/permissions/test.toml index a308726324..1e9000d21e 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions/test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions/test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RecordRequests = false +RunsOnDbr = true # The current user on GCP is not a service principal. This causes diverging output since we set permissions for the current user. Thus we skip this test on GCP. CloudEnvs.gcp = false diff --git a/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml b/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml index c86de121de..a9ae49e264 100644 --- a/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml +++ b/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +RunsOnDbr = false [CloudEnvs] gcp = false diff --git a/acceptance/bundle/resources/synced_database_tables/basic/test.toml b/acceptance/bundle/resources/synced_database_tables/basic/test.toml index d2bbdeaa58..d41d9b917c 100644 --- a/acceptance/bundle/resources/synced_database_tables/basic/test.toml +++ b/acceptance/bundle/resources/synced_database_tables/basic/test.toml @@ -4,6 +4,10 @@ Badness = "post deployment, bundle summary should print actual name that is full RecordRequests = false +# The test fails with this today: +# Error: cannot create resources.synced_database_tables.my_synced_table: Cannot create more than 20 synced database table(s) per source table. (400 BAD_REQUEST) +RunsOnDbr = false + [[Repls]] # clean up ?o= suffix after URL since not all workspaces have that Old = '\?o=\[(NUMID|ALPHANUMID)\]' diff --git a/acceptance/bundle/resources/volumes/recreate/out.test.toml b/acceptance/bundle/resources/volumes/recreate/out.test.toml index 7190c9b30b..6acc5cec9d 100644 --- a/acceptance/bundle/resources/volumes/recreate/out.test.toml +++ b/acceptance/bundle/resources/volumes/recreate/out.test.toml @@ -1,6 +1,7 @@ Local = false Cloud = true RequiresUnityCatalog = true +RunsOnDbr = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/recreate/test.toml b/acceptance/bundle/resources/volumes/recreate/test.toml index d3d2191055..d02128672c 100644 --- a/acceptance/bundle/resources/volumes/recreate/test.toml +++ b/acceptance/bundle/resources/volumes/recreate/test.toml @@ -1,6 +1,7 @@ Local = false Cloud = true RecordRequests = false +RunsOnDbr = true RequiresUnityCatalog = true Ignore = [ diff --git a/acceptance/dbr_runner.py b/acceptance/dbr_runner.py new file mode 100644 index 0000000000..ba646b183e --- /dev/null +++ b/acceptance/dbr_runner.py @@ -0,0 +1,356 @@ +# Databricks notebook source + +# This notebook runs CLI cloud acceptance tests on a DBR cluster. +# It is meant to be submitted as a job task from the TestDbrAcceptance* tests. +# +# The notebook expects the following parameters: +# - archive_path: Path to the archive.tar.gz file in the workspace +# - cloud_env: Cloud environment (e.g., "aws", "azure", "gcp") +# - test_filter: Optional regex filter for test names (e.g., "bundle/generate") +# - test_default_warehouse_id: Default SQL warehouse ID +# - test_default_cluster_id: Default cluster ID +# - test_instance_pool_id: Instance pool ID +# - test_metastore_id: Unity Catalog metastore ID +# - test_sp_application_id: Service principal application ID +# - debug_log_path: Workspace path for the debug log file (pre-created by the test runner) + +import os +import platform +import subprocess +import sys +import tarfile +from pathlib import Path + +# COMMAND ---------- + + +def get_workspace_client(): + """Get the workspace client using dbruntime context.""" + from databricks.sdk import WorkspaceClient + + return WorkspaceClient() + + +def get_auth_token(): + """Get the authentication token from the notebook context.""" + from dbruntime.databricks_repl_context import get_context + + ctx = get_context() + return ctx.apiToken + + +def get_workspace_url(): + """Get the workspace URL from spark config.""" + url = spark.conf.get("spark.databricks.workspaceUrl") + if not url.startswith("https://"): + url = "https://" + url + return url + + +def get_workspace_debug_log_path(debug_log_path: str) -> str: + """Convert workspace API path to FUSE path for direct file access.""" + # The debug_log_path is an API path (e.g., /Users/user@example.com/dbr_acceptance_tests/debug.log) + # We need to convert it to a FUSE path by adding /Workspace prefix + if debug_log_path.startswith("/Workspace"): + return debug_log_path + return f"/Workspace{debug_log_path}" + + +# COMMAND ---------- + + +def extract_archive(archive_path: str) -> Path: + """Extract the archive to the home directory.""" + import uuid + + home = Path.home() + unique_id = uuid.uuid4().hex[:8] + extract_dir = home / f"acceptance_test_{unique_id}" + + extract_dir.mkdir(parents=True, exist_ok=True) + + print(f"Extracting archive from {archive_path} to {extract_dir}") + + # The archive_path may be a FUSE path (with /Workspace prefix) or an API path. + # The workspace API expects paths without the /Workspace prefix. + api_path = archive_path + if api_path.startswith("/Workspace"): + api_path = api_path[len("/Workspace") :] + + w = get_workspace_client() + + # Download the archive to a temp location (use unique name to avoid conflicts) + temp_archive = home / f"archive_{unique_id}.tar.gz" + with open(temp_archive, "wb") as f: + resp = w.workspace.download(api_path) + f.write(resp.read()) + + # Extract the archive + with tarfile.open(temp_archive, "r:gz") as tar: + tar.extractall(path=extract_dir) + + # Clean up temp archive (use missing_ok for FUSE filesystem quirks) + temp_archive.unlink(missing_ok=True) + + print(f"Archive extracted to {extract_dir}") + return extract_dir + + +# COMMAND ---------- + + +def setup_environment(extract_dir: Path) -> dict: + """Set up the environment variables for running tests.""" + env = os.environ.copy() + + # Determine architecture + machine = platform.machine().lower() + if machine in ("x86_64", "amd64"): + arch = "amd64" + elif machine in ("aarch64", "arm64"): + arch = "arm64" + else: + raise ValueError(f"Unsupported architecture: {machine}") + + print(f"Detected architecture: {arch}") + + # Set up PATH to include our binaries + bin_dir = extract_dir / "bin" / arch + go_bin_dir = bin_dir / "go" / "bin" + + path_entries = [ + str(go_bin_dir), + str(bin_dir), + env.get("PATH", ""), + ] + env["PATH"] = os.pathsep.join(path_entries) + + # Set GOROOT for the Go installation + env["GOROOT"] = str(bin_dir / "go") + + # Set up authentication + env["DATABRICKS_HOST"] = get_workspace_url() + env["DATABRICKS_TOKEN"] = get_auth_token() + + # Disable telemetry for tests. Just for a marginally faster test execution. + env["DATABRICKS_CLI_TELEMETRY_DISABLED"] = "true" + + return env + + +# COMMAND ---------- + + +class TestResult: + """Holds the result of running tests.""" + + def __init__(self, return_code: int, stdout: str, stderr: str): + self.return_code = return_code + self.stdout = stdout + self.stderr = stderr + + +def run_tests( + extract_dir: Path, + env: dict, + test_filter: str = "", + cloud_env: str = "", + test_default_warehouse_id: str = "", + test_default_cluster_id: str = "", + test_instance_pool_id: str = "", + test_metastore_id: str = "", + test_user_email: str = "", + test_sp_application_id: str = "", + debug_log_path: str = "", +) -> TestResult: + """Run CLI cloud acceptance tests.""" + cli_dir = extract_dir / "cli" + + # Get the workspace FUSE path for the pre-created debug log file + workspace_log_path = get_workspace_debug_log_path(debug_log_path) + + cmd = [ + "go", + "test", + "./acceptance", + "-timeout", + "14400s", + "-v", + "-workspace-tmp-dir", + ] + + if test_filter: + cmd.extend(["-run", f"^TestAccept/{test_filter}"]) + else: + cmd.extend(["-run", "^TestAccept"]) + + # Cloud tests: run with CLOUD_ENV set and workspace access + env["CLOUD_ENV"] = cloud_env + # Only tests using direct deployment are run on DBR. + # Terraform based tests are out of scope for DBR. + env["ENVFILTER"] = "DATABRICKS_BUNDLE_ENGINE=direct" + + if test_default_warehouse_id: + env["TEST_DEFAULT_WAREHOUSE_ID"] = test_default_warehouse_id + if test_default_cluster_id: + env["TEST_DEFAULT_CLUSTER_ID"] = test_default_cluster_id + if test_instance_pool_id: + env["TEST_INSTANCE_POOL_ID"] = test_instance_pool_id + if test_metastore_id: + env["TEST_METASTORE_ID"] = test_metastore_id + if test_user_email: + env["TEST_USER_EMAIL"] = test_user_email + if test_sp_application_id: + env["TEST_SP_APPLICATION_ID"] = test_sp_application_id + + # Write header to debug log (write directly to workspace via FUSE) + with open(workspace_log_path, "w") as log_file: + log_file.write(f"Command: {' '.join(cmd)}\n") + log_file.write(f"Working directory: {cli_dir}\n") + log_file.write(f"CLOUD_ENV: {cloud_env}\n") + log_file.write(f"Test filter: {test_filter or '(all tests)'}\n") + log_file.write(f"PATH: {env.get('PATH', '')[:200]}...\n") + log_file.write(f"GOROOT: {env.get('GOROOT', '')}\n") + log_file.write("\n" + "=" * 60 + "\n") + log_file.write("TEST OUTPUT:\n") + log_file.write("=" * 60 + "\n") + print(f"Running command: {' '.join(cmd)}") + print(f"Working directory: {cli_dir}") + print(f"CLOUD_ENV: {cloud_env}") + print(f"Test filter: {test_filter or '(all tests)'}") + print(f"Go version: ", end="", flush=True) + + # Print Go version for debugging + subprocess.run(["go", "version"], cwd=cli_dir, env=env) + + print(f"PATH: {env.get('PATH', '')[:200]}...") + print(f"GOROOT: {env.get('GOROOT', '')}") + + print("\n" + "=" * 60) + print("TEST OUTPUT (streaming):") + print("=" * 60, flush=True) + + # Run tests with streaming output using Popen. + # Merge stderr into stdout for simpler streaming. + process = subprocess.Popen( + cmd, + cwd=cli_dir, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, # Line buffered + ) + + # Collect output while streaming it and write to debug log + output_lines = [] + with open(workspace_log_path, "a") as log_file: + for line in process.stdout: + print(line, end="", flush=True) + output_lines.append(line) + log_file.write(line) + log_file.flush() + + process.wait() + stdout = "".join(output_lines) + + # Write footer to debug log + with open(workspace_log_path, "a") as log_file: + log_file.write("\n" + "=" * 60 + "\n") + log_file.write(f"Tests finished with return code: {process.returncode}\n") + log_file.write("=" * 60 + "\n") + + print("\n" + "=" * 60) + print(f"Tests finished with return code: {process.returncode}") + print("=" * 60) + + return TestResult(process.returncode, stdout, "") + + +# COMMAND ---------- + + +def main(): + """Main entry point for the notebook.""" + # Get parameters from widgets + dbutils.widgets.text("archive_path", "") + dbutils.widgets.text("cloud_env", "") + dbutils.widgets.text("test_filter", "") + dbutils.widgets.text("test_default_warehouse_id", "") + dbutils.widgets.text("test_default_cluster_id", "") + dbutils.widgets.text("test_instance_pool_id", "") + dbutils.widgets.text("test_metastore_id", "") + dbutils.widgets.text("test_user_email", "") + dbutils.widgets.text("test_sp_application_id", "") + dbutils.widgets.text("debug_log_path", "") + + archive_path = dbutils.widgets.get("archive_path") + cloud_env = dbutils.widgets.get("cloud_env") + test_filter = dbutils.widgets.get("test_filter") + test_default_warehouse_id = dbutils.widgets.get("test_default_warehouse_id") + test_default_cluster_id = dbutils.widgets.get("test_default_cluster_id") + test_instance_pool_id = dbutils.widgets.get("test_instance_pool_id") + test_metastore_id = dbutils.widgets.get("test_metastore_id") + test_user_email = dbutils.widgets.get("test_user_email") + test_sp_application_id = dbutils.widgets.get("test_sp_application_id") + debug_log_path = dbutils.widgets.get("debug_log_path") + + if not archive_path: + raise ValueError("archive_path parameter is required") + if not cloud_env: + raise ValueError("cloud_env parameter is required") + if not debug_log_path: + raise ValueError("debug_log_path parameter is required") + + print("=" * 60) + print("DBR Cloud Test Runner") + print("=" * 60) + print(f"Archive path: {archive_path}") + print(f"Cloud environment: {cloud_env}") + print(f"Test filter: {test_filter or '(none)'}") + print("=" * 60) + + # Extract the archive + extract_dir = extract_archive(archive_path) + + # Set up the environment + env = setup_environment(extract_dir) + + # Run the tests + result = run_tests( + extract_dir=extract_dir, + env=env, + test_filter=test_filter, + cloud_env=cloud_env, + test_default_warehouse_id=test_default_warehouse_id, + test_default_cluster_id=test_default_cluster_id, + test_instance_pool_id=test_instance_pool_id, + test_metastore_id=test_metastore_id, + test_user_email=test_user_email, + test_sp_application_id=test_sp_application_id, + debug_log_path=debug_log_path, + ) + + print("=" * 60) + print(f"Tests completed with return code: {result.return_code}") + print("=" * 60) + + if result.return_code != 0: + # Include relevant output in the exception for debugging + stdout_preview = result.stdout[-100000:] if result.stdout else "(no stdout)" + stderr_preview = result.stderr[-100000:] if result.stderr else "(no stderr)" + error_msg = f"""Cloud tests failed with return code {result.return_code} + +=== STDOUT (last 100000 chars) === +{stdout_preview} + +=== STDERR (last 100000 chars) === +{stderr_preview} +""" + raise Exception(error_msg) + + +# COMMAND ---------- + +if __name__ == "__main__": + main() diff --git a/acceptance/dbr_test.go b/acceptance/dbr_test.go index ab0aea32ba..1a439e1fa4 100644 --- a/acceptance/dbr_test.go +++ b/acceptance/dbr_test.go @@ -2,19 +2,28 @@ package acceptance_test import ( "context" + "encoding/base64" "fmt" + "os" + "path" + "path/filepath" "testing" "time" + "github.com/databricks/cli/internal/testarchive" "github.com/databricks/cli/libs/filer" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func workspaceTmpDir(ctx context.Context, t *testing.T) (*databricks.WorkspaceClient, filer.Filer, string) { +// workspaceTmpDir creates a temporary directory in the workspace for running tests. +// This is used by acceptance tests when running with the -workspace-tmp-dir flag. +func workspaceTmpDir(ctx context.Context, t *testing.T) string { w, err := databricks.NewWorkspaceClient() require.NoError(t, err) @@ -22,7 +31,7 @@ func workspaceTmpDir(ctx context.Context, t *testing.T) (*databricks.WorkspaceCl require.NoError(t, err) timestamp := time.Now().Format("2006-01-02T15:04:05Z") - tmpDir := fmt.Sprintf( + path := fmt.Sprintf( "/Workspace/Users/%s/acceptance/%s/%s", currentUser.UserName, timestamp, @@ -30,18 +39,326 @@ func workspaceTmpDir(ctx context.Context, t *testing.T) (*databricks.WorkspaceCl ) t.Cleanup(func() { - err := w.Workspace.Delete(ctx, workspace.Delete{ - Path: tmpDir, - Recursive: true, - }) + // Use FUSE for cleanup to ensure proper operation ordering. + // Mixing FUSE (for writes) with API (for delete) can cause + // AsyncFlushFailedException because FUSE may have pending + // async writes that try to flush after API has deleted the directory. + err := os.RemoveAll(path) assert.NoError(t, err) }) - err = w.Workspace.MkdirsByPath(ctx, tmpDir) + // Create the directory using FUSE directly via os.MkdirAll. + // This ensures the directory is immediately visible through the FUSE mount. + // Using the SDK's MkdirsByPath can cause eventual consistency issues where + // FUSE doesn't see the directory immediately after creation. + err = os.MkdirAll(path, 0o755) + require.NoError(t, err, "Failed to create directory %s via FUSE", path) + + // Return the FUSE path for local file operations. + return path +} + +// dbrTestConfig holds the configuration for a DBR test run. +type dbrTestConfig struct { + // cloudTestFilter is a regex filter for cloud acceptance tests (Cloud=true). + // These tests run with CLOUD_ENV set and workspace access. + // If empty, all cloud tests are run. + cloudTestFilter string + + // timeout is the maximum duration to wait for the job to complete. + timeout time.Duration + + // verbose enables detailed output during test setup. + // If false, only essential information is printed. + verbose bool +} + +// setupDbrTestDir creates the test directory and returns the workspace client and filer. +// It returns the API path (without /Workspace prefix) for use with workspace APIs. +func setupDbrTestDir(ctx context.Context, t *testing.T, uniqueID string) (*databricks.WorkspaceClient, filer.Filer, string) { + w, err := databricks.NewWorkspaceClient() + require.NoError(t, err) + + currentUser, err := w.CurrentUser.Me(ctx) + require.NoError(t, err) + + // API path (without /Workspace prefix) for workspace API calls. + apiPath := path.Join("/Users", currentUser.UserName, "dbr-acceptance-test", uniqueID) + + err = w.Workspace.MkdirsByPath(ctx, apiPath) + require.NoError(t, err) + + // Note: We do not cleanup test directories created here. They are kept around + // to enable debugging of failures or analyzing the logs. + // They will automatically be cleaned by the nightly cleanup scripts. + + f, err := filer.NewWorkspaceFilesClient(w, apiPath) + require.NoError(t, err) + + return w, f, apiPath +} + +// buildAndUploadArchive builds the test archive and uploads it to the workspace. +func buildAndUploadArchive(ctx context.Context, t *testing.T, f filer.Filer, verbose bool) string { + // Control testarchive verbosity + testarchive.Verbose = verbose + + // Create temporary directories for the archive + archiveDir := t.TempDir() + binDir := t.TempDir() + + // Get the repo root (parent of acceptance directory) + cwd, err := os.Getwd() + require.NoError(t, err) + repoRoot := filepath.Join(cwd, "..") + + if verbose { + t.Log("Building test archive...") + } + err = testarchive.CreateArchive(archiveDir, binDir, repoRoot) + require.NoError(t, err) + + archivePath := filepath.Join(archiveDir, "archive.tar.gz") + archiveReader, err := os.Open(archivePath) + require.NoError(t, err) + defer archiveReader.Close() + + if verbose { + t.Log("Uploading archive to workspace...") + } + err = f.Write(ctx, "archive.tar.gz", archiveReader) + require.NoError(t, err) + + return "archive.tar.gz" +} + +// uploadRunner uploads the DBR runner notebook to the workspace using workspace.Import. +func uploadRunner(ctx context.Context, t *testing.T, w *databricks.WorkspaceClient, testDir string, verbose bool) string { + cwd, err := os.Getwd() + require.NoError(t, err) + + runnerPath := filepath.Join(cwd, "dbr_runner.py") + runnerContent, err := os.ReadFile(runnerPath) require.NoError(t, err) - f, err := filer.NewWorkspaceFilesClient(w, tmpDir) + notebookPath := path.Join(testDir, "dbr_runner") + + if verbose { + t.Log("Uploading DBR runner notebook...") + } + err = w.Workspace.Import(ctx, workspace.Import{ + Path: notebookPath, + Overwrite: true, + Language: workspace.LanguagePython, + Format: workspace.ImportFormatSource, + Content: base64.StdEncoding.EncodeToString(runnerContent), + }) require.NoError(t, err) - return w, f, tmpDir + return "dbr_runner" +} + +// buildBaseParams builds the common parameters for test tasks. +func buildBaseParams(testDir, archiveName, debugLogPath string) map[string]string { + return map[string]string{ + "archive_path": path.Join(testDir, archiveName), + "cloud_env": os.Getenv("CLOUD_ENV"), + "test_default_warehouse_id": os.Getenv("TEST_DEFAULT_WAREHOUSE_ID"), + "test_default_cluster_id": os.Getenv("TEST_DEFAULT_CLUSTER_ID"), + "test_instance_pool_id": os.Getenv("TEST_INSTANCE_POOL_ID"), + "test_metastore_id": os.Getenv("TEST_METASTORE_ID"), + "test_user_email": os.Getenv("TEST_USER_EMAIL"), + "test_sp_application_id": os.Getenv("TEST_SP_APPLICATION_ID"), + "debug_log_path": debugLogPath, + } +} + +// runDbrTests creates a job and runs it to execute cloud and local acceptance tests on DBR. +func runDbrTests(ctx context.Context, t *testing.T, w *databricks.WorkspaceClient, testDir, archiveName, runnerName string, config dbrTestConfig) { + cloudEnv := os.Getenv("CLOUD_ENV") + if cloudEnv == "" { + t.Fatal("CLOUD_ENV is not set. Please run DBR tests from a CI environment with deco env run.") + } + + currentUser, err := w.CurrentUser.Me(ctx) + require.NoError(t, err) + + // Create debug logs directory + debugLogsDir := path.Join("/Users", currentUser.UserName, "dbr_acceptance_tests") + err = w.Workspace.MkdirsByPath(ctx, debugLogsDir) + require.NoError(t, err) + + // Create an empty debug log file so we can get its URL before the job runs. + // This allows us to provide the URL upfront for users to follow along. + timestamp := time.Now().Format("2006-01-02_15-04-05") + debugLogFileName := fmt.Sprintf("debug-cloud-%s-%s.log", timestamp, uuid.New().String()[:8]) + debugLogPath := path.Join(debugLogsDir, debugLogFileName) + + // Create empty file via workspace API + err = w.Workspace.Import(ctx, workspace.Import{ + Path: debugLogPath, + Overwrite: true, + Format: workspace.ImportFormatAuto, + Content: base64.StdEncoding.EncodeToString([]byte("")), + }) + require.NoError(t, err) + + // Get the file's object ID for the URL + debugLogStatus, err := w.Workspace.GetStatusByPath(ctx, debugLogPath) + require.NoError(t, err) + + // Build cloud test parameters (Cloud=true tests, run with CLOUD_ENV set) + cloudParams := buildBaseParams(testDir, archiveName, debugLogPath) + cloudParams["test_type"] = "cloud" + cloudParams["test_filter"] = config.cloudTestFilter + + jobName := "DBR Tests" + if config.cloudTestFilter != "" { + jobName = fmt.Sprintf("DBR Tests (%s)", config.cloudTestFilter) + } + + // Print summary of what will run + t.Log("") + t.Log("=== DBR Test Run ===") + if config.cloudTestFilter != "" { + t.Logf(" Cloud tests: %s", config.cloudTestFilter) + } else { + t.Log(" Cloud tests: (all)") + } + + notebookPath := path.Join(testDir, runnerName) + + // Create a job (not a one-time run) so we can use MaxRetries on tasks. + // Always use serverless compute. + t.Log(" Cluster: serverless") + createJob := jobs.CreateJob{ + Name: jobName, + Environments: []jobs.JobEnvironment{ + { + EnvironmentKey: "default", + Spec: &compute.Environment{ + EnvironmentVersion: "4", + }, + }, + }, + Tasks: []jobs.Task{ + { + TaskKey: "cloud_tests", + EnvironmentKey: "default", + MaxRetries: 0, + NotebookTask: &jobs.NotebookTask{ + NotebookPath: notebookPath, + BaseParameters: cloudParams, + Source: jobs.SourceWorkspace, + }, + }, + }, + } + + // Create the job + job, err := w.Jobs.Create(ctx, createJob) + require.NoError(t, err) + + // The job is not deleted after the test completes. + // This is to enable debugging of failures or analyzing the logs. + // It will automatically be cleaned by the nightly cleanup scripts. + + // Trigger a run of the job + wait, err := w.Jobs.RunNow(ctx, jobs.RunNow{JobId: job.JobId}) + require.NoError(t, err) + + // Fetch run details immediately to get the URL so users can follow along + runDetails, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: wait.RunId}) + require.NoError(t, err) + + t.Log("") + t.Log("=== Job Started ===") + t.Logf(" Run URL: %s", runDetails.RunPageUrl) + t.Logf(" Debug logs: %s/editor/files/%d", w.Config.Host, debugLogStatus.ObjectId) + t.Log("") + t.Logf("Waiting for completion (timeout: %v)...", config.timeout) + + run, err := wait.GetWithTimeout(config.timeout) + if err != nil { + // Try to fetch the run details for the URL and task output + runDetails, fetchErr := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: wait.RunId}) + if fetchErr == nil { + // Try to get the task output for debugging + for _, task := range runDetails.Tasks { + output, outputErr := w.Jobs.GetRunOutput(ctx, jobs.GetRunOutputRequest{ + RunId: task.RunId, + }) + if outputErr == nil { + if output.Error != "" { + t.Logf("Task %s error: %s", task.TaskKey, output.Error) + } + if output.ErrorTrace != "" { + t.Logf("Task %s error trace:\n%s", task.TaskKey, output.ErrorTrace) + } + } + } + } + require.NoError(t, err) + } + + t.Logf("Job completed. Status: %s", run.State.ResultState) + + // Check if the job succeeded + if run.State.ResultState != jobs.RunResultStateSuccess { + // Try to get the task output for debugging + for _, task := range run.Tasks { + output, outputErr := w.Jobs.GetRunOutput(ctx, jobs.GetRunOutputRequest{ + RunId: task.RunId, + }) + if outputErr == nil && output.Error != "" { + t.Logf("Task %s error: %s", task.TaskKey, output.Error) + } + if outputErr == nil && output.ErrorTrace != "" { + t.Logf("Task %s error trace:\n%s", task.TaskKey, output.ErrorTrace) + } + } + t.Fatalf("Job failed with state: %s. Check the run URL for details: %s", run.State.ResultState, run.RunPageUrl) + } + + t.Log("All tests passed!") +} + +// runDbrAcceptanceTests is the main entry point for running DBR acceptance tests. +func runDbrAcceptanceTests(t *testing.T, config dbrTestConfig) { + ctx := context.Background() + uniqueID := uuid.New().String() + + w, f, testDir := setupDbrTestDir(ctx, t, uniqueID) + if config.verbose { + t.Logf("Test directory: %s", testDir) + } + + archiveName := buildAndUploadArchive(ctx, t, f, config.verbose) + runnerName := uploadRunner(ctx, t, w, testDir, config.verbose) + + runDbrTests(ctx, t, w, testDir, archiveName, runnerName, config) +} + +// TestDbrAcceptance runs all acceptance and integration tests on DBR using serverless compute. +// Only acceptance tests with RunsOnDbr=true in their test.toml will be executed. +// Both test types run in parallel tasks. +// +// Run with: +// +// deco env run -i -n aws-prod-ucws -- go test -v -timeout 4h -run TestDbrAcceptance$ ./acceptance +// OR +// make dbr-test +func TestDbrAcceptance(t *testing.T) { + if os.Getenv("DBR_ENABLED") != "true" { + t.Skip("Skipping DBR test: DBR_ENABLED not set") + } + + if os.Getenv("CLOUD_ENV") == "" { + t.Skip("Skipping DBR test: CLOUD_ENV not set") + } + + runDbrAcceptanceTests(t, dbrTestConfig{ + timeout: 3 * time.Hour, + verbose: os.Getenv("DBR_TEST_VERBOSE") != "", + }) } diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index fbf1ce3807..65fcadfbf3 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -129,8 +129,9 @@ type TestConfig struct { // On CI, we want to increase timeout, to account for slower environment TimeoutCIMultiplier float64 - // If true, skip this test when running on DBR / workspace file system. - SkipOnDbr *bool + // If true, run this test when running on DBR / workspace file system. + // Tests must explicitly opt-in to run on DBR. + RunsOnDbr *bool } type ServerStub struct { @@ -213,9 +214,22 @@ func LoadConfig(t *testing.T, dir string) (TestConfig, string) { result.Ignore = append(result.Ignore, ".cache") result.CompiledIgnoreObject = ignore.CompileIgnoreLines(result.Ignore...) + // Validate incompatible configuration combinations + validateConfig(t, result, strings.Join(configs, ", ")) + return result, strings.Join(configs, ", ") } +// validateConfig checks for incompatible configuration combinations. +func validateConfig(t *testing.T, config TestConfig, configPath string) { + // RunsOnDbr and RecordRequests are incompatible because serverless does not + // allow access to localhost ports, which the test proxy server requires. + if isTruePtr(config.RunsOnDbr) && isTruePtr(config.RecordRequests) { + t.Fatalf("Invalid config %s: RunsOnDbr and RecordRequests cannot both be true. "+ + "Serverless does not allow access to localhost ports, which the test proxy server requires.", configPath) + } +} + func DoLoadConfig(t *testing.T, path string) TestConfig { bytes, err := os.ReadFile(path) require.NoError(t, err, "Failed to read test config %s: %s", path, err) diff --git a/acceptance/internal/materialized_config.go b/acceptance/internal/materialized_config.go index 3233d3907c..9ab0dc18cb 100644 --- a/acceptance/internal/materialized_config.go +++ b/acceptance/internal/materialized_config.go @@ -17,6 +17,7 @@ type MaterializedConfig struct { RequiresUnityCatalog *bool `toml:"RequiresUnityCatalog,omitempty"` RequiresCluster *bool `toml:"RequiresCluster,omitempty"` RequiresWarehouse *bool `toml:"RequiresWarehouse,omitempty"` + RunsOnDbr *bool `toml:"RunsOnDbr,omitempty"` EnvMatrix map[string][]string `toml:"EnvMatrix,omitempty"` } @@ -32,6 +33,7 @@ func GenerateMaterializedConfig(config TestConfig) (string, error) { RequiresUnityCatalog: config.RequiresUnityCatalog, RequiresCluster: config.RequiresCluster, RequiresWarehouse: config.RequiresWarehouse, + RunsOnDbr: config.RunsOnDbr, EnvMatrix: config.EnvMatrix, } diff --git a/bundle/config/mutator/configure_wsfs.go b/bundle/config/mutator/configure_wsfs.go index 110e1a3819..4cd836014e 100644 --- a/bundle/config/mutator/configure_wsfs.go +++ b/bundle/config/mutator/configure_wsfs.go @@ -34,6 +34,16 @@ func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno return nil } + // On serverless (client version 2+), use the native sync root directly via FUSE. + // The FUSE provides capabitilies for both reading and writing notebooks. It also + // is much faster and enables running cloud tests on DBR, since otherwise the tests + // fail with an AsyncFlushError because of the conflict between writing to FUSE + // and via the workspace APIs simultaneously. + v := dbr.GetVersion(ctx) + if v.Type == dbr.ClusterTypeServerless && v.Major >= 2 { + return nil + } + // If so, swap out vfs.Path instance of the sync root with one that // makes all Workspace File System interactions extension aware. p, err := vfs.NewFilerPath(ctx, root, func(path string) (filer.Filer, error) { diff --git a/bundle/config/mutator/configure_wsfs_test.go b/bundle/config/mutator/configure_wsfs_test.go index 6762a446b4..53e1aa2a0d 100644 --- a/bundle/config/mutator/configure_wsfs_test.go +++ b/bundle/config/mutator/configure_wsfs_test.go @@ -2,6 +2,7 @@ package mutator_test import ( "context" + "reflect" "runtime" "testing" @@ -53,13 +54,50 @@ func TestConfigureWSFS_SkipsIfNotRunningOnRuntime(t *testing.T) { assert.Equal(t, originalSyncRoot, b.SyncRoot) } -func TestConfigureWSFS_SwapSyncRoot(t *testing.T) { - b := mockBundleForConfigureWSFS(t, "/Workspace/foo") - originalSyncRoot := b.SyncRoot +func TestConfigureWSFS_DBRVersions(t *testing.T) { + tests := []struct { + name string + version string + expectFUSE bool // true = osPath (uses FUSE), false = filerPath (uses wsfs extension) + }{ + // Serverless client version 2+ should use FUSE directly (osPath) + {"serverless_client_2", "client.2", true}, + {"serverless_client_2_1", "client.2.1", true}, + {"serverless_client_3", "client.3", true}, + {"serverless_client_3_6", "client.3.6", true}, + {"serverless_client_4_9", "client.4.9", true}, + {"serverless_client_4_10", "client.4.10", true}, - ctx := context.Background() - ctx = dbr.MockRuntime(ctx, dbr.Environment{IsDbr: true, Version: "15.4"}) - diags := bundle.Apply(ctx, b, mutator.ConfigureWSFS()) - assert.Empty(t, diags) - assert.NotEqual(t, originalSyncRoot, b.SyncRoot) + // Serverless client version 1 should use wsfs extension client (filerPath) + {"serverless_client_1", "client.1", false}, + {"serverless_client_1_13", "client.1.13", false}, + + // Interactive (non-serverless) versions should use wsfs extension client (filerPath) + {"interactive_15_4", "15.4", false}, + {"interactive_16_3", "16.3", false}, + {"interactive_16_4", "16.4", false}, + {"interactive_17_0", "17.0", false}, + {"interactive_17_1", "17.1", false}, + {"interactive_17_2", "17.2", false}, + {"interactive_17_3", "17.3", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := mockBundleForConfigureWSFS(t, "/Workspace/foo") + + ctx := context.Background() + ctx = dbr.MockRuntime(ctx, dbr.Environment{IsDbr: true, Version: tt.version}) + diags := bundle.Apply(ctx, b, mutator.ConfigureWSFS()) + assert.Empty(t, diags) + + // Check the underlying type of SyncRoot + typeName := reflect.TypeOf(b.SyncRoot).String() + if tt.expectFUSE { + assert.Equal(t, "*vfs.osPath", typeName, "expected osPath (FUSE) for version %s", tt.version) + } else { + assert.Equal(t, "*vfs.filerPath", typeName, "expected filerPath (wsfs extension) for version %s", tt.version) + } + }) + } } diff --git a/internal/testarchive/archive.go b/internal/testarchive/archive.go index f8da8e3ba7..4def785e90 100644 --- a/internal/testarchive/archive.go +++ b/internal/testarchive/archive.go @@ -112,9 +112,9 @@ func CreateArchive(archiveDir, binDir, repoRoot string) error { // TODO: Serverless clusters do not support arm64 yet. // Enable ARM64 once serverless clusters support it. - // goDownloader{arch: "arm64", binDir: binDir}, - // uvDownloader{arch: "arm64", binDir: binDir}, - // jqDownloader{arch: "arm64", binDir: binDir}, + // GoDownloader{Arch: "arm64", BinDir: binDir, RepoRoot: repoRoot}, + // UvDownloader{Arch: "arm64", BinDir: binDir}, + // JqDownloader{Arch: "arm64", BinDir: binDir}, } for _, downloader := range downloaders { @@ -135,7 +135,7 @@ func CreateArchive(archiveDir, binDir, repoRoot string) error { } totalFiles := len(gitFiles) + len(binFiles) - fmt.Printf("Found %d git-tracked files and %d downloaded files (%d total)\n", + logf("Found %d git-tracked files and %d downloaded files (%d total)\n", len(gitFiles), len(binFiles), totalFiles) // Create archive directory if it doesn't exist @@ -158,13 +158,13 @@ func CreateArchive(archiveDir, binDir, repoRoot string) error { tarWriter := tar.NewWriter(gzWriter) defer tarWriter.Close() - fmt.Printf("Creating archive %s...\n", archivePath) + logf("Creating archive %s...\n", archivePath) // Add git-tracked files to the archive for _, file := range gitFiles { err := addFileToArchive(tarWriter, filepath.Join(repoRoot, file), filepath.Join("cli", file)) if err != nil { - fmt.Printf("Warning: failed to add git file %s: %v\n", file, err) + logf("Warning: failed to add git file %s: %v\n", file, err) } } @@ -172,7 +172,7 @@ func CreateArchive(archiveDir, binDir, repoRoot string) error { for _, file := range binFiles { err := addFileToArchive(tarWriter, filepath.Join(binDir, file), filepath.Join("bin", file)) if err != nil { - fmt.Printf("Warning: failed to add downloaded file %s: %v\n", file, err) + logf("Warning: failed to add downloaded file %s: %v\n", file, err) } } @@ -181,6 +181,6 @@ func CreateArchive(archiveDir, binDir, repoRoot string) error { return fmt.Errorf("failed to stat archive: %w", err) } - fmt.Printf("✅ Successfully created comprehensive archive. Archive size: %.1f MB\n", float64(stat.Size())/(1024*1024)) + logf("✅ Archive created (%.1f MB)\n", float64(stat.Size())/(1024*1024)) return nil } diff --git a/internal/testarchive/utils.go b/internal/testarchive/utils.go index 2d76f13530..d450d92ff1 100644 --- a/internal/testarchive/utils.go +++ b/internal/testarchive/utils.go @@ -3,6 +3,8 @@ package testarchive import ( "archive/tar" "compress/gzip" + "crypto/sha256" + "encoding/hex" "fmt" "io" "net/http" @@ -10,7 +12,55 @@ import ( "path/filepath" ) +// Verbose controls whether detailed progress is printed. +// Set to true for detailed output, false for quiet operation. +var Verbose = true + +// logf prints a message only if Verbose is true. +func logf(format string, args ...any) { + if Verbose { + fmt.Printf(format, args...) + } +} + +// getCacheDir returns the cache directory for downloads. +// It uses ~/.cache/testarchive by default. +func getCacheDir() (string, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("failed to get home directory: %w", err) + } + cacheDir := filepath.Join(homeDir, ".cache", "testarchive") + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return "", fmt.Errorf("failed to create cache directory: %w", err) + } + return cacheDir, nil +} + +// getCacheKey returns a stable cache key for a URL. +func getCacheKey(url string) string { + hash := sha256.Sum256([]byte(url)) + return hex.EncodeToString(hash[:8]) +} + func downloadFile(url, outputPath string) error { + // Check if we have this file cached + cacheDir, err := getCacheDir() + if err == nil { + cacheKey := getCacheKey(url) + ext := filepath.Ext(outputPath) + if ext == "" { + ext = ".bin" + } + cachedFile := filepath.Join(cacheDir, cacheKey+ext) + if _, err := os.Stat(cachedFile); err == nil { + // Cache hit - copy from cache + logf("Using cached file for %s\n", url) + return copyFile(cachedFile, outputPath) + } + } + + // Download the file resp, err := http.Get(url) if err != nil { return fmt.Errorf("failed to download: %w", err) @@ -27,18 +77,48 @@ func downloadFile(url, outputPath string) error { } defer outFile.Close() - fmt.Printf("Downloading %s to %s\n", url, outputPath) + logf("Downloading %s to %s\n", url, outputPath) _, err = io.Copy(outFile, resp.Body) if err != nil { return fmt.Errorf("failed to save file: %w", err) } + // Cache the downloaded file for future use + if cacheDir != "" { + cacheKey := getCacheKey(url) + ext := filepath.Ext(outputPath) + if ext == "" { + ext = ".bin" + } + cachedFile := filepath.Join(cacheDir, cacheKey+ext) + if err := copyFile(outputPath, cachedFile); err != nil { + logf("Warning: failed to cache file: %v\n", err) + } + } + return nil } +func copyFile(src, dst string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + dstFile, err := os.Create(dst) + if err != nil { + return err + } + defer dstFile.Close() + + _, err = io.Copy(dstFile, srcFile) + return err +} + // ExtractTarGz extracts a tar.gz file to the specified directory func ExtractTarGz(archivePath, destDir string) error { - fmt.Printf("Extracting %s to %s\n", archivePath, destDir) + logf("Extracting %s to %s\n", archivePath, destDir) file, err := os.Open(archivePath) if err != nil { @@ -88,7 +168,7 @@ func ExtractTarGz(archivePath, destDir string) error { // Set file permissions if err := os.Chmod(targetPath, os.FileMode(header.Mode)); err != nil { - fmt.Printf("Warning: failed to set permissions for %s: %v\n", targetPath, err) + logf("Warning: failed to set permissions for %s: %v\n", targetPath, err) } } } diff --git a/libs/dbr/context.go b/libs/dbr/context.go index 303e911fba..dbf469e617 100644 --- a/libs/dbr/context.go +++ b/libs/dbr/context.go @@ -1,6 +1,10 @@ package dbr -import "context" +import ( + "context" + "strconv" + "strings" +) // key is a package-local type to use for context keys. // @@ -15,6 +19,71 @@ const ( dbrKey = key(1) ) +// ClusterType represents the type of Databricks cluster. +type ClusterType int + +const ( + ClusterTypeUnknown ClusterType = iota + ClusterTypeInteractive + ClusterTypeServerless +) + +func (t ClusterType) String() string { + switch t { + case ClusterTypeInteractive: + return "interactive" + case ClusterTypeServerless: + return "serverless" + default: + return "unknown" + } +} + +// Version represents a parsed DBR version. +type Version struct { + Type ClusterType + Major int + Minor int + Raw string +} + +// ParseVersion parses a DBR version string and returns structured version info. +// Examples: +// - "16.3" -> Interactive, Major=16, Minor=3 +// - "client.4.9" -> Serverless, Major=4, Minor=9 +func ParseVersion(version string) Version { + result := Version{Raw: version} + + if version == "" { + return result + } + + // Serverless versions have "client." prefix + if strings.HasPrefix(version, "client.") { + result.Type = ClusterTypeServerless + // Parse "client.X.Y" format + parts := strings.Split(strings.TrimPrefix(version, "client."), ".") + if len(parts) >= 1 { + result.Major, _ = strconv.Atoi(parts[0]) + } + if len(parts) >= 2 { + result.Minor, _ = strconv.Atoi(parts[1]) + } + return result + } + + // Interactive versions are "X.Y" format + result.Type = ClusterTypeInteractive + parts := strings.Split(version, ".") + if len(parts) >= 1 { + result.Major, _ = strconv.Atoi(parts[0]) + } + if len(parts) >= 2 { + result.Minor, _ = strconv.Atoi(parts[1]) + } + return result +} + type Environment struct { IsDbr bool Version string @@ -61,3 +130,9 @@ func RuntimeVersion(ctx context.Context) string { return v.(Environment).Version } + +// GetVersion returns the parsed runtime version from the context. +// It expects a context returned by [DetectRuntime] or [MockRuntime]. +func GetVersion(ctx context.Context) Version { + return ParseVersion(RuntimeVersion(ctx)) +} diff --git a/libs/dbr/context_test.go b/libs/dbr/context_test.go index 94e155ab59..a72202b9a6 100644 --- a/libs/dbr/context_test.go +++ b/libs/dbr/context_test.go @@ -84,3 +84,89 @@ func TestContext_RuntimeVersionWithMock(t *testing.T) { assert.Equal(t, "15.4", RuntimeVersion(MockRuntime(ctx, Environment{IsDbr: true, Version: "15.4"}))) assert.Empty(t, RuntimeVersion(MockRuntime(ctx, Environment{}))) } + +func TestParseVersion_Serverless(t *testing.T) { + tests := []struct { + version string + expectedType ClusterType + expectedMajor int + expectedMinor int + }{ + {"client.4.9", ClusterTypeServerless, 4, 9}, + {"client.4.10", ClusterTypeServerless, 4, 10}, + {"client.3.6", ClusterTypeServerless, 3, 6}, + {"client.2", ClusterTypeServerless, 2, 0}, + {"client.2.1", ClusterTypeServerless, 2, 1}, + {"client.1", ClusterTypeServerless, 1, 0}, + {"client.1.13", ClusterTypeServerless, 1, 13}, + } + + for _, tt := range tests { + t.Run(tt.version, func(t *testing.T) { + v := ParseVersion(tt.version) + assert.Equal(t, tt.expectedType, v.Type) + assert.Equal(t, tt.expectedMajor, v.Major) + assert.Equal(t, tt.expectedMinor, v.Minor) + assert.Equal(t, tt.version, v.Raw) + }) + } +} + +func TestParseVersion_Interactive(t *testing.T) { + tests := []struct { + version string + expectedType ClusterType + expectedMajor int + expectedMinor int + }{ + {"16.3", ClusterTypeInteractive, 16, 3}, + {"16.4", ClusterTypeInteractive, 16, 4}, + {"17.0", ClusterTypeInteractive, 17, 0}, + {"17.1", ClusterTypeInteractive, 17, 1}, + {"17.2", ClusterTypeInteractive, 17, 2}, + {"17.3", ClusterTypeInteractive, 17, 3}, + {"15.4", ClusterTypeInteractive, 15, 4}, + } + + for _, tt := range tests { + t.Run(tt.version, func(t *testing.T) { + v := ParseVersion(tt.version) + assert.Equal(t, tt.expectedType, v.Type) + assert.Equal(t, tt.expectedMajor, v.Major) + assert.Equal(t, tt.expectedMinor, v.Minor) + assert.Equal(t, tt.version, v.Raw) + }) + } +} + +func TestParseVersion_Empty(t *testing.T) { + v := ParseVersion("") + assert.Equal(t, ClusterTypeUnknown, v.Type) + assert.Equal(t, 0, v.Major) + assert.Equal(t, 0, v.Minor) + assert.Equal(t, "", v.Raw) +} + +func TestClusterType_String(t *testing.T) { + assert.Equal(t, "interactive", ClusterTypeInteractive.String()) + assert.Equal(t, "serverless", ClusterTypeServerless.String()) + assert.Equal(t, "unknown", ClusterTypeUnknown.String()) +} + +func TestContext_GetVersion(t *testing.T) { + ctx := context.Background() + + // Test serverless version + serverlessCtx := MockRuntime(ctx, Environment{IsDbr: true, Version: "client.4.9"}) + v := GetVersion(serverlessCtx) + assert.Equal(t, ClusterTypeServerless, v.Type) + assert.Equal(t, 4, v.Major) + assert.Equal(t, 9, v.Minor) + + // Test interactive version + interactiveCtx := MockRuntime(ctx, Environment{IsDbr: true, Version: "17.3"}) + v = GetVersion(interactiveCtx) + assert.Equal(t, ClusterTypeInteractive, v.Type) + assert.Equal(t, 17, v.Major) + assert.Equal(t, 3, v.Minor) +}