From fe106267350b940aeab4006d2dec412144609e36 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Tue, 16 Apr 2024 15:09:56 +0000 Subject: [PATCH 1/5] Fix protected default, add a general unit test for SQLX config parameters --- core/actions/table.ts | 12 ++- core/main_test.ts | 191 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 198 insertions(+), 5 deletions(-) diff --git a/core/actions/table.ts b/core/actions/table.ts index 537877374..d2ac6dace 100644 --- a/core/actions/table.ts +++ b/core/actions/table.ts @@ -453,8 +453,8 @@ export class Table extends ActionBuilder { if (config.disabled) { this.disabled(); } - if (config.protected) { - this.protected(); + if (config.type === "incremental") { + this.protected(config.protected); } if (config.bigquery && Object.keys(config.bigquery).length > 0) { this.bigquery(config.bigquery); @@ -520,8 +520,12 @@ export class Table extends ActionBuilder { return this; } - public protected() { - this.proto.protected = true; + public protected(defaultsToTrueProtected: boolean) { + // To prevent accidental data deletion, protected defaults to true if unspecified. + if (defaultsToTrueProtected === undefined || defaultsToTrueProtected === null) { + defaultsToTrueProtected = true; + } + this.proto.protected = defaultsToTrueProtected; return this; } diff --git a/core/main_test.ts b/core/main_test.ts index 1f5fe7049..d8b28468d 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -5,7 +5,7 @@ import { dump as dumpYaml } from "js-yaml"; import * as path from "path"; import { CompilerFunction, NodeVM } from "vm2"; -import { decode64, encode64 } from "df/common/protos"; +import { decode64, deepClone, encode64 } from "df/common/protos"; import { compile } from "df/core/compilers"; import { version } from "df/core/version"; import { dataform } from "df/protos/ts"; @@ -729,6 +729,195 @@ select 1 AS \${dataform.projectConfig.vars.columnVar}` }); }); + suite("incremental tables", () => { + test("sqlx config parameters are applied", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions/operation.sqlx"), "SELECT 1"); + fs.writeFileSync( + path.join(projectDir, "definitions/incremental_table.sqlx"), + ` +config { + type: "incremental", + disabled: true, + protected: false, + name: "name", + bigquery: { + partitionBy: "partitionBy", + clusterBy: ["clusterBy"], + updatePartitionFilter: "updatePartitionFilter", + labels: {"key": "val"}, + partitionExpirationDays: 1, + requirePartitionFilter: true, + additionalOptions: { + option1Key: "option1", + option2Key: "option2", + } + }, + tags: ["tag1", "tag2"], + uniqueKey: ["key1", "key2"], + dependencies: ["operation"], + hermetic: true, + schema: "schema", + assertions: { + uniqueKeys: [["uniqueKey1", "uniqueKey2"]], + nonNull: "nonNull", + rowConditions: ["rowConditions1", "rowConditions2"], + }, + database: "database", + columns: { + column1Key: "column1Val", + column2Key: { + description: "description", + columns: { + nestedColumnKey: "nestedColumnVal" + }, + displayName: "displayName", + tags: ["tag3", "tag4"], + bigqueryPolicyTags: ["bigqueryPolicyTag1", "bigqueryPolicyTag2"], + } + }, + description: "description", + materialized: false +} +SELECT 1` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(asPlainObject(result.compile.compiledGraph.graphErrors.compilationErrors)).deep.equals( + [] + ); + expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([ + { + type: "incremental", + disabled: true, + protected: false, + hermeticity: "HERMETIC", + target: { + database: "database", + name: "name", + schema: "schema" + }, + canonicalTarget: { + database: "database", + name: "name", + schema: "schema" + }, + bigquery: { + additionalOptions: { + option1Key: "option1", + option2Key: "option2" + }, + clusterBy: ["clusterBy"], + labels: { + key: "val" + }, + partitionBy: "partitionBy", + partitionExpirationDays: 1, + requirePartitionFilter: true, + updatePartitionFilter: "updatePartitionFilter" + }, + tags: ["tag1", "tag2"], + uniqueKey: ["key1", "key2"], + dependencyTargets: [ + { + database: "dataform", + name: "operation" + } + ], + actionDescriptor: { + bigqueryLabels: { + key: "val" + }, + columns: [ + { + description: "column1Val", + path: ["column1Key"] + }, + { + bigqueryPolicyTags: ["bigqueryPolicyTag1", "bigqueryPolicyTag2"], + description: "description", + displayName: "displayName", + path: ["column2Key"], + tags: ["tag3", "tag4"] + }, + { + description: "nestedColumnVal", + path: ["column2Key", "nestedColumnKey"] + } + ], + description: "description" + }, + enumType: "INCREMENTAL", + fileName: "definitions/incremental_table.sqlx", + query: "\n\nSELECT 1", + incrementalQuery: "\n\nSELECT 1" + } + ]); + expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals([ + { + target: { + database: "dataform", + name: "schema_name_assertions_uniqueKey_0" + }, + canonicalTarget: { + database: "dataform", + name: "schema_name_assertions_uniqueKey_0" + }, + dependencyTargets: [ + { + database: "database", + name: "name", + schema: "schema" + } + ], + disabled: true, + fileName: "definitions/incremental_table.sqlx", + parentAction: { + database: "database", + name: "name", + schema: "schema" + }, + query: + "\nSELECT\n *\nFROM (\n SELECT\n uniqueKey1, uniqueKey2,\n COUNT(1) AS index_row_count\n FROM `database.schema.name`\n GROUP BY uniqueKey1, uniqueKey2\n ) AS data\nWHERE index_row_count > 1\n", + tags: ["tag1", "tag2"] + }, + { + target: { + database: "dataform", + name: "schema_name_assertions_rowConditions" + }, + canonicalTarget: { + database: "dataform", + name: "schema_name_assertions_rowConditions" + }, + dependencyTargets: [ + { + database: "database", + name: "name", + schema: "schema" + } + ], + disabled: true, + fileName: "definitions/incremental_table.sqlx", + parentAction: { + database: "database", + name: "name", + schema: "schema" + }, + query: + "\nSELECT\n 'rowConditions1' AS failing_row_condition,\n *\nFROM `database.schema.name`\nWHERE NOT (rowConditions1)\nUNION ALL\nSELECT\n 'rowConditions2' AS failing_row_condition,\n *\nFROM `database.schema.name`\nWHERE NOT (rowConditions2)\nUNION ALL\nSELECT\n 'nonNull IS NOT NULL' AS failing_row_condition,\n *\nFROM `database.schema.name`\nWHERE NOT (nonNull IS NOT NULL)\n", + tags: ["tag1", "tag2"] + } + ]); + }); + }); + suite("notebooks", () => { const createSimpleNotebookProject = ( workflowSettingsYaml = VALID_WORKFLOW_SETTINGS_YAML From 195c04a56c9cdb6f48716214884a821de2167f79 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Tue, 16 Apr 2024 15:26:08 +0000 Subject: [PATCH 2/5] Tidy --- core/main_test.ts | 2 +- scripts/run_tests | 2 +- tests/core/core.spec.ts | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/main_test.ts b/core/main_test.ts index d8b28468d..4a3f26f35 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -5,7 +5,7 @@ import { dump as dumpYaml } from "js-yaml"; import * as path from "path"; import { CompilerFunction, NodeVM } from "vm2"; -import { decode64, deepClone, encode64 } from "df/common/protos"; +import { decode64, encode64 } from "df/common/protos"; import { compile } from "df/core/compilers"; import { version } from "df/core/version"; import { dataform } from "df/protos/ts"; diff --git a/scripts/run_tests b/scripts/run_tests index 960f16d65..6826abada 100755 --- a/scripts/run_tests +++ b/scripts/run_tests @@ -6,4 +6,4 @@ bazel run @nodejs//:yarn bazel build @npm//tslint/bin:tslint && bazel-bin/external/npm/tslint/bin/tslint.sh --project . # Run all the tests -bazel test --config=remote-cache ... --build_tests_only --test_env=USE_CLOUD_BUILD_NETWORK=true +bazel test ... --build_tests_only diff --git a/tests/core/core.spec.ts b/tests/core/core.spec.ts index fc0d5c17f..079dfb016 100644 --- a/tests/core/core.spec.ts +++ b/tests/core/core.spec.ts @@ -241,7 +241,8 @@ suite("@dataform/core", () => { disabled: false, fileName: path.basename(__filename), type: "incremental", - enumType: "INCREMENTAL" + enumType: "INCREMENTAL", + protected: true } ]); }); From 4c43d2c5a1386b9fb7eeb41e63d071c7eaf86f9f Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Wed, 26 Jun 2024 12:53:59 +0000 Subject: [PATCH 3/5] Remove test for incremental tables that is now duplicate --- core/main_test.ts | 189 ---------------------------------------------- 1 file changed, 189 deletions(-) diff --git a/core/main_test.ts b/core/main_test.ts index 35c644712..3e1448aa3 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -728,195 +728,6 @@ select 1 AS \${dataform.projectConfig.vars.columnVar}` }); }); - suite("incremental tables", () => { - test("sqlx config parameters are applied", () => { - const projectDir = tmpDirFixture.createNewTmpDir(); - fs.writeFileSync( - path.join(projectDir, "workflow_settings.yaml"), - VALID_WORKFLOW_SETTINGS_YAML - ); - fs.mkdirSync(path.join(projectDir, "definitions")); - fs.writeFileSync(path.join(projectDir, "definitions/operation.sqlx"), "SELECT 1"); - fs.writeFileSync( - path.join(projectDir, "definitions/incremental_table.sqlx"), - ` -config { - type: "incremental", - disabled: true, - protected: false, - name: "name", - bigquery: { - partitionBy: "partitionBy", - clusterBy: ["clusterBy"], - updatePartitionFilter: "updatePartitionFilter", - labels: {"key": "val"}, - partitionExpirationDays: 1, - requirePartitionFilter: true, - additionalOptions: { - option1Key: "option1", - option2Key: "option2", - } - }, - tags: ["tag1", "tag2"], - uniqueKey: ["key1", "key2"], - dependencies: ["operation"], - hermetic: true, - schema: "schema", - assertions: { - uniqueKeys: [["uniqueKey1", "uniqueKey2"]], - nonNull: "nonNull", - rowConditions: ["rowConditions1", "rowConditions2"], - }, - database: "database", - columns: { - column1Key: "column1Val", - column2Key: { - description: "description", - columns: { - nestedColumnKey: "nestedColumnVal" - }, - displayName: "displayName", - tags: ["tag3", "tag4"], - bigqueryPolicyTags: ["bigqueryPolicyTag1", "bigqueryPolicyTag2"], - } - }, - description: "description", - materialized: false -} -SELECT 1` - ); - - const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); - - expect(asPlainObject(result.compile.compiledGraph.graphErrors.compilationErrors)).deep.equals( - [] - ); - expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([ - { - type: "incremental", - disabled: true, - protected: false, - hermeticity: "HERMETIC", - target: { - database: "database", - name: "name", - schema: "schema" - }, - canonicalTarget: { - database: "database", - name: "name", - schema: "schema" - }, - bigquery: { - additionalOptions: { - option1Key: "option1", - option2Key: "option2" - }, - clusterBy: ["clusterBy"], - labels: { - key: "val" - }, - partitionBy: "partitionBy", - partitionExpirationDays: 1, - requirePartitionFilter: true, - updatePartitionFilter: "updatePartitionFilter" - }, - tags: ["tag1", "tag2"], - uniqueKey: ["key1", "key2"], - dependencyTargets: [ - { - database: "dataform", - name: "operation" - } - ], - actionDescriptor: { - bigqueryLabels: { - key: "val" - }, - columns: [ - { - description: "column1Val", - path: ["column1Key"] - }, - { - bigqueryPolicyTags: ["bigqueryPolicyTag1", "bigqueryPolicyTag2"], - description: "description", - displayName: "displayName", - path: ["column2Key"], - tags: ["tag3", "tag4"] - }, - { - description: "nestedColumnVal", - path: ["column2Key", "nestedColumnKey"] - } - ], - description: "description" - }, - enumType: "INCREMENTAL", - fileName: "definitions/incremental_table.sqlx", - query: "\n\nSELECT 1", - incrementalQuery: "\n\nSELECT 1" - } - ]); - expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals([ - { - target: { - database: "dataform", - name: "schema_name_assertions_uniqueKey_0" - }, - canonicalTarget: { - database: "dataform", - name: "schema_name_assertions_uniqueKey_0" - }, - dependencyTargets: [ - { - database: "database", - name: "name", - schema: "schema" - } - ], - disabled: true, - fileName: "definitions/incremental_table.sqlx", - parentAction: { - database: "database", - name: "name", - schema: "schema" - }, - query: - "\nSELECT\n *\nFROM (\n SELECT\n uniqueKey1, uniqueKey2,\n COUNT(1) AS index_row_count\n FROM `database.schema.name`\n GROUP BY uniqueKey1, uniqueKey2\n ) AS data\nWHERE index_row_count > 1\n", - tags: ["tag1", "tag2"] - }, - { - target: { - database: "dataform", - name: "schema_name_assertions_rowConditions" - }, - canonicalTarget: { - database: "dataform", - name: "schema_name_assertions_rowConditions" - }, - dependencyTargets: [ - { - database: "database", - name: "name", - schema: "schema" - } - ], - disabled: true, - fileName: "definitions/incremental_table.sqlx", - parentAction: { - database: "database", - name: "name", - schema: "schema" - }, - query: - "\nSELECT\n 'rowConditions1' AS failing_row_condition,\n *\nFROM `database.schema.name`\nWHERE NOT (rowConditions1)\nUNION ALL\nSELECT\n 'rowConditions2' AS failing_row_condition,\n *\nFROM `database.schema.name`\nWHERE NOT (rowConditions2)\nUNION ALL\nSELECT\n 'nonNull IS NOT NULL' AS failing_row_condition,\n *\nFROM `database.schema.name`\nWHERE NOT (nonNull IS NOT NULL)\n", - tags: ["tag1", "tag2"] - } - ]); - }); - }); - suite("notebooks", () => { const createSimpleNotebookProject = ( workflowSettingsYaml = VALID_WORKFLOW_SETTINGS_YAML From 30c4c052a0271d9f1651f4665e8237140173065a Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Wed, 26 Jun 2024 12:56:50 +0000 Subject: [PATCH 4/5] Remove protected field from being expected in actions where it is invalid --- core/main_test.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/core/main_test.ts b/core/main_test.ts index 3e1448aa3..f21d0c8c0 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -1603,8 +1603,6 @@ SELECT 1` }, type: "table", disabled: true, - // TODO(ekrekr): finish fixing this in https://github.com/dataform-co/dataform/pull/1718. - // protected: false, hermeticity: "HERMETIC", bigquery: { additionalOptions: { @@ -1697,8 +1695,6 @@ SELECT 1` }, type: "view", disabled: true, - // TODO(ekrekr): finish fixing this in https://github.com/dataform-co/dataform/pull/1718. - // protected: false, hermeticity: "HERMETIC", bigquery: { additionalOptions: { @@ -1794,8 +1790,7 @@ SELECT 1` }, type: "incremental", disabled: true, - // TODO(ekrekr): finish fixing this in https://github.com/dataform-co/dataform/pull/1718. - // protected: false, + protected: false, hermeticity: "HERMETIC", bigquery: { additionalOptions: { From 4a735feabfe19d03f1bddda0fa0ab4a8b6ad8264 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Wed, 26 Jun 2024 12:58:53 +0000 Subject: [PATCH 5/5] Reset run_tests script --- scripts/run_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_tests b/scripts/run_tests index 6826abada..f6e9093a9 100755 --- a/scripts/run_tests +++ b/scripts/run_tests @@ -6,4 +6,4 @@ bazel run @nodejs//:yarn bazel build @npm//tslint/bin:tslint && bazel-bin/external/npm/tslint/bin/tslint.sh --project . # Run all the tests -bazel test ... --build_tests_only +bazel test ... --build_tests_only --test_env=USE_CLOUD_BUILD_NETWORK=true