Add more test coverage of variables and project config overrides for …

…Dataform Core (#1603) * Contextable config test * Add a test for variable merging, and applying in SQLX config * Add override for default schema * Tidy * Update core tests to use the var from the project config request for auto assertion database
dataform-co · Dec 6, 2023 · b2cd68f · b2cd68f
1 parent 08be45f
commit b2cd68f
Showing 1 changed file with 109 additions and 21 deletions.
diff --git a/core/main_test.ts b/core/main_test.ts
@@ -152,6 +152,87 @@ suite("@dataform/core", ({ afterEach }) => {
       );
     });
 
+    test(`workflow settings and project config overrides are merged and applied within SQLX files`, () => {
+      const projectDir = tmpDirFixture.createNewTmpDir();
+      // tslint:disable-next-line: tsr-detect-non-literal-fs-filename
+      fs.writeFileSync(
+        path.join(projectDir, "workflow_settings.yaml"),
+        `
+defaultDatabase: dataform
+defaultLocation: locationInWorkflowSettings
+vars:
+  selectVar: selectVal
+`
+      );
+      // tslint:disable-next-line: tsr-detect-non-literal-fs-filename
+      fs.mkdirSync(path.join(projectDir, "definitions"));
+      // tslint:disable-next-line: tsr-detect-non-literal-fs-filename
+      fs.writeFileSync(
+        path.join(projectDir, "definitions/file.sqlx"),
+        `
+config {
+  type: "table",
+  database: dataform.projectConfig.vars.databaseVar,
+}
+select 1 AS \${dataform.projectConfig.vars.selectVar}`
+      );
+      const coreExecutionRequest = dataform.CoreExecutionRequest.create({
+        compile: {
+          compileConfig: {
+            projectDir,
+            filePaths: ["definitions/file.sqlx"],
+            projectConfigOverride: {
+              defaultLocation: "locationInOverride",
+              vars: {
+                databaseVar: "databaseVal"
+              }
+            }
+          }
+        }
+      });
+
+      const result = runMainInVm(coreExecutionRequest);
+
+      expect(asPlainObject(result.compile.compiledGraph)).deep.equals(
+        asPlainObject({
+          dataformCoreVersion: "3.0.0",
+          graphErrors: {},
+          projectConfig: {
+            defaultDatabase: "dataform",
+            defaultLocation: "locationInOverride",
+            vars: {
+              databaseVar: "databaseVal",
+              selectVar: "selectVal"
+            },
+            warehouse: "bigquery"
+          },
+          tables: [
+            {
+              canonicalTarget: {
+                database: "databaseVal",
+                name: "file"
+              },
+              disabled: false,
+              enumType: "TABLE",
+              fileName: "definitions/file.sqlx",
+              query: "\n\nselect 1 AS selectVal",
+              target: {
+                database: "databaseVal",
+                name: "file"
+              },
+              type: "table"
+            }
+          ],
+          targets: [
+            {
+              database: "databaseVal",
+              name: "file"
+            }
+          ]
+        })
+      );
+    });
+
     suite("variables", () => {
       test(`variables in workflow_settings.yaml must be strings`, () => {
         const projectDir = tmpDirFixture.createNewTmpDir();
@@ -196,9 +277,8 @@ vars:
           `
 defaultLocation: "us"
 vars:
-  var1: value1
-  var2: value2
-  var3: value3`
+  descriptionVar: descriptionValue
+  columnVar: columnValue`
         );
         // tslint:disable-next-line: tsr-detect-non-literal-fs-filename
         fs.mkdirSync(path.join(projectDir, "definitions"));
@@ -210,17 +290,27 @@ vars:
           `
 config {
   type: "table",
-  database: dataform.projectConfig.vars.var1,
+  database: dataform.projectConfig.vars.databaseVar,
   schema: "tableSchema",
-  description: dataform.projectConfig.vars.var2,
+  description: dataform.projectConfig.vars.descriptionVar,
   assertions: {
-    nonNull: [dataform.projectConfig.vars.var3],
+    nonNull: [dataform.projectConfig.vars.columnVar],
   }
 }
-select 1 AS \${dataform.projectConfig.vars.var3}`
+select 1 AS \${dataform.projectConfig.vars.columnVar}`
         );
         const coreExecutionRequest = dataform.CoreExecutionRequest.create({
-          compile: { compileConfig: { projectDir, filePaths: ["definitions/file.sqlx"] } }
+          compile: {
+            compileConfig: {
+              projectDir,
+              filePaths: ["definitions/file.sqlx"],
+              projectConfigOverride: {
+                vars: {
+                  databaseVar: "databaseVal"
+                }
+              }
+            }
+          }
         });
 
         const result = runMainInVm(coreExecutionRequest);
@@ -234,19 +324,19 @@ select 1 AS \${dataform.projectConfig.vars.var3}`
                 },
                 dependencyTargets: [
                   {
-                    database: "value1",
+                    database: "databaseVal",
                     name: "file",
                     schema: "tableSchema"
                   }
                 ],
                 fileName: "definitions/file.sqlx",
                 parentAction: {
-                  database: "value1",
+                  database: "databaseVal",
                   name: "file",
                   schema: "tableSchema"
                 },
                 query:
-                  "\nSELECT\n  'value3 IS NOT NULL' AS failing_row_condition,\n  *\nFROM `value1.tableSchema.file`\nWHERE NOT (value3 IS NOT NULL)\n",
+                  "\nSELECT\n  'columnValue IS NOT NULL' AS failing_row_condition,\n  *\nFROM `databaseVal.tableSchema.file`\nWHERE NOT (columnValue IS NOT NULL)\n",
                 target: {
                   name: "tableSchema_file_assertions_rowConditions"
                 }
@@ -257,28 +347,28 @@ select 1 AS \${dataform.projectConfig.vars.var3}`
             projectConfig: {
               defaultLocation: "us",
               vars: {
-                var1: "value1",
-                var2: "value2",
-                var3: "value3"
+                databaseVar: "databaseVal",
+                descriptionVar: "descriptionValue",
+                columnVar: "columnValue"
               },
               warehouse: "bigquery"
             },
             tables: [
               {
                 actionDescriptor: {
-                  description: "value2"
+                  description: "descriptionValue"
                 },
                 canonicalTarget: {
-                  database: "value1",
+                  database: "databaseVal",
                   name: "file",
                   schema: "tableSchema"
                 },
                 disabled: false,
                 enumType: "TABLE",
                 fileName: "definitions/file.sqlx",
-                query: "\n\nselect 1 AS value3",
+                query: "\n\nselect 1 AS columnValue",
                 target: {
-                  database: "value1",
+                  database: "databaseVal",
                   name: "file",
                   schema: "tableSchema"
                 },
@@ -287,7 +377,7 @@ select 1 AS \${dataform.projectConfig.vars.var3}`
             ],
             targets: [
               {
-                database: "value1",
+                database: "databaseVal",
                 name: "file",
                 schema: "tableSchema"
               },
@@ -299,8 +389,6 @@ select 1 AS \${dataform.projectConfig.vars.var3}`
         );
       });
     });
-
-    // TODO(ekrekr): add a test for nested fields, once they exist.
   });
 
   suite("notebooks", () => {