Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add more test coverage of variables and project config overrides for Dataform Core #1603

Merged
merged 5 commits into from Dec 6, 2023
Merged

Add more test coverage of variables and project config overrides for Dataform Core #1603

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
64b1a46
Contextable config test
Ekrekr Dec 4, 2023
5bac776
Add a test for variable merging, and applying in SQLX config
Ekrekr Dec 4, 2023
b603b80
Add override for default schema
Ekrekr Dec 4, 2023
417cb1c
Tidy
Ekrekr Dec 4, 2023
8289595
Update core tests to use the var from the project config request for …
Ekrekr Dec 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
130 changes: 109 additions & 21 deletions core/main_test.ts
View file
Open in desktop
Expand Up @@ -152,6 +152,87 @@ suite("@dataform/core", ({ afterEach }) => {
);
});

test(`workflow settings and project config overrides are merged and applied within SQLX files`, () => {
const projectDir = tmpDirFixture.createNewTmpDir();
// tslint:disable-next-line: tsr-detect-non-literal-fs-filename
fs.writeFileSync(
path.join(projectDir, "workflow_settings.yaml"),
`
defaultDatabase: dataform
defaultLocation: locationInWorkflowSettings
vars:
selectVar: selectVal
`
);
// tslint:disable-next-line: tsr-detect-non-literal-fs-filename
fs.mkdirSync(path.join(projectDir, "definitions"));
// tslint:disable-next-line: tsr-detect-non-literal-fs-filename
fs.writeFileSync(
path.join(projectDir, "definitions/file.sqlx"),
`
config {
type: "table",
database: dataform.projectConfig.vars.databaseVar,
}
select 1 AS \${dataform.projectConfig.vars.selectVar}`
);
const coreExecutionRequest = dataform.CoreExecutionRequest.create({
compile: {
compileConfig: {
projectDir,
filePaths: ["definitions/file.sqlx"],
projectConfigOverride: {
defaultLocation: "locationInOverride",
vars: {
databaseVar: "databaseVal"
}
}
}
}
});

const result = runMainInVm(coreExecutionRequest);

expect(asPlainObject(result.compile.compiledGraph)).deep.equals(
asPlainObject({
dataformCoreVersion: "3.0.0",
graphErrors: {},
projectConfig: {
defaultDatabase: "dataform",
defaultLocation: "locationInOverride",
vars: {
databaseVar: "databaseVal",
selectVar: "selectVal"
},
warehouse: "bigquery"
},
tables: [
{
canonicalTarget: {
database: "databaseVal",
name: "file"
},
disabled: false,
enumType: "TABLE",
fileName: "definitions/file.sqlx",
query: "\n\nselect 1 AS selectVal",
target: {
database: "databaseVal",
name: "file"
},
type: "table"
}
],
targets: [
{
database: "databaseVal",
name: "file"
}
]
})
);
});

suite("variables", () => {
test(`variables in workflow_settings.yaml must be strings`, () => {
const projectDir = tmpDirFixture.createNewTmpDir();
Expand Down Expand Up @@ -196,9 +277,8 @@ vars:
`
defaultLocation: "us"
vars:
var1: value1
var2: value2
var3: value3`
descriptionVar: descriptionValue
columnVar: columnValue`
);
// tslint:disable-next-line: tsr-detect-non-literal-fs-filename
fs.mkdirSync(path.join(projectDir, "definitions"));
Expand All @@ -210,17 +290,27 @@ vars:
`
config {
type: "table",
database: dataform.projectConfig.vars.var1,
database: dataform.projectConfig.vars.databaseVar,
schema: "tableSchema",
description: dataform.projectConfig.vars.var2,
description: dataform.projectConfig.vars.descriptionVar,
assertions: {
nonNull: [dataform.projectConfig.vars.var3],
nonNull: [dataform.projectConfig.vars.columnVar],
}
}
select 1 AS \${dataform.projectConfig.vars.var3}`
select 1 AS \${dataform.projectConfig.vars.columnVar}`
);
const coreExecutionRequest = dataform.CoreExecutionRequest.create({
compile: { compileConfig: { projectDir, filePaths: ["definitions/file.sqlx"] } }
compile: {
compileConfig: {
projectDir,
filePaths: ["definitions/file.sqlx"],
projectConfigOverride: {
vars: {
databaseVar: "databaseVal"
}
}
}
}
});

const result = runMainInVm(coreExecutionRequest);
Expand All @@ -234,19 +324,19 @@ select 1 AS \${dataform.projectConfig.vars.var3}`
},
dependencyTargets: [
{
database: "value1",
database: "databaseVal",
name: "file",
schema: "tableSchema"
}
],
fileName: "definitions/file.sqlx",
parentAction: {
database: "value1",
database: "databaseVal",
name: "file",
schema: "tableSchema"
},
query:
"\nSELECT\n 'value3 IS NOT NULL' AS failing_row_condition,\n *\nFROM `value1.tableSchema.file`\nWHERE NOT (value3 IS NOT NULL)\n",
"\nSELECT\n 'columnValue IS NOT NULL' AS failing_row_condition,\n *\nFROM `databaseVal.tableSchema.file`\nWHERE NOT (columnValue IS NOT NULL)\n",
target: {
name: "tableSchema_file_assertions_rowConditions"
}
Expand All @@ -257,28 +347,28 @@ select 1 AS \${dataform.projectConfig.vars.var3}`
projectConfig: {
defaultLocation: "us",
vars: {
var1: "value1",
var2: "value2",
var3: "value3"
databaseVar: "databaseVal",
descriptionVar: "descriptionValue",
columnVar: "columnValue"
},
warehouse: "bigquery"
},
tables: [
{
actionDescriptor: {
description: "value2"
description: "descriptionValue"
},
canonicalTarget: {
database: "value1",
database: "databaseVal",
name: "file",
schema: "tableSchema"
},
disabled: false,
enumType: "TABLE",
fileName: "definitions/file.sqlx",
query: "\n\nselect 1 AS value3",
query: "\n\nselect 1 AS columnValue",
target: {
database: "value1",
database: "databaseVal",
name: "file",
schema: "tableSchema"
},
Expand All @@ -287,7 +377,7 @@ select 1 AS \${dataform.projectConfig.vars.var3}`
],
targets: [
{
database: "value1",
database: "databaseVal",
name: "file",
schema: "tableSchema"
},
Expand All @@ -299,8 +389,6 @@ select 1 AS \${dataform.projectConfig.vars.var3}`
);
});
});

// TODO(ekrekr): add a test for nested fields, once they exist.
});

suite("notebooks", () => {
Expand Down