Skip to content

Commit

Permalink
feat: dbt 1.8 (#10180)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ZeRego
ZeRego committed May 24, 2024
1 parent 279a4f5 commit 379e902
Show file tree
Hide file tree
Showing 13 changed files with 20,667 additions and 30 deletions.
19 changes: 17 additions & 2 deletions dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,20 @@ RUN ln -s /usr/local/dbt1.4/bin/dbt /usr/local/bin/dbt\
"dbt-databricks~=1.7.0" \
"dbt-trino==1.7.0" \
"psycopg2-binary==2.9.6" \
&& ln -s /usr/local/dbt1.7/bin/dbt /usr/local/bin/dbt1.7
&& ln -s /usr/local/dbt1.7/bin/dbt /usr/local/bin/dbt1.7 \
&& python3 -m venv /usr/local/dbt1.8 \
&& /usr/local/dbt1.8/bin/pip install \
# from 1.8, dbt-core needs to be explicitly installed
"dbt-core~=1.8.0" \
# regression (https://github.com/dbt-labs/dbt-postgres/issues/96)
# "dbt-postgres~=1.8.0" \
# "dbt-redshift~=1.8.0" \
"dbt-snowflake~=1.8.0" \
"dbt-bigquery~=1.8.0" \
# databricks adaptor not available yet
# "dbt-databricks~=1.8.0" \
"dbt-trino~=1.8.0" \
&& ln -s /usr/local/dbt1.8/bin/dbt /usr/local/bin/dbt1.8

# -----------------------------
# Stage 1: stop here for dev environment
Expand Down Expand Up @@ -144,12 +157,14 @@ COPY --from=prod-builder /usr/local/dbt1.4 /usr/local/dbt1.4
COPY --from=prod-builder /usr/local/dbt1.5 /usr/local/dbt1.5
COPY --from=prod-builder /usr/local/dbt1.6 /usr/local/dbt1.6
COPY --from=prod-builder /usr/local/dbt1.7 /usr/local/dbt1.7
COPY --from=prod-builder /usr/local/dbt1.8 /usr/local/dbt1.8
COPY --from=prod-builder /usr/app /usr/app

RUN ln -s /usr/local/dbt1.4/bin/dbt /usr/local/bin/dbt \
&& ln -s /usr/local/dbt1.5/bin/dbt /usr/local/bin/dbt1.5 \
&& ln -s /usr/local/dbt1.6/bin/dbt /usr/local/bin/dbt1.6 \
&& ln -s /usr/local/dbt1.7/bin/dbt /usr/local/bin/dbt1.7
&& ln -s /usr/local/dbt1.7/bin/dbt /usr/local/bin/dbt1.7 \
&& ln -s /usr/local/dbt1.8/bin/dbt /usr/local/bin/dbt1.8


# Production config
Expand Down
15 changes: 14 additions & 1 deletion dockerfile-prs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,17 @@ RUN python3 -m venv /usr/local/dbt1.7 \
"psycopg2-binary==2.9.6" \
&& ln -s /usr/local/dbt1.7/bin/dbt /usr/local/bin/dbt1.7

RUN python3 -m venv /usr/local/dbt1.8 \
&& /usr/local/dbt1.8/bin/pip install \
"dbt-core~=1.8.0" \
# "dbt-postgres~=1.8.0" \
# "dbt-redshift~=1.8.0" \
"dbt-snowflake~=1.8.0" \
"dbt-bigquery~=1.8.0" \
# "dbt-databricks~=1.8.0" \
"dbt-trino~=1.8.0" \
&& ln -s /usr/local/dbt1.8/bin/dbt /usr/local/bin/dbt1.8

# -----------------------------
# Stage 1: stop here for dev environment
# -----------------------------
Expand Down Expand Up @@ -140,12 +151,14 @@ COPY --from=prod-builder /usr/local/dbt1.4 /usr/local/dbt1.4
COPY --from=prod-builder /usr/local/dbt1.5 /usr/local/dbt1.5
COPY --from=prod-builder /usr/local/dbt1.6 /usr/local/dbt1.6
COPY --from=prod-builder /usr/local/dbt1.7 /usr/local/dbt1.7
COPY --from=prod-builder /usr/local/dbt1.8 /usr/local/dbt1.8
COPY --from=prod-builder /usr/app /usr/app

RUN ln -s /usr/local/dbt1.4/bin/dbt /usr/local/bin/dbt \
&& ln -s /usr/local/dbt1.5/bin/dbt /usr/local/bin/dbt1.5 \
&& ln -s /usr/local/dbt1.6/bin/dbt /usr/local/bin/dbt1.6 \
&& ln -s /usr/local/dbt1.7/bin/dbt /usr/local/bin/dbt1.7
&& ln -s /usr/local/dbt1.7/bin/dbt /usr/local/bin/dbt1.7 \
&& ln -s /usr/local/dbt1.8/bin/dbt /usr/local/bin/dbt1.8


# Production config
Expand Down
3 changes: 3 additions & 0 deletions packages/backend/src/dbt/dbtCliClient.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ enum DbtCommands {
DBT_1_5 = 'dbt1.5',
DBT_1_6 = 'dbt1.6',
DBT_1_7 = 'dbt1.7',
DBT_1_8 = 'dbt1.8',
}

export class DbtCliClient implements DbtClient {
Expand Down Expand Up @@ -149,6 +150,8 @@ export class DbtCliClient implements DbtClient {
return DbtCommands.DBT_1_6;
case SupportedDbtVersions.V1_7:
return DbtCommands.DBT_1_7;
case SupportedDbtVersions.V1_8:
return DbtCommands.DBT_1_8;
default:
return assertUnreachable(
this.dbtVersion,
Expand Down
8 changes: 5 additions & 3 deletions packages/backend/src/projectAdapters/dbtBaseProjectAdapter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,11 @@ export class DbtBaseProjectAdapter implements ProjectAdapter {
// Validate metrics in the manifest - compile fails if any invalid
const metrics = DbtBaseProjectAdapter._validateDbtMetrics(
manifestVersion,
[DbtManifestVersion.V10, DbtManifestVersion.V11].includes(
manifestVersion,
)
[
DbtManifestVersion.V10,
DbtManifestVersion.V11,
DbtManifestVersion.V12,
].includes(manifestVersion)
? []
: Object.values(manifest.metrics),
);
Expand Down
1 change: 1 addition & 0 deletions packages/cli/src/dbt/manifest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ export const getDbtManifest = async (): Promise<DbtManifestVersion> => {
if (version.startsWith('1.5.')) return DbtManifestVersion.V9;
if (version.startsWith('1.6.')) return DbtManifestVersion.V10;
if (version.startsWith('1.7.')) return DbtManifestVersion.V11;
if (version.startsWith('1.8.')) return DbtManifestVersion.V12;
return DbtManifestVersion.V8;
};

Expand Down
12 changes: 7 additions & 5 deletions packages/cli/src/handlers/compile.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,15 +56,15 @@ export const compile = async (options: CompileHandlerOptions) => {
if (!isSupportedDbtVersion(dbtVersion)) {
if (process.env.CI === 'true') {
console.error(
`Your dbt version ${dbtVersion} does not match our supported versions (1.3.* - 1.7.*), this could cause problems on compile or validation.`,
`Your dbt version ${dbtVersion} does not match our supported versions (1.3.* - 1.8.*), this could cause problems on compile or validation.`,
);
} else {
const answers = await inquirer.prompt([
{
type: 'confirm',
name: 'isConfirm',
message: `${styles.warning(
`Your dbt version ${dbtVersion} does not match our supported version (1.3.* - 1.7.*), this could cause problems on compile or validation.`,
`Your dbt version ${dbtVersion} does not match our supported version (1.3.* - 1.8.*), this could cause problems on compile or validation.`,
)}\nDo you still want to continue?`,
},
]);
Expand Down Expand Up @@ -174,9 +174,11 @@ ${errors.join('')}`),
validModelsWithTypes,
false,
manifest.metadata.adapter_type,
[DbtManifestVersion.V10, DbtManifestVersion.V11].includes(
manifestVersion,
)
[
DbtManifestVersion.V10,
DbtManifestVersion.V11,
DbtManifestVersion.V12,
].includes(manifestVersion)
? []
: Object.values(manifest.metrics),
warehouseClient,
Expand Down
9 changes: 8 additions & 1 deletion packages/cli/src/handlers/dbt/compile.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,14 @@ export const dbtList = async (
const { stdout, stderr } = await execa('dbt', ['ls', ...args]);
const models = stdout
.split('\n')
.map<string>((line) => JSON.parse(line).unique_id)
.map<string>((line) => {
try {
return JSON.parse(line).unique_id;
} catch {
// ignore non-json lines
return '';
}
})
.filter((modelId) => modelId.startsWith('model.')); // filter models by name because "--models" and "--resource_type" are mutually exclusive arguments
GlobalState.debug(`> Models: ${models.join(' ')}`);
console.error(stderr);
Expand Down
3 changes: 2 additions & 1 deletion packages/cli/src/handlers/dbt/getDbtVersion.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ export const getSupportedDbtVersion = async () => {
if (version.startsWith('1.5.')) return SupportedDbtVersions.V1_5;
if (version.startsWith('1.6.')) return SupportedDbtVersions.V1_6;
if (version.startsWith('1.7.')) return SupportedDbtVersions.V1_7;
if (version.startsWith('1.8.')) return SupportedDbtVersions.V1_8;

console.error(
styles.warning(
Expand All @@ -40,7 +41,7 @@ export const getSupportedDbtVersion = async () => {
};

export const isSupportedDbtVersion = (version: string) => {
const supportedVersions = ['1.3.', '1.4.', '1.5.', '1.6.', '1.7'];
const supportedVersions = ['1.3.', '1.4.', '1.5.', '1.6.', '1.7', '1.8'];
return supportedVersions.some((supportedVersion) =>
version.startsWith(supportedVersion),
);
Expand Down
64 changes: 64 additions & 0 deletions packages/common/src/dbt/schemas/lightdashV12.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
{
"definitions": {
"LightdashNodeConfig": {
"allOf": [
{
"type": "object",
"properties": {
"meta": {
"$ref": "https://schemas.lightdash.com/lightdash/metadata.json#/definitions/LightdashModelMetadata"
}
}
}
]
},
"LightdashColumnInfo": {
"allOf": [
{
"type": "object",
"properties": {
"meta": {
"$ref": "https://schemas.lightdash.com/lightdash/metadata.json#/definitions/LightdashColumnMetadata"
}
}
}
]
},
"LightdashCompiledModelNode": {
"allOf": [
{
"type": "object",
"properties": {
"name": {
"type": "string",
"allOf": [
{
"not": { "enum": ["lightdash", "ld", ""] }
},
{
"pattern": "^[^.]*$"
}
]
},

"meta": {
"$ref": "https://schemas.lightdash.com/lightdash/metadata.json#/definitions/LightdashModelMetadata"
},
"config": {
"$ref": "#/definitions/LightdashNodeConfig"
},
"columns": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/LightdashColumnInfo"
},
"default": {}
}
}
}
]
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://schemas.lightdash.com/lightdash/v12.json"
}
Loading

0 comments on commit 379e902

Please sign in to comment.