diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts index fa4cf551ba2eb..1b0963f1f9db2 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts @@ -83,12 +83,23 @@ import { PROJECT_NAME, } from './lookml'; import { PRESTO, PRESTO_HOST_PORT, PRESTO_DATABASE, PRESTO_USERNAME, PRESTO_PASSWORD } from './presto'; -import { BIGQUERY_BETA, MYSQL } from '../constants'; +import { BIGQUERY_BETA, MYSQL, UNITY_CATALOG } from '../constants'; import { BIGQUERY_BETA_PROJECT_ID, DATASET_ALLOW, DATASET_DENY, PROJECT_ALLOW, PROJECT_DENY } from './bigqueryBeta'; import { MYSQL_HOST_PORT, MYSQL_PASSWORD, MYSQL_USERNAME } from './mysql'; import { MSSQL, MSSQL_DATABASE, MSSQL_HOST_PORT, MSSQL_PASSWORD, MSSQL_USERNAME } from './mssql'; import { TRINO, TRINO_DATABASE, TRINO_HOST_PORT, TRINO_PASSWORD, TRINO_USERNAME } from './trino'; import { MARIADB, MARIADB_DATABASE, MARIADB_HOST_PORT, MARIADB_PASSWORD, MARIADB_USERNAME } from './mariadb'; +import { + INCLUDE_COLUMN_LINEAGE, + TOKEN, + UNITY_CATALOG_ALLOW, + UNITY_CATALOG_DENY, + UNITY_METASTORE_ID_ALLOW, + UNITY_METASTORE_ID_DENY, + UNITY_TABLE_ALLOW, + UNITY_TABLE_DENY, + WORKSPACE_URL, +} from './unity_catalog'; export enum RecipeSections { Connection = 0, @@ -338,8 +349,23 @@ export const RECIPE_FIELDS: RecipeFields = { ], filterSectionTooltip: 'Include or exclude specific Schemas, Tables and Views from ingestion.', }, + [UNITY_CATALOG]: { + fields: [WORKSPACE_URL, TOKEN], + filterFields: [ + UNITY_METASTORE_ID_ALLOW, + UNITY_METASTORE_ID_DENY, + UNITY_CATALOG_ALLOW, + UNITY_CATALOG_DENY, + SCHEMA_ALLOW, + SCHEMA_DENY, + UNITY_TABLE_ALLOW, + UNITY_TABLE_DENY, + ], + advancedFields: [INCLUDE_TABLE_LINEAGE, INCLUDE_COLUMN_LINEAGE, STATEFUL_INGESTION_ENABLED], + filterSectionTooltip: 'Include or exclude specific Metastores, Catalogs, Schemas, and Tables from ingestion.', + }, }; export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS)); -export const CONNECTORS_WITH_TEST_CONNECTION = new Set([SNOWFLAKE, LOOKER, BIGQUERY_BETA, BIGQUERY]); +export const CONNECTORS_WITH_TEST_CONNECTION = new Set([SNOWFLAKE, LOOKER, BIGQUERY_BETA, BIGQUERY, UNITY_CATALOG]); diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/unity_catalog.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/unity_catalog.tsx new file mode 100644 index 0000000000000..3b5565ab1abd4 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/unity_catalog.tsx @@ -0,0 +1,153 @@ +import React from 'react'; +import { RecipeField, FieldType, setListValuesOnRecipe } from './common'; + +export const UNITY_CATALOG = 'unity-catalog'; + +export const TOKEN: RecipeField = { + name: 'token', + label: 'Token', + tooltip: 'A personal access token associated with the Databricks account used to extract metadata.', + type: FieldType.SECRET, + fieldPath: 'source.config.token', + placeholder: 'dapi1a2b3c45d67890e1f234567a8bc9012d', + required: true, + rules: null, +}; + +export const WORKSPACE_URL: RecipeField = { + name: 'workspace_url', + label: 'Workspace URL', + tooltip: 'The URL for the Databricks workspace from which to extract metadata.', + type: FieldType.TEXT, + fieldPath: 'source.config.workspace_url', + placeholder: 'https://abcsales.cloud.databricks.com', + required: true, + rules: null, +}; + +export const INCLUDE_TABLE_LINEAGE: RecipeField = { + name: 'include_table_lineage', + label: 'Include Table Lineage', + tooltip: ( +
+ Extract Table Lineage from Unity Catalog. Note that this requires that your Databricks accounts meets + certain requirements. View them{' '} + here +
+ ), + type: FieldType.BOOLEAN, + fieldPath: 'source.config.include_table_lineage', + rules: null, +}; + +export const INCLUDE_COLUMN_LINEAGE: RecipeField = { + name: 'include_column_lineage', + label: 'Include Column Lineage', + tooltip: ( +
+ Extract Column Lineage from Unity Catalog. Note that this requires that your Databricks accounts meets + certain requirements. View them{' '} + here. + Enabling this feature may increase the duration of ingestion. +
+ ), + type: FieldType.BOOLEAN, + fieldPath: 'source.config.include_column_lineage', + rules: null, +}; + +const metastoreIdAllowFieldPath = 'source.config.metastore_id_pattern.allow'; +export const UNITY_METASTORE_ID_ALLOW: RecipeField = { + name: 'metastore_id_pattern.allow', + label: 'Allow Patterns', + tooltip: + 'Only include specific Metastores by providing the id of a Metastore, or a Regular Expression (REGEX) to include specific Metastores. If not provided, all Metastores will be included.', + placeholder: '11111-2222-33333-44-555555', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: metastoreIdAllowFieldPath, + rules: null, + section: 'Metastores', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, metastoreIdAllowFieldPath), +}; + +const metastoreIdDenyFieldPath = 'source.config.metastore_id_pattern.deny'; +export const UNITY_METASTORE_ID_DENY: RecipeField = { + name: 'metastore_id_pattern.deny', + label: 'Deny Patterns', + tooltip: + 'Exclude specific Metastores by providing the id of a Metastores, or a Regular Expression (REGEX). If not provided, all Metastores will be included. Deny patterns always take precedence over Allow patterns.', + placeholder: '11111-2222-33333-44-555555', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: metastoreIdDenyFieldPath, + rules: null, + section: 'Metastores', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, metastoreIdDenyFieldPath), +}; + +const catalogAllowFieldPath = 'source.config.catalog_pattern.allow'; +export const UNITY_CATALOG_ALLOW: RecipeField = { + name: 'catalog_pattern.allow', + label: 'Allow Patterns', + tooltip: + 'Only include specific Catalogs by providing the name of a Catalog, or a Regular Expression (REGEX) to include specific Catalogs. If not provided, all Catalogs will be included.', + placeholder: 'my_catalog', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: catalogAllowFieldPath, + rules: null, + section: 'Catalogs', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, catalogAllowFieldPath), +}; + +const catalogDenyFieldPath = 'source.config.catalog_pattern.deny'; +export const UNITY_CATALOG_DENY: RecipeField = { + name: 'catalog_pattern.allow', + label: 'Allow Patterns', + tooltip: + 'Exclude specific Catalogs by providing the name of a Catalog, or a Regular Expression (REGEX) to exclude specific Catalogs. If not provided, all Catalogs will be included. Deny patterns always take precedence over Allow patterns.', + placeholder: 'my_catalog', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: catalogDenyFieldPath, + rules: null, + section: 'Catalogs', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, catalogDenyFieldPath), +}; + +const tableAllowFieldPath = 'source.config.metastore_id_pattern.allow'; +export const UNITY_TABLE_ALLOW: RecipeField = { + name: 'catalog_pattern.allow', + label: 'Allow Patterns', + tooltip: + 'Only include specific Tables by providing the fully-qualified name of a Table, or a Regular Expression (REGEX) to include specific Tables. If not provided, all Tables will be included.', + placeholder: 'catalog.schema.table', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: tableAllowFieldPath, + rules: null, + section: 'Tables', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, tableAllowFieldPath), +}; + +const tableDenyFieldPath = 'source.config.metastore_id_pattern.deny'; +export const UNITY_TABLE_DENY: RecipeField = { + name: 'catalog_pattern.allow', + label: 'Allow Patterns', + tooltip: + 'Exclude specific Tables by providing the fully-qualified name of a Table, or a Regular Expression (REGEX) to exclude specific Tables. If not provided, all Tables will be included. Deny patterns always take precedence over Allow patterns.', + placeholder: 'catalog.schema.table', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: tableDenyFieldPath, + rules: null, + section: 'Tables', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, tableDenyFieldPath), +}; diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts index 282be15e13226..ca925aa3a7821 100644 --- a/datahub-web-react/src/app/ingest/source/builder/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts @@ -25,6 +25,7 @@ import mariadbLogo from '../../../../images/mariadblogo.png'; import metabaseLogo from '../../../../images/metabaselogo.png'; import powerbiLogo from '../../../../images/powerbilogo.png'; import modeLogo from '../../../../images/modelogo.png'; +import databricksLogo from '../../../../images/databrickslogo.png'; export const ATHENA = 'athena'; export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`; @@ -91,6 +92,8 @@ export const TRINO = 'trino'; export const TRINO_URN = `urn:li:dataPlatform:${TRINO}`; export const CUSTOM = 'custom'; export const CUSTOM_URN = `urn:li:dataPlatform:${CUSTOM}`; +export const UNITY_CATALOG = 'unity-catalog'; +export const UNITY_CATALOG_URN = `urn:li:dataPlatform:${UNITY_CATALOG}`; export const PLATFORM_URN_TO_LOGO = { [ATHENA_URN]: athenaLogo, @@ -120,6 +123,7 @@ export const PLATFORM_URN_TO_LOGO = { [TABLEAU_URN]: tableauLogo, [TRINO_URN]: trinoLogo, [SUPERSET_URN]: supersetLogo, + [UNITY_CATALOG_URN]: databricksLogo, }; export const SOURCE_TO_PLATFORM_URN = { diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index 6f6c63672fb8a..939c0dda1c64b 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -97,6 +97,13 @@ "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mariadb/", "recipe": "source:\n type: mariadb\n config:\n # Coordinates\n host_port: null\n # The name\n database: null\n # Credentials\n username: null\n include_views: true\n include_tables: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true" }, + { + "urn": "urn:li:dataPlatform:unity-catalog", + "name": "unity-catalog", + "displayName": "Databricks Unity Catalog", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/databricks/#module-unity-catalog", + "recipe": "source:\n type: unity-catalog\n config:\n # Coordinates\n workspace_url: null\n include_table_lineage: true\n include_column_lineage: false\n stateful_ingestion:\n enabled: true" + }, { "urn": "urn:li:dataPlatform:mongodb", "name": "mongodb",