feat(ingest): add HBase source connector plugin #15257
+2,107
−0
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| import { SourceConfig } from '@app/ingest/source/conf/types'; | ||
|
|
||
| import hbaseLogo from '@images/hbaselogo.png'; | ||
|
|
||
| const placeholderRecipe = `\ | ||
| source: | ||
| type: hbase | ||
| config: | ||
| # Coordinates | ||
| host: # Your HBase Thrift server host, e.g. localhost | ||
| port: 9090 # Your HBase Thrift server port (default: 9090) | ||
|
|
||
| # Optional: Filter patterns | ||
| namespace_pattern: | ||
| allow: | ||
| - ".*" # Allow all namespaces | ||
| table_pattern: | ||
| allow: | ||
| - ".*" # Allow all tables | ||
|
|
||
| # Optional: Authentication | ||
| # auth_mechanism: # Authentication mechanism (e.g., KERBEROS) | ||
|
|
||
| # Optional: Schema extraction | ||
| include_column_families: true # Include column families in schema metadata | ||
| max_column_qualifiers: 100 # Maximum column qualifiers to sample | ||
|
|
||
| stateful_ingestion: | ||
| enabled: true | ||
| `; | ||
|
|
||
| export const HBASE = 'hbase'; | ||
|
|
||
| const hbaseConfig: SourceConfig = { | ||
| type: HBASE, | ||
| placeholderRecipe, | ||
| displayName: 'HBase', | ||
| docsUrl: 'https://docs.datahub.com/docs/generated/ingestion/sources/hbase/', | ||
| logoUrl: hbaseLogo, | ||
| }; | ||
|
|
||
| export default hbaseConfig; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| import { RecipeField, FieldType, setListValuesOnRecipe } from './common'; | ||
|
|
||
| export const HBASE_HOST: RecipeField = { | ||
| name: 'host', | ||
| label: 'Host', | ||
| tooltip: 'HBase Thrift server hostname or IP address', | ||
| type: FieldType.TEXT, | ||
| fieldPath: 'source.config.host', | ||
| placeholder: 'localhost', | ||
| required: true, | ||
| rules: null, | ||
| }; | ||
|
|
||
| export const HBASE_PORT: RecipeField = { | ||
| name: 'port', | ||
| label: 'Port', | ||
| tooltip: 'HBase Thrift server port (default: 9090 for Thrift1)', | ||
| type: FieldType.TEXT, | ||
| fieldPath: 'source.config.port', | ||
| placeholder: '9090', | ||
| required: false, | ||
| rules: null, | ||
| }; | ||
|
|
||
| export const HBASE_USE_SSL: RecipeField = { | ||
| name: 'use_ssl', | ||
| label: 'Use SSL', | ||
| tooltip: 'Whether to use SSL/TLS for connection', | ||
| type: FieldType.BOOLEAN, | ||
| fieldPath: 'source.config.use_ssl', | ||
| required: false, | ||
| rules: null, | ||
| }; | ||
|
|
||
| export const HBASE_AUTH_MECHANISM: RecipeField = { | ||
| name: 'auth_mechanism', | ||
| label: 'Authentication Mechanism', | ||
| tooltip: 'Authentication mechanism (None, KERBEROS, or custom)', | ||
| type: FieldType.TEXT, | ||
| fieldPath: 'source.config.auth_mechanism', | ||
| placeholder: 'KERBEROS', | ||
| required: false, | ||
| rules: null, | ||
| }; | ||
|
|
||
| export const NAMESPACE_ALLOW: RecipeField = { | ||
| name: 'namespace_pattern.allow', | ||
| label: 'Allow Patterns for Namespace', | ||
| tooltip: | ||
| 'Use regex here. e.g. to allow all namespaces, use ".*" or to allow namespaces starting with "production" use "production.*"', | ||
| placeholder: '.*', | ||
| type: FieldType.LIST, | ||
| buttonLabel: 'Add pattern', | ||
| fieldPath: 'source.config.namespace_pattern.allow', | ||
| rules: null, | ||
| section: 'Namespaces', | ||
| setValueOnRecipeOverride: (recipe: any, values: string[]) => | ||
| setListValuesOnRecipe(recipe, values, 'source.config.namespace_pattern.allow'), | ||
| }; | ||
|
|
||
| export const NAMESPACE_DENY: RecipeField = { | ||
| name: 'namespace_pattern.deny', | ||
| label: 'Deny Patterns for Namespace', | ||
| tooltip: | ||
| 'Use regex here. Deny patterns take precedence over allow patterns. e.g. to deny all system namespaces, use "system.*"', | ||
| placeholder: 'system.*', | ||
| type: FieldType.LIST, | ||
| buttonLabel: 'Add pattern', | ||
| fieldPath: 'source.config.namespace_pattern.deny', | ||
| rules: null, | ||
| section: 'Namespaces', | ||
| setValueOnRecipeOverride: (recipe: any, values: string[]) => | ||
| setListValuesOnRecipe(recipe, values, 'source.config.namespace_pattern.deny'), | ||
| }; | ||
|
|
||
| export const HBASE_INCLUDE_COLUMN_FAMILIES: RecipeField = { | ||
| name: 'include_column_families', | ||
| label: 'Include Column Families', | ||
| tooltip: 'Whether to include column families as schema metadata', | ||
| type: FieldType.BOOLEAN, | ||
| fieldPath: 'source.config.include_column_families', | ||
| required: false, | ||
| rules: null, | ||
| }; | ||
|
|
||
| export const HBASE_MAX_COLUMN_QUALIFIERS: RecipeField = { | ||
| name: 'max_column_qualifiers', | ||
| label: 'Max Column Qualifiers', | ||
| tooltip: 'Maximum number of column qualifiers to sample per column family', | ||
| type: FieldType.TEXT, | ||
| fieldPath: 'source.config.max_column_qualifiers', | ||
| placeholder: '100', | ||
| required: false, | ||
| rules: null, | ||
| }; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| import { SourceConfig } from '@app/ingestV2/source/conf/types'; | ||
|
|
||
| import hbaseLogo from '@images/hbaselogo.png'; | ||
|
|
||
| const placeholderRecipe = `\ | ||
| source: | ||
| type: hbase | ||
| config: | ||
| # Coordinates | ||
| host: # Your HBase Thrift server host, e.g. localhost | ||
| port: 9090 # Your HBase Thrift server port (default: 9090) | ||
|
|
||
| # Optional: Filter patterns | ||
| namespace_pattern: | ||
| allow: | ||
| - ".*" # Allow all namespaces | ||
| table_pattern: | ||
| allow: | ||
| - ".*" # Allow all tables | ||
|
|
||
| # Optional: Authentication | ||
| # auth_mechanism: # Authentication mechanism (e.g., KERBEROS) | ||
|
|
||
| # Optional: Schema extraction | ||
| include_column_families: true # Include column families in schema metadata | ||
| max_column_qualifiers: 100 # Maximum column qualifiers to sample | ||
|
|
||
| stateful_ingestion: | ||
| enabled: true | ||
| `; | ||
|
|
||
| export const HBASE = 'hbase'; | ||
|
|
||
| const hbaseConfig: SourceConfig = { | ||
| type: HBASE, | ||
| placeholderRecipe, | ||
| displayName: 'HBase', | ||
| docsUrl: 'https://docs.datahub.com/docs/generated/ingestion/sources/hbase/', | ||
| logoUrl: hbaseLogo, | ||
| }; | ||
|
|
||
| export default hbaseConfig; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
Contributor
There was a problem hiding this comment. this rectangle shape may not fit well in the UI, have you checked? |
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| ### Setup | ||
|
|
||
| This integration extracts metadata from Apache HBase via the Thrift API using the `happybase` Python library, including information about: | ||
|
|
||
| - Namespaces | ||
| - Tables | ||
| - Column families and their properties | ||
| - Table configurations | ||
|
|
||
| You'll need to have HBase Thrift server running and accessible with appropriate permissions. | ||
|
|
||
| #### Prerequisites | ||
|
|
||
| 1. **Install Required Python Packages**: | ||
|
|
||
| ```bash | ||
| pip install 'acryl-datahub[hbase]' | ||
| ``` | ||
|
|
||
| This will install the `happybase` package required for connecting to HBase. | ||
|
|
||
| 2. **HBase Thrift Server**: | ||
|
|
||
| - Ensure the HBase Thrift server is running (typically on port 9090). | ||
| - Start the Thrift server if not already running: | ||
| ```bash | ||
| hbase thrift start -p 9090 | ||
| ``` | ||
|
|
||
| 3. **Network Access**: | ||
|
|
||
| - The host running DataHub ingestion must have network access to the HBase Thrift server. | ||
| - Verify connectivity: | ||
| ```bash | ||
| telnet <hbase-host> 9090 | ||
| ``` | ||
|
|
||
| 4. **Permissions**: | ||
| - The user/service account must have read access to: | ||
| - System tables for metadata extraction | ||
| - Target namespaces and tables you want to ingest | ||
|
|
||
| #### Authentication | ||
|
|
||
| The current implementation supports unauthenticated connections to HBase Thrift server. | ||
|
|
||
| :::note | ||
|
|
||
| For production deployments, it's recommended to use secure connections and ensure your HBase Thrift server is properly secured with network-level access controls. | ||
|
|
||
| ::: | ||
|
|
||
| :::info | ||
|
|
||
| The connector extracts column family metadata but does not sample individual column qualifiers. This ensures efficient metadata extraction without impacting HBase performance. | ||
|
|
||
| ::: |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
v1 frontend was deprecated, so you can skip updating anything in
datahub-web-react/src/app/ingest/source/just keep udpates in
datahub-web-react/src/app/ingestV2/