🎉 Remove hash when it is not necessary from normalization outputs #3704
Conversation
| @@ -29,6 +29,7 @@ | |||
| import pkgutil | |||
| import shutil | |||
| from enum import Enum | |||
| from typing import Any, Dict | |||
There was a problem hiding this comment.
Changes in this file are made to comply to MyPy...
|
/test connector=bases/base-normalization
|
|
|
||
| class TableNameRegistry: | ||
| """ | ||
| A registry object that records table names being used during the run |
There was a problem hiding this comment.
Amazing! Definitely easier for others to follow with this!
...integrations/bases/base-normalization/normalization/transform_catalog/table_name_registry.py
Outdated
Show resolved
Hide resolved
...integrations/bases/base-normalization/normalization/transform_catalog/table_name_registry.py
Outdated
Show resolved
Hide resolved
...integrations/bases/base-normalization/normalization/transform_catalog/table_name_registry.py
Show resolved
Hide resolved
...integrations/bases/base-normalization/normalization/transform_catalog/table_name_registry.py
Outdated
Show resolved
Hide resolved
|
|
||
| stream_processor.collect_table_names() | ||
| for conflict in tables_registry.resolve_names(): | ||
| print(f"WARN: Resolving conflict: {conflict[0]}.{conflict[1]} from '{'.'.join(conflict[2])}' into {conflict[3]}") |
There was a problem hiding this comment.
good idea to print out the changed names!
There was a problem hiding this comment.
in the future, these "exceptions/conflicts" could be collected and displayed as warnings in the UI when setting up normalization for the connection
| # before reaching the database name length limit | ||
| # 2 characters for signaling truncate with '__' and 6 others for generating unique strings | ||
| TRUNCATE_RESERVED_SIZE: int = 8 | ||
| # we keep 4 characters for 1 underscore and 3 characters for suffix (_ab1, _ab2, etc) |
|
/publish connector=bases/base-normalization
|
avaidyanatha
changed the title
What
Closes #3523
Closes #2389
How
Introduce a two-stage loop through the catalog
In the future, we could separate the first stage when setting up the connection page, so the UI could have a chance to customize how to resolve naming conflicts. (The table name registry can then be imported from a file)
Pre-merge Checklist
Recommended reading order
Ignore .sql and .json files changes
airbyte-integrations/bases/base-normalization/normalization/transform_catalog/table_name_registry.pyairbyte-integrations/bases/base-normalization/normalization/transform_catalog/catalog_processor.pyairbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py