Performance improvements for large multi assets (#11782)

dagster-io · Jan 18, 2023 · aad6e21 · aad6e21
1 parent 7490d6b
commit aad6e21
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/asset_decorator.py b/python_modules/dagster/dagster/_core/definitions/decorators/asset_decorator.py
@@ -454,7 +454,7 @@ def inner(fn: Callable[..., Any]) -> AssetsDefinition:
                 (
                     f"Invalid out key '{out_name}' supplied to `internal_asset_deps` argument for"
                     f" multi-asset {op_name}. Must be one of the outs for this multi-asset"
-                    f" {list(outs.keys())}."
+                    f" {list(outs.keys())[:20]}."
                 ),
             )
             invalid_asset_deps = asset_keys.difference(valid_asset_deps)
@@ -464,7 +464,8 @@ def inner(fn: Callable[..., Any]) -> AssetsDefinition:
                     f"Invalid asset dependencies: {invalid_asset_deps} specified in"
                     f" `internal_asset_deps` argument for multi-asset '{op_name}' on key"
                     f" '{out_name}'. Each specified asset key must be associated with an input to"
-                    f" the asset or produced by this asset. Valid keys: {valid_asset_deps}"
+                    " the asset or produced by this asset. Valid keys:"
+                    f" {list(valid_asset_deps)[:20]}"
                 ),
             )
         with warnings.catch_warnings():

diff --git a/python_modules/dagster/dagster/_core/definitions/repository_definition.py b/python_modules/dagster/dagster/_core/definitions/repository_definition.py
@@ -1477,7 +1477,7 @@ def get_asset_value_loader(
     @property
     def asset_graph(self) -> AssetGraph:
         return AssetGraph.from_assets(
-            [*self._assets_defs_by_key.values(), *self.source_assets_by_key.values()]
+            [*set(self._assets_defs_by_key.values()), *self.source_assets_by_key.values()]
         )
 
     # If definition comes from the @repository decorator, then the __call__ method will be