Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
b878987
added ComponentConstructor class
darynaishchenko Jul 24, 2025
0c27b96
updated model_to_component.py, set up RecordFilter with resolve_depen…
darynaishchenko Jul 24, 2025
2d8129b
Auto-fix lint and format issues
Jul 24, 2025
e1a4e7e
fix mypy
darynaishchenko Jul 24, 2025
2b569f5
added check for function
darynaishchenko Jul 24, 2025
bd551fc
fix typo in error message
darynaishchenko Jul 25, 2025
01d2f87
added AdditionalFlags to ComponentConstructor
darynaishchenko Jul 25, 2025
fafab4d
added ComponentConstructor to QueryProperties
darynaishchenko Jul 25, 2025
57706a7
updated RecordFilter
darynaishchenko Jul 25, 2025
bc4a36f
updated ModelToComponent
darynaishchenko Jul 25, 2025
ba33479
updated SimpleRetriever with ComponentConstructor
darynaishchenko Jul 25, 2025
3cdd426
Auto-fix lint and format issues
Jul 25, 2025
06e05b9
updated check for name
darynaishchenko Jul 25, 2025
7d06719
fix unit tests
darynaishchenko Jul 25, 2025
218dabb
Auto-fix lint and format issues
Jul 25, 2025
b212ed9
fix mypy
darynaishchenko Jul 25, 2025
2d4fdae
Merge branch 'main' into daryna/low-code/refactor-model-to-component-…
darynaishchenko Aug 5, 2025
b9e26bd
updated simple retriever
darynaishchenko Aug 5, 2025
1b02cdc
Auto-fix lint and format issues
Aug 5, 2025
3d48eb0
Merge branch 'main' into daryna/low-code/refactor-model-to-component-…
darynaishchenko Aug 12, 2025
5e71790
deleted _json_schema_type_name_to_type from ComponentConstructor class
darynaishchenko Aug 12, 2025
cb93847
Merge branch 'main' into daryna/low-code/refactor-model-to-component-…
darynaishchenko Sep 11, 2025
906d587
format fix
darynaishchenko Sep 11, 2025
2fca25a
fix imports
darynaishchenko Sep 11, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions airbyte_cdk/sources/declarative/extractors/record_filter.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,22 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#
from dataclasses import InitVar, dataclass
from typing import Any, Iterable, Mapping, Optional, Union
from typing import Any, Callable, Iterable, Mapping, Optional, Union

from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
RecordFilter as RecordFilterModel,
)
from airbyte_cdk.sources.declarative.parsers.component_constructor import (
AdditionalFlags,
ComponentConstructor,
)
from airbyte_cdk.sources.streams.concurrent.cursor import Cursor
from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState


@dataclass
class RecordFilter:
class RecordFilter(ComponentConstructor[RecordFilterModel]):
"""
Filter applied on a list of Records

Expand All @@ -22,6 +29,21 @@ class RecordFilter:
config: Config
condition: str = ""

@classmethod
def resolve_dependencies(
cls,
model: RecordFilterModel,
config: Config,
dependency_constructor: Callable[..., Any],
additional_flags: AdditionalFlags,
**kwargs: Any,
) -> Mapping[str, Any]:
return {
"condition": model.condition or "",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm trying to better understand the pattern: In an ideal world where this isn't a dataclass that exposes everything as public, would the logic of having the condition as InterpolatedBoolean be done in resolve_dependencies or would we still have this in the init?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think for now we will still have this in __init__ just to have less thing to refactor and avoid missing something. But it's a good idea and I think we consider changing the place of interpolation here or in __init__

"config": config,
"parameters": model.parameters or {},
}

def __post_init__(self, parameters: Mapping[str, Any]) -> None:
self._filter_interpolator = InterpolatedBoolean(
condition=self.condition, parameters=parameters
Expand Down
90 changes: 90 additions & 0 deletions airbyte_cdk/sources/declarative/parsers/component_constructor.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
#
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
#

from dataclasses import dataclass
from typing import Any, Callable, Generic, Mapping, Optional, Type, TypeVar

from pydantic.v1 import BaseModel

from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType
from airbyte_cdk.sources.message import MessageRepository
from airbyte_cdk.sources.types import Config

M = TypeVar("M", bound=BaseModel)


@dataclass
class AdditionalFlags:
def __init__(
self,
emit_connector_builder_messages: bool,
disable_retries: bool,
message_repository: MessageRepository,
connector_state_manager: ConnectorStateManager,
limit_pages_fetched_per_slice: Optional[int],
limit_slices_fetched: Optional[int],
):
self.emit_connector_builder_messages = emit_connector_builder_messages
self.disable_retries = disable_retries
self.message_repository = message_repository
self.connector_state_manager = connector_state_manager
self.limit_pages_fetched_per_slice = limit_pages_fetched_per_slice
self.limit_slices_fetched = limit_slices_fetched

@property
def should_limit_slices_fetched(self) -> bool:
"""
Returns True if the number of slices fetched should be limited, False otherwise.
This is used to limit the number of slices fetched during tests.
"""
return bool(self.limit_slices_fetched or self.emit_connector_builder_messages)


@dataclass
class ComponentConstructor(Generic[M]):
@classmethod
def resolve_dependencies(
cls,
model: M,
config: Config,
dependency_constructor: Callable[..., Any],
additional_flags: AdditionalFlags,
**kwargs: Any,
) -> Mapping[str, Any]:
"""
Resolves the component's dependencies, this method should be created in the component,
if there are any dependencies on other components, or we need to adopt / change / adjust / fine-tune
specific component's behavior.
"""
return {}

@classmethod
def build(
cls,
model: M,
config: Config,
dependency_constructor: Callable[..., Any],
additional_flags: AdditionalFlags,
**kwargs: Any,
) -> "ComponentConstructor[M]":
"""
Builds up the Component and it's component-specific dependencies.
Order of operations:
- build the dependencies first
- build the component with the resolved dependencies
"""

# resolve the component dependencies first
resolved_dependencies: Mapping[str, Any] = cls.resolve_dependencies(
model=model,
config=config,
dependency_constructor=dependency_constructor,
additional_flags=additional_flags,
**kwargs,
)

# returns the instance of the component class,
# with resolved dependencies and model-specific arguments.
return cls(**resolved_dependencies)
Loading
Loading