Skip to content

Commit

Permalink
[ISSUE #34755] do not propagate parameters on InlineSchemaLoader (#34853
Browse files Browse the repository at this point in the history
)
  • Loading branch information
maxi297 committed Feb 7, 2024
1 parent 6e4ed76 commit 3d9f70f
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,11 @@ def propagate_types_and_parameters(
propagated_component["type"] = found_type

# When there is no resolved type, we're not processing a component (likely a regular object) and don't need to propagate parameters
if "type" not in propagated_component:
# When the type refers to a json schema, we're not processing a component as well. This check is currently imperfect as there could
# be json_schema are not objects but we believe this is not likely in our case because:
# * records are Mapping so objects hence SchemaLoader root should be an object
# * connection_specification is a Mapping
if "type" not in propagated_component or self._is_json_schema_object(propagated_component):
return propagated_component

# Combines parameters defined at the current level with parameters from parent components. Parameters at the current
Expand Down Expand Up @@ -140,3 +144,7 @@ def propagate_types_and_parameters(
if current_parameters:
propagated_component[PARAMETERS_STR] = current_parameters
return propagated_component

@staticmethod
def _is_json_schema_object(propagated_component: Mapping[str, Any]) -> bool:
return propagated_component.get("type") == "object"
Original file line number Diff line number Diff line change
Expand Up @@ -347,3 +347,60 @@ def test_only_propagate_parameters_to_components():
actual_component = transformer.propagate_types_and_parameters("", component, {})

assert actual_component == expected_component


def test_do_not_propagate_parameters_on_json_schema_object():
component = {
"type": "DeclarativeStream",
"streams": [
{
"type": "DeclarativeStream",
"schema_loader": {
"type": "InlineSchemaLoader",
"schema": {
"type": "object",
"$schema": "http://json-schema.org/schema#",
"properties": {"id": {"type": "string"}},
},
},
"$parameters": {
"name": "roasters",
"primary_key": "id",
},
}
],
}

expected_component = {
"type": "DeclarativeStream",
"streams": [
{
"type": "DeclarativeStream",
"name": "roasters",
"primary_key": "id",
"schema_loader": {
"type": "InlineSchemaLoader",
"name": "roasters",
"primary_key": "id",
"schema": {
"type": "object",
"$schema": "http://json-schema.org/schema#",
"properties": {"id": {"type": "string"}},
},
"$parameters": {
"name": "roasters",
"primary_key": "id",
},
},
"$parameters": {
"name": "roasters",
"primary_key": "id",
},
}
],
}

transformer = ManifestComponentTransformer()
actual_component = transformer.propagate_types_and_parameters("", component, {})

assert actual_component == expected_component

0 comments on commit 3d9f70f

Please sign in to comment.