Skip to content

Commit

Permalink
Add semantic_model_origin property to LinkableElement interface
Browse files Browse the repository at this point in the history 
LinkableElements have a commonality, which is that they always have
either a semantic model where they are defined, or they are virtual
constructs (either metric queries or virtual dimensions, such as
metric_time).

In order to do predicate pushdown evaluation against source nodes,
we require the linkable element to have a singular semantic model origin.
That may be a virtual model, as in the case of metric_time or metric
elements, or it may be a concrete model, as in the case of all other
entities and dimensions.

This change provides that accessor. It is separated out for ease of
review due to the need to rename the existing `semantic_model_origin`
property. We could not re-use it, as dataclasses do not allow a mixture
of @Property decorated functions and simple properties. This has the
added benefit of differentiating between the semantic model origin,
which may be virtual, and the semantic model containing the element
definition.
  • Loading branch information
@tlento
tlento committed Jun 12, 2024
1 parent 06a1467 commit 2e4a1e1
Show file tree
Hide file tree
Showing 37 changed files with 133 additions and 91 deletions.
49 changes: 44 additions & 5 deletions metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,24 @@ def element_type(self) -> LinkableElementType:
"""The LinkableElementType describing what this instance represents."""
raise NotImplementedError

@property
@abstractmethod
def semantic_model_origin(self) -> SemanticModelReference:
"""The semantic model where this element was defined, if one exists.
If no such model exists, the element will return the VIRTUAL_SEMANTIC_MODEL_REFERENCE, as it is
either a virtual construct (e.g., metric_time) or a composite of semantic model inputs that could be used as
a virtual semantic model at some point (e.g., metric queries requested as filter inputs).
"""
raise NotImplementedError


@dataclass(frozen=True)
class LinkableDimension(LinkableElement, SerializableDataclass):
"""Describes how a dimension can be realized by joining based on entity links."""

# The semantic model where this dimension was defined.
semantic_model_origin: Optional[SemanticModelReference]
defined_in_semantic_model: Optional[SemanticModelReference]
element_name: str
dimension_type: DimensionType
entity_links: Tuple[EntityReference, ...]
Expand Down Expand Up @@ -146,19 +157,33 @@ def reference(self) -> DimensionReference: # noqa: D102
@override
def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]:
semantic_model_references = set()
if self.semantic_model_origin:
semantic_model_references.add(self.semantic_model_origin)
if self.defined_in_semantic_model:
semantic_model_references.add(self.defined_in_semantic_model)
semantic_model_references.update(self.join_path.derived_from_semantic_models)

return sorted_semantic_model_references(semantic_model_references)

@property
@override
def semantic_model_origin(self) -> SemanticModelReference:
"""Returns the semantic model reference pointing to the model where the dimension is defined.
For virtual dimensions, such as metric_time, where there is no semantic model definition we return the
virtual semantic model reference.
"""
return (
self.defined_in_semantic_model
if self.defined_in_semantic_model
else SemanticModelDerivation.VIRTUAL_SEMANTIC_MODEL_REFERENCE
)


@dataclass(frozen=True)
class LinkableEntity(LinkableElement, SerializableDataclass):
"""Describes how an entity can be realized by joining based on entity links."""

# The semantic model where this entity was defined.
semantic_model_origin: SemanticModelReference
defined_in_semantic_model: SemanticModelReference
element_name: str
properties: FrozenSet[LinkableElementProperty]
entity_links: Tuple[EntityReference, ...]
Expand All @@ -182,10 +207,15 @@ def reference(self) -> EntityReference: # noqa: D102
@property
@override
def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]:
semantic_model_references = {self.semantic_model_origin}
semantic_model_references = {self.defined_in_semantic_model}
semantic_model_references.update(self.join_path.derived_from_semantic_models)
return sorted_semantic_model_references(semantic_model_references)

@property
@override
def semantic_model_origin(self) -> SemanticModelReference:
return self.defined_in_semantic_model


# TODO: add to DSI
@dataclass(frozen=True)
Expand Down Expand Up @@ -254,6 +284,15 @@ def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]:

return sorted_semantic_model_references(semantic_model_references)

@property
@override
def semantic_model_origin(self) -> SemanticModelReference:
"""Returns the virtual semantic model reference, as metrics are not defined in semantic models.
Metrics may be used as virtual source nodes for metrics as dimensions use cases, and we represent that here.
"""
return SemanticModelDerivation.VIRTUAL_SEMANTIC_MODEL_REFERENCE

@property
def metric_to_entity_join_path(self) -> Optional[SemanticModelJoinPath]:
"""Join path used in metric subquery to join entity to metric, if needed."""
Expand Down
5 changes: 3 additions & 2 deletions metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ def intersection_by_path_key(linkable_element_sets: Sequence[LinkableElementSet]
dimensions,
key=lambda linkable_dimension: (
linkable_dimension.semantic_model_origin.semantic_model_name
if linkable_dimension.semantic_model_origin
if linkable_dimension.defined_in_semantic_model
else ""
),
)
Expand All @@ -194,7 +194,8 @@ def intersection_by_path_key(linkable_element_sets: Sequence[LinkableElementSet]
path_key_to_linkable_entities={
path_key: tuple(
sorted(
entities, key=lambda linkable_entity: linkable_entity.semantic_model_origin.semantic_model_name
entities,
key=lambda linkable_entity: linkable_entity.defined_in_semantic_model.semantic_model_name,
)
)
for path_key, entities in join_path_to_linkable_entities.items()
Expand Down
24 changes: 13 additions & 11 deletions metricflow-semantics/metricflow_semantics/model/semantics/linkable_spec_resolver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def _generate_linkable_time_dimensions(

linkable_dimensions.append(
LinkableDimension(
semantic_model_origin=semantic_model_origin,
defined_in_semantic_model=semantic_model_origin,
element_name=dimension.reference.element_name,
dimension_type=DimensionType.TIME,
entity_links=entity_links,
Expand All @@ -88,7 +88,7 @@ def _generate_linkable_time_dimensions(
if time_granularity.to_int() <= date_part.to_int():
linkable_dimensions.append(
LinkableDimension(
semantic_model_origin=semantic_model_origin,
defined_in_semantic_model=semantic_model_origin,
element_name=dimension.reference.element_name,
dimension_type=DimensionType.TIME,
entity_links=entity_links,
Expand Down Expand Up @@ -328,7 +328,7 @@ def _get_elements_in_semantic_model(self, semantic_model: SemanticModel) -> Link
for entity in semantic_model.entities:
linkable_entities.append(
LinkableEntity(
semantic_model_origin=semantic_model.reference,
defined_in_semantic_model=semantic_model.reference,
element_name=entity.reference.element_name,
entity_links=(),
join_path=SemanticModelJoinPath(
Expand All @@ -343,7 +343,7 @@ def _get_elements_in_semantic_model(self, semantic_model: SemanticModel) -> Link
continue
linkable_entities.append(
LinkableEntity(
semantic_model_origin=semantic_model.reference,
defined_in_semantic_model=semantic_model.reference,
element_name=entity.reference.element_name,
entity_links=(entity_link,),
join_path=SemanticModelJoinPath(
Expand All @@ -360,7 +360,7 @@ def _get_elements_in_semantic_model(self, semantic_model: SemanticModel) -> Link
if dimension_type is DimensionType.CATEGORICAL:
linkable_dimensions.append(
LinkableDimension(
semantic_model_origin=semantic_model.reference,
defined_in_semantic_model=semantic_model.reference,
element_name=dimension.reference.element_name,
dimension_type=DimensionType.CATEGORICAL,
entity_links=(entity_link,),
Expand Down Expand Up @@ -492,14 +492,16 @@ def _get_metric_time_elements(self, measure_reference: Optional[MeasureReference
)
path_key_to_linkable_dimensions[path_key].append(
LinkableDimension(
semantic_model_origin=measure_semantic_model.reference if measure_semantic_model else None,
defined_in_semantic_model=measure_semantic_model.reference if measure_semantic_model else None,
element_name=MetricFlowReservedKeywords.METRIC_TIME.value,
dimension_type=DimensionType.TIME,
entity_links=(),
join_path=SemanticModelJoinPath(
left_semantic_model_reference=measure_semantic_model.reference
if measure_semantic_model
else SemanticModelDerivation.VIRTUAL_SEMANTIC_MODEL_REFERENCE,
left_semantic_model_reference=(
measure_semantic_model.reference
if measure_semantic_model
else SemanticModelDerivation.VIRTUAL_SEMANTIC_MODEL_REFERENCE
),
),
# Anything that's not at the base time granularity of the measure's aggregation time dimension
# should be considered derived.
Expand Down Expand Up @@ -717,7 +719,7 @@ def create_linkable_element_set_from_join_path(
if dimension_type == DimensionType.CATEGORICAL:
linkable_dimensions.append(
LinkableDimension(
semantic_model_origin=semantic_model.reference,
defined_in_semantic_model=semantic_model.reference,
element_name=dimension.reference.element_name,
dimension_type=DimensionType.CATEGORICAL,
entity_links=join_path.entity_links,
Expand Down Expand Up @@ -745,7 +747,7 @@ def create_linkable_element_set_from_join_path(
if entity.reference != join_path.last_entity_link:
linkable_entities.append(
LinkableEntity(
semantic_model_origin=semantic_model.reference,
defined_in_semantic_model=semantic_model.reference,
element_name=entity.reference.element_name,
entity_links=join_path.entity_links,
join_path=join_path,
Expand Down
20 changes: 10 additions & 10 deletions metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_element_set.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,22 +58,22 @@
# Entities
_base_entity = LinkableEntity(
element_name=_base_entity_reference.element_name,
semantic_model_origin=_base_semantic_model,
defined_in_semantic_model=_base_semantic_model,
entity_links=(),
join_path=SemanticModelJoinPath(left_semantic_model_reference=_measure_semantic_model),
properties=frozenset([LinkableElementProperty.ENTITY]),
)
_ambiguous_entity = LinkableEntity(
element_name=AMBIGUOUS_NAME,
semantic_model_origin=_base_semantic_model,
defined_in_semantic_model=_base_semantic_model,
entity_links=(_base_entity_reference,),
join_path=SemanticModelJoinPath(left_semantic_model_reference=_measure_semantic_model),
properties=frozenset([LinkableElementProperty.ENTITY, LinkableElementProperty.LOCAL_LINKED]),
)
# For testing deduplication on entities
_ambiguous_entity_with_join_path = LinkableEntity(
element_name=AMBIGUOUS_NAME,
semantic_model_origin=_base_semantic_model,
defined_in_semantic_model=_base_semantic_model,
entity_links=(_base_entity_reference,),
join_path=SemanticModelJoinPath(
left_semantic_model_reference=_measure_semantic_model,
Expand All @@ -92,7 +92,7 @@
element_name=_base_dimension_reference.element_name,
entity_links=(_base_entity_reference,),
dimension_type=DimensionType.CATEGORICAL,
semantic_model_origin=_base_semantic_model,
defined_in_semantic_model=_base_semantic_model,
join_path=SemanticModelJoinPath(left_semantic_model_reference=_measure_semantic_model),
properties=frozenset([LinkableElementProperty.LOCAL_LINKED]),
time_granularity=None,
Expand All @@ -102,7 +102,7 @@
element_name=_time_dimension_reference.element_name,
entity_links=(_base_entity_reference,),
dimension_type=DimensionType.TIME,
semantic_model_origin=_base_semantic_model,
defined_in_semantic_model=_base_semantic_model,
join_path=SemanticModelJoinPath(left_semantic_model_reference=_measure_semantic_model),
properties=frozenset([LinkableElementProperty.LOCAL_LINKED]),
time_granularity=TimeGranularity.DAY,
Expand All @@ -113,7 +113,7 @@
element_name=AMBIGUOUS_NAME,
entity_links=(_base_entity_reference,),
dimension_type=DimensionType.CATEGORICAL,
semantic_model_origin=_secondary_semantic_model,
defined_in_semantic_model=_secondary_semantic_model,
join_path=SemanticModelJoinPath(left_semantic_model_reference=_measure_semantic_model),
properties=frozenset([LinkableElementProperty.LOCAL_LINKED]),
time_granularity=None,
Expand All @@ -125,7 +125,7 @@
element_name=AMBIGUOUS_NAME,
entity_links=(_base_entity_reference,),
dimension_type=DimensionType.CATEGORICAL,
semantic_model_origin=_secondary_semantic_model,
defined_in_semantic_model=_secondary_semantic_model,
join_path=SemanticModelJoinPath(
left_semantic_model_reference=_measure_semantic_model,
path_elements=(
Expand Down Expand Up @@ -576,7 +576,7 @@ def linkable_set() -> LinkableElementSet: # noqa: D103
element_type=LinkableElementType.DIMENSION,
): (
LinkableDimension(
semantic_model_origin=SemanticModelReference("dimension_source"),
defined_in_semantic_model=SemanticModelReference("dimension_source"),
element_name="dimension_element",
dimension_type=DimensionType.CATEGORICAL,
entity_links=(entity_0,),
Expand All @@ -601,7 +601,7 @@ def linkable_set() -> LinkableElementSet: # noqa: D103
time_granularity=TimeGranularity.DAY,
): (
LinkableDimension(
semantic_model_origin=SemanticModelReference("time_dimension_source"),
defined_in_semantic_model=SemanticModelReference("time_dimension_source"),
element_name="time_dimension_element",
dimension_type=DimensionType.TIME,
entity_links=(entity_1,),
Expand All @@ -627,7 +627,7 @@ def linkable_set() -> LinkableElementSet: # noqa: D103
element_type=LinkableElementType.ENTITY,
): (
LinkableEntity(
semantic_model_origin=SemanticModelReference("entity_source"),
defined_in_semantic_model=SemanticModelReference("entity_source"),
element_name="entity_element",
entity_links=(entity_2,),
join_path=SemanticModelJoinPath(
Expand Down
14 changes: 7 additions & 7 deletions metricflow-semantics/tests_metricflow_semantics/model/test_where_filter_spec.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ def test_dimension_in_filter( # noqa: D103
date_part=None,
): (
LinkableDimension(
semantic_model_origin=SemanticModelReference("bookings"),
defined_in_semantic_model=SemanticModelReference("bookings"),
dimension_type=DimensionType.CATEGORICAL,
element_name="country_latest",
entity_links=(EntityReference("listing"),),
Expand Down Expand Up @@ -173,7 +173,7 @@ def test_dimension_in_filter_with_grain( # noqa: D103
date_part=None,
): (
LinkableDimension(
semantic_model_origin=SemanticModelReference("listings_source"),
defined_in_semantic_model=SemanticModelReference("listings_source"),
dimension_type=DimensionType.TIME,
element_name="created_at",
entity_links=(EntityReference("listing"),),
Expand Down Expand Up @@ -237,7 +237,7 @@ def test_time_dimension_in_filter( # noqa: D103
date_part=None,
): (
LinkableDimension(
semantic_model_origin=SemanticModelReference("listings_source"),
defined_in_semantic_model=SemanticModelReference("listings_source"),
dimension_type=DimensionType.CATEGORICAL,
element_name="created_at",
entity_links=(EntityReference("listing"),),
Expand Down Expand Up @@ -302,7 +302,7 @@ def test_date_part_in_filter( # noqa: D103
date_part=DatePart.YEAR,
): (
LinkableDimension(
semantic_model_origin=SemanticModelReference("bookings"),
defined_in_semantic_model=SemanticModelReference("bookings"),
dimension_type=DimensionType.TIME,
element_name="metric_time",
entity_links=(),
Expand Down Expand Up @@ -370,7 +370,7 @@ def resolved_spec_lookup() -> FilterSpecResolutionLookUp:
date_part=DatePart.YEAR,
): (
LinkableDimension(
semantic_model_origin=SemanticModelReference("bookings"),
defined_in_semantic_model=SemanticModelReference("bookings"),
dimension_type=DimensionType.TIME,
element_name="metric_time",
entity_links=(),
Expand Down Expand Up @@ -492,7 +492,7 @@ def test_entity_in_filter( # noqa: D103
date_part=DatePart.YEAR,
): (
LinkableEntity(
semantic_model_origin=SemanticModelReference("bookings"),
defined_in_semantic_model=SemanticModelReference("bookings"),
element_name="user",
entity_links=(EntityReference("listing"),),
join_path=SemanticModelJoinPath(
Expand Down Expand Up @@ -603,7 +603,7 @@ def get_spec(dimension: str) -> WhereFilterSpec:
date_part=DatePart.YEAR,
): (
LinkableDimension(
semantic_model_origin=SemanticModelReference("bookings"),
defined_in_semantic_model=SemanticModelReference("bookings"),
dimension_type=DimensionType.TIME,
element_name=METRIC_TIME_ELEMENT_NAME,
entity_links=(),
Expand Down
2 changes: 1 addition & 1 deletion ...n/test_ambiguous_metric_time_in_query_filter__accumulate_last_2_months_metric__result.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ GroupByItemResolution(
time_granularity=MONTH,
): (
LinkableDimension(
semantic_model_origin=SemanticModelReference(
defined_in_semantic_model=SemanticModelReference(
semantic_model_name='monthly_measures_source',
),
element_name='metric_time',
Expand Down
4 changes: 2 additions & 2 deletions ...metric_time_in_query_filter__derived_metric_with_different_parent_time_grains__result.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ GroupByItemResolution(
time_granularity=MONTH,
): (
LinkableDimension(
semantic_model_origin=SemanticModelReference(
defined_in_semantic_model=SemanticModelReference(
semantic_model_name='monthly_measures_source',
),
element_name='metric_time',
Expand Down Expand Up @@ -65,7 +65,7 @@ GroupByItemResolution(
time_granularity=YEAR,
): (
LinkableDimension(
semantic_model_origin=SemanticModelReference(
defined_in_semantic_model=SemanticModelReference(
semantic_model_name='yearly_measure_source',
),
element_name='metric_time',
Expand Down
Loading

0 comments on commit 2e4a1e1

Please sign in to comment.