From 533dd88a3cb21d3e29a62818734d2ba106aab64e Mon Sep 17 00:00:00 2001 From: Shenyang Cai Date: Wed, 3 Sep 2025 17:39:49 +0000 Subject: [PATCH 1/3] chore: add logging to bigframes.bigquery functions --- bigframes/bigquery/_operations/approx_agg.py | 2 ++ bigframes/bigquery/_operations/array.py | 4 ++++ bigframes/bigquery/_operations/datetime.py | 4 ++++ bigframes/bigquery/_operations/geo.py | 10 ++++++++++ bigframes/bigquery/_operations/json.py | 10 ++++++++++ bigframes/bigquery/_operations/search.py | 4 +++- bigframes/bigquery/_operations/sql.py | 4 ++-- bigframes/bigquery/_operations/struct.py | 2 ++ 8 files changed, 37 insertions(+), 3 deletions(-) diff --git a/bigframes/bigquery/_operations/approx_agg.py b/bigframes/bigquery/_operations/approx_agg.py index 696f8f5a66..00d67ce6bf 100644 --- a/bigframes/bigquery/_operations/approx_agg.py +++ b/bigframes/bigquery/_operations/approx_agg.py @@ -14,6 +14,7 @@ from __future__ import annotations +from bigframes.core import log_adapter import bigframes.operations.aggregations as agg_ops import bigframes.series as series @@ -23,6 +24,7 @@ """ +@log_adapter.method_logger def approx_top_count( series: series.Series, number: int, diff --git a/bigframes/bigquery/_operations/array.py b/bigframes/bigquery/_operations/array.py index 4af1416127..ddda62e09e 100644 --- a/bigframes/bigquery/_operations/array.py +++ b/bigframes/bigquery/_operations/array.py @@ -24,6 +24,7 @@ import bigframes_vendored.constants as constants +from bigframes.core import log_adapter import bigframes.core.groupby as groupby import bigframes.operations as ops import bigframes.operations.aggregations as agg_ops @@ -33,6 +34,7 @@ import bigframes.dataframe as dataframe +@log_adapter.method_logger def array_length(series: series.Series) -> series.Series: """Compute the length of each array element in the Series. @@ -68,6 +70,7 @@ def array_length(series: series.Series) -> series.Series: return series._apply_unary_op(ops.len_op) +@log_adapter.method_logger def array_agg( obj: groupby.SeriesGroupBy | groupby.DataFrameGroupBy, ) -> series.Series | dataframe.DataFrame: @@ -121,6 +124,7 @@ def array_agg( ) +@log_adapter.method_logger def array_to_string(series: series.Series, delimiter: str) -> series.Series: """Converts array elements within a Series into delimited strings. diff --git a/bigframes/bigquery/_operations/datetime.py b/bigframes/bigquery/_operations/datetime.py index f8767336dd..7375442a8f 100644 --- a/bigframes/bigquery/_operations/datetime.py +++ b/bigframes/bigquery/_operations/datetime.py @@ -14,8 +14,10 @@ from bigframes import operations as ops from bigframes import series +from bigframes.core import log_adapter +@log_adapter.method_logger def unix_seconds(input: series.Series) -> series.Series: """Converts a timestmap series to unix epoch seconds @@ -43,6 +45,7 @@ def unix_seconds(input: series.Series) -> series.Series: return input._apply_unary_op(ops.UnixSeconds()) +@log_adapter.method_logger def unix_millis(input: series.Series) -> series.Series: """Converts a timestmap series to unix epoch milliseconds @@ -70,6 +73,7 @@ def unix_millis(input: series.Series) -> series.Series: return input._apply_unary_op(ops.UnixMillis()) +@log_adapter.method_logger def unix_micros(input: series.Series) -> series.Series: """Converts a timestmap series to unix epoch microseconds diff --git a/bigframes/bigquery/_operations/geo.py b/bigframes/bigquery/_operations/geo.py index 9a92a8960d..2712091141 100644 --- a/bigframes/bigquery/_operations/geo.py +++ b/bigframes/bigquery/_operations/geo.py @@ -19,6 +19,7 @@ import shapely # type: ignore from bigframes import operations as ops +from bigframes.core import log_adapter import bigframes.geopandas import bigframes.series @@ -28,6 +29,7 @@ """ +@log_adapter.method_logger def st_area( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], ) -> bigframes.series.Series: @@ -103,6 +105,7 @@ def st_area( return series +@log_adapter.method_logger def st_buffer( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], buffer_radius: float, @@ -172,6 +175,7 @@ def st_buffer( return series +@log_adapter.method_logger def st_centroid( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], ) -> bigframes.series.Series: @@ -229,6 +233,7 @@ def st_centroid( return series +@log_adapter.method_logger def st_convexhull( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], ) -> bigframes.series.Series: @@ -284,6 +289,7 @@ def st_convexhull( return series +@log_adapter.method_logger def st_difference( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], other: Union[ @@ -387,6 +393,7 @@ def st_difference( return series._apply_binary_op(other, ops.geo_st_difference_op) +@log_adapter.method_logger def st_distance( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], other: Union[ @@ -464,6 +471,7 @@ def st_distance( ) +@log_adapter.method_logger def st_intersection( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], other: Union[ @@ -563,6 +571,7 @@ def st_intersection( return series._apply_binary_op(other, ops.geo_st_intersection_op) +@log_adapter.method_logger def st_isclosed( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], ) -> bigframes.series.Series: @@ -623,6 +632,7 @@ def st_isclosed( return series +@log_adapter.method_logger def st_length( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], *, diff --git a/bigframes/bigquery/_operations/json.py b/bigframes/bigquery/_operations/json.py index 7ad7855dba..f4e9591d05 100644 --- a/bigframes/bigquery/_operations/json.py +++ b/bigframes/bigquery/_operations/json.py @@ -24,6 +24,7 @@ from typing import Any, cast, Optional, Sequence, Tuple, Union import warnings +from bigframes.core import log_adapter import bigframes.core.utils as utils import bigframes.dtypes import bigframes.exceptions as bfe @@ -33,6 +34,7 @@ from . import array +@log_adapter.method_logger @utils.preview(name="The JSON-related API `json_set`") def json_set( input: series.Series, @@ -85,6 +87,7 @@ def json_set( return result +@log_adapter.method_logger def json_extract( input: series.Series, json_path: str, @@ -125,6 +128,7 @@ def json_extract( return input._apply_unary_op(ops.JSONExtract(json_path=json_path)) +@log_adapter.method_logger def json_extract_array( input: series.Series, json_path: str = "$", @@ -184,6 +188,7 @@ def json_extract_array( return input._apply_unary_op(ops.JSONExtractArray(json_path=json_path)) +@log_adapter.method_logger def json_extract_string_array( input: series.Series, json_path: str = "$", @@ -260,6 +265,7 @@ def json_extract_string_array( return array_series +@log_adapter.method_logger def json_query( input: series.Series, json_path: str, @@ -291,6 +297,7 @@ def json_query( return input._apply_unary_op(ops.JSONQuery(json_path=json_path)) +@log_adapter.method_logger def json_query_array( input: series.Series, json_path: str = "$", @@ -341,6 +348,7 @@ def json_query_array( return input._apply_unary_op(ops.JSONQueryArray(json_path=json_path)) +@log_adapter.method_logger def json_value( input: series.Series, json_path: str = "$", @@ -375,6 +383,7 @@ def json_value( return input._apply_unary_op(ops.JSONValue(json_path=json_path)) +@log_adapter.method_logger def json_value_array( input: series.Series, json_path: str = "$", @@ -430,6 +439,7 @@ def json_value_array( return input._apply_unary_op(ops.JSONValueArray(json_path=json_path)) +@log_adapter.method_logger @utils.preview(name="The JSON-related API `parse_json`") def parse_json( input: series.Series, diff --git a/bigframes/bigquery/_operations/search.py b/bigframes/bigquery/_operations/search.py index 5063fc9118..a0fc66471d 100644 --- a/bigframes/bigquery/_operations/search.py +++ b/bigframes/bigquery/_operations/search.py @@ -20,7 +20,7 @@ import google.cloud.bigquery as bigquery -import bigframes.core.sql +from bigframes.core import log_adapter import bigframes.ml.utils as utils if typing.TYPE_CHECKING: @@ -34,6 +34,7 @@ """ +@log_adapter.method_logger def create_vector_index( table_id: str, column_name: str, @@ -89,6 +90,7 @@ def create_vector_index( read_gbq_query(sql) +@log_adapter.method_logger def vector_search( base_table: str, column_to_search: str, diff --git a/bigframes/bigquery/_operations/sql.py b/bigframes/bigquery/_operations/sql.py index a84c074e01..486ed883e6 100644 --- a/bigframes/bigquery/_operations/sql.py +++ b/bigframes/bigquery/_operations/sql.py @@ -20,14 +20,14 @@ import google.cloud.bigquery +from bigframes.core import log_adapter import bigframes.core.compile.sqlglot.sqlglot_ir as sqlglot_ir -import bigframes.core.sql -import bigframes.dataframe import bigframes.dtypes import bigframes.operations import bigframes.series +@log_adapter.method_logger def sql_scalar( sql_template: str, columns: Sequence[bigframes.series.Series], diff --git a/bigframes/bigquery/_operations/struct.py b/bigframes/bigquery/_operations/struct.py index 7cb826351c..686c073852 100644 --- a/bigframes/bigquery/_operations/struct.py +++ b/bigframes/bigquery/_operations/struct.py @@ -22,6 +22,7 @@ import typing +from bigframes.core import log_adapter import bigframes.operations as ops import bigframes.series as series @@ -29,6 +30,7 @@ import bigframes.dataframe as dataframe +@log_adapter.method_logger def struct(value: dataframe.DataFrame) -> series.Series: """Takes a DataFrame and converts it into a Series of structs with each struct entry corresponding to a DataFrame row and each struct field From 03639ebb64a8842d05099d943e4d427d14545673 Mon Sep 17 00:00:00 2001 From: Shenyang Cai Date: Wed, 3 Sep 2025 18:04:11 +0000 Subject: [PATCH 2/3] Add logs in the __init__ file --- bigframes/bigquery/__init__.py | 78 ++++++++++---------- bigframes/bigquery/_operations/approx_agg.py | 2 - bigframes/bigquery/_operations/array.py | 4 - bigframes/bigquery/_operations/datetime.py | 4 - bigframes/bigquery/_operations/geo.py | 10 --- bigframes/bigquery/_operations/json.py | 10 --- bigframes/bigquery/_operations/search.py | 3 - bigframes/bigquery/_operations/sql.py | 2 - bigframes/bigquery/_operations/struct.py | 2 - 9 files changed, 40 insertions(+), 75 deletions(-) diff --git a/bigframes/bigquery/__init__.py b/bigframes/bigquery/__init__.py index dbaea57005..8b1756ee80 100644 --- a/bigframes/bigquery/__init__.py +++ b/bigframes/bigquery/__init__.py @@ -16,6 +16,8 @@ such as array functions: https://cloud.google.com/bigquery/docs/reference/standard-sql/array_functions. """ +import sys + from bigframes.bigquery._operations.approx_agg import approx_top_count from bigframes.bigquery._operations.array import ( array_agg, @@ -52,43 +54,43 @@ from bigframes.bigquery._operations.search import create_vector_index, vector_search from bigframes.bigquery._operations.sql import sql_scalar from bigframes.bigquery._operations.struct import struct +from bigframes.core import log_adapter -__all__ = [ - # approximate aggregate ops - "approx_top_count", - # array ops - "array_agg", - "array_length", - "array_to_string", - # datetime ops - "unix_micros", - "unix_millis", - "unix_seconds", - # geo ops - "st_area", - "st_buffer", - "st_centroid", - "st_convexhull", - "st_difference", - "st_distance", - "st_intersection", - "st_isclosed", - "st_length", - # json ops - "json_extract", - "json_extract_array", - "json_extract_string_array", - "json_query", - "json_query_array", - "json_set", - "json_value", - "json_value_array", - "parse_json", - # search ops - "create_vector_index", - "vector_search", - # sql ops - "sql_scalar", - # struct ops - "struct", +_functions = [ + approx_top_count, + array_agg, + array_length, + array_to_string, + unix_micros, + unix_millis, + unix_seconds, + st_area, + st_buffer, + st_centroid, + st_convexhull, + st_difference, + st_distance, + st_intersection, + st_isclosed, + st_length, + json_extract, + json_extract_array, + json_extract_string_array, + json_query, + json_query_array, + json_set, + json_value, + json_value_array, + parse_json, + create_vector_index, + vector_search, + sql_scalar, + struct, ] + +__all__ = [f.__name__ for f in _functions] + +_module = sys.modules[__name__] +for f in _functions: + _decorated_object = log_adapter.method_logger(f, custom_base_name="bigquery") + setattr(_module, f.__name__, _decorated_object) diff --git a/bigframes/bigquery/_operations/approx_agg.py b/bigframes/bigquery/_operations/approx_agg.py index 00d67ce6bf..696f8f5a66 100644 --- a/bigframes/bigquery/_operations/approx_agg.py +++ b/bigframes/bigquery/_operations/approx_agg.py @@ -14,7 +14,6 @@ from __future__ import annotations -from bigframes.core import log_adapter import bigframes.operations.aggregations as agg_ops import bigframes.series as series @@ -24,7 +23,6 @@ """ -@log_adapter.method_logger def approx_top_count( series: series.Series, number: int, diff --git a/bigframes/bigquery/_operations/array.py b/bigframes/bigquery/_operations/array.py index ddda62e09e..4af1416127 100644 --- a/bigframes/bigquery/_operations/array.py +++ b/bigframes/bigquery/_operations/array.py @@ -24,7 +24,6 @@ import bigframes_vendored.constants as constants -from bigframes.core import log_adapter import bigframes.core.groupby as groupby import bigframes.operations as ops import bigframes.operations.aggregations as agg_ops @@ -34,7 +33,6 @@ import bigframes.dataframe as dataframe -@log_adapter.method_logger def array_length(series: series.Series) -> series.Series: """Compute the length of each array element in the Series. @@ -70,7 +68,6 @@ def array_length(series: series.Series) -> series.Series: return series._apply_unary_op(ops.len_op) -@log_adapter.method_logger def array_agg( obj: groupby.SeriesGroupBy | groupby.DataFrameGroupBy, ) -> series.Series | dataframe.DataFrame: @@ -124,7 +121,6 @@ def array_agg( ) -@log_adapter.method_logger def array_to_string(series: series.Series, delimiter: str) -> series.Series: """Converts array elements within a Series into delimited strings. diff --git a/bigframes/bigquery/_operations/datetime.py b/bigframes/bigquery/_operations/datetime.py index 7375442a8f..f8767336dd 100644 --- a/bigframes/bigquery/_operations/datetime.py +++ b/bigframes/bigquery/_operations/datetime.py @@ -14,10 +14,8 @@ from bigframes import operations as ops from bigframes import series -from bigframes.core import log_adapter -@log_adapter.method_logger def unix_seconds(input: series.Series) -> series.Series: """Converts a timestmap series to unix epoch seconds @@ -45,7 +43,6 @@ def unix_seconds(input: series.Series) -> series.Series: return input._apply_unary_op(ops.UnixSeconds()) -@log_adapter.method_logger def unix_millis(input: series.Series) -> series.Series: """Converts a timestmap series to unix epoch milliseconds @@ -73,7 +70,6 @@ def unix_millis(input: series.Series) -> series.Series: return input._apply_unary_op(ops.UnixMillis()) -@log_adapter.method_logger def unix_micros(input: series.Series) -> series.Series: """Converts a timestmap series to unix epoch microseconds diff --git a/bigframes/bigquery/_operations/geo.py b/bigframes/bigquery/_operations/geo.py index 2712091141..9a92a8960d 100644 --- a/bigframes/bigquery/_operations/geo.py +++ b/bigframes/bigquery/_operations/geo.py @@ -19,7 +19,6 @@ import shapely # type: ignore from bigframes import operations as ops -from bigframes.core import log_adapter import bigframes.geopandas import bigframes.series @@ -29,7 +28,6 @@ """ -@log_adapter.method_logger def st_area( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], ) -> bigframes.series.Series: @@ -105,7 +103,6 @@ def st_area( return series -@log_adapter.method_logger def st_buffer( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], buffer_radius: float, @@ -175,7 +172,6 @@ def st_buffer( return series -@log_adapter.method_logger def st_centroid( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], ) -> bigframes.series.Series: @@ -233,7 +229,6 @@ def st_centroid( return series -@log_adapter.method_logger def st_convexhull( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], ) -> bigframes.series.Series: @@ -289,7 +284,6 @@ def st_convexhull( return series -@log_adapter.method_logger def st_difference( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], other: Union[ @@ -393,7 +387,6 @@ def st_difference( return series._apply_binary_op(other, ops.geo_st_difference_op) -@log_adapter.method_logger def st_distance( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], other: Union[ @@ -471,7 +464,6 @@ def st_distance( ) -@log_adapter.method_logger def st_intersection( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], other: Union[ @@ -571,7 +563,6 @@ def st_intersection( return series._apply_binary_op(other, ops.geo_st_intersection_op) -@log_adapter.method_logger def st_isclosed( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], ) -> bigframes.series.Series: @@ -632,7 +623,6 @@ def st_isclosed( return series -@log_adapter.method_logger def st_length( series: Union[bigframes.series.Series, bigframes.geopandas.GeoSeries], *, diff --git a/bigframes/bigquery/_operations/json.py b/bigframes/bigquery/_operations/json.py index f4e9591d05..7ad7855dba 100644 --- a/bigframes/bigquery/_operations/json.py +++ b/bigframes/bigquery/_operations/json.py @@ -24,7 +24,6 @@ from typing import Any, cast, Optional, Sequence, Tuple, Union import warnings -from bigframes.core import log_adapter import bigframes.core.utils as utils import bigframes.dtypes import bigframes.exceptions as bfe @@ -34,7 +33,6 @@ from . import array -@log_adapter.method_logger @utils.preview(name="The JSON-related API `json_set`") def json_set( input: series.Series, @@ -87,7 +85,6 @@ def json_set( return result -@log_adapter.method_logger def json_extract( input: series.Series, json_path: str, @@ -128,7 +125,6 @@ def json_extract( return input._apply_unary_op(ops.JSONExtract(json_path=json_path)) -@log_adapter.method_logger def json_extract_array( input: series.Series, json_path: str = "$", @@ -188,7 +184,6 @@ def json_extract_array( return input._apply_unary_op(ops.JSONExtractArray(json_path=json_path)) -@log_adapter.method_logger def json_extract_string_array( input: series.Series, json_path: str = "$", @@ -265,7 +260,6 @@ def json_extract_string_array( return array_series -@log_adapter.method_logger def json_query( input: series.Series, json_path: str, @@ -297,7 +291,6 @@ def json_query( return input._apply_unary_op(ops.JSONQuery(json_path=json_path)) -@log_adapter.method_logger def json_query_array( input: series.Series, json_path: str = "$", @@ -348,7 +341,6 @@ def json_query_array( return input._apply_unary_op(ops.JSONQueryArray(json_path=json_path)) -@log_adapter.method_logger def json_value( input: series.Series, json_path: str = "$", @@ -383,7 +375,6 @@ def json_value( return input._apply_unary_op(ops.JSONValue(json_path=json_path)) -@log_adapter.method_logger def json_value_array( input: series.Series, json_path: str = "$", @@ -439,7 +430,6 @@ def json_value_array( return input._apply_unary_op(ops.JSONValueArray(json_path=json_path)) -@log_adapter.method_logger @utils.preview(name="The JSON-related API `parse_json`") def parse_json( input: series.Series, diff --git a/bigframes/bigquery/_operations/search.py b/bigframes/bigquery/_operations/search.py index a0fc66471d..c16c2af1a9 100644 --- a/bigframes/bigquery/_operations/search.py +++ b/bigframes/bigquery/_operations/search.py @@ -20,7 +20,6 @@ import google.cloud.bigquery as bigquery -from bigframes.core import log_adapter import bigframes.ml.utils as utils if typing.TYPE_CHECKING: @@ -34,7 +33,6 @@ """ -@log_adapter.method_logger def create_vector_index( table_id: str, column_name: str, @@ -90,7 +88,6 @@ def create_vector_index( read_gbq_query(sql) -@log_adapter.method_logger def vector_search( base_table: str, column_to_search: str, diff --git a/bigframes/bigquery/_operations/sql.py b/bigframes/bigquery/_operations/sql.py index 486ed883e6..a2de61fc21 100644 --- a/bigframes/bigquery/_operations/sql.py +++ b/bigframes/bigquery/_operations/sql.py @@ -20,14 +20,12 @@ import google.cloud.bigquery -from bigframes.core import log_adapter import bigframes.core.compile.sqlglot.sqlglot_ir as sqlglot_ir import bigframes.dtypes import bigframes.operations import bigframes.series -@log_adapter.method_logger def sql_scalar( sql_template: str, columns: Sequence[bigframes.series.Series], diff --git a/bigframes/bigquery/_operations/struct.py b/bigframes/bigquery/_operations/struct.py index 686c073852..7cb826351c 100644 --- a/bigframes/bigquery/_operations/struct.py +++ b/bigframes/bigquery/_operations/struct.py @@ -22,7 +22,6 @@ import typing -from bigframes.core import log_adapter import bigframes.operations as ops import bigframes.series as series @@ -30,7 +29,6 @@ import bigframes.dataframe as dataframe -@log_adapter.method_logger def struct(value: dataframe.DataFrame) -> series.Series: """Takes a DataFrame and converts it into a Series of structs with each struct entry corresponding to a DataFrame row and each struct field From 76e72d6cbc0c7bddac7a1cbadcdede02603b160c Mon Sep 17 00:00:00 2001 From: Shenyang Cai Date: Wed, 3 Sep 2025 22:09:51 +0000 Subject: [PATCH 3/3] add comments --- bigframes/bigquery/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bigframes/bigquery/__init__.py b/bigframes/bigquery/__init__.py index 8b1756ee80..32412648d6 100644 --- a/bigframes/bigquery/__init__.py +++ b/bigframes/bigquery/__init__.py @@ -57,13 +57,17 @@ from bigframes.core import log_adapter _functions = [ + # approximate aggregate ops approx_top_count, + # array ops array_agg, array_length, array_to_string, + # datetime ops unix_micros, unix_millis, unix_seconds, + # geo ops st_area, st_buffer, st_centroid, @@ -73,6 +77,7 @@ st_intersection, st_isclosed, st_length, + # json ops json_extract, json_extract_array, json_extract_string_array, @@ -82,9 +87,12 @@ json_value, json_value_array, parse_json, + # search ops create_vector_index, vector_search, + # sql ops sql_scalar, + # struct ops struct, ]