Skip to content

Refactor: Remove duplicate _has_json_arrow_type from loader.py #2221

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 4, 2025
+3 −18
Merged

Refactor: Remove duplicate _has_json_arrow_type from loader.py #2221

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7c6b0cc
Refactor: Remove duplicate _has_json_arrow_type from loader.py
shuoweil Nov 1, 2025
18f2ca6
Merge branch 'main' into shuowei-refactor-loader-api
shuoweil Nov 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 3 additions & 18 deletions bigframes/session/loader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
import google.cloud.bigquery.table
from google.cloud.bigquery_storage_v1 import types as bq_storage_types
import pandas
import pyarrow as pa

import bigframes._tools
import bigframes._tools.strings
Expand Down Expand Up @@ -1307,22 +1306,6 @@ def _transform_read_gbq_configuration(configuration: Optional[dict]) -> dict:
return configuration


def _has_json_arrow_type(arrow_type: pa.DataType) -> bool:
"""
Searches recursively for JSON array type within a PyArrow DataType.
"""
if arrow_type == bigframes.dtypes.JSON_ARROW_TYPE:
return True
if pa.types.is_list(arrow_type):
return _has_json_arrow_type(arrow_type.value_type)
if pa.types.is_struct(arrow_type):
for i in range(arrow_type.num_fields):
if _has_json_arrow_type(arrow_type.field(i).type):
return True
return False
return False


def _validate_dtype_can_load(name: str, column_type: bigframes.dtypes.Dtype):
"""
Determines whether a datatype is supported by bq load jobs.
Expand All @@ -1339,7 +1322,9 @@ def _validate_dtype_can_load(name: str, column_type: bigframes.dtypes.Dtype):
if column_type == bigframes.dtypes.JSON_DTYPE:
return

if isinstance(column_type, pandas.ArrowDtype) and _has_json_arrow_type(
if isinstance(
column_type, pandas.ArrowDtype
) and bigframes.dtypes.contains_db_dtypes_json_arrow_type(
column_type.pyarrow_dtype
):
raise NotImplementedError(
Expand Down