From 19048cb76bf09bfd3606ca536a33627d3614066d Mon Sep 17 00:00:00 2001 From: Roja Reddy Sareddy Date: Thu, 27 Mar 2025 01:29:14 -0700 Subject: [PATCH 1/4] fix: Changes to address the field name error: json shadows a BaseModel attribute --- sample/sagemaker/2017-07-24/service-2.json | 5 ++-- .../main/code_injection/shape_dag.py | 7 ++++- src/sagemaker_core/main/shapes.py | 6 ++-- src/sagemaker_core/tools/shapes_codegen.py | 2 +- src/sagemaker_core/tools/shapes_extractor.py | 30 +++++++++++++++---- 5 files changed, 38 insertions(+), 12 deletions(-) diff --git a/sample/sagemaker/2017-07-24/service-2.json b/sample/sagemaker/2017-07-24/service-2.json index c1217419..e65deb4f 100644 --- a/sample/sagemaker/2017-07-24/service-2.json +++ b/sample/sagemaker/2017-07-24/service-2.json @@ -30859,9 +30859,10 @@ "shape":"MonitoringCsvDatasetFormat", "documentation":"

The CSV dataset used in the monitoring job.

" }, - "Json":{ + "JsonFormat":{ "shape":"MonitoringJsonDatasetFormat", - "documentation":"

The JSON dataset used in the monitoring job

" + "documentation":"

The JSON dataset used in the monitoring job

", + "alias": "json" }, "Parquet":{ "shape":"MonitoringParquetDatasetFormat", diff --git a/src/sagemaker_core/main/code_injection/shape_dag.py b/src/sagemaker_core/main/code_injection/shape_dag.py index f20540c5..07f9ed4a 100644 --- a/src/sagemaker_core/main/code_injection/shape_dag.py +++ b/src/sagemaker_core/main/code_injection/shape_dag.py @@ -11147,7 +11147,12 @@ "MonitoringDatasetFormat": { "members": [ {"name": "Csv", "shape": "MonitoringCsvDatasetFormat", "type": "structure"}, - {"name": "Json", "shape": "MonitoringJsonDatasetFormat", "type": "structure"}, + { + "alias": "json", + "name": "JsonFormat", + "shape": "MonitoringJsonDatasetFormat", + "type": "structure", + }, {"name": "Parquet", "shape": "MonitoringParquetDatasetFormat", "type": "structure"}, ], "type": "structure", diff --git a/src/sagemaker_core/main/shapes.py b/src/sagemaker_core/main/shapes.py index a1a96210..838b13a1 100644 --- a/src/sagemaker_core/main/shapes.py +++ b/src/sagemaker_core/main/shapes.py @@ -12,7 +12,7 @@ # language governing permissions and limitations under the License. import datetime -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field from typing import List, Dict, Optional, Any, Union from sagemaker_core.main.utils import Unassigned @@ -2541,12 +2541,12 @@ class MonitoringDatasetFormat(Base): Attributes ---------------------- csv: The CSV dataset used in the monitoring job. - json: The JSON dataset used in the monitoring job + json_format: The JSON dataset used in the monitoring job parquet: The Parquet dataset used in the monitoring job """ csv: Optional[MonitoringCsvDatasetFormat] = Unassigned() - json: Optional[MonitoringJsonDatasetFormat] = Unassigned() + json_format: Optional[MonitoringJsonDatasetFormat] = Field(default=Unassigned(), alias="json") parquet: Optional[MonitoringParquetDatasetFormat] = Unassigned() diff --git a/src/sagemaker_core/tools/shapes_codegen.py b/src/sagemaker_core/tools/shapes_codegen.py index 0e36a9e5..5c869eef 100644 --- a/src/sagemaker_core/tools/shapes_codegen.py +++ b/src/sagemaker_core/tools/shapes_codegen.py @@ -204,7 +204,7 @@ def generate_imports(self): """ imports = "import datetime\n" imports += "\n" - imports += "from pydantic import BaseModel, ConfigDict\n" + imports += "from pydantic import BaseModel, ConfigDict, Field\n" imports += "from typing import List, Dict, Optional, Any, Union\n" imports += "from sagemaker_core.main.utils import Unassigned" imports += "\n" diff --git a/src/sagemaker_core/tools/shapes_extractor.py b/src/sagemaker_core/tools/shapes_extractor.py index 7ca2a8d0..24d227d8 100644 --- a/src/sagemaker_core/tools/shapes_extractor.py +++ b/src/sagemaker_core/tools/shapes_extractor.py @@ -99,6 +99,9 @@ def get_shapes_dag(self): _dag[shape] = {"type": "structure", "members": []} for member, member_attrs in shape_data["members"].items(): shape_node_member = {"name": member, "shape": member_attrs["shape"]} + if member_attrs.get("alias") is not None: + shape_node_member["alias"] = member_attrs.get("alias") + member_shape_dict = _all_shapes[member_attrs["shape"]] shape_node_member["type"] = member_shape_dict["type"] _dag[shape]["members"].append(shape_node_member) @@ -218,6 +221,8 @@ def generate_shape_members(self, shape, required_override=()): # bring the required members in front ordered_members = {key: members[key] for key in required_args if key in members} ordered_members.update(members) + field_aliases = {} + for member_name, member_attrs in ordered_members.items(): member_shape_name = member_attrs["shape"] if self.combined_shapes[member_shape_name]: @@ -235,12 +240,27 @@ def generate_shape_members(self, shape, required_override=()): else: raise Exception("The Shape definition mush exist. The Json Data might be corrupt") member_name_snake_case = convert_to_snake_case(member_name) - if member_name in required_args: - init_data_body[f"{member_name_snake_case}"] = f"{member_type}" + + # Handle "alias" if present + if "alias" in member_attrs: + field_aliases[member_name_snake_case] = member_attrs["alias"] + if member_name in required_args: + init_data_body[f"{member_name_snake_case}"] = ( + f"{member_type} = Field(alias='{member_attrs['alias']}')" + ) + else: + init_data_body[f"{member_name_snake_case}"] = ( + f"Optional[{member_type}] = Field(default=Unassigned(), " + f"alias='{member_attrs['alias']}')" + ) else: - init_data_body[f"{member_name_snake_case}"] = ( - f"Optional[{member_type}] = Unassigned()" - ) + if member_name in required_args: + init_data_body[f"{member_name_snake_case}"] = f"{member_type}" + else: + init_data_body[f"{member_name_snake_case}"] = ( + f"Optional[{member_type}] = Unassigned()" + ) + return init_data_body @lru_cache From 66702cc47a8d5b40fb73a17b25b42a48540997eb Mon Sep 17 00:00:00 2001 From: Roja Reddy Sareddy Date: Thu, 27 Mar 2025 14:41:15 -0700 Subject: [PATCH 2/4] fix: Changes to address the field name error: json shadows a BaseModel attribute --- sample/sagemaker/2017-07-24/service-2.json | 5 ++- src/sagemaker_core/tools/constants.py | 2 ++ src/sagemaker_core/tools/shapes_codegen.py | 9 ++++- src/sagemaker_core/tools/shapes_extractor.py | 38 +++++++++++--------- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/sample/sagemaker/2017-07-24/service-2.json b/sample/sagemaker/2017-07-24/service-2.json index e65deb4f..c1217419 100644 --- a/sample/sagemaker/2017-07-24/service-2.json +++ b/sample/sagemaker/2017-07-24/service-2.json @@ -30859,10 +30859,9 @@ "shape":"MonitoringCsvDatasetFormat", "documentation":"

The CSV dataset used in the monitoring job.

" }, - "JsonFormat":{ + "Json":{ "shape":"MonitoringJsonDatasetFormat", - "documentation":"

The JSON dataset used in the monitoring job

", - "alias": "json" + "documentation":"

The JSON dataset used in the monitoring job

" }, "Parquet":{ "shape":"MonitoringParquetDatasetFormat", diff --git a/src/sagemaker_core/tools/constants.py b/src/sagemaker_core/tools/constants.py index bbfc3537..8986a572 100644 --- a/src/sagemaker_core/tools/constants.py +++ b/src/sagemaker_core/tools/constants.py @@ -97,3 +97,5 @@ CONFIG_SCHEMA_FILE_NAME = "config_schema.py" API_COVERAGE_JSON_FILE_PATH = os.getcwd() + "/src/sagemaker_core/tools/api_coverage.json" + +SHAPES_WITH_JSON_FIELD_ALIAS = ["MonitoringDatasetFormat"] # Shapes with field name with "json" diff --git a/src/sagemaker_core/tools/shapes_codegen.py b/src/sagemaker_core/tools/shapes_codegen.py index 5c869eef..ac5376ac 100644 --- a/src/sagemaker_core/tools/shapes_codegen.py +++ b/src/sagemaker_core/tools/shapes_codegen.py @@ -22,6 +22,7 @@ LICENCES_STRING, GENERATED_CLASSES_LOCATION, SHAPES_CODEGEN_FILE_NAME, + SHAPES_WITH_JSON_FIELD_ALIAS, ) from sagemaker_core.tools.shapes_extractor import ShapesExtractor from sagemaker_core.main.utils import ( @@ -180,7 +181,13 @@ def _generate_doc_string_for_shape(self, shape): if "members" in shape_dict: for member, member_attributes in shape_dict["members"].items(): - docstring += f"\n{convert_to_snake_case(member)}" + # Add alias if field name is json, to address the Bug: https://github.com/aws/sagemaker-python-sdk/issues/4944 + if shape in SHAPES_WITH_JSON_FIELD_ALIAS and member == "Json": + updated_member = "JsonFormat" + docstring += f"\n{convert_to_snake_case(updated_member)}" + else: + docstring += f"\n{convert_to_snake_case(member)}" + if "documentation" in member_attributes: docstring += f": {member_attributes['documentation']}" diff --git a/src/sagemaker_core/tools/shapes_extractor.py b/src/sagemaker_core/tools/shapes_extractor.py index 24d227d8..ff1f1ff2 100644 --- a/src/sagemaker_core/tools/shapes_extractor.py +++ b/src/sagemaker_core/tools/shapes_extractor.py @@ -16,7 +16,11 @@ from functools import lru_cache from typing import Optional, Any -from sagemaker_core.tools.constants import BASIC_JSON_TYPES_TO_PYTHON_TYPES, SHAPE_DAG_FILE_PATH +from sagemaker_core.tools.constants import ( + BASIC_JSON_TYPES_TO_PYTHON_TYPES, + SHAPE_DAG_FILE_PATH, + SHAPES_WITH_JSON_FIELD_ALIAS, +) from sagemaker_core.main.utils import ( reformat_file_with_black, convert_to_snake_case, @@ -99,8 +103,10 @@ def get_shapes_dag(self): _dag[shape] = {"type": "structure", "members": []} for member, member_attrs in shape_data["members"].items(): shape_node_member = {"name": member, "shape": member_attrs["shape"]} - if member_attrs.get("alias") is not None: - shape_node_member["alias"] = member_attrs.get("alias") + # Add alias if field name is json, to address the Bug: https://github.com/aws/sagemaker-python-sdk/issues/4944 + if shape in SHAPES_WITH_JSON_FIELD_ALIAS and member == "Json": + shape_node_member["name"] = "JsonFormat" + shape_node_member["alias"] = "json" member_shape_dict = _all_shapes[member_attrs["shape"]] shape_node_member["type"] = member_shape_dict["type"] @@ -239,22 +245,22 @@ def generate_shape_members(self, shape, required_override=()): member_type = BASIC_JSON_TYPES_TO_PYTHON_TYPES[member_shape_type] else: raise Exception("The Shape definition mush exist. The Json Data might be corrupt") - member_name_snake_case = convert_to_snake_case(member_name) - # Handle "alias" if present - if "alias" in member_attrs: - field_aliases[member_name_snake_case] = member_attrs["alias"] - if member_name in required_args: - init_data_body[f"{member_name_snake_case}"] = ( - f"{member_type} = Field(alias='{member_attrs['alias']}')" - ) - else: - init_data_body[f"{member_name_snake_case}"] = ( - f"Optional[{member_type}] = Field(default=Unassigned(), " - f"alias='{member_attrs['alias']}')" + is_required = member_name in required_args + # Add alias if field name is json, to address the Bug: https://github.com/aws/sagemaker-python-sdk/issues/4944 + if shape in SHAPES_WITH_JSON_FIELD_ALIAS and member_name == "Json": + updated_member_name_snake_case = "json_format" + field_aliases[updated_member_name_snake_case] = "json" + init_data_body[f"{updated_member_name_snake_case}"] = ( + ( + f"{member_type} = Field(alias='{field_aliases[updated_member_name_snake_case]}')" ) + if is_required + else f"Optional[{member_type}] = Field(default=Unassigned(), alias='json')" + ) else: - if member_name in required_args: + member_name_snake_case = convert_to_snake_case(member_name) + if is_required: init_data_body[f"{member_name_snake_case}"] = f"{member_type}" else: init_data_body[f"{member_name_snake_case}"] = ( From 54874644b0083fae50e2890ed8cfb30a6a2d7a28 Mon Sep 17 00:00:00 2001 From: Roja Reddy Sareddy Date: Thu, 27 Mar 2025 14:53:30 -0700 Subject: [PATCH 3/4] fix: rev-2 - Changes to address the field name error: json shadows a BaseModel attribute --- src/sagemaker_core/main/code_injection/shape_dag.py | 7 +------ src/sagemaker_core/main/shapes.py | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/sagemaker_core/main/code_injection/shape_dag.py b/src/sagemaker_core/main/code_injection/shape_dag.py index 07f9ed4a..f20540c5 100644 --- a/src/sagemaker_core/main/code_injection/shape_dag.py +++ b/src/sagemaker_core/main/code_injection/shape_dag.py @@ -11147,12 +11147,7 @@ "MonitoringDatasetFormat": { "members": [ {"name": "Csv", "shape": "MonitoringCsvDatasetFormat", "type": "structure"}, - { - "alias": "json", - "name": "JsonFormat", - "shape": "MonitoringJsonDatasetFormat", - "type": "structure", - }, + {"name": "Json", "shape": "MonitoringJsonDatasetFormat", "type": "structure"}, {"name": "Parquet", "shape": "MonitoringParquetDatasetFormat", "type": "structure"}, ], "type": "structure", diff --git a/src/sagemaker_core/main/shapes.py b/src/sagemaker_core/main/shapes.py index 838b13a1..914d493b 100644 --- a/src/sagemaker_core/main/shapes.py +++ b/src/sagemaker_core/main/shapes.py @@ -2541,12 +2541,12 @@ class MonitoringDatasetFormat(Base): Attributes ---------------------- csv: The CSV dataset used in the monitoring job. - json_format: The JSON dataset used in the monitoring job + json: The JSON dataset used in the monitoring job parquet: The Parquet dataset used in the monitoring job """ csv: Optional[MonitoringCsvDatasetFormat] = Unassigned() - json_format: Optional[MonitoringJsonDatasetFormat] = Field(default=Unassigned(), alias="json") + json: Optional[MonitoringJsonDatasetFormat] = Unassigned() parquet: Optional[MonitoringParquetDatasetFormat] = Unassigned() From d783bb1fc6c8632d9df690b4762c7bab2baba99e Mon Sep 17 00:00:00 2001 From: Roja Reddy Sareddy Date: Thu, 27 Mar 2025 14:56:53 -0700 Subject: [PATCH 4/4] fix: rev-2 - Changes to address the field name error: json shadows a BaseModel attribute --- src/sagemaker_core/main/shapes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sagemaker_core/main/shapes.py b/src/sagemaker_core/main/shapes.py index 914d493b..a1a96210 100644 --- a/src/sagemaker_core/main/shapes.py +++ b/src/sagemaker_core/main/shapes.py @@ -12,7 +12,7 @@ # language governing permissions and limitations under the License. import datetime -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict from typing import List, Dict, Optional, Any, Union from sagemaker_core.main.utils import Unassigned