From 99521306be0fc4d8985910739aa83bb250b6ad2e Mon Sep 17 00:00:00 2001 From: Ian Buss Date: Wed, 1 May 2024 21:16:46 +0100 Subject: [PATCH 1/7] Adjust sorting function for msgspec fields to account for aliasing --- datamodel_code_generator/model/msgspec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datamodel_code_generator/model/msgspec.py b/datamodel_code_generator/model/msgspec.py index a3f96665..09486e6c 100644 --- a/datamodel_code_generator/model/msgspec.py +++ b/datamodel_code_generator/model/msgspec.py @@ -33,7 +33,7 @@ def _has_field_assignment(field: DataModelFieldBase) -> bool: - return bool(field.field) or not ( + return bool(field.field and field.name == field.original_name) or not ( field.required or (field.represented_default == 'None' and field.strip_default_none) ) From cf683e4a89f3f2578264a6d0fe5042b78c0197b8 Mon Sep 17 00:00:00 2001 From: Ian Buss Date: Thu, 9 May 2024 20:22:17 +0100 Subject: [PATCH 2/7] Add UT --- .../main_msgspec_struct_snake_case/output.py | 66 +++++++ .../openapi/api_ordered_required_fields.yaml | 182 ++++++++++++++++++ tests/test_main.py | 27 +++ 3 files changed, 275 insertions(+) create mode 100644 tests/data/expected/main/main_msgspec_struct_snake_case/output.py create mode 100644 tests/data/openapi/api_ordered_required_fields.yaml diff --git a/tests/data/expected/main/main_msgspec_struct_snake_case/output.py b/tests/data/expected/main/main_msgspec_struct_snake_case/output.py new file mode 100644 index 00000000..2303b2c2 --- /dev/null +++ b/tests/data/expected/main/main_msgspec_struct_snake_case/output.py @@ -0,0 +1,66 @@ +# generated by datamodel-codegen: +# filename: api_ordered_required_fields.yaml +# timestamp: 2019-07-26T00:00:00+00:00 + +from __future__ import annotations + +from typing import List, Optional + +from msgspec import Meta, Struct, field +from typing_extensions import Annotated + + +class Pet(Struct): + id: int + name: str + z_last: str = field(name='zLast') + tag: Optional[str] = None + + +Pets = List[Pet] + + +class User(Struct): + id: int + name: str + tag: Optional[str] = None + + +Users = List[User] + + +Id = str + + +Rules = List[str] + + +class Error(Struct): + code: int + message: str + + +class Api(Struct): + api_key: Optional[ + Annotated[str, Meta(description='To be used as a dataset parameter value')] + ] = None + api_version_number: Optional[ + Annotated[str, Meta(description='To be used as a version parameter value')] + ] = None + api_url: Optional[ + Annotated[str, Meta(description="The URL describing the dataset's fields")] + ] = None + api_documentation_url: Optional[ + Annotated[str, Meta(description='A URL to the API console for each API')] + ] = None + + +Apis = List[Api] + + +class Event(Struct): + name: Optional[str] = None + + +class Result(Struct): + event: Optional[Event] = None diff --git a/tests/data/openapi/api_ordered_required_fields.yaml b/tests/data/openapi/api_ordered_required_fields.yaml new file mode 100644 index 00000000..f3530ea3 --- /dev/null +++ b/tests/data/openapi/api_ordered_required_fields.yaml @@ -0,0 +1,182 @@ +openapi: "3.0.0" +info: + version: 1.0.0 + title: Swagger Petstore + license: + name: MIT +servers: + - url: http://petstore.swagger.io/v1 +paths: + /pets: + get: + summary: List all pets + operationId: listPets + tags: + - pets + parameters: + - name: limit + in: query + description: How many items to return at one time (max 100) + required: false + schema: + type: integer + format: int32 + responses: + '200': + description: A paged array of pets + headers: + x-next: + description: A link to the next page of responses + schema: + type: string + content: + application/json: + schema: + $ref: "#/components/schemas/Pets" + default: + description: unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + x-amazon-apigateway-integration: + uri: + Fn::Sub: arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${PythonVersionFunction.Arn}/invocations + passthroughBehavior: when_no_templates + httpMethod: POST + type: aws_proxy + post: + summary: Create a pet + operationId: createPets + tags: + - pets + responses: + '201': + description: Null response + default: + description: unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + x-amazon-apigateway-integration: + uri: + Fn::Sub: arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${PythonVersionFunction.Arn}/invocations + passthroughBehavior: when_no_templates + httpMethod: POST + type: aws_proxy + /pets/{petId}: + get: + summary: Info for a specific pet + operationId: showPetById + tags: + - pets + parameters: + - name: petId + in: path + required: true + description: The id of the pet to retrieve + schema: + type: string + responses: + '200': + description: Expected response to a valid request + content: + application/json: + schema: + $ref: "#/components/schemas/Pets" + default: + description: unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + x-amazon-apigateway-integration: + uri: + Fn::Sub: arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${PythonVersionFunction.Arn}/invocations + passthroughBehavior: when_no_templates + httpMethod: POST + type: aws_proxy +components: + schemas: + Pet: + required: + - id + - name + - zLast + properties: + id: + type: integer + format: int64 + default: 1 + name: + type: string + tag: + type: string + zLast: + type: string + Pets: + type: array + items: + $ref: "#/components/schemas/Pet" + Users: + type: array + items: + required: + - id + - name + properties: + id: + type: integer + format: int64 + name: + type: string + tag: + type: string + Id: + type: string + Rules: + type: array + items: + type: string + Error: + description: error result + required: + - code + - message + properties: + code: + type: integer + format: int32 + message: + type: string + apis: + type: array + items: + type: object + properties: + apiKey: + type: string + description: To be used as a dataset parameter value + apiVersionNumber: + type: string + description: To be used as a version parameter value + apiUrl: + type: string + format: uri + description: "The URL describing the dataset's fields" + apiDocumentationUrl: + type: string + format: uri + description: A URL to the API console for each API + Event: + type: object + description: Event object + properties: + name: + type: string + Result: + type: object + properties: + event: + $ref: '#/components/schemas/Event' diff --git a/tests/test_main.py b/tests/test_main.py index 42c39cbe..9670478c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -6139,6 +6139,33 @@ def test_main_msgspec_struct(): ) +@freeze_time('2019-07-26') +def test_main_msgspec_struct_snake_case(): + with TemporaryDirectory() as output_dir: + output_file: Path = Path(output_dir) / 'output.py' + return_code: Exit = main( + [ + '--input', + str(OPEN_API_DATA_PATH / 'api_ordered_required_fields.yaml'), + '--output', + str(output_file), + # min msgspec python version is 3.8 + '--target-python-version', + '3.8', + '--snake-case-field', + '--output-model-type', + 'msgspec.Struct', + ] + ) + assert return_code == Exit.OK + assert ( + output_file.read_text() + == ( + EXPECTED_MAIN_PATH / 'main_msgspec_struct_snake_case' / 'output.py' + ).read_text() + ) + + @freeze_time('2019-07-26') @pytest.mark.skipif( black.__version__.split('.')[0] == '19', From 39ab749fcbaf712997b12da4fd36c15dbabcb7ac Mon Sep 17 00:00:00 2001 From: Ian Buss Date: Fri, 10 May 2024 11:09:21 +0100 Subject: [PATCH 3/7] Fix msgspec jinja template to add field if required --- datamodel_code_generator/model/template/msgspec.jinja2 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/datamodel_code_generator/model/template/msgspec.jinja2 b/datamodel_code_generator/model/template/msgspec.jinja2 index 6806975a..03801adf 100644 --- a/datamodel_code_generator/model/template/msgspec.jinja2 +++ b/datamodel_code_generator/model/template/msgspec.jinja2 @@ -18,12 +18,14 @@ class {{ class_name }}: {%- if not field.annotated and field.field %} {{ field.name }}: {{ field.type_hint }} = {{ field.field }} {%- else %} - {%- if field.annotated %} + {%- if field.annotated and not field.field %} {{ field.name }}: {{ field.annotated }} + {%- elif field.annotated and field.field %} + {{ field.name }}: {{ field.annotated }} = {{ field.field }} {%- else %} {{ field.name }}: {{ field.type_hint }} {%- endif %} - {%- if not field.required or field.data_type.is_optional or field.nullable + {%- if not field.field and (not field.required or field.data_type.is_optional or field.nullable) %} = {{ field.represented_default }} {%- endif -%} {%- endif %} From 771c0ac1df71ae8f71b6d5208b3c9b4ed8b96650 Mon Sep 17 00:00:00 2001 From: Ian Buss Date: Fri, 10 May 2024 12:05:17 +0100 Subject: [PATCH 4/7] Logic update after fixing jinja template --- datamodel_code_generator/model/msgspec.py | 2 +- .../main/main_msgspec_struct_snake_case/output.py | 10 +++++----- tests/data/openapi/api_ordered_required_fields.yaml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/datamodel_code_generator/model/msgspec.py b/datamodel_code_generator/model/msgspec.py index 09486e6c..a3f96665 100644 --- a/datamodel_code_generator/model/msgspec.py +++ b/datamodel_code_generator/model/msgspec.py @@ -33,7 +33,7 @@ def _has_field_assignment(field: DataModelFieldBase) -> bool: - return bool(field.field and field.name == field.original_name) or not ( + return bool(field.field) or not ( field.required or (field.represented_default == 'None' and field.strip_default_none) ) diff --git a/tests/data/expected/main/main_msgspec_struct_snake_case/output.py b/tests/data/expected/main/main_msgspec_struct_snake_case/output.py index 2303b2c2..2bf3bbdb 100644 --- a/tests/data/expected/main/main_msgspec_struct_snake_case/output.py +++ b/tests/data/expected/main/main_msgspec_struct_snake_case/output.py @@ -13,7 +13,7 @@ class Pet(Struct): id: int name: str - z_last: str = field(name='zLast') + before_tag: str = field(name='beforeTag') tag: Optional[str] = None @@ -43,16 +43,16 @@ class Error(Struct): class Api(Struct): api_key: Optional[ Annotated[str, Meta(description='To be used as a dataset parameter value')] - ] = None + ] = field(name='apiKey') api_version_number: Optional[ Annotated[str, Meta(description='To be used as a version parameter value')] - ] = None + ] = field(name='apiVersionNumber') api_url: Optional[ Annotated[str, Meta(description="The URL describing the dataset's fields")] - ] = None + ] = field(name='apiUrl') api_documentation_url: Optional[ Annotated[str, Meta(description='A URL to the API console for each API')] - ] = None + ] = field(name='apiDocumentationUrl') Apis = List[Api] diff --git a/tests/data/openapi/api_ordered_required_fields.yaml b/tests/data/openapi/api_ordered_required_fields.yaml index f3530ea3..5588b755 100644 --- a/tests/data/openapi/api_ordered_required_fields.yaml +++ b/tests/data/openapi/api_ordered_required_fields.yaml @@ -103,7 +103,7 @@ components: required: - id - name - - zLast + - beforeTag properties: id: type: integer @@ -111,9 +111,9 @@ components: default: 1 name: type: string - tag: + beforeTag: type: string - zLast: + tag: type: string Pets: type: array From ac120c3e441e313eaf2332a2e4e10191316b101f Mon Sep 17 00:00:00 2001 From: Ian Buss Date: Fri, 10 May 2024 13:10:15 +0100 Subject: [PATCH 5/7] Add default=None for optional fields with a field() --- datamodel_code_generator/model/msgspec.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datamodel_code_generator/model/msgspec.py b/datamodel_code_generator/model/msgspec.py index a3f96665..df109be1 100644 --- a/datamodel_code_generator/model/msgspec.py +++ b/datamodel_code_generator/model/msgspec.py @@ -33,7 +33,7 @@ def _has_field_assignment(field: DataModelFieldBase) -> bool: - return bool(field.field) or not ( + return not ( field.required or (field.represented_default == 'None' and field.strip_default_none) ) @@ -174,6 +174,8 @@ def __str__(self) -> str: } if self.alias: data['name'] = self.alias + if not self.required and self.default == UNDEFINED or self.default is None: + data['default'] = None if self.default != UNDEFINED and self.default is not None: data['default'] = self.default From 0d802a4fc36c936400d79c7480987bd365b59e20 Mon Sep 17 00:00:00 2001 From: Ian Buss Date: Fri, 10 May 2024 13:52:25 +0100 Subject: [PATCH 6/7] Add explicit default=None in msgspec for optional fields without a specified default --- datamodel_code_generator/model/msgspec.py | 4 ++-- tests/data/expected/main/main_msgspec_struct/output.py | 2 +- .../main/main_msgspec_struct_snake_case/output.py | 8 ++++---- tests/data/expected/main/main_pattern_msgspec/output.py | 2 +- .../output.py | 2 +- .../expected/main/main_with_aliases_msgspec/output.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/datamodel_code_generator/model/msgspec.py b/datamodel_code_generator/model/msgspec.py index df109be1..e69bc8b3 100644 --- a/datamodel_code_generator/model/msgspec.py +++ b/datamodel_code_generator/model/msgspec.py @@ -174,11 +174,11 @@ def __str__(self) -> str: } if self.alias: data['name'] = self.alias - if not self.required and self.default == UNDEFINED or self.default is None: - data['default'] = None if self.default != UNDEFINED and self.default is not None: data['default'] = self.default + elif not self.required: + data['default'] = None if self.required: data = { diff --git a/tests/data/expected/main/main_msgspec_struct/output.py b/tests/data/expected/main/main_msgspec_struct/output.py index 8bcb8dcf..d72c120b 100644 --- a/tests/data/expected/main/main_msgspec_struct/output.py +++ b/tests/data/expected/main/main_msgspec_struct/output.py @@ -6,7 +6,7 @@ from typing import List, Optional -from msgspec import Meta, Struct +from msgspec import Meta, Struct, field from typing_extensions import Annotated diff --git a/tests/data/expected/main/main_msgspec_struct_snake_case/output.py b/tests/data/expected/main/main_msgspec_struct_snake_case/output.py index 2bf3bbdb..02139291 100644 --- a/tests/data/expected/main/main_msgspec_struct_snake_case/output.py +++ b/tests/data/expected/main/main_msgspec_struct_snake_case/output.py @@ -43,16 +43,16 @@ class Error(Struct): class Api(Struct): api_key: Optional[ Annotated[str, Meta(description='To be used as a dataset parameter value')] - ] = field(name='apiKey') + ] = field(name='apiKey', default=None) api_version_number: Optional[ Annotated[str, Meta(description='To be used as a version parameter value')] - ] = field(name='apiVersionNumber') + ] = field(name='apiVersionNumber', default=None) api_url: Optional[ Annotated[str, Meta(description="The URL describing the dataset's fields")] - ] = field(name='apiUrl') + ] = field(name='apiUrl', default=None) api_documentation_url: Optional[ Annotated[str, Meta(description='A URL to the API console for each API')] - ] = field(name='apiDocumentationUrl') + ] = field(name='apiDocumentationUrl', default=None) Apis = List[Api] diff --git a/tests/data/expected/main/main_pattern_msgspec/output.py b/tests/data/expected/main/main_pattern_msgspec/output.py index ff718d9f..6bbed397 100644 --- a/tests/data/expected/main/main_pattern_msgspec/output.py +++ b/tests/data/expected/main/main_pattern_msgspec/output.py @@ -6,7 +6,7 @@ from typing import Annotated, Optional -from msgspec import Meta, Struct +from msgspec import Meta, Struct, field class Info(Struct): diff --git a/tests/data/expected/main/main_use_annotated_with_msgspec_meta_constraints/output.py b/tests/data/expected/main/main_use_annotated_with_msgspec_meta_constraints/output.py index 7d18208e..f5cb0114 100644 --- a/tests/data/expected/main/main_use_annotated_with_msgspec_meta_constraints/output.py +++ b/tests/data/expected/main/main_use_annotated_with_msgspec_meta_constraints/output.py @@ -6,7 +6,7 @@ from typing import Annotated, List, Optional, Union -from msgspec import Meta, Struct +from msgspec import Meta, Struct, field class Pet(Struct): diff --git a/tests/data/expected/main/main_with_aliases_msgspec/output.py b/tests/data/expected/main/main_with_aliases_msgspec/output.py index 448739c4..213ec585 100644 --- a/tests/data/expected/main/main_with_aliases_msgspec/output.py +++ b/tests/data/expected/main/main_with_aliases_msgspec/output.py @@ -57,7 +57,7 @@ class Api(Struct): class Event(Struct): - name_: Optional[str] = field(name='name') + name_: Optional[str] = field(name='name', default=None) class Result(Struct): From a485fc61ebafd7c3ccb7a87fe8b670ec8ae97f11 Mon Sep 17 00:00:00 2001 From: Koudai Aono Date: Mon, 3 Jun 2024 01:01:28 +0900 Subject: [PATCH 7/7] Remove unnecessary filed import --- datamodel_code_generator/model/msgspec.py | 4 +++- tests/data/expected/main/main_msgspec_struct/output.py | 2 +- tests/data/expected/main/main_pattern_msgspec/output.py | 2 +- .../output.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/datamodel_code_generator/model/msgspec.py b/datamodel_code_generator/model/msgspec.py index e69bc8b3..1614fb7a 100644 --- a/datamodel_code_generator/model/msgspec.py +++ b/datamodel_code_generator/model/msgspec.py @@ -48,7 +48,9 @@ def import_extender(cls: Type[DataModelFieldBaseT]) -> Type[DataModelFieldBaseT] @wraps(original_imports.fget) # type: ignore def new_imports(self: DataModelFieldBaseT) -> Tuple[Import, ...]: extra_imports = [] - if self.field: + field = self.field + # TODO: Improve field detection + if field and field.startswith('field('): extra_imports.append(IMPORT_MSGSPEC_FIELD) if self.field and 'lambda: convert' in self.field: extra_imports.append(IMPORT_MSGSPEC_CONVERT) diff --git a/tests/data/expected/main/main_msgspec_struct/output.py b/tests/data/expected/main/main_msgspec_struct/output.py index d72c120b..8bcb8dcf 100644 --- a/tests/data/expected/main/main_msgspec_struct/output.py +++ b/tests/data/expected/main/main_msgspec_struct/output.py @@ -6,7 +6,7 @@ from typing import List, Optional -from msgspec import Meta, Struct, field +from msgspec import Meta, Struct from typing_extensions import Annotated diff --git a/tests/data/expected/main/main_pattern_msgspec/output.py b/tests/data/expected/main/main_pattern_msgspec/output.py index 6bbed397..ff718d9f 100644 --- a/tests/data/expected/main/main_pattern_msgspec/output.py +++ b/tests/data/expected/main/main_pattern_msgspec/output.py @@ -6,7 +6,7 @@ from typing import Annotated, Optional -from msgspec import Meta, Struct, field +from msgspec import Meta, Struct class Info(Struct): diff --git a/tests/data/expected/main/main_use_annotated_with_msgspec_meta_constraints/output.py b/tests/data/expected/main/main_use_annotated_with_msgspec_meta_constraints/output.py index f5cb0114..7d18208e 100644 --- a/tests/data/expected/main/main_use_annotated_with_msgspec_meta_constraints/output.py +++ b/tests/data/expected/main/main_use_annotated_with_msgspec_meta_constraints/output.py @@ -6,7 +6,7 @@ from typing import Annotated, List, Optional, Union -from msgspec import Meta, Struct, field +from msgspec import Meta, Struct class Pet(Struct):