Advanced multipart implementation

.chronus/changes/advanced-multipart-implementation-2024-7-13-12-56-54.md

@@ -0,0 +1,7 @@
+---
+changeKind: feature
+packages:
+  - "@azure-tools/typespec-python"
+---
+
+Support advanced multipart for `@multipartBody`

package.json

@@ -31,7 +31,7 @@
   "homepage": "https://github.com/Azure/autorest.python#readme",
   "devDependencies": {
     "@actions/github": "6.0.0",
-    "@azure-tools/cadl-ranch": "~0.14.1",
+    "@azure-tools/cadl-ranch": "~0.14.2",
     "@chronus/chronus": "^0.10.2",
     "@chronus/github": "^0.3.2",
     "@typespec/prettier-plugin-typespec": "~0.58.0",

packages/typespec-python/generator/pygen/codegen/models/__init__.py

@@ -30,6 +30,7 @@
     UnixTimeType,
     SdkCoreType,
     DecimalType,
+    MultiPartFileType,
 )
 from .enum_type import EnumType, EnumValue
 from .base import BaseType
@@ -149,6 +150,7 @@
     "unixtime": UnixTimeType,
     "credential": StringType,
     "sdkcore": SdkCoreType,
+    "multipartfile": MultiPartFileType,
 }
 _LOGGER = logging.getLogger(__name__)
 

packages/typespec-python/generator/pygen/codegen/models/primitive_types.py

@@ -624,3 +624,29 @@ def instance_check_template(self) -> str:
     @property
     def serialization_type(self) -> str:
         return self.name
+
+
+class MultiPartFileType(PrimitiveType):
+    def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None:
+        super().__init__(yaml_data=yaml_data, code_model=code_model)
+        self.name = "FileType"
+
+    def type_annotation(self, **kwargs: Any) -> str:
+        return self.name
+
+    def docstring_type(self, **kwargs: Any) -> str:
+        return f"~{self.code_model.namespace}._vendor.{self.name}"
+
+    def imports(self, **kwargs: Any) -> FileImport:
+        file_import = super().imports(**kwargs)
+        relative_path = "..." if kwargs.get("async_mode") else ".."
+        file_import.add_submodule_import(f"{relative_path}_vendor", self.name, ImportType.LOCAL)
+        return file_import
+
+    @property
+    def default_template_representation_declaration(self) -> str:
+        return '"filetype"' if self.code_model.for_test else "filetype"
+
+    @property
+    def instance_check_template(self) -> str:
+        return f"isinstance({{}}, {self.name})"

packages/typespec-python/generator/pygen/codegen/models/property.py

@@ -97,12 +97,9 @@ def is_base_discriminator(self) -> bool:
         return self.is_discriminator and self.is_polymorphic and cast(ConstantType, self.type).value is None
 
     def type_annotation(self, *, is_operation_file: bool = False) -> str:
-        types_type_annotation = self.type.type_annotation(is_operation_file=is_operation_file)
-        if self.is_multipart_file_input:
-            # we only support FileType or list of FileType
-            types_type_annotation = types_type_annotation.replace("bytes", "FileType")
         if self.is_base_discriminator:
             return "str"
+        types_type_annotation = self.type.type_annotation(is_operation_file=is_operation_file)
         if self.optional and self.client_default_value is None:
             return f"Optional[{types_type_annotation}]"
         return types_type_annotation
@@ -115,9 +112,6 @@ def get_json_template_representation(
         *,
         client_default_value_declaration: Optional[str] = None,
     ) -> Any:
-        if self.is_multipart_file_input:
-            file_type_str = '"filetype"' if self.code_model.for_test else "filetype"
-            return f"[{file_type_str}]" if self.type.type == "list" else file_type_str
         if self.client_default_value:
             client_default_value_declaration = self.get_declaration(self.client_default_value)
         # make sure there is no \n otherwise the json template will be invalid
@@ -156,8 +150,6 @@ def imports(self, **kwargs) -> FileImport:
                 "rest_discriminator" if self.is_discriminator else "rest_field",
                 ImportType.LOCAL,
             )
-        if self.is_multipart_file_input:
-            file_import.add_submodule_import(".._vendor", "FileType", ImportType.LOCAL)
         return file_import
 
     @classmethod

packages/typespec-python/generator/pygen/codegen/serializers/general_serializer.py

@@ -124,7 +124,6 @@ def serialize_vendor_file(self, clients: List[Client]) -> str:
             file_import.add_submodule_import("typing", "Union", ImportType.STDLIB)
             file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB)
             file_import.add_submodule_import("typing", "Mapping", ImportType.STDLIB)
-            file_import.add_submodule_import("typing", "Sequence", ImportType.STDLIB)
             file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB)
             file_import.add_submodule_import("typing", "Any", ImportType.STDLIB)
             file_import.add_submodule_import("typing", "List", ImportType.STDLIB)

packages/typespec-python/generator/pygen/codegen/templates/vendor.py.jinja2

@@ -70,8 +70,6 @@ FileType = Union[
     Tuple[Optional[str], FileContent, Optional[str]],
 ]
 
-FilesType = Union[Mapping[str, FileType], Sequence[Tuple[str, FileType]]]
-
 def serialize_multipart_data_entry(data_entry: Any) -> Any:
     if isinstance(data_entry, (list, tuple, dict, Model)):
         return json.dumps(data_entry, cls=SdkJSONEncoder, exclude_readonly=True)

packages/typespec-python/package.json

@@ -65,7 +65,7 @@
     "@azure-tools/typespec-azure-resource-manager": "~0.45.0",
     "@azure-tools/typespec-autorest": "~0.45.0",
     "@azure-tools/cadl-ranch-expect": "~0.15.1",
-    "@azure-tools/cadl-ranch-specs": "~0.35.3",
+    "@azure-tools/cadl-ranch-specs": "~0.35.4",
     "@types/js-yaml": "~4.0.5",
     "@types/node": "^18.16.3",
     "@types/yargs": "17.0.32",

packages/typespec-python/scripts/regenerate.ts

@@ -165,6 +165,9 @@ async function getSubdirectories(baseDir: string, flags: RegenerateFlags): Promi
                 // after xml support, remove this check
                 if (mainTspRelativePath.includes("xml")) return;
 
+                // after fix test generation for nested operation group, remove this check
+                if (mainTspRelativePath.includes("client-operation-group")) return;
+
                 const hasMainTsp = await promises
                     .access(mainTspPath)
                     .then(() => true)

packages/typespec-python/src/code-model.ts

@@ -13,7 +13,7 @@ import {
     UsageFlags,
     getCrossLanguagePackageId,
 } from "@azure-tools/typespec-client-generator-core";
-import { KnownTypes, emitEndpointType, getType, simpleTypesMap, typesMap } from "./types.js";
+import { KnownTypes, disableGenerationMap, emitEndpointType, getType, simpleTypesMap, typesMap } from "./types.js";
 import { emitParamBase, getImplementation, removeUnderscoresFromNamespace } from "./utils.js";
 import { emitBasicHttpMethod, emitLroHttpMethod, emitLroPagingHttpMethod, emitPagingHttpMethod } from "./http.js";
 import { PythonSdkContext } from "./lib.js";
@@ -218,25 +218,28 @@ export function emitCodeModel<TServiceOperation extends SdkServiceOperation>(
         clients: [],
         subnamespaceToClients: {},
     };
+    for (const client of sdkPackage.clients) {
+        codeModel["clients"].push(emitClient(sdkContext, client));
+        if (client.nameSpace === sdkPackage.rootNamespace) {
+        } else {
+            codeModel["subnamespaceToClients"][client.nameSpace] = emitClient(sdkContext, client);
+        }
+    }
+    // loop through models and enums since there may be some orphaned models needs to be generated
     for (const model of sdkPackage.models) {
         if (model.name === "" || (model.usage & UsageFlags.Spread) > 0) {
             continue;
         }
-        getType(sdkContext, model);
+        if (!disableGenerationMap.has(model)) {
+            getType(sdkContext, model);
+        }
     }
     for (const sdkEnum of sdkPackage.enums) {
         if (sdkEnum.usage === UsageFlags.ApiVersionEnum) {
             continue;
         }
         getType(sdkContext, sdkEnum);
     }
-    for (const client of sdkPackage.clients) {
-        codeModel["clients"].push(emitClient(sdkContext, client));
-        if (client.nameSpace === sdkPackage.rootNamespace) {
-        } else {
-            codeModel["subnamespaceToClients"][client.nameSpace] = emitClient(sdkContext, client);
-        }
-    }
     codeModel["types"] = [...typesMap.values(), ...Object.values(KnownTypes), ...simpleTypesMap.values()];
     codeModel["crossLanguagePackageId"] = ignoreDiagnostics(getCrossLanguagePackageId(sdkContext));
     return codeModel;

packages/typespec-python/src/types.ts

@@ -23,6 +23,7 @@ import { PythonSdkContext } from "./lib.js";
 
 export const typesMap = new Map<SdkType, Record<string, any>>();
 export const simpleTypesMap = new Map<string | null, Record<string, any>>();
+export const disableGenerationMap = new Set<SdkType>();
 
 export interface CredentialType {
     kind: "Credential";
@@ -34,6 +35,11 @@ export interface CredentialTypeUnion {
     types: CredentialType[];
 }
 
+interface MultiPartFileType {
+    kind: "multipartfile";
+    type: SdkType;
+}
+
 function isEmptyModel(type: SdkType): boolean {
     // object, {} will be treated as empty model, user defined empty model will not
     return (
@@ -59,7 +65,7 @@ export function getSimpleTypeResult(result: Record<string, any>): Record<string,
 
 export function getType<TServiceOperation extends SdkServiceOperation>(
     context: PythonSdkContext<TServiceOperation>,
-    type: CredentialType | CredentialTypeUnion | Type | SdkType,
+    type: CredentialType | CredentialTypeUnion | Type | SdkType | MultiPartFileType,
     fromBody = false,
 ): Record<string, any> {
     switch (type.kind) {
@@ -109,11 +115,29 @@ export function getType<TServiceOperation extends SdkServiceOperation>(
             return KnownTypes.any;
         case "nullable":
             return getType(context, type.type);
+        case "multipartfile":
+            return emitMultiPartFile(context, type);
         default:
             throw Error(`Not supported ${type.kind}`);
     }
 }
 
+function emitMultiPartFile<TServiceOperation extends SdkServiceOperation>(
+    context: PythonSdkContext<TServiceOperation>,
+    type: MultiPartFileType,
+): Record<string, any> {
+    if (type.type.kind === "array") {
+        return getSimpleTypeResult({
+            type: "list",
+            elementType: getType(context, createMultiPartFileType(type.type.valueType)),
+        });
+    }
+    return getSimpleTypeResult({
+        type: type.kind,
+        description: type.type.description,
+    });
+}
+
 function emitCredential(credential: SdkCredentialType): Record<string, any> {
     let credential_type: Record<string, any> = {};
     const scheme = credential.scheme;
@@ -173,21 +197,42 @@ function visibilityMapping(visibility?: Visibility[]): string[] | undefined {
     return result;
 }
 
+function createMultiPartFileType(type: SdkType): MultiPartFileType {
+    return { kind: "multipartfile", type };
+}
+
+function addDisableGenerationMap(type: SdkType): void {
+    if (disableGenerationMap.has(type)) return;
+
+    disableGenerationMap.add(type);
+    if (type.kind === "model" && type.baseModel) {
+        addDisableGenerationMap(type.baseModel);
+    } else if (type.kind === "array") {
+        addDisableGenerationMap(type.valueType);
+    }
+}
+
 function emitProperty<TServiceOperation extends SdkServiceOperation>(
     context: PythonSdkContext<TServiceOperation>,
     property: SdkBodyModelPropertyType,
 ): Record<string, any> {
+    const isMultipartFileInput = property.multipartOptions?.isFilePart;
+    const sourceType = isMultipartFileInput ? createMultiPartFileType(property.type) : property.type;
+    if (isMultipartFileInput) {
+        // Python convert all the type of file part to FileType so clear these models' usage so that they won't be generated
+        addDisableGenerationMap(property.type);
+    }
     return {
         clientName: camelToSnakeCase(property.name),
         wireName: property.serializedName,
-        type: getType(context, property.type),
+        type: getType(context, sourceType),
         optional: property.optional,
         description: property.description,
         addedOn: getAddedOn(context, property),
         visibility: visibilityMapping(property.visibility),
         isDiscriminator: property.discriminator,
         flatten: property.flatten,
-        isMultipartFileInput: property.isMultipartFileInput,
+        isMultipartFileInput: isMultipartFileInput,
     };
 }
 

packages/typespec-python/test/azure/generated/payload-multipart/apiview_mapping_python.json

@@ -4,18 +4,24 @@
         "payload.multipart.models.Address": "Payload.MultiPart.Address",
         "payload.multipart.models.AnonymousModelRequest": "anonymousModel.Request.anonymous",
         "payload.multipart.models.BinaryArrayPartsRequest": "Payload.MultiPart.BinaryArrayPartsRequest",
+        "payload.multipart.models.ComplexHttpPartsModelRequest": "Payload.MultiPart.ComplexHttpPartsModelRequest",
         "payload.multipart.models.ComplexPartsRequest": "Payload.MultiPart.ComplexPartsRequest",
-        "payload.multipart.models.JsonArrayPartsRequest": "Payload.MultiPart.JsonArrayPartsRequest",
+        "payload.multipart.models.FileWithHttpPartOptionalContentTypeRequest": "Payload.MultiPart.FileWithHttpPartOptionalContentTypeRequest",
+        "payload.multipart.models.FileWithHttpPartRequiredContentTypeRequest": "Payload.MultiPart.FileWithHttpPartRequiredContentTypeRequest",
+        "payload.multipart.models.FileWithHttpPartSpecificContentTypeRequest": "Payload.MultiPart.FileWithHttpPartSpecificContentTypeRequest",
         "payload.multipart.models.JsonPartRequest": "Payload.MultiPart.JsonPartRequest",
         "payload.multipart.models.MultiBinaryPartsRequest": "Payload.MultiPart.MultiBinaryPartsRequest",
         "payload.multipart.models.MultiPartRequest": "Payload.MultiPart.MultiPartRequest",
         "payload.multipart.MultiPartClient.form_data.basic": "Payload.MultiPart.FormData.basic",
         "payload.multipart.MultiPartClient.form_data.complex": "Payload.MultiPart.FormData.complex",
         "payload.multipart.MultiPartClient.form_data.json_part": "Payload.MultiPart.FormData.jsonPart",
         "payload.multipart.MultiPartClient.form_data.binary_array_parts": "Payload.MultiPart.FormData.binaryArrayParts",
-        "payload.multipart.MultiPartClient.form_data.json_array_parts": "Payload.MultiPart.FormData.jsonArrayParts",
         "payload.multipart.MultiPartClient.form_data.multi_binary_parts": "Payload.MultiPart.FormData.multiBinaryParts",
         "payload.multipart.MultiPartClient.form_data.check_file_name_and_content_type": "Payload.MultiPart.FormData.checkFileNameAndContentType",
-        "payload.multipart.MultiPartClient.form_data.anonymous_model": "Payload.MultiPart.FormData.anonymousModel"
+        "payload.multipart.MultiPartClient.form_data.anonymous_model": "Payload.MultiPart.FormData.anonymousModel",
+        "payload.multipart.MultiPartClient.form_data.file_with_http_part_specific_content_type": "Payload.MultiPart.FormData.fileWithHttpPartSpecificContentType",
+        "payload.multipart.MultiPartClient.form_data.file_with_http_part_required_content_type": "Payload.MultiPart.FormData.fileWithHttpPartRequiredContentType",
+        "payload.multipart.MultiPartClient.form_data.file_with_http_part_optional_content_type": "Payload.MultiPart.FormData.fileWithHttpPartOptionalContentType",
+        "payload.multipart.MultiPartClient.form_data.complex_with_http_part": "Payload.MultiPart.FormData.complexWithHttpPart"
     }
 }