label unreferenced types and skip them when generating code (#849)

* label non-reference schemas and skip them when generating code * add comment, simplify dfs and remove error types * also remove the parent type which is not a discriminator because go SDK aggregate all the parents properties * do not generate polymorphics helper if no reference * pump version
Azure · Jul 19, 2022 · 8fd4fda · 8fd4fda
1 parent 11501a9
commit 8fd4fda
Show file tree

Hide file tree

Showing 85 changed files with 179 additions and 4,686 deletions.
diff --git a/.scripts/regeneration.js b/.scripts/regeneration.js
@@ -10,47 +10,47 @@ const sem = require('./semaphore')(8);
 const swaggerDir = 'src/node_modules/@microsoft.azure/autorest.testserver/swagger/';
 
 const goMappings = {
-    'additionalpropsgroup': ['additionalProperties.json'],
-    'arraygroup': ['body-array.json'],
-    'azurereportgroup': ['azure-report.json'],
-    'azurespecialsgroup': ['azure-special-properties.json', '--head-as-boolean'],
-    'binarygroup': ['body-binary.json'],
-    'booleangroup': ['body-boolean.json'],
-    'bytegroup': ['body-byte.json'],
-    'complexgroup': ['body-complex.json'],
-    'complexmodelgroup': ['complex-model.json'],
-    'custombaseurlgroup': ['custom-baseUrl.json'],
-    'dategroup': ['body-date.json'],
-    'datetimegroup': ['body-datetime.json'],
-    'datetimerfc1123group': ['body-datetime-rfc1123.json'],
-    'dictionarygroup': ['body-dictionary.json'],
-    'durationgroup': ['body-duration.json'],
-    'errorsgroup': ['xms-error-responses.json'],
-    'extenumsgroup': ['extensible-enums-swagger.json'],
-    'filegroup': ['body-file.json'],
-    'formdatagroup': ['body-formdata.json'],
-    'headergroup': ['header.json'],
-    'headgroup': ['head.json', '--head-as-boolean'],
-    'httpinfrastructuregroup': ['httpInfrastructure.json', '--head-as-boolean'],
-    'integergroup': ['body-integer.json'],
-    'lrogroup': ['lro.json'],
-    'mediatypesgroup': ['media_types.json'],
-    'migroup': ['multiple-inheritance.json'],
+    'additionalpropsgroup': ['additionalProperties.json', '--remove-unreferenced-types'],
+    'arraygroup': ['body-array.json', '--remove-unreferenced-types'],
+    'azurereportgroup': ['azure-report.json', '--remove-unreferenced-types'],
+    'azurespecialsgroup': ['azure-special-properties.json', '--head-as-boolean', '--remove-unreferenced-types'],
+    'binarygroup': ['body-binary.json', '--remove-unreferenced-types'],
+    'booleangroup': ['body-boolean.json', '--remove-unreferenced-types'],
+    'bytegroup': ['body-byte.json', '--remove-unreferenced-types'],
+    'complexgroup': ['body-complex.json', '--remove-unreferenced-types'],
+    'complexmodelgroup': ['complex-model.json', '--remove-unreferenced-types'],
+    'custombaseurlgroup': ['custom-baseUrl.json', '--remove-unreferenced-types'],
+    'dategroup': ['body-date.json', '--remove-unreferenced-types'],
+    'datetimegroup': ['body-datetime.json', '--remove-unreferenced-types'],
+    'datetimerfc1123group': ['body-datetime-rfc1123.json', '--remove-unreferenced-types'],
+    'dictionarygroup': ['body-dictionary.json', '--remove-unreferenced-types'],
+    'durationgroup': ['body-duration.json', '--remove-unreferenced-types'],
+    'errorsgroup': ['xms-error-responses.json', '--remove-unreferenced-types'],
+    'extenumsgroup': ['extensible-enums-swagger.json', '--remove-unreferenced-types'],
+    'filegroup': ['body-file.json', '--remove-unreferenced-types'],
+    'formdatagroup': ['body-formdata.json', '--remove-unreferenced-types'],
+    'headergroup': ['header.json', '--remove-unreferenced-types'],
+    'headgroup': ['head.json', '--head-as-boolean', '--remove-unreferenced-types'],
+    'httpinfrastructuregroup': ['httpInfrastructure.json', '--head-as-boolean', '--remove-unreferenced-types'],
+    'integergroup': ['body-integer.json', '--remove-unreferenced-types'],
+    'lrogroup': ['lro.json', '--remove-unreferenced-types'],
+    'mediatypesgroup': ['media_types.json', '--remove-unreferenced-types'],
+    'migroup': ['multiple-inheritance.json', '--remove-unreferenced-types'],
     //'modelflatteninggroup': ['model-flattening.json'],
-    'morecustombaseurigroup': ['custom-baseUrl-more-options.json'],
-    'nonstringenumgroup': ['non-string-enum.json'],
+    'morecustombaseurigroup': ['custom-baseUrl-more-options.json', '--remove-unreferenced-types'],
+    'nonstringenumgroup': ['non-string-enum.json', '--remove-unreferenced-types'],
     'noopsgroup': ['no-operations.json'],
-    'numbergroup': ['body-number.json'],
-    'objectgroup': ['object-type.json'],
-    'optionalgroup': ['required-optional.json'],
-    'paginggroup': ['paging.json'],
-    'paramgroupinggroup': ['azure-parameter-grouping.json'],
-    'reportgroup': ['report.json'],
-    'stringgroup': ['body-string.json'],
-    'urlgroup': ['url.json'],
-    'urlmultigroup': ['url-multi-collectionFormat.json'],
-    'validationgroup': ['validation.json'],
-    'xmlgroup': ['xml-service.json'],
+    'numbergroup': ['body-number.json', '--remove-unreferenced-types'],
+    'objectgroup': ['object-type.json', '--remove-unreferenced-types'],
+    'optionalgroup': ['required-optional.json', '--remove-unreferenced-types'],
+    'paginggroup': ['paging.json', '--remove-unreferenced-types'],
+    'paramgroupinggroup': ['azure-parameter-grouping.json', '--remove-unreferenced-types'],
+    'reportgroup': ['report.json', '--remove-unreferenced-types'],
+    'stringgroup': ['body-string.json', '--remove-unreferenced-types'],
+    'urlgroup': ['url.json', '--remove-unreferenced-types'],
+    'urlmultigroup': ['url-multi-collectionFormat.json', '--remove-unreferenced-types'],
+    'validationgroup': ['validation.json', '--remove-unreferenced-types'],
+    'xmlgroup': ['xml-service.json', '--remove-unreferenced-types'],
 };
 
 const args = process.argv.slice(2);
@@ -96,10 +96,10 @@ const blobStorage = 'https://raw.githubusercontent.com/Azure/azure-rest-api-spec
 generate("azstorage", blobStorage, 'test/storage/2020-06-12/azblob', '--security=AzureKey --module="azstorage" --openapi-type="data-plane" --honor-body-placement');
 
 const network = 'https://raw.githubusercontent.com/Azure/azure-rest-api-specs/228cf296647f6e41182cee7d1a403990e6a8fe3c/specification/network/resource-manager/readme.md';
-generateFromReadme("armnetwork", network, 'package-2020-03', 'test/network/2020-03-01/armnetwork', '--module=armnetwork --azure-arm=true');
+generateFromReadme("armnetwork", network, 'package-2020-03', 'test/network/2020-03-01/armnetwork', '--module=armnetwork --azure-arm=true --remove-unreferenced-types');
 
 const compute = 'https://raw.githubusercontent.com/Azure/azure-rest-api-specs/228cf296647f6e41182cee7d1a403990e6a8fe3c/specification/compute/resource-manager/readme.md';
-generateFromReadme("armcompute", compute, 'package-2019-12-01', 'test/compute/2019-12-01/armcompute', '--module=armcompute --azure-arm=true');
+generateFromReadme("armcompute", compute, 'package-2019-12-01', 'test/compute/2019-12-01/armcompute', '--module=armcompute --azure-arm=true --remove-unreferenced-types');
 
 const synapseArtifacts = 'https://raw.githubusercontent.com/Azure/azure-rest-api-specs/228cf296647f6e41182cee7d1a403990e6a8fe3c/specification/synapse/data-plane/readme.md';
 generateFromReadme("azartifacts", synapseArtifacts, 'package-artifacts-2019-06-01-preview', 'test/synapse/2019-06-01/azartifacts', '--security=AADToken --security-scopes="https://dev.azuresynapse.net/.default" --module="azartifacts" --openapi-type="data-plane"');
@@ -114,10 +114,10 @@ const keyvault = fullPath('test/keyvault/7.2/azkeyvault/autorest.md');
 generateFromReadme("azkeyvault", keyvault, 'package-7.2', 'test/keyvault/7.2/azkeyvault');
 
 const consumption = 'https://raw.githubusercontent.com/Azure/azure-rest-api-specs/3865f04d22e82db481be0727b406021d29cd2b70/specification/consumption/resource-manager/readme.md';
-generateFromReadme("armconsumption", consumption, 'package-2019-10', 'test/consumption/2019-10-01/armconsumption', '--module=armconsumption --azure-arm=true');
+generateFromReadme("armconsumption", consumption, 'package-2019-10', 'test/consumption/2019-10-01/armconsumption', '--module=armconsumption --azure-arm=true --remove-unreferenced-types');
 
 const databoxedge = fullPath('test/databoxedge/2021-02-01/armdataboxedge/autorest.md');
-generateFromReadme("armdataboxedge", databoxedge, 'package-2021-02-01', 'test/databoxedge/2021-02-01/armdataboxedge');
+generateFromReadme("armdataboxedge", databoxedge, 'package-2021-02-01', 'test/databoxedge/2021-02-01/armdataboxedge', '--remove-unreferenced-types');
 
 generate("azalias", 'test/swagger/alias.json', 'test/maps/azalias', '--security=AzureKey --module="azalias" --openapi-type="data-plane"');
 

diff --git a/src/common/helpers.ts b/src/common/helpers.ts
@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { ArraySchema, BinaryResponse, ConstantSchema, DictionarySchema, ObjectSchema, Operation, Parameter, Response, Schema, SchemaContext, SchemaResponse, SchemaType } from '@autorest/codemodel';
+import { ArraySchema, BinaryResponse, ConstantSchema, DictionarySchema, ObjectSchema, Operation, Parameter, Property, Response, Schema, SchemaContext, SchemaResponse, SchemaType } from '@autorest/codemodel';
 import { values } from '@azure-tools/linq';
 
 // variable to be used to determine comment length when calling comment from @azure-tools
@@ -199,3 +199,29 @@ export function formatConstantValue(schema: ConstantSchema): string {
 export function isOutputOnly(obj: ObjectSchema): boolean {
   return !values(obj.usage).any((u) => { return u === SchemaContext.Input});
 }
+
+// aggregate the properties from the provided type and its parent types
+export function aggregateProperties(obj: ObjectSchema): Array<Property> {
+  const allProps = new Array<Property>();
+  for (const prop of values(obj.properties)) {
+    allProps.push(prop);
+  }
+  for (const parent of values(obj.parents?.all)) {
+    if (isObjectSchema(parent)) {
+      for (const parentProp of values(parent.properties)) {
+        // ensure that the parent doesn't contain any properties with the same name but different type
+        const exists = values(allProps).where(p => { return p.language.go!.name === parentProp.language.go!.name; }).first();
+        if (exists) {
+          if (exists.schema.language.go!.name !== parentProp.schema.language.go!.name) {
+            const msg = `type ${obj.language.go!.name} contains duplicate property ${exists.language.go!.name} with mismatched types`;
+            throw new Error(msg);
+          }
+          // don't add the duplicate
+          continue;
+        }
+        allProps.push(parentProp);
+      }
+    }
+  }
+  return allProps;
+}
diff --git a/src/generator/constants.ts b/src/generator/constants.ts
@@ -74,6 +74,9 @@ function getEnums(schemas: Schemas): EnumEntry[] {
   // group all enum categories into a single array so they can be sorted
   const enums = new Array<EnumEntry>();
   for (const choice of values(schemas.choices)) {
+    if (choice.language.go!.omitType) {
+      continue;
+    }
     choice.choices.sort((a: ChoiceValue, b: ChoiceValue) => { return sortAscending(a.language.go!.name, b.language.go!.name); });
     const entry = new EnumEntry(choice.language.go!.name, choice.choiceType.language.go!.name, choice.language.go!.possibleValuesFunc, choice.choices);
     if (hasDescription(choice.language.go!)) {
@@ -82,6 +85,9 @@ function getEnums(schemas: Schemas): EnumEntry[] {
     enums.push(entry);
   }
   for (const choice of values(schemas.sealedChoices)) {
+    if (choice.language.go!.omitType) {
+      continue;
+    }
     if (choice.choices.length === 1) {
       continue;
     }

diff --git a/src/generator/models.ts b/src/generator/models.ts
@@ -5,9 +5,9 @@
 
 import { Session } from '@autorest/extension-base';
 import { capitalize, comment } from '@azure-tools/codegen';
-import { ByteArraySchema, CodeModel, ConstantSchema, DictionarySchema, GroupProperty, ObjectSchema, Language, SchemaType, Parameter, Property } from '@autorest/codemodel';
+import { ByteArraySchema, CodeModel, ConstantSchema, DictionarySchema, GroupProperty, ObjectSchema, Language, SchemaType, Parameter } from '@autorest/codemodel';
 import { values } from '@azure-tools/linq';
-import { formatConstantValue, isArraySchema, isDictionarySchema, isObjectSchema, commentLength } from '../common/helpers';
+import { formatConstantValue, isArraySchema, isDictionarySchema, isObjectSchema, commentLength, aggregateProperties } from '../common/helpers';
 import { contentPreamble, sortAscending } from './helpers';
 import { ImportManager } from './imports';
 import { generateStruct, getXMLSerialization, StructDef, StructMethod } from './structs';
@@ -183,32 +183,6 @@ function generateParamGroupStruct(imports: ImportManager, lang: Language, params
   return st;
 }
 
-// aggregate the properties from the provided type and its parent types
-function aggregateProperties(obj: ObjectSchema): Array<Property> {
-  const allProps = new Array<Property>();
-  for (const prop of values(obj.properties)) {
-    allProps.push(prop);
-  }
-  for (const parent of values(obj.parents?.all)) {
-    if (isObjectSchema(parent)) {
-      for (const parentProp of values(parent.properties)) {
-        // ensure that the parent doesn't contain any properties with the same name but different type
-        const exists = values(allProps).where(p => { return p.language.go!.name === parentProp.language.go!.name}).first();
-        if (exists) {
-          if (exists.schema.language.go!.name !== parentProp.schema.language.go!.name) {
-            const msg = `type ${obj.language.go!.name} contains duplicate property ${exists.language.go!.name} with mismatched types`;
-            throw new Error(msg);
-          }
-          // don't add the duplicate
-          continue;
-        }
-        allProps.push(parentProp);
-      }
-    }
-  }
-  return allProps;
-}
-
 // generates discriminator marker method
 function generateDiscriminatorMarkerMethod(obj: ObjectSchema, structDef: StructDef) {
   const typeName = obj.language.go!.name;

diff --git a/src/generator/polymorphics.ts b/src/generator/polymorphics.ts
@@ -16,11 +16,15 @@ export async function generatePolymorphicHelpers(session: Session<CodeModel>): P
     // no polymorphic types
     return '';
   }
+  const discriminators = <Array<ObjectSchema>>session.model.language.go!.discriminators.filter((d: ObjectSchema) => !d.language.go!.omitType);
+  if (discriminators.length === 0) {
+    // all polymorphic types omitted
+    return '';
+  }
   let text = await contentPreamble(session);
   const imports = new ImportManager();
   imports.add('encoding/json');
   text += imports.text();
-  const discriminators = <Array<ObjectSchema>>session.model.language.go!.discriminators;
   // add any sub-hierarchies (SalmonType, SharkType in the test server) to the list
   for (const disc of values(discriminators)) {
     for (const val of values(disc.discriminator!.all)) {

diff --git a/src/package.json b/src/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@autorest/go",
-  "version": "4.0.0-preview.43",
+  "version": "4.0.0-preview.44",
   "description": "AutoRest Go Generator",
   "main": "dist/exports.js",
   "typings": "dist/exports.d.ts",

diff --git a/src/transform/transform.ts b/src/transform/transform.ts
@@ -7,7 +7,7 @@ import { capitalize, KnownMediaType, serialize } from '@azure-tools/codegen';
 import { AutorestExtensionHost, startSession, Session } from '@autorest/extension-base';
 import { AnySchema, ObjectSchema, ArraySchema, ByteArraySchema, ChoiceValue, codeModelSchema, CodeModel, DateTimeSchema, GroupProperty, HttpHeader, HttpResponse, ImplementationLocation, Language, OperationGroup, SchemaType, NumberSchema, Operation, Parameter, Property, Protocols, Response, Schema, DictionarySchema, Protocol, ChoiceSchema, SealedChoiceSchema, ConstantSchema, Request, BooleanSchema, BinarySchema, StringSchema } from '@autorest/codemodel';
 import { clone, items, values } from '@azure-tools/linq';
-import { aggregateParameters, formatConstantValue, getSchemaResponse, hasAdditionalProperties, isBinaryResponseOperation, isMultiRespOperation, isTypePassedByValue, isObjectSchema, isSchemaResponse, isLROOperation, isOutputOnly, isPageableOperation } from '../common/helpers';
+import { aggregateParameters, formatConstantValue, getSchemaResponse, hasAdditionalProperties, isBinaryResponseOperation, isMultiRespOperation, isTypePassedByValue, isObjectSchema, isSchemaResponse, isLROOperation, isOutputOnly, isPageableOperation, isArraySchema, isDictionarySchema, aggregateProperties } from '../common/helpers';
 import { namer, protocolMethods } from './namer';
 import { fromString } from 'html-to-text';
 import { Converter } from 'showdown';
@@ -22,6 +22,7 @@ export async function transform(host: AutorestExtensionHost) {
     // run the namer first, so that any transformations are applied on proper names
     await namer(session);
     await process(session);
+    await labelUnreferencedTypes(session);
 
     // output the model to the pipeline
     host.writeFile({
@@ -938,3 +939,96 @@ function parseComments(comment: string): string {
     tables: true,
   });
 }
+
+
+// This function try to label all unreferenced types before doing transform.
+async function labelUnreferencedTypes(session: Session<CodeModel>) {
+  const model = session.model;
+
+  const isRemoveUnreferencedTypes = await session.getValue('remove-unreferenced-types', false);
+
+  if (!isRemoveUnreferencedTypes) return;
+
+  const referencedTypes = new Set<Schema>();
+
+  iterateOperations(model, referencedTypes);
+
+  labelOmitTypes(model.schemas.choices, referencedTypes);
+  labelOmitTypes(model.schemas.sealedChoices, referencedTypes);
+  labelOmitTypes(model.schemas.objects, referencedTypes);
+}
+
+// This function help to label omit types according to the referencedTypes set.
+function labelOmitTypes<Type extends Schema>(modelSchemas: Array<Type> | undefined, referencedSet: Set<Type>) {
+  if (modelSchemas){
+    for (const schema of modelSchemas){
+      if (!referencedSet.has(schema)){
+        schema.language.go!.omitType = true;
+      }
+    }
+  }
+}
+
+// This function iterate all the operations in all operation groups and try to find all the referenced types (including choices, sealedChoices and objects).
+// For each operation, we will aggregate all the parameters' type of the operation and put them into the referencedTypes set.
+// Also, all responses body types and header types will be put into the referencedTypes set.
+// Some special cases: exceptions response types, x-ms-odata types.
+function iterateOperations(model: CodeModel, referencedTypes: Set<Schema>) {
+  for (const group of values(model.operationGroups)) {
+    for (const op of values(group.operations)) {
+      for (const param of values(aggregateParameters(op))) {
+        dfsSchema(param.schema, referencedTypes);
+      }
+      // We do not use error responses' definition to unmarshal response. So exceptions will be ignored.
+      // const responses = values(op.responses).concat(values(op.exceptions));
+      const responses = values(op.responses);
+      for (const resp of values(responses)) {
+        if (isSchemaResponse(resp)) {
+          dfsSchema(resp.schema, referencedTypes);
+        }
+        const httpResponse = <HttpResponse>resp.protocol.http;
+        for (const header of values(httpResponse.headers)) {
+          dfsSchema(header.schema, referencedTypes);
+        }
+      }
+      // Such odata definition is not used directly in operation. But it seems the model is a detailed definition for some string param. Reserve for now.
+      if (op.extensions?.['x-ms-odata']) {
+        const schemaParts = op.extensions['x-ms-odata'].split('/');
+        const schema = values(model.schemas.objects).where((o) => o.language.default.name === schemaParts[schemaParts.length - 1]).first();
+        if (schema) {
+          dfsSchema(schema, referencedTypes);
+        }
+      }
+    }
+  }
+}
+
+// This function will do a depth first search for the root types.
+// All visited types will be put into referencedTypes set.
+// Objects children/parents will also be searched.
+function dfsSchema(schema: Schema, referencedTypes: Set<Schema>) {
+  if (referencedTypes.has(schema)) return;
+  referencedTypes.add(schema);
+  if (isObjectSchema(schema)) {
+    const allProps = aggregateProperties(schema);
+    for (const prop of allProps){
+      dfsSchema(prop.schema, referencedTypes);
+    }
+    // If schema is a discriminator, then reserve all the children
+    if (schema.discriminator) {
+      for (const child of values(schema.children?.all)) {
+        dfsSchema(child, referencedTypes);
+      }
+    }
+    // If schema is has allOf, then reserve all the parents
+    for (const parent of values(schema.parents?.immediate)) {
+      if (isObjectSchema(parent) && parent.discriminator) {
+        dfsSchema(parent, referencedTypes);
+      }
+    }
+  } else if (isArraySchema(schema)) {
+    dfsSchema(schema.elementType, referencedTypes);
+  } else if (isDictionarySchema(schema)) {
+    dfsSchema(schema.elementType, referencedTypes);
+  }
+}