Skip to content
This repository has been archived by the owner on Feb 23, 2024. It is now read-only.

Commit

Permalink
Restructure the API template generation, ordering revisions depends o…
Browse files Browse the repository at this point in the history 
…n property (#782)

* Add release template, when revisions generated for single api
* Add revision to api name, if not exist. Introduce ApisResource id naming helper. 
* Add local isCurrent property to api template
* Add api releasese file with release resource for each api
* Replace explicit dependson parameter for apis in all api related extractors
* Add tests for release extractor and api data processor
* Add apiRelease parameters file

Co-authored-by: Farhad Alizada <falizada@microsoft.com>
  • Loading branch information
@f-alizada
f-alizada and Farhad Alizada committed Aug 5, 2022
1 parent 3822fcf commit 3b04442
Show file tree
Hide file tree
Showing 32 changed files with 744 additions and 96 deletions.
84 changes: 77 additions & 7 deletions src/ArmTemplates/Commands/Executors/ExtractorExecutor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.FileHandlers;
using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Abstractions;
using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.ApiManagementService;
using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.ApiReleases;
using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Apis;
using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.ApiVersionSet;
using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.AuthorizationServer;
Expand Down Expand Up @@ -70,7 +71,8 @@ public class ExtractorExecutor
readonly ISchemaExtractor schemaExtractor;
readonly IOpenIdConnectProviderExtractor openIdConnectProviderExtractor;
readonly IPolicyFragmentsExtractor policyFragmentsExtractor;

readonly IApiReleaseExtractor apiReleaseExtractor;

public ExtractorExecutor(
ILogger<ExtractorExecutor> logger,
IApisClient apisClient,
Expand All @@ -95,7 +97,8 @@ public class ExtractorExecutor
IApiManagementServiceExtractor apiManagementServiceExtractor,
ISchemaExtractor schemaExtractor,
IOpenIdConnectProviderExtractor openIdConnectProviderExtractor,
IPolicyFragmentsExtractor policyFragmentsExtractor)
IPolicyFragmentsExtractor policyFragmentsExtractor,
IApiReleaseExtractor apiReleaseExtractor)
{
this.logger = logger;
this.apisClient = apisClient;
Expand All @@ -121,6 +124,7 @@ public class ExtractorExecutor
this.schemaExtractor = schemaExtractor;
this.openIdConnectProviderExtractor = openIdConnectProviderExtractor;
this.policyFragmentsExtractor = policyFragmentsExtractor;
this.apiReleaseExtractor = apiReleaseExtractor;
}

/// <summary>
Expand Down Expand Up @@ -151,7 +155,8 @@ public class ExtractorExecutor
IApiManagementServiceExtractor apiManagementServiceExtractor = null,
ISchemaExtractor schemaExtractor = null,
IOpenIdConnectProviderExtractor openIdConnectProviderExtractor = null,
IPolicyFragmentsExtractor policyFragmentsExtractor = null)
IPolicyFragmentsExtractor policyFragmentsExtractor = null,
IApiReleaseExtractor apiReleaseExtractor = null)
=> new ExtractorExecutor(
logger,
apisClient,
Expand All @@ -176,7 +181,8 @@ public class ExtractorExecutor
apiManagementServiceExtractor,
schemaExtractor,
openIdConnectProviderExtractor,
policyFragmentsExtractor);
policyFragmentsExtractor,
apiReleaseExtractor);

public void SetExtractorParameters(ExtractorParameters extractorParameters)
{
Expand Down Expand Up @@ -478,7 +484,8 @@ public async Task<Template<GroupTemplateResources>> GenerateGroupsTemplateAsync(
Template<IdentityProviderResources> identityProviderTemplate = null,
Template<SchemaTemplateResources> schemaTemplate = null,
Template<OpenIdConnectProviderResources> openIdConnectProviderTemplate = null,
Template<PolicyFragmentsResources> policyFragmentsTemplate = null)
Template<PolicyFragmentsResources> policyFragmentsTemplate = null,
Template<ApiReleaseTemplateResources> apiReleaseTemplate = null)
{
this.RenameExistingParametersDirectory(baseFilesGenerationDirectory);

Expand All @@ -498,6 +505,7 @@ public async Task<Template<GroupTemplateResources>> GenerateGroupsTemplateAsync(
await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.SchemaParameters, schemaTemplate, mainParametersTemplate);
await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.OpenIdConnectProvidersParameters, openIdConnectProviderTemplate, mainParametersTemplate);
await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.PolicyFragmentsParameters, policyFragmentsTemplate, mainParametersTemplate);
await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.ApiReleaseParameters, apiReleaseTemplate, mainParametersTemplate);
}

public async Task GenerateResourceParametersFile<TTemplateResource>(string baseFilesGenerationDirectory, string fileName, Template<TTemplateResource> resourceTemplate, Template mainParametersTemplate) where TTemplateResource : ITemplateResources, new()
Expand Down Expand Up @@ -901,6 +909,57 @@ public async Task<Template<SchemaTemplateResources>> GenerateSchemasTemplateAsyn
}

/// <summary>
/// Generates api release template in the desired folder
/// </summary>
/// <param name="baseFilesGenerationDirectory">name of base folder where to save output files</param>
/// <returns>generated api release template</returns>
public async Task<Template<ApiReleaseTemplateResources>> GenerateApiReleaseTemplateAsync(string apiName, string baseFilesGenerationDirectory)
{
this.logger.LogInformation("Started generation of api release template...");

var apiReleaseTemplate = this.apiReleaseExtractor.GenerateSingleApiReleaseTemplate(apiName, this.extractorParameters);

if (apiReleaseTemplate?.HasResources() == true)
{
await FileWriter.SaveAsJsonAsync(
apiReleaseTemplate,
directory: baseFilesGenerationDirectory,
fileName: this.extractorParameters.FileNames.ApiRelease);
}

this.logger.LogInformation("Finished generation of apiRelease template...");
return apiReleaseTemplate;
}

/// <summary>
/// Generates api release template in the desired folder
/// </summary>
/// <param name="baseFilesGenerationDirectory">name of base folder where to save output files</param>
/// <returns>generated api release template</returns>
public async Task<Template<ApiReleaseTemplateResources>> GenerateApiReleasesTemplateAsync(string baseFilesGenerationDirectory)
{
if (!string.IsNullOrEmpty(this.extractorParameters.SingleApiName) || !this.extractorParameters.MultipleApiNames.IsNullOrEmpty())
{
this.logger.LogInformation("Skip generation of api releases template: not all apis are extracted");
return null;
}

this.logger.LogInformation("Started generation of api releases template...");

var apiReleaseTemplate = await this.apiReleaseExtractor.GenerateCurrentApiReleaseTemplate(this.extractorParameters);

if (apiReleaseTemplate?.HasResources() == true)
{
await FileWriter.SaveAsJsonAsync(
apiReleaseTemplate,
directory: baseFilesGenerationDirectory,
fileName: this.extractorParameters.FileNames.ApiRelease);
}

this.logger.LogInformation("Finished generation of apiReleases template...");
return apiReleaseTemplate;
}

/// Generates policy fragments templates in the desired folder
/// </summary>
/// <param name="baseFilesGenerationDirectory">name of base folder where to save output files</param>
Expand Down Expand Up @@ -1034,14 +1093,17 @@ async Task GenerateSingleAPIWithRevisionsTemplates()
this.logger.LogInformation("Extracting singleAPI {0} with revisions", this.extractorParameters.SingleApiName);

string currentRevision = null;
bool generateSingleApiReleaseTemplate;
List<string> revList = new List<string>();

await foreach (var apiRevision in this.apiRevisionExtractor.GetApiRevisionsAsync(this.extractorParameters.SingleApiName, this.extractorParameters))
{
generateSingleApiReleaseTemplate = false;
var apiRevisionName = apiRevision.ApiId.Split("/")[2];
if (apiRevision.IsCurrent)
{
currentRevision = apiRevisionName;
generateSingleApiReleaseTemplate = true;
}

// creating a folder for this api revision
Expand All @@ -1050,6 +1112,11 @@ await foreach (var apiRevision in this.apiRevisionExtractor.GetApiRevisionsAsync
revList.Add(apiRevisionName);

await this.GenerateTemplates(revFileFolder, singleApiName: apiRevisionName);

if (generateSingleApiReleaseTemplate)
{
await this.GenerateApiReleaseTemplateAsync(apiRevisionName, revFileFolder);
}
}

if (currentRevision is null)
Expand Down Expand Up @@ -1088,9 +1155,9 @@ await foreach (var apiRevision in this.apiRevisionExtractor.GetApiRevisionsAsync
}

var apisToExtract = await this.GetApiNamesToExtract(singleApiName, multipleApiNames);

// generate different templates using extractors and write to output
apiTemplate = apiTemplate ?? await this.GenerateApiTemplateAsync(singleApiName, multipleApiNames, baseFilesGenerationDirectory);

var globalServicePolicyTemplate = await this.GeneratePolicyTemplateAsync(baseFilesGenerationDirectory);
var productApiTemplate = await this.GenerateProductApisTemplateAsync(singleApiName, multipleApiNames, baseFilesGenerationDirectory);
var productTemplate = await this.GenerateProductsTemplateAsync(singleApiName, baseFilesGenerationDirectory, apiTemplate.TypedResources.ApiProducts);
Expand All @@ -1106,9 +1173,11 @@ await foreach (var apiRevision in this.apiRevisionExtractor.GetApiRevisionsAsync
var openIdConnectProviderTemplate = await this.GenerateOpenIdConnectProviderTemplateAsync(baseFilesGenerationDirectory);
var schemasTempate = await this.GenerateSchemasTemplateAsync(baseFilesGenerationDirectory);
var policyFragmentTemplate = await this.GeneratePolicyFragmentsTemplateAsync(apiTemplate.TypedResources.GetAllPolicies(), baseFilesGenerationDirectory);
var apiReleasesTemplate = await this.GenerateApiReleasesTemplateAsync(baseFilesGenerationDirectory);
await this.GenerateGatewayTemplateAsync(singleApiName, baseFilesGenerationDirectory);
await this.GenerateGatewayApiTemplateAsync(singleApiName, multipleApiNames, baseFilesGenerationDirectory);
await this.GenerateApiManagementServiceTemplate(baseFilesGenerationDirectory);

var parametersTemplate = await this.GenerateParametersTemplateAsync(apisToExtract, loggerTemplate.TypedResources, backendTemplate.TypedResources, namedValueTemplate.TypedResources, identityProviderTemplate.TypedResources, openIdConnectProviderTemplate.TypedResources, baseFilesGenerationDirectory);

await this.GenerateResourceParametersFiles(
Expand All @@ -1129,7 +1198,8 @@ await foreach (var apiRevision in this.apiRevisionExtractor.GetApiRevisionsAsync
identityProviderTemplate: identityProviderTemplate,
openIdConnectProviderTemplate: openIdConnectProviderTemplate,
schemaTemplate: schemasTempate,
policyFragmentsTemplate: policyFragmentTemplate);
policyFragmentsTemplate: policyFragmentTemplate,
apiReleaseTemplate: apiReleasesTemplate);

await this.GenerateMasterTemplateAsync(
baseFilesGenerationDirectory,
Expand Down
2 changes: 2 additions & 0 deletions src/ArmTemplates/Common/API/Clients/Abstractions/IApisClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ public interface IApisClient

Task<List<ApiTemplateResource>> GetAllAsync(ExtractorParameters extractorParameters);

Task<List<ApiTemplateResource>> GetAllCurrentAsync(ExtractorParameters extractorParameters);

Task<List<ApiTemplateResource>> GetAllLinkedToProductAsync(string productName, ExtractorParameters extractorParameters);

Task<List<ApiTemplateResource>> GetAllLinkedToGatewayAsync(string gatewayName, ExtractorParameters extractorParameters);
Expand Down
40 changes: 35 additions & 5 deletions src/ArmTemplates/Common/API/Clients/Apis/ApisClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,23 @@
using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Constants;
using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Apis;
using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Models;
using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Utilities.DataProcessors.Absctraction;

namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.API.Clients.Apis
{
public class ApisClient : ApiClientBase, IApisClient
{
const string GetSingleApiRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/apis/{4}?api-version={5}";
const string GetAllApisRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/apis?api-version={4}";
const string GetAllCurrentApisRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/apis?api-version={4}&$filter=isCurrent";
const string GetAllApisLinkedToProductRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/products/{4}/apis?api-version={5}";
const string GetApisLinkedToGatewayRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/gateways/{4}/apis?api-version={5}";

public ApisClient(IHttpClientFactory httpClientFactory) : base(httpClientFactory)
readonly IApiDataProcessor apiDataProcessor;

public ApisClient(IHttpClientFactory httpClientFactory, IApiDataProcessor apiDataProcessor) : base(httpClientFactory)
{
this.apiDataProcessor = apiDataProcessor;
}

public async Task<ApiTemplateResource> GetSingleAsync(string apiName, ExtractorParameters extractorParameters)
Expand All @@ -30,7 +35,10 @@ public async Task<ApiTemplateResource> GetSingleAsync(string apiName, ExtractorP
string requestUrl = string.Format(GetSingleApiRequest,
this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, apiName, GlobalConstants.ApiVersion);

return await this.GetResponseAsync<ApiTemplateResource>(azToken, requestUrl);
var api = await this.GetResponseAsync<ApiTemplateResource>(azToken, requestUrl);
this.apiDataProcessor.ProcessSingleData(api);

return api;
}

public async Task<List<ApiTemplateResource>> GetAllAsync(ExtractorParameters extractorParameters)
Expand All @@ -40,7 +48,23 @@ public async Task<List<ApiTemplateResource>> GetAllAsync(ExtractorParameters ext
string requestUrl = string.Format(GetAllApisRequest,
this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, GlobalConstants.ApiVersion);

return await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
var apis = await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
this.apiDataProcessor.ProcessData(apis);

return apis;
}

public async Task<List<ApiTemplateResource>> GetAllCurrentAsync(ExtractorParameters extractorParameters)
{
var (azToken, azSubId) = await this.Auth.GetAccessToken();

string requestUrl = string.Format(GetAllCurrentApisRequest,
this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, GlobalConstants.ApiVersion);

var apis = await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
this.apiDataProcessor.ProcessData(apis);

return apis;
}

public async Task<List<ApiTemplateResource>> GetAllLinkedToProductAsync(string productName, ExtractorParameters extractorParameters)
Expand All @@ -49,7 +73,10 @@ public async Task<List<ApiTemplateResource>> GetAllLinkedToProductAsync(string p
string requestUrl = string.Format(GetAllApisLinkedToProductRequest,
this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, productName, GlobalConstants.ApiVersion);

return await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
var apis = await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
this.apiDataProcessor.ProcessData(apis);

return apis;
}

public async Task<List<ApiTemplateResource>> GetAllLinkedToGatewayAsync(string gatewayName, ExtractorParameters extractorParameters)
Expand All @@ -58,7 +85,10 @@ public async Task<List<ApiTemplateResource>> GetAllLinkedToGatewayAsync(string g
string requestUrl = string.Format(GetApisLinkedToGatewayRequest,
this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, gatewayName, GlobalConstants.ApiVersion);

return await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
var apis = await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
this.apiDataProcessor.ProcessData(apis);

return apis;
}
}
}
22 changes: 22 additions & 0 deletions src/ArmTemplates/Common/Extensions/NamingHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

using System.Linq;
using System.Text.RegularExpressions;
using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Constants;

namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Extensions
{
Expand All @@ -13,6 +14,11 @@ public static class NamingHelper
static readonly Regex ExcludeOtherFromLettersAndDigitsRegex = new Regex("[^a-zA-Z0-9]");
static readonly Regex ExcludeOtherFromAlphaNumericsAndHyphensRegex = new Regex("[^a-zA-Z0-9-]");

// ValidDeployment name is the string following the pattern: '^[-\w\._\(\)]+$'. https://docs.microsoft.com/en-us/rest/api/resources/deployments/create-or-update?tabs=HTTP#uri-parameters
// ExcludeOtherFromValidDeploymentNameCharsRegex matches any character other than [alphanumerics + '-' + '.' + '(' + ')' + '_'];
// Example: 'template;rev=1.json' string will match the ';' and '=' chars
static readonly Regex ExcludeOtherFromValidDeploymentNameCharsRegex = new Regex(@"[^-\w\._\(\)]");

public static string GetSubstringBetweenTwoCharacters(char left, char right, string fullString)
{
var regex = new Regex($"(?<={left})(.*?)(?={right})");
Expand Down Expand Up @@ -60,5 +66,21 @@ public static string GenerateParametrizedResourceName(string parameterName, stri
{
return $"[concat(parameters('{parameterName}'), '/{resourceName}')]";
}

public static string GenerateValidDeploymentFileName(string fileName)
{
if (string.IsNullOrEmpty(fileName))
{
return string.Empty;
}

var resourceName = ExcludeOtherFromValidDeploymentNameCharsRegex.Replace(fileName, "-");
return resourceName;
}

public static string GenerateApisResourceId(string apiName)
{
return $"[resourceId('{ResourceTypeConstants.API}', parameters('{ParameterNames.ApimServiceName}'), '{apiName}')]";
}
}
}

0 comments on commit 3b04442

Please sign in to comment.