From b179ebf76107a6285c978067b79f544e995df66f Mon Sep 17 00:00:00 2001 From: "mariano.pizarro" Date: Tue, 6 Jun 2023 10:01:06 -0300 Subject: [PATCH 01/20] feat(azure): Support missing storageBlob and storageContainer services --- src/services/storageBlob/data.ts | 4 ++++ src/services/storageBlob/format.ts | 10 ++++++++++ src/services/storageBlob/schema.graphql | 1 + src/services/storageContainer/format.ts | 8 ++++++++ src/services/storageContainer/schema.graphql | 1 + src/types/generated.ts | 2 ++ 6 files changed, 26 insertions(+) diff --git a/src/services/storageBlob/data.ts b/src/services/storageBlob/data.ts index 44b54186..70cc50a1 100644 --- a/src/services/storageBlob/data.ts +++ b/src/services/storageBlob/data.ts @@ -17,6 +17,8 @@ export interface RawAzureStorageBlob extends Omit { region: string resourceGroupId: string storageContainerId: string + storageAccountName: string + storageContainerName: string Tags: TagMap } @@ -75,6 +77,8 @@ export default async ({ storageBlobData.push({ ...blob, storageContainerId, + storageAccountName: accountName, + storageContainerName: containerName, resourceGroupId: storageContainer.resourceGroupId, region: storageContainer.region, Tags: tags, diff --git a/src/services/storageBlob/format.ts b/src/services/storageBlob/format.ts index 9ec2b313..1d934523 100644 --- a/src/services/storageBlob/format.ts +++ b/src/services/storageBlob/format.ts @@ -15,6 +15,13 @@ import { RawAzureStorageBlob } from './data' import t from '../../properties/translations' import { formatTagsFromMap } from '../../utils/format' +const formatBlobUrl = ( + storageAccountName: string, + storageContainerName: string, + blobName: string +): string => + `https://${storageAccountName}.blob.core.windows.net/${storageContainerName}/${blobName}` + const formatReplicationRule = ({ ruleId, replicationStatus, @@ -148,6 +155,8 @@ export default ({ isCurrentVersion, hasVersionsOnly, storageContainerId, + storageAccountName, + storageContainerName, Tags = {}, objectReplicationSourceProperties, resourceGroupId, @@ -156,6 +165,7 @@ export default ({ return { id: `${storageContainerId}/${name}`, name, + url: formatBlobUrl(storageAccountName, storageContainerName, name), region, subscriptionId: account, snapshot, diff --git a/src/services/storageBlob/schema.graphql b/src/services/storageBlob/schema.graphql index e8984500..10df18ce 100644 --- a/src/services/storageBlob/schema.graphql +++ b/src/services/storageBlob/schema.graphql @@ -74,6 +74,7 @@ type azureStorageBlob implements azureResource mutation: { add: true, delete: false } ) @key(fields: "id") { + url: String @search(by: [hash, regexp]) deleted: String @search(by: [hash, regexp]) snapshot: String @search(by: [hash, regexp]) versionId: String @search(by: [hash, regexp]) diff --git a/src/services/storageContainer/format.ts b/src/services/storageContainer/format.ts index 0db7f682..ff87548f 100644 --- a/src/services/storageContainer/format.ts +++ b/src/services/storageContainer/format.ts @@ -8,6 +8,12 @@ import { import { RawAzureStorageContainer } from './data' import t from '../../properties/translations' +const formatContainerUrl = ( + storageAccountName: string, + containerName: string +): string => + `https://${storageAccountName}.blob.core.windows.net/${containerName}/` + const formatImmutabilityPolicyUpdateHistory = ({ update, immutabilityPeriodSinceCreationInDays, @@ -84,11 +90,13 @@ export default ({ hasLegalHold, hasImmutabilityPolicy, resourceGroupId, + storageAccountName, } = service return { id, name, + url: formatContainerUrl(storageAccountName, name), type, region, subscriptionId: account, diff --git a/src/services/storageContainer/schema.graphql b/src/services/storageContainer/schema.graphql index 674b8e07..ea2cf378 100644 --- a/src/services/storageContainer/schema.graphql +++ b/src/services/storageContainer/schema.graphql @@ -35,6 +35,7 @@ type azureStorageContainer implements azureResource mutation: { add: true, delete: false } ) @key(fields: "id") { + url: String @search(by: [hash, regexp]) version: String @search(by: [hash, regexp]) deleted: String @search(by: [hash, regexp]) deletedTime: String @search(by: [hash, regexp]) diff --git a/src/types/generated.ts b/src/types/generated.ts index 84f4520f..4fe0b4d6 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -5544,6 +5544,7 @@ export type AzureStorageBlob = AzureResource & { resourceGroup?: Maybe>>; snapshot?: Maybe; storageContainer?: Maybe>>; + url?: Maybe; versionId?: Maybe; }; @@ -5620,6 +5621,7 @@ export type AzureStorageContainer = AzureResource & { resourceGroup?: Maybe>>; storageAccount?: Maybe>>; storageBlobs?: Maybe>>; + url?: Maybe; version?: Maybe; }; From 41bffd48c728303d0d697509c9efa9a5c7442166 Mon Sep 17 00:00:00 2001 From: "mariano.pizarro" Date: Tue, 6 Jun 2023 13:45:57 -0300 Subject: [PATCH 02/20] feat(azure): Add missing SQL Server services --- src/services/sqlServers/data.ts | 227 ++++++++++++++++++------- src/services/sqlServers/format.ts | 42 +++++ src/services/sqlServers/schema.graphql | 39 +++++ src/types/generated.ts | 21 +++ 4 files changed, 269 insertions(+), 60 deletions(-) diff --git a/src/services/sqlServers/data.ts b/src/services/sqlServers/data.ts index 3d0404f7..59e57602 100644 --- a/src/services/sqlServers/data.ts +++ b/src/services/sqlServers/data.ts @@ -1,19 +1,22 @@ import { - SqlManagementClient, - Server, + ElasticPool, + EncryptionProtector, + FailoverGroup, FirewallRule, - ServerSecurityAlertPolicy, + Server, ServerAzureADAdministrator, - EncryptionProtector, ServerBlobAuditingPolicy, + ServerSecurityAlertPolicy, ServerVulnerabilityAssessment, + SqlManagementClient, + VirtualNetworkRule, } from '@azure/arm-sql' import { PagedAsyncIterableIterator } from '@azure/core-paging' import CloudGraph from '@cloudgraph/sdk' import azureLoggerText from '../../properties/logger' import { AzureServiceInput, TagMap } from '../../types' -import { getResourceGroupFromEntity } from '../../utils/idParserUtils' import { lowerCaseLocation } from '../../utils/format' +import { getResourceGroupFromEntity } from '../../utils/idParserUtils' import { tryCatchWrapper } from '../../utils/index' const { logger } = CloudGraph @@ -24,7 +27,10 @@ export interface RawAzureServer extends Omit { region: string resourceGroupId: string Tags: TagMap + elasticPools: ElasticPool[] + failoverGroups: FailoverGroup[] firewallRules: FirewallRule[] + virtualNetworkRules: VirtualNetworkRule[] serverSecurityAlertPolicies: ServerSecurityAlertPolicy[] adAdministrators: ServerAzureADAdministrator[] encryptionProtectors: EncryptionProtector[] @@ -32,6 +38,90 @@ export interface RawAzureServer extends Omit { vulnerabilityAssessments: ServerVulnerabilityAssessment[] } +const listElasticPools = async ( + client: SqlManagementClient, + resourceGroup: string, + serverName: string +): Promise => { + const elasticPools: ElasticPool[] = [] + const elasticPoolsIterable = client.elasticPools.listByServer( + resourceGroup, + serverName + ) + await tryCatchWrapper( + async () => { + for await (const elasticPool of elasticPoolsIterable) { + if (elasticPool) { + elasticPools.push(elasticPool) + } + } + }, + { + service: serviceName, + client, + scope: 'elasticPools', + operation: 'listByServer', + } + ) + return elasticPools +} + +const listFileoverGroups = async ( + client: SqlManagementClient, + resourceGroup: string, + serverName: string +): Promise => { + const failoverGroups: FailoverGroup[] = [] + const failoverGroupsIterable = client.failoverGroups.listByServer( + resourceGroup, + serverName + ) + await tryCatchWrapper( + async () => { + for await (const failoverGroup of failoverGroupsIterable) { + if (failoverGroup) { + failoverGroups.push(failoverGroup) + } + } + }, + { + service: serviceName, + client, + scope: 'failoverGroups', + operation: 'listByServer', + } + ) + return failoverGroups +} + +const listVirtualNetworkRules = async ( + client: SqlManagementClient, + resourceGroup: string, + serverName: string +): Promise => { + const virtualNetworkRules: VirtualNetworkRule[] = [] + const virtualNetworkRulesIterable = client.virtualNetworkRules.listByServer( + resourceGroup, + serverName + ) + await tryCatchWrapper( + async () => { + for await (const virtualNetworkRule of virtualNetworkRulesIterable) { + if (virtualNetworkRule) { + virtualNetworkRules.push(virtualNetworkRule) + } + } + }, + { + service: serviceName, + client, + scope: 'virtualNetworkRules', + operation: 'listByServer', + } + ) + return virtualNetworkRules +} + const listFirewallRules = async ( client: SqlManagementClient, resourceGroup: string, @@ -143,15 +233,16 @@ const listEncryptionProtectors = async ( } const listServerVulnerabilityAssessments = async ( - client: SqlManagementClient, - resourceGroup: string, - serverName: string, + client: SqlManagementClient, + resourceGroup: string, + serverName: string ): Promise => { const databaseVulnerabilityAssessments: ServerVulnerabilityAssessment[] = [] - const vulnerabilityAssessmentIterable = client.serverVulnerabilityAssessments.listByServer( - resourceGroup, - serverName, - ) + const vulnerabilityAssessmentIterable = + client.serverVulnerabilityAssessments.listByServer( + resourceGroup, + serverName + ) await tryCatchWrapper( async () => { for await (const vulnerabilityAssessment of vulnerabilityAssessmentIterable) { @@ -171,15 +262,13 @@ const listServerVulnerabilityAssessments = async ( } const listServerBlobAuditingPolicies = async ( - client: SqlManagementClient, - resourceGroup: string, - serverName: string, + client: SqlManagementClient, + resourceGroup: string, + serverName: string ): Promise => { const serverBlobAuditingPolicies: ServerBlobAuditingPolicy[] = [] - const serverBlobAuditingPolicyIterable = client.serverBlobAuditingPolicies.listByServer( - resourceGroup, - serverName, - ) + const serverBlobAuditingPolicyIterable = + client.serverBlobAuditingPolicies.listByServer(resourceGroup, serverName) await tryCatchWrapper( async () => { for await (const policy of serverBlobAuditingPolicyIterable) { @@ -226,48 +315,66 @@ export default async ({ logger.debug(lt.foundSqlServers(sqlServers.length)) const result: { [property: string]: RawAzureServer[] } = {} - await Promise.all(sqlServers.map(async ({ name, tags, location, ...rest }) => { - const region = lowerCaseLocation(location) - if (regions.includes(region)) { - if (!result[region]) { - result[region] = [] - } - const resourceGroupId = getResourceGroupFromEntity(rest) - result[region].push({ - name, - ...rest, - resourceGroupId, - region, - Tags: tags || {}, - firewallRules: await listFirewallRules(client, resourceGroupId, name), - serverSecurityAlertPolicies: await listServerSecurityAlertPolicies( - client, - resourceGroupId, - name - ), - adAdministrators: await listADAdministrators( - client, - resourceGroupId, - name - ), - encryptionProtectors: await listEncryptionProtectors( - client, + await Promise.all( + sqlServers.map(async ({ name, tags, location, ...rest }) => { + const region = lowerCaseLocation(location) + if (regions.includes(region)) { + if (!result[region]) { + result[region] = [] + } + const resourceGroupId = getResourceGroupFromEntity(rest) + result[region].push({ + name, + ...rest, resourceGroupId, - name - ), - serverBlobAuditingPolicies: await listServerBlobAuditingPolicies( - client, - resourceGroupId, - name - ), - vulnerabilityAssessments: await listServerVulnerabilityAssessments( - client, - resourceGroupId, - name - ), - }) - } - })) + region, + Tags: tags || {}, + elasticPools: await listElasticPools(client, resourceGroupId, name), + failoverGroups: await listFileoverGroups( + client, + resourceGroupId, + name + ), + firewallRules: await listFirewallRules( + client, + resourceGroupId, + name + ), + virtualNetworkRules: await listVirtualNetworkRules( + client, + resourceGroupId, + name + ), + serverSecurityAlertPolicies: await listServerSecurityAlertPolicies( + client, + resourceGroupId, + name + ), + adAdministrators: await listADAdministrators( + client, + resourceGroupId, + name + ), + encryptionProtectors: await listEncryptionProtectors( + client, + resourceGroupId, + name + ), + serverBlobAuditingPolicies: await listServerBlobAuditingPolicies( + client, + resourceGroupId, + name + ), + vulnerabilityAssessments: await listServerVulnerabilityAssessments( + client, + resourceGroupId, + name + ), + }) + } + }) + ) + return result } catch (e) { logger.error(e) diff --git a/src/services/sqlServers/format.ts b/src/services/sqlServers/format.ts index 63531a37..470cda34 100644 --- a/src/services/sqlServers/format.ts +++ b/src/services/sqlServers/format.ts @@ -3,10 +3,13 @@ import { AzureSqlServer, AzureSqlServerAdAdministrator, AzureSqlServerBlobAuditingPolicy, + AzureSqlServerElasticPool, AzureSqlServerEncryptionProtector, + AzureSqlServerFailoverGroup, AzureSqlServerFirewallRule, AzureSqlServerPrivateEndpointConnection, AzureSqlServerSecurityAlertPolicy, + AzureSqlServerVirtualNetworkRule, AzureSqlServerVulnerabilityAssessment, } from '../../types/generated' import { formatTagsFromMap } from '../../utils/format' @@ -45,7 +48,10 @@ export default ({ workspaceFeature, restrictOutboundNetworkAccess, resourceGroupId, + elasticPools = [], + failoverGroups = [], firewallRules = [], + virtualNetworkRules = [], serverSecurityAlertPolicies = [], adAdministrators = [], encryptionProtectors = [], @@ -118,6 +124,30 @@ export default ({ workspaceFeature, restrictOutboundNetworkAccess, resourceGroupId, + elasticPools: + elasticPools?.map( + ({ + id: elasticPoolId, + name: elasticPoolName, + type: elasticPoolType, + }): AzureSqlServerElasticPool => ({ + id: elasticPoolId, + name: elasticPoolName, + type: elasticPoolType, + }) + ) ?? [], + failoverGroups: + failoverGroups?.map( + ({ + id: failoverGroupId, + name: failoverGroupName, + type: failoverGroupType, + }): AzureSqlServerFailoverGroup => ({ + id: failoverGroupId, + name: failoverGroupName, + type: failoverGroupType, + }) + ) ?? [], firewallRules: firewallRules?.map( ({ @@ -134,6 +164,18 @@ export default ({ endIpAddress, }) ) ?? [], + virtualNetworkRules: + virtualNetworkRules?.map( + ({ + id: virtualNetworkRuleId, + name: virtualNetworkRuleName, + type: virtualNetworkRuleType, + }): AzureSqlServerVirtualNetworkRule => ({ + id: virtualNetworkRuleId, + name: virtualNetworkRuleName, + type: virtualNetworkRuleType, + }) + ) ?? [], serverSecurityAlertPolicies: serverSecurityAlertPolicies?.map( ({ diff --git a/src/services/sqlServers/schema.graphql b/src/services/sqlServers/schema.graphql index f41cf8de..b498fec8 100644 --- a/src/services/sqlServers/schema.graphql +++ b/src/services/sqlServers/schema.graphql @@ -168,7 +168,10 @@ type azureSqlServer implements azureResource administrators: azureSqlServerExternalAdministrator workspaceFeature: String @search(by: [hash, regexp]) restrictOutboundNetworkAccess: String @search(by: [hash, regexp]) + elasticPools: [azureSqlServerElasticPool] + failoverGroups: [azureSqlServerFailoverGroup] firewallRules: [azureSqlServerFirewallRule] + virtualNetworkRules: [azureSqlServerVirtualNetworkRule] serverSecurityAlertPolicies: [azureSqlServerSecurityAlertPolicy] adAdministrators: [azureSqlServerADAdministrator] encryptionProtectors: [azureSqlServerEncryptionProtector] @@ -223,3 +226,39 @@ type azureSqlServerBlobAuditingPolicy storageEndpoint: String @search(by: [hash, regexp]) storageAccountSubscriptionId: String @search(by: [hash, regexp]) } + +type azureSqlServerElasticPool + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) +} + +type azureSqlServerFailoverGroup + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) +} + +type azureSqlServerVirtualNetworkRule + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) +} \ No newline at end of file diff --git a/src/types/generated.ts b/src/types/generated.ts index 84f4520f..1e077097 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -5230,7 +5230,9 @@ export type AzureSqlServer = AzureResource & { adAdministrators?: Maybe>>; administrators?: Maybe; databaseSql?: Maybe>>; + elasticPools?: Maybe>>; encryptionProtectors?: Maybe>>; + failoverGroups?: Maybe>>; federatedClientId?: Maybe; firewallRules?: Maybe>>; fullyQualifiedDomainName?: Maybe; @@ -5247,6 +5249,7 @@ export type AzureSqlServer = AzureResource & { state?: Maybe; subscriptionId?: Maybe; version?: Maybe; + virtualNetworkRules?: Maybe>>; vulnerabilityAssessments?: Maybe>>; workspaceFeature?: Maybe; }; @@ -5275,6 +5278,12 @@ export type AzureSqlServerBlobAuditingPolicy = { type?: Maybe; }; +export type AzureSqlServerElasticPool = { + id: Scalars['String']; + name?: Maybe; + type?: Maybe; +}; + export type AzureSqlServerEncryptionProtector = { autoRotationEnabled?: Maybe; id: Scalars['String']; @@ -5298,6 +5307,12 @@ export type AzureSqlServerExternalAdministrator = { tenantId?: Maybe; }; +export type AzureSqlServerFailoverGroup = { + id: Scalars['String']; + name?: Maybe; + type?: Maybe; +}; + export type AzureSqlServerFirewallRule = { endIpAddress?: Maybe; id: Scalars['String']; @@ -5355,6 +5370,12 @@ export type AzureSqlServerUserIdentity = { principalId?: Maybe; }; +export type AzureSqlServerVirtualNetworkRule = { + id: Scalars['String']; + name?: Maybe; + type?: Maybe; +}; + export type AzureSqlServerVulnerabilityAssessment = { id: Scalars['String']; name?: Maybe; From 8b4532048fc8137539489adb38eee101ff867039 Mon Sep 17 00:00:00 2001 From: Marco Franceschi Date: Tue, 6 Jun 2023 15:30:50 -0400 Subject: [PATCH 03/20] feat: Exposed containersIds for cosmosdb account --- src/services/cosmosDb/format.ts | 38 +++++++++++++++++++++------------ src/types/generated.ts | 1 + 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/services/cosmosDb/format.ts b/src/services/cosmosDb/format.ts index 5e139b8a..a62c93d0 100644 --- a/src/services/cosmosDb/format.ts +++ b/src/services/cosmosDb/format.ts @@ -31,9 +31,11 @@ const formatRestoreParameters = ( const { restoreTimestampInUtc, databasesToRestore, ...rest } = restoreParameters return { restoreTimestampInUtc: restoreTimestampInUtc?.toISOString(), - databasesToRestore: databasesToRestore?.map(db => ({ id: generateUniqueId({ - ...db - }), ...db })) || [], + databasesToRestore: databasesToRestore?.map(db => ({ + id: generateUniqueId({ + ...db + }), ...db + })) || [], ...rest, } } @@ -94,6 +96,20 @@ export default ({ tables = [], } = service + let containersIds: string[] = [] + const sqlDatabases = databases?.map(({ id: databaseId, options, data, ...rest }) => { + const containers = new Set(data.map(d => d.id)) + containersIds = Array.from(containers) + return ({ + id: databaseId, + ...rest, + options: { + throughput: options?.throughput, + maxThroughput: options?.autoscaleSettings?.maxThroughput, + }, + }) + }) || [] + return { id, name, @@ -133,10 +149,10 @@ export default ({ failoverPolicies?.map(fp => ({ id: fp.id, ...fp })) || [], virtualNetworkRules: virtualNetworkRules?.map(vn => ({ id: vn.id, ...vn })) || [], - privateEndpointConnections: privateEndpointConnections?.map(({ id: endpointId, privateEndpoint, ...pe}) => ({ - id: endpointId, + privateEndpointConnections: privateEndpointConnections?.map(({ id: endpointId, privateEndpoint, ...pe }) => ({ + id: endpointId, privateEndpointId: privateEndpoint?.id, - ...pe + ...pe })) || [], enableMultipleWriteLocations, enableCassandraConnector, @@ -160,14 +176,8 @@ export default ({ disableLocalAuth, capacityTotalThroughputLimit: capacity?.totalThroughputLimit, tags: formatTagsFromMap(Tags), - databases: databases?.map(({ id: databaseId, options, ...rest }) => ({ - id: databaseId, - ...rest, - options: { - throughput: options?.throughput, - maxThroughput: options?.autoscaleSettings?.maxThroughput, - }, - })) || [], + databases: sqlDatabases, azureTables: tables?.map(at => ({ id: at.id, ...at })) || [], + containersIds } } diff --git a/src/types/generated.ts b/src/types/generated.ts index 84f4520f..1588c2b9 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -2497,6 +2497,7 @@ export type AzureCosmosDb = AzureResource & { capacityTotalThroughputLimit?: Maybe; connectorOffer?: Maybe; consistencyPolicy?: Maybe; + containersIds?: Maybe>>; cors?: Maybe>>; createMode?: Maybe; createdAt?: Maybe; From fcdd3779aae1d5c2d80bcd093d282d54ba5e16f5 Mon Sep 17 00:00:00 2001 From: autocloud-deploy-bot Date: Tue, 6 Jun 2023 19:38:52 +0000 Subject: [PATCH 04/20] chore(release): 0.67.0-alpha.1 # [0.67.0-alpha.1](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.66.0...0.67.0-alpha.1) (2023-06-06) ### Features * **azure:** Add missing SQL Server services ([41bffd4](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/41bffd48c728303d0d697509c9efa9a5c7442166)) * **azure:** Support missing storageBlob and storageContainer services ([b179ebf](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/b179ebf76107a6285c978067b79f544e995df66f)) --- CHANGELOG.md | 8 ++++++++ package.json | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34f01c8e..547a1689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# [0.67.0-alpha.1](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.66.0...0.67.0-alpha.1) (2023-06-06) + + +### Features + +* **azure:** Add missing SQL Server services ([41bffd4](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/41bffd48c728303d0d697509c9efa9a5c7442166)) +* **azure:** Support missing storageBlob and storageContainer services ([b179ebf](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/b179ebf76107a6285c978067b79f544e995df66f)) + # [0.66.0](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.65.0...0.66.0) (2023-05-16) diff --git a/package.json b/package.json index 2b0271a3..ac80a527 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@cloudgraph/cg-provider-azure", - "version": "0.66.0", + "version": "0.67.0-alpha.1", "description": "CloudGraph provider plugin for Azure used to fetch Azure cloud data.", "publishConfig": { "registry": "https://registry.npmjs.org/", From cde0c791374d648f77541b1189ddbd3062ea67aa Mon Sep 17 00:00:00 2001 From: Marco Franceschi Date: Tue, 6 Jun 2023 15:56:18 -0400 Subject: [PATCH 05/20] fix: Added missing attribute for cosmosdb schema --- src/services/cosmosDb/schema.graphql | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/cosmosDb/schema.graphql b/src/services/cosmosDb/schema.graphql index 787134d8..6a0fbbe2 100644 --- a/src/services/cosmosDb/schema.graphql +++ b/src/services/cosmosDb/schema.graphql @@ -303,4 +303,5 @@ type azureCosmosDb implements azureResource databases: [azureCosmosDbDatabase] azureTables: [azureCosmosDbTable] resourceGroup: [azureResourceGroup] @hasInverse(field: cosmosDb) + containersIds: [String] @search(by: [hash]) } From bf9351013e04ae76c87ba8081f4b92c13634881d Mon Sep 17 00:00:00 2001 From: autocloud-deploy-bot Date: Tue, 6 Jun 2023 19:58:17 +0000 Subject: [PATCH 06/20] chore(release): 0.67.0-alpha.2 # [0.67.0-alpha.2](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.1...0.67.0-alpha.2) (2023-06-06) ### Bug Fixes * Added missing attribute for cosmosdb schema ([cde0c79](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/cde0c791374d648f77541b1189ddbd3062ea67aa)) ### Features * Exposed containersIds for cosmosdb account ([8b45320](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/8b4532048fc8137539489adb38eee101ff867039)) --- CHANGELOG.md | 12 ++++++++++++ package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 547a1689..1435b8f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# [0.67.0-alpha.2](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.1...0.67.0-alpha.2) (2023-06-06) + + +### Bug Fixes + +* Added missing attribute for cosmosdb schema ([cde0c79](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/cde0c791374d648f77541b1189ddbd3062ea67aa)) + + +### Features + +* Exposed containersIds for cosmosdb account ([8b45320](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/8b4532048fc8137539489adb38eee101ff867039)) + # [0.67.0-alpha.1](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.66.0...0.67.0-alpha.1) (2023-06-06) diff --git a/package.json b/package.json index ac80a527..726bfab1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@cloudgraph/cg-provider-azure", - "version": "0.67.0-alpha.1", + "version": "0.67.0-alpha.2", "description": "CloudGraph provider plugin for Azure used to fetch Azure cloud data.", "publishConfig": { "registry": "https://registry.npmjs.org/", From 90131c83d231a8ef3d2f29b23412e978688e32dc Mon Sep 17 00:00:00 2001 From: "mariano.pizarro" Date: Tue, 6 Jun 2023 17:59:08 -0300 Subject: [PATCH 07/20] feat(azure): add missing mySQL Server services --- src/services/mySqlServers/data.ts | 150 +++++++++++++++++++---- src/services/mySqlServers/format.ts | 54 +++++++- src/services/mySqlServers/schema.graphql | 41 +++++++ src/types/generated.ts | 23 ++++ 4 files changed, 244 insertions(+), 24 deletions(-) diff --git a/src/services/mySqlServers/data.ts b/src/services/mySqlServers/data.ts index 24b3d7f6..1938e29f 100644 --- a/src/services/mySqlServers/data.ts +++ b/src/services/mySqlServers/data.ts @@ -1,4 +1,10 @@ -import { MySQLManagementClient, Server } from '@azure/arm-mysql' +import { + Configuration, + FirewallRule, + MySQLManagementClient, + Server, + VirtualNetworkRule, +} from '@azure/arm-mysql' import { PagedAsyncIterableIterator } from '@azure/core-paging' import CloudGraph from '@cloudgraph/sdk' import azureLoggerText from '../../properties/logger' @@ -11,11 +17,97 @@ const { logger } = CloudGraph const lt = { ...azureLoggerText } const serviceName = 'MySQL Server' -export interface RawAzureMySqlServer - extends Omit { +export interface RawAzureMySqlServer extends Omit { region: string resourceGroupId: string Tags: TagMap + configurations: Configuration[] + firewallRules: FirewallRule[] + virtualNetworkRules: VirtualNetworkRule[] +} + +const listConfigurations = async ( + client: MySQLManagementClient, + resourceGroup: string, + serverName: string +): Promise => { + const configurations: Configuration[] = [] + const configurationsIterable = client.configurations.listByServer( + resourceGroup, + serverName + ) + await tryCatchWrapper( + async () => { + for await (const configuration of configurationsIterable) { + if (configuration) { + configurations.push(configuration) + } + } + }, + { + service: serviceName, + client, + scope: 'configurations', + operation: 'listByServer', + } + ) + return configurations +} + +const listFirewallRules = async ( + client: MySQLManagementClient, + resourceGroup: string, + serverName: string +): Promise => { + const firewallRules: FirewallRule[] = [] + const firewallRuleIterable = client.firewallRules.listByServer( + resourceGroup, + serverName + ) + await tryCatchWrapper( + async () => { + for await (const firewallRule of firewallRuleIterable) { + if (firewallRule) { + firewallRules.push(firewallRule) + } + } + }, + { + service: serviceName, + client, + scope: 'firewallRules', + operation: 'listByServer', + } + ) + return firewallRules +} + +const listVirtualNetworkRules = async ( + client: MySQLManagementClient, + resourceGroup: string, + serverName: string +): Promise => { + const virtualNetworkRules: VirtualNetworkRule[] = [] + const virtualNetworkRulesIterable = client.virtualNetworkRules.listByServer( + resourceGroup, + serverName + ) + await tryCatchWrapper( + async () => { + for await (const virtualNetworkRule of virtualNetworkRulesIterable) { + if (virtualNetworkRule) { + virtualNetworkRules.push(virtualNetworkRule) + } + } + }, + { + service: serviceName, + client, + scope: 'virtualNetworkRules', + operation: 'listByServer', + } + ) + return virtualNetworkRules } export default async ({ @@ -26,10 +118,7 @@ export default async ({ }> => { try { const { tokenCredentials, subscriptionId } = config - const client = new MySQLManagementClient( - tokenCredentials, - subscriptionId - ) + const client = new MySQLManagementClient(tokenCredentials, subscriptionId) const sqlServers: Server[] = [] const sqlServerIterable: PagedAsyncIterableIterator = client.servers.list() @@ -49,21 +138,40 @@ export default async ({ logger.debug(lt.foundMySqlServers(sqlServers.length)) const result: { [property: string]: RawAzureMySqlServer[] } = {} - sqlServers.map(({ location, tags, ...rest }) => { - const region = lowerCaseLocation(location) - if (regions.includes(region)) { - if (!result[region]) { - result[region] = [] + await Promise.all( + sqlServers.map(async ({ name, location, tags, ...rest }) => { + const region = lowerCaseLocation(location) + if (regions.includes(region)) { + if (!result[region]) { + result[region] = [] + } + const resourceGroupId = getResourceGroupFromEntity(rest) + result[region].push({ + name, + region, + ...rest, + resourceGroupId, + Tags: tags || {}, + configurations: await listConfigurations( + client, + resourceGroupId, + name + ), + firewallRules: await listFirewallRules( + client, + resourceGroupId, + name + ), + virtualNetworkRules: await listVirtualNetworkRules( + client, + resourceGroupId, + name + ), + }) } - const resourceGroupId = getResourceGroupFromEntity(rest) - result[region].push({ - region, - ...rest, - resourceGroupId, - Tags: tags || {}, - }) - } - }) + }) + ) + return result } catch (e) { logger.error(e) diff --git a/src/services/mySqlServers/format.ts b/src/services/mySqlServers/format.ts index cf324b24..bbe5983c 100644 --- a/src/services/mySqlServers/format.ts +++ b/src/services/mySqlServers/format.ts @@ -1,6 +1,11 @@ import { formatTagsFromMap } from '../../utils/format' import { RawAzureMySqlServer } from './data' -import { AzureMySqlServer } from '../../types/generated' +import { + AzureMySqlServer, + AzureMySqlServerConfiguration, + AzureMySqlServerFirewallRule, + AzureMySqlServerVirtualNetworkRule, +} from '../../types/generated' export default ({ service, @@ -33,6 +38,9 @@ export default ({ privateEndpointConnections, resourceGroupId, Tags, + configurations = [], + firewallRules = [], + virtualNetworkRules = [], } = service return { @@ -60,7 +68,47 @@ export default ({ privateEndpointConnections: privateEndpointConnections?.map(connection => ({ ...connection, id: connection.id, - })), + })), tags: formatTagsFromMap(Tags), + configurations: + configurations?.map( + ({ + id: configurationId, + name: configurationName, + type: configurationType, + }): AzureMySqlServerConfiguration => ({ + id: configurationId, + name: configurationName, + type: configurationType, + }) + ) ?? [], + firewallRules: + firewallRules?.map( + ({ + id: firewallRuleId, + name: firewallRuleName, + type: firewallRuleType, + startIpAddress, + endIpAddress, + }): AzureMySqlServerFirewallRule => ({ + id: firewallRuleId, + name: firewallRuleName, + type: firewallRuleType, + startIpAddress, + endIpAddress, + }) + ) ?? [], + virtualNetworkRules: + virtualNetworkRules?.map( + ({ + id: virtualNetworkRuleId, + name: virtualNetworkRuleName, + type: virtualNetworkRuleType, + }): AzureMySqlServerVirtualNetworkRule => ({ + id: virtualNetworkRuleId, + name: virtualNetworkRuleName, + type: virtualNetworkRuleType, + }) + ) ?? [], } -} \ No newline at end of file +} diff --git a/src/services/mySqlServers/schema.graphql b/src/services/mySqlServers/schema.graphql index 7f55001f..5c2b9af2 100644 --- a/src/services/mySqlServers/schema.graphql +++ b/src/services/mySqlServers/schema.graphql @@ -19,6 +19,9 @@ type azureMySqlServer implements azureResource replicaCapacity: Int @search publicNetworkAccess: String @search(by: [hash, regexp]) privateEndpointConnections: [azureMySqlServerServerPrivateEndpointConnection] + configurations: [azureMySqlServerConfiguration] + firewallRules: [azureMySqlServerFirewallRule] + virtualNetworkRules: [azureMySqlServerVirtualNetworkRule] databaseMySql: [azureDatabaseMySql] @hasInverse(field: mySqlServer) resourceGroup: [azureResourceGroup] @hasInverse(field: mySqlServer) } @@ -81,4 +84,42 @@ type azureMySqlServerServerPrivateLinkServiceConnectionStateProperty status: String @search(by: [hash, regexp]) description: String @search(by: [hash, regexp]) actionsRequired: String @search(by: [hash, regexp]) +} + +type azureMySqlServerConfiguration + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) +} + +type azureMySqlServerFirewallRule + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) + startIpAddress: String @search(by: [hash, regexp]) + endIpAddress: String @search(by: [hash, regexp]) +} + +type azureMySqlServerVirtualNetworkRule + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) } \ No newline at end of file diff --git a/src/types/generated.ts b/src/types/generated.ts index 5b35a3ab..8e82639e 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -4078,8 +4078,10 @@ export type AzureMetricAlertMultiMetricCriteria = { export type AzureMySqlServer = AzureResource & { administratorLogin?: Maybe; byokEnforcement?: Maybe; + configurations?: Maybe>>; databaseMySql?: Maybe>>; earliestRestoreDate?: Maybe; + firewallRules?: Maybe>>; fullyQualifiedDomainName?: Maybe; identity?: Maybe; infrastructureEncryption?: Maybe; @@ -4094,6 +4096,21 @@ export type AzureMySqlServer = AzureResource & { storageProfile?: Maybe; userVisibleState?: Maybe; version?: Maybe; + virtualNetworkRules?: Maybe>>; +}; + +export type AzureMySqlServerConfiguration = { + id: Scalars['String']; + name?: Maybe; + type?: Maybe; +}; + +export type AzureMySqlServerFirewallRule = { + endIpAddress?: Maybe; + id: Scalars['String']; + name?: Maybe; + startIpAddress?: Maybe; + type?: Maybe; }; export type AzureMySqlServerPrivateEndpointProperty = { @@ -4130,6 +4147,12 @@ export type AzureMySqlServerStorageProfile = { storageMB?: Maybe; }; +export type AzureMySqlServerVirtualNetworkRule = { + id: Scalars['String']; + name?: Maybe; + type?: Maybe; +}; + export type AzureNameValueProperty = { id: Scalars['String']; name?: Maybe; From 8aeff969f44cfa8751fefacbbca6597cf7213cd1 Mon Sep 17 00:00:00 2001 From: "mariano.pizarro" Date: Wed, 7 Jun 2023 10:17:59 -0300 Subject: [PATCH 08/20] feat(azure): Add missing postgreSql server services --- src/services/postgreSqlServers/data.ts | 21 +++++++++++++++ src/services/postgreSqlServers/format.ts | 26 +++++++++++++++---- src/services/postgreSqlServers/schema.graphql | 13 ++++++++++ src/types/generated.ts | 7 +++++ 4 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/services/postgreSqlServers/data.ts b/src/services/postgreSqlServers/data.ts index 97699b39..b4892ce9 100644 --- a/src/services/postgreSqlServers/data.ts +++ b/src/services/postgreSqlServers/data.ts @@ -3,6 +3,7 @@ import { FirewallRule, PostgreSQLManagementClient, Server, + VirtualNetworkRule, } from '@azure/arm-postgresql' import { PagedAsyncIterableIterator } from '@azure/core-paging' import CloudGraph from '@cloudgraph/sdk' @@ -23,6 +24,7 @@ export interface RawAzurePostgreSqlServer Tags: TagMap configurations: Configuration[] firewallRules: FirewallRule[] + virtualNetworkRules: VirtualNetworkRule[] } export default async ({ @@ -100,6 +102,24 @@ export default async ({ operation: `listByServer for ${name}`, } ) + const virtualNetworkRules: VirtualNetworkRule[] = [] + const virtualNetworkRulesIterable: PagedAsyncIterableIterator = + client.virtualNetworkRules.listByServer(resourceGroupId, name) + await tryCatchWrapper( + async () => { + for await (const virtualNetworkRule of virtualNetworkRulesIterable) { + if (virtualNetworkRule) { + virtualNetworkRules.push(virtualNetworkRule) + } + } + }, + { + service: serviceName, + client, + scope: 'virtualNetworkRules', + operation: `listByServer for ${name}`, + } + ) result[region].push({ name, region, @@ -107,6 +127,7 @@ export default async ({ resourceGroupId, configurations, firewallRules, + virtualNetworkRules, Tags: tags || {}, }) } diff --git a/src/services/postgreSqlServers/format.ts b/src/services/postgreSqlServers/format.ts index e62e4a76..336617fe 100644 --- a/src/services/postgreSqlServers/format.ts +++ b/src/services/postgreSqlServers/format.ts @@ -1,6 +1,9 @@ import { formatTagsFromMap } from '../../utils/format' import { RawAzurePostgreSqlServer } from './data' -import { AzurePostgreSqlServer } from '../../types/generated' +import { + AzurePostgreSqlServer, + AzurePostgreSqlServerVirtualNetworkRule, +} from '../../types/generated' export default ({ service, @@ -33,8 +36,9 @@ export default ({ privateEndpointConnections, resourceGroupId, Tags, - configurations, - firewallRules + configurations = [], + firewallRules = [], + virtualNetworkRules = [], } = service return { @@ -93,14 +97,26 @@ export default ({ name: confName, type: confType, startIpAddress, - endIpAddress + endIpAddress, }) => ({ id: confId, name: confName, type: confType, startIpAddress, - endIpAddress + endIpAddress, }) ), + virtualNetworkRules: + virtualNetworkRules?.map( + ({ + id: virtualNetworkRuleId, + name: virtualNetworkRuleName, + type: virtualNetworkRuleType, + }): AzurePostgreSqlServerVirtualNetworkRule => ({ + id: virtualNetworkRuleId, + name: virtualNetworkRuleName, + type: virtualNetworkRuleType, + }) + ) ?? [], } } diff --git a/src/services/postgreSqlServers/schema.graphql b/src/services/postgreSqlServers/schema.graphql index a8d1eefd..097543bd 100644 --- a/src/services/postgreSqlServers/schema.graphql +++ b/src/services/postgreSqlServers/schema.graphql @@ -21,6 +21,7 @@ type azurePostgreSqlServer implements azureResource publicNetworkAccess: String @search(by: [hash, regexp]) configurations: [azurePostgreSqlServerConfiguration] firewallRules: [azurePostgreSqlServerFirewallRule] + virtualNetworkRules: [azurePostgreSqlServerVirtualNetworkRule] privateEndpointConnections: [azurePostgreSqlServerPrivateEndpointConnection] databasePostgreSql: [azureDatabasePostgreSql] @hasInverse(field: postgreSqlServer) @@ -119,3 +120,15 @@ type azurePostgreSqlServerFirewallRule startIpAddress: String @search(by: [hash, regexp]) endIpAddress: String @search(by: [hash, regexp]) } + +type azurePostgreSqlServerVirtualNetworkRule + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) +} \ No newline at end of file diff --git a/src/types/generated.ts b/src/types/generated.ts index 5b35a3ab..25ac1891 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -4316,6 +4316,7 @@ export type AzurePostgreSqlServer = AzureResource & { storageProfile?: Maybe; userVisibleState?: Maybe; version?: Maybe; + virtualNetworkRules?: Maybe>>; }; export type AzurePostgreSqlServerConfiguration = { @@ -4372,6 +4373,12 @@ export type AzurePostgreSqlServerStorageProfile = { storageMB?: Maybe; }; +export type AzurePostgreSqlServerVirtualNetworkRule = { + id: Scalars['String']; + name?: Maybe; + type?: Maybe; +}; + export type AzurePrivateDnsZone = AzureResource & { etag?: Maybe; internalId?: Maybe; From dd63007928045eeab4c7b8c41ea07d519fdb6a86 Mon Sep 17 00:00:00 2001 From: autocloud-deploy-bot Date: Wed, 7 Jun 2023 13:40:32 +0000 Subject: [PATCH 09/20] chore(release): 0.67.0-alpha.3 # [0.67.0-alpha.3](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.2...0.67.0-alpha.3) (2023-06-07) ### Features * **azure:** Add missing postgreSql server services ([8aeff96](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/8aeff969f44cfa8751fefacbbca6597cf7213cd1)) --- CHANGELOG.md | 7 +++++++ package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1435b8f1..bcef0e71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# [0.67.0-alpha.3](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.2...0.67.0-alpha.3) (2023-06-07) + + +### Features + +* **azure:** Add missing postgreSql server services ([8aeff96](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/8aeff969f44cfa8751fefacbbca6597cf7213cd1)) + # [0.67.0-alpha.2](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.1...0.67.0-alpha.2) (2023-06-06) diff --git a/package.json b/package.json index 726bfab1..b302450a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@cloudgraph/cg-provider-azure", - "version": "0.67.0-alpha.2", + "version": "0.67.0-alpha.3", "description": "CloudGraph provider plugin for Azure used to fetch Azure cloud data.", "publishConfig": { "registry": "https://registry.npmjs.org/", From 83cbdd809f81e750d5abc83bf503dba47e12fe50 Mon Sep 17 00:00:00 2001 From: Marco Franceschi Date: Wed, 7 Jun 2023 14:01:42 -0400 Subject: [PATCH 10/20] feat: Fetched records for private dns --- src/services/privateDns/data.ts | 43 ++++++++++++++++++++++++++ src/services/privateDns/format.ts | 23 ++++++++++++-- src/services/privateDns/schema.graphql | 8 +++++ src/types/generated.ts | 8 +++++ 4 files changed, 80 insertions(+), 2 deletions(-) diff --git a/src/services/privateDns/data.ts b/src/services/privateDns/data.ts index 01fc5716..f6c4998d 100644 --- a/src/services/privateDns/data.ts +++ b/src/services/privateDns/data.ts @@ -16,9 +16,51 @@ export interface RawAzurePrivateDnsZone extends Omit { region: string resourceGroupId: string + records: RawAzureRecordSet[] Tags: TagMap } +export interface RawAzureRecordSet { + id: string + name: string + type: string +} + +export const listRecordSets = async ( + client: PrivateDnsManagementClient, + resourceGroup: string, + databaseAccountName: string +): Promise => { + const records: RawAzureRecordSet[] = [] + const recordsIterable = client.recordSets.list( + resourceGroup, + databaseAccountName + ) + await tryCatchWrapper( + async () => { + for await (const record of recordsIterable) { + if (record) { + const { id, name, type } = record + const recordType = type?.split('/').pop() + records.push({ + id, + name, + type: recordType, + } as RawAzureRecordSet) + } + + } + } + , { + service: 'Records Sets', + client, + scope: 'recordSets', + operation: 'listRecordSets', + } + ) + return records +} + export default async ({ regions, config, @@ -46,6 +88,7 @@ export default async ({ ...rest, region, resourceGroupId, + records: await listRecordSets(client, resourceGroupId, privateDnsZone.name), Tags: tags || {}, }) } diff --git a/src/services/privateDns/format.ts b/src/services/privateDns/format.ts index a2b2dee3..af647e2e 100644 --- a/src/services/privateDns/format.ts +++ b/src/services/privateDns/format.ts @@ -25,8 +25,19 @@ export default ({ numberOfVirtualNetworkLinksWithRegistration, provisioningState, internalId, - resourceGroupId + resourceGroupId, + records = [] } = service + + const aRecords = records.filter(r => r.type === 'A').map(r => r.id) + const aaaRecords = records.filter(r => r.type === 'AAA').map(r => r.id) + const mxRecords = records.filter(r => r.type === 'MX').map(r => r.id) + const ptrRecords = records.filter(r => r.type === 'PTR').map(r => r.id) + const soaRecord = records.filter(r => r.type === 'SOA').map(r => r.id) + const srvRecords = records.filter(r => r.type === 'SRV').map(r => r.id) + const txtRecords = records.filter(r => r.type === 'TXT').map(r => r.id) + const cnameRecord = records.filter(r => r.type === 'CNAME').pop()?.id + return { id, subscriptionId: account, @@ -43,6 +54,14 @@ export default ({ provisioningState, internalId, tags: formatTagsFromMap(Tags), - resourceGroupId + resourceGroupId, + aRecords, + aaaRecords, + mxRecords, + ptrRecords, + soaRecord, + srvRecords, + txtRecords, + cnameRecord, } } diff --git a/src/services/privateDns/schema.graphql b/src/services/privateDns/schema.graphql index 0751ccbb..d29770f5 100644 --- a/src/services/privateDns/schema.graphql +++ b/src/services/privateDns/schema.graphql @@ -13,5 +13,13 @@ type azurePrivateDnsZone implements azureResource numberOfVirtualNetworkLinksWithRegistration: Int @search provisioningState: String @search(by: [hash, regexp]) internalId: String @search(by: [hash, regexp]) + aRecords: [String] @search(by: [hash]) + aaaRecords: [String] @search(by: [hash]) + cnameRecord: String @search(by: [hash]) + mxRecords: [String] @search(by: [hash]) + ptrRecords: [String] @search(by: [hash]) + soaRecord: [String] @search(by: [hash]) + srvRecords: [String] @search(by: [hash]) + txtRecords: [String] @search(by: [hash]) resourceGroup: [azureResourceGroup] @hasInverse(field: privateDns) } diff --git a/src/types/generated.ts b/src/types/generated.ts index 25ac1891..16699565 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -4380,16 +4380,24 @@ export type AzurePostgreSqlServerVirtualNetworkRule = { }; export type AzurePrivateDnsZone = AzureResource & { + aRecords?: Maybe>>; + aaaRecords?: Maybe>>; + cnameRecord?: Maybe; etag?: Maybe; internalId?: Maybe; maxNumberOfRecordSets?: Maybe; maxNumberOfVirtualNetworkLinks?: Maybe; maxNumberOfVirtualNetworkLinksWithRegistration?: Maybe; + mxRecords?: Maybe>>; numberOfRecordSets?: Maybe; numberOfVirtualNetworkLinks?: Maybe; numberOfVirtualNetworkLinksWithRegistration?: Maybe; provisioningState?: Maybe; + ptrRecords?: Maybe>>; resourceGroup?: Maybe>>; + soaRecord?: Maybe>>; + srvRecords?: Maybe>>; + txtRecords?: Maybe>>; }; export type AzurePublicIp = AzureResource & { From 414ceece3af505c18a76d6141d63a8d85e20067b Mon Sep 17 00:00:00 2001 From: Marco Franceschi Date: Wed, 7 Jun 2023 14:45:10 -0400 Subject: [PATCH 11/20] feat: Fetched virtual network links for private zone --- src/services/privateDns/data.ts | 52 ++++++++++++++++++++++---- src/services/privateDns/format.ts | 9 +++-- src/services/privateDns/schema.graphql | 3 +- src/types/generated.ts | 3 +- 4 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/services/privateDns/data.ts b/src/services/privateDns/data.ts index f6c4998d..2083beba 100644 --- a/src/services/privateDns/data.ts +++ b/src/services/privateDns/data.ts @@ -16,25 +16,61 @@ export interface RawAzurePrivateDnsZone extends Omit { region: string resourceGroupId: string - records: RawAzureRecordSet[] + records: RawAzurePrivateDnsMetadata[] + virtualNetworkLinks: RawAzurePrivateDnsMetadata[] Tags: TagMap } -export interface RawAzureRecordSet { +export interface RawAzurePrivateDnsMetadata { id: string name: string type: string } +export const listVirtualNetworkLinks = async ( + client: PrivateDnsManagementClient, + resourceGroup: string, + privateZoneName: string +): Promise => { + const vnetworkLinks: RawAzurePrivateDnsMetadata[] = [] + const vnetworkLinksIterable = client.virtualNetworkLinks.list( + resourceGroup, + privateZoneName + ) + await tryCatchWrapper( + async () => { + for await (const vnetworkLink of vnetworkLinksIterable) { + if (vnetworkLink) { + const { id, name, type } = vnetworkLink + const recordType = type?.split('/').pop() + vnetworkLinks.push({ + id, + name, + type: recordType, + } as RawAzurePrivateDnsMetadata) + } + + } + } + , { + service: 'Virtual Network Links', + client, + scope: 'virtualNetworkLinks', + operation: 'listVirtualNetworkLinks', + } + ) + return vnetworkLinks +} + export const listRecordSets = async ( client: PrivateDnsManagementClient, resourceGroup: string, - databaseAccountName: string -): Promise => { - const records: RawAzureRecordSet[] = [] + privateZoneName: string +): Promise => { + const records: RawAzurePrivateDnsMetadata[] = [] const recordsIterable = client.recordSets.list( resourceGroup, - databaseAccountName + privateZoneName ) await tryCatchWrapper( async () => { @@ -46,7 +82,7 @@ export const listRecordSets = async ( id, name, type: recordType, - } as RawAzureRecordSet) + } as RawAzurePrivateDnsMetadata) } } @@ -77,6 +113,7 @@ export default async ({ const privateDnsZones: RawAzurePrivateDnsZone[] = [] const privateDnsZoneIterable: PagedAsyncIterableIterator = client.privateZones.list() + client.virtualNetworkLinks await tryCatchWrapper( async () => { for await (const privateDnsZone of privateDnsZoneIterable) { @@ -88,6 +125,7 @@ export default async ({ ...rest, region, resourceGroupId, + virtualNetworkLinks: await listVirtualNetworkLinks(client, resourceGroupId, privateDnsZone.name), records: await listRecordSets(client, resourceGroupId, privateDnsZone.name), Tags: tags || {}, }) diff --git a/src/services/privateDns/format.ts b/src/services/privateDns/format.ts index af647e2e..6afeb6fe 100644 --- a/src/services/privateDns/format.ts +++ b/src/services/privateDns/format.ts @@ -26,11 +26,13 @@ export default ({ provisioningState, internalId, resourceGroupId, - records = [] + records = [], + virtualNetworkLinks = [] } = service + // Records const aRecords = records.filter(r => r.type === 'A').map(r => r.id) - const aaaRecords = records.filter(r => r.type === 'AAA').map(r => r.id) + const aaaaRecords = records.filter(r => r.type === 'AAAA').map(r => r.id) const mxRecords = records.filter(r => r.type === 'MX').map(r => r.id) const ptrRecords = records.filter(r => r.type === 'PTR').map(r => r.id) const soaRecord = records.filter(r => r.type === 'SOA').map(r => r.id) @@ -56,12 +58,13 @@ export default ({ tags: formatTagsFromMap(Tags), resourceGroupId, aRecords, - aaaRecords, + aaaaRecords, mxRecords, ptrRecords, soaRecord, srvRecords, txtRecords, cnameRecord, + virtualNetworkLinks: virtualNetworkLinks.map(r => r.id) } } diff --git a/src/services/privateDns/schema.graphql b/src/services/privateDns/schema.graphql index d29770f5..c822fd61 100644 --- a/src/services/privateDns/schema.graphql +++ b/src/services/privateDns/schema.graphql @@ -14,12 +14,13 @@ type azurePrivateDnsZone implements azureResource provisioningState: String @search(by: [hash, regexp]) internalId: String @search(by: [hash, regexp]) aRecords: [String] @search(by: [hash]) - aaaRecords: [String] @search(by: [hash]) + aaaaRecords: [String] @search(by: [hash]) cnameRecord: String @search(by: [hash]) mxRecords: [String] @search(by: [hash]) ptrRecords: [String] @search(by: [hash]) soaRecord: [String] @search(by: [hash]) srvRecords: [String] @search(by: [hash]) txtRecords: [String] @search(by: [hash]) + virtualNetworkLinks: [String] @search(by: [hash]) resourceGroup: [azureResourceGroup] @hasInverse(field: privateDns) } diff --git a/src/types/generated.ts b/src/types/generated.ts index 16699565..30697fa3 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -4381,7 +4381,7 @@ export type AzurePostgreSqlServerVirtualNetworkRule = { export type AzurePrivateDnsZone = AzureResource & { aRecords?: Maybe>>; - aaaRecords?: Maybe>>; + aaaaRecords?: Maybe>>; cnameRecord?: Maybe; etag?: Maybe; internalId?: Maybe; @@ -4398,6 +4398,7 @@ export type AzurePrivateDnsZone = AzureResource & { soaRecord?: Maybe>>; srvRecords?: Maybe>>; txtRecords?: Maybe>>; + virtualNetworkLinks?: Maybe>>; }; export type AzurePublicIp = AzureResource & { From 5573e8eeee9387d50e440e7d1aecd4c529f3c531 Mon Sep 17 00:00:00 2001 From: "mariano.pizarro" Date: Thu, 8 Jun 2023 11:11:15 -0300 Subject: [PATCH 12/20] feat(azure): add route table services --- README.md | 2 + src/enums/schemasMap.ts | 2 + src/enums/serviceAliases.ts | 2 + src/enums/serviceMap.ts | 4 + src/enums/services.ts | 2 + src/properties/logger.ts | 4 + src/services/routeFilter/data.ts | 79 ++++++++++++++++++ src/services/routeFilter/format.ts | 22 +++++ src/services/routeFilter/index.ts | 13 +++ src/services/routeFilter/mutation.ts | 5 ++ src/services/routeFilter/schema.graphql | 9 ++ src/services/routeTable/data.ts | 105 ++++++++++++++++++++++++ src/services/routeTable/format.ts | 34 ++++++++ src/services/routeTable/index.ts | 13 +++ src/services/routeTable/mutation.ts | 5 ++ src/services/routeTable/schema.graphql | 22 +++++ src/types/generated.ts | 15 ++++ 17 files changed, 338 insertions(+) create mode 100644 src/services/routeFilter/data.ts create mode 100644 src/services/routeFilter/format.ts create mode 100644 src/services/routeFilter/index.ts create mode 100644 src/services/routeFilter/mutation.ts create mode 100644 src/services/routeFilter/schema.graphql create mode 100644 src/services/routeTable/data.ts create mode 100644 src/services/routeTable/format.ts create mode 100644 src/services/routeTable/index.ts create mode 100644 src/services/routeTable/mutation.ts create mode 100644 src/services/routeTable/schema.graphql diff --git a/README.md b/README.md index 01de1d6f..eb083ad3 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,8 @@ CloudGraph needs read permissions in order to ingest your data. To keep things e | replicationNetworks | resourceGroup | | replicationPolicies | resourceGroup | | resourceGroup | **all services** | +| routeFilter | | +| routeTable | | | securityAssessments | | | securityContacts | | | securityGroup | networkInterface, resourceGroup | diff --git a/src/enums/schemasMap.ts b/src/enums/schemasMap.ts index db58d622..9cb53e20 100644 --- a/src/enums/schemasMap.ts +++ b/src/enums/schemasMap.ts @@ -77,6 +77,8 @@ export default { [services.replicationNetworks]: 'azureReplicationNetwork', [services.replicationPolicies]: 'azureReplicationPolicy', [services.resourceGroup]: 'azureResourceGroup', + [services.routeFilter]: 'azureRouteFilter', + [services.routeTable]: 'azureRouteTable', [services.securityAssesments]: 'azureSecurityAssesment', [services.securityContacts]: 'azureSecurityContact', [services.securityGroup]: 'azureNetworkSecurityGroup', diff --git a/src/enums/serviceAliases.ts b/src/enums/serviceAliases.ts index 6ccdabbe..60df620f 100644 --- a/src/enums/serviceAliases.ts +++ b/src/enums/serviceAliases.ts @@ -65,6 +65,8 @@ export default { [services.replicationNetworks]: 'replicationNetworks', [services.replicationPolicies]: 'replicationPolicies', [services.resourceGroup]: 'resourceGroups', + [services.routeFilter]: 'routeFilters', + [services.routeTable]: 'routeTables', [services.securityAssesments]: 'securityAssesments', [services.securityContacts]: 'securityContacts', [services.securityGroup]: 'securityGroups', diff --git a/src/enums/serviceMap.ts b/src/enums/serviceMap.ts index fdeb7f2e..b421954a 100644 --- a/src/enums/serviceMap.ts +++ b/src/enums/serviceMap.ts @@ -89,6 +89,8 @@ import AzureBilling from '../services/billing' import AzureLogProfiles from '../services/logProfiles' import Subscription from '../services/subscription' import AzureNetworkWatcher from '../services/networkWatcher' +import AzureRouteTable from '../services/routeTable' +import AzureRouteFilter from '../services/routeFilter' /** * serviceMap is an object that contains all currently supported services for AZURE @@ -169,6 +171,8 @@ export default { [services.replicationNetworks]: AzureReplicationNetwork, [services.replicationPolicies]: AzureReplicationPolicy, [services.resourceGroup]: AzureResourceGroup, + [services.routeFilter]: AzureRouteFilter, + [services.routeTable]: AzureRouteTable, [services.securityAssesments]: AzureSecurityAssesments, [services.securityContacts]: AzureSecurityContacts, [services.securityGroup]: AzureNetworkSecurityGroup, diff --git a/src/enums/services.ts b/src/enums/services.ts index 006d416e..ee26660e 100644 --- a/src/enums/services.ts +++ b/src/enums/services.ts @@ -71,6 +71,8 @@ export default { replicationNetworks: 'replicationNetworks', replicationPolicies: 'replicationPolicies', resourceGroup: 'resourceGroup', + routeFilter: 'routeFilter', + routeTable: 'routeTable', securityAssesments: 'securityAssesments', securityContacts: 'securityContacts', securityGroup: 'securityGroup', diff --git a/src/properties/logger.ts b/src/properties/logger.ts index 8b14e71c..58cc7a3f 100644 --- a/src/properties/logger.ts +++ b/src/properties/logger.ts @@ -173,6 +173,10 @@ export default { `Found ${num} replication policies`, // Resource Groups foundResourceGroups: (num: number): string => `Found ${num} resource groups`, + // Route Filters + foundRouteFilters: (num: number): string => `Found ${num} route filters`, + // Route Tables + foundRouteTables: (num: number): string => `Found ${num} route tables`, // Security Assesments foundSecurityAssesments: (num: number): string => `Found ${num} security assesments`, diff --git a/src/services/routeFilter/data.ts b/src/services/routeFilter/data.ts new file mode 100644 index 00000000..23bcbb28 --- /dev/null +++ b/src/services/routeFilter/data.ts @@ -0,0 +1,79 @@ +import { NetworkManagementClient, RouteFilter } from '@azure/arm-network' +import { PagedAsyncIterableIterator } from '@azure/core-paging' +import CloudGraph from '@cloudgraph/sdk' + +import azureLoggerText from '../../properties/logger' +import { AzureServiceInput, TagMap } from '../../types' +import { tryCatchWrapper } from '../../utils' +import { lowerCaseLocation } from '../../utils/format' +import { getResourceGroupFromEntity } from '../../utils/idParserUtils' + +const { logger } = CloudGraph +const lt = { ...azureLoggerText } +const serviceName = 'RouteFilter' + +export interface RawAzureRouteFilter + extends Omit { + region: string + resourceGroupId: string + Tags: TagMap +} + +export default async ({ + regions, + config, +}: AzureServiceInput): Promise<{ + [property: string]: RawAzureRouteFilter[] +}> => { + try { + const { tokenCredentials, subscriptionId } = config + const client = new NetworkManagementClient(tokenCredentials, subscriptionId) + + const routeFilterData: RawAzureRouteFilter[] = [] + await tryCatchWrapper( + async () => { + const routeFilterIterable: PagedAsyncIterableIterator = + client.routeFilters.list() + for await (const routeFilter of routeFilterIterable) { + if (routeFilter) { + const { location, tags, ...rest } = routeFilter + const resourceGroupId = getResourceGroupFromEntity(rest) + const region = lowerCaseLocation(location) + routeFilterData.push({ + ...rest, + region, + resourceGroupId, + Tags: tags || {}, + }) + } + } + }, + { + service: serviceName, + client, + scope: 'routeFilters', + operation: 'list', + } + ) + const result: { [property: string]: RawAzureRouteFilter[] } = {} + let numOfGroups = 0 + routeFilterData.forEach(({ region, ...rest }) => { + if (regions.includes(region)) { + if (!result[region]) { + result[region] = [] + } + result[region].push({ + region, + ...rest, + }) + numOfGroups += 1 + } + }) + logger.debug(lt.foundRouteFilters(numOfGroups)) + + return result + } catch (e) { + logger.error(e) + return {} + } +} diff --git a/src/services/routeFilter/format.ts b/src/services/routeFilter/format.ts new file mode 100644 index 00000000..dabf5fe4 --- /dev/null +++ b/src/services/routeFilter/format.ts @@ -0,0 +1,22 @@ +import { RawAzureRouteFilter } from './data' +import { AzureRouteFilter } from '../../types/generated' +import { formatTagsFromMap } from '../../utils/format' + +export default ({ + service, + account: subscriptionId, +}: { + service: RawAzureRouteFilter + account: string +}): AzureRouteFilter => { + const { id, name, type, region, Tags } = service + + return { + id, + name, + type, + region, + subscriptionId, + tags: formatTagsFromMap(Tags), + } +} diff --git a/src/services/routeFilter/index.ts b/src/services/routeFilter/index.ts new file mode 100644 index 00000000..50f341bb --- /dev/null +++ b/src/services/routeFilter/index.ts @@ -0,0 +1,13 @@ +import { Service } from '@cloudgraph/sdk' +import BaseService from '../base' +import format from './format' +import mutation from './mutation' +import getData from './data' + +export default class AzureRouteFilter extends BaseService implements Service { + format = format.bind(this) + + getData = getData.bind(this) + + mutation = mutation +} diff --git a/src/services/routeFilter/mutation.ts b/src/services/routeFilter/mutation.ts new file mode 100644 index 00000000..3dbc99d7 --- /dev/null +++ b/src/services/routeFilter/mutation.ts @@ -0,0 +1,5 @@ +export default `mutation($input: [AddazureRouteFilterInput!]!) { + addazureRouteFilter(input: $input, upsert: true) { + numUids + } +}`; diff --git a/src/services/routeFilter/schema.graphql b/src/services/routeFilter/schema.graphql new file mode 100644 index 00000000..2b186487 --- /dev/null +++ b/src/services/routeFilter/schema.graphql @@ -0,0 +1,9 @@ +type azureRouteFilter implements azureResource + @generate( + query: { get: true, query: true, aggregate: true } + mutation: { add: true, delete: false } + subscription: false + ) + @key(fields: "id") { + name: String @search(by: [hash, regexp]) +} diff --git a/src/services/routeTable/data.ts b/src/services/routeTable/data.ts new file mode 100644 index 00000000..9b11ee35 --- /dev/null +++ b/src/services/routeTable/data.ts @@ -0,0 +1,105 @@ +import { NetworkManagementClient, Route, RouteTable } from '@azure/arm-network' +import { PagedAsyncIterableIterator } from '@azure/core-paging' +import CloudGraph from '@cloudgraph/sdk' + +import azureLoggerText from '../../properties/logger' +import { AzureServiceInput, TagMap } from '../../types' +import { tryCatchWrapper } from '../../utils' +import { lowerCaseLocation } from '../../utils/format' +import { getResourceGroupFromEntity } from '../../utils/idParserUtils' + +const { logger } = CloudGraph +const lt = { ...azureLoggerText } +const serviceName = 'RouteTable' + +export interface RawAzureRouteTable + extends Omit { + region: string + resourceGroupId: string + Tags: TagMap + routes: Route[] +} + +const listRouteTableRoute = async ( + client: NetworkManagementClient, + resourceGroup: string, + routeTableName: string +): Promise => { + const routes: Route[] = [] + const routesIterable = client.routes.list(resourceGroup, routeTableName) + await tryCatchWrapper( + async () => { + for await (const route of routesIterable) { + if (route) { + routes.push(route) + } + } + }, + { + service: serviceName, + client, + scope: 'routes', + operation: 'list', + } + ) + return routes +} + +export default async ({ + regions, + config, +}: AzureServiceInput): Promise<{ + [property: string]: RawAzureRouteTable[] +}> => { + try { + const { tokenCredentials, subscriptionId } = config + const client = new NetworkManagementClient(tokenCredentials, subscriptionId) + + const routeTableData: RouteTable[] = [] + await tryCatchWrapper( + async () => { + const routeTableIterable: PagedAsyncIterableIterator = + client.routeTables.listAll() + for await (const routeTable of routeTableIterable) { + routeTable && routeTableData.push(routeTable) + } + }, + { + service: serviceName, + client, + scope: 'routeTables', + operation: 'listAll', + } + ) + + const result: { + [property: string]: RawAzureRouteTable[] + } = {} + let numOfGroups = 0 + await Promise.all( + routeTableData.map(async ({ name, tags, location, ...rest }) => { + const region = lowerCaseLocation(location) + if (regions.includes(region)) { + if (!result[region]) { + result[region] = [] + } + const resourceGroupId = getResourceGroupFromEntity(rest) + result[region].push({ + ...rest, + region, + resourceGroupId, + Tags: tags || {}, + routes: await listRouteTableRoute(client, resourceGroupId, name), + }) + numOfGroups += 1 + } + }) + ) + logger.debug(lt.foundRouteTables(numOfGroups)) + + return result + } catch (e) { + logger.error(e) + return {} + } +} diff --git a/src/services/routeTable/format.ts b/src/services/routeTable/format.ts new file mode 100644 index 00000000..fa55470a --- /dev/null +++ b/src/services/routeTable/format.ts @@ -0,0 +1,34 @@ +import { RawAzureRouteTable } from './data' +import { AzureRouteTable, AzureRouteTableRoute } from '../../types/generated' +import { formatTagsFromMap } from '../../utils/format' + +export default ({ + service, + account: subscriptionId, +}: { + service: RawAzureRouteTable + account: string +}): AzureRouteTable => { + const { id, name, type, region, Tags, routes = [] } = service + + return { + id, + name, + type, + region, + subscriptionId, + tags: formatTagsFromMap(Tags), + routes: + routes?.map( + ({ + id: routeId, + name: routeName, + type: routeType, + }): AzureRouteTableRoute => ({ + id: routeId, + name: routeName, + type: routeType, + }) + ) ?? [], + } +} diff --git a/src/services/routeTable/index.ts b/src/services/routeTable/index.ts new file mode 100644 index 00000000..7c7c53fa --- /dev/null +++ b/src/services/routeTable/index.ts @@ -0,0 +1,13 @@ +import { Service } from '@cloudgraph/sdk' +import BaseService from '../base' +import format from './format' +import mutation from './mutation' +import getData from './data' + +export default class AzureRouteTable extends BaseService implements Service { + format = format.bind(this) + + getData = getData.bind(this) + + mutation = mutation +} diff --git a/src/services/routeTable/mutation.ts b/src/services/routeTable/mutation.ts new file mode 100644 index 00000000..40b5350d --- /dev/null +++ b/src/services/routeTable/mutation.ts @@ -0,0 +1,5 @@ +export default `mutation($input: [AddazureRouteTableInput!]!) { + addazureRouteTable(input: $input, upsert: true) { + numUids + } +}`; diff --git a/src/services/routeTable/schema.graphql b/src/services/routeTable/schema.graphql new file mode 100644 index 00000000..19f09818 --- /dev/null +++ b/src/services/routeTable/schema.graphql @@ -0,0 +1,22 @@ +type azureRouteTableRoute + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) +} + +type azureRouteTable implements azureResource + @generate( + query: { get: true, query: true, aggregate: true } + mutation: { add: true, delete: false } + subscription: false + ) + @key(fields: "id") { + name: String @search(by: [hash, regexp]) + routes: [azureRouteTableRoute] +} diff --git a/src/types/generated.ts b/src/types/generated.ts index 25ac1891..ab1a5d36 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -5075,6 +5075,21 @@ export type AzureResourceSystemData = { lastModifiedByType?: Maybe; }; +export type AzureRouteFilter = AzureResource & { + name?: Maybe; +}; + +export type AzureRouteTable = AzureResource & { + name?: Maybe; + routes?: Maybe>>; +}; + +export type AzureRouteTableRoute = { + id: Scalars['String']; + name?: Maybe; + type?: Maybe; +}; + export type AzureSecretBaseUnion = { secretName?: Maybe; secretVersion?: Maybe; From baa0a6e72510f70e217a5c516479db8d5d05294f Mon Sep 17 00:00:00 2001 From: Marco Franceschi Date: Thu, 8 Jun 2023 17:18:45 -0400 Subject: [PATCH 13/20] feat: Added missing subnets ids and endpoint policies for subnets --- src/services/virtualNetwork/format.ts | 12 ++++++++++++ src/services/virtualNetwork/schema.graphql | 2 ++ src/types/generated.ts | 2 ++ 3 files changed, 16 insertions(+) diff --git a/src/services/virtualNetwork/format.ts b/src/services/virtualNetwork/format.ts index ca5d8717..bc622801 100644 --- a/src/services/virtualNetwork/format.ts +++ b/src/services/virtualNetwork/format.ts @@ -25,7 +25,17 @@ export default ({ enableVmProtection, resourceGroupId, Tags, + subnets = [] } = service + const subnetsIds = [] + const serviceEndpointPoliciesIds = [] + + subnets.forEach(s => { + const { id, serviceEndpointPolicies = [] } = s + subnetsIds.push(id) + serviceEndpointPoliciesIds.push(...serviceEndpointPolicies.map(p => p.id)) + }) + return { id, name, @@ -55,6 +65,8 @@ export default ({ provisioningState, resourceGuid, resourceGroupId, + subnetsIds, + serviceEndpointPoliciesIds, tags: formatTagsFromMap(Tags), } } diff --git a/src/services/virtualNetwork/schema.graphql b/src/services/virtualNetwork/schema.graphql index 6ac5f637..54115069 100644 --- a/src/services/virtualNetwork/schema.graphql +++ b/src/services/virtualNetwork/schema.graphql @@ -22,6 +22,8 @@ type azureVirtualNetwork implements azureResource flowTimeoutInMinutes: Int @search provisioningState: String @search(by: [hash, regexp]) resourceGuid: String @search(by: [hash, regexp]) + subnetsIds: [String] @search(by: [hash]) + serviceEndpointPoliciesIds: [String] @search(by: [hash]) appServiceEnvironments: [azureAppServiceEnvironment] @hasInverse(field: virtualNetwork) firewalls: [azureFirewall] @hasInverse(field: virtualNetworks) lbVirtualNetworkOf: [azureLoadBalancer] @hasInverse(field: loadBalancerBackendVirtualNetworks) diff --git a/src/types/generated.ts b/src/types/generated.ts index 84f4520f..14d0334e 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -6306,6 +6306,8 @@ export type AzureVirtualNetwork = AzureResource & { provisioningState?: Maybe; resourceGroup?: Maybe>>; resourceGuid?: Maybe; + serviceEndpointPoliciesIds?: Maybe>>; + subnetsIds?: Maybe>>; virtualMachines?: Maybe>>; }; From 2b9a48b9e848aad2775069167dcb406ad0dc8f77 Mon Sep 17 00:00:00 2001 From: "mariano.pizarro" Date: Fri, 9 Jun 2023 13:46:14 -0300 Subject: [PATCH 14/20] feat(azure): Add missing dataFactory services --- src/services/dataFactory/data.ts | 241 +++++++++++++++++++++++- src/services/dataFactory/format.ts | 88 ++++++++- src/services/dataFactory/schema.graphql | 78 ++++++++ src/types/generated.ts | 41 ++++ 4 files changed, 445 insertions(+), 3 deletions(-) diff --git a/src/services/dataFactory/data.ts b/src/services/dataFactory/data.ts index 6f822434..95ad14ad 100644 --- a/src/services/dataFactory/data.ts +++ b/src/services/dataFactory/data.ts @@ -1,4 +1,13 @@ -import { DataFactoryManagementClient, Factory } from '@azure/arm-datafactory' +import { + DataFactoryManagementClient, + DataFlowResource, + DatasetResource, + Factory, + IntegrationRuntimeResource, + LinkedServiceResource, + PipelineResource, + TriggerResource, +} from '@azure/arm-datafactory' import { PagedAsyncIterableIterator } from '@azure/core-paging' import CloudGraph from '@cloudgraph/sdk' @@ -19,6 +28,198 @@ export interface RawAzureDataFactory region: string resourceGroupId: string Tags: TagMap + pipelines: PipelineResource[] + dataFlows: DataFlowResource[] + datasets: DatasetResource[] + integrationRuntimes: IntegrationRuntimeResource[] + linkedServices: LinkedServiceResource[] + triggers: TriggerResource[] +} + +const getPipelines = async ({ + client, + resourceGroupName, + factoryName, +}: { + client: DataFactoryManagementClient + resourceGroupName: string + factoryName: string +}): Promise => { + const pipelines: PipelineResource[] = [] + const pipelinesIterable: PagedAsyncIterableIterator = + client.pipelines.listByFactory(resourceGroupName, factoryName) + + await tryCatchWrapper( + async () => { + for await (const pipeline of pipelinesIterable) { + if (pipeline) { + pipelines.push(pipeline) + } + } + }, + { + service: serviceName, + client, + scope: 'pipelines', + operation: 'listByFactory', + } + ) + return pipelines +} + +const getDataFlows = async ({ + client, + resourceGroupName, + factoryName, +}: { + client: DataFactoryManagementClient + resourceGroupName: string + factoryName: string +}): Promise => { + const flows: DataFlowResource[] = [] + const flowsIterable: PagedAsyncIterableIterator = + client.dataFlows.listByFactory(resourceGroupName, factoryName) + + await tryCatchWrapper( + async () => { + for await (const flow of flowsIterable) { + if (flow) { + flows.push(flow) + } + } + }, + { + service: serviceName, + client, + scope: 'flows', + operation: 'listByFactory', + } + ) + return flows +} + +const getDatasets = async ({ + client, + resourceGroupName, + factoryName, +}: { + client: DataFactoryManagementClient + resourceGroupName: string + factoryName: string +}): Promise => { + const datasets: DatasetResource[] = [] + const datasetsIterable: PagedAsyncIterableIterator = + client.datasets.listByFactory(resourceGroupName, factoryName) + + await tryCatchWrapper( + async () => { + for await (const dataset of datasetsIterable) { + if (dataset) { + datasets.push(dataset) + } + } + }, + { + service: serviceName, + client, + scope: 'datasets', + operation: 'listByFactory', + } + ) + return datasets +} + +const getIntegrationRuntimes = async ({ + client, + resourceGroupName, + factoryName, +}: { + client: DataFactoryManagementClient + resourceGroupName: string + factoryName: string +}): Promise => { + const runtimes: IntegrationRuntimeResource[] = [] + const runtimesIterable: PagedAsyncIterableIterator = + client.integrationRuntimes.listByFactory(resourceGroupName, factoryName) + + await tryCatchWrapper( + async () => { + for await (const runtime of runtimesIterable) { + if (runtime) { + runtimes.push(runtime) + } + } + }, + { + service: serviceName, + client, + scope: 'integrationRuntimes', + operation: 'listByFactory', + } + ) + return runtimes +} + +const getLinkedServices = async ({ + client, + resourceGroupName, + factoryName, +}: { + client: DataFactoryManagementClient + resourceGroupName: string + factoryName: string +}): Promise => { + const linkedServices: LinkedServiceResource[] = [] + const linkedServicesIterable: PagedAsyncIterableIterator = + client.linkedServices.listByFactory(resourceGroupName, factoryName) + + await tryCatchWrapper( + async () => { + for await (const linkedService of linkedServicesIterable) { + if (linkedService) { + linkedServices.push(linkedService) + } + } + }, + { + service: serviceName, + client, + scope: 'linkedServices', + operation: 'listByFactory', + } + ) + return linkedServices +} + +const getTriggers = async ({ + client, + resourceGroupName, + factoryName, +}: { + client: DataFactoryManagementClient + resourceGroupName: string + factoryName: string +}): Promise => { + const triggers: TriggerResource[] = [] + const triggersIterable: PagedAsyncIterableIterator = + client.triggers.listByFactory(resourceGroupName, factoryName) + + await tryCatchWrapper( + async () => { + for await (const trigger of triggersIterable) { + if (trigger) { + triggers.push(trigger) + } + } + }, + { + service: serviceName, + client, + scope: 'triggers', + operation: 'listByFactory', + } + ) + return triggers } export default async ({ @@ -49,10 +250,46 @@ export default async ({ if (factory) { const { location, tags, ...restOfFactory } = factory const resourceGroupId = getResourceGroupFromEntity(restOfFactory) + const pipelines = await getPipelines({ + client, + resourceGroupName: resourceGroupId, + factoryName: factory.name, + }) + const dataFlows = await getDataFlows({ + client, + resourceGroupName: resourceGroupId, + factoryName: factory.name, + }) + const datasets = await getDatasets({ + client, + resourceGroupName: resourceGroupId, + factoryName: factory.name, + }) + const integrationRuntimes = await getIntegrationRuntimes({ + client, + resourceGroupName: resourceGroupId, + factoryName: factory.name, + }) + const linkedServices = await getLinkedServices({ + client, + resourceGroupName: resourceGroupId, + factoryName: factory.name, + }) + const triggers = await getTriggers({ + client, + resourceGroupName: resourceGroupId, + factoryName: factory.name, + }) factories.push({ region: lowerCaseLocation(location), Tags: tags || {}, resourceGroupId, + pipelines, + dataFlows, + datasets, + integrationRuntimes, + linkedServices, + triggers, ...restOfFactory, }) } @@ -68,7 +305,7 @@ export default async ({ logger.debug(lt.foundDataFactory(factories.length)) const result: { [property: string]: RawAzureDataFactory[] } = {} - factories.map(({ region, tags, ...rest }) => { + factories.forEach(({ region, tags, ...rest }) => { if (regions.includes(region)) { if (!result[region]) { result[region] = [] diff --git a/src/services/dataFactory/format.ts b/src/services/dataFactory/format.ts index b7e01259..f49887fa 100644 --- a/src/services/dataFactory/format.ts +++ b/src/services/dataFactory/format.ts @@ -1,4 +1,12 @@ -import { AzureDataFactory } from '../../types/generated' +import { + AzureDataFactory, + AzureDataFactoryDataFlow, + AzureDataFactoryDataset, + AzureDataFactoryIntegrationRuntime, + AzureDataFactoryLinkedService, + AzureDataFactoryPipeline, + AzureDataFactoryTrigger, +} from '../../types/generated' import { formatTagsFromMap } from '../../utils/format' import { RawAzureDataFactory } from './data' @@ -23,6 +31,12 @@ export default ({ createTime, version, publicNetworkAccess, + pipelines = [], + dataFlows = [], + datasets = [], + integrationRuntimes = [], + linkedServices = [], + triggers = [], } = service return { @@ -45,5 +59,77 @@ export default ({ createTime, version, publicNetworkAccess, + pipelines: + pipelines?.map( + ({ + id: pipelineId, + name: pipelineName, + type: pipelineType, + }): AzureDataFactoryPipeline => ({ + id: pipelineId, + name: pipelineName, + type: pipelineType, + }) + ) ?? [], + dataFlows: + dataFlows?.map( + ({ + id: dataFlowId, + name: dataFlowName, + type: dataFlowType, + }): AzureDataFactoryDataFlow => ({ + id: dataFlowId, + name: dataFlowName, + type: dataFlowType, + }) + ) ?? [], + datasets: + datasets?.map( + ({ + id: datasetId, + name: datasetName, + properties: { type: datasetType }, + }): AzureDataFactoryDataset => ({ + id: datasetId, + name: datasetName, + type: datasetType, + }) + ) ?? [], + integrationRuntimes: + integrationRuntimes?.map( + ({ + id: runtimeId, + name: runtimeName, + properties: { type: runtimeType }, + }): AzureDataFactoryIntegrationRuntime => ({ + id: runtimeId, + name: runtimeName, + type: runtimeType, + }) + ) ?? [], + linkedServices: + linkedServices?.map( + ({ + id: runtimeId, + name: runtimeName, + properties: { type: runtimeType }, + }): AzureDataFactoryLinkedService => ({ + id: runtimeId, + name: runtimeName, + type: runtimeType, + }) + ) ?? [], + triggers: + triggers?.map( + ({ + id: triggerId, + name: triggerName, + properties: { type: triggerType }, + }): AzureDataFactoryTrigger => ({ + id: triggerId, + name: triggerName, + type: triggerType, + }) + ) ?? [], } } diff --git a/src/services/dataFactory/schema.graphql b/src/services/dataFactory/schema.graphql index 9a6d5115..7f173776 100644 --- a/src/services/dataFactory/schema.graphql +++ b/src/services/dataFactory/schema.graphql @@ -1,3 +1,75 @@ +type azureDataFactoryIntegrationRuntime + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) +} + +type azureDataFactoryDataset + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) +} + +type azureDataFactoryDataFlow + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) +} + +type azureDataFactoryPipeline + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) +} + +type azureDataFactoryLinkedService + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) +} + +type azureDataFactoryTrigger + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + name: String @search(by: [hash, regexp]) + type: String @search(by: [hash, regexp]) +} + type azureDataFactoryIdentity { type: String @search(by: [hash, regexp]) principalId: String @search(by: [hash, regexp]) @@ -16,6 +88,12 @@ type azureDataFactory implements azureResource createTime: String @search(by: [hash, regexp]) version: String @search(by: [hash, regexp]) publicNetworkAccess: String @search(by: [hash, regexp]) + pipelines: [azureDataFactoryPipeline] + dataFlows: [azureDataFactoryDataFlow] + datasets: [azureDataFactoryDataset] + integrationRuntimes: [azureDataFactoryIntegrationRuntime] + linkedServices: [azureDataFactoryLinkedService] + triggers: [azureDataFactoryTrigger] resourceGroup: [azureResourceGroup] @hasInverse(field: dataFactories) integrationRuntimes: [azureIntegrationRuntime] @hasInverse(field: dataFactory) } diff --git a/src/types/generated.ts b/src/types/generated.ts index 25ac1891..b6b562dc 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -2772,21 +2772,62 @@ export type AzureDataCollectionRuleWindowsEventLogDataSource = { export type AzureDataFactory = AzureResource & { createTime?: Maybe; + dataFlows?: Maybe>>; + datasets?: Maybe>>; eTag?: Maybe; identity?: Maybe; integrationRuntimes?: Maybe>>; + linkedServices?: Maybe>>; + pipelines?: Maybe>>; provisioningState?: Maybe; publicNetworkAccess?: Maybe; resourceGroup?: Maybe>>; + triggers?: Maybe>>; version?: Maybe; }; +export type AzureDataFactoryDataFlow = { + id: Scalars['String']; + name?: Maybe; + type?: Maybe; +}; + +export type AzureDataFactoryDataset = { + id: Scalars['String']; + name?: Maybe; + type?: Maybe; +}; + export type AzureDataFactoryIdentity = { principalId?: Maybe; tenantId?: Maybe; type?: Maybe; }; +export type AzureDataFactoryIntegrationRuntime = { + id: Scalars['String']; + name?: Maybe; + type?: Maybe; +}; + +export type AzureDataFactoryLinkedService = { + id: Scalars['String']; + name?: Maybe; + type?: Maybe; +}; + +export type AzureDataFactoryPipeline = { + id: Scalars['String']; + name?: Maybe; + type?: Maybe; +}; + +export type AzureDataFactoryTrigger = { + id: Scalars['String']; + name?: Maybe; + type?: Maybe; +}; + export type AzureDataLakeStorageAccount = AzureResource & { resourceGroup?: Maybe>>; suffix?: Maybe; From 34b2a2cb69777b7ee6f314a2ffe36774b26b4230 Mon Sep 17 00:00:00 2001 From: autocloud-deploy-bot Date: Fri, 9 Jun 2023 16:54:12 +0000 Subject: [PATCH 15/20] chore(release): 0.67.0-alpha.4 # [0.67.0-alpha.4](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.3...0.67.0-alpha.4) (2023-06-09) ### Features * **azure:** add missing mySQL Server services ([90131c8](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/90131c83d231a8ef3d2f29b23412e978688e32dc)) --- CHANGELOG.md | 7 +++++++ package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcef0e71..bdf72a4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# [0.67.0-alpha.4](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.3...0.67.0-alpha.4) (2023-06-09) + + +### Features + +* **azure:** add missing mySQL Server services ([90131c8](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/90131c83d231a8ef3d2f29b23412e978688e32dc)) + # [0.67.0-alpha.3](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.2...0.67.0-alpha.3) (2023-06-07) diff --git a/package.json b/package.json index b302450a..ea2450d4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@cloudgraph/cg-provider-azure", - "version": "0.67.0-alpha.3", + "version": "0.67.0-alpha.4", "description": "CloudGraph provider plugin for Azure used to fetch Azure cloud data.", "publishConfig": { "registry": "https://registry.npmjs.org/", From 33ae7da24dd69264344016bf4af3b25a6264c43d Mon Sep 17 00:00:00 2001 From: Demian Rosas Date: Fri, 9 Jun 2023 17:04:37 -0300 Subject: [PATCH 16/20] feat: add publicIp missing services --- README.md | 12 ++-- src/enums/serviceAliases.ts | 1 + src/enums/serviceMap.ts | 2 + src/enums/services.ts | 1 + src/properties/logger.ts | 10 ++- src/services/publicIpPrefix/data.ts | 76 ++++++++++++++++++++++ src/services/publicIpPrefix/format.ts | 24 +++++++ src/services/publicIpPrefix/index.ts | 16 +++++ src/services/publicIpPrefix/mutation.ts | 5 ++ src/services/publicIpPrefix/schema.graphql | 18 +++++ src/types/generated.ts | 4 ++ 11 files changed, 162 insertions(+), 7 deletions(-) create mode 100644 src/services/publicIpPrefix/data.ts create mode 100644 src/services/publicIpPrefix/format.ts create mode 100644 src/services/publicIpPrefix/index.ts create mode 100644 src/services/publicIpPrefix/mutation.ts create mode 100644 src/services/publicIpPrefix/schema.graphql diff --git a/README.md b/README.md index 01de1d6f..ba0ee265 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,13 @@ Scan cloud infrastructure via the [Azure SDK](https://github.com/Azure/azure-sdk -- [Install](#install) -- [Authentication](#authentication) -- [Supported Services](#supported-services) - +- [CloudGraph Azure Provider](#cloudgraph-azure-provider) + - [Docs](#docs) + - [Install](#install) + - [Authentication](#authentication) + - [Supported Services](#supported-services) + - [Development](#development) + - [Testing](#testing) ## Docs @@ -95,6 +98,7 @@ CloudGraph needs read permissions in order to ingest your data. To keep things e | postgreSqlServers | resourceGroup, databasePostgreSql | | privateDns | resourceGroup | | publicIp | networkInterface, resourceGroup | +| publicIpPrefix | | | recoveryInstances | recoveryVaults, resourceGroup | | recoveryPolicies | recoveryVaults, resourceGroup | | recoveryVaults | recoveryInstances, recoveryPolicies, resourceGroup | diff --git a/src/enums/serviceAliases.ts b/src/enums/serviceAliases.ts index 6ccdabbe..53971bf3 100644 --- a/src/enums/serviceAliases.ts +++ b/src/enums/serviceAliases.ts @@ -56,6 +56,7 @@ export default { [services.postgreSqlServers]: 'postgreSqlServers', [services.privateDns]: 'privateDnsZones', [services.publicIp]: 'publicIps', + [services.publicIpPrefix]: 'publicIpPrefixes', [services.recoveryVaults]: 'recoveryVaults', [services.recoveryInstances]: 'recoveryInstances', [services.recoveryPolicies]: 'recoveryPolicies', diff --git a/src/enums/serviceMap.ts b/src/enums/serviceMap.ts index fdeb7f2e..282f0fe2 100644 --- a/src/enums/serviceMap.ts +++ b/src/enums/serviceMap.ts @@ -51,6 +51,7 @@ import AzurePolicyAssignment from '../services/policyAssignment' import AzurePostgreSqlServer from '../services/postgreSqlServers' import AzurePrivateDns from '../services/privateDns' import AzurePublicIp from '../services/publicIp' +import AzurePublicIpPrefix from '../services/publicIpPrefix' import AzureRecoveryVault from '../services/recoveryVaults' import AzureRecoveryInstance from '../services/recoveryInstances' import AzureRecoveryPolicy from '../services/recoveryPolicies' @@ -160,6 +161,7 @@ export default { [services.postgreSqlServers]: AzurePostgreSqlServer, [services.privateDns]: AzurePrivateDns, [services.publicIp]: AzurePublicIp, + [services.publicIpPrefix]: AzurePublicIpPrefix, [services.recoveryVaults]: AzureRecoveryVault, [services.recoveryInstances]: AzureRecoveryInstance, [services.recoveryPolicies]: AzureRecoveryPolicy, diff --git a/src/enums/services.ts b/src/enums/services.ts index 006d416e..fc15d0d1 100644 --- a/src/enums/services.ts +++ b/src/enums/services.ts @@ -62,6 +62,7 @@ export default { postgreSqlServers: 'postgreSqlServers', privateDns: 'privateDns', publicIp: 'publicIp', + publicIpPrefix: 'publicIpPrefix', recoveryVaults: 'recoveryVaults', recoveryInstances: 'recoveryInstances', recoveryPolicies: 'recoveryPolicies', diff --git a/src/properties/logger.ts b/src/properties/logger.ts index 8b14e71c..e48831cc 100644 --- a/src/properties/logger.ts +++ b/src/properties/logger.ts @@ -20,7 +20,8 @@ export default { foundAKSManagedClusters: (num: number): string => `Found ${num} AKS managed clusters`, /* App Gateway */ - foundApplicationGateway: (num: number): string => `Found ${num} Application Gateways`, + foundApplicationGateway: (num: number): string => + `Found ${num} Application Gateways`, /* App Service */ foundAppServiceEnvironments: (num: number): string => `Found ${num} app service environments`, @@ -152,11 +153,14 @@ export default { `Found ${num} private dns zones`, // Public Ips foundPublicIps: (num: number): string => `Found ${num} public ips`, + foundPublicIpPrefixes: (num: number): string => + `Found ${num} public ip prefixes`, // Recovery Vaults foundRecoveryVaults: (num: number): string => `Found ${num} recovery vaults`, foundRecoveryInstances: (num: number): string => - `Found ${num} recovery instances`, - foundRecoveryPolicies: (num: number): string => `Found ${num} recovery policies`, + `Found ${num} recovery instances`, + foundRecoveryPolicies: (num: number): string => + `Found ${num} recovery policies`, // RedisCache foundRedisCaches: (num: number): string => `Found ${num} Redis caches`, // Replication Appliances diff --git a/src/services/publicIpPrefix/data.ts b/src/services/publicIpPrefix/data.ts new file mode 100644 index 00000000..6649d493 --- /dev/null +++ b/src/services/publicIpPrefix/data.ts @@ -0,0 +1,76 @@ +import { NetworkManagementClient, PublicIPPrefix } from '@azure/arm-network' +import { PagedAsyncIterableIterator } from '@azure/core-paging' +import CloudGraph from '@cloudgraph/sdk' + +import azureLoggerText from '../../properties/logger' +import { AzureServiceInput, TagMap } from '../../types' +import { tryCatchWrapper } from '../../utils' +import { lowerCaseLocation } from '../../utils/format' +import { getResourceGroupFromEntity } from '../../utils/idParserUtils' + +const { logger } = CloudGraph +const lt = { ...azureLoggerText } +const serviceName = 'PublicIp' + +export interface RawAzurePublicIpPrefix + extends Omit { + region: string + resourceGroupId: string + Tags: TagMap +} + +export default async ({ + regions, + config, +}: AzureServiceInput): Promise<{ + [property: string]: RawAzurePublicIpPrefix[] +}> => { + try { + const { tokenCredentials, subscriptionId } = config + const client = new NetworkManagementClient(tokenCredentials, subscriptionId) + + const publicIpPrefixData: PublicIPPrefix[] = [] + await tryCatchWrapper( + async () => { + const publicIpPrefixIterable: PagedAsyncIterableIterator = + client.publicIPPrefixes.listAll() + for await (const publicIpPrefix of publicIpPrefixIterable) { + publicIpPrefix && publicIpPrefixData.push(publicIpPrefix) + } + }, + { + service: serviceName, + client, + scope: 'publicIPPrefixes', + operation: 'listAll', + } + ) + + const result: { + [property: string]: RawAzurePublicIpPrefix[] + } = {} + let numOfGroups = 0 + publicIpPrefixData.forEach(({ tags, location, ...rest }) => { + const region = lowerCaseLocation(location) + if (regions.includes(region)) { + if (!result[region]) { + result[region] = [] + } + const resourceGroupId = getResourceGroupFromEntity(rest) + result[region].push({ + ...rest, + region, + resourceGroupId, + Tags: tags || {}, + }) + numOfGroups += 1 + } + }) + logger.debug(lt.foundPublicIpPrefixes(numOfGroups)) + + return result + } catch (e) { + logger.error(e) + return {} + } +} diff --git a/src/services/publicIpPrefix/format.ts b/src/services/publicIpPrefix/format.ts new file mode 100644 index 00000000..bb3608e7 --- /dev/null +++ b/src/services/publicIpPrefix/format.ts @@ -0,0 +1,24 @@ +import { AzurePublicIpPrefix } from '../../types/generated' +import { formatTagsFromMap } from '../../utils/format' +import { RawAzurePublicIpPrefix } from './data' + +export default ({ + service, + account: subscriptionId, + region, +}: { + service: RawAzurePublicIpPrefix + account: string + region: string +}): AzurePublicIpPrefix => { + const { id, name, ipTags, resourceGroupId, Tags } = service + return { + id, + name, + region, + resourceGroupId, + subscriptionId, + ipTags, + tags: formatTagsFromMap(Tags), + } +} diff --git a/src/services/publicIpPrefix/index.ts b/src/services/publicIpPrefix/index.ts new file mode 100644 index 00000000..671f3cdd --- /dev/null +++ b/src/services/publicIpPrefix/index.ts @@ -0,0 +1,16 @@ +import { Service } from '@cloudgraph/sdk' +import BaseService from '../base' +import format from './format' +import mutation from './mutation' +import getData from './data' + +export default class AzurePublicIpPrefix + extends BaseService + implements Service +{ + format = format.bind(this) + + getData = getData.bind(this) + + mutation = mutation +} diff --git a/src/services/publicIpPrefix/mutation.ts b/src/services/publicIpPrefix/mutation.ts new file mode 100644 index 00000000..ec15bd9a --- /dev/null +++ b/src/services/publicIpPrefix/mutation.ts @@ -0,0 +1,5 @@ +export default `mutation($input: [AddazurePublicIpPrefixInput!]!) { + addazurePublicIpPrefix(input: $input, upsert: true) { + numUids + } +}` diff --git a/src/services/publicIpPrefix/schema.graphql b/src/services/publicIpPrefix/schema.graphql new file mode 100644 index 00000000..5ea40d8f --- /dev/null +++ b/src/services/publicIpPrefix/schema.graphql @@ -0,0 +1,18 @@ +type azurePublicIpTags + @generate( + query: { get: false, query: true, aggregate: false } + mutation: { add: false, delete: false } + subscription: false + ) { + ipTagType: String @search(by: [hash, regexp]) + tag: String @search(by: [hash, regexp]) +} + +type azurePublicIpPrefix implements azureResource + @generate( + query: { get: true, query: true, aggregate: true } + mutation: { add: true, delete: false } + ) + @key(fields: "id") { + ipTags: [azurePublicIpTags] +} diff --git a/src/types/generated.ts b/src/types/generated.ts index ae57b14a..8faebd6f 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -4438,6 +4438,10 @@ export type AzurePublicIpDnsSettings = { reverseFqdn?: Maybe; }; +export type AzurePublicIpPrefix = AzureResource & { + ipTags?: Maybe>>; +}; + export type AzurePublicIpTags = { ipTagType?: Maybe; tag?: Maybe; From 2f8cfb06d23e82977f0ce1ed266b2397741d0c44 Mon Sep 17 00:00:00 2001 From: autocloud-deploy-bot Date: Wed, 14 Jun 2023 15:26:24 +0000 Subject: [PATCH 17/20] chore(release): 0.67.0-alpha.5 # [0.67.0-alpha.5](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.4...0.67.0-alpha.5) (2023-06-14) ### Features * add publicIp missing services ([33ae7da](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/33ae7da24dd69264344016bf4af3b25a6264c43d)) * Added missing subnets ids and endpoint policies for subnets ([baa0a6e](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/baa0a6e72510f70e217a5c516479db8d5d05294f)) * **azure:** Add missing dataFactory services ([2b9a48b](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/2b9a48b9e848aad2775069167dcb406ad0dc8f77)) * **azure:** add route table services ([5573e8e](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/5573e8eeee9387d50e440e7d1aecd4c529f3c531)) * Fetched records for private dns ([83cbdd8](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/83cbdd809f81e750d5abc83bf503dba47e12fe50)) * Fetched virtual network links for private zone ([414ceec](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/414ceece3af505c18a76d6141d63a8d85e20067b)) --- CHANGELOG.md | 12 ++++++++++++ package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdf72a4b..89ff9e4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# [0.67.0-alpha.5](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.4...0.67.0-alpha.5) (2023-06-14) + + +### Features + +* add publicIp missing services ([33ae7da](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/33ae7da24dd69264344016bf4af3b25a6264c43d)) +* Added missing subnets ids and endpoint policies for subnets ([baa0a6e](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/baa0a6e72510f70e217a5c516479db8d5d05294f)) +* **azure:** Add missing dataFactory services ([2b9a48b](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/2b9a48b9e848aad2775069167dcb406ad0dc8f77)) +* **azure:** add route table services ([5573e8e](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/5573e8eeee9387d50e440e7d1aecd4c529f3c531)) +* Fetched records for private dns ([83cbdd8](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/83cbdd809f81e750d5abc83bf503dba47e12fe50)) +* Fetched virtual network links for private zone ([414ceec](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/414ceece3af505c18a76d6141d63a8d85e20067b)) + # [0.67.0-alpha.4](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.3...0.67.0-alpha.4) (2023-06-09) diff --git a/package.json b/package.json index ea2450d4..30a45ebd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@cloudgraph/cg-provider-azure", - "version": "0.67.0-alpha.4", + "version": "0.67.0-alpha.5", "description": "CloudGraph provider plugin for Azure used to fetch Azure cloud data.", "publishConfig": { "registry": "https://registry.npmjs.org/", From bd58d1dce0cd55d86828b0a463d41061352f2517 Mon Sep 17 00:00:00 2001 From: Alejandro Cotroneo Date: Tue, 18 Jul 2023 19:08:05 -0300 Subject: [PATCH 18/20] feat: collect crawl error logs --- package.json | 4 ++-- src/services/index.ts | 15 ++++++++++----- src/utils/index.ts | 14 ++++++++++++++ yarn.lock | 8 ++++---- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 30a45ebd..c9887231 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "@azure/identity": "^2.0.4", "@azure/storage-blob": "^12.8.0", "@azure/storage-queue": "^12.9.0", - "@cloudgraph/sdk": "0.22.1", + "@cloudgraph/sdk": "0.23.0", "@graphql-tools/load-files": "^6.5.3", "@graphql-tools/merge": "^8.2.3", "@microsoft/microsoft-graph-client": "^3.0.2", @@ -126,4 +126,4 @@ "**/glob-parent": "^5.1.2", "**/uri-js": "^3.0.1" } -} +} \ No newline at end of file diff --git a/src/services/index.ts b/src/services/index.ts index 59c2c1f1..38971180 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -25,7 +25,7 @@ import { GLOBAL_REGION, } from '../config/constants' import { obfuscateSensitiveString } from '../utils/format' -import { checkAndMergeConnections, sortResourcesDependencies } from '../utils' +import { getAllProviderErrors, sortResourcesDependencies } from '../utils' import { createDiffSecs } from '../utils/dateutils' import { getClientSecretCredentials, @@ -151,9 +151,10 @@ export default class Provider extends CloudGraph.Client { )}` ) this.logger.success( - `subscriptionIds: ${this.subscriptions.length > 1 - ? this.subscriptions.join(', ') - : this.subscriptions[0] + `subscriptionIds: ${ + this.subscriptions.length > 1 + ? this.subscriptions.join(', ') + : this.subscriptions[0] }` ) this.logger.success( @@ -700,11 +701,15 @@ export default class Provider extends CloudGraph.Client { } } - return this.enhanceData({ + const enhancedData = await this.enhanceData({ subscriptions: subscriptions.data[GLOBAL_REGION], configuredRegions, rawData, data: result, }) + + const errors = getAllProviderErrors() + + return { ...enhancedData, errors } } } diff --git a/src/utils/index.ts b/src/utils/index.ts index 290d98dc..6fe69455 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -3,6 +3,7 @@ import camelCase from 'lodash/camelCase' import isEmpty from 'lodash/isEmpty' import unionWith from 'lodash/unionWith' import isEqual from 'lodash/isEqual' +import { ProviderError } from '@cloudgraph/sdk/dist/src/types' import relations from '../enums/relations' import { AzureDebugScope, @@ -65,6 +66,12 @@ export function initTestConfig(): void { jest.setTimeout(900000) } +const errorsHistory: ProviderError[] = [] + +export function getAllProviderErrors(): ProviderError[] { + return errorsHistory +} + export function generateAzureErrorLog( service: string, functionName: string, @@ -87,6 +94,13 @@ export function generateAzureErrorLog( } else { logger.debug(err.message) } + if (err?.statusCode !== 429) { + errorsHistory.push({ + service, + function: functionName, + message: err?.message || 'Unknown error', + }) + } throw new Error() } diff --git a/yarn.lock b/yarn.lock index ddf2921d..14d849cc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1282,10 +1282,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@cloudgraph/sdk@0.22.1": - version "0.22.1" - resolved "https://registry.yarnpkg.com/@cloudgraph/sdk/-/sdk-0.22.1.tgz#bf5c2f263df17b085147a3ca234ac5b0a8806545" - integrity sha512-aMrnoTRRIODAVvgtmqGZPB7Z8x1kMgKjFtLjpiHa121o0GCEZCjbxDmU8T3DwmRwI1u4QD5iKbXf9dB0573mJA== +"@cloudgraph/sdk@0.23.0": + version "0.23.0" + resolved "https://registry.yarnpkg.com/@cloudgraph/sdk/-/sdk-0.23.0.tgz#3f6012c0e3573f527da27ebcccb8841bbeffd21b" + integrity sha512-c+Ozu8yLK3RlAcKy4d0fdTQc9JkdUJ2ESmrcW7oC5XdIARrDWQ5+Kr8AGPABOK1/UYtFyVR2JmUHnoSJdGAv6Q== dependencies: "@graphql-tools/load-files" "^6.5.3" "@graphql-tools/merge" "^8.2.1" From 62922f0e989259cef2c879b8d01d628a0114fec6 Mon Sep 17 00:00:00 2001 From: autocloud-deploy-bot Date: Wed, 19 Jul 2023 21:04:48 +0000 Subject: [PATCH 19/20] chore(release): 0.67.0-alpha.6 # [0.67.0-alpha.6](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.5...0.67.0-alpha.6) (2023-07-19) ### Features * collect crawl error logs ([bd58d1d](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/bd58d1dce0cd55d86828b0a463d41061352f2517)) --- CHANGELOG.md | 7 +++++++ package.json | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89ff9e4f..ac74307b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# [0.67.0-alpha.6](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.5...0.67.0-alpha.6) (2023-07-19) + + +### Features + +* collect crawl error logs ([bd58d1d](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/bd58d1dce0cd55d86828b0a463d41061352f2517)) + # [0.67.0-alpha.5](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.4...0.67.0-alpha.5) (2023-06-14) diff --git a/package.json b/package.json index c9887231..6a2459c0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@cloudgraph/cg-provider-azure", - "version": "0.67.0-alpha.5", + "version": "0.67.0-alpha.6", "description": "CloudGraph provider plugin for Azure used to fetch Azure cloud data.", "publishConfig": { "registry": "https://registry.npmjs.org/", @@ -126,4 +126,4 @@ "**/glob-parent": "^5.1.2", "**/uri-js": "^3.0.1" } -} \ No newline at end of file +} From 060a4db130925213f670bfe851c080d32318b463 Mon Sep 17 00:00:00 2001 From: autocloud-deploy-bot Date: Wed, 19 Jul 2023 21:07:57 +0000 Subject: [PATCH 20/20] chore(release): 0.67.0-beta.1 # [0.67.0-beta.1](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.66.0...0.67.0-beta.1) (2023-07-19) ### Bug Fixes * Added missing attribute for cosmosdb schema ([cde0c79](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/cde0c791374d648f77541b1189ddbd3062ea67aa)) ### Features * add publicIp missing services ([33ae7da](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/33ae7da24dd69264344016bf4af3b25a6264c43d)) * Added missing subnets ids and endpoint policies for subnets ([baa0a6e](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/baa0a6e72510f70e217a5c516479db8d5d05294f)) * **azure:** Add missing dataFactory services ([2b9a48b](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/2b9a48b9e848aad2775069167dcb406ad0dc8f77)) * **azure:** add missing mySQL Server services ([90131c8](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/90131c83d231a8ef3d2f29b23412e978688e32dc)) * **azure:** Add missing postgreSql server services ([8aeff96](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/8aeff969f44cfa8751fefacbbca6597cf7213cd1)) * **azure:** Add missing SQL Server services ([41bffd4](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/41bffd48c728303d0d697509c9efa9a5c7442166)) * **azure:** add route table services ([5573e8e](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/5573e8eeee9387d50e440e7d1aecd4c529f3c531)) * **azure:** Support missing storageBlob and storageContainer services ([b179ebf](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/b179ebf76107a6285c978067b79f544e995df66f)) * collect crawl error logs ([bd58d1d](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/bd58d1dce0cd55d86828b0a463d41061352f2517)) * Exposed containersIds for cosmosdb account ([8b45320](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/8b4532048fc8137539489adb38eee101ff867039)) * Fetched records for private dns ([83cbdd8](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/83cbdd809f81e750d5abc83bf503dba47e12fe50)) * Fetched virtual network links for private zone ([414ceec](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/414ceece3af505c18a76d6141d63a8d85e20067b)) --- CHANGELOG.md | 23 +++++++++++++++++++++++ package.json | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac74307b..994f270e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,26 @@ +# [0.67.0-beta.1](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.66.0...0.67.0-beta.1) (2023-07-19) + + +### Bug Fixes + +* Added missing attribute for cosmosdb schema ([cde0c79](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/cde0c791374d648f77541b1189ddbd3062ea67aa)) + + +### Features + +* add publicIp missing services ([33ae7da](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/33ae7da24dd69264344016bf4af3b25a6264c43d)) +* Added missing subnets ids and endpoint policies for subnets ([baa0a6e](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/baa0a6e72510f70e217a5c516479db8d5d05294f)) +* **azure:** Add missing dataFactory services ([2b9a48b](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/2b9a48b9e848aad2775069167dcb406ad0dc8f77)) +* **azure:** add missing mySQL Server services ([90131c8](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/90131c83d231a8ef3d2f29b23412e978688e32dc)) +* **azure:** Add missing postgreSql server services ([8aeff96](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/8aeff969f44cfa8751fefacbbca6597cf7213cd1)) +* **azure:** Add missing SQL Server services ([41bffd4](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/41bffd48c728303d0d697509c9efa9a5c7442166)) +* **azure:** add route table services ([5573e8e](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/5573e8eeee9387d50e440e7d1aecd4c529f3c531)) +* **azure:** Support missing storageBlob and storageContainer services ([b179ebf](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/b179ebf76107a6285c978067b79f544e995df66f)) +* collect crawl error logs ([bd58d1d](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/bd58d1dce0cd55d86828b0a463d41061352f2517)) +* Exposed containersIds for cosmosdb account ([8b45320](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/8b4532048fc8137539489adb38eee101ff867039)) +* Fetched records for private dns ([83cbdd8](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/83cbdd809f81e750d5abc83bf503dba47e12fe50)) +* Fetched virtual network links for private zone ([414ceec](https://github.com/cloudgraphdev/cloudgraph-provider-azure/commit/414ceece3af505c18a76d6141d63a8d85e20067b)) + # [0.67.0-alpha.6](https://github.com/cloudgraphdev/cloudgraph-provider-azure/compare/0.67.0-alpha.5...0.67.0-alpha.6) (2023-07-19) diff --git a/package.json b/package.json index 6a2459c0..9fe2929c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@cloudgraph/cg-provider-azure", - "version": "0.67.0-alpha.6", + "version": "0.67.0-beta.1", "description": "CloudGraph provider plugin for Azure used to fetch Azure cloud data.", "publishConfig": { "registry": "https://registry.npmjs.org/",