From 8c19d9065667dbba638b4932b1dea5628f0ff5bd Mon Sep 17 00:00:00 2001 From: Harold Date: Fri, 26 Aug 2022 17:31:51 -0400 Subject: [PATCH 1/2] fix(integrationRuntimes,logAnalyticsWorkspace,sqlServers): several fixes - Removed several sensitive data fields from schemas - Removed spread operations to prevent unmapped field errors - Format fixes --- src/services/integrationRuntimes/format.ts | 225 ++++++++++------- .../integrationRuntimes/schema.graphql | 8 - src/services/logAnalyticsWorkspace/format.ts | 20 +- src/services/sqlServers/format.ts | 228 +++++++++++++++--- src/services/sqlServers/schema.graphql | 6 - src/types/generated.ts | 14 -- 6 files changed, 339 insertions(+), 162 deletions(-) diff --git a/src/services/integrationRuntimes/format.ts b/src/services/integrationRuntimes/format.ts index 5588f389..41aa7f16 100644 --- a/src/services/integrationRuntimes/format.ts +++ b/src/services/integrationRuntimes/format.ts @@ -8,12 +8,12 @@ import { IntegrationRuntimeEdition, PackageStore, } from '@azure/arm-datafactory' -import { isArray, isEmpty, isObject, isString } from 'lodash' +import { isEmpty } from 'lodash' import { AzureIntegrationRuntime, AzureIntegrationRuntimeProperties, - AzureSecretBaseUnion, - AzureLinkedServiceReferencePatameters, + AzureCustomSetupBaseUnion, + AzurePackageStore, } from '../../types/generated' import { RawAzureIntegrationRuntimeResource } from './data' @@ -44,12 +44,9 @@ export interface RawAzureSecretBaseUnion { export interface RawAzureCustomSetupBaseUnion extends CustomSetupBase { targetName?: Record - userName?: Record - password?: RawAzureSecretBaseUnion variableName?: string variableValue?: string componentName?: string - licenseKey?: RawAzureSecretBaseUnion version?: string } @@ -113,47 +110,11 @@ export interface RawAzureIntegrationRuntimeUnion { referenceName?: string } -const formatParameters = ( - params: { [propertyName: string]: Record } -): AzureLinkedServiceReferencePatameters[] => { - if (isEmpty(params)) { - return [] - } - return Object.entries(params).map(([k, v]) => ({ - id: cuid(), - key: k, - value: Object.entries(v.value).map(([k2, v2]) => ({ - id: isObject(v) ? cuid() : `${k2}:${v2}`, - key: k, - value: - (isString(v2) && v2) || - (isArray(v2) && - (v2 as Array) - .map(i => (isString(i) && i) || JSON.stringify(i)) - .join(',')) || - JSON.stringify(v2), - })), - })) || [] -} - -const formatAzureSecret = ( - secret: RawAzureSecretBaseUnion -): AzureSecretBaseUnion => { - if (isEmpty(secret)) { - return {} - } - return { - type: secret?.type, - value: secret?.value, - store: { - type: secret?.store?.type, - referenceName: secret?.store?.referenceName, - parameters: formatParameters(secret?.store?.parameters), - }, - secretName: Object.values(secret?.secretName ?? {}).join(''), - secretVersion: Object.values(secret?.secretVersion ?? {}).join(''), - } -} +// function useInterfaceToExtractData( +// data: unknown +// ): ExpectedDataType { +// return data as ExpectedDataType +// } const formatProperties = ( runtimeProperties?: RawAzureIntegrationRuntimeUnion @@ -164,63 +125,151 @@ const formatProperties = ( const { type, - ssisProperties = {}, - customerVirtualNetwork = {}, - managedVirtualNetwork = {}, - computeProperties = {}, - linkedInfo = {}, - ...rest + ssisProperties: { + expressCustomSetupProperties = [], + packageStores = [], + catalogInfo: { + catalogServerEndpoint, + catalogPricingTier, + dualStandbyPairName, + } = {}, + credential: { + type: credentialType, + referenceName: credentialReferenceName, + } = {}, + licenseType, + customSetupScriptProperties: { blobContainerUri } = {}, + dataProxyProperties: { + connectVia: { + type: connectViaType, + referenceName: connectViaReferenceName, + } = {}, + stagingLinkedService: { + type: stagingLinkedServiceType, + referenceName: stagingLinkedServiceReferenceName, + } = {}, + path, + } = {}, + edition, + } = {}, + customerVirtualNetwork: { subnetId } = {}, + managedVirtualNetwork: { + id: mvId, + type: mvType, + referenceName: mvReferenceName, + } = {}, + computeProperties: { + location, + nodeSize, + numberOfNodes, + maxParallelExecutionsPerNode, + dataFlowProperties: { computeType, coreCount, timeToLive, cleanup } = {}, + vNetProperties: { + vNetId, + subnet, + publicIPs = [], + subnetId: vNetPropertiesSubnetId, + } = {}, + } = {}, + linkedInfo: { + key: { type: keyType, value: keyValue } = {}, + credential: { + type: linkedInfoCredentialType, + referenceName: linkedInfoCredentialReferenceName, + } = {}, + authorizationType, + resourceId, + } = {}, + description, + state, + referenceName, } = runtimeProperties - const { - expressCustomSetupProperties = [], - packageStores = [], - catalogInfo = {}, - credential = {}, - ...restSsisProperties - } = ssisProperties - const { catalogAdminPassword = {}, ...restCatalogInfo } = catalogInfo - const { - key = {}, - credential: linkedInfoCredential = {}, - ...restLinkedInfo - } = linkedInfo - return { integrationRuntimeType: type, ssisProperties: { expressCustomSetupProperties: expressCustomSetupProperties?.map( - ({ targetName, userName, password, licenseKey, ...sp }) => ({ + ({ + targetName, + variableName, + variableValue, + version, + type: expressCustomSetupPropertiesType, + componentName, + }): AzureCustomSetupBaseUnion => ({ id: cuid(), targetName: Object.values(targetName ?? {}).join(''), - userName: Object.values(userName ?? {}).join(''), - password: formatAzureSecret(password), - licenseKey: formatAzureSecret(licenseKey), - ...sp, + variableName, + variableValue, + version, + type: expressCustomSetupPropertiesType, + componentName, + }) + ) || [], + packageStores: + packageStores?.map( + ({ + name, + packageStoreLinkedService: { + type: packageStoreLinkedServiceType, + referenceName: packageStoreLinkedServiceReferenceName, + }, + }): AzurePackageStore => ({ + id: cuid(), + name, + packageStoreLinkedServiceType, + packageStoreLinkedServiceReferenceName, }) ) || [], - packageStores: packageStores?.map(ps => ({ id: cuid(), ...ps })) || [], catalogInfo: { - catalogAdminPasswordType: catalogAdminPassword?.type, - catalogAdminPasswordValue: catalogAdminPassword?.value, - ...restCatalogInfo, + catalogServerEndpoint, + catalogPricingTier, + dualStandbyPairName, + }, + credentialType, + credentialReferenceName, + licenseType, + customSetupScriptProperties: { blobContainerUri }, + dataProxyProperties: { + connectViaType, + connectViaReferenceName, + stagingLinkedServiceType, + stagingLinkedServiceReferenceName, + path, + }, + edition, + }, + customerVirtualNetworkSubnetId: subnetId, + managedVirtualNetwork: { + id: mvId, + type: mvType, + referenceName: mvReferenceName, + }, + computeProperties: { + location, + nodeSize, + numberOfNodes, + maxParallelExecutionsPerNode, + dataFlowProperties: { computeType, coreCount, timeToLive, cleanup }, + vNetProperties: { + vNetId, + subnet, + publicIPs, + subnetId: vNetPropertiesSubnetId, }, - credentialType: credential?.type, - credentialReferenceName: credential?.referenceName, - ...restSsisProperties, }, - customerVirtualNetworkSubnetId: customerVirtualNetwork?.subnetId, - managedVirtualNetwork, - computeProperties, linkedInfo: { - credentialType: linkedInfoCredential?.type, - credentialReferenceName: linkedInfoCredential?.referenceName, - keyType: key?.type, - keyValue: key?.value, - ...restLinkedInfo, + credentialType: linkedInfoCredentialType, + credentialReferenceName: linkedInfoCredentialReferenceName, + keyType, + keyValue, + authorizationType, + resourceId, }, - ...rest, + description, + state, + referenceName, } } diff --git a/src/services/integrationRuntimes/schema.graphql b/src/services/integrationRuntimes/schema.graphql index f147bdda..190cbb3a 100644 --- a/src/services/integrationRuntimes/schema.graphql +++ b/src/services/integrationRuntimes/schema.graphql @@ -66,9 +66,6 @@ type azureIntegrationRuntimeSsisCatalogInfo subscription: false ) { catalogServerEndpoint: String @search(by: [hash, regexp]) - catalogAdminUserName: String @search(by: [hash, regexp]) - catalogAdminPasswordType: String @search(by: [hash, regexp]) - catalogAdminPasswordValue: String @search(by: [hash, regexp]) catalogPricingTier: String @search(by: [hash, regexp]) dualStandbyPairName: String @search(by: [hash, regexp]) } @@ -80,8 +77,6 @@ type azureIntegrationRuntimeCustomSetupScriptProperties subscription: false ) { blobContainerUri: String @search(by: [hash, regexp]) - sasTokenType: String @search(by: [hash, regexp]) - sasTokenValue: String @search(by: [hash, regexp]) } type azureIntegrationRuntimeDataProxyProperties @@ -155,12 +150,9 @@ type azureCustomSetupBaseUnion id: String! @id @search(by: [hash]) type: String @search(by: [hash, regexp]) targetName: String @search(by: [hash, regexp]) - userName: String @search(by: [hash, regexp]) - password: azureSecretBaseUnion variableName: String @search(by: [hash, regexp]) variableValue: String @search(by: [hash, regexp]) componentName: String @search(by: [hash, regexp]) - licenseKey: azureSecretBaseUnion version: String @search(by: [hash, regexp]) } diff --git a/src/services/logAnalyticsWorkspace/format.ts b/src/services/logAnalyticsWorkspace/format.ts index 656dc7da..856d2d7e 100644 --- a/src/services/logAnalyticsWorkspace/format.ts +++ b/src/services/logAnalyticsWorkspace/format.ts @@ -18,9 +18,13 @@ export default ({ resourceGroupId, provisioningState, customerId, - sku, - features, - workspaceCapping, + sku: { name, lastSkuUpdate }, + features: { + enableLogAccessUsingOnlyResourcePermissions, + legacy, + searchVersion, + }, + workspaceCapping: { dailyQuotaGb, quotaNextResetTime, dataIngestionStatus }, retentionInDays, createdDate, modifiedDate, @@ -38,9 +42,13 @@ export default ({ region, provisioningState, customerId, - sku, - features, - workspaceCapping, + sku: { name, lastSkuUpdate }, + features: { + enableLogAccessUsingOnlyResourcePermissions, + legacy, + searchVersion, + }, + workspaceCapping: { dailyQuotaGb, quotaNextResetTime, dataIngestionStatus }, retentionInDays, createdDate, modifiedDate, diff --git a/src/services/sqlServers/format.ts b/src/services/sqlServers/format.ts index 8364d3ba..af717f45 100644 --- a/src/services/sqlServers/format.ts +++ b/src/services/sqlServers/format.ts @@ -1,5 +1,14 @@ import cuid from 'cuid' -import { AzureSqlServer } from '../../types/generated' +import { + AzureSqlServer, + AzureSqlServerAdAdministrator, + AzureSqlServerBlobAuditingPolicy, + AzureSqlServerEncryptionProtector, + AzureSqlServerFirewallRule, + AzureSqlServerPrivateEndpointConnection, + AzureSqlServerSecurityAlertPolicy, + AzureSqlServerVulnerabilityAssessment, +} from '../../types/generated' import { formatTagsFromMap } from '../../utils/format' import { RawAzureServer } from './data' @@ -16,10 +25,13 @@ export default ({ id, name, type, - identity, + identity: { + tenantId, + type: identityType, + principalId, + userAssignedIdentities = {}, + } = {}, kind, - administratorLogin, - administratorLoginPassword, version, state, fullyQualifiedDomainName, @@ -29,7 +41,7 @@ export default ({ primaryUserAssignedIdentityId, federatedClientId, keyId, - administrators, + administrators = {}, workspaceFeature, restrictOutboundNetworkAccess, resourceGroupId, @@ -49,31 +61,51 @@ export default ({ subscriptionId: account, type, identity: { - ...identity, - userAssignedIdentities: Object.keys( - identity?.userAssignedIdentities ?? {} - ).map(key => ({ - id: cuid(), - key, - value: identity?.userAssignedIdentities[key], - })), + tenantId, + type: identityType, + principalId, + ...(userAssignedIdentities + ? { + userAssignedIdentities: Object.keys(userAssignedIdentities).map( + key => ({ + id: cuid(), + key, + value: userAssignedIdentities[key], + }) + ), + } + : {}), }, kind, - administratorLogin, - administratorLoginPassword, version, state, fullyQualifiedDomainName, privateEndpointConnections: - privateEndpointConnections?.map(c => ({ - id: c.id || cuid(), - properties: { - privateEndpointId: c.properties?.privateEndpoint?.id, - privateLinkServiceConnectionState: - c.properties?.privateLinkServiceConnectionState, - provisioningState: c.properties?.provisioningState, - }, - })) || [], + privateEndpointConnections?.map( + ({ + id: privateEndpointConnectionId, + properties: { + privateEndpoint = {}, + privateLinkServiceConnectionState: { + description, + status, + actionsRequired, + } = {}, + provisioningState, + } = {}, + }): AzureSqlServerPrivateEndpointConnection => ({ + id: privateEndpointConnectionId ?? cuid(), + properties: { + privateEndpointId: privateEndpoint.id, + privateLinkServiceConnectionState: { + description, + status, + actionsRequired, + }, + provisioningState, + }, + }) + ) ?? [], minimalTlsVersion, publicNetworkAccess, primaryUserAssignedIdentityId, @@ -84,24 +116,140 @@ export default ({ restrictOutboundNetworkAccess, resourceGroupId, firewallRules: - firewallRules?.map(r => ({ id: r.id || cuid(), ...r })) || [], - serverSecurityAlertPolicies: serverSecurityAlertPolicies?.map(alertPolicy => ({ - ...alertPolicy, - id: alertPolicy.id || cuid(), - creationTime: alertPolicy?.creationTime?.toISOString(), - })), + firewallRules?.map( + ({ + id: firewallRuleId, + name: firewallRuleName, + type: firewallRuleType, + startIpAddress, + endIpAddress, + }): AzureSqlServerFirewallRule => ({ + id: firewallRuleId ?? cuid(), + name: firewallRuleName, + type: firewallRuleType, + startIpAddress, + endIpAddress, + }) + ) ?? [], + serverSecurityAlertPolicies: serverSecurityAlertPolicies?.map( + ({ + id: srvSecurityAlertPolicyId, + creationTime, + name: srvSecurityAlertPolicyName, + type: srvSecurityAlertPolicyType, + state: srvSecurityAlertPolicyState, + disabledAlerts = [], + emailAddresses = [], + emailAccountAdmins, + storageEndpoint, + retentionDays, + }): AzureSqlServerSecurityAlertPolicy => ({ + id: srvSecurityAlertPolicyId ?? cuid(), + name: srvSecurityAlertPolicyName, + type: srvSecurityAlertPolicyType, + state: srvSecurityAlertPolicyState, + disabledAlerts, + emailAddresses, + emailAccountAdmins, + storageEndpoint, + retentionDays, + creationTime: creationTime?.toISOString(), + }) + ) ?? [], adAdministrators: - adAdministrators?.map(a => ({ id: a.id || cuid(), ...a })) || [], + adAdministrators?.map( + ({ + id: adAdministratorId, + name: adAdministratorName, + type: adAdministratorType, + administratorType: adAdministratorAdminType, + sid, + tenantId: adAdministratorTenantId, + }): AzureSqlServerAdAdministrator => ({ + id: adAdministratorId ?? cuid(), + name: adAdministratorName, + type: adAdministratorType, + administratorType: adAdministratorAdminType, + sid, + tenantId: adAdministratorTenantId, + }) + ) ?? [], encryptionProtectors: - encryptionProtectors?.map(e => ({ id: e.id || cuid(), ...e })) || [], - serverBlobAuditingPolicies: serverBlobAuditingPolicies?.map(policy => ({ - ...policy, - id: policy.id || cuid(), - })), - vulnerabilityAssessments: vulnerabilityAssessments?.map(va => ({ - ...va, - id: va.id || cuid(), - })), + encryptionProtectors?.map( + ({ + id: encryptionProtectorId, + name: encryptionProtectorName, + type: encryptionProtectorType, + kind: encryptionProtectorKind, + location: encryptionProtectorLocation, + subregion, + serverKeyName, + serverKeyType, + uri, + thumbprint, + autoRotationEnabled, + }): AzureSqlServerEncryptionProtector => ({ + id: encryptionProtectorId ?? cuid(), + name: encryptionProtectorName, + type: encryptionProtectorType, + kind: encryptionProtectorKind, + location: encryptionProtectorLocation, + subregion, + serverKeyName, + serverKeyType, + uri, + thumbprint, + autoRotationEnabled, + }) + ) ?? [], + serverBlobAuditingPolicies: serverBlobAuditingPolicies?.map( + ({ + id: serverBlobAuditingPolicyId, + name: serverBlobAuditingPolicyName, + type: serverBlobAuditingPolicyType, + isDevopsAuditEnabled, + retentionDays, + auditActionsAndGroups, + isStorageSecondaryKeyInUse, + isAzureMonitorTargetEnabled, + queueDelayMs, + state: serverBlobAuditingPolicyState, + storageEndpoint, + storageAccountSubscriptionId, + }): AzureSqlServerBlobAuditingPolicy => ({ + id: serverBlobAuditingPolicyId || cuid(), + name: serverBlobAuditingPolicyName, + type: serverBlobAuditingPolicyType, + isDevopsAuditEnabled, + retentionDays, + auditActionsAndGroups, + isStorageSecondaryKeyInUse, + isAzureMonitorTargetEnabled, + queueDelayMs, + state: serverBlobAuditingPolicyState, + storageEndpoint, + storageAccountSubscriptionId, + }) + ) ?? [], + vulnerabilityAssessments: vulnerabilityAssessments?.map( + ({ + id: vaId, + name: vaName, + type: vaType, + storageContainerPath, + recurringScans: { isEnabled, emailSubscriptionAdmins, emails = [] }, + }): AzureSqlServerVulnerabilityAssessment => ({ + id: vaId ?? cuid(), + name: vaName, + type: vaType, + storageContainerPath, + recurringScans: { + isEnabled, + emailSubscriptionAdmins, + emails, + }, + }) + ) ?? [], tags: formatTagsFromMap(Tags), } } diff --git a/src/services/sqlServers/schema.graphql b/src/services/sqlServers/schema.graphql index d7e695cf..e56b95cf 100644 --- a/src/services/sqlServers/schema.graphql +++ b/src/services/sqlServers/schema.graphql @@ -124,7 +124,6 @@ type azureSqlServerADAdministrator name: String @search(by: [hash, regexp]) type: String @search(by: [hash, regexp]) administratorType: String @search(by: [hash, regexp]) - login: String @search(by: [hash, regexp]) sid: String @search(by: [hash, regexp]) tenantId: String @search(by: [hash, regexp]) } @@ -156,8 +155,6 @@ type azureSqlServer implements azureResource ) @key(fields: "id") { identity: azureSqlServerIdentity - administratorLogin: String @search(by: [hash, regexp]) - administratorLoginPassword: String @search(by: [hash, regexp]) version: String @search(by: [hash, regexp]) state: String @search(by: [hash, regexp]) fullyQualifiedDomainName: String @search(by: [hash, regexp]) @@ -191,8 +188,6 @@ type azureSqlServerVulnerabilityAssessment name: String @search(by: [hash, regexp]) type: String @search(by: [hash, regexp]) storageContainerPath: String @search(by: [hash, regexp]) - storageContainerSasKey: String @search(by: [hash, regexp]) - storageAccountAccessKey: String @search(by: [hash, regexp]) recurringScans: azureSqlServerVulnerabilityAssessmentRecurringScansProperties } @@ -225,6 +220,5 @@ type azureSqlServerBlobAuditingPolicy queueDelayMs: Int @search state: String @search(by: [hash, regexp]) storageEndpoint: String @search(by: [hash, regexp]) - storageAccountAccessKey: String @search(by: [hash, regexp]) storageAccountSubscriptionId: String @search(by: [hash, regexp]) } diff --git a/src/types/generated.ts b/src/types/generated.ts index 3cf3ce6a..df7d3660 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -1861,11 +1861,8 @@ export type AzureCredentialReference = { export type AzureCustomSetupBaseUnion = { componentName?: Maybe; id: Scalars['String']; - licenseKey?: Maybe; - password?: Maybe; targetName?: Maybe; type?: Maybe; - userName?: Maybe; variableName?: Maybe; variableValue?: Maybe; version?: Maybe; @@ -2706,8 +2703,6 @@ export type AzureIntegrationRuntimeComputeProperties = { export type AzureIntegrationRuntimeCustomSetupScriptProperties = { blobContainerUri?: Maybe; - sasTokenType?: Maybe; - sasTokenValue?: Maybe; }; export type AzureIntegrationRuntimeDataFlowProperties = { @@ -2744,9 +2739,6 @@ export type AzureIntegrationRuntimeProperties = { }; export type AzureIntegrationRuntimeSsisCatalogInfo = { - catalogAdminPasswordType?: Maybe; - catalogAdminPasswordValue?: Maybe; - catalogAdminUserName?: Maybe; catalogPricingTier?: Maybe; catalogServerEndpoint?: Maybe; dualStandbyPairName?: Maybe; @@ -4393,8 +4385,6 @@ export type AzureServiceBusUserAssignedIdentity = { export type AzureSqlServer = AzureResource & { adAdministrators?: Maybe>>; - administratorLogin?: Maybe; - administratorLoginPassword?: Maybe; administrators?: Maybe; databaseSql?: Maybe>>; encryptionProtectors?: Maybe>>; @@ -4420,7 +4410,6 @@ export type AzureSqlServer = AzureResource & { export type AzureSqlServerAdAdministrator = { administratorType?: Maybe; id: Scalars['String']; - login?: Maybe; name?: Maybe; sid?: Maybe; tenantId?: Maybe; @@ -4437,7 +4426,6 @@ export type AzureSqlServerBlobAuditingPolicy = { queueDelayMs?: Maybe; retentionDays?: Maybe; state?: Maybe; - storageAccountAccessKey?: Maybe; storageAccountSubscriptionId?: Maybe; storageEndpoint?: Maybe; type?: Maybe; @@ -4527,9 +4515,7 @@ export type AzureSqlServerVulnerabilityAssessment = { id: Scalars['String']; name?: Maybe; recurringScans?: Maybe; - storageAccountAccessKey?: Maybe; storageContainerPath?: Maybe; - storageContainerSasKey?: Maybe; type?: Maybe; }; From b635fe4ed73ee15fd85423de9eef84f27b5fe0dd Mon Sep 17 00:00:00 2001 From: Harold Date: Mon, 29 Aug 2022 09:18:14 -0400 Subject: [PATCH 2/2] remove comments --- src/services/integrationRuntimes/format.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/services/integrationRuntimes/format.ts b/src/services/integrationRuntimes/format.ts index 41aa7f16..fd05e05a 100644 --- a/src/services/integrationRuntimes/format.ts +++ b/src/services/integrationRuntimes/format.ts @@ -110,12 +110,6 @@ export interface RawAzureIntegrationRuntimeUnion { referenceName?: string } -// function useInterfaceToExtractData( -// data: unknown -// ): ExpectedDataType { -// return data as ExpectedDataType -// } - const formatProperties = ( runtimeProperties?: RawAzureIntegrationRuntimeUnion ): AzureIntegrationRuntimeProperties => {