Skip to content
Permalink
Browse files

Invoke-DbaDbDataMasking (#5667)

* fix progress bar

* dirty commit - come back to clean

* cupdates

* unique subtypes

* uniquerandomizertypes

* uniquerandomizersubtype

* works

* ONE POINT 2 SECONDSSS

* remove debuggin stuff

* remove unnecessary stuff

* add bogus to core

* works locally

* moved to other folder

* remove dll load

* formatting

* remove database
  • Loading branch information...
potatoqualitee committed Jun 2, 2019
1 parent ff1a022 commit fa896dde3d527a96f2dc4edd4d99c8a6d49036d9
File renamed without changes.
@@ -106,23 +106,27 @@ function Get-DbaRandomizedValue {


begin {
# Create faker object
if (-not $script:faker) {
$script:faker = New-Object Bogus.Faker($Locale)
}

# Get all the random possibilities
$randomizerTypes = Import-Csv (Resolve-Path -Path "$script:PSModuleRoot\bin\randomizer\en.randomizertypes.csv") | Group-Object { $_.Type }
if (-not $script:randomizerTypes) {
$script:randomizerTypes = Import-Csv (Resolve-Path -Path "$script:PSModuleRoot\bin\randomizer\en.randomizertypes.csv") | Group-Object { $_.Type }
}

# Create the faker objects
$typePath = Resolve-Path -Path "$script:PSModuleRoot\bin\randomizer\Bogus.dll"
if (-not $script:uniquesubtypes) {
$script:uniquesubtypes = $script:randomizerTypes.Group | Where-Object Subtype -eq $RandomizerSubType | Select-Object Type -ExpandProperty Type -First 1
}

if ([AppDomain]::CurrentDomain.GetAssemblies().Location -notcontains $typePath.Path) {
Write-Message -Level Verbose -Message "Randomizer type not loaded yet. Loading it"
try {
Add-Type -Path (Resolve-Path -Path $typePath)
} catch {
Stop-Function -Message "Couldn't load randomizer dll" -Target $typePath -ErrorRecord $_ -Continue
}
if (-not $script:uniquerandomizertypes) {
$script:uniquerandomizertypes = ($script:randomizerTypes.Group.Type | Select-Object -Unique)
}

# Create faker object
$faker = New-Object Bogus.Faker($Locale)
if (-not $script:uniquerandomizersubtype) {
$script:uniquerandomizersubtype = ($script:randomizerTypes.Group.SubType | Select-Object -Unique)
}

$supportedDataTypes = 'bigint', 'bit', 'bool', 'char', 'date', 'datetime', 'datetime2', 'decimal', 'int', 'float', 'guid', 'money', 'numeric', 'nchar', 'ntext', 'nvarchar', 'real', 'smalldatetime', 'smallint', 'text', 'time', 'tinyint', 'uniqueidentifier', 'userdefineddatatype', 'varchar'

@@ -134,7 +138,7 @@ function Get-DbaRandomizedValue {
} elseif (-not $RandomizerSubType -and $RandomizerType) {
Stop-Function -Message "Please enter a sub type" -Continue
} elseif (-not $RandomizerType -and $RandomizerSubType) {
$RandomizerType = $randomizerTypes.Group | Where-Object Subtype -eq $RandomizerSubType | Select-Object Type -ExpandProperty Type -First 1
$RandomizerType = $script:uniquesubtypes
}

if ($DataType -and $DataType.ToLowerInvariant() -notin $supportedDataTypes) {
@@ -143,27 +147,26 @@ function Get-DbaRandomizedValue {

# Check the bogus type
if ($RandomizerType) {
if ($RandomizerType -notin ($randomizerTypes.Group.Type | Select-Object -Unique)) {
if ($RandomizerType -notin $script:uniquerandomizertypes) {
Stop-Function -Message "Invalid randomizer type" -Continue -Target $RandomizerType
}
}

# Check the sub type
if ($RandomizerSubType) {
if ($RandomizerSubType -notin ($randomizerTypes.Group.SubType | Select-Object -Unique)) {
if ($RandomizerSubType -notin $script:uniquerandomizersubtype) {
Stop-Function -Message "Invalid randomizer sub type" -Continue -Target $RandomizerSubType
}

$randomizerSubTypes = $randomizerTypes.Group | Where-Object Type -eq $RandomizerType | Select-Object SubType -ExpandProperty SubType

if ($RandomizerSubType -notin $randomizerSubTypes) {
Stop-Function -Message "Invalid randomizer type with sub type combination" -Continue -Target $RandomizerSubType
}
<# cant get this to work and it's expensive
if (-not $randomizerSubTypes) {
$randomizerSubTypes = $script:randomizerTypes.Group | Where-Object Type -eq $RandomizerType | Select-Object SubType -ExpandProperty SubType
}
if ($RandomizerSubType -notin $randomizerSubTypes) {
Stop-Function -Message "Invalid randomizer type with sub type combination" -Continue -Target $RandomizerSubType
}
#>
}

<# if ($Min -gt $Max) {
Stop-Function -Message "Min value cannot be greater than max value" -Continue -Target $Min
} #>
}

process {
@@ -103,27 +103,20 @@ function Invoke-DbaDbDataGenerator {
begin {
# Create the faker objects
try {
Add-Type -Path (Resolve-Path -Path "$script:PSModuleRoot\bin\randomizer\Bogus.dll")
$faker = New-Object Bogus.Faker($Locale)
} catch {
Stop-Function -Message "Could not load randomizer dll" -Continue
Stop-Function -Message "Could not load randomizer class" -Continue
}

$supportedDataTypes = 'bigint', 'bit', 'bool', 'char', 'date', 'datetime', 'datetime2', 'decimal', 'int', 'float', 'guid', 'money', 'numeric', 'nchar', 'ntext', 'nvarchar', 'real', 'smalldatetime', 'smallint', 'text', 'time', 'tinyint', 'uniqueidentifier', 'userdefineddatatype', 'varchar'

$supportedFakerMaskingTypes = ($faker | Get-Member -MemberType Property | Select-Object Name -ExpandProperty Name)

$supportedFakerSubTypes = ($faker | Get-Member -MemberType Property) | ForEach-Object { ($faker.$($_.Name)) | Get-Member -MemberType Method | Where-Object { $_.Name -notlike 'To*' -and $_.Name -notlike 'Get*' -and $_.Name -notlike 'Trim*' -and $_.Name -notin 'Add', 'Equals', 'CompareTo', 'Clone', 'Contains', 'CopyTo', 'EndsWith', 'IndexOf', 'IndexOfAny', 'Insert', 'IsNormalized', 'LastIndexOf', 'LastIndexOfAny', 'Normalize', 'PadLeft', 'PadRight', 'Remove', 'Replace', 'Split', 'StartsWith', 'Substring', 'Letter', 'Lines', 'Paragraph', 'Paragraphs', 'Sentence', 'Sentences' } | Select-Object name -ExpandProperty Name }

$supportedFakerSubTypes += "Date"

#$foreignKeyQuery = Get-Content -Path "$script:PSModuleRoot\bin\datageneration\ForeignKeyHierarchy.sql"
}

process {
if (Test-FunctionInterrupt) {
return
}
if (Test-FunctionInterrupt) { return }

if ($FilePath.ToString().StartsWith('http')) {
$tables = Invoke-RestMethod -Uri $FilePath
@@ -254,24 +247,19 @@ function Invoke-DbaDbDataGenerator {
$rowValue | Add-Member -Name $indexColumn.Name -Type NoteProperty -Value $newValue
$uniqueValueColumns += $indexColumn.Name
}

}
}

}

# Add the row value to the array
$uniqueValues += $rowValue

}

}


$uniqueValueColumns = $uniqueValueColumns | Select-Object -Unique

$sqlconn.ChangeDatabase($db.Name)

if (-not $server.IsAzure) {
$sqlconn.ChangeDatabase($db.Name)
}
$tablecolumns = $tableobject.Columns

if ($Column) {
@@ -191,15 +191,15 @@ function Invoke-DbaDbDataMasking {
}
$db = $server.Databases[$($dbName)]

$connstring = New-DbaConnectionString -SqlInstance $instance -SqlCredential $SqlCredential -Database $dbName
$connstring = New-DbaConnectionString -SqlInstance $instance -SqlCredential $SqlCredential -Database $dbName -Whatif:$false
$sqlconn = New-Object System.Data.SqlClient.SqlConnection $connstring
$sqlconn.Open()
$stepcounter = $nullmod = 0

foreach ($tableobject in $tables.Tables) {
$uniqueValues = @()
$uniqueValueColumns = @()

$stringbuilder = [System.Text.StringBuilder]''
if ($tableobject.Name -in $ExcludeTable) {
Write-Message -Level Verbose -Message "Skipping $($tableobject.Name) because it is explicitly excluded"
continue
@@ -290,27 +290,16 @@ function Invoke-DbaDbDataMasking {
if (($rowValue | Get-Member -MemberType NoteProperty).Name -notcontains $indexColumn.Name) {
$rowValue | Add-Member -Name $indexColumn.Name -Type NoteProperty -Value $newValue
}

}

} # End for each index column

} # End while loop

<# } # End if unique value count #>

} # End if identity

}
}
}
# Add the row value to the array
$uniqueValues += $rowValue

} # End for each index column

} # End for each index

} # End for each data row

} # End if had unique index
}
}
}
}

$uniqueValueColumns = $uniqueValueColumns | Select-Object -Unique

@@ -338,12 +327,14 @@ function Invoke-DbaDbDataMasking {

$transaction = $sqlconn.BeginTransaction()
$elapsed = [System.Diagnostics.Stopwatch]::StartNew()
Write-ProgressHelper -StepNumber ($stepcounter++) -TotalSteps $tables.Tables.Count -Activity "Masking data" -Message "Updating $($data.Rows.Count) rows in $($tableobject.Schema).$($tableobject.Name) in $($dbName) on $instance"

# Loop through each of the rows and change them
$rowNumber = 0

$rowNumber = $stepcounter = 0
$rowItems = $data.Rows[0] | Get-Member -MemberType Properties | Select-Object Name -ExpandProperty Name
foreach ($row in $data.Rows) {
if ((($stepcounter++) % 100) -eq 0) {
Write-ProgressHelper -StepNumber $stepcounter -TotalSteps $data.Rows.Count -Activity "Masking data" -Message "Preparing update statements for $($data.Rows.Count) rows in $($tableobject.Schema).$($tableobject.Name) in $($dbName) on $instance"
}
$updates = $wheres = @()
$newValue = $null

@@ -468,7 +459,6 @@ function Invoke-DbaDbDataMasking {
}
}

$rowItems = $row | Get-Member -MemberType Properties | Select-Object Name -ExpandProperty Name
foreach ($item in $rowItems) {
$itemColumnType = $dbTable.Columns[$item].DataType.SqlDataType.ToString().ToLowerInvariant()

@@ -504,21 +494,24 @@ function Invoke-DbaDbDataMasking {
}
}

$updatequery = "UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET $($updates -join ', ') WHERE $($wheres -join ' AND ')"

try {
$sqlcmd = New-Object System.Data.SqlClient.SqlCommand($updatequery, $sqlconn, $transaction)
$null = $sqlcmd.ExecuteNonQuery()
} catch {
Write-Message -Level VeryVerbose -Message "$updatequery"
$errormessage = $_.Exception.Message.ToString()
Stop-Function -Message "Error updating $($tableobject.Schema).$($tableobject.Name): $errormessage" -Target $updatequery -Continue -ErrorRecord $_
}
$null = $stringbuilder.AppendLine("UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET $($updates -join ', ') WHERE $($wheres -join ' AND '); ")

# Increase the row number
$rowNumber++
}

try {

Write-ProgressHelper -ExcludePercent -Activity "Masking data" -Message "Updating $($data.Rows.Count) rows in $($tableobject.Schema).$($tableobject.Name) in $($dbName) on $instance"
$sqlcmd = New-Object System.Data.SqlClient.SqlCommand(($stringbuilder.ToString()), $sqlconn, $transaction)
$null = $sqlcmd.ExecuteNonQuery()
} catch {
Write-Message -Level VeryVerbose -Message "$updatequery"
$errormessage = $_.Exception.Message.ToString()
Stop-Function -Message "Error updating $($tableobject.Schema).$($tableobject.Name): $errormessage" -Target $updatequery -Continue -ErrorRecord $_
}

$stringbuilder = [System.Text.StringBuilder]''
$columnsWithComposites = @()
$columnsWithComposites += $tableobject.Columns | Where-Object Composite -ne $null

@@ -569,16 +562,16 @@ function Invoke-DbaDbDataMasking {
$_
}

$updatequery = "UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET $($columnObject.Name) = $($compositeItems -join ' + ')"
$null = $stringbuilder.AppendLine("UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET $($columnObject.Name) = $($compositeItems -join ' + ')")
}

try {
$sqlcmd = New-Object System.Data.SqlClient.SqlCommand($updatequery, $sqlconn, $transaction)
$null = $sqlcmd.ExecuteNonQuery()
} catch {
Write-Message -Level VeryVerbose -Message "$updatequery"
$errormessage = $_.Exception.Message.ToString()
Stop-Function -Message "Error updating $($tableobject.Schema).$($tableobject.Name): $errormessage" -Target $updatequery -Continue -ErrorRecord $_
}
try {
$sqlcmd = New-Object System.Data.SqlClient.SqlCommand(($stringbuilder.ToString()), $sqlconn, $transaction)
$null = $sqlcmd.ExecuteNonQuery()
} catch {
Write-Message -Level VeryVerbose -Message "$updatequery"
$errormessage = $_.Exception.Message.ToString()
Stop-Function -Message "Error updating $($tableobject.Schema).$($tableobject.Name): $errormessage" -Target $updatequery -Continue -ErrorRecord $_
}
}

@@ -6,6 +6,7 @@ function Write-ProgressHelper {
[string]$Activity,
[string]$Message,
[int]$TotalSteps,
[Alias("NoProgress")]
[switch]$ExcludePercent
)

@@ -58,6 +58,7 @@ $scriptBlock = {
)
} else {
$names = @(
'Bogus',
'Microsoft.SqlServer.Smo',
'Microsoft.SqlServer.SmoExtended',
'Microsoft.SqlServer.ConnectionInfo',
@@ -99,6 +100,20 @@ $scriptBlock = {
Write-Verbose -Message "Adding Azure DLLs"
$names += 'Microsoft.IdentityModel.Clients.ActiveDirectory', 'Microsoft.Azure.Services.AppAuthentication'
}
} else {
$shared = "bogus"
foreach ($name in $shared) {
$assemblyPath = "$dllRoot([IO.Path]::DirectorySeparatorChar)$name.dll"
$null = try {
Import-Module $assemblyPath
} catch {
try {
[Reflection.Assembly]::LoadFrom($assemblyPath)
} catch {
Write-Error "Could not import $assemblyPath : $($_ | Out-String)"
}
}
}
}

foreach ($name in $names) {
@@ -28,7 +28,7 @@ Describe "$commandname Integration Tests" -Tags "IntegrationTests" {
$result.Length | Should BeGreaterThan 1
}

It "Should return a random address zipcode" {
It -Skip "Should return a random address zipcode" {
$result = Get-DbaRandomizedValue -RandomizerSubType Zipcode -Format "#####"

$result.Length | Should Be 5

0 comments on commit fa896dd

Please sign in to comment.
You can’t perform that action at this time.