From 469924d855aba51a539b8a1ac8a2f6c9442283e4 Mon Sep 17 00:00:00 2001 From: iSazonov Date: Fri, 18 Nov 2016 06:29:04 +0600 Subject: [PATCH] Add support W3C Extended Log File Format in Import-Csv (#2482) * Add support W3C Extended Log File Format 1. Add support W3C Extended Log File Format 2. Refactoring Import-Csv tests 3. Add #Type tests * Fx test after review * Fix tests after review * Remove unneeded test --- .../commands/utility/CSVCommands.cs | 45 ++++--- .../Import-Csv.Tests.ps1 | 126 ++++++++++-------- .../assets/TestImportCsv_NoHeader.csv | 4 + .../assets/TestImportCsv_W3C_ELF.csv | 9 ++ ...stCsv.csv => TestImportCsv_WithHeader.csv} | 11 +- 5 files changed, 118 insertions(+), 77 deletions(-) create mode 100644 test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_NoHeader.csv create mode 100644 test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_W3C_ELF.csv rename test/powershell/Modules/Microsoft.PowerShell.Utility/assets/{TestCsv.csv => TestImportCsv_WithHeader.csv} (78%) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs index 377607e976d8..78a7825ef643 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs @@ -1190,11 +1190,31 @@ internal void ReadHeader() TypeName = ReadTypeInformation(); } - if ((Header == null) && (!this.EOF)) + while ((Header == null) && (!this.EOF)) { - Collection values = ParseNextRecord(true); - if (values.Count != 0) + Collection values = ParseNextRecord(); + + // Trim all trailing blankspaces and delimiters ( single/multiple ). + // If there is only one element in the row and if its a blankspace we dont trim it. + // A trailing delimiter is represented as a blankspace while being added to result collection + // which is getting trimmed along with blankspaces supplied through the CSV in the below loop. + while (values.Count > 1 && values[values.Count - 1].Equals(string.Empty)) + { + values.RemoveAt(values.Count - 1); + } + + // File starts with '#' and contains '#Fields:' is W3C Extended Log File Format + if (values.Count != 0 && values[0].StartsWith("#Fields: ")) + { + values[0] = values[0].Substring(9); + Header = values; + } else if (values.Count != 0 && values[0].StartsWith("#")) + { + // Skip all lines starting with '#' + } else { + // This is not W3C Extended Log File Format + // By default first line is Header Header = values; } } @@ -1213,7 +1233,7 @@ internal void ReadHeader() ReadHeader(); while (true) { - Collection values = ParseNextRecord(false); + Collection values = ParseNextRecord(); if (values.Count == 0) break; @@ -1300,14 +1320,11 @@ private string /// Reads the next record from the file and returns parsed collection /// of string. /// - /// - /// Indicates if the parsed row is a header row or a values row. - /// /// /// Parsed collection of strings. /// private Collection - ParseNextRecord(bool isHeaderRow) + ParseNextRecord() { //Collection of strings to return Collection result = new Collection(); @@ -1471,18 +1488,6 @@ private string result.Add(current.ToString()); } - //Trim all trailing blankspaces and delimiters ( single/multiple ). - // If there is only one element in the row and if its a blankspace we dont trim it. - // A trailing delimiter is represented as a blankspace while being added to result collection - // which is getting trimmed along with blankspaces supplied through the CSV in the below loop. - if (isHeaderRow) - { - while (result.Count > 1 && result[result.Count - 1].Equals(string.Empty)) - { - result.RemoveAt(result.Count - 1); - } - } - return result; } diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/Import-Csv.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/Import-Csv.Tests.ps1 index ff8290d97d59..11f6f099f3e2 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/Import-Csv.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/Import-Csv.Tests.ps1 @@ -1,55 +1,3 @@ -Describe "Import-Csv" -Tags "CI" { - $testCsv = Join-Path -Path (Join-Path $PSScriptRoot -ChildPath assets) -ChildPath TestCsv.csv - - It "Should be able to call without error" { - { Import-Csv $testCsv } | Should Not Throw - } - - It "Should be able to assign to a variable" { - $actual = Import-Csv $testCsv - - $actual | Should Not BeNullOrEmpty - $actual.GetType().BaseType | Should Be array - } - - It "Should have the data from the csv file" { - $actualContent = $(Get-Content $testCsv)[0] - $testContent = $($(Import-Csv $testCsv) | Get-Member) | ? { $_.MemberType -eq "NoteProperty" } | % { $_.Name } | Select-Object -First 1 - - $actualContent.IndexOf($testContent) | Should BeGreaterThan -1 - } - - It "Should be able to prepend a custom header" { - $header = "test1","test2","test3" - - $originalContent = $($(Import-Csv $testCsv) | Get-Member) | ? { $_.MemberType -eq "NoteProperty" } | % { $_.Name } | Select-Object -First 1 - - $testContent = $($(Import-Csv $testCsv -Header $header) | Get-Member) | ? { $_.MemberType -eq "NoteProperty" } | % { $_.Name } | Select-Object -First 3 - - # the original csv file doesn't contain the headers - $originalContent.IndexOf($header[0]) | Should Be -1 - - # but it does with the -Header switch! - $testContent[0] | Should Be $header[0] - $testContent[1] | Should Be $header[1] - $testContent[2] | Should Be $header[2] - } - - It "Should be able to use the alias without error" { - { Import-Csv $testCsv } | Should Not Throw - } - - It "Should have the same output between the alias and the full cmdlet name" { - $alias = $($(ipcsv $testCsv) | Get-Member) | ? { $_.MemberType -eq "NoteProperty" } | % { $_.Name } | Select-Object -First 1 - $cmdlet = $($(Import-Csv $testCsv) | Get-Member) | ? { $_.MemberType -eq "NoteProperty" } | % { $_.Name } | Select-Object -First 1 - - $alias[0] | Should Be $cmdlet[0] - $alias[1] | Should Be $cmdlet[1] - $alias[2] | Should Be $cmdlet[2] - - } -} - Describe "Import-Csv DRT Unit Tests" -Tags "CI" { BeforeAll { $fileToGenerate = Join-Path $TestDrive -ChildPath "importCSVTest.csv" @@ -71,3 +19,77 @@ Describe "Import-Csv DRT Unit Tests" -Tags "CI" { $returnObject.Second | Should Be 2 } } + +Describe "Import-Csv File Format Tests" -Tags "CI" { + BeforeAll { + # The file is w/o header + $TestImportCsv_NoHeader = Join-Path -Path (Join-Path $PSScriptRoot -ChildPath assets) -ChildPath TestImportCsv_NoHeader.csv + # The file is with header + $TestImportCsv_WithHeader = Join-Path -Path (Join-Path $PSScriptRoot -ChildPath assets) -ChildPath TestImportCsv_WithHeader.csv + # The file is W3C Extended Log File Format + $TestImportCsv_W3C_ELF = Join-Path -Path (Join-Path $PSScriptRoot -ChildPath assets) -ChildPath TestImportCsv_W3C_ELF.csv + + $testCSVfiles = $TestImportCsv_NoHeader, $TestImportCsv_WithHeader, $TestImportCsv_W3C_ELF + $orginalHeader = "Column1","Column2","Column 3" + $customHeader = "test1","test2","test3" + } + # Test set is the same for all file formats + foreach ($testCsv in $testCSVfiles) { + $FileName = (dir $testCsv).Name + Context "Next test file: $FileName" { + BeforeAll { + $CustomHeaderParams = @{Header = $customHeader; Delimiter = ","} + if ($FileName -eq "TestImportCsv_NoHeader.csv") { + # The file does not have header + # (w/o Delimiter here we get throw (bug?)) + $HeaderParams = @{Header = $orginalHeader; Delimiter = ","} + } else { + # The files have header + $HeaderParams = @{Delimiter = ","} + } + + } + + It "Should be able to import all fields" { + $actual = Import-Csv -Path $testCsv @HeaderParams + $actualfields = $actual[0].psobject.Properties.Name + $actualfields | Should Be $orginalHeader + } + + It "Should be able to import all fields with custom header" { + $actual = Import-Csv -Path $testCsv @CustomHeaderParams + $actualfields = $actual[0].psobject.Properties.Name + $actualfields | Should Be $customHeader + } + + It "Should be able to import correct values" { + $actual = Import-Csv -Path $testCsv @HeaderParams + $actual.count | Should Be 4 + $actual[0].'Column1' | Should Be "data1" + $actual[0].'Column2' | Should Be "1" + $actual[0].'Column 3' | Should Be "A" + } + + } + } +} + +Describe "Import-Csv #Type Tests" -Tags "CI" { + BeforeAll { + $testfile = Join-Path $TestDrive -ChildPath "testfile.csv" + Remove-Item -Path $testfile -Force -ErrorAction SilentlyContinue + $processlist = (Get-Process)[0..1] + $processlist | Export-Csv -Path $testfile -Force + # Import-Csv add "CSV:" before actual type + # (Why #HandleCount ? See Issue #1812) + $expectedProcessType = "CSV:System.Diagnostics.Process#HandleCount" + } + + It "Test import-csv import Object" { + $importObjectList = Import-Csv -Path $testfile + $processlist.Count | Should Be $importObjectList.Count + + $importType = $importObjectList[0].psobject.TypeNames[0] + $importType | Should Be $expectedProcessType + } +} diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_NoHeader.csv b/test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_NoHeader.csv new file mode 100644 index 000000000000..192b997de195 --- /dev/null +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_NoHeader.csv @@ -0,0 +1,4 @@ +data1,1,A +data2,2,B +data3,3,C +data4,4,D diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_W3C_ELF.csv b/test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_W3C_ELF.csv new file mode 100644 index 000000000000..e1f13f34063b --- /dev/null +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_W3C_ELF.csv @@ -0,0 +1,9 @@ +#Software: Microsoft Exchange Server +#Version: 15.0.0.0 +#Log-type: Transport Connectivity Log +#Date: 2016-09-16T23:30:07.338Z +#Fields: Column1,Column2,Column 3 +data1,1,A +data2,2,B +data3,3,C +data4,4,D diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestCsv.csv b/test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_WithHeader.csv similarity index 78% rename from test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestCsv.csv rename to test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_WithHeader.csv index 73a63d8992dd..1517b022998f 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestCsv.csv +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_WithHeader.csv @@ -1,5 +1,6 @@ -Column1,Column2,Column 3 -data1,1,A -data2,2,B -data3,3,C -data4,4,D +# Test comment +Column1,Column2,Column 3 +data1,1,A +data2,2,B +data3,3,C +data4,4,D