Add support W3C Extended Log File Format in Import-Csv (#2482)

* Add support W3C Extended Log File Format

1. Add support W3C Extended Log File Format
2. Refactoring Import-Csv tests

3. Add #Type tests

* Fx test after review

* Fix tests after review

* Remove unneeded test

src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs

@@ -1190,11 +1190,31 @@ internal void ReadHeader()
                 TypeName = ReadTypeInformation();
             }
 
-            if ((Header == null) && (!this.EOF))
+            while ((Header == null) && (!this.EOF))
             {
-                Collection<string> values = ParseNextRecord(true);
-                if (values.Count != 0)
+                Collection<string> values = ParseNextRecord();
+
+                // Trim all trailing blankspaces and delimiters ( single/multiple ).
+                // If there is only one element in the row and if its a blankspace we dont trim it.
+                // A trailing delimiter is represented as a blankspace while being added to result collection
+                // which is getting trimmed along with blankspaces supplied through the CSV in the below loop.
+                while (values.Count > 1 && values[values.Count - 1].Equals(string.Empty))
+                {
+                    values.RemoveAt(values.Count - 1);
+                }
+
+                // File starts with '#' and contains '#Fields:' is W3C Extended Log File Format
+                if (values.Count != 0 && values[0].StartsWith("#Fields: "))
+                {
+                    values[0] = values[0].Substring(9);
+                    Header = values;
+                } else if (values.Count != 0 && values[0].StartsWith("#"))
+                {
+                    // Skip all lines starting with '#'
+                } else 
                 {
+                    // This is not W3C Extended Log File Format
+                    // By default first line is Header
                     Header = values;
                 }
             }
@@ -1213,7 +1233,7 @@ internal void ReadHeader()
             ReadHeader();
             while (true)
             {
-                Collection<string> values = ParseNextRecord(false);
+                Collection<string> values = ParseNextRecord();
                 if (values.Count == 0)
                     break;
 
@@ -1300,14 +1320,11 @@ private string
         /// Reads the next record from the file and returns parsed collection
         /// of string.
         /// </summary>
-        /// <param name="isHeaderRow">
-        /// Indicates if the parsed row is a header row or a values row.
-        /// </param>
         /// <returns>
         /// Parsed collection of strings.
         /// </returns>
         private Collection<string>
-        ParseNextRecord(bool isHeaderRow)
+        ParseNextRecord()
         {
             //Collection of strings to return
             Collection<string> result = new Collection<string>();
@@ -1471,18 +1488,6 @@ private string
                 result.Add(current.ToString());
             }
 
-            //Trim all trailing blankspaces and delimiters ( single/multiple ).
-            // If there is only one element in the row and if its a blankspace we dont trim it.
-            // A trailing delimiter is represented as a blankspace while being added to result collection
-            // which is getting trimmed along with blankspaces supplied through the CSV in the below loop.
-            if (isHeaderRow)
-            {
-                while (result.Count > 1 && result[result.Count - 1].Equals(string.Empty))
-                {
-                    result.RemoveAt(result.Count - 1);
-                }
-            }
-
             return result;
         }
 

test/powershell/Modules/Microsoft.PowerShell.Utility/Import-Csv.Tests.ps1

@@ -1,55 +1,3 @@
-Describe "Import-Csv" -Tags "CI" {
-    $testCsv = Join-Path -Path (Join-Path $PSScriptRoot -ChildPath assets) -ChildPath TestCsv.csv
-
-    It "Should be able to call without error" {
-	    { Import-Csv $testCsv } | Should Not Throw
-        }
-
-    It "Should be able to assign to a variable" {
-	    $actual = Import-Csv $testCsv
-
-	    $actual                    | Should Not BeNullOrEmpty
-	    $actual.GetType().BaseType | Should Be array
-    }
-
-    It "Should have the data from the csv file" {
-	    $actualContent = $(Get-Content $testCsv)[0]
-	    $testContent   = $($(Import-Csv $testCsv) | Get-Member) | ? { $_.MemberType -eq "NoteProperty" } | % { $_.Name } | Select-Object -First 1
-
-	    $actualContent.IndexOf($testContent) | Should BeGreaterThan -1
-    }
-
-    It "Should be able to prepend a custom header" {
-	    $header = "test1","test2","test3"
-
-	    $originalContent = $($(Import-Csv $testCsv) | Get-Member) | ? { $_.MemberType -eq "NoteProperty" } | % { $_.Name } | Select-Object -First 1
-
-	    $testContent = $($(Import-Csv $testCsv -Header $header) | Get-Member) | ? { $_.MemberType -eq "NoteProperty" } | % { $_.Name } | Select-Object -First 3
-
-	    # the original csv file doesn't contain the headers
-        $originalContent.IndexOf($header[0]) | Should Be -1
-
-        # but it does with the -Header switch!
-        $testContent[0] | Should Be $header[0]
-        $testContent[1] | Should Be $header[1]
-        $testContent[2] | Should Be $header[2]
-    }
-
-    It "Should be able to use the alias without error" {
-        { Import-Csv $testCsv } | Should Not Throw
-    }
-
-    It "Should have the same output between the alias and the full cmdlet name" {
-        $alias  = $($(ipcsv $testCsv) | Get-Member) | ? { $_.MemberType -eq "NoteProperty" } | % { $_.Name } | Select-Object -First 1
-        $cmdlet = $($(Import-Csv $testCsv) | Get-Member) | ? { $_.MemberType -eq "NoteProperty" } | % { $_.Name } | Select-Object -First 1
-
-        $alias[0] | Should Be $cmdlet[0]
-        $alias[1] | Should Be $cmdlet[1]
-        $alias[2] | Should Be $cmdlet[2]
-
-    }
-}
-
 Describe "Import-Csv DRT Unit Tests" -Tags "CI" {
     BeforeAll {
         $fileToGenerate = Join-Path $TestDrive -ChildPath "importCSVTest.csv"
@@ -71,3 +19,77 @@ Describe "Import-Csv DRT Unit Tests" -Tags "CI" {
         $returnObject.Second | Should Be 2
     }
 }
+
+Describe "Import-Csv File Format Tests" -Tags "CI" {
+    BeforeAll {
+        # The file is w/o header
+        $TestImportCsv_NoHeader = Join-Path -Path (Join-Path $PSScriptRoot -ChildPath assets) -ChildPath TestImportCsv_NoHeader.csv
+        # The file is with header
+        $TestImportCsv_WithHeader = Join-Path -Path (Join-Path $PSScriptRoot -ChildPath assets) -ChildPath TestImportCsv_WithHeader.csv
+        # The file is W3C Extended Log File Format
+        $TestImportCsv_W3C_ELF = Join-Path -Path (Join-Path $PSScriptRoot -ChildPath assets) -ChildPath TestImportCsv_W3C_ELF.csv
+
+        $testCSVfiles = $TestImportCsv_NoHeader, $TestImportCsv_WithHeader, $TestImportCsv_W3C_ELF
+        $orginalHeader = "Column1","Column2","Column 3"
+        $customHeader = "test1","test2","test3"
+    }
+    # Test set is the same for all file formats
+    foreach ($testCsv in $testCSVfiles) {
+       $FileName = (dir $testCsv).Name
+        Context "Next test file: $FileName" {
+            BeforeAll {
+                $CustomHeaderParams = @{Header = $customHeader; Delimiter = ","}
+                if ($FileName -eq "TestImportCsv_NoHeader.csv") {
+                    # The file does not have header
+                    # (w/o Delimiter here we get throw (bug?)) 
+                    $HeaderParams = @{Header = $orginalHeader; Delimiter = ","}
+                } else {
+                    # The files have header
+                    $HeaderParams = @{Delimiter = ","}
+                }
+
+            }
+
+            It "Should be able to import all fields" {
+                $actual = Import-Csv -Path $testCsv @HeaderParams
+                $actualfields = $actual[0].psobject.Properties.Name
+                $actualfields | Should Be $orginalHeader
+            }
+
+            It "Should be able to import all fields with custom header" {
+                $actual = Import-Csv -Path $testCsv @CustomHeaderParams
+                $actualfields = $actual[0].psobject.Properties.Name
+                $actualfields | Should Be $customHeader
+            }
+
+            It "Should be able to import correct values" {
+                $actual = Import-Csv -Path $testCsv @HeaderParams
+                $actual.count         | Should Be 4
+                $actual[0].'Column1'  | Should Be "data1"
+                $actual[0].'Column2'  | Should Be "1"
+                $actual[0].'Column 3' | Should Be "A"
+            }
+
+        }
+    }
+}
+
+Describe "Import-Csv #Type Tests" -Tags "CI" {
+    BeforeAll {
+        $testfile = Join-Path $TestDrive -ChildPath "testfile.csv"
+        Remove-Item -Path $testfile -Force -ErrorAction SilentlyContinue
+        $processlist = (Get-Process)[0..1]
+        $processlist | Export-Csv -Path $testfile -Force
+        # Import-Csv add "CSV:" before actual type
+        # (Why #HandleCount ? See Issue #1812)
+        $expectedProcessType = "CSV:System.Diagnostics.Process#HandleCount"
+    }
+
+    It "Test import-csv import Object" {
+        $importObjectList = Import-Csv -Path $testfile
+        $processlist.Count | Should Be $importObjectList.Count
+
+        $importType = $importObjectList[0].psobject.TypeNames[0]
+        $importType | Should Be $expectedProcessType
+    }
+}

test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_NoHeader.csv

@@ -0,0 +1,4 @@
+data1,1,A
+data2,2,B
+data3,3,C
+data4,4,D

test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_W3C_ELF.csv

@@ -0,0 +1,9 @@
+#Software: Microsoft Exchange Server
+#Version: 15.0.0.0
+#Log-type: Transport Connectivity Log
+#Date: 2016-09-16T23:30:07.338Z
+#Fields: Column1,Column2,Column 3
+data1,1,A
+data2,2,B
+data3,3,C
+data4,4,D

test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestCsv.csv → test/powershell/Modules/Microsoft.PowerShell.Utility/assets/TestImportCsv_WithHeader.csv

@@ -1,5 +1,6 @@
-Column1,Column2,Column 3
-data1,1,A
-data2,2,B
-data3,3,C
-data4,4,D
+# Test comment
+Column1,Column2,Column 3
+data1,1,A
+data2,2,B
+data3,3,C
+data4,4,D