Split unit and integration test execution in CI process so that integ…

…ration tests - Fixes #184 (#189) * Split unit and integration test execution in CI process so that integration tests * Ensure Code Coverage is uploaded to Azure DevOps CI
PlagueHO · Oct 20, 2018 · 32c6da2 · 32c6da2
1 parent 92aeebf
commit 32c6da2
Show file tree

Hide file tree

Showing 18 changed files with 122 additions and 31 deletions.
diff --git a/.gitignore b/.gitignore
@@ -36,7 +36,8 @@ bld/
 
 # NUNIT
 *.VisualState.xml
-test/TestResults.xml
+test/**/TestResults.*.xml
+test/**/CodeCoverage.xml
 
 # Build Results of an ATL Project
 [Dd]ebugPS/

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,8 @@
 - Fixed 'Unable to find type \[Microsoft.PowerShell.Commands.HttpResponseException\]' exception
   being thrown in `Invoke-CosmosDbRequest` when error is returned by Cosmos DB in PowerShell 5.x
   or earlier - fixes [Issue #186](https://github.com/PlagueHO/CosmosDB/issues/186).
+- Split unit and integration test execution in CI process so that integration tests do
+  not run when unit tests fail - fixes [Issue #184](https://github.com/PlagueHO/CosmosDB/issues/184).
 
 ## 2.1.8.59
 

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -17,9 +17,24 @@ jobs:
     - task: PublishTestResults@2
       inputs:
         testRunner: 'NUnit'
-        testResultsFiles: '**/TestResults.xml'
-        testRunTitle: 'PS_Win2016'
-      displayName: 'Publish Test Results'
+        testResultsFiles: '**/Unit/TestResults.unit.xml'
+        testRunTitle: 'PS_Win2016_Unit'
+      displayName: 'Publish Unit Test Results'
+      condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues', 'Failed')
+
+    - task: PublishCodeCoverageResults@1
+      inputs:
+        summaryFileLocation: '**/Unit/CodeCoverage.xml'
+        failIfCoverageEmpty: true
+      displayName: 'Publish Unit Test Code Coverage'
+      condition: and(in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues', 'Failed'), eq(variables['System.PullRequest.IsFork'], false))
+
+    - task: PublishTestResults@2
+      inputs:
+        testRunner: 'NUnit'
+        testResultsFiles: '**/Integration/TestResults.integration.xml'
+        testRunTitle: 'PS_Win2016_Integration'
+      displayName: 'Publish Integration Test Results'
       condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues', 'Failed')
 
     - task: PublishBuildArtifacts@1
@@ -76,9 +91,17 @@ jobs:
     - task: PublishTestResults@2
       inputs:
         testRunner: 'NUnit'
-        testResultsFiles: '**/TestResults.xml'
-        testRunTitle: 'PSCore_Ubuntu1604'
-      displayName: 'Publish Test Results'
+        testResultsFiles: '**/Unit/TestResults.unit.xml'
+        testRunTitle: 'PSCore_Ubuntu1604_Unit'
+      displayName: 'Publish Unit Test Results'
+      condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues', 'Failed')
+
+    - task: PublishTestResults@2
+      inputs:
+        testRunner: 'NUnit'
+        testResultsFiles: '**/Integration/TestResults.integration.xml'
+        testRunTitle: 'PSCore_Ubuntu1604_Integration'
+      displayName: 'Publish Integration Test Results'
       condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues', 'Failed')
 
   - job: Build_PSCore_MacOS1013
@@ -104,7 +127,15 @@ jobs:
     - task: PublishTestResults@2
       inputs:
         testRunner: 'NUnit'
-        testResultsFiles: '**/TestResults.xml'
-        testRunTitle: 'PSCore_MacOS1013'
-      displayName: 'Publish Test Results'
+        testResultsFiles: '**/TestResults.unit.xml'
+        testRunTitle: 'PSCore_MacOS1013_Unit'
+      displayName: 'Publish Unit Test Results'
+      condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues', 'Failed')
+
+    - task: PublishTestResults@2
+      inputs:
+        testRunner: 'NUnit'
+        testResultsFiles: '**/TestResults.integration.xml'
+        testRunTitle: 'PSCore_MacOS1013_Integration'
+      displayName: 'Publish Integration Test Results'
       condition: in(variables['Agent.JobStatus'], 'Succeeded', 'SucceededWithIssues', 'Failed')
diff --git a/psakefile.ps1 b/psakefile.ps1
@@ -13,7 +13,7 @@ Properties {
     $separator = '----------------------------------------------------------------------'
 }
 
-Task Default -Depends Test,Build
+Task Default -Depends Test, Build
 
 Task Init {
     Set-Location -Path $ProjectRoot
@@ -44,7 +44,19 @@ Task Init {
     "`n"
 }
 
-Task Test -Depends Init {
+Task PrepareTest -Depends Init {
+    # Install any dependencies required for testing
+    Invoke-PSDepend `
+        -Path $PSScriptRoot `
+        -Force `
+        -Import `
+        -Install `
+        -Tags 'Test',('Test_{0}' -f $PSVersionTable.PSEdition)
+}
+
+Task Test -Depends UnitTest, IntegrationTest
+
+Task UnitTest -Depends Init, PrepareTest {
     $separator
 
     # Install any dependencies required for the Test stage
@@ -56,13 +68,18 @@ Task Test -Depends Init {
         -Tags 'Test',('Test_{0}' -f $PSVersionTable.PSEdition)
 
     # Execute tests
-    $testResultsFile = Join-Path -Path $ProjectRoot -ChildPath 'test\TestResults.xml'
+    $testScriptsPath = Join-Path -Path $ProjectRoot -ChildPath 'test\Unit'
+    $testResultsFile = Join-Path -Path $testScriptsPath -ChildPath 'TestResults.unit.xml'
+    $codeCoverageFile = Join-Path -Path $testScriptsPath -ChildPath 'CodeCoverage.xml'
     $testResults = Invoke-Pester `
+        -Script $testScriptsPath `
         -OutputFormat NUnitXml `
         -OutputFile $testResultsFile `
         -PassThru `
         -ExcludeTag Incomplete `
-        -CodeCoverage @( Join-Path -Path $ProjectRoot -ChildPath 'src\lib\*.ps1' )
+        -CodeCoverage @( Join-Path -Path $ProjectRoot -ChildPath 'src\lib\*.ps1' ) `
+        -CodeCoverageOutputFile $codeCoverageFile `
+        -CodeCoverageOutputFileFormat JaCoCo
 
     # Prepare and uploade code coverage
     if ($testResults.CodeCoverage)
@@ -109,14 +126,54 @@ Task Test -Depends Init {
 
         if ($testResults.FailedCount -gt 0)
         {
-            throw "$($testResults.FailedCount) tests failed."
+            throw "$($testResults.FailedCount) unit tests failed."
+        }
+    }
+    else
+    {
+        if ($testResults.FailedCount -gt 0)
+        {
+            Write-Error -Exception "$($testResults.FailedCount) unit tests failed."
+        }
+    }
+
+    "`n"
+}
+
+Task IntegrationTest -Depends Init, PrepareTest {
+    $separator
+
+    # Execute tests
+    $testScriptsPath = Join-Path -Path $ProjectRoot -ChildPath 'test\Integration'
+    $testResultsFile = Join-Path -Path $testScriptsPath -ChildPath 'TestResults.integration.xml'
+    $testResults = Invoke-Pester `
+        -Script $testScriptsPath `
+        -OutputFormat NUnitXml `
+        -OutputFile $testResultsFile `
+        -PassThru `
+        -ExcludeTag Incomplete
+
+    # Upload tests
+    if ($ENV:BHBuildSystem -eq 'AppVeyor')
+    {
+        'Publishing test results to AppVeyor'
+        (New-Object 'System.Net.WebClient').UploadFile(
+            "https://ci.appveyor.com/api/testresults/nunit/$($env:APPVEYOR_JOB_ID)",
+            (Resolve-Path $testResultsFile))
+
+        "Publishing test results to AppVeyor as Artifact"
+        Push-AppveyorArtifact $testResultsFile
+
+        if ($testResults.FailedCount -gt 0)
+        {
+            throw "$($testResults.FailedCount) integration tests failed."
         }
     }
     else
     {
         if ($testResults.FailedCount -gt 0)
         {
-            Write-Error -Exception "$($testResults.FailedCount) tests failed."
+            Write-Error -Exception "$($testResults.FailedCount) integration tests failed."
         }
     }
 

diff --git a/test/CosmosDB.integration.Tests.ps1 → ...ntegration/CosmosDB.integration.Tests.ps1 b/test/CosmosDB.integration.Tests.ps1 → ...ntegration/CosmosDB.integration.Tests.ps1
@@ -4,8 +4,8 @@ param (
 )
 
 $ModuleManifestName = 'CosmosDB.psd1'
-$ModuleManifestPath = "$PSScriptRoot\..\src\$ModuleManifestName"
-$TestHelperPath = "$PSScriptRoot\TestHelper"
+$ModuleManifestPath = "$PSScriptRoot\..\..\src\$ModuleManifestName"
+$TestHelperPath = "$PSScriptRoot\..\TestHelper"
 
 Import-Module -Name $ModuleManifestPath -Force
 Import-Module -Name $TestHelperPath -Force

diff --git a/test/CosmosDB.accounts.Tests.ps1 → test/Unit/CosmosDB.accounts.Tests.ps1 b/test/CosmosDB.accounts.Tests.ps1 → test/Unit/CosmosDB.accounts.Tests.ps1
@@ -4,7 +4,7 @@ param (
 )
 
 $ModuleManifestName = 'CosmosDB.psd1'
-$ModuleManifestPath = "$PSScriptRoot\..\src\$ModuleManifestName"
+$ModuleManifestPath = "$PSScriptRoot\..\..\src\$ModuleManifestName"
 
 Import-Module -Name $ModuleManifestPath -Force
 

diff --git a/test/CosmosDB.attachments.Tests.ps1 → test/Unit/CosmosDB.attachments.Tests.ps1 b/test/CosmosDB.attachments.Tests.ps1 → test/Unit/CosmosDB.attachments.Tests.ps1
@@ -4,7 +4,7 @@ param (
 )
 
 $ModuleManifestName = 'CosmosDB.psd1'
-$ModuleManifestPath = "$PSScriptRoot\..\src\$ModuleManifestName"
+$ModuleManifestPath = "$PSScriptRoot\..\..\src\$ModuleManifestName"
 
 Import-Module -Name $ModuleManifestPath -Force
 

diff --git a/test/CosmosDB.collections.Tests.ps1 → test/Unit/CosmosDB.collections.Tests.ps1 b/test/CosmosDB.collections.Tests.ps1 → test/Unit/CosmosDB.collections.Tests.ps1
@@ -4,7 +4,7 @@ param (
 )
 
 $ModuleManifestName = 'CosmosDB.psd1'
-$ModuleManifestPath = "$PSScriptRoot\..\src\$ModuleManifestName"
+$ModuleManifestPath = "$PSScriptRoot\..\..\src\$ModuleManifestName"
 
 Import-Module -Name $ModuleManifestPath -Force
 

diff --git a/test/CosmosDB.databases.Tests.ps1 → test/Unit/CosmosDB.databases.Tests.ps1 b/test/CosmosDB.databases.Tests.ps1 → test/Unit/CosmosDB.databases.Tests.ps1
@@ -4,7 +4,7 @@ param (
 )
 
 $ModuleManifestName = 'CosmosDB.psd1'
-$ModuleManifestPath = "$PSScriptRoot\..\src\$ModuleManifestName"
+$ModuleManifestPath = "$PSScriptRoot\..\..\src\$ModuleManifestName"
 
 Import-Module -Name $ModuleManifestPath -Force
 

diff --git a/test/CosmosDB.documents.Tests.ps1 → test/Unit/CosmosDB.documents.Tests.ps1 b/test/CosmosDB.documents.Tests.ps1 → test/Unit/CosmosDB.documents.Tests.ps1
@@ -4,7 +4,7 @@ param (
 )
 
 $ModuleManifestName = 'CosmosDB.psd1'
-$ModuleManifestPath = "$PSScriptRoot\..\src\$ModuleManifestName"
+$ModuleManifestPath = "$PSScriptRoot\..\..\src\$ModuleManifestName"
 
 Import-Module -Name $ModuleManifestPath -Force
 

diff --git a/test/CosmosDB.generic.Tests.ps1 → test/Unit/CosmosDB.generic.Tests.ps1 b/test/CosmosDB.generic.Tests.ps1 → test/Unit/CosmosDB.generic.Tests.ps1
@@ -4,7 +4,7 @@ param (
 )
 
 $moduleManifestName = 'CosmosDB.psd1'
-$moduleRootPath = "$PSScriptRoot\..\src\"
+$moduleRootPath = "$PSScriptRoot\..\..\src\"
 $moduleManifestPath = Join-Path -Path $moduleRootPath -ChildPath $moduleManifestName
 
 Describe 'Module Manifest Tests' {

diff --git a/test/CosmosDB.offers.Tests.ps1 → test/Unit/CosmosDB.offers.Tests.ps1 b/test/CosmosDB.offers.Tests.ps1 → test/Unit/CosmosDB.offers.Tests.ps1
@@ -4,7 +4,7 @@ param (
 )
 
 $ModuleManifestName = 'CosmosDB.psd1'
-$ModuleManifestPath = "$PSScriptRoot\..\src\$ModuleManifestName"
+$ModuleManifestPath = "$PSScriptRoot\..\..\src\$ModuleManifestName"
 
 Import-Module -Name $ModuleManifestPath -Force
 

diff --git a/test/CosmosDB.permissions.Tests.ps1 → test/Unit/CosmosDB.permissions.Tests.ps1 b/test/CosmosDB.permissions.Tests.ps1 → test/Unit/CosmosDB.permissions.Tests.ps1
@@ -4,7 +4,7 @@ param (
 )
 
 $ModuleManifestName = 'CosmosDB.psd1'
-$ModuleManifestPath = "$PSScriptRoot\..\src\$ModuleManifestName"
+$ModuleManifestPath = "$PSScriptRoot\..\..\src\$ModuleManifestName"
 
 Import-Module -Name $ModuleManifestPath -Force
 

diff --git a/test/CosmosDB.storedprocedures.Tests.ps1 → .../Unit/CosmosDB.storedprocedures.Tests.ps1 b/test/CosmosDB.storedprocedures.Tests.ps1 → .../Unit/CosmosDB.storedprocedures.Tests.ps1
@@ -4,7 +4,7 @@ param (
 )
 
 $ModuleManifestName = 'CosmosDB.psd1'
-$ModuleManifestPath = "$PSScriptRoot\..\src\$ModuleManifestName"
+$ModuleManifestPath = "$PSScriptRoot\..\..\src\$ModuleManifestName"
 
 Import-Module -Name $ModuleManifestPath -Force
 

diff --git a/test/CosmosDB.triggers.Tests.ps1 → test/Unit/CosmosDB.triggers.Tests.ps1 b/test/CosmosDB.triggers.Tests.ps1 → test/Unit/CosmosDB.triggers.Tests.ps1
@@ -4,7 +4,7 @@ param (
 )
 
 $ModuleManifestName = 'CosmosDB.psd1'
-$ModuleManifestPath = "$PSScriptRoot\..\src\$ModuleManifestName"
+$ModuleManifestPath = "$PSScriptRoot\..\..\src\$ModuleManifestName"
 
 Import-Module -Name $ModuleManifestPath -Force
 

diff --git a/test/CosmosDB.userdefinedfunctions.Tests.ps1 → ...t/CosmosDB.userdefinedfunctions.Tests.ps1 b/test/CosmosDB.userdefinedfunctions.Tests.ps1 → ...t/CosmosDB.userdefinedfunctions.Tests.ps1
@@ -4,7 +4,7 @@ param (
 )
 
 $ModuleManifestName = 'CosmosDB.psd1'
-$ModuleManifestPath = "$PSScriptRoot\..\src\$ModuleManifestName"
+$ModuleManifestPath = "$PSScriptRoot\..\..\src\$ModuleManifestName"
 
 Import-Module -Name $ModuleManifestPath -Force
 

diff --git a/test/CosmosDB.users.Tests.ps1 → test/Unit/CosmosDB.users.Tests.ps1 b/test/CosmosDB.users.Tests.ps1 → test/Unit/CosmosDB.users.Tests.ps1
@@ -4,7 +4,7 @@ param (
 )
 
 $ModuleManifestName = 'CosmosDB.psd1'
-$ModuleManifestPath = "$PSScriptRoot\..\src\$ModuleManifestName"
+$ModuleManifestPath = "$PSScriptRoot\..\..\src\$ModuleManifestName"
 
 Import-Module -Name $ModuleManifestPath -Force
 

diff --git a/test/CosmosDB.utils.Tests.ps1 → test/Unit/CosmosDB.utils.Tests.ps1 b/test/CosmosDB.utils.Tests.ps1 → test/Unit/CosmosDB.utils.Tests.ps1
@@ -4,7 +4,7 @@ param (
 )
 
 $ModuleManifestName = 'CosmosDB.psd1'
-$ModuleManifestPath = "$PSScriptRoot\..\src\$ModuleManifestName"
+$ModuleManifestPath = "$PSScriptRoot\..\..\src\$ModuleManifestName"
 
 Import-Module -Name $ModuleManifestPath -Force