From 0aa2dfc102791d95a4bbd79754d2b0e30f40fb60 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 4 Jul 2025 13:32:01 +0000 Subject: [PATCH 01/50] Initial plan From 4d3f4ffc7b38c2b5877f5fd79a2393c40b8ffff7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 4 Jul 2025 13:38:37 +0000 Subject: [PATCH 02/50] Add BeforeAll and AfterAll job support for Test-ModuleLocal workflows Co-authored-by: MariusStorhaug <17722253+MariusStorhaug@users.noreply.github.com> --- .github/workflows/workflow.yml | 121 ++++++++++++++++++ tests/srcTestRepo/tests/AfterAll.ps1 | 12 ++ tests/srcTestRepo/tests/BeforeAll.ps1 | 12 ++ .../tests/MyTests/AfterAll.ps1 | 8 ++ .../tests/MyTests/BeforeAll.ps1 | 8 ++ 5 files changed, 161 insertions(+) create mode 100644 tests/srcTestRepo/tests/AfterAll.ps1 create mode 100644 tests/srcTestRepo/tests/BeforeAll.ps1 create mode 100644 tests/srcWithManifestTestRepo/tests/MyTests/AfterAll.ps1 create mode 100644 tests/srcWithManifestTestRepo/tests/MyTests/BeforeAll.ps1 diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index c2f7ac8c..f74b7dfa 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -178,11 +178,88 @@ jobs: Version: ${{ inputs.Version }} WorkingDirectory: ${{ inputs.WorkingDirectory }} + BeforeAll-ModuleLocal: + if: ${{ needs.Build-Module.result == 'success' && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} + name: BeforeAll Setup + runs-on: ubuntu-latest + needs: + - Build-Module + - Get-Settings + outputs: + hasBeforeAll: ${{ steps.check-beforeall.outputs.hasBeforeAll }} + hasAfterAll: ${{ steps.check-afterall.outputs.hasAfterAll }} + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Check for BeforeAll/AfterAll scripts + id: check-beforeall + shell: bash + working-directory: ${{ inputs.WorkingDirectory }} + run: | + hasBeforeAll=false + hasAfterAll=false + + # Check each test path for BeforeAll/AfterAll scripts + testSuites='${{ needs.Get-Settings.outputs.ModuleTestSuites }}' + echo "Test suites: $testSuites" + + if echo "$testSuites" | jq -e '.[] | select(.TestPath)' > /dev/null 2>&1; then + for testPath in $(echo "$testSuites" | jq -r '.[].TestPath' | sort -u); do + echo "Checking test path: $testPath" + if [ -f "$testPath/BeforeAll.ps1" ]; then + echo "BeforeAll.ps1 found in $testPath" + hasBeforeAll=true + fi + if [ -f "$testPath/AfterAll.ps1" ]; then + echo "AfterAll.ps1 found in $testPath" + hasAfterAll=true + fi + done + fi + + echo "hasBeforeAll=$hasBeforeAll" >> $GITHUB_OUTPUT + echo "hasAfterAll=$hasAfterAll" >> $GITHUB_OUTPUT + + - name: Install-PSModuleHelpers + if: steps.check-beforeall.outputs.hasBeforeAll == 'true' + uses: PSModule/Install-PSModuleHelpers@v1 + + - name: Run BeforeAll scripts + if: steps.check-beforeall.outputs.hasBeforeAll == 'true' + shell: pwsh + working-directory: ${{ inputs.WorkingDirectory }} + env: + TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} + TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} + TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} + TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} + TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} + TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} + TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} + GITHUB_TOKEN: ${{ github.token }} + run: | + $testSuites = '${{ needs.Get-Settings.outputs.ModuleTestSuites }}' | ConvertFrom-Json + $processedPaths = @() + + foreach ($suite in $testSuites) { + $testPath = $suite.TestPath + if ($testPath -and $testPath -notin $processedPaths) { + $beforeAllScript = Join-Path $testPath "BeforeAll.ps1" + if (Test-Path $beforeAllScript) { + Write-Host "Running BeforeAll setup script: $beforeAllScript" + & $beforeAllScript + $processedPaths += $testPath + } + } + } + Test-ModuleLocal: if: ${{ needs.Build-Module.result == 'success' && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} needs: - Build-Module - Get-Settings + - BeforeAll-ModuleLocal strategy: fail-fast: false matrix: @@ -201,6 +278,48 @@ jobs: Version: ${{ inputs.Version }} WorkingDirectory: ${{ inputs.WorkingDirectory }} + AfterAll-ModuleLocal: + if: ${{ always() && needs.BeforeAll-ModuleLocal.outputs.hasAfterAll == 'true' }} + name: AfterAll Teardown + runs-on: ubuntu-latest + needs: + - BeforeAll-ModuleLocal + - Test-ModuleLocal + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Install-PSModuleHelpers + uses: PSModule/Install-PSModuleHelpers@v1 + + - name: Run AfterAll scripts + shell: pwsh + working-directory: ${{ inputs.WorkingDirectory }} + env: + TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} + TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} + TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} + TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} + TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} + TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} + TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} + GITHUB_TOKEN: ${{ github.token }} + run: | + $testSuites = '${{ needs.Get-Settings.outputs.ModuleTestSuites }}' | ConvertFrom-Json + $processedPaths = @() + + foreach ($suite in $testSuites) { + $testPath = $suite.TestPath + if ($testPath -and $testPath -notin $processedPaths) { + $afterAllScript = Join-Path $testPath "AfterAll.ps1" + if (Test-Path $afterAllScript) { + Write-Host "Running AfterAll teardown script: $afterAllScript" + & $afterAllScript + $processedPaths += $testPath + } + } + } + Get-TestResults: if: needs.Get-Settings.result == 'success' && !fromJson(needs.Get-Settings.outputs.Settings).Test.TestResults.Skip && (needs.Get-Settings.outputs.SourceCodeTestSuites != '[]' || needs.Get-Settings.outputs.PSModuleTestSuites != '[]' || needs.Get-Settings.outputs.ModuleTestSuites != '[]') && (always() && !cancelled()) needs: @@ -209,6 +328,7 @@ jobs: - Lint-SourceCode - Test-Module - Test-ModuleLocal + - AfterAll-ModuleLocal uses: ./.github/workflows/Get-TestResults.yml secrets: inherit with: @@ -226,6 +346,7 @@ jobs: - Get-Settings - Test-Module - Test-ModuleLocal + - AfterAll-ModuleLocal uses: ./.github/workflows/Get-CodeCoverage.yml secrets: inherit with: diff --git a/tests/srcTestRepo/tests/AfterAll.ps1 b/tests/srcTestRepo/tests/AfterAll.ps1 new file mode 100644 index 00000000..6c9b667e --- /dev/null +++ b/tests/srcTestRepo/tests/AfterAll.ps1 @@ -0,0 +1,12 @@ +Write-Host "=== AFTERALL TEARDOWN SCRIPT EXECUTING ===" +Write-Host "Tearing down test environment..." + +# Example teardown tasks: +# - Clean up test infrastructure +# - Remove test data +# - Cleanup test environment +# - Drop test databases +# - Stop test services + +Write-Host "Test environment teardown completed successfully!" +Write-Host "=== AFTERALL TEARDOWN SCRIPT COMPLETED ===" \ No newline at end of file diff --git a/tests/srcTestRepo/tests/BeforeAll.ps1 b/tests/srcTestRepo/tests/BeforeAll.ps1 new file mode 100644 index 00000000..90740257 --- /dev/null +++ b/tests/srcTestRepo/tests/BeforeAll.ps1 @@ -0,0 +1,12 @@ +Write-Host "=== BEFOREALL SETUP SCRIPT EXECUTING ===" +Write-Host "Setting up test environment..." + +# Example setup tasks: +# - Deploy test infrastructure +# - Download test data +# - Initialize test environment +# - Set up test databases +# - Configure test services + +Write-Host "Test environment setup completed successfully!" +Write-Host "=== BEFOREALL SETUP SCRIPT COMPLETED ===" \ No newline at end of file diff --git a/tests/srcWithManifestTestRepo/tests/MyTests/AfterAll.ps1 b/tests/srcWithManifestTestRepo/tests/MyTests/AfterAll.ps1 new file mode 100644 index 00000000..72c4ea7a --- /dev/null +++ b/tests/srcWithManifestTestRepo/tests/MyTests/AfterAll.ps1 @@ -0,0 +1,8 @@ +Write-Host "=== AFTERALL TEARDOWN SCRIPT (MyTests) EXECUTING ===" +Write-Host "Tearing down test environment for MyTests..." + +# Example teardown tasks for MyTests directory: +Write-Host "Cleaning up MyTests test environment..." + +Write-Host "MyTests environment teardown completed successfully!" +Write-Host "=== AFTERALL TEARDOWN SCRIPT (MyTests) COMPLETED ===" \ No newline at end of file diff --git a/tests/srcWithManifestTestRepo/tests/MyTests/BeforeAll.ps1 b/tests/srcWithManifestTestRepo/tests/MyTests/BeforeAll.ps1 new file mode 100644 index 00000000..8f2f74c5 --- /dev/null +++ b/tests/srcWithManifestTestRepo/tests/MyTests/BeforeAll.ps1 @@ -0,0 +1,8 @@ +Write-Host "=== BEFOREALL SETUP SCRIPT (MyTests) EXECUTING ===" +Write-Host "Setting up test environment for MyTests..." + +# Example setup tasks for MyTests directory: +Write-Host "Initializing MyTests test environment..." + +Write-Host "MyTests environment setup completed successfully!" +Write-Host "=== BEFOREALL SETUP SCRIPT (MyTests) COMPLETED ===" \ No newline at end of file From 9eeb8c7e0a9088bc013d5136434c3a6eb1b3be7b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 4 Jul 2025 13:40:29 +0000 Subject: [PATCH 03/50] Add additional test BeforeAll/AfterAll scripts for comprehensive testing Co-authored-by: MariusStorhaug <17722253+MariusStorhaug@users.noreply.github.com> --- .../tests/Environments/AfterAll.ps1 | 8 ++++++++ .../tests/Environments/BeforeAll.ps1 | 8 ++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/srcWithManifestTestRepo/tests/Environments/AfterAll.ps1 create mode 100644 tests/srcWithManifestTestRepo/tests/Environments/BeforeAll.ps1 diff --git a/tests/srcWithManifestTestRepo/tests/Environments/AfterAll.ps1 b/tests/srcWithManifestTestRepo/tests/Environments/AfterAll.ps1 new file mode 100644 index 00000000..20f4fe51 --- /dev/null +++ b/tests/srcWithManifestTestRepo/tests/Environments/AfterAll.ps1 @@ -0,0 +1,8 @@ +Write-Host "=== AFTERALL TEARDOWN SCRIPT (Environments) EXECUTING ===" +Write-Host "Tearing down test environment for Environments..." + +# Example teardown tasks for Environments directory: +Write-Host "Cleaning up Environments test environment..." + +Write-Host "Environments environment teardown completed successfully!" +Write-Host "=== AFTERALL TEARDOWN SCRIPT (Environments) COMPLETED ===" \ No newline at end of file diff --git a/tests/srcWithManifestTestRepo/tests/Environments/BeforeAll.ps1 b/tests/srcWithManifestTestRepo/tests/Environments/BeforeAll.ps1 new file mode 100644 index 00000000..d6e18a0e --- /dev/null +++ b/tests/srcWithManifestTestRepo/tests/Environments/BeforeAll.ps1 @@ -0,0 +1,8 @@ +Write-Host "=== BEFOREALL SETUP SCRIPT (Environments) EXECUTING ===" +Write-Host "Setting up test environment for Environments..." + +# Example setup tasks for Environments directory: +Write-Host "Initializing Environments test environment..." + +Write-Host "Environments environment setup completed successfully!" +Write-Host "=== BEFOREALL SETUP SCRIPT (Environments) COMPLETED ===" \ No newline at end of file From 2a30035676e640205ad9dc9236612be3c5505cf2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 4 Jul 2025 13:41:55 +0000 Subject: [PATCH 04/50] Add documentation for BeforeAll/AfterAll setup and teardown scripts Co-authored-by: MariusStorhaug <17722253+MariusStorhaug@users.noreply.github.com> --- README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/README.md b/README.md index 28259b48..935965e6 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,10 @@ Depending on the labels in the pull requests, the workflow will result in differ - This produces a json based report that is used to later evaluate the results of the tests. - [Test module](./.github/workflows/Test-ModuleLocal.yml) - Import and tests the module in parallel (matrix) using Pester tests from the module repository. + - Supports setup and teardown scripts: + - **BeforeAll.ps1**: Runs once before all test matrix jobs to set up test environment (e.g., deploy infrastructure, download test data) + - **AfterAll.ps1**: Runs once after all test matrix jobs complete to clean up test environment (e.g., remove test resources, cleanup databases) + - Setup/teardown scripts are automatically detected in test directories and executed with the same environment variables as tests - This produces a json based report that is used to later evaluate the results of the tests. - [Get test results](./.github/workflows/Get-TestResults.yml) - Gathers the test results from the previous steps and creates a summary of the results. @@ -257,6 +261,46 @@ Build: | `Verbose` | `boolean` | Whether to enable verbose output. | `false` | `false` | | `WorkingDirectory` | `string` | The path to the root of the repo. | `false` | `.` | +### Setup and Teardown Scripts + +The workflow supports automatic execution of setup and teardown scripts for module tests: + +#### BeforeAll.ps1 +- **Location**: Place in your test directories (e.g., `tests/BeforeAll.ps1`) +- **Purpose**: Runs once before all test matrix jobs to prepare the test environment +- **Use cases**: Deploy test infrastructure, download test data, initialize databases, configure services +- **Environment**: Has access to the same environment variables as your tests (secrets, GitHub token, etc.) + +#### AfterAll.ps1 +- **Location**: Place in your test directories (e.g., `tests/AfterAll.ps1`) +- **Purpose**: Runs once after all test matrix jobs complete to clean up the test environment +- **Use cases**: Remove test resources, cleanup databases, stop services, upload artifacts +- **Environment**: Has access to the same environment variables as your tests + +**Example BeforeAll.ps1:** +```powershell +Write-Host "Setting up test environment..." +# Deploy test infrastructure +# Download test data +# Initialize test databases +Write-Host "Test environment ready!" +``` + +**Example AfterAll.ps1:** +```powershell +Write-Host "Cleaning up test environment..." +# Remove test resources +# Cleanup databases +# Stop services +Write-Host "Cleanup completed!" +``` + +**Notes:** +- Scripts are automatically detected and executed if present +- Each unique test directory path is processed only once +- Scripts run with PowerShell and have access to PSModuleHelpers +- If no scripts are found, the workflow continues normally + ### Secrets The following secrets are used by the workflow. They can be automatically provided (if available) by setting the `secrets: inherit` From 2027a0bb3330bb4f31ac9b876ebf2744e342c419 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 6 Jul 2025 06:21:19 +0000 Subject: [PATCH 05/50] Rework BeforeAll/AfterAll to use PowerShell-based steps Co-authored-by: MariusStorhaug <17722253+MariusStorhaug@users.noreply.github.com> --- .github/workflows/Test-ModuleLocal.yml | 45 +++++++++ .github/workflows/workflow.yml | 121 ------------------------- 2 files changed, 45 insertions(+), 121 deletions(-) diff --git a/.github/workflows/Test-ModuleLocal.yml b/.github/workflows/Test-ModuleLocal.yml index b7b9654b..f51c6b2c 100644 --- a/.github/workflows/Test-ModuleLocal.yml +++ b/.github/workflows/Test-ModuleLocal.yml @@ -104,6 +104,28 @@ jobs: "name=$name" >> $env:GITHUB_OUTPUT "path=$path" >> $env:GITHUB_OUTPUT + - name: Run BeforeAll Setup Script + shell: pwsh + working-directory: ${{ inputs.WorkingDirectory }} + env: + TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} + TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} + TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} + TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} + TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} + TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} + TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} + GITHUB_TOKEN: ${{ github.token }} + run: | + $testPath = '${{ inputs.TestPath }}' + $beforeAllScript = Join-Path $testPath "BeforeAll.ps1" + if (Test-Path $beforeAllScript) { + Write-Host "Running BeforeAll setup script: $beforeAllScript" + & $beforeAllScript + } else { + Write-Host "No BeforeAll.ps1 script found in $testPath" + } + - name: Test-ModuleLocal uses: PSModule/Invoke-Pester@v4 env: @@ -133,3 +155,26 @@ jobs: Prescript: | # This is to speed up module loading in Pester. Install-PSResource -Repository PSGallery -TrustRepository -Name PSCustomObject Import-Module -Name '${{ steps.import-module.outputs.name }}' -RequiredVersion 999.0.0 + + - name: Run AfterAll Teardown Script + if: always() + shell: pwsh + working-directory: ${{ inputs.WorkingDirectory }} + env: + TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} + TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} + TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} + TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} + TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} + TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} + TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} + GITHUB_TOKEN: ${{ github.token }} + run: | + $testPath = '${{ inputs.TestPath }}' + $afterAllScript = Join-Path $testPath "AfterAll.ps1" + if (Test-Path $afterAllScript) { + Write-Host "Running AfterAll teardown script: $afterAllScript" + & $afterAllScript + } else { + Write-Host "No AfterAll.ps1 script found in $testPath" + } diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index f74b7dfa..c2f7ac8c 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -178,88 +178,11 @@ jobs: Version: ${{ inputs.Version }} WorkingDirectory: ${{ inputs.WorkingDirectory }} - BeforeAll-ModuleLocal: - if: ${{ needs.Build-Module.result == 'success' && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} - name: BeforeAll Setup - runs-on: ubuntu-latest - needs: - - Build-Module - - Get-Settings - outputs: - hasBeforeAll: ${{ steps.check-beforeall.outputs.hasBeforeAll }} - hasAfterAll: ${{ steps.check-afterall.outputs.hasAfterAll }} - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Check for BeforeAll/AfterAll scripts - id: check-beforeall - shell: bash - working-directory: ${{ inputs.WorkingDirectory }} - run: | - hasBeforeAll=false - hasAfterAll=false - - # Check each test path for BeforeAll/AfterAll scripts - testSuites='${{ needs.Get-Settings.outputs.ModuleTestSuites }}' - echo "Test suites: $testSuites" - - if echo "$testSuites" | jq -e '.[] | select(.TestPath)' > /dev/null 2>&1; then - for testPath in $(echo "$testSuites" | jq -r '.[].TestPath' | sort -u); do - echo "Checking test path: $testPath" - if [ -f "$testPath/BeforeAll.ps1" ]; then - echo "BeforeAll.ps1 found in $testPath" - hasBeforeAll=true - fi - if [ -f "$testPath/AfterAll.ps1" ]; then - echo "AfterAll.ps1 found in $testPath" - hasAfterAll=true - fi - done - fi - - echo "hasBeforeAll=$hasBeforeAll" >> $GITHUB_OUTPUT - echo "hasAfterAll=$hasAfterAll" >> $GITHUB_OUTPUT - - - name: Install-PSModuleHelpers - if: steps.check-beforeall.outputs.hasBeforeAll == 'true' - uses: PSModule/Install-PSModuleHelpers@v1 - - - name: Run BeforeAll scripts - if: steps.check-beforeall.outputs.hasBeforeAll == 'true' - shell: pwsh - working-directory: ${{ inputs.WorkingDirectory }} - env: - TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} - TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} - TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} - TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} - TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} - TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} - TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} - GITHUB_TOKEN: ${{ github.token }} - run: | - $testSuites = '${{ needs.Get-Settings.outputs.ModuleTestSuites }}' | ConvertFrom-Json - $processedPaths = @() - - foreach ($suite in $testSuites) { - $testPath = $suite.TestPath - if ($testPath -and $testPath -notin $processedPaths) { - $beforeAllScript = Join-Path $testPath "BeforeAll.ps1" - if (Test-Path $beforeAllScript) { - Write-Host "Running BeforeAll setup script: $beforeAllScript" - & $beforeAllScript - $processedPaths += $testPath - } - } - } - Test-ModuleLocal: if: ${{ needs.Build-Module.result == 'success' && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} needs: - Build-Module - Get-Settings - - BeforeAll-ModuleLocal strategy: fail-fast: false matrix: @@ -278,48 +201,6 @@ jobs: Version: ${{ inputs.Version }} WorkingDirectory: ${{ inputs.WorkingDirectory }} - AfterAll-ModuleLocal: - if: ${{ always() && needs.BeforeAll-ModuleLocal.outputs.hasAfterAll == 'true' }} - name: AfterAll Teardown - runs-on: ubuntu-latest - needs: - - BeforeAll-ModuleLocal - - Test-ModuleLocal - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Install-PSModuleHelpers - uses: PSModule/Install-PSModuleHelpers@v1 - - - name: Run AfterAll scripts - shell: pwsh - working-directory: ${{ inputs.WorkingDirectory }} - env: - TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} - TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} - TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} - TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} - TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} - TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} - TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} - GITHUB_TOKEN: ${{ github.token }} - run: | - $testSuites = '${{ needs.Get-Settings.outputs.ModuleTestSuites }}' | ConvertFrom-Json - $processedPaths = @() - - foreach ($suite in $testSuites) { - $testPath = $suite.TestPath - if ($testPath -and $testPath -notin $processedPaths) { - $afterAllScript = Join-Path $testPath "AfterAll.ps1" - if (Test-Path $afterAllScript) { - Write-Host "Running AfterAll teardown script: $afterAllScript" - & $afterAllScript - $processedPaths += $testPath - } - } - } - Get-TestResults: if: needs.Get-Settings.result == 'success' && !fromJson(needs.Get-Settings.outputs.Settings).Test.TestResults.Skip && (needs.Get-Settings.outputs.SourceCodeTestSuites != '[]' || needs.Get-Settings.outputs.PSModuleTestSuites != '[]' || needs.Get-Settings.outputs.ModuleTestSuites != '[]') && (always() && !cancelled()) needs: @@ -328,7 +209,6 @@ jobs: - Lint-SourceCode - Test-Module - Test-ModuleLocal - - AfterAll-ModuleLocal uses: ./.github/workflows/Get-TestResults.yml secrets: inherit with: @@ -346,7 +226,6 @@ jobs: - Get-Settings - Test-Module - Test-ModuleLocal - - AfterAll-ModuleLocal uses: ./.github/workflows/Get-CodeCoverage.yml secrets: inherit with: From 92d0958a422cf74e3f809e8d145c94e4463b5982 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 6 Jul 2025 06:24:06 +0000 Subject: [PATCH 06/50] Update documentation for PowerShell-based BeforeAll/AfterAll implementation Co-authored-by: MariusStorhaug <17722253+MariusStorhaug@users.noreply.github.com> --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 935965e6..3b92d32e 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ Depending on the labels in the pull requests, the workflow will result in differ - [Test module](./.github/workflows/Test-ModuleLocal.yml) - Import and tests the module in parallel (matrix) using Pester tests from the module repository. - Supports setup and teardown scripts: - - **BeforeAll.ps1**: Runs once before all test matrix jobs to set up test environment (e.g., deploy infrastructure, download test data) - - **AfterAll.ps1**: Runs once after all test matrix jobs complete to clean up test environment (e.g., remove test resources, cleanup databases) + - **BeforeAll.ps1**: Runs before each test execution to set up test environment (e.g., deploy infrastructure, download test data) + - **AfterAll.ps1**: Runs after each test execution to clean up test environment (e.g., remove test resources, cleanup databases) - Setup/teardown scripts are automatically detected in test directories and executed with the same environment variables as tests - This produces a json based report that is used to later evaluate the results of the tests. - [Get test results](./.github/workflows/Get-TestResults.yml) @@ -267,13 +267,13 @@ The workflow supports automatic execution of setup and teardown scripts for modu #### BeforeAll.ps1 - **Location**: Place in your test directories (e.g., `tests/BeforeAll.ps1`) -- **Purpose**: Runs once before all test matrix jobs to prepare the test environment +- **Purpose**: Runs before each test execution to prepare the test environment - **Use cases**: Deploy test infrastructure, download test data, initialize databases, configure services - **Environment**: Has access to the same environment variables as your tests (secrets, GitHub token, etc.) #### AfterAll.ps1 - **Location**: Place in your test directories (e.g., `tests/AfterAll.ps1`) -- **Purpose**: Runs once after all test matrix jobs complete to clean up the test environment +- **Purpose**: Runs after each test execution to clean up the test environment - **Use cases**: Remove test resources, cleanup databases, stop services, upload artifacts - **Environment**: Has access to the same environment variables as your tests From 7b69fe7b761629142d17e5048cff9dfe15f3fb9b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 11 Sep 2025 23:54:39 +0000 Subject: [PATCH 07/50] Implement separate BeforeAll/AfterAll jobs with conditional execution Co-authored-by: MariusStorhaug <17722253+MariusStorhaug@users.noreply.github.com> --- .github/workflows/AfterAll-ModuleLocal.yml | 152 ++++++++++++++++++++ .github/workflows/BeforeAll-ModuleLocal.yml | 151 +++++++++++++++++++ .github/workflows/CI.yml | 33 +++++ .github/workflows/Get-Settings.yml | 45 ++++++ .github/workflows/Test-ModuleLocal.yml | 45 ------ .github/workflows/workflow.yml | 33 +++++ 6 files changed, 414 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/AfterAll-ModuleLocal.yml create mode 100644 .github/workflows/BeforeAll-ModuleLocal.yml diff --git a/.github/workflows/AfterAll-ModuleLocal.yml b/.github/workflows/AfterAll-ModuleLocal.yml new file mode 100644 index 00000000..1d1f006a --- /dev/null +++ b/.github/workflows/AfterAll-ModuleLocal.yml @@ -0,0 +1,152 @@ +name: AfterAll-ModuleLocal + +on: + workflow_call: + secrets: + TEST_APP_ENT_CLIENT_ID: + description: The client ID of an Enterprise GitHub App for running tests. + required: false + TEST_APP_ENT_PRIVATE_KEY: + description: The private key of an Enterprise GitHub App for running tests. + required: false + TEST_APP_ORG_CLIENT_ID: + description: The client ID of an Organization GitHub App for running tests. + required: false + TEST_APP_ORG_PRIVATE_KEY: + description: The private key of an Organization GitHub App for running tests. + required: false + TEST_USER_ORG_FG_PAT: + description: The fine-grained personal access token with org access for running tests. + required: false + TEST_USER_USER_FG_PAT: + description: The fine-grained personal access token with user account access for running tests. + required: false + TEST_USER_PAT: + description: The classic personal access token for running tests. + required: false + inputs: + Name: + type: string + description: The name of the module to process. Scripts default to the repository name if nothing is specified. + required: false + Debug: + type: boolean + description: Enable debug output. + required: false + default: false + Verbose: + type: boolean + description: Enable verbose output. + required: false + default: false + Version: + type: string + description: Specifies the version of the GitHub module to be installed. The value must be an exact version. + required: false + default: '' + Prerelease: + type: boolean + description: Whether to use a prerelease version of the 'GitHub' module. + required: false + default: false + WorkingDirectory: + type: string + description: The working directory where the script will run from. + required: false + default: '.' + +permissions: + contents: read # to checkout the repo + +jobs: + AfterAll-ModuleLocal: + name: AfterAll-ModuleLocal + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v5 + + - name: Install-PSModuleHelpers + uses: PSModule/Install-PSModuleHelpers@v1 + + - name: Run AfterAll Teardown Scripts + if: always() + uses: PSModule/GitHub-Script@v1 + env: + TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} + TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} + TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} + TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} + TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} + TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} + TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} + GITHUB_TOKEN: ${{ github.token }} + with: + Name: AfterAll-ModuleLocal + ShowInfo: false + ShowOutput: true + Debug: ${{ inputs.Debug }} + Prerelease: ${{ inputs.Prerelease }} + Verbose: ${{ inputs.Verbose }} + Version: ${{ inputs.Version }} + WorkingDirectory: ${{ inputs.WorkingDirectory }} + Script: | + LogGroup "Running AfterAll Teardown Scripts" { + function Find-TestDirectories { + param ([string]$Path) + + $directories = @() + $childDirs = Get-ChildItem -Path $Path -Directory + + foreach ($dir in $childDirs) { + $directories += $dir.FullName + $directories += Find-TestDirectories -Path $dir.FullName + } + + return $directories + } + + # Locate the tests directory. + $testsPath = Resolve-Path 'tests' -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path + if (-not $testsPath) { + Write-Warning 'No tests directory found' + exit 0 + } + Write-Host "Tests found at [$testsPath]" + + $allTestFolders = @($testsPath) + (Find-TestDirectories -Path $testsPath) + $processedDirectories = @() + + foreach ($folder in $allTestFolders) { + $afterAllScript = Join-Path $folder "AfterAll.ps1" + + if (Test-Path $afterAllScript -PathType Leaf) { + # Get unique directory path to avoid duplicate execution + $uniqueDirectory = Resolve-Path $folder -Relative + if ($processedDirectories -notcontains $uniqueDirectory) { + $processedDirectories += $uniqueDirectory + + Write-Host "Running AfterAll teardown script: $afterAllScript" + try { + Push-Location $folder + & $afterAllScript + Write-Host "AfterAll script completed successfully: $afterAllScript" + } + catch { + Write-Warning "AfterAll script failed: $afterAllScript - $_" + # Don't throw for teardown scripts to ensure other cleanup scripts can run + } + finally { + Pop-Location + } + } + } + } + + if ($processedDirectories.Count -eq 0) { + Write-Host "No AfterAll.ps1 scripts found in test directories" + } else { + Write-Host "Processed AfterAll scripts in $($processedDirectories.Count) directories:" + $processedDirectories | ForEach-Object { Write-Host " - $_" } + } + } \ No newline at end of file diff --git a/.github/workflows/BeforeAll-ModuleLocal.yml b/.github/workflows/BeforeAll-ModuleLocal.yml new file mode 100644 index 00000000..ea212acc --- /dev/null +++ b/.github/workflows/BeforeAll-ModuleLocal.yml @@ -0,0 +1,151 @@ +name: BeforeAll-ModuleLocal + +on: + workflow_call: + secrets: + TEST_APP_ENT_CLIENT_ID: + description: The client ID of an Enterprise GitHub App for running tests. + required: false + TEST_APP_ENT_PRIVATE_KEY: + description: The private key of an Enterprise GitHub App for running tests. + required: false + TEST_APP_ORG_CLIENT_ID: + description: The client ID of an Organization GitHub App for running tests. + required: false + TEST_APP_ORG_PRIVATE_KEY: + description: The private key of an Organization GitHub App for running tests. + required: false + TEST_USER_ORG_FG_PAT: + description: The fine-grained personal access token with org access for running tests. + required: false + TEST_USER_USER_FG_PAT: + description: The fine-grained personal access token with user account access for running tests. + required: false + TEST_USER_PAT: + description: The classic personal access token for running tests. + required: false + inputs: + Name: + type: string + description: The name of the module to process. Scripts default to the repository name if nothing is specified. + required: false + Debug: + type: boolean + description: Enable debug output. + required: false + default: false + Verbose: + type: boolean + description: Enable verbose output. + required: false + default: false + Version: + type: string + description: Specifies the version of the GitHub module to be installed. The value must be an exact version. + required: false + default: '' + Prerelease: + type: boolean + description: Whether to use a prerelease version of the 'GitHub' module. + required: false + default: false + WorkingDirectory: + type: string + description: The working directory where the script will run from. + required: false + default: '.' + +permissions: + contents: read # to checkout the repo + +jobs: + BeforeAll-ModuleLocal: + name: BeforeAll-ModuleLocal + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v5 + + - name: Install-PSModuleHelpers + uses: PSModule/Install-PSModuleHelpers@v1 + + - name: Run BeforeAll Setup Scripts + uses: PSModule/GitHub-Script@v1 + env: + TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} + TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} + TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} + TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} + TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} + TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} + TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} + GITHUB_TOKEN: ${{ github.token }} + with: + Name: BeforeAll-ModuleLocal + ShowInfo: false + ShowOutput: true + Debug: ${{ inputs.Debug }} + Prerelease: ${{ inputs.Prerelease }} + Verbose: ${{ inputs.Verbose }} + Version: ${{ inputs.Version }} + WorkingDirectory: ${{ inputs.WorkingDirectory }} + Script: | + LogGroup "Running BeforeAll Setup Scripts" { + function Find-TestDirectories { + param ([string]$Path) + + $directories = @() + $childDirs = Get-ChildItem -Path $Path -Directory + + foreach ($dir in $childDirs) { + $directories += $dir.FullName + $directories += Find-TestDirectories -Path $dir.FullName + } + + return $directories + } + + # Locate the tests directory. + $testsPath = Resolve-Path 'tests' -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path + if (-not $testsPath) { + Write-Warning 'No tests directory found' + exit 0 + } + Write-Host "Tests found at [$testsPath]" + + $allTestFolders = @($testsPath) + (Find-TestDirectories -Path $testsPath) + $processedDirectories = @() + + foreach ($folder in $allTestFolders) { + $beforeAllScript = Join-Path $folder "BeforeAll.ps1" + + if (Test-Path $beforeAllScript -PathType Leaf) { + # Get unique directory path to avoid duplicate execution + $uniqueDirectory = Resolve-Path $folder -Relative + if ($processedDirectories -notcontains $uniqueDirectory) { + $processedDirectories += $uniqueDirectory + + Write-Host "Running BeforeAll setup script: $beforeAllScript" + try { + Push-Location $folder + & $beforeAllScript + Write-Host "BeforeAll script completed successfully: $beforeAllScript" + } + catch { + Write-Error "BeforeAll script failed: $beforeAllScript - $_" + throw + } + finally { + Pop-Location + } + } + } + } + + if ($processedDirectories.Count -eq 0) { + Write-Host "No BeforeAll.ps1 scripts found in test directories" + } else { + Write-Host "Processed BeforeAll scripts in $($processedDirectories.Count) directories:" + $processedDirectories | ForEach-Object { Write-Host " - $_" } + } + } \ No newline at end of file diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 30ab8859..aef6ee1d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -176,11 +176,27 @@ jobs: Version: ${{ inputs.Version }} WorkingDirectory: ${{ inputs.WorkingDirectory }} + BeforeAll-ModuleLocal: + if: ${{ needs.Build-Module.result == 'success' && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} + needs: + - Build-Module + - Get-Settings + uses: ./.github/workflows/BeforeAll-ModuleLocal.yml + secrets: inherit + with: + Name: ${{ fromJson(needs.Get-Settings.outputs.Settings).Name }} + Debug: ${{ inputs.Debug }} + Prerelease: ${{ inputs.Prerelease }} + Verbose: ${{ inputs.Verbose }} + Version: ${{ inputs.Version }} + WorkingDirectory: ${{ inputs.WorkingDirectory }} + Test-ModuleLocal: if: ${{ needs.Build-Module.result == 'success' && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} needs: - Build-Module - Get-Settings + - BeforeAll-ModuleLocal strategy: fail-fast: false matrix: @@ -199,6 +215,21 @@ jobs: Version: ${{ inputs.Version }} WorkingDirectory: ${{ inputs.WorkingDirectory }} + AfterAll-ModuleLocal: + if: ${{ always() && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} + needs: + - Get-Settings + - Test-ModuleLocal + uses: ./.github/workflows/AfterAll-ModuleLocal.yml + secrets: inherit + with: + Name: ${{ fromJson(needs.Get-Settings.outputs.Settings).Name }} + Debug: ${{ inputs.Debug }} + Prerelease: ${{ inputs.Prerelease }} + Verbose: ${{ inputs.Verbose }} + Version: ${{ inputs.Version }} + WorkingDirectory: ${{ inputs.WorkingDirectory }} + Get-TestResults: if: needs.Get-Settings.result == 'success' && !fromJson(needs.Get-Settings.outputs.Settings).Test.TestResults.Skip && (needs.Get-Settings.outputs.SourceCodeTestSuites != '[]' || needs.Get-Settings.outputs.PSModuleTestSuites != '[]' || needs.Get-Settings.outputs.ModuleTestSuites != '[]') && (always() && !cancelled()) needs: @@ -207,6 +238,7 @@ jobs: - Lint-SourceCode - Test-Module - Test-ModuleLocal + - AfterAll-ModuleLocal uses: ./.github/workflows/Get-TestResults.yml secrets: inherit with: @@ -224,6 +256,7 @@ jobs: - Get-Settings - Test-Module - Test-ModuleLocal + - AfterAll-ModuleLocal uses: ./.github/workflows/Get-CodeCoverage.yml secrets: inherit with: diff --git a/.github/workflows/Get-Settings.yml b/.github/workflows/Get-Settings.yml index 1c79b4e5..28b93905 100644 --- a/.github/workflows/Get-Settings.yml +++ b/.github/workflows/Get-Settings.yml @@ -50,6 +50,12 @@ on: ModuleTestSuites: description: Module local test suites to run. value: ${{ jobs.Get-Settings.outputs.ModuleTestSuites }} + HasBeforeAllScripts: + description: Whether BeforeAll.ps1 scripts are found in test directories. + value: ${{ jobs.Get-Settings.outputs.HasBeforeAllScripts }} + HasAfterAllScripts: + description: Whether AfterAll.ps1 scripts are found in test directories. + value: ${{ jobs.Get-Settings.outputs.HasAfterAllScripts }} permissions: contents: read # to checkout the repo @@ -63,6 +69,8 @@ jobs: SourceCodeTestSuites: ${{ fromJson(steps.Get-Settings.outputs.result).SourceCodeTestSuites }} PSModuleTestSuites: ${{ fromJson(steps.Get-Settings.outputs.result).PSModuleTestSuites }} ModuleTestSuites: ${{ fromJson(steps.Get-Settings.outputs.result).ModuleTestSuites }} + HasBeforeAllScripts: ${{ fromJson(steps.Get-Settings.outputs.result).HasBeforeAllScripts }} + HasAfterAllScripts: ${{ fromJson(steps.Get-Settings.outputs.result).HasAfterAllScripts }} steps: - name: Checkout Code uses: actions/checkout@v5 @@ -364,3 +372,40 @@ jobs: $moduleTestSuites = ($null -ne $moduleTestSuites) ? ($moduleTestSuites | ConvertTo-Json -AsArray) : '[]' Set-GitHubOutput -Name ModuleTestSuites -Value $moduleTestSuites } + + # Check for BeforeAll.ps1 and AfterAll.ps1 scripts + LogGroup 'Setup/Teardown Scripts Detection:' { + $hasBeforeAllScripts = $false + $hasAfterAllScripts = $false + + if (-not $settings.Test.Module.Skip) { + # Locate the tests directory. + $testsPath = Resolve-Path 'tests' -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path + if ($testsPath) { + Write-Host "Checking for setup/teardown scripts in [$testsPath]" + + $allTestFolders = @($testsPath) + (Find-TestDirectories -Path $testsPath) + + foreach ($folder in $allTestFolders) { + $beforeAllScript = Join-Path $folder "BeforeAll.ps1" + $afterAllScript = Join-Path $folder "AfterAll.ps1" + + if (Test-Path $beforeAllScript) { + Write-Host "Found BeforeAll.ps1 in: $folder" + $hasBeforeAllScripts = $true + } + + if (Test-Path $afterAllScript) { + Write-Host "Found AfterAll.ps1 in: $folder" + $hasAfterAllScripts = $true + } + } + } + } + + Write-Host "BeforeAll scripts found: $hasBeforeAllScripts" + Write-Host "AfterAll scripts found: $hasAfterAllScripts" + + Set-GitHubOutput -Name HasBeforeAllScripts -Value $hasBeforeAllScripts.ToString().ToLower() + Set-GitHubOutput -Name HasAfterAllScripts -Value $hasAfterAllScripts.ToString().ToLower() + } diff --git a/.github/workflows/Test-ModuleLocal.yml b/.github/workflows/Test-ModuleLocal.yml index 71d32434..9ea51de5 100644 --- a/.github/workflows/Test-ModuleLocal.yml +++ b/.github/workflows/Test-ModuleLocal.yml @@ -104,28 +104,6 @@ jobs: "name=$name" >> $env:GITHUB_OUTPUT "path=$path" >> $env:GITHUB_OUTPUT - - name: Run BeforeAll Setup Script - shell: pwsh - working-directory: ${{ inputs.WorkingDirectory }} - env: - TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} - TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} - TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} - TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} - TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} - TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} - TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} - GITHUB_TOKEN: ${{ github.token }} - run: | - $testPath = '${{ inputs.TestPath }}' - $beforeAllScript = Join-Path $testPath "BeforeAll.ps1" - if (Test-Path $beforeAllScript) { - Write-Host "Running BeforeAll setup script: $beforeAllScript" - & $beforeAllScript - } else { - Write-Host "No BeforeAll.ps1 script found in $testPath" - } - - name: Test-ModuleLocal uses: PSModule/Invoke-Pester@v4 env: @@ -155,26 +133,3 @@ jobs: Prescript: | # This is to speed up module loading in Pester. Install-PSResource -Repository PSGallery -TrustRepository -Name PSCustomObject Import-Module -Name '${{ steps.import-module.outputs.name }}' -RequiredVersion 999.0.0 - - - name: Run AfterAll Teardown Script - if: always() - shell: pwsh - working-directory: ${{ inputs.WorkingDirectory }} - env: - TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} - TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} - TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} - TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} - TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} - TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} - TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} - GITHUB_TOKEN: ${{ github.token }} - run: | - $testPath = '${{ inputs.TestPath }}' - $afterAllScript = Join-Path $testPath "AfterAll.ps1" - if (Test-Path $afterAllScript) { - Write-Host "Running AfterAll teardown script: $afterAllScript" - & $afterAllScript - } else { - Write-Host "No AfterAll.ps1 script found in $testPath" - } diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index c2ade8ee..198059cc 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -178,11 +178,27 @@ jobs: Version: ${{ inputs.Version }} WorkingDirectory: ${{ inputs.WorkingDirectory }} + BeforeAll-ModuleLocal: + if: ${{ needs.Build-Module.result == 'success' && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} + needs: + - Build-Module + - Get-Settings + uses: ./.github/workflows/BeforeAll-ModuleLocal.yml + secrets: inherit + with: + Name: ${{ fromJson(needs.Get-Settings.outputs.Settings).Name }} + Debug: ${{ inputs.Debug }} + Prerelease: ${{ inputs.Prerelease }} + Verbose: ${{ inputs.Verbose }} + Version: ${{ inputs.Version }} + WorkingDirectory: ${{ inputs.WorkingDirectory }} + Test-ModuleLocal: if: ${{ needs.Build-Module.result == 'success' && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} needs: - Build-Module - Get-Settings + - BeforeAll-ModuleLocal strategy: fail-fast: false matrix: @@ -201,6 +217,21 @@ jobs: Version: ${{ inputs.Version }} WorkingDirectory: ${{ inputs.WorkingDirectory }} + AfterAll-ModuleLocal: + if: ${{ always() && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} + needs: + - Get-Settings + - Test-ModuleLocal + uses: ./.github/workflows/AfterAll-ModuleLocal.yml + secrets: inherit + with: + Name: ${{ fromJson(needs.Get-Settings.outputs.Settings).Name }} + Debug: ${{ inputs.Debug }} + Prerelease: ${{ inputs.Prerelease }} + Verbose: ${{ inputs.Verbose }} + Version: ${{ inputs.Version }} + WorkingDirectory: ${{ inputs.WorkingDirectory }} + Get-TestResults: if: needs.Get-Settings.result == 'success' && !fromJson(needs.Get-Settings.outputs.Settings).Test.TestResults.Skip && (needs.Get-Settings.outputs.SourceCodeTestSuites != '[]' || needs.Get-Settings.outputs.PSModuleTestSuites != '[]' || needs.Get-Settings.outputs.ModuleTestSuites != '[]') && (always() && !cancelled()) needs: @@ -209,6 +240,7 @@ jobs: - Lint-SourceCode - Test-Module - Test-ModuleLocal + - AfterAll-ModuleLocal uses: ./.github/workflows/Get-TestResults.yml secrets: inherit with: @@ -226,6 +258,7 @@ jobs: - Get-Settings - Test-Module - Test-ModuleLocal + - AfterAll-ModuleLocal uses: ./.github/workflows/Get-CodeCoverage.yml secrets: inherit with: From 87ac2eb8cfbde046d8155d7fb4e260fd10b962fc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 11 Sep 2025 23:57:13 +0000 Subject: [PATCH 08/50] Update documentation to reflect separate BeforeAll/AfterAll job architecture Co-authored-by: MariusStorhaug <17722253+MariusStorhaug@users.noreply.github.com> --- README.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3b92d32e..e8fcdf3c 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,9 @@ Depending on the labels in the pull requests, the workflow will result in differ - This produces a json based report that is used to later evaluate the results of the tests. - [Test module](./.github/workflows/Test-ModuleLocal.yml) - Import and tests the module in parallel (matrix) using Pester tests from the module repository. - - Supports setup and teardown scripts: - - **BeforeAll.ps1**: Runs before each test execution to set up test environment (e.g., deploy infrastructure, download test data) - - **AfterAll.ps1**: Runs after each test execution to clean up test environment (e.g., remove test resources, cleanup databases) + - Supports setup and teardown scripts executed via separate dedicated jobs: + - **BeforeAll.ps1**: Runs once before all test matrix jobs to set up test environment (e.g., deploy infrastructure, download test data) + - **AfterAll.ps1**: Runs once after all test matrix jobs complete to clean up test environment (e.g., remove test resources, cleanup databases) - Setup/teardown scripts are automatically detected in test directories and executed with the same environment variables as tests - This produces a json based report that is used to later evaluate the results of the tests. - [Get test results](./.github/workflows/Get-TestResults.yml) @@ -267,13 +267,13 @@ The workflow supports automatic execution of setup and teardown scripts for modu #### BeforeAll.ps1 - **Location**: Place in your test directories (e.g., `tests/BeforeAll.ps1`) -- **Purpose**: Runs before each test execution to prepare the test environment +- **Purpose**: Runs once before all test matrix jobs to prepare the test environment - **Use cases**: Deploy test infrastructure, download test data, initialize databases, configure services - **Environment**: Has access to the same environment variables as your tests (secrets, GitHub token, etc.) #### AfterAll.ps1 - **Location**: Place in your test directories (e.g., `tests/AfterAll.ps1`) -- **Purpose**: Runs after each test execution to clean up the test environment +- **Purpose**: Runs once after all test matrix jobs complete to clean up the test environment - **Use cases**: Remove test resources, cleanup databases, stop services, upload artifacts - **Environment**: Has access to the same environment variables as your tests @@ -301,6 +301,17 @@ Write-Host "Cleanup completed!" - Scripts run with PowerShell and have access to PSModuleHelpers - If no scripts are found, the workflow continues normally +**Execution Order:** +The workflow executes setup and teardown scripts using separate dedicated jobs with the following order: +``` +BeforeAll-ModuleLocal → Test-ModuleLocal (matrix) → AfterAll-ModuleLocal → Get-TestResults/Get-CodeCoverage +``` + +- **BeforeAll-ModuleLocal**: Runs once before all test matrix jobs +- **Test-ModuleLocal**: Runs tests in parallel matrix configuration +- **AfterAll-ModuleLocal**: Runs once after all test matrix jobs complete (always executes for cleanup, even if tests fail) +- **Get-TestResults/Get-CodeCoverage**: Process results after cleanup is complete + ### Secrets The following secrets are used by the workflow. They can be automatically provided (if available) by setting the `secrets: inherit` From f6fef54262260758f48b4129f512430a8b70f082 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 1 Oct 2025 07:14:26 +0000 Subject: [PATCH 09/50] Move BeforeAll/AfterAll jobs into Test-ModuleLocal workflow and remove detection logic Co-authored-by: MariusStorhaug <17722253+MariusStorhaug@users.noreply.github.com> --- .github/workflows/AfterAll-ModuleLocal.yml | 152 ---------------- .github/workflows/BeforeAll-ModuleLocal.yml | 151 ---------------- .github/workflows/CI.yml | 33 ---- .github/workflows/Get-Settings.yml | 45 ----- .github/workflows/Test-ModuleLocal.yml | 188 ++++++++++++++++++++ .github/workflows/workflow.yml | 33 ---- 6 files changed, 188 insertions(+), 414 deletions(-) delete mode 100644 .github/workflows/AfterAll-ModuleLocal.yml delete mode 100644 .github/workflows/BeforeAll-ModuleLocal.yml diff --git a/.github/workflows/AfterAll-ModuleLocal.yml b/.github/workflows/AfterAll-ModuleLocal.yml deleted file mode 100644 index 1d1f006a..00000000 --- a/.github/workflows/AfterAll-ModuleLocal.yml +++ /dev/null @@ -1,152 +0,0 @@ -name: AfterAll-ModuleLocal - -on: - workflow_call: - secrets: - TEST_APP_ENT_CLIENT_ID: - description: The client ID of an Enterprise GitHub App for running tests. - required: false - TEST_APP_ENT_PRIVATE_KEY: - description: The private key of an Enterprise GitHub App for running tests. - required: false - TEST_APP_ORG_CLIENT_ID: - description: The client ID of an Organization GitHub App for running tests. - required: false - TEST_APP_ORG_PRIVATE_KEY: - description: The private key of an Organization GitHub App for running tests. - required: false - TEST_USER_ORG_FG_PAT: - description: The fine-grained personal access token with org access for running tests. - required: false - TEST_USER_USER_FG_PAT: - description: The fine-grained personal access token with user account access for running tests. - required: false - TEST_USER_PAT: - description: The classic personal access token for running tests. - required: false - inputs: - Name: - type: string - description: The name of the module to process. Scripts default to the repository name if nothing is specified. - required: false - Debug: - type: boolean - description: Enable debug output. - required: false - default: false - Verbose: - type: boolean - description: Enable verbose output. - required: false - default: false - Version: - type: string - description: Specifies the version of the GitHub module to be installed. The value must be an exact version. - required: false - default: '' - Prerelease: - type: boolean - description: Whether to use a prerelease version of the 'GitHub' module. - required: false - default: false - WorkingDirectory: - type: string - description: The working directory where the script will run from. - required: false - default: '.' - -permissions: - contents: read # to checkout the repo - -jobs: - AfterAll-ModuleLocal: - name: AfterAll-ModuleLocal - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v5 - - - name: Install-PSModuleHelpers - uses: PSModule/Install-PSModuleHelpers@v1 - - - name: Run AfterAll Teardown Scripts - if: always() - uses: PSModule/GitHub-Script@v1 - env: - TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} - TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} - TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} - TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} - TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} - TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} - TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} - GITHUB_TOKEN: ${{ github.token }} - with: - Name: AfterAll-ModuleLocal - ShowInfo: false - ShowOutput: true - Debug: ${{ inputs.Debug }} - Prerelease: ${{ inputs.Prerelease }} - Verbose: ${{ inputs.Verbose }} - Version: ${{ inputs.Version }} - WorkingDirectory: ${{ inputs.WorkingDirectory }} - Script: | - LogGroup "Running AfterAll Teardown Scripts" { - function Find-TestDirectories { - param ([string]$Path) - - $directories = @() - $childDirs = Get-ChildItem -Path $Path -Directory - - foreach ($dir in $childDirs) { - $directories += $dir.FullName - $directories += Find-TestDirectories -Path $dir.FullName - } - - return $directories - } - - # Locate the tests directory. - $testsPath = Resolve-Path 'tests' -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path - if (-not $testsPath) { - Write-Warning 'No tests directory found' - exit 0 - } - Write-Host "Tests found at [$testsPath]" - - $allTestFolders = @($testsPath) + (Find-TestDirectories -Path $testsPath) - $processedDirectories = @() - - foreach ($folder in $allTestFolders) { - $afterAllScript = Join-Path $folder "AfterAll.ps1" - - if (Test-Path $afterAllScript -PathType Leaf) { - # Get unique directory path to avoid duplicate execution - $uniqueDirectory = Resolve-Path $folder -Relative - if ($processedDirectories -notcontains $uniqueDirectory) { - $processedDirectories += $uniqueDirectory - - Write-Host "Running AfterAll teardown script: $afterAllScript" - try { - Push-Location $folder - & $afterAllScript - Write-Host "AfterAll script completed successfully: $afterAllScript" - } - catch { - Write-Warning "AfterAll script failed: $afterAllScript - $_" - # Don't throw for teardown scripts to ensure other cleanup scripts can run - } - finally { - Pop-Location - } - } - } - } - - if ($processedDirectories.Count -eq 0) { - Write-Host "No AfterAll.ps1 scripts found in test directories" - } else { - Write-Host "Processed AfterAll scripts in $($processedDirectories.Count) directories:" - $processedDirectories | ForEach-Object { Write-Host " - $_" } - } - } \ No newline at end of file diff --git a/.github/workflows/BeforeAll-ModuleLocal.yml b/.github/workflows/BeforeAll-ModuleLocal.yml deleted file mode 100644 index ea212acc..00000000 --- a/.github/workflows/BeforeAll-ModuleLocal.yml +++ /dev/null @@ -1,151 +0,0 @@ -name: BeforeAll-ModuleLocal - -on: - workflow_call: - secrets: - TEST_APP_ENT_CLIENT_ID: - description: The client ID of an Enterprise GitHub App for running tests. - required: false - TEST_APP_ENT_PRIVATE_KEY: - description: The private key of an Enterprise GitHub App for running tests. - required: false - TEST_APP_ORG_CLIENT_ID: - description: The client ID of an Organization GitHub App for running tests. - required: false - TEST_APP_ORG_PRIVATE_KEY: - description: The private key of an Organization GitHub App for running tests. - required: false - TEST_USER_ORG_FG_PAT: - description: The fine-grained personal access token with org access for running tests. - required: false - TEST_USER_USER_FG_PAT: - description: The fine-grained personal access token with user account access for running tests. - required: false - TEST_USER_PAT: - description: The classic personal access token for running tests. - required: false - inputs: - Name: - type: string - description: The name of the module to process. Scripts default to the repository name if nothing is specified. - required: false - Debug: - type: boolean - description: Enable debug output. - required: false - default: false - Verbose: - type: boolean - description: Enable verbose output. - required: false - default: false - Version: - type: string - description: Specifies the version of the GitHub module to be installed. The value must be an exact version. - required: false - default: '' - Prerelease: - type: boolean - description: Whether to use a prerelease version of the 'GitHub' module. - required: false - default: false - WorkingDirectory: - type: string - description: The working directory where the script will run from. - required: false - default: '.' - -permissions: - contents: read # to checkout the repo - -jobs: - BeforeAll-ModuleLocal: - name: BeforeAll-ModuleLocal - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v5 - - - name: Install-PSModuleHelpers - uses: PSModule/Install-PSModuleHelpers@v1 - - - name: Run BeforeAll Setup Scripts - uses: PSModule/GitHub-Script@v1 - env: - TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} - TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} - TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} - TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} - TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} - TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} - TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} - GITHUB_TOKEN: ${{ github.token }} - with: - Name: BeforeAll-ModuleLocal - ShowInfo: false - ShowOutput: true - Debug: ${{ inputs.Debug }} - Prerelease: ${{ inputs.Prerelease }} - Verbose: ${{ inputs.Verbose }} - Version: ${{ inputs.Version }} - WorkingDirectory: ${{ inputs.WorkingDirectory }} - Script: | - LogGroup "Running BeforeAll Setup Scripts" { - function Find-TestDirectories { - param ([string]$Path) - - $directories = @() - $childDirs = Get-ChildItem -Path $Path -Directory - - foreach ($dir in $childDirs) { - $directories += $dir.FullName - $directories += Find-TestDirectories -Path $dir.FullName - } - - return $directories - } - - # Locate the tests directory. - $testsPath = Resolve-Path 'tests' -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path - if (-not $testsPath) { - Write-Warning 'No tests directory found' - exit 0 - } - Write-Host "Tests found at [$testsPath]" - - $allTestFolders = @($testsPath) + (Find-TestDirectories -Path $testsPath) - $processedDirectories = @() - - foreach ($folder in $allTestFolders) { - $beforeAllScript = Join-Path $folder "BeforeAll.ps1" - - if (Test-Path $beforeAllScript -PathType Leaf) { - # Get unique directory path to avoid duplicate execution - $uniqueDirectory = Resolve-Path $folder -Relative - if ($processedDirectories -notcontains $uniqueDirectory) { - $processedDirectories += $uniqueDirectory - - Write-Host "Running BeforeAll setup script: $beforeAllScript" - try { - Push-Location $folder - & $beforeAllScript - Write-Host "BeforeAll script completed successfully: $beforeAllScript" - } - catch { - Write-Error "BeforeAll script failed: $beforeAllScript - $_" - throw - } - finally { - Pop-Location - } - } - } - } - - if ($processedDirectories.Count -eq 0) { - Write-Host "No BeforeAll.ps1 scripts found in test directories" - } else { - Write-Host "Processed BeforeAll scripts in $($processedDirectories.Count) directories:" - $processedDirectories | ForEach-Object { Write-Host " - $_" } - } - } \ No newline at end of file diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index aef6ee1d..30ab8859 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -176,27 +176,11 @@ jobs: Version: ${{ inputs.Version }} WorkingDirectory: ${{ inputs.WorkingDirectory }} - BeforeAll-ModuleLocal: - if: ${{ needs.Build-Module.result == 'success' && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} - needs: - - Build-Module - - Get-Settings - uses: ./.github/workflows/BeforeAll-ModuleLocal.yml - secrets: inherit - with: - Name: ${{ fromJson(needs.Get-Settings.outputs.Settings).Name }} - Debug: ${{ inputs.Debug }} - Prerelease: ${{ inputs.Prerelease }} - Verbose: ${{ inputs.Verbose }} - Version: ${{ inputs.Version }} - WorkingDirectory: ${{ inputs.WorkingDirectory }} - Test-ModuleLocal: if: ${{ needs.Build-Module.result == 'success' && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} needs: - Build-Module - Get-Settings - - BeforeAll-ModuleLocal strategy: fail-fast: false matrix: @@ -215,21 +199,6 @@ jobs: Version: ${{ inputs.Version }} WorkingDirectory: ${{ inputs.WorkingDirectory }} - AfterAll-ModuleLocal: - if: ${{ always() && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} - needs: - - Get-Settings - - Test-ModuleLocal - uses: ./.github/workflows/AfterAll-ModuleLocal.yml - secrets: inherit - with: - Name: ${{ fromJson(needs.Get-Settings.outputs.Settings).Name }} - Debug: ${{ inputs.Debug }} - Prerelease: ${{ inputs.Prerelease }} - Verbose: ${{ inputs.Verbose }} - Version: ${{ inputs.Version }} - WorkingDirectory: ${{ inputs.WorkingDirectory }} - Get-TestResults: if: needs.Get-Settings.result == 'success' && !fromJson(needs.Get-Settings.outputs.Settings).Test.TestResults.Skip && (needs.Get-Settings.outputs.SourceCodeTestSuites != '[]' || needs.Get-Settings.outputs.PSModuleTestSuites != '[]' || needs.Get-Settings.outputs.ModuleTestSuites != '[]') && (always() && !cancelled()) needs: @@ -238,7 +207,6 @@ jobs: - Lint-SourceCode - Test-Module - Test-ModuleLocal - - AfterAll-ModuleLocal uses: ./.github/workflows/Get-TestResults.yml secrets: inherit with: @@ -256,7 +224,6 @@ jobs: - Get-Settings - Test-Module - Test-ModuleLocal - - AfterAll-ModuleLocal uses: ./.github/workflows/Get-CodeCoverage.yml secrets: inherit with: diff --git a/.github/workflows/Get-Settings.yml b/.github/workflows/Get-Settings.yml index 28b93905..1c79b4e5 100644 --- a/.github/workflows/Get-Settings.yml +++ b/.github/workflows/Get-Settings.yml @@ -50,12 +50,6 @@ on: ModuleTestSuites: description: Module local test suites to run. value: ${{ jobs.Get-Settings.outputs.ModuleTestSuites }} - HasBeforeAllScripts: - description: Whether BeforeAll.ps1 scripts are found in test directories. - value: ${{ jobs.Get-Settings.outputs.HasBeforeAllScripts }} - HasAfterAllScripts: - description: Whether AfterAll.ps1 scripts are found in test directories. - value: ${{ jobs.Get-Settings.outputs.HasAfterAllScripts }} permissions: contents: read # to checkout the repo @@ -69,8 +63,6 @@ jobs: SourceCodeTestSuites: ${{ fromJson(steps.Get-Settings.outputs.result).SourceCodeTestSuites }} PSModuleTestSuites: ${{ fromJson(steps.Get-Settings.outputs.result).PSModuleTestSuites }} ModuleTestSuites: ${{ fromJson(steps.Get-Settings.outputs.result).ModuleTestSuites }} - HasBeforeAllScripts: ${{ fromJson(steps.Get-Settings.outputs.result).HasBeforeAllScripts }} - HasAfterAllScripts: ${{ fromJson(steps.Get-Settings.outputs.result).HasAfterAllScripts }} steps: - name: Checkout Code uses: actions/checkout@v5 @@ -372,40 +364,3 @@ jobs: $moduleTestSuites = ($null -ne $moduleTestSuites) ? ($moduleTestSuites | ConvertTo-Json -AsArray) : '[]' Set-GitHubOutput -Name ModuleTestSuites -Value $moduleTestSuites } - - # Check for BeforeAll.ps1 and AfterAll.ps1 scripts - LogGroup 'Setup/Teardown Scripts Detection:' { - $hasBeforeAllScripts = $false - $hasAfterAllScripts = $false - - if (-not $settings.Test.Module.Skip) { - # Locate the tests directory. - $testsPath = Resolve-Path 'tests' -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path - if ($testsPath) { - Write-Host "Checking for setup/teardown scripts in [$testsPath]" - - $allTestFolders = @($testsPath) + (Find-TestDirectories -Path $testsPath) - - foreach ($folder in $allTestFolders) { - $beforeAllScript = Join-Path $folder "BeforeAll.ps1" - $afterAllScript = Join-Path $folder "AfterAll.ps1" - - if (Test-Path $beforeAllScript) { - Write-Host "Found BeforeAll.ps1 in: $folder" - $hasBeforeAllScripts = $true - } - - if (Test-Path $afterAllScript) { - Write-Host "Found AfterAll.ps1 in: $folder" - $hasAfterAllScripts = $true - } - } - } - } - - Write-Host "BeforeAll scripts found: $hasBeforeAllScripts" - Write-Host "AfterAll scripts found: $hasAfterAllScripts" - - Set-GitHubOutput -Name HasBeforeAllScripts -Value $hasBeforeAllScripts.ToString().ToLower() - Set-GitHubOutput -Name HasAfterAllScripts -Value $hasAfterAllScripts.ToString().ToLower() - } diff --git a/.github/workflows/Test-ModuleLocal.yml b/.github/workflows/Test-ModuleLocal.yml index 9ea51de5..83bf176d 100644 --- a/.github/workflows/Test-ModuleLocal.yml +++ b/.github/workflows/Test-ModuleLocal.yml @@ -77,9 +77,102 @@ permissions: contents: read # to checkout the repo and create releases on the repo jobs: + BeforeAll-ModuleLocal: + name: BeforeAll-ModuleLocal + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v5 + + - name: Install-PSModuleHelpers + uses: PSModule/Install-PSModuleHelpers@v1 + + - name: Run BeforeAll Setup Scripts + uses: PSModule/GitHub-Script@v1 + env: + TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} + TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} + TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} + TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} + TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} + TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} + TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} + GITHUB_TOKEN: ${{ github.token }} + with: + Name: BeforeAll-ModuleLocal + ShowInfo: false + ShowOutput: true + Debug: ${{ inputs.Debug }} + Prerelease: ${{ inputs.Prerelease }} + Verbose: ${{ inputs.Verbose }} + Version: ${{ inputs.Version }} + WorkingDirectory: ${{ inputs.WorkingDirectory }} + Script: | + LogGroup "Running BeforeAll Setup Scripts" { + function Find-TestDirectories { + param ([string]$Path) + + $directories = @() + $childDirs = Get-ChildItem -Path $Path -Directory + + foreach ($dir in $childDirs) { + $directories += $dir.FullName + $directories += Find-TestDirectories -Path $dir.FullName + } + + return $directories + } + + # Locate the tests directory. + $testsPath = Resolve-Path 'tests' -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path + if (-not $testsPath) { + Write-Host 'No tests directory found - exiting successfully' + exit 0 + } + Write-Host "Tests found at [$testsPath]" + + $allTestFolders = @($testsPath) + (Find-TestDirectories -Path $testsPath) + $processedDirectories = @() + + foreach ($folder in $allTestFolders) { + $beforeAllScript = Join-Path $folder "BeforeAll.ps1" + + if (Test-Path $beforeAllScript -PathType Leaf) { + # Get unique directory path to avoid duplicate execution + $uniqueDirectory = Resolve-Path $folder -Relative + if ($processedDirectories -notcontains $uniqueDirectory) { + $processedDirectories += $uniqueDirectory + + Write-Host "Running BeforeAll setup script: $beforeAllScript" + try { + Push-Location $folder + & $beforeAllScript + Write-Host "BeforeAll script completed successfully: $beforeAllScript" + } + catch { + Write-Error "BeforeAll script failed: $beforeAllScript - $_" + throw + } + finally { + Pop-Location + } + } + } + } + + if ($processedDirectories.Count -eq 0) { + Write-Host "No BeforeAll.ps1 scripts found in test directories - exiting successfully" + } else { + Write-Host "Processed BeforeAll scripts in $($processedDirectories.Count) directories:" + $processedDirectories | ForEach-Object { Write-Host " - $_" } + } + } + Test-ModuleLocal: name: Test-${{ inputs.TestName }} (${{ inputs.RunsOn }}) runs-on: ${{ inputs.RunsOn }} + needs: + - BeforeAll-ModuleLocal steps: - name: Checkout Code uses: actions/checkout@v5 @@ -133,3 +226,98 @@ jobs: Prescript: | # This is to speed up module loading in Pester. Install-PSResource -Repository PSGallery -TrustRepository -Name PSCustomObject Import-Module -Name '${{ steps.import-module.outputs.name }}' -RequiredVersion 999.0.0 + + AfterAll-ModuleLocal: + name: AfterAll-ModuleLocal + runs-on: ubuntu-latest + needs: + - Test-ModuleLocal + if: always() + steps: + - name: Checkout Code + uses: actions/checkout@v5 + + - name: Install-PSModuleHelpers + uses: PSModule/Install-PSModuleHelpers@v1 + + - name: Run AfterAll Teardown Scripts + if: always() + uses: PSModule/GitHub-Script@v1 + env: + TEST_APP_ENT_CLIENT_ID: ${{ secrets.TEST_APP_ENT_CLIENT_ID }} + TEST_APP_ENT_PRIVATE_KEY: ${{ secrets.TEST_APP_ENT_PRIVATE_KEY }} + TEST_APP_ORG_CLIENT_ID: ${{ secrets.TEST_APP_ORG_CLIENT_ID }} + TEST_APP_ORG_PRIVATE_KEY: ${{ secrets.TEST_APP_ORG_PRIVATE_KEY }} + TEST_USER_ORG_FG_PAT: ${{ secrets.TEST_USER_ORG_FG_PAT }} + TEST_USER_USER_FG_PAT: ${{ secrets.TEST_USER_USER_FG_PAT }} + TEST_USER_PAT: ${{ secrets.TEST_USER_PAT }} + GITHUB_TOKEN: ${{ github.token }} + with: + Name: AfterAll-ModuleLocal + ShowInfo: false + ShowOutput: true + Debug: ${{ inputs.Debug }} + Prerelease: ${{ inputs.Prerelease }} + Verbose: ${{ inputs.Verbose }} + Version: ${{ inputs.Version }} + WorkingDirectory: ${{ inputs.WorkingDirectory }} + Script: | + LogGroup "Running AfterAll Teardown Scripts" { + function Find-TestDirectories { + param ([string]$Path) + + $directories = @() + $childDirs = Get-ChildItem -Path $Path -Directory + + foreach ($dir in $childDirs) { + $directories += $dir.FullName + $directories += Find-TestDirectories -Path $dir.FullName + } + + return $directories + } + + # Locate the tests directory. + $testsPath = Resolve-Path 'tests' -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path + if (-not $testsPath) { + Write-Host 'No tests directory found - exiting successfully' + exit 0 + } + Write-Host "Tests found at [$testsPath]" + + $allTestFolders = @($testsPath) + (Find-TestDirectories -Path $testsPath) + $processedDirectories = @() + + foreach ($folder in $allTestFolders) { + $afterAllScript = Join-Path $folder "AfterAll.ps1" + + if (Test-Path $afterAllScript -PathType Leaf) { + # Get unique directory path to avoid duplicate execution + $uniqueDirectory = Resolve-Path $folder -Relative + if ($processedDirectories -notcontains $uniqueDirectory) { + $processedDirectories += $uniqueDirectory + + Write-Host "Running AfterAll teardown script: $afterAllScript" + try { + Push-Location $folder + & $afterAllScript + Write-Host "AfterAll script completed successfully: $afterAllScript" + } + catch { + Write-Warning "AfterAll script failed: $afterAllScript - $_" + # Don't throw for teardown scripts to ensure other cleanup scripts can run + } + finally { + Pop-Location + } + } + } + } + + if ($processedDirectories.Count -eq 0) { + Write-Host "No AfterAll.ps1 scripts found in test directories - exiting successfully" + } else { + Write-Host "Processed AfterAll scripts in $($processedDirectories.Count) directories:" + $processedDirectories | ForEach-Object { Write-Host " - $_" } + } + } diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 198059cc..c2ade8ee 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -178,27 +178,11 @@ jobs: Version: ${{ inputs.Version }} WorkingDirectory: ${{ inputs.WorkingDirectory }} - BeforeAll-ModuleLocal: - if: ${{ needs.Build-Module.result == 'success' && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} - needs: - - Build-Module - - Get-Settings - uses: ./.github/workflows/BeforeAll-ModuleLocal.yml - secrets: inherit - with: - Name: ${{ fromJson(needs.Get-Settings.outputs.Settings).Name }} - Debug: ${{ inputs.Debug }} - Prerelease: ${{ inputs.Prerelease }} - Verbose: ${{ inputs.Verbose }} - Version: ${{ inputs.Version }} - WorkingDirectory: ${{ inputs.WorkingDirectory }} - Test-ModuleLocal: if: ${{ needs.Build-Module.result == 'success' && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} needs: - Build-Module - Get-Settings - - BeforeAll-ModuleLocal strategy: fail-fast: false matrix: @@ -217,21 +201,6 @@ jobs: Version: ${{ inputs.Version }} WorkingDirectory: ${{ inputs.WorkingDirectory }} - AfterAll-ModuleLocal: - if: ${{ always() && !cancelled() && needs.Get-Settings.outputs.ModuleTestSuites != '[]' }} - needs: - - Get-Settings - - Test-ModuleLocal - uses: ./.github/workflows/AfterAll-ModuleLocal.yml - secrets: inherit - with: - Name: ${{ fromJson(needs.Get-Settings.outputs.Settings).Name }} - Debug: ${{ inputs.Debug }} - Prerelease: ${{ inputs.Prerelease }} - Verbose: ${{ inputs.Verbose }} - Version: ${{ inputs.Version }} - WorkingDirectory: ${{ inputs.WorkingDirectory }} - Get-TestResults: if: needs.Get-Settings.result == 'success' && !fromJson(needs.Get-Settings.outputs.Settings).Test.TestResults.Skip && (needs.Get-Settings.outputs.SourceCodeTestSuites != '[]' || needs.Get-Settings.outputs.PSModuleTestSuites != '[]' || needs.Get-Settings.outputs.ModuleTestSuites != '[]') && (always() && !cancelled()) needs: @@ -240,7 +209,6 @@ jobs: - Lint-SourceCode - Test-Module - Test-ModuleLocal - - AfterAll-ModuleLocal uses: ./.github/workflows/Get-TestResults.yml secrets: inherit with: @@ -258,7 +226,6 @@ jobs: - Get-Settings - Test-Module - Test-ModuleLocal - - AfterAll-ModuleLocal uses: ./.github/workflows/Get-CodeCoverage.yml secrets: inherit with: From 3e25e4cbb5ef21b3af84a174ae7e3ccc2dda04ae Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 1 Oct 2025 08:13:00 +0000 Subject: [PATCH 10/50] Move matrix strategy into Test-ModuleLocal workflow and update calling workflows Co-authored-by: MariusStorhaug <17722253+MariusStorhaug@users.noreply.github.com> --- .github/workflows/CI.yml | 9 +------- .github/workflows/Test-ModuleLocal.yml | 30 +++++++++----------------- .github/workflows/workflow.yml | 9 +------- 3 files changed, 12 insertions(+), 36 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 30ab8859..549358c4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -181,17 +181,10 @@ jobs: needs: - Build-Module - Get-Settings - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.Get-Settings.outputs.ModuleTestSuites) }} uses: ./.github/workflows/Test-ModuleLocal.yml secrets: inherit with: - RunsOn: ${{ matrix.RunsOn }} - OS: ${{ matrix.OSName }} - TestPath: ${{ matrix.TestPath }} - TestName: ${{ matrix.TestName }} + ModuleTestSuites: ${{ needs.Get-Settings.outputs.ModuleTestSuites }} Name: ${{ fromJson(needs.Get-Settings.outputs.Settings).Name }} Debug: ${{ inputs.Debug }} Prerelease: ${{ inputs.Prerelease }} diff --git a/.github/workflows/Test-ModuleLocal.yml b/.github/workflows/Test-ModuleLocal.yml index 83bf176d..a8e5f785 100644 --- a/.github/workflows/Test-ModuleLocal.yml +++ b/.github/workflows/Test-ModuleLocal.yml @@ -25,28 +25,14 @@ on: description: The classic personal access token for running tests. required: false inputs: - RunsOn: + ModuleTestSuites: type: string - description: The type of runner to use for the job. - required: true - OS: - type: string - description: The operating system name. + description: JSON array of module test suites to run (from Get-Settings output). required: true Name: type: string description: The name of the module to process. Scripts default to the repository name if nothing is specified. required: false - TestPath: - type: string - description: The path to the tests folder. - required: false - default: tests - TestName: - type: string - description: The path to the tests folder. - required: false - default: tests Debug: type: boolean description: Enable debug output. @@ -169,10 +155,14 @@ jobs: } Test-ModuleLocal: - name: Test-${{ inputs.TestName }} (${{ inputs.RunsOn }}) - runs-on: ${{ inputs.RunsOn }} + name: Test-${{ matrix.TestName }} (${{ matrix.OSName }}) + runs-on: ${{ matrix.RunsOn }} needs: - BeforeAll-ModuleLocal + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(inputs.ModuleTestSuites) }} steps: - name: Checkout Code uses: actions/checkout@v5 @@ -213,14 +203,14 @@ jobs: Prerelease: ${{ inputs.Prerelease }} Verbose: ${{ inputs.Verbose }} Version: ${{ inputs.Version }} - TestResult_TestSuiteName: ${{ inputs.TestName }}-${{ inputs.OS }} + TestResult_TestSuiteName: ${{ matrix.TestName }}-${{ matrix.OSName }} TestResult_Enabled: true CodeCoverage_Enabled: true Output_Verbosity: Detailed CodeCoverage_OutputFormat: JaCoCo CodeCoverage_CoveragePercentTarget: 0 Filter_ExcludeTag: Flaky - Path: ${{ inputs.TestPath }} + Path: ${{ matrix.TestPath }} Run_Path: ${{ steps.import-module.outputs.path }} WorkingDirectory: ${{ inputs.WorkingDirectory }} Prescript: | # This is to speed up module loading in Pester. diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index c2ade8ee..c325109b 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -183,17 +183,10 @@ jobs: needs: - Build-Module - Get-Settings - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.Get-Settings.outputs.ModuleTestSuites) }} uses: ./.github/workflows/Test-ModuleLocal.yml secrets: inherit with: - RunsOn: ${{ matrix.RunsOn }} - OS: ${{ matrix.OSName }} - TestPath: ${{ matrix.TestPath }} - TestName: ${{ matrix.TestName }} + ModuleTestSuites: ${{ needs.Get-Settings.outputs.ModuleTestSuites }} Name: ${{ fromJson(needs.Get-Settings.outputs.Settings).Name }} Debug: ${{ inputs.Debug }} Prerelease: ${{ inputs.Prerelease }} From 0ffc613c63bef6f21710dc0e77e337289741697d Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Wed, 1 Oct 2025 12:23:55 +0200 Subject: [PATCH 11/50] feat: Add PowerShell scripts for setup and agent context management - Implemented `setup-plan.ps1` to create an implementation plan for features, including JSON output option and template handling. - Developed `update-agent-context.ps1` to update agent context files based on `plan.md`, supporting multiple agent types and ensuring environment validation. - Created templates for agent files and implementation plans to standardize documentation and facilitate easier updates. - Introduced a comprehensive feature specification template to guide feature development and ensure clarity in requirements. - Added a task template to streamline task generation for feature implementation, ensuring proper sequencing and dependencies. --- .github/prompts/analyze.prompt.md | 101 ++++ .github/prompts/clarify.prompt.md | 158 +++++++ .github/prompts/constitution.prompt.md | 73 +++ .github/prompts/implement.prompt.md | 56 +++ .github/prompts/plan.prompt.md | 43 ++ .github/prompts/specify.prompt.md | 21 + .github/prompts/tasks.prompt.md | 62 +++ .specify/memory/constitution.md | 50 ++ .../powershell/check-prerequisites.ps1 | 148 ++++++ .specify/scripts/powershell/common.ps1 | 136 ++++++ .../scripts/powershell/create-new-feature.ps1 | 117 +++++ .specify/scripts/powershell/setup-plan.ps1 | 61 +++ .../powershell/update-agent-context.ps1 | 430 ++++++++++++++++++ .specify/templates/agent-file-template.md | 23 + .specify/templates/plan-template.md | 219 +++++++++ .specify/templates/spec-template.md | 116 +++++ .specify/templates/tasks-template.md | 127 ++++++ 17 files changed, 1941 insertions(+) create mode 100644 .github/prompts/analyze.prompt.md create mode 100644 .github/prompts/clarify.prompt.md create mode 100644 .github/prompts/constitution.prompt.md create mode 100644 .github/prompts/implement.prompt.md create mode 100644 .github/prompts/plan.prompt.md create mode 100644 .github/prompts/specify.prompt.md create mode 100644 .github/prompts/tasks.prompt.md create mode 100644 .specify/memory/constitution.md create mode 100644 .specify/scripts/powershell/check-prerequisites.ps1 create mode 100644 .specify/scripts/powershell/common.ps1 create mode 100644 .specify/scripts/powershell/create-new-feature.ps1 create mode 100644 .specify/scripts/powershell/setup-plan.ps1 create mode 100644 .specify/scripts/powershell/update-agent-context.ps1 create mode 100644 .specify/templates/agent-file-template.md create mode 100644 .specify/templates/plan-template.md create mode 100644 .specify/templates/spec-template.md create mode 100644 .specify/templates/tasks-template.md diff --git a/.github/prompts/analyze.prompt.md b/.github/prompts/analyze.prompt.md new file mode 100644 index 00000000..b9940175 --- /dev/null +++ b/.github/prompts/analyze.prompt.md @@ -0,0 +1,101 @@ +--- +description: Perform a non-destructive cross-artifact consistency and quality analysis across spec.md, plan.md, and tasks.md after task generation. +--- + +The user input to you can be provided directly by the agent or as a command argument - you **MUST** consider it before proceeding with the prompt (if not empty). + +User input: + +$ARGUMENTS + +Goal: Identify inconsistencies, duplications, ambiguities, and underspecified items across the three core artifacts (`spec.md`, `plan.md`, `tasks.md`) before implementation. This command MUST run only after `/tasks` has successfully produced a complete `tasks.md`. + +STRICTLY READ-ONLY: Do **not** modify any files. Output a structured analysis report. Offer an optional remediation plan (user must explicitly approve before any follow-up editing commands would be invoked manually). + +Constitution Authority: The project constitution (`.specify/memory/constitution.md`) is **non-negotiable** within this analysis scope. Constitution conflicts are automatically CRITICAL and require adjustment of the spec, plan, or tasks—not dilution, reinterpretation, or silent ignoring of the principle. If a principle itself needs to change, that must occur in a separate, explicit constitution update outside `/analyze`. + +Execution steps: + +1. Run `.specify/scripts/powershell/check-prerequisites.ps1 -Json -RequireTasks -IncludeTasks` once from repo root and parse JSON for FEATURE_DIR and AVAILABLE_DOCS. Derive absolute paths: + - SPEC = FEATURE_DIR/spec.md + - PLAN = FEATURE_DIR/plan.md + - TASKS = FEATURE_DIR/tasks.md + Abort with an error message if any required file is missing (instruct the user to run missing prerequisite command). + +2. Load artifacts: + - Parse spec.md sections: Overview/Context, Functional Requirements, Non-Functional Requirements, User Stories, Edge Cases (if present). + - Parse plan.md: Architecture/stack choices, Data Model references, Phases, Technical constraints. + - Parse tasks.md: Task IDs, descriptions, phase grouping, parallel markers [P], referenced file paths. + - Load constitution `.specify/memory/constitution.md` for principle validation. + +3. Build internal semantic models: + - Requirements inventory: Each functional + non-functional requirement with a stable key (derive slug based on imperative phrase; e.g., "User can upload file" -> `user-can-upload-file`). + - User story/action inventory. + - Task coverage mapping: Map each task to one or more requirements or stories (inference by keyword / explicit reference patterns like IDs or key phrases). + - Constitution rule set: Extract principle names and any MUST/SHOULD normative statements. + +4. Detection passes: + A. Duplication detection: + - Identify near-duplicate requirements. Mark lower-quality phrasing for consolidation. + B. Ambiguity detection: + - Flag vague adjectives (fast, scalable, secure, intuitive, robust) lacking measurable criteria. + - Flag unresolved placeholders (TODO, TKTK, ???, , etc.). + C. Underspecification: + - Requirements with verbs but missing object or measurable outcome. + - User stories missing acceptance criteria alignment. + - Tasks referencing files or components not defined in spec/plan. + D. Constitution alignment: + - Any requirement or plan element conflicting with a MUST principle. + - Missing mandated sections or quality gates from constitution. + E. Coverage gaps: + - Requirements with zero associated tasks. + - Tasks with no mapped requirement/story. + - Non-functional requirements not reflected in tasks (e.g., performance, security). + F. Inconsistency: + - Terminology drift (same concept named differently across files). + - Data entities referenced in plan but absent in spec (or vice versa). + - Task ordering contradictions (e.g., integration tasks before foundational setup tasks without dependency note). + - Conflicting requirements (e.g., one requires to use Next.js while other says to use Vue as the framework). + +5. Severity assignment heuristic: + - CRITICAL: Violates constitution MUST, missing core spec artifact, or requirement with zero coverage that blocks baseline functionality. + - HIGH: Duplicate or conflicting requirement, ambiguous security/performance attribute, untestable acceptance criterion. + - MEDIUM: Terminology drift, missing non-functional task coverage, underspecified edge case. + - LOW: Style/wording improvements, minor redundancy not affecting execution order. + +6. Produce a Markdown report (no file writes) with sections: + + ### Specification Analysis Report + | ID | Category | Severity | Location(s) | Summary | Recommendation | + |----|----------|----------|-------------|---------|----------------| + | A1 | Duplication | HIGH | spec.md:L120-134 | Two similar requirements ... | Merge phrasing; keep clearer version | + (Add one row per finding; generate stable IDs prefixed by category initial.) + + Additional subsections: + - Coverage Summary Table: + | Requirement Key | Has Task? | Task IDs | Notes | + - Constitution Alignment Issues (if any) + - Unmapped Tasks (if any) + - Metrics: + * Total Requirements + * Total Tasks + * Coverage % (requirements with >=1 task) + * Ambiguity Count + * Duplication Count + * Critical Issues Count + +7. At end of report, output a concise Next Actions block: + - If CRITICAL issues exist: Recommend resolving before `/implement`. + - If only LOW/MEDIUM: User may proceed, but provide improvement suggestions. + - Provide explicit command suggestions: e.g., "Run /specify with refinement", "Run /plan to adjust architecture", "Manually edit tasks.md to add coverage for 'performance-metrics'". + +8. Ask the user: "Would you like me to suggest concrete remediation edits for the top N issues?" (Do NOT apply them automatically.) + +Behavior rules: +- NEVER modify files. +- NEVER hallucinate missing sections—if absent, report them. +- KEEP findings deterministic: if rerun without changes, produce consistent IDs and counts. +- LIMIT total findings in the main table to 50; aggregate remainder in a summarized overflow note. +- If zero issues found, emit a success report with coverage statistics and proceed recommendation. + +Context: $ARGUMENTS diff --git a/.github/prompts/clarify.prompt.md b/.github/prompts/clarify.prompt.md new file mode 100644 index 00000000..25ec9561 --- /dev/null +++ b/.github/prompts/clarify.prompt.md @@ -0,0 +1,158 @@ +--- +description: Identify underspecified areas in the current feature spec by asking up to 5 highly targeted clarification questions and encoding answers back into the spec. +--- + +The user input to you can be provided directly by the agent or as a command argument - you **MUST** consider it before proceeding with the prompt (if not empty). + +User input: + +$ARGUMENTS + +Goal: Detect and reduce ambiguity or missing decision points in the active feature specification and record the clarifications directly in the spec file. + +Note: This clarification workflow is expected to run (and be completed) BEFORE invoking `/plan`. If the user explicitly states they are skipping clarification (e.g., exploratory spike), you may proceed, but must warn that downstream rework risk increases. + +Execution steps: + +1. Run `.specify/scripts/powershell/check-prerequisites.ps1 -Json -PathsOnly` from repo root **once** (combined `--json --paths-only` mode / `-Json -PathsOnly`). Parse minimal JSON payload fields: + - `FEATURE_DIR` + - `FEATURE_SPEC` + - (Optionally capture `IMPL_PLAN`, `TASKS` for future chained flows.) + - If JSON parsing fails, abort and instruct user to re-run `/specify` or verify feature branch environment. + +2. Load the current spec file. Perform a structured ambiguity & coverage scan using this taxonomy. For each category, mark status: Clear / Partial / Missing. Produce an internal coverage map used for prioritization (do not output raw map unless no questions will be asked). + + Functional Scope & Behavior: + - Core user goals & success criteria + - Explicit out-of-scope declarations + - User roles / personas differentiation + + Domain & Data Model: + - Entities, attributes, relationships + - Identity & uniqueness rules + - Lifecycle/state transitions + - Data volume / scale assumptions + + Interaction & UX Flow: + - Critical user journeys / sequences + - Error/empty/loading states + - Accessibility or localization notes + + Non-Functional Quality Attributes: + - Performance (latency, throughput targets) + - Scalability (horizontal/vertical, limits) + - Reliability & availability (uptime, recovery expectations) + - Observability (logging, metrics, tracing signals) + - Security & privacy (authN/Z, data protection, threat assumptions) + - Compliance / regulatory constraints (if any) + + Integration & External Dependencies: + - External services/APIs and failure modes + - Data import/export formats + - Protocol/versioning assumptions + + Edge Cases & Failure Handling: + - Negative scenarios + - Rate limiting / throttling + - Conflict resolution (e.g., concurrent edits) + + Constraints & Tradeoffs: + - Technical constraints (language, storage, hosting) + - Explicit tradeoffs or rejected alternatives + + Terminology & Consistency: + - Canonical glossary terms + - Avoided synonyms / deprecated terms + + Completion Signals: + - Acceptance criteria testability + - Measurable Definition of Done style indicators + + Misc / Placeholders: + - TODO markers / unresolved decisions + - Ambiguous adjectives ("robust", "intuitive") lacking quantification + + For each category with Partial or Missing status, add a candidate question opportunity unless: + - Clarification would not materially change implementation or validation strategy + - Information is better deferred to planning phase (note internally) + +3. Generate (internally) a prioritized queue of candidate clarification questions (maximum 5). Do NOT output them all at once. Apply these constraints: + - Maximum of 5 total questions across the whole session. + - Each question must be answerable with EITHER: + * A short multiple‑choice selection (2–5 distinct, mutually exclusive options), OR + * A one-word / short‑phrase answer (explicitly constrain: "Answer in <=5 words"). + - Only include questions whose answers materially impact architecture, data modeling, task decomposition, test design, UX behavior, operational readiness, or compliance validation. + - Ensure category coverage balance: attempt to cover the highest impact unresolved categories first; avoid asking two low-impact questions when a single high-impact area (e.g., security posture) is unresolved. + - Exclude questions already answered, trivial stylistic preferences, or plan-level execution details (unless blocking correctness). + - Favor clarifications that reduce downstream rework risk or prevent misaligned acceptance tests. + - If more than 5 categories remain unresolved, select the top 5 by (Impact * Uncertainty) heuristic. + +4. Sequential questioning loop (interactive): + - Present EXACTLY ONE question at a time. + - For multiple‑choice questions render options as a Markdown table: + + | Option | Description | + |--------|-------------| + | A |