diff --git a/.azure/pipelines/identitymodel-helix-matrix.yml b/.azure/pipelines/identitymodel-helix-matrix.yml
index a302954c7619..1d1137fe08a1 100644
--- a/.azure/pipelines/identitymodel-helix-matrix.yml
+++ b/.azure/pipelines/identitymodel-helix-matrix.yml
@@ -49,46 +49,48 @@ extends:
           timeoutInMinutes: 300
           steps:
           - task: NuGetAuthenticate@1
+            inputs:
+              forceReinstallCredentialProvider: true
           - task: NuGetCommand@2
             displayName: Install Microsoft.IdentityModel.Logging
             inputs:
               command: 'custom'
               arguments: 'install Microsoft.IdentityModel.Logging
                           -Source https://pkgs.dev.azure.com/dnceng/internal/_packaging/identitymodel-nightlies/nuget/v3/index.json 
-                          -DependencyVersion Highest -PreRelease'
+                          -DependencyVersion Highest -OutputDirectory $(Build.StagingDirectory) -PreRelease'
           - task: NuGetCommand@2
-            inputs:
             displayName: Install Microsoft.IdentityModel.Protocols.OpenIdConnect
+            inputs:
               command: 'custom'
               arguments: 'install Microsoft.IdentityModel.Protocols.OpenIdConnect
                           -Source https://pkgs.dev.azure.com/dnceng/internal/_packaging/identitymodel-nightlies/nuget/v3/index.json 
-                          -DependencyVersion Highest -PreRelease'
+                          -DependencyVersion Highest -OutputDirectory $(Build.StagingDirectory) -PreRelease'
           - task: NuGetCommand@2
-            inputs:
             displayName: Install Microsoft.IdentityModel.Protocols.WsFederation
+            inputs:
               command: 'custom'
               arguments: 'install Microsoft.IdentityModel.Protocols.WsFederation
                           -Source https://pkgs.dev.azure.com/dnceng/internal/_packaging/identitymodel-nightlies/nuget/v3/index.json 
-                          -DependencyVersion Highest -PreRelease'
+                          -DependencyVersion Highest -OutputDirectory $(Build.StagingDirectory) -PreRelease'
           - task: NuGetCommand@2
-            inputs:
             displayName: System.IdentityModel.Tokens.Jwt
+            inputs:
               command: 'custom'
               arguments: 'install System.IdentityModel.Tokens.Jwt
                           -Source https://pkgs.dev.azure.com/dnceng/internal/_packaging/identitymodel-nightlies/nuget/v3/index.json 
-                          -DependencyVersion Highest -PreRelease'
+                          -DependencyVersion Highest -OutputDirectory $(Build.StagingDirectory) -PreRelease'
           - task: PowerShell@2
             displayName: Add IdentityModel feel to NuGet.config
             inputs:
               filePath: $(Build.SourcesDirectory)/eng/scripts/SetupIdentitySources.ps1
-              arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config
+              arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -IdentityModelPackageSource $(Build.StagingDirectory)
           # Build the shared framework
           - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -pack -arch x64
                     /p:CrossgenOutput=false /p:IsIdentityModelTestJob=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
             displayName: Build shared fx
           # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step.
           - script: .\eng\build.cmd -ci -prepareMachine -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test
-                    -projects eng\helix\helix.proj /p:IsHelixJob=true
+                    -projects eng\helix\helix.proj /p:IsHelixJob=true /p:RunTemplateTests=false
                     /p:CrossgenOutput=false /p:IsIdentityModelTestJob=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
             displayName: Run build.cmd helix target
             env:
diff --git a/Directory.Build.props b/Directory.Build.props
index 17b7a04db056..a727412840a4 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -121,6 +121,8 @@
     <!-- Ensure API docs are available. -->
     <NoWarn>$(NoWarn.Replace('1591', ''))</NoWarn>
     <NoWarn Condition=" '$(BuildingInsideVisualStudio)' == 'true' ">$(NoWarn);0105</NoWarn>
+    <!-- Allow prerelease dependencies in stable packages when we're running the IdentityModel nightly tests -->
+    <NoWarn Condition=" '$(IsIdentityModelTestJob)' == 'true' ">$(NoWarn);NU5104</NoWarn>
 
     <!-- For local builds, don't make missing XML docs a fatal build error, but still surface so we have visibility into undocumented APIs. -->
     <WarningsNotAsErrors Condition=" '$(ContinuousIntegrationBuild)' != 'true' ">$(WarningsNotAsErrors);CS1591</WarningsNotAsErrors>
diff --git a/eng/Versions.props b/eng/Versions.props
index 0caf0c3d49ab..3a76d50f3cb9 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -12,7 +12,7 @@
     <PreReleaseVersionIteration>5</PreReleaseVersionIteration>
     <ValidateBaseline>true</ValidateBaseline>
     <IdentityModelVersion Condition="'$(IsIdentityModelTestJob)' != 'true'">7.4.1</IdentityModelVersion>
-    <IdentityModelVersion Condition="'$(IsIdentityModelTestJob)' == 'true'">7.*</IdentityModelVersion>
+    <IdentityModelVersion Condition="'$(IsIdentityModelTestJob)' == 'true'">*-*</IdentityModelVersion>
     <!--
         When StabilizePackageVersion is set to 'true', this branch will produce stable outputs for 'Shipping' packages
     -->
diff --git a/eng/helix/helix.proj b/eng/helix/helix.proj
index 1806b2be72dd..f31e201d516e 100644
--- a/eng/helix/helix.proj
+++ b/eng/helix/helix.proj
@@ -21,7 +21,7 @@
   <!-- Microsoft.DotNet.Helix.Sdk.MultiQueue.targets splits $(HelixTargetQueues) into @(HelixTargetQueue) items. -->
   <ItemGroup Condition=" '$(HelixTargetQueues)' == '' ">
     <HelixTargetQueue Condition="'$(_UseHelixOpenQueues)' == 'true'" Include="@(HelixAvailableTargetQueue)" />
-    <HelixTargetQueue Condition="'$(_UseHelixOpenQueues)' != 'true'" Include="@(HelixAvailableTargetQueue->'%(Identity)'->Replace('.Open',''))" />
+    <HelixTargetQueue Condition="'$(_UseHelixOpenQueues)' != 'true'" Include="@(HelixAvailableTargetQueue->'%(Identity)'->Replace('.Open','')->Replace('.open',''))" />
   </ItemGroup>
 
   <PropertyGroup>
diff --git a/eng/scripts/SetupIdentitySources.ps1 b/eng/scripts/SetupIdentitySources.ps1
index 415bc17c644d..58a4e690d7b1 100644
--- a/eng/scripts/SetupIdentitySources.ps1
+++ b/eng/scripts/SetupIdentitySources.ps1
@@ -1,6 +1,7 @@
 [CmdletBinding()]
 param (
-    [Parameter(Mandatory = $true)][string]$ConfigFile
+    [Parameter(Mandatory = $true)][string]$ConfigFile,
+    [Parameter(Mandatory = $true)][string]$IdentityModelPackageSource
 )
 
 $ErrorActionPreference = "Stop"
@@ -40,6 +41,6 @@ if ($sources -eq $null) {
     $doc.DocumentElement.AppendChild($sources) | Out-Null
 }
 
-AddPackageSource -Sources $sources -SourceName "identitymodel-nightlies" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/identitymodel-nightlies/nuget/v3/index.json"
+AddPackageSource -Sources $sources -SourceName "identitymodel-nightlies" -SourceEndPoint $IdentityModelPackageSource
 
 $doc.Save($filename)
\ No newline at end of file
diff --git a/eng/targets/Helix.targets b/eng/targets/Helix.targets
index 7c1be4555643..43bf0cb7f7f1 100644
--- a/eng/targets/Helix.targets
+++ b/eng/targets/Helix.targets
@@ -141,7 +141,7 @@
     <ItemGroup Condition="'$(_UseHelixOpenQueues)' != 'true'">
       <_Temp Include="@(HelixAvailableTargetQueue)" />
       <HelixAvailableTargetQueue Remove="@(HelixAvailableTargetQueue)" />
-      <HelixAvailableTargetQueue Include="@(_Temp->'%(Identity)'->Replace('.Open',''))" />
+      <HelixAvailableTargetQueue Include="@(_Temp->'%(Identity)'->Replace('.Open','')->Replace('.open',''))" />
     </ItemGroup>
 
     <ItemGroup>
diff --git a/src/ProjectTemplates/test/Templates.Blazor.Tests/Templates.Blazor.Tests.csproj b/src/ProjectTemplates/test/Templates.Blazor.Tests/Templates.Blazor.Tests.csproj
index 368459e98c90..1db85d5286fe 100644
--- a/src/ProjectTemplates/test/Templates.Blazor.Tests/Templates.Blazor.Tests.csproj
+++ b/src/ProjectTemplates/test/Templates.Blazor.Tests/Templates.Blazor.Tests.csproj
@@ -9,6 +9,7 @@
     <RunTemplateTests Condition="'$(RunTemplateTests)' == ''">true</RunTemplateTests>
     <RunBlazorPlaywrightTemplateTests Condition="'$(RunBlazorPlaywrightTemplateTests)' == ''">$(RunTemplateTests)</RunBlazorPlaywrightTemplateTests>
     <SkipTests Condition="'$(RunBlazorPlaywrightTemplateTests)' != 'true'">true</SkipTests>
+    <BuildHelixPayload Condition="'$(RunBlazorPlaywrightTemplateTests)' != 'true'">false</BuildHelixPayload>
 
     <BaseOutputPath />
     <OutputPath />
diff --git a/src/ProjectTemplates/test/Templates.Blazor.WebAssembly.Auth.Tests/Templates.Blazor.WebAssembly.Auth.Tests.csproj b/src/ProjectTemplates/test/Templates.Blazor.WebAssembly.Auth.Tests/Templates.Blazor.WebAssembly.Auth.Tests.csproj
index 3754aee3abd4..3d0831a1001a 100644
--- a/src/ProjectTemplates/test/Templates.Blazor.WebAssembly.Auth.Tests/Templates.Blazor.WebAssembly.Auth.Tests.csproj
+++ b/src/ProjectTemplates/test/Templates.Blazor.WebAssembly.Auth.Tests/Templates.Blazor.WebAssembly.Auth.Tests.csproj
@@ -9,6 +9,7 @@
 
     <RunTemplateTests Condition="'$(RunTemplateTests)' == ''">true</RunTemplateTests>
     <SkipTests Condition="'$(RunTemplateTests)' != 'true'">true</SkipTests>
+    <BuildHelixPayload Condition="'$(RunTemplateTests)' != 'true'">false</BuildHelixPayload>
 
     <BaseOutputPath />
     <OutputPath />
diff --git a/src/ProjectTemplates/test/Templates.Blazor.WebAssembly.Tests/Templates.Blazor.WebAssembly.Tests.csproj b/src/ProjectTemplates/test/Templates.Blazor.WebAssembly.Tests/Templates.Blazor.WebAssembly.Tests.csproj
index 44348f90847a..b42866c3a78f 100644
--- a/src/ProjectTemplates/test/Templates.Blazor.WebAssembly.Tests/Templates.Blazor.WebAssembly.Tests.csproj
+++ b/src/ProjectTemplates/test/Templates.Blazor.WebAssembly.Tests/Templates.Blazor.WebAssembly.Tests.csproj
@@ -9,6 +9,7 @@
 
     <RunTemplateTests Condition="'$(RunTemplateTests)' == ''">true</RunTemplateTests>
     <SkipTests Condition="'$(RunTemplateTests)' != 'true'">true</SkipTests>
+    <BuildHelixPayload Condition="'$(RunTemplateTests)' != 'true'">false</BuildHelixPayload>
 
     <BaseOutputPath />
     <OutputPath />
diff --git a/src/ProjectTemplates/test/Templates.Mvc.Tests/Templates.Mvc.Tests.csproj b/src/ProjectTemplates/test/Templates.Mvc.Tests/Templates.Mvc.Tests.csproj
index f65dbaa59476..66aa65b43735 100644
--- a/src/ProjectTemplates/test/Templates.Mvc.Tests/Templates.Mvc.Tests.csproj
+++ b/src/ProjectTemplates/test/Templates.Mvc.Tests/Templates.Mvc.Tests.csproj
@@ -9,6 +9,7 @@
 
     <RunTemplateTests Condition="'$(RunTemplateTests)' == ''">true</RunTemplateTests>
     <SkipTests Condition="'$(RunTemplateTests)' != 'true'">true</SkipTests>
+    <BuildHelixPayload Condition="'$(RunTemplateTests)' != 'true'">false</BuildHelixPayload>
 
     <BaseOutputPath />
     <OutputPath />
diff --git a/src/ProjectTemplates/test/Templates.Tests/Templates.Tests.csproj b/src/ProjectTemplates/test/Templates.Tests/Templates.Tests.csproj
index d7c3c354d144..fb297bfae7c5 100644
--- a/src/ProjectTemplates/test/Templates.Tests/Templates.Tests.csproj
+++ b/src/ProjectTemplates/test/Templates.Tests/Templates.Tests.csproj
@@ -9,6 +9,7 @@
 
     <RunTemplateTests Condition="'$(RunTemplateTests)' == ''">true</RunTemplateTests>
     <SkipTests Condition="'$(RunTemplateTests)' != 'true'">true</SkipTests>
+    <BuildHelixPayload Condition="'$(RunTemplateTests)' != 'true'">false</BuildHelixPayload>
 
     <BaseOutputPath />
     <OutputPath />