diff --git a/com.unity.render-pipelines.core/Runtime/Textures/BufferedRTHandleSystem.cs b/com.unity.render-pipelines.core/Runtime/Textures/BufferedRTHandleSystem.cs
index b8e89f04c6b..ff40f3fe260 100644
--- a/com.unity.render-pipelines.core/Runtime/Textures/BufferedRTHandleSystem.cs
+++ b/com.unity.render-pipelines.core/Runtime/Textures/BufferedRTHandleSystem.cs
@@ -143,6 +143,18 @@ public void ResetReferenceSize(int width, int height)
             m_RTHandleSystem.ResetReferenceSize(width, height);
         }
 
+        /// <summary>
+        /// Queries the number of RT handle buffers allocated for a buffer ID.
+        /// </summary>
+        /// <param name="bufferId">The buffer ID to query.</param>
+        public int GetNumFramesAllocated(int bufferId)
+        {
+            if (!m_RTHandles.ContainsKey(bufferId))
+                return 0;
+
+            return m_RTHandles[bufferId].Length;
+        }
+
         void Swap()
         {
             foreach (var item in m_RTHandles)
diff --git a/com.unity.render-pipelines.high-definition/CHANGELOG.md b/com.unity.render-pipelines.high-definition/CHANGELOG.md
index e44f541de40..2f025d8efdb 100644
--- a/com.unity.render-pipelines.high-definition/CHANGELOG.md
+++ b/com.unity.render-pipelines.high-definition/CHANGELOG.md
@@ -173,6 +173,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Fixed material Emission properties not begin animated when recording an animation (case 1328108).
 - Fixed fog precision in some camera positions (case 1329603).
 - Fixed contact shadows tile coordinates calculations.
+- Fixed issue with history buffer allocation for AOVs when the request does not come in first frame.
 
 ### Changed
 - Changed Window/Render Pipeline/HD Render Pipeline Wizard to Window/Rendering/HDRP Wizard
diff --git a/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/Camera/HDCamera.cs b/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/Camera/HDCamera.cs
index 746620132b4..17879c80f9a 100644
--- a/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/Camera/HDCamera.cs
+++ b/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/Camera/HDCamera.cs
@@ -731,9 +731,11 @@ internal void Update(FrameSettings currentFrameSettings, HDRenderPipeline hdrp,
                 // If we have a mismatch with color buffer format we need to reallocate the pyramid
                 var hdPipeline = (HDRenderPipeline)(RenderPipelineManager.currentPipeline);
                 bool forceReallocPyramid = false;
-                if (m_NumColorPyramidBuffersAllocated > 0)
+                int colorBufferID = (int)HDCameraFrameHistoryType.ColorBufferMipChain;
+                int numColorPyramidBuffersAllocated = m_HistoryRTSystem.GetNumFramesAllocated(colorBufferID);
+                if (numColorPyramidBuffersAllocated > 0)
                 {
-                    var currPyramid = GetCurrentFrameRT((int)HDCameraFrameHistoryType.ColorBufferMipChain);
+                    var currPyramid = GetCurrentFrameRT(colorBufferID);
                     if (currPyramid != null && currPyramid.rt.graphicsFormat != hdPipeline.GetColorBufferFormat())
                     {
                         forceReallocPyramid = true;
@@ -746,8 +748,19 @@ internal void Update(FrameSettings currentFrameSettings, HDRenderPipeline hdrp,
                 if (isHistoryColorPyramidRequired) // Superset of case above
                     numColorPyramidBuffersRequired = 2;
 
+                // Check if we have any AOV requests that require history buffer allocations (the actual allocation happens later in this function)
+                foreach (var aovRequest in aovRequests)
+                {
+                    var aovHistory = GetHistoryRTHandleSystem(aovRequest);
+                    if (aovHistory.GetNumFramesAllocated(colorBufferID) != numColorPyramidBuffersRequired)
+                    {
+                        forceReallocPyramid = true;
+                        break;
+                    }
+                }
+
                 // Handle the color buffers
-                if (m_NumColorPyramidBuffersAllocated != numColorPyramidBuffersRequired || forceReallocPyramid)
+                if (numColorPyramidBuffersAllocated != numColorPyramidBuffersRequired || forceReallocPyramid)
                 {
                     // Reinit the system.
                     colorPyramidHistoryIsValid = false;
@@ -775,9 +788,6 @@ internal void Update(FrameSettings currentFrameSettings, HDRenderPipeline hdrp,
                         }
                         BindHistoryRTHandleSystem(cameraHistory);
                     }
-
-                    // Mark as init.
-                    m_NumColorPyramidBuffersAllocated = numColorPyramidBuffersRequired;
                 }
 
                 // Handle the volumetric fog buffers
@@ -1210,7 +1220,6 @@ public RTHandle Allocator(string id, int frameIndex, RTHandleSystem rtHandleSyst
 
         HDAdditionalCameraData  m_AdditionalCameraData = null; // Init in Update
         BufferedRTHandleSystem  m_HistoryRTSystem = new BufferedRTHandleSystem();
-        int                     m_NumColorPyramidBuffersAllocated = 0;
         int                     m_NumVolumetricBuffersAllocated   = 0;
         float                   m_AmbientOcclusionResolutionScale = 0.0f; // Factor used to track if history should be reallocated for Ambient Occlusion
         float                   m_ScreenSpaceAccumulationResolutionScale = 0.0f; // Use another scale if AO & SSR don't have the same resolution