diff --git a/OpenRA.Game/Graphics/Sprite.cs b/OpenRA.Game/Graphics/Sprite.cs
index 0ee1aa91dfe8..8646521c9281 100644
--- a/OpenRA.Game/Graphics/Sprite.cs
+++ b/OpenRA.Game/Graphics/Sprite.cs
@@ -50,13 +50,15 @@ public Sprite(Sheet sheet, Rectangle bounds, float zRamp, float3 offset, Texture
 
 	public class SpriteWithSecondaryData : Sprite
 	{
+		public readonly Sheet SecondarySheet;
 		public readonly Rectangle SecondaryBounds;
 		public readonly TextureChannel SecondaryChannel;
 		public readonly float SecondaryTop, SecondaryLeft, SecondaryBottom, SecondaryRight;
 
-		public SpriteWithSecondaryData(Sprite s, Rectangle secondaryBounds, TextureChannel secondaryChannel)
+		public SpriteWithSecondaryData(Sprite s, Sheet secondarySheet, Rectangle secondaryBounds, TextureChannel secondaryChannel)
 			: base(s.Sheet, s.Bounds, s.ZRamp, s.Offset, s.Channel, s.BlendMode)
 		{
+			SecondarySheet = secondarySheet;
 			SecondaryBounds = secondaryBounds;
 			SecondaryChannel = secondaryChannel;
 			SecondaryLeft = (float)Math.Min(secondaryBounds.Left, secondaryBounds.Right) / s.Sheet.Size.Width;
diff --git a/OpenRA.Game/Graphics/SpriteLoader.cs b/OpenRA.Game/Graphics/SpriteLoader.cs
index 648eb95da6e5..186e9addb71b 100644
--- a/OpenRA.Game/Graphics/SpriteLoader.cs
+++ b/OpenRA.Game/Graphics/SpriteLoader.cs
@@ -111,23 +111,6 @@ public SpriteCache(IReadOnlyFileSystem fileSystem, ISpriteLoader[] loaders, Shee
 				return sprite;
 			}
 		}
-
-		/// <summary>Returns all instances of sets of sprites with the given filename</summary>
-		public IEnumerable<Sprite[]> AllCached(string filename)
-		{
-			return sprites.GetOrAdd(filename);
-		}
-
-		/// <summary>Loads and caches a new instance of sprites with the given filename</summary>
-		public Sprite[] Reload(string filename)
-		{
-			var sprite = FrameLoader.GetFrames(fileSystem, filename, loaders)
-				.Select(a => SheetBuilder.Add(a))
-				.ToArray();
-
-			sprites.GetOrAdd(filename).Add(sprite);
-			return sprite;
-		}
 	}
 
 	public class FrameCache
diff --git a/OpenRA.Game/Graphics/SpriteRenderer.cs b/OpenRA.Game/Graphics/SpriteRenderer.cs
index 150820143136..5f412f90d2ea 100644
--- a/OpenRA.Game/Graphics/SpriteRenderer.cs
+++ b/OpenRA.Game/Graphics/SpriteRenderer.cs
@@ -62,26 +62,46 @@ int2 SetRenderStateForSprite(Sprite s)
 
 			currentBlend = s.BlendMode;
 
+			// Check if the sheet (or secondary data sheet) have already been mapped
 			var sheet = s.Sheet;
 			var sheetIndex = 0;
 			for (; sheetIndex < ns; sheetIndex++)
 				if (sheets[sheetIndex] == sheet)
 					break;
 
-			if (sheetIndex == ns)
+			var secondarySheetIndex = 0;
+			var ss = s as SpriteWithSecondaryData;
+			if (ss != null)
 			{
-				if (sheetIndex == sheets.Length)
-				{
-					Flush();
-					sheetIndex = 0;
-				}
+				var secondarySheet = ss.SecondarySheet;
+				for (; secondarySheetIndex < ns; secondarySheetIndex++)
+					if (sheets[secondarySheetIndex] == secondarySheet)
+						break;
+			}
+
+			// Make sure that we have enough free samplers to map both if needed, otherwise flush
+			var needSamplers = (sheetIndex == ns ? 1 : 0) + (secondarySheetIndex == ns ? 1 : 0);
+			if (ns + needSamplers >= sheets.Length)
+			{
+				Flush();
+				sheetIndex = 0;
+				if (ss != null)
+					secondarySheetIndex = 1;
+			}
 
+			if (sheetIndex >= ns)
+			{
 				sheets[sheetIndex] = sheet;
 				ns += 1;
 			}
 
-			// TODO: Add support for secondary channels on different sheets
-			return new int2(sheetIndex, sheetIndex);
+			if (secondarySheetIndex >= ns && ss != null)
+			{
+				sheets[secondarySheetIndex] = ss.SecondarySheet;
+				ns += 1;
+			}
+
+			return new int2(sheetIndex, secondarySheetIndex);
 		}
 
 		internal void DrawSprite(Sprite s, float3 location, float paletteTextureIndex, float3 size)
diff --git a/OpenRA.Game/Graphics/Theater.cs b/OpenRA.Game/Graphics/Theater.cs
index ed01fe9f8a23..924e7bb122ae 100644
--- a/OpenRA.Game/Graphics/Theater.cs
+++ b/OpenRA.Game/Graphics/Theater.cs
@@ -85,7 +85,7 @@ public Theater(TileSet tileset)
 
 							// s and ss are guaranteed to use the same sheet
 							// because of the custom terrain sheet allocation
-							s = new SpriteWithSecondaryData(s, ss.Bounds, ss.Channel);
+							s = new SpriteWithSecondaryData(s, s.Sheet, ss.Bounds, ss.Channel);
 						}
 
 						return s;
diff --git a/OpenRA.Mods.Common/Graphics/DefaultSpriteSequence.cs b/OpenRA.Mods.Common/Graphics/DefaultSpriteSequence.cs
index 4073ef3a6dc5..37ff76c14eef 100644
--- a/OpenRA.Mods.Common/Graphics/DefaultSpriteSequence.cs
+++ b/OpenRA.Mods.Common/Graphics/DefaultSpriteSequence.cs
@@ -268,38 +268,21 @@ public DefaultSpriteSequence(ModData modData, TileSet tileSet, SpriteCache cache
 				{
 					var depthSpriteFrame = LoadField(d, "DepthSpriteFrame", 0);
 					var depthOffset = LoadField(d, "DepthSpriteOffset", float2.Zero);
-					var depthSprites = cache.AllCached(depthSprite)
-						.Select(s => s[depthSpriteFrame]);
+					Func<int, IEnumerable<int>> getDepthFrame = _ => new int[] { depthSpriteFrame };
+					var ds = cache[depthSprite, getDepthFrame][depthSpriteFrame];
 
 					sprites = sprites.Select(s =>
 					{
 						if (s == null)
 							return null;
 
-						// The depth sprite must live on the same sheet as the main sprite
-						var ds = depthSprites.FirstOrDefault(dss => dss.Sheet == s.Sheet);
-						if (ds == null)
-						{
-							// The sequence has probably overflowed onto a new sheet.
-							// Allocating a new depth sprite on this sheet will almost certainly work
-							ds = cache.Reload(depthSprite)[depthSpriteFrame];
-							depthSprites = cache.AllCached(depthSprite)
-								.Select(ss => ss[depthSpriteFrame]);
-
-							// If that doesn't work then we may be referencing a cached sprite from an earlier sheet
-							// TODO: We could try and reallocate the main sprite, but that requires more complicated code and a perf hit
-							// We'll only cross that bridge if this becomes a problem in reality
-							if (ds.Sheet != s.Sheet)
-								throw new SheetOverflowException("Cross-sheet depth sprite reference: {0}.{1}: {2}");
-						}
-
 						var cw = (ds.Bounds.Left + ds.Bounds.Right) / 2 + (int)(s.Offset.X + depthOffset.X);
 						var ch = (ds.Bounds.Top + ds.Bounds.Bottom) / 2 + (int)(s.Offset.Y + depthOffset.Y);
 						var w = s.Bounds.Width / 2;
 						var h = s.Bounds.Height / 2;
 
 						var r = Rectangle.FromLTRB(cw - w, ch - h, cw + w, ch + h);
-						return new SpriteWithSecondaryData(s, r, ds.Channel);
+						return new SpriteWithSecondaryData(s, ds.Sheet, r, ds.Channel);
 					}).ToArray();
 				}