GBA Video: Fix broken sprite blending hack (fixes #532)

CHANGES

@@ -26,6 +26,7 @@ Bugfixes:
  - GBA Savedata: Fix 512 byte EEPROM saving as 8kB (fixes mgba.io/i/877)
  - SDL: Fix potential race condition when pressing keys (fixes mgba.io/i/872)
  - GBA: Fix keypad IRQs not firing when extra buttons are pressed
+ - GBA Video: Fix broken sprite blending hack (fixes mgba.io/i/532)
 Misc:
  - Qt: Don't rebuild library view if style hasn't changed
  - SDL: Fix 2.0.5 build on macOS under some circumstances

src/gba/renderers/software-obj.c

@@ -166,9 +166,13 @@ int GBAVideoSoftwareRendererPreprocessSprite(struct GBAVideoSoftwareRenderer* re
 		target2 |= renderer->bg[2].target2 << (renderer->bg[2].priority);
 		target2 |= renderer->bg[3].target2 << (renderer->bg[3].priority);
 		if ((1 << GBAObjAttributesCGetPriority(sprite->c)) <= target2) {
+			flags |= FLAG_REBLEND;
 			variant = 0;
+		} else if (!target2) {
+			flags &= ~FLAG_TARGET_1;
 		}
 	}
+
 	color_t* palette = &renderer->normalPalette[0x100];
 	color_t* objwinPalette = palette;
 	int objwinSlowPath = GBARegisterDISPCNTIsObjwinEnable(renderer->dispcnt) && GBAWindowControlGetBlendEnable(renderer->objwin.packed) != GBAWindowControlIsBlendEnable(renderer->currentWindow.packed);

src/gba/renderers/software-private.h

@@ -43,7 +43,7 @@ static inline void _compositeBlendObjwin(struct GBAVideoSoftwareRenderer* render
 		if (current & FLAG_TARGET_1 && color & FLAG_TARGET_2) {
 			color = _mix(renderer->blda, current, renderer->bldb, color);
 		} else {
-			color = (current & 0x00FFFFFF) | ((current << 1) & FLAG_REBLEND);
+			color = (current & 0x00FFFFFF) | (current & FLAG_REBLEND);
 		}
 	} else {
 		color = (color & ~FLAG_TARGET_2) | (current & FLAG_OBJWIN);
@@ -59,7 +59,7 @@ static inline void _compositeBlendNoObjwin(struct GBAVideoSoftwareRenderer* rend
 		if (current & FLAG_TARGET_1 && color & FLAG_TARGET_2) {
 			color = _mix(renderer->blda, current, renderer->bldb, color);
 		} else {
-			color = (current & 0x00FFFFFF) | ((current << 1) & FLAG_REBLEND);
+			color = (current & 0x00FFFFFF) | (current & FLAG_REBLEND);
 		}
 	} else {
 		color = color & ~FLAG_TARGET_2;
@@ -73,7 +73,7 @@ static inline void _compositeNoBlendObjwin(struct GBAVideoSoftwareRenderer* rend
 	if (color < current) {
 		color |= (current & FLAG_OBJWIN);
 	} else {
-		color = (current & 0x00FFFFFF) | ((current << 1) & FLAG_REBLEND);
+		color = (current & 0x00FFFFFF) | (current & FLAG_REBLEND);
 	}
 	*pixel = color;
 }
@@ -82,7 +82,7 @@ static inline void _compositeNoBlendNoObjwin(struct GBAVideoSoftwareRenderer* re
                                              uint32_t current) {
 	UNUSED(renderer);
 	if (color >= current) {
-		color = (current & 0x00FFFFFF) | ((current << 1) & FLAG_REBLEND);
+		color = (current & 0x00FFFFFF) | (current & FLAG_REBLEND);
 	}
 	*pixel = color;
 }