DS GX: Properly center cross product in polygon normal calculations

CHANGES

@@ -6,6 +6,7 @@ Bugfixes:
  - DS GX: Automatically normalize winding culling calculations (fixes mgba.io/i/699)
  - DS GX: Fixed viewport calculations (fixes mgba.io/i/709)
  - DS Video: Fix display capture blending value 16 (fixes mgba.io/i/757)
+ - DS GX: Properly center cross product in polygon normal calculations
 Misc:
  - DS GX: Clean up and unify texture mapping
  - DS Core: Add symbol loading

include/mgba/internal/ds/gx.h

@@ -50,8 +50,8 @@ DECL_BITS(DSGXTexParams, CoordTfMode, 30, 2);
 DECL_BITFIELD(DSGXPolygonAttrs, uint32_t);
 DECL_BITS(DSGXPolygonAttrs, Lights, 0, 4);
 DECL_BITS(DSGXPolygonAttrs, Mode, 4, 2);
-DECL_BIT(DSGXPolygonAttrs, FrontFace, 6);
-DECL_BIT(DSGXPolygonAttrs, BackFace, 7);
+DECL_BIT(DSGXPolygonAttrs, BackFace, 6);
+DECL_BIT(DSGXPolygonAttrs, FrontFace, 7);
 DECL_BIT(DSGXPolygonAttrs, UpdateDepth, 11);
 // TODO
 DECL_BITS(DSGXPolygonAttrs, Alpha, 16, 5);

src/ds/gx.c

@@ -238,24 +238,32 @@ static bool _clipPolygon(struct DSGX* gx, struct DSGXPolygon* poly) {
 		int64_t ny = 0;
 		int64_t nz = 0;
 		int64_t dot = 0;
-		int rank = 30;
+		int rank = 29;
 		for (v = 0; v < poly->verts; ++v) {
-			struct DSGXVertex* v0 = &gx->pendingVertices[poly->vertIds[v]];
+			struct DSGXVertex v0 = gx->pendingVertices[poly->vertIds[v]];
 			struct DSGXVertex* v1;
-			struct DSGXVertex* v2;
+			struct DSGXVertex v2;
 			if (v < poly->verts - 2) {
 				v1 = &gx->pendingVertices[poly->vertIds[v + 1]];
-				v2 = &gx->pendingVertices[poly->vertIds[v + 2]];
+				v2 = gx->pendingVertices[poly->vertIds[v + 2]];
 			} else if (v < poly->verts - 1) {
 				v1 = &gx->pendingVertices[poly->vertIds[v + 1]];
-				v2 = &gx->pendingVertices[poly->vertIds[v + 2 - poly->verts]];
+				v2 = gx->pendingVertices[poly->vertIds[v + 2 - poly->verts]];
 			} else {
 				v1 = &gx->pendingVertices[poly->vertIds[v + 1 - poly->verts]];
-				v2 = &gx->pendingVertices[poly->vertIds[v + 2 - poly->verts]];
+				v2 = gx->pendingVertices[poly->vertIds[v + 2 - poly->verts]];
 			}
-			nx = ((int64_t) v0->viewCoord[1] * v2->viewCoord[3] - (int64_t) v0->viewCoord[3] * v2->viewCoord[1]);
-			ny = ((int64_t) v0->viewCoord[3] * v2->viewCoord[0] - (int64_t) v0->viewCoord[0] * v2->viewCoord[3]);
-			nz = ((int64_t) v0->viewCoord[0] * v2->viewCoord[1] - (int64_t) v0->viewCoord[1] * v2->viewCoord[0]);
+			// Center around v1
+			v0.viewCoord[0] -= v1->viewCoord[0];
+			v0.viewCoord[1] -= v1->viewCoord[1];
+			v0.viewCoord[3] -= v1->viewCoord[3];
+			v2.viewCoord[0] -= v1->viewCoord[0];
+			v2.viewCoord[1] -= v1->viewCoord[1];
+			v2.viewCoord[3] -= v1->viewCoord[3];
+			// Calculate cross product
+			nx = ((int64_t) v0.viewCoord[1] * v2.viewCoord[3] - (int64_t) v0.viewCoord[3] * v2.viewCoord[1]);
+			ny = ((int64_t) v0.viewCoord[3] * v2.viewCoord[0] - (int64_t) v0.viewCoord[0] * v2.viewCoord[3]);
+			nz = ((int64_t) v0.viewCoord[0] * v2.viewCoord[1] - (int64_t) v0.viewCoord[1] * v2.viewCoord[0]);
 			int rx = 64 - clz64(nx >= 0 ? nx : -nx);
 			int ry = 64 - clz64(ny >= 0 ? ny : -ny);
 			int rz = 64 - clz64(nz >= 0 ? nz : -nz);
@@ -265,22 +273,22 @@ static bool _clipPolygon(struct DSGX* gx, struct DSGXPolygon* poly) {
 			if (rz > rx) {
 				rx = rz;
 			}
-			if (rx > 30) {
-				nx >>= rx - 30;
-				ny >>= rx - 30;
-				nz >>= rx - 30;
-			}
 			if (rx > rank) {
 				dot >>= rx - rank;
 				rank = rx;
 			}
+			if (rank > 29) {
+				nx >>= rank - 29;
+				ny >>= rank - 29;
+				nz >>= rank - 29;
+			}
 
 			dot += nx * v1->viewCoord[0] + ny * v1->viewCoord[1] + nz * v1->viewCoord[3];
 		}
-		if (!DSGXPolygonAttrsIsBackFace(poly->polyParams) && dot < 0) {
+		if (!DSGXPolygonAttrsIsFrontFace(poly->polyParams) && dot < 0) {
 			return false;
 		}
-		if (!DSGXPolygonAttrsIsFrontFace(poly->polyParams) && dot > 0) {
+		if (!DSGXPolygonAttrsIsBackFace(poly->polyParams) && dot > 0) {
 			return false;
 		}
 	}