fix(otel): prevent infinite retry loops on unicode hex escape errors

.github/workflows/unit-tests-webapp.yml

@@ -12,8 +12,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-        shardTotal: [10]
+        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
+        shardTotal: [8]
     env:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       SHARD_INDEX: ${{ matrix.shardIndex }}

apps/webapp/app/db.server.ts

@@ -122,30 +122,89 @@ function getClient() {
         url: databaseUrl.href,
       },
     },
-    // @ts-expect-error
     log: [
+      // events
       {
-        emit: "stdout",
+        emit: "event",
         level: "error",
       },
       {
-        emit: "stdout",
+        emit: "event",
         level: "info",
       },
       {
-        emit: "stdout",
+        emit: "event",
         level: "warn",
       },
-    ].concat(
-      process.env.VERBOSE_PRISMA_LOGS === "1"
+      // stdout
+      ...((process.env.PRISMA_LOG_TO_STDOUT === "1"
         ? [
-            { emit: "event", level: "query" },
-            { emit: "stdout", level: "query" },
+            {
+              emit: "stdout",
+              level: "error",
+            },
+            {
+              emit: "stdout",
+              level: "info",
+            },
+            {
+              emit: "stdout",
+              level: "warn",
+            },
           ]
-        : []
-    ),
+        : []) satisfies Prisma.LogDefinition[]),
+      // verbose
+      ...((process.env.VERBOSE_PRISMA_LOGS === "1"
+        ? [
+            {
+              emit: "event",
+              level: "query",
+            },
+            {
+              emit: "stdout",
+              level: "query",
+            },
+          ]
+        : []) satisfies Prisma.LogDefinition[]),
+    ],
   });
 
+  // Only use structured logging if we're not already logging to stdout
+  if (process.env.PRISMA_LOG_TO_STDOUT !== "1") {
+    client.$on("info", (log) => {
+      logger.info("PrismaClient info", {
+        clientType: "writer",
+        event: {
+          timestamp: log.timestamp,
+          message: log.message,
+          target: log.target,
+        },
+      });
+    });
+
+    client.$on("warn", (log) => {
+      logger.warn("PrismaClient warn", {
+        clientType: "writer",
+        event: {
+          timestamp: log.timestamp,
+          message: log.message,
+          target: log.target,
+        },
+      });
+    });
+
+    client.$on("error", (log) => {
+      logger.error("PrismaClient error", {
+        clientType: "writer",
+        event: {
+          timestamp: log.timestamp,
+          message: log.message,
+          target: log.target,
+        },
+      });
+    });
+  }
+
   // connect eagerly
   client.$connect();
 
@@ -174,30 +233,89 @@ function getReplicaClient() {
         url: replicaUrl.href,
       },
     },
-    // @ts-expect-error
     log: [
+      // events
       {
-        emit: "stdout",
+        emit: "event",
         level: "error",
       },
       {
-        emit: "stdout",
+        emit: "event",
         level: "info",
       },
       {
-        emit: "stdout",
+        emit: "event",
         level: "warn",
       },
-    ].concat(
-      process.env.VERBOSE_PRISMA_LOGS === "1"
+      // stdout
+      ...((process.env.PRISMA_LOG_TO_STDOUT === "1"
         ? [
-            { emit: "event", level: "query" },
-            { emit: "stdout", level: "query" },
+            {
+              emit: "stdout",
+              level: "error",
+            },
+            {
+              emit: "stdout",
+              level: "info",
+            },
+            {
+              emit: "stdout",
+              level: "warn",
+            },
           ]
-        : []
-    ),
+        : []) satisfies Prisma.LogDefinition[]),
+      // verbose
+      ...((process.env.VERBOSE_PRISMA_LOGS === "1"
+        ? [
+            {
+              emit: "event",
+              level: "query",
+            },
+            {
+              emit: "stdout",
+              level: "query",
+            },
+          ]
+        : []) satisfies Prisma.LogDefinition[]),
+    ],
   });
 
+  // Only use structured logging if we're not already logging to stdout
+  if (process.env.PRISMA_LOG_TO_STDOUT !== "1") {
+    replicaClient.$on("info", (log) => {
+      logger.info("PrismaClient info", {
+        clientType: "reader",
+        event: {
+          timestamp: log.timestamp,
+          message: log.message,
+          target: log.target,
+        },
+      });
+    });
+
+    replicaClient.$on("warn", (log) => {
+      logger.warn("PrismaClient warn", {
+        clientType: "reader",
+        event: {
+          timestamp: log.timestamp,
+          message: log.message,
+          target: log.target,
+        },
+      });
+    });
+
+    replicaClient.$on("error", (log) => {
+      logger.error("PrismaClient error", {
+        clientType: "reader",
+        event: {
+          timestamp: log.timestamp,
+          message: log.message,
+          target: log.target,
+        },
+      });
+    });
+  }
+
   // connect eagerly
   replicaClient.$connect();
 

apps/webapp/app/v3/eventRepository.server.ts

@@ -1230,25 +1230,23 @@ export class EventRepository {
 
         return events;
       } catch (error) {
-        if (error instanceof Prisma.PrismaClientUnknownRequestError) {
-          logger.error("Failed to insert events, most likely because of null characters", {
-            error: {
-              name: error.name,
-              message: error.message,
-              stack: error.stack,
-              clientVersion: error.clientVersion,
-            },
+        if (isRetriablePrismaError(error)) {
+          const isKnownError = error instanceof Prisma.PrismaClientKnownRequestError;
+          span.setAttribute("prisma_error_type", isKnownError ? "known" : "unknown");
+
+          const errorDetails = getPrismaErrorDetails(error);
+          if (errorDetails.code) {
+            span.setAttribute("prisma_error_code", errorDetails.code);
+          }
+
+          logger.error("Failed to insert events, will attempt bisection", {
+            error: errorDetails,
           });
 
           if (events.length === 1) {
             logger.debug("Attempting to insert event individually and it failed", {
               event: events[0],
-              error: {
-                name: error.name,
-                message: error.message,
-                stack: error.stack,
-                clientVersion: error.clientVersion,
-              },
+              error: errorDetails,
             });
 
             span.setAttribute("failed_event_count", 1);
@@ -1258,12 +1256,7 @@ export class EventRepository {
 
           if (depth > MAX_FLUSH_DEPTH) {
             logger.error("Failed to insert events, reached maximum depth", {
-              error: {
-                name: error.name,
-                message: error.message,
-                stack: error.stack,
-                clientVersion: error.clientVersion,
-              },
+              error: errorDetails,
               depth,
               eventsCount: events.length,
             });
@@ -1917,3 +1910,62 @@ export async function recordRunDebugLog(
     },
   });
 }
+
+/**
+ * Extracts error details from Prisma errors in a type-safe way.
+ * Only includes 'code' property for PrismaClientKnownRequestError.
+ */
+function getPrismaErrorDetails(
+  error: Prisma.PrismaClientUnknownRequestError | Prisma.PrismaClientKnownRequestError
+): {
+  name: string;
+  message: string;
+  stack: string | undefined;
+  clientVersion: string;
+  code?: string;
+} {
+  const base = {
+    name: error.name,
+    message: error.message,
+    stack: error.stack,
+    clientVersion: error.clientVersion,
+  };
+
+  if (error instanceof Prisma.PrismaClientKnownRequestError) {
+    return { ...base, code: error.code };
+  }
+
+  return base;
+}
+
+/**
+ * Checks if a PrismaClientKnownRequestError is a Unicode/hex escape error.
+ */
+function isUnicodeError(error: Prisma.PrismaClientKnownRequestError): boolean {
+  return (
+    error.message.includes("lone leading surrogate in hex escape") ||
+    error.message.includes("unexpected end of hex escape") ||
+    error.message.includes("invalid Unicode") ||
+    error.message.includes("invalid escape sequence")
+  );
+}
+
+/**
+ * Determines if a Prisma error should be retried with bisection logic.
+ * Returns true for errors that might be resolved by splitting the batch.
+ */
+function isRetriablePrismaError(
+  error: unknown
+): error is Prisma.PrismaClientUnknownRequestError | Prisma.PrismaClientKnownRequestError {
+  if (error instanceof Prisma.PrismaClientUnknownRequestError) {
+    // Always retry unknown errors with bisection
+    return true;
+  }
+
+  if (error instanceof Prisma.PrismaClientKnownRequestError) {
+    // Only retry known errors if they're Unicode/hex escape related
+    return isUnicodeError(error);
+  }
+
+  return false;
+}