CCM-16073 - Integration test fixes

infrastructure/terraform/components/callbacks/elasticache_delivery_state.tf

@@ -1,11 +1,17 @@
+resource "random_password" "elasticache_default_user" {
+  length  = 32
+  special = false
+}
+
 resource "aws_elasticache_user" "delivery_state_default" {
-  user_id       = "${local.csi}-delivery-state-default"
+  user_id       = "${local.csi}-valkey-default"
   user_name     = "default"
   engine        = "valkey"
   access_string = "off -@all"
 
   authentication_mode {
-    type = "no-password-required"
+    type      = "password"
+    passwords = [random_password.elasticache_default_user.result]
   }
 
   tags = local.default_tags

infrastructure/terraform/components/callbacks/locals.tf

@@ -21,7 +21,7 @@ locals {
       targets = [
         for target in try(client.targets, []) :
         merge(target, {
-          invocationEndpoint = try(target.delivery.mtls.enabled, false) ? "https://${aws_lb.mock_webhook_mtls[0].dns_name}/${target.targetId}" : "http://${aws_lb.mock_webhook_mtls[0].dns_name}/${target.targetId}"
+          invocationEndpoint = "https://${aws_lb.mock_webhook_mtls[0].dns_name}/${target.targetId}"
           apiKey             = merge(target.apiKey, { headerValue = random_password.mock_webhook_api_key[0].result })
         })
       ]

infrastructure/terraform/components/callbacks/module_mock_webhook_alb_mtls.tf

@@ -19,18 +19,7 @@ resource "aws_vpc_security_group_ingress_rule" "mock_webhook_alb_https" {
   from_port                    = 443
   to_port                      = 443
   ip_protocol                  = "tcp"
-  description                  = "Allow HTTPS Client Lambda to reach mock webhook via mTLS"
-  tags                         = local.default_tags
-}
-
-resource "aws_vpc_security_group_ingress_rule" "mock_webhook_alb_http" {
-  count                        = var.deploy_mock_clients ? 1 : 0
-  security_group_id            = aws_security_group.mock_webhook_alb[0].id
-  referenced_security_group_id = aws_security_group.https_client_lambda.id
-  from_port                    = 80
-  to_port                      = 80
-  ip_protocol                  = "tcp"
-  description                  = "Allow HTTPS Client Lambda to reach mock webhook without mTLS"
+  description                  = "Allow HTTPS Client Lambda to reach mock webhook (mTLS and non-mTLS)"
   tags                         = local.default_tags
 }
 
@@ -102,17 +91,3 @@ resource "aws_lb_listener" "mock_webhook_mtls" {
 
   tags = local.default_tags
 }
-
-resource "aws_lb_listener" "mock_webhook_http" {
-  count             = var.deploy_mock_clients ? 1 : 0
-  load_balancer_arn = aws_lb.mock_webhook_mtls[0].arn
-  port              = 80
-  protocol          = "HTTP"
-
-  default_action {
-    type             = "forward"
-    target_group_arn = aws_lb_target_group.mock_webhook_mtls[0].arn
-  }
-
-  tags = local.default_tags
-}

lambdas/https-client-lambda/src/__tests__/dlq-sender.test.ts

@@ -54,4 +54,81 @@ describe("sendToDlq", () => {
 
     process.env.DLQ_URL = saved;
   });
+
+  it("includes ERROR_CODE and ERROR_MESSAGE for HTTP error with JSON body", async () => {
+    mockSend.mockResolvedValue({});
+
+    await sendToDlq('{"test":"message"}', {
+      statusCode: 400,
+      responseBody: JSON.stringify({ message: "Bad request" }),
+    });
+
+    const command = mockSend.mock.calls[0][0];
+    expect(command).toBeInstanceOf(SendMessageCommand);
+    expect(command.input.MessageAttributes).toEqual({
+      ERROR_CODE: { DataType: "String", StringValue: "HTTP_CLIENT_ERROR" },
+      ERROR_MESSAGE: { DataType: "String", StringValue: "Bad request" },
+    });
+  });
+
+  it("uses raw response body as ERROR_MESSAGE when not valid JSON", async () => {
+    mockSend.mockResolvedValue({});
+
+    await sendToDlq('{"test":"message"}', {
+      statusCode: 400,
+      responseBody: "Bad request",
+    });
+
+    const command = mockSend.mock.calls[0][0];
+    expect(command.input.MessageAttributes).toEqual({
+      ERROR_CODE: { DataType: "String", StringValue: "HTTP_CLIENT_ERROR" },
+      ERROR_MESSAGE: { DataType: "String", StringValue: "Bad request" },
+    });
+  });
+
+  it("uses errorCode as ERROR_CODE when provided", async () => {
+    mockSend.mockResolvedValue({});
+
+    await sendToDlq('{"test":"message"}', {
+      errorCode: "CERT_HAS_EXPIRED",
+    });
+
+    const command = mockSend.mock.calls[0][0];
+    expect(command.input.MessageAttributes).toEqual({
+      ERROR_CODE: { DataType: "String", StringValue: "CERT_HAS_EXPIRED" },
+    });
+  });
+
+  it("sends empty MessageAttributes when errorInfo has no relevant fields", async () => {
+    mockSend.mockResolvedValue({});
+
+    await sendToDlq('{"test":"message"}', {});
+
+    const command = mockSend.mock.calls[0][0];
+    expect(command.input.MessageAttributes).toEqual({});
+  });
+
+  it("sends no MessageAttributes when errorInfo is omitted", async () => {
+    mockSend.mockResolvedValue({});
+
+    await sendToDlq('{"test":"message"}');
+
+    const command = mockSend.mock.calls[0][0];
+    expect(command.input.MessageAttributes).toBeUndefined();
+  });
+
+  it("uses JSON body message field when present in responseBody", async () => {
+    mockSend.mockResolvedValue({});
+
+    await sendToDlq('{"test":"message"}', {
+      statusCode: 422,
+      responseBody: JSON.stringify({ message: "Validation failed", code: 42 }),
+    });
+
+    const command = mockSend.mock.calls[0][0];
+    expect(command.input.MessageAttributes?.ERROR_MESSAGE).toEqual({
+      DataType: "String",
+      StringValue: "Validation failed",
+    });
+  });
 });

lambdas/https-client-lambda/src/__tests__/endpoint-gate.test.ts

@@ -98,14 +98,6 @@ describe("admit", () => {
     );
   });
 
-  it("propagates non-NOSCRIPT Redis errors", async () => {
-    mockSendCommand.mockRejectedValueOnce(new Error("Connection refused"));
-
-    await expect(
-      admit(mockRedis, "target-1", 10, true, defaultConfig),
-    ).rejects.toThrow("Connection refused");
-  });
-
   it("passes cbProbeIntervalMs=0 when circuit breaker is disabled", async () => {
     mockSendCommand.mockResolvedValueOnce([1, "allowed", 0, 10]);
 
@@ -123,8 +115,46 @@ describe("admit", () => {
     await admit(mockRedis, "my-target", 5, true, defaultConfig);
 
     const args = mockSendCommand.mock.calls[0]![0] as string[];
-    expect(args[3]).toBe("cb:my-target");
-    expect(args[4]).toBe("rl:my-target");
+    expect(args[3]).toBe("cb:{my-target}");
+    expect(args[4]).toBe("rl:{my-target}");
+  });
+});
+
+describe("evalScript", () => {
+  it("throws a wrapped error including the original message when EVALSHA fails with a non-NOSCRIPT Error", async () => {
+    const redisError = new Error("WRONGTYPE Operation against a key");
+    mockSendCommand.mockRejectedValueOnce(redisError);
+
+    const thrown = await admit(
+      mockRedis,
+      "target-1",
+      10,
+      true,
+      defaultConfig,
+    ).catch((error: unknown) => error);
+
+    expect(thrown).toBeInstanceOf(Error);
+    expect((thrown as Error).message).toContain("Redis error in script");
+    expect((thrown as Error).message).toContain(
+      "WRONGTYPE Operation against a key",
+    );
+    expect((thrown as Error & { cause: unknown }).cause).toBe(redisError);
+  });
+
+  it("throws a wrapped error using String() when EVALSHA rejects with a non-Error value", async () => {
+    mockSendCommand.mockRejectedValueOnce("connection refused");
+
+    const thrown = await admit(
+      mockRedis,
+      "target-1",
+      10,
+      true,
+      defaultConfig,
+    ).catch((error: unknown) => error);
+
+    expect(thrown).toBeInstanceOf(Error);
+    expect((thrown as Error).message).toContain("Redis error in script");
+    expect((thrown as Error).message).toContain("connection refused");
   });
 });
 
@@ -187,20 +217,12 @@ describe("recordResult", () => {
     expect(mockSendCommand).toHaveBeenCalledTimes(2);
   });
 
-  it("propagates non-NOSCRIPT Redis errors", async () => {
-    mockSendCommand.mockRejectedValueOnce(new Error("Connection refused"));
-
-    await expect(
-      recordResult(mockRedis, "target-1", false, defaultConfig),
-    ).rejects.toThrow("Connection refused");
-  });
-
   it("passes correct cb key for target", async () => {
     mockSendCommand.mockResolvedValueOnce([1, "closed"]);
 
     await recordResult(mockRedis, "my-target", true, defaultConfig);
 
     const args = mockSendCommand.mock.calls[0]![0] as string[];
-    expect(args[3]).toBe("cb:my-target");
+    expect(args[3]).toBe("cb:{my-target}");
   });
 });

lambdas/https-client-lambda/src/__tests__/handler.test.ts

@@ -142,7 +142,9 @@ describe("processRecords", () => {
     const failures = await processRecords([makeRecord()]);
 
     expect(failures).toEqual([]);
-    expect(mockSendToDlq).toHaveBeenCalledWith(makeRecord().body);
+    expect(mockSendToDlq).toHaveBeenCalledWith(makeRecord().body, {
+      outcome: "permanent_failure",
+    });
   });
 
   it("returns failure for transient 5xx errors", async () => {
@@ -538,4 +540,21 @@ describe("processRecords", () => {
       3_600_000,
     );
   });
+
+  it("returns no failure when handleRateLimitedRecord resolves without throwing", async () => {
+    mockDeliverPayload.mockResolvedValue({
+      outcome: "permanent_failure",
+      statusCode: 429,
+      retryAfterHeader: "60",
+    });
+    mockHandleRateLimitedRecord.mockResolvedValueOnce(undefined);
+
+    const failures = await processRecords([makeRecord()]);
+
+    expect(failures).toEqual([]);
+    expect(mockIsWindowExhausted).toHaveBeenCalledWith(
+      expect.any(Number),
+      7_200_000,
+    );
+  });
 });

lambdas/https-client-lambda/src/__tests__/https-client.test.ts

@@ -36,6 +36,7 @@ type MockResponse = EventEmitter & {
 function mockHttpsRequest(
   statusCode: number,
   headers: Record<string, string | undefined> = {},
+  body = "",
 ) {
   const mockReq = new EventEmitter() as EventEmitter & {
     end: jest.Mock;
@@ -56,7 +57,13 @@ function mockHttpsRequest(
     });
 
     if (callback) {
-      process.nextTick(() => callback(res));
+      process.nextTick(() => {
+        callback(res);
+        process.nextTick(() => {
+          if (body) res.emit("data", Buffer.from(body));
+          res.emit("end");
+        });
+      });
     }
 
     return mockReq as unknown as ReturnType<typeof https.request>;
@@ -125,7 +132,7 @@ describe("deliverPayload", () => {
   });
 
   it("returns permanent_failure on 4xx non-429", async () => {
-    mockHttpsRequest(400);
+    mockHttpsRequest(400, {}, JSON.stringify({ message: "Bad request" }));
 
     const result = await deliverPayload(
       createTarget(),
@@ -134,7 +141,11 @@ describe("deliverPayload", () => {
       createMockAgent(),
     );
 
-    expect(result).toEqual({ outcome: "permanent_failure" });
+    expect(result).toEqual({
+      outcome: "permanent_failure",
+      statusCode: 400,
+      responseBody: JSON.stringify({ message: "Bad request" }),
+    });
   });
 
   it("returns permanent_failure on TLS error CERT_HAS_EXPIRED", async () => {
@@ -147,7 +158,10 @@ describe("deliverPayload", () => {
       createMockAgent(),
     );
 
-    expect(result).toEqual({ outcome: "permanent_failure" });
+    expect(result).toEqual({
+      outcome: "permanent_failure",
+      errorCode: "CERT_HAS_EXPIRED",
+    });
   });
 
   it("returns permanent_failure on TLS pinning error", async () => {
@@ -160,7 +174,10 @@ describe("deliverPayload", () => {
       createMockAgent(),
     );
 
-    expect(result).toEqual({ outcome: "permanent_failure" });
+    expect(result).toEqual({
+      outcome: "permanent_failure",
+      errorCode: "ERR_CERT_PINNING_FAILED",
+    });
   });
 
   it("returns transient_failure on 5xx", async () => {
@@ -189,6 +206,7 @@ describe("deliverPayload", () => {
     expect(result).toEqual({
       outcome: "rate_limited",
       retryAfterHeader: "60",
+      statusCode: 429,
     });
   });
 
@@ -205,6 +223,7 @@ describe("deliverPayload", () => {
     expect(result).toEqual({
       outcome: "rate_limited",
       retryAfterHeader: undefined,
+      statusCode: 429,
     });
   });
 
@@ -287,4 +306,43 @@ describe("deliverPayload", () => {
 
     expect(result).toEqual({ outcome: "transient_failure", statusCode: 0 });
   });
+
+  it("treats undefined statusCode as 0", async () => {
+    const mockReq = new EventEmitter() as EventEmitter & {
+      end: jest.Mock;
+      destroy: jest.Mock;
+    };
+    mockReq.end = jest.fn();
+    mockReq.destroy = jest.fn();
+
+    jest.spyOn(https, "request").mockImplementation((...args: unknown[]) => {
+      const callback = args.find((a) => typeof a === "function") as
+        | ((res: MockResponse) => void)
+        | undefined;
+
+      const res = Object.assign(new EventEmitter(), {
+        statusCode: undefined as unknown as number,
+        headers: {},
+        resume: jest.fn(),
+      }) as MockResponse;
+
+      if (callback) {
+        process.nextTick(() => {
+          callback(res);
+          process.nextTick(() => (res as EventEmitter).emit("end"));
+        });
+      }
+
+      return mockReq as unknown as ReturnType<typeof https.request>;
+    });
+
+    const result = await deliverPayload(
+      createTarget(),
+      '{"test":true}',
+      "sig-abc",
+      createMockAgent(),
+    );
+
+    expect(result).toEqual({ outcome: "transient_failure", statusCode: 0 });
+  });
 });