Add signing key rotation support (#541)

.changeset/slimy-pears-tan.md

@@ -0,0 +1,5 @@
+---
+"inngest": minor
+---
+
+Add signing key rotation support

packages/inngest/package.json

@@ -212,6 +212,7 @@
     "glob": "^10.3.10",
     "inquirer": "^9.2.10",
     "jest": "^29.3.1",
+    "jest-fetch-mock": "^3.0.3",
     "koa": "^2.14.2",
     "minimist": "^1.2.8",
     "next": "^13.5.4",

packages/inngest/src/api/api.ts

@@ -2,6 +2,7 @@ import { type fetch } from "cross-fetch";
 import { type ExecutionVersion } from "../components/execution/InngestExecution";
 import { getFetch } from "../helpers/env";
 import { getErrorMessage } from "../helpers/errors";
+import { fetchWithAuthFallback } from "../helpers/net";
 import { hashSigningKey } from "../helpers/strings";
 import { err, ok, type Result } from "../types";
 import {
@@ -18,43 +19,64 @@ type FetchT = typeof fetch;
 interface InngestApiConstructorOpts {
   baseUrl?: string;
   signingKey: string;
+  signingKeyFallback: string | undefined;
   fetch?: FetchT;
 }
 
 export class InngestApi {
   public readonly baseUrl: string;
   private signingKey: string;
+  private signingKeyFallback: string | undefined;
   private readonly fetch: FetchT;
 
   constructor({
     baseUrl = "https://api.inngest.com",
     signingKey,
+    signingKeyFallback,
     fetch,
   }: InngestApiConstructorOpts) {
     this.baseUrl = baseUrl;
     this.signingKey = signingKey;
+    this.signingKeyFallback = signingKeyFallback;
     this.fetch = getFetch(fetch);
   }
 
   private get hashedKey(): string {
     return hashSigningKey(this.signingKey);
   }
 
+  private get hashedFallbackKey(): string | undefined {
+    if (!this.signingKeyFallback) {
+      return;
+    }
+
+    return hashSigningKey(this.signingKeyFallback);
+  }
+
   // set the signing key in case it was not instantiated previously
   setSigningKey(key: string | undefined) {
     if (typeof key === "string" && this.signingKey === "") {
       this.signingKey = key;
     }
   }
 
+  setSigningKeyFallback(key: string | undefined) {
+    if (typeof key === "string" && !this.signingKeyFallback) {
+      this.signingKeyFallback = key;
+    }
+  }
+
   async getRunSteps(
     runId: string,
     version: ExecutionVersion
   ): Promise<Result<StepsResponse, ErrorResponse>> {
     const url = new URL(`/v0/runs/${runId}/actions`, this.baseUrl);
 
-    return this.fetch(url, {
-      headers: { Authorization: `Bearer ${this.hashedKey}` },
+    return fetchWithAuthFallback({
+      authToken: this.hashedKey,
+      authTokenFallback: this.hashedFallbackKey,
+      fetch: this.fetch,
+      url,
     })
       .then(async (resp) => {
         const data: unknown = await resp.json();
@@ -78,8 +100,11 @@ export class InngestApi {
   ): Promise<Result<BatchResponse, ErrorResponse>> {
     const url = new URL(`/v0/runs/${runId}/batch`, this.baseUrl);
 
-    return this.fetch(url, {
-      headers: { Authorization: `Bearer ${this.hashedKey}` },
+    return fetchWithAuthFallback({
+      authToken: this.hashedKey,
+      authTokenFallback: this.hashedFallbackKey,
+      fetch: this.fetch,
+      url,
     })
       .then(async (resp) => {
         const data: unknown = await resp.json();

packages/inngest/src/components/Inngest.ts

@@ -208,6 +208,7 @@ export class Inngest<TClientOpts extends ClientOptions = ClientOptions> {
     this.inngestApi = new InngestApi({
       baseUrl: this.apiBaseUrl || defaultInngestApiBaseUrl,
       signingKey: processEnv(envKeys.InngestSigningKey) || "",
+      signingKeyFallback: processEnv(envKeys.InngestSigningKeyFallback),
       fetch: this.fetch,
     });
 
@@ -459,6 +460,8 @@ export class Inngest<TClientOpts extends ClientOptions = ClientOptions> {
       }
     }
 
+    // We don't need to do fallback auth here because this uses event keys and
+    // not signing keys
     const response = await this.fetch(url, {
       method: "POST",
       body: stringify(payloads),

packages/inngest/src/components/InngestCommHandler.ts

@@ -29,6 +29,7 @@ import {
   undefinedToNull,
   type FnData,
 } from "../helpers/functions";
+import { fetchWithAuthFallback } from "../helpers/net";
 import { runAsPromise } from "../helpers/promises";
 import { createStream } from "../helpers/stream";
 import { hashSigningKey, stringify } from "../helpers/strings";
@@ -37,11 +38,12 @@ import {
   logLevels,
   type EventPayload,
   type FunctionConfig,
-  type IntrospectRequest,
+  type InsecureIntrospection,
   type LogLevel,
   type OutgoingOp,
   type RegisterOptions,
   type RegisterRequest,
+  type SecureIntrospection,
   type SupportedFrameworkName,
 } from "../types";
 import { version } from "../version";
@@ -246,6 +248,12 @@ export class InngestCommHandler<
    */
   protected signingKey: string | undefined;
 
+  /**
+   * The same as signingKey, except used as a fallback when auth fails using the
+   * primary signing key.
+   */
+  protected signingKeyFallback: string | undefined;
+
   /**
    * A property that can be set to indicate whether we believe we are in
    * production mode.
@@ -394,6 +402,7 @@ export class InngestCommHandler<
     );
 
     this.signingKey = options.signingKey;
+    this.signingKeyFallback = options.signingKeyFallback;
     this.serveHost = options.serveHost || this.env[envKeys.InngestServeHost];
     this.servePath = options.servePath || this.env[envKeys.InngestServePath];
 
@@ -442,6 +451,10 @@ export class InngestCommHandler<
     return hashSigningKey(this.signingKey);
   }
 
+  private get hashedSigningKeyFallback(): string {
+    return hashSigningKey(this.signingKeyFallback);
+  }
+
   /**
    * `createHandler` should be used to return a type-equivalent version of the
    * `handler` specified during instantiation.
@@ -830,14 +843,39 @@ export class InngestCommHandler<
           deployId: null,
         });
 
-        const introspection: IntrospectRequest = {
-          message: "Inngest endpoint configured correctly.",
-          hasEventKey: this.client["eventKeySet"](),
-          hasSigningKey: Boolean(this.signingKey),
-          functionsFound: registerBody.functions.length,
-          mode: this._mode,
+        const signature = await actions.headers(
+          "checking signature for run request",
+          headerKeys.Signature
+        );
+
+        let introspection: InsecureIntrospection | SecureIntrospection = {
+          extra: {
+            is_mode_explicit: this._mode.isExplicit,
+            message: "Inngest endpoint configured correctly.",
+          },
+          has_event_key: this.client["eventKeySet"](),
+          has_signing_key: Boolean(this.signingKey),
+          function_count: registerBody.functions.length,
+          mode: this._mode.type,
         };
 
+        // Only allow secure introspection in Cloud mode, since Dev mode skips
+        // signature validation
+        if (this._mode.type === "cloud") {
+          try {
+            this.validateSignature(signature ?? undefined, "");
+
+            introspection = {
+              ...introspection,
+              signing_key_fallback_hash: this.hashedSigningKeyFallback,
+              signing_key_hash: this.hashedSigningKey,
+            };
+          } catch {
+            // Swallow signature validation error since we'll just return the
+            // insecure introspection
+          }
+        }
+
         return {
           status: 200,
           body: stringify(introspection),
@@ -1112,14 +1150,17 @@ export class InngestCommHandler<
     }
 
     try {
-      res = await this.fetch(registerURL.href, {
-        method: "POST",
-        body: stringify(body),
-        headers: {
-          ...getHeaders(),
-          Authorization: `Bearer ${this.hashedSigningKey}`,
+      res = await fetchWithAuthFallback({
+        authToken: this.hashedSigningKey,
+        authTokenFallback: this.hashedSigningKeyFallback,
+        fetch: this.fetch,
+        url: registerURL.href,
+        options: {
+          method: "POST",
+          body: stringify(body),
+          headers: getHeaders(),
+          redirect: "follow",
         },
-        redirect: "follow",
       });
     } catch (err: unknown) {
       this.log("error", err);
@@ -1177,6 +1218,16 @@ export class InngestCommHandler<
       this.client["inngestApi"].setSigningKey(this.signingKey);
     }
 
+    if (this.env[envKeys.InngestSigningKeyFallback]) {
+      if (!this.signingKeyFallback) {
+        this.signingKeyFallback = String(
+          this.env[envKeys.InngestSigningKeyFallback]
+        );
+      }
+
+      this.client["inngestApi"].setSigningKeyFallback(this.signingKeyFallback);
+    }
+
     if (!this.client["eventKeySet"]() && this.env[envKeys.InngestEventKey]) {
       this.client.setEventKey(String(this.env[envKeys.InngestEventKey]));
     }
@@ -1217,6 +1268,7 @@ export class InngestCommHandler<
       body,
       allowExpiredSignatures: this.allowExpiredSignatures,
       signingKey: this.signingKey,
+      signingKeyFallback: this.signingKeyFallback,
     });
   }
 
@@ -1282,7 +1334,7 @@ class RequestSignature {
     return delta > 1000 * 60 * 5;
   }
 
-  public verifySignature({
+  #verifySignature({
     body,
     signingKey,
     allowExpiredSignatures,
@@ -1314,6 +1366,32 @@ class RequestSignature {
       throw new Error("Invalid signature");
     }
   }
+
+  public verifySignature({
+    body,
+    signingKey,
+    signingKeyFallback,
+    allowExpiredSignatures,
+  }: {
+    body: unknown;
+    signingKey: string;
+    signingKeyFallback: string | undefined;
+    allowExpiredSignatures: boolean;
+  }): void {
+    try {
+      this.#verifySignature({ body, signingKey, allowExpiredSignatures });
+    } catch (err) {
+      if (!signingKeyFallback) {
+        throw err;
+      }
+
+      this.#verifySignature({
+        body,
+        signingKey: signingKeyFallback,
+        allowExpiredSignatures,
+      });
+    }
+  }
 }
 
 /**

packages/inngest/src/helpers/consts.ts

@@ -17,6 +17,7 @@ export enum queryKeys {
 
 export enum envKeys {
   InngestSigningKey = "INNGEST_SIGNING_KEY",
+  InngestSigningKeyFallback = "INNGEST_SIGNING_KEY_FALLBACK",
   InngestEventKey = "INNGEST_EVENT_KEY",
 
   /**

packages/inngest/src/helpers/env.ts

@@ -124,7 +124,7 @@ export interface ModeOptions {
 }
 
 export class Mode {
-  private readonly type: "cloud" | "dev";
+  public readonly type: "cloud" | "dev";
 
   /**
    * Whether the mode was explicitly set, or inferred from other sources.

packages/inngest/src/helpers/net.test.ts

@@ -0,0 +1,65 @@
+import fetchMock from "jest-fetch-mock";
+import { fetchWithAuthFallback } from "./net";
+
+describe("fetchWithAuthFallback", () => {
+  beforeEach(() => {
+    fetchMock.resetMocks();
+  });
+
+  it("should make a fetch request with the provided auth token", async () => {
+    fetchMock.mockResponseOnce(JSON.stringify({ data: "12345" }));
+
+    const response = await fetchWithAuthFallback({
+      authToken: "testToken",
+      fetch: fetchMock as typeof fetch,
+      url: "https://example.com",
+    });
+
+    expect(fetchMock).toHaveBeenCalledWith("https://example.com", {
+      headers: {
+        Authorization: "Bearer testToken",
+      },
+    });
+    expect(response.status).toEqual(200);
+  });
+
+  it("should retry with the fallback token if the first request fails with 401", async () => {
+    fetchMock.mockResponses(
+      [JSON.stringify({}), { status: 401 }],
+      [JSON.stringify({ data: "12345" }), { status: 200 }]
+    );
+
+    const response = await fetchWithAuthFallback({
+      authToken: "testToken",
+      authTokenFallback: "fallbackToken",
+      fetch: fetchMock as typeof fetch,
+      url: "https://example.com",
+    });
+
+    expect(fetchMock).toHaveBeenNthCalledWith(1, "https://example.com", {
+      headers: {
+        Authorization: "Bearer testToken",
+      },
+    });
+    expect(fetchMock).toHaveBeenNthCalledWith(2, "https://example.com", {
+      headers: {
+        Authorization: "Bearer fallbackToken",
+      },
+    });
+    expect(response.status).toEqual(200);
+  });
+
+  it("should not retry with the fallback token if the first request fails with a non-401/403 status", async () => {
+    fetchMock.mockResponseOnce(JSON.stringify({}), { status: 500 });
+
+    const response = await fetchWithAuthFallback({
+      authToken: "testToken",
+      authTokenFallback: "fallbackToken",
+      fetch: fetchMock as typeof fetch,
+      url: "https://example.com",
+    });
+
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+    expect(response.status).toEqual(500);
+  });
+});