feat(security): multiple encryption options, API tokens, easier setup

.github/workflows/test.yml

@@ -102,6 +102,7 @@ jobs:
 
   e2e:
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     env:
       DATABASE_URL: postgresql://hatchet:hatchet@127.0.0.1:5431/hatchet
 
@@ -153,14 +154,17 @@ jobs:
           . .env
           set +a
 
-          go run ./cmd/hatchet-admin seed
+          go run ./cmd/hatchet-admin quickstart
 
-          go run ./cmd/hatchet-engine &
-          go run ./cmd/hatchet-api &
+          go run ./cmd/hatchet-engine --config ./generated/ &
+          go run ./cmd/hatchet-api --config ./generated/ &
           sleep 30
 
       - name: Test
-        run: go test -tags e2e ./... -p 1 -v -failfast
+        run: |
+          export HATCHET_CLIENT_TOKEN="$(go run ./cmd/hatchet-admin token create --config ./generated/ --tenant-id 707d0855-80ab-4e1f-a156-f1c4546cbf52)"
+
+          go test -tags e2e ./... -p 1 -v -failfast
 
       - name: Teardown
         run: docker compose down

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/gitguardian/ggshield
+    rev: v1.23.0
+    hooks:
+      - id: ggshield
+        language_version: python3
+        stages: [commit]

CONTRIBUTING.md

@@ -8,7 +8,7 @@
 - `docker-compose`
 - [`Taskfile`](https://taskfile.dev/installation/)
 - The following additional devtools:
-  - `protoc`: `brew install protobuf@25` 
+  - `protoc`: `brew install protobuf@25`
   - `caddy` and `nss`: `brew install caddy nss`
 
 ### Setup
@@ -83,3 +83,37 @@ OTEL_EXPORTER_OTLP_HEADERS=<optional-headers>
 # optional
 OTEL_EXPORTER_OTLP_ENDPOINT=<collector-url>
 ```
+
+### CloudKMS
+
+CloudKMS can be used to generate master encryption keys:
+
+```
+gcloud kms keyrings create "development" --location "global"
+gcloud kms keys create "development" --location "global" --keyring "development" --purpose "encryption"
+gcloud kms keys list --location "global" --keyring "development"
+```
+
+From the last step, copy the Key URI and set the following environment variable:
+
+```
+SERVER_ENCRYPTION_CLOUDKMS_KEY_URI=gcp-kms://projects/<PROJECT>/locations/global/keyRings/development/cryptoKeys/development
+```
+
+Generate a service account in GCP which can encrypt/decrypt on CloudKMS, then download a service account JSON file and set it via:
+
+```
+SERVER_ENCRYPTION_CLOUDKMS_CREDENTIALS_JSON='{...}'
+```
+
+## Issues
+
+### Query engine leakage
+
+Sometimes the spawned query engines from Prisma don't get killed when hot reloading. You can run `task kill-query-engines` on OSX to kill the query engines.
+
+Make sure you call `.Disconnect` on the database config object when writing CLI commands which interact with the database. If you don't, and you try to wrap these CLI commands in a new command, it will never exit, for example:
+
+```
+export HATCHET_CLIENT_TOKEN="$(go run ./cmd/hatchet-admin token create --tenant-id <tenant>)"
+```

api-contracts/dispatcher/dispatcher.proto

@@ -15,17 +15,14 @@ service Dispatcher {
 }
 
 message WorkerRegisterRequest {
-    // the tenant id
-    string tenantId = 1;
-
     // the name of the worker
-    string workerName = 2;
+    string workerName = 1;
 
     // a list of actions that this worker can run
-    repeated string actions = 3;
+    repeated string actions = 2;
 
     // (optional) the services for this worker
-    repeated string services = 4;
+    repeated string services = 3;
 }
 
 message WorkerRegisterResponse {
@@ -74,19 +71,13 @@ message AssignedAction {
 }
 
 message WorkerListenRequest {
-    // the tenant id
-    string tenantId = 1;
-
     // the id of the worker
-    string workerId = 2;
+    string workerId = 1;
 }
 
 message WorkerUnsubscribeRequest {
-    // the tenant id to unsubscribe from
-    string tenantId = 1;
-
     // the id of the worker
-    string workerId = 2;
+    string workerId = 1;
 }
 
 message WorkerUnsubscribeResponse {
@@ -105,34 +96,31 @@ enum ActionEventType {
 }
 
 message ActionEvent {
-    // the tenant id
-    string tenantId = 1;
-
     // the id of the worker
-    string workerId = 2;
+    string workerId = 1;
 
     // the id of the job
-    string jobId = 3;
+    string jobId = 2;
 
     // the job run id
-    string jobRunId = 4;
+    string jobRunId = 3;
 
     // the id of the step
-    string stepId = 5;
+    string stepId = 4;
 
     // the step run id
-    string stepRunId = 6;
+    string stepRunId = 5;
 
     // the action id
-    string actionId = 7;
+    string actionId = 6;
 
-    google.protobuf.Timestamp eventTimestamp = 8;
+    google.protobuf.Timestamp eventTimestamp = 7;
 
     // the step event type
-    ActionEventType eventType = 9;
+    ActionEventType eventType = 8;
 
     // the event payload
-    string eventPayload = 10;
+    string eventPayload = 9;
 }
 
 message ActionEventResponse {

api-contracts/events/events.proto

@@ -30,28 +30,22 @@ message Event {
 }
 
 message PushEventRequest {
-    // the tenant id
-    string tenantId = 1;
-
     // the key for the event
-    string key = 2;
+    string key = 1;
 
     // the payload for the event
-    string payload = 3;
+    string payload = 2;
 
     // when the event was generated
-    google.protobuf.Timestamp eventTimestamp = 4;
+    google.protobuf.Timestamp eventTimestamp = 3;
 }
 
 message ListEventRequest {
-    // (required) the tenant id
-    string tenantId = 1;
-
     // (optional) the number of events to skip
-    int32 offset = 2; 
+    int32 offset = 1; 
 
     // (optional) the key for the event
-    string key = 3;
+    string key = 2;
 }
 
 message ListEventResponse {
@@ -60,9 +54,6 @@ message ListEventResponse {
 }
 
 message ReplayEventRequest {
-    // the tenant id
-    string tenantId = 1;
-
     // the event id to replay
-    string eventId = 2;
+    string eventId = 1;
 }

api-contracts/openapi/components/schemas/_index.yaml

@@ -108,3 +108,11 @@ WorkerList:
   $ref: "./worker.yaml#/WorkerList"
 Worker:
   $ref: "./worker.yaml#/Worker"
+APIToken:
+  $ref: "./api_tokens.yaml#/APIToken"
+CreateAPITokenRequest:
+  $ref: "./api_tokens.yaml#/CreateAPITokenRequest"
+CreateAPITokenResponse:
+  $ref: "./api_tokens.yaml#/CreateAPITokenResponse"
+ListAPITokensResponse:
+  $ref: "./api_tokens.yaml#/ListAPITokensResponse"

api-contracts/openapi/components/schemas/api_tokens.yaml

@@ -0,0 +1,45 @@
+APIToken:
+  type: object
+  properties:
+    metadata:
+      $ref: "./metadata.yaml#/APIResourceMeta"
+    name:
+      type: string
+      description: The name of the API token.
+      maxLength: 255
+    expiresAt:
+      type: string
+      format: date-time
+      description: When the API token expires.
+  required:
+    - metadata
+    - name
+    - expiresAt
+
+CreateAPITokenRequest:
+  type: object
+  properties:
+    name:
+      type: string
+      description: A name for the API token.
+      maxLength: 255
+  required:
+    - name
+
+CreateAPITokenResponse:
+  type: object
+  properties:
+    token:
+      type: string
+      description: The API token.
+  required:
+    - token
+
+ListAPITokensResponse:
+  properties:
+    pagination:
+      $ref: "./metadata.yaml#/PaginationResponse"
+    rows:
+      items:
+        $ref: "#/APIToken"
+      type: array

api-contracts/openapi/openapi.yaml

@@ -48,6 +48,10 @@ paths:
     $ref: "./paths/tenant/tenant.yaml#/invites"
   /api/v1/tenants/{tenant}/invites/{tenant-invite}:
     $ref: "./paths/tenant/tenant.yaml#/inviteScoped"
+  /api/v1/tenants/{tenant}/api-tokens:
+    $ref: "./paths/api-tokens/api_tokens.yaml#/withTenant"
+  /api/v1/api-tokens/{api-token}:
+    $ref: "./paths/api-tokens/api_tokens.yaml#/revoke"
   /api/v1/tenants/{tenant}/events:
     $ref: "./paths/event/event.yaml#/withTenant"
   /api/v1/tenants/{tenant}/events/replay:

api-contracts/openapi/paths/api-tokens/api_tokens.yaml

@@ -0,0 +1,111 @@
+withTenant:
+  post:
+    x-resources: ["tenant"]
+    description: Create an API token for a tenant
+    operationId: api-token:create
+    parameters:
+      - description: The tenant id
+        in: path
+        name: tenant
+        required: true
+        schema:
+          type: string
+          format: uuid
+          minLength: 36
+          maxLength: 36
+    requestBody:
+      content:
+        application/json:
+          schema:
+            $ref: "../../components/schemas/_index.yaml#/CreateAPITokenRequest"
+    responses:
+      "200":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/CreateAPITokenResponse"
+        description: Successfully retrieved the workflows
+      "400":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: A malformed or bad request
+      "403":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: Forbidden
+    summary: Create API Token
+    tags:
+      - API Token
+  get:
+    x-resources: ["tenant"]
+    description: List API tokens for a tenant
+    operationId: api-token:list
+    parameters:
+      - description: The tenant id
+        in: path
+        name: tenant
+        required: true
+        schema:
+          type: string
+          format: uuid
+          minLength: 36
+          maxLength: 36
+    responses:
+      "200":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/ListAPITokensResponse"
+        description: Successfully retrieved the workflows
+      "400":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: A malformed or bad request
+      "403":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: Forbidden
+    summary: List API Tokens
+    tags:
+      - API Token
+revoke:
+  post:
+    x-resources: ["tenant", "api-token"]
+    description: Revoke an API token for a tenant
+    operationId: api-token:update:revoke
+    parameters:
+      - description: The API token
+        in: path
+        name: api-token
+        required: true
+        schema:
+          type: string
+          format: uuid
+          minLength: 36
+          maxLength: 36
+    responses:
+      "204":
+        description: Successfully revoked the token
+      "400":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: A malformed or bad request
+      "403":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: Forbidden
+    summary: Revoke API Token
+    tags:
+      - API Token