feat(security): multiple encryption options, API tokens, easier setup

.github/workflows/test.yml

@@ -102,6 +102,7 @@ jobs:
 
   e2e:
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     env:
       DATABASE_URL: postgresql://hatchet:hatchet@127.0.0.1:5431/hatchet
 
@@ -153,14 +154,17 @@ jobs:
           . .env
           set +a
 
-          go run ./cmd/hatchet-admin seed
+          go run ./cmd/hatchet-admin quickstart
 
-          go run ./cmd/hatchet-engine &
-          go run ./cmd/hatchet-api &
+          go run ./cmd/hatchet-engine --config ./generated/ &
+          go run ./cmd/hatchet-api --config ./generated/ &
           sleep 30
 
       - name: Test
-        run: go test -tags e2e ./... -p 1 -v -failfast
+        run: |
+          export HATCHET_CLIENT_TOKEN="$(go run ./cmd/hatchet-admin token create --config ./generated/ --tenant-id 707d0855-80ab-4e1f-a156-f1c4546cbf52)"
+
+          go test -tags e2e ./... -p 1 -v -failfast
 
       - name: Teardown
         run: docker compose down

.gitignore

@@ -85,3 +85,5 @@ tmp
 
 postgres-data
 rabbitmq.conf
+
+*encryption-keys

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/gitguardian/ggshield
+    rev: v1.23.0
+    hooks:
+      - id: ggshield
+        language_version: python3
+        stages: [commit]

CONTRIBUTING.md

@@ -8,7 +8,7 @@
 - `docker-compose`
 - [`Taskfile`](https://taskfile.dev/installation/)
 - The following additional devtools:
-  - `protoc`: `brew install protobuf@25` 
+  - `protoc`: `brew install protobuf@25`
   - `caddy` and `nss`: `brew install caddy nss`
 
 ### Setup
@@ -19,7 +19,9 @@
 
 3. Generate certificates needed for communicating between the Hatchet client and engine: `task generate-certs`
 
-4. Create environment variables:
+4. Generate keysets for encryption: `task generate-local-encryption-keys`
+
+5. Create environment variables:
 
 ```sh
 alias randstring='f() { openssl rand -base64 69 | tr -d "\n" | tr -d "=+/" | cut -c1-$1 };f'
@@ -30,29 +32,55 @@ SERVER_TLS_CERT_FILE=./hack/dev/certs/cluster.pem
 SERVER_TLS_KEY_FILE=./hack/dev/certs/cluster.key
 SERVER_TLS_ROOT_CA_FILE=./hack/dev/certs/ca.cert
 
+SERVER_ENCRYPTION_MASTER_KEYSET_FILE=./hack/dev/encryption-keys/master.key
+SERVER_ENCRYPTION_JWT_PRIVATE_KEYSET_FILE=./hack/dev/encryption-keys/private_ec256.key
+SERVER_ENCRYPTION_JWT_PUBLIC_KEYSET_FILE=./hack/dev/encryption-keys/public_ec256.key
+
 SERVER_PORT=8080
 SERVER_URL=https://app.dev.hatchet-tools.com
 
 SERVER_AUTH_COOKIE_SECRETS="$(randstring 16) $(randstring 16)"
 SERVER_AUTH_COOKIE_DOMAIN=app.dev.hatchet-tools.com
 SERVER_AUTH_COOKIE_INSECURE=false
 SERVER_AUTH_SET_EMAIL_VERIFIED=true
+
+SERVER_LOGGER_LEVEL=debug
+SERVER_LOGGER_FORMAT=console
+DATABASE_LOGGER_LEVEL=debug
+DATABASE_LOGGER_FORMAT=console
 EOF
 ```
 
-5. Migrate the database: `task prisma-migrate`
+6. Migrate the database: `task prisma-migrate`
 
-6. Generate all files: `task generate`
+7. Generate all files: `task generate`
 
-7. Seed the database: `task seed-dev`
+8. Seed the database: `task seed-dev`
 
-8. Start the Hatchet engine, API server, dashboard, and Prisma studio:
+9. Start the Hatchet engine, API server, dashboard, and Prisma studio:
 
 ```sh
 task start-dev
 ```
 
-10. To create and test workflows, run the examples in the `./examples` directory. You will need to add the tenant (output from the `task seed-dev` command) to the `.env` file in each example directory.
+### Creating and testing workflows
+
+To create and test workflows, run the examples in the `./examples` directory.
+
+You will need to add the tenant (output from the `task seed-dev` command) to the `.env` file in each example directory. An example `.env` file for the `./examples/simple` directory can be generated via:
+
+```sh
+alias get_token='go run ./cmd/hatchet-admin token create --name local --tenant-id 707d0855-80ab-4e1f-a156-f1c4546cbf52'
+
+cat > ./examples/simple/.env <<EOF
+HATCHET_CLIENT_TENANT_ID=707d0855-80ab-4e1f-a156-f1c4546cbf52
+HATCHET_CLIENT_TLS_ROOT_CA_FILE=../../hack/dev/certs/ca.cert
+HATCHET_CLIENT_TLS_SERVER_NAME=cluster
+HATCHET_CLIENT_TOKEN="$(get_token)"
+EOF
+```
+
+This example can then be run via `go run main.go` from the `./examples/simple` directory.
 
 ### Logging
 
@@ -83,3 +111,37 @@ OTEL_EXPORTER_OTLP_HEADERS=<optional-headers>
 # optional
 OTEL_EXPORTER_OTLP_ENDPOINT=<collector-url>
 ```
+
+### CloudKMS
+
+CloudKMS can be used to generate master encryption keys:
+
+```
+gcloud kms keyrings create "development" --location "global"
+gcloud kms keys create "development" --location "global" --keyring "development" --purpose "encryption"
+gcloud kms keys list --location "global" --keyring "development"
+```
+
+From the last step, copy the Key URI and set the following environment variable:
+
+```
+SERVER_ENCRYPTION_CLOUDKMS_KEY_URI=gcp-kms://projects/<PROJECT>/locations/global/keyRings/development/cryptoKeys/development
+```
+
+Generate a service account in GCP which can encrypt/decrypt on CloudKMS, then download a service account JSON file and set it via:
+
+```
+SERVER_ENCRYPTION_CLOUDKMS_CREDENTIALS_JSON='{...}'
+```
+
+## Issues
+
+### Query engine leakage
+
+Sometimes the spawned query engines from Prisma don't get killed when hot reloading. You can run `task kill-query-engines` on OSX to kill the query engines.
+
+Make sure you call `.Disconnect` on the database config object when writing CLI commands which interact with the database. If you don't, and you try to wrap these CLI commands in a new command, it will never exit, for example:
+
+```
+export HATCHET_CLIENT_TOKEN="$(go run ./cmd/hatchet-admin token create --tenant-id <tenant>)"
+```

Taskfile.yaml

@@ -1,9 +1,6 @@
 version: "3"
 
 tasks:
-  write-default-env:
-    cmds:
-      - sh ./hack/dev/write-default-env.sh
   set-etc-hosts:
     cmds:
       - sudo sh ./hack/dev/manage-hosts.sh add 127.0.0.1 app.dev.hatchet-tools.com
@@ -46,7 +43,13 @@ tasks:
       - task: generate-api-client
   generate-certs:
     cmds:
-      - sh ./hack/dev/generate-temporal-certs.sh ./hack/dev/certs
+      - sh ./hack/dev/generate-x509-certs.sh ./hack/dev/certs
+  generate-local-encryption-keys:
+    cmds:
+      - sh ./hack/dev/generate-local-encryption-keys.sh ./hack/dev/encryption-keys
+  generate-dev-api-token:
+    cmds:
+      - sh ./hack/dev/generate-dev-api-token.sh
   generate-api-server:
     cmds:
       - sh ./hack/oas/generate-server.sh

api-contracts/dispatcher/dispatcher.proto

@@ -15,17 +15,14 @@ service Dispatcher {
 }
 
 message WorkerRegisterRequest {
-    // the tenant id
-    string tenantId = 1;
-
     // the name of the worker
-    string workerName = 2;
+    string workerName = 1;
 
     // a list of actions that this worker can run
-    repeated string actions = 3;
+    repeated string actions = 2;
 
     // (optional) the services for this worker
-    repeated string services = 4;
+    repeated string services = 3;
 }
 
 message WorkerRegisterResponse {
@@ -74,19 +71,13 @@ message AssignedAction {
 }
 
 message WorkerListenRequest {
-    // the tenant id
-    string tenantId = 1;
-
     // the id of the worker
-    string workerId = 2;
+    string workerId = 1;
 }
 
 message WorkerUnsubscribeRequest {
-    // the tenant id to unsubscribe from
-    string tenantId = 1;
-
     // the id of the worker
-    string workerId = 2;
+    string workerId = 1;
 }
 
 message WorkerUnsubscribeResponse {
@@ -105,34 +96,31 @@ enum ActionEventType {
 }
 
 message ActionEvent {
-    // the tenant id
-    string tenantId = 1;
-
     // the id of the worker
-    string workerId = 2;
+    string workerId = 1;
 
     // the id of the job
-    string jobId = 3;
+    string jobId = 2;
 
     // the job run id
-    string jobRunId = 4;
+    string jobRunId = 3;
 
     // the id of the step
-    string stepId = 5;
+    string stepId = 4;
 
     // the step run id
-    string stepRunId = 6;
+    string stepRunId = 5;
 
     // the action id
-    string actionId = 7;
+    string actionId = 6;
 
-    google.protobuf.Timestamp eventTimestamp = 8;
+    google.protobuf.Timestamp eventTimestamp = 7;
 
     // the step event type
-    ActionEventType eventType = 9;
+    ActionEventType eventType = 8;
 
     // the event payload
-    string eventPayload = 10;
+    string eventPayload = 9;
 }
 
 message ActionEventResponse {

api-contracts/events/events.proto

@@ -30,28 +30,22 @@ message Event {
 }
 
 message PushEventRequest {
-    // the tenant id
-    string tenantId = 1;
-
     // the key for the event
-    string key = 2;
+    string key = 1;
 
     // the payload for the event
-    string payload = 3;
+    string payload = 2;
 
     // when the event was generated
-    google.protobuf.Timestamp eventTimestamp = 4;
+    google.protobuf.Timestamp eventTimestamp = 3;
 }
 
 message ListEventRequest {
-    // (required) the tenant id
-    string tenantId = 1;
-
     // (optional) the number of events to skip
-    int32 offset = 2; 
+    int32 offset = 1; 
 
     // (optional) the key for the event
-    string key = 3;
+    string key = 2;
 }
 
 message ListEventResponse {
@@ -60,9 +54,6 @@ message ListEventResponse {
 }
 
 message ReplayEventRequest {
-    // the tenant id
-    string tenantId = 1;
-
     // the event id to replay
-    string eventId = 2;
+    string eventId = 1;
 }

api-contracts/openapi/components/schemas/_index.yaml

@@ -108,3 +108,11 @@ WorkerList:
   $ref: "./worker.yaml#/WorkerList"
 Worker:
   $ref: "./worker.yaml#/Worker"
+APIToken:
+  $ref: "./api_tokens.yaml#/APIToken"
+CreateAPITokenRequest:
+  $ref: "./api_tokens.yaml#/CreateAPITokenRequest"
+CreateAPITokenResponse:
+  $ref: "./api_tokens.yaml#/CreateAPITokenResponse"
+ListAPITokensResponse:
+  $ref: "./api_tokens.yaml#/ListAPITokensResponse"

api-contracts/openapi/components/schemas/api_tokens.yaml

@@ -0,0 +1,45 @@
+APIToken:
+  type: object
+  properties:
+    metadata:
+      $ref: "./metadata.yaml#/APIResourceMeta"
+    name:
+      type: string
+      description: The name of the API token.
+      maxLength: 255
+    expiresAt:
+      type: string
+      format: date-time
+      description: When the API token expires.
+  required:
+    - metadata
+    - name
+    - expiresAt
+
+CreateAPITokenRequest:
+  type: object
+  properties:
+    name:
+      type: string
+      description: A name for the API token.
+      maxLength: 255
+  required:
+    - name
+
+CreateAPITokenResponse:
+  type: object
+  properties:
+    token:
+      type: string
+      description: The API token.
+  required:
+    - token
+
+ListAPITokensResponse:
+  properties:
+    pagination:
+      $ref: "./metadata.yaml#/PaginationResponse"
+    rows:
+      items:
+        $ref: "#/APIToken"
+      type: array

api-contracts/openapi/openapi.yaml

@@ -48,6 +48,10 @@ paths:
     $ref: "./paths/tenant/tenant.yaml#/invites"
   /api/v1/tenants/{tenant}/invites/{tenant-invite}:
     $ref: "./paths/tenant/tenant.yaml#/inviteScoped"
+  /api/v1/tenants/{tenant}/api-tokens:
+    $ref: "./paths/api-tokens/api_tokens.yaml#/withTenant"
+  /api/v1/api-tokens/{api-token}:
+    $ref: "./paths/api-tokens/api_tokens.yaml#/revoke"
   /api/v1/tenants/{tenant}/events:
     $ref: "./paths/event/event.yaml#/withTenant"
   /api/v1/tenants/{tenant}/events/replay: