From 9a84ab3b24c694b5c8aac7df0b37db29c22a6e97 Mon Sep 17 00:00:00 2001 From: Evan Corkrean <62527488+corkrean@users.noreply.github.com> Date: Tue, 17 Jun 2025 08:56:08 -0600 Subject: [PATCH 1/2] initial data doc additions and refactor --- pages/spicedb/_meta.json | 1 + pages/spicedb/concepts/datastores.mdx | 8 +- pages/spicedb/data/_meta.json | 6 + .../spicedb/{ops => data}/bulk-operations.mdx | 0 pages/spicedb/data/migrations.mdx | 40 ++++++ pages/spicedb/data/writing-relationships.mdx | 116 ++++++++++++++++++ pages/spicedb/ops/_meta.json | 1 - 7 files changed, 167 insertions(+), 5 deletions(-) create mode 100644 pages/spicedb/data/_meta.json rename pages/spicedb/{ops => data}/bulk-operations.mdx (100%) create mode 100644 pages/spicedb/data/migrations.mdx create mode 100644 pages/spicedb/data/writing-relationships.mdx diff --git a/pages/spicedb/_meta.json b/pages/spicedb/_meta.json index 6da5a260..8cdf38d0 100644 --- a/pages/spicedb/_meta.json +++ b/pages/spicedb/_meta.json @@ -3,6 +3,7 @@ "concepts": "Concepts", "modeling": "Modeling & Integrating", "ops": "Operations", + "data": "Managing Data", "api": "API Reference", "links": "Links" } diff --git a/pages/spicedb/concepts/datastores.mdx b/pages/spicedb/concepts/datastores.mdx index 49baf2d6..769178d9 100644 --- a/pages/spicedb/concepts/datastores.mdx +++ b/pages/spicedb/concepts/datastores.mdx @@ -4,11 +4,11 @@ import { Callout } from 'nextra/components' In order to reduce operational complexity, SpiceDB leverages existing, popular systems for persisting data. -Each datastore implementation comes with various trade-offs. +AuthzZed has standardized our managed services on CockroachDB, but we give self-hosted users the option to pick the datastore that best suits their operational requirements. -- [CockroachDB](#cockroachdb) - Recommended for planet-scale, multi-region deployments -- [Cloud Spanner](#cloud-spanner) - Recommended for Google Cloud deployments -- [PostgreSQL](#postgresql) - Recommended for single-region deployments +- [CockroachDB](#cockroachdb) - Recommended for self-hosted deployments with high throughput and/or multi-region requirements +- [Cloud Spanner](#cloud-spanner) - Recommended for self-hosted Google Cloud deployments +- [PostgreSQL](#postgresql) - Recommended for self-hosted single-region deployments - [MySQL](#mysql) - Not recommended; only use if you cannot use PostgreSQL - [memdb](#memdb) - Recommended for local development and integration testing against applications diff --git a/pages/spicedb/data/_meta.json b/pages/spicedb/data/_meta.json new file mode 100644 index 00000000..4ce10e73 --- /dev/null +++ b/pages/spicedb/data/_meta.json @@ -0,0 +1,6 @@ +{ + "writing-relationships": "Writing Relationships", + "data-syncronization": "Data Syncronization", + "bulk-operations": "Bulk Importing Relationships", + "migrations": "Migrations" + } \ No newline at end of file diff --git a/pages/spicedb/ops/bulk-operations.mdx b/pages/spicedb/data/bulk-operations.mdx similarity index 100% rename from pages/spicedb/ops/bulk-operations.mdx rename to pages/spicedb/data/bulk-operations.mdx diff --git a/pages/spicedb/data/migrations.mdx b/pages/spicedb/data/migrations.mdx new file mode 100644 index 00000000..af83eeea --- /dev/null +++ b/pages/spicedb/data/migrations.mdx @@ -0,0 +1,40 @@ +import { Callout } from 'nextra/components' + +# Migrations + +## Migrating from SpiceDB to SpiceDB + + + Migrating data at the underlying database level is not recommended and impossible in some cases. Using tools like `pg_dump`/`pg_restore` will break SpiceDB MVCC. Additionally, if you are migrating to a SpiceDB with a different datastore type (e.g. Postgres -> CockroachDB), you ***must*** use the SpiceDB APIs (`exportBulk`/`importBulk` or `zed backup`) to backup and restore. + + +The options provided below enable you to consistently migrate between datastores with minimal downtime. + +All options utilize the [Zed CLI tool](https://github.com/authzed/zed?tab=readme-ov-file#zed). + +### Write Downtime Migration + +A simple migration that will incur write downtime (not read downtime) for the duration between starting the export and finishing the import. + +1. Spin up your new SpiceDB. +2. Stop writes to your old SpiceDB. +3. Run `zed backup create ` against the old SpiceDB. +4. Run `zed backup restore ` against the new SpiceDB with the backup file generated by the previous command. +5. Switch reads to the new SpiceDB. +6. Start writing to the new SpiceDB. + +### Near Zero Downtime Migration + +This option involves more complexity than the above option. + +1. Spin up your new SpiceDB. +2. Run `zed backup create ` against your old SpiceDB. +3. Run `zed backup restore ` against the new SpiceDB with the backup file generated by the previous command. +4. Obtain the zed token that points to the backup revision with `zed backup parse-revision `. +5. Using the [watch API](https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.WatchRequest), consume relationship changes from the old SpiceDB and write them to the new SpiceDB. +You should provide the zed token that you obtained in the previous request as the `optional_start_cursor` in the `WatchRequest`. +You will need to build a small worker to consume changes and write them to the new SpiceDB. +The worker needs to continuously consume changes until you are ready to turn off writes on the old SpiceDB. +6. Stop writes to the old SpiceDB. +7. Wait for the worker to process all changes. (this is when write downtime will occur) +8. Switch reads and writes to the new SpiceDB. diff --git a/pages/spicedb/data/writing-relationships.mdx b/pages/spicedb/data/writing-relationships.mdx new file mode 100644 index 00000000..0d8f0716 --- /dev/null +++ b/pages/spicedb/data/writing-relationships.mdx @@ -0,0 +1,116 @@ +import { Tabs } from 'nextra/components' +import { Callout } from 'nextra/components' + +# Writing relationships + +This page will provide some practical recommendations for writing relationships to SpiceDB. +If you are interested in relationships as a concept, check out this [page](/spicedb/concepts/relationships). + +## Retries + +When making requests to SpiceDB, it's important to implement proper retry logic to handle transient failures. [SpiceDB APIs use gRPC*](/spicedb/getting-started/client-libraries), which can experience various types of temporary failures that can be resolved through retries. + +Retries are recommended for all gRPC methods, not just WriteRelationships. + +*SpiceDB can also expose an [HTTP API](/spicedb/getting-started/client-libraries#http-clients); however, gRPC is recommended. + +### Implementing Retry Policies + +You can implement your own retry policies using the gRPC Service Config. +Below, you will find a recommended Retry Policy. + +``` +"retryPolicy": { + "maxAttempts": 3, + "initialBackoff": "1s", + "maxBackoff": "4s", + "backoffMultiplier": 2, + "retryableStatusCodes": [ + 'UNAVAILABLE', 'RESOURCE_EXHAUSTED', 'DEADLINE_EXCEEDED', 'ABORTED', + ] +} +``` + +This retry policy configuration provides exponential backoff with the following behavior: + +**`maxAttempts: 3`** - Allows for a maximum of 3 total attempts (1 initial request + 2 retries). +This prevents infinite retry loops while giving sufficient opportunity for transient issues to resolve. + +**`initialBackoff: "1s"`** - Sets the initial delay to 1 second before the first retry attempt. +This gives the system time to recover from temporary issues. + +**`maxBackoff: "4s"`** - Caps the maximum delay between retries at 4 seconds to prevent excessively long waits that could impact user experience. + +**`backoffMultiplier: 2`** - Doubles the backoff time with each retry attempt. +Combined with the other settings, this creates a retry pattern of: 1s → 2s → 4s. + +**`retryableStatusCodes`** - Only retries on specific gRPC status codes that indicate transient failures: +-`UNAVAILABLE`: SpiceDB is temporarily unavailable +-`RESOURCE_EXHAUSTED`: SpiceDB is overloaded +-`DEADLINE_EXCEEDED`: Request timed out +-`ABORTED`: Operation was aborted, often due to conflicts that may resolve on retry + +You can find a python retry example [here](https://github.com/authzed/examples/blob/main/data/retry/main.py). + +## Writes: Touch vs Create + +A SpiceDB [relationship update](https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.RelationshipUpdate) can use one of three operation types `CREATE`, `TOUCH`, OR `DELETE`. +This section will cover `CREATE` and `TOUCH`. +You can read more about `DELETE` in [the section below](#deleting-relationships). + +### Understanding the Operations + +**`CREATE`** - Inserts a new relationship. +If the relationship already exists, the operation will fail with an error. + +**`TOUCH`** - Upserts a relationship. +If the relationship already exists, it will do nothing. +If it doesn't exist, it will create it. + +### Key Differences + +| Operation | Behavior on Existing Relationship | Performance | Use Case | +|-----------|-----------------------------------|-------------|----------| +| `CREATE` | Fails with error | Faster (single insert) | Initial relationship creation | +| `TOUCH` | Updates/overwrites | Slower (delete + insert) | Idempotent operations | + +### Special Considerations + +**Expiring Relationships:** When working with [expiring relationships](/spicedb/concepts/expiring-relationships), always use `TOUCH`. +If a relationship has expired but hasn't been garbage collected yet, using `CREATE` will return an error. + +**Error Handling:** When using `CREATE`, be prepared to handle duplicate relationship errors appropriately in your application logic. + +## Deleting Relationships + +SpiceDB provides two methods for deleting relationships: using the `WriteRelationships` API with the `DELETE` operation or using the `DeleteRelationships` API. +Each approach has different behaviors and use cases. + +### WriteRelationships with the `DELETE` Operation + +The [`WriteRelationships`](https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.WriteRelationshipsRequest) API supports a `DELETE` operation type that allows you to remove specific relationships as part of a batch of relationship updates. + +**`DELETE`** - Removes a relationship. +If the relationship does not exist, the operation will silently succeed (no-op). + +#### Characteristics + +- **Atomic Operations**: Can be combined with other relationship operations (`CREATE`, `TOUCH`) in a single atomic transaction +- **Granular Control**: Delete specific relationships alongside creating or updating others +- **Silent Failure**: Does not fail if the relationship doesn't exist +- **Batch Limit**: Subject to the same batch size limits as other `WriteRelationships` operations (1,000 updates by default) + +### DeleteRelationships API + +The [`DeleteRelationships`](https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.DeleteRelationshipsRequest) API is a dedicated method for bulk deletion of relationships based on filters rather than specifying individual relationships. + +#### Characteristics + +- **Filter-Based**: Delete relationships based on resource type, relation, subject type, or combinations thereof +- **Bulk Operations**: Can delete many relationships matching the filter criteria in a single call +- **Separate Transaction**: Operates independently from `WriteRelationships` +- **Efficient for Mass Deletion**: Optimized for removing large numbers of relationships + +## Bulk Import + +Check out [Bulk Importing Relationships](/spicedb/modeling/Data/writing-relationships) diff --git a/pages/spicedb/ops/_meta.json b/pages/spicedb/ops/_meta.json index e35888ee..38268186 100644 --- a/pages/spicedb/ops/_meta.json +++ b/pages/spicedb/ops/_meta.json @@ -1,6 +1,5 @@ { "observability": "Observability Tooling", "deploying-spicedb-operator": "Deploying the SpiceDB Operator", - "bulk-operations": "Bulk Importing Relationships", "secure-rag-pipelines": "Secure Your RAG Pipelines with Fine Grained Authorization" } From 4efae0c7259030a7ffc1178f2cac2b3c94ab9f66 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 18 Jun 2025 15:58:30 -0700 Subject: [PATCH 2/2] data/writing-relationships: fix broken link --- pages/spicedb/data/writing-relationships.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/spicedb/data/writing-relationships.mdx b/pages/spicedb/data/writing-relationships.mdx index 0d8f0716..7852d116 100644 --- a/pages/spicedb/data/writing-relationships.mdx +++ b/pages/spicedb/data/writing-relationships.mdx @@ -113,4 +113,4 @@ The [`DeleteRelationships`](https://buf.build/authzed/api/docs/main:authzed.api. ## Bulk Import -Check out [Bulk Importing Relationships](/spicedb/modeling/Data/writing-relationships) +Check out [Bulk Importing Relationships](./bulk-operations)