WIP: feat: initial implementation of the LookupWatch API

.github/labeler.yml

@@ -1,23 +1,35 @@
 ---
-area/dependencies:
+"area/api devtools":
+  - "cmd/spicedb/developer.go"
+"area/api http":
+  - "internal/gateway/**/*"
+"area/api v0":
+  - "internal/services/v0/**/*"
+"area/api v1":
+  - "internal/services/v1/**/*"
+  - "internal/services/v1alpha1/**/*"
+"area/cli":
+  - "cmd/**/*"
+"area/dashboard":
+  - "internal/dashboard/**/*"
+"area/datastore":
+  - "internal/datastore/**/*"
+"area/dependencies":
   - "Dockerfile"
   - "go.mod"
   - "go.sum"
-area/tooling:
-  - "**/*_test.go"
-  - ".github/**/*"
-  - ".*"
-  - "Dockerfile*"
-area/docs:
+"area/dispatch":
+  - "internal/dispatch/**/*"
+"area/docs":
   - "CODE-OF-CONDUCT.md"
   - "CONTRIBUTING.md"
   - "DCO"
   - "LICENSE"
   - "README.md"
-"area/api v0":
-  - "internal/services/v0/**/*"
-"area/api v1":
-  - "internal/services/v1/**/*"
-  - "internal/services/v1alpha1/**/*"
-"area/datastore":
-  - "internal/datastore/**/*"
+"area/schema":
+  - "pkg/schemadsl/**/*"
+"area/tooling":
+  - "**/*_test.go"
+  - ".github/**/*"
+  - ".*"
+  - "Dockerfile*"

.github/workflows/build.yaml

@@ -73,6 +73,7 @@ jobs:
           # chaosd doesn't yet include time modification, install it separately
           git clone https://github.com/chaos-mesh/chaos-mesh/
           pushd chaos-mesh
+          git reset --hard 72d2bc17febc7f2a4a10c97417f11c5eb1d86a13
           CGO_ENABLED=1 go build ./cmd/watchmaker/
           popd
           mv ./chaos-mesh/watchmaker ./watchmaker
@@ -115,9 +116,9 @@ jobs:
           go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26
           go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1
           go install github.com/envoyproxy/protoc-gen-validate@v0.6.1
-      - uses: "bufbuild/buf-setup-action@v0.3.1"
+      - uses: "bufbuild/buf-setup-action@v0.6.0"
         with:
-          version: "0.52.0"
+          version: "1.0.0-rc8"
       - name: "Generate & Diff Protos"
         run: "./buf.gen.yaml && git diff && bash -c '[ $(git status --porcelain | tee /dev/fd/2 | wc -c) -eq 0 ]'"
       - uses: "bufbuild/buf-breaking-action@v0.4.0"

.github/workflows/lint.yaml

@@ -18,8 +18,7 @@ jobs:
       - name: "Install linting tools"
         run: |
           # This is done before checking out, as to not modify go.mod
-          go install mvdan.cc/gofumpt/gofumports@latest
-          go install github.com/mgechev/revive@latest
+          go install mvdan.cc/gofumpt/gofumports@v0.1.1
           go install golang.org/x/tools/cmd/stringer@latest
       - uses: "actions/checkout@v2"
       - name: "Go Mod Tidy"
@@ -34,7 +33,7 @@ jobs:
           fi
       - uses: "golangci/golangci-lint-action@v2"
         with:
-          version: "v1.42"
+          version: "v1.43"
           skip-go-installation: true
           skip-pkg-cache: true
           skip-build-cache: false

.github/workflows/release.yaml

@@ -18,6 +18,11 @@ jobs:
           registry: "quay.io"
           username: "${{ secrets.QUAYIO_USER }}"
           password: "${{ secrets.QUAYIO_PASSWORD }}"
+      - uses: "docker/login-action@v1"
+        with:
+          registry: "ghcr.io"
+          username: "${{ github.actor }}"
+          password: "${{ secrets.GHCR_TOKEN }}"
       - uses: "actions/setup-go@v2"
         with:
           go-version: "^1.17"

.golangci.yaml

@@ -1,4 +1,6 @@
 ---
+run:
+  timeout: "5m"
 output:
   sort-results: true
 linters-settings:

.goreleaser.yml

@@ -58,6 +58,7 @@ brews:
 dockers:
   - image_templates:
       - &amd_image "quay.io/authzed/spicedb:v{{ .Version }}-amd64"
+      - &amd_image_gh "ghcr.io/authzed/spicedb:v{{ .Version }}-amd64"
     dockerfile: &dockerfile "Dockerfile.release"
     goos: "linux"
     goarch: "amd64"
@@ -66,6 +67,7 @@ dockers:
       - "--platform=linux/amd64"
   - image_templates:
       - &arm_image "quay.io/authzed/spicedb:v{{ .Version }}-arm64"
+      - &arm_image_gh "ghcr.io/authzed/spicedb:v{{ .Version }}-arm64"
     dockerfile: *dockerfile
     goos: "linux"
     goarch: "arm64"
@@ -81,6 +83,14 @@ docker_manifests:
     image_templates:
       - *amd_image
       - *arm_image
+  - name_template: "ghcr.io/authzed/spicedb:v{{ .Version }}"
+    image_templates:
+      - *amd_image_gh
+      - *arm_image_gh
+  - name_template: "ghcr.io/authzed/spicedb:latest"
+    image_templates:
+      - *amd_image_gh
+      - *arm_image_gh
 checksum:
   name_template: "checksums.txt"
 snapshot:
@@ -92,4 +102,4 @@ release:
   prerelease: "auto"
   footer: |
     ## Docker Images
-    This release is available at `quay.io/authzed/spicedb:{{ .Version }}`
+    This release is available at `quay.io/authzed/spicedb:v{{ .Version }}` and `ghcr.io/authzed/spicedb:v{{ .Version }}`

Dockerfile

@@ -1,10 +1,4 @@
-FROM golang:1.17.2-alpine3.13 AS spicedb-builder
-
-ARG GRPC_HEALTH_PROBE_VERSION=0.3.6
-RUN apk add curl
-RUN curl -Lo /go/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64
-RUN chmod +x /go/bin/grpc_health_probe
-
+FROM golang:1.17.3-alpine3.13 AS spicedb-builder
 WORKDIR /go/src/app
 
 # Prepare dependencies
@@ -14,9 +8,9 @@ RUN go mod download
 COPY . .
 RUN go build ./cmd/spicedb/
 
-FROM alpine:3.14.2
+FROM alpine:3.15.0
 
 RUN [ ! -e /etc/nsswitch.conf ] && echo 'hosts: files dns' > /etc/nsswitch.conf
-COPY --from=spicedb-builder /go/bin/grpc_health_probe /usr/local/bin/
+COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:v0.4.6 /ko-app/grpc-health-probe /usr/local/bin/grpc_health_probe
 COPY --from=spicedb-builder /go/src/app/spicedb /usr/local/bin/spicedb
 ENTRYPOINT ["spicedb"]

Dockerfile.release

@@ -1,14 +1,5 @@
 # vim: syntax=dockerfile
-FROM alpine AS grpc
-ARG TARGETARCH
-ARG GRPC_HEALTH_PROBE_VERSION=0.3.6
-RUN apk update && \
-	apk add curl && \
-	curl -Lo /grpc_health_probe \
-	https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v$GRPC_HEALTH_PROBE_VERSION/grpc_health_probe-linux-$TARGETARCH && \
-	chmod +x /grpc_health_probe
-
 FROM gcr.io/distroless/base
-COPY --from=grpc /grpc_health_probe /usr/local/bin/grpc_health_probe
+COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:v0.4.6 /ko-app/grpc-health-probe /usr/local/bin/grpc_health_probe
 COPY spicedb /usr/local/bin/spicedb
 ENTRYPOINT ["spicedb"]

README.md

@@ -7,14 +7,14 @@
 [![Discord Server](https://img.shields.io/discord/844600078504951838?color=7289da&logo=discord "Discord Server")](https://discord.gg/jTysUaxXzM)
 [![Twitter](https://img.shields.io/twitter/follow/authzed?color=%23179CF0&logo=twitter&style=flat-square "@authzed on Twitter")](https://twitter.com/authzed)
 
-SpiceDB is a [Zanzibar]-inspired database that stores, computes, and validates application permissions.
+SpiceDB is a database system for managing security-critical application permissions.
 
 Developers create a schema that models their permissions requirements and use a [client library] to apply the schema to the database, insert data into the database, and query the data to efficiently check permissions in their applications.
 
 Features that distinguish SpiceDB from other systems include:
 
 - Expressive [gRPC] and [HTTP] APIs for checking permissions, listing access, and powering devtools
-- An architecture faithful to the [Google Zanzibar] paper, including resistance to the [New Enemy Problem]
+- An architecture faithful to [Google's Zanzibar paper], including resistance to the [New Enemy Problem]
 - An intuitive and expressive [schema language] complete with a [playground] dev environment
 - A powerful graph engine that supports distributed, parallel evaluation
 - Pluggable storage that supports [in-memory], [PostgreSQL], and [CockroachDB]
@@ -25,8 +25,7 @@ See [CONTRIBUTING.md] for instructions on how to contribute and perform common t
 [client library]: https://docs.authzed.com/reference/api#client-libraries
 [gRPC]: https://buf.build/authzed/api
 [HTTP]: https://petstore.swagger.io/?url=https://raw.githubusercontent.com/authzed/authzed-go/main/proto/apidocs.swagger.json
-[Zanzibar]: https://authzed.com/blog/what-is-zanzibar/
-[Google Zanzibar]: https://authzed.com/blog/what-is-zanzibar/
+[Google's Zanzibar paper]: https://authzed.com/blog/what-is-zanzibar/
 [New Enemy Problem]: https://authzed.com/blog/new-enemies/
 [schema language]: https://docs.authzed.com/guides/schema
 [playground]: https://play.authzed.com
@@ -81,14 +80,14 @@ SpiceDB is also available as a container image:
 
 ```sh
 docker pull quay.io/authzed/spicedb:latest
-docker run quay.io/authzed/spicedb serve --grpc-preshared-key "somerandomkeyhere" --grpc-no-tls --http-no-tls
+docker run quay.io/authzed/spicedb serve --grpc-preshared-key "somerandomkeyhere"
 ```
 
 SpiceDB supports environment variables. You can replace any command's argument with an environment variable by adding the `SPICEDB` prefix.  
 For example `--log-level` becomes `SPICEDB_LOG_LEVEL`.
 
 ```sh
-docker run -e SPICEDB_GRPC_PRESHARED_KEY=somerandomkeyhere -e SPICEDB_GRPC_NO_TLS=1 -e SPICEDB_HTTP_NO_TLS=1 quay.io/authzed/spicedb serve
+docker run -e SPICEDB_GRPC_PRESHARED_KEY=somerandomkeyhere quay.io/authzed/spicedb serve
 ```
 
 For production usage, we **highly** recommend using a tag that corresponds to the [latest release], rather than `latest`.
@@ -98,7 +97,7 @@ For production usage, we **highly** recommend using a tag that corresponds to th
 ### Running SpiceDB locally
 
 ```sh
-spicedb serve --grpc-preshared-key "somerandomkeyhere" --grpc-no-tls --http-no-tls
+spicedb serve --grpc-preshared-key "somerandomkeyhere"
 ```
 
 Visit [http://localhost:8080](http://localhost:8080) to see next steps, including loading the schema
@@ -114,7 +113,7 @@ By using unique tokens in each of your application's integration tests, they can
 
 A [SpiceDB GitHub action] is also available to run SpiceDB as part of your integration test workflows.
 
-[Bearer Token]: https://docs.authzed.com/concepts/terminology/#api-token
+[Bearer Token]: https://docs.authzed.com/reference/api#authentication
 [SpiceDB GitHub action]: https://github.com/authzed/action-spicedb
 
 ### Developing your own schema

buf.gen.yaml

@@ -1,4 +1,4 @@
-#!/usr/bin/env -S ./proto/buf-generate.sh
+#!/usr/bin/env -S buf generate -o internal/proto proto/internal --template
 ---
 version: "v1"
 managed:

cmd/spicedb/developer.go

@@ -4,8 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net"
-	"net/http"
 	"os"
 	"os/signal"
 
@@ -18,6 +16,7 @@ import (
 	grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/jzelinskie/cobrautil"
 	"github.com/jzelinskie/stringz"
+	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
@@ -30,15 +29,17 @@ import (
 
 func registerDeveloperServiceCmd(rootCmd *cobra.Command) {
 	developerServiceCmd := &cobra.Command{
-		Use:   "serve-devtools",
-		Short: "runs the developer tools service",
-		Long:  "Serves the authzed.api.v0.DeveloperService which is used for development tooling such as the Authzed Playground",
-		Run:   developerServiceRun,
-		Args:  cobra.ExactArgs(0),
+		Use:     "serve-devtools",
+		Short:   "runs the developer tools service",
+		Long:    "Serves the authzed.api.v0.DeveloperService which is used for development tooling such as the Authzed Playground",
+		PreRunE: defaultPreRunE,
+		Run:     developerServiceRun,
+		Args:    cobra.ExactArgs(0),
 	}
 
-	cobrautil.RegisterGrpcServerFlags(developerServiceCmd.Flags())
-	cobrautil.RegisterMetricsServerFlags(developerServiceCmd.Flags())
+	cobrautil.RegisterGrpcServerFlags(developerServiceCmd.Flags(), "grpc", "gRPC", ":50051", true)
+	cobrautil.RegisterHttpServerFlags(developerServiceCmd.Flags(), "metrics", "metrics", ":9090", true)
+	cobrautil.RegisterHttpServerFlags(developerServiceCmd.Flags(), "http", "download", ":8443", false)
 
 	developerServiceCmd.Flags().String("share-store", "inmemory", "kind of share store to use")
 	developerServiceCmd.Flags().String("share-store-salt", "", "salt for share store hashing")
@@ -47,13 +48,12 @@ func registerDeveloperServiceCmd(rootCmd *cobra.Command) {
 	developerServiceCmd.Flags().String("s3-bucket", "", "s3 bucket name for s3 share store")
 	developerServiceCmd.Flags().String("s3-endpoint", "", "s3 endpoint for s3 share store")
 	developerServiceCmd.Flags().String("s3-region", "auto", "s3 region for s3 share store")
-	developerServiceCmd.Flags().String("download-addr", ":8443", "address to listen for download requests")
 
 	rootCmd.AddCommand(developerServiceCmd)
 }
 
 func developerServiceRun(cmd *cobra.Command, args []string) {
-	grpcServer, err := cobrautil.GrpcServerFromFlags(cmd, grpc.ChainUnaryInterceptor(
+	grpcServer, err := cobrautil.GrpcServerFromFlags(cmd, "grpc", grpc.ChainUnaryInterceptor(
 		grpclog.UnaryServerInterceptor(grpczerolog.InterceptorLogger(log.Logger)),
 		otelgrpc.UnaryServerInterceptor(),
 		grpcprom.UnaryServerInterceptor,
@@ -70,39 +70,33 @@ func developerServiceRun(cmd *cobra.Command, args []string) {
 	registerDeveloperGrpcServices(grpcServer, shareStore)
 
 	go func() {
-		addr := cobrautil.MustGetStringExpanded(cmd, "grpc-addr")
-		l, err := net.Listen("tcp", addr)
-		if err != nil {
-			log.Fatal().Str("addr", addr).Msg("failed to listen on addr for gRPC server")
-		}
-
-		log.Info().Str("addr", addr).Msg("gRPC server started listening")
-		if err := grpcServer.Serve(l); err != nil {
+		if err := cobrautil.GrpcListenFromFlags(cmd, "grpc", grpcServer, zerolog.InfoLevel); err != nil {
 			log.Warn().Err(err).Msg("gRPC service did not shutdown cleanly")
 		}
 	}()
 
-	metricsrv := cobrautil.MetricsServerFromFlags(cmd)
+	// Start the metrics endpoint.
+	metricsSrv := cobrautil.HttpServerFromFlags(cmd, "metrics")
+	metricsSrv.Handler = metricsHandler()
 	go func() {
-		log.Info().Str("addr", metricsrv.Addr).Msg("metrics server started listening")
-		if err := metricsrv.ListenAndServe(); err != http.ErrServerClosed {
+		if err := cobrautil.HttpListenFromFlags(cmd, "metrics", metricsSrv, zerolog.InfoLevel); err != nil {
 			log.Fatal().Err(err).Msg("failed while serving metrics")
 		}
 	}()
 
-	downloadSrv := v0svc.NewHTTPDownloadServer(cobrautil.MustGetString(cmd, "download-addr"), shareStore)
+	// start the http download api
+	downloadSrv := cobrautil.HttpServerFromFlags(cmd, "http")
+	downloadSrv.Handler = v0svc.NewHTTPDownloadServer(cobrautil.MustGetString(cmd, "http-addr"), shareStore).Handler
 	go func() {
-		log.Info().Str("addr", downloadSrv.Addr).Msg("download server started listening")
-		if err := downloadSrv.ListenAndServe(); err != http.ErrServerClosed {
-			log.Fatal().Err(err).Msg("failed while serving http api")
+		if err := cobrautil.HttpListenFromFlags(cmd, "http", downloadSrv, zerolog.InfoLevel); err != nil {
+			log.Fatal().Err(err).Msg("failed while serving download http api")
 		}
 	}()
-
 	signalctx, _ := signal.NotifyContext(context.Background(), os.Interrupt)
 	<-signalctx.Done()
 	log.Info().Msg("received interrupt")
 	grpcServer.GracefulStop()
-	if err := metricsrv.Close(); err != nil {
+	if err := metricsSrv.Close(); err != nil {
 		log.Fatal().Err(err).Msg("failed while shutting down metrics server")
 	}
 }