From 84c1f73c1b3479b19a5e9a77371d7d0712ba3848 Mon Sep 17 00:00:00 2001 From: Lakshan Perera Date: Tue, 31 Jan 2023 07:39:24 +0700 Subject: [PATCH 1/4] feat: use edge runtime for functions serve --- cmd/functions.go | 10 +- internal/functions/serve/serve.go | 138 ++++++++++++++++++++++++- internal/functions/serve/serve_test.go | 2 +- internal/utils/misc.go | 31 ++++-- 4 files changed, 168 insertions(+), 13 deletions(-) diff --git a/cmd/functions.go b/cmd/functions.go index f7294267f..772392d55 100644 --- a/cmd/functions.go +++ b/cmd/functions.go @@ -95,18 +95,23 @@ var ( } envFilePath string + serveAll bool functionsServeCmd = &cobra.Command{ Use: "serve ", Short: "Serve a Function locally", - Args: cobra.ExactArgs(1), + Args: cobra.RangeArgs(0, 1), RunE: func(cmd *cobra.Command, args []string) error { ctx, _ := signal.NotifyContext(cmd.Context(), os.Interrupt) // Fallback to config if user did not set the flag. if !cmd.Flags().Changed("no-verify-jwt") { noVerifyJWT = nil } - return serve.Run(ctx, args[0], envFilePath, noVerifyJWT, importMapPath, afero.NewOsFs()) + slug := "" + if len(args) > 1 { + slug = args[0] + } + return serve.Run(ctx, slug, envFilePath, noVerifyJWT, importMapPath, serveAll, afero.NewOsFs()) }, } ) @@ -120,6 +125,7 @@ func init() { functionsServeCmd.Flags().BoolVar(noVerifyJWT, "no-verify-jwt", false, "Disable JWT verification for the Function.") functionsServeCmd.Flags().StringVar(&envFilePath, "env-file", "", "Path to an env file to be populated to the Function environment.") functionsServeCmd.Flags().StringVar(&importMapPath, "import-map", "", "Path to import map file.") + functionsServeCmd.Flags().BoolVar(&serveAll, "all", false, "Serve all functions (caution: Experimental feature)") functionsDownloadCmd.Flags().StringVar(&projectRef, "project-ref", "", "Project ref of the Supabase project.") functionsCmd.AddCommand(functionsDeleteCmd) functionsCmd.AddCommand(functionsDeployCmd) diff --git a/internal/functions/serve/serve.go b/internal/functions/serve/serve.go index 687952206..7b42bbc78 100644 --- a/internal/functions/serve/serve.go +++ b/internal/functions/serve/serve.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "os" "path/filepath" "strconv" @@ -40,7 +41,11 @@ func ParseEnvFile(envFilePath string) ([]string, error) { return env, nil } -func Run(ctx context.Context, slug string, envFilePath string, noVerifyJWT *bool, importMapPath string, fsys afero.Fs) error { +func Run(ctx context.Context, slug string, envFilePath string, noVerifyJWT *bool, importMapPath string, serveAll bool, fsys afero.Fs) error { + if serveAll { + return runServeAll(ctx, envFilePath, noVerifyJWT, importMapPath, fsys) + } + // 1. Sanity checks. { if err := utils.LoadConfigFS(fsys); err != nil { @@ -216,3 +221,134 @@ func Run(ctx context.Context, slug string, envFilePath string, noVerifyJWT *bool fmt.Println("Stopped serving " + utils.Bold(localFuncDir)) return nil } + +func runServeAll(ctx context.Context, envFilePath string, noVerifyJWT *bool, importMapPath string, fsys afero.Fs) error { + // 1. Sanity checks. + { + if err := utils.LoadConfigFS(fsys); err != nil { + return err + } + if err := utils.AssertSupabaseDbIsRunning(); err != nil { + return err + } + if envFilePath != "" { + if _, err := fsys.Stat(envFilePath); err != nil { + return fmt.Errorf("Failed to read env file: %w", err) + } + } + if importMapPath != "" { + // skip + } else if f, err := fsys.Stat(utils.FallbackImportMapPath); err == nil && !f.IsDir() { + importMapPath = utils.FallbackImportMapPath + } + if importMapPath != "" { + if _, err := fsys.Stat(importMapPath); err != nil { + return fmt.Errorf("Failed to read import map: %w", err) + } + } + } + + // 2. Parse user defined env + userEnv, err := ParseEnvFile(envFilePath) + if err != nil { + return err + } + + // 3. Start container + { + _ = utils.Docker.ContainerRemove(ctx, utils.DenoRelayId, types.ContainerRemoveOptions{ + RemoveVolumes: true, + Force: true, + }) + + env := []string{ + "JWT_SECRET=" + utils.JWTSecret, + "SUPABASE_URL=http://" + utils.KongId + ":8000", + "SUPABASE_ANON_KEY=" + utils.AnonKey, + "SUPABASE_SERVICE_ROLE_KEY=" + utils.ServiceRoleKey, + "SUPABASE_DB_URL=postgresql://postgres:postgres@localhost:" + strconv.FormatUint(uint64(utils.Config.Db.Port), 10) + "/postgres", + } + verifyJWTEnv := "VERIFY_JWT=true" + if noVerifyJWT != nil { + verifyJWTEnv = "VERIFY_JWT=false" + } + env = append(env, verifyJWTEnv) + + cwd, err := os.Getwd() + if err != nil { + return err + } + + binds := []string{filepath.Join(cwd, utils.FunctionsDir) + ":" + relayFuncDir + ":ro,z"} + // If a import map path is explcitly provided, mount it as a separate file + if importMapPath != "" { + binds = append(binds, filepath.Join(cwd, importMapPath)+":"+customDockerImportMapPath+":ro,z") + } + + // bind deno cache directory + cachePath, err := utils.GetDenoCachePath() + if err != nil { + return err + } + binds = append(binds, cachePath+":/root/.cache/deno:rw,z") + + containerID, err := utils.DockerStart( + ctx, + container.Config{ + Image: utils.EdgeRuntimeImage, + Env: append(env, userEnv...), + Cmd: []string{"start", "--dir", relayFuncDir, "-p", "8081"}, + OpenStdin: true, + AttachStdin: true, + AttachStderr: true, + AttachStdout: true, + Tty: true, + }, + container.HostConfig{ + Binds: binds, + }, + utils.DenoRelayId, + ) + if err != nil { + return err + } + + fmt.Println("Serving " + utils.Bold(utils.FunctionsDir)) + + // TODO: pipe the OS signals to the container + resp, err := utils.Docker.ContainerAttach(ctx, containerID, types.ContainerAttachOptions{ + Stream: true, + Stdin: true, + Stdout: true, + Stderr: true, + Logs: true, + }) + if err != nil { + return err + } + + go func() { + _, _ = io.Copy(os.Stdout, resp.Reader) + }() + + statusCh, errCh := utils.Docker.ContainerWait(ctx, containerID, container.WaitConditionNotRunning) + select { + case err := <-errCh: + if err != nil { + return err + } + case <-statusCh: + } + + go func() { + <-ctx.Done() + if ctx.Err() != nil { + utils.DockerRemove(utils.DenoRelayId) + } + }() + } + + fmt.Println("Stopped serving " + utils.Bold(utils.FunctionsDir)) + return nil + +} diff --git a/internal/functions/serve/serve_test.go b/internal/functions/serve/serve_test.go index 50cba2fbc..845bbea02 100644 --- a/internal/functions/serve/serve_test.go +++ b/internal/functions/serve/serve_test.go @@ -35,7 +35,7 @@ func TestServeCommand(t *testing.T) { Post("/v" + utils.Docker.ClientVersion() + "/containers"). Reply(http.StatusServiceUnavailable) // Run test - err := Run(context.Background(), "test-func", "", nil, "", fsys) + err := Run(context.Background(), "test-func", "", nil, "", false, fsys) // Check error assert.ErrorContains(t, err, "request returned Service Unavailable for API route and version http://localhost/v1.41/containers/supabase_deno_relay_serve/exec") assert.Empty(t, apitest.ListUnmatchedRequests()) diff --git a/internal/utils/misc.go b/internal/utils/misc.go index 4d6941f7b..ff7637a89 100644 --- a/internal/utils/misc.go +++ b/internal/utils/misc.go @@ -28,15 +28,16 @@ const ( Pg14Image = "supabase/postgres:14.1.0.89" Pg15Image = "supabase/postgres:15.1.0.33" // Append to ServiceImages when adding new dependencies below - KongImage = "library/kong:2.8.1" - InbucketImage = "inbucket/inbucket:3.0.3" - PostgrestImage = "postgrest/postgrest:v10.1.1.20221215" - DifferImage = "supabase/pgadmin-schema-diff:cli-0.0.5" - MigraImage = "djrobstep/migra:3.0.1621480950" - PgmetaImage = "supabase/postgres-meta:v0.60.3" - StudioImage = "supabase/studio:20230127-6bfd87b" - DenoRelayImage = "supabase/deno-relay:v1.5.0" - ImageProxyImage = "darthsim/imgproxy:v3.8.0" + KongImage = "library/kong:2.8.1" + InbucketImage = "inbucket/inbucket:3.0.3" + PostgrestImage = "postgrest/postgrest:v10.1.1.20221215" + DifferImage = "supabase/pgadmin-schema-diff:cli-0.0.5" + MigraImage = "djrobstep/migra:3.0.1621480950" + PgmetaImage = "supabase/postgres-meta:v0.60.3" + StudioImage = "supabase/studio:20230127-6bfd87b" + DenoRelayImage = "supabase/deno-relay:v1.5.0" + ImageProxyImage = "darthsim/imgproxy:v3.8.0" + EdgeRuntimeImage = "supabase/edge-runtime:latest" // Update initial schemas in internal/utils/templates/initial_schemas when // updating any one of these. GotrueImage = "supabase/gotrue:v2.40.1" @@ -322,6 +323,18 @@ func GetDenoPath() (string, error) { return denoPath, nil } +func GetDenoCachePath() (string, error) { + if len(DenoPathOverride) > 0 { + return DenoPathOverride, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + cachePath := filepath.Join(home, ".supabase", "deno_cache") + return cachePath, nil +} + func InstallOrUpgradeDeno(ctx context.Context, fsys afero.Fs) error { denoPath, err := GetDenoPath() if err != nil { From c5b32917b0fdc35041161119d1fbd78f79b76f1e Mon Sep 17 00:00:00 2001 From: Lakshan Perera Date: Wed, 1 Feb 2023 10:59:53 +0700 Subject: [PATCH 2/4] fix: use docker volume for deno cache --- internal/functions/serve/serve.go | 8 ++------ internal/utils/misc.go | 12 ------------ 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/internal/functions/serve/serve.go b/internal/functions/serve/serve.go index 7b42bbc78..758c0d773 100644 --- a/internal/functions/serve/serve.go +++ b/internal/functions/serve/serve.go @@ -285,12 +285,8 @@ func runServeAll(ctx context.Context, envFilePath string, noVerifyJWT *bool, imp binds = append(binds, filepath.Join(cwd, importMapPath)+":"+customDockerImportMapPath+":ro,z") } - // bind deno cache directory - cachePath, err := utils.GetDenoCachePath() - if err != nil { - return err - } - binds = append(binds, cachePath+":/root/.cache/deno:rw,z") + // bind deno cache volume + binds = append(binds, utils.DenoRelayId+":/root/.cache/deno:rw,z") containerID, err := utils.DockerStart( ctx, diff --git a/internal/utils/misc.go b/internal/utils/misc.go index ff7637a89..b24d32231 100644 --- a/internal/utils/misc.go +++ b/internal/utils/misc.go @@ -323,18 +323,6 @@ func GetDenoPath() (string, error) { return denoPath, nil } -func GetDenoCachePath() (string, error) { - if len(DenoPathOverride) > 0 { - return DenoPathOverride, nil - } - home, err := os.UserHomeDir() - if err != nil { - return "", err - } - cachePath := filepath.Join(home, ".supabase", "deno_cache") - return cachePath, nil -} - func InstallOrUpgradeDeno(ctx context.Context, fsys afero.Fs) error { denoPath, err := GetDenoPath() if err != nil { From 1140dce36ebf2304702baaebaa9e8163fb072777 Mon Sep 17 00:00:00 2001 From: Lakshan Perera Date: Wed, 1 Feb 2023 12:32:39 +0700 Subject: [PATCH 3/4] fix: use edge-runtime 1.0.7 image --- internal/utils/misc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/utils/misc.go b/internal/utils/misc.go index b24d32231..9bd756123 100644 --- a/internal/utils/misc.go +++ b/internal/utils/misc.go @@ -37,7 +37,7 @@ const ( StudioImage = "supabase/studio:20230127-6bfd87b" DenoRelayImage = "supabase/deno-relay:v1.5.0" ImageProxyImage = "darthsim/imgproxy:v3.8.0" - EdgeRuntimeImage = "supabase/edge-runtime:latest" + EdgeRuntimeImage = "supabase/edge-runtime:v1.0.7" // Update initial schemas in internal/utils/templates/initial_schemas when // updating any one of these. GotrueImage = "supabase/gotrue:v2.40.1" From 4aa793a9e990e1b2bc2d8e07dc3c900c55ddb285 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Wed, 1 Feb 2023 13:24:19 +0800 Subject: [PATCH 4/4] chore: support binds in run with stream --- internal/db/diff/diff.go | 2 + internal/functions/serve/serve.go | 68 ++++++------------------------- internal/utils/docker.go | 10 +++-- 3 files changed, 22 insertions(+), 58 deletions(-) diff --git a/internal/db/diff/diff.go b/internal/db/diff/diff.go index 655f80337..7c1691c04 100644 --- a/internal/db/diff/diff.go +++ b/internal/db/diff/diff.go @@ -82,6 +82,7 @@ func DiffSchema(ctx context.Context, source, target string, schema []string, p u utils.DifferImage, nil, args, + nil, stream.Stdout(), stream.Stderr(), ); err != nil { @@ -95,6 +96,7 @@ func DiffSchema(ctx context.Context, source, target string, schema []string, p u utils.DifferImage, nil, append([]string{"--schema", s}, args...), + nil, stream.Stdout(), stream.Stderr(), ); err != nil { diff --git a/internal/functions/serve/serve.go b/internal/functions/serve/serve.go index 758c0d773..88f30e394 100644 --- a/internal/functions/serve/serve.go +++ b/internal/functions/serve/serve.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io" "os" "path/filepath" "strconv" @@ -279,69 +278,28 @@ func runServeAll(ctx context.Context, envFilePath string, noVerifyJWT *bool, imp return err } - binds := []string{filepath.Join(cwd, utils.FunctionsDir) + ":" + relayFuncDir + ":ro,z"} + binds := []string{ + filepath.Join(cwd, utils.FunctionsDir) + ":" + relayFuncDir + ":ro,z", + utils.DenoRelayId + ":/root/.cache/deno:rw,z", + } // If a import map path is explcitly provided, mount it as a separate file if importMapPath != "" { binds = append(binds, filepath.Join(cwd, importMapPath)+":"+customDockerImportMapPath+":ro,z") } - // bind deno cache volume - binds = append(binds, utils.DenoRelayId+":/root/.cache/deno:rw,z") - - containerID, err := utils.DockerStart( - ctx, - container.Config{ - Image: utils.EdgeRuntimeImage, - Env: append(env, userEnv...), - Cmd: []string{"start", "--dir", relayFuncDir, "-p", "8081"}, - OpenStdin: true, - AttachStdin: true, - AttachStderr: true, - AttachStdout: true, - Tty: true, - }, - container.HostConfig{ - Binds: binds, - }, - utils.DenoRelayId, - ) - if err != nil { - return err - } - fmt.Println("Serving " + utils.Bold(utils.FunctionsDir)) - // TODO: pipe the OS signals to the container - resp, err := utils.Docker.ContainerAttach(ctx, containerID, types.ContainerAttachOptions{ - Stream: true, - Stdin: true, - Stdout: true, - Stderr: true, - Logs: true, - }) - if err != nil { + if err := utils.DockerRunOnceWithStream( + ctx, + utils.EdgeRuntimeImage, + append(env, userEnv...), + []string{"start", "--dir", relayFuncDir, "-p", "8081"}, + binds, + os.Stdout, + os.Stderr, + ); err != nil { return err } - - go func() { - _, _ = io.Copy(os.Stdout, resp.Reader) - }() - - statusCh, errCh := utils.Docker.ContainerWait(ctx, containerID, container.WaitConditionNotRunning) - select { - case err := <-errCh: - if err != nil { - return err - } - case <-statusCh: - } - - go func() { - <-ctx.Done() - if ctx.Err() != nil { - utils.DockerRemove(utils.DenoRelayId) - } - }() } fmt.Println("Stopped serving " + utils.Bold(utils.FunctionsDir)) diff --git a/internal/utils/docker.go b/internal/utils/docker.go index 3a35151ae..428d3adbf 100644 --- a/internal/utils/docker.go +++ b/internal/utils/docker.go @@ -302,11 +302,11 @@ func DockerRunOnce(ctx context.Context, image string, env []string, cmd []string stderr = os.Stderr } var out bytes.Buffer - err := DockerRunOnceWithStream(ctx, image, env, cmd, &out, stderr) + err := DockerRunOnceWithStream(ctx, image, env, cmd, nil, &out, stderr) return out.String(), err } -func DockerRunOnceWithStream(ctx context.Context, image string, env []string, cmd []string, stdout, stderr io.Writer) error { +func DockerRunOnceWithStream(ctx context.Context, image string, env, cmd, binds []string, stdout, stderr io.Writer) error { // Cannot rely on docker's auto remove because // 1. We must inspect exit code after container stops // 2. Context cancellation may happen after start @@ -314,7 +314,11 @@ func DockerRunOnceWithStream(ctx context.Context, image string, env []string, cm Image: image, Env: env, Cmd: cmd, - }, container.HostConfig{}, "") + }, container.HostConfig{ + Binds: binds, + // Allows containerized functions on Linux to reach host OS + ExtraHosts: []string{"host.docker.internal:host-gateway"}, + }, "") if err != nil { return err }