satellite/console: optional separate web app server

This change creates the ability to run a server separate from the
console web server to serve the front end app. You can run it with
`satellite run ui`. Since there are now potentially two servers instead
of one, the UI server has the option to act as a reverse proxy to the
api server for local development by setting `--console.address` to the
console backend address and `--console.backend-reverse-proxy` to the
console backend's http url. Also, a feature flag has been implemented
on the api server to retain the ability to serve the front end app. It
is toggled with `--console.frontend-enable`.

github issue: #5843

Change-Id: I0d30451a20636e3184110dbe28c8a2a8a9505804

cmd/satellite/main.go

@@ -100,6 +100,11 @@ var (
 		Short: "Run the satellite API",
 		RunE:  cmdAPIRun,
 	}
+	runUICmd = &cobra.Command{
+		Use:   "ui",
+		Short: "Run the satellite UI",
+		RunE:  cmdUIRun,
+	}
 	runRepairerCmd = &cobra.Command{
 		Use:   "repair",
 		Short: "Run the repair service",
@@ -366,6 +371,7 @@ func init() {
 	rootCmd.AddCommand(runCmd)
 	runCmd.AddCommand(runMigrationCmd)
 	runCmd.AddCommand(runAPICmd)
+	runCmd.AddCommand(runUICmd)
 	runCmd.AddCommand(runAdminCmd)
 	runCmd.AddCommand(runRepairerCmd)
 	runCmd.AddCommand(runAuditorCmd)
@@ -404,6 +410,7 @@ func init() {
 	process.Bind(runCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
 	process.Bind(runMigrationCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
 	process.Bind(runAPICmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
+	process.Bind(runUICmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
 	process.Bind(runAdminCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
 	process.Bind(runRepairerCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))
 	process.Bind(runAuditorCmd, &runCfg, defaults, cfgstruct.ConfDir(confDir), cfgstruct.IdentityDir(identityDir))

cmd/satellite/ui.go

@@ -0,0 +1,47 @@
+// Copyright (C) 2023 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/zeebo/errs"
+	"go.uber.org/zap"
+
+	"storj.io/private/process"
+	"storj.io/storj/satellite"
+)
+
+func cmdUIRun(cmd *cobra.Command, args []string) (err error) {
+	ctx, _ := process.Ctx(cmd)
+	log := zap.L()
+
+	runCfg.Debug.Address = *process.DebugAddrFlag
+
+	identity, err := runCfg.Identity.Load()
+	if err != nil {
+		log.Error("Failed to load identity.", zap.Error(err))
+		return errs.New("Failed to load identity: %+v", err)
+	}
+
+	satAddr := runCfg.Config.Contact.ExternalAddress
+	if satAddr == "" {
+		return errs.New("cannot run satellite ui if contact.external-address is not set")
+	}
+	apiAddress := runCfg.Config.Console.ExternalAddress
+	if apiAddress == "" {
+		apiAddress = runCfg.Config.Console.Address
+	}
+	peer, err := satellite.NewUI(log, identity, &runCfg.Config, process.AtomicLevel(cmd), satAddr, apiAddress)
+	if err != nil {
+		return err
+	}
+
+	if err := process.InitMetricsWithHostname(ctx, log, nil); err != nil {
+		log.Warn("Failed to initialize telemetry batcher on satellite api", zap.Error(err))
+	}
+
+	runError := peer.Run(ctx)
+	closeError := peer.Close()
+	return errs.Combine(runError, closeError)
+}

private/testplanet/satellite.go

@@ -66,6 +66,7 @@ type Satellite struct {
 
 	Core       *satellite.Core
 	API        *satellite.API
+	UI         *satellite.UI
 	Repairer   *satellite.Repairer
 	Auditor    *satellite.Auditor
 	Admin      *satellite.Admin
@@ -172,12 +173,17 @@ type Satellite struct {
 		Service *mailservice.Service
 	}
 
-	Console struct {
+	ConsoleBackend struct {
 		Listener net.Listener
 		Service  *console.Service
 		Endpoint *consoleweb.Server
 	}
 
+	ConsoleFrontend struct {
+		Listener net.Listener
+		Endpoint *consoleweb.Server
+	}
+
 	NodeStats struct {
 		Endpoint *nodestats.Endpoint
 	}
@@ -298,6 +304,11 @@ func (system *Satellite) Run(ctx context.Context) (err error) {
 	group.Go(func() error {
 		return errs2.IgnoreCanceled(system.API.Run(ctx))
 	})
+	if system.UI != nil {
+		group.Go(func() error {
+			return errs2.IgnoreCanceled(system.UI.Run(ctx))
+		})
+	}
 	group.Go(func() error {
 		return errs2.IgnoreCanceled(system.Repairer.Run(ctx))
 	})
@@ -519,6 +530,15 @@ func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int
 		return nil, errs.Wrap(err)
 	}
 
+	// only run if front-end endpoints on console back-end server are disabled.
+	var ui *satellite.UI
+	if !config.Console.FrontendEnable {
+		ui, err = planet.newUI(ctx, index, identity, config, api.ExternalAddress, api.Console.Listener.Addr().String())
+		if err != nil {
+			return nil, errs.Wrap(err)
+		}
+	}
+
 	adminPeer, err := planet.newAdmin(ctx, index, identity, db, metabaseDB, config, versionInfo)
 	if err != nil {
 		return nil, errs.Wrap(err)
@@ -548,19 +568,20 @@ func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int
 		peer.Mail.EmailReminders.TestSetLinkAddress("http://" + api.Console.Listener.Addr().String() + "/")
 	}
 
-	return createNewSystem(prefix, log, config, peer, api, repairerPeer, auditorPeer, adminPeer, gcBFPeer, rangedLoopPeer), nil
+	return createNewSystem(prefix, log, config, peer, api, ui, repairerPeer, auditorPeer, adminPeer, gcBFPeer, rangedLoopPeer), nil
 }
 
 // createNewSystem makes a new Satellite System and exposes the same interface from
 // before we split out the API. In the short term this will help keep all the tests passing
 // without much modification needed. However long term, we probably want to rework this
 // so it represents how the satellite will run when it is made up of many processes.
-func createNewSystem(name string, log *zap.Logger, config satellite.Config, peer *satellite.Core, api *satellite.API, repairerPeer *satellite.Repairer, auditorPeer *satellite.Auditor, adminPeer *satellite.Admin, gcBFPeer *satellite.GarbageCollectionBF, rangedLoopPeer *satellite.RangedLoop) *Satellite {
+func createNewSystem(name string, log *zap.Logger, config satellite.Config, peer *satellite.Core, api *satellite.API, ui *satellite.UI, repairerPeer *satellite.Repairer, auditorPeer *satellite.Auditor, adminPeer *satellite.Admin, gcBFPeer *satellite.GarbageCollectionBF, rangedLoopPeer *satellite.RangedLoop) *Satellite {
 	system := &Satellite{
 		Name:       name,
 		Config:     config,
 		Core:       peer,
 		API:        api,
+		UI:         ui,
 		Repairer:   repairerPeer,
 		Auditor:    auditorPeer,
 		Admin:      adminPeer,
@@ -655,6 +676,15 @@ func (planet *Planet) newAPI(ctx context.Context, index int, identity *identity.
 	return satellite.NewAPI(log, identity, db, metabaseDB, revocationDB, liveAccounting, rollupsWriteCache, &config, versionInfo, nil)
 }
 
+func (planet *Planet) newUI(ctx context.Context, index int, identity *identity.FullIdentity, config satellite.Config, satelliteAddr, consoleAPIAddr string) (_ *satellite.UI, err error) {
+	defer mon.Task()(&ctx)(&err)
+
+	prefix := "satellite-ui" + strconv.Itoa(index)
+	log := planet.log.Named(prefix)
+
+	return satellite.NewUI(log, identity, &config, nil, satelliteAddr, consoleAPIAddr)
+}
+
 func (planet *Planet) newAdmin(ctx context.Context, index int, identity *identity.FullIdentity, db satellite.DB, metabaseDB *metabase.DB, config satellite.Config, versionInfo version.Info) (_ *satellite.Admin, err error) {
 	defer mon.Task()(&ctx)(&err)
 

satellite/console/consoleweb/consoleapi/auth.go

@@ -53,7 +53,7 @@ type Auth struct {
 }
 
 // NewAuth is a constructor for api auth controller.
-func NewAuth(log *zap.Logger, service *console.Service, accountFreezeService *console.AccountFreezeService, mailService *mailservice.Service, cookieAuth *consolewebauth.CookieAuth, analytics *analytics.Service, satelliteName string, externalAddress string, letUsKnowURL string, termsAndConditionsURL string, contactInfoURL string, generalRequestURL string) *Auth {
+func NewAuth(log *zap.Logger, service *console.Service, accountFreezeService *console.AccountFreezeService, mailService *mailservice.Service, cookieAuth *consolewebauth.CookieAuth, analytics *analytics.Service, satelliteName, externalAddress, letUsKnowURL, termsAndConditionsURL, contactInfoURL, generalRequestURL string) *Auth {
 	return &Auth{
 		log:                       log,
 		ExternalAddress:           externalAddress,

satellite/console/consoleweb/server.go

@@ -15,6 +15,7 @@ import (
 	"mime"
 	"net"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -62,10 +63,14 @@ var (
 
 // Config contains configuration for console web server.
 type Config struct {
-	Address         string `help:"server address of the graphql api gateway and frontend app" devDefault:"127.0.0.1:0" releaseDefault:":10100"`
-	StaticDir       string `help:"path to static resources" default:""`
-	Watch           bool   `help:"whether to load templates on each request" default:"false" devDefault:"true"`
-	ExternalAddress string `help:"external endpoint of the satellite if hosted" default:""`
+	Address             string `help:"server address of the graphql api gateway and frontend app" devDefault:"127.0.0.1:0" releaseDefault:":10100"`
+	FrontendAddress     string `help:"server address of the front-end app" devDefault:"127.0.0.1:0" releaseDefault:":10200"`
+	ExternalAddress     string `help:"external endpoint of the satellite if hosted" default:""`
+	FrontendEnable      bool   `help:"feature flag to toggle whether console back-end server should also serve front-end endpoints" default:"true"`
+	BackendReverseProxy string `help:"the target URL of console back-end reverse proxy for local development when running a UI server" default:""`
+
+	StaticDir string `help:"path to static resources" default:""`
+	Watch     bool   `help:"whether to load templates on each request" default:"false" devDefault:"true"`
 
 	AuthToken       string `help:"auth token needed for access to registration token creation endpoint" default:"" testDefault:"very-secret-token"`
 	AuthTokenSecret string `help:"secret used to sign auth tokens" releaseDefault:"" devDefault:"my-suppa-secret-key"`
@@ -220,7 +225,7 @@ func NewServer(logger *zap.Logger, config Config, service *console.Service, oidc
 		packagePlans:      packagePlans,
 	}
 
-	logger.Debug("Starting Satellite UI.", zap.Stringer("Address", server.listener.Addr()))
+	logger.Debug("Starting Satellite Console server.", zap.Stringer("Address", server.listener.Addr()))
 
 	server.cookieAuth = consolewebauth.NewCookieAuth(consolewebauth.CookieSettings{
 		Name: "_tokenKey",
@@ -353,30 +358,26 @@ func NewServer(logger *zap.Logger, config Config, service *console.Service, oidc
 	analyticsRouter.HandleFunc("/event", analyticsController.EventTriggered).Methods(http.MethodPost, http.MethodOptions)
 	analyticsRouter.HandleFunc("/page", analyticsController.PageEventTriggered).Methods(http.MethodPost, http.MethodOptions)
 
-	if server.config.StaticDir != "" {
-		oidc := oidc.NewEndpoint(
-			server.nodeURL, server.config.ExternalAddress,
-			logger, oidcService, service,
-			server.config.OauthCodeExpiry, server.config.OauthAccessTokenExpiry, server.config.OauthRefreshTokenExpiry,
-		)
+	oidc := oidc.NewEndpoint(
+		server.nodeURL, server.config.ExternalAddress,
+		logger, oidcService, service,
+		server.config.OauthCodeExpiry, server.config.OauthAccessTokenExpiry, server.config.OauthRefreshTokenExpiry,
+	)
+
+	router.HandleFunc("/.well-known/openid-configuration", oidc.WellKnownConfiguration)
+	router.Handle("/oauth/v2/authorize", server.withAuth(http.HandlerFunc(oidc.AuthorizeUser))).Methods(http.MethodPost)
+	router.Handle("/oauth/v2/tokens", server.ipRateLimiter.Limit(http.HandlerFunc(oidc.Tokens))).Methods(http.MethodPost)
+	router.Handle("/oauth/v2/userinfo", server.ipRateLimiter.Limit(http.HandlerFunc(oidc.UserInfo))).Methods(http.MethodGet)
+	router.Handle("/oauth/v2/clients/{id}", server.withAuth(http.HandlerFunc(oidc.GetClient))).Methods(http.MethodGet)
 
-		router.HandleFunc("/.well-known/openid-configuration", oidc.WellKnownConfiguration)
-		router.Handle("/oauth/v2/authorize", server.withAuth(http.HandlerFunc(oidc.AuthorizeUser))).Methods(http.MethodPost)
-		router.Handle("/oauth/v2/tokens", server.ipRateLimiter.Limit(http.HandlerFunc(oidc.Tokens))).Methods(http.MethodPost)
-		router.Handle("/oauth/v2/userinfo", server.ipRateLimiter.Limit(http.HandlerFunc(oidc.UserInfo))).Methods(http.MethodGet)
-		router.Handle("/oauth/v2/clients/{id}", server.withAuth(http.HandlerFunc(oidc.GetClient))).Methods(http.MethodGet)
+	router.HandleFunc("/invited", server.handleInvited)
+	router.HandleFunc("/activation", server.accountActivationHandler)
+	router.HandleFunc("/cancel-password-recovery", server.cancelPasswordRecoveryHandler)
 
+	if server.config.StaticDir != "" && server.config.FrontendEnable {
 		fs := http.FileServer(http.Dir(server.config.StaticDir))
 		router.PathPrefix("/static/").Handler(server.withCORS(server.brotliMiddleware(http.StripPrefix("/static", fs))))
 
-		router.HandleFunc("/invited", server.handleInvited)
-
-		// These paths previously required a trailing slash, so we support both forms for now
-		slashRouter := router.NewRoute().Subrouter()
-		slashRouter.StrictSlash(true)
-		slashRouter.HandleFunc("/activation", server.accountActivationHandler)
-		slashRouter.HandleFunc("/cancel-password-recovery", server.cancelPasswordRecoveryHandler)
-
 		if server.config.UseVuetifyProject {
 			router.PathPrefix("/vuetifypoc").Handler(server.withCORS(http.HandlerFunc(server.vuetifyAppHandler)))
 		}
@@ -427,6 +428,100 @@ func (server *Server) Run(ctx context.Context) (err error) {
 	return group.Wait()
 }
 
+// NewFrontendServer creates new instance of console front-end server.
+// NB: The return type is currently consoleweb.Server, but it does not contain all the dependencies.
+// It should only be used with RunFrontEnd and Close. We plan on moving this to its own type, but
+// right now since we have a feature flag to allow the backend server to continue serving the frontend, it
+// makes it easier if they are the same type.
+func NewFrontendServer(logger *zap.Logger, config Config, listener net.Listener, nodeURL storj.NodeURL, stripePublicKey string) (server *Server, err error) {
+	server = &Server{
+		log:             logger,
+		config:          config,
+		listener:        listener,
+		nodeURL:         nodeURL,
+		stripePublicKey: stripePublicKey,
+	}
+
+	logger.Debug("Starting Satellite UI server.", zap.Stringer("Address", server.listener.Addr()))
+
+	router := mux.NewRouter()
+
+	// N.B. This middleware has to be the first one because it has to be called
+	// the earliest in the HTTP chain.
+	router.Use(newTraceRequestMiddleware(logger, router))
+
+	// in local development, proxy certain requests to the console back-end server
+	if config.BackendReverseProxy != "" {
+		target, err := url.Parse(config.BackendReverseProxy)
+		if err != nil {
+			return nil, Error.Wrap(err)
+		}
+		proxy := httputil.NewSingleHostReverseProxy(target)
+		logger.Debug("Reverse proxy targeting", zap.String("address", config.BackendReverseProxy))
+
+		router.PathPrefix("/api").Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			proxy.ServeHTTP(w, r)
+		}))
+		router.PathPrefix("/oauth").Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			proxy.ServeHTTP(w, r)
+		}))
+		router.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {
+			proxy.ServeHTTP(w, r)
+		})
+		router.HandleFunc("/invited", func(w http.ResponseWriter, r *http.Request) {
+			proxy.ServeHTTP(w, r)
+		})
+		router.HandleFunc("/activation", func(w http.ResponseWriter, r *http.Request) {
+			proxy.ServeHTTP(w, r)
+		})
+		router.HandleFunc("/cancel-password-recovery", func(w http.ResponseWriter, r *http.Request) {
+			proxy.ServeHTTP(w, r)
+		})
+		router.HandleFunc("/registrationToken/", func(w http.ResponseWriter, r *http.Request) {
+			proxy.ServeHTTP(w, r)
+		})
+		router.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+			proxy.ServeHTTP(w, r)
+		})
+	}
+
+	fs := http.FileServer(http.Dir(server.config.StaticDir))
+
+	router.HandleFunc("/robots.txt", server.seoHandler)
+	router.PathPrefix("/static/").Handler(server.brotliMiddleware(http.StripPrefix("/static", fs)))
+	router.HandleFunc("/config", server.frontendConfigHandler)
+	if server.config.UseVuetifyProject {
+		router.PathPrefix("/vuetifypoc").Handler(http.HandlerFunc(server.vuetifyAppHandler))
+	}
+	router.PathPrefix("/").Handler(http.HandlerFunc(server.appHandler))
+	server.server = http.Server{
+		Handler:        server.withRequest(router),
+		MaxHeaderBytes: ContentLengthLimit.Int(),
+	}
+	return server, nil
+}
+
+// RunFrontend starts the server that runs the webapp.
+func (server *Server) RunFrontend(ctx context.Context) (err error) {
+	defer mon.Task()(&ctx)(&err)
+
+	ctx, cancel := context.WithCancel(ctx)
+	var group errgroup.Group
+	group.Go(func() error {
+		<-ctx.Done()
+		return server.server.Shutdown(context.Background())
+	})
+	group.Go(func() error {
+		defer cancel()
+		err := server.server.Serve(server.listener)
+		if errs2.IsCanceled(err) || errors.Is(err, http.ErrServerClosed) {
+			err = nil
+		}
+		return err
+	})
+	return group.Wait()
+}
+
 // Close closes server and underlying listener.
 func (server *Server) Close() error {
 	return server.server.Close()