From 53e21e60af83705854ce2473fa9f26a5a88a8895 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Sat, 26 Jun 2021 12:24:28 +0800 Subject: [PATCH 1/4] refactor: server manager --- api/cmd/root.go | 141 +++----------------------- api/cmd/version.go | 11 +-- api/internal/core/server/http.go | 69 +++++++++++++ api/internal/core/server/server.go | 152 +++++++++++++++++++++++++++++ api/internal/core/server/store.go | 36 +++++++ api/internal/utils/version.go | 14 ++- api/test/shell/cli_test.sh | 2 +- 7 files changed, 286 insertions(+), 139 deletions(-) create mode 100644 api/internal/core/server/http.go create mode 100644 api/internal/core/server/server.go create mode 100644 api/internal/core/server/store.go diff --git a/api/cmd/root.go b/api/cmd/root.go index 05e16a232d..f0557e7175 100644 --- a/api/cmd/root.go +++ b/api/cmd/root.go @@ -17,28 +17,16 @@ package cmd import ( - "context" - "crypto/tls" "fmt" - "net" - "net/http" "os" "os/signal" - "strconv" "syscall" - "time" - "github.com/shiningrush/droplet" "github.com/spf13/cobra" - "github.com/apisix/manager-api/internal" "github.com/apisix/manager-api/internal/conf" - "github.com/apisix/manager-api/internal/core/storage" - "github.com/apisix/manager-api/internal/core/store" - "github.com/apisix/manager-api/internal/filter" - "github.com/apisix/manager-api/internal/handler" + "github.com/apisix/manager-api/internal/core/server" "github.com/apisix/manager-api/internal/log" - "github.com/apisix/manager-api/internal/utils" ) var ( @@ -88,126 +76,25 @@ func manageAPI() error { conf.InitConf() log.InitLogger() - if err := utils.WritePID(conf.PIDPath, forceStart); err != nil { - log.Errorf("failed to write pid: %s", err) - return err - } - utils.AppendToClosers(func() error { - if err := os.Remove(conf.PIDPath); err != nil { - log.Errorf("failed to remove pid path: %s", err) - return err - } - return nil + s, err := server.NewServer(&server.Options{ + ForceStart: forceStart, }) - - // Signal received to the process externally. - quit := make(chan os.Signal, 1) - signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) - - defer func() { - utils.CloseAll() - signal.Stop(quit) - }() - - droplet.Option.Orchestrator = func(mws []droplet.Middleware) []droplet.Middleware { - var newMws []droplet.Middleware - // default middleware order: resp_reshape, auto_input, traffic_log - // We should put err_transform at second to catch all error - newMws = append(newMws, mws[0], &handler.ErrorTransformMiddleware{}, &filter.AuthenticationMiddleware{}) - newMws = append(newMws, mws[1:]...) - return newMws - } - - if err := storage.InitETCDClient(conf.ETCDConfig); err != nil { - log.Errorf("init etcd client fail: %w", err) - return err - } - if err := store.InitStores(); err != nil { - log.Errorf("init stores fail: %w", err) + if err != nil { return err } - var server, serverSSL *http.Server - // For internal error handling across multiple goroutines. - errsig := make(chan error, 1) - - // routes - r := internal.SetUpRouter() - addr := net.JoinHostPort(conf.ServerHost, strconv.Itoa(conf.ServerPort)) - server = &http.Server{ - Addr: addr, - Handler: r, - ReadTimeout: time.Duration(1000) * time.Millisecond, - WriteTimeout: time.Duration(5000) * time.Millisecond, - } - - log.Infof("The Manager API is listening on %s", addr) - - go func() { - if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { - log.Errorf("listen and serv fail: %s", err) - errsig <- err - } - }() - - // HTTPS - if conf.SSLCert != "" && conf.SSLKey != "" { - addrSSL := net.JoinHostPort(conf.ServerHost, strconv.Itoa(conf.SSLPort)) - serverSSL = &http.Server{ - Addr: addrSSL, - Handler: r, - ReadTimeout: time.Duration(1000) * time.Millisecond, - WriteTimeout: time.Duration(5000) * time.Millisecond, - TLSConfig: &tls.Config{ - // Causes servers to use Go's default ciphersuite preferences, - // which are tuned to avoid attacks. Does nothing on clients. - PreferServerCipherSuites: true, - }, - } - go func() { - err := serverSSL.ListenAndServeTLS(conf.SSLCert, conf.SSLKey) - if err != nil && err != http.ErrServerClosed { - log.Errorf("listen and serve for HTTPS failed: %s", err) - errsig <- err - } - }() - } - - printInfo() - - select { - case err := <-errsig: + err = s.Start() + if err != nil { return err - - case sig := <-quit: - log.Infof("The Manager API server receive %s and start shutting down", sig.String()) - - shutdownServer(server) - shutdownServer(serverSSL) - log.Infof("The Manager API server exited") - return nil } -} -func shutdownServer(server *http.Server) { - if server != nil { - ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) - defer cancel() - - if err := server.Shutdown(ctx); err != nil { - log.Errorf("Shutting down server error: %s", err) - } - } -} + // Signal received to the process externally. + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + sig := <-quit -func printInfo() { - fmt.Fprint(os.Stdout, "The manager-api is running successfully!\n\n") - printVersion() - fmt.Fprintf(os.Stdout, "%-8s: %s:%d\n", "Listen", conf.ServerHost, conf.ServerPort) - if conf.SSLCert != "" && conf.SSLKey != "" { - fmt.Fprintf(os.Stdout, "%-8s: %s:%d\n", "HTTPS Listen", conf.SSLHost, conf.SSLPort) - } - fmt.Fprintf(os.Stdout, "%-8s: %s\n", "Loglevel", conf.ErrorLogLevel) - fmt.Fprintf(os.Stdout, "%-8s: %s\n", "ErrorLogFile", conf.ErrorLogPath) - fmt.Fprintf(os.Stdout, "%-8s: %s\n\n", "AccessLogFile", conf.AccessLogPath) + log.Infof("The Manager API server receive %s and start shutting down", sig.String()) + s.Stop() + log.Infof("The Manager API server exited") + return nil } diff --git a/api/cmd/version.go b/api/cmd/version.go index df7abc19a4..c064502bd3 100644 --- a/api/cmd/version.go +++ b/api/cmd/version.go @@ -17,9 +17,6 @@ package cmd import ( - "fmt" - "os" - "github.com/spf13/cobra" "github.com/apisix/manager-api/internal/utils" @@ -30,13 +27,7 @@ func newVersionCommand() *cobra.Command { Use: "version", Short: "show manager-api version", Run: func(cmd *cobra.Command, args []string) { - printVersion() + utils.PrintVersion() }, } } - -func printVersion() { - gitHash, version := utils.GetHashAndVersion() - fmt.Fprintf(os.Stdout, "%-8s: %s\n", "Version", version) - fmt.Fprintf(os.Stdout, "%-8s: %s\n", "GitHash", gitHash) -} diff --git a/api/internal/core/server/http.go b/api/internal/core/server/http.go new file mode 100644 index 0000000000..6b1661dd91 --- /dev/null +++ b/api/internal/core/server/http.go @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package server + +import ( + "crypto/tls" + "net" + "net/http" + "strconv" + "time" + + "github.com/shiningrush/droplet" + + "github.com/apisix/manager-api/internal" + "github.com/apisix/manager-api/internal/conf" + "github.com/apisix/manager-api/internal/filter" + "github.com/apisix/manager-api/internal/handler" +) + +func (s *server) setupHTTPServer() { + droplet.Option.Orchestrator = func(mws []droplet.Middleware) []droplet.Middleware { + var newMws []droplet.Middleware + // default middleware order: resp_reshape, auto_input, traffic_log + // We should put err_transform at second to catch all error + newMws = append(newMws, mws[0], &handler.ErrorTransformMiddleware{}, &filter.AuthenticationMiddleware{}) + newMws = append(newMws, mws[1:]...) + return newMws + } + + // routes + r := internal.SetUpRouter() + addr := net.JoinHostPort(conf.ServerHost, strconv.Itoa(conf.ServerPort)) + s.server = &http.Server{ + Addr: addr, + Handler: r, + ReadTimeout: time.Duration(1000) * time.Millisecond, + WriteTimeout: time.Duration(5000) * time.Millisecond, + } + + // HTTPS + if conf.SSLCert != "" && conf.SSLKey != "" { + addrSSL := net.JoinHostPort(conf.SSLHost, strconv.Itoa(conf.SSLPort)) + s.serverSSL = &http.Server{ + Addr: addrSSL, + Handler: r, + ReadTimeout: time.Duration(1000) * time.Millisecond, + WriteTimeout: time.Duration(5000) * time.Millisecond, + TLSConfig: &tls.Config{ + // Causes servers to use Go's default ciphersuite preferences, + // which are tuned to avoid attacks. Does nothing on clients. + PreferServerCipherSuites: true, + }, + } + } +} diff --git a/api/internal/core/server/server.go b/api/internal/core/server/server.go new file mode 100644 index 0000000000..5459eb7f3e --- /dev/null +++ b/api/internal/core/server/server.go @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package server + +import ( + "context" + "fmt" + "net/http" + "os" + "time" + + "github.com/apisix/manager-api/internal/conf" + "github.com/apisix/manager-api/internal/log" + "github.com/apisix/manager-api/internal/utils" +) + +// server is the manager of Manager API, which is responsible for managing the life cycle of Manager API, including initialization, start, stop and so on +type server struct { + server *http.Server + serverSSL *http.Server + options *Options +} + +type Options struct { + ForceStart bool // force start new instance +} + +// NewServer Create a server manager +func NewServer(options *Options) (*server, error) { + return &server{options: options}, nil +} + +func (s *server) Start() error { + // initialize server + err := s.init() + if err != nil { + return err + } + + // write daemon pid file + err = s.writePID() + if err != nil { + return err + } + + // print server info to stdout + s.printInfo() + + // For internal error handling across multiple goroutines. + errSig := make(chan error, 1) + + // start HTTP server + log.Infof("The Manager API is listening on %s", s.server.Addr) + go func() { + if err := s.server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Errorf("listen and serv fail: %s", err) + errSig <- err + } + }() + + // start HTTPs server + if conf.SSLCert != "" && conf.SSLKey != "" { + go func() { + err := s.serverSSL.ListenAndServeTLS(conf.SSLCert, conf.SSLKey) + if err != nil && err != http.ErrServerClosed { + log.Errorf("listen and serve for HTTPS failed: %s", err) + errSig <- err + } + }() + } + + // handle HTTP(s) server error + select { + case err := <-errSig: + return err + default: + return nil + } +} + +func (s *server) Stop() { + utils.CloseAll() + + s.shutdownServer(s.server) + s.shutdownServer(s.serverSSL) +} + +func (s *server) init() error { + log.Info("Initialize Manager API store") + err := s.setupStore() + if err != nil { + return err + } + + log.Info("Initialize Manager API server") + s.setupHTTPServer() + + return nil +} + +func (s *server) writePID() error { + if err := utils.WritePID(conf.PIDPath, s.options.ForceStart); err != nil { + log.Errorf("failed to write pid: %s", err) + return err + } + utils.AppendToClosers(func() error { + if err := os.Remove(conf.PIDPath); err != nil { + log.Errorf("failed to remove pid path: %s", err) + return err + } + return nil + }) + + return nil +} + +func (s *server) shutdownServer(server *http.Server) { + if server != nil { + ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) + defer cancel() + + if err := server.Shutdown(ctx); err != nil { + log.Errorf("Shutting down server error: %s", err) + } + } +} + +func (s *server) printInfo() { + fmt.Fprint(os.Stdout, "The manager-api is running successfully!\n\n") + utils.PrintVersion() + fmt.Fprintf(os.Stdout, "%-8s: %s:%d\n", "Listen", conf.ServerHost, conf.ServerPort) + if conf.SSLCert != "" && conf.SSLKey != "" { + fmt.Fprintf(os.Stdout, "%-8s: %s:%d\n", "HTTPS Listen", conf.SSLHost, conf.SSLPort) + } + fmt.Fprintf(os.Stdout, "%-8s: %s\n", "Loglevel", conf.ErrorLogLevel) + fmt.Fprintf(os.Stdout, "%-8s: %s\n", "ErrorLogFile", conf.ErrorLogPath) + fmt.Fprintf(os.Stdout, "%-8s: %s\n\n", "AccessLogFile", conf.AccessLogPath) +} diff --git a/api/internal/core/server/store.go b/api/internal/core/server/store.go new file mode 100644 index 0000000000..6130a19711 --- /dev/null +++ b/api/internal/core/server/store.go @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package server + +import ( + "github.com/apisix/manager-api/internal/conf" + "github.com/apisix/manager-api/internal/core/storage" + "github.com/apisix/manager-api/internal/core/store" + "github.com/apisix/manager-api/internal/log" +) + +func (s *server) setupStore() error { + if err := storage.InitETCDClient(conf.ETCDConfig); err != nil { + log.Errorf("init etcd client fail: %w", err) + return err + } + if err := store.InitStores(); err != nil { + log.Errorf("init stores fail: %w", err) + return err + } + return nil +} diff --git a/api/internal/utils/version.go b/api/internal/utils/version.go index d4bec4ab00..c061156406 100644 --- a/api/internal/utils/version.go +++ b/api/internal/utils/version.go @@ -16,12 +16,24 @@ */ package utils +import ( + "fmt" + "os" +) + var ( gitHash string version string ) -// get the hash and version +// GetHashAndVersion get the hash and version func GetHashAndVersion() (string, string) { return gitHash, version } + +// PrintVersion print version and git hash to stdout +func PrintVersion() { + gitHash, version := GetHashAndVersion() + fmt.Fprintf(os.Stdout, "%-8s: %s\n", "Version", version) + fmt.Fprintf(os.Stdout, "%-8s: %s\n", "GitHash", gitHash) +} diff --git a/api/test/shell/cli_test.sh b/api/test/shell/cli_test.sh index 325b0a4e5a..1bbb66f728 100755 --- a/api/test/shell/cli_test.sh +++ b/api/test/shell/cli_test.sh @@ -223,7 +223,7 @@ sleep 6 cat ${logfile} -if [[ `grep -c "cmd/root.go" ${logfile}` -ne '1' ]]; then +if [[ `grep -c "server/store.go" ${logfile}` -ne '1' ]]; then echo "failed: failed to write the correct caller" exit 1 fi From e0ce9d8a3b26d0cae24dcb382f4a5810389c675d Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Sun, 27 Jun 2021 12:16:56 +0800 Subject: [PATCH 2/4] chore: add comment --- api/internal/core/server/http.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/internal/core/server/http.go b/api/internal/core/server/http.go index 6b1661dd91..582cfa3c0f 100644 --- a/api/internal/core/server/http.go +++ b/api/internal/core/server/http.go @@ -32,6 +32,7 @@ import ( ) func (s *server) setupHTTPServer() { + // orchestrator droplet.Option.Orchestrator = func(mws []droplet.Middleware) []droplet.Middleware { var newMws []droplet.Middleware // default middleware order: resp_reshape, auto_input, traffic_log @@ -43,6 +44,8 @@ func (s *server) setupHTTPServer() { // routes r := internal.SetUpRouter() + + // HTTP addr := net.JoinHostPort(conf.ServerHost, strconv.Itoa(conf.ServerPort)) s.server = &http.Server{ Addr: addr, From 74ebcf9df72c061ebe94b173b9a3ba380a8a3445 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Mon, 28 Jun 2021 14:07:43 +0800 Subject: [PATCH 3/4] chore: change function name --- api/internal/core/server/http.go | 2 +- api/internal/core/server/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/internal/core/server/http.go b/api/internal/core/server/http.go index 582cfa3c0f..53a4216174 100644 --- a/api/internal/core/server/http.go +++ b/api/internal/core/server/http.go @@ -31,7 +31,7 @@ import ( "github.com/apisix/manager-api/internal/handler" ) -func (s *server) setupHTTPServer() { +func (s *server) setupAPI() { // orchestrator droplet.Option.Orchestrator = func(mws []droplet.Middleware) []droplet.Middleware { var newMws []droplet.Middleware diff --git a/api/internal/core/server/server.go b/api/internal/core/server/server.go index 5459eb7f3e..5e72744813 100644 --- a/api/internal/core/server/server.go +++ b/api/internal/core/server/server.go @@ -107,7 +107,7 @@ func (s *server) init() error { } log.Info("Initialize Manager API server") - s.setupHTTPServer() + s.setupAPI() return nil } From eeb8bbcc295e0ac0c76b35146d463aa6bf9eacba Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Mon, 28 Jun 2021 15:09:42 +0800 Subject: [PATCH 4/4] fix: error handler --- api/cmd/root.go | 20 ++++++++++++-------- api/internal/core/server/server.go | 22 +++++++--------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/api/cmd/root.go b/api/cmd/root.go index f0557e7175..b9d2aa48a0 100644 --- a/api/cmd/root.go +++ b/api/cmd/root.go @@ -83,18 +83,22 @@ func manageAPI() error { return err } - err = s.Start() - if err != nil { - return err - } + // start Manager API server + errSig := make(chan error, 5) + s.Start(errSig) // Signal received to the process externally. quit := make(chan os.Signal, 1) signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) - sig := <-quit - log.Infof("The Manager API server receive %s and start shutting down", sig.String()) - s.Stop() - log.Infof("The Manager API server exited") + select { + case sig := <-quit: + log.Infof("The Manager API server receive %s and start shutting down", sig.String()) + s.Stop() + log.Infof("The Manager API server exited") + case err := <-errSig: + log.Errorf("The Manager API server start failed: %s", err.Error()) + return err + } return nil } diff --git a/api/internal/core/server/server.go b/api/internal/core/server/server.go index 5e72744813..07c4b1cd41 100644 --- a/api/internal/core/server/server.go +++ b/api/internal/core/server/server.go @@ -44,29 +44,29 @@ func NewServer(options *Options) (*server, error) { return &server{options: options}, nil } -func (s *server) Start() error { +func (s *server) Start(errSig chan error) { // initialize server err := s.init() if err != nil { - return err + errSig <- err + return } // write daemon pid file err = s.writePID() if err != nil { - return err + errSig <- err + return } // print server info to stdout s.printInfo() - // For internal error handling across multiple goroutines. - errSig := make(chan error, 1) - // start HTTP server log.Infof("The Manager API is listening on %s", s.server.Addr) go func() { - if err := s.server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + err := s.server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { log.Errorf("listen and serv fail: %s", err) errSig <- err } @@ -82,14 +82,6 @@ func (s *server) Start() error { } }() } - - // handle HTTP(s) server error - select { - case err := <-errSig: - return err - default: - return nil - } } func (s *server) Stop() {