add server options

cmd/config.go

@@ -4,7 +4,6 @@ import (
 	"context"
 
 	"github.com/pkg/errors"
-	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/spf13/cobra"
@@ -24,7 +23,7 @@ var (
 backends and manage existing ones.
 `,
 		Run: func(command *cobra.Command, args []string) {
-			cmd.CheckArgs(0, 0, command, args)
+			CheckArgs(0, 0, command, args)
 			config.EditConfig(context.Background())
 		},
 	}
@@ -44,7 +43,7 @@ you would do:
     honey config create c1 aws region us-east-1
 `,
 		RunE: func(command *cobra.Command, args []string) error {
-			cmd.CheckArgs(2, 256, command, args)
+			CheckArgs(2, 256, command, args)
 			in, err := argsToMap(args[2:])
 			if err != nil {
 				return err

cmd/obscure.go

@@ -5,9 +5,9 @@ import (
 	"fmt"
 	"os"
 
-	rcloneCmd "github.com/rclone/rclone/cmd"
-	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/spf13/cobra"
+
+	"github.com/rclone/rclone/fs/config/obscure"
 )
 
 var obscureCmd = &cobra.Command{
@@ -33,7 +33,7 @@ echo "secretpassword" | honey obscure -
 If there is no data on STDIN to read, honey obscure will default to
 obfuscating the hyphen itself.`,
 	RunE: func(command *cobra.Command, args []string) error {
-		rcloneCmd.CheckArgs(1, 1, command, args)
+		CheckArgs(1, 1, command, args)
 		var password string
 		fi, _ := os.Stdin.Stat()
 		if args[0] == "-" && (fi.Mode()&os.ModeCharDevice) == 0 {

cmd/root.go

@@ -53,7 +53,7 @@ var (
 	flagsRe *regexp.Regexp
 
 	Root = &cobra.Command{
-		Use:           "honey filter",
+		Use:           "honey",
 		SilenceUsage:  true,
 		SilenceErrors: true,
 		Short:         "DevOps tool to help find an instance in sea of clouds",
@@ -412,3 +412,14 @@ Use "honey [command] --help" for more information about a command.
 Use "honey help flags" for to see the global flags.
 Use "honey help backends" for a list of supported services.
 `
+
+// CheckArgs checks there are enough arguments and prints a message if not
+func CheckArgs(MinArgs, MaxArgs int, cmd *cobra.Command, args []string) {
+	if len(args) < MinArgs {
+		_ = cmd.Usage()
+		log.Fatalf("Command %s needs %d arguments minimum: you provided %d non flag arguments: %q\n", cmd.Name(), MinArgs, len(args), args)
+	} else if len(args) > MaxArgs {
+		_ = cmd.Usage()
+		log.Fatalf("Command %s needs %d arguments maximum: you provided %d non flag arguments: %q\n", cmd.Name(), MaxArgs, len(args), args)
+	}
+}

cmd/serve.go

@@ -4,14 +4,19 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/bringg/honey/pkg/resthttp"
+	"github.com/bringg/honey/pkg/resthttp/httpflags"
 )
 
 var (
 	serveCmd = &cobra.Command{
 		Use:   "serve",
 		Short: "Serve over a http protocol",
 		RunE: func(command *cobra.Command, args []string) error {
-			return resthttp.NewServer(&resthttp.Options{}).Serve()
+			return resthttp.NewServer(&httpflags.Opt).Serve()
 		},
 	}
 )
+
+func init() {
+	httpflags.AddFlags(serveCmd.Flags())
+}

go.sum

@@ -218,6 +218,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
 github.com/denis-tingajkin/go-header v0.4.2 h1:jEeSF4sdv8/3cT/WY8AgDHUoItNSoEZ7qg9dX7pc218=
 github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCFFnBUn4RN0nRcs1LJA=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=

pkg/config/configflags/configflags.go

@@ -21,6 +21,7 @@ func AddFlags(ci *place.ConfigInfo, flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &ci.NoColor, "no-color", "", ci.NoColor, "disable colorize the json for outputing to the screen")
 	flags.BoolVarP(flagSet, &quiet, "quiet", "q", quiet, "Print as little stuff as possible")
 	flags.BoolVarP(flagSet, &ci.NoCache, "no-cache", "", ci.NoCache, "no-cache will skip lookup in cache")
+	flags.Uint32VarP(flagSet, &ci.CacheTTL, "cache-ttl", "", ci.CacheTTL, "cache-ttl cache duration in seconds")
 	flags.StringVarP(flagSet, &config.ConfigPath, "config", "c", config.ConfigPath, "config file")
 	flags.StringVarP(flagSet, &ci.OutFormat, "output", "o", ci.OutFormat, "")
 	flags.StringVarP(flagSet, &ci.BackendsString, "backends", "b", ci.BackendsString, "")

pkg/place/cache/store.go

@@ -46,7 +46,7 @@ func (s *Store) Close() error {
 }
 
 // Put _
-func (s *Store) Put(bucket string, key []byte, value interface{}) error {
+func (s *Store) Put(bucket string, key []byte, value interface{}, ttl uint32) error {
 	if err := s.db.Update(
 		func(tx *nutsdb.Tx) error {
 			data, err := msgpack.Marshal(value)
@@ -55,8 +55,7 @@ func (s *Store) Put(bucket string, key []byte, value interface{}) error {
 			}
 
 			// If set ttl = 0 or Persistent, this key will nerver expired.
-			// Set ttl = 600 , after 600 seconds, this key will expired.
-			if err := tx.Put(bucket, cacheKeyName(key), data, 600); err != nil {
+			if err := tx.Put(bucket, cacheKeyName(key), data, ttl); err != nil {
 				return err
 			}
 

pkg/place/config.go

@@ -3,7 +3,6 @@ package place
 import (
 	"context"
 	"strings"
-	"time"
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
@@ -32,19 +31,19 @@ var (
 type (
 	// ConfigInfo is honey config options
 	ConfigInfo struct {
-		Timeout        time.Duration // Data channel timeout
 		NoCache        bool
 		NoColor        bool
 		OutFormat      string
 		BackendsString string
+		CacheTTL       uint32
 	}
 )
 
 func NewConfig() *ConfigInfo {
 	c := new(ConfigInfo)
 
-	c.Timeout = 5 * 60 * time.Second
 	c.OutFormat = "table"
+	c.CacheTTL = 600 // Set ttl = 600 , after 600 seconds, cache key will be expired.
 
 	return c
 }

pkg/place/operations/find.go

@@ -32,6 +32,10 @@ func (cs *ConcurrentSlice) Append(item place.Printable) {
 
 // Find _
 func Find(ctx context.Context, backendNames []string, pattern string) (place.Printable, error) {
+	if pattern == "" {
+		return nil, errors.New("filter text is missing")
+	}
+
 	backends := make(map[string]place.Backend)
 
 	cacheDB, err := cache.NewStore()
@@ -94,7 +98,7 @@ func Find(ctx context.Context, backendNames []string, pattern string) (place.Pri
 				log.Debugf("using backend: %s, provider %s, pattern `%s`, found: %d items", bucketName, backend.Name(), pattern, len(ins))
 
 				// store to cache
-				if err := cacheDB.Put(bucketName, []byte(backend.CacheKeyName(pattern)), ins); err != nil {
+				if err := cacheDB.Put(bucketName, []byte(backend.CacheKeyName(pattern)), ins, ci.CacheTTL); err != nil {
 					log.Debugf("can't store cache for (%s) backend: %v", bucketName, err)
 				}
 

pkg/resthttp/httpflags/httpflags.go

@@ -0,0 +1,33 @@
+package httpflags
+
+import (
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/spf13/pflag"
+
+	"github.com/bringg/honey/pkg/resthttp"
+)
+
+// Options set by command line flags
+var (
+	Opt = resthttp.DefaultOpt
+)
+
+// AddFlagsPrefix adds flags for the resthttp
+func AddFlagsPrefix(flagSet *pflag.FlagSet, Opt *resthttp.Options) {
+	flags.StringVarP(flagSet, &Opt.ListenAddr, "addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to.")
+	flags.DurationVarP(flagSet, &Opt.ServerReadTimeout, "server-read-timeout", "", Opt.ServerReadTimeout, "Timeout for server reading data")
+	flags.DurationVarP(flagSet, &Opt.ServerWriteTimeout, "server-write-timeout", "", Opt.ServerWriteTimeout, "Timeout for server writing data")
+	flags.IntVarP(flagSet, &Opt.MaxHeaderBytes, "max-header-bytes", "", Opt.MaxHeaderBytes, "Maximum size of request header")
+	flags.StringVarP(flagSet, &Opt.SslCert, "cert", "", Opt.SslCert, "SSL PEM key (concatenation of certificate and CA certificate)")
+	flags.StringVarP(flagSet, &Opt.SslKey, "key", "", Opt.SslKey, "SSL PEM Private key")
+	flags.StringVarP(flagSet, &Opt.ClientCA, "client-ca", "", Opt.ClientCA, "Client certificate authority to verify clients with")
+	flags.StringVarP(flagSet, &Opt.Realm, "realm", "", Opt.Realm, "realm for authentication")
+	flags.StringVarP(flagSet, &Opt.BasicUser, "user", "", Opt.BasicUser, "User name for authentication.")
+	flags.StringVarP(flagSet, &Opt.BasicPass, "pass", "", Opt.BasicPass, "Password for authentication.")
+	flags.StringVarP(flagSet, &Opt.BaseURL, "baseurl", "", Opt.BaseURL, "Prefix for URLs - leave blank for root.")
+}
+
+// AddFlags adds flags for the resthttp
+func AddFlags(flagSet *pflag.FlagSet) {
+	AddFlagsPrefix(flagSet, &Opt)
+}

pkg/resthttp/server.go

@@ -1,17 +1,41 @@
 package resthttp
 
 import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
 	"time"
 
 	"github.com/labstack/echo/v4"
+	"github.com/labstack/echo/v4/middleware"
+	"github.com/sirupsen/logrus"
 
 	"github.com/bringg/honey/pkg/place"
 	"github.com/bringg/honey/pkg/resthttp/handlers"
 )
 
+var (
+	DefaultOpt = Options{
+		ListenAddr:         "localhost:8080",
+		Realm:              "honey",
+		ServerReadTimeout:  60 * time.Second,
+		ServerWriteTimeout: 60 * time.Second,
+		MaxHeaderBytes:     4096,
+	}
+
+	log = logrus.WithField("where", "server")
+)
+
 type (
 	Server struct {
 		echo *echo.Echo
+		Opt  *Options
 	}
 
 	Options struct {
@@ -23,36 +47,117 @@ type (
 		SslCert            string        // SSL PEM key (concatenation of certificate and CA certificate)
 		SslKey             string        // SSL PEM Private key
 		ClientCA           string        // Client certificate authority to verify clients with
-		HtPasswd           string        // htpasswd file - if not provided no authentication is done
 		Realm              string        // realm for authentication
-		BasicUser          string        // single username for basic auth if not using Htpasswd
+		BasicUser          string        // single username for basic auth
 		BasicPass          string        // password for BasicUser
 	}
 )
 
-func NewServer(opts *Options) *Server {
+// NewServer _
+func NewServer(opt *Options) *Server {
 	e := echo.New()
+
 	e.HideBanner = true
+	e.Server.MaxHeaderBytes = opt.MaxHeaderBytes
+	e.Server.ReadTimeout = opt.ServerReadTimeout
+	e.Server.WriteTimeout = opt.ServerWriteTimeout
+
+	useSSL := opt.SslKey != ""
+	if (opt.SslCert != "") != useSSL {
+		log.Fatalf("Need both -cert and -key to use SSL")
+	}
+
+	if opt.ClientCA != "" {
+		if !useSSL {
+			log.Fatalf("Can't use --client-ca without --cert and --key")
+		}
+
+		certpool := x509.NewCertPool()
+		pem, err := ioutil.ReadFile(opt.ClientCA)
+		if err != nil {
+			log.Fatalf("Failed to read client certificate authority: %v", err)
+		}
+
+		if !certpool.AppendCertsFromPEM(pem) {
+			log.Fatalf("Can't parse client certificate authority")
+		}
+
+		e.Server.TLSConfig.ClientCAs = certpool
+		e.Server.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert
+	}
+
+	// If a Base URL is set then serve from there
+	opt.BaseURL = strings.Trim(opt.BaseURL, "/")
+	if opt.BaseURL != "" {
+		opt.BaseURL = "/" + opt.BaseURL
+	}
 
 	return &Server{
 		echo: e,
+		Opt:  opt,
 	}
 }
 
 func (s *Server) Serve() error {
+	api := s.echo.Group(s.Opt.BaseURL + "/api/v1")
+
 	// Middlewares
 	// set copy of config to request context
-	s.echo.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
+	api.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
 		return func(c echo.Context) error {
 			ctx, _ := place.AddConfig(c.Request().Context())
 			c.SetRequest(c.Request().WithContext(ctx))
 			return next(c)
 		}
 	})
 
+	// set compresses HTTP response using gzip compression scheme
+	api.Use(middleware.GzipWithConfig(middleware.GzipConfig{
+		Level: 5,
+	}))
+
+	// set basic auth
+	if s.Opt.BasicUser != "" {
+		api.Use(middleware.BasicAuthWithConfig(middleware.BasicAuthConfig{
+			Realm: s.Opt.Realm,
+			Validator: func(username, password string, c echo.Context) (bool, error) {
+				if strings.Compare(username, s.Opt.BasicUser) == 0 &&
+					strings.Compare(username, s.Opt.BasicUser) == 0 {
+					return true, nil
+				}
+
+				return false, nil
+			},
+		}))
+	}
+
 	// Routes
-	s.echo.GET("/backends", handlers.Backends())
-	s.echo.GET("/search", handlers.Search())
+	api.GET("/backends", handlers.Backends())
+	api.GET("/search", handlers.Search())
+
+	// Start server
+	go func() {
+		if err := s.echo.Start(s.Opt.ListenAddr); err != nil && err != http.ErrServerClosed {
+			log.Fatal("shutting down the server")
+		}
+	}()
+
+	quit := make(chan os.Signal, 1)
+
+	// interrupt signal sent from terminal
+	// sigterm signal sent from kubernetes
+	signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
+
+	<-quit
+
+	log.Debug("gracefully shutting down the server")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	if err := s.echo.Shutdown(ctx); err != nil {
+		return err
+	}
 
-	return s.echo.Start(":1323")
+	return nil
 }