From 57cea2cd3d76a8ed5871f265e7bcf5fca4661821 Mon Sep 17 00:00:00 2001 From: klahaha Date: Thu, 23 Sep 2021 18:17:49 +0200 Subject: [PATCH 01/56] Remove hardcoded paths. Add HTTP error responses, some logging. To run, do ./build && bin/transcode from the root of the repo. The basedir to look up scripts is currently **not** configurable. --- hls/manager.go | 9 ++++++++- internal/api/hls.go | 11 ++++++++++- internal/api/http.go | 10 +++++++++- internal/api/router.go | 7 ++++--- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/hls/manager.go b/hls/manager.go index 001b91f..e6263dd 100644 --- a/hls/manager.go +++ b/hls/manager.go @@ -2,6 +2,7 @@ package hls import ( "errors" + "fmt" "io" "net/http" "os" @@ -21,7 +22,7 @@ import ( const cleanupPeriod = 4 * time.Second // timeot for first playlist, when it waits for new data -const playlistTimeout = 20 * time.Second +const playlistTimeout = 60 * time.Second // minimum segments available to consider stream as active const hlsMinimumSegments = 2 @@ -82,6 +83,7 @@ func (m *ManagerCtx) Start() error { m.cmd = m.cmdFactory() m.cmd.Dir = m.tempdir + //fmt.Println(m.cmd.Dir) if m.events.onCmdLog != nil { m.cmd.Stderr = utils.LogEvent(m.events.onCmdLog) @@ -89,6 +91,9 @@ func (m *ManagerCtx) Start() error { m.cmd.Stderr = utils.LogWriter(m.logger) } + cwd, _ := os.Getwd() + m.cmd.Path = fmt.Sprintf("%s/%s", cwd, m.cmd.Path) + read, write := io.Pipe() m.cmd.Stdout = write @@ -126,6 +131,7 @@ func (m *ManagerCtx) Start() error { } if err != nil { + // When command fails to start, we reach this branch after a while m.logger.Err(err).Msg("cmd read failed") return } @@ -229,6 +235,7 @@ func (m *ManagerCtx) ServePlaylist(w http.ResponseWriter, r *http.Request) { case <-m.shutdown: m.logger.Warn().Msg("playlist load failed because of shutdown") w.WriteHeader(http.StatusNotFound) + // When command failed to start and timeout has been increased we reach this branch after a while w.Write([]byte("404 playlist not found")) return case <-time.After(playlistTimeout): diff --git a/internal/api/hls.go b/internal/api/hls.go index 2f9673c..95f8f93 100644 --- a/internal/api/hls.go +++ b/internal/api/hls.go @@ -30,6 +30,13 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { return } + _, stream_exists := a.Conf.Streams[input] + if !stream_exists { + w.WriteHeader(http.StatusNotFound) + w.Write([]byte("404 not found")) + return + } + ID := fmt.Sprintf("%s/%s", profile, input) manager, ok := hlsManagers[ID] @@ -39,7 +46,9 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { // get transcode cmd cmd, err := transcodeStart("profiles/hls", profile, input) if err != nil { - logger.Panic().Err(err).Msg("transcode could not be started") + logger.Error().Err(err).Msg("transcode could not be started") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(fmt.Sprintf("%v\n", err))) } return cmd diff --git a/internal/api/http.go b/internal/api/http.go index 9f36971..f7be9ab 100644 --- a/internal/api/http.go +++ b/internal/api/http.go @@ -21,7 +21,8 @@ func (a *ApiManagerCtx) Http(r chi.Router) { Logger() logger.Info().Msg("command startred") - cmd := exec.Command("/app/data/http-test.sh") + // WTF is this for? + cmd := exec.Command("data/http-test.sh") read, write := io.Pipe() cmd.Stdout = write @@ -47,6 +48,13 @@ func (a *ApiManagerCtx) Http(r chi.Router) { profile := chi.URLParam(r, "profile") input := chi.URLParam(r, "input") + _, stream_exists := a.Conf.Streams[input] + if !stream_exists { + w.WriteHeader(http.StatusNotFound) + w.Write([]byte("404 not found")) + return + } + cmd, err := transcodeStart("profiles/http", profile, input) if err != nil { logger.Warn().Err(err).Msg("transcode could not be started") diff --git a/internal/api/router.go b/internal/api/router.go index ed56d63..67857f1 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -21,11 +21,12 @@ func init() { } } -type ApiManagerCtx struct{} +type ApiManagerCtx struct { + Conf *YamlConf +} func New() *ApiManagerCtx { - - return &ApiManagerCtx{} + return &ApiManagerCtx{Conf: conf} } func (a *ApiManagerCtx) Mount(r *chi.Mux) { From 8684cae3cebde7421395df67a59e212bd9e68cb2 Mon Sep 17 00:00:00 2001 From: klahaha Date: Thu, 23 Sep 2021 18:41:08 +0200 Subject: [PATCH 02/56] CMD "main" package is built from repo root Can now do simple go build && ./go-transcode --- .gitignore | 1 + build | 3 +- cmd/root.go | 2 +- cmd/serve.go | 2 +- cmd/transcode/main.go | 13 -------- internal/api/hls.go | 2 +- internal/api/router.go | 4 +-- internal/main.go | 68 ++++++++++++++++++++++++++++++++++++++++++ main.go | 65 ++++------------------------------------ 9 files changed, 81 insertions(+), 79 deletions(-) delete mode 100644 cmd/transcode/main.go create mode 100644 internal/main.go diff --git a/.gitignore b/.gitignore index 88a2a87..73fef16 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ bin/ streams.yaml +go-transcode diff --git a/build b/build index 0793609..58ea400 100755 --- a/build +++ b/build @@ -1,3 +1,4 @@ #!/bin/sh -go build -o bin/transcode cmd/transcode/main.go +#go build -o bin/transcode cmd/transcode/main.go +go build diff --git a/cmd/root.go b/cmd/root.go index 6e7074d..d98821b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/m1k1o/go-transcode" + "github.com/m1k1o/go-transcode/internal" ) func Execute() error { diff --git a/cmd/serve.go b/cmd/serve.go index 7165c0c..62647c4 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -4,7 +4,7 @@ import ( "github.com/rs/zerolog/log" "github.com/spf13/cobra" - "github.com/m1k1o/go-transcode" + "github.com/m1k1o/go-transcode/internal" "github.com/m1k1o/go-transcode/internal/config" ) diff --git a/cmd/transcode/main.go b/cmd/transcode/main.go deleted file mode 100644 index b7ab534..0000000 --- a/cmd/transcode/main.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import ( - "github.com/rs/zerolog/log" - - "github.com/m1k1o/go-transcode/cmd" -) - -func main() { - if err := cmd.Execute(); err != nil { - log.Panic().Err(err).Msg("failed to execute command") - } -} diff --git a/internal/api/hls.go b/internal/api/hls.go index 95f8f93..32beafc 100644 --- a/internal/api/hls.go +++ b/internal/api/hls.go @@ -86,6 +86,6 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { r.Get("/{profile}/{input}/play.html", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") - http.ServeFile(w, r, "/app/data/play.html") + http.ServeFile(w, r, "data/play.html") }) } diff --git a/internal/api/router.go b/internal/api/router.go index 67857f1..088d044 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -15,7 +15,7 @@ var conf *YamlConf func init() { var err error - conf, err = loadConf("/app/streams.yaml") + conf, err = loadConf("streams.yaml") if err != nil { panic(err) } @@ -50,7 +50,7 @@ func transcodeStart(folder string, profile string, input string) (*exec.Cmd, err return nil, fmt.Errorf("invalid profile path") } - profilePath := fmt.Sprintf("/app/%s/%s.sh", folder, profile) + profilePath := fmt.Sprintf("%s/%s.sh", folder, profile) if _, err := os.Stat(profilePath); os.IsNotExist(err) { return nil, err } diff --git a/internal/main.go b/internal/main.go new file mode 100644 index 0000000..a58d2de --- /dev/null +++ b/internal/main.go @@ -0,0 +1,68 @@ +package transcode + +import ( + "os" + "os/signal" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + + "github.com/m1k1o/go-transcode/internal/api" + "github.com/m1k1o/go-transcode/internal/config" + "github.com/m1k1o/go-transcode/internal/http" +) + +var Service *Main + +func init() { + Service = &Main{ + RootConfig: &config.Root{}, + ServerConfig: &config.Server{}, + } +} + +type Main struct { + RootConfig *config.Root + ServerConfig *config.Server + + logger zerolog.Logger + apiManager *api.ApiManagerCtx + server *http.ServerCtx +} + +func (main *Main) Preflight() { + main.logger = log.With().Str("service", "main").Logger() +} + +func (main *Main) Start() { + main.apiManager = api.New() + + main.server = http.New( + main.apiManager, + main.ServerConfig, + ) + main.server.Start() +} + +func (main *Main) Shutdown() { + if err := main.server.Shutdown(); err != nil { + main.logger.Err(err).Msg("server shutdown with an error") + } else { + main.logger.Debug().Msg("server shutdown") + } +} + +func (main *Main) ServeCommand(cmd *cobra.Command, args []string) { + main.logger.Info().Msg("starting main server") + main.Start() + main.logger.Info().Msg("main ready") + + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt) + sig := <-quit + + main.logger.Warn().Msgf("received %s, attempting graceful shutdown: \n", sig) + main.Shutdown() + main.logger.Info().Msg("shutdown complete") +} diff --git a/main.go b/main.go index a58d2de..b7ab534 100644 --- a/main.go +++ b/main.go @@ -1,68 +1,13 @@ -package transcode +package main import ( - "os" - "os/signal" - - "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - "github.com/m1k1o/go-transcode/internal/api" - "github.com/m1k1o/go-transcode/internal/config" - "github.com/m1k1o/go-transcode/internal/http" + "github.com/m1k1o/go-transcode/cmd" ) -var Service *Main - -func init() { - Service = &Main{ - RootConfig: &config.Root{}, - ServerConfig: &config.Server{}, +func main() { + if err := cmd.Execute(); err != nil { + log.Panic().Err(err).Msg("failed to execute command") } } - -type Main struct { - RootConfig *config.Root - ServerConfig *config.Server - - logger zerolog.Logger - apiManager *api.ApiManagerCtx - server *http.ServerCtx -} - -func (main *Main) Preflight() { - main.logger = log.With().Str("service", "main").Logger() -} - -func (main *Main) Start() { - main.apiManager = api.New() - - main.server = http.New( - main.apiManager, - main.ServerConfig, - ) - main.server.Start() -} - -func (main *Main) Shutdown() { - if err := main.server.Shutdown(); err != nil { - main.logger.Err(err).Msg("server shutdown with an error") - } else { - main.logger.Debug().Msg("server shutdown") - } -} - -func (main *Main) ServeCommand(cmd *cobra.Command, args []string) { - main.logger.Info().Msg("starting main server") - main.Start() - main.logger.Info().Msg("main ready") - - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt) - sig := <-quit - - main.logger.Warn().Msgf("received %s, attempting graceful shutdown: \n", sig) - main.Shutdown() - main.logger.Info().Msg("shutdown complete") -} From c02ea72b349cdec5d7a637b0672306b46fe3c754 Mon Sep 17 00:00:00 2001 From: klahaha Date: Thu, 23 Sep 2021 19:06:38 +0200 Subject: [PATCH 03/56] BaseDir is now part of YamlConfig --- hls/manager.go | 10 +++++++--- internal/api/config.go | 26 -------------------------- internal/api/hls.go | 4 ++-- internal/api/http.go | 2 +- internal/api/router.go | 10 ++++++---- internal/config/config.go | 33 ++++++++++++++++++++++++++++++++- 6 files changed, 48 insertions(+), 37 deletions(-) delete mode 100644 internal/api/config.go diff --git a/hls/manager.go b/hls/manager.go index e6263dd..e7744e7 100644 --- a/hls/manager.go +++ b/hls/manager.go @@ -16,6 +16,7 @@ import ( "github.com/rs/zerolog/log" "github.com/m1k1o/go-transcode/internal/utils" + "github.com/m1k1o/go-transcode/internal/config" ) // how often should be cleanup called @@ -53,10 +54,12 @@ type ManagerCtx struct { playlistLoad chan string shutdown chan interface{} + Conf *config.YamlConf } -func New(cmdFactory func() *exec.Cmd) *ManagerCtx { +func New(cmdFactory func() *exec.Cmd, conf *config.YamlConf) *ManagerCtx { return &ManagerCtx{ + Conf: conf, logger: log.With().Str("module", "hls").Str("submodule", "manager").Logger(), cmdFactory: cmdFactory, @@ -91,8 +94,9 @@ func (m *ManagerCtx) Start() error { m.cmd.Stderr = utils.LogWriter(m.logger) } - cwd, _ := os.Getwd() - m.cmd.Path = fmt.Sprintf("%s/%s", cwd, m.cmd.Path) + //cwd, _ := os.Getwd() + //m.cmd.Path = fmt.Sprintf("%s/%s", cwd, m.cmd.Path) + m.cmd.Path = fmt.Sprintf("%s/%s", m.Conf.BaseDir, m.cmd.Path) read, write := io.Pipe() m.cmd.Stdout = write diff --git a/internal/api/config.go b/internal/api/config.go deleted file mode 100644 index 5223221..0000000 --- a/internal/api/config.go +++ /dev/null @@ -1,26 +0,0 @@ -package api - -import ( - "io/ioutil" - - "gopkg.in/yaml.v2" -) - -type YamlConf struct { - Streams map[string]string `yaml:"streams"` -} - -func loadConf(path string) (*YamlConf, error) { - yamlFile, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - - conf := &YamlConf{} - err = yaml.Unmarshal(yamlFile, conf) - if err != nil { - return nil, err - } - - return conf, nil -} diff --git a/internal/api/hls.go b/internal/api/hls.go index 32beafc..d385d44 100644 --- a/internal/api/hls.go +++ b/internal/api/hls.go @@ -52,7 +52,7 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { } return cmd - }) + }, a.Conf) hlsManagers[ID] = manager } @@ -86,6 +86,6 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { r.Get("/{profile}/{input}/play.html", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") - http.ServeFile(w, r, "data/play.html") + http.ServeFile(w, r, fmt.Sprintf("%s/%s", a.Conf.BaseDir, "data/play.html")) }) } diff --git a/internal/api/http.go b/internal/api/http.go index f7be9ab..9379474 100644 --- a/internal/api/http.go +++ b/internal/api/http.go @@ -22,7 +22,7 @@ func (a *ApiManagerCtx) Http(r chi.Router) { logger.Info().Msg("command startred") // WTF is this for? - cmd := exec.Command("data/http-test.sh") + cmd := exec.Command(fmt.Sprintf("%s/%s", a.Conf.BaseDir, "data/http-test.sh")) read, write := io.Pipe() cmd.Stdout = write diff --git a/internal/api/router.go b/internal/api/router.go index 088d044..6ca4aa3 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -9,20 +9,22 @@ import ( "github.com/go-chi/chi" "github.com/rs/zerolog/log" + + "github.com/m1k1o/go-transcode/internal/config" ) -var conf *YamlConf +var conf *config.YamlConf func init() { var err error - conf, err = loadConf("streams.yaml") + conf, err = config.LoadConf("streams.yaml") if err != nil { panic(err) } } type ApiManagerCtx struct { - Conf *YamlConf + Conf *config.YamlConf } func New() *ApiManagerCtx { @@ -50,7 +52,7 @@ func transcodeStart(folder string, profile string, input string) (*exec.Cmd, err return nil, fmt.Errorf("invalid profile path") } - profilePath := fmt.Sprintf("%s/%s.sh", folder, profile) + profilePath := fmt.Sprintf("%s/%s/%s.sh", conf.BaseDir, folder, profile) if _, err := os.Stat(profilePath); os.IsNotExist(err) { return nil, err } diff --git a/internal/config/config.go b/internal/config/config.go index 68a2abc..acff0fd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,8 +1,39 @@ package config -import "github.com/spf13/cobra" +import ( + "io/ioutil" + "os" + + "gopkg.in/yaml.v2" + "github.com/spf13/cobra" +) type Config interface { Init(cmd *cobra.Command) error Set() } + +type YamlConf struct { + BaseDir string `yaml:"basedir",omitempty` + Streams map[string]string `yaml:"streams"` +} + +func LoadConf(path string) (*YamlConf, error) { + yamlFile, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + conf := &YamlConf{} + err = yaml.Unmarshal(yamlFile, conf) + if err != nil { + return nil, err + } + + if conf.BaseDir == "" { + cwd, _ := os.Getwd() + conf.BaseDir = cwd + } + + return conf, nil +} From 9101df50e59aae66708c642995d21c073175f296 Mon Sep 17 00:00:00 2001 From: klahaha Date: Thu, 23 Sep 2021 19:30:02 +0200 Subject: [PATCH 04/56] Try basedir from /etc/go-transcode/ if it exists, and basedir isn't set in config --- internal/config/config.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index acff0fd..98e47a4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,9 +30,15 @@ func LoadConf(path string) (*YamlConf, error) { return nil, err } + // If basedir is not explicit in config, try /etc/go-transcode/, + // fallback to the current working directory if conf.BaseDir == "" { - cwd, _ := os.Getwd() - conf.BaseDir = cwd + if _, err := os.Stat("/etc/go-transcode"); os.IsNotExist(err) { + cwd, _ := os.Getwd() + conf.BaseDir = cwd + } else { + conf.BaseDir = "/etc/go-transcode" + } } return conf, nil From aaf10458b3ab47355fbc9b9e29695941f67b03c7 Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 15:50:02 +0200 Subject: [PATCH 05/56] config is together --- internal/config/config.go | 70 +++++++++++++++++++++++++++++++++++++++ internal/config/root.go | 30 ----------------- internal/config/server.go | 51 ---------------------------- 3 files changed, 70 insertions(+), 81 deletions(-) delete mode 100644 internal/config/root.go delete mode 100644 internal/config/server.go diff --git a/internal/config/config.go b/internal/config/config.go index 98e47a4..6e4357b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,6 +6,7 @@ import ( "gopkg.in/yaml.v2" "github.com/spf13/cobra" + "github.com/spf13/viper" ) type Config interface { @@ -43,3 +44,72 @@ func LoadConf(path string) (*YamlConf, error) { return conf, nil } + +type Root struct { + Debug bool + CfgFile string +} + +func (Root) Init(cmd *cobra.Command) error { + cmd.PersistentFlags().BoolP("debug", "d", false, "enable debug mode") + if err := viper.BindPFlag("debug", cmd.PersistentFlags().Lookup("debug")); err != nil { + return err + } + + cmd.PersistentFlags().String("config", "", "configuration file path") + if err := viper.BindPFlag("config", cmd.PersistentFlags().Lookup("config")); err != nil { + return err + } + + return nil +} + +func (s *Root) Set() { + s.Debug = viper.GetBool("debug") + s.CfgFile = viper.GetString("config") +} + +type Server struct { + Cert string + Key string + Bind string + Static string + Proxy bool +} + +func (Server) Init(cmd *cobra.Command) error { + cmd.PersistentFlags().String("bind", "127.0.0.1:8080", "address/port/socket to serve neko") + if err := viper.BindPFlag("bind", cmd.PersistentFlags().Lookup("bind")); err != nil { + return err + } + + cmd.PersistentFlags().String("cert", "", "path to the SSL cert used to secure the neko server") + if err := viper.BindPFlag("cert", cmd.PersistentFlags().Lookup("cert")); err != nil { + return err + } + + cmd.PersistentFlags().String("key", "", "path to the SSL key used to secure the neko server") + if err := viper.BindPFlag("key", cmd.PersistentFlags().Lookup("key")); err != nil { + return err + } + + cmd.PersistentFlags().String("static", "", "path to neko client files to serve") + if err := viper.BindPFlag("static", cmd.PersistentFlags().Lookup("static")); err != nil { + return err + } + + cmd.PersistentFlags().Bool("proxy", false, "allow reverse proxies") + if err := viper.BindPFlag("proxy", cmd.PersistentFlags().Lookup("proxy")); err != nil { + return err + } + + return nil +} + +func (s *Server) Set() { + s.Cert = viper.GetString("cert") + s.Key = viper.GetString("key") + s.Bind = viper.GetString("bind") + s.Static = viper.GetString("static") + s.Proxy = viper.GetBool("proxy") +} diff --git a/internal/config/root.go b/internal/config/root.go deleted file mode 100644 index 2f2b497..0000000 --- a/internal/config/root.go +++ /dev/null @@ -1,30 +0,0 @@ -package config - -import ( - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -type Root struct { - Debug bool - CfgFile string -} - -func (Root) Init(cmd *cobra.Command) error { - cmd.PersistentFlags().BoolP("debug", "d", false, "enable debug mode") - if err := viper.BindPFlag("debug", cmd.PersistentFlags().Lookup("debug")); err != nil { - return err - } - - cmd.PersistentFlags().String("config", "", "configuration file path") - if err := viper.BindPFlag("config", cmd.PersistentFlags().Lookup("config")); err != nil { - return err - } - - return nil -} - -func (s *Root) Set() { - s.Debug = viper.GetBool("debug") - s.CfgFile = viper.GetString("config") -} diff --git a/internal/config/server.go b/internal/config/server.go deleted file mode 100644 index 9e270ca..0000000 --- a/internal/config/server.go +++ /dev/null @@ -1,51 +0,0 @@ -package config - -import ( - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -type Server struct { - Cert string - Key string - Bind string - Static string - Proxy bool -} - -func (Server) Init(cmd *cobra.Command) error { - cmd.PersistentFlags().String("bind", "127.0.0.1:8080", "address/port/socket to serve neko") - if err := viper.BindPFlag("bind", cmd.PersistentFlags().Lookup("bind")); err != nil { - return err - } - - cmd.PersistentFlags().String("cert", "", "path to the SSL cert used to secure the neko server") - if err := viper.BindPFlag("cert", cmd.PersistentFlags().Lookup("cert")); err != nil { - return err - } - - cmd.PersistentFlags().String("key", "", "path to the SSL key used to secure the neko server") - if err := viper.BindPFlag("key", cmd.PersistentFlags().Lookup("key")); err != nil { - return err - } - - cmd.PersistentFlags().String("static", "", "path to neko client files to serve") - if err := viper.BindPFlag("static", cmd.PersistentFlags().Lookup("static")); err != nil { - return err - } - - cmd.PersistentFlags().Bool("proxy", false, "allow reverse proxies") - if err := viper.BindPFlag("proxy", cmd.PersistentFlags().Lookup("proxy")); err != nil { - return err - } - - return nil -} - -func (s *Server) Set() { - s.Cert = viper.GetString("cert") - s.Key = viper.GetString("key") - s.Bind = viper.GetString("bind") - s.Static = viper.GetString("static") - s.Proxy = viper.GetBool("proxy") -} From b26400f0e7146c817fd3d36dbeba55dd68b20f0d Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 16:16:13 +0200 Subject: [PATCH 06/56] Remove unused type --- cmd/serve.go | 15 +++------------ internal/config/config.go | 5 ----- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index 62647c4..9be9266 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -5,7 +5,6 @@ import ( "github.com/spf13/cobra" "github.com/m1k1o/go-transcode/internal" - "github.com/m1k1o/go-transcode/internal/config" ) func init() { @@ -16,21 +15,13 @@ func init() { Run: transcode.Service.ServeCommand, } - configs := []config.Config{ - transcode.Service.ServerConfig, - } - cobra.OnInitialize(func() { - for _, cfg := range configs { - cfg.Set() - } + transcode.Service.ServerConfig.Set() transcode.Service.Preflight() }) - for _, cfg := range configs { - if err := cfg.Init(command); err != nil { - log.Panic().Err(err).Msg("unable to run serve command") - } + if err := transcode.Service.ServerConfig.Init(command); err !=nil { + log.Panic().Err(err).Msg("unable to run serve command") } root.AddCommand(command) diff --git a/internal/config/config.go b/internal/config/config.go index 6e4357b..953df7c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,11 +9,6 @@ import ( "github.com/spf13/viper" ) -type Config interface { - Init(cmd *cobra.Command) error - Set() -} - type YamlConf struct { BaseDir string `yaml:"basedir",omitempty` Streams map[string]string `yaml:"streams"` From 40942e30c278bb95fba8d09dfc7ce89281ba8340 Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 16:16:20 +0200 Subject: [PATCH 07/56] quick test for settings cascade --- test_args.sh | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 test_args.sh diff --git a/test_args.sh b/test_args.sh new file mode 100755 index 0000000..6a2bac1 --- /dev/null +++ b/test_args.sh @@ -0,0 +1,37 @@ +#! /usr/bin/env bash + +# Number of seconds to wait for HTTP server to start +# Test fails after this timeout +TIMEOUT=1 + +go build + +# Test default settings (:8080) +output="$(timeout --preserve-status $TIMEOUT ./go-transcode serve 2>&1 3>&1)" +if echo "$output" | grep ":8080" > /dev/null; then + echo "Default settings work" +else + echo "Default settings failed:" + echo "$output" +fi + +# Test env settings (:8889) +output="$(TRANSCODE_BIND=":8889" timeout --preserve-status $TIMEOUT ./go-transcode serve 2>&1)" +if echo "$output" | grep ":8889" > /dev/null; then + echo "Env settings work" +else + echo "Env settings failed:" + echo "$output" +fi + +# Test CLI settings (:8890) +# We also check that CLI settings have higher priority than Env +output="$(TRANSCODE_BIND=":8889" timeout --preserve-status $TIMEOUT ./go-transcode serve --bind ":8890" 2>&1)" +if echo "$output" | grep ":8890" > /dev/null; then + echo "CLI settings work" +else + echo "CLI settings failed:" + echo "$output" +fi + + From 331746a6d61d310effe68deb023131c6acc29c4b Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 17:08:26 +0200 Subject: [PATCH 08/56] ignore env TRANSCODE_BIND in default settings test --- test_args.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_args.sh b/test_args.sh index 6a2bac1..3dbe042 100755 --- a/test_args.sh +++ b/test_args.sh @@ -7,7 +7,7 @@ TIMEOUT=1 go build # Test default settings (:8080) -output="$(timeout --preserve-status $TIMEOUT ./go-transcode serve 2>&1 3>&1)" +output="$(TRANSCODE_BIND= timeout --preserve-status $TIMEOUT ./go-transcode serve 2>&1 3>&1)" if echo "$output" | grep ":8080" > /dev/null; then echo "Default settings work" else From 502c288c1131ec89bc8bf20be3faee12094482a2 Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 17:09:08 +0200 Subject: [PATCH 09/56] move YamlConf to Server config, loaded by viper --- hls/manager.go | 4 ++-- internal/api/hls.go | 2 +- internal/api/http.go | 4 ++-- internal/api/router.go | 20 ++++------------ internal/config/config.go | 48 ++++++++++++--------------------------- internal/http/http.go | 3 +++ internal/main.go | 2 +- 7 files changed, 29 insertions(+), 54 deletions(-) diff --git a/hls/manager.go b/hls/manager.go index e7744e7..e638fa9 100644 --- a/hls/manager.go +++ b/hls/manager.go @@ -54,10 +54,10 @@ type ManagerCtx struct { playlistLoad chan string shutdown chan interface{} - Conf *config.YamlConf + Conf *config.Server } -func New(cmdFactory func() *exec.Cmd, conf *config.YamlConf) *ManagerCtx { +func New(cmdFactory func() *exec.Cmd, conf *config.Server) *ManagerCtx { return &ManagerCtx{ Conf: conf, logger: log.With().Str("module", "hls").Str("submodule", "manager").Logger(), diff --git a/internal/api/hls.go b/internal/api/hls.go index d385d44..0b18483 100644 --- a/internal/api/hls.go +++ b/internal/api/hls.go @@ -44,7 +44,7 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { // create new manager manager = hls.New(func() *exec.Cmd { // get transcode cmd - cmd, err := transcodeStart("profiles/hls", profile, input) + cmd, err := a.transcodeStart("profiles/hls", profile, input) if err != nil { logger.Error().Err(err).Msg("transcode could not be started") w.WriteHeader(http.StatusInternalServerError) diff --git a/internal/api/http.go b/internal/api/http.go index 9379474..dcf3df6 100644 --- a/internal/api/http.go +++ b/internal/api/http.go @@ -55,7 +55,7 @@ func (a *ApiManagerCtx) Http(r chi.Router) { return } - cmd, err := transcodeStart("profiles/http", profile, input) + cmd, err := a.transcodeStart("profiles/http", profile, input) if err != nil { logger.Warn().Err(err).Msg("transcode could not be started") w.WriteHeader(http.StatusInternalServerError) @@ -90,7 +90,7 @@ func (a *ApiManagerCtx) Http(r chi.Router) { profile := chi.URLParam(r, "profile") input := chi.URLParam(r, "input") - cmd, err := transcodeStart("profiles", profile, input) + cmd, err := a.transcodeStart("profiles", profile, input) if err != nil { logger.Warn().Err(err).Msg("transcode could not be started") w.WriteHeader(http.StatusInternalServerError) diff --git a/internal/api/router.go b/internal/api/router.go index 6ca4aa3..dc7f80c 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -13,21 +13,11 @@ import ( "github.com/m1k1o/go-transcode/internal/config" ) -var conf *config.YamlConf - -func init() { - var err error - conf, err = config.LoadConf("streams.yaml") - if err != nil { - panic(err) - } -} - type ApiManagerCtx struct { - Conf *config.YamlConf + Conf *config.Server } -func New() *ApiManagerCtx { +func New(conf *config.Server) *ApiManagerCtx { return &ApiManagerCtx{Conf: conf} } @@ -41,8 +31,8 @@ func (a *ApiManagerCtx) Mount(r *chi.Mux) { r.Group(a.Http) } -func transcodeStart(folder string, profile string, input string) (*exec.Cmd, error) { - url, ok := conf.Streams[input] +func (a *ApiManagerCtx) transcodeStart(folder string, profile string, input string) (*exec.Cmd, error) { + url, ok := a.Conf.Streams[input] if !ok { return nil, fmt.Errorf("stream not found") } @@ -52,7 +42,7 @@ func transcodeStart(folder string, profile string, input string) (*exec.Cmd, err return nil, fmt.Errorf("invalid profile path") } - profilePath := fmt.Sprintf("%s/%s/%s.sh", conf.BaseDir, folder, profile) + profilePath := fmt.Sprintf("%s/%s/%s.sh", a.Conf.BaseDir, folder, profile) if _, err := os.Stat(profilePath); os.IsNotExist(err) { return nil, err } diff --git a/internal/config/config.go b/internal/config/config.go index 953df7c..58d569f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,45 +1,12 @@ package config import ( - "io/ioutil" "os" - "gopkg.in/yaml.v2" "github.com/spf13/cobra" "github.com/spf13/viper" ) -type YamlConf struct { - BaseDir string `yaml:"basedir",omitempty` - Streams map[string]string `yaml:"streams"` -} - -func LoadConf(path string) (*YamlConf, error) { - yamlFile, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - - conf := &YamlConf{} - err = yaml.Unmarshal(yamlFile, conf) - if err != nil { - return nil, err - } - - // If basedir is not explicit in config, try /etc/go-transcode/, - // fallback to the current working directory - if conf.BaseDir == "" { - if _, err := os.Stat("/etc/go-transcode"); os.IsNotExist(err) { - cwd, _ := os.Getwd() - conf.BaseDir = cwd - } else { - conf.BaseDir = "/etc/go-transcode" - } - } - - return conf, nil -} - type Root struct { Debug bool CfgFile string @@ -70,6 +37,8 @@ type Server struct { Bind string Static string Proxy bool + BaseDir string `yaml:"basedir",omitempty` + Streams map[string]string `yaml:"streams"` } func (Server) Init(cmd *cobra.Command) error { @@ -98,6 +67,15 @@ func (Server) Init(cmd *cobra.Command) error { return err } + var basedir_fallback string + if _, err := os.Stat("/etc/transcode"); os.IsNotExist(err) { + cwd, _ := os.Getwd() + basedir_fallback = cwd + } else { + basedir_fallback = "/etc/transcode" + } + cmd.PersistentFlags().String("basedir", basedir_fallback, "The base directory for assets and profiles") + return nil } @@ -107,4 +85,8 @@ func (s *Server) Set() { s.Bind = viper.GetString("bind") s.Static = viper.GetString("static") s.Proxy = viper.GetBool("proxy") + s.BaseDir = viper.GetString("basedir") + // TODO: potential serve /etc/transcode basedir + if s.BaseDir == "" { s.BaseDir = "." } + s.Streams = viper.GetStringMapString("streams") } diff --git a/internal/http/http.go b/internal/http/http.go index 824a6dc..d0a6efe 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -28,6 +28,7 @@ func New(ApiManager types.ApiManager, conf *config.Server) *ServerCtx { router := chi.NewRouter() router.Use(middleware.Recoverer) // Recover from panics without crashing server router.Use(middleware.RequestID) // Create a request ID for each request + // TODO: Why Logger not logger? router.Use(Logger) // Log API request calls using custom logger function ApiManager.Mount(router) @@ -53,6 +54,8 @@ func New(ApiManager types.ApiManager, conf *config.Server) *ServerCtx { Handler: router, } + logger.Info().Msgf("Serving streams from basedir %v: %v", conf.BaseDir, conf.Streams) + return &ServerCtx{ logger: logger, router: router, diff --git a/internal/main.go b/internal/main.go index a58d2de..d55e3d4 100644 --- a/internal/main.go +++ b/internal/main.go @@ -36,7 +36,7 @@ func (main *Main) Preflight() { } func (main *Main) Start() { - main.apiManager = api.New() + main.apiManager = api.New(main.ServerConfig) main.server = http.New( main.apiManager, From a2b678e0bd477e0576aad302e1706b2fee202306 Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 17:25:52 +0200 Subject: [PATCH 10/56] Fail test_args if go build fails --- test_args.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_args.sh b/test_args.sh index 3dbe042..8cd9f5c 100755 --- a/test_args.sh +++ b/test_args.sh @@ -4,7 +4,8 @@ # Test fails after this timeout TIMEOUT=1 -go build +# Fail if go build fails +go build || exit 1 # Test default settings (:8080) output="$(TRANSCODE_BIND= timeout --preserve-status $TIMEOUT ./go-transcode serve 2>&1 3>&1)" From 1613aa037f267297476ce3cd4ba0e0a3bc3d4963 Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 17:42:18 +0200 Subject: [PATCH 11/56] auto reload config file --- cmd/root.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/root.go b/cmd/root.go index d98821b..2eb9c6e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -8,6 +8,7 @@ import ( "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" + "github.com/fsnotify/fsnotify" "github.com/m1k1o/go-transcode/internal" ) @@ -63,6 +64,13 @@ func init() { } } + viper.OnConfigChange(func(e fsnotify.Event) { + // TODO: list config change + log.Logger.Info().Msg("Config file reloaded") + transcode.Service.ServerConfig.Set() + }) + viper.WatchConfig() + file := viper.ConfigFileUsed() logger := log.With(). Bool("debug", viper.GetBool("debug")). From 9f121175e494778409247a4544f030b38b5d7533 Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 19:10:22 +0200 Subject: [PATCH 12/56] Removed unused "logging" --- cmd/root.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 2eb9c6e..178d951 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -74,7 +74,6 @@ func init() { file := viper.ConfigFileUsed() logger := log.With(). Bool("debug", viper.GetBool("debug")). - Str("logging", viper.GetString("logs")). Str("config", file). Logger() From e51a38b1771da1db3cd78a86792709a1ff17c4ab Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 19:10:33 +0200 Subject: [PATCH 13/56] Add TLS warning message --- internal/http/http.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/http/http.go b/internal/http/http.go index d0a6efe..79c715f 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -70,6 +70,7 @@ func (s *ServerCtx) Start() { if err := s.http.ListenAndServeTLS(s.conf.Cert, s.conf.Key); err != http.ErrServerClosed { s.logger.Panic().Err(err).Msg("unable to start https server") } + s.logger.Warn().Msg("TLS support is provided for convenience, but you should never use it in production. Use a reverse proxy (apache nginx caddy) instead!") }() s.logger.Info().Msgf("https listening on %s", s.http.Addr) } else { From 8019523ca07de1d6e753f9df64c3dad7173fc481 Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 19:46:04 +0200 Subject: [PATCH 14/56] add info to readme --- README.md | 95 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 2e5d799..94f73ad 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,59 @@ -# Go live HTTP on-demand transcoding -Transcoding is expensive and resource consuming operation on CPU and GPU. For big companies with thousands of customers it is essential, to have a dedicated 24/7 transcoding servers. But we, single sporadic users of transcoding, need to have different approach. Transcoding should be done only when its output is really needed. This tool is trying to solve this problem by offering transcoding on demand. +# go-transcode HTTP on-demand transcoding API -This tool is intended to be used with live streams only. Seeking is not supported, yet. +## Why + +Transcoding is expensive and resource consuming operation on CPU and GPU. For big companies with thousands of customers it is essential, to have a dedicated 24/7 transcoding servers which can store all the transcoded versions. + +For the rest of us who don't have infinite resources and cannot have 3 times bigger media library because of transcoding, we should only transcode when it is needed. This tool is trying to solve this problem by offering transcoding on demand. + +This feature is common in media centers (plex, jellyfin) but there was no simple transcoding server without all other media center features. Now there is one! go-transcode is simple and extensible, and will probably not add features unrelated to transcoding. + +## Features + +- Sources + - [x] Live streams + - [ ] At-rest files (basic support) + - [x] Any codec/container supported by ffmpeg +- Outputs + - [x] Basic MP4 over HTTP (h264+aac) : `http://go-transcode/[profile]/[stream-id]` + - [x] Basic HLS over HTTP (h264+aac) : `http://go-transcode/[profile]/[stream-id]/index.m3u8` + - [x] Demo HTML player : `http://go-transcode/[profile]/[stream-id]/play.html` +- [ ] Seeking (index) +- [ ] Audio/Subtitles tracks +- [ ] Private mode (serve users authenticated by reverse proxy) ## Config -Specify streams as object in yaml file. -### Streams -Create `streams.yaml` file, with your streams: +Place your config file in `transcode.yml` (or `/etc/transcode/transcode.yml`). The streams are defined like this: ```yaml streams: : ``` -Example: +Full configuration example: + ```yaml +debug: true # debug logs +bind: localhost:8888 # IP/port to bind to +static: # TODO: what is this? +proxy: true # TODO: issue #4 streams: cam: rtmp://localhost/live/cam ch1_hd: http://192.168.1.34:9981/stream/channelid/85 ch2_hd: http://192.168.1.34:9981/stream/channelid/43 ``` -HTTP streaming is accessible via: -- `http://localhost:8080//` - -HLS is accessible via: -- `http://localhost:8080///index.m3u8` -- `http://localhost:8080///play.html` - -## CPU Profiles -Profiles (HTTP and HLS) with CPU transcoding can be found in `profiles`: +## Transcoding profiles -* h264_360p -* h264_540p -* h264_720p -* h264_1080p +go-transcode supports any formats that ffmpeg likes. We provide profiles out-of-the-box for h264+aac (mp4 container) for 360p, 540p, 720p and 1080p resolutions: `h264_360p`, `h264_540p`, `h264_720p` and `h264_1080p`. Profiles can have any name, but must match regex: `^[0-9A-Za-z_-]+$` -Profile names must match flowing regex: `^[0-9A-Za-z_-]+$` +We provide two different profiles directories: -## GPU Profiles -Profiles (HTTP and HLS) with GPU transcoding can be found in `profiles_nvidia`: +- profiles/ for CPU transcoding +- profiles_nvidia/ for NVENC support (proprietary Nvidia driver) -* h264_360p -* h264_540p -* h264_720p -* h264_1080p - -Profile names must match flowing regex: `^[0-9A-Za-z_-]+$` +In these profile directories, actual profiles are located in hls/ and http/, depending on the output format requested. ## Docker @@ -103,3 +109,34 @@ Input codec will be automatically determined from given stream. Please check you | vc1 | vc1_cuvid | SMPTE VC-1 | | vp8 | vp8_cuvid | On2 VP8 | | vp9 | vp9_cuvid | Google VP9 | + +## Alternatives + +- [nginx-vod-module](https://github.com/kaltura/nginx-vod-module): only supports MP4 sources +- [tvheadend](https://tvheadend.org/): intended for TV sources, not media library ; nvidia support hard to compile +- [jellyfin](https://github.com/jellyfin/jellyfin): doesn't support live streams ; cannot run standalone transcoding service (without media library) +- suggestions? + +## Contribute + +Join us in the [Matrix space](https://matrix.to/#/#go-transcode:proxychat.net) or [via XMPP bridge](xmpp:#go-transcode#proxychat.net@matrix.org). + +## Architecture + +The source code is in the following files/folders: + +- cmd/ and main.go : source for the command-line interface +- internal/ : actual source code logic +- hls/ : process runner for HLS transcoding + +TODO: document different modules/packages and dependencies + +Other files/folders in the repositories are: + +- data/ : files used/served by go-transcode +- profiles/ and profiles_nvidia/ : the ffmpeg profiles for transcoding +- dev/ : some docker helper scripts +- tests/ : some tests for the project +- god.mod and go.sum : golang dependencies/modules tracking +- Dockerfile, Dockerfile.nvidia and docker-compose.yaml : for the docker lovers +- LICENSE : licensing information (Apache 2.0) From 88f9c4a9a90d32767474cb1adb6e4f8d0932edfe Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 19:46:16 +0200 Subject: [PATCH 15/56] Moved args test to tests/ folder --- test_args.sh => tests/args.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test_args.sh => tests/args.sh (100%) diff --git a/test_args.sh b/tests/args.sh similarity index 100% rename from test_args.sh rename to tests/args.sh From cdcda5b036f279604a86c9b939f4c50e07262d8b Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 20:08:47 +0200 Subject: [PATCH 16/56] add test for config reload --- tests/reload.sh | 75 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100755 tests/reload.sh diff --git a/tests/reload.sh b/tests/reload.sh new file mode 100755 index 0000000..078f99a --- /dev/null +++ b/tests/reload.sh @@ -0,0 +1,75 @@ +#! /usr/bin/env bash + +go build || exit 1 + +if [ ! -f $1 ]; then + echo "Please give test video first argument" + exit 1 +fi + +tmpfile=$(mktemp --suffix .yml) +log=$(mktemp) + +BASE_PORT=8888 +INCREMENT=1 + +port=$BASE_PORT +isfree=$(netstat -taln | grep $port) + +while [[ -n "$isfree" ]]; do + port=$[port+INCREMENT] + isfree=$(netstat -taln | grep $port) +done + +if [[ -n "$isfree" ]]; then + echo "Could not find free port for test" + exit 1 +fi + +baseurl="http://localhost:$port" + +echo "Using port: $port. Logging to $log" + +echo -e "streams:\n test: $1" > $tmpfile + +./go-transcode serve --config $tmpfile --bind :$port >> $log 2>&1 & +pid=$! + +output="$(curl -o /dev/null -s -I -XGET -w "%{http_code}" $baseurl/h264_720p/test)" +if [[ "$output" != "200" ]]; then + echo "$output" + echo "Serve 1 failed" + exit 1 +else + echo "Serve 1 success" +fi + +output="$(curl -o /dev/null -s -I -XGET -w "%{http_code}" $baseurl/h264_720p/test2)" +if [[ "$output" != "404" ]]; then + echo "Serve 2 failed" + exit 1 +else + echo "Serve 2 success" +fi + +# Change config and try again test2 +echo -e "streams:\n test2: $1" > $tmpfile +sleep 1 + +output="$(curl -o /dev/null -s -I -XGET -w "%{http_code}" $baseurl/h264_720p/test2)" +if [[ "$output" != "200" ]]; then + echo "Serve 3 failed" + exit 1 +else + echo "Serve 3 success" +fi + +output="$(curl -o /dev/null -s -I -XGET -w "%{http_code}" $baseurl/h264_720p/test)" +if [[ "$output" != "404" ]]; then + echo "Serve 4 failed" + exit 1 +else + echo "Serve 4 success" +fi + + From dc50d78da2d3fdf7d70271a793382659b0c83de7 Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 20:19:06 +0200 Subject: [PATCH 17/56] Fail test when empty argument --- tests/reload.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/reload.sh b/tests/reload.sh index 078f99a..5e867de 100755 --- a/tests/reload.sh +++ b/tests/reload.sh @@ -2,7 +2,7 @@ go build || exit 1 -if [ ! -f $1 ]; then +if [ $# -lt 1 ] || [ ! -f $1 ]; then echo "Please give test video first argument" exit 1 fi From ed7ae519ae341b0abdef865f717590362addf3dc Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 20:26:35 +0200 Subject: [PATCH 18/56] BaseDir can be /etc/transcode if it exists --- internal/config/config.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 58d569f..df11362 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -67,14 +67,7 @@ func (Server) Init(cmd *cobra.Command) error { return err } - var basedir_fallback string - if _, err := os.Stat("/etc/transcode"); os.IsNotExist(err) { - cwd, _ := os.Getwd() - basedir_fallback = cwd - } else { - basedir_fallback = "/etc/transcode" - } - cmd.PersistentFlags().String("basedir", basedir_fallback, "The base directory for assets and profiles") + cmd.PersistentFlags().String("basedir", "", "The base directory for assets and profiles") return nil } @@ -86,7 +79,13 @@ func (s *Server) Set() { s.Static = viper.GetString("static") s.Proxy = viper.GetBool("proxy") s.BaseDir = viper.GetString("basedir") - // TODO: potential serve /etc/transcode basedir - if s.BaseDir == "" { s.BaseDir = "." } + if s.BaseDir == "" { + if _, err := os.Stat("/etc/transcode"); os.IsNotExist(err) { + cwd, _ := os.Getwd() + s.BaseDir = cwd + } else { + s.BaseDir = "/etc/transcode" + } + } s.Streams = viper.GetStringMapString("streams") } From 4db1a14b90a725361c013472499d500de9985077 Mon Sep 17 00:00:00 2001 From: klahaha Date: Fri, 24 Sep 2021 20:37:29 +0200 Subject: [PATCH 19/56] Profiles are merged --- internal/api/hls.go | 2 +- internal/api/http.go | 3 ++- internal/api/router.go | 3 ++- internal/config/config.go | 5 +++++ profiles/{ => default}/hls/copy.sh | 0 profiles/{ => default}/hls/h264_1080p.sh | 0 profiles/{ => default}/hls/h264_360p.sh | 0 profiles/{ => default}/hls/h264_540p.sh | 0 profiles/{ => default}/hls/h264_720p.sh | 0 profiles/{ => default}/http/copy.sh | 0 profiles/{ => default}/http/h264_1080p.sh | 0 profiles/{ => default}/http/h264_360p.sh | 0 profiles/{ => default}/http/h264_540p.sh | 0 profiles/{ => default}/http/h264_720p.sh | 0 {profiles_nvidia => profiles/nvidia}/.helpers.sh | 0 {profiles_nvidia => profiles/nvidia}/hls/copy.sh | 0 {profiles_nvidia => profiles/nvidia}/hls/h264_1080p.sh | 0 {profiles_nvidia => profiles/nvidia}/hls/h264_360p.sh | 0 {profiles_nvidia => profiles/nvidia}/hls/h264_540p.sh | 0 {profiles_nvidia => profiles/nvidia}/hls/h264_720p.sh | 0 {profiles_nvidia => profiles/nvidia}/http/copy.sh | 0 {profiles_nvidia => profiles/nvidia}/http/h264_1080p.sh | 0 {profiles_nvidia => profiles/nvidia}/http/h264_360p.sh | 0 {profiles_nvidia => profiles/nvidia}/http/h264_540p.sh | 0 {profiles_nvidia => profiles/nvidia}/http/h264_720p.sh | 0 25 files changed, 10 insertions(+), 3 deletions(-) rename profiles/{ => default}/hls/copy.sh (100%) rename profiles/{ => default}/hls/h264_1080p.sh (100%) rename profiles/{ => default}/hls/h264_360p.sh (100%) rename profiles/{ => default}/hls/h264_540p.sh (100%) rename profiles/{ => default}/hls/h264_720p.sh (100%) rename profiles/{ => default}/http/copy.sh (100%) rename profiles/{ => default}/http/h264_1080p.sh (100%) rename profiles/{ => default}/http/h264_360p.sh (100%) rename profiles/{ => default}/http/h264_540p.sh (100%) rename profiles/{ => default}/http/h264_720p.sh (100%) rename {profiles_nvidia => profiles/nvidia}/.helpers.sh (100%) rename {profiles_nvidia => profiles/nvidia}/hls/copy.sh (100%) rename {profiles_nvidia => profiles/nvidia}/hls/h264_1080p.sh (100%) rename {profiles_nvidia => profiles/nvidia}/hls/h264_360p.sh (100%) rename {profiles_nvidia => profiles/nvidia}/hls/h264_540p.sh (100%) rename {profiles_nvidia => profiles/nvidia}/hls/h264_720p.sh (100%) rename {profiles_nvidia => profiles/nvidia}/http/copy.sh (100%) rename {profiles_nvidia => profiles/nvidia}/http/h264_1080p.sh (100%) rename {profiles_nvidia => profiles/nvidia}/http/h264_360p.sh (100%) rename {profiles_nvidia => profiles/nvidia}/http/h264_540p.sh (100%) rename {profiles_nvidia => profiles/nvidia}/http/h264_720p.sh (100%) diff --git a/internal/api/hls.go b/internal/api/hls.go index 0b18483..e7ccb6d 100644 --- a/internal/api/hls.go +++ b/internal/api/hls.go @@ -44,7 +44,7 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { // create new manager manager = hls.New(func() *exec.Cmd { // get transcode cmd - cmd, err := a.transcodeStart("profiles/hls", profile, input) + cmd, err := a.transcodeStart("hls", profile, input) if err != nil { logger.Error().Err(err).Msg("transcode could not be started") w.WriteHeader(http.StatusInternalServerError) diff --git a/internal/api/http.go b/internal/api/http.go index dcf3df6..ea1e36c 100644 --- a/internal/api/http.go +++ b/internal/api/http.go @@ -55,7 +55,7 @@ func (a *ApiManagerCtx) Http(r chi.Router) { return } - cmd, err := a.transcodeStart("profiles/http", profile, input) + cmd, err := a.transcodeStart("http", profile, input) if err != nil { logger.Warn().Err(err).Msg("transcode could not be started") w.WriteHeader(http.StatusInternalServerError) @@ -90,6 +90,7 @@ func (a *ApiManagerCtx) Http(r chi.Router) { profile := chi.URLParam(r, "profile") input := chi.URLParam(r, "input") + // TODO: what is this? why no http/hls? cmd, err := a.transcodeStart("profiles", profile, input) if err != nil { logger.Warn().Err(err).Msg("transcode could not be started") diff --git a/internal/api/router.go b/internal/api/router.go index dc7f80c..f2b270a 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -42,7 +42,8 @@ func (a *ApiManagerCtx) transcodeStart(folder string, profile string, input stri return nil, fmt.Errorf("invalid profile path") } - profilePath := fmt.Sprintf("%s/%s/%s.sh", a.Conf.BaseDir, folder, profile) + // [basedir]/profiles/[profiles]/hls,http/[profile] + profilePath := fmt.Sprintf("%s/profiles/%s/%s/%s.sh", a.Conf.BaseDir, a.Conf.Profiles, folder, profile) if _, err := os.Stat(profilePath); os.IsNotExist(err) { return nil, err } diff --git a/internal/config/config.go b/internal/config/config.go index df11362..a47acd2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -39,6 +39,7 @@ type Server struct { Proxy bool BaseDir string `yaml:"basedir",omitempty` Streams map[string]string `yaml:"streams"` + Profiles string `yaml:"profiles",omitempty` } func (Server) Init(cmd *cobra.Command) error { @@ -69,6 +70,8 @@ func (Server) Init(cmd *cobra.Command) error { cmd.PersistentFlags().String("basedir", "", "The base directory for assets and profiles") + cmd.PersistentFlags().String("profiles", "default", "The hardware encoding profiles to load for ffmpeg (default, nvidia)") + return nil } @@ -87,5 +90,7 @@ func (s *Server) Set() { s.BaseDir = "/etc/transcode" } } + s.Profiles = viper.GetString("profiles") + if s.Profiles == "" { s.Profiles = "default" } s.Streams = viper.GetStringMapString("streams") } diff --git a/profiles/hls/copy.sh b/profiles/default/hls/copy.sh similarity index 100% rename from profiles/hls/copy.sh rename to profiles/default/hls/copy.sh diff --git a/profiles/hls/h264_1080p.sh b/profiles/default/hls/h264_1080p.sh similarity index 100% rename from profiles/hls/h264_1080p.sh rename to profiles/default/hls/h264_1080p.sh diff --git a/profiles/hls/h264_360p.sh b/profiles/default/hls/h264_360p.sh similarity index 100% rename from profiles/hls/h264_360p.sh rename to profiles/default/hls/h264_360p.sh diff --git a/profiles/hls/h264_540p.sh b/profiles/default/hls/h264_540p.sh similarity index 100% rename from profiles/hls/h264_540p.sh rename to profiles/default/hls/h264_540p.sh diff --git a/profiles/hls/h264_720p.sh b/profiles/default/hls/h264_720p.sh similarity index 100% rename from profiles/hls/h264_720p.sh rename to profiles/default/hls/h264_720p.sh diff --git a/profiles/http/copy.sh b/profiles/default/http/copy.sh similarity index 100% rename from profiles/http/copy.sh rename to profiles/default/http/copy.sh diff --git a/profiles/http/h264_1080p.sh b/profiles/default/http/h264_1080p.sh similarity index 100% rename from profiles/http/h264_1080p.sh rename to profiles/default/http/h264_1080p.sh diff --git a/profiles/http/h264_360p.sh b/profiles/default/http/h264_360p.sh similarity index 100% rename from profiles/http/h264_360p.sh rename to profiles/default/http/h264_360p.sh diff --git a/profiles/http/h264_540p.sh b/profiles/default/http/h264_540p.sh similarity index 100% rename from profiles/http/h264_540p.sh rename to profiles/default/http/h264_540p.sh diff --git a/profiles/http/h264_720p.sh b/profiles/default/http/h264_720p.sh similarity index 100% rename from profiles/http/h264_720p.sh rename to profiles/default/http/h264_720p.sh diff --git a/profiles_nvidia/.helpers.sh b/profiles/nvidia/.helpers.sh similarity index 100% rename from profiles_nvidia/.helpers.sh rename to profiles/nvidia/.helpers.sh diff --git a/profiles_nvidia/hls/copy.sh b/profiles/nvidia/hls/copy.sh similarity index 100% rename from profiles_nvidia/hls/copy.sh rename to profiles/nvidia/hls/copy.sh diff --git a/profiles_nvidia/hls/h264_1080p.sh b/profiles/nvidia/hls/h264_1080p.sh similarity index 100% rename from profiles_nvidia/hls/h264_1080p.sh rename to profiles/nvidia/hls/h264_1080p.sh diff --git a/profiles_nvidia/hls/h264_360p.sh b/profiles/nvidia/hls/h264_360p.sh similarity index 100% rename from profiles_nvidia/hls/h264_360p.sh rename to profiles/nvidia/hls/h264_360p.sh diff --git a/profiles_nvidia/hls/h264_540p.sh b/profiles/nvidia/hls/h264_540p.sh similarity index 100% rename from profiles_nvidia/hls/h264_540p.sh rename to profiles/nvidia/hls/h264_540p.sh diff --git a/profiles_nvidia/hls/h264_720p.sh b/profiles/nvidia/hls/h264_720p.sh similarity index 100% rename from profiles_nvidia/hls/h264_720p.sh rename to profiles/nvidia/hls/h264_720p.sh diff --git a/profiles_nvidia/http/copy.sh b/profiles/nvidia/http/copy.sh similarity index 100% rename from profiles_nvidia/http/copy.sh rename to profiles/nvidia/http/copy.sh diff --git a/profiles_nvidia/http/h264_1080p.sh b/profiles/nvidia/http/h264_1080p.sh similarity index 100% rename from profiles_nvidia/http/h264_1080p.sh rename to profiles/nvidia/http/h264_1080p.sh diff --git a/profiles_nvidia/http/h264_360p.sh b/profiles/nvidia/http/h264_360p.sh similarity index 100% rename from profiles_nvidia/http/h264_360p.sh rename to profiles/nvidia/http/h264_360p.sh diff --git a/profiles_nvidia/http/h264_540p.sh b/profiles/nvidia/http/h264_540p.sh similarity index 100% rename from profiles_nvidia/http/h264_540p.sh rename to profiles/nvidia/http/h264_540p.sh diff --git a/profiles_nvidia/http/h264_720p.sh b/profiles/nvidia/http/h264_720p.sh similarity index 100% rename from profiles_nvidia/http/h264_720p.sh rename to profiles/nvidia/http/h264_720p.sh From 2489df75c85fec6d77bff1f121968d050858bf67 Mon Sep 17 00:00:00 2001 From: klahaha <91276599+klahaha@users.noreply.github.com> Date: Fri, 24 Sep 2021 20:42:59 +0200 Subject: [PATCH 20/56] Update README.md --- README.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 94f73ad..aea2cf2 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,17 @@ This feature is common in media centers (plex, jellyfin) but there was no simple ## Features -- Sources - - [x] Live streams - - [ ] At-rest files (basic support) - - [x] Any codec/container supported by ffmpeg -- Outputs - - [x] Basic MP4 over HTTP (h264+aac) : `http://go-transcode/[profile]/[stream-id]` - - [x] Basic HLS over HTTP (h264+aac) : `http://go-transcode/[profile]/[stream-id]/index.m3u8` - - [x] Demo HTML player : `http://go-transcode/[profile]/[stream-id]/play.html` +Sources: +- [x] Live streams +- [ ] At-rest files (basic support) +- [x] Any codec/container supported by ffmpeg + +Outputs: +- [x] Basic MP4 over HTTP (h264+aac) : `http://go-transcode/[profile]/[stream-id]` +- [x] Basic HLS over HTTP (h264+aac) : `http://go-transcode/[profile]/[stream-id]/index.m3u8` +- [x] Demo HTML player : `http://go-transcode/[profile]/[stream-id]/play.html` + +Features: - [ ] Seeking (index) - [ ] Audio/Subtitles tracks - [ ] Private mode (serve users authenticated by reverse proxy) @@ -50,13 +53,15 @@ go-transcode supports any formats that ffmpeg likes. We provide profiles out-of- We provide two different profiles directories: -- profiles/ for CPU transcoding -- profiles_nvidia/ for NVENC support (proprietary Nvidia driver) +- profiles/default for CPU transcoding +- profiles/nvidia for NVENC support (proprietary Nvidia driver) In these profile directories, actual profiles are located in hls/ and http/, depending on the output format requested. ## Docker +TODO: outdated docker section + ### Build ```sh From c189ef485b43998cbb866155e3bd6a81fc2235cf Mon Sep 17 00:00:00 2001 From: klahaha Date: Sat, 25 Sep 2021 14:01:34 +0200 Subject: [PATCH 21/56] Add the general room to the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aea2cf2..0167f99 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ Input codec will be automatically determined from given stream. Please check you ## Contribute -Join us in the [Matrix space](https://matrix.to/#/#go-transcode:proxychat.net) or [via XMPP bridge](xmpp:#go-transcode#proxychat.net@matrix.org). +Join us in the [Matrix space](https://matrix.to/#/#go-transcode:proxychat.net) (or the [#go-transcode-general](https://matrix.to/#/#go-transcode-general:proxychat.net) room directly) or [via XMPP bridge](xmpp:#go-transcode-general#proxychat.net@matrix.org). ## Architecture From 8b0d5366fbeab72b2a48435f8e97acf14cc1946c Mon Sep 17 00:00:00 2001 From: klahaha Date: Sat, 25 Sep 2021 17:23:33 +0200 Subject: [PATCH 22/56] fix basedir twice in cmd path before basedir config it was necessary to add it here --- hls/manager.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/hls/manager.go b/hls/manager.go index e638fa9..a58293f 100644 --- a/hls/manager.go +++ b/hls/manager.go @@ -2,7 +2,6 @@ package hls import ( "errors" - "fmt" "io" "net/http" "os" @@ -86,7 +85,6 @@ func (m *ManagerCtx) Start() error { m.cmd = m.cmdFactory() m.cmd.Dir = m.tempdir - //fmt.Println(m.cmd.Dir) if m.events.onCmdLog != nil { m.cmd.Stderr = utils.LogEvent(m.events.onCmdLog) @@ -94,10 +92,6 @@ func (m *ManagerCtx) Start() error { m.cmd.Stderr = utils.LogWriter(m.logger) } - //cwd, _ := os.Getwd() - //m.cmd.Path = fmt.Sprintf("%s/%s", cwd, m.cmd.Path) - m.cmd.Path = fmt.Sprintf("%s/%s", m.Conf.BaseDir, m.cmd.Path) - read, write := io.Pipe() m.cmd.Stdout = write From 5bb40772be53db43473b1190fe6961070a0e7312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 19:42:51 +0200 Subject: [PATCH 23/56] update README. --- README.md | 58 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 0167f99..1ccb628 100644 --- a/README.md +++ b/README.md @@ -12,22 +12,22 @@ This feature is common in media centers (plex, jellyfin) but there was no simple Sources: - [x] Live streams -- [ ] At-rest files (basic support) +- [ ] Static files (basic support) - [x] Any codec/container supported by ffmpeg Outputs: - [x] Basic MP4 over HTTP (h264+aac) : `http://go-transcode/[profile]/[stream-id]` - [x] Basic HLS over HTTP (h264+aac) : `http://go-transcode/[profile]/[stream-id]/index.m3u8` -- [x] Demo HTML player : `http://go-transcode/[profile]/[stream-id]/play.html` +- [x] Demo HTML player (for HLS) : `http://go-transcode/[profile]/[stream-id]/play.html` Features: -- [ ] Seeking (index) +- [ ] Seeking for static files (index) - [ ] Audio/Subtitles tracks - [ ] Private mode (serve users authenticated by reverse proxy) ## Config -Place your config file in `transcode.yml` (or `/etc/transcode/transcode.yml`). The streams are defined like this: +Place your config file in `./transcode.yml` (or `/etc/transcode/transcode.yml`). The streams are defined like this: ```yaml streams: @@ -37,10 +37,18 @@ streams: Full configuration example: ```yaml -debug: true # debug logs -bind: localhost:8888 # IP/port to bind to -static: # TODO: what is this? -proxy: true # TODO: issue #4 +# allow debug outputs +debug: true + +# bind server to IP:PORT (use :8888 for all connections) +bind: localhost:8888 + +# serve static files from this directory (optional) +static: /var/www/html + +# TODO: issue #4 +proxy: true + streams: cam: rtmp://localhost/live/cam ch1_hd: http://192.168.1.34:9981/stream/channelid/85 @@ -56,11 +64,11 @@ We provide two different profiles directories: - profiles/default for CPU transcoding - profiles/nvidia for NVENC support (proprietary Nvidia driver) -In these profile directories, actual profiles are located in hls/ and http/, depending on the output format requested. +In these profile directories, actual profiles are located in `hls/` and `http/`, depending on the output format requested. ## Docker -TODO: outdated docker section +*TODO: outdated docker section* ### Build @@ -117,10 +125,10 @@ Input codec will be automatically determined from given stream. Please check you ## Alternatives -- [nginx-vod-module](https://github.com/kaltura/nginx-vod-module): only supports MP4 sources -- [tvheadend](https://tvheadend.org/): intended for TV sources, not media library ; nvidia support hard to compile -- [jellyfin](https://github.com/jellyfin/jellyfin): doesn't support live streams ; cannot run standalone transcoding service (without media library) -- suggestions? +- [nginx-vod-module](https://github.com/kaltura/nginx-vod-module): Only supports MP4 sources. +- [tvheadend](https://tvheadend.org/): Intended for various live sources (IPTV or DVB), not media library - although it can record TV. Supports Nvidia acceleration, but it is hard to compile. +- [jellyfin](https://github.com/jellyfin/jellyfin): Supports live TV sources, although does not work realiably. Cannot run standalone transcoding service (without media library). +- Any suggestions? ## Contribute @@ -130,18 +138,18 @@ Join us in the [Matrix space](https://matrix.to/#/#go-transcode:proxychat.net) ( The source code is in the following files/folders: -- cmd/ and main.go : source for the command-line interface -- internal/ : actual source code logic -- hls/ : process runner for HLS transcoding +- `cmd/` and `main.go`: source for the command-line interface +- `internal/`: actual source code logic +- `hls/`: process runner for HLS transcoding -TODO: document different modules/packages and dependencies +*TODO: document different modules/packages and dependencies* Other files/folders in the repositories are: -- data/ : files used/served by go-transcode -- profiles/ and profiles_nvidia/ : the ffmpeg profiles for transcoding -- dev/ : some docker helper scripts -- tests/ : some tests for the project -- god.mod and go.sum : golang dependencies/modules tracking -- Dockerfile, Dockerfile.nvidia and docker-compose.yaml : for the docker lovers -- LICENSE : licensing information (Apache 2.0) +- `data/`: files used/served by go-transcode +- `profiles/ and profiles_nvidia/`: the ffmpeg profiles for transcoding +- `dev/`: some docker helper scripts +- `tests/`: some tests for the project +- `god.mod` and `go.sum`: golang dependencies/modules tracking +- `Dockerfile`, `Dockerfile.nvidia` and `docker-compose.yaml`: for the docker lovers +- `LICENSE`: licensing information (Apache 2.0) From 8a597fb1bfb6dade2ce29a753f6d309c78052291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 19:49:01 +0200 Subject: [PATCH 24/56] remove build script, since it is replaced by go build. --- .gitignore | 1 - Dockerfile | 4 ++-- build | 4 ---- dev/start | 3 ++- 4 files changed, 4 insertions(+), 8 deletions(-) delete mode 100755 build diff --git a/.gitignore b/.gitignore index 73fef16..210bc26 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -bin/ streams.yaml go-transcode diff --git a/Dockerfile b/Dockerfile index 2c1e2b1..e21ea33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,9 +17,9 @@ RUN set -eux; apt update; \ COPY . . RUN go get -v -t -d .; \ - ./build + go build -o bin/go-transcode ENV TRANSCODE_BIND=:8080 -ENTRYPOINT [ "bin/transcode" ] +ENTRYPOINT [ "bin/go-transcode" ] CMD [ "serve" ] diff --git a/build b/build deleted file mode 100755 index 58ea400..0000000 --- a/build +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -#go build -o bin/transcode cmd/transcode/main.go -go build diff --git a/dev/start b/dev/start index 9154cf4..65d4ac1 100755 --- a/dev/start +++ b/dev/start @@ -6,4 +6,5 @@ docker run --rm -it \ -p "3005:8080" \ -v "${PWD}/../:/app" \ --entrypoint="/bin/bash" \ - transcode_server_img -c '/app/build && ./bin/transcode serve --bind :8080'; + --workdir="/app" \ + transcode_server_img -c 'go build && ./go-transcode serve --bind :8080'; From 7a51d87d284cd0d53e5ba7cd90eac0537fdf102e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 19:55:47 +0200 Subject: [PATCH 25/56] remove internal config dependency from hls package. --- hls/manager.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hls/manager.go b/hls/manager.go index a58293f..52884c9 100644 --- a/hls/manager.go +++ b/hls/manager.go @@ -15,7 +15,6 @@ import ( "github.com/rs/zerolog/log" "github.com/m1k1o/go-transcode/internal/utils" - "github.com/m1k1o/go-transcode/internal/config" ) // how often should be cleanup called @@ -53,12 +52,10 @@ type ManagerCtx struct { playlistLoad chan string shutdown chan interface{} - Conf *config.Server } -func New(cmdFactory func() *exec.Cmd, conf *config.Server) *ManagerCtx { +func New(cmdFactory func() *exec.Cmd) *ManagerCtx { return &ManagerCtx{ - Conf: conf, logger: log.With().Str("module", "hls").Str("submodule", "manager").Logger(), cmdFactory: cmdFactory, From 8cce4ca5a0c9f537ee54d090fa21665bf9681b9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 19:59:34 +0200 Subject: [PATCH 26/56] remove internal config dependency from hls package. --- internal/api/hls.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/api/hls.go b/internal/api/hls.go index e7ccb6d..e7a6a5a 100644 --- a/internal/api/hls.go +++ b/internal/api/hls.go @@ -52,7 +52,7 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { } return cmd - }, a.Conf) + }) hlsManagers[ID] = manager } From ad97723c3861761f7fafc64d8e206c7f1755b04d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 20:11:31 +0200 Subject: [PATCH 27/56] use http.NotFound. --- internal/api/hls.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/api/hls.go b/internal/api/hls.go index e7a6a5a..28b1fe3 100644 --- a/internal/api/hls.go +++ b/internal/api/hls.go @@ -30,10 +30,10 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { return } - _, stream_exists := a.Conf.Streams[input] - if !stream_exists { - w.WriteHeader(http.StatusNotFound) - w.Write([]byte("404 not found")) + // check if stream exists + _, ok := a.Conf.Streams[input] + if !ok { + http.NotFound(w, r) return } From 6b517fe33bd560016d752cdf060c3cc0134c1a21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 20:14:26 +0200 Subject: [PATCH 28/56] use path.Join, fixes #5. --- internal/api/hls.go | 3 ++- internal/api/http.go | 3 ++- internal/api/router.go | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/api/hls.go b/internal/api/hls.go index 28b1fe3..ab8391c 100644 --- a/internal/api/hls.go +++ b/internal/api/hls.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "os/exec" + "path" "regexp" "github.com/go-chi/chi" @@ -86,6 +87,6 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { r.Get("/{profile}/{input}/play.html", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") - http.ServeFile(w, r, fmt.Sprintf("%s/%s", a.Conf.BaseDir, "data/play.html")) + http.ServeFile(w, r, path.Join(a.Conf.BaseDir, "data/play.html")) }) } diff --git a/internal/api/http.go b/internal/api/http.go index ea1e36c..66720ce 100644 --- a/internal/api/http.go +++ b/internal/api/http.go @@ -5,6 +5,7 @@ import ( "io" "net/http" "os/exec" + "path" "github.com/go-chi/chi" "github.com/rs/zerolog/log" @@ -22,7 +23,7 @@ func (a *ApiManagerCtx) Http(r chi.Router) { logger.Info().Msg("command startred") // WTF is this for? - cmd := exec.Command(fmt.Sprintf("%s/%s", a.Conf.BaseDir, "data/http-test.sh")) + cmd := exec.Command(path.Join(a.Conf.BaseDir, "data/http-test.sh")) read, write := io.Pipe() cmd.Stdout = write diff --git a/internal/api/router.go b/internal/api/router.go index f2b270a..deb8821 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -5,6 +5,7 @@ import ( "net/http" "os" "os/exec" + "path" "regexp" "github.com/go-chi/chi" @@ -42,8 +43,8 @@ func (a *ApiManagerCtx) transcodeStart(folder string, profile string, input stri return nil, fmt.Errorf("invalid profile path") } - // [basedir]/profiles/[profiles]/hls,http/[profile] - profilePath := fmt.Sprintf("%s/profiles/%s/%s/%s.sh", a.Conf.BaseDir, a.Conf.Profiles, folder, profile) + // [basedir]/profiles/[profiles]/hls,http/[profile].sh + profilePath := path.Join(a.Conf.BaseDir, "profiles", a.Conf.Profiles, folder, fmt.Sprintf("%s.sh", profile)) if _, err := os.Stat(profilePath); os.IsNotExist(err) { return nil, err } From 0a9f7a5da33528da51ddca82aef213034e485774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 20:19:59 +0200 Subject: [PATCH 29/56] API minor changes. --- internal/api/http.go | 23 +++++++++++++++-------- internal/api/router.go | 4 +++- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/internal/api/http.go b/internal/api/http.go index 66720ce..dba0fdf 100644 --- a/internal/api/http.go +++ b/internal/api/http.go @@ -21,9 +21,9 @@ func (a *ApiManagerCtx) Http(r chi.Router) { Str("module", "ffmpeg"). Logger() - logger.Info().Msg("command startred") - // WTF is this for? + // dummy input for testing purposes cmd := exec.Command(path.Join(a.Conf.BaseDir, "data/http-test.sh")) + logger.Info().Msg("command startred") read, write := io.Pipe() cmd.Stdout = write @@ -49,10 +49,10 @@ func (a *ApiManagerCtx) Http(r chi.Router) { profile := chi.URLParam(r, "profile") input := chi.URLParam(r, "input") - _, stream_exists := a.Conf.Streams[input] - if !stream_exists { - w.WriteHeader(http.StatusNotFound) - w.Write([]byte("404 not found")) + // check if stream exists + _, ok := a.Conf.Streams[input] + if !ok { + http.NotFound(w, r) return } @@ -82,6 +82,7 @@ func (a *ApiManagerCtx) Http(r chi.Router) { io.Copy(w, read) }) + // buffered http streaming (alternative to prervious type) r.Get("/{profile}/{input}/buf", func(w http.ResponseWriter, r *http.Request) { logger := log.With(). Str("path", r.URL.Path). @@ -91,8 +92,14 @@ func (a *ApiManagerCtx) Http(r chi.Router) { profile := chi.URLParam(r, "profile") input := chi.URLParam(r, "input") - // TODO: what is this? why no http/hls? - cmd, err := a.transcodeStart("profiles", profile, input) + // check if stream exists + _, ok := a.Conf.Streams[input] + if !ok { + http.NotFound(w, r) + return + } + + cmd, err := a.transcodeStart("http", profile, input) if err != nil { logger.Warn().Err(err).Msg("transcode could not be started") w.WriteHeader(http.StatusInternalServerError) diff --git a/internal/api/router.go b/internal/api/router.go index deb8821..9946b3a 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -19,7 +19,9 @@ type ApiManagerCtx struct { } func New(conf *config.Server) *ApiManagerCtx { - return &ApiManagerCtx{Conf: conf} + return &ApiManagerCtx{ + Conf: conf, + } } func (a *ApiManagerCtx) Mount(r *chi.Mux) { From 03c0b3fbffe17aa10ef0787bde7863c6154d6e60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 20:20:43 +0200 Subject: [PATCH 30/56] add transcode config to gitignore. --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 210bc26..c234a0d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ -streams.yaml +transcode.yml +transcode.yaml go-transcode From af098b4b5ed905936193c4ed1735d809131823e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 20:29:24 +0200 Subject: [PATCH 31/56] edit config. --- internal/config/config.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index a47acd2..042aed7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -37,9 +37,10 @@ type Server struct { Bind string Static string Proxy bool - BaseDir string `yaml:"basedir",omitempty` - Streams map[string]string `yaml:"streams"` - Profiles string `yaml:"profiles",omitempty` + + BaseDir string `yaml:"basedir,omitempty"` + Streams map[string]string `yaml:"streams"` + Profiles string `yaml:"profiles,omitempty"` } func (Server) Init(cmd *cobra.Command) error { @@ -68,9 +69,15 @@ func (Server) Init(cmd *cobra.Command) error { return err } - cmd.PersistentFlags().String("basedir", "", "The base directory for assets and profiles") + cmd.PersistentFlags().String("basedir", "", "base directory for assets and profiles") + if err := viper.BindPFlag("basedir", cmd.PersistentFlags().Lookup("basedir")); err != nil { + return err + } - cmd.PersistentFlags().String("profiles", "default", "The hardware encoding profiles to load for ffmpeg (default, nvidia)") + cmd.PersistentFlags().String("profiles", "default", "hardware encoding profiles to load for ffmpeg (default, nvidia)") + if err := viper.BindPFlag("profiles", cmd.PersistentFlags().Lookup("profiles")); err != nil { + return err + } return nil } @@ -81,6 +88,7 @@ func (s *Server) Set() { s.Bind = viper.GetString("bind") s.Static = viper.GetString("static") s.Proxy = viper.GetBool("proxy") + s.BaseDir = viper.GetString("basedir") if s.BaseDir == "" { if _, err := os.Stat("/etc/transcode"); os.IsNotExist(err) { @@ -90,7 +98,11 @@ func (s *Server) Set() { s.BaseDir = "/etc/transcode" } } + s.Profiles = viper.GetString("profiles") - if s.Profiles == "" { s.Profiles = "default" } + if s.Profiles == "" { + s.Profiles = "default" + } + s.Streams = viper.GetStringMapString("streams") } From 9284c6f7523eb760b387edb99a91f35d69728e4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 20:30:01 +0200 Subject: [PATCH 32/56] use existing logger for HTTP. --- internal/http/http.go | 5 ++-- internal/http/logger.go | 66 +++++++++++++++++++---------------------- 2 files changed, 32 insertions(+), 39 deletions(-) diff --git a/internal/http/http.go b/internal/http/http.go index 79c715f..b44268a 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -26,10 +26,9 @@ func New(ApiManager types.ApiManager, conf *config.Server) *ServerCtx { logger := log.With().Str("module", "http").Logger() router := chi.NewRouter() - router.Use(middleware.Recoverer) // Recover from panics without crashing server router.Use(middleware.RequestID) // Create a request ID for each request - // TODO: Why Logger not logger? - router.Use(Logger) // Log API request calls using custom logger function + router.Use(middleware.RequestLogger(&logformatter{logger})) + router.Use(middleware.Recoverer) // Recover from panics without crashing server ApiManager.Mount(router) diff --git a/internal/http/logger.go b/internal/http/logger.go index dabffea..acc20ef 100644 --- a/internal/http/logger.go +++ b/internal/http/logger.go @@ -6,54 +6,48 @@ import ( "time" "github.com/go-chi/chi/middleware" - "github.com/rs/zerolog/log" + "github.com/rs/zerolog" ) -func Logger(next http.Handler) http.Handler { - fn := func(w http.ResponseWriter, r *http.Request) { - req := map[string]interface{}{} - - if reqID := middleware.GetReqID(r.Context()); reqID != "" { - req["id"] = reqID - } - - scheme := "http" - if r.TLS != nil { - scheme = "https" - } +type logformatter struct { + logger zerolog.Logger +} - req["scheme"] = scheme - req["proto"] = r.Proto - req["method"] = r.Method - req["remote"] = r.RemoteAddr - req["agent"] = r.UserAgent() - req["uri"] = fmt.Sprintf("%s://%s%s", scheme, r.Host, r.RequestURI) +func (l *logformatter) NewLogEntry(r *http.Request) middleware.LogEntry { + req := map[string]interface{}{} - fields := map[string]interface{}{} - fields["req"] = req + if reqID := middleware.GetReqID(r.Context()); reqID != "" { + req["id"] = reqID + } - entry := &entry{ - fields: fields, - } + scheme := "http" + if r.TLS != nil { + scheme = "https" + } - ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor) - t1 := time.Now() + req["scheme"] = scheme + req["proto"] = r.Proto + req["method"] = r.Method + req["remote"] = r.RemoteAddr + req["agent"] = r.UserAgent() + req["uri"] = fmt.Sprintf("%s://%s%s", scheme, r.Host, r.RequestURI) - defer func() { - entry.Write(ww.Status(), ww.BytesWritten(), time.Since(t1)) - }() + fields := map[string]interface{}{} + fields["req"] = req - next.ServeHTTP(ww, r) + return &logentry{ + fields: fields, + logger: l.logger, } - return http.HandlerFunc(fn) } -type entry struct { +type logentry struct { + logger zerolog.Logger fields map[string]interface{} errors []map[string]interface{} } -func (e *entry) Write(status, bytes int, elapsed time.Duration) { +func (e *logentry) Write(status, bytes int, header http.Header, elapsed time.Duration, extra interface{}) { res := map[string]interface{}{} res["time"] = time.Now().UTC().Format(time.RFC1123) res["status"] = status @@ -65,13 +59,13 @@ func (e *entry) Write(status, bytes int, elapsed time.Duration) { if len(e.errors) > 0 { e.fields["errors"] = e.errors - log.Error().Fields(e.fields).Msgf("request failed (%d)", status) + e.logger.Error().Fields(e.fields).Msgf("request failed (%d)", status) } else { - log.Debug().Fields(e.fields).Msgf("request complete (%d)", status) + e.logger.Debug().Fields(e.fields).Msgf("request complete (%d)", status) } } -func (e *entry) Panic(v interface{}, stack []byte) { +func (e *logentry) Panic(v interface{}, stack []byte) { err := map[string]interface{}{} err["message"] = fmt.Sprintf("%+v", v) err["stack"] = string(stack) From d9017512b93d85a15a741c7517413e618fad54c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 20:33:18 +0200 Subject: [PATCH 33/56] edit HTTP module. --- internal/http/http.go | 23 +++++++++++------------ internal/main.go | 8 ++++++-- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/internal/http/http.go b/internal/http/http.go index b44268a..b4b91ca 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -17,12 +17,12 @@ import ( type ServerCtx struct { logger zerolog.Logger + config *config.Server router *chi.Mux http *http.Server - conf *config.Server } -func New(ApiManager types.ApiManager, conf *config.Server) *ServerCtx { +func New(ApiManager types.ApiManager, config *config.Server) *ServerCtx { logger := log.With().Str("module", "http").Logger() router := chi.NewRouter() @@ -32,10 +32,11 @@ func New(ApiManager types.ApiManager, conf *config.Server) *ServerCtx { ApiManager.Mount(router) - if conf.Static != "" { - fs := http.FileServer(http.Dir(conf.Static)) + // serve static files + if config.Static != "" { + fs := http.FileServer(http.Dir(config.Static)) router.Get("/*", func(w http.ResponseWriter, r *http.Request) { - if _, err := os.Stat(conf.Static + r.RequestURI); os.IsNotExist(err) { + if _, err := os.Stat(config.Static + r.RequestURI); os.IsNotExist(err) { http.StripPrefix(r.RequestURI, fs).ServeHTTP(w, r) } else { fs.ServeHTTP(w, r) @@ -49,27 +50,25 @@ func New(ApiManager types.ApiManager, conf *config.Server) *ServerCtx { }) http := &http.Server{ - Addr: conf.Bind, + Addr: config.Bind, Handler: router, } - logger.Info().Msgf("Serving streams from basedir %v: %v", conf.BaseDir, conf.Streams) - return &ServerCtx{ logger: logger, + config: config, router: router, http: http, - conf: conf, } } func (s *ServerCtx) Start() { - if s.conf.Cert != "" && s.conf.Key != "" { + if s.config.Cert != "" && s.config.Key != "" { + s.logger.Warn().Msg("TLS support is provided for convenience, but you should never use it in production. Use a reverse proxy (apache nginx caddy) instead!") go func() { - if err := s.http.ListenAndServeTLS(s.conf.Cert, s.conf.Key); err != http.ErrServerClosed { + if err := s.http.ListenAndServeTLS(s.config.Cert, s.config.Key); err != http.ErrServerClosed { s.logger.Panic().Err(err).Msg("unable to start https server") } - s.logger.Warn().Msg("TLS support is provided for convenience, but you should never use it in production. Use a reverse proxy (apache nginx caddy) instead!") }() s.logger.Info().Msgf("https listening on %s", s.http.Addr) } else { diff --git a/internal/main.go b/internal/main.go index d55e3d4..4cac78a 100644 --- a/internal/main.go +++ b/internal/main.go @@ -36,13 +36,17 @@ func (main *Main) Preflight() { } func (main *Main) Start() { - main.apiManager = api.New(main.ServerConfig) + config := main.ServerConfig + + main.apiManager = api.New(config) main.server = http.New( main.apiManager, - main.ServerConfig, + config, ) main.server.Start() + + main.logger.Info().Msgf("serving streams from basedir %s: %s", config.BaseDir, config.Streams) } func (main *Main) Shutdown() { From f1c575610d57d774c14cc6a3072a643fb5523400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 20:35:39 +0200 Subject: [PATCH 34/56] remove unused newline in log. --- internal/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/main.go b/internal/main.go index 4cac78a..20818c0 100644 --- a/internal/main.go +++ b/internal/main.go @@ -66,7 +66,7 @@ func (main *Main) ServeCommand(cmd *cobra.Command, args []string) { signal.Notify(quit, os.Interrupt) sig := <-quit - main.logger.Warn().Msgf("received %s, attempting graceful shutdown: \n", sig) + main.logger.Warn().Msgf("received %s, attempting graceful shutdown", sig) main.Shutdown() main.logger.Info().Msg("shutdown complete") } From 38d2012c90ef8dd579d48ae0df9ef5272926b36e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 20:46:00 +0200 Subject: [PATCH 35/56] fix outdated docker build. --- README.md | 6 ++---- docker-compose.yaml | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1ccb628..d71a5d3 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,6 @@ In these profile directories, actual profiles are located in `hls/` and `http/`, ## Docker -*TODO: outdated docker section* - ### Build ```sh @@ -82,7 +80,7 @@ docker build -t go-transcode:latest . docker run --rm -d \ --name="go-transcode" \ -p "8080:8080" \ - -v "${PWD}/streams.yaml:/app/streams.yaml" go-transcode:latest + -v "${PWD}/transcode.yaml:/app/transcode.yaml" go-transcode:latest ``` ## Nvidia GPU support (docker) @@ -104,7 +102,7 @@ docker run --rm -d \ --gpus=all \ --name="go-transcode-nvidia" \ -p "8080:8080" \ - -v "${PWD}/streams.yaml:/app/streams.yaml" go-transcode-nvidia:latest + -v "${PWD}/transcode.yaml:/app/transcode.yaml" go-transcode-nvidia:latest ``` ### Supported inputs diff --git a/docker-compose.yaml b/docker-compose.yaml index 08ffca7..b008a33 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,5 +8,5 @@ services: ports: - "8080:8080" volumes: - - ./streams.yaml:/app/streams.yaml + - ./transcode.yaml:/app/transcode.yaml command: serve -d From 85c6c25989fd7d1c1ba52d66941a816789da9a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 20:49:49 +0200 Subject: [PATCH 36/56] upgrade go dependencies. --- go.mod | 16 ++++---- go.sum | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 122 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index a017992..6c08c69 100644 --- a/go.mod +++ b/go.mod @@ -3,27 +3,29 @@ module github.com/m1k1o/go-transcode go 1.17 require ( - github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/fsnotify/fsnotify v1.5.1 github.com/go-chi/chi v1.5.4 - github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/mapstructure v1.4.2 // indirect github.com/pelletier/go-toml v1.9.4 // indirect - github.com/rs/zerolog v1.24.0 + github.com/rs/zerolog v1.25.0 github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/cobra v1.2.1 github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.8.1 - golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34 // indirect + github.com/spf13/viper v1.9.0 + golang.org/x/sys v0.0.0-20210925032602-92d5a993a665 // indirect golang.org/x/text v0.3.7 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/ini.v1 v1.63.0 // indirect - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/ini.v1 v1.63.2 // indirect ) require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/kr/pretty v0.3.0 // indirect github.com/magiconair/properties v1.8.5 // indirect + github.com/rogpeppe/go-internal v1.8.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 4a156d3..09cf700 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,11 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -27,6 +32,7 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -39,13 +45,16 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -53,9 +62,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -65,8 +76,10 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= @@ -90,6 +103,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -107,6 +121,7 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -120,10 +135,12 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -135,21 +152,29 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -161,8 +186,11 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -170,31 +198,43 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= +github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -202,26 +242,31 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.24.0 h1:76ivFxmVSRs1u2wUwJVg5VZDYQgeH1JpoS6ndgr9Wy8= -github.com/rs/zerolog v1.24.0/go.mod h1:7KHcEGe0QZPOm2IE4Kpb5rTh6n1h2hIgS5OOnu1rUaI= +github.com/rs/zerolog v1.25.0 h1:Rj7XygbUHKUlDPcVdoLyR91fJBsduXj5fRxyqIQj/II= +github.com/rs/zerolog v1.25.0/go.mod h1:7KHcEGe0QZPOm2IE4Kpb5rTh6n1h2hIgS5OOnu1rUaI= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -233,8 +278,9 @@ github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmq github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk= +github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -260,6 +306,7 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= @@ -268,8 +315,10 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -319,6 +368,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -341,6 +391,7 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -353,6 +404,10 @@ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -368,6 +423,7 @@ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -375,12 +431,18 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -404,10 +466,17 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34 h1:GkvMjFtXUmahfDtashnc1mnrCtuBVcwse5QV2lUk/tI= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210925032602-92d5a993a665 h1:QOQNt6vCjMpXE7JSK5VvAzJC1byuN3FgTNSBwf+CJgI= +golang.org/x/sys v0.0.0-20210925032602-92d5a993a665/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -416,6 +485,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -435,6 +505,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -471,7 +542,10 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -499,6 +573,12 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -546,7 +626,18 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -566,7 +657,13 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -579,14 +676,16 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.63.0 h1:2t0h8NA59dpVQpa5Yh8cIcR6nHAeBIEk0zlLVqfw4N4= -gopkg.in/ini.v1 v1.63.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= +gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 1d8985aa91741bfb5f674d3a77142fa33b3f6118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 20:56:29 +0200 Subject: [PATCH 37/56] hls wait when killing proccess group, fixes #2. --- hls/manager.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hls/manager.go b/hls/manager.go index 52884c9..9d692db 100644 --- a/hls/manager.go +++ b/hls/manager.go @@ -176,13 +176,12 @@ func (m *ManagerCtx) Stop() { err := m.cmd.Process.Kill() m.logger.Err(err).Msg("killing proccess") } + m.cmd.Wait() m.cmd = nil } - time.AfterFunc(2*time.Second, func() { - err := os.RemoveAll(m.tempdir) - m.logger.Err(err).Msg("removing tempdir") - }) + err := os.RemoveAll(m.tempdir) + m.logger.Err(err).Msg("removing tempdir") if m.events.onStop != nil { m.events.onStop() From 1a9a6c93b901f43cd6bcd6ccf99704effe9e6618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 20:58:23 +0200 Subject: [PATCH 38/56] go fmt ./... --- cmd/root.go | 2 +- cmd/serve.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 178d951..6ab4b1f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,11 +4,11 @@ import ( "os" "runtime" + "github.com/fsnotify/fsnotify" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/fsnotify/fsnotify" "github.com/m1k1o/go-transcode/internal" ) diff --git a/cmd/serve.go b/cmd/serve.go index 9be9266..75530af 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -20,7 +20,7 @@ func init() { transcode.Service.Preflight() }) - if err := transcode.Service.ServerConfig.Init(command); err !=nil { + if err := transcode.Service.ServerConfig.Init(command); err != nil { log.Panic().Err(err).Msg("unable to run serve command") } From 1710c1da41330952bc7746df8028576dbfa99408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 21:24:25 +0200 Subject: [PATCH 39/56] use single resourceRegex in api. --- internal/api/hls.go | 7 ++----- internal/api/router.go | 5 +++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/internal/api/hls.go b/internal/api/hls.go index ab8391c..1d5bd18 100644 --- a/internal/api/hls.go +++ b/internal/api/hls.go @@ -5,7 +5,6 @@ import ( "net/http" "os/exec" "path" - "regexp" "github.com/go-chi/chi" "github.com/rs/zerolog/log" @@ -24,8 +23,7 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { profile := chi.URLParam(r, "profile") input := chi.URLParam(r, "input") - re := regexp.MustCompile(`^[0-9A-Za-z_-]+$`) - if !re.MatchString(profile) || !re.MatchString(input) { + if !resourceRegex.MatchString(profile) || !resourceRegex.MatchString(input) { w.WriteHeader(http.StatusBadRequest) w.Write([]byte("400 invalid parameters")) return @@ -66,8 +64,7 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { input := chi.URLParam(r, "input") file := chi.URLParam(r, "file") - re := regexp.MustCompile(`^[0-9A-Za-z_-]+$`) - if !re.MatchString(profile) || !re.MatchString(input) || !re.MatchString(file) { + if !resourceRegex.MatchString(profile) || !resourceRegex.MatchString(input) || !resourceRegex.MatchString(file) { w.WriteHeader(http.StatusBadRequest) w.Write([]byte("400 invalid parameters")) return diff --git a/internal/api/router.go b/internal/api/router.go index 9946b3a..a0926cb 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -14,6 +14,8 @@ import ( "github.com/m1k1o/go-transcode/internal/config" ) +var resourceRegex = regexp.MustCompile(`^[0-9A-Za-z_-]+$`) + type ApiManagerCtx struct { Conf *config.Server } @@ -40,8 +42,7 @@ func (a *ApiManagerCtx) transcodeStart(folder string, profile string, input stri return nil, fmt.Errorf("stream not found") } - re := regexp.MustCompile(`^[0-9A-Za-z_-]+$`) - if !re.MatchString(profile) { + if !resourceRegex.MatchString(profile) { return nil, fmt.Errorf("invalid profile path") } From 080affb5042f3b4e96679cfb5a80bcc2c326ecd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 21:44:17 +0200 Subject: [PATCH 40/56] add api shutdown. --- internal/api/router.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/api/router.go b/internal/api/router.go index a0926cb..919a1b1 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -26,6 +26,18 @@ func New(conf *config.Server) *ApiManagerCtx { } } +func (manager *ApiManagerCtx) Start() { +} + +func (manager *ApiManagerCtx) Shutdown() error { + // close all hls managers + for _, hls := range hlsManagers { + hls.Stop() + } + + return nil +} + func (a *ApiManagerCtx) Mount(r *chi.Mux) { r.Get("/ping", func(w http.ResponseWriter, r *http.Request) { //nolint From 9a2c7414537611c721e85f970d0952d40655355c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 21:44:32 +0200 Subject: [PATCH 41/56] move router mount outside of http module. --- internal/http/http.go | 19 +++++++++---------- internal/main.go | 19 ++++++++++--------- internal/types/api.go | 9 --------- 3 files changed, 19 insertions(+), 28 deletions(-) delete mode 100644 internal/types/api.go diff --git a/internal/http/http.go b/internal/http/http.go index b4b91ca..1f5f940 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -12,7 +12,6 @@ import ( "github.com/rs/zerolog/log" "github.com/m1k1o/go-transcode/internal/config" - "github.com/m1k1o/go-transcode/internal/types" ) type ServerCtx struct { @@ -22,7 +21,7 @@ type ServerCtx struct { http *http.Server } -func New(ApiManager types.ApiManager, config *config.Server) *ServerCtx { +func New(config *config.Server) *ServerCtx { logger := log.With().Str("module", "http").Logger() router := chi.NewRouter() @@ -30,8 +29,6 @@ func New(ApiManager types.ApiManager, config *config.Server) *ServerCtx { router.Use(middleware.RequestLogger(&logformatter{logger})) router.Use(middleware.Recoverer) // Recover from panics without crashing server - ApiManager.Mount(router) - // serve static files if config.Static != "" { fs := http.FileServer(http.Dir(config.Static)) @@ -49,16 +46,14 @@ func New(ApiManager types.ApiManager, config *config.Server) *ServerCtx { w.Write([]byte("404")) }) - http := &http.Server{ - Addr: config.Bind, - Handler: router, - } - return &ServerCtx{ logger: logger, config: config, router: router, - http: http, + http: &http.Server{ + Addr: config.Bind, + Handler: router, + }, } } @@ -87,3 +82,7 @@ func (s *ServerCtx) Shutdown() error { return s.http.Shutdown(ctx) } + +func (s *ServerCtx) Mount(fn func(r *chi.Mux)) { + fn(s.router) +} diff --git a/internal/main.go b/internal/main.go index 20818c0..45e2a5b 100644 --- a/internal/main.go +++ b/internal/main.go @@ -39,22 +39,23 @@ func (main *Main) Start() { config := main.ServerConfig main.apiManager = api.New(config) + main.apiManager.Start() - main.server = http.New( - main.apiManager, - config, - ) + main.server = http.New(config) + main.server.Mount(main.apiManager.Mount) main.server.Start() main.logger.Info().Msgf("serving streams from basedir %s: %s", config.BaseDir, config.Streams) } func (main *Main) Shutdown() { - if err := main.server.Shutdown(); err != nil { - main.logger.Err(err).Msg("server shutdown with an error") - } else { - main.logger.Debug().Msg("server shutdown") - } + var err error + + err = main.server.Shutdown() + main.logger.Err(err).Msg("server shutdown") + + err = main.apiManager.Shutdown() + main.logger.Err(err).Msg("api shutdown") } func (main *Main) ServeCommand(cmd *cobra.Command, args []string) { diff --git a/internal/types/api.go b/internal/types/api.go deleted file mode 100644 index f0efae6..0000000 --- a/internal/types/api.go +++ /dev/null @@ -1,9 +0,0 @@ -package types - -import ( - "github.com/go-chi/chi" -) - -type ApiManager interface { - Mount(r *chi.Mux) -} From 869b50800d153db9840eb92a11edf437dfd1bed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 22:05:32 +0200 Subject: [PATCH 42/56] rename server to http manager. --- internal/http/http.go | 12 ++++++------ internal/main.go | 18 +++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/internal/http/http.go b/internal/http/http.go index 1f5f940..91c82a9 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -14,14 +14,14 @@ import ( "github.com/m1k1o/go-transcode/internal/config" ) -type ServerCtx struct { +type HttpCtx struct { logger zerolog.Logger config *config.Server router *chi.Mux http *http.Server } -func New(config *config.Server) *ServerCtx { +func New(config *config.Server) *HttpCtx { logger := log.With().Str("module", "http").Logger() router := chi.NewRouter() @@ -46,7 +46,7 @@ func New(config *config.Server) *ServerCtx { w.Write([]byte("404")) }) - return &ServerCtx{ + return &HttpCtx{ logger: logger, config: config, router: router, @@ -57,7 +57,7 @@ func New(config *config.Server) *ServerCtx { } } -func (s *ServerCtx) Start() { +func (s *HttpCtx) Start() { if s.config.Cert != "" && s.config.Key != "" { s.logger.Warn().Msg("TLS support is provided for convenience, but you should never use it in production. Use a reverse proxy (apache nginx caddy) instead!") go func() { @@ -76,13 +76,13 @@ func (s *ServerCtx) Start() { } } -func (s *ServerCtx) Shutdown() error { +func (s *HttpCtx) Shutdown() error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() return s.http.Shutdown(ctx) } -func (s *ServerCtx) Mount(fn func(r *chi.Mux)) { +func (s *HttpCtx) Mount(fn func(r *chi.Mux)) { fn(s.router) } diff --git a/internal/main.go b/internal/main.go index 45e2a5b..2c98812 100644 --- a/internal/main.go +++ b/internal/main.go @@ -26,9 +26,9 @@ type Main struct { RootConfig *config.Root ServerConfig *config.Server - logger zerolog.Logger - apiManager *api.ApiManagerCtx - server *http.ServerCtx + logger zerolog.Logger + apiManager *api.ApiManagerCtx + httpManager *http.HttpCtx } func (main *Main) Preflight() { @@ -41,9 +41,9 @@ func (main *Main) Start() { main.apiManager = api.New(config) main.apiManager.Start() - main.server = http.New(config) - main.server.Mount(main.apiManager.Mount) - main.server.Start() + main.httpManager = http.New(config) + main.httpManager.Mount(main.apiManager.Mount) + main.httpManager.Start() main.logger.Info().Msgf("serving streams from basedir %s: %s", config.BaseDir, config.Streams) } @@ -51,11 +51,11 @@ func (main *Main) Start() { func (main *Main) Shutdown() { var err error - err = main.server.Shutdown() - main.logger.Err(err).Msg("server shutdown") + err = main.httpManager.Shutdown() + main.logger.Err(err).Msg("http manager shutdown") err = main.apiManager.Shutdown() - main.logger.Err(err).Msg("api shutdown") + main.logger.Err(err).Msg("api manager shutdown") } func (main *Main) ServeCommand(cmd *cobra.Command, args []string) { From d918d666e1efcebedf379a25cd31449cc6f84420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 25 Sep 2021 22:06:21 +0200 Subject: [PATCH 43/56] rename server to http manager. --- internal/http/http.go | 12 ++++++------ internal/main.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/http/http.go b/internal/http/http.go index 91c82a9..0ef7694 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -14,14 +14,14 @@ import ( "github.com/m1k1o/go-transcode/internal/config" ) -type HttpCtx struct { +type HttpManagerCtx struct { logger zerolog.Logger config *config.Server router *chi.Mux http *http.Server } -func New(config *config.Server) *HttpCtx { +func New(config *config.Server) *HttpManagerCtx { logger := log.With().Str("module", "http").Logger() router := chi.NewRouter() @@ -46,7 +46,7 @@ func New(config *config.Server) *HttpCtx { w.Write([]byte("404")) }) - return &HttpCtx{ + return &HttpManagerCtx{ logger: logger, config: config, router: router, @@ -57,7 +57,7 @@ func New(config *config.Server) *HttpCtx { } } -func (s *HttpCtx) Start() { +func (s *HttpManagerCtx) Start() { if s.config.Cert != "" && s.config.Key != "" { s.logger.Warn().Msg("TLS support is provided for convenience, but you should never use it in production. Use a reverse proxy (apache nginx caddy) instead!") go func() { @@ -76,13 +76,13 @@ func (s *HttpCtx) Start() { } } -func (s *HttpCtx) Shutdown() error { +func (s *HttpManagerCtx) Shutdown() error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() return s.http.Shutdown(ctx) } -func (s *HttpCtx) Mount(fn func(r *chi.Mux)) { +func (s *HttpManagerCtx) Mount(fn func(r *chi.Mux)) { fn(s.router) } diff --git a/internal/main.go b/internal/main.go index 2c98812..8b96b73 100644 --- a/internal/main.go +++ b/internal/main.go @@ -28,7 +28,7 @@ type Main struct { logger zerolog.Logger apiManager *api.ApiManagerCtx - httpManager *http.HttpCtx + httpManager *http.HttpManagerCtx } func (main *Main) Preflight() { From 9a769085096234c556bfc987f0e8d2a553d5f284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sun, 26 Sep 2021 00:35:16 +0200 Subject: [PATCH 44/56] use RootConfig in root. --- cmd/root.go | 56 +++++++++++++++++++++--------------------------- internal/main.go | 4 ++++ 2 files changed, 29 insertions(+), 31 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 6ab4b1f..fcdc759 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -10,7 +10,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/m1k1o/go-transcode/internal" + transcode "github.com/m1k1o/go-transcode/internal" ) func Execute() error { @@ -25,24 +25,26 @@ var root = &cobra.Command{ func init() { cobra.OnInitialize(func() { + config := transcode.Service.RootConfig + config.Set() + ////// // logs ////// - zerolog.TimeFieldFormat = "" - zerolog.SetGlobalLevel(zerolog.InfoLevel) + zerolog.TimeFieldFormat = zerolog.TimeFormatUnix + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout}) - if viper.GetBool("debug") { + if config.Debug { zerolog.SetGlobalLevel(zerolog.DebugLevel) + } else { + zerolog.SetGlobalLevel(zerolog.InfoLevel) } - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout}) - ////// // configs ////// - config := viper.GetString("config") - if config != "" { - viper.SetConfigFile(config) // Use config file from the flag. + if config.CfgFile != "" { + viper.SetConfigFile(config.CfgFile) // use config file from the flag } else { if runtime.GOOS == "linux" { viper.AddConfigPath("/etc/transcode/") @@ -55,35 +57,27 @@ func init() { viper.SetEnvPrefix("transcode") viper.AutomaticEnv() // read in environment variables that match - if err := viper.ReadInConfig(); err != nil { - if _, ok := err.(viper.ConfigFileNotFoundError); !ok { - log.Error().Err(err) - } - if config != "" { - log.Error().Err(err) - } + err := viper.ReadInConfig() + if err != nil && config.CfgFile != "" { + log.Err(err) } - viper.OnConfigChange(func(e fsnotify.Event) { - // TODO: list config change - log.Logger.Info().Msg("Config file reloaded") - transcode.Service.ServerConfig.Set() - }) - viper.WatchConfig() - - file := viper.ConfigFileUsed() logger := log.With(). - Bool("debug", viper.GetBool("debug")). - Str("config", file). + Bool("debug", config.Debug). Logger() - if file == "" { - logger.Warn().Msg("preflight complete without config file") + file := viper.ConfigFileUsed() + if file != "" { + viper.OnConfigChange(func(e fsnotify.Event) { + log.Info().Msg("config file reloaded") + transcode.Service.ConfigReload() + }) + + viper.WatchConfig() + logger.Info().Str("config", file).Msg("preflight complete with config file") } else { - logger.Info().Msg("preflight complete") + logger.Warn().Msg("preflight complete without config file") } - - transcode.Service.RootConfig.Set() }) if err := transcode.Service.RootConfig.Init(root); err != nil { diff --git a/internal/main.go b/internal/main.go index 8b96b73..d56c23c 100644 --- a/internal/main.go +++ b/internal/main.go @@ -71,3 +71,7 @@ func (main *Main) ServeCommand(cmd *cobra.Command, args []string) { main.Shutdown() main.logger.Info().Msg("shutdown complete") } + +func (main *Main) ConfigReload() { + // TODO: Implement. +} From 5b8af74e705be7c241b9a2fc98488d95373e9bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sun, 26 Sep 2021 00:45:53 +0200 Subject: [PATCH 45/56] add AbsPath func to config. --- internal/api/hls.go | 7 ++++--- internal/api/http.go | 8 ++++---- internal/api/router.go | 11 +++++------ internal/config/config.go | 7 +++++++ 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/internal/api/hls.go b/internal/api/hls.go index 1d5bd18..5b0e98f 100644 --- a/internal/api/hls.go +++ b/internal/api/hls.go @@ -4,7 +4,6 @@ import ( "fmt" "net/http" "os/exec" - "path" "github.com/go-chi/chi" "github.com/rs/zerolog/log" @@ -30,7 +29,7 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { } // check if stream exists - _, ok := a.Conf.Streams[input] + _, ok := a.config.Streams[input] if !ok { http.NotFound(w, r) return @@ -84,6 +83,8 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { r.Get("/{profile}/{input}/play.html", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") - http.ServeFile(w, r, path.Join(a.Conf.BaseDir, "data/play.html")) + + file := a.config.AbsPath("data", "play.html") + http.ServeFile(w, r, file) }) } diff --git a/internal/api/http.go b/internal/api/http.go index dba0fdf..ad8a96c 100644 --- a/internal/api/http.go +++ b/internal/api/http.go @@ -5,7 +5,6 @@ import ( "io" "net/http" "os/exec" - "path" "github.com/go-chi/chi" "github.com/rs/zerolog/log" @@ -22,7 +21,8 @@ func (a *ApiManagerCtx) Http(r chi.Router) { Logger() // dummy input for testing purposes - cmd := exec.Command(path.Join(a.Conf.BaseDir, "data/http-test.sh")) + file := a.config.AbsPath("data", "http-test.sh") + cmd := exec.Command(file) logger.Info().Msg("command startred") read, write := io.Pipe() @@ -50,7 +50,7 @@ func (a *ApiManagerCtx) Http(r chi.Router) { input := chi.URLParam(r, "input") // check if stream exists - _, ok := a.Conf.Streams[input] + _, ok := a.config.Streams[input] if !ok { http.NotFound(w, r) return @@ -93,7 +93,7 @@ func (a *ApiManagerCtx) Http(r chi.Router) { input := chi.URLParam(r, "input") // check if stream exists - _, ok := a.Conf.Streams[input] + _, ok := a.config.Streams[input] if !ok { http.NotFound(w, r) return diff --git a/internal/api/router.go b/internal/api/router.go index 919a1b1..e0bb1d5 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -5,7 +5,6 @@ import ( "net/http" "os" "os/exec" - "path" "regexp" "github.com/go-chi/chi" @@ -17,12 +16,12 @@ import ( var resourceRegex = regexp.MustCompile(`^[0-9A-Za-z_-]+$`) type ApiManagerCtx struct { - Conf *config.Server + config *config.Server } -func New(conf *config.Server) *ApiManagerCtx { +func New(config *config.Server) *ApiManagerCtx { return &ApiManagerCtx{ - Conf: conf, + config: config, } } @@ -49,7 +48,7 @@ func (a *ApiManagerCtx) Mount(r *chi.Mux) { } func (a *ApiManagerCtx) transcodeStart(folder string, profile string, input string) (*exec.Cmd, error) { - url, ok := a.Conf.Streams[input] + url, ok := a.config.Streams[input] if !ok { return nil, fmt.Errorf("stream not found") } @@ -59,7 +58,7 @@ func (a *ApiManagerCtx) transcodeStart(folder string, profile string, input stri } // [basedir]/profiles/[profiles]/hls,http/[profile].sh - profilePath := path.Join(a.Conf.BaseDir, "profiles", a.Conf.Profiles, folder, fmt.Sprintf("%s.sh", profile)) + profilePath := a.config.AbsPath("profiles", a.config.Profiles, folder, fmt.Sprintf("%s.sh", profile)) if _, err := os.Stat(profilePath); os.IsNotExist(err) { return nil, err } diff --git a/internal/config/config.go b/internal/config/config.go index 042aed7..81c627c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,6 +2,7 @@ package config import ( "os" + "path" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -106,3 +107,9 @@ func (s *Server) Set() { s.Streams = viper.GetStringMapString("streams") } + +func (s *Server) AbsPath(elem ...string) string { + // prepend base path + elem = append([]string{s.BaseDir}, elem...) + return path.Join(elem...) +} From e050a78a7f741fc021d36f889056cca8c99af46d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sun, 26 Sep 2021 00:57:38 +0200 Subject: [PATCH 46/56] edit Architecture, change to A-Z order. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d71a5d3..fbc0292 100644 --- a/README.md +++ b/README.md @@ -137,17 +137,17 @@ Join us in the [Matrix space](https://matrix.to/#/#go-transcode:proxychat.net) ( The source code is in the following files/folders: - `cmd/` and `main.go`: source for the command-line interface -- `internal/`: actual source code logic - `hls/`: process runner for HLS transcoding +- `internal/`: actual source code logic *TODO: document different modules/packages and dependencies* Other files/folders in the repositories are: - `data/`: files used/served by go-transcode -- `profiles/ and profiles_nvidia/`: the ffmpeg profiles for transcoding - `dev/`: some docker helper scripts +- `profiles/`: the ffmpeg profiles for transcoding - `tests/`: some tests for the project -- `god.mod` and `go.sum`: golang dependencies/modules tracking - `Dockerfile`, `Dockerfile.nvidia` and `docker-compose.yaml`: for the docker lovers +- `god.mod` and `go.sum`: golang dependencies/modules tracking - `LICENSE`: licensing information (Apache 2.0) From e4699e67c815c532bdc60b8c073f87672515217f Mon Sep 17 00:00:00 2001 From: klahaha Date: Sat, 25 Sep 2021 17:25:41 +0200 Subject: [PATCH 47/56] one profile dir with nvidia/cpu hardware autodetect --- internal/api/router.go | 6 +- internal/config/config.go | 7 ++- .../{nvidia/.helpers.sh => .helpers.cuda.sh} | 0 profiles/default/hls/h264_1080p.sh | 26 --------- profiles/default/hls/h264_360p.sh | 26 --------- profiles/default/hls/h264_540p.sh | 26 --------- profiles/default/hls/h264_720p.sh | 26 --------- profiles/default/http/h264_1080p.sh | 18 ------ profiles/default/http/h264_360p.sh | 18 ------ profiles/default/http/h264_540p.sh | 18 ------ profiles/default/http/h264_720p.sh | 18 ------ profiles/{default => }/hls/copy.sh | 0 profiles/hls/h264_1080p.sh | 10 ++++ profiles/hls/h264_360p.sh | 10 ++++ profiles/hls/h264_540p.sh | 10 ++++ profiles/hls/h264_720p.sh | 10 ++++ profiles/hls_h264.sh | 56 +++++++++++++++++++ profiles/{default => }/http/copy.sh | 0 profiles/http/h264_1080p.sh | 10 ++++ profiles/http/h264_360p.sh | 10 ++++ profiles/http/h264_540p.sh | 10 ++++ profiles/http/h264_720p.sh | 10 ++++ profiles/http_h264.sh | 48 ++++++++++++++++ profiles/nvidia/hls/copy.sh | 19 ------- profiles/nvidia/hls/h264_1080p.sh | 30 ---------- profiles/nvidia/hls/h264_360p.sh | 30 ---------- profiles/nvidia/hls/h264_540p.sh | 30 ---------- profiles/nvidia/hls/h264_720p.sh | 30 ---------- profiles/nvidia/http/copy.sh | 11 ---- profiles/nvidia/http/h264_1080p.sh | 22 -------- profiles/nvidia/http/h264_360p.sh | 22 -------- profiles/nvidia/http/h264_540p.sh | 22 -------- profiles/nvidia/http/h264_720p.sh | 22 -------- 33 files changed, 192 insertions(+), 419 deletions(-) rename profiles/{nvidia/.helpers.sh => .helpers.cuda.sh} (100%) delete mode 100755 profiles/default/hls/h264_1080p.sh delete mode 100755 profiles/default/hls/h264_360p.sh delete mode 100755 profiles/default/hls/h264_540p.sh delete mode 100755 profiles/default/hls/h264_720p.sh delete mode 100755 profiles/default/http/h264_1080p.sh delete mode 100755 profiles/default/http/h264_360p.sh delete mode 100755 profiles/default/http/h264_540p.sh delete mode 100755 profiles/default/http/h264_720p.sh rename profiles/{default => }/hls/copy.sh (100%) create mode 100755 profiles/hls/h264_1080p.sh create mode 100755 profiles/hls/h264_360p.sh create mode 100755 profiles/hls/h264_540p.sh create mode 100755 profiles/hls/h264_720p.sh create mode 100755 profiles/hls_h264.sh rename profiles/{default => }/http/copy.sh (100%) create mode 100755 profiles/http/h264_1080p.sh create mode 100755 profiles/http/h264_360p.sh create mode 100755 profiles/http/h264_540p.sh create mode 100755 profiles/http/h264_720p.sh create mode 100755 profiles/http_h264.sh delete mode 100755 profiles/nvidia/hls/copy.sh delete mode 100755 profiles/nvidia/hls/h264_1080p.sh delete mode 100755 profiles/nvidia/hls/h264_360p.sh delete mode 100755 profiles/nvidia/hls/h264_540p.sh delete mode 100755 profiles/nvidia/hls/h264_720p.sh delete mode 100755 profiles/nvidia/http/copy.sh delete mode 100755 profiles/nvidia/http/h264_1080p.sh delete mode 100755 profiles/nvidia/http/h264_360p.sh delete mode 100755 profiles/nvidia/http/h264_540p.sh delete mode 100755 profiles/nvidia/http/h264_720p.sh diff --git a/internal/api/router.go b/internal/api/router.go index e0bb1d5..c18de15 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -5,6 +5,7 @@ import ( "net/http" "os" "os/exec" + "path" "regexp" "github.com/go-chi/chi" @@ -57,8 +58,9 @@ func (a *ApiManagerCtx) transcodeStart(folder string, profile string, input stri return nil, fmt.Errorf("invalid profile path") } - // [basedir]/profiles/[profiles]/hls,http/[profile].sh - profilePath := a.config.AbsPath("profiles", a.config.Profiles, folder, fmt.Sprintf("%s.sh", profile)) + // [profiles]/hls,http/[profile].sh + // [profiles] defaults to [basedir]/profiles + profilePath := path.Join(a.config.Profiles, folder, fmt.Sprintf("%s.sh", profile)) if _, err := os.Stat(profilePath); os.IsNotExist(err) { return nil, err } diff --git a/internal/config/config.go b/internal/config/config.go index 81c627c..b4ac81d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "os" "path" @@ -75,7 +76,7 @@ func (Server) Init(cmd *cobra.Command) error { return err } - cmd.PersistentFlags().String("profiles", "default", "hardware encoding profiles to load for ffmpeg (default, nvidia)") + cmd.PersistentFlags().String("profiles", "", "hardware encoding profiles to load for ffmpeg (default, nvidia)") if err := viper.BindPFlag("profiles", cmd.PersistentFlags().Lookup("profiles")); err != nil { return err } @@ -102,9 +103,9 @@ func (s *Server) Set() { s.Profiles = viper.GetString("profiles") if s.Profiles == "" { - s.Profiles = "default" + // TODO: issue #5 + s.Profiles = fmt.Sprintf("%s/profiles", s.BaseDir) } - s.Streams = viper.GetStringMapString("streams") } diff --git a/profiles/nvidia/.helpers.sh b/profiles/.helpers.cuda.sh similarity index 100% rename from profiles/nvidia/.helpers.sh rename to profiles/.helpers.cuda.sh diff --git a/profiles/default/hls/h264_1080p.sh b/profiles/default/hls/h264_1080p.sh deleted file mode 100755 index 8fff30a..0000000 --- a/profiles/default/hls/h264_1080p.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -exec ffmpeg -hide_banner -loglevel warning \ - -i "${1}" \ - -map 0:v:0 -map 0:a:0 \ - -vf scale=w=1920:h=1080:force_original_aspect_ratio=decrease \ - -c:a aac \ - -ar 48000 \ - -b:a 192k \ - -c:v h264 \ - -profile:v main \ - -b:v 5000k \ - -maxrate 5350k \ - -bufsize 7500k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f hls \ - -hls_time 2 \ - -hls_list_size 5 \ - -hls_wrap 10 \ - -hls_delete_threshold 1 \ - -hls_flags delete_segments \ - -hls_start_number_source datetime \ - -hls_segment_filename "live_%03d.ts" - diff --git a/profiles/default/hls/h264_360p.sh b/profiles/default/hls/h264_360p.sh deleted file mode 100755 index 9c12015..0000000 --- a/profiles/default/hls/h264_360p.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -exec ffmpeg -hide_banner -loglevel warning \ - -i "${1}" \ - -map 0:v:0 -map 0:a:0 \ - -vf scale=w=640:h=360:force_original_aspect_ratio=decrease \ - -c:a aac \ - -ar 48000 \ - -b:a 96k \ - -c:v h264 \ - -profile:v main \ - -b:v 800k \ - -maxrate 856k \ - -bufsize 1200k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f hls \ - -hls_time 2 \ - -hls_list_size 5 \ - -hls_wrap 10 \ - -hls_delete_threshold 1 \ - -hls_flags delete_segments \ - -hls_start_number_source datetime \ - -hls_segment_filename "live_%03d.ts" - diff --git a/profiles/default/hls/h264_540p.sh b/profiles/default/hls/h264_540p.sh deleted file mode 100755 index ab3f994..0000000 --- a/profiles/default/hls/h264_540p.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -exec ffmpeg -hide_banner -loglevel warning \ - -i "${1}" \ - -map 0:v:0 -map 0:a:0 \ - -vf scale=w=960:h=540:force_original_aspect_ratio=decrease \ - -c:a aac \ - -ar 48000 \ - -b:a 128k \ - -c:v h264 \ - -profile:v main \ - -b:v 1800k \ - -maxrate 1800k \ - -bufsize 3100k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f hls \ - -hls_time 2 \ - -hls_list_size 5 \ - -hls_wrap 10 \ - -hls_delete_threshold 1 \ - -hls_flags delete_segments \ - -hls_start_number_source datetime \ - -hls_segment_filename "live_%03d.ts" - diff --git a/profiles/default/hls/h264_720p.sh b/profiles/default/hls/h264_720p.sh deleted file mode 100755 index 461f4bd..0000000 --- a/profiles/default/hls/h264_720p.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -exec ffmpeg -hide_banner -loglevel warning \ - -i "${1}" \ - -map 0:v:0 -map 0:a:0 \ - -vf scale=w=1280:h=720:force_original_aspect_ratio=decrease \ - -c:a aac \ - -ar 48000 \ - -b:a 128k \ - -c:v h264 \ - -profile:v main \ - -b:v 2800k \ - -maxrate 2996k \ - -bufsize 4200k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f hls \ - -hls_time 2 \ - -hls_list_size 5 \ - -hls_wrap 10 \ - -hls_delete_threshold 1 \ - -hls_flags delete_segments \ - -hls_start_number_source datetime \ - -hls_segment_filename "live_%03d.ts" - diff --git a/profiles/default/http/h264_1080p.sh b/profiles/default/http/h264_1080p.sh deleted file mode 100755 index 47cf02d..0000000 --- a/profiles/default/http/h264_1080p.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -exec ffmpeg -hide_banner -loglevel warning \ - -i "${1}" \ - -vf scale=w=1920:h=1080:force_original_aspect_ratio=decrease \ - -c:a aac \ - -ar 48000 \ - -b:a 192k \ - -c:v h264 \ - -profile:v main \ - -b:v 5000k \ - -maxrate 5350k \ - -bufsize 7500k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f mpegts - diff --git a/profiles/default/http/h264_360p.sh b/profiles/default/http/h264_360p.sh deleted file mode 100755 index adbb953..0000000 --- a/profiles/default/http/h264_360p.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -exec ffmpeg -hide_banner -loglevel warning \ - -i "${1}" \ - -vf scale=w=640:h=360:force_original_aspect_ratio=decrease \ - -c:a aac \ - -ar 48000 \ - -b:a 96k \ - -c:v h264 \ - -profile:v main \ - -b:v 800k \ - -maxrate 856k \ - -bufsize 1200k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f mpegts - diff --git a/profiles/default/http/h264_540p.sh b/profiles/default/http/h264_540p.sh deleted file mode 100755 index 8fb4242..0000000 --- a/profiles/default/http/h264_540p.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -exec ffmpeg -hide_banner -loglevel warning \ - -i "${1}" \ - -vf scale=w=960:h=540:force_original_aspect_ratio=decrease \ - -c:a aac \ - -ar 48000 \ - -b:a 128k \ - -c:v h264 \ - -profile:v main \ - -b:v 1800k \ - -maxrate 1800k \ - -bufsize 3100k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f mpegts - diff --git a/profiles/default/http/h264_720p.sh b/profiles/default/http/h264_720p.sh deleted file mode 100755 index 2117f74..0000000 --- a/profiles/default/http/h264_720p.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -exec ffmpeg -hide_banner -loglevel warning \ - -i "${1}" \ - -vf scale=w=1280:h=720:force_original_aspect_ratio=decrease \ - -c:a aac \ - -ar 48000 \ - -b:a 128k \ - -c:v h264 \ - -profile:v main \ - -b:v 2800k \ - -maxrate 2996k \ - -bufsize 4200k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f mpegts - diff --git a/profiles/default/hls/copy.sh b/profiles/hls/copy.sh similarity index 100% rename from profiles/default/hls/copy.sh rename to profiles/hls/copy.sh diff --git a/profiles/hls/h264_1080p.sh b/profiles/hls/h264_1080p.sh new file mode 100755 index 0000000..da54644 --- /dev/null +++ b/profiles/hls/h264_1080p.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +export VW="1920" +export VH="1080" +export ABANDWIDTH="192k" +export VBANDWIDTH="5000k" +export VMAXRATE="5350k" +export VBUFSIZE="7500k" + +"$(dirname "$0")"/../hls_h264.sh "$1" diff --git a/profiles/hls/h264_360p.sh b/profiles/hls/h264_360p.sh new file mode 100755 index 0000000..d186474 --- /dev/null +++ b/profiles/hls/h264_360p.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +export VW="640" +export VH="360" +export ABANDWIDTH="96k" +export VBANDWIDTH="800k" +export VMAXRATE="856k" +export VBUFSIZE="1200k" + +"$(dirname "$0")"/../hls_h264.sh "$1" diff --git a/profiles/hls/h264_540p.sh b/profiles/hls/h264_540p.sh new file mode 100755 index 0000000..9e71eda --- /dev/null +++ b/profiles/hls/h264_540p.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +export VW="960" +export VH="540" +export ABANDWIDTH="128k" +export VBANDWIDTH="1800k" +export VMAXRATE="1800k" +export VBUFSIZE="3100k" + +"$(dirname "$0")"/../hls_h264.sh "$1" diff --git a/profiles/hls/h264_720p.sh b/profiles/hls/h264_720p.sh new file mode 100755 index 0000000..2a83f23 --- /dev/null +++ b/profiles/hls/h264_720p.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +export VW="1280" +export VH="720" +export ABANDWIDTH="128k" +export VBANDWIDTH="2800k" +export VMAXRATE="2996k" +export VBUFSIZE="4200k" + +"$(dirname "$0")"/../hls_h264.sh "$1" diff --git a/profiles/hls_h264.sh b/profiles/hls_h264.sh new file mode 100755 index 0000000..7eee3be --- /dev/null +++ b/profiles/hls_h264.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +if [[ "$VW" = "" ]]; then echo "Missing \$VW"; exit 1; fi +if [[ "$VH" = "" ]]; then echo "Missing \$VH"; exit 1; fi +if [[ "$ABANDWIDTH" = "" ]]; then echo "Missing \$ABANDWIDTH"; exit 1; fi +if [[ "$VBANDWIDTH" = "" ]]; then echo "Missing \$VBANDWIDTH"; exit 1; fi +if [[ "$VMAXRATE" = "" ]]; then echo "Missing \$VMAXRATE"; exit 1; fi +if [[ "$VBUFSIZE" = "" ]]; then echo "Missing \$VBUFSIZE"; exit 1; fi + +HWSUPPORT="$(ffmpeg -init_hw_device list 2> /dev/null)" + +if echo $HWSUPPORT | grep "^vaapi" > /dev/null; then + # TODO: vaapi support + #source "$(dirname "$0")/../.helpers.vaapi.sh" + echo "NOT using VAAPI hardware (CPU fallback)." + VF="scale=w=$VW:h=$VH:force_original_aspect_ratio=decrease" + CV="h264" +elif echo $HWSUPPORT | grep "^cuda" > /dev/null; then + echo "Using CUDA hardware." + source "$(dirname "$0")/.helpers.cuda.sh" + INPUT="$(cuvid_codec "${1}")" + # ffmpeg parameters + EXTRAPARAMS="-hwaccel_output_format cuda -c:v "$INPUT"" + # TODO: Why no force_original_aspect_ratio here? + VF="hwupload_cuda,yadif_cuda=0:-1:0,scale_npp=$VW:$VH:interp_algo=super" + CV="h264_nvenc" +else + echo "Using CPU hardware." + VF="scale=w=$VW:h=$VH:force_original_aspect_ratio=decrease" + CV="h264" +fi + +exec ffmpeg -hide_banner -loglevel warning \ + -i "${1}" $EXTRAPARAMS \ + -map 0:v:0 -map 0:a:0 \ + -vf $VF \ + -c:a aac \ + -ar 48000 \ + -b:a $ABANDWIDTH \ + -c:v $CV \ + -profile:v main \ + -b:v $VBANDWIDTH \ + -maxrate $VMAXRATE \ + -bufsize $VBUFSIZE \ + -crf 20 \ + -sc_threshold 0 \ + -g 48 \ + -keyint_min 48 \ + -f hls \ + -hls_time 2 \ + -hls_list_size 5 \ + -hls_wrap 10 \ + -hls_delete_threshold 1 \ + -hls_flags delete_segments \ + -hls_start_number_source datetime \ + -hls_segment_filename "live_%03d.ts" - diff --git a/profiles/default/http/copy.sh b/profiles/http/copy.sh similarity index 100% rename from profiles/default/http/copy.sh rename to profiles/http/copy.sh diff --git a/profiles/http/h264_1080p.sh b/profiles/http/h264_1080p.sh new file mode 100755 index 0000000..81e360e --- /dev/null +++ b/profiles/http/h264_1080p.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +export VW="1920" +export VH="1080" +export ABANDWIDTH="192k" +export VBANDWIDTH="5000k" +export VMAXRATE="5350k" +export VBUFSIZE="7500k" + +"$(dirname "$0")"/../http_h264.sh "$1" diff --git a/profiles/http/h264_360p.sh b/profiles/http/h264_360p.sh new file mode 100755 index 0000000..ae52f1a --- /dev/null +++ b/profiles/http/h264_360p.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +export VW="640" +export VH="360" +export ABANDWIDTH="96k" +export VBANDWIDTH="800k" +export VMAXRATE="856k" +export VBUFSIZE="1200k" + +"$(dirname "$0")"/../http_h264.sh "$1" diff --git a/profiles/http/h264_540p.sh b/profiles/http/h264_540p.sh new file mode 100755 index 0000000..07f94ce --- /dev/null +++ b/profiles/http/h264_540p.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +export VW="960" +export VH="540" +export ABANDWIDTH="128k" +export VBANDWIDTH="1800k" +export VMAXRATE="1800k" +export VBUFSIZE="3100k" + +"$(dirname "$0")"/../http_h264.sh "$1" diff --git a/profiles/http/h264_720p.sh b/profiles/http/h264_720p.sh new file mode 100755 index 0000000..a2a964d --- /dev/null +++ b/profiles/http/h264_720p.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +export VW="1280" +export VH="720" +export ABANDWIDTH="128k" +export VBANDWIDTH="2800k" +export VMAXRATE="2996k" +export VBUFSIZE="4200k" + +"$(dirname "$0")"/../http_h264.sh "$1" diff --git a/profiles/http_h264.sh b/profiles/http_h264.sh new file mode 100755 index 0000000..798a7b5 --- /dev/null +++ b/profiles/http_h264.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +if [[ "$VW" = "" ]]; then echo "Missing \$VW"; exit 1; fi +if [[ "$VH" = "" ]]; then echo "Missing \$VH"; exit 1; fi +if [[ "$ABANDWIDTH" = "" ]]; then echo "Missing \$ABANDWIDTH"; exit 1; fi +if [[ "$VBANDWIDTH" = "" ]]; then echo "Missing \$VBANDWIDTH"; exit 1; fi +if [[ "$VMAXRATE" = "" ]]; then echo "Missing \$VMAXRATE"; exit 1; fi +if [[ "$VBUFSIZE" = "" ]]; then echo "Missing \$VBUFSIZE"; exit 1; fi + +HWSUPPORT="$(ffmpeg -init_hw_device list 2> /dev/null)" + +if echo $HWSUPPORT | grep "^vaapi" > /dev/null; then + # TODO: vaapi support + #source "$(dirname "$0")/../.helpers.vaapi.sh" + echo "NOT using VAAPI hardware (CPU fallback)." + VF="scale=w=$VW:h=$VH:force_original_aspect_ratio=decrease" + CV="h264" +elif echo $HWSUPPORT | grep "^cuda" > /dev/null; then + echo "Using CUDA hardware." + source "$(dirname "$0")/.helpers.cuda.sh" + INPUT="$(cuvid_codec "${1}")" + # ffmpeg parameters + EXTRAPARAMS="-hwaccel_output_format cuda -c:v "$INPUT"" + # TODO: Why no force_original_aspect_ratio here? + VF="hwupload_cuda,yadif_cuda=0:-1:0,scale_npp=$VW:$VH:interp_algo=super" + CV="h264_nvenc" +else + echo "Using CPU hardware." + VF="scale=w=$VW:h=$VH:force_original_aspect_ratio=decrease" + CV="h264" +fi + +exec ffmpeg -hide_banner -loglevel warning \ + -i "${1}" $EXTRAPARAMS \ + -vf $VF \ + -c:a aac \ + -ar 48000 \ + -b:a $ABANDWIDTH \ + -c:v $CV \ + -profile:v main \ + -b:v $VBANDWIDTH \ + -maxrate $VMAXRATE \ + -bufsize $VBUFSIZE \ + -crf 20 \ + -sc_threshold 0 \ + -g 48 \ + -keyint_min 48 \ + -f mpegts - diff --git a/profiles/nvidia/hls/copy.sh b/profiles/nvidia/hls/copy.sh deleted file mode 100755 index 1481c5a..0000000 --- a/profiles/nvidia/hls/copy.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -source "$(dirname "$0")/../.helpers.sh" - -exec ffmpeg -hide_banner -loglevel warning \ - -hwaccel_output_format cuda \ - -c:v "$(cuvid_codec "${1}")" \ - -i "${1}" \ - -map 0:v:0 -map 0:a:0 \ - -c:a copy \ - -c:v copy \ - -f hls \ - -hls_time 2 \ - -hls_list_size 5 \ - -hls_wrap 10 \ - -hls_delete_threshold 1 \ - -hls_flags delete_segments \ - -hls_start_number_source datetime \ - -hls_segment_filename "live_%03d.ts" - diff --git a/profiles/nvidia/hls/h264_1080p.sh b/profiles/nvidia/hls/h264_1080p.sh deleted file mode 100755 index c48e777..0000000 --- a/profiles/nvidia/hls/h264_1080p.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -source "$(dirname "$0")/../.helpers.sh" - -exec ffmpeg -hide_banner -loglevel warning \ - -hwaccel_output_format cuda \ - -c:v "$(cuvid_codec "${1}")" \ - -i "${1}" \ - -map 0:v:0 -map 0:a:0 \ - -vf hwupload_cuda,yadif_cuda=0:-1:0,scale_npp=1920:1080:interp_algo=super \ - -c:a aac \ - -ar 48000 \ - -b:a 192k \ - -c:v h264_nvenc \ - -profile:v main \ - -b:v 5000k \ - -maxrate 5350k \ - -bufsize 7500k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f hls \ - -hls_time 2 \ - -hls_list_size 5 \ - -hls_wrap 10 \ - -hls_delete_threshold 1 \ - -hls_flags delete_segments \ - -hls_start_number_source datetime \ - -hls_segment_filename "live_%03d.ts" - diff --git a/profiles/nvidia/hls/h264_360p.sh b/profiles/nvidia/hls/h264_360p.sh deleted file mode 100755 index a40c969..0000000 --- a/profiles/nvidia/hls/h264_360p.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -source "$(dirname "$0")/../.helpers.sh" - -exec ffmpeg -hide_banner -loglevel warning \ - -hwaccel_output_format cuda \ - -c:v "$(cuvid_codec "${1}")" \ - -i "${1}" \ - -map 0:v:0 -map 0:a:0 \ - -vf hwupload_cuda,yadif_cuda=0:-1:0,scale_npp=640:360:interp_algo=super \ - -c:a aac \ - -ar 48000 \ - -b:a 96k \ - -c:v h264_nvenc \ - -profile:v main \ - -b:v 800k \ - -maxrate 856k \ - -bufsize 1200k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f hls \ - -hls_time 2 \ - -hls_list_size 5 \ - -hls_wrap 10 \ - -hls_delete_threshold 1 \ - -hls_flags delete_segments \ - -hls_start_number_source datetime \ - -hls_segment_filename "live_%03d.ts" - diff --git a/profiles/nvidia/hls/h264_540p.sh b/profiles/nvidia/hls/h264_540p.sh deleted file mode 100755 index 79491f8..0000000 --- a/profiles/nvidia/hls/h264_540p.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -source "$(dirname "$0")/../.helpers.sh" - -exec ffmpeg -hide_banner -loglevel warning \ - -hwaccel_output_format cuda \ - -c:v "$(cuvid_codec "${1}")" \ - -i "${1}" \ - -map 0:v:0 -map 0:a:0 \ - -vf hwupload_cuda,yadif_cuda=0:-1:0,scale_npp=960:540:interp_algo=super \ - -c:a aac \ - -ar 48000 \ - -b:a 128k \ - -c:v h264_nvenc \ - -profile:v main \ - -b:v 1800k \ - -maxrate 1800k \ - -bufsize 3100k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f hls \ - -hls_time 2 \ - -hls_list_size 5 \ - -hls_wrap 10 \ - -hls_delete_threshold 1 \ - -hls_flags delete_segments \ - -hls_start_number_source datetime \ - -hls_segment_filename "live_%03d.ts" - diff --git a/profiles/nvidia/hls/h264_720p.sh b/profiles/nvidia/hls/h264_720p.sh deleted file mode 100755 index 1fbafc9..0000000 --- a/profiles/nvidia/hls/h264_720p.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -source "$(dirname "$0")/../.helpers.sh" - -exec ffmpeg -hide_banner -loglevel warning \ - -hwaccel_output_format cuda \ - -c:v "$(cuvid_codec "${1}")" \ - -i "${1}" \ - -map 0:v:0 -map 0:a:0 \ - -vf hwupload_cuda,yadif_cuda=0:-1:0,scale_npp=1280:720:interp_algo=super \ - -c:a aac \ - -ar 48000 \ - -b:a 128k \ - -c:v h264_nvenc \ - -profile:v main \ - -b:v 2800k \ - -maxrate 2996k \ - -bufsize 4200k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f hls \ - -hls_time 2 \ - -hls_list_size 5 \ - -hls_wrap 10 \ - -hls_delete_threshold 1 \ - -hls_flags delete_segments \ - -hls_start_number_source datetime \ - -hls_segment_filename "live_%03d.ts" - diff --git a/profiles/nvidia/http/copy.sh b/profiles/nvidia/http/copy.sh deleted file mode 100755 index 4e6da57..0000000 --- a/profiles/nvidia/http/copy.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -source "$(dirname "$0")/../.helpers.sh" - -exec ffmpeg -hide_banner -loglevel warning \ - -hwaccel_output_format cuda \ - -c:v "$(cuvid_codec "${1}")" \ - -i "${1}" \ - -c:a copy \ - -c:v copy \ - -f mpegts - diff --git a/profiles/nvidia/http/h264_1080p.sh b/profiles/nvidia/http/h264_1080p.sh deleted file mode 100755 index fbc02ee..0000000 --- a/profiles/nvidia/http/h264_1080p.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -source "$(dirname "$0")/../.helpers.sh" - -exec ffmpeg -hide_banner -loglevel warning \ - -hwaccel_output_format cuda \ - -c:v "$(cuvid_codec "${1}")" \ - -i "${1}" \ - -vf hwupload_cuda,yadif_cuda=0:-1:0,scale_npp=1920:1080:interp_algo=super \ - -c:a aac \ - -ar 48000 \ - -b:a 192k \ - -c:v h264_nvenc \ - -profile:v main \ - -b:v 5000k \ - -maxrate 5350k \ - -bufsize 7500k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f mpegts - diff --git a/profiles/nvidia/http/h264_360p.sh b/profiles/nvidia/http/h264_360p.sh deleted file mode 100755 index 40bfd6a..0000000 --- a/profiles/nvidia/http/h264_360p.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -source "$(dirname "$0")/../.helpers.sh" - -exec ffmpeg -hide_banner -loglevel warning \ - -hwaccel_output_format cuda \ - -c:v "$(cuvid_codec "${1}")" \ - -i "${1}" \ - -vf hwupload_cuda,yadif_cuda=0:-1:0,scale_npp=640:360:interp_algo=super \ - -c:a aac \ - -ar 48000 \ - -b:a 96k \ - -c:v h264_nvenc \ - -profile:v main \ - -b:v 800k \ - -maxrate 856k \ - -bufsize 1200k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f mpegts - diff --git a/profiles/nvidia/http/h264_540p.sh b/profiles/nvidia/http/h264_540p.sh deleted file mode 100755 index 06f7a88..0000000 --- a/profiles/nvidia/http/h264_540p.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -source "$(dirname "$0")/../.helpers.sh" - -exec ffmpeg -hide_banner -loglevel warning \ - -hwaccel_output_format cuda \ - -c:v "$(cuvid_codec "${1}")" \ - -i "${1}" \ - -vf hwupload_cuda,yadif_cuda=0:-1:0,scale_npp=960:540:interp_algo=super \ - -c:a aac \ - -ar 48000 \ - -b:a 128k \ - -c:v h264_nvenc \ - -profile:v main \ - -b:v 1800k \ - -maxrate 1800k \ - -bufsize 3100k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f mpegts - diff --git a/profiles/nvidia/http/h264_720p.sh b/profiles/nvidia/http/h264_720p.sh deleted file mode 100755 index 4b945b1..0000000 --- a/profiles/nvidia/http/h264_720p.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -source "$(dirname "$0")/../.helpers.sh" - -exec ffmpeg -hide_banner -loglevel warning \ - -hwaccel_output_format cuda \ - -c:v "$(cuvid_codec "${1}")" \ - -i "${1}" \ - -vf hwupload_cuda,yadif_cuda=0:-1:0,scale_npp=1280:720:interp_algo=super \ - -c:a aac \ - -ar 48000 \ - -b:a 128k \ - -c:v h264_nvenc \ - -profile:v main \ - -b:v 2800k \ - -maxrate 2996k \ - -bufsize 4200k \ - -crf 20 \ - -sc_threshold 0 \ - -g 48 \ - -keyint_min 48 \ - -f mpegts - From 1300680667955ff2e86b8d0822926eec13ed8653 Mon Sep 17 00:00:00 2001 From: klahaha Date: Sun, 26 Sep 2021 15:12:22 +0200 Subject: [PATCH 48/56] README and Dockerfile.nvidia for unique profiles dir --- Dockerfile.nvidia | 2 +- README.md | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/Dockerfile.nvidia b/Dockerfile.nvidia index d1d273f..a206662 100644 --- a/Dockerfile.nvidia +++ b/Dockerfile.nvidia @@ -14,7 +14,7 @@ RUN groupadd --gid $USER_GID $USERNAME; \ useradd --uid $USER_UID --gid $USERNAME --shell /bin/bash --create-home $USERNAME; COPY --from=build /app/bin bin -COPY profiles_nvidia profiles +COPY profiles profiles COPY data data ENV USER=$USERNAME diff --git a/README.md b/README.md index fbc0292..c50fd93 100644 --- a/README.md +++ b/README.md @@ -59,12 +59,7 @@ streams: go-transcode supports any formats that ffmpeg likes. We provide profiles out-of-the-box for h264+aac (mp4 container) for 360p, 540p, 720p and 1080p resolutions: `h264_360p`, `h264_540p`, `h264_720p` and `h264_1080p`. Profiles can have any name, but must match regex: `^[0-9A-Za-z_-]+$` -We provide two different profiles directories: - -- profiles/default for CPU transcoding -- profiles/nvidia for NVENC support (proprietary Nvidia driver) - -In these profile directories, actual profiles are located in `hls/` and `http/`, depending on the output format requested. +In these profile directories, actual profiles are located in `hls/` and `http/`, depending on the output format requested. The profiles scripts detect hardware support by running ffmpeg. No special config needed to use hardware acceleration. ## Docker From fead3b5fc7acc69d1edb4bbc30b26aeedbccc72b Mon Sep 17 00:00:00 2001 From: klahaha Date: Sun, 26 Sep 2021 15:14:55 +0200 Subject: [PATCH 49/56] fix config reload tests/reload.sh works with spaces in file --- internal/main.go | 3 ++- tests/reload.sh | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/main.go b/internal/main.go index d56c23c..ea35a87 100644 --- a/internal/main.go +++ b/internal/main.go @@ -73,5 +73,6 @@ func (main *Main) ServeCommand(cmd *cobra.Command, args []string) { } func (main *Main) ConfigReload() { - // TODO: Implement. + main.RootConfig.Set() + main.ServerConfig.Set() } diff --git a/tests/reload.sh b/tests/reload.sh index 5e867de..d82c1ae 100755 --- a/tests/reload.sh +++ b/tests/reload.sh @@ -2,7 +2,7 @@ go build || exit 1 -if [ $# -lt 1 ] || [ ! -f $1 ]; then +if [ $# -lt 1 ] || [ ! -f "$1" ]; then echo "Please give test video first argument" exit 1 fi @@ -30,7 +30,7 @@ baseurl="http://localhost:$port" echo "Using port: $port. Logging to $log" -echo -e "streams:\n test: $1" > $tmpfile +echo -e "streams:\n test: "$1"" > $tmpfile ./go-transcode serve --config $tmpfile --bind :$port >> $log 2>&1 & pid=$! @@ -53,7 +53,7 @@ else fi # Change config and try again test2 -echo -e "streams:\n test2: $1" > $tmpfile +echo -e "streams:\n test2: "$1"" > $tmpfile sleep 1 output="$(curl -o /dev/null -s -I -XGET -w "%{http_code}" $baseurl/h264_720p/test2)" From afb0aaefbbc5892235eeea9848be2d65eae70cd7 Mon Sep 17 00:00:00 2001 From: klahaha Date: Sun, 26 Sep 2021 15:55:19 +0200 Subject: [PATCH 50/56] move data/play.html to internal/api/ --- internal/api/hls.go | 9 ++++++--- {data => internal/api}/play.html | 0 2 files changed, 6 insertions(+), 3 deletions(-) rename {data => internal/api}/play.html (100%) diff --git a/internal/api/hls.go b/internal/api/hls.go index 5b0e98f..16a0688 100644 --- a/internal/api/hls.go +++ b/internal/api/hls.go @@ -1,6 +1,7 @@ package api import ( + _ "embed" "fmt" "net/http" "os/exec" @@ -13,6 +14,9 @@ import ( var hlsManagers map[string]hls.Manager = make(map[string]hls.Manager) +//go:embed play.html +var playHTML string + func (a *ApiManagerCtx) HLS(r chi.Router) { r.Get("/{profile}/{input}/index.m3u8", func(w http.ResponseWriter, r *http.Request) { logger := log.With(). @@ -83,8 +87,7 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { r.Get("/{profile}/{input}/play.html", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") - - file := a.config.AbsPath("data", "play.html") - http.ServeFile(w, r, file) + w.Write([]byte(playHTML)) + return }) } diff --git a/data/play.html b/internal/api/play.html similarity index 100% rename from data/play.html rename to internal/api/play.html From 0524b3f659bf22e153edaf6faacf079074a48af2 Mon Sep 17 00:00:00 2001 From: klahaha Date: Sun, 26 Sep 2021 15:57:18 +0200 Subject: [PATCH 51/56] move data/*-test.sh to profiles/ --- internal/api/http.go | 2 +- {data => profiles}/hls-test.sh | 0 {data => profiles}/http-test.sh | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename {data => profiles}/hls-test.sh (100%) rename {data => profiles}/http-test.sh (100%) diff --git a/internal/api/http.go b/internal/api/http.go index ad8a96c..6cea160 100644 --- a/internal/api/http.go +++ b/internal/api/http.go @@ -21,7 +21,7 @@ func (a *ApiManagerCtx) Http(r chi.Router) { Logger() // dummy input for testing purposes - file := a.config.AbsPath("data", "http-test.sh") + file := a.config.AbsPath("profiles", "http-test.sh") cmd := exec.Command(file) logger.Info().Msg("command startred") diff --git a/data/hls-test.sh b/profiles/hls-test.sh similarity index 100% rename from data/hls-test.sh rename to profiles/hls-test.sh diff --git a/data/http-test.sh b/profiles/http-test.sh similarity index 100% rename from data/http-test.sh rename to profiles/http-test.sh From 28bbcabcd63aeb22f8c39031db0da6272aabd213 Mon Sep 17 00:00:00 2001 From: klahaha Date: Sun, 26 Sep 2021 16:02:49 +0200 Subject: [PATCH 52/56] add bin/ and .env to .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index c234a0d..c0bc540 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ transcode.yml transcode.yaml go-transcode +bin +.env From 39840d367feadf0e88aa527e269160d85a1a9f34 Mon Sep 17 00:00:00 2001 From: klahaha Date: Sun, 26 Sep 2021 16:27:20 +0200 Subject: [PATCH 53/56] fix panic when missing profile --- internal/api/hls.go | 13 ++++++++++--- internal/api/http.go | 22 ++++++++++++++++++++-- internal/api/router.go | 23 ++++++++++++++--------- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/internal/api/hls.go b/internal/api/hls.go index 16a0688..bae7b90 100644 --- a/internal/api/hls.go +++ b/internal/api/hls.go @@ -39,6 +39,15 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { return } + // check if profile exists + profilePath, err := a.ProfilePath("hls", profile) + if err != nil { + logger.Error().Err(err).Msg("Failed to find profile") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(fmt.Sprintf("%v\n", err))) + return + } + ID := fmt.Sprintf("%s/%s", profile, input) manager, ok := hlsManagers[ID] @@ -46,11 +55,9 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { // create new manager manager = hls.New(func() *exec.Cmd { // get transcode cmd - cmd, err := a.transcodeStart("hls", profile, input) + cmd, err := a.transcodeStart(profilePath, input) if err != nil { logger.Error().Err(err).Msg("transcode could not be started") - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("%v\n", err))) } return cmd diff --git a/internal/api/http.go b/internal/api/http.go index 6cea160..837a821 100644 --- a/internal/api/http.go +++ b/internal/api/http.go @@ -56,7 +56,16 @@ func (a *ApiManagerCtx) Http(r chi.Router) { return } - cmd, err := a.transcodeStart("http", profile, input) + // check if profile exists + profilePath, err := a.ProfilePath("hls", profile) + if err != nil { + logger.Error().Err(err).Msg("Failed to find profile") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(fmt.Sprintf("%v\n", err))) + return + } + + cmd, err := a.transcodeStart(profilePath, input) if err != nil { logger.Warn().Err(err).Msg("transcode could not be started") w.WriteHeader(http.StatusInternalServerError) @@ -99,7 +108,16 @@ func (a *ApiManagerCtx) Http(r chi.Router) { return } - cmd, err := a.transcodeStart("http", profile, input) + // check if profile exists + profilePath, err := a.ProfilePath("hls", profile) + if err != nil { + logger.Error().Err(err).Msg("Failed to find profile") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(fmt.Sprintf("%v\n", err))) + return + } + + cmd, err := a.transcodeStart(profilePath, input) if err != nil { logger.Warn().Err(err).Msg("transcode could not be started") w.WriteHeader(http.StatusInternalServerError) diff --git a/internal/api/router.go b/internal/api/router.go index c18de15..6d05bf5 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -48,21 +48,26 @@ func (a *ApiManagerCtx) Mount(r *chi.Mux) { r.Group(a.Http) } -func (a *ApiManagerCtx) transcodeStart(folder string, profile string, input string) (*exec.Cmd, error) { - url, ok := a.config.Streams[input] - if !ok { - return nil, fmt.Errorf("stream not found") - } +func (a *ApiManagerCtx) ProfilePath(folder string, profile string) (string, error) { + // [profiles]/hls,http/[profile].sh + // [profiles] defaults to [basedir]/profiles if !resourceRegex.MatchString(profile) { - return nil, fmt.Errorf("invalid profile path") + return "", fmt.Errorf("invalid profile path") } - // [profiles]/hls,http/[profile].sh - // [profiles] defaults to [basedir]/profiles profilePath := path.Join(a.config.Profiles, folder, fmt.Sprintf("%s.sh", profile)) if _, err := os.Stat(profilePath); os.IsNotExist(err) { - return nil, err + return "", err + } + return profilePath, nil +} + +// Call ProfilePath before +func (a *ApiManagerCtx) transcodeStart(profilePath string, input string) (*exec.Cmd, error) { + url, ok := a.config.Streams[input] + if !ok { + return nil, fmt.Errorf("stream not found") } log.Info().Str("profilePath", profilePath).Str("url", url).Msg("command startred") From 8bc115b1e24dc6995a04e46760f067940c320675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sun, 26 Sep 2021 17:37:48 +0200 Subject: [PATCH 54/56] lint and fix HTTP responses. --- hls/manager.go | 16 ++++++---------- internal/api/hls.go | 19 +++++++------------ internal/api/http.go | 35 +++++++++++++++++------------------ internal/api/router.go | 2 +- internal/http/http.go | 2 +- 5 files changed, 32 insertions(+), 42 deletions(-) diff --git a/hls/manager.go b/hls/manager.go index 9d692db..6a21535 100644 --- a/hls/manager.go +++ b/hls/manager.go @@ -176,7 +176,7 @@ func (m *ManagerCtx) Stop() { err := m.cmd.Process.Kill() m.logger.Err(err).Msg("killing proccess") } - m.cmd.Wait() + _ = m.cmd.Wait() m.cmd = nil } @@ -217,8 +217,7 @@ func (m *ManagerCtx) ServePlaylist(w http.ResponseWriter, r *http.Request) { err := m.Start() if err != nil { m.logger.Warn().Err(err).Msg("transcode could not be started") - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(err.Error())) + http.Error(w, "500 not available", http.StatusInternalServerError) return } } @@ -228,21 +227,19 @@ func (m *ManagerCtx) ServePlaylist(w http.ResponseWriter, r *http.Request) { case playlist = <-m.playlistLoad: case <-m.shutdown: m.logger.Warn().Msg("playlist load failed because of shutdown") - w.WriteHeader(http.StatusNotFound) // When command failed to start and timeout has been increased we reach this branch after a while - w.Write([]byte("404 playlist not found")) + http.Error(w, "404 playlist not found", http.StatusNotFound) return case <-time.After(playlistTimeout): m.logger.Warn().Msg("playlist load channel timeouted") - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("500 not available")) + http.Error(w, "500 not available", http.StatusInternalServerError) return } } w.Header().Set("Content-Type", "application/vnd.apple.mpegurl") w.Header().Set("Cache-Control", "no-cache") - w.Write([]byte(playlist)) + _, _ = w.Write([]byte(playlist)) } func (m *ManagerCtx) ServeMedia(w http.ResponseWriter, r *http.Request) { @@ -251,8 +248,7 @@ func (m *ManagerCtx) ServeMedia(w http.ResponseWriter, r *http.Request) { if _, err := os.Stat(path); os.IsNotExist(err) { m.logger.Warn().Str("path", path).Msg("media file not found") - w.WriteHeader(http.StatusNotFound) - w.Write([]byte("404 media not found")) + http.Error(w, "404 media not found", http.StatusNotFound) return } diff --git a/internal/api/hls.go b/internal/api/hls.go index bae7b90..bf227a9 100644 --- a/internal/api/hls.go +++ b/internal/api/hls.go @@ -27,24 +27,22 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { input := chi.URLParam(r, "input") if !resourceRegex.MatchString(profile) || !resourceRegex.MatchString(input) { - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte("400 invalid parameters")) + http.Error(w, "400 invalid parameters", http.StatusBadRequest) return } // check if stream exists _, ok := a.config.Streams[input] if !ok { - http.NotFound(w, r) + http.Error(w, "404 stream not found", http.StatusNotFound) return } // check if profile exists profilePath, err := a.ProfilePath("hls", profile) if err != nil { - logger.Error().Err(err).Msg("Failed to find profile") - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("%v\n", err))) + logger.Warn().Err(err).Msg("profile path could not be found") + http.Error(w, "404 profile not found", http.StatusNotFound) return } @@ -75,8 +73,7 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { file := chi.URLParam(r, "file") if !resourceRegex.MatchString(profile) || !resourceRegex.MatchString(input) || !resourceRegex.MatchString(file) { - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte("400 invalid parameters")) + http.Error(w, "400 invalid parameters", http.StatusBadRequest) return } @@ -84,8 +81,7 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { manager, ok := hlsManagers[ID] if !ok { - w.WriteHeader(http.StatusNotFound) - w.Write([]byte("404 transcode not found")) + http.Error(w, "404 transcode not found", http.StatusNotFound) return } @@ -94,7 +90,6 @@ func (a *ApiManagerCtx) HLS(r chi.Router) { r.Get("/{profile}/{input}/play.html", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") - w.Write([]byte(playHTML)) - return + _, _ = w.Write([]byte(playHTML)) }) } diff --git a/internal/api/http.go b/internal/api/http.go index 837a821..f190dab 100644 --- a/internal/api/http.go +++ b/internal/api/http.go @@ -1,7 +1,6 @@ package api import ( - "fmt" "io" "net/http" "os/exec" @@ -36,8 +35,10 @@ func (a *ApiManagerCtx) Http(r chi.Router) { write.Close() }() - go cmd.Run() - io.Copy(w, read) + go func() { + _ = cmd.Run() + }() + _, _ = io.Copy(w, read) }) r.Get("/{profile}/{input}", func(w http.ResponseWriter, r *http.Request) { @@ -52,24 +53,22 @@ func (a *ApiManagerCtx) Http(r chi.Router) { // check if stream exists _, ok := a.config.Streams[input] if !ok { - http.NotFound(w, r) + http.Error(w, "404 stream not found", http.StatusNotFound) return } // check if profile exists profilePath, err := a.ProfilePath("hls", profile) if err != nil { - logger.Error().Err(err).Msg("Failed to find profile") - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("%v\n", err))) + logger.Warn().Err(err).Msg("profile path could not be found") + http.Error(w, "404 profile not found", http.StatusNotFound) return } cmd, err := a.transcodeStart(profilePath, input) if err != nil { logger.Warn().Err(err).Msg("transcode could not be started") - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("%v", err))) + http.Error(w, "500 not available", http.StatusInternalServerError) return } @@ -87,8 +86,10 @@ func (a *ApiManagerCtx) Http(r chi.Router) { write.Close() }() - go cmd.Run() - io.Copy(w, read) + go func() { + _ = cmd.Run() + }() + _, _ = io.Copy(w, read) }) // buffered http streaming (alternative to prervious type) @@ -104,24 +105,22 @@ func (a *ApiManagerCtx) Http(r chi.Router) { // check if stream exists _, ok := a.config.Streams[input] if !ok { - http.NotFound(w, r) + http.Error(w, "404 stream not found", http.StatusNotFound) return } // check if profile exists profilePath, err := a.ProfilePath("hls", profile) if err != nil { - logger.Error().Err(err).Msg("Failed to find profile") - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("%v\n", err))) + logger.Warn().Err(err).Msg("profile path could not be found") + http.Error(w, "404 profile not found", http.StatusNotFound) return } cmd, err := a.transcodeStart(profilePath, input) if err != nil { logger.Warn().Err(err).Msg("transcode could not be started") - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("%v", err))) + http.Error(w, "500 not available", http.StatusInternalServerError) return } @@ -133,7 +132,7 @@ func (a *ApiManagerCtx) Http(r chi.Router) { cmd.Stderr = utils.LogWriter(logger) go utils.IOPipeToHTTP(w, read) - cmd.Run() + _ = cmd.Run() write.Close() logger.Info().Msg("command stopped") }) diff --git a/internal/api/router.go b/internal/api/router.go index 6d05bf5..d229396 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -41,7 +41,7 @@ func (manager *ApiManagerCtx) Shutdown() error { func (a *ApiManagerCtx) Mount(r *chi.Mux) { r.Get("/ping", func(w http.ResponseWriter, r *http.Request) { //nolint - w.Write([]byte("pong")) + _, _ = w.Write([]byte("pong")) }) r.Group(a.HLS) diff --git a/internal/http/http.go b/internal/http/http.go index 0ef7694..f58a8ef 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -43,7 +43,7 @@ func New(config *config.Server) *HttpManagerCtx { router.NotFound(func(w http.ResponseWriter, r *http.Request) { //nolint - w.Write([]byte("404")) + _, _ = w.Write([]byte("404")) }) return &HttpManagerCtx{ From 5548b689b0c515f52d424661a35f243811376106 Mon Sep 17 00:00:00 2001 From: klahaha Date: Sun, 26 Sep 2021 17:43:08 +0200 Subject: [PATCH 55/56] config file is config.yml or config.yaml --- .gitignore | 4 ++-- cmd/root.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index c0bc540..dea0fff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -transcode.yml -transcode.yaml +config.yml +config.yaml go-transcode bin .env diff --git a/cmd/root.go b/cmd/root.go index fcdc759..352b282 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -51,7 +51,7 @@ func init() { } viper.AddConfigPath(".") - viper.SetConfigName("transcode") + viper.SetConfigName("config") } viper.SetEnvPrefix("transcode") From e03ddf9f42767df301bf1d073f0ed0eaa5d8f068 Mon Sep 17 00:00:00 2001 From: klahaha Date: Sun, 26 Sep 2021 17:49:46 +0200 Subject: [PATCH 56/56] update to use config.yaml --- README.md | 6 +++--- docker-compose.yaml | 2 +- tests/reload.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c50fd93..b63045d 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Features: ## Config -Place your config file in `./transcode.yml` (or `/etc/transcode/transcode.yml`). The streams are defined like this: +Place your config file in `./config.yaml` (or `/etc/transcode/config.yaml`). The streams are defined like this: ```yaml streams: @@ -75,7 +75,7 @@ docker build -t go-transcode:latest . docker run --rm -d \ --name="go-transcode" \ -p "8080:8080" \ - -v "${PWD}/transcode.yaml:/app/transcode.yaml" go-transcode:latest + -v "${PWD}/config.yaml:/app/config.yaml" go-transcode:latest ``` ## Nvidia GPU support (docker) @@ -97,7 +97,7 @@ docker run --rm -d \ --gpus=all \ --name="go-transcode-nvidia" \ -p "8080:8080" \ - -v "${PWD}/transcode.yaml:/app/transcode.yaml" go-transcode-nvidia:latest + -v "${PWD}/config.yaml:/app/config.yaml" go-transcode-nvidia:latest ``` ### Supported inputs diff --git a/docker-compose.yaml b/docker-compose.yaml index b008a33..7437295 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,5 +8,5 @@ services: ports: - "8080:8080" volumes: - - ./transcode.yaml:/app/transcode.yaml + - ./config.yaml:/app/config.yaml command: serve -d diff --git a/tests/reload.sh b/tests/reload.sh index d82c1ae..56a8552 100755 --- a/tests/reload.sh +++ b/tests/reload.sh @@ -7,7 +7,7 @@ if [ $# -lt 1 ] || [ ! -f "$1" ]; then exit 1 fi -tmpfile=$(mktemp --suffix .yml) +tmpfile=$(mktemp --suffix .yaml) log=$(mktemp) BASE_PORT=8888