From 28286660bb86f86f472be4f4f9ea82e09f71f40c Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 13 May 2023 16:29:17 +0800 Subject: [PATCH 01/24] rewrite --- cmd/cmd.go | 23 +- cmd/doctor.go | 90 ++- cmd/dump.go | 5 +- cmd/embedded.go | 8 +- cmd/manager_logging.go | 120 +--- cmd/serv.go | 5 +- cmd/web.go | 10 +- custom/conf/app.example.ini | 3 +- .../administration/logging-config.en-us.md | 299 ++++++++++ .../logging-documentation.en-us.md | 524 ------------------ main.go | 2 + models/db/log.go | 52 +- models/issues/pull.go | 39 +- models/migrations/base/tests.go | 6 +- models/organization/team.go | 16 +- models/perm/access/repo_permission.go | 55 +- models/perm/access_mode.go | 9 +- models/repo/repo.go | 16 +- models/unit/unit.go | 7 +- models/user/user.go | 14 +- modules/context/access_log.go | 5 +- modules/doctor/doctor.go | 74 ++- modules/git/git_test.go | 3 - modules/graceful/manager.go | 2 +- modules/graceful/manager_unix.go | 3 +- .../graceful/releasereopen/releasereopen.go | 60 ++ modules/indexer/code/elastic_search.go | 20 +- modules/indexer/issues/elastic_search.go | 21 +- modules/lfs/pointer.go | 12 +- modules/log/color.go | 115 ++++ modules/log/color_console.go | 14 + ...onsole_other.go => color_console_other.go} | 0 ...le_windows.go => color_console_windows.go} | 2 +- .../log/{colors_router.go => color_router.go} | 60 +- modules/log/colors.go | 435 --------------- modules/log/conn.go | 137 ----- modules/log/conn_test.go | 230 -------- modules/log/console.go | 93 ---- modules/log/console_test.go | 137 ----- modules/log/errors.go | 61 -- modules/log/event.go | 460 --------------- modules/log/event_format.go | 253 +++++++++ modules/log/event_writer.go | 51 ++ modules/log/event_writer_base.go | 136 +++++ modules/log/event_writer_conn.go | 112 ++++ modules/log/event_writer_console.go | 41 ++ modules/log/event_writer_file.go | 49 ++ modules/log/file.go | 283 ---------- modules/log/file_test.go | 235 -------- modules/log/flags.go | 13 +- modules/log/init.go | 32 ++ modules/log/level.go | 96 ++-- modules/log/log.go | 305 ---------- modules/log/log_test.go | 152 ----- modules/log/logger.go | 141 ----- modules/log/logger_global.go | 84 +++ modules/log/logger_impl.go | 234 ++++++++ modules/log/logger_types.go | 75 +++ modules/log/manager.go | 93 ++++ modules/log/misc.go | 33 ++ modules/log/multichannel.go | 104 ---- modules/log/package.go | 24 + modules/log/provider.go | 25 - modules/log/smtp.go | 114 ---- modules/log/smtp_test.go | 85 --- modules/log/stack.go | 6 +- modules/log/writer.go | 269 --------- modules/log/writer_test.go | 275 --------- modules/private/manager.go | 14 +- modules/setting/config_provider.go | 23 + modules/setting/database.go | 2 +- modules/setting/log.go | 469 ++++++---------- modules/setting/repository.go | 1 - modules/setting/server.go | 1 - modules/setting/setting.go | 15 +- modules/ssh/ssh.go | 16 +- modules/templates/htmlrenderer.go | 2 +- modules/test/logchecker.go | 67 +-- modules/test/logchecker_test.go | 2 - modules/testlogger/testlogger.go | 45 +- modules/util/rotatingfilewriter/writer.go | 226 ++++++++ .../util/rotatingfilewriter/writer_test.go | 48 ++ modules/web/routing/logger.go | 32 +- options/locale/locale_en-US.ini | 9 +- routers/common/middleware.go | 4 +- routers/private/internal.go | 2 +- routers/private/manager.go | 126 +++-- routers/web/admin/config.go | 7 +- routers/web/repo/issue.go | 6 +- routers/web/repo/issue_watch.go | 2 +- services/migrations/codebase.go | 8 +- services/migrations/gitbucket.go | 8 +- services/migrations/gitea_downloader.go | 8 +- services/migrations/github.go | 8 +- services/migrations/gitlab.go | 8 +- services/migrations/gogs.go | 8 +- services/migrations/onedev.go | 8 +- templates/admin/config.tmpl | 65 +-- tests/e2e/e2e_test.go | 2 +- tests/integration/integration_test.go | 2 +- .../migration-test/migration_test.go | 8 +- tests/test_utils.go | 6 +- 102 files changed, 2663 insertions(+), 5162 deletions(-) create mode 100644 docs/content/doc/administration/logging-config.en-us.md delete mode 100644 docs/content/doc/administration/logging-documentation.en-us.md create mode 100644 modules/graceful/releasereopen/releasereopen.go create mode 100644 modules/log/color.go create mode 100644 modules/log/color_console.go rename modules/log/{console_other.go => color_console_other.go} (100%) rename modules/log/{console_windows.go => color_console_windows.go} (94%) rename modules/log/{colors_router.go => color_router.go} (54%) delete mode 100644 modules/log/colors.go delete mode 100644 modules/log/conn.go delete mode 100644 modules/log/conn_test.go delete mode 100644 modules/log/console.go delete mode 100644 modules/log/console_test.go delete mode 100644 modules/log/errors.go delete mode 100644 modules/log/event.go create mode 100644 modules/log/event_format.go create mode 100644 modules/log/event_writer.go create mode 100644 modules/log/event_writer_base.go create mode 100644 modules/log/event_writer_conn.go create mode 100644 modules/log/event_writer_console.go create mode 100644 modules/log/event_writer_file.go delete mode 100644 modules/log/file.go delete mode 100644 modules/log/file_test.go create mode 100644 modules/log/init.go delete mode 100644 modules/log/log.go delete mode 100644 modules/log/log_test.go delete mode 100644 modules/log/logger.go create mode 100644 modules/log/logger_global.go create mode 100644 modules/log/logger_impl.go create mode 100644 modules/log/logger_types.go create mode 100644 modules/log/manager.go create mode 100644 modules/log/misc.go delete mode 100644 modules/log/multichannel.go create mode 100644 modules/log/package.go delete mode 100644 modules/log/provider.go delete mode 100644 modules/log/smtp.go delete mode 100644 modules/log/smtp_test.go delete mode 100644 modules/log/writer.go delete mode 100644 modules/log/writer_test.go create mode 100644 modules/util/rotatingfilewriter/writer.go create mode 100644 modules/util/rotatingfilewriter/writer_test.go diff --git a/cmd/cmd.go b/cmd/cmd.go index cf2d9ef89e83..afe403d502ff 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "io" "os" "os/signal" "strings" @@ -59,7 +60,7 @@ func confirm() (bool, error) { func initDB(ctx context.Context) error { setting.Init(&setting.Options{}) setting.LoadDBSetting() - setting.InitSQLLog(false) + setting.InitSQLLoggersForCli() if setting.Database.Type == "" { log.Fatal(`Database settings are missing from the configuration file: %q. @@ -93,3 +94,23 @@ func installSignals() (context.Context, context.CancelFunc) { return ctx, cancel } + +func setupConsoleLogger(level log.Level, colorize bool, out io.Writer) { + if out != os.Stdout && out != os.Stderr { + panic("setupConsoleLogger can only be used with os.Stdout or os.Stderr") + } + + writeMode := log.WriterMode{ + WriterType: "console", + Level: level, + Colorize: colorize, + WriterOption: log.WriterConsoleOption{Stderr: out == os.Stderr}, + } + writer, err := log.NewEventWriter("console", writeMode) + if err != nil { + log.FallbackErrorf("unable to create console log writer: %v", err) + return + } + + log.GetManager().GetLogger(log.DEFAULT).RemoveAllWriters().AddWriters(writer) +} diff --git a/cmd/doctor.go b/cmd/doctor.go index 65c028c5ed19..1bf40bd4c8b7 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -4,10 +4,10 @@ package cmd import ( - "errors" "fmt" golog "log" "os" + "path/filepath" "strings" "text/tabwriter" @@ -82,23 +82,25 @@ You should back-up your database before doing this and ensure that your database } func runRecreateTable(ctx *cli.Context) error { + stdCtx, cancel := installSignals() + defer cancel() + // Redirect the default golog to here golog.SetFlags(0) golog.SetPrefix("") - golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT))) + golog.SetOutput(log.LoggerToWriter(log.GetLogger(log.DEFAULT).Info)) + debug := ctx.Bool("debug") setting.Init(&setting.Options{}) setting.LoadDBSetting() - setting.Log.EnableXORMLog = ctx.Bool("debug") - setting.Database.LogSQL = ctx.Bool("debug") - // FIXME: don't use CfgProvider directly - setting.CfgProvider.Section("log").Key("XORM").SetValue(",") - - setting.InitSQLLog(!ctx.Bool("debug")) - stdCtx, cancel := installSignals() - defer cancel() + if debug { + setting.InitSQLLoggersForCliDebug() + } else { + setting.InitSQLLoggersForCli() + } + setting.Database.LogSQL = debug if err := db.InitEngine(stdCtx); err != nil { fmt.Println(err) fmt.Println("Check if you are using the right config file. You can use a --config directive to specify one.") @@ -125,44 +127,31 @@ func runRecreateTable(ctx *cli.Context) error { }) } -func setDoctorLogger(ctx *cli.Context) { +func setupDoctorDefaultLogger(ctx *cli.Context, colorize bool) { + // Silence the default loggers + setupConsoleLogger(log.FATAL, log.CanColorStderr, os.Stderr) + logFile := ctx.String("log-file") if !ctx.IsSet("log-file") { logFile = "doctor.log" } - colorize := log.CanColorStdout - if ctx.IsSet("color") { - colorize = ctx.Bool("color") - } if len(logFile) == 0 { - log.NewLogger(1000, "doctor", "console", fmt.Sprintf(`{"level":"NONE","stacktracelevel":"NONE","colorize":%t}`, colorize)) + // if no doctor log-file is set, do not show any log from default logger return } - defer func() { - recovered := recover() - if recovered == nil { - return - } - - err, ok := recovered.(error) - if !ok { - panic(recovered) - } - if errors.Is(err, os.ErrPermission) { - fmt.Fprintf(os.Stderr, "ERROR: Unable to write logs to provided file due to permissions error: %s\n %v\n", logFile, err) - } else { - fmt.Fprintf(os.Stderr, "ERROR: Unable to write logs to provided file: %s\n %v\n", logFile, err) - } - fmt.Fprintf(os.Stderr, "WARN: Logging will be disabled\n Use `--log-file` to configure log file location\n") - log.NewLogger(1000, "doctor", "console", fmt.Sprintf(`{"level":"NONE","stacktracelevel":"NONE","colorize":%t}`, colorize)) - }() - if logFile == "-" { - log.NewLogger(1000, "doctor", "console", fmt.Sprintf(`{"level":"trace","stacktracelevel":"NONE","colorize":%t}`, colorize)) + setupConsoleLogger(log.TRACE, colorize, os.Stdout) } else { - log.NewLogger(1000, "doctor", "file", fmt.Sprintf(`{"filename":%q,"level":"trace","stacktracelevel":"NONE"}`, logFile)) + logFile, _ = filepath.Abs(logFile) + writeMode := log.WriterMode{WriterType: "file", Level: log.TRACE, WriterOption: log.WriterFileOption{FileName: logFile}} + writer, err := log.NewEventWriter("console-to-file", writeMode) + if err != nil { + log.FallbackErrorf("unable to create file log writer: %v", err) + return + } + log.GetManager().GetLogger(log.DEFAULT).RemoveAllWriters().AddWriters(writer) } } @@ -170,22 +159,17 @@ func runDoctor(ctx *cli.Context) error { stdCtx, cancel := installSignals() defer cancel() - // Silence the default loggers - log.DelNamedLogger("console") - log.DelNamedLogger(log.DEFAULT) - - // Now setup our own - setDoctorLogger(ctx) - colorize := log.CanColorStdout if ctx.IsSet("color") { colorize = ctx.Bool("color") } - // Finally redirect the default golog to here + setupDoctorDefaultLogger(ctx, colorize) + + // Finally redirect the default golang's log to here golog.SetFlags(0) golog.SetPrefix("") - golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT))) + golog.SetOutput(log.LoggerToWriter(log.GetLogger(log.DEFAULT).Info)) if ctx.IsSet("list") { w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0) @@ -233,17 +217,5 @@ func runDoctor(ctx *cli.Context) error { } } - // Now we can set up our own logger to return information about what the doctor is doing - if err := log.NewNamedLogger("doctorouter", - 0, - "console", - "console", - fmt.Sprintf(`{"level":"INFO","stacktracelevel":"NONE","colorize":%t,"flags":-1}`, colorize)); err != nil { - fmt.Println(err) - return err - } - - logger := log.GetLogger("doctorouter") - defer logger.Close() - return doctor.RunChecks(stdCtx, logger, ctx.Bool("fix"), checks) + return doctor.RunChecks(stdCtx, colorize, ctx.Bool("fix"), checks) } diff --git a/cmd/dump.go b/cmd/dump.go index 32ccc5566c8a..40524f48d4d1 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -172,10 +172,7 @@ func runDump(ctx *cli.Context) error { outType := ctx.String("type") if fileName == "-" { file = os.Stdout - err := log.DelLogger("console") - if err != nil { - fatal("Deleting default logger failed. Can not write to stdout: %v", err) - } + setupConsoleLogger(log.FATAL, log.CanColorStderr, os.Stderr) } else { for _, suffix := range outputTypeEnum.Enum { if strings.HasSuffix(fileName, "."+suffix) { diff --git a/cmd/embedded.go b/cmd/embedded.go index 3f849bea0a26..e51f8477b445 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -97,13 +97,7 @@ type assetFile struct { } func initEmbeddedExtractor(c *cli.Context) error { - // FIXME: there is a bug, if the user runs `gitea embedded` with a different user or root, - // The setting.Init (loadRunModeFrom) will fail and do log.Fatal - // But the console logger has been deleted, so nothing is printed, the user sees nothing and Gitea just exits. - - // Silence the console logger - log.DelNamedLogger("console") - log.DelNamedLogger(log.DEFAULT) + setupConsoleLogger(log.ERROR, log.CanColorStderr, os.Stderr) // Read configuration file setting.Init(&setting.Options{ diff --git a/cmd/manager_logging.go b/cmd/manager_logging.go index 914210d37037..dd85cc26d808 100644 --- a/cmd/manager_logging.go +++ b/cmd/manager_logging.go @@ -16,13 +16,13 @@ import ( var ( defaultLoggingFlags = []cli.Flag{ cli.StringFlag{ - Name: "group, g", - Usage: "Group to add logger to - will default to \"default\"", + Name: "logger", + Usage: `Logger name - will default to "default"`, }, cli.StringFlag{ - Name: "name, n", - Usage: "Name of the new logger - will default to mode", + Name: "writer", + Usage: "Name of the log writer - will default to mode", }, cli.StringFlag{ - Name: "level, l", + Name: "level", Usage: "Logging level for the new logger", }, cli.StringFlag{ Name: "stacktrace-level, L", @@ -83,8 +83,8 @@ var ( cli.BoolFlag{ Name: "debug", }, cli.StringFlag{ - Name: "group, g", - Usage: "Group to add logger to - will default to \"default\"", + Name: "logger", + Usage: `Logger name - will default to "default"`, }, }, Action: runRemoveLogger, @@ -93,15 +93,6 @@ var ( Usage: "Add a logger", Subcommands: []cli.Command{ { - Name: "console", - Usage: "Add a console logger", - Flags: append(defaultLoggingFlags, - cli.BoolFlag{ - Name: "stderr", - Usage: "Output console logs to stderr - only relevant for console", - }), - Action: runAddConsoleLogger, - }, { Name: "file", Usage: "Add a file logger", Flags: append(defaultLoggingFlags, []cli.Flag{ @@ -148,28 +139,6 @@ var ( }, }...), Action: runAddConnLogger, - }, { - Name: "smtp", - Usage: "Add an SMTP logger", - Flags: append(defaultLoggingFlags, []cli.Flag{ - cli.StringFlag{ - Name: "username, u", - Usage: "Mail server username", - }, cli.StringFlag{ - Name: "password, P", - Usage: "Mail server password", - }, cli.StringFlag{ - Name: "host, H", - Usage: "Mail server host (defaults to: 127.0.0.1:25)", - }, cli.StringSliceFlag{ - Name: "send-to, s", - Usage: "Email address(es) to send to", - }, cli.StringFlag{ - Name: "subject, S", - Usage: "Subject header of sent emails", - }, - }...), - Action: runAddSMTPLogger, }, }, }, { @@ -194,50 +163,16 @@ func runRemoveLogger(c *cli.Context) error { defer cancel() setup(ctx, c.Bool("debug")) - group := c.String("group") - if len(group) == 0 { - group = log.DEFAULT + logger := c.String("logger") + if len(logger) == 0 { + logger = log.DEFAULT } - name := c.Args().First() + writer := c.Args().First() - extra := private.RemoveLogger(ctx, group, name) + extra := private.RemoveLogger(ctx, logger, writer) return handleCliResponseExtra(extra) } -func runAddSMTPLogger(c *cli.Context) error { - ctx, cancel := installSignals() - defer cancel() - - setup(ctx, c.Bool("debug")) - vals := map[string]interface{}{} - mode := "smtp" - if c.IsSet("host") { - vals["host"] = c.String("host") - } else { - vals["host"] = "127.0.0.1:25" - } - - if c.IsSet("username") { - vals["username"] = c.String("username") - } - if c.IsSet("password") { - vals["password"] = c.String("password") - } - - if !c.IsSet("send-to") { - return fmt.Errorf("Some recipients must be provided") - } - vals["sendTos"] = c.StringSlice("send-to") - - if c.IsSet("subject") { - vals["subject"] = c.String("subject") - } else { - vals["subject"] = "Diagnostic message from Gitea" - } - - return commonAddLogger(c, mode, vals) -} - func runAddConnLogger(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() @@ -301,25 +236,12 @@ func runAddFileLogger(c *cli.Context) error { return commonAddLogger(c, mode, vals) } -func runAddConsoleLogger(c *cli.Context) error { - ctx, cancel := installSignals() - defer cancel() - - setup(ctx, c.Bool("debug")) - vals := map[string]interface{}{} - mode := "console" - if c.IsSet("stderr") && c.Bool("stderr") { - vals["stderr"] = c.Bool("stderr") - } - return commonAddLogger(c, mode, vals) -} - func commonAddLogger(c *cli.Context, mode string, vals map[string]interface{}) error { if len(c.String("level")) > 0 { - vals["level"] = log.FromString(c.String("level")).String() + vals["level"] = log.LevelFromString(c.String("level")).String() } if len(c.String("stacktrace-level")) > 0 { - vals["stacktraceLevel"] = log.FromString(c.String("stacktrace-level")).String() + vals["stacktraceLevel"] = log.LevelFromString(c.String("stacktrace-level")).String() } if len(c.String("expression")) > 0 { vals["expression"] = c.String("expression") @@ -333,18 +255,18 @@ func commonAddLogger(c *cli.Context, mode string, vals map[string]interface{}) e if c.IsSet("color") { vals["colorize"] = c.Bool("color") } - group := "default" - if c.IsSet("group") { - group = c.String("group") + logger := log.DEFAULT + if c.IsSet("logger") { + logger = c.String("logger") } - name := mode - if c.IsSet("name") { - name = c.String("name") + writer := mode + if c.IsSet("writer") { + writer = c.String("writer") } ctx, cancel := installSignals() defer cancel() - extra := private.AddLogger(ctx, group, name, mode, vals) + extra := private.AddLogger(ctx, logger, writer, mode, vals) return handleCliResponseExtra(extra) } diff --git a/cmd/serv.go b/cmd/serv.go index a79f314d00b6..87bf1cce20e4 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -56,11 +56,10 @@ var CmdServ = cli.Command{ } func setup(ctx context.Context, debug bool) { - _ = log.DelLogger("console") if debug { - _ = log.NewLogger(1000, "console", "console", `{"level":"trace","stacktracelevel":"NONE","stderr":true}`) + setupConsoleLogger(log.TRACE, false, os.Stderr) } else { - _ = log.NewLogger(1000, "console", "console", `{"level":"fatal","stacktracelevel":"NONE","stderr":true}`) + setupConsoleLogger(log.FATAL, false, os.Stderr) } setting.Init(&setting.Options{}) if debug { diff --git a/cmd/web.go b/cmd/web.go index 3a01d07b05f5..bc344db54017 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -103,11 +103,9 @@ func createPIDFile(pidPath string) { func runWeb(ctx *cli.Context) error { if ctx.Bool("verbose") { - _ = log.DelLogger("console") - log.NewLogger(0, "console", "console", fmt.Sprintf(`{"level": "trace", "colorize": %t, "stacktraceLevel": "none"}`, log.CanColorStdout)) + setupConsoleLogger(log.TRACE, log.CanColorStdout, os.Stdout) } else if ctx.Bool("quiet") { - _ = log.DelLogger("console") - log.NewLogger(0, "console", "console", fmt.Sprintf(`{"level": "fatal", "colorize": %t, "stacktraceLevel": "none"}`, log.CanColorStdout)) + setupConsoleLogger(log.FATAL, log.CanColorStdout, os.Stdout) } defer func() { if panicked := recover(); panicked != nil { @@ -156,7 +154,7 @@ func runWeb(ctx *cli.Context) error { case <-graceful.GetManager().IsShutdown(): <-graceful.GetManager().Done() log.Info("PID: %d Gitea Web Finished", os.Getpid()) - log.Close() + log.GetManager().Close() return err default: } @@ -199,7 +197,7 @@ func runWeb(ctx *cli.Context) error { err := listen(c, true) <-graceful.GetManager().Done() log.Info("PID: %d Gitea Web Finished", os.Getpid()) - log.Close() + log.GetManager().Close() return err } diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 9046f5473447..0055929d995e 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -230,7 +230,6 @@ RUN_MODE = ; prod ;; ;; Disable CDN even in "prod" mode ;OFFLINE_MODE = false -;DISABLE_ROUTER_LOG = false ;; ;; TLS Settings: Either ACME or manual ;; (Other common TLS configuration are found before) @@ -387,7 +386,7 @@ USER = root ;ITERATE_BUFFER_SIZE = 50 ;; ;; Show the database generated SQL -LOG_SQL = false ; if unset defaults to true +;LOG_SQL = false ;; ;; Maximum number of DB Connect retries ;DB_RETRIES = 10 diff --git a/docs/content/doc/administration/logging-config.en-us.md b/docs/content/doc/administration/logging-config.en-us.md new file mode 100644 index 000000000000..20a730973408 --- /dev/null +++ b/docs/content/doc/administration/logging-config.en-us.md @@ -0,0 +1,299 @@ +--- +date: "2019-04-02T17:06:00+01:00" +title: "Logging Configuration" +slug: "logging-config" +weight: 40 +toc: false +draft: false +aliases: + - /en-us/logging-configuration +menu: + sidebar: + parent: "administration" + name: "Logging Configuration" + weight: 40 + identifier: "logging-config" +--- + +# Logging Configuration + +The logging configuration of Gitea mainly consists of 3 types of components: + +- The `[log]` section for general configuration +- `[log.]` sections for the configuration of different log writers to output logs, aka: "writer mode", the mode name is also used as "writer name". +- `[log]` section could contain sub-loggers like`logger.LOGGER-NAME.CONFIG-KEY` + +There is a fully functional log output by default, so it is not necessary to define one. + +**Table of Contents** + +{{< toc >}} + +## Collecting Logs for Help + +To collect logs for help and issue report, see [Support Options]({{< relref "doc/help/support.en-us.md" >}}). + +## The `[log]` section + +Configuration of logging facilities in Gitea happen in the `[log]` section and it's subsections. + +In the top level `[log]` section the following configurations can be placed: + +- `ROOT_PATH`: (Default: **%(GITEA_WORK_DIR)/log**): Base path for log files +- `MODE`: (Default: **console**) List of log outputs to use for the Default logger. +- `LEVEL`: (Default: **Info**) Least severe log events to persist, case-insensitive. Possible values are: `Trace`, `Debug`, `Info`, `Warn`, `Error`, `Fatal`. +- `STACKTRACE_LEVEL`: (Default: **None**) For this and more severe events the stacktrace will be printed upon getting logged. + +And it can contain the following sub-loggers: + +- `logger.ROUTER.MODE`: (Default: **,**): List of log outputs to use for the Router logger. +- `logger.ACCESS.MODE`: (Default: **\**) List of log outputs to use for the Access logger. By default, the access logger is disabled. +- `logger.XORM.MODE`: (Default: **,**) List of log outputs to use for the XORM logger. + +Setting a comma (`,`) to sub-logger's mode means making it use the default global `MODE`. + +## Quick samples + +### Default (empty) Configuration + +The empty configuration is equivalent to default: + +```ini +[log] +ROOT_PATH = %(GITEA_WORK_DIR)/log +MODE = console +LEVEL = Info +STACKTRACE_LEVEL = None +logger.ROUTER.MODE = , +logger.XORM.MODE = , +logger.ACCESS.MODE = + +; this is the config options of "console" mode (used by MODE=console above) +[log.console] +MODE = console +FLAGS = stdflags +PREFIX = +COLORIZE = true +``` + +This is equivalent to sending all logs to the console, with default Golang log being sent to the console log too. + +This is only a sample, and it is the default, do not need to write it into your configuration file. + +### Disable Router logs and record some access logs to file + +The Router logger is disabled, the access logs (>=Warn) goes into `access.log`: + +```ini +[log] +logger.ROUTER.MODE = +logger.ACCESS.MODE = access-file + +[log.access-file] +MODE = file +LEVEL = Warn +FILE_NAME = access.log +``` + +### Set different log levels for different modes + +Default logs (>=Warn) goes into `gitea.log`, while Error logs goes into `file-error.log`: + +```ini +[log] +LEVEL = Warn +MODE = file, file-error + +; by default, the "file" mode will record logs to %(log.ROOT_PATH)/gitea.log, so we don't need to set it +; [log.file] + +[log.file-error] +LEVEL = Error +FILE_NAME = file-error.log +``` + +## Log outputs (mode and writer) + +Gitea provides the following log output writers: + +- `console` - Log to `stdout` (or `stderr` if it is set in the config) +- `file` - Log to a file +- `conn` - Log to a socket (network or unix) + +### Common configuration + +Certain configuration is common to all modes of log output: + +- `MODE` is the mode of the log output writer. It will default to the mode name in the ini section. Thus `[log.console]` will default to `MODE = console`. +- `LEVEL` is the lowest level that this output will log. +- `STACKTRACE_LEVEL` is the lowest level that this output will print a stacktrace. +- `COLORIZE` will default to `true` for `console` as described, otherwise it will default to `false`. + +#### `EXPRESSION` + +`EXPRESSION` represents a regular expression that log events must match to be logged by the output writer. +Either the log message, (with colors removed), must match or the `longfilename:linenumber:functionname` must match. +NB: the whole message or string doesn't need to completely match. + +Please note this expression will be run in the writer's goroutine but not the logging event goroutine. + +#### `FLAGS` + +`FLAGS` represents the preceding logging context information that is +printed before each message. It is a comma-separated string set. The order of values does not matter. + +It is default to `stdflags` (Equal to `date,time,medfile,shortfuncname,levelinitial`) + +Possible values are: + +- `none` or `,` - No flags. +- `date` - the date in the local time zone: `2009/01/23`. +- `time` - the time in the local time zone: `01:23:23`. +- `microseconds` - microsecond resolution: `01:23:23.123123`. Assumes time. +- `longfile` - full file name and line number: `/a/b/c/d.go:23`. +- `shortfile` - final file name element and line number: `d.go:23`. +- `funcname` - function name of the caller: `runtime.Caller()`. +- `shortfuncname` - last part of the function name. Overrides `funcname`. +- `utc` - if date or time is set, use UTC rather than the local time zone. +- `levelinitial` - Initial character of the provided level in brackets eg. `[I]` for info. +- `level` - Provided level in brackets `[INFO]`. +- `gopid` - The Goroutine-PID of the context. +- `medfile` - Last 20 characters of the filename - equivalent to `shortfile,longfile`. +- `stdflags` - Equivalent to `date,time,medfile,shortfuncname,levelinitial`. + +### Console mode + +In this mode the logger will forward log messages to the stdout and +stderr streams attached to the Gitea process. + +For loggers in console mode, `COLORIZE` will default to `true` if not +on windows, or the Windows terminal can be set into ANSI mode or is a +cygwin or Msys pipe. + +Settings: + +- `STDERR`: **false**: Whether the logger should print to `stderr` instead of `stdout`. + +### File mode + +In this mode the logger will save log messages to a file. + +Settings: + +- `FILE_NAME`: The file to write the log events to. Default to `%(ROOT_PATH)/gitea.log` if the section name is `log.file`, or `%(ROOT_PATH)/.log` otherwise. +- `MAX_SIZE_SHIFT`: **28**: Maximum size shift of a single file. 28 represents 256Mb. For details see below. +- `LOG_ROTATE` **true**: Whether to rotate the log files. TODO: if false, will it delete instead on daily rotate, or do nothing?. +- `DAILY_ROTATE`: **true**: Whether to rotate logs daily. +- `MAX_DAYS`: **7**: Delete rotated log files after this number of days. +- `COMPRESS`: **true**: Whether to compress old log files by default with gzip. +- `COMPRESSION_LEVEL`: **-1**: Compression level. For details see below. + +The default value of `FILE_NAME` depends on the respective logger facility. +If unset, their own default will be used. +If set it will be relative to the provided `ROOT_PATH` in the master `[log]` section. + +`MAX_SIZE_SHIFT` defines the maximum size of a file by left shifting 1 the given number of times (`1 << x`). +The exact behavior at the time of v1.17.3 can be seen [here](https://github.com/go-gitea/gitea/blob/v1.17.3/modules/setting/log.go#L185). + +The useful values of `COMPRESSION_LEVEL` are from 1 to (and including) 9, where higher numbers mean better compression. +Beware that better compression might come with higher resource usage. +Must be preceded with a `-` sign. + +### Conn mode + +In this mode the logger will send log messages over a network socket. + +Settings: + +- `ADDR`: **:7020**: Sets the address to connect to. +- `PROTOCOL`: **tcp**: Set the protocol, either "tcp", "unix" or "udp". +- `RECONNECT`: **false**: Try to reconnect when connection is lost. +- `RECONNECT_ON_MSG`: **false**: Reconnect host for every single message. + +### The "Router" logger + +The Router logger logs the following message types when Gitea's route handlers work: + +- `started` messages will be logged at TRACE level +- `polling`/`completed` routers will be logged at INFO +- `slow` routers will be logged at WARN +- `failed` routers will be logged at WARN + +### The "XORM" logger + +To make XORM outputs SQL logs, the `LOG_SQL` in `[database]` section should also be set to `true`. + +### The "Access" logger + +The Access logger is a new logger for version 1.9. It provides a NCSA +Common Log compliant log format. It's highly configurable but caution +should be taken when changing its template. The main benefit of this +logger is that Gitea can now log accesses in a standard log format so +standard tools may be used. + +You can enable this logger using `logger.ACCESS.MODE = ...`. + +If desired the format of the Access logger can be changed by changing +the value of the `ACCESS_LOG_TEMPLATE`. + +Please note, the access logger will log at `INFO` level, setting the +`LEVEL` of this logger to `WARN` or above will result in no access logs. + +#### The ACCESS_LOG_TEMPLATE + +This value represent a go template. It's default value is: + +``` +{{.Ctx.RemoteHost}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}" "{{.Ctx.Req.UserAgent}}"` +``` + +The template is passed following options: + +- `Ctx` is the `context.Context` +- `Identity` is the `SignedUserName` or `"-"` if the user is not logged in +- `Start` is the start time of the request +- `ResponseWriter` is the `http.ResponseWriter` + +Caution must be taken when changing this template as it runs outside of +the standard panic recovery trap. The template should also be as simple +as it runs for every request. + +## Releasing-and-Reopening, Pausing and Resuming logging + +If you are running on Unix you may wish to release-and-reopen logs in order to use `logrotate` or other tools. +It is possible force Gitea to release and reopen it's logging files and connections by sending `SIGUSR1` to the +running process, or running `gitea manager logging release-and-reopen`. + +Alternatively, you may wish to pause and resume logging - this can be accomplished through the use of the +`gitea manager logging pause` and `gitea manager logging resume` commands. Please note that whilst logging +is paused log events below INFO level will not be stored and only a limited number of events will be stored. +Logging may block, albeit temporarily, slowing Gitea considerably whilst paused - therefore it is +recommended that pausing only done for a very short period of time. + +## Adding and removing logging whilst Gitea is running + +It is possible to add and remove logging whilst Gitea is running using the `gitea manager logging add` and `remove` subcommands. +This functionality can only adjust running log systems and cannot be used to start the access or router loggers if they +were not already initialized. If you wish to start these systems you are advised to adjust the app.ini and (gracefully) restart +the Gitea service. + +The main intention of these commands is to easily add a temporary logger to investigate problems on running systems where a restart +may cause the issue to disappear. + +## Using `logrotate` instead of built-in log rotation + +Gitea includes built-in log rotation, which should be enough for most deployments. However, if you instead want to use the `logrotate` utility: + +- Disable built-in log rotation by setting `LOG_ROTATE` to `false` in your `app.ini`. +- Install `logrotate`. +- Configure `logrotate` to match your deployment requirements, see `man 8 logrotate` for configuration syntax details. + In the `postrotate/endscript` block send Gitea a `USR1` signal via `kill -USR1` or `kill -10` to the `gitea` process itself, + or run `gitea manager logging release-and-reopen` (with the appropriate environment). + Ensure that your configurations apply to all files emitted by Gitea loggers as described in the above sections. +- Always do `logrotate /etc/logrotate.conf --debug` to test your configurations. +- If you are using docker and are running from outside the container you can use + `docker exec -u $OS_USER $CONTAINER_NAME sh -c 'gitea manager logging release-and-reopen'` + or `docker exec $CONTAINER_NAME sh -c '/bin/s6-svc -1 /etc/s6/gitea/'` or send `USR1` directly to the Gitea process itself. + +The next `logrotate` jobs will include your configurations, so no restart is needed. +You can also immediately reload `logrotate` with `logrotate /etc/logrotate.conf --force`. diff --git a/docs/content/doc/administration/logging-documentation.en-us.md b/docs/content/doc/administration/logging-documentation.en-us.md deleted file mode 100644 index 3b2bf8076922..000000000000 --- a/docs/content/doc/administration/logging-documentation.en-us.md +++ /dev/null @@ -1,524 +0,0 @@ ---- -date: "2019-04-02T17:06:00+01:00" -title: "Logging Configuration" -slug: "logging-configuration" -weight: 40 -toc: false -draft: false -aliases: - - /en-us/logging-configuration -menu: - sidebar: - parent: "administration" - name: "Logging Configuration" - weight: 40 - identifier: "logging-configuration" ---- - -# Logging Configuration - -The logging configuration of Gitea mainly consists of 3 types of components: - -- The `[log]` section for general configuration -- `[log.]` sections for the configuration of different log outputs -- `[log..]` sections for output specific configuration of a log group - -As mentioned below, there is a fully functional log output by default, so it is not necessary to define one. - -**Table of Contents** - -{{< toc >}} - -## Collecting Logs for Help - -To collect logs for help and issue report, see [Support Options]({{< relref "doc/help/support.en-us.md" >}}). - -## The `[log]` section - -Configuration of logging facilities in Gitea happen in the `[log]` section and it's subsections. - -In the top level `[log]` section the following configurations can be placed: - -- `ROOT_PATH`: (Default: **%(GITEA_WORK_DIR)/log**): Base path for log files -- `MODE`: (Default: **console**) List of log outputs to use for the Default logger. -- `ROUTER`: (Default: **console**): List of log outputs to use for the Router logger. -- `ACCESS`: List of log outputs to use for the Access logger. -- `XORM`: (Default: **,**) List of log outputs to use for the XORM logger. -- `ENABLE_ACCESS_LOG`: (Default: **false**): whether the Access logger is allowed to emit logs -- `ENABLE_XORM_LOG`: (Default: **true**): whether the XORM logger is allowed to emit logs - -For details on the loggers check the "Log Groups" section. -Important: log outputs won't be used if you don't enable them for the desired loggers in the corresponding list value. - -Lists are specified as comma separated values. This format also works in subsection. - -This section may be used for defining default values for subsections. -Examples: - -- `LEVEL`: (Default: **Info**) Least severe log events to persist. Case insensitive. The full list of levels as of v1.17.3 can be read [here](https://github.com/go-gitea/gitea/blob/v1.17.3/custom/conf/app.example.ini#L507). -- `STACKTRACE_LEVEL`: (Default: **None**) For this and more severe events the stacktrace will be printed upon getting logged. - -Some values are not inherited by subsections. For details see the "Non-inherited default values" section. - -## Log outputs - -Log outputs are the targets to which log messages will be sent. -The content and the format of the log messages to be saved can be configured in these. - -Log outputs are also called subloggers. - -Gitea provides 4 possible log outputs: - -- `console` - Log to `os.Stdout` or `os.Stderr` -- `file` - Log to a file -- `conn` - Log to a socket (network or unix) -- `smtp` - Log via email - -By default, Gitea has a `console` output configured, which is used by the loggers as seen in the section "The log section" above. - -### Common configuration - -Certain configuration is common to all modes of log output: - -- `MODE` is the mode of the log output. It will default to the sublogger - name, thus `[log.console.router]` will default to `MODE = console`. - For mode specific confgurations read further. -- `LEVEL` is the lowest level that this output will log. This value - is inherited from `[log]` and in the case of the non-default loggers - from `[log.sublogger]`. -- `STACKTRACE_LEVEL` is the lowest level that this output will print - a stacktrace. This value is inherited. -- `COLORIZE` will default to `true` for `console` as - described, otherwise it will default to `false`. - -### Non-inherited default values - -There are several values which are not inherited as described above but -rather default to those specific to type of logger, these are: -`EXPRESSION`, `FLAGS`, `PREFIX` and `FILE_NAME`. - -#### `EXPRESSION` - -`EXPRESSION` represents a regular expression that log events must match to be logged by the sublogger. Either the log message, (with colors removed), must match or the `longfilename:linenumber:functionname` must match. NB: the whole message or string doesn't need to completely match. - -Please note this expression will be run in the sublogger's goroutine -not the logging event subroutine. Therefore it can be complicated. - -#### `FLAGS` - -`FLAGS` represents the preceding logging context information that is -printed before each message. It is a comma-separated string set. The order of values does not matter. - -Possible values are: - -- `none` or `,` - No flags. -- `date` - the date in the local time zone: `2009/01/23`. -- `time` - the time in the local time zone: `01:23:23`. -- `microseconds` - microsecond resolution: `01:23:23.123123`. Assumes - time. -- `longfile` - full file name and line number: `/a/b/c/d.go:23`. -- `shortfile` - final file name element and line number: `d.go:23`. -- `funcname` - function name of the caller: `runtime.Caller()`. -- `shortfuncname` - last part of the function name. Overrides - `funcname`. -- `utc` - if date or time is set, use UTC rather than the local time - zone. -- `levelinitial` - Initial character of the provided level in brackets eg. `[I]` for info. -- `level` - Provided level in brackets `[INFO]` -- `medfile` - Last 20 characters of the filename - equivalent to - `shortfile,longfile`. -- `stdflags` - Equivalent to `date,time,medfile,shortfuncname,levelinitial` - -### Console mode - -In this mode the logger will forward log messages to the stdout and -stderr streams attached to the Gitea process. - -For loggers in console mode, `COLORIZE` will default to `true` if not -on windows, or the windows terminal can be set into ANSI mode or is a -cygwin or Msys pipe. - -Settings: - -- `STDERR`: **false**: Whether the logger should print to `stderr` instead of `stdout`. - -### File mode - -In this mode the logger will save log messages to a file. - -Settings: - -- `FILE_NAME`: The file to write the log events to. For details see below. -- `MAX_SIZE_SHIFT`: **28**: Maximum size shift of a single file. 28 represents 256Mb. For details see below. -- `LOG_ROTATE` **true**: Whether to rotate the log files. TODO: if false, will it delete instead on daily rotate, or do nothing?. -- `DAILY_ROTATE`: **true**: Whether to rotate logs daily. -- `MAX_DAYS`: **7**: Delete rotated log files after this number of days. -- `COMPRESS`: **true**: Whether to compress old log files by default with gzip. -- `COMPRESSION_LEVEL`: **-1**: Compression level. For details see below. - -The default value of `FILE_NAME` depends on the respective logger facility. -If unset, their own default will be used. -If set it will be relative to the provided `ROOT_PATH` in the master `[log]` section. - -`MAX_SIZE_SHIFT` defines the maximum size of a file by left shifting 1 the given number of times (`1 << x`). -The exact behavior at the time of v1.17.3 can be seen [here](https://github.com/go-gitea/gitea/blob/v1.17.3/modules/setting/log.go#L185). - -The useful values of `COMPRESSION_LEVEL` are from 1 to (and including) 9, where higher numbers mean better compression. -Beware that better compression might come with higher resource usage. -Must be preceded with a `-` sign. - -### Conn mode - -In this mode the logger will send log messages over a network socket. - -Settings: - -- `ADDR`: **:7020**: Sets the address to connect to. -- `PROTOCOL`: **tcp**: Set the protocol, either "tcp", "unix" or "udp". -- `RECONNECT`: **false**: Try to reconnect when connection is lost. -- `RECONNECT_ON_MSG`: **false**: Reconnect host for every single message. - -### SMTP mode - -In this mode the logger will send log messages in email. - -It is not recommended to use this logger to send general logging -messages. However, you could perhaps set this logger to work on `FATAL` messages only. - -Settings: - -- `HOST`: **127.0.0.1:25**: The SMTP host to connect to. -- `USER`: User email address to send from. -- `PASSWD`: Password for the smtp server. -- `RECEIVERS`: Email addresses to send to. -- `SUBJECT`: **Diagnostic message from Gitea**. The content of the email's subject field. - -## Log Groups - -The fundamental thing to be aware of in Gitea is that there are several -log groups: - -- The "Default" logger -- The Router logger -- The Access logger -- The XORM logger - -There is also the go log logger. - -### The go log logger - -Go provides its own extremely basic logger in the `log` package, -however, this is not sufficient for our purposes as it does not provide -a way of logging at multiple levels, nor does it provide a good way of -controlling where these logs are logged except through setting of a -writer. - -We have therefore redirected this logger to our Default logger, and we -will log anything that is logged using the go logger at the INFO level. - -### The "Default" logger - -Calls to `log.Info`, `log.Debug`, `log.Error` etc. from the `code.gitea.io/gitea/modules/log` package will log to this logger. - -You can configure the outputs of this logger by setting the `MODE` -value in the `[log]` section of the configuration. - -Each output sublogger is configured in a separate `[log.sublogger.default]` -which inherits from the sublogger `[log.sublogger]` section and from the -generic `[log]` section, but there are certain default values. These will -not be inherited from the `[log]` section: - -- `FLAGS` is `stdflags` (Equal to - `date,time,medfile,shortfuncname,levelinitial`) -- `FILE_NAME` will default to `%(ROOT_PATH)/gitea.log` -- `EXPRESSION` will default to `""` -- `PREFIX` will default to `""` - -The provider type of the sublogger can be set using the `MODE` value in -its subsection, but will default to the name. This allows you to have -multiple subloggers that will log to files. - -### The "Router" logger - -The Router logger has been substantially changed in v1.17. If you are using the router logger for fail2ban or other monitoring -you will need to update this configuration. - -You can disable Router log by setting `DISABLE_ROUTER_LOG` or by setting all of its sublogger configurations to `none`. - -You can configure the outputs of this -router log by setting the `ROUTER` value in the `[log]` section of the -configuration. `ROUTER` will default to `console` if unset and will default to same level as main logger. - -The Router logger logs the following: - -- `started` messages will be logged at TRACE level -- `polling`/`completed` routers will be logged at INFO -- `slow` routers will be logged at WARN -- `failed` routers will be logged at WARN - -The logging level for the router will default to that of the main configuration. Set `[log..router]` `LEVEL` to change this. - -Each output sublogger for this logger is configured in -`[log.sublogger.router]` sections. There are certain default values -which will not be inherited from the `[log]` or relevant -`[log.sublogger]` sections: - -- `FILE_NAME` will default to `%(ROOT_PATH)/router.log` -- `FLAGS` defaults to `date,time` -- `EXPRESSION` will default to `""` -- `PREFIX` will default to `""` - -NB: You can redirect the router logger to send its events to the Gitea -log using the value: `ROUTER = ,` - -### The "Access" logger - -The Access logger is a new logger for version 1.9. It provides a NCSA -Common Log compliant log format. It's highly configurable but caution -should be taken when changing its template. The main benefit of this -logger is that Gitea can now log accesses in a standard log format so -standard tools may be used. - -You can enable this logger using `ENABLE_ACCESS_LOG`. Its outputs are -configured by setting the `ACCESS` value in the `[log]` section of the -configuration. `ACCESS` defaults to `file` if unset. - -Each output sublogger for this logger is configured in -`[log.sublogger.access]` sections. There are certain default values -which will not be inherited from the `[log]` or relevant -`[log.sublogger]` sections: - -- `FILE_NAME` will default to `%(ROOT_PATH)/access.log` -- `FLAGS` defaults to `` or None -- `EXPRESSION` will default to `""` -- `PREFIX` will default to `""` - -If desired the format of the Access logger can be changed by changing -the value of the `ACCESS_LOG_TEMPLATE`. - -Please note, the access logger will log at `INFO` level, setting the -`LEVEL` of this logger to `WARN` or above will result in no access logs. - -NB: You can redirect the access logger to send its events to the Gitea -log using the value: `ACCESS = ,` - -#### The ACCESS_LOG_TEMPLATE - -This value represent a go template. It's default value is: - -`{{.Ctx.RemoteHost}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}" "{{.Ctx.Req.UserAgent}}"` - -The template is passed following options: - -- `Ctx` is the `context.Context` -- `Identity` is the `SignedUserName` or `"-"` if the user is not logged - in -- `Start` is the start time of the request -- `ResponseWriter` is the `http.ResponseWriter` - -Caution must be taken when changing this template as it runs outside of -the standard panic recovery trap. The template should also be as simple -as it runs for every request. - -### The "XORM" logger - -The XORM logger is a long-standing logger that exists to collect XORM -log events. It is enabled by default but can be switched off by setting -`ENABLE_XORM_LOG` to `false` in the `[log]` section. Its outputs are -configured by setting the `XORM` value in the `[log]` section of the -configuration. `XORM` defaults to `,` if unset, meaning it is redirected -to the main Gitea log. - -XORM will log SQL events by default. This can be changed by setting -the `LOG_SQL` value to `false` in the `[database]` section. - -Each output sublogger for this logger is configured in -`[log.sublogger.xorm]` sections. There are certain default values -which will not be inherited from the `[log]` or relevant -`[log.sublogger]` sections: - -- `FILE_NAME` will default to `%(ROOT_PATH)/xorm.log` -- `FLAGS` defaults to `date,time` -- `EXPRESSION` will default to `""` -- `PREFIX` will default to `""` - -## Debugging problems - -When submitting logs in Gitea issues it is often helpful to submit -merged logs obtained by either by redirecting the console log to a file or -copying and pasting it. To that end it is recommended to set your logging to: - -```ini -[database] -LOG_SQL = false ; SQL logs are rarely helpful unless we specifically ask for them - -... - -[log] -MODE = console -LEVEL = debug ; please set the level to debug when we are debugging a problem -ROUTER = console -COLORIZE = false ; this can be true if you can strip out the ansi coloring -ENABLE_SSH_LOG = true ; shows logs related to git over SSH. -``` - -Sometimes it will be helpful get some specific `TRACE` level logging restricted -to messages that match a specific `EXPRESSION`. Adjusting the `MODE` in the -`[log]` section to `MODE = console,traceconsole` to add a new logger output -`traceconsole` and then adding its corresponding section would be helpful: - -```ini -[log.traceconsole] ; traceconsole here is just a name -MODE = console ; this is the output that the traceconsole writes to -LEVEL = trace -EXPRESSION = ; putting a string here will restrict this logger to logging only those messages that match this expression -``` - -(It's worth noting that log messages that match the expression at or above debug -level will get logged twice so don't worry about that.) - -`STACKTRACE_LEVEL` should generally be left unconfigured (and hence kept at -`none`). There are only very specific occasions when it useful. - -## Empty Configuration - -The empty configuration is equivalent to: - -```ini -[log] -ROOT_PATH = %(GITEA_WORK_DIR)/log -MODE = console -LEVEL = Info -STACKTRACE_LEVEL = None -ENABLE_ACCESS_LOG = false -ENABLE_XORM_LOG = true -XORM = , - -[log.console] -MODE = console -LEVEL = %(LEVEL) -STACKTRACE_LEVEL = %(STACKTRACE_LEVEL) -FLAGS = stdflags -PREFIX = -COLORIZE = true # Or false if your windows terminal cannot color -``` - -This is equivalent to sending all logs to the console, with default go log being sent to the console log too. - -## Releasing-and-Reopening, Pausing and Resuming logging - -If you are running on Unix you may wish to release-and-reopen logs in order to use `logrotate` or other tools. -It is possible force Gitea to release and reopen it's logging files and connections by sending `SIGUSR1` to the -running process, or running `gitea manager logging release-and-reopen`. - -Alternatively, you may wish to pause and resume logging - this can be accomplished through the use of the -`gitea manager logging pause` and `gitea manager logging resume` commands. Please note that whilst logging -is paused log events below INFO level will not be stored and only a limited number of events will be stored. -Logging may block, albeit temporarily, slowing Gitea considerably whilst paused - therefore it is -recommended that pausing only done for a very short period of time. - -## Adding and removing logging whilst Gitea is running - -It is possible to add and remove logging whilst Gitea is running using the `gitea manager logging add` and `remove` subcommands. -This functionality can only adjust running log systems and cannot be used to start the access or router loggers if they -were not already initialized. If you wish to start these systems you are advised to adjust the app.ini and (gracefully) restart -the Gitea service. - -The main intention of these commands is to easily add a temporary logger to investigate problems on running systems where a restart -may cause the issue to disappear. - -## Log colorization - -Logs to the console will be colorized by default when not running on -Windows. Terminal sniffing will occur on Windows and if it is -determined that we are running on a terminal capable of color we will -colorize. - -Further, on \*nix it is becoming common to have file logs that are -colored by default. Therefore file logs will be colorised by default -when not running on Windows. - -You can switch on or off colorization by using the `COLORIZE` value. - -From a development point of view. If you write -`log.Info("A %s string", "formatted")` the `formatted` part of the log -message will be Bolded on colorized logs. - -You can change this by either rendering the formatted string yourself. -Or you can wrap the value in a `log.ColoredValue` struct. - -The `log.ColoredValue` struct contains a pointer to value, a pointer to -string of bytes which should represent a color and second set of reset -bytes. Pointers were chosen to prevent copying of large numbers of -values. There are several helper methods: - -- `log.NewColoredValue` takes a value and 0 or more color attributes - that represent the color. If 0 are provided it will default to a cached - bold. Note, it is recommended that color bytes constructed from - attributes should be cached if this is a commonly used log message. -- `log.NewColoredValuePointer` takes a pointer to a value, and - 0 or more color attributes that represent the color. -- `log.NewColoredValueBytes` takes a value and a pointer to an array - of bytes representing the color. - -These functions will not double wrap a `log.ColoredValue`. They will -also set the `resetBytes` to the cached `resetBytes`. - -The `colorBytes` and `resetBytes` are not exported to prevent -accidental overwriting of internal values. - -## ColorFormat & ColorFormatted - -Structs may implement the `log.ColorFormatted` interface by implementing the `ColorFormat(fmt.State)` function. - -If a `log.ColorFormatted` struct is logged with `%-v` format, its `ColorFormat` will be used instead of the usual `%v`. The full `fmt.State` will be passed to allow implementers to look at additional flags. - -In order to help implementers provide `ColorFormat` methods. There is a -`log.ColorFprintf(...)` function in the log module that will wrap values in `log.ColoredValue` and recognise `%-v`. - -In general it is recommended not to make the results of this function too verbose to help increase its versatility. Usually this should simply be an `ID`:`Name`. If you wish to make a more verbose result, it is recommended to use `%-+v` as your marker. - -## Log Spoofing protection - -In order to protect the logs from being spoofed with cleverly -constructed messages. Newlines are now prefixed with a tab and control -characters except those used in an ANSI CSI are escaped with a -preceding `\` and their octal value. - -## Creating a new named logger group - -Should a developer wish to create a new named logger, `NEWONE`. It is -recommended to add an `ENABLE_NEWONE_LOG` value to the `[log]` -section, and to add a new `NEWONE` value for the modes. - -A function like `func newNewOneLogService()` is recommended to manage -construction of the named logger. e.g. - -```go -func newNewoneLogService() { - EnableNewoneLog = Cfg.Section("log").Key("ENABLE_NEWONE_LOG").MustBool(false) - Cfg.Section("log").Key("NEWONE").MustString("file") // or console? or "," if you want to send this to default logger by default - if EnableNewoneLog { - options := newDefaultLogOptions() - options.filename = filepath.Join(LogRootPath, "newone.log") - options.flags = "stdflags" - options.bufferLength = Cfg.Section("log").Key("BUFFER_LEN").MustInt64(10000) - generateNamedLogger("newone", options) - } -} -``` - -You should then add `newOneLogService` to `NewServices()` in -`modules/setting/setting.go` - -## Using `logrotate` instead of built-in log rotation - -Gitea includes built-in log rotation, which should be enough for most deployments. However, if you instead want to use the `logrotate` utility: - -- Disable built-in log rotation by setting `LOG_ROTATE` to `false` in your `app.ini`. -- Install `logrotate`. -- Configure `logrotate` to match your deployment requirements, see `man 8 logrotate` for configuration syntax details. In the `postrotate/endscript` block send Gitea a `USR1` signal via `kill -USR1` or `kill -10` to the `gitea` process itself, or run `gitea manager logging release-and-reopen` (with the appropriate environment). Ensure that your configurations apply to all files emitted by Gitea loggers as described in the above sections. -- Always do `logrotate /etc/logrotate.conf --debug` to test your configurations. -- If you are using docker and are running from outside of the container you can use `docker exec -u $OS_USER $CONTAINER_NAME sh -c 'gitea manager logging release-and-reopen'` or `docker exec $CONTAINER_NAME sh -c '/bin/s6-svc -1 /etc/s6/gitea/'` or send `USR1` directly to the Gitea process itself. - -The next `logrotate` jobs will include your configurations, so no restart is needed. You can also immediately reload `logrotate` with `logrotate /etc/logrotate.conf --force`. diff --git a/main.go b/main.go index 1589fa97db43..49093eb8a703 100644 --- a/main.go +++ b/main.go @@ -120,6 +120,8 @@ arguments - which can alternatively be run by running the subcommand web.` if err != nil { log.Fatal("Failed to run app with %s: %v", os.Args, err) } + + log.GetManager().Close() } func setFlagsAndBeforeOnSubcommands(command *cli.Command, defaultFlags []cli.Flag, before cli.BeforeFunc) { diff --git a/models/db/log.go b/models/db/log.go index fec2ea3c3d38..dd95f64ca80b 100644 --- a/models/db/log.go +++ b/models/db/log.go @@ -14,67 +14,62 @@ import ( // XORMLogBridge a logger bridge from Logger to xorm type XORMLogBridge struct { - showSQLint *int32 - logger log.Logger + showSQL atomic.Bool + logger log.Logger } // NewXORMLogger inits a log bridge for xorm func NewXORMLogger(showSQL bool) xormlog.Logger { - showSQLint := int32(0) - if showSQL { - showSQLint = 1 - } - return &XORMLogBridge{ - showSQLint: &showSQLint, - logger: log.GetLogger("xorm"), - } + l := &XORMLogBridge{logger: log.GetLogger("xorm")} + l.showSQL.Store(showSQL) + return l } const stackLevel = 8 // Log a message with defined skip and at logging level -func (l *XORMLogBridge) Log(skip int, level log.Level, format string, v ...interface{}) error { - return l.logger.Log(skip+1, level, format, v...) +func (l *XORMLogBridge) Log(skip int, level log.Level, format string, v ...interface{}) { + l.logger.Log(skip+1, level, format, v...) } // Debug show debug log func (l *XORMLogBridge) Debug(v ...interface{}) { - _ = l.Log(stackLevel, log.DEBUG, fmt.Sprint(v...)) + l.Log(stackLevel, log.DEBUG, "%s", fmt.Sprint(v...)) } // Debugf show debug log func (l *XORMLogBridge) Debugf(format string, v ...interface{}) { - _ = l.Log(stackLevel, log.DEBUG, format, v...) + l.Log(stackLevel, log.DEBUG, format, v...) } // Error show error log func (l *XORMLogBridge) Error(v ...interface{}) { - _ = l.Log(stackLevel, log.ERROR, fmt.Sprint(v...)) + l.Log(stackLevel, log.ERROR, "%s", fmt.Sprint(v...)) } // Errorf show error log func (l *XORMLogBridge) Errorf(format string, v ...interface{}) { - _ = l.Log(stackLevel, log.ERROR, format, v...) + l.Log(stackLevel, log.ERROR, format, v...) } // Info show information level log func (l *XORMLogBridge) Info(v ...interface{}) { - _ = l.Log(stackLevel, log.INFO, fmt.Sprint(v...)) + l.Log(stackLevel, log.INFO, "%s", fmt.Sprint(v...)) } // Infof show information level log func (l *XORMLogBridge) Infof(format string, v ...interface{}) { - _ = l.Log(stackLevel, log.INFO, format, v...) + l.Log(stackLevel, log.INFO, format, v...) } // Warn show warning log func (l *XORMLogBridge) Warn(v ...interface{}) { - _ = l.Log(stackLevel, log.WARN, fmt.Sprint(v...)) + l.Log(stackLevel, log.WARN, "%s", fmt.Sprint(v...)) } // Warnf show warnning log func (l *XORMLogBridge) Warnf(format string, v ...interface{}) { - _ = l.Log(stackLevel, log.WARN, format, v...) + l.Log(stackLevel, log.WARN, format, v...) } // Level get logger level @@ -86,10 +81,12 @@ func (l *XORMLogBridge) Level() xormlog.LogLevel { return xormlog.LOG_INFO case log.WARN: return xormlog.LOG_WARNING - case log.ERROR, log.CRITICAL: + case log.ERROR: return xormlog.LOG_ERR + case log.NONE: + return xormlog.LOG_OFF } - return xormlog.LOG_OFF + return xormlog.LOG_UNKNOWN } // SetLevel set the logger level @@ -98,16 +95,13 @@ func (l *XORMLogBridge) SetLevel(lvl xormlog.LogLevel) { // ShowSQL set if record SQL func (l *XORMLogBridge) ShowSQL(show ...bool) { - showSQL := int32(1) - if len(show) > 0 && !show[0] { - showSQL = 0 + if len(show) == 0 { + show = []bool{true} } - atomic.StoreInt32(l.showSQLint, showSQL) + l.showSQL.Store(show[0]) } // IsShowSQL if record SQL func (l *XORMLogBridge) IsShowSQL() bool { - showSQL := atomic.LoadInt32(l.showSQLint) - - return showSQL == 1 + return l.showSQL.Load() } diff --git a/models/issues/pull.go b/models/issues/pull.go index 855d37867d81..218a265741e2 100644 --- a/models/issues/pull.go +++ b/models/issues/pull.go @@ -224,40 +224,27 @@ func DeletePullsByBaseRepoID(ctx context.Context, repoID int64) error { return err } -// ColorFormat writes a colored string to identify this struct -func (pr *PullRequest) ColorFormat(s fmt.State) { +func (pr *PullRequest) String() string { if pr == nil { - log.ColorFprintf(s, "PR[%d]%s#%d[%s...%s:%s]", - log.NewColoredIDValue(0), - log.NewColoredValue("/"), - log.NewColoredIDValue(0), - log.NewColoredValue(""), - log.NewColoredValue("/"), - log.NewColoredValue(""), - ) - return - } - - log.ColorFprintf(s, "PR[%d]", log.NewColoredIDValue(pr.ID)) + return "" + } + + s := new(strings.Builder) + fmt.Fprintf(s, "') + return s.String() } // MustHeadUserName returns the HeadRepo's username if failed return blank diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index 08eded7fdc02..dd99a1eda280 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -24,6 +24,8 @@ import ( "xorm.io/xorm" ) +// FIXME: this file shouldn't be in a normal package, it should only be compiled for tests + // PrepareTestEnv prepares the test environment and reset the database. The skip parameter should usually be 0. // Provide models to be sync'd with the database - in particular any models you expect fixtures to be loaded from. // @@ -110,7 +112,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En } func MainTest(m *testing.M) { - log.Register("test", testlogger.NewTestLogger) + log.RegisterEventWriter("test", testlogger.NewTestLoggerWriter) giteaRoot := base.SetupGiteaRoot() if giteaRoot == "" { @@ -154,7 +156,7 @@ func MainTest(m *testing.M) { os.Exit(1) } setting.LoadDBSetting() - setting.InitLogs(true) + setting.InitLoggersForTest() exitStatus := m.Run() diff --git a/models/organization/team.go b/models/organization/team.go index 5e3c9ecffe22..3624fdc46663 100644 --- a/models/organization/team.go +++ b/models/organization/team.go @@ -94,21 +94,11 @@ func init() { db.RegisterModel(new(TeamInvite)) } -// ColorFormat provides a basic color format for a Team -func (t *Team) ColorFormat(s fmt.State) { +func (t *Team) LogString() string { if t == nil { - log.ColorFprintf(s, "%d:%s (OrgID: %d) %-v", - log.NewColoredIDValue(0), - "", - log.NewColoredIDValue(0), - 0) - return + return "" } - log.ColorFprintf(s, "%d:%s (OrgID: %d) %-v", - log.NewColoredIDValue(t.ID), - t.Name, - log.NewColoredIDValue(t.OrgID), - t.AccessMode) + return fmt.Sprintf("", t.ID, t.Name, t.OrgID, t.AccessMode.LogString()) } // LoadUnits load a list of available units for a team diff --git a/models/perm/access/repo_permission.go b/models/perm/access/repo_permission.go index ee76e482001a..64df5355bba6 100644 --- a/models/perm/access/repo_permission.go +++ b/models/perm/access/repo_permission.go @@ -102,45 +102,28 @@ func (p *Permission) CanWriteIssuesOrPulls(isPull bool) bool { return p.CanWrite(unit.TypeIssues) } -// ColorFormat writes a colored string for these Permissions -func (p *Permission) ColorFormat(s fmt.State) { - noColor := log.ColorBytes(log.Reset) - - format := "perm_model.AccessMode: %-v, %d Units, %d UnitsMode(s): [ " - args := []interface{}{ - p.AccessMode, - log.NewColoredValueBytes(len(p.Units), &noColor), - log.NewColoredValueBytes(len(p.UnitsMode), &noColor), - } - if s.Flag('+') { - for i, unit := range p.Units { - config := "" - if unit.Config != nil { - configBytes, err := unit.Config.ToDB() - config = string(configBytes) - if err != nil { - config = err.Error() - } +func (p *Permission) LogString() string { + format := "", mode, mode.String()) } // ParseAccessMode returns corresponding access mode to given permission string. diff --git a/models/repo/repo.go b/models/repo/repo.go index 7cbd5867b7fb..ffa108a0555f 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -196,19 +196,11 @@ func (repo *Repository) SanitizedOriginalURL() string { return u.String() } -// ColorFormat returns a colored string to represent this repo -func (repo *Repository) ColorFormat(s fmt.State) { +func (repo *Repository) LogString() string { if repo == nil { - log.ColorFprintf(s, "%d:%s/%s", - log.NewColoredIDValue(0), - "", - "") - return - } - log.ColorFprintf(s, "%d:%s/%s", - log.NewColoredIDValue(repo.ID), - repo.OwnerName, - repo.Name) + return "" + } + return fmt.Sprintf("", repo.ID, repo.OwnerName, repo.Name) } // IsBeingMigrated indicates that repository is being migrated diff --git a/models/unit/unit.go b/models/unit/unit.go index 7cd679116f1b..5f5e20de1e0e 100644 --- a/models/unit/unit.go +++ b/models/unit/unit.go @@ -62,11 +62,8 @@ func (u Type) String() string { return fmt.Sprintf("Unknown Type %d", u) } -// ColorFormat provides a ColorFormatted version of this Type -func (u Type) ColorFormat(s fmt.State) { - log.ColorFprintf(s, "%d:%s", - log.NewColoredIDValue(u), - u) +func (u Type) LogString() string { + return fmt.Sprintf("", u, u.String()) } var ( diff --git a/models/user/user.go b/models/user/user.go index 46c4440e5f07..fa4b848a5fec 100644 --- a/models/user/user.go +++ b/models/user/user.go @@ -152,17 +152,11 @@ type SearchOrganizationsOptions struct { All bool } -// ColorFormat writes a colored string to identify this struct -func (u *User) ColorFormat(s fmt.State) { +func (u *User) LogString() string { if u == nil { - log.ColorFprintf(s, "%d:%s", - log.NewColoredIDValue(0), - log.NewColoredValue("")) - return - } - log.ColorFprintf(s, "%d:%s", - log.NewColoredIDValue(u.ID), - log.NewColoredValue(u.Name)) + return "" + } + return fmt.Sprintf("", u.ID, u.Name) } // BeforeUpdate is invoked from XORM before updating this object. diff --git a/modules/context/access_log.go b/modules/context/access_log.go index b6468d139bf9..9b649a6a0133 100644 --- a/modules/context/access_log.go +++ b/modules/context/access_log.go @@ -95,10 +95,7 @@ func AccessLogger() func(http.Handler) http.Handler { log.Error("Could not set up chi access logger: %v", err.Error()) } - err = logger.SendLog(log.INFO, "", "", 0, buf.String(), "") - if err != nil { - log.Error("Could not set up chi access logger: %v", err.Error()) - } + logger.Info("%s", buf.String()) }) } } diff --git a/modules/doctor/doctor.go b/modules/doctor/doctor.go index 32eb5938c307..10838a751217 100644 --- a/modules/doctor/doctor.go +++ b/modules/doctor/doctor.go @@ -6,6 +6,7 @@ package doctor import ( "context" "fmt" + "os" "sort" "strings" @@ -26,27 +27,9 @@ type Check struct { Priority int } -type wrappedLevelLogger struct { - log.LevelLogger -} - -func (w *wrappedLevelLogger) Log(skip int, level log.Level, format string, v ...interface{}) error { - return w.LevelLogger.Log( - skip+1, - level, - " - %s "+format, - append( - []interface{}{ - log.NewColoredValueBytes( - fmt.Sprintf("[%s]", strings.ToUpper(level.String()[0:1])), - level.Color()), - }, v...)...) -} - -func initDBDisableConsole(ctx context.Context, disableConsole bool) error { +func initDBSkipLogger(ctx context.Context) error { setting.Init(&setting.Options{}) setting.LoadDBSetting() - setting.InitSQLLog(disableConsole) if err := db.InitEngine(ctx); err != nil { return fmt.Errorf("db.InitEngine: %w", err) } @@ -57,30 +40,61 @@ func initDBDisableConsole(ctx context.Context, disableConsole bool) error { return nil } +type doctorCheckLogger struct { + colorize bool +} + +var _ log.BaseLogger = (*doctorCheckLogger)(nil) + +func (d *doctorCheckLogger) Log(skip int, level log.Level, format string, v ...any) { + _, _ = fmt.Fprintf(os.Stdout, format+"\n", v...) +} + +func (d *doctorCheckLogger) GetLevel() log.Level { + return log.TRACE +} + +type doctorCheckStepLogger struct { + colorize bool +} + +var _ log.BaseLogger = (*doctorCheckStepLogger)(nil) + +func (d *doctorCheckStepLogger) Log(skip int, level log.Level, format string, v ...any) { + levelChar := fmt.Sprintf("[%s]", strings.ToUpper(level.String()[0:1])) + var levelArg any = levelChar + if d.colorize { + levelArg = log.NewColoredValue(levelChar, level.ColorAttributes()...) + } + args := append([]any{levelArg}, v...) + _, _ = fmt.Fprintf(os.Stdout, " - %s "+format+"\n", args...) +} + +func (d *doctorCheckStepLogger) GetLevel() log.Level { + return log.TRACE +} + // Checks is the list of available commands var Checks []*Check // RunChecks runs the doctor checks for the provided list -func RunChecks(ctx context.Context, logger log.Logger, autofix bool, checks []*Check) error { - wrappedLogger := log.LevelLoggerLogger{ - LevelLogger: &wrappedLevelLogger{logger}, - } - +func RunChecks(ctx context.Context, colorize, autofix bool, checks []*Check) error { + // the checks output logs by a special logger, they do not use the default logger + logger := log.BaseLoggerToGeneralLogger(&doctorCheckLogger{colorize: colorize}) + loggerStep := log.BaseLoggerToGeneralLogger(&doctorCheckStepLogger{colorize: colorize}) dbIsInit := false for i, check := range checks { if !dbIsInit && !check.SkipDatabaseInitialization { // Only open database after the most basic configuration check - setting.Log.EnableXORMLog = false - if err := initDBDisableConsole(ctx, true); err != nil { + if err := initDBSkipLogger(ctx); err != nil { logger.Error("Error whilst initializing the database: %v", err) logger.Error("Check if you are using the right config file. You can use a --config directive to specify one.") return nil } dbIsInit = true } - logger.Info("[%d] %s", log.NewColoredIDValue(i+1), check.Title) - logger.Flush() - if err := check.Run(ctx, &wrappedLogger, autofix); err != nil { + logger.Info("\n[%d] %s", i+1, check.Title) + if err := check.Run(ctx, loggerStep, autofix); err != nil { if check.AbortIfFailed { logger.Critical("FAIL") return err @@ -88,9 +102,9 @@ func RunChecks(ctx context.Context, logger log.Logger, autofix bool, checks []*C logger.Error("ERROR") } else { logger.Info("OK") - logger.Flush() } } + logger.Info("\nAll done.") return nil } diff --git a/modules/git/git_test.go b/modules/git/git_test.go index e3bfe496da70..25eb3085319d 100644 --- a/modules/git/git_test.go +++ b/modules/git/git_test.go @@ -10,7 +10,6 @@ import ( "strings" "testing" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -18,8 +17,6 @@ import ( ) func testRun(m *testing.M) error { - _ = log.NewLogger(1000, "console", "console", `{"level":"trace","stacktracelevel":"NONE","stderr":true}`) - gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") if err != nil { return fmt.Errorf("unable to create temp dir: %w", err) diff --git a/modules/graceful/manager.go b/modules/graceful/manager.go index c7b4c101ef28..d32788092d4f 100644 --- a/modules/graceful/manager.go +++ b/modules/graceful/manager.go @@ -30,7 +30,7 @@ const ( // * HTTP redirection fallback // * Builtin SSH listener // -// If you add an additional place you must increment this number +// If you add a new place you must increment this number // and add a function to call manager.InformCleanup if it's not going to be used const numberOfServersToCreate = 4 diff --git a/modules/graceful/manager_unix.go b/modules/graceful/manager_unix.go index ca6ccc1b6648..edd9b9f74762 100644 --- a/modules/graceful/manager_unix.go +++ b/modules/graceful/manager_unix.go @@ -15,6 +15,7 @@ import ( "syscall" "time" + "code.gitea.io/gitea/modules/graceful/releasereopen" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" @@ -136,7 +137,7 @@ func (g *Manager) handleSignals(ctx context.Context) { g.DoGracefulRestart() case syscall.SIGUSR1: log.Warn("PID %d. Received SIGUSR1. Releasing and reopening logs", pid) - if err := log.ReleaseReopen(); err != nil { + if err := releasereopen.GetManager().ReleaseReopen(); err != nil { log.Error("Error whilst releasing and reopening logs: %v", err) } case syscall.SIGUSR2: diff --git a/modules/graceful/releasereopen/releasereopen.go b/modules/graceful/releasereopen/releasereopen.go new file mode 100644 index 000000000000..36c02ab81747 --- /dev/null +++ b/modules/graceful/releasereopen/releasereopen.go @@ -0,0 +1,60 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package releasereopen + +import ( + "errors" + "sync" +) + +type ReleaseReopener interface { + ReleaseReopen() error +} + +type Manager struct { + mu sync.Mutex + counter int64 + + releaseReopeners map[int64]ReleaseReopener +} + +func (r *Manager) Register(rr ReleaseReopener) (cancel func()) { + r.mu.Lock() + defer r.mu.Unlock() + + r.counter++ + r.releaseReopeners[r.counter] = rr + + return func() { + r.mu.Lock() + defer r.mu.Unlock() + + delete(r.releaseReopeners, r.counter) + } +} + +func (r *Manager) ReleaseReopen() error { + r.mu.Lock() + defer r.mu.Unlock() + + var errs []error + for _, rr := range r.releaseReopeners { + if err := rr.ReleaseReopen(); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func GetManager() *Manager { + return manager +} + +var manager *Manager + +func init() { + manager = &Manager{ + releaseReopeners: make(map[int64]ReleaseReopener), + } +} diff --git a/modules/indexer/code/elastic_search.go b/modules/indexer/code/elastic_search.go index 609753800910..0e56a865880e 100644 --- a/modules/indexer/code/elastic_search.go +++ b/modules/indexer/code/elastic_search.go @@ -49,14 +49,6 @@ type ElasticSearchIndexer struct { lock sync.RWMutex } -type elasticLogger struct { - log.Logger -} - -func (l elasticLogger) Printf(format string, args ...interface{}) { - _ = l.Logger.Log(2, l.Logger.GetLevel(), format, args...) -} - // NewElasticSearchIndexer creates a new elasticsearch indexer func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, bool, error) { opts := []elastic.ClientOptionFunc{ @@ -66,15 +58,11 @@ func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, bo elastic.SetGzip(false), } - logger := elasticLogger{log.GetLogger(log.DEFAULT)} + logger := log.GetLogger(log.DEFAULT) - if logger.GetLevel() == log.TRACE || logger.GetLevel() == log.DEBUG { - opts = append(opts, elastic.SetTraceLog(logger)) - } else if logger.GetLevel() == log.ERROR || logger.GetLevel() == log.CRITICAL || logger.GetLevel() == log.FATAL { - opts = append(opts, elastic.SetErrorLog(logger)) - } else if logger.GetLevel() == log.INFO || logger.GetLevel() == log.WARN { - opts = append(opts, elastic.SetInfoLog(logger)) - } + opts = append(opts, elastic.SetTraceLog(&log.PrintfLogger{Logf: logger.Trace})) + opts = append(opts, elastic.SetInfoLog(&log.PrintfLogger{Logf: logger.Info})) + opts = append(opts, elastic.SetErrorLog(&log.PrintfLogger{Logf: logger.Error})) client, err := elastic.NewClient(opts...) if err != nil { diff --git a/modules/indexer/issues/elastic_search.go b/modules/indexer/issues/elastic_search.go index fd1dd4b45232..ec62f857adac 100644 --- a/modules/indexer/issues/elastic_search.go +++ b/modules/indexer/issues/elastic_search.go @@ -29,14 +29,6 @@ type ElasticSearchIndexer struct { lock sync.RWMutex } -type elasticLogger struct { - log.LevelLogger -} - -func (l elasticLogger) Printf(format string, args ...interface{}) { - _ = l.Log(2, l.GetLevel(), format, args...) -} - // NewElasticSearchIndexer creates a new elasticsearch indexer func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, error) { opts := []elastic.ClientOptionFunc{ @@ -46,15 +38,10 @@ func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, er elastic.SetGzip(false), } - logger := elasticLogger{log.GetLogger(log.DEFAULT)} - - if logger.GetLevel() == log.TRACE || logger.GetLevel() == log.DEBUG { - opts = append(opts, elastic.SetTraceLog(logger)) - } else if logger.GetLevel() == log.ERROR || logger.GetLevel() == log.CRITICAL || logger.GetLevel() == log.FATAL { - opts = append(opts, elastic.SetErrorLog(logger)) - } else if logger.GetLevel() == log.INFO || logger.GetLevel() == log.WARN { - opts = append(opts, elastic.SetInfoLog(logger)) - } + logger := log.GetLogger(log.DEFAULT) + opts = append(opts, elastic.SetTraceLog(&log.PrintfLogger{Logf: logger.Trace})) + opts = append(opts, elastic.SetInfoLog(&log.PrintfLogger{Logf: logger.Info})) + opts = append(opts, elastic.SetErrorLog(&log.PrintfLogger{Logf: logger.Error})) client, err := elastic.NewClient(opts...) if err != nil { diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go index f7f225bf1c1e..649c81a0cfcd 100644 --- a/modules/lfs/pointer.go +++ b/modules/lfs/pointer.go @@ -13,8 +13,6 @@ import ( "strconv" "strings" - "code.gitea.io/gitea/modules/log" - "github.com/minio/sha256-simd" ) @@ -113,15 +111,11 @@ func (p Pointer) RelativePath() string { return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:]) } -// ColorFormat provides a basic color format for a Team -func (p Pointer) ColorFormat(s fmt.State) { +func (p Pointer) LogString() string { if p.Oid == "" && p.Size == 0 { - log.ColorFprintf(s, "") - return + return "" } - log.ColorFprintf(s, "%s:%d", - log.NewColoredIDValue(p.Oid), - p.Size) + return fmt.Sprintf("", p.Oid, p.Size) } // GeneratePointer generates a pointer for arbitrary content diff --git a/modules/log/color.go b/modules/log/color.go new file mode 100644 index 000000000000..dcbba5f6d65d --- /dev/null +++ b/modules/log/color.go @@ -0,0 +1,115 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "fmt" + "strconv" +) + +const escape = "\033" + +// ColorAttribute defines a single SGR Code +type ColorAttribute int + +// Base ColorAttributes +const ( + Reset ColorAttribute = iota + Bold + Faint + Italic + Underline + BlinkSlow + BlinkRapid + ReverseVideo + Concealed + CrossedOut +) + +// Foreground text colors +const ( + FgBlack ColorAttribute = iota + 30 + FgRed + FgGreen + FgYellow + FgBlue + FgMagenta + FgCyan + FgWhite +) + +// Foreground Hi-Intensity text colors +const ( + FgHiBlack ColorAttribute = iota + 90 + FgHiRed + FgHiGreen + FgHiYellow + FgHiBlue + FgHiMagenta + FgHiCyan + FgHiWhite +) + +// Background text colors +const ( + BgBlack ColorAttribute = iota + 40 + BgRed + BgGreen + BgYellow + BgBlue + BgMagenta + BgCyan + BgWhite +) + +// Background Hi-Intensity text colors +const ( + BgHiBlack ColorAttribute = iota + 100 + BgHiRed + BgHiGreen + BgHiYellow + BgHiBlue + BgHiMagenta + BgHiCyan + BgHiWhite +) + +var ( + resetBytes = ColorBytes(Reset) + fgCyanBytes = ColorBytes(FgCyan) + fgGreenBytes = ColorBytes(FgGreen) +) + +type ColoredValue struct { + v any + colors []ColorAttribute +} + +func (c *ColoredValue) Format(f fmt.State, verb rune) { + _, _ = f.Write(ColorBytes(c.colors...)) + s := fmt.Sprintf(fmt.FormatString(f, verb), c.v) + _, _ = f.Write([]byte(s)) + _, _ = f.Write(resetBytes) +} + +func NewColoredValue(v any, color ...ColorAttribute) *ColoredValue { + return &ColoredValue{v: v, colors: color} +} + +// ColorBytes converts a list of ColorAttributes to a byte array +func ColorBytes(attrs ...ColorAttribute) []byte { + bytes := make([]byte, 0, 20) + bytes = append(bytes, escape[0], '[') + if len(attrs) > 0 { + bytes = append(bytes, strconv.Itoa(int(attrs[0]))...) + for _, a := range attrs[1:] { + bytes = append(bytes, ';') + bytes = append(bytes, strconv.Itoa(int(a))...) + } + } else { + bytes = append(bytes, strconv.Itoa(int(Bold))...) + } + bytes = append(bytes, 'm') + return bytes +} diff --git a/modules/log/color_console.go b/modules/log/color_console.go new file mode 100644 index 000000000000..1a08bbdc41c5 --- /dev/null +++ b/modules/log/color_console.go @@ -0,0 +1,14 @@ +// Copyright 2014 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +// CanColorStdout reports if we can color the Stdout +// Although we could do terminal sniffing and the like - in reality +// most tools on *nix are happy to display ansi colors. +// We will terminal sniff on Windows in console_windows.go +var CanColorStdout = true + +// CanColorStderr reports if we can color the Stderr +var CanColorStderr = true diff --git a/modules/log/console_other.go b/modules/log/color_console_other.go similarity index 100% rename from modules/log/console_other.go rename to modules/log/color_console_other.go diff --git a/modules/log/console_windows.go b/modules/log/color_console_windows.go similarity index 94% rename from modules/log/console_windows.go rename to modules/log/color_console_windows.go index 54dac12fa0d2..3f59e934dadd 100644 --- a/modules/log/console_windows.go +++ b/modules/log/color_console_windows.go @@ -20,7 +20,7 @@ func enableVTMode(console windows.Handle) bool { // EnableVirtualTerminalProcessing is the console mode to allow ANSI code // interpretation on the console. See: // https://docs.microsoft.com/en-us/windows/console/setconsolemode - // It only works on windows 10. Earlier terminals will fail with an err which we will + // It only works on Windows 10. Earlier terminals will fail with an err which we will // handle to say don't color mode |= windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING err = windows.SetConsoleMode(console, mode) diff --git a/modules/log/colors_router.go b/modules/log/color_router.go similarity index 54% rename from modules/log/colors_router.go rename to modules/log/color_router.go index efc7337b6b25..80e7e02079e6 100644 --- a/modules/log/colors_router.go +++ b/modules/log/color_router.go @@ -8,15 +8,15 @@ import ( "time" ) -var statusToColor = map[int][]byte{ - 100: ColorBytes(Bold), - 200: ColorBytes(FgGreen), - 300: ColorBytes(FgYellow), - 304: ColorBytes(FgCyan), - 400: ColorBytes(Bold, FgRed), - 401: ColorBytes(Bold, FgMagenta), - 403: ColorBytes(Bold, FgMagenta), - 500: ColorBytes(Bold, BgRed), +var statusToColor = map[int][]ColorAttribute{ + 100: {Bold}, + 200: {FgGreen}, + 300: {FgYellow}, + 304: {FgCyan}, + 400: {Bold, FgRed}, + 401: {Bold, FgMagenta}, + 403: {Bold, FgMagenta}, + 500: {Bold, BgRed}, } // ColoredStatus adds colors for HTTP status @@ -26,30 +26,30 @@ func ColoredStatus(status int, s ...string) *ColoredValue { color, ok = statusToColor[(status/100)*100] } if !ok { - color = fgBoldBytes + color = []ColorAttribute{Bold} } if len(s) > 0 { - return NewColoredValueBytes(s[0], &color) + return NewColoredValue(s[0], color...) } - return NewColoredValueBytes(status, &color) + return NewColoredValue(status, color...) } -var methodToColor = map[string][]byte{ - "GET": ColorBytes(FgBlue), - "POST": ColorBytes(FgGreen), - "DELETE": ColorBytes(FgRed), - "PATCH": ColorBytes(FgCyan), - "PUT": ColorBytes(FgYellow, Faint), - "HEAD": ColorBytes(FgBlue, Faint), +var methodToColor = map[string][]ColorAttribute{ + "GET": {FgBlue}, + "POST": {FgGreen}, + "DELETE": {FgRed}, + "PATCH": {FgCyan}, + "PUT": {FgYellow, Faint}, + "HEAD": {FgBlue, Faint}, } // ColoredMethod adds colors for HTTP methods on log func ColoredMethod(method string) *ColoredValue { color, ok := methodToColor[method] if !ok { - return NewColoredValueBytes(method, &fgBoldBytes) + return NewColoredValue(method, Bold) } - return NewColoredValueBytes(method, &color) + return NewColoredValue(method, color...) } var ( @@ -61,15 +61,15 @@ var ( 10 * time.Second, } - durationColors = [][]byte{ - ColorBytes(FgGreen), - ColorBytes(Bold), - ColorBytes(FgYellow), - ColorBytes(FgRed, Bold), - ColorBytes(BgRed), + durationColors = [][]ColorAttribute{ + {FgGreen}, + {Bold}, + {FgYellow}, + {FgRed, Bold}, + {BgRed}, } - wayTooLong = ColorBytes(BgMagenta) + wayTooLong = BgMagenta ) // ColoredTime converts the provided time to a ColoredValue for logging. The duration is always formatted in milliseconds. @@ -80,8 +80,8 @@ func ColoredTime(duration time.Duration) *ColoredValue { str := fmt.Sprintf("%.1fms", float64(duration.Microseconds())/1000) for i, k := range durations { if duration < k { - return NewColoredValueBytes(str, &durationColors[i]) + return NewColoredValue(str, durationColors[i]...) } } - return NewColoredValueBytes(str, &wayTooLong) + return NewColoredValue(str, wayTooLong) } diff --git a/modules/log/colors.go b/modules/log/colors.go deleted file mode 100644 index 85e205cb6764..000000000000 --- a/modules/log/colors.go +++ /dev/null @@ -1,435 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "fmt" - "io" - "reflect" - "strconv" - "strings" -) - -const escape = "\033" - -// ColorAttribute defines a single SGR Code -type ColorAttribute int - -// Base ColorAttributes -const ( - Reset ColorAttribute = iota - Bold - Faint - Italic - Underline - BlinkSlow - BlinkRapid - ReverseVideo - Concealed - CrossedOut -) - -// Foreground text colors -const ( - FgBlack ColorAttribute = iota + 30 - FgRed - FgGreen - FgYellow - FgBlue - FgMagenta - FgCyan - FgWhite -) - -// Foreground Hi-Intensity text colors -const ( - FgHiBlack ColorAttribute = iota + 90 - FgHiRed - FgHiGreen - FgHiYellow - FgHiBlue - FgHiMagenta - FgHiCyan - FgHiWhite -) - -// Background text colors -const ( - BgBlack ColorAttribute = iota + 40 - BgRed - BgGreen - BgYellow - BgBlue - BgMagenta - BgCyan - BgWhite -) - -// Background Hi-Intensity text colors -const ( - BgHiBlack ColorAttribute = iota + 100 - BgHiRed - BgHiGreen - BgHiYellow - BgHiBlue - BgHiMagenta - BgHiCyan - BgHiWhite -) - -var colorAttributeToString = map[ColorAttribute]string{ - Reset: "Reset", - Bold: "Bold", - Faint: "Faint", - Italic: "Italic", - Underline: "Underline", - BlinkSlow: "BlinkSlow", - BlinkRapid: "BlinkRapid", - ReverseVideo: "ReverseVideo", - Concealed: "Concealed", - CrossedOut: "CrossedOut", - FgBlack: "FgBlack", - FgRed: "FgRed", - FgGreen: "FgGreen", - FgYellow: "FgYellow", - FgBlue: "FgBlue", - FgMagenta: "FgMagenta", - FgCyan: "FgCyan", - FgWhite: "FgWhite", - FgHiBlack: "FgHiBlack", - FgHiRed: "FgHiRed", - FgHiGreen: "FgHiGreen", - FgHiYellow: "FgHiYellow", - FgHiBlue: "FgHiBlue", - FgHiMagenta: "FgHiMagenta", - FgHiCyan: "FgHiCyan", - FgHiWhite: "FgHiWhite", - BgBlack: "BgBlack", - BgRed: "BgRed", - BgGreen: "BgGreen", - BgYellow: "BgYellow", - BgBlue: "BgBlue", - BgMagenta: "BgMagenta", - BgCyan: "BgCyan", - BgWhite: "BgWhite", - BgHiBlack: "BgHiBlack", - BgHiRed: "BgHiRed", - BgHiGreen: "BgHiGreen", - BgHiYellow: "BgHiYellow", - BgHiBlue: "BgHiBlue", - BgHiMagenta: "BgHiMagenta", - BgHiCyan: "BgHiCyan", - BgHiWhite: "BgHiWhite", -} - -func (c *ColorAttribute) String() string { - return colorAttributeToString[*c] -} - -var colorAttributeFromString = map[string]ColorAttribute{} - -// ColorAttributeFromString will return a ColorAttribute given a string -func ColorAttributeFromString(from string) ColorAttribute { - lowerFrom := strings.TrimSpace(strings.ToLower(from)) - return colorAttributeFromString[lowerFrom] -} - -// ColorString converts a list of ColorAttributes to a color string -func ColorString(attrs ...ColorAttribute) string { - return string(ColorBytes(attrs...)) -} - -// ColorBytes converts a list of ColorAttributes to a byte array -func ColorBytes(attrs ...ColorAttribute) []byte { - bytes := make([]byte, 0, 20) - bytes = append(bytes, escape[0], '[') - if len(attrs) > 0 { - bytes = append(bytes, strconv.Itoa(int(attrs[0]))...) - for _, a := range attrs[1:] { - bytes = append(bytes, ';') - bytes = append(bytes, strconv.Itoa(int(a))...) - } - } else { - bytes = append(bytes, strconv.Itoa(int(Bold))...) - } - bytes = append(bytes, 'm') - return bytes -} - -var levelToColor = map[Level][]byte{ - TRACE: ColorBytes(Bold, FgCyan), - DEBUG: ColorBytes(Bold, FgBlue), - INFO: ColorBytes(Bold, FgGreen), - WARN: ColorBytes(Bold, FgYellow), - ERROR: ColorBytes(Bold, FgRed), - CRITICAL: ColorBytes(Bold, BgMagenta), - FATAL: ColorBytes(Bold, BgRed), - NONE: ColorBytes(Reset), -} - -var ( - resetBytes = ColorBytes(Reset) - fgCyanBytes = ColorBytes(FgCyan) - fgGreenBytes = ColorBytes(FgGreen) - fgBoldBytes = ColorBytes(Bold) -) - -type protectedANSIWriterMode int - -const ( - escapeAll protectedANSIWriterMode = iota - allowColor - removeColor -) - -type protectedANSIWriter struct { - w io.Writer - mode protectedANSIWriterMode -} - -// Write will protect against unusual characters -func (c *protectedANSIWriter) Write(bytes []byte) (int, error) { - end := len(bytes) - totalWritten := 0 -normalLoop: - for i := 0; i < end; { - lasti := i - - if c.mode == escapeAll { - for i < end && (bytes[i] >= ' ' || bytes[i] == '\n' || bytes[i] == '\t') { - i++ - } - } else { - // Allow tabs if we're not escaping everything - for i < end && (bytes[i] >= ' ' || bytes[i] == '\t') { - i++ - } - } - - if i > lasti { - written, err := c.w.Write(bytes[lasti:i]) - totalWritten += written - if err != nil { - return totalWritten, err - } - - } - if i >= end { - break - } - - // If we're not just escaping all we should prefix all newlines with a \t - if c.mode != escapeAll { - if bytes[i] == '\n' { - written, err := c.w.Write([]byte{'\n', '\t'}) - if written > 0 { - totalWritten++ - } - if err != nil { - return totalWritten, err - } - i++ - continue normalLoop - } - - if bytes[i] == escape[0] && i+1 < end && bytes[i+1] == '[' { - for j := i + 2; j < end; j++ { - if bytes[j] >= '0' && bytes[j] <= '9' { - continue - } - if bytes[j] == ';' { - continue - } - if bytes[j] == 'm' { - if c.mode == allowColor { - written, err := c.w.Write(bytes[i : j+1]) - totalWritten += written - if err != nil { - return totalWritten, err - } - } else { - totalWritten = j - } - i = j + 1 - continue normalLoop - } - break - } - } - } - - // Process naughty character - if _, err := fmt.Fprintf(c.w, `\%#03o`, bytes[i]); err != nil { - return totalWritten, err - } - i++ - totalWritten++ - } - return totalWritten, nil -} - -// ColorSprintf returns a colored string from a format and arguments -// arguments will be wrapped in ColoredValues to protect against color spoofing -func ColorSprintf(format string, args ...interface{}) string { - if len(args) > 0 { - v := make([]interface{}, len(args)) - for i := 0; i < len(v); i++ { - v[i] = NewColoredValuePointer(&args[i]) - } - return fmt.Sprintf(format, v...) - } - return format -} - -// ColorFprintf will write to the provided writer similar to ColorSprintf -func ColorFprintf(w io.Writer, format string, args ...interface{}) (int, error) { - if len(args) > 0 { - v := make([]interface{}, len(args)) - for i := 0; i < len(v); i++ { - v[i] = NewColoredValuePointer(&args[i]) - } - return fmt.Fprintf(w, format, v...) - } - return fmt.Fprint(w, format) -} - -// ColorFormatted structs provide their own colored string when formatted with ColorSprintf -type ColorFormatted interface { - // ColorFormat provides the colored representation of the value - ColorFormat(s fmt.State) -} - -var colorFormattedType = reflect.TypeOf((*ColorFormatted)(nil)).Elem() - -// ColoredValue will Color the provided value -type ColoredValue struct { - colorBytes *[]byte - resetBytes *[]byte - Value *interface{} -} - -// NewColoredValue is a helper function to create a ColoredValue from a Value -// If no color is provided it defaults to Bold with standard Reset -// If a ColoredValue is provided it is not changed -func NewColoredValue(value interface{}, color ...ColorAttribute) *ColoredValue { - return NewColoredValuePointer(&value, color...) -} - -// NewColoredValuePointer is a helper function to create a ColoredValue from a Value Pointer -// If no color is provided it defaults to Bold with standard Reset -// If a ColoredValue is provided it is not changed -func NewColoredValuePointer(value *interface{}, color ...ColorAttribute) *ColoredValue { - if val, ok := (*value).(*ColoredValue); ok { - return val - } - if len(color) > 0 { - bytes := ColorBytes(color...) - return &ColoredValue{ - colorBytes: &bytes, - resetBytes: &resetBytes, - Value: value, - } - } - return &ColoredValue{ - colorBytes: &fgBoldBytes, - resetBytes: &resetBytes, - Value: value, - } -} - -// NewColoredValueBytes creates a value from the provided value with color bytes -// If a ColoredValue is provided it is not changed -func NewColoredValueBytes(value interface{}, colorBytes *[]byte) *ColoredValue { - if val, ok := value.(*ColoredValue); ok { - return val - } - return &ColoredValue{ - colorBytes: colorBytes, - resetBytes: &resetBytes, - Value: &value, - } -} - -// NewColoredIDValue is a helper function to create a ColoredValue from a Value -// The Value will be colored with FgCyan -// If a ColoredValue is provided it is not changed -func NewColoredIDValue(value interface{}) *ColoredValue { - return NewColoredValueBytes(value, &fgCyanBytes) -} - -// Format will format the provided value and protect against ANSI color spoofing within the value -// If the wrapped value is ColorFormatted and the format is "%-v" then its ColorString will -// be used. It is presumed that this ColorString is safe. -func (cv *ColoredValue) Format(s fmt.State, c rune) { - if c == 'v' && s.Flag('-') { - if val, ok := (*cv.Value).(ColorFormatted); ok { - val.ColorFormat(s) - return - } - v := reflect.ValueOf(*cv.Value) - t := v.Type() - - if reflect.PtrTo(t).Implements(colorFormattedType) { - vp := reflect.New(t) - vp.Elem().Set(v) - val := vp.Interface().(ColorFormatted) - val.ColorFormat(s) - return - } - } - s.Write(*cv.colorBytes) - fmt.Fprintf(&protectedANSIWriter{w: s}, fmtString(s, c), *(cv.Value)) - s.Write(*cv.resetBytes) -} - -// ColorFormatAsString returns the result of the ColorFormat without the color -func ColorFormatAsString(colorVal ColorFormatted) string { - s := new(strings.Builder) - _, _ = ColorFprintf(&protectedANSIWriter{w: s, mode: removeColor}, "%-v", colorVal) - return s.String() -} - -// SetColorBytes will allow a user to set the colorBytes of a colored value -func (cv *ColoredValue) SetColorBytes(colorBytes []byte) { - cv.colorBytes = &colorBytes -} - -// SetColorBytesPointer will allow a user to set the colorBytes pointer of a colored value -func (cv *ColoredValue) SetColorBytesPointer(colorBytes *[]byte) { - cv.colorBytes = colorBytes -} - -// SetResetBytes will allow a user to set the resetBytes pointer of a colored value -func (cv *ColoredValue) SetResetBytes(resetBytes []byte) { - cv.resetBytes = &resetBytes -} - -// SetResetBytesPointer will allow a user to set the resetBytes pointer of a colored value -func (cv *ColoredValue) SetResetBytesPointer(resetBytes *[]byte) { - cv.resetBytes = resetBytes -} - -func fmtString(s fmt.State, c rune) string { - var width, precision string - base := make([]byte, 0, 8) - base = append(base, '%') - for _, c := range []byte(" +-#0") { - if s.Flag(int(c)) { - base = append(base, c) - } - } - if w, ok := s.Width(); ok { - width = strconv.Itoa(w) - } - if p, ok := s.Precision(); ok { - precision = "." + strconv.Itoa(p) - } - return fmt.Sprintf("%s%s%s%c", base, width, precision, c) -} - -func init() { - for attr, from := range colorAttributeToString { - colorAttributeFromString[strings.ToLower(from)] = attr - } -} diff --git a/modules/log/conn.go b/modules/log/conn.go deleted file mode 100644 index b21a744037bc..000000000000 --- a/modules/log/conn.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2014 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "fmt" - "io" - "net" - - "code.gitea.io/gitea/modules/json" -) - -type connWriter struct { - innerWriter io.WriteCloser - ReconnectOnMsg bool `json:"reconnectOnMsg"` - Reconnect bool `json:"reconnect"` - Net string `json:"net"` - Addr string `json:"addr"` -} - -// Close the inner writer -func (i *connWriter) Close() error { - if i.innerWriter != nil { - return i.innerWriter.Close() - } - return nil -} - -// Write the data to the connection -func (i *connWriter) Write(p []byte) (int, error) { - if i.neededConnectOnMsg() { - if err := i.connect(); err != nil { - return 0, err - } - } - - if i.ReconnectOnMsg { - defer i.innerWriter.Close() - } - - return i.innerWriter.Write(p) -} - -func (i *connWriter) neededConnectOnMsg() bool { - if i.Reconnect { - i.Reconnect = false - return true - } - - if i.innerWriter == nil { - return true - } - - return i.ReconnectOnMsg -} - -func (i *connWriter) connect() error { - if i.innerWriter != nil { - i.innerWriter.Close() - i.innerWriter = nil - } - - conn, err := net.Dial(i.Net, i.Addr) - if err != nil { - return err - } - - if tcpConn, ok := conn.(*net.TCPConn); ok { - err = tcpConn.SetKeepAlive(true) - if err != nil { - return err - } - } - - i.innerWriter = conn - return nil -} - -func (i *connWriter) releaseReopen() error { - if i.innerWriter != nil { - return i.connect() - } - return nil -} - -// ConnLogger implements LoggerProvider. -// it writes messages in keep-live tcp connection. -type ConnLogger struct { - WriterLogger - ReconnectOnMsg bool `json:"reconnectOnMsg"` - Reconnect bool `json:"reconnect"` - Net string `json:"net"` - Addr string `json:"addr"` -} - -// NewConn creates new ConnLogger returning as LoggerProvider. -func NewConn() LoggerProvider { - conn := new(ConnLogger) - conn.Level = TRACE - return conn -} - -// Init inits connection writer with json config. -// json config only need key "level". -func (log *ConnLogger) Init(jsonconfig string) error { - err := json.Unmarshal([]byte(jsonconfig), log) - if err != nil { - return fmt.Errorf("Unable to parse JSON: %w", err) - } - log.NewWriterLogger(&connWriter{ - ReconnectOnMsg: log.ReconnectOnMsg, - Reconnect: log.Reconnect, - Net: log.Net, - Addr: log.Addr, - }, log.Level) - return nil -} - -// Flush does nothing for this implementation -func (log *ConnLogger) Flush() { -} - -// GetName returns the default name for this implementation -func (log *ConnLogger) GetName() string { - return "conn" -} - -// ReleaseReopen causes the ConnLogger to reconnect to the server -func (log *ConnLogger) ReleaseReopen() error { - return log.out.(*connWriter).releaseReopen() -} - -func init() { - Register("conn", NewConn) -} diff --git a/modules/log/conn_test.go b/modules/log/conn_test.go deleted file mode 100644 index 445bd7765341..000000000000 --- a/modules/log/conn_test.go +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "fmt" - "io" - "net" - "strings" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func listenReadAndClose(t *testing.T, l net.Listener, expected string) { - conn, err := l.Accept() - assert.NoError(t, err) - defer conn.Close() - written, err := io.ReadAll(conn) - - assert.NoError(t, err) - assert.Equal(t, expected, string(written)) -} - -func TestConnLogger(t *testing.T) { - protocol := "tcp" - address := ":3099" - - l, err := net.Listen(protocol, address) - if err != nil { - t.Fatal(err) - } - defer l.Close() - - prefix := "TestPrefix " - level := INFO - flags := LstdFlags | LUTC | Lfuncname - - logger := NewConn() - connLogger := logger.(*ConnLogger) - - logger.Init(fmt.Sprintf("{\"prefix\":\"%s\",\"level\":\"%s\",\"flags\":%d,\"reconnectOnMsg\":%t,\"reconnect\":%t,\"net\":\"%s\",\"addr\":\"%s\"}", prefix, level.String(), flags, true, true, protocol, address)) - - assert.Equal(t, flags, connLogger.Flags) - assert.Equal(t, level, connLogger.Level) - assert.Equal(t, level, logger.GetLevel()) - - location, _ := time.LoadLocation("EST") - - date := time.Date(2019, time.January, 13, 22, 3, 30, 15, location) - - dateString := date.UTC().Format("2006/01/02 15:04:05") - - event := Event{ - level: INFO, - msg: "TEST MSG", - caller: "CALLER", - filename: "FULL/FILENAME", - line: 1, - time: date, - } - expected := fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - var wg sync.WaitGroup - wg.Add(2) - go func() { - defer wg.Done() - listenReadAndClose(t, l, expected) - }() - go func() { - defer wg.Done() - err := logger.LogEvent(&event) - assert.NoError(t, err) - }() - wg.Wait() - - event.level = WARN - expected = fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - wg.Add(2) - go func() { - defer wg.Done() - listenReadAndClose(t, l, expected) - }() - go func() { - defer wg.Done() - err := logger.LogEvent(&event) - assert.NoError(t, err) - }() - wg.Wait() - - logger.Close() -} - -func TestConnLoggerBadConfig(t *testing.T) { - logger := NewConn() - - err := logger.Init("{") - assert.Error(t, err) - assert.Contains(t, err.Error(), "Unable to parse JSON") - logger.Close() -} - -func TestConnLoggerCloseBeforeSend(t *testing.T) { - protocol := "tcp" - address := ":3099" - - prefix := "TestPrefix " - level := INFO - flags := LstdFlags | LUTC | Lfuncname - - logger := NewConn() - - logger.Init(fmt.Sprintf("{\"prefix\":\"%s\",\"level\":\"%s\",\"flags\":%d,\"reconnectOnMsg\":%t,\"reconnect\":%t,\"net\":\"%s\",\"addr\":\"%s\"}", prefix, level.String(), flags, false, false, protocol, address)) - logger.Close() -} - -func TestConnLoggerFailConnect(t *testing.T) { - protocol := "tcp" - address := ":3099" - - prefix := "TestPrefix " - level := INFO - flags := LstdFlags | LUTC | Lfuncname - - logger := NewConn() - - logger.Init(fmt.Sprintf("{\"prefix\":\"%s\",\"level\":\"%s\",\"flags\":%d,\"reconnectOnMsg\":%t,\"reconnect\":%t,\"net\":\"%s\",\"addr\":\"%s\"}", prefix, level.String(), flags, false, false, protocol, address)) - - assert.Equal(t, level, logger.GetLevel()) - - location, _ := time.LoadLocation("EST") - - date := time.Date(2019, time.January, 13, 22, 3, 30, 15, location) - - // dateString := date.UTC().Format("2006/01/02 15:04:05") - - event := Event{ - level: INFO, - msg: "TEST MSG", - caller: "CALLER", - filename: "FULL/FILENAME", - line: 1, - time: date, - } - - err := logger.LogEvent(&event) - assert.Error(t, err) - - logger.Close() -} - -func TestConnLoggerClose(t *testing.T) { - protocol := "tcp" - address := ":3099" - - l, err := net.Listen(protocol, address) - if err != nil { - t.Fatal(err) - } - defer l.Close() - - prefix := "TestPrefix " - level := INFO - flags := LstdFlags | LUTC | Lfuncname - - logger := NewConn() - connLogger := logger.(*ConnLogger) - - logger.Init(fmt.Sprintf("{\"prefix\":\"%s\",\"level\":\"%s\",\"flags\":%d,\"reconnectOnMsg\":%t,\"reconnect\":%t,\"net\":\"%s\",\"addr\":\"%s\"}", prefix, level.String(), flags, false, false, protocol, address)) - - assert.Equal(t, flags, connLogger.Flags) - assert.Equal(t, level, connLogger.Level) - assert.Equal(t, level, logger.GetLevel()) - location, _ := time.LoadLocation("EST") - - date := time.Date(2019, time.January, 13, 22, 3, 30, 15, location) - - dateString := date.UTC().Format("2006/01/02 15:04:05") - - event := Event{ - level: INFO, - msg: "TEST MSG", - caller: "CALLER", - filename: "FULL/FILENAME", - line: 1, - time: date, - } - expected := fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - var wg sync.WaitGroup - wg.Add(2) - go func() { - defer wg.Done() - err := logger.LogEvent(&event) - assert.NoError(t, err) - logger.Close() - }() - go func() { - defer wg.Done() - listenReadAndClose(t, l, expected) - }() - wg.Wait() - - logger = NewConn() - connLogger = logger.(*ConnLogger) - - logger.Init(fmt.Sprintf("{\"prefix\":\"%s\",\"level\":\"%s\",\"flags\":%d,\"reconnectOnMsg\":%t,\"reconnect\":%t,\"net\":\"%s\",\"addr\":\"%s\"}", prefix, level.String(), flags, false, true, protocol, address)) - - assert.Equal(t, flags, connLogger.Flags) - assert.Equal(t, level, connLogger.Level) - assert.Equal(t, level, logger.GetLevel()) - - event.level = WARN - expected = fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - wg.Add(2) - go func() { - defer wg.Done() - listenReadAndClose(t, l, expected) - }() - go func() { - defer wg.Done() - err := logger.LogEvent(&event) - assert.NoError(t, err) - logger.Close() - }() - wg.Wait() - logger.Flush() - logger.Close() -} diff --git a/modules/log/console.go b/modules/log/console.go deleted file mode 100644 index ce0415d1390c..000000000000 --- a/modules/log/console.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2014 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "fmt" - "io" - "os" - - "code.gitea.io/gitea/modules/json" -) - -// CanColorStdout reports if we can color the Stdout -// Although we could do terminal sniffing and the like - in reality -// most tools on *nix are happy to display ansi colors. -// We will terminal sniff on Windows in console_windows.go -var CanColorStdout = true - -// CanColorStderr reports if we can color the Stderr -var CanColorStderr = true - -type nopWriteCloser struct { - w io.WriteCloser -} - -func (n *nopWriteCloser) Write(p []byte) (int, error) { - return n.w.Write(p) -} - -func (n *nopWriteCloser) Close() error { - return nil -} - -// ConsoleLogger implements LoggerProvider and writes messages to terminal. -type ConsoleLogger struct { - WriterLogger - Stderr bool `json:"stderr"` -} - -// NewConsoleLogger create ConsoleLogger returning as LoggerProvider. -func NewConsoleLogger() LoggerProvider { - log := &ConsoleLogger{} - log.NewWriterLogger(&nopWriteCloser{ - w: os.Stdout, - }) - return log -} - -// Init inits connection writer with json config. -// json config only need key "level". -func (log *ConsoleLogger) Init(config string) error { - err := json.Unmarshal([]byte(config), log) - if err != nil { - return fmt.Errorf("Unable to parse JSON: %w", err) - } - if log.Stderr { - log.NewWriterLogger(&nopWriteCloser{ - w: os.Stderr, - }) - } else { - log.NewWriterLogger(log.out) - } - return nil -} - -// Flush when log should be flushed -func (log *ConsoleLogger) Flush() { -} - -// ReleaseReopen causes the console logger to reconnect to os.Stdout -func (log *ConsoleLogger) ReleaseReopen() error { - if log.Stderr { - log.NewWriterLogger(&nopWriteCloser{ - w: os.Stderr, - }) - } else { - log.NewWriterLogger(&nopWriteCloser{ - w: os.Stdout, - }) - } - return nil -} - -// GetName returns the default name for this implementation -func (log *ConsoleLogger) GetName() string { - return "console" -} - -func init() { - Register("console", NewConsoleLogger) -} diff --git a/modules/log/console_test.go b/modules/log/console_test.go deleted file mode 100644 index e4c3882d4f8a..000000000000 --- a/modules/log/console_test.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestConsoleLoggerBadConfig(t *testing.T) { - logger := NewConsoleLogger() - - err := logger.Init("{") - assert.Error(t, err) - assert.Contains(t, err.Error(), "Unable to parse JSON") - logger.Close() -} - -func TestConsoleLoggerMinimalConfig(t *testing.T) { - for _, level := range Levels() { - var written []byte - var closed bool - - c := CallbackWriteCloser{ - callback: func(p []byte, close bool) { - written = p - closed = close - }, - } - prefix := "" - flags := LstdFlags - - cw := NewConsoleLogger() - realCW := cw.(*ConsoleLogger) - cw.Init(fmt.Sprintf("{\"level\":\"%s\"}", level)) - nwc := realCW.out.(*nopWriteCloser) - nwc.w = c - - assert.Equal(t, flags, realCW.Flags) - assert.Equal(t, FromString(level), realCW.Level) - assert.Equal(t, FromString(level), cw.GetLevel()) - assert.Equal(t, prefix, realCW.Prefix) - assert.Equal(t, "", string(written)) - cw.Close() - assert.False(t, closed) - - } -} - -func TestConsoleLogger(t *testing.T) { - var written []byte - var closed bool - - c := CallbackWriteCloser{ - callback: func(p []byte, close bool) { - written = p - closed = close - }, - } - prefix := "TestPrefix " - level := INFO - flags := LstdFlags | LUTC | Lfuncname - - cw := NewConsoleLogger() - realCW := cw.(*ConsoleLogger) - realCW.Colorize = false - nwc := realCW.out.(*nopWriteCloser) - nwc.w = c - - cw.Init(fmt.Sprintf("{\"expression\":\"FILENAME\",\"prefix\":\"%s\",\"level\":\"%s\",\"flags\":%d}", prefix, level.String(), flags)) - - assert.Equal(t, flags, realCW.Flags) - assert.Equal(t, level, realCW.Level) - assert.Equal(t, level, cw.GetLevel()) - - location, _ := time.LoadLocation("EST") - - date := time.Date(2019, time.January, 13, 22, 3, 30, 15, location) - - dateString := date.UTC().Format("2006/01/02 15:04:05") - - event := Event{ - level: INFO, - msg: "TEST MSG", - caller: "CALLER", - filename: "FULL/FILENAME", - line: 1, - time: date, - } - - expected := fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - cw.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - event.level = DEBUG - expected = "" - cw.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - - event.level = TRACE - expected = "" - cw.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - - nonMatchEvent := Event{ - level: INFO, - msg: "TEST MSG", - caller: "CALLER", - filename: "FULL/FI_LENAME", - line: 1, - time: date, - } - event.level = INFO - expected = "" - cw.LogEvent(&nonMatchEvent) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - - event.level = WARN - expected = fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - cw.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - cw.Close() - assert.False(t, closed) -} diff --git a/modules/log/errors.go b/modules/log/errors.go deleted file mode 100644 index 942639a43444..000000000000 --- a/modules/log/errors.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import "fmt" - -// ErrTimeout represents a "Timeout" kind of error. -type ErrTimeout struct { - Name string - Provider string -} - -// IsErrTimeout checks if an error is a ErrTimeout. -func IsErrTimeout(err error) bool { - if err == nil { - return false - } - _, ok := err.(ErrTimeout) - return ok -} - -func (err ErrTimeout) Error() string { - return fmt.Sprintf("Log Timeout for %s (%s)", err.Name, err.Provider) -} - -// ErrUnknownProvider represents a "Unknown Provider" kind of error. -type ErrUnknownProvider struct { - Provider string -} - -// IsErrUnknownProvider checks if an error is a ErrUnknownProvider. -func IsErrUnknownProvider(err error) bool { - if err == nil { - return false - } - _, ok := err.(ErrUnknownProvider) - return ok -} - -func (err ErrUnknownProvider) Error() string { - return fmt.Sprintf("Unknown Log Provider \"%s\" (Was it registered?)", err.Provider) -} - -// ErrDuplicateName represents a Duplicate Name error -type ErrDuplicateName struct { - Name string -} - -// IsErrDuplicateName checks if an error is a ErrDuplicateName. -func IsErrDuplicateName(err error) bool { - if err == nil { - return false - } - _, ok := err.(ErrDuplicateName) - return ok -} - -func (err ErrDuplicateName) Error() string { - return fmt.Sprintf("Duplicate named logger: %s", err.Name) -} diff --git a/modules/log/event.go b/modules/log/event.go deleted file mode 100644 index 723c8810bc42..000000000000 --- a/modules/log/event.go +++ /dev/null @@ -1,460 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "context" - "fmt" - "runtime/pprof" - "sync" - "time" - - "code.gitea.io/gitea/modules/process" -) - -// Event represents a logging event -type Event struct { - level Level - msg string - caller string - filename string - line int - time time.Time - stacktrace string -} - -// EventLogger represents the behaviours of a logger -type EventLogger interface { - LogEvent(event *Event) error - Close() - Flush() - GetLevel() Level - GetStacktraceLevel() Level - GetName() string - ReleaseReopen() error -} - -// ChannelledLog represents a cached channel to a LoggerProvider -type ChannelledLog struct { - ctx context.Context - finished context.CancelFunc - name string - provider string - queue chan *Event - loggerProvider LoggerProvider - flush chan bool - close chan bool - closed chan bool -} - -// NewChannelledLog a new logger instance with given logger provider and config. -func NewChannelledLog(parent context.Context, name, provider, config string, bufferLength int64) (*ChannelledLog, error) { - if log, ok := providers[provider]; ok { - - l := &ChannelledLog{ - queue: make(chan *Event, bufferLength), - flush: make(chan bool), - close: make(chan bool), - closed: make(chan bool), - } - l.loggerProvider = log() - if err := l.loggerProvider.Init(config); err != nil { - return nil, err - } - l.name = name - l.provider = provider - l.ctx, _, l.finished = process.GetManager().AddTypedContext(parent, fmt.Sprintf("Logger: %s(%s)", l.name, l.provider), process.SystemProcessType, false) - go l.Start() - return l, nil - } - return nil, ErrUnknownProvider{provider} -} - -// Start processing the ChannelledLog -func (l *ChannelledLog) Start() { - pprof.SetGoroutineLabels(l.ctx) - defer l.finished() - for { - select { - case event, ok := <-l.queue: - if !ok { - l.closeLogger() - return - } - l.loggerProvider.LogEvent(event) //nolint:errcheck - case _, ok := <-l.flush: - if !ok { - l.closeLogger() - return - } - l.emptyQueue() - l.loggerProvider.Flush() - case <-l.close: - l.emptyQueue() - l.closeLogger() - return - } - } -} - -// LogEvent logs an event to this ChannelledLog -func (l *ChannelledLog) LogEvent(event *Event) error { - select { - case l.queue <- event: - return nil - case <-time.After(60 * time.Second): - // We're blocked! - return ErrTimeout{ - Name: l.name, - Provider: l.provider, - } - } -} - -func (l *ChannelledLog) emptyQueue() bool { - for { - select { - case event, ok := <-l.queue: - if !ok { - return false - } - l.loggerProvider.LogEvent(event) //nolint:errcheck - default: - return true - } - } -} - -func (l *ChannelledLog) closeLogger() { - l.loggerProvider.Flush() - l.loggerProvider.Close() - l.closed <- true -} - -// Close this ChannelledLog -func (l *ChannelledLog) Close() { - l.close <- true - <-l.closed -} - -// Flush this ChannelledLog -func (l *ChannelledLog) Flush() { - l.flush <- true -} - -// ReleaseReopen this ChannelledLog -func (l *ChannelledLog) ReleaseReopen() error { - return l.loggerProvider.ReleaseReopen() -} - -// GetLevel gets the level of this ChannelledLog -func (l *ChannelledLog) GetLevel() Level { - return l.loggerProvider.GetLevel() -} - -// GetStacktraceLevel gets the level of this ChannelledLog -func (l *ChannelledLog) GetStacktraceLevel() Level { - return l.loggerProvider.GetStacktraceLevel() -} - -// GetName returns the name of this ChannelledLog -func (l *ChannelledLog) GetName() string { - return l.name -} - -// MultiChannelledLog represents a cached channel to a LoggerProvider -type MultiChannelledLog struct { - ctx context.Context - finished context.CancelFunc - name string - bufferLength int64 - queue chan *Event - rwmutex sync.RWMutex - loggers map[string]EventLogger - flush chan bool - close chan bool - started bool - level Level - stacktraceLevel Level - closed chan bool - paused chan bool -} - -// NewMultiChannelledLog a new logger instance with given logger provider and config. -func NewMultiChannelledLog(name string, bufferLength int64) *MultiChannelledLog { - ctx, _, finished := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Logger: %s", name), process.SystemProcessType, false) - - m := &MultiChannelledLog{ - ctx: ctx, - finished: finished, - name: name, - queue: make(chan *Event, bufferLength), - flush: make(chan bool), - bufferLength: bufferLength, - loggers: make(map[string]EventLogger), - level: NONE, - stacktraceLevel: NONE, - close: make(chan bool), - closed: make(chan bool), - paused: make(chan bool), - } - return m -} - -// AddLogger adds a logger to this MultiChannelledLog -func (m *MultiChannelledLog) AddLogger(logger EventLogger) error { - m.rwmutex.Lock() - name := logger.GetName() - if _, has := m.loggers[name]; has { - m.rwmutex.Unlock() - return ErrDuplicateName{name} - } - m.loggers[name] = logger - if logger.GetLevel() < m.level { - m.level = logger.GetLevel() - } - if logger.GetStacktraceLevel() < m.stacktraceLevel { - m.stacktraceLevel = logger.GetStacktraceLevel() - } - m.rwmutex.Unlock() - go m.Start() - return nil -} - -// DelLogger removes a sub logger from this MultiChannelledLog -// NB: If you delete the last sublogger this logger will simply drop -// log events -func (m *MultiChannelledLog) DelLogger(name string) bool { - m.rwmutex.Lock() - logger, has := m.loggers[name] - if !has { - m.rwmutex.Unlock() - return false - } - delete(m.loggers, name) - m.internalResetLevel() - m.rwmutex.Unlock() - logger.Flush() - logger.Close() - return true -} - -// GetEventLogger returns a sub logger from this MultiChannelledLog -func (m *MultiChannelledLog) GetEventLogger(name string) EventLogger { - m.rwmutex.RLock() - defer m.rwmutex.RUnlock() - return m.loggers[name] -} - -// GetEventLoggerNames returns a list of names -func (m *MultiChannelledLog) GetEventLoggerNames() []string { - m.rwmutex.RLock() - defer m.rwmutex.RUnlock() - var keys []string - for k := range m.loggers { - keys = append(keys, k) - } - return keys -} - -func (m *MultiChannelledLog) closeLoggers() { - m.rwmutex.Lock() - for _, logger := range m.loggers { - logger.Flush() - logger.Close() - } - m.rwmutex.Unlock() - m.closed <- true -} - -// Pause pauses this Logger -func (m *MultiChannelledLog) Pause() { - m.paused <- true -} - -// Resume resumes this Logger -func (m *MultiChannelledLog) Resume() { - m.paused <- false -} - -// ReleaseReopen causes this logger to tell its subloggers to release and reopen -func (m *MultiChannelledLog) ReleaseReopen() error { - m.rwmutex.Lock() - defer m.rwmutex.Unlock() - var accumulatedErr error - for _, logger := range m.loggers { - if err := logger.ReleaseReopen(); err != nil { - if accumulatedErr == nil { - accumulatedErr = fmt.Errorf("Error whilst reopening: %s Error: %w", logger.GetName(), err) - } else { - accumulatedErr = fmt.Errorf("Error whilst reopening: %s Error: %v & %w", logger.GetName(), err, accumulatedErr) - } - } - } - return accumulatedErr -} - -// Start processing the MultiChannelledLog -func (m *MultiChannelledLog) Start() { - m.rwmutex.Lock() - if m.started { - m.rwmutex.Unlock() - return - } - pprof.SetGoroutineLabels(m.ctx) - defer m.finished() - - m.started = true - m.rwmutex.Unlock() - paused := false - for { - if paused { - select { - case paused = <-m.paused: - if !paused { - m.ResetLevel() - } - case _, ok := <-m.flush: - if !ok { - m.closeLoggers() - return - } - m.rwmutex.RLock() - for _, logger := range m.loggers { - logger.Flush() - } - m.rwmutex.RUnlock() - case <-m.close: - m.closeLoggers() - return - } - continue - } - select { - case paused = <-m.paused: - if paused && m.level < INFO { - m.level = INFO - } - case event, ok := <-m.queue: - if !ok { - m.closeLoggers() - return - } - m.rwmutex.RLock() - for _, logger := range m.loggers { - err := logger.LogEvent(event) - if err != nil { - fmt.Println(err) //nolint:forbidigo - } - } - m.rwmutex.RUnlock() - case _, ok := <-m.flush: - if !ok { - m.closeLoggers() - return - } - m.emptyQueue() - m.rwmutex.RLock() - for _, logger := range m.loggers { - logger.Flush() - } - m.rwmutex.RUnlock() - case <-m.close: - m.emptyQueue() - m.closeLoggers() - return - } - } -} - -func (m *MultiChannelledLog) emptyQueue() bool { - for { - select { - case event, ok := <-m.queue: - if !ok { - return false - } - m.rwmutex.RLock() - for _, logger := range m.loggers { - err := logger.LogEvent(event) - if err != nil { - fmt.Println(err) //nolint:forbidigo - } - } - m.rwmutex.RUnlock() - default: - return true - } - } -} - -// LogEvent logs an event to this MultiChannelledLog -func (m *MultiChannelledLog) LogEvent(event *Event) error { - select { - case m.queue <- event: - return nil - case <-time.After(100 * time.Millisecond): - // We're blocked! - return ErrTimeout{ - Name: m.name, - Provider: "MultiChannelledLog", - } - } -} - -// Close this MultiChannelledLog -func (m *MultiChannelledLog) Close() { - m.close <- true - <-m.closed -} - -// Flush this ChannelledLog -func (m *MultiChannelledLog) Flush() { - m.flush <- true -} - -// GetLevel gets the level of this MultiChannelledLog -func (m *MultiChannelledLog) GetLevel() Level { - m.rwmutex.RLock() - defer m.rwmutex.RUnlock() - return m.level -} - -// GetStacktraceLevel gets the level of this MultiChannelledLog -func (m *MultiChannelledLog) GetStacktraceLevel() Level { - m.rwmutex.RLock() - defer m.rwmutex.RUnlock() - return m.stacktraceLevel -} - -func (m *MultiChannelledLog) internalResetLevel() Level { - m.level = NONE - for _, logger := range m.loggers { - level := logger.GetLevel() - if level < m.level { - m.level = level - } - level = logger.GetStacktraceLevel() - if level < m.stacktraceLevel { - m.stacktraceLevel = level - } - } - return m.level -} - -// ResetLevel will reset the level of this MultiChannelledLog -func (m *MultiChannelledLog) ResetLevel() Level { - m.rwmutex.Lock() - defer m.rwmutex.Unlock() - return m.internalResetLevel() -} - -// GetName gets the name of this MultiChannelledLog -func (m *MultiChannelledLog) GetName() string { - return m.name -} - -func (e *Event) GetMsg() string { - return e.msg -} diff --git a/modules/log/event_format.go b/modules/log/event_format.go new file mode 100644 index 000000000000..208e8ab3971f --- /dev/null +++ b/modules/log/event_format.go @@ -0,0 +1,253 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "bytes" + "fmt" + "strings" + "time" +) + +type Event struct { + Time time.Time + + GoroutinePid string + Caller string + Filename string + Line int + + Level Level + + Msg string + MsgFormat string + MsgFrozenArgs []any // it may contains *ColorValue + + Stacktrace string +} + +type EventFormatter func(mode *WriterMode, event *Event, reuse []byte) []byte + +type frozenMsgArg struct { + m *frozenMsgFormatter + v any + s string + + processed bool +} + +func (a *frozenMsgArg) Format(s fmt.State, c rune) { + a.s = fmt.Sprintf(fmt.FormatString(s, c), a.v) + _, _ = s.Write([]byte(a.s)) + a.processed = true +} + +type frozenMsgFormatter struct { + format string + args []any +} + +func (m *frozenMsgFormatter) addArgs(args ...any) { + for _, v := range args { + switch v := v.(type) { + case fmt.Stringer, fmt.GoStringer, LogStringer: + m.args = append(m.args, &frozenMsgArg{m: m, v: v}) + default: + m.args = append(m.args, v) + } + } +} + +func (m *frozenMsgFormatter) doFormat() string { + res := fmt.Sprintf(m.format, m.args...) + for i := range m.args { + if arg, ok := m.args[i].(*frozenMsgArg); ok { + if arg.processed { + m.args[i] = arg.s + } else { + switch v := arg.v.(type) { + case LogStringer: + m.args[i] = v.LogString() + case fmt.GoStringer: // GoString() is for "%#v" only, but it's also fine to freeze the argument by it + m.args[i] = v.GoString() + case fmt.Stringer: + m.args[i] = v.String() + default: + m.args[i] = v + } + } + } + } + return res +} + +func frozenMsgFormat(format string, args ...any) (msg string, frozenArgs []any) { + m := frozenMsgFormatter{format: format} + m.addArgs(args...) + msg = m.doFormat() + return msg, m.args +} + +// Copy of cheap integer to fixed-width decimal to ascii from logger. +func itoa(buf []byte, i, wid int) []byte { + var s [20]byte + bp := len(s) - 1 + for i >= 10 || wid > 1 { + wid-- + q := i / 10 + s[bp] = byte('0' + i - q*10) + bp-- + i = q + } + // i < 10 + s[bp] = byte('0' + i) + return append(buf, s[bp:]...) +} + +// EventFormatTextMessage makes the log message for a writer with its mode. This function is a copy of the original package +func EventFormatTextMessage(mode *WriterMode, event *Event, buf []byte) []byte { + buf = append(buf, mode.Prefix...) + t := event.Time + if mode.Flags&(Ldate|Ltime|Lmicroseconds) != 0 { + if mode.Colorize { + buf = append(buf, fgCyanBytes...) + } + if mode.Flags&LUTC != 0 { + t = t.UTC() + } + if mode.Flags&Ldate != 0 { + year, month, day := t.Date() + buf = itoa(buf, year, 4) + buf = append(buf, '/') + buf = itoa(buf, int(month), 2) + buf = append(buf, '/') + buf = itoa(buf, day, 2) + buf = append(buf, ' ') + } + if mode.Flags&(Ltime|Lmicroseconds) != 0 { + hour, min, sec := t.Clock() + buf = itoa(buf, hour, 2) + buf = append(buf, ':') + buf = itoa(buf, min, 2) + buf = append(buf, ':') + buf = itoa(buf, sec, 2) + if mode.Flags&Lmicroseconds != 0 { + buf = append(buf, '.') + buf = itoa(buf, t.Nanosecond()/1e3, 6) + } + buf = append(buf, ' ') + } + if mode.Colorize { + buf = append(buf, resetBytes...) + } + + } + if mode.Flags&(Lshortfile|Llongfile) != 0 { + if mode.Colorize { + buf = append(buf, fgGreenBytes...) + } + file := event.Filename + if mode.Flags&Lmedfile == Lmedfile { + startIndex := len(file) - 20 + if startIndex > 0 { + file = "..." + file[startIndex:] + } + } else if mode.Flags&Lshortfile != 0 { + startIndex := strings.LastIndexByte(file, '/') + if startIndex > 0 && startIndex < len(file) { + file = file[startIndex+1:] + } + } + buf = append(buf, file...) + buf = append(buf, ':') + buf = itoa(buf, event.Line, -1) + if mode.Flags&(Lfuncname|Lshortfuncname) != 0 { + buf = append(buf, ':') + } else { + if mode.Colorize { + buf = append(buf, resetBytes...) + } + buf = append(buf, ' ') + } + } + if mode.Flags&(Lfuncname|Lshortfuncname) != 0 { + if mode.Colorize { + buf = append(buf, fgGreenBytes...) + } + funcname := event.Caller + if mode.Flags&Lshortfuncname != 0 { + lastIndex := strings.LastIndexByte(funcname, '.') + if lastIndex > 0 && len(funcname) > lastIndex+1 { + funcname = funcname[lastIndex+1:] + } + } + buf = append(buf, funcname...) + if mode.Colorize { + buf = append(buf, resetBytes...) + } + buf = append(buf, ' ') + } + + if mode.Flags&(Llevel|Llevelinitial) != 0 { + level := strings.ToUpper(event.Level.String()) + if mode.Colorize { + buf = append(buf, ColorBytes(levelToColor[event.Level]...)...) + } + buf = append(buf, '[') + if mode.Flags&Llevelinitial != 0 { + buf = append(buf, level[0]) + } else { + buf = append(buf, level...) + } + buf = append(buf, ']') + if mode.Colorize { + buf = append(buf, resetBytes...) + } + buf = append(buf, ' ') + } + + msg := []byte(event.Msg) + if mode.Colorize { + hasColorValue := false + for _, v := range event.MsgFrozenArgs { + if _, hasColorValue = v.(*ColoredValue); hasColorValue { + break + } + } + if hasColorValue { + msg = []byte(fmt.Sprintf(event.MsgFormat, event.MsgFrozenArgs...)) + } + } + if len(msg) > 0 && msg[len(msg)-1] == '\n' { + msg = msg[:len(msg)-1] + } + + if mode.Flags&Lgopid == Lgopid { + if event.GoroutinePid != "" { + buf = append(buf, '[') + if mode.Colorize { + buf = append(buf, ColorBytes(FgHiYellow)...) + } + buf = append(buf, event.GoroutinePid...) + if mode.Colorize { + buf = append(buf, resetBytes...) + } + buf = append(buf, ']', ' ') + } + } + buf = append(buf, msg...) + + if event.Stacktrace != "" && mode.StacktraceLevel <= event.Level { + lines := bytes.Split([]byte(event.Stacktrace), []byte("\n")) + if len(lines) > 1 { + for _, line := range lines { + buf = append(buf, "\n\t"...) + buf = append(buf, line...) + } + } + buf = append(buf, '\n') + } + buf = append(buf, '\n') + return buf +} diff --git a/modules/log/event_writer.go b/modules/log/event_writer.go new file mode 100644 index 000000000000..9568df1ca328 --- /dev/null +++ b/modules/log/event_writer.go @@ -0,0 +1,51 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "fmt" +) + +type EventWriter interface { + EventWriterBase +} + +type EventWriterProvider func(name string, mode WriterMode) EventWriter + +var eventWriterProviders = map[string]EventWriterProvider{} + +func RegisterEventWriter(writerType string, p EventWriterProvider) { + eventWriterProviders[writerType] = p +} + +func HasEventWriter(writerType string) bool { + _, ok := eventWriterProviders[writerType] + return ok +} + +type WriterMode struct { + WriterType string + // ModeName string + + BufferLen int + + Level Level + + Prefix string + Colorize bool + Flags int + + Expression string + + StacktraceLevel Level + + WriterOption any +} + +func NewEventWriter(name string, mode WriterMode) (EventWriter, error) { + if p, ok := eventWriterProviders[mode.WriterType]; ok { + return p(name, mode), nil + } + return nil, fmt.Errorf("unknown event writer type %q for writer %q", mode.WriterType, name) +} diff --git a/modules/log/event_writer_base.go b/modules/log/event_writer_base.go new file mode 100644 index 000000000000..770b4a1e00d5 --- /dev/null +++ b/modules/log/event_writer_base.go @@ -0,0 +1,136 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "context" + "fmt" + "io" + "regexp" + "time" +) + +type EventWriterBase interface { + Base() *EventWriterBaseImpl + GetWriterType() string + GetWriterName() string + GetLevel() Level + + Run(ctx context.Context) +} + +type EventWriterBaseImpl struct { + LoggerImpl *LoggerImpl + + Name string + Mode *WriterMode + Queue chan *Event + + Formatter EventFormatter // format the Event to a message and write it to output + OutputWriteCloser io.WriteCloser // it will be closed when the event writer is stopped + + stopped chan struct{} +} + +var _ EventWriterBase = (*EventWriterBaseImpl)(nil) + +func (b *EventWriterBaseImpl) Base() *EventWriterBaseImpl { + return b +} + +func (b *EventWriterBaseImpl) GetWriterType() string { + return b.Mode.WriterType +} + +func (b *EventWriterBaseImpl) GetWriterName() string { + return b.Name +} + +func (b *EventWriterBaseImpl) GetLevel() Level { + return b.Mode.Level +} + +func (b *EventWriterBaseImpl) Run(ctx context.Context) { + defer b.OutputWriteCloser.Close() + + var exprRegexp *regexp.Regexp + var err error + if b.Mode.Expression != "" { + if exprRegexp, err = regexp.Compile(b.Mode.Expression); err != nil { + FallbackErrorf("unable to compile expression %q for writer %q: %v", b.Mode.Expression, b.Name, err) + } + } + + var buf []byte + for { + pause := b.LoggerImpl.GetPauseChan() + if pause != nil { + select { + case <-pause: + case <-ctx.Done(): + return + } + } + select { + case <-ctx.Done(): + return + case event, ok := <-b.Queue: + if !ok { + return + } + + if exprRegexp != nil { + matched := exprRegexp.Match([]byte(fmt.Sprintf("%s:%d:%s", event.Filename, event.Line, event.Caller))) || + exprRegexp.Match([]byte(event.Msg)) + if !matched { + continue + } + } + + buf = EventFormatTextMessage(b.Mode, event, buf[:0]) + _, err := b.OutputWriteCloser.Write(buf) + if err != nil { + FallbackErrorf("unable to write log message of %q (%v): %s", b.Name, err, string(buf)) + } + if len(buf) > 2048 { + buf = nil // do not waste too much memory + } + } + } +} + +func NewEventWriterBase(name string, mode WriterMode) *EventWriterBaseImpl { + if mode.BufferLen == 0 { + mode.BufferLen = 1000 + } + if mode.Level == UNDEFINED { + mode.Level = INFO + } + if mode.StacktraceLevel == UNDEFINED { + mode.StacktraceLevel = NONE + } + b := &EventWriterBaseImpl{ + Name: name, + Mode: &mode, + Queue: make(chan *Event, mode.BufferLen), + stopped: make(chan struct{}), + } + return b +} + +func eventWriterStartGo(ctx context.Context, w EventWriter) { + go func() { + defer close(w.Base().stopped) + w.Run(ctx) + }() +} + +func eventWriterStopWait(w EventWriter) { + close(w.Base().Queue) + select { + case <-w.Base().stopped: + case <-time.After(2 * time.Second): + FallbackErrorf("unable to stop log writer %q in time, skip", w.GetWriterName()) + } +} diff --git a/modules/log/event_writer_conn.go b/modules/log/event_writer_conn.go new file mode 100644 index 000000000000..68fdf2451b42 --- /dev/null +++ b/modules/log/event_writer_conn.go @@ -0,0 +1,112 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "io" + "net" +) + +type WriterConnOption struct { + Addr string + Protocol string + Reconnect bool + ReconnectOnMsg bool +} + +type eventWriterConn struct { + *EventWriterBaseImpl + connWriter connWriter +} + +var _ EventWriter = (*eventWriterConn)(nil) + +func NewEventWriterConn(name string, mode WriterMode) EventWriter { + w := &eventWriterConn{EventWriterBaseImpl: NewEventWriterBase(name, mode)} + opt := mode.WriterOption.(WriterConnOption) + w.connWriter = connWriter{ + ReconnectOnMsg: opt.ReconnectOnMsg, + Reconnect: opt.Reconnect, + Net: opt.Protocol, + Addr: opt.Addr, + } + w.Formatter = EventFormatTextMessage + w.OutputWriteCloser = &w.connWriter + return w +} + +func init() { + RegisterEventWriter("conn", NewEventWriterConn) +} + +// below is copied from old code + +type connWriter struct { + innerWriter io.WriteCloser + + ReconnectOnMsg bool + Reconnect bool + Net string `json:"net"` + Addr string `json:"addr"` +} + +var _ io.WriteCloser = (*connWriter)(nil) + +// Close the inner writer +func (i *connWriter) Close() error { + if i.innerWriter != nil { + return i.innerWriter.Close() + } + return nil +} + +// Write the data to the connection +func (i *connWriter) Write(p []byte) (int, error) { + if i.neededConnectOnMsg() { + if err := i.connect(); err != nil { + return 0, err + } + } + + if i.ReconnectOnMsg { + defer i.innerWriter.Close() + } + + return i.innerWriter.Write(p) +} + +func (i *connWriter) neededConnectOnMsg() bool { + if i.Reconnect { + i.Reconnect = false + return true + } + + if i.innerWriter == nil { + return true + } + + return i.ReconnectOnMsg +} + +func (i *connWriter) connect() error { + if i.innerWriter != nil { + _ = i.innerWriter.Close() + i.innerWriter = nil + } + + conn, err := net.Dial(i.Net, i.Addr) + if err != nil { + return err + } + + if tcpConn, ok := conn.(*net.TCPConn); ok { + err = tcpConn.SetKeepAlive(true) + if err != nil { + return err + } + } + + i.innerWriter = conn + return nil +} diff --git a/modules/log/event_writer_console.go b/modules/log/event_writer_console.go new file mode 100644 index 000000000000..5b0445053e07 --- /dev/null +++ b/modules/log/event_writer_console.go @@ -0,0 +1,41 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "io" + "os" +) + +type WriterConsoleOption struct { + Stderr bool +} + +type eventWriterConsole struct { + *EventWriterBaseImpl +} + +var _ EventWriter = (*eventWriterConsole)(nil) + +type nopCloser struct { + io.Writer +} + +func (nopCloser) Close() error { return nil } + +func NewEventWriterConsole(name string, mode WriterMode) EventWriter { + w := &eventWriterConsole{EventWriterBaseImpl: NewEventWriterBase(name, mode)} + opt := mode.WriterOption.(WriterConsoleOption) + w.Formatter = EventFormatTextMessage + if opt.Stderr { + w.OutputWriteCloser = nopCloser{os.Stderr} + } else { + w.OutputWriteCloser = nopCloser{os.Stdout} + } + return w +} + +func init() { + RegisterEventWriter("console", NewEventWriterConsole) +} diff --git a/modules/log/event_writer_file.go b/modules/log/event_writer_file.go new file mode 100644 index 000000000000..779d6f714dae --- /dev/null +++ b/modules/log/event_writer_file.go @@ -0,0 +1,49 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "code.gitea.io/gitea/modules/util/rotatingfilewriter" +) + +type WriterFileOption struct { + FileName string + MaxSize int64 + LogRotate bool + DailyRotate bool + MaxDays int + Compress bool + CompressionLevel int +} + +type eventWriterFile struct { + *EventWriterBaseImpl + fileWriter *rotatingfilewriter.RotatingFileWriter +} + +var _ EventWriter = (*eventWriterFile)(nil) + +func NewEventWriterFile(name string, mode WriterMode) EventWriter { + w := &eventWriterFile{EventWriterBaseImpl: NewEventWriterBase(name, mode)} + opt := mode.WriterOption.(WriterFileOption) + var err error + w.fileWriter, err = rotatingfilewriter.Open(opt.FileName, &rotatingfilewriter.Options{ + Rotate: opt.LogRotate, + MaximumSize: opt.MaxSize, + RotateDaily: opt.DailyRotate, + KeepDays: opt.MaxDays, + Compress: opt.Compress, + CompressionLevel: opt.CompressionLevel, + }) + if err != nil { + FallbackErrorf("unable to open log file %q: %v", opt.FileName, err) + } + w.Formatter = EventFormatTextMessage + w.OutputWriteCloser = w.fileWriter + return w +} + +func init() { + RegisterEventWriter("file", NewEventWriterFile) +} diff --git a/modules/log/file.go b/modules/log/file.go deleted file mode 100644 index 2ec6de450c7e..000000000000 --- a/modules/log/file.go +++ /dev/null @@ -1,283 +0,0 @@ -// Copyright 2014 The Gogs Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "bufio" - "compress/gzip" - "errors" - "fmt" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/util" -) - -// FileLogger implements LoggerProvider. -// It writes messages by lines limit, file size limit, or time frequency. -type FileLogger struct { - WriterLogger - mw *MuxWriter - // The opened file - Filename string `json:"filename"` - - // Rotate at size - Maxsize int `json:"maxsize"` - maxsizeCursize int - - // Rotate daily - Daily bool `json:"daily"` - Maxdays int64 `json:"maxdays"` - dailyOpenDate int - - Rotate bool `json:"rotate"` - - Compress bool `json:"compress"` - CompressionLevel int `json:"compressionLevel"` - - startLock sync.Mutex // Only one log can write to the file -} - -// MuxWriter an *os.File writer with locker. -type MuxWriter struct { - mu sync.Mutex - fd *os.File - owner *FileLogger -} - -// Write writes to os.File. -func (mw *MuxWriter) Write(b []byte) (int, error) { - mw.mu.Lock() - defer mw.mu.Unlock() - mw.owner.docheck(len(b)) - return mw.fd.Write(b) -} - -// Close the internal writer -func (mw *MuxWriter) Close() error { - return mw.fd.Close() -} - -// SetFd sets os.File in writer. -func (mw *MuxWriter) SetFd(fd *os.File) { - if mw.fd != nil { - mw.fd.Close() - } - mw.fd = fd -} - -// NewFileLogger create a FileLogger returning as LoggerProvider. -func NewFileLogger() LoggerProvider { - log := &FileLogger{ - Filename: "", - Maxsize: 1 << 28, // 256 MB - Daily: true, - Maxdays: 7, - Rotate: true, - Compress: true, - CompressionLevel: gzip.DefaultCompression, - } - log.Level = TRACE - // use MuxWriter instead direct use os.File for lock write when rotate - log.mw = new(MuxWriter) - log.mw.owner = log - - return log -} - -// Init file logger with json config. -// config like: -// -// { -// "filename":"log/gogs.log", -// "maxsize":1<<30, -// "daily":true, -// "maxdays":15, -// "rotate":true -// } -func (log *FileLogger) Init(config string) error { - if err := json.Unmarshal([]byte(config), log); err != nil { - return fmt.Errorf("Unable to parse JSON: %w", err) - } - if len(log.Filename) == 0 { - return errors.New("config must have filename") - } - // set MuxWriter as Logger's io.Writer - log.NewWriterLogger(log.mw) - return log.StartLogger() -} - -// StartLogger start file logger. create log file and set to locker-inside file writer. -func (log *FileLogger) StartLogger() error { - fd, err := log.createLogFile() - if err != nil { - return err - } - log.mw.SetFd(fd) - return log.initFd() -} - -func (log *FileLogger) docheck(size int) { - log.startLock.Lock() - defer log.startLock.Unlock() - if log.Rotate && ((log.Maxsize > 0 && log.maxsizeCursize >= log.Maxsize) || - (log.Daily && time.Now().Day() != log.dailyOpenDate)) { - if err := log.DoRotate(); err != nil { - fmt.Fprintf(os.Stderr, "FileLogger(%q): %s\n", log.Filename, err) - return - } - } - log.maxsizeCursize += size -} - -func (log *FileLogger) createLogFile() (*os.File, error) { - // Open the log file - return os.OpenFile(log.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o660) -} - -func (log *FileLogger) initFd() error { - fd := log.mw.fd - finfo, err := fd.Stat() - if err != nil { - return fmt.Errorf("get stat: %w", err) - } - log.maxsizeCursize = int(finfo.Size()) - log.dailyOpenDate = time.Now().Day() - return nil -} - -// DoRotate means it need to write file in new file. -// new file name like xx.log.2013-01-01.2 -func (log *FileLogger) DoRotate() error { - _, err := os.Lstat(log.Filename) - if err == nil { // file exists - // Find the next available number - num := 1 - fname := "" - for ; err == nil && num <= 999; num++ { - fname = log.Filename + fmt.Sprintf(".%s.%03d", time.Now().Format("2006-01-02"), num) - _, err = os.Lstat(fname) - if log.Compress && err != nil { - _, err = os.Lstat(fname + ".gz") - } - } - // return error if the last file checked still existed - if err == nil { - return fmt.Errorf("rotate: cannot find free log number to rename %s", log.Filename) - } - - fd := log.mw.fd - fd.Close() - - // close fd before rename - // Rename the file to its newfound home - if err = util.Rename(log.Filename, fname); err != nil { - return fmt.Errorf("Rotate: %w", err) - } - - if log.Compress { - go compressOldLogFile(fname, log.CompressionLevel) //nolint:errcheck - } - - // re-start logger - if err = log.StartLogger(); err != nil { - return fmt.Errorf("Rotate StartLogger: %w", err) - } - - go log.deleteOldLog() - } - - return nil -} - -func compressOldLogFile(fname string, compressionLevel int) error { - reader, err := os.Open(fname) - if err != nil { - return err - } - defer reader.Close() - buffer := bufio.NewReader(reader) - fw, err := os.OpenFile(fname+".gz", os.O_WRONLY|os.O_CREATE, 0o660) - if err != nil { - return err - } - defer fw.Close() - zw, err := gzip.NewWriterLevel(fw, compressionLevel) - if err != nil { - return err - } - defer zw.Close() - _, err = buffer.WriteTo(zw) - if err != nil { - zw.Close() - fw.Close() - util.Remove(fname + ".gz") //nolint:errcheck - return err - } - reader.Close() - return util.Remove(fname) -} - -func (log *FileLogger) deleteOldLog() { - dir := filepath.Dir(log.Filename) - _ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) (returnErr error) { - defer func() { - if r := recover(); r != nil { - returnErr = fmt.Errorf("Unable to delete old log '%s', error: %+v", path, r) - } - }() - - if err != nil { - return err - } - if d.IsDir() { - return nil - } - info, err := d.Info() - if err != nil { - return err - } - if info.ModTime().Unix() < (time.Now().Unix() - 60*60*24*log.Maxdays) { - if strings.HasPrefix(filepath.Base(path), filepath.Base(log.Filename)) { - if err := util.Remove(path); err != nil { - returnErr = fmt.Errorf("Failed to remove %s: %w", path, err) - } - } - } - return returnErr - }) -} - -// Flush flush file logger. -// there are no buffering messages in file logger in memory. -// flush file means sync file from disk. -func (log *FileLogger) Flush() { - _ = log.mw.fd.Sync() -} - -// ReleaseReopen releases and reopens log files -func (log *FileLogger) ReleaseReopen() error { - closingErr := log.mw.fd.Close() - startingErr := log.StartLogger() - if startingErr != nil { - if closingErr != nil { - return fmt.Errorf("Error during closing: %v Error during starting: %v", closingErr, startingErr) - } - return startingErr - } - return closingErr -} - -// GetName returns the default name for this implementation -func (log *FileLogger) GetName() string { - return "file" -} - -func init() { - Register("file", NewFileLogger) -} diff --git a/modules/log/file_test.go b/modules/log/file_test.go deleted file mode 100644 index 34f74598067c..000000000000 --- a/modules/log/file_test.go +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "compress/gzip" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestFileLoggerFails(t *testing.T) { - tmpDir := t.TempDir() - - prefix := "TestPrefix " - level := INFO - flags := LstdFlags | LUTC | Lfuncname - // filename := filepath.Join(tmpDir, "test.log") - - fileLogger := NewFileLogger() - // realFileLogger, ok := fileLogger.(*FileLogger) - // assert.True(t, ok) - - // Fail if there is bad json - err := fileLogger.Init("{") - assert.Error(t, err) - - // Fail if there is no filename - err = fileLogger.Init(fmt.Sprintf("{\"prefix\":\"%s\",\"level\":\"%s\",\"flags\":%d,\"filename\":\"%s\"}", prefix, level.String(), flags, "")) - assert.Error(t, err) - - // Fail if the file isn't a filename - err = fileLogger.Init(fmt.Sprintf("{\"prefix\":\"%s\",\"level\":\"%s\",\"flags\":%d,\"filename\":\"%s\"}", prefix, level.String(), flags, filepath.ToSlash(tmpDir))) - assert.Error(t, err) -} - -func TestFileLogger(t *testing.T) { - tmpDir := t.TempDir() - - prefix := "TestPrefix " - level := INFO - flags := LstdFlags | LUTC | Lfuncname - filename := filepath.Join(tmpDir, "test.log") - - fileLogger := NewFileLogger() - realFileLogger, ok := fileLogger.(*FileLogger) - assert.True(t, ok) - - location, _ := time.LoadLocation("EST") - - date := time.Date(2019, time.January, 13, 22, 3, 30, 15, location) - - dateString := date.UTC().Format("2006/01/02 15:04:05") - - event := Event{ - level: INFO, - msg: "TEST MSG", - caller: "CALLER", - filename: "FULL/FILENAME", - line: 1, - time: date, - } - - expected := fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - - fileLogger.Init(fmt.Sprintf("{\"prefix\":\"%s\",\"level\":\"%s\",\"flags\":%d,\"filename\":\"%s\",\"maxsize\":%d,\"compress\":false}", prefix, level.String(), flags, filepath.ToSlash(filename), len(expected)*2)) - - assert.Equal(t, flags, realFileLogger.Flags) - assert.Equal(t, level, realFileLogger.Level) - assert.Equal(t, level, fileLogger.GetLevel()) - - fileLogger.LogEvent(&event) - fileLogger.Flush() - logData, err := os.ReadFile(filename) - assert.NoError(t, err) - assert.Equal(t, expected, string(logData)) - - event.level = DEBUG - fileLogger.LogEvent(&event) - fileLogger.Flush() - logData, err = os.ReadFile(filename) - assert.NoError(t, err) - assert.Equal(t, expected, string(logData)) - - event.level = TRACE - fileLogger.LogEvent(&event) - fileLogger.Flush() - logData, err = os.ReadFile(filename) - assert.NoError(t, err) - assert.Equal(t, expected, string(logData)) - - event.level = WARN - expected += fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - fileLogger.LogEvent(&event) - fileLogger.Flush() - logData, err = os.ReadFile(filename) - assert.NoError(t, err) - assert.Equal(t, expected, string(logData)) - - // Should rotate - fileLogger.LogEvent(&event) - fileLogger.Flush() - logData, err = os.ReadFile(filename + fmt.Sprintf(".%s.%03d", time.Now().Format("2006-01-02"), 1)) - assert.NoError(t, err) - assert.Equal(t, expected, string(logData)) - - logData, err = os.ReadFile(filename) - assert.NoError(t, err) - expected = fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - assert.Equal(t, expected, string(logData)) - - for num := 2; num <= 999; num++ { - file, err := os.OpenFile(filename+fmt.Sprintf(".%s.%03d", time.Now().Format("2006-01-02"), num), os.O_RDONLY|os.O_CREATE, 0o666) - assert.NoError(t, err) - file.Close() - } - err = realFileLogger.DoRotate() - assert.Error(t, err) - - expected += fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - fileLogger.LogEvent(&event) - fileLogger.Flush() - logData, err = os.ReadFile(filename) - assert.NoError(t, err) - assert.Equal(t, expected, string(logData)) - - // Should fail to rotate - expected += fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - fileLogger.LogEvent(&event) - fileLogger.Flush() - logData, err = os.ReadFile(filename) - assert.NoError(t, err) - assert.Equal(t, expected, string(logData)) - - fileLogger.Close() -} - -func TestCompressFileLogger(t *testing.T) { - tmpDir := t.TempDir() - - prefix := "TestPrefix " - level := INFO - flags := LstdFlags | LUTC | Lfuncname - filename := filepath.Join(tmpDir, "test.log") - - fileLogger := NewFileLogger() - realFileLogger, ok := fileLogger.(*FileLogger) - assert.True(t, ok) - - location, _ := time.LoadLocation("EST") - - date := time.Date(2019, time.January, 13, 22, 3, 30, 15, location) - - dateString := date.UTC().Format("2006/01/02 15:04:05") - - event := Event{ - level: INFO, - msg: "TEST MSG", - caller: "CALLER", - filename: "FULL/FILENAME", - line: 1, - time: date, - } - - expected := fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - - fileLogger.Init(fmt.Sprintf("{\"prefix\":\"%s\",\"level\":\"%s\",\"flags\":%d,\"filename\":\"%s\",\"maxsize\":%d,\"compress\":true}", prefix, level.String(), flags, filepath.ToSlash(filename), len(expected)*2)) - - fileLogger.LogEvent(&event) - fileLogger.Flush() - logData, err := os.ReadFile(filename) - assert.NoError(t, err) - assert.Equal(t, expected, string(logData)) - - event.level = WARN - expected += fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - fileLogger.LogEvent(&event) - fileLogger.Flush() - logData, err = os.ReadFile(filename) - assert.NoError(t, err) - assert.Equal(t, expected, string(logData)) - - // Should rotate - fileLogger.LogEvent(&event) - fileLogger.Flush() - - for num := 2; num <= 999; num++ { - file, err := os.OpenFile(filename+fmt.Sprintf(".%s.%03d.gz", time.Now().Format("2006-01-02"), num), os.O_RDONLY|os.O_CREATE, 0o666) - assert.NoError(t, err) - file.Close() - } - err = realFileLogger.DoRotate() - assert.Error(t, err) -} - -func TestCompressOldFile(t *testing.T) { - tmpDir := t.TempDir() - fname := filepath.Join(tmpDir, "test") - nonGzip := filepath.Join(tmpDir, "test-nonGzip") - - f, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, 0o660) - assert.NoError(t, err) - ng, err := os.OpenFile(nonGzip, os.O_CREATE|os.O_WRONLY, 0o660) - assert.NoError(t, err) - - for i := 0; i < 999; i++ { - f.WriteString("This is a test file\n") - ng.WriteString("This is a test file\n") - } - f.Close() - ng.Close() - - err = compressOldLogFile(fname, -1) - assert.NoError(t, err) - - _, err = os.Lstat(fname + ".gz") - assert.NoError(t, err) - - f, err = os.Open(fname + ".gz") - assert.NoError(t, err) - zr, err := gzip.NewReader(f) - assert.NoError(t, err) - data, err := io.ReadAll(zr) - assert.NoError(t, err) - original, err := os.ReadFile(nonGzip) - assert.NoError(t, err) - assert.Equal(t, original, data) -} diff --git a/modules/log/flags.go b/modules/log/flags.go index 4a3732600bb9..0a1304bfce88 100644 --- a/modules/log/flags.go +++ b/modules/log/flags.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2023 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package log @@ -25,12 +25,11 @@ const ( LUTC // if Ldate or Ltime is set, use UTC rather than the local time zone Llevelinitial // Initial character of the provided level in brackets eg. [I] for info Llevel // Provided level in brackets [INFO] + Lgopid - // Last 20 characters of the filename - Lmedfile = Lshortfile | Llongfile + Lmedfile = Lshortfile | Llongfile // last 20 characters of the filename - // LstdFlags is the initial value for the standard logger - LstdFlags = Ldate | Ltime | Lmedfile | Lshortfuncname | Llevelinitial + LstdFlags = Ldate | Ltime | Lmedfile | Lshortfuncname | Llevelinitial // default ) var flagFromString = map[string]int{ @@ -47,10 +46,10 @@ var flagFromString = map[string]int{ "level": Llevel, "medfile": Lmedfile, "stdflags": LstdFlags, + "gopid": Lgopid, } -// FlagsFromString takes a comma separated list of flags and returns -// the flags for this string +// FlagsFromString takes a comma separated list of flags and returns the flags for this string func FlagsFromString(from string) int { flags := 0 for _, flag := range strings.Split(strings.ToLower(from), ",") { diff --git a/modules/log/init.go b/modules/log/init.go new file mode 100644 index 000000000000..91ee81447618 --- /dev/null +++ b/modules/log/init.go @@ -0,0 +1,32 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "runtime" + "strings" + + "code.gitea.io/gitea/modules/process" +) + +var prefix string + +func init() { + _, filename, _, _ := runtime.Caller(0) + prefix = strings.TrimSuffix(filename, "modules/log/init.go") + if prefix == filename { + // in case the source code file is moved, we can not trim the suffix, the code above should also be updated. + panic("unable to detect correct package prefix, please update file: " + filename) + } + + process.Trace = func(start bool, pid process.IDType, description string, parentPID process.IDType, typ string) { + if start && parentPID != "" { + Log(1, TRACE, "Start %s: %s (from %s) (%s)", NewColoredValue(pid, FgHiYellow), description, NewColoredValue(parentPID, FgYellow), NewColoredValue(typ, Reset)) + } else if start { + Log(1, TRACE, "Start %s: %s (%s)", NewColoredValue(pid, FgHiYellow), description, NewColoredValue(typ, Reset)) + } else { + Log(1, TRACE, "Done %s: %s", NewColoredValue(pid, FgHiYellow), NewColoredValue(description, Reset)) + } + } +} diff --git a/modules/log/level.go b/modules/log/level.go index 3c8a736b30a6..0a870e1249dc 100644 --- a/modules/log/level.go +++ b/modules/log/level.go @@ -5,8 +5,6 @@ package log import ( "bytes" - "fmt" - "os" "strings" "code.gitea.io/gitea/modules/json" @@ -16,53 +14,50 @@ import ( type Level int const ( - // TRACE represents the lowest log level - TRACE Level = iota - // DEBUG is for debug logging + UNDEFINED Level = iota + TRACE DEBUG - // INFO is for information INFO - // WARN is for warning information WARN - // ERROR is for error reporting ERROR - // CRITICAL is for critical errors - CRITICAL - // FATAL is for fatal errors FATAL - // NONE is for no logging NONE ) +const CRITICAL = ERROR // most logger frameworks doesn't support CRITICAL, and it doesn't seem useful + var toString = map[Level]string{ - TRACE: "trace", - DEBUG: "debug", - INFO: "info", - WARN: "warn", - ERROR: "error", - CRITICAL: "critical", - FATAL: "fatal", - NONE: "none", + UNDEFINED: "undefined", + + TRACE: "trace", + DEBUG: "debug", + INFO: "info", + WARN: "warn", + ERROR: "error", + FATAL: "fatal", + NONE: "none", } var toLevel = map[string]Level{ - "trace": TRACE, - "debug": DEBUG, - "info": INFO, - "warn": WARN, - "error": ERROR, - "critical": CRITICAL, - "fatal": FATAL, - "none": NONE, + "undefined": UNDEFINED, + + "trace": TRACE, + "debug": DEBUG, + "info": INFO, + "warn": WARN, + "error": ERROR, + "fatal": FATAL, + "none": NONE, } -// Levels returns all the possible logging levels -func Levels() []string { - keys := make([]string, 0) - for key := range toLevel { - keys = append(keys, key) - } - return keys +var levelToColor = map[Level][]ColorAttribute{ + TRACE: {Bold, FgCyan}, + DEBUG: {Bold, FgBlue}, + INFO: {Bold, FgGreen}, + WARN: {Bold, FgYellow}, + ERROR: {Bold, FgRed}, + FATAL: {Bold, BgRed}, + NONE: {Reset}, } func (l Level) String() string { @@ -73,14 +68,13 @@ func (l Level) String() string { return "info" } -// Color returns the color string for this Level -func (l Level) Color() *[]byte { +func (l Level) ColorAttributes() []ColorAttribute { color, ok := levelToColor[l] if ok { - return &(color) + return color } none := levelToColor[NONE] - return &none + return none } // MarshalJSON takes a Level and turns it into text @@ -91,31 +85,29 @@ func (l Level) MarshalJSON() ([]byte, error) { return buffer.Bytes(), nil } -// FromString takes a level string and returns a Level -func FromString(level string) Level { - temp, ok := toLevel[strings.ToLower(level)] - if !ok { - return INFO - } - return temp -} - // UnmarshalJSON takes text and turns it into a Level func (l *Level) UnmarshalJSON(b []byte) error { - var tmp interface{} + var tmp any err := json.Unmarshal(b, &tmp) if err != nil { - fmt.Fprintf(os.Stderr, "Err: %v", err) return err } switch v := tmp.(type) { case string: - *l = FromString(v) + *l = LevelFromString(v) case int: - *l = FromString(Level(v).String()) + *l = LevelFromString(Level(v).String()) default: *l = INFO } return nil } + +// LevelFromString takes a level string and returns a Level +func LevelFromString(level string) Level { + if l, ok := toLevel[strings.ToLower(level)]; ok { + return l + } + return INFO +} diff --git a/modules/log/log.go b/modules/log/log.go deleted file mode 100644 index eee2728bf68d..000000000000 --- a/modules/log/log.go +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright 2014 The Gogs Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "fmt" - "os" - "runtime" - "strings" - "sync" - - "code.gitea.io/gitea/modules/process" -) - -type loggerMap struct { - sync.Map -} - -func (m *loggerMap) Load(k string) (*MultiChannelledLogger, bool) { - v, ok := m.Map.Load(k) - if !ok { - return nil, false - } - l, ok := v.(*MultiChannelledLogger) - return l, ok -} - -func (m *loggerMap) Store(k string, v *MultiChannelledLogger) { - m.Map.Store(k, v) -} - -func (m *loggerMap) Delete(k string) { - m.Map.Delete(k) -} - -var ( - // DEFAULT is the name of the default logger - DEFAULT = "default" - // NamedLoggers map of named loggers - NamedLoggers loggerMap - prefix string -) - -// NewLogger create a logger for the default logger -func NewLogger(bufLen int64, name, provider, config string) *MultiChannelledLogger { - err := NewNamedLogger(DEFAULT, bufLen, name, provider, config) - if err != nil { - CriticalWithSkip(1, "Unable to create default logger: %v", err) - panic(err) - } - l, _ := NamedLoggers.Load(DEFAULT) - return l -} - -// NewNamedLogger creates a new named logger for a given configuration -func NewNamedLogger(name string, bufLen int64, subname, provider, config string) error { - logger, ok := NamedLoggers.Load(name) - if !ok { - logger = newLogger(name, bufLen) - NamedLoggers.Store(name, logger) - } - - return logger.SetLogger(subname, provider, config) -} - -// DelNamedLogger closes and deletes the named logger -func DelNamedLogger(name string) { - l, ok := NamedLoggers.Load(name) - if ok { - NamedLoggers.Delete(name) - l.Close() - } -} - -// DelLogger removes the named sublogger from the default logger -func DelLogger(name string) error { - logger, _ := NamedLoggers.Load(DEFAULT) - found, err := logger.DelLogger(name) - if !found { - Trace("Log %s not found, no need to delete", name) - } - return err -} - -// GetLogger returns either a named logger or the default logger -func GetLogger(name string) *MultiChannelledLogger { - logger, ok := NamedLoggers.Load(name) - if ok { - return logger - } - logger, _ = NamedLoggers.Load(DEFAULT) - return logger -} - -// GetLevel returns the minimum logger level -func GetLevel() Level { - l, _ := NamedLoggers.Load(DEFAULT) - return l.GetLevel() -} - -// GetStacktraceLevel returns the minimum logger level -func GetStacktraceLevel() Level { - l, _ := NamedLoggers.Load(DEFAULT) - return l.GetStacktraceLevel() -} - -// Trace records trace log -func Trace(format string, v ...interface{}) { - Log(1, TRACE, format, v...) -} - -// IsTrace returns true if at least one logger is TRACE -func IsTrace() bool { - return GetLevel() <= TRACE -} - -// Debug records debug log -func Debug(format string, v ...interface{}) { - Log(1, DEBUG, format, v...) -} - -// IsDebug returns true if at least one logger is DEBUG -func IsDebug() bool { - return GetLevel() <= DEBUG -} - -// Info records info log -func Info(format string, v ...interface{}) { - Log(1, INFO, format, v...) -} - -// IsInfo returns true if at least one logger is INFO -func IsInfo() bool { - return GetLevel() <= INFO -} - -// Warn records warning log -func Warn(format string, v ...interface{}) { - Log(1, WARN, format, v...) -} - -// IsWarn returns true if at least one logger is WARN -func IsWarn() bool { - return GetLevel() <= WARN -} - -// Error records error log -func Error(format string, v ...interface{}) { - Log(1, ERROR, format, v...) -} - -// ErrorWithSkip records error log from "skip" calls back from this function -func ErrorWithSkip(skip int, format string, v ...interface{}) { - Log(skip+1, ERROR, format, v...) -} - -// IsError returns true if at least one logger is ERROR -func IsError() bool { - return GetLevel() <= ERROR -} - -// Critical records critical log -func Critical(format string, v ...interface{}) { - Log(1, CRITICAL, format, v...) -} - -// CriticalWithSkip records critical log from "skip" calls back from this function -func CriticalWithSkip(skip int, format string, v ...interface{}) { - Log(skip+1, CRITICAL, format, v...) -} - -// IsCritical returns true if at least one logger is CRITICAL -func IsCritical() bool { - return GetLevel() <= CRITICAL -} - -// Fatal records fatal log and exit process -func Fatal(format string, v ...interface{}) { - Log(1, FATAL, format, v...) - Close() - os.Exit(1) -} - -// FatalWithSkip records fatal log from "skip" calls back from this function -func FatalWithSkip(skip int, format string, v ...interface{}) { - Log(skip+1, FATAL, format, v...) - Close() - os.Exit(1) -} - -// IsFatal returns true if at least one logger is FATAL -func IsFatal() bool { - return GetLevel() <= FATAL -} - -// Pause pauses all the loggers -func Pause() { - NamedLoggers.Range(func(key, value interface{}) bool { - logger := value.(*MultiChannelledLogger) - logger.Pause() - logger.Flush() - return true - }) -} - -// Resume resumes all the loggers -func Resume() { - NamedLoggers.Range(func(key, value interface{}) bool { - logger := value.(*MultiChannelledLogger) - logger.Resume() - return true - }) -} - -// ReleaseReopen releases and reopens logging files -func ReleaseReopen() error { - var accumulatedErr error - NamedLoggers.Range(func(key, value interface{}) bool { - logger := value.(*MultiChannelledLogger) - if err := logger.ReleaseReopen(); err != nil { - if accumulatedErr == nil { - accumulatedErr = fmt.Errorf("Error reopening %s: %w", key.(string), err) - } else { - accumulatedErr = fmt.Errorf("Error reopening %s: %v & %w", key.(string), err, accumulatedErr) - } - } - return true - }) - return accumulatedErr -} - -// Close closes all the loggers -func Close() { - l, ok := NamedLoggers.Load(DEFAULT) - if !ok { - return - } - NamedLoggers.Delete(DEFAULT) - l.Close() -} - -// Log a message with defined skip and at logging level -// A skip of 0 refers to the caller of this command -func Log(skip int, level Level, format string, v ...interface{}) { - l, ok := NamedLoggers.Load(DEFAULT) - if ok { - l.Log(skip+1, level, format, v...) //nolint:errcheck - } -} - -// LoggerAsWriter is a io.Writer shim around the gitea log -type LoggerAsWriter struct { - ourLoggers []*MultiChannelledLogger - level Level -} - -// NewLoggerAsWriter creates a Writer representation of the logger with setable log level -func NewLoggerAsWriter(level string, ourLoggers ...*MultiChannelledLogger) *LoggerAsWriter { - if len(ourLoggers) == 0 { - l, _ := NamedLoggers.Load(DEFAULT) - ourLoggers = []*MultiChannelledLogger{l} - } - l := &LoggerAsWriter{ - ourLoggers: ourLoggers, - level: FromString(level), - } - return l -} - -// Write implements the io.Writer interface to allow spoofing of chi -func (l *LoggerAsWriter) Write(p []byte) (int, error) { - for _, logger := range l.ourLoggers { - // Skip = 3 because this presumes that we have been called by log.Println() - // If the caller has used log.Output or the like this will be wrong - logger.Log(3, l.level, string(p)) //nolint:errcheck - } - return len(p), nil -} - -// Log takes a given string and logs it at the set log-level -func (l *LoggerAsWriter) Log(msg string) { - for _, logger := range l.ourLoggers { - // Set the skip to reference the call just above this - _ = logger.Log(1, l.level, msg) - } -} - -func init() { - process.Trace = func(start bool, pid process.IDType, description string, parentPID process.IDType, typ string) { - if start && parentPID != "" { - Log(1, TRACE, "Start %s: %s (from %s) (%s)", NewColoredValue(pid, FgHiYellow), description, NewColoredValue(parentPID, FgYellow), NewColoredValue(typ, Reset)) - } else if start { - Log(1, TRACE, "Start %s: %s (%s)", NewColoredValue(pid, FgHiYellow), description, NewColoredValue(typ, Reset)) - } else { - Log(1, TRACE, "Done %s: %s", NewColoredValue(pid, FgHiYellow), NewColoredValue(description, Reset)) - } - } - _, filename, _, _ := runtime.Caller(0) - prefix = strings.TrimSuffix(filename, "modules/log/log.go") - if prefix == filename { - // in case the source code file is moved, we can not trim the suffix, the code above should also be updated. - panic("unable to detect correct package prefix, please update file: " + filename) - } -} diff --git a/modules/log/log_test.go b/modules/log/log_test.go deleted file mode 100644 index 819cdb521fa5..000000000000 --- a/modules/log/log_test.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -func baseConsoleTest(t *testing.T, logger *MultiChannelledLogger) (chan []byte, chan bool) { - written := make(chan []byte) - closed := make(chan bool) - - c := CallbackWriteCloser{ - callback: func(p []byte, close bool) { - written <- p - closed <- close - }, - } - m := logger.MultiChannelledLog - - channelledLog := m.GetEventLogger("console") - assert.NotEmpty(t, channelledLog) - realChanLog, ok := channelledLog.(*ChannelledLog) - assert.True(t, ok) - realCL, ok := realChanLog.loggerProvider.(*ConsoleLogger) - assert.True(t, ok) - assert.Equal(t, INFO, realCL.Level) - realCL.out = c - - format := "test: %s" - args := []interface{}{"A"} - - logger.Log(0, INFO, format, args...) - line := <-written - assert.Contains(t, string(line), fmt.Sprintf(format, args...)) - assert.False(t, <-closed) - - format = "test2: %s" - logger.Warn(format, args...) - line = <-written - - assert.Contains(t, string(line), fmt.Sprintf(format, args...)) - assert.False(t, <-closed) - - format = "testerror: %s" - logger.Error(format, args...) - line = <-written - assert.Contains(t, string(line), fmt.Sprintf(format, args...)) - assert.False(t, <-closed) - return written, closed -} - -func TestNewLoggerUnexported(t *testing.T) { - level := INFO - logger := newLogger("UNEXPORTED", 0) - err := logger.SetLogger("console", "console", fmt.Sprintf(`{"level":"%s"}`, level.String())) - assert.NoError(t, err) - out := logger.MultiChannelledLog.GetEventLogger("console") - assert.NotEmpty(t, out) - chanlog, ok := out.(*ChannelledLog) - assert.True(t, ok) - assert.Equal(t, "console", chanlog.provider) - assert.Equal(t, INFO, logger.GetLevel()) - baseConsoleTest(t, logger) -} - -func TestNewLoggger(t *testing.T) { - level := INFO - logger := NewLogger(0, "console", "console", fmt.Sprintf(`{"level":"%s"}`, level.String())) - - assert.Equal(t, INFO, GetLevel()) - assert.False(t, IsTrace()) - assert.False(t, IsDebug()) - assert.True(t, IsInfo()) - assert.True(t, IsWarn()) - assert.True(t, IsError()) - - written, closed := baseConsoleTest(t, logger) - - format := "test: %s" - args := []interface{}{"A"} - - Log(0, INFO, format, args...) - line := <-written - assert.Contains(t, string(line), fmt.Sprintf(format, args...)) - assert.False(t, <-closed) - - Info(format, args...) - line = <-written - assert.Contains(t, string(line), fmt.Sprintf(format, args...)) - assert.False(t, <-closed) - - go DelLogger("console") - line = <-written - assert.Equal(t, "", string(line)) - assert.True(t, <-closed) -} - -func TestNewLogggerRecreate(t *testing.T) { - level := INFO - NewLogger(0, "console", "console", fmt.Sprintf(`{"level":"%s"}`, level.String())) - - assert.Equal(t, INFO, GetLevel()) - assert.False(t, IsTrace()) - assert.False(t, IsDebug()) - assert.True(t, IsInfo()) - assert.True(t, IsWarn()) - assert.True(t, IsError()) - - format := "test: %s" - args := []interface{}{"A"} - - Log(0, INFO, format, args...) - - NewLogger(0, "console", "console", fmt.Sprintf(`{"level":"%s"}`, level.String())) - - assert.Equal(t, INFO, GetLevel()) - assert.False(t, IsTrace()) - assert.False(t, IsDebug()) - assert.True(t, IsInfo()) - assert.True(t, IsWarn()) - assert.True(t, IsError()) - - Log(0, INFO, format, args...) - - assert.Panics(t, func() { - NewLogger(0, "console", "console", fmt.Sprintf(`{"level":"%s"`, level.String())) - }) - - go DelLogger("console") - - // We should be able to redelete without a problem - go DelLogger("console") -} - -func TestNewNamedLogger(t *testing.T) { - level := INFO - err := NewNamedLogger("test", 0, "console", "console", fmt.Sprintf(`{"level":"%s"}`, level.String())) - assert.NoError(t, err) - logger, _ := NamedLoggers.Load("test") - assert.Equal(t, level, logger.GetLevel()) - - written, closed := baseConsoleTest(t, logger) - go DelNamedLogger("test") - line := <-written - assert.Equal(t, "", string(line)) - assert.True(t, <-closed) -} diff --git a/modules/log/logger.go b/modules/log/logger.go deleted file mode 100644 index 71949e29b813..000000000000 --- a/modules/log/logger.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import "os" - -// Logger is the basic interface for logging -type Logger interface { - LevelLogger - Trace(format string, v ...interface{}) - IsTrace() bool - Debug(format string, v ...interface{}) - IsDebug() bool - Info(format string, v ...interface{}) - IsInfo() bool - Warn(format string, v ...interface{}) - IsWarn() bool - Error(format string, v ...interface{}) - ErrorWithSkip(skip int, format string, v ...interface{}) - IsError() bool - Critical(format string, v ...interface{}) - CriticalWithSkip(skip int, format string, v ...interface{}) - IsCritical() bool - Fatal(format string, v ...interface{}) - FatalWithSkip(skip int, format string, v ...interface{}) - IsFatal() bool -} - -// LevelLogger is the simplest logging interface -type LevelLogger interface { - Flush() - Close() - GetLevel() Level - Log(skip int, level Level, format string, v ...interface{}) error -} - -// SettableLogger is the interface of loggers which have subloggers -type SettableLogger interface { - SetLogger(name, provider, config string) error - DelLogger(name string) (bool, error) -} - -// StacktraceLogger is a logger that can log stacktraces -type StacktraceLogger interface { - GetStacktraceLevel() Level -} - -// LevelLoggerLogger wraps a LevelLogger as a Logger -type LevelLoggerLogger struct { - LevelLogger -} - -// Trace records trace log -func (l *LevelLoggerLogger) Trace(format string, v ...interface{}) { - l.Log(1, TRACE, format, v...) //nolint:errcheck -} - -// IsTrace returns true if the logger is TRACE -func (l *LevelLoggerLogger) IsTrace() bool { - return l.GetLevel() <= TRACE -} - -// Debug records debug log -func (l *LevelLoggerLogger) Debug(format string, v ...interface{}) { - l.Log(1, DEBUG, format, v...) //nolint:errcheck -} - -// IsDebug returns true if the logger is DEBUG -func (l *LevelLoggerLogger) IsDebug() bool { - return l.GetLevel() <= DEBUG -} - -// Info records information log -func (l *LevelLoggerLogger) Info(format string, v ...interface{}) { - l.Log(1, INFO, format, v...) //nolint:errcheck -} - -// IsInfo returns true if the logger is INFO -func (l *LevelLoggerLogger) IsInfo() bool { - return l.GetLevel() <= INFO -} - -// Warn records warning log -func (l *LevelLoggerLogger) Warn(format string, v ...interface{}) { - l.Log(1, WARN, format, v...) //nolint:errcheck -} - -// IsWarn returns true if the logger is WARN -func (l *LevelLoggerLogger) IsWarn() bool { - return l.GetLevel() <= WARN -} - -// Error records error log -func (l *LevelLoggerLogger) Error(format string, v ...interface{}) { - l.Log(1, ERROR, format, v...) //nolint:errcheck -} - -// ErrorWithSkip records error log from "skip" calls back from this function -func (l *LevelLoggerLogger) ErrorWithSkip(skip int, format string, v ...interface{}) { - l.Log(skip+1, ERROR, format, v...) //nolint:errcheck -} - -// IsError returns true if the logger is ERROR -func (l *LevelLoggerLogger) IsError() bool { - return l.GetLevel() <= ERROR -} - -// Critical records critical log -func (l *LevelLoggerLogger) Critical(format string, v ...interface{}) { - l.Log(1, CRITICAL, format, v...) //nolint:errcheck -} - -// CriticalWithSkip records critical log from "skip" calls back from this function -func (l *LevelLoggerLogger) CriticalWithSkip(skip int, format string, v ...interface{}) { - l.Log(skip+1, CRITICAL, format, v...) //nolint:errcheck -} - -// IsCritical returns true if the logger is CRITICAL -func (l *LevelLoggerLogger) IsCritical() bool { - return l.GetLevel() <= CRITICAL -} - -// Fatal records fatal log and exit the process -func (l *LevelLoggerLogger) Fatal(format string, v ...interface{}) { - l.Log(1, FATAL, format, v...) //nolint:errcheck - l.Close() - os.Exit(1) -} - -// FatalWithSkip records fatal log from "skip" calls back from this function and exits the process -func (l *LevelLoggerLogger) FatalWithSkip(skip int, format string, v ...interface{}) { - l.Log(skip+1, FATAL, format, v...) //nolint:errcheck - l.Close() - os.Exit(1) -} - -// IsFatal returns true if the logger is FATAL -func (l *LevelLoggerLogger) IsFatal() bool { - return l.GetLevel() <= FATAL -} diff --git a/modules/log/logger_global.go b/modules/log/logger_global.go new file mode 100644 index 000000000000..f4fcb0f516ab --- /dev/null +++ b/modules/log/logger_global.go @@ -0,0 +1,84 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "fmt" + "os" +) + +func FallbackErrorf(format string, args ...any) { + s := fmt.Sprintf(format, args...) + _, _ = fmt.Fprintln(os.Stderr, s) +} + +func GetLevel() Level { + return GetLogger(DEFAULT).GetLevel() +} + +func Log(skip int, level Level, format string, v ...any) { + GetLogger(DEFAULT).Log(skip+1, level, format, v...) +} + +func Trace(format string, v ...any) { + Log(1, TRACE, format, v...) +} + +func IsTrace() bool { + return GetLevel() <= TRACE +} + +func Debug(format string, v ...any) { + Log(1, DEBUG, format, v...) +} + +func IsDebug() bool { + return GetLevel() <= DEBUG +} + +func Info(format string, v ...any) { + Log(1, INFO, format, v...) +} + +func Warn(format string, v ...any) { + Log(1, WARN, format, v...) +} + +func Error(format string, v ...any) { + Log(1, ERROR, format, v...) +} + +func ErrorWithSkip(skip int, format string, v ...any) { + Log(skip+1, ERROR, format, v...) +} + +func Critical(format string, v ...any) { + Log(1, ERROR, format, v...) +} + +// Fatal records fatal log and exit process +func Fatal(format string, v ...any) { + Log(1, FATAL, format, v...) + GetManager().Close() + os.Exit(1) +} + +func GetLogger(name string) Logger { + return GetManager().GetLogger(name) +} + +func IsLoggerEnabled(name string) bool { + return GetManager().GetLogger(name).IsEnabled() +} + +func SetConsoleLogger(loggerName, writerName string, level Level) { + writer := NewEventWriterConsole(writerName, WriterMode{ + WriterType: "console", + Level: level, + Flags: LstdFlags, + Colorize: CanColorStdout, + WriterOption: WriterConsoleOption{}, + }) + GetManager().GetLogger(loggerName).RemoveAllWriters().AddWriters(writer) +} diff --git a/modules/log/logger_impl.go b/modules/log/logger_impl.go new file mode 100644 index 000000000000..a7ab67e6da5e --- /dev/null +++ b/modules/log/logger_impl.go @@ -0,0 +1,234 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "context" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/util" +) + +type LoggerImpl struct { + LevelLogger + + ctx context.Context + ctxCancel context.CancelFunc + + level atomic.Int32 + stacktraceLevel atomic.Int32 + + eventWriterMu sync.RWMutex + eventWriters map[string]EventWriter + + pauseMu sync.RWMutex + pauseChan chan struct{} +} + +var ( + _ BaseLogger = (*LoggerImpl)(nil) + _ LevelLogger = (*LoggerImpl)(nil) +) + +func (l *LoggerImpl) SendLogEvent(event *Event) { + l.eventWriterMu.RLock() + defer l.eventWriterMu.RUnlock() + + if len(l.eventWriters) == 0 { + FallbackErrorf("[no logger writer]: %s", event.Msg) + return + } + + for _, w := range l.eventWriters { + if event.Level < w.GetLevel() { + continue + } + select { + case w.Base().Queue <- event: + default: + bs, _ := json.Marshal(event) + FallbackErrorf("log writer %q queue is full, event: %v", w.GetWriterName(), string(bs)) + } + } +} + +func (l *LoggerImpl) syncLevelInternal() { + lowestLevel := NONE + for _, w := range l.eventWriters { + if w.GetLevel() < lowestLevel { + lowestLevel = w.GetLevel() + } + } + l.level.Store(int32(lowestLevel)) + + lowestLevel = NONE + for _, w := range l.eventWriters { + if w.Base().Mode.StacktraceLevel < lowestLevel { + lowestLevel = w.GetLevel() + } + } + l.stacktraceLevel.Store(int32(lowestLevel)) +} + +func (l *LoggerImpl) AddWriters(writer ...EventWriter) { + l.eventWriterMu.Lock() + defer l.eventWriterMu.Unlock() + + for _, w := range writer { + if old, ok := l.eventWriters[w.GetWriterName()]; ok { + eventWriterStopWait(old) + delete(l.eventWriters, old.GetWriterName()) + } + } + + for _, w := range writer { + l.eventWriters[w.GetWriterName()] = w + w.Base().LoggerImpl = l + eventWriterStartGo(l.ctx, w) + } + + l.syncLevelInternal() +} + +func (l *LoggerImpl) RemoveWriter(modeName string) error { + l.eventWriterMu.Lock() + defer l.eventWriterMu.Unlock() + + w, ok := l.eventWriters[modeName] + if !ok { + return util.ErrNotExist + } + + eventWriterStopWait(w) + delete(l.eventWriters, w.GetWriterName()) + l.syncLevelInternal() + return nil +} + +func (l *LoggerImpl) RemoveAllWriters() *LoggerImpl { + l.eventWriterMu.Lock() + defer l.eventWriterMu.Unlock() + + for _, w := range l.eventWriters { + eventWriterStopWait(w) + } + l.eventWriters = map[string]EventWriter{} + l.syncLevelInternal() + return l +} + +func (l *LoggerImpl) DumpWriters() map[string]any { + l.eventWriterMu.RLock() + defer l.eventWriterMu.RUnlock() + + writers := make(map[string]any, len(l.eventWriters)) + for k, w := range l.eventWriters { + bs, err := json.Marshal(w.Base().Mode) + if err != nil { + FallbackErrorf("marshal writer %q to dump failed: %v", k, err) + continue + } + m := map[string]any{} + _ = json.Unmarshal(bs, &m) + m["WriterType"] = w.GetWriterType() + writers[k] = m + } + return writers +} + +func (l *LoggerImpl) Pause() { + l.pauseMu.Lock() + l.pauseChan = make(chan struct{}) + l.pauseMu.Unlock() +} + +func (l *LoggerImpl) Resume() { + l.pauseMu.Lock() + close(l.pauseChan) + l.pauseChan = nil + l.pauseMu.Unlock() +} + +func (l *LoggerImpl) GetPauseChan() chan struct{} { + l.pauseMu.RLock() + defer l.pauseMu.RUnlock() + return l.pauseChan +} + +func (l *LoggerImpl) IsEnabled() bool { + l.eventWriterMu.RLock() + defer l.eventWriterMu.RUnlock() + return l.level.Load() < int32(FATAL) && len(l.eventWriters) > 0 +} + +func (l *LoggerImpl) Log(skip int, level Level, format string, logArgs ...any) { + if Level(l.level.Load()) > level { + return + } + + event := &Event{ + Time: time.Now(), + Level: level, + Caller: "?()", + } + + pc, filename, line, ok := runtime.Caller(skip + 1) + if ok { + fn := runtime.FuncForPC(pc) + if fn != nil { + event.Caller = fn.Name() + "()" + } + } + event.Filename, event.Line = strings.TrimPrefix(filename, prefix), line + + if l.stacktraceLevel.Load() <= int32(level) { + event.Stacktrace = Stack(skip + 1) + } + + labels := getGoroutineLabels() + if labels != nil { + event.GoroutinePid = labels["pid"] + } + + msgArgs := make([]any, len(logArgs)) + copy(msgArgs, logArgs) + for i, v := range logArgs { + if cv, ok := v.(*ColoredValue); ok { + msgArgs[i] = cv.v + } + } + msg, frozenArgs := frozenMsgFormat(format, msgArgs...) + for i, v := range logArgs { + if cv, ok := v.(*ColoredValue); ok { + newCV := *cv + newCV.v = frozenArgs[i] + msgArgs[i] = &newCV + } else { + msgArgs[i] = frozenArgs[i] + } + } + + event.Msg, event.MsgFormat, event.MsgFrozenArgs = msg, format, msgArgs + + l.SendLogEvent(event) +} + +func (l *LoggerImpl) GetLevel() Level { + return Level(l.level.Load()) +} + +func NewLoggerWithWriters(writer ...EventWriter) *LoggerImpl { + l := &LoggerImpl{} + l.ctx, l.ctxCancel = context.WithCancel(context.Background()) + l.LevelLogger = BaseLoggerToGeneralLogger(l) + l.eventWriters = map[string]EventWriter{} + l.syncLevelInternal() + l.AddWriters(writer...) + return l +} diff --git a/modules/log/logger_types.go b/modules/log/logger_types.go new file mode 100644 index 000000000000..600eee5dbae0 --- /dev/null +++ b/modules/log/logger_types.go @@ -0,0 +1,75 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +// BaseLogger provides the basic logging functions +type BaseLogger interface { + Log(skip int, level Level, format string, v ...any) + GetLevel() Level +} + +// LevelLogger provides level-related logging functions +type LevelLogger interface { + LevelEnabled(level Level) bool + + Trace(format string, v ...any) + Debug(format string, v ...any) + Info(format string, v ...any) + Warn(format string, v ...any) + Error(format string, v ...any) + Critical(format string, v ...any) +} + +type Logger interface { + BaseLogger + LevelLogger +} + +type baseToLogger struct { + base BaseLogger +} + +var _ Logger = (*baseToLogger)(nil) + +func (s *baseToLogger) Log(skip int, level Level, format string, v ...any) { + s.base.Log(skip+1, level, format, v...) +} + +func (s *baseToLogger) GetLevel() Level { + return s.base.GetLevel() +} + +func (s *baseToLogger) LevelEnabled(level Level) bool { + return s.base.GetLevel() <= level +} + +func (s *baseToLogger) Trace(format string, v ...any) { + s.base.Log(1, TRACE, format, v...) +} + +func (s *baseToLogger) Debug(format string, v ...any) { + s.base.Log(1, DEBUG, format, v...) +} + +func (s *baseToLogger) Info(format string, v ...any) { + s.base.Log(1, INFO, format, v...) +} + +func (s *baseToLogger) Warn(format string, v ...any) { + s.base.Log(1, WARN, format, v...) +} + +func (s *baseToLogger) Error(format string, v ...any) { + s.base.Log(1, ERROR, format, v...) +} + +func (s *baseToLogger) Critical(format string, v ...any) { + s.base.Log(1, CRITICAL, format, v...) +} + +// BaseLoggerToGeneralLogger wraps a BaseLogger (which only has Log() function) to a Logger (which has Info() function) +func BaseLoggerToGeneralLogger(b BaseLogger) Logger { + l := &baseToLogger{base: b} + return l +} diff --git a/modules/log/manager.go b/modules/log/manager.go new file mode 100644 index 000000000000..cdd5603e38a7 --- /dev/null +++ b/modules/log/manager.go @@ -0,0 +1,93 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "sync" + "sync/atomic" +) + +const DEFAULT = "default" + +type LoggerManager struct { + mu sync.Mutex + loggers map[string]*LoggerImpl + defaultLogger atomic.Pointer[LoggerImpl] +} + +func (m *LoggerManager) GetLogger(name string) *LoggerImpl { + if name == DEFAULT { + if logger := m.defaultLogger.Load(); logger != nil { + return logger + } + } + + m.mu.Lock() + defer m.mu.Unlock() + + logger := m.loggers[name] + if logger == nil { + logger = NewLoggerWithWriters() + m.loggers[name] = logger + if name == DEFAULT { + m.defaultLogger.Store(logger) + } + } + + return logger +} + +func (m *LoggerManager) PauseAll() { + m.mu.Lock() + defer m.mu.Unlock() + + for _, logger := range m.loggers { + logger.Pause() + } +} + +func (m *LoggerManager) ResumeAll() { + m.mu.Lock() + defer m.mu.Unlock() + + for _, logger := range m.loggers { + logger.Resume() + } +} + +func (m *LoggerManager) Close() { + m.mu.Lock() + defer m.mu.Unlock() + + for _, logger := range m.loggers { + logger.RemoveAllWriters() + } +} + +func (m *LoggerManager) DumpLoggers() map[string]any { + m.mu.Lock() + defer m.mu.Unlock() + + dump := map[string]any{} + for name, logger := range m.loggers { + m := map[string]any{ + "IsEnabled": logger.IsEnabled(), + "EventWriters": logger.DumpWriters(), + } + dump[name] = m + } + return dump +} + +var manager *LoggerManager + +func GetManager() *LoggerManager { + return manager +} + +func init() { + manager = &LoggerManager{ + loggers: map[string]*LoggerImpl{}, + } +} diff --git a/modules/log/misc.go b/modules/log/misc.go new file mode 100644 index 000000000000..20d760dac43d --- /dev/null +++ b/modules/log/misc.go @@ -0,0 +1,33 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "io" +) + +type LogStringer interface { //nolint:revive + LogString() string +} + +type PrintfLogger struct { + Logf func(format string, args ...any) +} + +func (p *PrintfLogger) Printf(format string, args ...any) { + p.Logf(format, args...) +} + +type loggerToWriter struct { + logf func(format string, args ...any) +} + +func (p *loggerToWriter) Write(bs []byte) (int, error) { + p.logf("%s", string(bs)) + return len(bs), nil +} + +func LoggerToWriter(logf func(format string, args ...any)) io.Writer { + return &loggerToWriter{logf: logf} +} diff --git a/modules/log/multichannel.go b/modules/log/multichannel.go deleted file mode 100644 index 6b8a9b82464a..000000000000 --- a/modules/log/multichannel.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2020 The Gogs Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "fmt" - "runtime" - "strings" - "time" -) - -// MultiChannelledLogger is default logger in the Gitea application. -// it can contain several providers and log message into all providers. -type MultiChannelledLogger struct { - LevelLoggerLogger - *MultiChannelledLog - bufferLength int64 -} - -// newLogger initializes and returns a new logger. -func newLogger(name string, buffer int64) *MultiChannelledLogger { - l := &MultiChannelledLogger{ - MultiChannelledLog: NewMultiChannelledLog(name, buffer), - bufferLength: buffer, - } - l.LevelLogger = l - return l -} - -// SetLogger sets new logger instance with given logger provider and config. -func (l *MultiChannelledLogger) SetLogger(name, provider, config string) error { - eventLogger, err := NewChannelledLog(l.ctx, name, provider, config, l.bufferLength) - if err != nil { - return fmt.Errorf("failed to create sublogger (%s): %w", name, err) - } - - l.MultiChannelledLog.DelLogger(name) - - err = l.MultiChannelledLog.AddLogger(eventLogger) - if err != nil { - if IsErrDuplicateName(err) { - return fmt.Errorf("%w other names: %v", err, l.MultiChannelledLog.GetEventLoggerNames()) - } - return fmt.Errorf("failed to add sublogger (%s): %w", name, err) - } - - return nil -} - -// DelLogger deletes a sublogger from this logger. -func (l *MultiChannelledLogger) DelLogger(name string) (bool, error) { - return l.MultiChannelledLog.DelLogger(name), nil -} - -// Log msg at the provided level with the provided caller defined by skip (0 being the function that calls this function) -func (l *MultiChannelledLogger) Log(skip int, level Level, format string, v ...interface{}) error { - if l.GetLevel() > level { - return nil - } - caller := "?()" - pc, filename, line, ok := runtime.Caller(skip + 1) - if ok { - // Get caller function name. - fn := runtime.FuncForPC(pc) - if fn != nil { - caller = fn.Name() + "()" - } - } - msg := format - if len(v) > 0 { - msg = ColorSprintf(format, v...) - } - labels := getGoroutineLabels() - if labels != nil { - pid, ok := labels["pid"] - if ok { - msg = "[" + ColorString(FgHiYellow) + pid + ColorString(Reset) + "] " + msg - } - } - stack := "" - if l.GetStacktraceLevel() <= level { - stack = Stack(skip + 1) - } - return l.SendLog(level, caller, strings.TrimPrefix(filename, prefix), line, msg, stack) -} - -// SendLog sends a log event at the provided level with the information given -func (l *MultiChannelledLogger) SendLog(level Level, caller, filename string, line int, msg, stack string) error { - if l.GetLevel() > level { - return nil - } - event := &Event{ - level: level, - caller: caller, - filename: filename, - line: line, - msg: msg, - time: time.Now(), - stacktrace: stack, - } - l.LogEvent(event) //nolint:errcheck - return nil -} diff --git a/modules/log/package.go b/modules/log/package.go new file mode 100644 index 000000000000..7d5c3d289db1 --- /dev/null +++ b/modules/log/package.go @@ -0,0 +1,24 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +// Package log provides logging capabilities for Gitea. +// Concepts: +// +// * Logger: a Logger provides logging functions and dispatches log events to all its writers +// +// * EventWriter: written log Event to a destination (eg: file, console) +// - EventWriterBase: the base struct of a writer, it contains common fields and functions for all writers +// - WriterType: the type name of a writer, eg: console, file +// - WriterName: aka Mode Name in document, the name of a writer instance, it's usually defined by the config file. +// It is called "mode name" because old code use MODE as config key, to keep compatibility, keep this concept. +// +// * WriterMode: the common options for all writers, eg: log level. +// - WriterConsoleOption and others: the specified options for a writer, eg: file path, remote address. +// +// Call graph: +// -> log.Info() +// -> LoggerImpl.Log() +// -> prepare log event, freeze all Stringer arguments (because the Event might be used in another goroutine) +// -> LoggerImpl.SendLogEvent, then the event goes into writer's goroutines +// -> EventWriter.Run() handles the events +package log diff --git a/modules/log/provider.go b/modules/log/provider.go deleted file mode 100644 index b5058139d705..000000000000 --- a/modules/log/provider.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -// LoggerProvider represents behaviors of a logger provider. -type LoggerProvider interface { - Init(config string) error - EventLogger -} - -type loggerProvider func() LoggerProvider - -var providers = make(map[string]loggerProvider) - -// Register registers given logger provider to providers. -func Register(name string, log loggerProvider) { - if log == nil { - panic("log: register provider is nil") - } - if _, dup := providers[name]; dup { - panic("log: register called twice for provider \"" + name + "\"") - } - providers[name] = log -} diff --git a/modules/log/smtp.go b/modules/log/smtp.go deleted file mode 100644 index 4e896496d75f..000000000000 --- a/modules/log/smtp.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2014 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "fmt" - "net/smtp" - "strings" - - "code.gitea.io/gitea/modules/json" -) - -type smtpWriter struct { - owner *SMTPLogger -} - -// Write sends the message as an email -func (s *smtpWriter) Write(p []byte) (int, error) { - return s.owner.sendMail(p) -} - -// Close does nothing -func (s *smtpWriter) Close() error { - return nil -} - -// SMTPLogger implements LoggerProvider and is used to send emails via given SMTP-server. -type SMTPLogger struct { - WriterLogger - Username string `json:"Username"` - Password string `json:"password"` - Host string `json:"host"` - Subject string `json:"subject"` - RecipientAddresses []string `json:"sendTos"` - sendMailFn func(string, smtp.Auth, string, []string, []byte) error -} - -// NewSMTPLogger creates smtp writer. -func NewSMTPLogger() LoggerProvider { - s := &SMTPLogger{} - s.Level = TRACE - s.sendMailFn = smtp.SendMail - return s -} - -// Init smtp writer with json config. -// config like: -// -// { -// "Username":"example@gmail.com", -// "password:"password", -// "host":"smtp.gmail.com:465", -// "subject":"email title", -// "sendTos":["email1","email2"], -// "level":LevelError -// } -func (log *SMTPLogger) Init(jsonconfig string) error { - err := json.Unmarshal([]byte(jsonconfig), log) - if err != nil { - return fmt.Errorf("Unable to parse JSON: %w", err) - } - log.NewWriterLogger(&smtpWriter{ - owner: log, - }) - log.sendMailFn = smtp.SendMail - return nil -} - -// WriteMsg writes message in smtp writer. -// it will send an email with subject and only this message. -func (log *SMTPLogger) sendMail(p []byte) (int, error) { - hp := strings.Split(log.Host, ":") - - // Set up authentication information. - auth := smtp.PlainAuth( - "", - log.Username, - log.Password, - hp[0], - ) - // Connect to the server, authenticate, set the sender and recipient, - // and send the email all in one step. - contentType := "Content-Type: text/plain" + "; charset=UTF-8" - mailmsg := []byte("To: " + strings.Join(log.RecipientAddresses, ";") + "\r\nFrom: " + log.Username + "<" + log.Username + - ">\r\nSubject: " + log.Subject + "\r\n" + contentType + "\r\n\r\n") - mailmsg = append(mailmsg, p...) - return len(p), log.sendMailFn( - log.Host, - auth, - log.Username, - log.RecipientAddresses, - mailmsg, - ) -} - -// Flush when log should be flushed -func (log *SMTPLogger) Flush() { -} - -// ReleaseReopen does nothing -func (log *SMTPLogger) ReleaseReopen() error { - return nil -} - -// GetName returns the default name for this implementation -func (log *SMTPLogger) GetName() string { - return "smtp" -} - -func init() { - Register("smtp", NewSMTPLogger) -} diff --git a/modules/log/smtp_test.go b/modules/log/smtp_test.go deleted file mode 100644 index d7d28f28f83b..000000000000 --- a/modules/log/smtp_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "fmt" - "net/smtp" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestSMTPLogger(t *testing.T) { - prefix := "TestPrefix " - level := INFO - flags := LstdFlags | LUTC | Lfuncname - username := "testuser" - password := "testpassword" - host := "testhost" - subject := "testsubject" - sendTos := []string{"testto1", "testto2"} - - logger := NewSMTPLogger() - smtpLogger, ok := logger.(*SMTPLogger) - assert.True(t, ok) - - err := logger.Init(fmt.Sprintf("{\"prefix\":\"%s\",\"level\":\"%s\",\"flags\":%d,\"username\":\"%s\",\"password\":\"%s\",\"host\":\"%s\",\"subject\":\"%s\",\"sendTos\":[\"%s\",\"%s\"]}", prefix, level.String(), flags, username, password, host, subject, sendTos[0], sendTos[1])) - assert.NoError(t, err) - - assert.Equal(t, flags, smtpLogger.Flags) - assert.Equal(t, level, smtpLogger.Level) - assert.Equal(t, level, logger.GetLevel()) - - location, _ := time.LoadLocation("EST") - - date := time.Date(2019, time.January, 13, 22, 3, 30, 15, location) - - dateString := date.UTC().Format("2006/01/02 15:04:05") - - event := Event{ - level: INFO, - msg: "TEST MSG", - caller: "CALLER", - filename: "FULL/FILENAME", - line: 1, - time: date, - } - - expected := fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - - var envToHost string - var envFrom string - var envTo []string - var envMsg []byte - smtpLogger.sendMailFn = func(addr string, a smtp.Auth, from string, to []string, msg []byte) error { - envToHost = addr - envFrom = from - envTo = to - envMsg = msg - return nil - } - - err = logger.LogEvent(&event) - assert.NoError(t, err) - assert.Equal(t, host, envToHost) - assert.Equal(t, username, envFrom) - assert.Equal(t, sendTos, envTo) - assert.Contains(t, string(envMsg), expected) - - logger.Flush() - - event.level = WARN - expected = fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - err = logger.LogEvent(&event) - assert.NoError(t, err) - assert.Equal(t, host, envToHost) - assert.Equal(t, username, envFrom) - assert.Equal(t, sendTos, envTo) - assert.Contains(t, string(envMsg), expected) - - logger.Close() -} diff --git a/modules/log/stack.go b/modules/log/stack.go index d4496cff03fd..9b22e92867c3 100644 --- a/modules/log/stack.go +++ b/modules/log/stack.go @@ -32,19 +32,19 @@ func Stack(skip int) string { } // Print equivalent of debug.Stack() - fmt.Fprintf(buf, "%s:%d (0x%x)\n", filename, lineNumber, programCounter) + _, _ = fmt.Fprintf(buf, "%s:%d (0x%x)\n", filename, lineNumber, programCounter) // Now try to print the offending line if filename != lastFilename { data, err := os.ReadFile(filename) if err != nil { - // can't read this sourcefile + // can't read this source file // likely we don't have the sourcecode available continue } lines = bytes.Split(data, []byte{'\n'}) lastFilename = filename } - fmt.Fprintf(buf, "\t%s: %s\n", functionName(programCounter), source(lines, lineNumber)) + _, _ = fmt.Fprintf(buf, "\t%s: %s\n", functionName(programCounter), source(lines, lineNumber)) } return buf.String() } diff --git a/modules/log/writer.go b/modules/log/writer.go deleted file mode 100644 index 61f1d866ee54..000000000000 --- a/modules/log/writer.go +++ /dev/null @@ -1,269 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "bytes" - "fmt" - "io" - "regexp" - "strings" - "sync" -) - -type byteArrayWriter []byte - -func (b *byteArrayWriter) Write(p []byte) (int, error) { - *b = append(*b, p...) - return len(p), nil -} - -// WriterLogger represent a basic logger for Gitea -type WriterLogger struct { - out io.WriteCloser - mu sync.Mutex - - Level Level `json:"level"` - StacktraceLevel Level `json:"stacktraceLevel"` - Flags int `json:"flags"` - Prefix string `json:"prefix"` - Colorize bool `json:"colorize"` - Expression string `json:"expression"` - regexp *regexp.Regexp -} - -// NewWriterLogger creates a new WriterLogger from the provided WriteCloser. -// Optionally the level can be changed at the same time. -func (logger *WriterLogger) NewWriterLogger(out io.WriteCloser, level ...Level) { - logger.mu.Lock() - defer logger.mu.Unlock() - logger.out = out - switch logger.Flags { - case 0: - logger.Flags = LstdFlags - case -1: - logger.Flags = 0 - } - if len(level) > 0 { - logger.Level = level[0] - } - logger.createExpression() -} - -func (logger *WriterLogger) createExpression() { - if len(logger.Expression) > 0 { - var err error - logger.regexp, err = regexp.Compile(logger.Expression) - if err != nil { - logger.regexp = nil - } - } -} - -// GetLevel returns the logging level for this logger -func (logger *WriterLogger) GetLevel() Level { - return logger.Level -} - -// GetStacktraceLevel returns the stacktrace logging level for this logger -func (logger *WriterLogger) GetStacktraceLevel() Level { - return logger.StacktraceLevel -} - -// Copy of cheap integer to fixed-width decimal to ascii from logger. -func itoa(buf *[]byte, i, wid int) { - var logger [20]byte - bp := len(logger) - 1 - for i >= 10 || wid > 1 { - wid-- - q := i / 10 - logger[bp] = byte('0' + i - q*10) - bp-- - i = q - } - // i < 10 - logger[bp] = byte('0' + i) - *buf = append(*buf, logger[bp:]...) -} - -func (logger *WriterLogger) createMsg(buf *[]byte, event *Event) { - *buf = append(*buf, logger.Prefix...) - t := event.time - if logger.Flags&(Ldate|Ltime|Lmicroseconds) != 0 { - if logger.Colorize { - *buf = append(*buf, fgCyanBytes...) - } - if logger.Flags&LUTC != 0 { - t = t.UTC() - } - if logger.Flags&Ldate != 0 { - year, month, day := t.Date() - itoa(buf, year, 4) - *buf = append(*buf, '/') - itoa(buf, int(month), 2) - *buf = append(*buf, '/') - itoa(buf, day, 2) - *buf = append(*buf, ' ') - } - if logger.Flags&(Ltime|Lmicroseconds) != 0 { - hour, min, sec := t.Clock() - itoa(buf, hour, 2) - *buf = append(*buf, ':') - itoa(buf, min, 2) - *buf = append(*buf, ':') - itoa(buf, sec, 2) - if logger.Flags&Lmicroseconds != 0 { - *buf = append(*buf, '.') - itoa(buf, t.Nanosecond()/1e3, 6) - } - *buf = append(*buf, ' ') - } - if logger.Colorize { - *buf = append(*buf, resetBytes...) - } - - } - if logger.Flags&(Lshortfile|Llongfile) != 0 { - if logger.Colorize { - *buf = append(*buf, fgGreenBytes...) - } - file := event.filename - if logger.Flags&Lmedfile == Lmedfile { - startIndex := len(file) - 20 - if startIndex > 0 { - file = "..." + file[startIndex:] - } - } else if logger.Flags&Lshortfile != 0 { - startIndex := strings.LastIndexByte(file, '/') - if startIndex > 0 && startIndex < len(file) { - file = file[startIndex+1:] - } - } - *buf = append(*buf, file...) - *buf = append(*buf, ':') - itoa(buf, event.line, -1) - if logger.Flags&(Lfuncname|Lshortfuncname) != 0 { - *buf = append(*buf, ':') - } else { - if logger.Colorize { - *buf = append(*buf, resetBytes...) - } - *buf = append(*buf, ' ') - } - } - if logger.Flags&(Lfuncname|Lshortfuncname) != 0 { - if logger.Colorize { - *buf = append(*buf, fgGreenBytes...) - } - funcname := event.caller - if logger.Flags&Lshortfuncname != 0 { - lastIndex := strings.LastIndexByte(funcname, '.') - if lastIndex > 0 && len(funcname) > lastIndex+1 { - funcname = funcname[lastIndex+1:] - } - } - *buf = append(*buf, funcname...) - if logger.Colorize { - *buf = append(*buf, resetBytes...) - } - *buf = append(*buf, ' ') - - } - if logger.Flags&(Llevel|Llevelinitial) != 0 { - level := strings.ToUpper(event.level.String()) - if logger.Colorize { - *buf = append(*buf, levelToColor[event.level]...) - } - *buf = append(*buf, '[') - if logger.Flags&Llevelinitial != 0 { - *buf = append(*buf, level[0]) - } else { - *buf = append(*buf, level...) - } - *buf = append(*buf, ']') - if logger.Colorize { - *buf = append(*buf, resetBytes...) - } - *buf = append(*buf, ' ') - } - - msg := []byte(event.msg) - if len(msg) > 0 && msg[len(msg)-1] == '\n' { - msg = msg[:len(msg)-1] - } - - pawMode := allowColor - if !logger.Colorize { - pawMode = removeColor - } - - baw := byteArrayWriter(*buf) - (&protectedANSIWriter{ - w: &baw, - mode: pawMode, - }).Write(msg) //nolint:errcheck - *buf = baw - - if event.stacktrace != "" && logger.StacktraceLevel <= event.level { - lines := bytes.Split([]byte(event.stacktrace), []byte("\n")) - if len(lines) > 1 { - for _, line := range lines { - *buf = append(*buf, "\n\t"...) - *buf = append(*buf, line...) - } - } - *buf = append(*buf, '\n') - } - *buf = append(*buf, '\n') -} - -// LogEvent logs the event to the internal writer -func (logger *WriterLogger) LogEvent(event *Event) error { - if logger.Level > event.level { - return nil - } - - logger.mu.Lock() - defer logger.mu.Unlock() - if !logger.Match(event) { - return nil - } - var buf []byte - logger.createMsg(&buf, event) - _, err := logger.out.Write(buf) - return err -} - -// Match checks if the given event matches the logger's regexp expression -func (logger *WriterLogger) Match(event *Event) bool { - if logger.regexp == nil { - return true - } - if logger.regexp.Match([]byte(fmt.Sprintf("%s:%d:%s", event.filename, event.line, event.caller))) { - return true - } - // Match on the non-colored msg - therefore strip out colors - var msg []byte - baw := byteArrayWriter(msg) - (&protectedANSIWriter{ - w: &baw, - mode: removeColor, - }).Write([]byte(event.msg)) //nolint:errcheck - msg = baw - return logger.regexp.Match(msg) -} - -// Close the base logger -func (logger *WriterLogger) Close() { - logger.mu.Lock() - defer logger.mu.Unlock() - if logger.out != nil { - logger.out.Close() - } -} - -// GetName returns empty for these provider loggers -func (logger *WriterLogger) GetName() string { - return "" -} diff --git a/modules/log/writer_test.go b/modules/log/writer_test.go deleted file mode 100644 index 8c03f87d907c..000000000000 --- a/modules/log/writer_test.go +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -type CallbackWriteCloser struct { - callback func([]byte, bool) -} - -func (c CallbackWriteCloser) Write(p []byte) (int, error) { - c.callback(p, false) - return len(p), nil -} - -func (c CallbackWriteCloser) Close() error { - c.callback(nil, true) - return nil -} - -func TestBaseLogger(t *testing.T) { - var written []byte - var closed bool - - c := CallbackWriteCloser{ - callback: func(p []byte, close bool) { - written = p - closed = close - }, - } - prefix := "TestPrefix " - b := WriterLogger{ - out: c, - Level: INFO, - Flags: LstdFlags | LUTC, - Prefix: prefix, - } - location, _ := time.LoadLocation("EST") - - date := time.Date(2019, time.January, 13, 22, 3, 30, 15, location) - - dateString := date.UTC().Format("2006/01/02 15:04:05") - - event := Event{ - level: INFO, - msg: "TEST MSG", - caller: "CALLER", - filename: "FULL/FILENAME", - line: 1, - time: date, - } - - assert.Equal(t, INFO, b.GetLevel()) - - expected := fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - event.level = DEBUG - expected = "" - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - - event.level = TRACE - expected = "" - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - - event.level = WARN - expected = fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - event.level = ERROR - expected = fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - event.level = CRITICAL - expected = fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.filename, event.line, event.caller, strings.ToUpper(event.level.String())[0], event.msg) - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - b.Close() - assert.True(t, closed) -} - -func TestBaseLoggerDated(t *testing.T) { - var written []byte - var closed bool - - c := CallbackWriteCloser{ - callback: func(p []byte, close bool) { - written = p - closed = close - }, - } - prefix := "" - b := WriterLogger{ - out: c, - Level: WARN, - Flags: Ldate | Ltime | Lmicroseconds | Lshortfile | Llevel, - Prefix: prefix, - } - - location, _ := time.LoadLocation("EST") - - date := time.Date(2019, time.January, 13, 22, 3, 30, 115, location) - - dateString := date.Format("2006/01/02 15:04:05.000000") - - event := Event{ - level: WARN, - msg: "TEST MESSAGE TEST\n", - caller: "CALLER", - filename: "FULL/FILENAME", - line: 1, - time: date, - } - - assert.Equal(t, WARN, b.GetLevel()) - - expected := fmt.Sprintf("%s%s %s:%d [%s] %s", prefix, dateString, "FILENAME", event.line, strings.ToUpper(event.level.String()), event.msg) - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - event.level = INFO - expected = "" - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - event.level = ERROR - expected = fmt.Sprintf("%s%s %s:%d [%s] %s", prefix, dateString, "FILENAME", event.line, strings.ToUpper(event.level.String()), event.msg) - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - event.level = DEBUG - expected = "" - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - event.level = CRITICAL - expected = fmt.Sprintf("%s%s %s:%d [%s] %s", prefix, dateString, "FILENAME", event.line, strings.ToUpper(event.level.String()), event.msg) - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - event.level = TRACE - expected = "" - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - b.Close() - assert.True(t, closed) -} - -func TestBaseLoggerMultiLineNoFlagsRegexp(t *testing.T) { - var written []byte - var closed bool - - c := CallbackWriteCloser{ - callback: func(p []byte, close bool) { - written = p - closed = close - }, - } - prefix := "" - b := WriterLogger{ - Level: DEBUG, - StacktraceLevel: ERROR, - Flags: -1, - Prefix: prefix, - Expression: "FILENAME", - } - b.NewWriterLogger(c) - - location, _ := time.LoadLocation("EST") - - date := time.Date(2019, time.January, 13, 22, 3, 30, 115, location) - - event := Event{ - level: DEBUG, - msg: "TEST\nMESSAGE\nTEST", - caller: "CALLER", - filename: "FULL/FILENAME", - line: 1, - time: date, - } - - assert.Equal(t, DEBUG, b.GetLevel()) - - expected := "TEST\n\tMESSAGE\n\tTEST\n" - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - event.filename = "ELSEWHERE" - - b.LogEvent(&event) - assert.Equal(t, "", string(written)) - assert.False(t, closed) - written = written[:0] - - event.caller = "FILENAME" - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] - - event = Event{ - level: DEBUG, - msg: "TEST\nFILENAME\nTEST", - caller: "CALLER", - filename: "FULL/ELSEWHERE", - line: 1, - time: date, - } - expected = "TEST\n\tFILENAME\n\tTEST\n" - b.LogEvent(&event) - assert.Equal(t, expected, string(written)) - assert.False(t, closed) - written = written[:0] -} - -func TestBrokenRegexp(t *testing.T) { - var closed bool - - c := CallbackWriteCloser{ - callback: func(p []byte, close bool) { - closed = close - }, - } - - b := WriterLogger{ - Level: DEBUG, - StacktraceLevel: ERROR, - Flags: -1, - Prefix: prefix, - Expression: "\\", - } - b.NewWriterLogger(c) - assert.Empty(t, b.regexp) - b.Close() - assert.True(t, closed) -} diff --git a/modules/private/manager.go b/modules/private/manager.go index 5853db34e4b7..3448f2e34c53 100644 --- a/modules/private/manager.go +++ b/modules/private/manager.go @@ -75,18 +75,18 @@ func SetLogSQL(ctx context.Context, on bool) ResponseExtra { // LoggerOptions represents the options for the add logger call type LoggerOptions struct { - Group string - Name string + Logger string + Writer string Mode string Config map[string]interface{} } // AddLogger adds a logger -func AddLogger(ctx context.Context, group, name, mode string, config map[string]interface{}) ResponseExtra { +func AddLogger(ctx context.Context, logger, writer, mode string, config map[string]interface{}) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/add-logger" req := newInternalRequest(ctx, reqURL, "POST", LoggerOptions{ - Group: group, - Name: name, + Logger: logger, + Writer: writer, Mode: mode, Config: config, }) @@ -94,8 +94,8 @@ func AddLogger(ctx context.Context, group, name, mode string, config map[string] } // RemoveLogger removes a logger -func RemoveLogger(ctx context.Context, group, name string) ResponseExtra { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/remove-logger/%s/%s", url.PathEscape(group), url.PathEscape(name)) +func RemoveLogger(ctx context.Context, logger, writer string) ResponseExtra { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/remove-logger/%s/%s", url.PathEscape(logger), url.PathEscape(writer)) req := newInternalRequest(ctx, reqURL, "POST") return requestJSONUserMsg(req, "Removed") } diff --git a/modules/setting/config_provider.go b/modules/setting/config_provider.go index ce9ef72248ad..1830715a79f8 100644 --- a/modules/setting/config_provider.go +++ b/modules/setting/config_provider.go @@ -33,6 +33,29 @@ type ConfigProvider interface { Save() error } +// KeyInSection only searches the keys in the given section. +// ini package has a special behavior: with "[sec] a=1" and an empty "[sec.sub]", +// then in "[sec.sub]", Key()/HasKey() can always see "a=1" because it always tries parent sections. +func KeyInSection(sec ConfigSection, key string) *ini.Key { + if sec == nil { + return nil + } + for _, k := range sec.Keys() { + if k.Name() == key { + return k + } + } + return nil +} + +func KeyInSectionString(sec ConfigSection, key string) string { + k := KeyInSection(sec, key) + if k != nil { + return k.String() + } + return "" +} + type iniFileConfigProvider struct { opts *Options *ini.File diff --git a/modules/setting/database.go b/modules/setting/database.go index 8c4dfb21d7ef..7a7c7029a430 100644 --- a/modules/setting/database.go +++ b/modules/setting/database.go @@ -92,7 +92,7 @@ func loadDBSetting(rootCfg ConfigProvider) { Database.MaxOpenConns = sec.Key("MAX_OPEN_CONNS").MustInt(0) Database.IterateBufferSize = sec.Key("ITERATE_BUFFER_SIZE").MustInt(50) - Database.LogSQL = sec.Key("LOG_SQL").MustBool(true) + Database.LogSQL = sec.Key("LOG_SQL").MustBool(false) Database.DBConnectRetries = sec.Key("DB_RETRIES").MustInt(10) Database.DBConnectBackoff = sec.Key("DB_RETRY_BACKOFF").MustDuration(3 * time.Second) Database.AutoMigration = sec.Key("AUTO_MIGRATION").MustBool(true) diff --git a/modules/setting/log.go b/modules/setting/log.go index d9a9e5af8fd5..ff869c3af298 100644 --- a/modules/setting/log.go +++ b/modules/setting/log.go @@ -10,384 +10,233 @@ import ( "path" "path/filepath" "strings" - "sync" - "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" ) -var ( - filenameSuffix = "" - descriptionLock = sync.RWMutex{} - logDescriptions = make(map[string]*LogDescription) -) - // Log settings var Log struct { + RootPath string + + Mode string Level log.Level - StacktraceLogLevel string - RootPath string - EnableSSHLog bool - EnableXORMLog bool + StacktraceLogLevel log.Level + BufferLen int - DisableRouterLog bool + EnableSSHLog bool - EnableAccessLog bool AccessLogTemplate string - BufferLength int64 RequestIDHeaders []string } -// GetLogDescriptions returns a race safe set of descriptions -func GetLogDescriptions() map[string]*LogDescription { - descriptionLock.RLock() - defer descriptionLock.RUnlock() - descs := make(map[string]*LogDescription, len(logDescriptions)) - for k, v := range logDescriptions { - subLogDescriptions := make([]SubLogDescription, len(v.SubLogDescriptions)) - copy(subLogDescriptions, v.SubLogDescriptions) - - descs[k] = &LogDescription{ - Name: v.Name, - SubLogDescriptions: subLogDescriptions, - } - } - return descs -} - -// AddLogDescription adds a set of descriptions to the complete description -func AddLogDescription(key string, description *LogDescription) { - descriptionLock.Lock() - defer descriptionLock.Unlock() - logDescriptions[key] = description -} +const accessLogTemplateDefault = `{{.Ctx.RemoteHost}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}" "{{.Ctx.Req.UserAgent}}"` -// AddSubLogDescription adds a sub log description -func AddSubLogDescription(key string, subLogDescription SubLogDescription) bool { - descriptionLock.Lock() - defer descriptionLock.Unlock() - desc, ok := logDescriptions[key] - if !ok { - return false - } - for i, sub := range desc.SubLogDescriptions { - if sub.Name == subLogDescription.Name { - desc.SubLogDescriptions[i] = subLogDescription - return true - } - } - desc.SubLogDescriptions = append(desc.SubLogDescriptions, subLogDescription) - return true -} +func loadLogGlobalFrom(rootCfg ConfigProvider) { + sec := rootCfg.Section("log") -// RemoveSubLogDescription removes a sub log description -func RemoveSubLogDescription(key, name string) bool { - descriptionLock.Lock() - defer descriptionLock.Unlock() - desc, ok := logDescriptions[key] - if !ok { - return false - } - for i, sub := range desc.SubLogDescriptions { - if sub.Name == name { - desc.SubLogDescriptions = append(desc.SubLogDescriptions[:i], desc.SubLogDescriptions[i+1:]...) - return true - } - } - return false -} + Log.Level = log.LevelFromString(sec.Key("LEVEL").MustString(log.INFO.String())) + Log.StacktraceLogLevel = log.LevelFromString(sec.Key("STACKTRACE_LEVEL").MustString(log.NONE.String())) + Log.BufferLen = sec.Key("BUFFER_LEN").MustInt(10000) + Log.Mode = sec.Key("MODE").MustString("console") -type defaultLogOptions struct { - levelName string // LogLevel - flags string - filename string // path.Join(LogRootPath, "gitea.log") - bufferLength int64 - disableConsole bool -} - -func newDefaultLogOptions() defaultLogOptions { - return defaultLogOptions{ - levelName: Log.Level.String(), - flags: "stdflags", - filename: filepath.Join(Log.RootPath, "gitea.log"), - bufferLength: 10000, - disableConsole: false, + Log.RootPath = sec.Key("ROOT_PATH").MustString(path.Join(AppWorkPath, "log")) + if !filepath.IsAbs(Log.RootPath) { + Log.RootPath = filepath.Join(AppWorkPath, Log.RootPath) } -} - -// SubLogDescription describes a sublogger -type SubLogDescription struct { - Name string - Provider string - Config string -} + Log.RootPath = util.FilePathJoinAbs(Log.RootPath) -// LogDescription describes a named logger -type LogDescription struct { - Name string - SubLogDescriptions []SubLogDescription -} - -func getLogLevel(section ConfigSection, key string, defaultValue log.Level) log.Level { - value := section.Key(key).MustString(defaultValue.String()) - return log.FromString(value) -} + Log.EnableSSHLog = sec.Key("ENABLE_SSH_LOG").MustBool(false) -func getStacktraceLogLevel(section ConfigSection, key, defaultValue string) string { - value := section.Key(key).MustString(defaultValue) - return log.FromString(value).String() + Log.AccessLogTemplate = sec.Key("ACCESS_LOG_TEMPLATE").MustString(accessLogTemplateDefault) + Log.RequestIDHeaders = sec.Key("REQUEST_ID_HEADERS").Strings(",") } -func loadLogFrom(rootCfg ConfigProvider) { +func prepareLoggerConfig(rootCfg ConfigProvider) { sec := rootCfg.Section("log") - Log.Level = getLogLevel(sec, "LEVEL", log.INFO) - Log.StacktraceLogLevel = getStacktraceLogLevel(sec, "STACKTRACE_LEVEL", "None") - Log.RootPath = sec.Key("ROOT_PATH").MustString(path.Join(AppWorkPath, "log")) - forcePathSeparator(Log.RootPath) - Log.BufferLength = sec.Key("BUFFER_LEN").MustInt64(10000) - Log.EnableSSHLog = sec.Key("ENABLE_SSH_LOG").MustBool(false) - Log.EnableAccessLog = sec.Key("ENABLE_ACCESS_LOG").MustBool(false) - Log.AccessLogTemplate = sec.Key("ACCESS_LOG_TEMPLATE").MustString( - `{{.Ctx.RemoteHost}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}" "{{.Ctx.Req.UserAgent}}"`, - ) - Log.RequestIDHeaders = sec.Key("REQUEST_ID_HEADERS").Strings(",") - // the `MustString` updates the default value, and `log.ACCESS` is used by `generateNamedLogger("access")` later - _ = rootCfg.Section("log").Key("ACCESS").MustString("file") + if !sec.HasKey("logger.DEFAULT.MODE") { + sec.Key("logger.DEFAULT.MODE").MustString(",") + } - sec.Key("ROUTER").MustString("console") - // Allow [log] DISABLE_ROUTER_LOG to override [server] DISABLE_ROUTER_LOG - Log.DisableRouterLog = sec.Key("DISABLE_ROUTER_LOG").MustBool(Log.DisableRouterLog) + deprecatedSetting(rootCfg, "log", "ACCESS", "log", "logger.ACCESS.MODE", "1.21") + deprecatedSetting(rootCfg, "log", "ENABLE_ACCESS_LOG", "log", "logger.ACCESS.MODE", "1.21") + if val := sec.Key("ACCESS").String(); val != "" { + sec.Key("logger.ACCESS.MODE").MustString(val) + } + if sec.HasKey("ENABLE_ACCESS_LOG") && !sec.Key("ENABLE_ACCESS_LOG").MustBool() { + sec.Key("logger.ACCESS.MODE").SetValue("") + } - Log.EnableXORMLog = rootCfg.Section("log").Key("ENABLE_XORM_LOG").MustBool(true) + deprecatedSetting(rootCfg, "log", "ROUTER", "log", "logger.ROUTER.MODE", "1.21") + deprecatedSetting(rootCfg, "log", "DISABLE_ROUTER_LOG", "log", "logger.ROUTER.MODE", "1.21") + if val := sec.Key("ROUTER").String(); val != "" { + sec.Key("logger.ROUTER.MODE").MustString(val) + } + if !sec.HasKey("logger.ROUTER.MODE") { + sec.Key("logger.ROUTER.MODE").MustString(",") // use default logger + } + if sec.HasKey("DISABLE_ROUTER_LOG") && sec.Key("DISABLE_ROUTER_LOG").MustBool() { + sec.Key("logger.ROUTER.MODE").SetValue("") + } + + deprecatedSetting(rootCfg, "log", "XORM", "log", "logger.XORM.MODE", "1.21") + deprecatedSetting(rootCfg, "log", "ENABLE_XORM_LOG", "log", "logger.XORM.MODE", "1.21") + if val := sec.Key("XORM").String(); val != "" { + sec.Key("logger.XORM.MODE").MustString(val) + } + if !sec.HasKey("logger.XORM.MODE") { + sec.Key("logger.XORM.MODE").MustString(",") // use default logger + } + if sec.HasKey("ENABLE_XORM_LOG") && !sec.Key("ENABLE_XORM_LOG").MustBool() { + sec.Key("logger.XORM.MODE").SetValue("") + } } -func generateLogConfig(sec ConfigSection, name string, defaults defaultLogOptions) (mode, jsonConfig, levelName string) { - level := getLogLevel(sec, "LEVEL", Log.Level) - levelName = level.String() - stacktraceLevelName := getStacktraceLogLevel(sec, "STACKTRACE_LEVEL", Log.StacktraceLogLevel) - stacktraceLevel := log.FromString(stacktraceLevelName) - mode = name - keys := sec.Keys() - logPath := defaults.filename - flags := log.FlagsFromString(defaults.flags) - expression := "" - prefix := "" - for _, key := range keys { - switch key.Name() { - case "MODE": - mode = key.MustString(name) - case "FILE_NAME": - logPath = key.MustString(defaults.filename) - forcePathSeparator(logPath) - if !filepath.IsAbs(logPath) { - logPath = path.Join(Log.RootPath, logPath) - } - case "FLAGS": - flags = log.FlagsFromString(key.MustString(defaults.flags)) - case "EXPRESSION": - expression = key.MustString("") - case "PREFIX": - prefix = key.MustString("") - } +func LogPrepareFilenameForWriter(modeName, fileName string) string { + defaultFileName := "gitea.log" + if modeName != "file" { + defaultFileName = modeName + ".log" + } + if fileName == "" { + fileName = defaultFileName + } + if !filepath.IsAbs(fileName) { + fileName = filepath.Join(Log.RootPath, fileName) + } else { + fileName = filepath.Clean(fileName) } + if err := os.MkdirAll(filepath.Dir(fileName), os.ModePerm); err != nil { + panic(fmt.Sprintf("unable to create directory for log %q: %v", fileName, err.Error())) + } + return fileName +} - logConfig := map[string]interface{}{ - "level": level.String(), - "expression": expression, - "prefix": prefix, - "flags": flags, - "stacktraceLevel": stacktraceLevel.String(), +func loadLogModeByName(rootCfg ConfigProvider, modeName string) log.WriterMode { + sec := rootCfg.Section("log." + modeName) + + writerMode := log.WriterMode{} + writerMode.WriterType = KeyInSectionString(sec, "MODE") + if writerMode.WriterType == "" { + writerMode.WriterType = modeName } + writerMode.Level = log.LevelFromString(sec.Key("LEVEL").MustString(Log.Level.String())) + writerMode.StacktraceLevel = log.LevelFromString(sec.Key("STACKTRACE_LEVEL").MustString(Log.StacktraceLogLevel.String())) + writerMode.Prefix = sec.Key("PREFIX").MustString("") + writerMode.Flags = log.FlagsFromString(sec.Key("FLAGS").MustString("stdflags")) + writerMode.Expression = sec.Key("EXPRESSION").MustString("") - // Generate log configuration. - switch mode { + switch writerMode.WriterType { case "console": useStderr := sec.Key("STDERR").MustBool(false) - logConfig["stderr"] = useStderr + writerOption := log.WriterConsoleOption{Stderr: useStderr} if useStderr { - logConfig["colorize"] = sec.Key("COLORIZE").MustBool(log.CanColorStderr) + writerMode.Colorize = sec.Key("COLORIZE").MustBool(log.CanColorStderr) } else { - logConfig["colorize"] = sec.Key("COLORIZE").MustBool(log.CanColorStdout) + writerMode.Colorize = sec.Key("COLORIZE").MustBool(log.CanColorStdout) } - + writerMode.WriterOption = writerOption case "file": - if err := os.MkdirAll(path.Dir(logPath), os.ModePerm); err != nil { - panic(err.Error()) - } - - logConfig["filename"] = logPath + filenameSuffix - logConfig["rotate"] = sec.Key("LOG_ROTATE").MustBool(true) - logConfig["maxsize"] = 1 << uint(sec.Key("MAX_SIZE_SHIFT").MustInt(28)) - logConfig["daily"] = sec.Key("DAILY_ROTATE").MustBool(true) - logConfig["maxdays"] = sec.Key("MAX_DAYS").MustInt(7) - logConfig["compress"] = sec.Key("COMPRESS").MustBool(true) - logConfig["compressionLevel"] = sec.Key("COMPRESSION_LEVEL").MustInt(-1) + fileName := LogPrepareFilenameForWriter(modeName, sec.Key("FILE_NAME").String()) + writerOption := log.WriterFileOption{} + writerOption.FileName = fileName + filenameSuffix // FIXME: the suffix doesn't seem right, see its related comments + writerOption.LogRotate = sec.Key("LOG_ROTATE").MustBool(true) + writerOption.MaxSize = 1 << uint(sec.Key("MAX_SIZE_SHIFT").MustInt(28)) + writerOption.DailyRotate = sec.Key("DAILY_ROTATE").MustBool(true) + writerOption.MaxDays = sec.Key("MAX_DAYS").MustInt(7) + writerOption.Compress = sec.Key("COMPRESS").MustBool(true) + writerOption.CompressionLevel = sec.Key("COMPRESSION_LEVEL").MustInt(-1) + writerMode.WriterOption = writerOption case "conn": - logConfig["reconnectOnMsg"] = sec.Key("RECONNECT_ON_MSG").MustBool() - logConfig["reconnect"] = sec.Key("RECONNECT").MustBool() - logConfig["net"] = sec.Key("PROTOCOL").In("tcp", []string{"tcp", "unix", "udp"}) - logConfig["addr"] = sec.Key("ADDR").MustString(":7020") - case "smtp": - logConfig["username"] = sec.Key("USER").MustString("example@example.com") - logConfig["password"] = sec.Key("PASSWD").MustString("******") - logConfig["host"] = sec.Key("HOST").MustString("127.0.0.1:25") - sendTos := strings.Split(sec.Key("RECEIVERS").MustString(""), ",") - for i, address := range sendTos { - sendTos[i] = strings.TrimSpace(address) + writerOption := log.WriterConnOption{} + writerOption.ReconnectOnMsg = sec.Key("RECONNECT_ON_MSG").MustBool() + writerOption.Reconnect = sec.Key("RECONNECT").MustBool() + writerOption.Protocol = sec.Key("PROTOCOL").In("tcp", []string{"tcp", "unix", "udp"}) + writerOption.Addr = sec.Key("ADDR").MustString(":7020") + writerMode.WriterOption = writerOption + default: + if !log.HasEventWriter(writerMode.WriterType) { + panic(fmt.Sprintf("invalid log writer type (mode): %s", writerMode.WriterType)) } - logConfig["sendTos"] = sendTos - logConfig["subject"] = sec.Key("SUBJECT").MustString("Diagnostic message from Gitea") } - logConfig["colorize"] = sec.Key("COLORIZE").MustBool(false) - byteConfig, err := json.Marshal(logConfig) - if err != nil { - log.Error("Failed to marshal log configuration: %v %v", logConfig, err) - return - } - jsonConfig = string(byteConfig) - return mode, jsonConfig, levelName + return writerMode } -func generateNamedLogger(rootCfg ConfigProvider, key string, options defaultLogOptions) *LogDescription { - description := LogDescription{ - Name: key, - } - - sections := strings.Split(rootCfg.Section("log").Key(strings.ToUpper(key)).MustString(""), ",") - - for i := 0; i < len(sections); i++ { - sections[i] = strings.TrimSpace(sections[i]) - } +var filenameSuffix = "" - for _, name := range sections { - if len(name) == 0 || (name == "console" && options.disableConsole) { - continue - } - sec, err := rootCfg.GetSection("log." + name + "." + key) - if err != nil { - sec, _ = rootCfg.NewSection("log." + name + "." + key) - } - - provider, config, levelName := generateLogConfig(sec, name, options) +// RestartLogsWithPIDSuffix restarts the logs with a PID suffix on files +// FIXME: it seems not right, it breaks log rotating or log collectors +func RestartLogsWithPIDSuffix() { + filenameSuffix = fmt.Sprintf(".%d", os.Getpid()) + initAllLoggers() // when forking, before restarting, rename logger file and re-init all loggers +} - if err := log.NewNamedLogger(key, options.bufferLength, name, provider, config); err != nil { - // Maybe panic here? - log.Error("Could not create new named logger: %v", err.Error()) - } +func InitLoggersForTest() { + initAllLoggers() +} - description.SubLogDescriptions = append(description.SubLogDescriptions, SubLogDescription{ - Name: name, - Provider: provider, - Config: config, - }) - log.Info("%s Log: %s(%s:%s)", util.ToTitleCase(key), util.ToTitleCase(name), provider, levelName) - } +// initAllLoggers creates all the log services +func initAllLoggers() { + loadLogGlobalFrom(CfgProvider) + prepareLoggerConfig(CfgProvider) - AddLogDescription(key, &description) + initLoggerByName(CfgProvider, log.DEFAULT) // default + initLoggerByName(CfgProvider, "access") + initLoggerByName(CfgProvider, "router") + initLoggerByName(CfgProvider, "xorm") - return &description + golog.SetFlags(0) + golog.SetPrefix("") + golog.SetOutput(log.LoggerToWriter(log.GetLogger(log.DEFAULT).Info)) } -// initLogFrom initializes logging with settings from configuration provider -func initLogFrom(rootCfg ConfigProvider) { +func initLoggerByName(rootCfg ConfigProvider, loggerName string) { sec := rootCfg.Section("log") - options := newDefaultLogOptions() - options.bufferLength = Log.BufferLength + keyPrefix := "logger." + strings.ToUpper(loggerName) - description := LogDescription{ - Name: log.DEFAULT, + disabled := sec.HasKey(keyPrefix+".MODE") && sec.Key(keyPrefix+".MODE").String() == "" + if disabled { + return } - sections := strings.Split(sec.Key("MODE").MustString("console"), ",") + modeVal := sec.Key(keyPrefix + ".MODE").String() + if modeVal == "," { + modeVal = Log.Mode + } - useConsole := false - for _, name := range sections { - name = strings.TrimSpace(name) - if name == "" { + var eventWriters []log.EventWriter + modes := strings.Split(modeVal, ",") + for _, modeName := range modes { + modeName = strings.TrimSpace(modeName) + if modeName == "" { continue } - if name == "console" { - useConsole = true - } - - sec, err := rootCfg.GetSection("log." + name + ".default") + opt := loadLogModeByName(rootCfg, modeName) + opt.BufferLen = Log.BufferLen + eventWriter, err := log.NewEventWriter(modeName, opt) if err != nil { - sec, err = rootCfg.GetSection("log." + name) - if err != nil { - sec, _ = rootCfg.NewSection("log." + name) - } - } - - provider, config, levelName := generateLogConfig(sec, name, options) - log.NewLogger(options.bufferLength, name, provider, config) - description.SubLogDescriptions = append(description.SubLogDescriptions, SubLogDescription{ - Name: name, - Provider: provider, - Config: config, - }) - log.Info("Gitea Log Mode: %s(%s:%s)", util.ToTitleCase(name), util.ToTitleCase(provider), levelName) - } - - AddLogDescription(log.DEFAULT, &description) - - if !useConsole { - log.Info("According to the configuration, subsequent logs will not be printed to the console") - if err := log.DelLogger("console"); err != nil { - log.Fatal("Cannot delete console logger: %v", err) + log.FallbackErrorf("Failed to create event writer for logger %s: %v", loggerName, err) + continue } + eventWriters = append(eventWriters, eventWriter) } - // Finally redirect the default golog to here - golog.SetFlags(0) - golog.SetPrefix("") - golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT))) + log.GetManager().GetLogger(loggerName).RemoveAllWriters().AddWriters(eventWriters...) } -// RestartLogsWithPIDSuffix restarts the logs with a PID suffix on files -func RestartLogsWithPIDSuffix() { - filenameSuffix = fmt.Sprintf(".%d", os.Getpid()) - InitLogs(false) +func InitSQLLoggersForCli() { + log.SetConsoleLogger("xorm", "console", log.INFO) } -// InitLogs creates all the log services -func InitLogs(disableConsole bool) { - initLogFrom(CfgProvider) - - if !Log.DisableRouterLog { - options := newDefaultLogOptions() - options.filename = filepath.Join(Log.RootPath, "router.log") - options.flags = "date,time" // For the router we don't want any prefixed flags - options.bufferLength = Log.BufferLength - generateNamedLogger(CfgProvider, "router", options) - } - - if Log.EnableAccessLog { - options := newDefaultLogOptions() - options.filename = filepath.Join(Log.RootPath, "access.log") - options.flags = "" // For the router we don't want any prefixed flags - options.bufferLength = Log.BufferLength - generateNamedLogger(CfgProvider, "access", options) - } - - initSQLLogFrom(CfgProvider, disableConsole) +func InitSQLLoggersForCliDebug() { + log.SetConsoleLogger("xorm", "console", log.DEBUG) } -// InitSQLLog initializes xorm logger setting -func InitSQLLog(disableConsole bool) { - initSQLLogFrom(CfgProvider, disableConsole) +func IsAccessLogEnabled() bool { + return log.IsLoggerEnabled("access") } -func initSQLLogFrom(rootCfg ConfigProvider, disableConsole bool) { - if Log.EnableXORMLog { - options := newDefaultLogOptions() - options.filename = filepath.Join(Log.RootPath, "xorm.log") - options.bufferLength = Log.BufferLength - options.disableConsole = disableConsole - - rootCfg.Section("log").Key("XORM").MustString(",") - generateNamedLogger(rootCfg, "xorm", options) - } +func IsRouteLogEnabled() bool { + return log.IsLoggerEnabled("router") } diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 153307a0b63d..f680e160c4ae 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -278,7 +278,6 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { Repository.MaxCreationLimit = sec.Key("MAX_CREATION_LIMIT").MustInt(-1) Repository.DefaultBranch = sec.Key("DEFAULT_BRANCH").MustString(Repository.DefaultBranch) RepoRootPath = sec.Key("ROOT").MustString(path.Join(AppDataPath, "gitea-repositories")) - forcePathSeparator(RepoRootPath) if !filepath.IsAbs(RepoRootPath) { RepoRootPath = filepath.Join(AppWorkPath, RepoRootPath) } else { diff --git a/modules/setting/server.go b/modules/setting/server.go index 183906268576..d937faca1012 100644 --- a/modules/setting/server.go +++ b/modules/setting/server.go @@ -317,7 +317,6 @@ func loadServerFrom(rootCfg ConfigProvider) { PortToRedirect = sec.Key("PORT_TO_REDIRECT").MustString("80") RedirectorUseProxyProtocol = sec.Key("REDIRECTOR_USE_PROXY_PROTOCOL").MustBool(UseProxyProtocol) OfflineMode = sec.Key("OFFLINE_MODE").MustBool() - Log.DisableRouterLog = sec.Key("DISABLE_ROUTER_LOG").MustBool() if len(StaticRootPath) == 0 { StaticRootPath = AppWorkPath } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index b085a7b32149..8f20ef085690 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -115,7 +115,7 @@ func init() { // We can rely on log.CanColorStdout being set properly because modules/log/console_windows.go comes before modules/setting/setting.go lexicographically // By default set this logger at Info - we'll change it later, but we need to start with something. - log.NewLogger(0, "console", "console", fmt.Sprintf(`{"level": "info", "colorize": %t, "stacktraceLevel": "none"}`, log.CanColorStdout)) + log.SetConsoleLogger(log.DEFAULT, "console", log.INFO) var err error if AppPath, err = getAppPath(); err != nil { @@ -124,12 +124,6 @@ func init() { AppWorkPath = getWorkPath(AppPath) } -func forcePathSeparator(path string) { - if strings.Contains(path, "\\") { - log.Fatal("Do not use '\\' or '\\\\' in paths, instead, please use '/' in all places") - } -} - // IsRunUserMatchCurrentUser returns false if configured run user does not match // actual user that runs the app. The first return value is the actual user name. // This check is ignored under Windows since SSH remote login is not the main @@ -218,9 +212,9 @@ func Init(opts *Options) { // loadCommonSettingsFrom loads common configurations from a configuration provider. func loadCommonSettingsFrom(cfg ConfigProvider) { - // WARNNING: don't change the sequence except you know what you are doing. + // WARNING: don't change the sequence except you know what you are doing. loadRunModeFrom(cfg) - loadLogFrom(cfg) + loadLogGlobalFrom(cfg) loadServerFrom(cfg) loadSSHFrom(cfg) @@ -282,10 +276,11 @@ func mustCurrentRunUserMatch(rootCfg ConfigProvider) { // LoadSettings initializes the settings for normal start up func LoadSettings() { + initAllLoggers() + loadDBSetting(CfgProvider) loadServiceFrom(CfgProvider) loadOAuth2ClientFrom(CfgProvider) - InitLogs(false) loadCacheFrom(CfgProvider) loadSessionFrom(CfgProvider) loadCorsFrom(CfgProvider) diff --git a/modules/ssh/ssh.go b/modules/ssh/ssh.go index 9ec14f2caa74..4bf57eafb721 100644 --- a/modules/ssh/ssh.go +++ b/modules/ssh/ssh.go @@ -223,9 +223,7 @@ func publicKeyHandler(ctx ssh.Context, key ssh.PublicKey) bool { // validate the cert for this principal if err := c.CheckCert(principal, cert); err != nil { // User is presenting an invalid certificate - STOP any further processing - if log.IsError() { - log.Error("Invalid Certificate KeyID %s with Signature Fingerprint %s presented for Principal: %s from %s", cert.KeyId, gossh.FingerprintSHA256(cert.SignatureKey), principal, ctx.RemoteAddr()) - } + log.Error("Invalid Certificate KeyID %s with Signature Fingerprint %s presented for Principal: %s from %s", cert.KeyId, gossh.FingerprintSHA256(cert.SignatureKey), principal, ctx.RemoteAddr()) log.Warn("Failed authentication attempt from %s", ctx.RemoteAddr()) return false @@ -239,10 +237,8 @@ func publicKeyHandler(ctx ssh.Context, key ssh.PublicKey) bool { return true } - if log.IsWarn() { - log.Warn("From %s Fingerprint: %s is a certificate, but no valid principals found", ctx.RemoteAddr(), gossh.FingerprintSHA256(key)) - log.Warn("Failed authentication attempt from %s", ctx.RemoteAddr()) - } + log.Warn("From %s Fingerprint: %s is a certificate, but no valid principals found", ctx.RemoteAddr(), gossh.FingerprintSHA256(key)) + log.Warn("Failed authentication attempt from %s", ctx.RemoteAddr()) return false } @@ -253,10 +249,8 @@ func publicKeyHandler(ctx ssh.Context, key ssh.PublicKey) bool { pkey, err := asymkey_model.SearchPublicKeyByContent(ctx, strings.TrimSpace(string(gossh.MarshalAuthorizedKey(key)))) if err != nil { if asymkey_model.IsErrKeyNotExist(err) { - if log.IsWarn() { - log.Warn("Unknown public key: %s from %s", gossh.FingerprintSHA256(key), ctx.RemoteAddr()) - log.Warn("Failed authentication attempt from %s", ctx.RemoteAddr()) - } + log.Warn("Unknown public key: %s from %s", gossh.FingerprintSHA256(key), ctx.RemoteAddr()) + log.Warn("Failed authentication attempt from %s", ctx.RemoteAddr()) return false } log.Error("SearchPublicKeyByContent: %v", err) diff --git a/modules/templates/htmlrenderer.go b/modules/templates/htmlrenderer.go index d60be887278a..311e5b741d43 100644 --- a/modules/templates/htmlrenderer.go +++ b/modules/templates/htmlrenderer.go @@ -126,7 +126,7 @@ func wrapFatal(msg string) { if msg == "" { return } - log.FatalWithSkip(1, "Unable to compile templates, %s", msg) + log.Fatal("Unable to compile templates, %s", msg) } type templateErrorPrettier struct { diff --git a/modules/test/logchecker.go b/modules/test/logchecker.go index 8f8c753c76fc..b85ee3df5f54 100644 --- a/modules/test/logchecker.go +++ b/modules/test/logchecker.go @@ -4,7 +4,8 @@ package test import ( - "strconv" + "context" + "fmt" "strings" "sync" "sync/atomic" @@ -14,9 +15,7 @@ import ( ) type LogChecker struct { - logger *log.MultiChannelledLogger - loggerName string - eventLoggerName string + *log.EventWriterBaseImpl filterMessages []string filtered []bool @@ -27,54 +26,44 @@ type LogChecker struct { mu sync.Mutex } -func (lc *LogChecker) LogEvent(event *log.Event) error { +func (lc *LogChecker) Run(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case event, ok := <-lc.Queue: + if !ok { + return + } + lc.checkLogEvent(event) + } + } +} + +func (lc *LogChecker) checkLogEvent(event *log.Event) { lc.mu.Lock() defer lc.mu.Unlock() for i, msg := range lc.filterMessages { - if strings.Contains(event.GetMsg(), msg) { + if strings.Contains(event.Msg, msg) { lc.filtered[i] = true } } - if strings.Contains(event.GetMsg(), lc.stopMark) { + if strings.Contains(event.Msg, lc.stopMark) { lc.stopped = true } - return nil -} - -func (lc *LogChecker) Close() {} - -func (lc *LogChecker) Flush() {} - -func (lc *LogChecker) GetLevel() log.Level { - return log.TRACE -} - -func (lc *LogChecker) GetStacktraceLevel() log.Level { - return log.NONE -} - -func (lc *LogChecker) GetName() string { - return lc.eventLoggerName -} - -func (lc *LogChecker) ReleaseReopen() error { - return nil } var checkerIndex int64 -func NewLogChecker(loggerName string) (logChecker *LogChecker, cancel func()) { - logger := log.GetLogger(loggerName) +func NewLogChecker(namePrefix string) (logChecker *LogChecker, cancel func()) { + logger := log.GetManager().GetLogger(namePrefix) newCheckerIndex := atomic.AddInt64(&checkerIndex, 1) - lc := &LogChecker{ - logger: logger, - loggerName: loggerName, - eventLoggerName: "TestLogChecker-" + strconv.FormatInt(newCheckerIndex, 10), - } - if err := logger.AddLogger(lc); err != nil { - panic(err) // it's impossible - } - return lc, func() { _, _ = logger.DelLogger(lc.GetName()) } + writerName := namePrefix + "-" + fmt.Sprint(newCheckerIndex) + + lc := &LogChecker{} + lc.EventWriterBaseImpl = log.NewEventWriterBase(writerName, log.WriterMode{}) + logger.AddWriters(lc) + return lc, func() { _ = logger.RemoveWriter(writerName) } } // Filter will make the `Check` function to check if these logs are outputted. diff --git a/modules/test/logchecker_test.go b/modules/test/logchecker_test.go index 4dfea8c3e382..6b093ab1b36a 100644 --- a/modules/test/logchecker_test.go +++ b/modules/test/logchecker_test.go @@ -13,8 +13,6 @@ import ( ) func TestLogChecker(t *testing.T) { - _ = log.NewLogger(1000, "console", "console", `{"level":"info","stacktracelevel":"NONE","stderr":true}`) - lc, cleanup := NewLogChecker(log.DEFAULT) defer cleanup() diff --git a/modules/testlogger/testlogger.go b/modules/testlogger/testlogger.go index cc80e86c81cb..356eff65fd14 100644 --- a/modules/testlogger/testlogger.go +++ b/modules/testlogger/testlogger.go @@ -13,7 +13,6 @@ import ( "testing" "time" - "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/queue" ) @@ -24,11 +23,6 @@ var ( SlowFlush = 5 * time.Second ) -// TestLogger is a logger which will write to the testing log -type TestLogger struct { - log.WriterLogger -} - var WriterCloser = &testLoggerWriterCloser{} type testLoggerWriterCloser struct { @@ -171,37 +165,18 @@ func Printf(format string, args ...interface{}) { fmt.Fprintf(os.Stdout, "\t"+format, args...) } -// NewTestLogger creates a TestLogger as a log.LoggerProvider -func NewTestLogger() log.LoggerProvider { - logger := &TestLogger{} - logger.Colorize = log.CanColorStdout - logger.Level = log.TRACE - return logger -} - -// Init inits connection writer with json config. -// json config only need key "level". -func (log *TestLogger) Init(config string) error { - err := json.Unmarshal([]byte(config), log) - if err != nil { - return err - } - log.NewWriterLogger(WriterCloser) - return nil -} - -// Flush when log should be flushed -func (log *TestLogger) Flush() { -} - -// ReleaseReopen does nothing -func (log *TestLogger) ReleaseReopen() error { - return nil +// TestLogEventWriter is a logger which will write to the testing log +type TestLogEventWriter struct { + *log.EventWriterBaseImpl } -// GetName returns the default name for this implementation -func (log *TestLogger) GetName() string { - return "test" +// NewTestLoggerWriter creates a TestLogEventWriter as a log.LoggerProvider +func NewTestLoggerWriter(name string, mode log.WriterMode) log.EventWriter { + w := &TestLogEventWriter{} + w.EventWriterBaseImpl = log.NewEventWriterBase(name, mode) + w.Formatter = log.EventFormatTextMessage + w.OutputWriteCloser = WriterCloser + return w } func init() { diff --git a/modules/util/rotatingfilewriter/writer.go b/modules/util/rotatingfilewriter/writer.go new file mode 100644 index 000000000000..a38e8ba3cd8b --- /dev/null +++ b/modules/util/rotatingfilewriter/writer.go @@ -0,0 +1,226 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rotatingfilewriter + +import ( + "bufio" + "compress/gzip" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "code.gitea.io/gitea/modules/graceful/releasereopen" + "code.gitea.io/gitea/modules/util" +) + +type Options struct { + Rotate bool + MaximumSize int64 + RotateDaily bool + KeepDays int + Compress bool + CompressionLevel int +} + +type RotatingFileWriter struct { + mu sync.Mutex + fd *os.File + + currentSize int64 + openDate int + + options Options + + cancelReleaseReopen func() +} + +// Open creates a new rotating file writer. +// Notice: if a file is opened by two rotators, there will be conflicts when rotating. +// In the future, there should be "rotating file manager" +func Open(filename string, options *Options) (*RotatingFileWriter, error) { + if options == nil { + options = &Options{} + } + + rfw := &RotatingFileWriter{ + options: *options, + } + + if err := rfw.open(filename); err != nil { + return nil, err + } + + rfw.cancelReleaseReopen = releasereopen.GetManager().Register(rfw) + return rfw, nil +} + +func (rfw *RotatingFileWriter) Write(b []byte) (int, error) { + if rfw.options.Rotate && ((rfw.options.MaximumSize > 0 && rfw.currentSize >= rfw.options.MaximumSize) || (rfw.options.RotateDaily && time.Now().Day() != rfw.openDate)) { + if err := rfw.DoRotate(); err != nil { + // This should be + // return 0, err + // but the old behaviour does not return. This may lead to other errors. + fmt.Fprintf(os.Stderr, "RotatingFileWriter: %s\n", err) + } + } + + n, err := rfw.fd.Write(b) + if err == nil { + rfw.currentSize += int64(n) + } + return n, err +} + +func (rfw *RotatingFileWriter) Flush() error { + return rfw.fd.Sync() +} + +func (rfw *RotatingFileWriter) Close() error { + rfw.mu.Lock() + if rfw.cancelReleaseReopen != nil { + rfw.cancelReleaseReopen() + rfw.cancelReleaseReopen = nil + } + rfw.mu.Unlock() + return rfw.fd.Close() +} + +func (rfw *RotatingFileWriter) open(filename string) error { + fd, err := os.OpenFile(filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o660) + if err != nil { + return err + } + + rfw.fd = fd + + finfo, err := fd.Stat() + if err != nil { + return err + } + rfw.currentSize = finfo.Size() + rfw.openDate = time.Now().Day() + + return nil +} + +func (rfw *RotatingFileWriter) ReleaseReopen() error { + return errors.Join( + rfw.fd.Close(), + rfw.open(rfw.fd.Name()), + ) +} + +// DoRotate the log file creating a backup like xx.2013-01-01.2 +func (rfw *RotatingFileWriter) DoRotate() error { + if !rfw.options.Rotate { + return nil + } + + rfw.mu.Lock() + defer rfw.mu.Unlock() + + prefix := fmt.Sprintf("%s.%s.", rfw.fd.Name(), time.Now().Format("2006-01-02")) + + var err error + fname := "" + for i := 1; err == nil && i <= 999; i++ { + fname = prefix + fmt.Sprintf("%03d", i) + _, err = os.Lstat(fname) + if rfw.options.Compress && err != nil { + _, err = os.Lstat(fname + ".gz") + } + } + // return error if the last file checked still existed + if err == nil { + return fmt.Errorf("cannot find free file to rename %s", rfw.fd.Name()) + } + + fd := rfw.fd + if err := fd.Close(); err != nil { // close file before rename + return err + } + + if err := util.Rename(fd.Name(), fname); err != nil { + return err + } + + if rfw.options.Compress { + go compressOldFile(fname, rfw.options.CompressionLevel) //nolint:errcheck + } + + if err := rfw.open(fd.Name()); err != nil { + return err + } + + go deleteOldFiles( + filepath.Dir(fd.Name()), + filepath.Base(fd.Name()), + time.Now().AddDate(0, 0, -rfw.options.KeepDays), + ) + + return nil +} + +func compressOldFile(fname string, compressionLevel int) error { + reader, err := os.Open(fname) + if err != nil { + return err + } + defer reader.Close() + + buffer := bufio.NewReader(reader) + fw, err := os.OpenFile(fname+".gz", os.O_WRONLY|os.O_CREATE, 0o660) + if err != nil { + return err + } + defer fw.Close() + + zw, err := gzip.NewWriterLevel(fw, compressionLevel) + if err != nil { + return err + } + defer zw.Close() + + _, err = buffer.WriteTo(zw) + if err != nil { + zw.Close() + fw.Close() + util.Remove(fname + ".gz") //nolint:errcheck + return err + } + reader.Close() + + return util.Remove(fname) +} + +func deleteOldFiles(dir, prefix string, removeBefore time.Time) { + _ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) (returnErr error) { + defer func() { + if r := recover(); r != nil { + returnErr = fmt.Errorf("unable to delete old file '%s', error: %+v", path, r) + } + }() + + if err != nil { + return err + } + if d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + if info.ModTime().Before(removeBefore) { + if strings.HasPrefix(filepath.Base(path), prefix) { + return util.Remove(path) + } + } + return nil + }) +} diff --git a/modules/util/rotatingfilewriter/writer_test.go b/modules/util/rotatingfilewriter/writer_test.go new file mode 100644 index 000000000000..88392797b375 --- /dev/null +++ b/modules/util/rotatingfilewriter/writer_test.go @@ -0,0 +1,48 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rotatingfilewriter + +import ( + "compress/gzip" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCompressOldFile(t *testing.T) { + tmpDir := t.TempDir() + fname := filepath.Join(tmpDir, "test") + nonGzip := filepath.Join(tmpDir, "test-nonGzip") + + f, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, 0o660) + assert.NoError(t, err) + ng, err := os.OpenFile(nonGzip, os.O_CREATE|os.O_WRONLY, 0o660) + assert.NoError(t, err) + + for i := 0; i < 999; i++ { + f.WriteString("This is a test file\n") + ng.WriteString("This is a test file\n") + } + f.Close() + ng.Close() + + err = compressOldFile(fname, gzip.DefaultCompression) + assert.NoError(t, err) + + _, err = os.Lstat(fname + ".gz") + assert.NoError(t, err) + + f, err = os.Open(fname + ".gz") + assert.NoError(t, err) + zr, err := gzip.NewReader(f) + assert.NoError(t, err) + data, err := io.ReadAll(zr) + assert.NoError(t, err) + original, err := os.ReadFile(nonGzip) + assert.NoError(t, err) + assert.Equal(t, original, data) +} diff --git a/modules/web/routing/logger.go b/modules/web/routing/logger.go index d1b0ff0cda6d..b58065aa7365 100644 --- a/modules/web/routing/logger.go +++ b/modules/web/routing/logger.go @@ -5,6 +5,7 @@ package routing import ( "net/http" + "strings" "time" "code.gitea.io/gitea/modules/context" @@ -25,18 +26,18 @@ func NewLoggerHandler() func(next http.Handler) http.Handler { } var ( - startMessage = log.NewColoredValueBytes("started ", log.DEBUG.Color()) - slowMessage = log.NewColoredValueBytes("slow ", log.WARN.Color()) - pollingMessage = log.NewColoredValueBytes("polling ", log.INFO.Color()) - failedMessage = log.NewColoredValueBytes("failed ", log.WARN.Color()) - completedMessage = log.NewColoredValueBytes("completed", log.INFO.Color()) - unknownHandlerMessage = log.NewColoredValueBytes("completed", log.ERROR.Color()) + startMessage = log.NewColoredValue("started ", log.DEBUG.ColorAttributes()...) + slowMessage = log.NewColoredValue("slow ", log.WARN.ColorAttributes()...) + pollingMessage = log.NewColoredValue("polling ", log.INFO.ColorAttributes()...) + failedMessage = log.NewColoredValue("failed ", log.WARN.ColorAttributes()...) + completedMessage = log.NewColoredValue("completed", log.INFO.ColorAttributes()...) + unknownHandlerMessage = log.NewColoredValue("completed", log.ERROR.ColorAttributes()...) ) func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { return func(trigger Event, record *requestRecord) { if trigger == StartEvent { - if !logger.IsTrace() { + if !logger.LevelEnabled(log.TRACE) { // for performance, if the "started" message shouldn't be logged, we just return as early as possible // developers can set the router log level to TRACE to get the "started" request messages. return @@ -59,12 +60,12 @@ func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { if trigger == StillExecutingEvent { message := slowMessage - level := log.WARN + logf := logger.Warn if isLongPolling { - level = log.INFO + logf = logger.Info message = pollingMessage } - _ = logger.Log(0, level, "router: %s %v %s for %s, elapsed %v @ %s", + logf("router: %s %v %s for %s, elapsed %v @ %s", message, log.ColoredMethod(req.Method), req.RequestURI, req.RemoteAddr, log.ColoredTime(time.Since(record.startTime)), @@ -74,7 +75,7 @@ func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { } if panicErr != nil { - _ = logger.Log(0, log.WARN, "router: %s %v %s for %s, panic in %v @ %s, err=%v", + logger.Warn("router: %s %v %s for %s, panic in %v @ %s, err=%v", failedMessage, log.ColoredMethod(req.Method), req.RequestURI, req.RemoteAddr, log.ColoredTime(time.Since(record.startTime)), @@ -88,14 +89,17 @@ func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { if v, ok := record.responseWriter.(context.ResponseWriter); ok { status = v.Status() } - level := log.INFO + logf := log.Info + if strings.HasPrefix(req.RequestURI, "/assets/") { + logf = log.Trace + } message := completedMessage if isUnknownHandler { - level = log.ERROR + logf = log.Error message = unknownHandlerMessage } - _ = logger.Log(0, level, "router: %s %v %s for %s, %v %v in %v @ %s", + logf("router: %s %v %s for %s, %v %v in %v @ %s", message, log.ColoredMethod(req.Method), req.RequestURI, req.RemoteAddr, log.ColoredStatus(status), log.ColoredStatus(status, http.StatusText(status)), log.ColoredTime(time.Since(record.startTime)), diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 906a32dc2d73..c83bdb860d6e 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3026,15 +3026,10 @@ config.git_pull_timeout = Pull Operation Timeout config.git_gc_timeout = GC Operation Timeout config.log_config = Log Configuration -config.log_mode = Log Mode -config.own_named_logger = Named Logger -config.routes_to_default_logger = Routes To Default Logger -config.go_log = Uses Go Log (redirected to default) -config.router_log_mode = Router Log Mode +config.logger_name_fmt = Logger: %s config.disabled_logger = Disabled config.access_log_mode = Access Log Mode -config.access_log_template = Template -config.xorm_log_mode = XORM Log Mode +config.access_log_template = Access Log Template config.xorm_log_sql = Log SQL config.get_setting_failed = Get setting %s failed diff --git a/routers/common/middleware.go b/routers/common/middleware.go index c1ee9dd765af..23a88922d03f 100644 --- a/routers/common/middleware.go +++ b/routers/common/middleware.go @@ -60,11 +60,11 @@ func ProtocolMiddlewares() (handlers []any) { handlers = append(handlers, proxy.ForwardedHeaders(opt)) } - if !setting.Log.DisableRouterLog { + if setting.IsRouteLogEnabled() { handlers = append(handlers, routing.NewLoggerHandler()) } - if setting.Log.EnableAccessLog { + if setting.IsAccessLogEnabled() { handlers = append(handlers, context.AccessLogger()) } diff --git a/routers/private/internal.go b/routers/private/internal.go index c6ea9e026383..b09fb58d05dd 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -73,7 +73,7 @@ func Routes() *web.Route { r.Post("/manager/release-and-reopen-logging", ReleaseReopenLogging) r.Post("/manager/set-log-sql", SetLogSQL) r.Post("/manager/add-logger", bind(private.LoggerOptions{}), AddLogger) - r.Post("/manager/remove-logger/{group}/{name}", RemoveLogger) + r.Post("/manager/remove-logger/{logger}/{writer}", RemoveLogger) r.Get("/manager/processes", Processes) r.Post("/mail/send", SendEmail) r.Post("/restore_repo", RestoreRepo) diff --git a/routers/private/manager.go b/routers/private/manager.go index 38ad83326fe1..872e65b2a1b6 100644 --- a/routers/private/manager.go +++ b/routers/private/manager.go @@ -10,7 +10,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/graceful" - "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/graceful/releasereopen" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/queue" @@ -46,19 +46,19 @@ func FlushQueues(ctx *context.PrivateContext) { // PauseLogging pauses logging func PauseLogging(ctx *context.PrivateContext) { - log.Pause() + log.GetManager().PauseAll() ctx.PlainText(http.StatusOK, "success") } // ResumeLogging resumes logging func ResumeLogging(ctx *context.PrivateContext) { - log.Resume() + log.GetManager().ResumeAll() ctx.PlainText(http.StatusOK, "success") } // ReleaseReopenLogging releases and reopens logging files func ReleaseReopenLogging(ctx *context.PrivateContext) { - if err := log.ReleaseReopen(); err != nil { + if err := releasereopen.GetManager().ReleaseReopen(); err != nil { ctx.JSON(http.StatusInternalServerError, private.Response{ Err: fmt.Sprintf("Error during release and reopen: %v", err), }) @@ -75,90 +75,108 @@ func SetLogSQL(ctx *context.PrivateContext) { // RemoveLogger removes a logger func RemoveLogger(ctx *context.PrivateContext) { - group := ctx.Params("group") - name := ctx.Params("name") - ok, err := log.GetLogger(group).DelLogger(name) + logger := ctx.Params("logger") + writer := ctx.Params("writer") + err := log.GetManager().GetLogger(logger).RemoveWriter(writer) if err != nil { ctx.JSON(http.StatusInternalServerError, private.Response{ - Err: fmt.Sprintf("Failed to remove logger: %s %s %v", group, name, err), + Err: fmt.Sprintf("Failed to remove log writer: %s %s %v", logger, writer, err), }) return } - if ok { - setting.RemoveSubLogDescription(group, name) - } - ctx.PlainText(http.StatusOK, fmt.Sprintf("Removed %s %s", group, name)) + ctx.PlainText(http.StatusOK, fmt.Sprintf("Removed %s %s", logger, writer)) } // AddLogger adds a logger func AddLogger(ctx *context.PrivateContext) { opts := web.GetForm(ctx).(*private.LoggerOptions) - if len(opts.Group) == 0 { - opts.Group = log.DEFAULT + + if len(opts.Logger) == 0 { + opts.Logger = log.DEFAULT } - if _, ok := opts.Config["flags"]; !ok { - switch opts.Group { + + writerMode := log.WriterMode{} + writerMode.WriterType = opts.Mode + + var flags string + var ok bool + if flags, ok = opts.Config["flags"].(string); !ok { + switch opts.Logger { case "access": - opts.Config["flags"] = log.FlagsFromString("") + flags = "" case "router": - opts.Config["flags"] = log.FlagsFromString("date,time") + flags = "date,time" default: - opts.Config["flags"] = log.FlagsFromString("stdflags") + flags = "stdflags" } } + writerMode.Flags = log.FlagsFromString(flags) - if _, ok := opts.Config["colorize"]; !ok && opts.Mode == "console" { + if writerMode.Colorize, ok = opts.Config["colorize"].(bool); !ok && opts.Mode == "console" { if _, ok := opts.Config["stderr"]; ok { - opts.Config["colorize"] = log.CanColorStderr + writerMode.Colorize = log.CanColorStderr } else { - opts.Config["colorize"] = log.CanColorStdout + writerMode.Colorize = log.CanColorStdout } } - if _, ok := opts.Config["level"]; !ok { - opts.Config["level"] = setting.Log.Level + writerMode.Level = setting.Log.Level + if level, ok := opts.Config["level"].(string); ok { + writerMode.Level = log.LevelFromString(level) } - if _, ok := opts.Config["stacktraceLevel"]; !ok { - opts.Config["stacktraceLevel"] = setting.Log.StacktraceLogLevel + writerMode.StacktraceLevel = setting.Log.StacktraceLogLevel + if stacktraceLevel, ok := opts.Config["level"].(string); ok { + writerMode.StacktraceLevel = log.LevelFromString(stacktraceLevel) } - if opts.Mode == "file" { - if _, ok := opts.Config["maxsize"]; !ok { - opts.Config["maxsize"] = 1 << 28 + writerMode.Prefix, _ = opts.Config["prefix"].(string) + writerMode.Expression, _ = opts.Config["expression"].(string) + + switch opts.Mode { + case "console": + writerOption := log.WriterConsoleOption{} + writerOption.Stderr, _ = opts.Config["stderr"].(bool) + writerMode.WriterOption = writerOption + case "file": + writerOption := log.WriterFileOption{} + fileName, _ := opts.Config["filename"].(string) + writerOption.FileName = setting.LogPrepareFilenameForWriter(opts.Writer, fileName) + writerOption.LogRotate = opts.Config["rotate"].(bool) + maxSizeShift, _ := opts.Config["maxsize"].(int) + if maxSizeShift == 0 { + maxSizeShift = 28 } - if _, ok := opts.Config["maxdays"]; !ok { - opts.Config["maxdays"] = 7 + writerOption.MaxSize = 1 << maxSizeShift + writerOption.DailyRotate, _ = opts.Config["daily"].(bool) + writerOption.MaxDays, _ = opts.Config["maxdays"].(int) + if writerOption.MaxDays == 0 { + writerOption.MaxDays = 7 } - if _, ok := opts.Config["compressionLevel"]; !ok { - opts.Config["compressionLevel"] = -1 + writerOption.Compress, _ = opts.Config["compress"].(bool) + writerOption.CompressionLevel, _ = opts.Config["compressionLevel"].(int) + if writerOption.CompressionLevel == 0 { + writerOption.CompressionLevel = -1 } + writerMode.WriterOption = writerOption + case "conn": + writerOption := log.WriterConnOption{} + writerOption.ReconnectOnMsg, _ = opts.Config["reconnectOnMsg"].(bool) + writerOption.Reconnect, _ = opts.Config["reconnect"].(bool) + writerOption.Protocol, _ = opts.Config["net"].(string) + writerOption.Addr, _ = opts.Config["address"].(string) + writerMode.WriterOption = writerOption + default: + panic(fmt.Sprintf("invalid log writer mode: %s", writerMode.WriterType)) } - - bufferLen := setting.Log.BufferLength - byteConfig, err := json.Marshal(opts.Config) + writer, err := log.NewEventWriter(opts.Writer, writerMode) if err != nil { - log.Error("Failed to marshal log configuration: %v %v", opts.Config, err) - ctx.JSON(http.StatusInternalServerError, private.Response{ - Err: fmt.Sprintf("Failed to marshal log configuration: %v %v", opts.Config, err), - }) - return - } - config := string(byteConfig) - - if err := log.NewNamedLogger(opts.Group, bufferLen, opts.Name, opts.Mode, config); err != nil { - log.Error("Failed to create new named logger: %s %v", config, err) + log.Error("Failed to create new log writer: %v", err) ctx.JSON(http.StatusInternalServerError, private.Response{ - Err: fmt.Sprintf("Failed to create new named logger: %s %v", config, err), + Err: fmt.Sprintf("Failed to create new log writer: %v", err), }) return } - - setting.AddSubLogDescription(opts.Group, setting.SubLogDescription{ - Name: opts.Name, - Provider: opts.Mode, - Config: config, - }) - + log.GetManager().GetLogger(opts.Logger).AddWriters(writer) ctx.PlainText(http.StatusOK, "success") } diff --git a/routers/web/admin/config.go b/routers/web/admin/config.go index 7460ea24a745..be662c22efdc 100644 --- a/routers/web/admin/config.go +++ b/routers/web/admin/config.go @@ -117,7 +117,6 @@ func Config(ctx *context.Context) { ctx.Data["AppBuiltWith"] = setting.AppBuiltWith ctx.Data["Domain"] = setting.Domain ctx.Data["OfflineMode"] = setting.OfflineMode - ctx.Data["DisableRouterLog"] = setting.Log.DisableRouterLog ctx.Data["RunUser"] = setting.RunUser ctx.Data["RunMode"] = util.ToTitleCase(setting.RunMode) ctx.Data["GitVersion"] = git.VersionInfo() @@ -182,13 +181,11 @@ func Config(ctx *context.Context) { } ctx.Data["EnvVars"] = envVars - ctx.Data["Loggers"] = setting.GetLogDescriptions() - ctx.Data["EnableAccessLog"] = setting.Log.EnableAccessLog ctx.Data["AccessLogTemplate"] = setting.Log.AccessLogTemplate - ctx.Data["DisableRouterLog"] = setting.Log.DisableRouterLog - ctx.Data["EnableXORMLog"] = setting.Log.EnableXORMLog ctx.Data["LogSQL"] = setting.Database.LogSQL + ctx.Data["Loggers"] = log.GetManager().DumpLoggers() + ctx.HTML(http.StatusOK, tplConfig) } diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index c2f30a01f473..6e9b1f91434f 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -2739,7 +2739,7 @@ func NewComment(ctx *context.Context) { log.Trace("Permission Denied: User %-v not the Poster (ID: %d) and cannot read %s in Repo %-v.\n"+ "User in Repo has Permissions: %-+v", ctx.Doer, - log.NewColoredIDValue(issue.PosterID), + issue.PosterID, issueType, ctx.Repo.Repository, ctx.Repo.Permission) @@ -3017,7 +3017,7 @@ func ChangeIssueReaction(ctx *context.Context) { log.Trace("Permission Denied: User %-v not the Poster (ID: %d) and cannot read %s in Repo %-v.\n"+ "User in Repo has Permissions: %-+v", ctx.Doer, - log.NewColoredIDValue(issue.PosterID), + issue.PosterID, issueType, ctx.Repo.Repository, ctx.Repo.Permission) @@ -3119,7 +3119,7 @@ func ChangeCommentReaction(ctx *context.Context) { log.Trace("Permission Denied: User %-v not the Poster (ID: %d) and cannot read %s in Repo %-v.\n"+ "User in Repo has Permissions: %-+v", ctx.Doer, - log.NewColoredIDValue(comment.Issue.PosterID), + comment.Issue.PosterID, issueType, ctx.Repo.Repository, ctx.Repo.Permission) diff --git a/routers/web/repo/issue_watch.go b/routers/web/repo/issue_watch.go index 1837c2b632c9..d3d3a2af21e3 100644 --- a/routers/web/repo/issue_watch.go +++ b/routers/web/repo/issue_watch.go @@ -29,7 +29,7 @@ func IssueWatch(ctx *context.Context) { log.Trace("Permission Denied: User %-v not the Poster (ID: %d) and cannot read %s in Repo %-v.\n"+ "User in Repo has Permissions: %-+v", ctx.Doer, - log.NewColoredIDValue(issue.PosterID), + issue.PosterID, issueType, ctx.Repo.Repository, ctx.Repo.Permission) diff --git a/services/migrations/codebase.go b/services/migrations/codebase.go index c02c8e13a2e3..22bf0f73de71 100644 --- a/services/migrations/codebase.go +++ b/services/migrations/codebase.go @@ -115,13 +115,11 @@ func (d *CodebaseDownloader) String() string { return fmt.Sprintf("migration from codebase server %s %s/%s", d.baseURL, d.project, d.repoName) } -// ColorFormat provides a basic color format for a GogsDownloader -func (d *CodebaseDownloader) ColorFormat(s fmt.State) { +func (d *CodebaseDownloader) LogString() string { if d == nil { - log.ColorFprintf(s, "") - return + return "" } - log.ColorFprintf(s, "migration from codebase server %s %s/%s", d.baseURL, d.project, d.repoName) + return fmt.Sprintf("", d.baseURL, d.project, d.repoName) } // FormatCloneURL add authentication into remote URLs diff --git a/services/migrations/gitbucket.go b/services/migrations/gitbucket.go index cc3d4fc93674..38dc2ac77c65 100644 --- a/services/migrations/gitbucket.go +++ b/services/migrations/gitbucket.go @@ -62,13 +62,11 @@ func (g *GitBucketDownloader) String() string { return fmt.Sprintf("migration from gitbucket server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) } -// ColorFormat provides a basic color format for a GitBucketDownloader -func (g *GitBucketDownloader) ColorFormat(s fmt.State) { +func (g *GitBucketDownloader) LogString() string { if g == nil { - log.ColorFprintf(s, "") - return + return "" } - log.ColorFprintf(s, "migration from gitbucket server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) + return fmt.Sprintf("", g.baseURL, g.repoOwner, g.repoName) } // NewGitBucketDownloader creates a GitBucket downloader diff --git a/services/migrations/gitea_downloader.go b/services/migrations/gitea_downloader.go index 470090b5010a..b9ba93325b91 100644 --- a/services/migrations/gitea_downloader.go +++ b/services/migrations/gitea_downloader.go @@ -134,13 +134,11 @@ func (g *GiteaDownloader) String() string { return fmt.Sprintf("migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) } -// ColorFormat provides a basic color format for a GiteaDownloader -func (g *GiteaDownloader) ColorFormat(s fmt.State) { +func (g *GiteaDownloader) LogString() string { if g == nil { - log.ColorFprintf(s, "") - return + return "" } - log.ColorFprintf(s, "migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) + return fmt.Sprintf("", g.baseURL, g.repoOwner, g.repoName) } // GetRepoInfo returns a repository information diff --git a/services/migrations/github.go b/services/migrations/github.go index 3e63fddb6a5d..ca6e037b68cd 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -125,13 +125,11 @@ func (g *GithubDownloaderV3) String() string { return fmt.Sprintf("migration from github server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) } -// ColorFormat provides a basic color format for a GithubDownloader -func (g *GithubDownloaderV3) ColorFormat(s fmt.State) { +func (g *GithubDownloaderV3) LogString() string { if g == nil { - log.ColorFprintf(s, "") - return + return "" } - log.ColorFprintf(s, "migration from github server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) + return fmt.Sprintf("", g.baseURL, g.repoOwner, g.repoName) } func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) { diff --git a/services/migrations/gitlab.go b/services/migrations/gitlab.go index 8034869a4ae4..015c38cd3b0c 100644 --- a/services/migrations/gitlab.go +++ b/services/migrations/gitlab.go @@ -137,13 +137,11 @@ func (g *GitlabDownloader) String() string { return fmt.Sprintf("migration from gitlab server %s [%d]/%s", g.baseURL, g.repoID, g.repoName) } -// ColorFormat provides a basic color format for a GitlabDownloader -func (g *GitlabDownloader) ColorFormat(s fmt.State) { +func (g *GitlabDownloader) LogString() string { if g == nil { - log.ColorFprintf(s, "") - return + return "" } - log.ColorFprintf(s, "migration from gitlab server %s [%d]/%s", g.baseURL, g.repoID, g.repoName) + return fmt.Sprintf("", g.baseURL, g.repoID, g.repoName) } // SetContext set context diff --git a/services/migrations/gogs.go b/services/migrations/gogs.go index d01934ac6d52..72c52d180b9b 100644 --- a/services/migrations/gogs.go +++ b/services/migrations/gogs.go @@ -77,13 +77,11 @@ func (g *GogsDownloader) String() string { return fmt.Sprintf("migration from gogs server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) } -// ColorFormat provides a basic color format for a GogsDownloader -func (g *GogsDownloader) ColorFormat(s fmt.State) { +func (g *GogsDownloader) LogString() string { if g == nil { - log.ColorFprintf(s, "") - return + return "" } - log.ColorFprintf(s, "migration from gogs server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) + return fmt.Sprintf("", g.baseURL, g.repoOwner, g.repoName) } // SetContext set context diff --git a/services/migrations/onedev.go b/services/migrations/onedev.go index d4b1b73d37c2..33fc43c34961 100644 --- a/services/migrations/onedev.go +++ b/services/migrations/onedev.go @@ -114,13 +114,11 @@ func (d *OneDevDownloader) String() string { return fmt.Sprintf("migration from oneDev server %s [%d]/%s", d.baseURL, d.repoID, d.repoName) } -// ColorFormat provides a basic color format for a OneDevDownloader -func (d *OneDevDownloader) ColorFormat(s fmt.State) { +func (d *OneDevDownloader) LogString() string { if d == nil { - log.ColorFprintf(s, "") - return + return "" } - log.ColorFprintf(s, "migration from oneDev server %s [%d]/%s", d.baseURL, d.repoID, d.repoName) + return fmt.Sprintf("", d.baseURL, d.repoID, d.repoName) } func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result interface{}) error { diff --git a/templates/admin/config.tmpl b/templates/admin/config.tmpl index c4f77ec1ae23..629632a7187a 100644 --- a/templates/admin/config.tmpl +++ b/templates/admin/config.tmpl @@ -361,66 +361,23 @@
- {{range .Loggers.default.SubLogDescriptions}} -
{{$.locale.Tr "admin.config.log_mode"}}
-
{{.Name}} ({{.Provider}})
-
{{$.locale.Tr "admin.config.log_config"}}
- 
{{JsonUtils.PrettyIndent .Config}}
- {{end}} -
-
{{$.locale.Tr "admin.config.router_log_mode"}}
- {{if .DisableRouterLog}} -
{{$.locale.Tr "admin.config.disabled_logger"}}
- {{else}} - {{if .Loggers.router.SubLogDescriptions}} -
{{$.locale.Tr "admin.config.own_named_logger"}}
- {{range .Loggers.router.SubLogDescriptions}} -
{{$.locale.Tr "admin.config.log_mode"}}
-
{{.Name}} ({{.Provider}})
-
{{$.locale.Tr "admin.config.log_config"}}
- 
{{JsonUtils.PrettyIndent .Config}}
- {{end}} - {{else}} -
{{$.locale.Tr "admin.config.routes_to_default_logger"}}
- {{end}} + {{if .Loggers.xorm.IsEnabled}} +
{{$.locale.Tr "admin.config.xorm_log_sql"}}
+
{{if $.LogSQL}}{{svg "octicon-check"}}{{else}}{{svg "octicon-x"}}{{end}}
{{end}} -
-
{{$.locale.Tr "admin.config.access_log_mode"}}
- {{if .EnableAccessLog}} - {{if .Loggers.access.SubLogDescriptions}} -
{{$.locale.Tr "admin.config.own_named_logger"}}
- {{range .Loggers.access.SubLogDescriptions}} -
{{$.locale.Tr "admin.config.log_mode"}}
-
{{.Name}} ({{.Provider}})
-
{{$.locale.Tr "admin.config.log_config"}}
- 
{{JsonUtils.PrettyIndent .Config}}
- {{end}} - {{else}} -
{{$.locale.Tr "admin.config.routes_to_default_logger"}}
- {{end}} + + {{if .Loggers.access.IsEnabled}}
{{$.locale.Tr "admin.config.access_log_template"}}
{{$.AccessLogTemplate}}
- {{else}} -
{{$.locale.Tr "admin.config.disabled_logger"}}
{{end}} -
-
{{$.locale.Tr "admin.config.xorm_log_mode"}}
- {{if .EnableXORMLog}} - {{if .Loggers.xorm.SubLogDescriptions}} -
{{$.locale.Tr "admin.config.own_named_logger"}}
- {{range .Loggers.xorm.SubLogDescriptions}} -
{{$.locale.Tr "admin.config.log_mode"}}
-
{{.Name}} ({{.Provider}})
-
{{$.locale.Tr "admin.config.log_config"}}
- 
{{JsonUtils.PrettyIndent .Config}}
- {{end}} + + {{range $loggerName, $loggerDetail := .Loggers}} +
{{$.locale.Tr "admin.config.logger_name_fmt" $loggerName}}
+ {{if $loggerDetail.IsEnabled}} + 
{{$loggerDetail.EventWriters | JsonUtils.EncodeToString | JsonUtils.PrettyIndent}}
{{else}} -
{{$.locale.Tr "admin.config.routes_to_default_logger"}}
+
{{$.locale.Tr "admin.config.disabled_logger"}}
{{end}} -
{{$.locale.Tr "admin.config.xorm_log_sql"}}
-
{{if $.LogSQL}}{{svg "octicon-check"}}{{else}}{{svg "octicon-x"}}{{end}}
- {{else}} -
{{$.locale.Tr "admin.config.disabled_logger"}}
{{end}}
diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index e75b3db38ffb..fbcb0e7da56a 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -31,7 +31,7 @@ import ( var c *web.Route func TestMain(m *testing.M) { - defer log.Close() + defer log.GetManager().Close() managerCtx, cancel := context.WithCancel(context.Background()) graceful.InitManager(managerCtx) diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 01f26d567fd3..27918a9cccac 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -80,7 +80,7 @@ func NewNilResponseHashSumRecorder() *NilResponseHashSumRecorder { } func TestMain(m *testing.M) { - defer log.Close() + defer log.GetManager().Close() managerCtx, cancel := context.WithCancel(context.Background()) graceful.InitManager(managerCtx) diff --git a/tests/integration/migration-test/migration_test.go b/tests/integration/migration-test/migration_test.go index a68d458a0b1e..22c2326288ae 100644 --- a/tests/integration/migration-test/migration_test.go +++ b/tests/integration/migration-test/migration_test.go @@ -24,7 +24,9 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/testlogger" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/tests" @@ -35,6 +37,8 @@ import ( var currentEngine *xorm.Engine func initMigrationTest(t *testing.T) func() { + log.RegisterEventWriter("test", testlogger.NewTestLoggerWriter) + deferFn := tests.PrintCurrentTest(t, 2) giteaRoot := base.SetupGiteaRoot() if giteaRoot == "" { @@ -84,7 +88,7 @@ func initMigrationTest(t *testing.T) func() { assert.NoError(t, git.InitFull(context.Background())) setting.LoadDBSetting() - setting.InitLogs(true) + setting.InitLoggersForTest() return deferFn } @@ -292,7 +296,7 @@ func doMigrationTest(t *testing.T, version string) { return } - setting.InitSQLLog(false) + setting.InitSQLLoggersForCli() err := db.InitEngineWithMigration(context.Background(), wrappedMigrate) assert.NoError(t, err) diff --git a/tests/test_utils.go b/tests/test_utils.go index de022fde0a1f..6f0fb25560d9 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -36,6 +36,8 @@ func exitf(format string, args ...interface{}) { } func InitTest(requireGitea bool) { + log.RegisterEventWriter("test", testlogger.NewTestLoggerWriter) + giteaRoot := base.SetupGiteaRoot() if giteaRoot == "" { exitf("Environment variable $GITEA_ROOT not set") @@ -246,7 +248,3 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() { func Printf(format string, args ...interface{}) { testlogger.Printf(format, args...) } - -func init() { - log.Register("test", testlogger.NewTestLogger) -} From 12417a8b05b6b16199cf1ae448123732ac8623d5 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 16 May 2023 10:47:18 +0800 Subject: [PATCH 02/24] fix merge --- modules/graceful/manager_unix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/graceful/manager_unix.go b/modules/graceful/manager_unix.go index a8f89a63432f..d89f6fc725bc 100644 --- a/modules/graceful/manager_unix.go +++ b/modules/graceful/manager_unix.go @@ -186,7 +186,7 @@ func (g *Manager) handleSignals(ctx context.Context) { case syscall.SIGUSR1: log.Warn("PID %d. Received SIGUSR1. Releasing and reopening logs", pid) g.notify(statusMsg("Releasing and reopening logs")) - if err := log.ReleaseReopen(); err != nil { + if err := releasereopen.GetManager().ReleaseReopen(); err != nil { log.Error("Error whilst releasing and reopening logs: %v", err) } case syscall.SIGUSR2: From a1c229dea1545bcaf3fcfddd9c4b3a346c160dea Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 16 May 2023 10:48:57 +0800 Subject: [PATCH 03/24] fine tune --- cmd/cmd.go | 10 +- cmd/doctor.go | 8 +- modules/log/event_format.go | 151 ++++++++---------- modules/log/event_format_test.go | 57 +++++++ modules/log/event_writer.go | 14 +- modules/log/event_writer_base.go | 46 ++++-- modules/log/event_writer_conn.go | 4 +- modules/log/event_writer_conn_test.go | 74 +++++++++ modules/log/event_writer_console.go | 4 +- modules/log/event_writer_file.go | 4 +- modules/log/flags.go | 115 +++++++++---- modules/log/flags_test.go | 27 ++++ modules/log/init.go | 6 +- modules/log/level.go | 13 +- modules/log/{package.go => logger.go} | 28 +++- modules/log/logger_global.go | 3 +- modules/log/logger_impl.go | 46 ++++-- modules/log/logger_test.go | 109 +++++++++++++ modules/log/logger_types.go | 75 --------- modules/log/manager.go | 2 +- modules/log/misc.go | 48 +++++- modules/setting/log.go | 35 ++-- modules/test/logchecker.go | 8 +- modules/testlogger/testlogger.go | 4 +- routers/private/manager.go | 8 +- services/migrations/gitbucket.go | 2 +- .../migration-test/migration_test.go | 2 +- 27 files changed, 610 insertions(+), 293 deletions(-) create mode 100644 modules/log/event_format_test.go create mode 100644 modules/log/event_writer_conn_test.go create mode 100644 modules/log/flags_test.go rename modules/log/{package.go => logger.go} (65%) create mode 100644 modules/log/logger_test.go delete mode 100644 modules/log/logger_types.go diff --git a/cmd/cmd.go b/cmd/cmd.go index afe403d502ff..b148007fbe3b 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -60,7 +60,7 @@ func confirm() (bool, error) { func initDB(ctx context.Context) error { setting.Init(&setting.Options{}) setting.LoadDBSetting() - setting.InitSQLLoggersForCli() + setting.InitSQLLoggersForCli(log.INFO) if setting.Database.Type == "" { log.Fatal(`Database settings are missing from the configuration file: %q. @@ -101,16 +101,10 @@ func setupConsoleLogger(level log.Level, colorize bool, out io.Writer) { } writeMode := log.WriterMode{ - WriterType: "console", Level: level, Colorize: colorize, WriterOption: log.WriterConsoleOption{Stderr: out == os.Stderr}, } - writer, err := log.NewEventWriter("console", writeMode) - if err != nil { - log.FallbackErrorf("unable to create console log writer: %v", err) - return - } - + writer := log.NewEventWriterConsole("console-default", writeMode) log.GetManager().GetLogger(log.DEFAULT).RemoveAllWriters().AddWriters(writer) } diff --git a/cmd/doctor.go b/cmd/doctor.go index 1bf40bd4c8b7..b596e9ac0cb7 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -95,9 +95,9 @@ func runRecreateTable(ctx *cli.Context) error { setting.LoadDBSetting() if debug { - setting.InitSQLLoggersForCliDebug() + setting.InitSQLLoggersForCli(log.DEBUG) } else { - setting.InitSQLLoggersForCli() + setting.InitSQLLoggersForCli(log.INFO) } setting.Database.LogSQL = debug @@ -145,8 +145,8 @@ func setupDoctorDefaultLogger(ctx *cli.Context, colorize bool) { setupConsoleLogger(log.TRACE, colorize, os.Stdout) } else { logFile, _ = filepath.Abs(logFile) - writeMode := log.WriterMode{WriterType: "file", Level: log.TRACE, WriterOption: log.WriterFileOption{FileName: logFile}} - writer, err := log.NewEventWriter("console-to-file", writeMode) + writeMode := log.WriterMode{Level: log.TRACE, WriterOption: log.WriterFileOption{FileName: logFile}} + writer, err := log.NewEventWriter("console-to-file", "file", writeMode) if err != nil { log.FallbackErrorf("unable to create file log writer: %v", err) return diff --git a/modules/log/event_format.go b/modules/log/event_format.go index 208e8ab3971f..b90e90e0f949 100644 --- a/modules/log/event_format.go +++ b/modules/log/event_format.go @@ -20,76 +20,23 @@ type Event struct { Level Level - Msg string - MsgFormat string - MsgFrozenArgs []any // it may contains *ColorValue + MsgSimpleText string - Stacktrace string -} - -type EventFormatter func(mode *WriterMode, event *Event, reuse []byte) []byte - -type frozenMsgArg struct { - m *frozenMsgFormatter - v any - s string - - processed bool -} - -func (a *frozenMsgArg) Format(s fmt.State, c rune) { - a.s = fmt.Sprintf(fmt.FormatString(s, c), a.v) - _, _ = s.Write([]byte(a.s)) - a.processed = true -} + msgFormat string + msgArgs []any -type frozenMsgFormatter struct { - format string - args []any + Stacktrace string } -func (m *frozenMsgFormatter) addArgs(args ...any) { - for _, v := range args { - switch v := v.(type) { - case fmt.Stringer, fmt.GoStringer, LogStringer: - m.args = append(m.args, &frozenMsgArg{m: m, v: v}) - default: - m.args = append(m.args, v) - } - } +type EventFormatted struct { + Origin *Event + Msg any } -func (m *frozenMsgFormatter) doFormat() string { - res := fmt.Sprintf(m.format, m.args...) - for i := range m.args { - if arg, ok := m.args[i].(*frozenMsgArg); ok { - if arg.processed { - m.args[i] = arg.s - } else { - switch v := arg.v.(type) { - case LogStringer: - m.args[i] = v.LogString() - case fmt.GoStringer: // GoString() is for "%#v" only, but it's also fine to freeze the argument by it - m.args[i] = v.GoString() - case fmt.Stringer: - m.args[i] = v.String() - default: - m.args[i] = v - } - } - } - } - return res -} - -func frozenMsgFormat(format string, args ...any) (msg string, frozenArgs []any) { - m := frozenMsgFormatter{format: format} - m.addArgs(args...) - msg = m.doFormat() - return msg, m.args -} +type EventFormatter func(mode *WriterMode, event *Event, msgFormat string, msgArgs ...any) []byte // Copy of cheap integer to fixed-width decimal to ascii from logger. +// TODO: legacy bugs: doesn't support negative number, overflow if wid it too large. func itoa(buf []byte, i, wid int) []byte { var s [20]byte bp := len(s) - 1 @@ -105,18 +52,41 @@ func itoa(buf []byte, i, wid int) []byte { return append(buf, s[bp:]...) } +func colorSprintf(colorize bool, format string, args ...any) string { + hasColorValue := false + for _, v := range args { + if _, hasColorValue = v.(*ColoredValue); hasColorValue { + break + } + } + if colorize || !hasColorValue { + return fmt.Sprintf(format, args...) + } + + noColors := make([]any, len(args)) + copy(noColors, args) + for i, v := range args { + if cv, ok := v.(*ColoredValue); ok { + noColors[i] = cv.v + } + } + return fmt.Sprintf(format, noColors...) +} + // EventFormatTextMessage makes the log message for a writer with its mode. This function is a copy of the original package -func EventFormatTextMessage(mode *WriterMode, event *Event, buf []byte) []byte { +func EventFormatTextMessage(mode *WriterMode, event *Event, msgFormat string, msgArgs ...any) []byte { + buf := make([]byte, 0, 1024) buf = append(buf, mode.Prefix...) t := event.Time - if mode.Flags&(Ldate|Ltime|Lmicroseconds) != 0 { + flags := mode.Flags.Bits() + if flags&(Ldate|Ltime|Lmicroseconds) != 0 { if mode.Colorize { buf = append(buf, fgCyanBytes...) } - if mode.Flags&LUTC != 0 { + if flags&LUTC != 0 { t = t.UTC() } - if mode.Flags&Ldate != 0 { + if flags&Ldate != 0 { year, month, day := t.Date() buf = itoa(buf, year, 4) buf = append(buf, '/') @@ -125,14 +95,14 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, buf []byte) []byte { buf = itoa(buf, day, 2) buf = append(buf, ' ') } - if mode.Flags&(Ltime|Lmicroseconds) != 0 { + if flags&(Ltime|Lmicroseconds) != 0 { hour, min, sec := t.Clock() buf = itoa(buf, hour, 2) buf = append(buf, ':') buf = itoa(buf, min, 2) buf = append(buf, ':') buf = itoa(buf, sec, 2) - if mode.Flags&Lmicroseconds != 0 { + if flags&Lmicroseconds != 0 { buf = append(buf, '.') buf = itoa(buf, t.Nanosecond()/1e3, 6) } @@ -143,17 +113,17 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, buf []byte) []byte { } } - if mode.Flags&(Lshortfile|Llongfile) != 0 { + if flags&(Lshortfile|Llongfile) != 0 { if mode.Colorize { buf = append(buf, fgGreenBytes...) } file := event.Filename - if mode.Flags&Lmedfile == Lmedfile { + if flags&Lmedfile == Lmedfile { startIndex := len(file) - 20 if startIndex > 0 { file = "..." + file[startIndex:] } - } else if mode.Flags&Lshortfile != 0 { + } else if flags&Lshortfile != 0 { startIndex := strings.LastIndexByte(file, '/') if startIndex > 0 && startIndex < len(file) { file = file[startIndex+1:] @@ -162,7 +132,7 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, buf []byte) []byte { buf = append(buf, file...) buf = append(buf, ':') buf = itoa(buf, event.Line, -1) - if mode.Flags&(Lfuncname|Lshortfuncname) != 0 { + if flags&(Lfuncname|Lshortfuncname) != 0 { buf = append(buf, ':') } else { if mode.Colorize { @@ -171,12 +141,12 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, buf []byte) []byte { buf = append(buf, ' ') } } - if mode.Flags&(Lfuncname|Lshortfuncname) != 0 { + if flags&(Lfuncname|Lshortfuncname) != 0 { if mode.Colorize { buf = append(buf, fgGreenBytes...) } funcname := event.Caller - if mode.Flags&Lshortfuncname != 0 { + if flags&Lshortfuncname != 0 { lastIndex := strings.LastIndexByte(funcname, '.') if lastIndex > 0 && len(funcname) > lastIndex+1 { funcname = funcname[lastIndex+1:] @@ -189,13 +159,13 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, buf []byte) []byte { buf = append(buf, ' ') } - if mode.Flags&(Llevel|Llevelinitial) != 0 { + if flags&(Llevel|Llevelinitial) != 0 { level := strings.ToUpper(event.Level.String()) if mode.Colorize { buf = append(buf, ColorBytes(levelToColor[event.Level]...)...) } buf = append(buf, '[') - if mode.Flags&Llevelinitial != 0 { + if flags&Llevelinitial != 0 { buf = append(buf, level[0]) } else { buf = append(buf, level...) @@ -207,23 +177,34 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, buf []byte) []byte { buf = append(buf, ' ') } - msg := []byte(event.Msg) - if mode.Colorize { + var msg []byte + + // if the log needs colorizing, do it + if mode.Colorize && len(msgArgs) > 0 { hasColorValue := false - for _, v := range event.MsgFrozenArgs { + for _, v := range msgArgs { if _, hasColorValue = v.(*ColoredValue); hasColorValue { break } } if hasColorValue { - msg = []byte(fmt.Sprintf(event.MsgFormat, event.MsgFrozenArgs...)) + msg = []byte(fmt.Sprintf(msgFormat, msgArgs...)) } } + // try to re-use the pre-formatted simple text message + if len(msg) == 0 { + msg = []byte(event.MsgSimpleText) + } + // if still no message, do the normal Sprintf for the message + if len(msg) == 0 { + msg = []byte(colorSprintf(mode.Colorize, msgFormat, msgArgs...)) + } + // remove at most one trailing new line if len(msg) > 0 && msg[len(msg)-1] == '\n' { msg = msg[:len(msg)-1] } - if mode.Flags&Lgopid == Lgopid { + if flags&Lgopid == Lgopid { if event.GoroutinePid != "" { buf = append(buf, '[') if mode.Colorize { @@ -240,11 +221,9 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, buf []byte) []byte { if event.Stacktrace != "" && mode.StacktraceLevel <= event.Level { lines := bytes.Split([]byte(event.Stacktrace), []byte("\n")) - if len(lines) > 1 { - for _, line := range lines { - buf = append(buf, "\n\t"...) - buf = append(buf, line...) - } + for _, line := range lines { + buf = append(buf, "\n\t"...) + buf = append(buf, line...) } buf = append(buf, '\n') } diff --git a/modules/log/event_format_test.go b/modules/log/event_format_test.go new file mode 100644 index 000000000000..7c299a607d86 --- /dev/null +++ b/modules/log/event_format_test.go @@ -0,0 +1,57 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestItoa(t *testing.T) { + b := itoa(nil, 0, 0) + assert.Equal(t, "0", string(b)) + + b = itoa(nil, 0, 1) + assert.Equal(t, "0", string(b)) + + b = itoa(nil, 0, 2) + assert.Equal(t, "00", string(b)) +} + +func TestEventFormatTextMessage(t *testing.T) { + res := EventFormatTextMessage(&WriterMode{Prefix: "[PREFIX] ", Colorize: false, Flags: Flags{defined: true, flags: 0xffffffff}}, + &Event{ + Time: time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC), + Caller: "caller", + Filename: "filename", + Line: 123, + GoroutinePid: "pid", + Level: ERROR, + Stacktrace: "stacktrace", + }, + "msg format: %v %v", "arg0", NewColoredValue("arg1", FgBlue), + ) + + assert.Equal(t, `[PREFIX] 2020/01/02 03:04:05.000000 filename:123:caller [E] [pid] msg format: arg0 arg1 + stacktrace + +`, string(res)) + + res = EventFormatTextMessage(&WriterMode{Prefix: "[PREFIX] ", Colorize: true, Flags: Flags{defined: true, flags: 0xffffffff}}, + &Event{ + Time: time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC), + Caller: "caller", + Filename: "filename", + Line: 123, + GoroutinePid: "pid", + Level: ERROR, + Stacktrace: "stacktrace", + }, + "msg format: %v %v", "arg0", NewColoredValue("arg1", FgBlue), + ) + + assert.Equal(t, "[PREFIX] \x1b[36m2020/01/02 03:04:05.000000 \x1b[0m\x1b[32mfilename:123:\x1b[32mcaller\x1b[0m \x1b[1;31m[E]\x1b[0m [\x1b[93mpid\x1b[0m] msg format: arg0 \x1b[34marg1\x1b[0m\n\tstacktrace\n\n", string(res)) +} diff --git a/modules/log/event_writer.go b/modules/log/event_writer.go index 9568df1ca328..3ca729ff7d95 100644 --- a/modules/log/event_writer.go +++ b/modules/log/event_writer.go @@ -25,16 +25,12 @@ func HasEventWriter(writerType string) bool { } type WriterMode struct { - WriterType string - // ModeName string - BufferLen int - Level Level - + Level Level Prefix string Colorize bool - Flags int + Flags Flags Expression string @@ -43,9 +39,9 @@ type WriterMode struct { WriterOption any } -func NewEventWriter(name string, mode WriterMode) (EventWriter, error) { - if p, ok := eventWriterProviders[mode.WriterType]; ok { +func NewEventWriter(name, writerType string, mode WriterMode) (EventWriter, error) { + if p, ok := eventWriterProviders[writerType]; ok { return p(name, mode), nil } - return nil, fmt.Errorf("unknown event writer type %q for writer %q", mode.WriterType, name) + return nil, fmt.Errorf("unknown event writer type %q for writer %q", writerType, name) } diff --git a/modules/log/event_writer_base.go b/modules/log/event_writer_base.go index 770b4a1e00d5..3ce946371754 100644 --- a/modules/log/event_writer_base.go +++ b/modules/log/event_writer_base.go @@ -23,11 +23,13 @@ type EventWriterBase interface { type EventWriterBaseImpl struct { LoggerImpl *LoggerImpl + writerType string + Name string Mode *WriterMode - Queue chan *Event + Queue chan *EventFormatted - Formatter EventFormatter // format the Event to a message and write it to output + FormatMessage EventFormatter // format the Event to a message and write it to output OutputWriteCloser io.WriteCloser // it will be closed when the event writer is stopped stopped chan struct{} @@ -40,7 +42,7 @@ func (b *EventWriterBaseImpl) Base() *EventWriterBaseImpl { } func (b *EventWriterBaseImpl) GetWriterType() string { - return b.Mode.WriterType + return b.writerType } func (b *EventWriterBaseImpl) GetWriterName() string { @@ -55,14 +57,13 @@ func (b *EventWriterBaseImpl) Run(ctx context.Context) { defer b.OutputWriteCloser.Close() var exprRegexp *regexp.Regexp - var err error if b.Mode.Expression != "" { + var err error if exprRegexp, err = regexp.Compile(b.Mode.Expression); err != nil { FallbackErrorf("unable to compile expression %q for writer %q: %v", b.Mode.Expression, b.Name, err) } } - var buf []byte for { pause := b.LoggerImpl.GetPauseChan() if pause != nil { @@ -81,26 +82,32 @@ func (b *EventWriterBaseImpl) Run(ctx context.Context) { } if exprRegexp != nil { - matched := exprRegexp.Match([]byte(fmt.Sprintf("%s:%d:%s", event.Filename, event.Line, event.Caller))) || - exprRegexp.Match([]byte(event.Msg)) + matched := exprRegexp.Match([]byte(fmt.Sprintf("%s:%d:%s", event.Origin.Filename, event.Origin.Line, event.Origin.Caller))) || + exprRegexp.Match([]byte(event.Origin.MsgSimpleText)) if !matched { continue } } - buf = EventFormatTextMessage(b.Mode, event, buf[:0]) - _, err := b.OutputWriteCloser.Write(buf) - if err != nil { - FallbackErrorf("unable to write log message of %q (%v): %s", b.Name, err, string(buf)) + var err error + switch msg := event.Msg.(type) { + case string: + _, err = b.OutputWriteCloser.Write([]byte(msg)) + case []byte: + _, err = b.OutputWriteCloser.Write(msg) + case io.WriterTo: + _, err = msg.WriteTo(b.OutputWriteCloser) + default: + _, err = b.OutputWriteCloser.Write([]byte(fmt.Sprint(msg))) } - if len(buf) > 2048 { - buf = nil // do not waste too much memory + if err != nil { + FallbackErrorf("unable to write log message of %q (%v): %v", b.Name, err, event.Msg) } } } } -func NewEventWriterBase(name string, mode WriterMode) *EventWriterBaseImpl { +func NewEventWriterBase(name, writerType string, mode WriterMode) *EventWriterBaseImpl { if mode.BufferLen == 0 { mode.BufferLen = 1000 } @@ -111,9 +118,14 @@ func NewEventWriterBase(name string, mode WriterMode) *EventWriterBaseImpl { mode.StacktraceLevel = NONE } b := &EventWriterBaseImpl{ - Name: name, - Mode: &mode, - Queue: make(chan *Event, mode.BufferLen), + writerType: writerType, + + Name: name, + Mode: &mode, + Queue: make(chan *EventFormatted, mode.BufferLen), + + FormatMessage: EventFormatTextMessage, + stopped: make(chan struct{}), } return b diff --git a/modules/log/event_writer_conn.go b/modules/log/event_writer_conn.go index 68fdf2451b42..12c81f46a8cb 100644 --- a/modules/log/event_writer_conn.go +++ b/modules/log/event_writer_conn.go @@ -23,7 +23,7 @@ type eventWriterConn struct { var _ EventWriter = (*eventWriterConn)(nil) func NewEventWriterConn(name string, mode WriterMode) EventWriter { - w := &eventWriterConn{EventWriterBaseImpl: NewEventWriterBase(name, mode)} + w := &eventWriterConn{EventWriterBaseImpl: NewEventWriterBase(name, "conn", mode)} opt := mode.WriterOption.(WriterConnOption) w.connWriter = connWriter{ ReconnectOnMsg: opt.ReconnectOnMsg, @@ -31,7 +31,7 @@ func NewEventWriterConn(name string, mode WriterMode) EventWriter { Net: opt.Protocol, Addr: opt.Addr, } - w.Formatter = EventFormatTextMessage + w.FormatMessage = EventFormatTextMessage w.OutputWriteCloser = &w.connWriter return w } diff --git a/modules/log/event_writer_conn_test.go b/modules/log/event_writer_conn_test.go new file mode 100644 index 000000000000..30280cb1c55d --- /dev/null +++ b/modules/log/event_writer_conn_test.go @@ -0,0 +1,74 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "fmt" + "io" + "net" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func listenReadAndClose(t *testing.T, l net.Listener, expected string) { + conn, err := l.Accept() + assert.NoError(t, err) + defer conn.Close() + written, err := io.ReadAll(conn) + + assert.NoError(t, err) + assert.Equal(t, expected, string(written)) +} + +func TestConnLogger(t *testing.T) { + protocol := "tcp" + address := ":3099" + + l, err := net.Listen(protocol, address) + if err != nil { + t.Fatal(err) + } + defer l.Close() + + prefix := "TestPrefix " + level := INFO + flags := LstdFlags | LUTC | Lfuncname + + logger := NewLoggerWithWriters(NewEventWriterConn("test-conn", WriterMode{ + Level: level, + Prefix: prefix, + Flags: FlagsFromBits(flags), + WriterOption: WriterConnOption{Addr: address, Protocol: protocol, Reconnect: true, ReconnectOnMsg: true}, + })) + + location, _ := time.LoadLocation("EST") + + date := time.Date(2019, time.January, 13, 22, 3, 30, 15, location) + + dateString := date.UTC().Format("2006/01/02 15:04:05") + + event := Event{ + Level: INFO, + MsgSimpleText: "TEST MSG", + Caller: "CALLER", + Filename: "FULL/FILENAME", + Line: 1, + Time: date, + } + expected := fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.Filename, event.Line, event.Caller, strings.ToUpper(event.Level.String())[0], event.MsgSimpleText) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + listenReadAndClose(t, l, expected) + }() + logger.SendLogEvent(&event) + wg.Wait() + + logger.Close() +} diff --git a/modules/log/event_writer_console.go b/modules/log/event_writer_console.go index 5b0445053e07..e54b8020e36b 100644 --- a/modules/log/event_writer_console.go +++ b/modules/log/event_writer_console.go @@ -25,9 +25,9 @@ type nopCloser struct { func (nopCloser) Close() error { return nil } func NewEventWriterConsole(name string, mode WriterMode) EventWriter { - w := &eventWriterConsole{EventWriterBaseImpl: NewEventWriterBase(name, mode)} + w := &eventWriterConsole{EventWriterBaseImpl: NewEventWriterBase(name, "console", mode)} opt := mode.WriterOption.(WriterConsoleOption) - w.Formatter = EventFormatTextMessage + w.FormatMessage = EventFormatTextMessage if opt.Stderr { w.OutputWriteCloser = nopCloser{os.Stderr} } else { diff --git a/modules/log/event_writer_file.go b/modules/log/event_writer_file.go index 779d6f714dae..71f9effb556e 100644 --- a/modules/log/event_writer_file.go +++ b/modules/log/event_writer_file.go @@ -25,7 +25,7 @@ type eventWriterFile struct { var _ EventWriter = (*eventWriterFile)(nil) func NewEventWriterFile(name string, mode WriterMode) EventWriter { - w := &eventWriterFile{EventWriterBaseImpl: NewEventWriterBase(name, mode)} + w := &eventWriterFile{EventWriterBaseImpl: NewEventWriterBase(name, "file", mode)} opt := mode.WriterOption.(WriterFileOption) var err error w.fileWriter, err = rotatingfilewriter.Open(opt.FileName, &rotatingfilewriter.Options{ @@ -39,7 +39,7 @@ func NewEventWriterFile(name string, mode WriterMode) EventWriter { if err != nil { FallbackErrorf("unable to open log file %q: %v", opt.FileName, err) } - w.Formatter = EventFormatTextMessage + w.FormatMessage = EventFormatTextMessage w.OutputWriteCloser = w.fileWriter return w } diff --git a/modules/log/flags.go b/modules/log/flags.go index 0a1304bfce88..edb96a486089 100644 --- a/modules/log/flags.go +++ b/modules/log/flags.go @@ -3,7 +3,12 @@ package log -import "strings" +import ( + "sort" + "strings" + + "code.gitea.io/gitea/modules/json" +) // These flags define which text to prefix to each log entry generated // by the Logger. Bits are or'ed together to control what's printed. @@ -15,25 +20,30 @@ import "strings" // The standard is: // 2009/01/23 01:23:23 ...a/logger/c/d.go:23:runtime.Caller() [I]: message const ( - Ldate = 1 << iota // the date in the local time zone: 2009/01/23 - Ltime // the time in the local time zone: 01:23:23 - Lmicroseconds // microsecond resolution: 01:23:23.123123. assumes Ltime. - Llongfile // full file name and line number: /a/logger/c/d.go:23 - Lshortfile // final file name element and line number: d.go:23. overrides Llongfile - Lfuncname // function name of the caller: runtime.Caller() - Lshortfuncname // last part of the function name - LUTC // if Ldate or Ltime is set, use UTC rather than the local time zone - Llevelinitial // Initial character of the provided level in brackets eg. [I] for info - Llevel // Provided level in brackets [INFO] + Ldate uint32 = 1 << iota // the date in the local time zone: 2009/01/23 + Ltime // the time in the local time zone: 01:23:23 + Lmicroseconds // microsecond resolution: 01:23:23.123123. assumes Ltime. + Llongfile // full file name and line number: /a/logger/c/d.go:23 + Lshortfile // final file name element and line number: d.go:23. overrides Llongfile + Lfuncname // function name of the caller: runtime.Caller() + Lshortfuncname // last part of the function name + LUTC // if Ldate or Ltime is set, use UTC rather than the local time zone + Llevelinitial // Initial character of the provided level in brackets eg. [I] for info + Llevel // Provided level in brackets [INFO] Lgopid - Lmedfile = Lshortfile | Llongfile // last 20 characters of the filename - + Lmedfile = Lshortfile | Llongfile // last 20 characters of the filename LstdFlags = Ldate | Ltime | Lmedfile | Lshortfuncname | Llevelinitial // default ) -var flagFromString = map[string]int{ - "none": 0, +const Ldefault = LstdFlags + +type Flags struct { + defined bool + flags uint32 +} + +var flagFromString = map[string]uint32{ "date": Ldate, "time": Ltime, "microseconds": Lmicroseconds, @@ -44,22 +54,73 @@ var flagFromString = map[string]int{ "utc": LUTC, "levelinitial": Llevelinitial, "level": Llevel, - "medfile": Lmedfile, - "stdflags": LstdFlags, "gopid": Lgopid, + + "medfile": Lmedfile, + "stdflags": LstdFlags, } -// FlagsFromString takes a comma separated list of flags and returns the flags for this string -func FlagsFromString(from string) int { - flags := 0 - for _, flag := range strings.Split(strings.ToLower(from), ",") { - f, ok := flagFromString[strings.TrimSpace(flag)] - if ok { - flags |= f +var flagToString = map[uint32]string{ + Ldate: "date", + Ltime: "time", + Lmicroseconds: "microseconds", + Llongfile: "longfile", + Lshortfile: "shortfile", + Lfuncname: "funcname", + Lshortfuncname: "shortfuncname", + LUTC: "utc", + Llevelinitial: "levelinitial", + Llevel: "level", + Lgopid: "gopid", +} + +func (f Flags) Bits() uint32 { + if !f.defined { + return Ldefault + } + return f.flags +} + +func (f Flags) String() string { + flags := f.Bits() + var flagNames []string + for flag, name := range flagToString { + if flags&flag != 0 { + flagNames = append(flagNames, name) } } - if flags == 0 { - return -1 + if len(flagNames) == 0 { + return "none" } - return flags + sort.Strings(flagNames) + return strings.Join(flagNames, ",") +} + +func (f *Flags) UnmarshalJSON(bytes []byte) error { + var s string + if err := json.Unmarshal(bytes, &s); err != nil { + return err + } + *f = FlagsFromString(s) + return nil +} + +func (f Flags) MarshalJSON() ([]byte, error) { + return []byte(`"` + f.String() + `"`), nil +} + +func FlagsFromString(from string, def ...uint32) Flags { + from = strings.TrimSpace(from) + if from == "" && len(def) > 0 { + return Flags{defined: true, flags: def[0]} + } + flags := uint32(0) + for _, flag := range strings.Split(strings.ToLower(from), ",") { + flags |= flagFromString[strings.TrimSpace(flag)] + } + return Flags{defined: true, flags: flags} +} + +func FlagsFromBits(flags uint32) Flags { + return Flags{defined: true, flags: flags} } diff --git a/modules/log/flags_test.go b/modules/log/flags_test.go new file mode 100644 index 000000000000..f17ce9a302c0 --- /dev/null +++ b/modules/log/flags_test.go @@ -0,0 +1,27 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "testing" + + "code.gitea.io/gitea/modules/json" + + "github.com/stretchr/testify/assert" +) + +func TestFlags(t *testing.T) { + assert.EqualValues(t, Ldefault, Flags{}.Bits()) + assert.EqualValues(t, 0, FlagsFromString("").Bits()) + assert.EqualValues(t, Lgopid, FlagsFromString("", Lgopid).Bits()) + assert.EqualValues(t, 0, FlagsFromString("none", Lgopid).Bits()) + assert.EqualValues(t, Ldate|Ltime, FlagsFromString("date,time", Lgopid).Bits()) + + bs, err := json.Marshal(FlagsFromString("utc,level")) + assert.NoError(t, err) + assert.EqualValues(t, `"level,utc"`, string(bs)) + var flags Flags + assert.NoError(t, json.Unmarshal(bs, &flags)) + assert.EqualValues(t, LUTC|Llevel, flags.Bits()) +} diff --git a/modules/log/init.go b/modules/log/init.go index 91ee81447618..3508039060b5 100644 --- a/modules/log/init.go +++ b/modules/log/init.go @@ -10,12 +10,12 @@ import ( "code.gitea.io/gitea/modules/process" ) -var prefix string +var projectPackagePrefix string func init() { _, filename, _, _ := runtime.Caller(0) - prefix = strings.TrimSuffix(filename, "modules/log/init.go") - if prefix == filename { + projectPackagePrefix = strings.TrimSuffix(filename, "modules/log/init.go") + if projectPackagePrefix == filename { // in case the source code file is moved, we can not trim the suffix, the code above should also be updated. panic("unable to detect correct package prefix, please update file: " + filename) } diff --git a/modules/log/level.go b/modules/log/level.go index 0a870e1249dc..01fa3f5e46b0 100644 --- a/modules/log/level.go +++ b/modules/log/level.go @@ -34,6 +34,7 @@ var toString = map[Level]string{ INFO: "info", WARN: "warn", ERROR: "error", + FATAL: "fatal", NONE: "none", } @@ -41,11 +42,13 @@ var toString = map[Level]string{ var toLevel = map[string]Level{ "undefined": UNDEFINED, - "trace": TRACE, - "debug": DEBUG, - "info": INFO, - "warn": WARN, - "error": ERROR, + "trace": TRACE, + "debug": DEBUG, + "info": INFO, + "warn": WARN, + "warning": WARN, + "error": ERROR, + "fatal": FATAL, "none": NONE, } diff --git a/modules/log/package.go b/modules/log/logger.go similarity index 65% rename from modules/log/package.go rename to modules/log/logger.go index 7d5c3d289db1..a833b6ef0fa7 100644 --- a/modules/log/package.go +++ b/modules/log/logger.go @@ -18,7 +18,33 @@ // Call graph: // -> log.Info() // -> LoggerImpl.Log() -// -> prepare log event, freeze all Stringer arguments (because the Event might be used in another goroutine) // -> LoggerImpl.SendLogEvent, then the event goes into writer's goroutines // -> EventWriter.Run() handles the events package log + +// BaseLogger provides the basic logging functions +type BaseLogger interface { + Log(skip int, level Level, format string, v ...any) + GetLevel() Level +} + +// LevelLogger provides level-related logging functions +type LevelLogger interface { + LevelEnabled(level Level) bool + + Trace(format string, v ...any) + Debug(format string, v ...any) + Info(format string, v ...any) + Warn(format string, v ...any) + Error(format string, v ...any) + Critical(format string, v ...any) +} + +type Logger interface { + BaseLogger + LevelLogger +} + +type LogStringer interface { //nolint:revive + LogString() string +} diff --git a/modules/log/logger_global.go b/modules/log/logger_global.go index f4fcb0f516ab..b08c4224939d 100644 --- a/modules/log/logger_global.go +++ b/modules/log/logger_global.go @@ -74,9 +74,8 @@ func IsLoggerEnabled(name string) bool { func SetConsoleLogger(loggerName, writerName string, level Level) { writer := NewEventWriterConsole(writerName, WriterMode{ - WriterType: "console", Level: level, - Flags: LstdFlags, + Flags: FlagsFromBits(LstdFlags), Colorize: CanColorStdout, WriterOption: WriterConsoleOption{}, }) diff --git a/modules/log/logger_impl.go b/modules/log/logger_impl.go index a7ab67e6da5e..34f287c556cb 100644 --- a/modules/log/logger_impl.go +++ b/modules/log/logger_impl.go @@ -41,16 +41,25 @@ func (l *LoggerImpl) SendLogEvent(event *Event) { defer l.eventWriterMu.RUnlock() if len(l.eventWriters) == 0 { - FallbackErrorf("[no logger writer]: %s", event.Msg) + FallbackErrorf("[no logger writer]: %s", event.MsgSimpleText) return } + // the writers have their own goroutines, the message arguments (with Stringer) shouldn't be used in other goroutines + // so the event message must be formatted here + msgFormat, msgArgs := event.msgFormat, event.msgArgs + event.msgFormat, event.msgArgs = "(already processed by formatters)", nil + for _, w := range l.eventWriters { if event.Level < w.GetLevel() { continue } + formatted := &EventFormatted{ + Origin: event, + Msg: w.Base().FormatMessage(w.Base().Mode, event, msgFormat, msgArgs), + } select { - case w.Base().Queue <- event: + case w.Base().Queue <- formatted: default: bs, _ := json.Marshal(event) FallbackErrorf("log writer %q queue is full, event: %v", w.GetWriterName(), string(bs)) @@ -155,6 +164,11 @@ func (l *LoggerImpl) Resume() { l.pauseMu.Unlock() } +func (l *LoggerImpl) Close() { + l.RemoveAllWriters() + l.ctxCancel() +} + func (l *LoggerImpl) GetPauseChan() chan struct{} { l.pauseMu.RLock() defer l.pauseMu.RUnlock() @@ -185,7 +199,7 @@ func (l *LoggerImpl) Log(skip int, level Level, format string, logArgs ...any) { event.Caller = fn.Name() + "()" } } - event.Filename, event.Line = strings.TrimPrefix(filename, prefix), line + event.Filename, event.Line = strings.TrimPrefix(filename, projectPackagePrefix), line if l.stacktraceLevel.Load() <= int32(level) { event.Stacktrace = Stack(skip + 1) @@ -196,26 +210,24 @@ func (l *LoggerImpl) Log(skip int, level Level, format string, logArgs ...any) { event.GoroutinePid = labels["pid"] } + // get a simple text message without color msgArgs := make([]any, len(logArgs)) copy(msgArgs, logArgs) - for i, v := range logArgs { - if cv, ok := v.(*ColoredValue); ok { - msgArgs[i] = cv.v - } - } - msg, frozenArgs := frozenMsgFormat(format, msgArgs...) - for i, v := range logArgs { + + // handle LogStringer values + for i, v := range msgArgs { if cv, ok := v.(*ColoredValue); ok { - newCV := *cv - newCV.v = frozenArgs[i] - msgArgs[i] = &newCV - } else { - msgArgs[i] = frozenArgs[i] + if s, ok := cv.v.(LogStringer); ok { + cv.v = s.LogString() + } + } else if s, ok := v.(LogStringer); ok { + msgArgs[i] = s.LogString() } } - event.Msg, event.MsgFormat, event.MsgFrozenArgs = msg, format, msgArgs - + event.MsgSimpleText = colorSprintf(false, format, msgArgs...) + event.msgFormat = format + event.msgArgs = msgArgs l.SendLogEvent(event) } diff --git a/modules/log/logger_test.go b/modules/log/logger_test.go new file mode 100644 index 000000000000..75b10b014654 --- /dev/null +++ b/modules/log/logger_test.go @@ -0,0 +1,109 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type dummyWriter struct { + *EventWriterBaseImpl + + delay time.Duration + + mu sync.Mutex + logs []string +} + +func (d *dummyWriter) Write(p []byte) (n int, err error) { + if d.delay > 0 { + time.Sleep(d.delay) + } + d.mu.Lock() + defer d.mu.Unlock() + d.logs = append(d.logs, string(p)) + return len(p), nil +} + +func (d *dummyWriter) Close() error { + return nil +} + +func (d *dummyWriter) GetLogs() []string { + d.mu.Lock() + defer d.mu.Unlock() + logs := make([]string, len(d.logs)) + copy(logs, d.logs) + return logs +} + +func newDummyWriter(name string, level Level, delay time.Duration) *dummyWriter { + w := &dummyWriter{ + EventWriterBaseImpl: NewEventWriterBase(name, "dummy", WriterMode{Level: level, Flags: FlagsFromBits(0)}), + + delay: delay, + } + w.Base().OutputWriteCloser = w + return w +} + +func TestLogger(t *testing.T) { + logger := NewLoggerWithWriters() + + dump := logger.DumpWriters() + assert.EqualValues(t, 0, len(dump)) + assert.EqualValues(t, NONE, logger.GetLevel()) + assert.False(t, logger.IsEnabled()) + + w1 := newDummyWriter("dummy-1", DEBUG, 0) + logger.AddWriters(w1) + assert.EqualValues(t, DEBUG, logger.GetLevel()) + + w2 := newDummyWriter("dummy-2", WARN, 200*time.Millisecond) + logger.AddWriters(w2) + assert.EqualValues(t, DEBUG, logger.GetLevel()) + + dump = logger.DumpWriters() + assert.EqualValues(t, 2, len(dump)) + + logger.Trace("trace-level") // this level is not logged + logger.Debug("debug-level") + logger.Error("error-level") + + // w2 is slow, so only w1 has logs + time.Sleep(100 * time.Millisecond) + assert.Equal(t, []string{"debug-level\n", "error-level\n"}, w1.GetLogs()) + assert.Equal(t, []string{}, w2.GetLogs()) + + logger.Close() + + // after Close, all logs are flushed + assert.Equal(t, []string{"debug-level\n", "error-level\n"}, w1.GetLogs()) + assert.Equal(t, []string{"error-level\n"}, w2.GetLogs()) +} + +func TestLoggerPause(t *testing.T) { + logger := NewLoggerWithWriters() + + w1 := newDummyWriter("dummy-1", DEBUG, 0) + logger.AddWriters(w1) + assert.EqualValues(t, DEBUG, logger.GetLevel()) + + logger.Pause() + + time.Sleep(100 * time.Millisecond) + logger.Info("info-level") + assert.Equal(t, []string{}, w1.GetLogs()) + + logger.Resume() + + time.Sleep(100 * time.Millisecond) + assert.Equal(t, []string{"info-level\n"}, w1.GetLogs()) + + logger.Close() +} diff --git a/modules/log/logger_types.go b/modules/log/logger_types.go deleted file mode 100644 index 600eee5dbae0..000000000000 --- a/modules/log/logger_types.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package log - -// BaseLogger provides the basic logging functions -type BaseLogger interface { - Log(skip int, level Level, format string, v ...any) - GetLevel() Level -} - -// LevelLogger provides level-related logging functions -type LevelLogger interface { - LevelEnabled(level Level) bool - - Trace(format string, v ...any) - Debug(format string, v ...any) - Info(format string, v ...any) - Warn(format string, v ...any) - Error(format string, v ...any) - Critical(format string, v ...any) -} - -type Logger interface { - BaseLogger - LevelLogger -} - -type baseToLogger struct { - base BaseLogger -} - -var _ Logger = (*baseToLogger)(nil) - -func (s *baseToLogger) Log(skip int, level Level, format string, v ...any) { - s.base.Log(skip+1, level, format, v...) -} - -func (s *baseToLogger) GetLevel() Level { - return s.base.GetLevel() -} - -func (s *baseToLogger) LevelEnabled(level Level) bool { - return s.base.GetLevel() <= level -} - -func (s *baseToLogger) Trace(format string, v ...any) { - s.base.Log(1, TRACE, format, v...) -} - -func (s *baseToLogger) Debug(format string, v ...any) { - s.base.Log(1, DEBUG, format, v...) -} - -func (s *baseToLogger) Info(format string, v ...any) { - s.base.Log(1, INFO, format, v...) -} - -func (s *baseToLogger) Warn(format string, v ...any) { - s.base.Log(1, WARN, format, v...) -} - -func (s *baseToLogger) Error(format string, v ...any) { - s.base.Log(1, ERROR, format, v...) -} - -func (s *baseToLogger) Critical(format string, v ...any) { - s.base.Log(1, CRITICAL, format, v...) -} - -// BaseLoggerToGeneralLogger wraps a BaseLogger (which only has Log() function) to a Logger (which has Info() function) -func BaseLoggerToGeneralLogger(b BaseLogger) Logger { - l := &baseToLogger{base: b} - return l -} diff --git a/modules/log/manager.go b/modules/log/manager.go index cdd5603e38a7..49c6c5972c00 100644 --- a/modules/log/manager.go +++ b/modules/log/manager.go @@ -61,7 +61,7 @@ func (m *LoggerManager) Close() { defer m.mu.Unlock() for _, logger := range m.loggers { - logger.RemoveAllWriters() + logger.Close() } } diff --git a/modules/log/misc.go b/modules/log/misc.go index 20d760dac43d..089f679cf69f 100644 --- a/modules/log/misc.go +++ b/modules/log/misc.go @@ -7,8 +7,52 @@ import ( "io" ) -type LogStringer interface { //nolint:revive - LogString() string +type baseToLogger struct { + base BaseLogger +} + +// BaseLoggerToGeneralLogger wraps a BaseLogger (which only has Log() function) to a Logger (which has Info() function) +func BaseLoggerToGeneralLogger(b BaseLogger) Logger { + l := &baseToLogger{base: b} + return l +} + +var _ Logger = (*baseToLogger)(nil) + +func (s *baseToLogger) Log(skip int, level Level, format string, v ...any) { + s.base.Log(skip+1, level, format, v...) +} + +func (s *baseToLogger) GetLevel() Level { + return s.base.GetLevel() +} + +func (s *baseToLogger) LevelEnabled(level Level) bool { + return s.base.GetLevel() <= level +} + +func (s *baseToLogger) Trace(format string, v ...any) { + s.base.Log(1, TRACE, format, v...) +} + +func (s *baseToLogger) Debug(format string, v ...any) { + s.base.Log(1, DEBUG, format, v...) +} + +func (s *baseToLogger) Info(format string, v ...any) { + s.base.Log(1, INFO, format, v...) +} + +func (s *baseToLogger) Warn(format string, v ...any) { + s.base.Log(1, WARN, format, v...) +} + +func (s *baseToLogger) Error(format string, v ...any) { + s.base.Log(1, ERROR, format, v...) +} + +func (s *baseToLogger) Critical(format string, v ...any) { + s.base.Log(1, CRITICAL, format, v...) } type PrintfLogger struct { diff --git a/modules/setting/log.go b/modules/setting/log.go index ff869c3af298..4ed2d1e68aab 100644 --- a/modules/setting/log.go +++ b/modules/setting/log.go @@ -112,13 +112,13 @@ func LogPrepareFilenameForWriter(modeName, fileName string) string { return fileName } -func loadLogModeByName(rootCfg ConfigProvider, modeName string) log.WriterMode { +func loadLogModeByName(rootCfg ConfigProvider, modeName string) (writerType string, writerMode log.WriterMode) { sec := rootCfg.Section("log." + modeName) - writerMode := log.WriterMode{} - writerMode.WriterType = KeyInSectionString(sec, "MODE") - if writerMode.WriterType == "" { - writerMode.WriterType = modeName + writerMode = log.WriterMode{} + writerType = KeyInSectionString(sec, "MODE") + if writerType == "" { + writerType = modeName } writerMode.Level = log.LevelFromString(sec.Key("LEVEL").MustString(Log.Level.String())) writerMode.StacktraceLevel = log.LevelFromString(sec.Key("STACKTRACE_LEVEL").MustString(Log.StacktraceLogLevel.String())) @@ -126,7 +126,7 @@ func loadLogModeByName(rootCfg ConfigProvider, modeName string) log.WriterMode { writerMode.Flags = log.FlagsFromString(sec.Key("FLAGS").MustString("stdflags")) writerMode.Expression = sec.Key("EXPRESSION").MustString("") - switch writerMode.WriterType { + switch writerType { case "console": useStderr := sec.Key("STDERR").MustBool(false) writerOption := log.WriterConsoleOption{Stderr: useStderr} @@ -155,12 +155,12 @@ func loadLogModeByName(rootCfg ConfigProvider, modeName string) log.WriterMode { writerOption.Addr = sec.Key("ADDR").MustString(":7020") writerMode.WriterOption = writerOption default: - if !log.HasEventWriter(writerMode.WriterType) { - panic(fmt.Sprintf("invalid log writer type (mode): %s", writerMode.WriterType)) + if !log.HasEventWriter(writerType) { + panic(fmt.Sprintf("invalid log writer type (mode): %s", writerType)) } } - return writerMode + return writerType, writerMode } var filenameSuffix = "" @@ -212,9 +212,12 @@ func initLoggerByName(rootCfg ConfigProvider, loggerName string) { if modeName == "" { continue } - opt := loadLogModeByName(rootCfg, modeName) - opt.BufferLen = Log.BufferLen - eventWriter, err := log.NewEventWriter(modeName, opt) + writerName := modeName + writerType, writerMode := loadLogModeByName(rootCfg, modeName) + if writerMode.BufferLen == 0 { + writerMode.BufferLen = Log.BufferLen + } + eventWriter, err := log.NewEventWriter(writerName, writerType, writerMode) if err != nil { log.FallbackErrorf("Failed to create event writer for logger %s: %v", loggerName, err) continue @@ -225,12 +228,8 @@ func initLoggerByName(rootCfg ConfigProvider, loggerName string) { log.GetManager().GetLogger(loggerName).RemoveAllWriters().AddWriters(eventWriters...) } -func InitSQLLoggersForCli() { - log.SetConsoleLogger("xorm", "console", log.INFO) -} - -func InitSQLLoggersForCliDebug() { - log.SetConsoleLogger("xorm", "console", log.DEBUG) +func InitSQLLoggersForCli(level log.Level) { + log.SetConsoleLogger("xorm", "console", level) } func IsAccessLogEnabled() bool { diff --git a/modules/test/logchecker.go b/modules/test/logchecker.go index b85ee3df5f54..7bf234f5604a 100644 --- a/modules/test/logchecker.go +++ b/modules/test/logchecker.go @@ -40,15 +40,15 @@ func (lc *LogChecker) Run(ctx context.Context) { } } -func (lc *LogChecker) checkLogEvent(event *log.Event) { +func (lc *LogChecker) checkLogEvent(event *log.EventFormatted) { lc.mu.Lock() defer lc.mu.Unlock() for i, msg := range lc.filterMessages { - if strings.Contains(event.Msg, msg) { + if strings.Contains(event.Origin.MsgSimpleText, msg) { lc.filtered[i] = true } } - if strings.Contains(event.Msg, lc.stopMark) { + if strings.Contains(event.Origin.MsgSimpleText, lc.stopMark) { lc.stopped = true } } @@ -61,7 +61,7 @@ func NewLogChecker(namePrefix string) (logChecker *LogChecker, cancel func()) { writerName := namePrefix + "-" + fmt.Sprint(newCheckerIndex) lc := &LogChecker{} - lc.EventWriterBaseImpl = log.NewEventWriterBase(writerName, log.WriterMode{}) + lc.EventWriterBaseImpl = log.NewEventWriterBase(writerName, "test-log-checker", log.WriterMode{}) logger.AddWriters(lc) return lc, func() { _ = logger.RemoveWriter(writerName) } } diff --git a/modules/testlogger/testlogger.go b/modules/testlogger/testlogger.go index 356eff65fd14..117c45c50cb2 100644 --- a/modules/testlogger/testlogger.go +++ b/modules/testlogger/testlogger.go @@ -173,8 +173,8 @@ type TestLogEventWriter struct { // NewTestLoggerWriter creates a TestLogEventWriter as a log.LoggerProvider func NewTestLoggerWriter(name string, mode log.WriterMode) log.EventWriter { w := &TestLogEventWriter{} - w.EventWriterBaseImpl = log.NewEventWriterBase(name, mode) - w.Formatter = log.EventFormatTextMessage + w.EventWriterBaseImpl = log.NewEventWriterBase(name, "test-log-writer", mode) + w.FormatMessage = log.EventFormatTextMessage w.OutputWriteCloser = WriterCloser return w } diff --git a/routers/private/manager.go b/routers/private/manager.go index 872e65b2a1b6..4762ebb289b5 100644 --- a/routers/private/manager.go +++ b/routers/private/manager.go @@ -96,7 +96,7 @@ func AddLogger(ctx *context.PrivateContext) { } writerMode := log.WriterMode{} - writerMode.WriterType = opts.Mode + writerType := opts.Mode var flags string var ok bool @@ -133,7 +133,7 @@ func AddLogger(ctx *context.PrivateContext) { writerMode.Prefix, _ = opts.Config["prefix"].(string) writerMode.Expression, _ = opts.Config["expression"].(string) - switch opts.Mode { + switch writerType { case "console": writerOption := log.WriterConsoleOption{} writerOption.Stderr, _ = opts.Config["stderr"].(bool) @@ -167,9 +167,9 @@ func AddLogger(ctx *context.PrivateContext) { writerOption.Addr, _ = opts.Config["address"].(string) writerMode.WriterOption = writerOption default: - panic(fmt.Sprintf("invalid log writer mode: %s", writerMode.WriterType)) + panic(fmt.Sprintf("invalid log writer mode: %s", writerType)) } - writer, err := log.NewEventWriter(opts.Writer, writerMode) + writer, err := log.NewEventWriter(opts.Writer, writerType, writerMode) if err != nil { log.Error("Failed to create new log writer: %v", err) ctx.JSON(http.StatusInternalServerError, private.Response{ diff --git a/services/migrations/gitbucket.go b/services/migrations/gitbucket.go index 38dc2ac77c65..5f11555839b5 100644 --- a/services/migrations/gitbucket.go +++ b/services/migrations/gitbucket.go @@ -66,7 +66,7 @@ func (g *GitBucketDownloader) LogString() string { if g == nil { return "" } - return fmt.Sprintf("", g.baseURL, g.repoOwner, g.repoName) + return fmt.Sprintf("", g.baseURL, g.repoOwner, g.repoName) } // NewGitBucketDownloader creates a GitBucket downloader diff --git a/tests/integration/migration-test/migration_test.go b/tests/integration/migration-test/migration_test.go index 22c2326288ae..556a54015b4f 100644 --- a/tests/integration/migration-test/migration_test.go +++ b/tests/integration/migration-test/migration_test.go @@ -296,7 +296,7 @@ func doMigrationTest(t *testing.T, version string) { return } - setting.InitSQLLoggersForCli() + setting.InitSQLLoggersForCli(log.INFO) err := db.InitEngineWithMigration(context.Background(), wrappedMigrate) assert.NoError(t, err) From 6b21f681567bdc70ff5244d8d45da6575bfd99ba Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 16 May 2023 15:25:41 +0800 Subject: [PATCH 04/24] fine tune tests and documents --- custom/conf/app.example.ini | 85 ++---- .../config-cheat-sheet.en-us.md | 38 +-- .../administration/logging-config.en-us.md | 20 +- modules/log/event_writer_base.go | 4 +- modules/log/logger_test.go | 35 ++- modules/log/manager.go | 6 +- modules/setting/log.go | 85 +++--- modules/setting/log_test.go | 264 ++++++++++++++++++ modules/setting/mailer.go | 16 +- 9 files changed, 404 insertions(+), 149 deletions(-) create mode 100644 modules/setting/log_test.go diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 0055929d995e..76446f1f3361 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -549,34 +549,32 @@ ENABLE = true ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Main Logger ;; -;; Either "console", "file", "conn", "smtp" or "database", default is "console" +;; Either "console", "file" or "conn", default is "console" ;; Use comma to separate multiple modes, e.g. "console, file" MODE = console ;; ;; Either "Trace", "Debug", "Info", "Warn", "Error", "Critical" or "None", default is "Info" LEVEL = Info ;; -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; Router Logger -;; -;; Switch off the router log -;DISABLE_ROUTER_LOG=false +;; Print Stacktrace with logs (rarely helpful, do not set) Either "Trace", "Debug", "Info", "Warn", "Error", "Critical", default is "None" +;STACKTRACE_LEVEL = None ;; -;; Set the log "modes" for the router log (if file is set the log file will default to router.log) -ROUTER = console +;; Buffer length of the channel, keep it as it is if you don't know what it is. +;BUFFER_LEN = 10000 ;; -;; The router will log different things at different levels. +;; Sub logger modes, a single comma means use default MODE above, empty means disable it +;logger.access.MODE= +;logger.router.MODE=, +;logger.xorm.MODE=, ;; -;; * started messages will be logged at TRACE level -;; * polling/completed routers will be logged at INFO -;; * slow routers will be logged at WARN -;; * failed routers will be logged at WARN +;; Collect SSH logs (Creates log from ssh git request) ;; -;; The routing level will default to that of the system but individual router level can be set in -;; [log..router] LEVEL +;ENABLE_SSH_LOG = false ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; +;; Access Logger (Creates log in NCSA common log format) +;; ;; Print request id which parsed from request headers in access log, when access log is enabled. ;; * E.g: ;; * In request Header: X-Request-ID: test-id-123 @@ -590,53 +588,28 @@ ROUTER = console ;; * Configuration in app.ini: REQUEST_ID_HEADERS = X-Request-ID, X-Trace-ID, X-Req-ID ;; * Print in log: 127.0.0.1:58384 - - [14/Feb/2023:16:33:51 +0800] "trace-id-1q2w3e4r" ;; -;; REQUEST_ID_HEADERS = - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; -;; Access Logger (Creates log in NCSA common log format) -;; -;ENABLE_ACCESS_LOG = false -;; -;; Set the log "modes" for the access log (if file is set the log file will default to access.log) -;ACCESS = file +;REQUEST_ID_HEADERS = ;; ;; Sets the template used to create the access log. ;ACCESS_LOG_TEMPLATE = {{.Ctx.RemoteHost}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}" "{{.Ctx.Req.UserAgent}}" -;; -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; -;; SSH log (Creates log from ssh git request) -;; -;ENABLE_SSH_LOG = false -;; -;; Other Settings -;; -;; Print Stacktraces with logs. (Rarely helpful.) Either "Trace", "Debug", "Info", "Warn", "Error", "Critical", default is "None" -;STACKTRACE_LEVEL = None -;; -;; Buffer length of the channel, keep it as it is if you don't know what it is. -;BUFFER_LEN = 10000 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; Creating specific log configuration ;; -;; You can set specific configuration for individual modes and subloggers +;; Log modes (aka log writers) ;; -;; Configuration available to all log modes/subloggers +;[log.%(WriterMode}] +;MODE=console/file/conn/... ;LEVEL= ;FLAGS = stdflags ;EXPRESSION = ;PREFIX = ;COLORIZE = false ;; -;; For "console" mode only +;[log.console] ;STDERR = false ;; -;; For "file" mode only -;LEVEL = -;; Set the file_name for the logger. If this is a relative path this -;; will be relative to ROOT_PATH +;[log.file] +;; Set the file_name for the logger. If this is a relative path this will be relative to ROOT_PATH ;FILE_NAME = ;; This enables automated log rotate(switch of following options), default is true ;LOG_ROTATE = true @@ -650,9 +623,8 @@ ROUTER = console ;COMPRESS = true ;; compression level see godoc for compress/gzip ;COMPRESSION_LEVEL = -1 -; -;; For "conn" mode only -;LEVEL = +;; +;[log.conn] ;; Reconnect host for every single message, default is false ;RECONNECT_ON_MSG = false ;; Try to reconnect when connection is lost, default is false @@ -661,19 +633,6 @@ ROUTER = console ;PROTOCOL = tcp ;; Host address ;ADDR = -; -;; For "smtp" mode only -;LEVEL = -;; Name displayed in mail title, default is "Diagnostic message from server" -;SUBJECT = Diagnostic message from server -;; Mail server -;HOST = -;; Mailer user name and password -;USER = -;; Use PASSWD = `your password` for quoting if you use special characters in the password. -;PASSWD = -;; Receivers, can be one or more, e.g. 1@example.com,2@example.com -;RECEIVERS = ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index 624c59e87cf5..1e6e1535f727 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -835,22 +835,16 @@ Default templates for project boards: ## Log (`log`) - `ROOT_PATH`: **\**: Root path for log files. -- `MODE`: **console**: Logging mode. For multiple modes, use a comma to separate values. You can configure each mode in per mode log subsections `\[log.modename\]`. By default the file mode will log to `$ROOT_PATH/gitea.log`. +- `MODE`: **console**: Logging mode. For multiple modes, use a comma to separate values. You can configure each mode in per mode log subsections `\[log.writer-mode-name\]`. - `LEVEL`: **Info**: General log level. \[Trace, Debug, Info, Warn, Error, Critical, Fatal, None\] -- `STACKTRACE_LEVEL`: **None**: Default log level at which to log create stack traces. \[Trace, Debug, Info, Warn, Error, Critical, Fatal, None\] +- `STACKTRACE_LEVEL`: **None**: Default log level at which to log create stack traces (rarely useful, do not set it). \[Trace, Debug, Info, Warn, Error, Critical, Fatal, None\] - `ENABLE_SSH_LOG`: **false**: save ssh log to log file -- `ENABLE_XORM_LOG`: **true**: Set whether to perform XORM logging. Please note SQL statement logging can be disabled by setting `LOG_SQL` to false in the `[database]` section. - -### Router Log (`log`) - -- `DISABLE_ROUTER_LOG`: **false**: Mute printing of the router log. -- `ROUTER`: **console**: The mode or name of the log the router should log to. (If you set this to `,` it will log to default Gitea logger.) - NB: You must have `DISABLE_ROUTER_LOG` set to `false` for this option to take effect. Configure each mode in per mode log subsections `\[log.modename.router\]`. +- `logger.access.MODE`: **\**: The "access" logger +- `logger.router.MODE`: **,**: The "router" logger, a single comma means it will use the default MODE above +- `logger.xorm.MODE`: **,**: The "xorm" logger ### Access Log (`log`) -- `ENABLE_ACCESS_LOG`: **false**: Creates an access.log in NCSA common log format, or as per the following template -- `ACCESS`: **file**: Logging mode for the access logger, use a comma to separate values. Configure each mode in per mode log subsections `\[log.modename.access\]`. By default the file mode will log to `$ROOT_PATH/access.log`. (If you set this to `,` it will log to the default Gitea logger.) - `ACCESS_LOG_TEMPLATE`: **`{{.Ctx.RemoteHost}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}" "{{.Ctx.Req.UserAgent}}"`**: Sets the template used to create the access log. - The following variables are available: - `Ctx`: the `context.Context` of the request. @@ -858,29 +852,29 @@ Default templates for project boards: - `Start`: the start time of the request. - `ResponseWriter`: the responseWriter from the request. - `RequestID`: the value matching REQUEST_ID_HEADERS（default: `-`, if not matched）. - - You must be very careful to ensure that this template does not throw errors or panics as this template runs outside of the panic/recovery script. + - You must be very careful to ensure that this template does not throw errors or panics as this template runs outside the panic/recovery script. - `REQUEST_ID_HEADERS`: **\**: You can configure multiple values that are splited by comma here. It will match in the order of configuration, and the first match will be finally printed in the access log. - e.g. - In the Request Header: X-Request-ID: **test-id-123** - Configuration in app.ini: REQUEST_ID_HEADERS = X-Request-ID - Print in log: 127.0.0.1:58384 - - [14/Feb/2023:16:33:51 +0800] "**test-id-123**" ... -### Log subsections (`log.name`, `log.name.*`) +### Log subsections (`log.writer-mode-name`) -- `LEVEL`: **log.LEVEL**: Sets the log-level of this sublogger. Defaults to the `LEVEL` set in the global `[log]` section. +- `MODE`: **name**: Sets the mode of this log writer - Defaults to the provided subsection name. This allows you to have two different file loggers at different levels. +- `LEVEL`: **log.LEVEL**: Sets the log-level of this writer. Defaults to the `LEVEL` set in the global `[log]` section. - `STACKTRACE_LEVEL`: **log.STACKTRACE_LEVEL**: Sets the log level at which to log stack traces. -- `MODE`: **name**: Sets the mode of this sublogger - Defaults to the provided subsection name. This allows you to have two different file loggers at different levels. - `EXPRESSION`: **""**: A regular expression to match either the function name, file or message. Defaults to empty. Only log messages that match the expression will be saved in the logger. - `FLAGS`: **stdflags**: A comma separated string representing the log flags. Defaults to `stdflags` which represents the prefix: `2009/01/23 01:23:23 ...a/b/c/d.go:23:runtime.Caller() [I]: message`. `none` means don't prefix log lines. See `modules/log/flags.go` for more information. - `PREFIX`: **""**: An additional prefix for every log line in this logger. Defaults to empty. - `COLORIZE`: **false**: Whether to colorize the log lines -### Console log mode (`log.console`, `log.console.*`, or `MODE=console`) +### Console log mode (`log.console`, or `MODE=console`) - For the console logger `COLORIZE` will default to `true` if not on windows or the terminal is determined to be able to color. - `STDERR`: **false**: Use Stderr instead of Stdout. -### File log mode (`log.file`, `log.file.*` or `MODE=file`) +### File log mode (`log.file`, or `MODE=file`) - `FILE_NAME`: Set the file name for this logger. Defaults as described above. If relative will be relative to the `ROOT_PATH` - `LOG_ROTATE`: **true**: Rotate the log files. @@ -890,21 +884,13 @@ Default templates for project boards: - `COMPRESS`: **true**: Compress old log files by default with gzip - `COMPRESSION_LEVEL`: **-1**: Compression level -### Conn log mode (`log.conn`, `log.conn.*` or `MODE=conn`) +### Conn log mode (`log.conn`, or `MODE=conn`) - `RECONNECT_ON_MSG`: **false**: Reconnect host for every single message. - `RECONNECT`: **false**: Try to reconnect when connection is lost. - `PROTOCOL`: **tcp**: Set the protocol, either "tcp", "unix" or "udp". - `ADDR`: **:7020**: Sets the address to connect to. -### SMTP log mode (`log.smtp`, `log.smtp.*` or `MODE=smtp`) - -- `USER`: User email address to send from. -- `PASSWD`: Password for the smtp server. -- `HOST`: **127.0.0.1:25**: The SMTP host to connect to. -- `RECEIVERS`: Email addresses to send to. -- `SUBJECT`: **Diagnostic message from Gitea** - ## Cron (`cron`) - `ENABLED`: **false**: Enable to run all cron tasks periodically with default settings. diff --git a/docs/content/doc/administration/logging-config.en-us.md b/docs/content/doc/administration/logging-config.en-us.md index 20a730973408..5f3ca30c9a2d 100644 --- a/docs/content/doc/administration/logging-config.en-us.md +++ b/docs/content/doc/administration/logging-config.en-us.md @@ -21,7 +21,7 @@ The logging configuration of Gitea mainly consists of 3 types of components: - The `[log]` section for general configuration - `[log.]` sections for the configuration of different log writers to output logs, aka: "writer mode", the mode name is also used as "writer name". -- `[log]` section could contain sub-loggers like`logger.LOGGER-NAME.CONFIG-KEY` +- `[log]` section could contain sub-loggers like`logger..` There is a fully functional log output by default, so it is not necessary to define one. @@ -46,9 +46,9 @@ In the top level `[log]` section the following configurations can be placed: And it can contain the following sub-loggers: -- `logger.ROUTER.MODE`: (Default: **,**): List of log outputs to use for the Router logger. -- `logger.ACCESS.MODE`: (Default: **\**) List of log outputs to use for the Access logger. By default, the access logger is disabled. -- `logger.XORM.MODE`: (Default: **,**) List of log outputs to use for the XORM logger. +- `logger.router.MODE`: (Default: **,**): List of log outputs to use for the Router logger. +- `logger.access.MODE`: (Default: **\**) List of log outputs to use for the Access logger. By default, the access logger is disabled. +- `logger.xorm.MODE`: (Default: **,**) List of log outputs to use for the XORM logger. Setting a comma (`,`) to sub-logger's mode means making it use the default global `MODE`. @@ -64,9 +64,9 @@ ROOT_PATH = %(GITEA_WORK_DIR)/log MODE = console LEVEL = Info STACKTRACE_LEVEL = None -logger.ROUTER.MODE = , -logger.XORM.MODE = , -logger.ACCESS.MODE = +logger.router.MODE = , +logger.xorm.MODE = , +logger.access.MODE = ; this is the config options of "console" mode (used by MODE=console above) [log.console] @@ -86,8 +86,8 @@ The Router logger is disabled, the access logs (>=Warn) goes into `access.log`: ```ini [log] -logger.ROUTER.MODE = -logger.ACCESS.MODE = access-file +logger.router.MODE = +logger.access.MODE = access-file [log.access-file] MODE = file @@ -231,7 +231,7 @@ should be taken when changing its template. The main benefit of this logger is that Gitea can now log accesses in a standard log format so standard tools may be used. -You can enable this logger using `logger.ACCESS.MODE = ...`. +You can enable this logger using `logger.access.MODE = ...`. If desired the format of the Access logger can be changed by changing the value of the `ACCESS_LOG_TEMPLATE`. diff --git a/modules/log/event_writer_base.go b/modules/log/event_writer_base.go index 3ce946371754..3b56df33c2e9 100644 --- a/modules/log/event_writer_base.go +++ b/modules/log/event_writer_base.go @@ -82,8 +82,8 @@ func (b *EventWriterBaseImpl) Run(ctx context.Context) { } if exprRegexp != nil { - matched := exprRegexp.Match([]byte(fmt.Sprintf("%s:%d:%s", event.Origin.Filename, event.Origin.Line, event.Origin.Caller))) || - exprRegexp.Match([]byte(event.Origin.MsgSimpleText)) + fileLineCaller := fmt.Sprintf("%s:%d:%s", event.Origin.Filename, event.Origin.Line, event.Origin.Caller) + matched := exprRegexp.Match([]byte(fileLineCaller)) || exprRegexp.Match([]byte(event.Origin.MsgSimpleText)) if !matched { continue } diff --git a/modules/log/logger_test.go b/modules/log/logger_test.go index 75b10b014654..0afe8b43c442 100644 --- a/modules/log/logger_test.go +++ b/modules/log/logger_test.go @@ -92,7 +92,6 @@ func TestLoggerPause(t *testing.T) { w1 := newDummyWriter("dummy-1", DEBUG, 0) logger.AddWriters(w1) - assert.EqualValues(t, DEBUG, logger.GetLevel()) logger.Pause() @@ -107,3 +106,37 @@ func TestLoggerPause(t *testing.T) { logger.Close() } + +type testLogString struct{} + +func (t testLogString) LogString() string { + return "log-string" +} + +func TestLoggerLogString(t *testing.T) { + logger := NewLoggerWithWriters() + + w1 := newDummyWriter("dummy-1", DEBUG, 0) + logger.AddWriters(w1) + + logger.Info("%s %s", testLogString{}, &testLogString{}) + logger.Close() + + assert.Equal(t, []string{"log-string log-string\n"}, w1.GetLogs()) +} + +func TestLoggerExpressionFilter(t *testing.T) { + logger := NewLoggerWithWriters() + + w1 := newDummyWriter("dummy-1", DEBUG, 0) + w1.Mode.Expression = "foo.*" + logger.AddWriters(w1) + + logger.Info("foo") + logger.Info("bar") + logger.Info("foo bar") + logger.SendLogEvent(&Event{Level: INFO, Filename: "foo.go", MsgSimpleText: "by filename"}) + logger.Close() + + assert.Equal(t, []string{"foo\n", "foo bar\n", "by filename\n"}, w1.GetLogs()) +} diff --git a/modules/log/manager.go b/modules/log/manager.go index 49c6c5972c00..2572afb22345 100644 --- a/modules/log/manager.go +++ b/modules/log/manager.go @@ -80,14 +80,14 @@ func (m *LoggerManager) DumpLoggers() map[string]any { return dump } -var manager *LoggerManager +var manager = NewManager() func GetManager() *LoggerManager { return manager } -func init() { - manager = &LoggerManager{ +func NewManager() *LoggerManager { + return &LoggerManager{ loggers: map[string]*LoggerImpl{}, } } diff --git a/modules/setting/log.go b/modules/setting/log.go index 4ed2d1e68aab..bfa8a13ed3d1 100644 --- a/modules/setting/log.go +++ b/modules/setting/log.go @@ -15,8 +15,7 @@ import ( "code.gitea.io/gitea/modules/util" ) -// Log settings -var Log struct { +type LogGlobalConfig struct { RootPath string Mode string @@ -30,6 +29,8 @@ var Log struct { RequestIDHeaders []string } +var Log LogGlobalConfig + const accessLogTemplateDefault = `{{.Ctx.RemoteHost}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}" "{{.Ctx.Req.UserAgent}}"` func loadLogGlobalFrom(rootCfg ConfigProvider) { @@ -55,48 +56,48 @@ func loadLogGlobalFrom(rootCfg ConfigProvider) { func prepareLoggerConfig(rootCfg ConfigProvider) { sec := rootCfg.Section("log") - if !sec.HasKey("logger.DEFAULT.MODE") { - sec.Key("logger.DEFAULT.MODE").MustString(",") + if !sec.HasKey("logger.default.MODE") { + sec.Key("logger.default.MODE").MustString(",") } - deprecatedSetting(rootCfg, "log", "ACCESS", "log", "logger.ACCESS.MODE", "1.21") - deprecatedSetting(rootCfg, "log", "ENABLE_ACCESS_LOG", "log", "logger.ACCESS.MODE", "1.21") + deprecatedSetting(rootCfg, "log", "ACCESS", "log", "logger.access.MODE", "1.21") + deprecatedSetting(rootCfg, "log", "ENABLE_ACCESS_LOG", "log", "logger.access.MODE", "1.21") if val := sec.Key("ACCESS").String(); val != "" { - sec.Key("logger.ACCESS.MODE").MustString(val) + sec.Key("logger.access.MODE").MustString(val) } if sec.HasKey("ENABLE_ACCESS_LOG") && !sec.Key("ENABLE_ACCESS_LOG").MustBool() { - sec.Key("logger.ACCESS.MODE").SetValue("") + sec.Key("logger.access.MODE").SetValue("") } - deprecatedSetting(rootCfg, "log", "ROUTER", "log", "logger.ROUTER.MODE", "1.21") - deprecatedSetting(rootCfg, "log", "DISABLE_ROUTER_LOG", "log", "logger.ROUTER.MODE", "1.21") + deprecatedSetting(rootCfg, "log", "ROUTER", "log", "logger.router.MODE", "1.21") + deprecatedSetting(rootCfg, "log", "DISABLE_ROUTER_LOG", "log", "logger.router.MODE", "1.21") if val := sec.Key("ROUTER").String(); val != "" { - sec.Key("logger.ROUTER.MODE").MustString(val) + sec.Key("logger.router.MODE").MustString(val) } - if !sec.HasKey("logger.ROUTER.MODE") { - sec.Key("logger.ROUTER.MODE").MustString(",") // use default logger + if !sec.HasKey("logger.router.MODE") { + sec.Key("logger.router.MODE").MustString(",") // use default logger } if sec.HasKey("DISABLE_ROUTER_LOG") && sec.Key("DISABLE_ROUTER_LOG").MustBool() { - sec.Key("logger.ROUTER.MODE").SetValue("") + sec.Key("logger.router.MODE").SetValue("") } - deprecatedSetting(rootCfg, "log", "XORM", "log", "logger.XORM.MODE", "1.21") - deprecatedSetting(rootCfg, "log", "ENABLE_XORM_LOG", "log", "logger.XORM.MODE", "1.21") + deprecatedSetting(rootCfg, "log", "XORM", "log", "logger.xorm.MODE", "1.21") + deprecatedSetting(rootCfg, "log", "ENABLE_XORM_LOG", "log", "logger.xorm.MODE", "1.21") if val := sec.Key("XORM").String(); val != "" { - sec.Key("logger.XORM.MODE").MustString(val) + sec.Key("logger.xorm.MODE").MustString(val) } - if !sec.HasKey("logger.XORM.MODE") { - sec.Key("logger.XORM.MODE").MustString(",") // use default logger + if !sec.HasKey("logger.xorm.MODE") { + sec.Key("logger.xorm.MODE").MustString(",") // use default logger } if sec.HasKey("ENABLE_XORM_LOG") && !sec.Key("ENABLE_XORM_LOG").MustBool() { - sec.Key("logger.XORM.MODE").SetValue("") + sec.Key("logger.xorm.MODE").SetValue("") } } -func LogPrepareFilenameForWriter(modeName, fileName string) string { +func LogPrepareFilenameForWriter(loggerName, fileName string) string { defaultFileName := "gitea.log" - if modeName != "file" { - defaultFileName = modeName + ".log" + if loggerName != "default" { + defaultFileName = loggerName + ".log" } if fileName == "" { fileName = defaultFileName @@ -112,7 +113,7 @@ func LogPrepareFilenameForWriter(modeName, fileName string) string { return fileName } -func loadLogModeByName(rootCfg ConfigProvider, modeName string) (writerType string, writerMode log.WriterMode) { +func loadLogModeByName(rootCfg ConfigProvider, loggerName, modeName string) (writerType string, writerMode log.WriterMode, err error) { sec := rootCfg.Section("log." + modeName) writerMode = log.WriterMode{} @@ -137,7 +138,7 @@ func loadLogModeByName(rootCfg ConfigProvider, modeName string) (writerType stri } writerMode.WriterOption = writerOption case "file": - fileName := LogPrepareFilenameForWriter(modeName, sec.Key("FILE_NAME").String()) + fileName := LogPrepareFilenameForWriter(loggerName, sec.Key("FILE_NAME").String()) writerOption := log.WriterFileOption{} writerOption.FileName = fileName + filenameSuffix // FIXME: the suffix doesn't seem right, see its related comments writerOption.LogRotate = sec.Key("LOG_ROTATE").MustBool(true) @@ -156,11 +157,11 @@ func loadLogModeByName(rootCfg ConfigProvider, modeName string) (writerType stri writerMode.WriterOption = writerOption default: if !log.HasEventWriter(writerType) { - panic(fmt.Sprintf("invalid log writer type (mode): %s", writerType)) + return "", writerMode, fmt.Errorf("invalid log writer type (mode): %s", writerType) } } - return writerType, writerMode + return writerType, writerMode, nil } var filenameSuffix = "" @@ -178,22 +179,26 @@ func InitLoggersForTest() { // initAllLoggers creates all the log services func initAllLoggers() { - loadLogGlobalFrom(CfgProvider) - prepareLoggerConfig(CfgProvider) - - initLoggerByName(CfgProvider, log.DEFAULT) // default - initLoggerByName(CfgProvider, "access") - initLoggerByName(CfgProvider, "router") - initLoggerByName(CfgProvider, "xorm") + initManagedLoggers(log.GetManager(), CfgProvider) golog.SetFlags(0) golog.SetPrefix("") golog.SetOutput(log.LoggerToWriter(log.GetLogger(log.DEFAULT).Info)) } -func initLoggerByName(rootCfg ConfigProvider, loggerName string) { +func initManagedLoggers(manager *log.LoggerManager, cfg ConfigProvider) { + loadLogGlobalFrom(cfg) + prepareLoggerConfig(cfg) + + initLoggerByName(manager, cfg, log.DEFAULT) // default + initLoggerByName(manager, cfg, "access") + initLoggerByName(manager, cfg, "router") + initLoggerByName(manager, cfg, "xorm") +} + +func initLoggerByName(manager *log.LoggerManager, rootCfg ConfigProvider, loggerName string) { sec := rootCfg.Section("log") - keyPrefix := "logger." + strings.ToUpper(loggerName) + keyPrefix := "logger." + loggerName disabled := sec.HasKey(keyPrefix+".MODE") && sec.Key(keyPrefix+".MODE").String() == "" if disabled { @@ -213,7 +218,11 @@ func initLoggerByName(rootCfg ConfigProvider, loggerName string) { continue } writerName := modeName - writerType, writerMode := loadLogModeByName(rootCfg, modeName) + writerType, writerMode, err := loadLogModeByName(rootCfg, loggerName, modeName) + if err != nil { + log.FallbackErrorf("Failed to load writer mode %q for logger %s: %v", modeName, loggerName, err) + continue + } if writerMode.BufferLen == 0 { writerMode.BufferLen = Log.BufferLen } @@ -225,7 +234,7 @@ func initLoggerByName(rootCfg ConfigProvider, loggerName string) { eventWriters = append(eventWriters, eventWriter) } - log.GetManager().GetLogger(loggerName).RemoveAllWriters().AddWriters(eventWriters...) + manager.GetLogger(loggerName).RemoveAllWriters().AddWriters(eventWriters...) } func InitSQLLoggersForCli(level log.Level) { diff --git a/modules/setting/log_test.go b/modules/setting/log_test.go new file mode 100644 index 000000000000..b65f533a8c5c --- /dev/null +++ b/modules/setting/log_test.go @@ -0,0 +1,264 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "path/filepath" + "strings" + "testing" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func initLoggersByConfig(t *testing.T, config string) (*log.LoggerManager, func()) { + oldLogConfig := Log + Log = LogGlobalConfig{} + defer func() { + Log = oldLogConfig + }() + + cfg, err := NewConfigProviderFromData(config) + assert.NoError(t, err) + + manager := log.NewManager() + initManagedLoggers(manager, cfg) + return manager, manager.Close +} + +func toJSON(v interface{}) string { + b, _ := json.MarshalIndent(v, "", "\t") + return string(b) +} + +func TestLogConfigDefault(t *testing.T) { + manager, managerClose := initLoggersByConfig(t, ``) + + writerDump := ` +{ + "console": { + "BufferLen": 10000, + "Colorize": false, + "Expression": "", + "Flags": "date,levelinitial,longfile,shortfile,shortfuncname,time", + "Level": "info", + "Prefix": "", + "StacktraceLevel": "none", + "WriterOption": { + "Stderr": false + }, + "WriterType": "console" + } +} +` + defer managerClose() + + dump := manager.GetLogger(log.DEFAULT).DumpWriters() + require.JSONEq(t, writerDump, toJSON(dump)) + + dump = manager.GetLogger("access").DumpWriters() + require.JSONEq(t, "{}", toJSON(dump)) + + dump = manager.GetLogger("router").DumpWriters() + require.JSONEq(t, writerDump, toJSON(dump)) + + dump = manager.GetLogger("xorm").DumpWriters() + require.JSONEq(t, writerDump, toJSON(dump)) +} + +func TestLogConfigDisable(t *testing.T) { + manager, managerClose := initLoggersByConfig(t, ` +[log] +logger.router.MODE = +logger.xorm.MODE = +`) + + writerDump := ` +{ + "console": { + "BufferLen": 10000, + "Colorize": false, + "Expression": "", + "Flags": "date,levelinitial,longfile,shortfile,shortfuncname,time", + "Level": "info", + "Prefix": "", + "StacktraceLevel": "none", + "WriterOption": { + "Stderr": false + }, + "WriterType": "console" + } +} +` + defer managerClose() + + dump := manager.GetLogger(log.DEFAULT).DumpWriters() + require.JSONEq(t, writerDump, toJSON(dump)) + + dump = manager.GetLogger("access").DumpWriters() + require.JSONEq(t, "{}", toJSON(dump)) + + dump = manager.GetLogger("router").DumpWriters() + require.JSONEq(t, "{}", toJSON(dump)) + + dump = manager.GetLogger("xorm").DumpWriters() + require.JSONEq(t, "{}", toJSON(dump)) +} + +func TestLogConfigLegacyDefault(t *testing.T) { + manager, managerClose := initLoggersByConfig(t, ` +[log] +MODE = console +`) + + writerDump := ` +{ + "console": { + "BufferLen": 10000, + "Colorize": false, + "Expression": "", + "Flags": "date,levelinitial,longfile,shortfile,shortfuncname,time", + "Level": "info", + "Prefix": "", + "StacktraceLevel": "none", + "WriterOption": { + "Stderr": false + }, + "WriterType": "console" + } +} +` + defer managerClose() + + dump := manager.GetLogger(log.DEFAULT).DumpWriters() + require.JSONEq(t, writerDump, toJSON(dump)) + + dump = manager.GetLogger("access").DumpWriters() + require.JSONEq(t, "{}", toJSON(dump)) + + dump = manager.GetLogger("router").DumpWriters() + require.JSONEq(t, writerDump, toJSON(dump)) + + dump = manager.GetLogger("xorm").DumpWriters() + require.JSONEq(t, writerDump, toJSON(dump)) +} + +func TestLogConfigLegacyMode(t *testing.T) { + tempDir := t.TempDir() + + tempPath := func(file string) string { + return filepath.Join(tempDir, file) + } + + manager, managerClose := initLoggersByConfig(t, ` +[log] +ROOT_PATH = `+tempDir+` +MODE = file +ROUTER = file +ACCESS = file +`) + defer managerClose() + + writerDump := ` +{ + "file": { + "BufferLen": 10000, + "Colorize": false, + "Expression": "", + "Flags": "date,levelinitial,longfile,shortfile,shortfuncname,time", + "Level": "info", + "Prefix": "", + "StacktraceLevel": "none", + "WriterOption": { + "Compress": true, + "CompressionLevel": -1, + "DailyRotate": true, + "FileName": "$FILENAME", + "LogRotate": true, + "MaxDays": 7, + "MaxSize": 268435456 + }, + "WriterType": "file" + } +} +` + dump := manager.GetLogger(log.DEFAULT).DumpWriters() + require.JSONEq(t, strings.ReplaceAll(writerDump, "$FILENAME", tempPath("gitea.log")), toJSON(dump)) + + dump = manager.GetLogger("access").DumpWriters() + require.JSONEq(t, strings.ReplaceAll(writerDump, "$FILENAME", tempPath("access.log")), toJSON(dump)) + + dump = manager.GetLogger("router").DumpWriters() + require.JSONEq(t, strings.ReplaceAll(writerDump, "$FILENAME", tempPath("router.log")), toJSON(dump)) +} + +func TestLogConfigLegacyModeDisable(t *testing.T) { + manager, managerClose := initLoggersByConfig(t, ` +[log] +ROUTER = file +ACCESS = file +DISABLE_ROUTER_LOG = true +ENABLE_ACCESS_LOG = false +`) + defer managerClose() + + dump := manager.GetLogger("access").DumpWriters() + require.JSONEq(t, "{}", toJSON(dump)) + + dump = manager.GetLogger("router").DumpWriters() + require.JSONEq(t, "{}", toJSON(dump)) +} + +func TestLogConfigNewConfig(t *testing.T) { + manager, managerClose := initLoggersByConfig(t, ` +[log] +logger.xorm.MODE = console, console-1 + +[log.console] +LEVEL = warn + +[log.console-1] +MODE = console +LEVEL = error +STDERR = true +`) + + writerDump := ` +{ + "console": { + "BufferLen": 10000, + "Colorize": false, + "Expression": "", + "Flags": "date,levelinitial,longfile,shortfile,shortfuncname,time", + "Level": "warn", + "Prefix": "", + "StacktraceLevel": "none", + "WriterOption": { + "Stderr": false + }, + "WriterType": "console" + }, + "console-1": { + "BufferLen": 10000, + "Colorize": false, + "Expression": "", + "Flags": "date,levelinitial,longfile,shortfile,shortfuncname,time", + "Level": "error", + "Prefix": "", + "StacktraceLevel": "none", + "WriterOption": { + "Stderr": true + }, + "WriterType": "console" + } +} +` + defer managerClose() + + dump := manager.GetLogger("xorm").DumpWriters() + require.JSONEq(t, writerDump, toJSON(dump)) +} diff --git a/modules/setting/mailer.go b/modules/setting/mailer.go index 39afce7d4645..d6ecd6b81608 100644 --- a/modules/setting/mailer.go +++ b/modules/setting/mailer.go @@ -4,6 +4,7 @@ package setting import ( + "context" "net" "net/mail" "strings" @@ -198,7 +199,7 @@ func loadMailerFrom(rootCfg ConfigProvider) { ips := tryResolveAddr(MailService.SMTPAddr) if MailService.Protocol == "smtp" { for _, ip := range ips { - if !ip.IsLoopback() { + if !ip.IP.IsLoopback() { log.Warn("connecting over insecure SMTP protocol to non-local address is not recommended") break } @@ -258,20 +259,23 @@ func loadNotifyMailFrom(rootCfg ConfigProvider) { log.Info("Notify Mail Service Enabled") } -func tryResolveAddr(addr string) []net.IP { +func tryResolveAddr(addr string) []net.IPAddr { if strings.HasPrefix(addr, "[") && strings.HasSuffix(addr, "]") { addr = addr[1 : len(addr)-1] } ip := net.ParseIP(addr) if ip != nil { - ips := make([]net.IP, 1) - ips[0] = ip + ips := make([]net.IPAddr, 1) + ips[0] = net.IPAddr{IP: ip} return ips } - ips, err := net.LookupIP(addr) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + ips, err := net.DefaultResolver.LookupIPAddr(ctx, addr) if err != nil { log.Warn("could not look up mailer.SMTP_ADDR: %v", err) - return make([]net.IP, 0) + return nil } return ips } From a3b7b288afb75b31e724de2fa81adbaa6443fe5b Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 16 May 2023 19:29:14 +0800 Subject: [PATCH 05/24] optimize flags --- modules/log/flags.go | 14 ++++++++++++++ modules/log/flags_test.go | 3 +++ modules/setting/log_test.go | 12 ++++++------ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/modules/log/flags.go b/modules/log/flags.go index edb96a486089..109efed04ebe 100644 --- a/modules/log/flags.go +++ b/modules/log/flags.go @@ -74,6 +74,14 @@ var flagToString = map[uint32]string{ Lgopid: "gopid", } +var flagComboToString = []struct { + flag uint32 + name string +}{ + {LstdFlags, "stdflags"}, + {Lmedfile, "medfile"}, +} + func (f Flags) Bits() uint32 { if !f.defined { return Ldefault @@ -84,6 +92,12 @@ func (f Flags) Bits() uint32 { func (f Flags) String() string { flags := f.Bits() var flagNames []string + for _, it := range flagComboToString { + if flags&it.flag == it.flag { + flags = flags &^ it.flag + flagNames = append(flagNames, it.name) + } + } for flag, name := range flagToString { if flags&flag != 0 { flagNames = append(flagNames, name) diff --git a/modules/log/flags_test.go b/modules/log/flags_test.go index f17ce9a302c0..03972a9fb06b 100644 --- a/modules/log/flags_test.go +++ b/modules/log/flags_test.go @@ -18,6 +18,9 @@ func TestFlags(t *testing.T) { assert.EqualValues(t, 0, FlagsFromString("none", Lgopid).Bits()) assert.EqualValues(t, Ldate|Ltime, FlagsFromString("date,time", Lgopid).Bits()) + assert.EqualValues(t, "stdflags", FlagsFromString("stdflags").String()) + assert.EqualValues(t, "medfile", FlagsFromString("medfile").String()) + bs, err := json.Marshal(FlagsFromString("utc,level")) assert.NoError(t, err) assert.EqualValues(t, `"level,utc"`, string(bs)) diff --git a/modules/setting/log_test.go b/modules/setting/log_test.go index b65f533a8c5c..52de16b9cdfb 100644 --- a/modules/setting/log_test.go +++ b/modules/setting/log_test.go @@ -44,7 +44,7 @@ func TestLogConfigDefault(t *testing.T) { "BufferLen": 10000, "Colorize": false, "Expression": "", - "Flags": "date,levelinitial,longfile,shortfile,shortfuncname,time", + "Flags": "stdflags", "Level": "info", "Prefix": "", "StacktraceLevel": "none", @@ -83,7 +83,7 @@ logger.xorm.MODE = "BufferLen": 10000, "Colorize": false, "Expression": "", - "Flags": "date,levelinitial,longfile,shortfile,shortfuncname,time", + "Flags": "stdflags", "Level": "info", "Prefix": "", "StacktraceLevel": "none", @@ -121,7 +121,7 @@ MODE = console "BufferLen": 10000, "Colorize": false, "Expression": "", - "Flags": "date,levelinitial,longfile,shortfile,shortfuncname,time", + "Flags": "stdflags", "Level": "info", "Prefix": "", "StacktraceLevel": "none", @@ -169,7 +169,7 @@ ACCESS = file "BufferLen": 10000, "Colorize": false, "Expression": "", - "Flags": "date,levelinitial,longfile,shortfile,shortfuncname,time", + "Flags": "stdflags", "Level": "info", "Prefix": "", "StacktraceLevel": "none", @@ -233,7 +233,7 @@ STDERR = true "BufferLen": 10000, "Colorize": false, "Expression": "", - "Flags": "date,levelinitial,longfile,shortfile,shortfuncname,time", + "Flags": "stdflags", "Level": "warn", "Prefix": "", "StacktraceLevel": "none", @@ -246,7 +246,7 @@ STDERR = true "BufferLen": 10000, "Colorize": false, "Expression": "", - "Flags": "date,levelinitial,longfile,shortfile,shortfuncname,time", + "Flags": "stdflags", "Level": "error", "Prefix": "", "StacktraceLevel": "none", From 2aea30ba8dd7c0ad0679b3b9e9b87de5b5a2ebd8 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 16 May 2023 20:14:46 +0800 Subject: [PATCH 06/24] add one more test --- modules/log/event_writer_conn.go | 1 - modules/log/event_writer_console.go | 1 - modules/log/event_writer_file.go | 1 - modules/log/flags.go | 34 +++++------- modules/log/logger_impl.go | 2 +- modules/log/logger_test.go | 8 +-- modules/setting/log_test.go | 80 +++++++++++++++++++++++++++++ modules/testlogger/testlogger.go | 1 - 8 files changed, 99 insertions(+), 29 deletions(-) diff --git a/modules/log/event_writer_conn.go b/modules/log/event_writer_conn.go index 12c81f46a8cb..b52481986d71 100644 --- a/modules/log/event_writer_conn.go +++ b/modules/log/event_writer_conn.go @@ -31,7 +31,6 @@ func NewEventWriterConn(name string, mode WriterMode) EventWriter { Net: opt.Protocol, Addr: opt.Addr, } - w.FormatMessage = EventFormatTextMessage w.OutputWriteCloser = &w.connWriter return w } diff --git a/modules/log/event_writer_console.go b/modules/log/event_writer_console.go index e54b8020e36b..78183de644ba 100644 --- a/modules/log/event_writer_console.go +++ b/modules/log/event_writer_console.go @@ -27,7 +27,6 @@ func (nopCloser) Close() error { return nil } func NewEventWriterConsole(name string, mode WriterMode) EventWriter { w := &eventWriterConsole{EventWriterBaseImpl: NewEventWriterBase(name, "console", mode)} opt := mode.WriterOption.(WriterConsoleOption) - w.FormatMessage = EventFormatTextMessage if opt.Stderr { w.OutputWriteCloser = nopCloser{os.Stderr} } else { diff --git a/modules/log/event_writer_file.go b/modules/log/event_writer_file.go index 71f9effb556e..4f41b96453a0 100644 --- a/modules/log/event_writer_file.go +++ b/modules/log/event_writer_file.go @@ -39,7 +39,6 @@ func NewEventWriterFile(name string, mode WriterMode) EventWriter { if err != nil { FallbackErrorf("unable to open log file %q: %v", opt.FileName, err) } - w.FormatMessage = EventFormatTextMessage w.OutputWriteCloser = w.fileWriter return w } diff --git a/modules/log/flags.go b/modules/log/flags.go index 109efed04ebe..d27bd87867e2 100644 --- a/modules/log/flags.go +++ b/modules/log/flags.go @@ -60,26 +60,25 @@ var flagFromString = map[string]uint32{ "stdflags": LstdFlags, } -var flagToString = map[uint32]string{ - Ldate: "date", - Ltime: "time", - Lmicroseconds: "microseconds", - Llongfile: "longfile", - Lshortfile: "shortfile", - Lfuncname: "funcname", - Lshortfuncname: "shortfuncname", - LUTC: "utc", - Llevelinitial: "levelinitial", - Llevel: "level", - Lgopid: "gopid", -} - var flagComboToString = []struct { flag uint32 name string }{ + // name with more bits comes first {LstdFlags, "stdflags"}, {Lmedfile, "medfile"}, + + {Ldate, "date"}, + {Ltime, "time"}, + {Lmicroseconds, "microseconds"}, + {Llongfile, "longfile"}, + {Lshortfile, "shortfile"}, + {Lfuncname, "funcname"}, + {Lshortfuncname, "shortfuncname"}, + {LUTC, "utc"}, + {Llevelinitial, "levelinitial"}, + {Llevel, "level"}, + {Lgopid, "gopid"}, } func (f Flags) Bits() uint32 { @@ -94,15 +93,10 @@ func (f Flags) String() string { var flagNames []string for _, it := range flagComboToString { if flags&it.flag == it.flag { - flags = flags &^ it.flag + flags &^= it.flag flagNames = append(flagNames, it.name) } } - for flag, name := range flagToString { - if flags&flag != 0 { - flagNames = append(flagNames, name) - } - } if len(flagNames) == 0 { return "none" } diff --git a/modules/log/logger_impl.go b/modules/log/logger_impl.go index 34f287c556cb..eb55d845754e 100644 --- a/modules/log/logger_impl.go +++ b/modules/log/logger_impl.go @@ -56,7 +56,7 @@ func (l *LoggerImpl) SendLogEvent(event *Event) { } formatted := &EventFormatted{ Origin: event, - Msg: w.Base().FormatMessage(w.Base().Mode, event, msgFormat, msgArgs), + Msg: w.Base().FormatMessage(w.Base().Mode, event, msgFormat, msgArgs...), } select { case w.Base().Queue <- formatted: diff --git a/modules/log/logger_test.go b/modules/log/logger_test.go index 0afe8b43c442..fd7ebe67ca82 100644 --- a/modules/log/logger_test.go +++ b/modules/log/logger_test.go @@ -45,9 +45,8 @@ func (d *dummyWriter) GetLogs() []string { func newDummyWriter(name string, level Level, delay time.Duration) *dummyWriter { w := &dummyWriter{ EventWriterBaseImpl: NewEventWriterBase(name, "dummy", WriterMode{Level: level, Flags: FlagsFromBits(0)}), - - delay: delay, } + w.delay = delay w.Base().OutputWriteCloser = w return w } @@ -117,12 +116,13 @@ func TestLoggerLogString(t *testing.T) { logger := NewLoggerWithWriters() w1 := newDummyWriter("dummy-1", DEBUG, 0) + w1.Mode.Colorize = true logger.AddWriters(w1) - logger.Info("%s %s", testLogString{}, &testLogString{}) + logger.Info("%s %s %s", testLogString{}, &testLogString{}, NewColoredValue(testLogString{}, FgRed)) logger.Close() - assert.Equal(t, []string{"log-string log-string\n"}, w1.GetLogs()) + assert.Equal(t, []string{"log-string log-string \x1b[31mlog-string\x1b[0m\n"}, w1.GetLogs()) } func TestLoggerExpressionFilter(t *testing.T) { diff --git a/modules/setting/log_test.go b/modules/setting/log_test.go index 52de16b9cdfb..71fe0bb15569 100644 --- a/modules/setting/log_test.go +++ b/modules/setting/log_test.go @@ -262,3 +262,83 @@ STDERR = true dump := manager.GetLogger("xorm").DumpWriters() require.JSONEq(t, writerDump, toJSON(dump)) } + +func TestLogConfigModeFile(t *testing.T) { + tempDir := t.TempDir() + + tempPath := func(file string) string { + return filepath.Join(tempDir, file) + } + + manager, managerClose := initLoggersByConfig(t, ` +[log] +ROOT_PATH = `+tempDir+` +BUFFER_LEN = 10 +MODE = file, file1 + +[log.file1] +MODE = file +LEVEL = error +STACKTRACE_LEVEL = fatal +EXPRESSION = filter +FLAGS = medfile +PREFIX = "[Prefix] " +FILE_NAME = file-xxx.log +LOG_ROTATE = false +MAX_SIZE_SHIFT = 1 +DAILY_ROTATE = false +MAX_DAYS = 90 +COMPRESS = false +COMPRESSION_LEVEL = 4 +`) + + writerDump := ` +{ + "file": { + "BufferLen": 10, + "Colorize": false, + "Expression": "", + "Flags": "stdflags", + "Level": "info", + "Prefix": "", + "StacktraceLevel": "none", + "WriterOption": { + "Compress": true, + "CompressionLevel": -1, + "DailyRotate": true, + "FileName": "$FILENAME-0", + "LogRotate": true, + "MaxDays": 7, + "MaxSize": 268435456 + }, + "WriterType": "file" + }, + "file1": { + "BufferLen": 10, + "Colorize": false, + "Expression": "filter", + "Flags": "medfile", + "Level": "error", + "Prefix": "[Prefix] ", + "StacktraceLevel": "fatal", + "WriterOption": { + "Compress": false, + "CompressionLevel": 4, + "DailyRotate": false, + "FileName": "$FILENAME-1", + "LogRotate": false, + "MaxDays": 90, + "MaxSize": 2 + }, + "WriterType": "file" + } +} +` + defer managerClose() + + dump := manager.GetLogger(log.DEFAULT).DumpWriters() + expected := writerDump + expected = strings.ReplaceAll(expected, "$FILENAME-0", tempPath("gitea.log")) + expected = strings.ReplaceAll(expected, "$FILENAME-1", tempPath("file-xxx.log")) + require.JSONEq(t, expected, toJSON(dump)) +} diff --git a/modules/testlogger/testlogger.go b/modules/testlogger/testlogger.go index 117c45c50cb2..2c8d327f83cc 100644 --- a/modules/testlogger/testlogger.go +++ b/modules/testlogger/testlogger.go @@ -174,7 +174,6 @@ type TestLogEventWriter struct { func NewTestLoggerWriter(name string, mode log.WriterMode) log.EventWriter { w := &TestLogEventWriter{} w.EventWriterBaseImpl = log.NewEventWriterBase(name, "test-log-writer", mode) - w.FormatMessage = log.EventFormatTextMessage w.OutputWriteCloser = WriterCloser return w } From 9d59a8668d424225b2f4ba1294eabb731e7f7e62 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 16 May 2023 22:31:54 +0800 Subject: [PATCH 07/24] improve legacy testlogger --- modules/testlogger/testlogger.go | 54 ++++++++++---------------------- 1 file changed, 17 insertions(+), 37 deletions(-) diff --git a/modules/testlogger/testlogger.go b/modules/testlogger/testlogger.go index 2c8d327f83cc..b4275e600570 100644 --- a/modules/testlogger/testlogger.go +++ b/modules/testlogger/testlogger.go @@ -27,10 +27,10 @@ var WriterCloser = &testLoggerWriterCloser{} type testLoggerWriterCloser struct { sync.RWMutex - t []*testing.TB + t []testing.TB } -func (w *testLoggerWriterCloser) pushT(t *testing.TB) { +func (w *testLoggerWriterCloser) pushT(t testing.TB) { w.Lock() w.t = append(w.t, t) w.Unlock() @@ -42,7 +42,7 @@ func (w *testLoggerWriterCloser) Write(p []byte) (int, error) { w.RLock() defer w.RUnlock() - var t *testing.TB + var t testing.TB if len(w.t) > 0 { t = w.t[len(w.t)-1] } @@ -51,33 +51,13 @@ func (w *testLoggerWriterCloser) Write(p []byte) (int, error) { p = p[:len(p)-1] } - if t == nil || *t == nil { + if t == nil { // if there is no running test, the log message should be outputted to console, to avoid losing important information. // the "???" prefix is used to match the "===" and "+++" in PrintCurrentTest return fmt.Fprintf(os.Stdout, "??? [TestLogger] %s\n", p) } - defer func() { - err := recover() - if err == nil { - return - } - var errString string - errErr, ok := err.(error) - if ok { - errString = errErr.Error() - } else { - errString, ok = err.(string) - } - if !ok { - panic(err) - } - if !strings.HasPrefix(errString, "Log in goroutine after ") { - panic(err) - } - }() - - (*t).Log(string(p)) + t.Log(string(p)) return len(p), nil } @@ -100,8 +80,8 @@ func (w *testLoggerWriterCloser) Reset() { if t == nil { continue } - fmt.Fprintf(os.Stdout, "Unclosed logger writer in test: %s", (*t).Name()) - (*t).Errorf("Unclosed logger writer in test: %s", (*t).Name()) + _, _ = fmt.Fprintf(os.Stdout, "Unclosed logger writer in test: %s", t.Name()) + t.Errorf("Unclosed logger writer in test: %s", t.Name()) } w.t = nil } @@ -118,25 +98,25 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() { _, filename, line, _ := runtime.Caller(actualSkip) if log.CanColorStdout { - fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", fmt.Formatter(log.NewColoredValue(t.Name())), strings.TrimPrefix(filename, prefix), line) + _, _ = fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", fmt.Formatter(log.NewColoredValue(t.Name())), strings.TrimPrefix(filename, prefix), line) } else { - fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", t.Name(), strings.TrimPrefix(filename, prefix), line) + _, _ = fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", t.Name(), strings.TrimPrefix(filename, prefix), line) } - WriterCloser.pushT(&t) + WriterCloser.pushT(t) return func() { took := time.Since(start) if took > SlowTest { if log.CanColorStdout { - fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgYellow)), fmt.Formatter(log.NewColoredValue(took, log.Bold, log.FgYellow))) + _, _ = fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgYellow)), fmt.Formatter(log.NewColoredValue(took, log.Bold, log.FgYellow))) } else { - fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", t.Name(), took) + _, _ = fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", t.Name(), took) } } timer := time.AfterFunc(SlowFlush, func() { if log.CanColorStdout { - fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), SlowFlush) + _, _ = fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), SlowFlush) } else { - fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), SlowFlush) + _, _ = fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), SlowFlush) } }) if err := queue.GetManager().FlushAll(context.Background(), time.Minute); err != nil { @@ -146,9 +126,9 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() { flushTook := time.Since(start) - took if flushTook > SlowFlush { if log.CanColorStdout { - fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), fmt.Formatter(log.NewColoredValue(flushTook, log.Bold, log.FgRed))) + _, _ = fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), fmt.Formatter(log.NewColoredValue(flushTook, log.Bold, log.FgRed))) } else { - fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", t.Name(), flushTook) + _, _ = fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", t.Name(), flushTook) } } WriterCloser.popT() @@ -162,7 +142,7 @@ func Printf(format string, args ...interface{}) { args[i] = log.NewColoredValue(args[i]) } } - fmt.Fprintf(os.Stdout, "\t"+format, args...) + _, _ = fmt.Fprintf(os.Stdout, "\t"+format, args...) } // TestLogEventWriter is a logger which will write to the testing log From f54bfd5b8de248ac413973336dc2f727425d9a52 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 16 May 2023 22:41:02 +0800 Subject: [PATCH 08/24] revert mailer --- modules/setting/mailer.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/modules/setting/mailer.go b/modules/setting/mailer.go index d6ecd6b81608..39afce7d4645 100644 --- a/modules/setting/mailer.go +++ b/modules/setting/mailer.go @@ -4,7 +4,6 @@ package setting import ( - "context" "net" "net/mail" "strings" @@ -199,7 +198,7 @@ func loadMailerFrom(rootCfg ConfigProvider) { ips := tryResolveAddr(MailService.SMTPAddr) if MailService.Protocol == "smtp" { for _, ip := range ips { - if !ip.IP.IsLoopback() { + if !ip.IsLoopback() { log.Warn("connecting over insecure SMTP protocol to non-local address is not recommended") break } @@ -259,23 +258,20 @@ func loadNotifyMailFrom(rootCfg ConfigProvider) { log.Info("Notify Mail Service Enabled") } -func tryResolveAddr(addr string) []net.IPAddr { +func tryResolveAddr(addr string) []net.IP { if strings.HasPrefix(addr, "[") && strings.HasSuffix(addr, "]") { addr = addr[1 : len(addr)-1] } ip := net.ParseIP(addr) if ip != nil { - ips := make([]net.IPAddr, 1) - ips[0] = net.IPAddr{IP: ip} + ips := make([]net.IP, 1) + ips[0] = ip return ips } - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - ips, err := net.DefaultResolver.LookupIPAddr(ctx, addr) + ips, err := net.LookupIP(addr) if err != nil { log.Warn("could not look up mailer.SMTP_ADDR: %v", err) - return nil + return make([]net.IP, 0) } return ips } From 069148858686cc95fe4645865e4c4f3afa32bda4 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 17 May 2023 00:23:19 +0800 Subject: [PATCH 09/24] test buggy ini package --- modules/setting/config_provider.go | 24 +++++++++--- modules/setting/config_provider_test.go | 46 ++++++++++++++++++++++ modules/setting/log.go | 18 ++++++--- modules/setting/log_test.go | 51 +++++++++++++++++++++++-- 4 files changed, 125 insertions(+), 14 deletions(-) create mode 100644 modules/setting/config_provider_test.go diff --git a/modules/setting/config_provider.go b/modules/setting/config_provider.go index 1830715a79f8..40562f04022d 100644 --- a/modules/setting/config_provider.go +++ b/modules/setting/config_provider.go @@ -33,10 +33,10 @@ type ConfigProvider interface { Save() error } -// KeyInSection only searches the keys in the given section. +// ConfigSectionKey only searches the keys in the given section. // ini package has a special behavior: with "[sec] a=1" and an empty "[sec.sub]", // then in "[sec.sub]", Key()/HasKey() can always see "a=1" because it always tries parent sections. -func KeyInSection(sec ConfigSection, key string) *ini.Key { +func ConfigSectionKey(sec ConfigSection, key string) *ini.Key { if sec == nil { return nil } @@ -48,11 +48,25 @@ func KeyInSection(sec ConfigSection, key string) *ini.Key { return nil } -func KeyInSectionString(sec ConfigSection, key string) string { - k := KeyInSection(sec, key) - if k != nil { +func ConfigSectionKeyString(sec ConfigSection, key string, def ...string) string { + k := ConfigSectionKey(sec, key) + if k != nil && k.String() != "" { return k.String() } + if len(def) > 0 { + return def[0] + } + return "" +} + +func ConfigInheritedKeyString(sec ConfigSection, key string, def ...string) string { + k := sec.Key(key) + if k != nil && k.String() != "" { + return k.String() + } + if len(def) > 0 { + return def[0] + } return "" } diff --git a/modules/setting/config_provider_test.go b/modules/setting/config_provider_test.go new file mode 100644 index 000000000000..ceb9446c13f1 --- /dev/null +++ b/modules/setting/config_provider_test.go @@ -0,0 +1,46 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfigProviderBehaviors(t *testing.T) { + // buggy overwritten behavior + cfg, _ := NewConfigProviderFromData(` +[foo] +key = +`) + cfg.Section("foo.bar").Key("key").MustString("1") // try to read a key from subsection + assert.Equal(t, "1", cfg.Section("foo").Key("key").String()) // TODO: BUGGY! the key in [foo] is overwritten + + // subsection can see parent keys + cfg, _ = NewConfigProviderFromData(` +[foo] +key = 123 +`) + assert.Equal(t, "123", cfg.Section("foo.bar.xxx").Key("key").String()) +} + +func TestConfigProviderHelper(t *testing.T) { + cfg, _ := NewConfigProviderFromData(` +[foo] +empty = +key = 123 +`) + + assert.Equal(t, "def", ConfigSectionKeyString(cfg.Section("foo"), "empty", "def")) + + assert.NotNil(t, ConfigSectionKey(cfg.Section("foo"), "key")) + assert.Nil(t, ConfigSectionKey(cfg.Section("foo.bar"), "key")) + + assert.Equal(t, "123", ConfigSectionKeyString(cfg.Section("foo"), "key")) + assert.Equal(t, "", ConfigSectionKeyString(cfg.Section("foo.bar"), "key")) + assert.Equal(t, "def", ConfigSectionKeyString(cfg.Section("foo.bar"), "key", "def")) + + assert.Equal(t, "123", ConfigInheritedKeyString(cfg.Section("foo.bar"), "key")) +} diff --git a/modules/setting/log.go b/modules/setting/log.go index bfa8a13ed3d1..186cc65a3897 100644 --- a/modules/setting/log.go +++ b/modules/setting/log.go @@ -117,15 +117,21 @@ func loadLogModeByName(rootCfg ConfigProvider, loggerName, modeName string) (wri sec := rootCfg.Section("log." + modeName) writerMode = log.WriterMode{} - writerType = KeyInSectionString(sec, "MODE") + writerType = ConfigSectionKeyString(sec, "MODE") if writerType == "" { writerType = modeName } - writerMode.Level = log.LevelFromString(sec.Key("LEVEL").MustString(Log.Level.String())) - writerMode.StacktraceLevel = log.LevelFromString(sec.Key("STACKTRACE_LEVEL").MustString(Log.StacktraceLogLevel.String())) - writerMode.Prefix = sec.Key("PREFIX").MustString("") - writerMode.Flags = log.FlagsFromString(sec.Key("FLAGS").MustString("stdflags")) - writerMode.Expression = sec.Key("EXPRESSION").MustString("") + + writerMode.Level = log.LevelFromString(ConfigInheritedKeyString(sec, "LEVEL", Log.Level.String())) + writerMode.StacktraceLevel = log.LevelFromString(ConfigInheritedKeyString(sec, "STACKTRACE_LEVEL", Log.StacktraceLogLevel.String())) + writerMode.Prefix = ConfigInheritedKeyString(sec, "PREFIX") + writerMode.Expression = ConfigInheritedKeyString(sec, "EXPRESSION") + + defaultFlags := "stdflags" + if loggerName == "access" { + defaultFlags = "none" // "access" logger is special, by default it doesn't have output flags + } + writerMode.Flags = log.FlagsFromString(ConfigInheritedKeyString(sec, "FLAGS", defaultFlags)) switch writerType { case "console": diff --git a/modules/setting/log_test.go b/modules/setting/log_test.go index 71fe0bb15569..fc1db8f38725 100644 --- a/modules/setting/log_test.go +++ b/modules/setting/log_test.go @@ -185,12 +185,35 @@ ACCESS = file "WriterType": "file" } } +` + writerDumpAccess := ` +{ + "file": { + "BufferLen": 10000, + "Colorize": false, + "Expression": "", + "Flags": "none", + "Level": "info", + "Prefix": "", + "StacktraceLevel": "none", + "WriterOption": { + "Compress": true, + "CompressionLevel": -1, + "DailyRotate": true, + "FileName": "$FILENAME", + "LogRotate": true, + "MaxDays": 7, + "MaxSize": 268435456 + }, + "WriterType": "file" + } +} ` dump := manager.GetLogger(log.DEFAULT).DumpWriters() require.JSONEq(t, strings.ReplaceAll(writerDump, "$FILENAME", tempPath("gitea.log")), toJSON(dump)) dump = manager.GetLogger("access").DumpWriters() - require.JSONEq(t, strings.ReplaceAll(writerDump, "$FILENAME", tempPath("access.log")), toJSON(dump)) + require.JSONEq(t, strings.ReplaceAll(writerDumpAccess, "$FILENAME", tempPath("access.log")), toJSON(dump)) dump = manager.GetLogger("router").DumpWriters() require.JSONEq(t, strings.ReplaceAll(writerDump, "$FILENAME", tempPath("router.log")), toJSON(dump)) @@ -216,6 +239,7 @@ ENABLE_ACCESS_LOG = false func TestLogConfigNewConfig(t *testing.T) { manager, managerClose := initLoggersByConfig(t, ` [log] +logger.access.MODE = console logger.xorm.MODE = console, console-1 [log.console] @@ -227,6 +251,8 @@ LEVEL = error STDERR = true `) + defer managerClose() + writerDump := ` { "console": { @@ -257,10 +283,29 @@ STDERR = true } } ` - defer managerClose() - + writerDumpAccess := ` +{ + "console": { + "BufferLen": 10000, + "Colorize": false, + "Expression": "", + "Flags": "none", + "Level": "warn", + "Prefix": "", + "StacktraceLevel": "none", + "WriterOption": { + "Stderr": false + }, + "WriterType": "console" + } +} +` dump := manager.GetLogger("xorm").DumpWriters() require.JSONEq(t, writerDump, toJSON(dump)) + + dump = manager.GetLogger("access").DumpWriters() + require.JSONEq(t, writerDumpAccess, toJSON(dump)) + } func TestLogConfigModeFile(t *testing.T) { From 090a803819802f6ae36ea191e4aa1bc76c84f5f1 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 17 May 2023 00:25:59 +0800 Subject: [PATCH 10/24] doc & lint --- docs/content/doc/administration/logging-config.en-us.md | 2 +- modules/setting/log_test.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/content/doc/administration/logging-config.en-us.md b/docs/content/doc/administration/logging-config.en-us.md index 5f3ca30c9a2d..5897b1ff16bb 100644 --- a/docs/content/doc/administration/logging-config.en-us.md +++ b/docs/content/doc/administration/logging-config.en-us.md @@ -215,7 +215,7 @@ Settings: The Router logger logs the following message types when Gitea's route handlers work: - `started` messages will be logged at TRACE level -- `polling`/`completed` routers will be logged at INFO +- `polling`/`completed` routers will be logged at INFO. Exception: "/assets" static resource requests are also logged at TRACE. - `slow` routers will be logged at WARN - `failed` routers will be logged at WARN diff --git a/modules/setting/log_test.go b/modules/setting/log_test.go index fc1db8f38725..7211db7b55be 100644 --- a/modules/setting/log_test.go +++ b/modules/setting/log_test.go @@ -305,7 +305,6 @@ STDERR = true dump = manager.GetLogger("access").DumpWriters() require.JSONEq(t, writerDumpAccess, toJSON(dump)) - } func TestLogConfigModeFile(t *testing.T) { From fe67dc4a36a5342eef34868c270d2f2991c391b9 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 17 May 2023 12:11:02 +0800 Subject: [PATCH 11/24] improve rotatingfilewriter --- modules/log/init.go | 3 ++ modules/log/logger_global.go | 4 +- modules/util/rotatingfilewriter/writer.go | 50 +++++++++++++++-------- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/modules/log/init.go b/modules/log/init.go index 3508039060b5..798ba86410d2 100644 --- a/modules/log/init.go +++ b/modules/log/init.go @@ -8,6 +8,7 @@ import ( "strings" "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/util/rotatingfilewriter" ) var projectPackagePrefix string @@ -20,6 +21,8 @@ func init() { panic("unable to detect correct package prefix, please update file: " + filename) } + rotatingfilewriter.ErrorPrintf = FallbackErrorf + process.Trace = func(start bool, pid process.IDType, description string, parentPID process.IDType, typ string) { if start && parentPID != "" { Log(1, TRACE, "Start %s: %s (from %s) (%s)", NewColoredValue(pid, FgHiYellow), description, NewColoredValue(parentPID, FgYellow), NewColoredValue(typ, Reset)) diff --git a/modules/log/logger_global.go b/modules/log/logger_global.go index b08c4224939d..f100341254a5 100644 --- a/modules/log/logger_global.go +++ b/modules/log/logger_global.go @@ -8,9 +8,9 @@ import ( "os" ) +// FallbackErrorf is the last chance to show an error if the logger has internal errors func FallbackErrorf(format string, args ...any) { - s := fmt.Sprintf(format, args...) - _, _ = fmt.Fprintln(os.Stderr, s) + _, _ = fmt.Fprintf(os.Stderr, format+"\n", args) } func GetLevel() Level { diff --git a/modules/util/rotatingfilewriter/writer.go b/modules/util/rotatingfilewriter/writer.go index a38e8ba3cd8b..0fc24093cdb6 100644 --- a/modules/util/rotatingfilewriter/writer.go +++ b/modules/util/rotatingfilewriter/writer.go @@ -39,6 +39,15 @@ type RotatingFileWriter struct { cancelReleaseReopen func() } +var ErrorPrintf func(format string, args ...interface{}) + +// errorf tries to print error messages. Since this writer could be used by a logger system, this is the last chance to show the error in some cases +func errorf(format string, args ...interface{}) { + if ErrorPrintf != nil { + ErrorPrintf("rotatingfilewriter: "+format+"\n", args...) + } +} + // Open creates a new rotating file writer. // Notice: if a file is opened by two rotators, there will be conflicts when rotating. // In the future, there should be "rotating file manager" @@ -62,10 +71,8 @@ func Open(filename string, options *Options) (*RotatingFileWriter, error) { func (rfw *RotatingFileWriter) Write(b []byte) (int, error) { if rfw.options.Rotate && ((rfw.options.MaximumSize > 0 && rfw.currentSize >= rfw.options.MaximumSize) || (rfw.options.RotateDaily && time.Now().Day() != rfw.openDate)) { if err := rfw.DoRotate(); err != nil { - // This should be - // return 0, err - // but the old behaviour does not return. This may lead to other errors. - fmt.Fprintf(os.Stderr, "RotatingFileWriter: %s\n", err) + // if this writer is used by a logger system, it's the logger system's responsibility to handle/show the error + return 0, err } } @@ -150,7 +157,7 @@ func (rfw *RotatingFileWriter) DoRotate() error { } if rfw.options.Compress { - go compressOldFile(fname, rfw.options.CompressionLevel) //nolint:errcheck + go compressOldFile(fname, rfw.options.CompressionLevel) } if err := rfw.open(fd.Name()); err != nil { @@ -166,40 +173,46 @@ func (rfw *RotatingFileWriter) DoRotate() error { return nil } -func compressOldFile(fname string, compressionLevel int) error { +func compressOldFile(fname string, compressionLevel int) { reader, err := os.Open(fname) if err != nil { - return err + errorf("compressOldFile: failed to open existing file %s: %v", fname, err) + return } defer reader.Close() buffer := bufio.NewReader(reader) - fw, err := os.OpenFile(fname+".gz", os.O_WRONLY|os.O_CREATE, 0o660) + fnameGz := fname + ".gz" + fw, err := os.OpenFile(fnameGz, os.O_WRONLY|os.O_CREATE, 0o660) if err != nil { - return err + errorf("compressOldFile: failed to open new file %s: %v", fnameGz, err) + return } defer fw.Close() zw, err := gzip.NewWriterLevel(fw, compressionLevel) if err != nil { - return err + errorf("compressOldFile: failed to create gzip writer: %v", err) + return } defer zw.Close() _, err = buffer.WriteTo(zw) if err != nil { - zw.Close() - fw.Close() - util.Remove(fname + ".gz") //nolint:errcheck - return err + _ = zw.Close() + _ = fw.Close() + _ = util.Remove(fname + ".gz") + errorf("compressOldFile: failed to write to gz file: %v", err) + return } - reader.Close() + _ = reader.Close() - return util.Remove(fname) + err = util.Remove(fname) + errorf("compressOldFile: failed to delete old file: %v", err) } func deleteOldFiles(dir, prefix string, removeBefore time.Time) { - _ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) (returnErr error) { + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) (returnErr error) { defer func() { if r := recover(); r != nil { returnErr = fmt.Errorf("unable to delete old file '%s', error: %+v", path, r) @@ -223,4 +236,7 @@ func deleteOldFiles(dir, prefix string, removeBefore time.Time) { } return nil }) + if err != nil { + errorf("deleteOldFiles: failed to delete old file: %v", err) + } } From 63e169a03a82bb67589c8887767bf62d2e841231 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 17 May 2023 13:18:20 +0800 Subject: [PATCH 12/24] clarify ini package behavior --- modules/setting/config_provider.go | 17 ++++++++- modules/setting/config_provider_test.go | 48 +++++++++++++++++-------- modules/setting/log.go | 28 +++++++-------- 3 files changed, 64 insertions(+), 29 deletions(-) diff --git a/modules/setting/config_provider.go b/modules/setting/config_provider.go index 40562f04022d..37f5754ffdb0 100644 --- a/modules/setting/config_provider.go +++ b/modules/setting/config_provider.go @@ -33,9 +33,10 @@ type ConfigProvider interface { Save() error } -// ConfigSectionKey only searches the keys in the given section. +// ConfigSectionKey only searches the keys in the given section, but it is O(n). // ini package has a special behavior: with "[sec] a=1" and an empty "[sec.sub]", // then in "[sec.sub]", Key()/HasKey() can always see "a=1" because it always tries parent sections. +// It returns nil if the key doesn't exist. func ConfigSectionKey(sec ConfigSection, key string) *ini.Key { if sec == nil { return nil @@ -59,6 +60,20 @@ func ConfigSectionKeyString(sec ConfigSection, key string, def ...string) string return "" } +// ConfigInheritedKey works like ini.Section.Key(), but it always returns a new key instance, it is O(n) because NewKey is O(n) +// and the returned key is safe to be used with "MustXxx", it doesn't change the parent's values. +// Otherwise, ini.Section.Key().MustXxx would pollute the parent section's keys. +// It never returns nil. +func ConfigInheritedKey(sec ConfigSection, key string) *ini.Key { + k := sec.Key(key) + if k != nil && k.String() != "" { + newKey, _ := sec.NewKey(k.Name(), k.String()) + return newKey + } + newKey, _ := sec.NewKey(key, "") + return newKey +} + func ConfigInheritedKeyString(sec ConfigSection, key string, def ...string) string { k := sec.Key(key) if k != nil && k.String() != "" { diff --git a/modules/setting/config_provider_test.go b/modules/setting/config_provider_test.go index ceb9446c13f1..76f7048d59c9 100644 --- a/modules/setting/config_provider_test.go +++ b/modules/setting/config_provider_test.go @@ -10,20 +10,25 @@ import ( ) func TestConfigProviderBehaviors(t *testing.T) { - // buggy overwritten behavior - cfg, _ := NewConfigProviderFromData(` + t.Run("BuggyKeyOverwritten", func(t *testing.T) { + cfg, _ := NewConfigProviderFromData(` [foo] key = `) - cfg.Section("foo.bar").Key("key").MustString("1") // try to read a key from subsection - assert.Equal(t, "1", cfg.Section("foo").Key("key").String()) // TODO: BUGGY! the key in [foo] is overwritten + sec := cfg.Section("foo") + secSub := cfg.Section("foo.bar") + secSub.Key("key").MustString("1") // try to read a key from subsection + assert.Equal(t, "1", sec.Key("key").String()) // TODO: BUGGY! the key in [foo] is overwritten + }) - // subsection can see parent keys - cfg, _ = NewConfigProviderFromData(` + t.Run("SubsectionSeeParentKeys", func(t *testing.T) { + cfg, _ := NewConfigProviderFromData(` [foo] key = 123 `) - assert.Equal(t, "123", cfg.Section("foo.bar.xxx").Key("key").String()) + secSub := cfg.Section("foo.bar.xxx") + assert.Equal(t, "123", secSub.Key("key").String()) + }) } func TestConfigProviderHelper(t *testing.T) { @@ -33,14 +38,29 @@ empty = key = 123 `) - assert.Equal(t, "def", ConfigSectionKeyString(cfg.Section("foo"), "empty", "def")) + sec := cfg.Section("foo") + secSub := cfg.Section("foo.bar") + + // test empty key + assert.Equal(t, "def", ConfigSectionKeyString(sec, "empty", "def")) + assert.Equal(t, "xyz", ConfigSectionKeyString(secSub, "empty", "xyz")) + + // test non-inherited key, only see the keys in current section + assert.NotNil(t, ConfigSectionKey(sec, "key")) + assert.Nil(t, ConfigSectionKey(secSub, "key")) - assert.NotNil(t, ConfigSectionKey(cfg.Section("foo"), "key")) - assert.Nil(t, ConfigSectionKey(cfg.Section("foo.bar"), "key")) + // test default behavior + assert.Equal(t, "123", ConfigSectionKeyString(sec, "key")) + assert.Equal(t, "", ConfigSectionKeyString(secSub, "key")) + assert.Equal(t, "def", ConfigSectionKeyString(secSub, "key", "def")) - assert.Equal(t, "123", ConfigSectionKeyString(cfg.Section("foo"), "key")) - assert.Equal(t, "", ConfigSectionKeyString(cfg.Section("foo.bar"), "key")) - assert.Equal(t, "def", ConfigSectionKeyString(cfg.Section("foo.bar"), "key", "def")) + assert.Equal(t, "123", ConfigInheritedKeyString(secSub, "key")) - assert.Equal(t, "123", ConfigInheritedKeyString(cfg.Section("foo.bar"), "key")) + // Workaround for ini package's BuggyKeyOverwritten behavior + assert.Equal(t, "", ConfigSectionKeyString(sec, "empty")) + assert.Equal(t, "", ConfigSectionKeyString(secSub, "empty")) + assert.Equal(t, "def", ConfigInheritedKey(secSub, "empty").MustString("def")) + assert.Equal(t, "def", ConfigInheritedKey(secSub, "empty").MustString("xyz")) + assert.Equal(t, "", ConfigSectionKeyString(sec, "empty")) + assert.Equal(t, "def", ConfigSectionKeyString(secSub, "empty")) } diff --git a/modules/setting/log.go b/modules/setting/log.go index 186cc65a3897..4c62c749efea 100644 --- a/modules/setting/log.go +++ b/modules/setting/log.go @@ -135,31 +135,31 @@ func loadLogModeByName(rootCfg ConfigProvider, loggerName, modeName string) (wri switch writerType { case "console": - useStderr := sec.Key("STDERR").MustBool(false) + useStderr := ConfigInheritedKey(sec, "STDERR").MustBool(false) writerOption := log.WriterConsoleOption{Stderr: useStderr} if useStderr { - writerMode.Colorize = sec.Key("COLORIZE").MustBool(log.CanColorStderr) + writerMode.Colorize = ConfigInheritedKey(sec, "COLORIZE").MustBool(log.CanColorStderr) } else { - writerMode.Colorize = sec.Key("COLORIZE").MustBool(log.CanColorStdout) + writerMode.Colorize = ConfigInheritedKey(sec, "COLORIZE").MustBool(log.CanColorStdout) } writerMode.WriterOption = writerOption case "file": - fileName := LogPrepareFilenameForWriter(loggerName, sec.Key("FILE_NAME").String()) + fileName := LogPrepareFilenameForWriter(loggerName, ConfigInheritedKey(sec, "FILE_NAME").String()) writerOption := log.WriterFileOption{} writerOption.FileName = fileName + filenameSuffix // FIXME: the suffix doesn't seem right, see its related comments - writerOption.LogRotate = sec.Key("LOG_ROTATE").MustBool(true) - writerOption.MaxSize = 1 << uint(sec.Key("MAX_SIZE_SHIFT").MustInt(28)) - writerOption.DailyRotate = sec.Key("DAILY_ROTATE").MustBool(true) - writerOption.MaxDays = sec.Key("MAX_DAYS").MustInt(7) - writerOption.Compress = sec.Key("COMPRESS").MustBool(true) - writerOption.CompressionLevel = sec.Key("COMPRESSION_LEVEL").MustInt(-1) + writerOption.LogRotate = ConfigInheritedKey(sec, "LOG_ROTATE").MustBool(true) + writerOption.MaxSize = 1 << uint(ConfigInheritedKey(sec, "MAX_SIZE_SHIFT").MustInt(28)) + writerOption.DailyRotate = ConfigInheritedKey(sec, "DAILY_ROTATE").MustBool(true) + writerOption.MaxDays = ConfigInheritedKey(sec, "MAX_DAYS").MustInt(7) + writerOption.Compress = ConfigInheritedKey(sec, "COMPRESS").MustBool(true) + writerOption.CompressionLevel = ConfigInheritedKey(sec, "COMPRESSION_LEVEL").MustInt(-1) writerMode.WriterOption = writerOption case "conn": writerOption := log.WriterConnOption{} - writerOption.ReconnectOnMsg = sec.Key("RECONNECT_ON_MSG").MustBool() - writerOption.Reconnect = sec.Key("RECONNECT").MustBool() - writerOption.Protocol = sec.Key("PROTOCOL").In("tcp", []string{"tcp", "unix", "udp"}) - writerOption.Addr = sec.Key("ADDR").MustString(":7020") + writerOption.ReconnectOnMsg = ConfigInheritedKey(sec, "RECONNECT_ON_MSG").MustBool() + writerOption.Reconnect = ConfigInheritedKey(sec, "RECONNECT").MustBool() + writerOption.Protocol = ConfigInheritedKey(sec, "PROTOCOL").In("tcp", []string{"tcp", "unix", "udp"}) + writerOption.Addr = ConfigInheritedKey(sec, "ADDR").MustString(":7020") writerMode.WriterOption = writerOption default: if !log.HasEventWriter(writerType) { From 8b6b2d3b415796081951eeac6d5ab89a1201b2d7 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 17 May 2023 13:27:37 +0800 Subject: [PATCH 13/24] fix rotatingfilewriter error handling --- modules/util/rotatingfilewriter/writer.go | 26 +++++++++++++---------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/modules/util/rotatingfilewriter/writer.go b/modules/util/rotatingfilewriter/writer.go index 0fc24093cdb6..65e1b3b22bb7 100644 --- a/modules/util/rotatingfilewriter/writer.go +++ b/modules/util/rotatingfilewriter/writer.go @@ -157,7 +157,12 @@ func (rfw *RotatingFileWriter) DoRotate() error { } if rfw.options.Compress { - go compressOldFile(fname, rfw.options.CompressionLevel) + go func() { + err := compressOldFile(fname, rfw.options.CompressionLevel) + if err != nil { + errorf("DoRotate: %v", err) + } + }() } if err := rfw.open(fd.Name()); err != nil { @@ -173,11 +178,10 @@ func (rfw *RotatingFileWriter) DoRotate() error { return nil } -func compressOldFile(fname string, compressionLevel int) { +func compressOldFile(fname string, compressionLevel int) error { reader, err := os.Open(fname) if err != nil { - errorf("compressOldFile: failed to open existing file %s: %v", fname, err) - return + return fmt.Errorf("compressOldFile: failed to open existing file %s: %w", fname, err) } defer reader.Close() @@ -185,15 +189,13 @@ func compressOldFile(fname string, compressionLevel int) { fnameGz := fname + ".gz" fw, err := os.OpenFile(fnameGz, os.O_WRONLY|os.O_CREATE, 0o660) if err != nil { - errorf("compressOldFile: failed to open new file %s: %v", fnameGz, err) - return + return fmt.Errorf("compressOldFile: failed to open new file %s: %w", fnameGz, err) } defer fw.Close() zw, err := gzip.NewWriterLevel(fw, compressionLevel) if err != nil { - errorf("compressOldFile: failed to create gzip writer: %v", err) - return + return fmt.Errorf("compressOldFile: failed to create gzip writer: %w", err) } defer zw.Close() @@ -202,13 +204,15 @@ func compressOldFile(fname string, compressionLevel int) { _ = zw.Close() _ = fw.Close() _ = util.Remove(fname + ".gz") - errorf("compressOldFile: failed to write to gz file: %v", err) - return + return fmt.Errorf("compressOldFile: failed to write to gz file: %w", err) } _ = reader.Close() err = util.Remove(fname) - errorf("compressOldFile: failed to delete old file: %v", err) + if err != nil { + return fmt.Errorf("compressOldFile: failed to delete old file: %w", err) + } + return nil } func deleteOldFiles(dir, prefix string, removeBefore time.Time) { From ac094e3cc543f99282d9a67d35b1454d2bc7414e Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 17 May 2023 15:42:31 +0800 Subject: [PATCH 14/24] clarify access log behavior, make event writer shareable, improve tests --- .../config-cheat-sheet.en-us.md | 2 +- .../administration/logging-config.en-us.md | 2 +- modules/log/event_writer.go | 2 +- modules/log/event_writer_base.go | 29 ++++--- modules/log/event_writer_conn.go | 6 +- modules/log/event_writer_conn_test.go | 3 +- modules/log/logger_impl.go | 44 ++++------- modules/log/logger_test.go | 15 ++-- modules/log/manager.go | 77 ++++++++++++++----- modules/log/manager_test.go | 42 ++++++++++ modules/setting/log.go | 52 +++++++------ modules/setting/log_test.go | 15 ++-- routers/private/manager.go | 2 +- 13 files changed, 183 insertions(+), 108 deletions(-) create mode 100644 modules/log/manager_test.go diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index 1e6e1535f727..c8d338991d59 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -876,7 +876,7 @@ Default templates for project boards: ### File log mode (`log.file`, or `MODE=file`) -- `FILE_NAME`: Set the file name for this logger. Defaults as described above. If relative will be relative to the `ROOT_PATH` +- `FILE_NAME`: Set the file name for this logger. Defaults to "gitea.log" (exception: access log defaults to "access.log"). If relative will be relative to the `ROOT_PATH` - `LOG_ROTATE`: **true**: Rotate the log files. - `MAX_SIZE_SHIFT`: **28**: Maximum size shift of a single file, 28 represents 256Mb. - `DAILY_ROTATE`: **true**: Rotate logs daily. diff --git a/docs/content/doc/administration/logging-config.en-us.md b/docs/content/doc/administration/logging-config.en-us.md index 5897b1ff16bb..53a914e10273 100644 --- a/docs/content/doc/administration/logging-config.en-us.md +++ b/docs/content/doc/administration/logging-config.en-us.md @@ -180,7 +180,7 @@ In this mode the logger will save log messages to a file. Settings: -- `FILE_NAME`: The file to write the log events to. Default to `%(ROOT_PATH)/gitea.log` if the section name is `log.file`, or `%(ROOT_PATH)/.log` otherwise. +- `FILE_NAME`: The file to write the log events to. Default to `%(ROOT_PATH)/gitea.log`. Exception: access log will default to `%(ROOT_PATH)/access.log`. - `MAX_SIZE_SHIFT`: **28**: Maximum size shift of a single file. 28 represents 256Mb. For details see below. - `LOG_ROTATE` **true**: Whether to rotate the log files. TODO: if false, will it delete instead on daily rotate, or do nothing?. - `DAILY_ROTATE`: **true**: Whether to rotate logs daily. diff --git a/modules/log/event_writer.go b/modules/log/event_writer.go index 3ca729ff7d95..c76b208c78db 100644 --- a/modules/log/event_writer.go +++ b/modules/log/event_writer.go @@ -11,7 +11,7 @@ type EventWriter interface { EventWriterBase } -type EventWriterProvider func(name string, mode WriterMode) EventWriter +type EventWriterProvider func(writerName string, writerMode WriterMode) EventWriter var eventWriterProviders = map[string]EventWriterProvider{} diff --git a/modules/log/event_writer_base.go b/modules/log/event_writer_base.go index 3b56df33c2e9..e3467c2ac679 100644 --- a/modules/log/event_writer_base.go +++ b/modules/log/event_writer_base.go @@ -21,8 +21,6 @@ type EventWriterBase interface { } type EventWriterBaseImpl struct { - LoggerImpl *LoggerImpl - writerType string Name string @@ -31,7 +29,9 @@ type EventWriterBaseImpl struct { FormatMessage EventFormatter // format the Event to a message and write it to output OutputWriteCloser io.WriteCloser // it will be closed when the event writer is stopped + GetPauseChan func() chan struct{} + shared bool stopped chan struct{} } @@ -65,14 +65,17 @@ func (b *EventWriterBaseImpl) Run(ctx context.Context) { } for { - pause := b.LoggerImpl.GetPauseChan() - if pause != nil { - select { - case <-pause: - case <-ctx.Done(): - return + if b.GetPauseChan != nil { + pause := b.GetPauseChan() + if pause != nil { + select { + case <-pause: + case <-ctx.Done(): + return + } } } + select { case <-ctx.Done(): return @@ -124,14 +127,18 @@ func NewEventWriterBase(name, writerType string, mode WriterMode) *EventWriterBa Mode: &mode, Queue: make(chan *EventFormatted, mode.BufferLen), + GetPauseChan: GetManager().GetPauseChan, // by default, use the global pause channel FormatMessage: EventFormatTextMessage, - - stopped: make(chan struct{}), } return b } -func eventWriterStartGo(ctx context.Context, w EventWriter) { +func eventWriterStartGo(ctx context.Context, w EventWriter, shared bool) { + if w.Base().stopped != nil { + return // already started + } + w.Base().shared = shared + w.Base().stopped = make(chan struct{}) go func() { defer close(w.Base().stopped) w.Run(ctx) diff --git a/modules/log/event_writer_conn.go b/modules/log/event_writer_conn.go index b52481986d71..022206aa4d51 100644 --- a/modules/log/event_writer_conn.go +++ b/modules/log/event_writer_conn.go @@ -22,9 +22,9 @@ type eventWriterConn struct { var _ EventWriter = (*eventWriterConn)(nil) -func NewEventWriterConn(name string, mode WriterMode) EventWriter { - w := &eventWriterConn{EventWriterBaseImpl: NewEventWriterBase(name, "conn", mode)} - opt := mode.WriterOption.(WriterConnOption) +func NewEventWriterConn(writerName string, writerMode WriterMode) EventWriter { + w := &eventWriterConn{EventWriterBaseImpl: NewEventWriterBase(writerName, "conn", writerMode)} + opt := writerMode.WriterOption.(WriterConnOption) w.connWriter = connWriter{ ReconnectOnMsg: opt.ReconnectOnMsg, Reconnect: opt.Reconnect, diff --git a/modules/log/event_writer_conn_test.go b/modules/log/event_writer_conn_test.go index 30280cb1c55d..e08ec025a328 100644 --- a/modules/log/event_writer_conn_test.go +++ b/modules/log/event_writer_conn_test.go @@ -4,6 +4,7 @@ package log import ( + "context" "fmt" "io" "net" @@ -39,7 +40,7 @@ func TestConnLogger(t *testing.T) { level := INFO flags := LstdFlags | LUTC | Lfuncname - logger := NewLoggerWithWriters(NewEventWriterConn("test-conn", WriterMode{ + logger := NewLoggerWithWriters(context.Background(), NewEventWriterConn("test-conn", WriterMode{ Level: level, Prefix: prefix, Flags: FlagsFromBits(flags), diff --git a/modules/log/logger_impl.go b/modules/log/logger_impl.go index eb55d845754e..f9bb897a8063 100644 --- a/modules/log/logger_impl.go +++ b/modules/log/logger_impl.go @@ -26,9 +26,6 @@ type LoggerImpl struct { eventWriterMu sync.RWMutex eventWriters map[string]EventWriter - - pauseMu sync.RWMutex - pauseChan chan struct{} } var ( @@ -85,21 +82,26 @@ func (l *LoggerImpl) syncLevelInternal() { l.stacktraceLevel.Store(int32(lowestLevel)) } +func (l *LoggerImpl) removeWriterInternal(w EventWriter) { + if !w.Base().shared { + eventWriterStopWait(w) // only stop non-shared writers, shared writers are managed by the manager + } + delete(l.eventWriters, w.GetWriterName()) +} + func (l *LoggerImpl) AddWriters(writer ...EventWriter) { l.eventWriterMu.Lock() defer l.eventWriterMu.Unlock() for _, w := range writer { if old, ok := l.eventWriters[w.GetWriterName()]; ok { - eventWriterStopWait(old) - delete(l.eventWriters, old.GetWriterName()) + l.removeWriterInternal(old) } } for _, w := range writer { l.eventWriters[w.GetWriterName()] = w - w.Base().LoggerImpl = l - eventWriterStartGo(l.ctx, w) + eventWriterStartGo(l.ctx, w, false) } l.syncLevelInternal() @@ -114,8 +116,7 @@ func (l *LoggerImpl) RemoveWriter(modeName string) error { return util.ErrNotExist } - eventWriterStopWait(w) - delete(l.eventWriters, w.GetWriterName()) + l.removeWriterInternal(w) l.syncLevelInternal() return nil } @@ -125,7 +126,7 @@ func (l *LoggerImpl) RemoveAllWriters() *LoggerImpl { defer l.eventWriterMu.Unlock() for _, w := range l.eventWriters { - eventWriterStopWait(w) + l.removeWriterInternal(w) } l.eventWriters = map[string]EventWriter{} l.syncLevelInternal() @@ -151,30 +152,11 @@ func (l *LoggerImpl) DumpWriters() map[string]any { return writers } -func (l *LoggerImpl) Pause() { - l.pauseMu.Lock() - l.pauseChan = make(chan struct{}) - l.pauseMu.Unlock() -} - -func (l *LoggerImpl) Resume() { - l.pauseMu.Lock() - close(l.pauseChan) - l.pauseChan = nil - l.pauseMu.Unlock() -} - func (l *LoggerImpl) Close() { l.RemoveAllWriters() l.ctxCancel() } -func (l *LoggerImpl) GetPauseChan() chan struct{} { - l.pauseMu.RLock() - defer l.pauseMu.RUnlock() - return l.pauseChan -} - func (l *LoggerImpl) IsEnabled() bool { l.eventWriterMu.RLock() defer l.eventWriterMu.RUnlock() @@ -235,9 +217,9 @@ func (l *LoggerImpl) GetLevel() Level { return Level(l.level.Load()) } -func NewLoggerWithWriters(writer ...EventWriter) *LoggerImpl { +func NewLoggerWithWriters(ctx context.Context, writer ...EventWriter) *LoggerImpl { l := &LoggerImpl{} - l.ctx, l.ctxCancel = context.WithCancel(context.Background()) + l.ctx, l.ctxCancel = context.WithCancel(ctx) l.LevelLogger = BaseLoggerToGeneralLogger(l) l.eventWriters = map[string]EventWriter{} l.syncLevelInternal() diff --git a/modules/log/logger_test.go b/modules/log/logger_test.go index fd7ebe67ca82..f8b072e304c2 100644 --- a/modules/log/logger_test.go +++ b/modules/log/logger_test.go @@ -4,6 +4,7 @@ package log import ( + "context" "sync" "testing" "time" @@ -52,7 +53,7 @@ func newDummyWriter(name string, level Level, delay time.Duration) *dummyWriter } func TestLogger(t *testing.T) { - logger := NewLoggerWithWriters() + logger := NewLoggerWithWriters(context.Background()) dump := logger.DumpWriters() assert.EqualValues(t, 0, len(dump)) @@ -87,18 +88,18 @@ func TestLogger(t *testing.T) { } func TestLoggerPause(t *testing.T) { - logger := NewLoggerWithWriters() + logger := NewLoggerWithWriters(context.Background()) w1 := newDummyWriter("dummy-1", DEBUG, 0) logger.AddWriters(w1) - logger.Pause() + GetManager().PauseAll() - time.Sleep(100 * time.Millisecond) logger.Info("info-level") + time.Sleep(100 * time.Millisecond) assert.Equal(t, []string{}, w1.GetLogs()) - logger.Resume() + GetManager().ResumeAll() time.Sleep(100 * time.Millisecond) assert.Equal(t, []string{"info-level\n"}, w1.GetLogs()) @@ -113,7 +114,7 @@ func (t testLogString) LogString() string { } func TestLoggerLogString(t *testing.T) { - logger := NewLoggerWithWriters() + logger := NewLoggerWithWriters(context.Background()) w1 := newDummyWriter("dummy-1", DEBUG, 0) w1.Mode.Colorize = true @@ -126,7 +127,7 @@ func TestLoggerLogString(t *testing.T) { } func TestLoggerExpressionFilter(t *testing.T) { - logger := NewLoggerWithWriters() + logger := NewLoggerWithWriters(context.Background()) w1 := newDummyWriter("dummy-1", DEBUG, 0) w1.Mode.Expression = "foo.*" diff --git a/modules/log/manager.go b/modules/log/manager.go index 2572afb22345..c6cda3bd1874 100644 --- a/modules/log/manager.go +++ b/modules/log/manager.go @@ -4,6 +4,8 @@ package log import ( + "context" + "fmt" "sync" "sync/atomic" ) @@ -11,9 +13,16 @@ import ( const DEFAULT = "default" type LoggerManager struct { + ctx context.Context + ctxCancel context.CancelFunc + mu sync.Mutex + writers map[string]EventWriter loggers map[string]*LoggerImpl defaultLogger atomic.Pointer[LoggerImpl] + + pauseMu sync.RWMutex + pauseChan chan struct{} } func (m *LoggerManager) GetLogger(name string) *LoggerImpl { @@ -28,7 +37,7 @@ func (m *LoggerManager) GetLogger(name string) *LoggerImpl { logger := m.loggers[name] if logger == nil { - logger = NewLoggerWithWriters() + logger = NewLoggerWithWriters(m.ctx) m.loggers[name] = logger if name == DEFAULT { m.defaultLogger.Store(logger) @@ -39,21 +48,22 @@ func (m *LoggerManager) GetLogger(name string) *LoggerImpl { } func (m *LoggerManager) PauseAll() { - m.mu.Lock() - defer m.mu.Unlock() - - for _, logger := range m.loggers { - logger.Pause() - } + m.pauseMu.Lock() + m.pauseChan = make(chan struct{}) + m.pauseMu.Unlock() } func (m *LoggerManager) ResumeAll() { - m.mu.Lock() - defer m.mu.Unlock() + m.pauseMu.Lock() + close(m.pauseChan) + m.pauseChan = nil + m.pauseMu.Unlock() +} - for _, logger := range m.loggers { - logger.Resume() - } +func (m *LoggerManager) GetPauseChan() chan struct{} { + m.pauseMu.RLock() + defer m.pauseMu.RUnlock() + return m.pauseChan } func (m *LoggerManager) Close() { @@ -63,6 +73,12 @@ func (m *LoggerManager) Close() { for _, logger := range m.loggers { logger.Close() } + m.loggers = map[string]*LoggerImpl{} + + for _, writer := range m.writers { + eventWriterStopWait(writer) + } + m.writers = map[string]EventWriter{} } func (m *LoggerManager) DumpLoggers() map[string]any { @@ -71,23 +87,46 @@ func (m *LoggerManager) DumpLoggers() map[string]any { dump := map[string]any{} for name, logger := range m.loggers { - m := map[string]any{ + loggerDump := map[string]any{ "IsEnabled": logger.IsEnabled(), "EventWriters": logger.DumpWriters(), } - dump[name] = m + dump[name] = loggerDump } return dump } -var manager = NewManager() +func (m *LoggerManager) GetSharedWriter(writerName string) EventWriter { + m.mu.Lock() + defer m.mu.Unlock() + return m.writers[writerName] +} + +func (m *LoggerManager) NewSharedWriter(writerName, writerType string, mode WriterMode) (writer EventWriter, err error) { + m.mu.Lock() + defer m.mu.Unlock() + + if _, ok := m.writers[writerName]; ok { + return nil, fmt.Errorf("log event writer %q has been added before", writerName) + } + + if writer, err = NewEventWriter(writerName, writerType, mode); err != nil { + return nil, err + } + + m.writers[writerName] = writer + eventWriterStartGo(m.ctx, writer, true) + return writer, nil +} + +var loggerManager = NewManager() func GetManager() *LoggerManager { - return manager + return loggerManager } func NewManager() *LoggerManager { - return &LoggerManager{ - loggers: map[string]*LoggerImpl{}, - } + m := &LoggerManager{writers: map[string]EventWriter{}, loggers: map[string]*LoggerImpl{}} + m.ctx, m.ctxCancel = context.WithCancel(context.Background()) + return m } diff --git a/modules/log/manager_test.go b/modules/log/manager_test.go new file mode 100644 index 000000000000..aa01f79980c5 --- /dev/null +++ b/modules/log/manager_test.go @@ -0,0 +1,42 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package log + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSharedWorker(t *testing.T) { + RegisterEventWriter("dummy", func(writerName string, writerMode WriterMode) EventWriter { + return newDummyWriter(writerName, writerMode.Level, 0) + }) + + m := NewManager() + _, err := m.NewSharedWriter("dummy-1", "dummy", WriterMode{Level: DEBUG, Flags: FlagsFromBits(0)}) + assert.NoError(t, err) + + w := m.GetSharedWriter("dummy-1") + assert.NotNil(t, w) + loggerTest := m.GetLogger("test") + loggerTest.AddWriters(w) + loggerTest.Info("msg-1") + loggerTest.RemoveAllWriters() // the shared writer is not closed here + loggerTest.Info("never seen") + + // the shared writer can still be used later + w = m.GetSharedWriter("dummy-1") + assert.NotNil(t, w) + loggerTest.AddWriters(w) + loggerTest.Info("msg-2") + + m.GetLogger("test-another").AddWriters(w) + m.GetLogger("test-another").Info("msg-3") + + m.Close() + + logs := w.(*dummyWriter).GetLogs() + assert.Equal(t, []string{"msg-1\n", "msg-2\n", "msg-3\n"}, logs) +} diff --git a/modules/setting/log.go b/modules/setting/log.go index 4c62c749efea..af64ea8d8537 100644 --- a/modules/setting/log.go +++ b/modules/setting/log.go @@ -94,11 +94,7 @@ func prepareLoggerConfig(rootCfg ConfigProvider) { } } -func LogPrepareFilenameForWriter(loggerName, fileName string) string { - defaultFileName := "gitea.log" - if loggerName != "default" { - defaultFileName = loggerName + ".log" - } +func LogPrepareFilenameForWriter(fileName, defaultFileName string) string { if fileName == "" { fileName = defaultFileName } @@ -113,7 +109,7 @@ func LogPrepareFilenameForWriter(loggerName, fileName string) string { return fileName } -func loadLogModeByName(rootCfg ConfigProvider, loggerName, modeName string) (writerType string, writerMode log.WriterMode, err error) { +func loadLogModeByName(rootCfg ConfigProvider, loggerName, modeName string) (writerName, writerType string, writerMode log.WriterMode, err error) { sec := rootCfg.Section("log." + modeName) writerMode = log.WriterMode{} @@ -122,29 +118,35 @@ func loadLogModeByName(rootCfg ConfigProvider, loggerName, modeName string) (wri writerType = modeName } + writerName = modeName + defaultFlags := "stdflags" + defaultFilaName := "gitea.log" + if loggerName == "access" { + // "access" logger is special, by default it doesn't have output flags, so it also needs a new writer name to avoid conflicting with other writers. + // so "access" logger's writer name is usually "file.access" or "console.access" + writerName += ".access" + defaultFlags = "none" + defaultFilaName = "access.log" + } + writerMode.Level = log.LevelFromString(ConfigInheritedKeyString(sec, "LEVEL", Log.Level.String())) writerMode.StacktraceLevel = log.LevelFromString(ConfigInheritedKeyString(sec, "STACKTRACE_LEVEL", Log.StacktraceLogLevel.String())) writerMode.Prefix = ConfigInheritedKeyString(sec, "PREFIX") writerMode.Expression = ConfigInheritedKeyString(sec, "EXPRESSION") - - defaultFlags := "stdflags" - if loggerName == "access" { - defaultFlags = "none" // "access" logger is special, by default it doesn't have output flags - } writerMode.Flags = log.FlagsFromString(ConfigInheritedKeyString(sec, "FLAGS", defaultFlags)) switch writerType { case "console": useStderr := ConfigInheritedKey(sec, "STDERR").MustBool(false) - writerOption := log.WriterConsoleOption{Stderr: useStderr} + defaultCanColor := log.CanColorStdout if useStderr { - writerMode.Colorize = ConfigInheritedKey(sec, "COLORIZE").MustBool(log.CanColorStderr) - } else { - writerMode.Colorize = ConfigInheritedKey(sec, "COLORIZE").MustBool(log.CanColorStdout) + defaultCanColor = log.CanColorStderr } + writerOption := log.WriterConsoleOption{Stderr: useStderr} + writerMode.Colorize = ConfigInheritedKey(sec, "COLORIZE").MustBool(defaultCanColor) writerMode.WriterOption = writerOption case "file": - fileName := LogPrepareFilenameForWriter(loggerName, ConfigInheritedKey(sec, "FILE_NAME").String()) + fileName := LogPrepareFilenameForWriter(ConfigInheritedKey(sec, "FILE_NAME").String(), defaultFilaName) writerOption := log.WriterFileOption{} writerOption.FileName = fileName + filenameSuffix // FIXME: the suffix doesn't seem right, see its related comments writerOption.LogRotate = ConfigInheritedKey(sec, "LOG_ROTATE").MustBool(true) @@ -163,11 +165,11 @@ func loadLogModeByName(rootCfg ConfigProvider, loggerName, modeName string) (wri writerMode.WriterOption = writerOption default: if !log.HasEventWriter(writerType) { - return "", writerMode, fmt.Errorf("invalid log writer type (mode): %s", writerType) + return "", "", writerMode, fmt.Errorf("invalid log writer type (mode): %s", writerType) } } - return writerType, writerMode, nil + return writerName, writerType, writerMode, nil } var filenameSuffix = "" @@ -223,8 +225,7 @@ func initLoggerByName(manager *log.LoggerManager, rootCfg ConfigProvider, logger if modeName == "" { continue } - writerName := modeName - writerType, writerMode, err := loadLogModeByName(rootCfg, loggerName, modeName) + writerName, writerType, writerMode, err := loadLogModeByName(rootCfg, loggerName, modeName) if err != nil { log.FallbackErrorf("Failed to load writer mode %q for logger %s: %v", modeName, loggerName, err) continue @@ -232,10 +233,13 @@ func initLoggerByName(manager *log.LoggerManager, rootCfg ConfigProvider, logger if writerMode.BufferLen == 0 { writerMode.BufferLen = Log.BufferLen } - eventWriter, err := log.NewEventWriter(writerName, writerType, writerMode) - if err != nil { - log.FallbackErrorf("Failed to create event writer for logger %s: %v", loggerName, err) - continue + eventWriter := manager.GetSharedWriter(writerName) + if eventWriter == nil { + eventWriter, err = manager.NewSharedWriter(writerName, writerType, writerMode) + if err != nil { + log.FallbackErrorf("Failed to create event writer for logger %s: %v", loggerName, err) + continue + } } eventWriters = append(eventWriters, eventWriter) } diff --git a/modules/setting/log_test.go b/modules/setting/log_test.go index 7211db7b55be..c07651f5488f 100644 --- a/modules/setting/log_test.go +++ b/modules/setting/log_test.go @@ -37,6 +37,7 @@ func toJSON(v interface{}) string { func TestLogConfigDefault(t *testing.T) { manager, managerClose := initLoggersByConfig(t, ``) + defer managerClose() writerDump := ` { @@ -55,7 +56,6 @@ func TestLogConfigDefault(t *testing.T) { } } ` - defer managerClose() dump := manager.GetLogger(log.DEFAULT).DumpWriters() require.JSONEq(t, writerDump, toJSON(dump)) @@ -76,6 +76,7 @@ func TestLogConfigDisable(t *testing.T) { logger.router.MODE = logger.xorm.MODE = `) + defer managerClose() writerDump := ` { @@ -94,7 +95,6 @@ logger.xorm.MODE = } } ` - defer managerClose() dump := manager.GetLogger(log.DEFAULT).DumpWriters() require.JSONEq(t, writerDump, toJSON(dump)) @@ -114,6 +114,7 @@ func TestLogConfigLegacyDefault(t *testing.T) { [log] MODE = console `) + defer managerClose() writerDump := ` { @@ -132,7 +133,6 @@ MODE = console } } ` - defer managerClose() dump := manager.GetLogger(log.DEFAULT).DumpWriters() require.JSONEq(t, writerDump, toJSON(dump)) @@ -188,7 +188,7 @@ ACCESS = file ` writerDumpAccess := ` { - "file": { + "file.access": { "BufferLen": 10000, "Colorize": false, "Expression": "", @@ -216,7 +216,7 @@ ACCESS = file require.JSONEq(t, strings.ReplaceAll(writerDumpAccess, "$FILENAME", tempPath("access.log")), toJSON(dump)) dump = manager.GetLogger("router").DumpWriters() - require.JSONEq(t, strings.ReplaceAll(writerDump, "$FILENAME", tempPath("router.log")), toJSON(dump)) + require.JSONEq(t, strings.ReplaceAll(writerDump, "$FILENAME", tempPath("gitea.log")), toJSON(dump)) } func TestLogConfigLegacyModeDisable(t *testing.T) { @@ -250,7 +250,6 @@ MODE = console LEVEL = error STDERR = true `) - defer managerClose() writerDump := ` @@ -285,7 +284,7 @@ STDERR = true ` writerDumpAccess := ` { - "console": { + "console.access": { "BufferLen": 10000, "Colorize": false, "Expression": "", @@ -335,6 +334,7 @@ MAX_DAYS = 90 COMPRESS = false COMPRESSION_LEVEL = 4 `) + defer managerClose() writerDump := ` { @@ -378,7 +378,6 @@ COMPRESSION_LEVEL = 4 } } ` - defer managerClose() dump := manager.GetLogger(log.DEFAULT).DumpWriters() expected := writerDump diff --git a/routers/private/manager.go b/routers/private/manager.go index 4762ebb289b5..8ed05da6a593 100644 --- a/routers/private/manager.go +++ b/routers/private/manager.go @@ -141,7 +141,7 @@ func AddLogger(ctx *context.PrivateContext) { case "file": writerOption := log.WriterFileOption{} fileName, _ := opts.Config["filename"].(string) - writerOption.FileName = setting.LogPrepareFilenameForWriter(opts.Writer, fileName) + writerOption.FileName = setting.LogPrepareFilenameForWriter(fileName, opts.Writer+".log") writerOption.LogRotate = opts.Config["rotate"].(bool) maxSizeShift, _ := opts.Config["maxsize"].(int) if maxSizeShift == 0 { From 1bb5e96e6abbdd3cc920add7ddc822e7777bdfa5 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 17 May 2023 16:02:05 +0800 Subject: [PATCH 15/24] add more comments --- modules/log/event_format.go | 6 +++--- modules/log/event_writer.go | 33 +++++++++++++++++++------------- modules/log/event_writer_base.go | 5 +++++ modules/log/logger_impl.go | 11 +++++++++++ modules/log/manager.go | 22 +++++++++++++++------ modules/log/misc.go | 1 + 6 files changed, 56 insertions(+), 22 deletions(-) diff --git a/modules/log/event_format.go b/modules/log/event_format.go index b90e90e0f949..02ed1f5f8eb7 100644 --- a/modules/log/event_format.go +++ b/modules/log/event_format.go @@ -22,15 +22,15 @@ type Event struct { MsgSimpleText string - msgFormat string - msgArgs []any + msgFormat string // the format and args is only valid in the caller's goroutine + msgArgs []any // they are discarded before the event is passed to the writer's channel Stacktrace string } type EventFormatted struct { Origin *Event - Msg any + Msg any // the message formatted by the writer's formatter, the writer knows its type } type EventFormatter func(mode *WriterMode, event *Event, msgFormat string, msgArgs ...any) []byte diff --git a/modules/log/event_writer.go b/modules/log/event_writer.go index c76b208c78db..4b77e488de7b 100644 --- a/modules/log/event_writer.go +++ b/modules/log/event_writer.go @@ -7,23 +7,16 @@ import ( "fmt" ) +// EventWriter is the general interface for all event writers +// EventWriterBase is only used as its base interface +// A writer implementation could override the default EventWriterBase functions +// eg: a writer can override the Run to handle events in its own way with its own goroutine type EventWriter interface { EventWriterBase } -type EventWriterProvider func(writerName string, writerMode WriterMode) EventWriter - -var eventWriterProviders = map[string]EventWriterProvider{} - -func RegisterEventWriter(writerType string, p EventWriterProvider) { - eventWriterProviders[writerType] = p -} - -func HasEventWriter(writerType string) bool { - _, ok := eventWriterProviders[writerType] - return ok -} - +// WriterMode is the mode for creating a new EventWriter, it contains common options for all writers +// Its WriterOption field is the specified options for a writer, it should be passed by value but not by pointer type WriterMode struct { BufferLen int @@ -39,6 +32,20 @@ type WriterMode struct { WriterOption any } +// EventWriterProvider is the function for creating a new EventWriter +type EventWriterProvider func(writerName string, writerMode WriterMode) EventWriter + +var eventWriterProviders = map[string]EventWriterProvider{} + +func RegisterEventWriter(writerType string, p EventWriterProvider) { + eventWriterProviders[writerType] = p +} + +func HasEventWriter(writerType string) bool { + _, ok := eventWriterProviders[writerType] + return ok +} + func NewEventWriter(name, writerType string, mode WriterMode) (EventWriter, error) { if p, ok := eventWriterProviders[writerType]; ok { return p(name, mode), nil diff --git a/modules/log/event_writer_base.go b/modules/log/event_writer_base.go index e3467c2ac679..f61d9a7b9d8a 100644 --- a/modules/log/event_writer_base.go +++ b/modules/log/event_writer_base.go @@ -11,6 +11,8 @@ import ( "time" ) +// EventWriterBase is the base interface for most event writers +// It provides default implementations for most methods type EventWriterBase interface { Base() *EventWriterBaseImpl GetWriterType() string @@ -53,6 +55,7 @@ func (b *EventWriterBaseImpl) GetLevel() Level { return b.Mode.Level } +// Run is the default implementation for EventWriter.Run func (b *EventWriterBaseImpl) Run(ctx context.Context) { defer b.OutputWriteCloser.Close() @@ -133,6 +136,7 @@ func NewEventWriterBase(name, writerType string, mode WriterMode) *EventWriterBa return b } +// eventWriterStartGo use "go" to start an event worker's Run method func eventWriterStartGo(ctx context.Context, w EventWriter, shared bool) { if w.Base().stopped != nil { return // already started @@ -145,6 +149,7 @@ func eventWriterStartGo(ctx context.Context, w EventWriter, shared bool) { }() } +// eventWriterStopWait stops an event writer and waits for it to finish flushing (with a timeout) func eventWriterStopWait(w EventWriter) { close(w.Base().Queue) select { diff --git a/modules/log/logger_impl.go b/modules/log/logger_impl.go index f9bb897a8063..2ea9d644a968 100644 --- a/modules/log/logger_impl.go +++ b/modules/log/logger_impl.go @@ -33,6 +33,7 @@ var ( _ LevelLogger = (*LoggerImpl)(nil) ) +// SendLogEvent sends a log event to all writers func (l *LoggerImpl) SendLogEvent(event *Event) { l.eventWriterMu.RLock() defer l.eventWriterMu.RUnlock() @@ -64,6 +65,7 @@ func (l *LoggerImpl) SendLogEvent(event *Event) { } } +// syncLevelInternal syncs the level of the logger with the levels of the writers func (l *LoggerImpl) syncLevelInternal() { lowestLevel := NONE for _, w := range l.eventWriters { @@ -82,6 +84,7 @@ func (l *LoggerImpl) syncLevelInternal() { l.stacktraceLevel.Store(int32(lowestLevel)) } +// removeWriterInternal removes a writer from the logger, and stops it if it's not shared func (l *LoggerImpl) removeWriterInternal(w EventWriter) { if !w.Base().shared { eventWriterStopWait(w) // only stop non-shared writers, shared writers are managed by the manager @@ -89,6 +92,7 @@ func (l *LoggerImpl) removeWriterInternal(w EventWriter) { delete(l.eventWriters, w.GetWriterName()) } +// AddWriters adds writers to the logger, and starts them. Existing writers will be replaced by new ones. func (l *LoggerImpl) AddWriters(writer ...EventWriter) { l.eventWriterMu.Lock() defer l.eventWriterMu.Unlock() @@ -107,6 +111,7 @@ func (l *LoggerImpl) AddWriters(writer ...EventWriter) { l.syncLevelInternal() } +// RemoveWriter removes a writer from the logger, and the writer is closed and flushed if it is not shared func (l *LoggerImpl) RemoveWriter(modeName string) error { l.eventWriterMu.Lock() defer l.eventWriterMu.Unlock() @@ -121,6 +126,7 @@ func (l *LoggerImpl) RemoveWriter(modeName string) error { return nil } +// RemoveAllWriters removes all writers from the logger, non-shared writers are closed and flushed func (l *LoggerImpl) RemoveAllWriters() *LoggerImpl { l.eventWriterMu.Lock() defer l.eventWriterMu.Unlock() @@ -133,6 +139,7 @@ func (l *LoggerImpl) RemoveAllWriters() *LoggerImpl { return l } +// DumpWriters dumps the writers as a JSON map, it's used for debugging and display purposes. func (l *LoggerImpl) DumpWriters() map[string]any { l.eventWriterMu.RLock() defer l.eventWriterMu.RUnlock() @@ -152,17 +159,21 @@ func (l *LoggerImpl) DumpWriters() map[string]any { return writers } +// Close closes the logger, non-shared writers are closed and flushed func (l *LoggerImpl) Close() { l.RemoveAllWriters() l.ctxCancel() } +// IsEnabled returns true if the logger is enabled: it has a working level and has writers +// Fatal is not considered as enabled, because it's a special case and the process just exits func (l *LoggerImpl) IsEnabled() bool { l.eventWriterMu.RLock() defer l.eventWriterMu.RUnlock() return l.level.Load() < int32(FATAL) && len(l.eventWriters) > 0 } +// Log prepares the log event, if the level matches, the event will be sent to the writers func (l *LoggerImpl) Log(skip int, level Level, format string, logArgs ...any) { if Level(l.level.Load()) > level { return diff --git a/modules/log/manager.go b/modules/log/manager.go index c6cda3bd1874..bbaef7eb20ad 100644 --- a/modules/log/manager.go +++ b/modules/log/manager.go @@ -12,6 +12,7 @@ import ( const DEFAULT = "default" +// LoggerManager manages loggers and shared event writers type LoggerManager struct { ctx context.Context ctxCancel context.CancelFunc @@ -25,6 +26,7 @@ type LoggerManager struct { pauseChan chan struct{} } +// GetLogger returns a logger with the given name. If the logger doesn't exist, a new empty one will be created. func (m *LoggerManager) GetLogger(name string) *LoggerImpl { if name == DEFAULT { if logger := m.defaultLogger.Load(); logger != nil { @@ -47,12 +49,14 @@ func (m *LoggerManager) GetLogger(name string) *LoggerImpl { return logger } +// PauseAll pauses all event writers func (m *LoggerManager) PauseAll() { m.pauseMu.Lock() m.pauseChan = make(chan struct{}) m.pauseMu.Unlock() } +// ResumeAll resumes all event writers func (m *LoggerManager) ResumeAll() { m.pauseMu.Lock() close(m.pauseChan) @@ -60,12 +64,14 @@ func (m *LoggerManager) ResumeAll() { m.pauseMu.Unlock() } +// GetPauseChan returns a channel for writer pausing func (m *LoggerManager) GetPauseChan() chan struct{} { m.pauseMu.RLock() defer m.pauseMu.RUnlock() return m.pauseChan } +// Close closes the logger manager, all loggers and writers will be closed, the messages are flushed. func (m *LoggerManager) Close() { m.mu.Lock() defer m.mu.Unlock() @@ -79,8 +85,11 @@ func (m *LoggerManager) Close() { eventWriterStopWait(writer) } m.writers = map[string]EventWriter{} + + m.ctxCancel() } +// DumpLoggers returns a map of all loggers and their event writers, for debugging and display purposes. func (m *LoggerManager) DumpLoggers() map[string]any { m.mu.Lock() defer m.mu.Unlock() @@ -96,12 +105,7 @@ func (m *LoggerManager) DumpLoggers() map[string]any { return dump } -func (m *LoggerManager) GetSharedWriter(writerName string) EventWriter { - m.mu.Lock() - defer m.mu.Unlock() - return m.writers[writerName] -} - +// NewSharedWriter creates a new shared event writer, it can be used by multiple loggers, and a shared writer won't be closed if a logger is closed. func (m *LoggerManager) NewSharedWriter(writerName, writerType string, mode WriterMode) (writer EventWriter, err error) { m.mu.Lock() defer m.mu.Unlock() @@ -119,6 +123,12 @@ func (m *LoggerManager) NewSharedWriter(writerName, writerType string, mode Writ return writer, nil } +func (m *LoggerManager) GetSharedWriter(writerName string) EventWriter { + m.mu.Lock() + defer m.mu.Unlock() + return m.writers[writerName] +} + var loggerManager = NewManager() func GetManager() *LoggerManager { diff --git a/modules/log/misc.go b/modules/log/misc.go index 089f679cf69f..ae4ce04cf32b 100644 --- a/modules/log/misc.go +++ b/modules/log/misc.go @@ -72,6 +72,7 @@ func (p *loggerToWriter) Write(bs []byte) (int, error) { return len(bs), nil } +// LoggerToWriter wraps a log function to an io.Writer func LoggerToWriter(logf func(format string, args ...any)) io.Writer { return &loggerToWriter{logf: logf} } From cc3f7af008172d6cbf9d75db53aa42382bd6a179 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 17 May 2023 16:55:00 +0800 Subject: [PATCH 16/24] fix doc --- custom/conf/app.example.ini | 8 ++++---- .../doc/administration/logging-config.en-us.md | 16 ++++++---------- modules/log/flags.go | 4 ++-- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 76446f1f3361..795b6602024f 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -553,10 +553,10 @@ ENABLE = true ;; Use comma to separate multiple modes, e.g. "console, file" MODE = console ;; -;; Either "Trace", "Debug", "Info", "Warn", "Error", "Critical" or "None", default is "Info" +;; Either "Trace", "Debug", "Info", "Warn", "Error" or "None", default is "Info" LEVEL = Info ;; -;; Print Stacktrace with logs (rarely helpful, do not set) Either "Trace", "Debug", "Info", "Warn", "Error", "Critical", default is "None" +;; Print Stacktrace with logs (rarely helpful, do not set) Either "Trace", "Debug", "Info", "Warn", "Error", default is "None" ;STACKTRACE_LEVEL = None ;; ;; Buffer length of the channel, keep it as it is if you don't know what it is. @@ -584,7 +584,7 @@ LEVEL = Info ;; If you configure more than one in the .ini file, it will match in the order of configuration, ;; and the first match will be finally printed in the log. ;; * E.g: -;; * In reuqest Header: X-Trace-ID: trace-id-1q2w3e4r +;; * In request Header: X-Trace-ID: trace-id-1q2w3e4r ;; * Configuration in app.ini: REQUEST_ID_HEADERS = X-Request-ID, X-Trace-ID, X-Req-ID ;; * Print in log: 127.0.0.1:58384 - - [14/Feb/2023:16:33:51 +0800] "trace-id-1q2w3e4r" ;; @@ -597,7 +597,7 @@ LEVEL = Info ;; ;; Log modes (aka log writers) ;; -;[log.%(WriterMode}] +;[log.%(WriterMode)] ;MODE=console/file/conn/... ;LEVEL= ;FLAGS = stdflags diff --git a/docs/content/doc/administration/logging-config.en-us.md b/docs/content/doc/administration/logging-config.en-us.md index 53a914e10273..3d1236e941e7 100644 --- a/docs/content/doc/administration/logging-config.en-us.md +++ b/docs/content/doc/administration/logging-config.en-us.md @@ -155,11 +155,11 @@ Possible values are: - `funcname` - function name of the caller: `runtime.Caller()`. - `shortfuncname` - last part of the function name. Overrides `funcname`. - `utc` - if date or time is set, use UTC rather than the local time zone. -- `levelinitial` - Initial character of the provided level in brackets eg. `[I]` for info. -- `level` - Provided level in brackets `[INFO]`. -- `gopid` - The Goroutine-PID of the context. -- `medfile` - Last 20 characters of the filename - equivalent to `shortfile,longfile`. -- `stdflags` - Equivalent to `date,time,medfile,shortfuncname,levelinitial`. +- `levelinitial` - initial character of the provided level in brackets eg. `[I]` for info. +- `level` - level in brackets `[INFO]`. +- `gopid` - the Goroutine-PID of the context. +- `medfile` - last 20 characters of the filename - equivalent to `shortfile,longfile`. +- `stdflags` - equivalent to `date,time,medfile,shortfuncname,levelinitial`. ### Console mode @@ -180,7 +180,7 @@ In this mode the logger will save log messages to a file. Settings: -- `FILE_NAME`: The file to write the log events to. Default to `%(ROOT_PATH)/gitea.log`. Exception: access log will default to `%(ROOT_PATH)/access.log`. +- `FILE_NAME`: The file to write the log events to, relative to `ROOT_PATH`, Default to `%(ROOT_PATH)/gitea.log`. Exception: access log will default to `%(ROOT_PATH)/access.log`. - `MAX_SIZE_SHIFT`: **28**: Maximum size shift of a single file. 28 represents 256Mb. For details see below. - `LOG_ROTATE` **true**: Whether to rotate the log files. TODO: if false, will it delete instead on daily rotate, or do nothing?. - `DAILY_ROTATE`: **true**: Whether to rotate logs daily. @@ -188,10 +188,6 @@ Settings: - `COMPRESS`: **true**: Whether to compress old log files by default with gzip. - `COMPRESSION_LEVEL`: **-1**: Compression level. For details see below. -The default value of `FILE_NAME` depends on the respective logger facility. -If unset, their own default will be used. -If set it will be relative to the provided `ROOT_PATH` in the master `[log]` section. - `MAX_SIZE_SHIFT` defines the maximum size of a file by left shifting 1 the given number of times (`1 << x`). The exact behavior at the time of v1.17.3 can be seen [here](https://github.com/go-gitea/gitea/blob/v1.17.3/modules/setting/log.go#L185). diff --git a/modules/log/flags.go b/modules/log/flags.go index d27bd87867e2..f025159d5394 100644 --- a/modules/log/flags.go +++ b/modules/log/flags.go @@ -28,9 +28,9 @@ const ( Lfuncname // function name of the caller: runtime.Caller() Lshortfuncname // last part of the function name LUTC // if Ldate or Ltime is set, use UTC rather than the local time zone - Llevelinitial // Initial character of the provided level in brackets eg. [I] for info + Llevelinitial // Initial character of the provided level in brackets, eg. [I] for info Llevel // Provided level in brackets [INFO] - Lgopid + Lgopid // the Goroutine-PID of the context Lmedfile = Lshortfile | Llongfile // last 20 characters of the filename LstdFlags = Ldate | Ltime | Lmedfile | Lshortfuncname | Llevelinitial // default From bef01a1fc0525d2c48b4426795f6ca9efa83b4ec Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 18 May 2023 12:39:09 +0800 Subject: [PATCH 17/24] Update modules/util/rotatingfilewriter/writer.go Co-authored-by: Jason Song --- modules/util/rotatingfilewriter/writer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/util/rotatingfilewriter/writer.go b/modules/util/rotatingfilewriter/writer.go index 65e1b3b22bb7..5243bfe35386 100644 --- a/modules/util/rotatingfilewriter/writer.go +++ b/modules/util/rotatingfilewriter/writer.go @@ -110,7 +110,7 @@ func (rfw *RotatingFileWriter) open(filename string) error { return err } rfw.currentSize = finfo.Size() - rfw.openDate = time.Now().Day() + rfw.openDate = finfo.ModTime().Day() return nil } From 90535c10fe71f7ff01608c9d66d5a320be01c333 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 18 May 2023 23:45:14 +0800 Subject: [PATCH 18/24] Update docs/content/doc/administration/config-cheat-sheet.en-us.md Co-authored-by: delvh --- docs/content/doc/administration/config-cheat-sheet.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index 1dafa2b6d4a9..da3502236fff 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -876,7 +876,7 @@ Default templates for project boards: ### File log mode (`log.file`, or `MODE=file`) -- `FILE_NAME`: Set the file name for this logger. Defaults to "gitea.log" (exception: access log defaults to "access.log"). If relative will be relative to the `ROOT_PATH` +- `FILE_NAME`: Set the file name for this logger. Defaults to `gitea.log` (exception: access log defaults to `access.log`). If relative will be relative to the `ROOT_PATH` - `LOG_ROTATE`: **true**: Rotate the log files. - `MAX_SIZE_SHIFT`: **28**: Maximum size shift of a single file, 28 represents 256Mb. - `DAILY_ROTATE`: **true**: Rotate logs daily. From f05ea76139c0bbe7d8275587bec28871131ff23c Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 18 May 2023 23:46:09 +0800 Subject: [PATCH 19/24] Update docs/content/doc/administration/logging-config.en-us.md Co-authored-by: delvh --- docs/content/doc/administration/logging-config.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/administration/logging-config.en-us.md b/docs/content/doc/administration/logging-config.en-us.md index 3d1236e941e7..8c23edb3d936 100644 --- a/docs/content/doc/administration/logging-config.en-us.md +++ b/docs/content/doc/administration/logging-config.en-us.md @@ -21,7 +21,7 @@ The logging configuration of Gitea mainly consists of 3 types of components: - The `[log]` section for general configuration - `[log.]` sections for the configuration of different log writers to output logs, aka: "writer mode", the mode name is also used as "writer name". -- `[log]` section could contain sub-loggers like`logger..` +- The `[log]` section can also contain sub-logger configurations following the key schema `logger..` There is a fully functional log output by default, so it is not necessary to define one. From 03d69b946e5e35a4890dffc3823be37a904f154c Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 18 May 2023 23:50:09 +0800 Subject: [PATCH 20/24] Update docs/content/doc/administration/config-cheat-sheet.en-us.md Co-authored-by: delvh --- docs/content/doc/administration/config-cheat-sheet.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index da3502236fff..f48c81661577 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -859,7 +859,7 @@ Default templates for project boards: - Configuration in app.ini: REQUEST_ID_HEADERS = X-Request-ID - Print in log: 127.0.0.1:58384 - - [14/Feb/2023:16:33:51 +0800] "**test-id-123**" ... -### Log subsections (`log.writer-mode-name`) +### Log subsections (`log.`) - `MODE`: **name**: Sets the mode of this log writer - Defaults to the provided subsection name. This allows you to have two different file loggers at different levels. - `LEVEL`: **log.LEVEL**: Sets the log-level of this writer. Defaults to the `LEVEL` set in the global `[log]` section. From 9b5c93edfca4ed6c7ed1deea009567abbe1c3cd9 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 18 May 2023 23:53:10 +0800 Subject: [PATCH 21/24] Apply suggestions from code review Co-authored-by: delvh --- .../content/doc/administration/logging-config.en-us.md | 10 +++++----- modules/lfs/pointer.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/content/doc/administration/logging-config.en-us.md b/docs/content/doc/administration/logging-config.en-us.md index 8c23edb3d936..857eb19b563f 100644 --- a/docs/content/doc/administration/logging-config.en-us.md +++ b/docs/content/doc/administration/logging-config.en-us.md @@ -35,7 +35,7 @@ To collect logs for help and issue report, see [Support Options]({{< relref "doc ## The `[log]` section -Configuration of logging facilities in Gitea happen in the `[log]` section and it's subsections. +Configuration of logging facilities in Gitea happen in the `[log]` section and its subsections. In the top level `[log]` section the following configurations can be placed: @@ -142,7 +142,7 @@ Please note this expression will be run in the writer's goroutine but not the lo `FLAGS` represents the preceding logging context information that is printed before each message. It is a comma-separated string set. The order of values does not matter. -It is default to `stdflags` (Equal to `date,time,medfile,shortfuncname,levelinitial`) +It defaults to `stdflags` (= `date,time,medfile,shortfuncname,levelinitial`) Possible values are: @@ -221,7 +221,7 @@ To make XORM outputs SQL logs, the `LOG_SQL` in `[database]` section should also ### The "Access" logger -The Access logger is a new logger for version 1.9. It provides a NCSA +The Access logger is a new logger since Gitea 1.9. It provides a NCSA Common Log compliant log format. It's highly configurable but caution should be taken when changing its template. The main benefit of this logger is that Gitea can now log accesses in a standard log format so @@ -237,9 +237,9 @@ Please note, the access logger will log at `INFO` level, setting the #### The ACCESS_LOG_TEMPLATE -This value represent a go template. It's default value is: +This value represents a go template. Its default value is -``` +```tmpl {{.Ctx.RemoteHost}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}" "{{.Ctx.Req.UserAgent}}"` ``` diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go index 649c81a0cfcd..d7653e836c91 100644 --- a/modules/lfs/pointer.go +++ b/modules/lfs/pointer.go @@ -113,9 +113,9 @@ func (p Pointer) RelativePath() string { func (p Pointer) LogString() string { if p.Oid == "" && p.Size == 0 { - return "" + return "" } - return fmt.Sprintf("", p.Oid, p.Size) + return fmt.Sprintf("", p.Oid, p.Size) } // GeneratePointer generates a pointer for arbitrary content From 9aceb066d166c495080f8baa4289921cacd73784 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 18 May 2023 23:59:10 +0800 Subject: [PATCH 22/24] Update modules/log/color_console.go Co-authored-by: delvh --- modules/log/color_console.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/log/color_console.go b/modules/log/color_console.go index 1a08bbdc41c5..2658652ec600 100644 --- a/modules/log/color_console.go +++ b/modules/log/color_console.go @@ -1,5 +1,5 @@ // Copyright 2014 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2023 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package log From f985c0f277bd2733a59d5e4bc2a5efc45413fe95 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 19 May 2023 23:03:39 +0800 Subject: [PATCH 23/24] add test for ReleaseReopen manager --- .../graceful/releasereopen/releasereopen.go | 11 ++--- .../releasereopen/releasereopen_test.go | 43 +++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 modules/graceful/releasereopen/releasereopen_test.go diff --git a/modules/graceful/releasereopen/releasereopen.go b/modules/graceful/releasereopen/releasereopen.go index 36c02ab81747..de5b07c0a6c8 100644 --- a/modules/graceful/releasereopen/releasereopen.go +++ b/modules/graceful/releasereopen/releasereopen.go @@ -24,13 +24,14 @@ func (r *Manager) Register(rr ReleaseReopener) (cancel func()) { defer r.mu.Unlock() r.counter++ + currentCounter := r.counter r.releaseReopeners[r.counter] = rr return func() { r.mu.Lock() defer r.mu.Unlock() - delete(r.releaseReopeners, r.counter) + delete(r.releaseReopeners, currentCounter) } } @@ -51,10 +52,10 @@ func GetManager() *Manager { return manager } -var manager *Manager - -func init() { - manager = &Manager{ +func NewManager() *Manager { + return &Manager{ releaseReopeners: make(map[int64]ReleaseReopener), } } + +var manager = NewManager() diff --git a/modules/graceful/releasereopen/releasereopen_test.go b/modules/graceful/releasereopen/releasereopen_test.go new file mode 100644 index 000000000000..0e8b48257d64 --- /dev/null +++ b/modules/graceful/releasereopen/releasereopen_test.go @@ -0,0 +1,43 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package releasereopen + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +type testReleaseReopener struct { + count int +} + +func (t *testReleaseReopener) ReleaseReopen() error { + t.count++ + return nil +} + +func TestManager(t *testing.T) { + m := NewManager() + + t1 := &testReleaseReopener{} + t2 := &testReleaseReopener{} + t3 := &testReleaseReopener{} + + _ = m.Register(t1) + c2 := m.Register(t2) + _ = m.Register(t3) + + assert.NoError(t, m.ReleaseReopen()) + assert.EqualValues(t, 1, t1.count) + assert.EqualValues(t, 1, t2.count) + assert.EqualValues(t, 1, t3.count) + + c2() + + assert.NoError(t, m.ReleaseReopen()) + assert.EqualValues(t, 2, t1.count) + assert.EqualValues(t, 1, t2.count) + assert.EqualValues(t, 2, t3.count) +} From 60079bad3e8309b92a43fc5562ec8d7469140b2c Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 19 May 2023 23:36:19 +0800 Subject: [PATCH 24/24] support "%#v" --- modules/log/event_format.go | 14 ++++++++++++++ modules/log/logger_impl.go | 4 ++-- modules/log/logger_test.go | 8 +++++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/modules/log/event_format.go b/modules/log/event_format.go index 02ed1f5f8eb7..524ca3dd872e 100644 --- a/modules/log/event_format.go +++ b/modules/log/event_format.go @@ -35,6 +35,20 @@ type EventFormatted struct { type EventFormatter func(mode *WriterMode, event *Event, msgFormat string, msgArgs ...any) []byte +type logStringFormatter struct { + v LogStringer +} + +var _ fmt.Formatter = logStringFormatter{} + +func (l logStringFormatter) Format(f fmt.State, verb rune) { + if f.Flag('#') && verb == 'v' { + _, _ = fmt.Fprintf(f, "%#v", l.v) + return + } + _, _ = f.Write([]byte(l.v.LogString())) +} + // Copy of cheap integer to fixed-width decimal to ascii from logger. // TODO: legacy bugs: doesn't support negative number, overflow if wid it too large. func itoa(buf []byte, i, wid int) []byte { diff --git a/modules/log/logger_impl.go b/modules/log/logger_impl.go index 2ea9d644a968..c7e8fde3c082 100644 --- a/modules/log/logger_impl.go +++ b/modules/log/logger_impl.go @@ -211,10 +211,10 @@ func (l *LoggerImpl) Log(skip int, level Level, format string, logArgs ...any) { for i, v := range msgArgs { if cv, ok := v.(*ColoredValue); ok { if s, ok := cv.v.(LogStringer); ok { - cv.v = s.LogString() + cv.v = logStringFormatter{v: s} } } else if s, ok := v.(LogStringer); ok { - msgArgs[i] = s.LogString() + msgArgs[i] = logStringFormatter{v: s} } } diff --git a/modules/log/logger_test.go b/modules/log/logger_test.go index f8b072e304c2..1fb63bf629ea 100644 --- a/modules/log/logger_test.go +++ b/modules/log/logger_test.go @@ -107,7 +107,9 @@ func TestLoggerPause(t *testing.T) { logger.Close() } -type testLogString struct{} +type testLogString struct { + Field string +} func (t testLogString) LogString() string { return "log-string" @@ -120,10 +122,10 @@ func TestLoggerLogString(t *testing.T) { w1.Mode.Colorize = true logger.AddWriters(w1) - logger.Info("%s %s %s", testLogString{}, &testLogString{}, NewColoredValue(testLogString{}, FgRed)) + logger.Info("%s %s %#v %v", testLogString{}, &testLogString{}, testLogString{Field: "detail"}, NewColoredValue(testLogString{}, FgRed)) logger.Close() - assert.Equal(t, []string{"log-string log-string \x1b[31mlog-string\x1b[0m\n"}, w1.GetLogs()) + assert.Equal(t, []string{"log-string log-string log.testLogString{Field:\"detail\"} \x1b[31mlog-string\x1b[0m\n"}, w1.GetLogs()) } func TestLoggerExpressionFilter(t *testing.T) {