From 72620db8dfa654630603b9e8d49d994823cbe8d3 Mon Sep 17 00:00:00 2001 From: zokki Date: Fri, 4 Jul 2025 00:08:23 +0200 Subject: [PATCH] feat: add a `EXCLUSION` to the logger (#8212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This feature is intended to help reduce noisy logs generated by routine Kubernetes probes and Prometheus scraping. While logs are essential, these specific requests (e.g., to /metrics and /api/healthz) generally don't provide useful information and tend to clutter the output. The goal is to introduce functionality that effectively acts as the inverse of the existing EXPRESSION mode—allowing logging to be excluded based on a condition, rather than included. Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/8212 Reviewed-by: Gusted Co-authored-by: zokki Co-committed-by: zokki --- cmd/manager_logging.go | 8 +++++ custom/conf/app.example.ini | 1 + modules/log/event_writer.go | 1 + modules/log/event_writer_base.go | 15 ++++++++ modules/log/event_writer_buffer_test.go | 46 +++++++++++++++++++++++++ modules/log/logger_test.go | 16 +++++++++ modules/setting/log.go | 1 + modules/setting/log_test.go | 11 ++++++ routers/private/manager.go | 1 + 9 files changed, 100 insertions(+) diff --git a/cmd/manager_logging.go b/cmd/manager_logging.go index c543afe872..c18bfa919b 100644 --- a/cmd/manager_logging.go +++ b/cmd/manager_logging.go @@ -44,6 +44,11 @@ func defaultLoggingFlags() []cli.Flag { Aliases: []string{"e"}, Usage: "Matching expression for the logger", }, + &cli.StringFlag{ + Name: "exclusion", + Aliases: []string{"x"}, + Usage: "Exclusion for the logger", + }, &cli.StringFlag{ Name: "prefix", Aliases: []string{"p"}, @@ -286,6 +291,9 @@ func commonAddLogger(ctx context.Context, c *cli.Command, mode string, vals map[ if len(c.String("expression")) > 0 { vals["expression"] = c.String("expression") } + if len(c.String("exclusion")) > 0 { + vals["exclusion"] = c.String("exclusion") + } if len(c.String("prefix")) > 0 { vals["prefix"] = c.String("prefix") } diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 1b8d4c6697..37d67df5f0 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -631,6 +631,7 @@ LEVEL = Info ;LEVEL= ;FLAGS = stdflags or journald ;EXPRESSION = +;EXCLUSION = ;PREFIX = ;COLORIZE = false ;; diff --git a/modules/log/event_writer.go b/modules/log/event_writer.go index 4b77e488de..32b5b582c5 100644 --- a/modules/log/event_writer.go +++ b/modules/log/event_writer.go @@ -26,6 +26,7 @@ type WriterMode struct { Flags Flags Expression string + Exclusion string StacktraceLevel Level diff --git a/modules/log/event_writer_base.go b/modules/log/event_writer_base.go index 9189ca4e90..4de2b953c7 100644 --- a/modules/log/event_writer_base.go +++ b/modules/log/event_writer_base.go @@ -68,6 +68,14 @@ func (b *EventWriterBaseImpl) Run(ctx context.Context) { } } + var exclusionRegexp *regexp.Regexp + if b.Mode.Exclusion != "" { + var err error + if exclusionRegexp, err = regexp.Compile(b.Mode.Exclusion); err != nil { + FallbackErrorf("unable to compile exclusion %q for writer %q: %v", b.Mode.Exclusion, b.Name, err) + } + } + handlePaused := func() { if pause := b.GetPauseChan(); pause != nil { select { @@ -95,6 +103,13 @@ func (b *EventWriterBaseImpl) Run(ctx context.Context) { continue } } + if exclusionRegexp != nil { + fileLineCaller := fmt.Sprintf("%s:%d:%s", event.Origin.Filename, event.Origin.Line, event.Origin.Caller) + matched := exclusionRegexp.MatchString(fileLineCaller) || exclusionRegexp.MatchString(event.Origin.MsgSimpleText) + if matched { + continue + } + } var err error switch msg := event.Msg.(type) { diff --git a/modules/log/event_writer_buffer_test.go b/modules/log/event_writer_buffer_test.go index ba9455ba69..d1e37c3673 100644 --- a/modules/log/event_writer_buffer_test.go +++ b/modules/log/event_writer_buffer_test.go @@ -31,3 +31,49 @@ func TestBufferLogger(t *testing.T) { logger.Close() assert.Contains(t, bufferWriter.Buffer.String(), expected) } + +func TestBufferLoggerWithExclusion(t *testing.T) { + prefix := "ExclusionPrefix " + level := log.INFO + message := "something" + + bufferWriter := log.NewEventWriterBuffer("test-buffer", log.WriterMode{ + Level: level, + Prefix: prefix, + Exclusion: message, + }) + + logger := log.NewLoggerWithWriters(t.Context(), "test", bufferWriter) + + logger.SendLogEvent(&log.Event{ + Level: log.INFO, + MsgSimpleText: message, + }) + logger.Close() + assert.NotContains(t, bufferWriter.Buffer.String(), message) +} + +func TestBufferLoggerWithExpressionAndExclusion(t *testing.T) { + prefix := "BothPrefix " + level := log.INFO + expression := ".*foo.*" + exclusion := ".*bar.*" + + bufferWriter := log.NewEventWriterBuffer("test-buffer", log.WriterMode{ + Level: level, + Prefix: prefix, + Expression: expression, + Exclusion: exclusion, + }) + + logger := log.NewLoggerWithWriters(t.Context(), "test", bufferWriter) + + logger.SendLogEvent(&log.Event{Level: log.INFO, MsgSimpleText: "foo expression"}) + logger.SendLogEvent(&log.Event{Level: log.INFO, MsgSimpleText: "bar exclusion"}) + logger.SendLogEvent(&log.Event{Level: log.INFO, MsgSimpleText: "foo bar both"}) + logger.SendLogEvent(&log.Event{Level: log.INFO, MsgSimpleText: "none"}) + logger.Close() + + assert.Contains(t, bufferWriter.Buffer.String(), "foo expression") + assert.NotContains(t, bufferWriter.Buffer.String(), "bar") +} diff --git a/modules/log/logger_test.go b/modules/log/logger_test.go index 6d6ceb69d7..99045b0f4f 100644 --- a/modules/log/logger_test.go +++ b/modules/log/logger_test.go @@ -143,3 +143,19 @@ func TestLoggerExpressionFilter(t *testing.T) { assert.Equal(t, []string{"foo\n", "foo bar\n", "by filename\n"}, w1.GetLogs()) } + +func TestLoggerExclusionFilter(t *testing.T) { + logger := NewLoggerWithWriters(t.Context(), "test") + + w1 := newDummyWriter("dummy-1", DEBUG, 0) + w1.Mode.Exclusion = "foo.*" + logger.AddWriters(w1) + + logger.Info("foo") + logger.Info("bar") + logger.Info("foo bar") + logger.SendLogEvent(&Event{Level: INFO, Filename: "foo.go", MsgSimpleText: "by filename"}) + logger.Close() + + assert.Equal(t, []string{"bar\n"}, w1.GetLogs()) +} diff --git a/modules/setting/log.go b/modules/setting/log.go index 0747ac4dac..6d069d0e9c 100644 --- a/modules/setting/log.go +++ b/modules/setting/log.go @@ -133,6 +133,7 @@ func loadLogModeByName(rootCfg ConfigProvider, loggerName, modeName string) (wri writerMode.StacktraceLevel = log.LevelFromString(ConfigInheritedKeyString(sec, "STACKTRACE_LEVEL", Log.StacktraceLogLevel.String())) writerMode.Prefix = ConfigInheritedKeyString(sec, "PREFIX") writerMode.Expression = ConfigInheritedKeyString(sec, "EXPRESSION") + writerMode.Exclusion = ConfigInheritedKeyString(sec, "EXCLUSION") // flags are updated and set below switch writerType { diff --git a/modules/setting/log_test.go b/modules/setting/log_test.go index eda6dc36af..223bd68285 100644 --- a/modules/setting/log_test.go +++ b/modules/setting/log_test.go @@ -44,6 +44,7 @@ func TestLogConfigDefault(t *testing.T) { "BufferLen": 10000, "Colorize": false, "Expression": "", + "Exclusion": "", "Flags": "stdflags", "Level": "info", "Prefix": "", @@ -83,6 +84,7 @@ logger.xorm.MODE = "BufferLen": 10000, "Colorize": false, "Expression": "", + "Exclusion": "", "Flags": "stdflags", "Level": "info", "Prefix": "", @@ -121,6 +123,7 @@ MODE = console "BufferLen": 10000, "Colorize": false, "Expression": "", + "Exclusion": "", "Flags": "stdflags", "Level": "info", "Prefix": "", @@ -168,6 +171,7 @@ ACCESS = file "BufferLen": 10000, "Colorize": false, "Expression": "", + "Exclusion": "", "Flags": "stdflags", "Level": "info", "Prefix": "", @@ -191,6 +195,7 @@ ACCESS = file "BufferLen": 10000, "Colorize": false, "Expression": "", + "Exclusion": "", "Flags": "none", "Level": "info", "Prefix": "", @@ -257,6 +262,7 @@ STDERR = true "BufferLen": 10000, "Colorize": false, "Expression": "", + "Exclusion": "", "Flags": "stdflags", "Level": "warn", "Prefix": "", @@ -270,6 +276,7 @@ STDERR = true "BufferLen": 10000, "Colorize": false, "Expression": "", + "Exclusion": "", "Flags": "stdflags", "Level": "error", "Prefix": "", @@ -287,6 +294,7 @@ STDERR = true "BufferLen": 10000, "Colorize": false, "Expression": "", + "Exclusion": "", "Flags": "none", "Level": "warn", "Prefix": "", @@ -323,6 +331,7 @@ MODE = file LEVEL = error STACKTRACE_LEVEL = fatal EXPRESSION = filter +EXCLUSION = not FLAGS = medfile PREFIX = "[Prefix] " FILE_NAME = file-xxx.log @@ -341,6 +350,7 @@ COMPRESSION_LEVEL = 4 "BufferLen": 10, "Colorize": false, "Expression": "", + "Exclusion": "", "Flags": "stdflags", "Level": "info", "Prefix": "", @@ -360,6 +370,7 @@ COMPRESSION_LEVEL = 4 "BufferLen": 10, "Colorize": false, "Expression": "filter", + "Exclusion": "not", "Flags": "medfile", "Level": "error", "Prefix": "[Prefix] ", diff --git a/routers/private/manager.go b/routers/private/manager.go index 7ab198f71b..90b48256df 100644 --- a/routers/private/manager.go +++ b/routers/private/manager.go @@ -145,6 +145,7 @@ func AddLogger(ctx *context.PrivateContext) { writerMode.Prefix, _ = opts.Config["prefix"].(string) writerMode.Expression, _ = opts.Config["expression"].(string) + writerMode.Exclusion, _ = opts.Config["exclusion"].(string) switch writerType { case "console":