feat: add a EXCLUSION to the logger (#8212)

This feature is intended to help reduce noisy logs generated by routine Kubernetes probes and Prometheus scraping. While logs are essential, these specific requests (e.g., to /metrics and /api/healthz) generally don't provide useful information and tend to clutter the output.

The goal is to introduce functionality that effectively acts as the inverse of the existing EXPRESSION mode—allowing logging to be excluded based on a condition, rather than included.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/8212
Reviewed-by: Gusted <gusted@noreply.codeberg.org>
Co-authored-by: zokki <zokki.softwareschmiede@gmail.com>
Co-committed-by: zokki <zokki.softwareschmiede@gmail.com>

cmd/manager_logging.go

@@ -44,6 +44,11 @@ func defaultLoggingFlags() []cli.Flag {
 			Aliases: []string{"e"},
 			Usage:   "Matching expression for the logger",
 		},
+		&cli.StringFlag{
+			Name:    "exclusion",
+			Aliases: []string{"x"},
+			Usage:   "Exclusion for the logger",
+		},
 		&cli.StringFlag{
 			Name:    "prefix",
 			Aliases: []string{"p"},
@@ -286,6 +291,9 @@ func commonAddLogger(ctx context.Context, c *cli.Command, mode string, vals map[
 	if len(c.String("expression")) > 0 {
 		vals["expression"] = c.String("expression")
 	}
+	if len(c.String("exclusion")) > 0 {
+		vals["exclusion"] = c.String("exclusion")
+	}
 	if len(c.String("prefix")) > 0 {
 		vals["prefix"] = c.String("prefix")
 	}

custom/conf/app.example.ini

@@ -631,6 +631,7 @@ LEVEL = Info
 ;LEVEL=
 ;FLAGS = stdflags or journald
 ;EXPRESSION =
+;EXCLUSION =
 ;PREFIX =
 ;COLORIZE = false
 ;;

modules/log/event_writer.go

@@ -26,6 +26,7 @@ type WriterMode struct {
 	Flags    Flags
 
 	Expression string
+	Exclusion  string
 
 	StacktraceLevel Level
 

modules/log/event_writer_base.go

@@ -68,6 +68,14 @@ func (b *EventWriterBaseImpl) Run(ctx context.Context) {
 		}
 	}
 
+	var exclusionRegexp *regexp.Regexp
+	if b.Mode.Exclusion != "" {
+		var err error
+		if exclusionRegexp, err = regexp.Compile(b.Mode.Exclusion); err != nil {
+			FallbackErrorf("unable to compile exclusion %q for writer %q: %v", b.Mode.Exclusion, b.Name, err)
+		}
+	}
+
 	handlePaused := func() {
 		if pause := b.GetPauseChan(); pause != nil {
 			select {
@@ -95,6 +103,13 @@ func (b *EventWriterBaseImpl) Run(ctx context.Context) {
 					continue
 				}
 			}
+			if exclusionRegexp != nil {
+				fileLineCaller := fmt.Sprintf("%s:%d:%s", event.Origin.Filename, event.Origin.Line, event.Origin.Caller)
+				matched := exclusionRegexp.MatchString(fileLineCaller) || exclusionRegexp.MatchString(event.Origin.MsgSimpleText)
+				if matched {
+					continue
+				}
+			}
 
 			var err error
 			switch msg := event.Msg.(type) {

modules/log/event_writer_buffer_test.go

@@ -31,3 +31,49 @@ func TestBufferLogger(t *testing.T) {
 	logger.Close()
 	assert.Contains(t, bufferWriter.Buffer.String(), expected)
 }
+
+func TestBufferLoggerWithExclusion(t *testing.T) {
+	prefix := "ExclusionPrefix "
+	level := log.INFO
+	message := "something"
+
+	bufferWriter := log.NewEventWriterBuffer("test-buffer", log.WriterMode{
+		Level:     level,
+		Prefix:    prefix,
+		Exclusion: message,
+	})
+
+	logger := log.NewLoggerWithWriters(t.Context(), "test", bufferWriter)
+
+	logger.SendLogEvent(&log.Event{
+		Level:         log.INFO,
+		MsgSimpleText: message,
+	})
+	logger.Close()
+	assert.NotContains(t, bufferWriter.Buffer.String(), message)
+}
+
+func TestBufferLoggerWithExpressionAndExclusion(t *testing.T) {
+	prefix := "BothPrefix "
+	level := log.INFO
+	expression := ".*foo.*"
+	exclusion := ".*bar.*"
+
+	bufferWriter := log.NewEventWriterBuffer("test-buffer", log.WriterMode{
+		Level:      level,
+		Prefix:     prefix,
+		Expression: expression,
+		Exclusion:  exclusion,
+	})
+
+	logger := log.NewLoggerWithWriters(t.Context(), "test", bufferWriter)
+
+	logger.SendLogEvent(&log.Event{Level: log.INFO, MsgSimpleText: "foo expression"})
+	logger.SendLogEvent(&log.Event{Level: log.INFO, MsgSimpleText: "bar exclusion"})
+	logger.SendLogEvent(&log.Event{Level: log.INFO, MsgSimpleText: "foo bar both"})
+	logger.SendLogEvent(&log.Event{Level: log.INFO, MsgSimpleText: "none"})
+	logger.Close()
+
+	assert.Contains(t, bufferWriter.Buffer.String(), "foo expression")
+	assert.NotContains(t, bufferWriter.Buffer.String(), "bar")
+}

modules/log/logger_test.go

@@ -143,3 +143,19 @@ func TestLoggerExpressionFilter(t *testing.T) {
 
 	assert.Equal(t, []string{"foo\n", "foo bar\n", "by filename\n"}, w1.GetLogs())
 }
+
+func TestLoggerExclusionFilter(t *testing.T) {
+	logger := NewLoggerWithWriters(t.Context(), "test")
+
+	w1 := newDummyWriter("dummy-1", DEBUG, 0)
+	w1.Mode.Exclusion = "foo.*"
+	logger.AddWriters(w1)
+
+	logger.Info("foo")
+	logger.Info("bar")
+	logger.Info("foo bar")
+	logger.SendLogEvent(&Event{Level: INFO, Filename: "foo.go", MsgSimpleText: "by filename"})
+	logger.Close()
+
+	assert.Equal(t, []string{"bar\n"}, w1.GetLogs())
+}

modules/setting/log.go

@@ -133,6 +133,7 @@ func loadLogModeByName(rootCfg ConfigProvider, loggerName, modeName string) (wri
 	writerMode.StacktraceLevel = log.LevelFromString(ConfigInheritedKeyString(sec, "STACKTRACE_LEVEL", Log.StacktraceLogLevel.String()))
 	writerMode.Prefix = ConfigInheritedKeyString(sec, "PREFIX")
 	writerMode.Expression = ConfigInheritedKeyString(sec, "EXPRESSION")
+	writerMode.Exclusion = ConfigInheritedKeyString(sec, "EXCLUSION")
 	// flags are updated and set below
 
 	switch writerType {

modules/setting/log_test.go

@@ -44,6 +44,7 @@ func TestLogConfigDefault(t *testing.T) {
 		"BufferLen": 10000,
 		"Colorize": false,
 		"Expression": "",
+		"Exclusion": "",
 		"Flags": "stdflags",
 		"Level": "info",
 		"Prefix": "",
@@ -83,6 +84,7 @@ logger.xorm.MODE =
 		"BufferLen": 10000,
 		"Colorize": false,
 		"Expression": "",
+		"Exclusion": "",
 		"Flags": "stdflags",
 		"Level": "info",
 		"Prefix": "",
@@ -121,6 +123,7 @@ MODE = console
 		"BufferLen": 10000,
 		"Colorize": false,
 		"Expression": "",
+		"Exclusion": "",
 		"Flags": "stdflags",
 		"Level": "info",
 		"Prefix": "",
@@ -168,6 +171,7 @@ ACCESS = file
 		"BufferLen": 10000,
 		"Colorize": false,
 		"Expression": "",
+		"Exclusion": "",
 		"Flags": "stdflags",
 		"Level": "info",
 		"Prefix": "",
@@ -191,6 +195,7 @@ ACCESS = file
 		"BufferLen": 10000,
 		"Colorize": false,
 		"Expression": "",
+		"Exclusion": "",
 		"Flags": "none",
 		"Level": "info",
 		"Prefix": "",
@@ -257,6 +262,7 @@ STDERR = true
 		"BufferLen": 10000,
 		"Colorize": false,
 		"Expression": "",
+		"Exclusion": "",
 		"Flags": "stdflags",
 		"Level": "warn",
 		"Prefix": "",
@@ -270,6 +276,7 @@ STDERR = true
 		"BufferLen": 10000,
 		"Colorize": false,
 		"Expression": "",
+		"Exclusion": "",
 		"Flags": "stdflags",
 		"Level": "error",
 		"Prefix": "",
@@ -287,6 +294,7 @@ STDERR = true
 		"BufferLen": 10000,
 		"Colorize": false,
 		"Expression": "",
+		"Exclusion": "",
 		"Flags": "none",
 		"Level": "warn",
 		"Prefix": "",
@@ -323,6 +331,7 @@ MODE = file
 LEVEL = error
 STACKTRACE_LEVEL = fatal
 EXPRESSION = filter
+EXCLUSION = not
 FLAGS = medfile
 PREFIX = "[Prefix] "
 FILE_NAME = file-xxx.log
@@ -341,6 +350,7 @@ COMPRESSION_LEVEL = 4
 		"BufferLen": 10,
 		"Colorize": false,
 		"Expression": "",
+		"Exclusion": "",
 		"Flags": "stdflags",
 		"Level": "info",
 		"Prefix": "",
@@ -360,6 +370,7 @@ COMPRESSION_LEVEL = 4
 		"BufferLen": 10,
 		"Colorize": false,
 		"Expression": "filter",
+		"Exclusion": "not",
 		"Flags": "medfile",
 		"Level": "error",
 		"Prefix": "[Prefix] ",

routers/private/manager.go

@@ -145,6 +145,7 @@ func AddLogger(ctx *context.PrivateContext) {
 
 	writerMode.Prefix, _ = opts.Config["prefix"].(string)
 	writerMode.Expression, _ = opts.Config["expression"].(string)
+	writerMode.Exclusion, _ = opts.Config["exclusion"].(string)
 
 	switch writerType {
 	case "console":