refactor(log):Refactor log filtering to use centralized configuration and add server-specific filtering (#798)

* feat(log):Add configurable log filtering middleware for HTTP requests

Implement a comprehensive log filtering system that allows selective suppression of HTTP request logs based on paths, methods, and prefixes. The system includes environment variable configuration support and filters health checks, WebDAV requests, and HEAD requests by default to reduce log noise.

* fix(log):Replace gin.DefaultLogFormatter with custom implementation

* Remove filtered logger test file

* fix(log):Refactor log filtering to use centralized configuration and add server-specific filtering

* fix(log):Add documentation comments for log filtering configuration

cmd/server.go

@@ -48,7 +48,7 @@ the address is defined in config file`,
 			gin.SetMode(gin.ReleaseMode)
 		}
 		r := gin.New()
-		r.Use(middlewares.ConfigurableFilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
+		r.Use(middlewares.HTTPFilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
 		server.Init(r)
 		var httpHandler http.Handler = r
 		if conf.Conf.Scheme.EnableH2c {
@@ -103,7 +103,7 @@ the address is defined in config file`,
 		}
 		if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable {
 			s3r := gin.New()
-			s3r.Use(middlewares.FilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
+			s3r.Use(middlewares.S3FilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
 			server.InitS3(s3r)
 			s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port)
 			utils.Log.Infof("start S3 server @ %s", s3Base)

internal/conf/config.go

@@ -44,6 +44,32 @@ type LogConfig struct {
 	MaxBackups int    `json:"max_backups" env:"MAX_BACKUPS"`
 	MaxAge     int    `json:"max_age" env:"MAX_AGE"`
 	Compress   bool   `json:"compress" env:"COMPRESS"`
+	Filter     LogFilterConfig `json:"filter"` // Log filtering configuration (config file only, no env support)
+}
+
+// LogFilterConfig holds configuration for log filtering
+// Note: This configuration is only supported via config file, not environment variables
+type LogFilterConfig struct {
+	// EnableFiltering controls whether log filtering is enabled
+	EnableFiltering bool `json:"enable_filtering"`
+	
+	// FilterHealthChecks controls whether to filter health check requests
+	FilterHealthChecks bool `json:"filter_health_checks"`
+	
+	// FilterWebDAV controls whether to filter WebDAV requests (only for HTTP server)
+	FilterWebDAV bool `json:"filter_webdav"`
+	
+	// FilterHEADRequests controls whether to filter HEAD requests
+	FilterHEADRequests bool `json:"filter_head_requests"`
+	
+	// CustomSkipPaths allows adding custom paths to skip
+	CustomSkipPaths []string `json:"custom_skip_paths"`
+	
+	// CustomSkipMethods allows adding custom methods to skip
+	CustomSkipMethods []string `json:"custom_skip_methods"`
+	
+	// CustomSkipPrefixes allows adding custom path prefixes to skip
+	CustomSkipPrefixes []string `json:"custom_skip_prefixes"`
 }
 
 type TaskConfig struct {
@@ -152,6 +178,15 @@ func DefaultConfig(dataDir string) *Config {
 			MaxSize:    50,
 			MaxBackups: 30,
 			MaxAge:     28,
+			Filter: LogFilterConfig{
+				EnableFiltering:    true,
+				FilterHealthChecks: true,
+				FilterWebDAV:       true,
+				FilterHEADRequests: true,
+				CustomSkipPaths:    []string{},
+				CustomSkipMethods:  []string{},
+				CustomSkipPrefixes: []string{},
+			},
 		},
 		MaxConnections:        0,
 		MaxConcurrency:        64,

server/middlewares/config.go

@@ -1,145 +1,55 @@
 package middlewares
 
 import (
-	"os"
+	"github.com/OpenListTeam/OpenList/v4/internal/conf"
-	"strings"
-
 	"github.com/gin-gonic/gin"
 	log "github.com/sirupsen/logrus"
 )
 
-// LogFilterConfig holds configuration for log filtering
+// UnifiedFilteredLogger returns a filtered logger using global configuration
-type LogFilterConfig struct {
+// serverType: "http" for main HTTP server, "s3" for S3 server
-	// EnableFiltering controls whether log filtering is enabled
+func UnifiedFilteredLogger(serverType string) gin.HandlerFunc {
-	EnableFiltering bool
+	config := conf.Conf.Log.Filter
-	
-	// FilterHealthChecks controls whether to filter health check requests
-	FilterHealthChecks bool
-	
-	// FilterWebDAV controls whether to filter WebDAV requests
-	FilterWebDAV bool
-	
-	// FilterHEADRequests controls whether to filter HEAD requests
-	FilterHEADRequests bool
-	
-	// CustomSkipPaths allows adding custom paths to skip
-	CustomSkipPaths []string
-	
-	// CustomSkipMethods allows adding custom methods to skip
-	CustomSkipMethods []string
-	
-	// CustomSkipPrefixes allows adding custom path prefixes to skip
-	CustomSkipPrefixes []string
-}
-
-// DefaultLogFilterConfig returns the default configuration
-func DefaultLogFilterConfig() LogFilterConfig {
-	return LogFilterConfig{
-		EnableFiltering:    true,
-		FilterHealthChecks: true,
-		FilterWebDAV:       true,
-		FilterHEADRequests: true,
-		CustomSkipPaths:    []string{},
-		CustomSkipMethods:  []string{},
-		CustomSkipPrefixes: []string{},
-	}
-}
-
-// LoadLogFilterConfigFromEnv loads configuration from environment variables
-func LoadLogFilterConfigFromEnv() LogFilterConfig {
-	config := DefaultLogFilterConfig()
-	
-	// Check if filtering is enabled
-	if env := os.Getenv("OPENLIST_LOG_FILTER_ENABLED"); env != "" {
-		config.EnableFiltering = strings.ToLower(env) == "true"
-	}
-	
-	// Check individual filter options
-	if env := os.Getenv("OPENLIST_LOG_FILTER_HEALTH_CHECKS"); env != "" {
-		config.FilterHealthChecks = strings.ToLower(env) == "true"
-	}
-	
-	if env := os.Getenv("OPENLIST_LOG_FILTER_WEBDAV"); env != "" {
-		config.FilterWebDAV = strings.ToLower(env) == "true"
-	}
-	
-	if env := os.Getenv("OPENLIST_LOG_FILTER_HEAD_REQUESTS"); env != "" {
-		config.FilterHEADRequests = strings.ToLower(env) == "true"
-	}
-	
-	// Load custom skip paths
-	if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_PATHS"); env != "" {
-		config.CustomSkipPaths = strings.Split(env, ",")
-		for i, path := range config.CustomSkipPaths {
-			config.CustomSkipPaths[i] = strings.TrimSpace(path)
-		}
-	}
-	
-	// Load custom skip methods
-	if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_METHODS"); env != "" {
-		config.CustomSkipMethods = strings.Split(env, ",")
-		for i, method := range config.CustomSkipMethods {
-			config.CustomSkipMethods[i] = strings.TrimSpace(strings.ToUpper(method))
-		}
-	}
-	
-	// Load custom skip prefixes
-	if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_PREFIXES"); env != "" {
-		config.CustomSkipPrefixes = strings.Split(env, ",")
-		for i, prefix := range config.CustomSkipPrefixes {
-			config.CustomSkipPrefixes[i] = strings.TrimSpace(prefix)
-		}
-	}
-	
-	return config
-}
-
-// ToFilteredLoggerConfig converts LogFilterConfig to FilteredLoggerConfig
-func (c LogFilterConfig) ToFilteredLoggerConfig() FilteredLoggerConfig {
-	if !c.EnableFiltering {
-		// Return empty config to disable filtering
-		return FilteredLoggerConfig{
-			Output: log.StandardLogger().Out,
-		}
-	}
-	
-	config := FilteredLoggerConfig{
-		Output: log.StandardLogger().Out,
-	}
-	
-	// Add health check paths
-	if c.FilterHealthChecks {
-		config.SkipPaths = append(config.SkipPaths, "/ping")
-	}
-	
-	// Add HEAD method filtering
-	if c.FilterHEADRequests {
-		config.SkipMethods = append(config.SkipMethods, "HEAD")
-	}
-	
-	// Add WebDAV filtering
-	if c.FilterWebDAV {
-		config.SkipPathPrefixes = append(config.SkipPathPrefixes, "/dav/")
-		config.SkipMethods = append(config.SkipMethods, "PROPFIND")
-	}
-	
-	// Add custom configurations
-	config.SkipPaths = append(config.SkipPaths, c.CustomSkipPaths...)
-	config.SkipMethods = append(config.SkipMethods, c.CustomSkipMethods...)
-	config.SkipPathPrefixes = append(config.SkipPathPrefixes, c.CustomSkipPrefixes...)
-	
-	return config
-}
-
-// ConfigurableFilteredLogger returns a filtered logger with configuration loaded from environment
-func ConfigurableFilteredLogger() gin.HandlerFunc {
-	config := LoadLogFilterConfigFromEnv()
-	loggerConfig := config.ToFilteredLoggerConfig()
 	
 	if !config.EnableFiltering {
 		// Return standard Gin logger if filtering is disabled
 		return gin.LoggerWithWriter(log.StandardLogger().Out)
 	}
 	
+	loggerConfig := FilteredLoggerConfig{
+		Output: log.StandardLogger().Out,
+	}
+	
+	// Add health check paths
+	if config.FilterHealthChecks {
+		loggerConfig.SkipPaths = append(loggerConfig.SkipPaths, "/ping")
+	}
+	
+	// Add HEAD method filtering
+	if config.FilterHEADRequests {
+		loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, "HEAD")
+	}
+	
+	// Add WebDAV filtering only for HTTP server (not for S3)
+	if config.FilterWebDAV && serverType == "http" {
+		loggerConfig.SkipPathPrefixes = append(loggerConfig.SkipPathPrefixes, "/dav/")
+		loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, "PROPFIND")
+	}
+	
+	// Add custom configurations
+	loggerConfig.SkipPaths = append(loggerConfig.SkipPaths, config.CustomSkipPaths...)
+	loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, config.CustomSkipMethods...)
+	loggerConfig.SkipPathPrefixes = append(loggerConfig.SkipPathPrefixes, config.CustomSkipPrefixes...)
+	
 	return FilteredLoggerWithConfig(loggerConfig)
+}
+
+// HTTPFilteredLogger returns a filtered logger for the main HTTP server
+func HTTPFilteredLogger() gin.HandlerFunc {
+	return UnifiedFilteredLogger("http")
+}
+
+// S3FilteredLogger returns a filtered logger for the S3 server
+func S3FilteredLogger() gin.HandlerFunc {
+	return UnifiedFilteredLogger("s3")
 }

server/middlewares/filtered_logger.go

@@ -44,24 +44,6 @@ func FilteredLoggerWithConfig(config FilteredLoggerConfig) gin.HandlerFunc {
 	})
 }
 
-// FilteredLogger returns a gin.HandlerFunc (middleware) that logs requests
-// but filters out health check and PROPFIND requests
-func FilteredLogger() gin.HandlerFunc {
-	config := FilteredLoggerConfig{
-		SkipPaths: []string{
-			"/ping",
-		},
-		SkipMethods: []string{
-			"HEAD", // Skip HEAD requests for health checks
-		},
-		SkipPathPrefixes: []string{
-			"/dav/", // Skip WebDAV PROPFIND requests
-		},
-		Output: log.StandardLogger().Out,
-	}
-
-	return FilteredLoggerWithConfig(config)
-}
 
 // shouldSkipLogging determines if a request should be skipped from logging
 func shouldSkipLogging(path, method string, config FilteredLoggerConfig) bool {