From a9f02ecdac5dc1de5b9094b85634e9ca30bc6d73 Mon Sep 17 00:00:00 2001 From: Suyunjing <69945917+Suyunmeng@users.noreply.github.com> Date: Thu, 24 Jul 2025 16:10:47 +0800 Subject: [PATCH] refactor(log):Refactor log filtering to use centralized configuration and add server-specific filtering (#798) * feat(log):Add configurable log filtering middleware for HTTP requests Implement a comprehensive log filtering system that allows selective suppression of HTTP request logs based on paths, methods, and prefixes. The system includes environment variable configuration support and filters health checks, WebDAV requests, and HEAD requests by default to reduce log noise. * fix(log):Replace gin.DefaultLogFormatter with custom implementation * Remove filtered logger test file * fix(log):Refactor log filtering to use centralized configuration and add server-specific filtering * fix(log):Add documentation comments for log filtering configuration --- cmd/server.go | 4 +- internal/conf/config.go | 35 ++++++ server/middlewares/config.go | 170 ++++++-------------------- server/middlewares/filtered_logger.go | 18 --- 4 files changed, 77 insertions(+), 150 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index db0ac841..3bba2e1b 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -48,7 +48,7 @@ the address is defined in config file`, gin.SetMode(gin.ReleaseMode) } r := gin.New() - r.Use(middlewares.ConfigurableFilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out)) + r.Use(middlewares.HTTPFilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out)) server.Init(r) var httpHandler http.Handler = r if conf.Conf.Scheme.EnableH2c { @@ -103,7 +103,7 @@ the address is defined in config file`, } if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable { s3r := gin.New() - s3r.Use(middlewares.FilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out)) + s3r.Use(middlewares.S3FilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out)) server.InitS3(s3r) s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port) utils.Log.Infof("start S3 server @ %s", s3Base) diff --git a/internal/conf/config.go b/internal/conf/config.go index dea66082..e34f3f99 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -44,6 +44,32 @@ type LogConfig struct { MaxBackups int `json:"max_backups" env:"MAX_BACKUPS"` MaxAge int `json:"max_age" env:"MAX_AGE"` Compress bool `json:"compress" env:"COMPRESS"` + Filter LogFilterConfig `json:"filter"` // Log filtering configuration (config file only, no env support) +} + +// LogFilterConfig holds configuration for log filtering +// Note: This configuration is only supported via config file, not environment variables +type LogFilterConfig struct { + // EnableFiltering controls whether log filtering is enabled + EnableFiltering bool `json:"enable_filtering"` + + // FilterHealthChecks controls whether to filter health check requests + FilterHealthChecks bool `json:"filter_health_checks"` + + // FilterWebDAV controls whether to filter WebDAV requests (only for HTTP server) + FilterWebDAV bool `json:"filter_webdav"` + + // FilterHEADRequests controls whether to filter HEAD requests + FilterHEADRequests bool `json:"filter_head_requests"` + + // CustomSkipPaths allows adding custom paths to skip + CustomSkipPaths []string `json:"custom_skip_paths"` + + // CustomSkipMethods allows adding custom methods to skip + CustomSkipMethods []string `json:"custom_skip_methods"` + + // CustomSkipPrefixes allows adding custom path prefixes to skip + CustomSkipPrefixes []string `json:"custom_skip_prefixes"` } type TaskConfig struct { @@ -152,6 +178,15 @@ func DefaultConfig(dataDir string) *Config { MaxSize: 50, MaxBackups: 30, MaxAge: 28, + Filter: LogFilterConfig{ + EnableFiltering: true, + FilterHealthChecks: true, + FilterWebDAV: true, + FilterHEADRequests: true, + CustomSkipPaths: []string{}, + CustomSkipMethods: []string{}, + CustomSkipPrefixes: []string{}, + }, }, MaxConnections: 0, MaxConcurrency: 64, diff --git a/server/middlewares/config.go b/server/middlewares/config.go index 2ffb91c4..bf314079 100644 --- a/server/middlewares/config.go +++ b/server/middlewares/config.go @@ -1,145 +1,55 @@ package middlewares import ( - "os" - "strings" - + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) -// LogFilterConfig holds configuration for log filtering -type LogFilterConfig struct { - // EnableFiltering controls whether log filtering is enabled - EnableFiltering bool - - // FilterHealthChecks controls whether to filter health check requests - FilterHealthChecks bool - - // FilterWebDAV controls whether to filter WebDAV requests - FilterWebDAV bool - - // FilterHEADRequests controls whether to filter HEAD requests - FilterHEADRequests bool - - // CustomSkipPaths allows adding custom paths to skip - CustomSkipPaths []string - - // CustomSkipMethods allows adding custom methods to skip - CustomSkipMethods []string - - // CustomSkipPrefixes allows adding custom path prefixes to skip - CustomSkipPrefixes []string -} - -// DefaultLogFilterConfig returns the default configuration -func DefaultLogFilterConfig() LogFilterConfig { - return LogFilterConfig{ - EnableFiltering: true, - FilterHealthChecks: true, - FilterWebDAV: true, - FilterHEADRequests: true, - CustomSkipPaths: []string{}, - CustomSkipMethods: []string{}, - CustomSkipPrefixes: []string{}, - } -} - -// LoadLogFilterConfigFromEnv loads configuration from environment variables -func LoadLogFilterConfigFromEnv() LogFilterConfig { - config := DefaultLogFilterConfig() - - // Check if filtering is enabled - if env := os.Getenv("OPENLIST_LOG_FILTER_ENABLED"); env != "" { - config.EnableFiltering = strings.ToLower(env) == "true" - } - - // Check individual filter options - if env := os.Getenv("OPENLIST_LOG_FILTER_HEALTH_CHECKS"); env != "" { - config.FilterHealthChecks = strings.ToLower(env) == "true" - } - - if env := os.Getenv("OPENLIST_LOG_FILTER_WEBDAV"); env != "" { - config.FilterWebDAV = strings.ToLower(env) == "true" - } - - if env := os.Getenv("OPENLIST_LOG_FILTER_HEAD_REQUESTS"); env != "" { - config.FilterHEADRequests = strings.ToLower(env) == "true" - } - - // Load custom skip paths - if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_PATHS"); env != "" { - config.CustomSkipPaths = strings.Split(env, ",") - for i, path := range config.CustomSkipPaths { - config.CustomSkipPaths[i] = strings.TrimSpace(path) - } - } - - // Load custom skip methods - if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_METHODS"); env != "" { - config.CustomSkipMethods = strings.Split(env, ",") - for i, method := range config.CustomSkipMethods { - config.CustomSkipMethods[i] = strings.TrimSpace(strings.ToUpper(method)) - } - } - - // Load custom skip prefixes - if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_PREFIXES"); env != "" { - config.CustomSkipPrefixes = strings.Split(env, ",") - for i, prefix := range config.CustomSkipPrefixes { - config.CustomSkipPrefixes[i] = strings.TrimSpace(prefix) - } - } - - return config -} - -// ToFilteredLoggerConfig converts LogFilterConfig to FilteredLoggerConfig -func (c LogFilterConfig) ToFilteredLoggerConfig() FilteredLoggerConfig { - if !c.EnableFiltering { - // Return empty config to disable filtering - return FilteredLoggerConfig{ - Output: log.StandardLogger().Out, - } - } - - config := FilteredLoggerConfig{ - Output: log.StandardLogger().Out, - } - - // Add health check paths - if c.FilterHealthChecks { - config.SkipPaths = append(config.SkipPaths, "/ping") - } - - // Add HEAD method filtering - if c.FilterHEADRequests { - config.SkipMethods = append(config.SkipMethods, "HEAD") - } - - // Add WebDAV filtering - if c.FilterWebDAV { - config.SkipPathPrefixes = append(config.SkipPathPrefixes, "/dav/") - config.SkipMethods = append(config.SkipMethods, "PROPFIND") - } - - // Add custom configurations - config.SkipPaths = append(config.SkipPaths, c.CustomSkipPaths...) - config.SkipMethods = append(config.SkipMethods, c.CustomSkipMethods...) - config.SkipPathPrefixes = append(config.SkipPathPrefixes, c.CustomSkipPrefixes...) - - return config -} - -// ConfigurableFilteredLogger returns a filtered logger with configuration loaded from environment -func ConfigurableFilteredLogger() gin.HandlerFunc { - config := LoadLogFilterConfigFromEnv() - loggerConfig := config.ToFilteredLoggerConfig() +// UnifiedFilteredLogger returns a filtered logger using global configuration +// serverType: "http" for main HTTP server, "s3" for S3 server +func UnifiedFilteredLogger(serverType string) gin.HandlerFunc { + config := conf.Conf.Log.Filter if !config.EnableFiltering { // Return standard Gin logger if filtering is disabled return gin.LoggerWithWriter(log.StandardLogger().Out) } + loggerConfig := FilteredLoggerConfig{ + Output: log.StandardLogger().Out, + } + + // Add health check paths + if config.FilterHealthChecks { + loggerConfig.SkipPaths = append(loggerConfig.SkipPaths, "/ping") + } + + // Add HEAD method filtering + if config.FilterHEADRequests { + loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, "HEAD") + } + + // Add WebDAV filtering only for HTTP server (not for S3) + if config.FilterWebDAV && serverType == "http" { + loggerConfig.SkipPathPrefixes = append(loggerConfig.SkipPathPrefixes, "/dav/") + loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, "PROPFIND") + } + + // Add custom configurations + loggerConfig.SkipPaths = append(loggerConfig.SkipPaths, config.CustomSkipPaths...) + loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, config.CustomSkipMethods...) + loggerConfig.SkipPathPrefixes = append(loggerConfig.SkipPathPrefixes, config.CustomSkipPrefixes...) + return FilteredLoggerWithConfig(loggerConfig) +} + +// HTTPFilteredLogger returns a filtered logger for the main HTTP server +func HTTPFilteredLogger() gin.HandlerFunc { + return UnifiedFilteredLogger("http") +} + +// S3FilteredLogger returns a filtered logger for the S3 server +func S3FilteredLogger() gin.HandlerFunc { + return UnifiedFilteredLogger("s3") } \ No newline at end of file diff --git a/server/middlewares/filtered_logger.go b/server/middlewares/filtered_logger.go index 812aac69..ffa5a770 100644 --- a/server/middlewares/filtered_logger.go +++ b/server/middlewares/filtered_logger.go @@ -44,24 +44,6 @@ func FilteredLoggerWithConfig(config FilteredLoggerConfig) gin.HandlerFunc { }) } -// FilteredLogger returns a gin.HandlerFunc (middleware) that logs requests -// but filters out health check and PROPFIND requests -func FilteredLogger() gin.HandlerFunc { - config := FilteredLoggerConfig{ - SkipPaths: []string{ - "/ping", - }, - SkipMethods: []string{ - "HEAD", // Skip HEAD requests for health checks - }, - SkipPathPrefixes: []string{ - "/dav/", // Skip WebDAV PROPFIND requests - }, - Output: log.StandardLogger().Out, - } - - return FilteredLoggerWithConfig(config) -} // shouldSkipLogging determines if a request should be skipped from logging func shouldSkipLogging(path, method string, config FilteredLoggerConfig) bool {