feat(log):Add configurable log filtering middleware for HTTP requests (#782)

* feat(log):Add configurable log filtering middleware for HTTP requests

Implement a comprehensive log filtering system that allows selective suppression of HTTP request logs based on paths, methods, and prefixes. The system includes environment variable configuration support and filters health checks, WebDAV requests, and HEAD requests by default to reduce log noise.

cmd/server.go

@@ -19,6 +19,7 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/internal/fs"
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
 	"github.com/OpenListTeam/OpenList/v4/server"
+	"github.com/OpenListTeam/OpenList/v4/server/middlewares"
 	"github.com/OpenListTeam/sftpd-openlist"
 	ftpserver "github.com/fclairamb/ftpserverlib"
 	"github.com/gin-gonic/gin"
@@ -47,7 +48,7 @@ the address is defined in config file`,
 			gin.SetMode(gin.ReleaseMode)
 		}
 		r := gin.New()
-		r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out))
+		r.Use(middlewares.ConfigurableFilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
 		server.Init(r)
 		var httpHandler http.Handler = r
 		if conf.Conf.Scheme.EnableH2c {
@@ -102,7 +103,7 @@ the address is defined in config file`,
 		}
 		if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable {
 			s3r := gin.New()
-			s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out))
+			s3r.Use(middlewares.FilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
 			server.InitS3(s3r)
 			s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port)
 			utils.Log.Infof("start S3 server @ %s", s3Base)

server/middlewares/config.go

@@ -0,0 +1,145 @@
+package middlewares
+
+import (
+	"os"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+	log "github.com/sirupsen/logrus"
+)
+
+// LogFilterConfig holds configuration for log filtering
+type LogFilterConfig struct {
+	// EnableFiltering controls whether log filtering is enabled
+	EnableFiltering bool
+	
+	// FilterHealthChecks controls whether to filter health check requests
+	FilterHealthChecks bool
+	
+	// FilterWebDAV controls whether to filter WebDAV requests
+	FilterWebDAV bool
+	
+	// FilterHEADRequests controls whether to filter HEAD requests
+	FilterHEADRequests bool
+	
+	// CustomSkipPaths allows adding custom paths to skip
+	CustomSkipPaths []string
+	
+	// CustomSkipMethods allows adding custom methods to skip
+	CustomSkipMethods []string
+	
+	// CustomSkipPrefixes allows adding custom path prefixes to skip
+	CustomSkipPrefixes []string
+}
+
+// DefaultLogFilterConfig returns the default configuration
+func DefaultLogFilterConfig() LogFilterConfig {
+	return LogFilterConfig{
+		EnableFiltering:    true,
+		FilterHealthChecks: true,
+		FilterWebDAV:       true,
+		FilterHEADRequests: true,
+		CustomSkipPaths:    []string{},
+		CustomSkipMethods:  []string{},
+		CustomSkipPrefixes: []string{},
+	}
+}
+
+// LoadLogFilterConfigFromEnv loads configuration from environment variables
+func LoadLogFilterConfigFromEnv() LogFilterConfig {
+	config := DefaultLogFilterConfig()
+	
+	// Check if filtering is enabled
+	if env := os.Getenv("OPENLIST_LOG_FILTER_ENABLED"); env != "" {
+		config.EnableFiltering = strings.ToLower(env) == "true"
+	}
+	
+	// Check individual filter options
+	if env := os.Getenv("OPENLIST_LOG_FILTER_HEALTH_CHECKS"); env != "" {
+		config.FilterHealthChecks = strings.ToLower(env) == "true"
+	}
+	
+	if env := os.Getenv("OPENLIST_LOG_FILTER_WEBDAV"); env != "" {
+		config.FilterWebDAV = strings.ToLower(env) == "true"
+	}
+	
+	if env := os.Getenv("OPENLIST_LOG_FILTER_HEAD_REQUESTS"); env != "" {
+		config.FilterHEADRequests = strings.ToLower(env) == "true"
+	}
+	
+	// Load custom skip paths
+	if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_PATHS"); env != "" {
+		config.CustomSkipPaths = strings.Split(env, ",")
+		for i, path := range config.CustomSkipPaths {
+			config.CustomSkipPaths[i] = strings.TrimSpace(path)
+		}
+	}
+	
+	// Load custom skip methods
+	if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_METHODS"); env != "" {
+		config.CustomSkipMethods = strings.Split(env, ",")
+		for i, method := range config.CustomSkipMethods {
+			config.CustomSkipMethods[i] = strings.TrimSpace(strings.ToUpper(method))
+		}
+	}
+	
+	// Load custom skip prefixes
+	if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_PREFIXES"); env != "" {
+		config.CustomSkipPrefixes = strings.Split(env, ",")
+		for i, prefix := range config.CustomSkipPrefixes {
+			config.CustomSkipPrefixes[i] = strings.TrimSpace(prefix)
+		}
+	}
+	
+	return config
+}
+
+// ToFilteredLoggerConfig converts LogFilterConfig to FilteredLoggerConfig
+func (c LogFilterConfig) ToFilteredLoggerConfig() FilteredLoggerConfig {
+	if !c.EnableFiltering {
+		// Return empty config to disable filtering
+		return FilteredLoggerConfig{
+			Output: log.StandardLogger().Out,
+		}
+	}
+	
+	config := FilteredLoggerConfig{
+		Output: log.StandardLogger().Out,
+	}
+	
+	// Add health check paths
+	if c.FilterHealthChecks {
+		config.SkipPaths = append(config.SkipPaths, "/ping")
+	}
+	
+	// Add HEAD method filtering
+	if c.FilterHEADRequests {
+		config.SkipMethods = append(config.SkipMethods, "HEAD")
+	}
+	
+	// Add WebDAV filtering
+	if c.FilterWebDAV {
+		config.SkipPathPrefixes = append(config.SkipPathPrefixes, "/dav/")
+		config.SkipMethods = append(config.SkipMethods, "PROPFIND")
+	}
+	
+	// Add custom configurations
+	config.SkipPaths = append(config.SkipPaths, c.CustomSkipPaths...)
+	config.SkipMethods = append(config.SkipMethods, c.CustomSkipMethods...)
+	config.SkipPathPrefixes = append(config.SkipPathPrefixes, c.CustomSkipPrefixes...)
+	
+	return config
+}
+
+// ConfigurableFilteredLogger returns a filtered logger with configuration loaded from environment
+func ConfigurableFilteredLogger() gin.HandlerFunc {
+	config := LoadLogFilterConfigFromEnv()
+	loggerConfig := config.ToFilteredLoggerConfig()
+	
+	if !config.EnableFiltering {
+		// Return standard Gin logger if filtering is disabled
+		return gin.LoggerWithWriter(log.StandardLogger().Out)
+	}
+	
+	return FilteredLoggerWithConfig(loggerConfig)
+}

server/middlewares/filtered_logger.go

@@ -0,0 +1,119 @@
+package middlewares
+
+import (
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	log "github.com/sirupsen/logrus"
+)
+
+// FilteredLoggerConfig defines the configuration for the filtered logger
+type FilteredLoggerConfig struct {
+	// SkipPaths is a list of URL paths to skip logging
+	SkipPaths []string
+	// SkipMethods is a list of HTTP methods to skip logging
+	SkipMethods []string
+	// SkipPathPrefixes is a list of URL path prefixes to skip logging
+	SkipPathPrefixes []string
+	// Output is the writer where logs will be written
+	Output io.Writer
+}
+
+// FilteredLoggerWithConfig returns a gin.HandlerFunc (middleware) that logs requests
+// but skips logging for specified paths, methods, or path prefixes
+func FilteredLoggerWithConfig(config FilteredLoggerConfig) gin.HandlerFunc {
+	if config.Output == nil {
+		config.Output = log.StandardLogger().Out
+	}
+
+	return gin.LoggerWithConfig(gin.LoggerConfig{
+		Output: config.Output,
+		SkipPaths: config.SkipPaths,
+		Formatter: func(param gin.LogFormatterParams) string {
+			// Skip logging for health check endpoints
+			if shouldSkipLogging(param.Path, param.Method, config) {
+				return ""
+			}
+
+			// Use a custom log format similar to Gin's default
+			return defaultLogFormatter(param)
+		},
+	})
+}
+
+// FilteredLogger returns a gin.HandlerFunc (middleware) that logs requests
+// but filters out health check and PROPFIND requests
+func FilteredLogger() gin.HandlerFunc {
+	config := FilteredLoggerConfig{
+		SkipPaths: []string{
+			"/ping",
+		},
+		SkipMethods: []string{
+			"HEAD", // Skip HEAD requests for health checks
+		},
+		SkipPathPrefixes: []string{
+			"/dav/", // Skip WebDAV PROPFIND requests
+		},
+		Output: log.StandardLogger().Out,
+	}
+
+	return FilteredLoggerWithConfig(config)
+}
+
+// shouldSkipLogging determines if a request should be skipped from logging
+func shouldSkipLogging(path, method string, config FilteredLoggerConfig) bool {
+	// Check if path should be skipped
+	for _, skipPath := range config.SkipPaths {
+		if path == skipPath {
+			return true
+		}
+	}
+
+	// Check if method should be skipped
+	for _, skipMethod := range config.SkipMethods {
+		if method == skipMethod {
+			return true
+		}
+	}
+
+	// Check if path prefix should be skipped
+	for _, skipPrefix := range config.SkipPathPrefixes {
+		if strings.HasPrefix(path, skipPrefix) {
+			return true
+		}
+	}
+
+	// Special case: Skip PROPFIND requests (common in WebDAV)
+	if method == "PROPFIND" {
+		return true
+	}
+
+	return false
+}
+
+// defaultLogFormatter provides a default log format similar to Gin's built-in formatter
+func defaultLogFormatter(param gin.LogFormatterParams) string {
+	var statusColor, methodColor, resetColor string
+	if param.IsOutputColor() {
+		statusColor = param.StatusCodeColor()
+		methodColor = param.MethodColor()
+		resetColor = param.ResetColor()
+	}
+
+	if param.Latency > time.Minute {
+		param.Latency = param.Latency.Truncate(time.Second)
+	}
+
+	return fmt.Sprintf("[GIN] %v |%s %3d %s| %13v | %15s |%s %-7s %s %#v\n%s",
+		param.TimeStamp.Format("2006/01/02 - 15:04:05"),
+		statusColor, param.StatusCode, resetColor,
+		param.Latency,
+		param.ClientIP,
+		methodColor, param.Method, resetColor,
+		param.Path,
+		param.ErrorMessage,
+	)
+}