diff --git a/cmd/server.go b/cmd/server.go index 3bba2e1b..3447ee50 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -48,7 +48,15 @@ the address is defined in config file`, gin.SetMode(gin.ReleaseMode) } r := gin.New() - r.Use(middlewares.HTTPFilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out)) + + // gin log + if conf.Conf.Log.Filter.Enable { + r.Use(middlewares.FilteredLogger()) + } else { + r.Use(gin.LoggerWithWriter(log.StandardLogger().Out)) + } + r.Use(gin.RecoveryWithWriter(log.StandardLogger().Out)) + server.Init(r) var httpHandler http.Handler = r if conf.Conf.Scheme.EnableH2c { @@ -103,7 +111,7 @@ the address is defined in config file`, } if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable { s3r := gin.New() - s3r.Use(middlewares.S3FilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out)) + s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out)) server.InitS3(s3r) s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port) utils.Log.Infof("start S3 server @ %s", s3Base) diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index f1f0767f..db8dcf28 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -82,6 +82,9 @@ func InitConfig() { if !conf.Conf.Force { confFromEnv() } + if len(conf.Conf.Log.Filter.Filters) == 0 { + conf.Conf.Log.Filter.Enable = false + } // convert abs path convertAbsPath := func(path *string) { if !filepath.IsAbs(*path) { diff --git a/internal/conf/config.go b/internal/conf/config.go index e34f3f99..e4e3ab0e 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -38,38 +38,24 @@ type Scheme struct { } type LogConfig struct { - Enable bool `json:"enable" env:"LOG_ENABLE"` - Name string `json:"name" env:"LOG_NAME"` - MaxSize int `json:"max_size" env:"MAX_SIZE"` - MaxBackups int `json:"max_backups" env:"MAX_BACKUPS"` - MaxAge int `json:"max_age" env:"MAX_AGE"` - Compress bool `json:"compress" env:"COMPRESS"` - Filter LogFilterConfig `json:"filter"` // Log filtering configuration (config file only, no env support) + Enable bool `json:"enable" env:"ENABLE"` + Name string `json:"name" env:"NAME"` + MaxSize int `json:"max_size" env:"MAX_SIZE"` + MaxBackups int `json:"max_backups" env:"MAX_BACKUPS"` + MaxAge int `json:"max_age" env:"MAX_AGE"` + Compress bool `json:"compress" env:"COMPRESS"` + Filter LogFilterConfig `json:"filter" envPrefix:"FILTER_"` } -// LogFilterConfig holds configuration for log filtering -// Note: This configuration is only supported via config file, not environment variables type LogFilterConfig struct { - // EnableFiltering controls whether log filtering is enabled - EnableFiltering bool `json:"enable_filtering"` - - // FilterHealthChecks controls whether to filter health check requests - FilterHealthChecks bool `json:"filter_health_checks"` - - // FilterWebDAV controls whether to filter WebDAV requests (only for HTTP server) - FilterWebDAV bool `json:"filter_webdav"` - - // FilterHEADRequests controls whether to filter HEAD requests - FilterHEADRequests bool `json:"filter_head_requests"` - - // CustomSkipPaths allows adding custom paths to skip - CustomSkipPaths []string `json:"custom_skip_paths"` - - // CustomSkipMethods allows adding custom methods to skip - CustomSkipMethods []string `json:"custom_skip_methods"` - - // CustomSkipPrefixes allows adding custom path prefixes to skip - CustomSkipPrefixes []string `json:"custom_skip_prefixes"` + Enable bool `json:"enable" env:"ENABLE"` + Filters []Filter `json:"filters"` +} + +type Filter struct { + CIDR string `json:"cidr"` + Path string `json:"path"` + Method string `json:"method"` } type TaskConfig struct { @@ -131,7 +117,7 @@ type Config struct { TempDir string `json:"temp_dir" env:"TEMP_DIR"` BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` DistDir string `json:"dist_dir"` - Log LogConfig `json:"log"` + Log LogConfig `json:"log" envPrefix:"LOG_"` DelayedStart int `json:"delayed_start" env:"DELAYED_START"` MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"` MaxConcurrency int `json:"max_concurrency" env:"MAX_CONCURRENCY"` @@ -179,13 +165,12 @@ func DefaultConfig(dataDir string) *Config { MaxBackups: 30, MaxAge: 28, Filter: LogFilterConfig{ - EnableFiltering: true, - FilterHealthChecks: true, - FilterWebDAV: true, - FilterHEADRequests: true, - CustomSkipPaths: []string{}, - CustomSkipMethods: []string{}, - CustomSkipPrefixes: []string{}, + Enable: false, + Filters: []Filter{ + {Path: "/ping"}, + {Method: "HEAD"}, + {Path: "/dav/", Method: "PROPFIND"}, + }, }, }, MaxConnections: 0, diff --git a/server/middlewares/config.go b/server/middlewares/config.go deleted file mode 100644 index bf314079..00000000 --- a/server/middlewares/config.go +++ /dev/null @@ -1,55 +0,0 @@ -package middlewares - -import ( - "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/gin-gonic/gin" - log "github.com/sirupsen/logrus" -) - -// UnifiedFilteredLogger returns a filtered logger using global configuration -// serverType: "http" for main HTTP server, "s3" for S3 server -func UnifiedFilteredLogger(serverType string) gin.HandlerFunc { - config := conf.Conf.Log.Filter - - if !config.EnableFiltering { - // Return standard Gin logger if filtering is disabled - return gin.LoggerWithWriter(log.StandardLogger().Out) - } - - loggerConfig := FilteredLoggerConfig{ - Output: log.StandardLogger().Out, - } - - // Add health check paths - if config.FilterHealthChecks { - loggerConfig.SkipPaths = append(loggerConfig.SkipPaths, "/ping") - } - - // Add HEAD method filtering - if config.FilterHEADRequests { - loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, "HEAD") - } - - // Add WebDAV filtering only for HTTP server (not for S3) - if config.FilterWebDAV && serverType == "http" { - loggerConfig.SkipPathPrefixes = append(loggerConfig.SkipPathPrefixes, "/dav/") - loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, "PROPFIND") - } - - // Add custom configurations - loggerConfig.SkipPaths = append(loggerConfig.SkipPaths, config.CustomSkipPaths...) - loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, config.CustomSkipMethods...) - loggerConfig.SkipPathPrefixes = append(loggerConfig.SkipPathPrefixes, config.CustomSkipPrefixes...) - - return FilteredLoggerWithConfig(loggerConfig) -} - -// HTTPFilteredLogger returns a filtered logger for the main HTTP server -func HTTPFilteredLogger() gin.HandlerFunc { - return UnifiedFilteredLogger("http") -} - -// S3FilteredLogger returns a filtered logger for the S3 server -func S3FilteredLogger() gin.HandlerFunc { - return UnifiedFilteredLogger("s3") -} \ No newline at end of file diff --git a/server/middlewares/filtered_logger.go b/server/middlewares/filtered_logger.go index ffa5a770..2a51a14c 100644 --- a/server/middlewares/filtered_logger.go +++ b/server/middlewares/filtered_logger.go @@ -1,101 +1,99 @@ package middlewares import ( - "fmt" - "io" + "net/netip" "strings" - "time" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) -// FilteredLoggerConfig defines the configuration for the filtered logger -type FilteredLoggerConfig struct { - // SkipPaths is a list of URL paths to skip logging - SkipPaths []string - // SkipMethods is a list of HTTP methods to skip logging - SkipMethods []string - // SkipPathPrefixes is a list of URL path prefixes to skip logging - SkipPathPrefixes []string - // Output is the writer where logs will be written - Output io.Writer +type filter struct { + CIDR *netip.Prefix `json:"cidr,omitempty"` + Path *string `json:"path,omitempty"` + Method *string `json:"method,omitempty"` } -// FilteredLoggerWithConfig returns a gin.HandlerFunc (middleware) that logs requests -// but skips logging for specified paths, methods, or path prefixes -func FilteredLoggerWithConfig(config FilteredLoggerConfig) gin.HandlerFunc { - if config.Output == nil { - config.Output = log.StandardLogger().Out - } +var filterList []*filter - return gin.LoggerWithConfig(gin.LoggerConfig{ - Output: config.Output, - SkipPaths: config.SkipPaths, - Formatter: func(param gin.LogFormatterParams) string { - // Skip logging for health check endpoints - if shouldSkipLogging(param.Path, param.Method, config) { - return "" +func initFilterList() { + for _, s := range conf.Conf.Log.Filter.Filters { + f := new(filter) + + if s.CIDR != "" { + cidr, err := netip.ParsePrefix(s.CIDR) + if err != nil { + log.Errorf("failed to parse CIDR %s: %v", s.CIDR, err) + continue } + f.CIDR = &cidr + } - // Use a custom log format similar to Gin's default - return defaultLogFormatter(param) - }, - }) + if s.Path != "" { + f.Path = &s.Path + } + + if s.Method != "" { + f.Method = &s.Method + } + + if f.CIDR == nil && f.Path == nil && f.Method == nil { + log.Warnf("filter %s is empty, skipping", s) + continue + } + + filterList = append(filterList, f) + log.Debugf("added filter: %+v", f) + } + + log.Infof("Loaded %d log filters.", len(filterList)) } +func skiperDecider(c *gin.Context) bool { + // every filter need metch all condithon as filter match + // so if any condithon not metch, skip this filter + // all filters misatch, log this request -// shouldSkipLogging determines if a request should be skipped from logging -func shouldSkipLogging(path, method string, config FilteredLoggerConfig) bool { - // Check if path should be skipped - for _, skipPath := range config.SkipPaths { - if path == skipPath { - return true + for _, f := range filterList { + if f.CIDR != nil { + cip := netip.MustParseAddr(c.ClientIP()) + if !f.CIDR.Contains(cip) { + continue + } } - } - // Check if method should be skipped - for _, skipMethod := range config.SkipMethods { - if method == skipMethod { - return true + if f.Path != nil { + if (*f.Path)[0] == '/' { + // match path as prefix/exact path + if !strings.HasPrefix(c.Request.URL.Path, *f.Path) { + continue + } + } else { + // match path as relative path + if !strings.Contains(c.Request.URL.Path, "/"+*f.Path) { + continue + } + } } - } - // Check if path prefix should be skipped - for _, skipPrefix := range config.SkipPathPrefixes { - if strings.HasPrefix(path, skipPrefix) { - return true + if f.Method != nil { + if *f.Method != c.Request.Method { + continue + } } - } - // Special case: Skip PROPFIND requests (common in WebDAV) - if method == "PROPFIND" { return true } return false } -// defaultLogFormatter provides a default log format similar to Gin's built-in formatter -func defaultLogFormatter(param gin.LogFormatterParams) string { - var statusColor, methodColor, resetColor string - if param.IsOutputColor() { - statusColor = param.StatusCodeColor() - methodColor = param.MethodColor() - resetColor = param.ResetColor() - } +func FilteredLogger() gin.HandlerFunc { + initFilterList() - if param.Latency > time.Minute { - param.Latency = param.Latency.Truncate(time.Second) - } - - return fmt.Sprintf("[GIN] %v |%s %3d %s| %13v | %15s |%s %-7s %s %#v\n%s", - param.TimeStamp.Format("2006/01/02 - 15:04:05"), - statusColor, param.StatusCode, resetColor, - param.Latency, - param.ClientIP, - methodColor, param.Method, resetColor, - param.Path, - param.ErrorMessage, - ) -} \ No newline at end of file + return gin.LoggerWithConfig(gin.LoggerConfig{ + Output: log.StandardLogger().Out, + Skip: skiperDecider, + }) +}