Backups/OpenList
1
0
Fork 0
mirror of https://github.com/OpenListTeam/OpenList.git synced 2025-09-19 12:16:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(log): filter (#816)

Browse Source
This commit is contained in:
Ljcbaby
2025-07-25 11:33:27 +08:00
committed by GitHub
parent e8a1ed638a
commit 358e4d851e
5 changed files with 104 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/server.go
View File

@ -48,7 +48,15 @@ the address is defined in config file`,
gin.SetMode(gin.ReleaseMode) gin.SetMode(gin.ReleaseMode)
} }
r := gin.New() r := gin.New()
r.Use(middlewares.HTTPFilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
// gin log
if conf.Conf.Log.Filter.Enable {
r.Use(middlewares.FilteredLogger())
} else {
r.Use(gin.LoggerWithWriter(log.StandardLogger().Out))
}
r.Use(gin.RecoveryWithWriter(log.StandardLogger().Out))
server.Init(r) server.Init(r)
var httpHandler http.Handler = r var httpHandler http.Handler = r
if conf.Conf.Scheme.EnableH2c { if conf.Conf.Scheme.EnableH2c {
@ -103,7 +111,7 @@ the address is defined in config file`,
} }
if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable { if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable {
s3r := gin.New() s3r := gin.New()
s3r.Use(middlewares.S3FilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out)) s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out))
server.InitS3(s3r) server.InitS3(s3r)
s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port) s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port)
utils.Log.Infof("start S3 server @ %s", s3Base) utils.Log.Infof("start S3 server @ %s", s3Base)

3
internal/bootstrap/config.go
View File

@ -82,6 +82,9 @@ func InitConfig() {
if !conf.Conf.Force { if !conf.Conf.Force {
confFromEnv() confFromEnv()
} }
if len(conf.Conf.Log.Filter.Filters) == 0 {
conf.Conf.Log.Filter.Enable = false
}
// convert abs path // convert abs path
convertAbsPath := func(path *string) { convertAbsPath := func(path *string) {
if !filepath.IsAbs(*path) { if !filepath.IsAbs(*path) {

49
internal/conf/config.go
View File

@ -38,38 +38,24 @@ type Scheme struct {
} }
type LogConfig struct { type LogConfig struct {
Enable bool `json:"enable" env:"LOG_ENABLE"` Enable bool `json:"enable" env:"ENABLE"`
Name string `json:"name" env:"LOG_NAME"` Name string `json:"name" env:"NAME"`
MaxSize int `json:"max_size" env:"MAX_SIZE"` MaxSize int `json:"max_size" env:"MAX_SIZE"`
MaxBackups int `json:"max_backups" env:"MAX_BACKUPS"` MaxBackups int `json:"max_backups" env:"MAX_BACKUPS"`
MaxAge int `json:"max_age" env:"MAX_AGE"` MaxAge int `json:"max_age" env:"MAX_AGE"`
Compress bool `json:"compress" env:"COMPRESS"` Compress bool `json:"compress" env:"COMPRESS"`
Filter LogFilterConfig `json:"filter"` // Log filtering configuration (config file only, no env support) Filter LogFilterConfig `json:"filter" envPrefix:"FILTER_"`
} }
// LogFilterConfig holds configuration for log filtering
// Note: This configuration is only supported via config file, not environment variables
type LogFilterConfig struct { type LogFilterConfig struct {
// EnableFiltering controls whether log filtering is enabled Enable bool `json:"enable" env:"ENABLE"`
EnableFiltering bool `json:"enable_filtering"` Filters []Filter `json:"filters"`
}
// FilterHealthChecks controls whether to filter health check requests type Filter struct {
FilterHealthChecks bool `json:"filter_health_checks"` CIDR string `json:"cidr"`
Path string `json:"path"`
// FilterWebDAV controls whether to filter WebDAV requests (only for HTTP server) Method string `json:"method"`
FilterWebDAV bool `json:"filter_webdav"`
// FilterHEADRequests controls whether to filter HEAD requests
FilterHEADRequests bool `json:"filter_head_requests"`
// CustomSkipPaths allows adding custom paths to skip
CustomSkipPaths []string `json:"custom_skip_paths"`
// CustomSkipMethods allows adding custom methods to skip
CustomSkipMethods []string `json:"custom_skip_methods"`
// CustomSkipPrefixes allows adding custom path prefixes to skip
CustomSkipPrefixes []string `json:"custom_skip_prefixes"`
} }
type TaskConfig struct { type TaskConfig struct {
@ -131,7 +117,7 @@ type Config struct {
TempDir string `json:"temp_dir" env:"TEMP_DIR"` TempDir string `json:"temp_dir" env:"TEMP_DIR"`
BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"`
DistDir string `json:"dist_dir"` DistDir string `json:"dist_dir"`
Log LogConfig `json:"log"` Log LogConfig `json:"log" envPrefix:"LOG_"`
DelayedStart int `json:"delayed_start" env:"DELAYED_START"` DelayedStart int `json:"delayed_start" env:"DELAYED_START"`
MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"` MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"`
MaxConcurrency int `json:"max_concurrency" env:"MAX_CONCURRENCY"` MaxConcurrency int `json:"max_concurrency" env:"MAX_CONCURRENCY"`
@ -179,13 +165,12 @@ func DefaultConfig(dataDir string) *Config {
MaxBackups: 30, MaxBackups: 30,
MaxAge: 28, MaxAge: 28,
Filter: LogFilterConfig{ Filter: LogFilterConfig{
EnableFiltering: true, Enable: false,
FilterHealthChecks: true, Filters: []Filter{
FilterWebDAV: true, {Path: "/ping"},
FilterHEADRequests: true, {Method: "HEAD"},
CustomSkipPaths: []string{}, {Path: "/dav/", Method: "PROPFIND"},
CustomSkipMethods: []string{}, },
CustomSkipPrefixes: []string{},
}, },
}, },
MaxConnections: 0, MaxConnections: 0,

55
server/middlewares/config.go
View File

@ -1,55 +0,0 @@
package middlewares
import (
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
)
// UnifiedFilteredLogger returns a filtered logger using global configuration
// serverType: "http" for main HTTP server, "s3" for S3 server
func UnifiedFilteredLogger(serverType string) gin.HandlerFunc {
config := conf.Conf.Log.Filter
if !config.EnableFiltering {
// Return standard Gin logger if filtering is disabled
return gin.LoggerWithWriter(log.StandardLogger().Out)
}
loggerConfig := FilteredLoggerConfig{
Output: log.StandardLogger().Out,
}
// Add health check paths
if config.FilterHealthChecks {
loggerConfig.SkipPaths = append(loggerConfig.SkipPaths, "/ping")
}
// Add HEAD method filtering
if config.FilterHEADRequests {
loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, "HEAD")
}
// Add WebDAV filtering only for HTTP server (not for S3)
if config.FilterWebDAV && serverType == "http" {
loggerConfig.SkipPathPrefixes = append(loggerConfig.SkipPathPrefixes, "/dav/")
loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, "PROPFIND")
}
// Add custom configurations
loggerConfig.SkipPaths = append(loggerConfig.SkipPaths, config.CustomSkipPaths...)
loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, config.CustomSkipMethods...)
loggerConfig.SkipPathPrefixes = append(loggerConfig.SkipPathPrefixes, config.CustomSkipPrefixes...)
return FilteredLoggerWithConfig(loggerConfig)
}
// HTTPFilteredLogger returns a filtered logger for the main HTTP server
func HTTPFilteredLogger() gin.HandlerFunc {
return UnifiedFilteredLogger("http")
}
// S3FilteredLogger returns a filtered logger for the S3 server
func S3FilteredLogger() gin.HandlerFunc {
return UnifiedFilteredLogger("s3")
}

130
server/middlewares/filtered_logger.go
View File

@ -1,101 +1,99 @@
package middlewares package middlewares
import ( import (
"fmt" "net/netip"
"io"
"strings" "strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
// FilteredLoggerConfig defines the configuration for the filtered logger type filter struct {
type FilteredLoggerConfig struct { CIDR *netip.Prefix `json:"cidr,omitempty"`
// SkipPaths is a list of URL paths to skip logging Path *string `json:"path,omitempty"`
SkipPaths []string Method *string `json:"method,omitempty"`
// SkipMethods is a list of HTTP methods to skip logging
SkipMethods []string
// SkipPathPrefixes is a list of URL path prefixes to skip logging
SkipPathPrefixes []string
// Output is the writer where logs will be written
Output io.Writer
} }
// FilteredLoggerWithConfig returns a gin.HandlerFunc (middleware) that logs requests var filterList []*filter
// but skips logging for specified paths, methods, or path prefixes
func FilteredLoggerWithConfig(config FilteredLoggerConfig) gin.HandlerFunc { func initFilterList() {
if config.Output == nil { for _, s := range conf.Conf.Log.Filter.Filters {
config.Output = log.StandardLogger().Out f := new(filter)
if s.CIDR != "" {
cidr, err := netip.ParsePrefix(s.CIDR)
if err != nil {
log.Errorf("failed to parse CIDR %s: %v", s.CIDR, err)
continue
}
f.CIDR = &cidr
} }
return gin.LoggerWithConfig(gin.LoggerConfig{ if s.Path != "" {
Output: config.Output, f.Path = &s.Path
SkipPaths: config.SkipPaths,
Formatter: func(param gin.LogFormatterParams) string {
// Skip logging for health check endpoints
if shouldSkipLogging(param.Path, param.Method, config) {
return ""
} }
// Use a custom log format similar to Gin's default if s.Method != "" {
return defaultLogFormatter(param) f.Method = &s.Method
},
})
} }
if f.CIDR == nil && f.Path == nil && f.Method == nil {
log.Warnf("filter %s is empty, skipping", s)
continue
}
// shouldSkipLogging determines if a request should be skipped from logging filterList = append(filterList, f)
func shouldSkipLogging(path, method string, config FilteredLoggerConfig) bool { log.Debugf("added filter: %+v", f)
// Check if path should be skipped }
for _, skipPath := range config.SkipPaths {
if path == skipPath { log.Infof("Loaded %d log filters.", len(filterList))
return true }
func skiperDecider(c *gin.Context) bool {
// every filter need metch all condithon as filter match
// so if any condithon not metch, skip this filter
// all filters misatch, log this request
for _, f := range filterList {
if f.CIDR != nil {
cip := netip.MustParseAddr(c.ClientIP())
if !f.CIDR.Contains(cip) {
continue
} }
} }
// Check if method should be skipped if f.Path != nil {
for _, skipMethod := range config.SkipMethods { if (*f.Path)[0] == '/' {
if method == skipMethod { // match path as prefix/exact path
return true if !strings.HasPrefix(c.Request.URL.Path, *f.Path) {
continue
}
} else {
// match path as relative path
if !strings.Contains(c.Request.URL.Path, "/"+*f.Path) {
continue
}
} }
} }
// Check if path prefix should be skipped if f.Method != nil {
for _, skipPrefix := range config.SkipPathPrefixes { if *f.Method != c.Request.Method {
if strings.HasPrefix(path, skipPrefix) { continue
return true
} }
} }
// Special case: Skip PROPFIND requests (common in WebDAV)
if method == "PROPFIND" {
return true return true
} }
return false return false
} }
// defaultLogFormatter provides a default log format similar to Gin's built-in formatter func FilteredLogger() gin.HandlerFunc {
func defaultLogFormatter(param gin.LogFormatterParams) string { initFilterList()
var statusColor, methodColor, resetColor string
if param.IsOutputColor() {
statusColor = param.StatusCodeColor()
methodColor = param.MethodColor()
resetColor = param.ResetColor()
}
if param.Latency > time.Minute { return gin.LoggerWithConfig(gin.LoggerConfig{
param.Latency = param.Latency.Truncate(time.Second) Output: log.StandardLogger().Out,
} Skip: skiperDecider,
})
return fmt.Sprintf("[GIN] %v |%s %3d %s| %13v | %15s |%s %-7s %s %#v\n%s",
param.TimeStamp.Format("2006/01/02 - 15:04:05"),
statusColor, param.StatusCode, resetColor,
param.Latency,
param.ClientIP,
methodColor, param.Method, resetColor,
param.Path,
param.ErrorMessage,
)
} }