feat(docker): Update docker-compose configuration (#1081)

* feat(docker): Update docker-compose configuration

* Update docker-compose.yml

Co-authored-by: MadDogOwner <xiaoran@xrgzs.top>
Signed-off-by: huancun _- <huancun@hc26.org>

---------

Signed-off-by: huancun _- <huancun@hc26.org>
Co-authored-by: MadDogOwner <xiaoran@xrgzs.top>

.github/workflows/sync_repo.yml

@@ -0,0 +1,38 @@
+name: Sync to Gitee
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    name: Sync GitHub to Gitee
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.GITEE_SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          ssh-keyscan gitee.com >> ~/.ssh/known_hosts
+
+      - name: Create single commit and push
+        run: |
+          git config user.name "GitHub Actions"
+          git config user.email "actions@github.com"
+          
+          # Create a new branch
+          git checkout --orphan new-main
+          git add .
+          git commit -m "Sync from GitHub: $(date)"
+
+          # Add Gitee remote and force push
+          git remote add gitee ${{ vars.GITEE_REPO_URL }}
+          git push --force gitee new-main:main

buf.gen.yaml

@@ -1,11 +0,0 @@
-version: v1
-plugins:
-  - plugin: buf.build/protocolbuffers/go:v1.36.7
-    out: .
-    opt:
-      - paths=source_relative
-  - plugin: buf.build/grpc/go:v1.5.1
-    out: .
-    opt:
-      - paths=source_relative
-      - require_unimplemented_servers=false

buf.yaml

@@ -1 +0,0 @@
-version: v1

cmd/common.go

@@ -1,42 +1,51 @@
 package cmd
 
 import (
-	"context"
+	"os"
+	"path/filepath"
+	"strconv"
 
-	"github.com/OpenListTeam/OpenList/v5/cmd/flags"
-	"github.com/OpenListTeam/OpenList/v5/internal/bootstrap"
-	"github.com/sirupsen/logrus"
+	"github.com/OpenListTeam/OpenList/v4/internal/bootstrap"
+	"github.com/OpenListTeam/OpenList/v4/internal/bootstrap/data"
+	"github.com/OpenListTeam/OpenList/v4/internal/db"
+	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
+	log "github.com/sirupsen/logrus"
 )
 
-func Init(ctx context.Context) {
-	if flags.Dev {
-		flags.Debug = true
-	}
-	initLogrus()
+func Init() {
 	bootstrap.InitConfig()
-	bootstrap.InitDriverPlugins()
+	bootstrap.Log()
+	bootstrap.InitDB()
+	data.InitData()
+	bootstrap.InitStreamLimit()
+	bootstrap.InitIndex()
+	bootstrap.InitUpgradePatch()
 }
 
 func Release() {
-
+	db.Close()
 }
 
-func initLog(l *logrus.Logger) {
-	if flags.Debug {
-		l.SetLevel(logrus.DebugLevel)
-		l.SetReportCaller(true)
-	} else {
-		l.SetLevel(logrus.InfoLevel)
-		l.SetReportCaller(false)
+var pid = -1
+var pidFile string
+
+func initDaemon() {
+	ex, err := os.Executable()
+	if err != nil {
+		log.Fatal(err)
+	}
+	exPath := filepath.Dir(ex)
+	_ = os.MkdirAll(filepath.Join(exPath, "daemon"), 0700)
+	pidFile = filepath.Join(exPath, "daemon/pid")
+	if utils.Exists(pidFile) {
+		bytes, err := os.ReadFile(pidFile)
+		if err != nil {
+			log.Fatal("failed to read pid file", err)
+		}
+		id, err := strconv.Atoi(string(bytes))
+		if err != nil {
+			log.Fatal("failed to parse pid data", err)
+		}
+		pid = id
 	}
 }
-func initLogrus() {
-	formatter := logrus.TextFormatter{
-		ForceColors:               true,
-		EnvironmentOverrideColors: true,
-		TimestampFormat:           "2006-01-02 15:04:05",
-		FullTimestamp:             true,
-	}
-	logrus.SetFormatter(&formatter)
-	initLog(logrus.StandardLogger())
-}

cmd/flags/config.go

@@ -1,40 +1,10 @@
 package flags
 
-import (
-	"os"
-	"path/filepath"
-
-	"github.com/sirupsen/logrus"
-)
-
 var (
-	ConfigFile  string
+	DataDir     string
 	Debug       bool
 	NoPrefix    bool
 	Dev         bool
 	ForceBinDir bool
 	LogStd      bool
-
-	pwd string
 )
-
-// Program working directory
-func PWD() string {
-	if pwd != "" {
-		return pwd
-	}
-	if ForceBinDir {
-		ex, err := os.Executable()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		pwd = filepath.Dir(ex)
-		return pwd
-	}
-	d, err := os.Getwd()
-	if err != nil {
-		logrus.Fatal(err)
-	}
-	pwd = d
-	return d
-}

cmd/root.go

@@ -4,7 +4,10 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/OpenListTeam/OpenList/v5/cmd/flags"
+	"github.com/OpenListTeam/OpenList/v4/cmd/flags"
+	_ "github.com/OpenListTeam/OpenList/v4/drivers"
+	_ "github.com/OpenListTeam/OpenList/v4/internal/archive"
+	_ "github.com/OpenListTeam/OpenList/v4/internal/offline_download"
 	"github.com/spf13/cobra"
 )
 
@@ -24,10 +27,10 @@ func Execute() {
 }
 
 func init() {
-	RootCmd.PersistentFlags().StringVarP(&flags.ConfigFile, "config", "c", "data/config.json", "config file")
+	RootCmd.PersistentFlags().StringVar(&flags.DataDir, "data", "data", "data folder")
 	RootCmd.PersistentFlags().BoolVar(&flags.Debug, "debug", false, "start with debug mode")
 	RootCmd.PersistentFlags().BoolVar(&flags.NoPrefix, "no-prefix", false, "disable env prefix")
 	RootCmd.PersistentFlags().BoolVar(&flags.Dev, "dev", false, "start with dev mode")
-	RootCmd.PersistentFlags().BoolVarP(&flags.ForceBinDir, "force-bin-dir", "f", false, "force to use the directory where the binary file is located as data directory")
-	RootCmd.PersistentFlags().BoolVar(&flags.LogStd, "log-std", false, "force to log to std")
+	RootCmd.PersistentFlags().BoolVar(&flags.ForceBinDir, "force-bin-dir", false, "Force to use the directory where the binary file is located as data directory")
+	RootCmd.PersistentFlags().BoolVar(&flags.LogStd, "log-std", false, "Force to log to std")
 }

cmd/server.go

@@ -13,9 +13,15 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/OpenListTeam/OpenList/v5/cmd/flags"
-	"github.com/OpenListTeam/OpenList/v5/internal/conf"
-	"github.com/OpenListTeam/OpenList/v5/server"
+	"github.com/OpenListTeam/OpenList/v4/cmd/flags"
+	"github.com/OpenListTeam/OpenList/v4/internal/bootstrap"
+	"github.com/OpenListTeam/OpenList/v4/internal/conf"
+	"github.com/OpenListTeam/OpenList/v4/internal/fs"
+	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
+	"github.com/OpenListTeam/OpenList/v4/server"
+	"github.com/OpenListTeam/OpenList/v4/server/middlewares"
+	"github.com/OpenListTeam/sftpd-openlist"
+	ftpserver "github.com/fclairamb/ftpserverlib"
 	"github.com/gin-gonic/gin"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -29,127 +35,220 @@ var ServerCmd = &cobra.Command{
 	Short: "Start the server at the specified address",
 	Long: `Start the server at the specified address
 the address is defined in config file`,
-	Run: func(_ *cobra.Command, args []string) {
-		serverCtx, serverCancel := context.WithCancel(context.Background())
-		defer serverCancel()
-		Init(serverCtx)
-
-		if !flags.Debug {
+	Run: func(cmd *cobra.Command, args []string) {
+		Init()
+		if conf.Conf.DelayedStart != 0 {
+			utils.Log.Infof("delayed start for %d seconds", conf.Conf.DelayedStart)
+			time.Sleep(time.Duration(conf.Conf.DelayedStart) * time.Second)
+		}
+		bootstrap.InitOfflineDownloadTools()
+		bootstrap.LoadStorages()
+		bootstrap.InitTaskManager()
+		if !flags.Debug && !flags.Dev {
 			gin.SetMode(gin.ReleaseMode)
 		}
 		r := gin.New()
-		r.Use(gin.LoggerWithWriter(log.StandardLogger().Out))
-		r.Use(gin.RecoveryWithWriter(log.StandardLogger().Out))
-		server.Init(r)
 
+		// gin log
+		if conf.Conf.Log.Filter.Enable {
+			r.Use(middlewares.FilteredLogger())
+		} else {
+			r.Use(gin.LoggerWithWriter(log.StandardLogger().Out))
+		}
+		r.Use(gin.RecoveryWithWriter(log.StandardLogger().Out))
+
+		server.Init(r)
 		var httpHandler http.Handler = r
 		if conf.Conf.Scheme.EnableH2c {
 			httpHandler = h2c.NewHandler(r, &http2.Server{})
 		}
 		var httpSrv, httpsSrv, unixSrv *http.Server
-		if conf.Conf.Scheme.HttpPort > 0 {
+		if conf.Conf.Scheme.HttpPort != -1 {
 			httpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpPort)
-			log.Infoln("start HTTP server", "@", httpBase)
+			fmt.Printf("start HTTP server @ %s\n", httpBase)
+			utils.Log.Infof("start HTTP server @ %s", httpBase)
 			httpSrv = &http.Server{Addr: httpBase, Handler: httpHandler}
 			go func() {
 				err := httpSrv.ListenAndServe()
 				if err != nil && !errors.Is(err, http.ErrServerClosed) {
-					log.Errorln("start HTTP server", ":", err)
-					serverCancel()
+					utils.Log.Fatalf("failed to start http: %s", err.Error())
 				}
 			}()
 		}
-		if conf.Conf.Scheme.HttpsPort > 0 {
+		if conf.Conf.Scheme.HttpsPort != -1 {
 			httpsBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpsPort)
-			log.Infoln("start HTTPS server", "@", httpsBase)
+			fmt.Printf("start HTTPS server @ %s\n", httpsBase)
+			utils.Log.Infof("start HTTPS server @ %s", httpsBase)
 			httpsSrv = &http.Server{Addr: httpsBase, Handler: r}
 			go func() {
 				err := httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile)
 				if err != nil && !errors.Is(err, http.ErrServerClosed) {
-					log.Errorln("start HTTPS server", ":", err)
-					serverCancel()
+					utils.Log.Fatalf("failed to start https: %s", err.Error())
 				}
 			}()
 		}
 		if conf.Conf.Scheme.UnixFile != "" {
-			log.Infoln("start Unix server", "@", conf.Conf.Scheme.UnixFile)
+			fmt.Printf("start unix server @ %s\n", conf.Conf.Scheme.UnixFile)
+			utils.Log.Infof("start unix server @ %s", conf.Conf.Scheme.UnixFile)
 			unixSrv = &http.Server{Handler: httpHandler}
 			go func() {
 				listener, err := net.Listen("unix", conf.Conf.Scheme.UnixFile)
 				if err != nil {
-					log.Errorln("start Unix server", ":", err)
-					serverCancel()
-					return
+					utils.Log.Fatalf("failed to listen unix: %+v", err)
 				}
-
+				// set socket file permission
 				mode, err := strconv.ParseUint(conf.Conf.Scheme.UnixFilePerm, 8, 32)
 				if err != nil {
-					log.Errorln("parse unix_file_perm", ":", err)
+					utils.Log.Errorf("failed to parse socket file permission: %+v", err)
 				} else {
 					err = os.Chmod(conf.Conf.Scheme.UnixFile, os.FileMode(mode))
 					if err != nil {
-						log.Errorln("chmod socket file", ":", err)
+						utils.Log.Errorf("failed to chmod socket file: %+v", err)
 					}
 				}
-
 				err = unixSrv.Serve(listener)
 				if err != nil && !errors.Is(err, http.ErrServerClosed) {
-					log.Errorln("start Unix server", ":", err)
-					serverCancel()
+					utils.Log.Fatalf("failed to start unix: %s", err.Error())
 				}
 			}()
 		}
-
+		if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable {
+			s3r := gin.New()
+			s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out))
+			server.InitS3(s3r)
+			s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port)
+			fmt.Printf("start S3 server @ %s\n", s3Base)
+			utils.Log.Infof("start S3 server @ %s", s3Base)
+			go func() {
+				var err error
+				if conf.Conf.S3.SSL {
+					httpsSrv = &http.Server{Addr: s3Base, Handler: s3r}
+					err = httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile)
+				}
+				if !conf.Conf.S3.SSL {
+					httpSrv = &http.Server{Addr: s3Base, Handler: s3r}
+					err = httpSrv.ListenAndServe()
+				}
+				if err != nil && !errors.Is(err, http.ErrServerClosed) {
+					utils.Log.Fatalf("failed to start s3 server: %s", err.Error())
+				}
+			}()
+		}
+		var ftpDriver *server.FtpMainDriver
+		var ftpServer *ftpserver.FtpServer
+		if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable {
+			var err error
+			ftpDriver, err = server.NewMainDriver()
+			if err != nil {
+				utils.Log.Fatalf("failed to start ftp driver: %s", err.Error())
+			} else {
+				fmt.Printf("start ftp server on %s\n", conf.Conf.FTP.Listen)
+				utils.Log.Infof("start ftp server on %s", conf.Conf.FTP.Listen)
+				go func() {
+					ftpServer = ftpserver.NewFtpServer(ftpDriver)
+					err = ftpServer.ListenAndServe()
+					if err != nil {
+						utils.Log.Fatalf("problem ftp server listening: %s", err.Error())
+					}
+				}()
+			}
+		}
+		var sftpDriver *server.SftpDriver
+		var sftpServer *sftpd.SftpServer
+		if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable {
+			var err error
+			sftpDriver, err = server.NewSftpDriver()
+			if err != nil {
+				utils.Log.Fatalf("failed to start sftp driver: %s", err.Error())
+			} else {
+				fmt.Printf("start sftp server on %s", conf.Conf.SFTP.Listen)
+				utils.Log.Infof("start sftp server on %s", conf.Conf.SFTP.Listen)
+				go func() {
+					sftpServer = sftpd.NewSftpServer(sftpDriver)
+					err = sftpServer.RunServer()
+					if err != nil {
+						utils.Log.Fatalf("problem sftp server listening: %s", err.Error())
+					}
+				}()
+			}
+		}
+		// Wait for interrupt signal to gracefully shutdown the server with
+		// a timeout of 1 second.
 		quit := make(chan os.Signal, 1)
 		// kill (no param) default send syscanll.SIGTERM
 		// kill -2 is syscall.SIGINT
 		// kill -9 is syscall. SIGKILL but can"t be catch, so don't need add it
 		signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
-		select {
-		case <-quit:
-		case <-serverCtx.Done():
-		}
-
-		log.Println("shutdown server...")
+		<-quit
+		utils.Log.Println("Shutdown server...")
+		fs.ArchiveContentUploadTaskManager.RemoveAll()
 		Release()
-
-		quitCtx, quitCancel := context.WithTimeout(context.Background(), time.Second)
-		defer quitCancel()
+		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+		defer cancel()
 		var wg sync.WaitGroup
-		if httpSrv != nil {
+		if conf.Conf.Scheme.HttpPort != -1 {
 			wg.Add(1)
 			go func() {
 				defer wg.Done()
-				if err := httpSrv.Shutdown(quitCtx); err != nil {
-					log.Errorln("shutdown HTTP server", ":", err)
+				if err := httpSrv.Shutdown(ctx); err != nil {
+					utils.Log.Fatal("HTTP server shutdown err: ", err)
 				}
 			}()
 		}
-		if httpsSrv != nil {
+		if conf.Conf.Scheme.HttpsPort != -1 {
 			wg.Add(1)
 			go func() {
 				defer wg.Done()
-				if err := httpsSrv.Shutdown(quitCtx); err != nil {
-					log.Errorln("shutdown HTTPS server", ":", err)
+				if err := httpsSrv.Shutdown(ctx); err != nil {
+					utils.Log.Fatal("HTTPS server shutdown err: ", err)
 				}
 			}()
 		}
-		if unixSrv != nil {
+		if conf.Conf.Scheme.UnixFile != "" {
 			wg.Add(1)
 			go func() {
 				defer wg.Done()
-				if err := unixSrv.Shutdown(quitCtx); err != nil {
-					log.Errorln("shutdown Unix server", ":", err)
+				if err := unixSrv.Shutdown(ctx); err != nil {
+					utils.Log.Fatal("Unix server shutdown err: ", err)
+				}
+			}()
+		}
+		if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable && ftpServer != nil && ftpDriver != nil {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				ftpDriver.Stop()
+				if err := ftpServer.Stop(); err != nil {
+					utils.Log.Fatal("FTP server shutdown err: ", err)
+				}
+			}()
+		}
+		if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable && sftpServer != nil && sftpDriver != nil {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				if err := sftpServer.Close(); err != nil {
+					utils.Log.Fatal("SFTP server shutdown err: ", err)
 				}
 			}()
 		}
 		wg.Wait()
-		log.Println("server exit")
+		utils.Log.Println("Server exit")
 	},
 }
 
 func init() {
 	RootCmd.AddCommand(ServerCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// serverCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// serverCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
 }
 
 // OutOpenListInit 暴露用于外部启动server的函数

docker-compose.yml

@@ -6,10 +6,9 @@ services:
     ports:
       - '5244:5244'
       - '5245:5245'
+    user: '0:0'
     environment:
-      - PUID=0
-      - PGID=0
       - UMASK=022
-      - TZ=UTC
+      - TZ=Asia/Shanghai
     container_name: openlist
     image: 'openlistteam/openlist:latest'

drivers/123_open/upload.go

@@ -70,6 +70,8 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes
 		var reader *stream.SectionReader
 		var rateLimitedRd io.Reader
 		sliceMD5 := ""
+		// 表单
+		b := bytes.NewBuffer(make([]byte, 0, 2048))
 		threadG.GoWithLifecycle(errgroup.Lifecycle{
 			Before: func(ctx context.Context) error {
 				if reader == nil {
@@ -84,7 +86,6 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes
 					if err != nil {
 						return err
 					}
-					rateLimitedRd = driver.NewLimitedUploadStream(ctx, reader)
 				}
 				return nil
 			},
@@ -92,9 +93,8 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes
 				// 重置分片reader位置，因为HashReader、上一次失败已经读取到分片EOF
 				reader.Seek(0, io.SeekStart)
 
-				// 创建表单数据
-				var b bytes.Buffer
-				w := multipart.NewWriter(&b)
+				b.Reset()
+				w := multipart.NewWriter(b)
 				// 添加表单字段
 				err = w.WriteField("preuploadID", createResp.Data.PreuploadID)
 				if err != nil {
@@ -109,21 +109,20 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes
 					return err
 				}
 				// 写入文件内容
-				fw, err := w.CreateFormFile("slice", fmt.Sprintf("%s.part%d", file.GetName(), partNumber))
-				if err != nil {
-					return err
-				}
-				_, err = utils.CopyWithBuffer(fw, rateLimitedRd)
+				_, err = w.CreateFormFile("slice", fmt.Sprintf("%s.part%d", file.GetName(), partNumber))
 				if err != nil {
 					return err
 				}
+				headSize := b.Len()
 				err = w.Close()
 				if err != nil {
 					return err
 				}
-
+				head := bytes.NewReader(b.Bytes()[:headSize])
+				tail := bytes.NewReader(b.Bytes()[headSize:])
+				rateLimitedRd = driver.NewLimitedUploadStream(ctx, io.MultiReader(head, reader, tail))
 				// 创建请求并设置header
-				req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadDomain+"/upload/v2/file/slice", &b)
+				req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadDomain+"/upload/v2/file/slice", rateLimitedRd)
 				if err != nil {
 					return err
 				}