feat(docker): Update docker-compose configuration (#1081)

* feat(docker): Update docker-compose configuration

* Update docker-compose.yml

Co-authored-by: MadDogOwner <xiaoran@xrgzs.top>
Signed-off-by: huancun _- <huancun@hc26.org>

---------

Signed-off-by: huancun _- <huancun@hc26.org>
Co-authored-by: MadDogOwner <xiaoran@xrgzs.top>

.github/workflows/sync_repo.yml

@@ -0,0 +1,38 @@
+name: Sync to Gitee
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    name: Sync GitHub to Gitee
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.GITEE_SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          ssh-keyscan gitee.com >> ~/.ssh/known_hosts
+
+      - name: Create single commit and push
+        run: |
+          git config user.name "GitHub Actions"
+          git config user.email "actions@github.com"
+          
+          # Create a new branch
+          git checkout --orphan new-main
+          git add .
+          git commit -m "Sync from GitHub: $(date)"
+
+          # Add Gitee remote and force push
+          git remote add gitee ${{ vars.GITEE_REPO_URL }}
+          git push --force gitee new-main:main

Dockerfile

@@ -1,3 +1,6 @@
+### Default image is base. You can add other support by modifying BASE_IMAGE_TAG. The following parameters are supported: base (default), aria2, ffmpeg, aio
+ARG BASE_IMAGE_TAG=base
+
 FROM alpine:edge AS builder
 LABEL stage=go-builder
 WORKDIR /app/
@@ -7,21 +10,26 @@ RUN go mod download
 COPY ./ ./
 RUN bash build.sh release docker
 
-### Default image is base. You can add other support by modifying BASE_IMAGE_TAG. The following parameters are supported: base (default), aria2, ffmpeg, aio
-ARG BASE_IMAGE_TAG=base
 FROM openlistteam/openlist-base-image:${BASE_IMAGE_TAG}
-
+LABEL MAINTAINER="OpenList"
 ARG INSTALL_FFMPEG=false
 ARG INSTALL_ARIA2=false
-LABEL MAINTAINER="OpenList"
+ARG USER=openlist
+ARG UID=1001
+ARG GID=1001
 
 WORKDIR /opt/openlist/
 
 COPY --chmod=755 --from=builder /app/bin/openlist ./
 COPY --chmod=755 entrypoint.sh /entrypoint.sh
+RUN adduser -u ${UID} -g ${GID} -h /opt/openlist/data -D -s /bin/sh ${USER} \
+    && chown -R ${UID}:${GID} /opt \
+    && chown -R ${UID}:${GID} /entrypoint.sh
+
+USER ${USER}
 RUN /entrypoint.sh version
 
-ENV PUID=0 PGID=0 UMASK=022 RUN_ARIA2=${INSTALL_ARIA2}
+ENV UMASK=022 RUN_ARIA2=${INSTALL_ARIA2}
 VOLUME /opt/openlist/data/
 EXPOSE 5244 5245
 CMD [ "/entrypoint.sh" ]

Dockerfile.ci

@@ -1,18 +1,26 @@
 ARG BASE_IMAGE_TAG=base
 FROM ghcr.io/openlistteam/openlist-base-image:${BASE_IMAGE_TAG}
-
+LABEL MAINTAINER="OpenList"
 ARG TARGETPLATFORM
 ARG INSTALL_FFMPEG=false
 ARG INSTALL_ARIA2=false
-LABEL MAINTAINER="OpenList"
+ARG USER=openlist
+ARG UID=1001
+ARG GID=1001
 
 WORKDIR /opt/openlist/
 
 COPY --chmod=755 /build/${TARGETPLATFORM}/openlist ./
 COPY --chmod=755 entrypoint.sh /entrypoint.sh
+
+RUN adduser -u ${UID} -g ${GID} -h /opt/openlist/data -D -s /bin/sh ${USER} \
+    && chown -R ${UID}:${GID} /opt \
+    && chown -R ${UID}:${GID} /entrypoint.sh
+
+USER ${USER}
 RUN /entrypoint.sh version
 
-ENV PUID=0 PGID=0 UMASK=022 RUN_ARIA2=${INSTALL_ARIA2}
+ENV UMASK=022 RUN_ARIA2=${INSTALL_ARIA2}
 VOLUME /opt/openlist/data/
 EXPOSE 5244 5245
 CMD [ "/entrypoint.sh" ]

docker-compose.yml

@@ -6,10 +6,9 @@ services:
     ports:
       - '5244:5244'
       - '5245:5245'
+    user: '0:0'
     environment:
-      - PUID=0
-      - PGID=0
       - UMASK=022
-      - TZ=UTC
+      - TZ=Asia/Shanghai
     container_name: openlist
     image: 'openlistteam/openlist:latest'

drivers/115/driver.go

@@ -186,9 +186,7 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
 	preHash = strings.ToUpper(preHash)
 	fullHash := stream.GetHash().GetHash(utils.SHA1)
 	if len(fullHash) != utils.SHA1.Width {
-		cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
-		up = model.UpdateProgressWithRange(up, 50, 100)
-		_, fullHash, err = streamPkg.CacheFullInTempFileAndHash(stream, cacheFileProgress, utils.SHA1)
+		_, fullHash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1)
 		if err != nil {
 			return nil, err
 		}

drivers/115/util.go

@@ -321,7 +321,7 @@ func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.Upload
 		err       error
 	)
 
-	tmpF, err := s.CacheFullInTempFile()
+	tmpF, err := s.CacheFullAndWriter(&up, nil)
 	if err != nil {
 		return nil, err
 	}

drivers/115_open/driver.go

@@ -239,9 +239,7 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre
 	}
 	sha1 := file.GetHash().GetHash(utils.SHA1)
 	if len(sha1) != utils.SHA1.Width {
-		cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
-		up = model.UpdateProgressWithRange(up, 50, 100)
-		_, sha1, err = stream.CacheFullInTempFileAndHash(file, cacheFileProgress, utils.SHA1)
+		_, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1)
 		if err != nil {
 			return err
 		}

drivers/115_open/upload.go

@@ -86,13 +86,14 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer,
 
 	fileSize := stream.GetSize()
 	chunkSize := calPartSize(fileSize)
-	partNum := (stream.GetSize() + chunkSize - 1) / chunkSize
-	parts := make([]oss.UploadPart, partNum)
-	offset := int64(0)
-	ss, err := streamPkg.NewStreamSectionReader(stream, int(chunkSize))
+	ss, err := streamPkg.NewStreamSectionReader(stream, int(chunkSize), &up)
 	if err != nil {
 		return err
 	}
+
+	partNum := (stream.GetSize() + chunkSize - 1) / chunkSize
+	parts := make([]oss.UploadPart, partNum)
+	offset := int64(0)
 	for i := int64(1); i <= partNum; i++ {
 		if utils.IsCanceled(ctx) {
 			return ctx.Err()
@@ -119,7 +120,7 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer,
 			retry.Attempts(3),
 			retry.DelayType(retry.BackOffDelay),
 			retry.Delay(time.Second))
-		ss.RecycleSectionReader(rd)
+		ss.FreeSectionReader(rd)
 		if err != nil {
 			return err
 		}

drivers/123/driver.go

@@ -182,9 +182,7 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, file model.FileStrea
 	etag := file.GetHash().GetHash(utils.MD5)
 	var err error
 	if len(etag) < utils.MD5.Width {
-		cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
-		up = model.UpdateProgressWithRange(up, 50, 100)
-		_, etag, err = stream.CacheFullInTempFileAndHash(file, cacheFileProgress, utils.MD5)
+		_, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5)
 		if err != nil {
 			return err
 		}

drivers/123/upload.go

@@ -81,6 +81,12 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi
 	if size > chunkSize {
 		chunkCount = int((size + chunkSize - 1) / chunkSize)
 	}
+
+	ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up)
+	if err != nil {
+		return err
+	}
+
 	lastChunkSize := size % chunkSize
 	if lastChunkSize == 0 {
 		lastChunkSize = chunkSize
@@ -92,10 +98,6 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi
 		batchSize = 10
 		getS3UploadUrl = d.getS3PreSignedUrls
 	}
-	ss, err := stream.NewStreamSectionReader(file, int(chunkSize))
-	if err != nil {
-		return err
-	}
 
 	thread := min(int(chunkCount), d.UploadThread)
 	threadG, uploadCtx := errgroup.NewOrderedGroupWithContext(ctx, thread,
@@ -180,7 +182,7 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi
 					return nil
 				},
 				After: func(err error) {
-					ss.RecycleSectionReader(reader)
+					ss.FreeSectionReader(reader)
 				},
 			})
 		}

drivers/123_open/driver.go

@@ -132,9 +132,7 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre
 	// etag 文件md5
 	etag := file.GetHash().GetHash(utils.MD5)
 	if len(etag) < utils.MD5.Width {
-		cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
-		up = model.UpdateProgressWithRange(up, 50, 100)
-		_, etag, err = stream.CacheFullInTempFileAndHash(file, cacheFileProgress, utils.MD5)
+		_, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5)
 		if err != nil {
 			return nil, err
 		}

drivers/123_open/upload.go

@@ -46,6 +46,12 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes
 	uploadDomain := createResp.Data.Servers[0]
 	size := file.GetSize()
 	chunkSize := createResp.Data.SliceSize
+
+	ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up)
+	if err != nil {
+		return err
+	}
+
 	uploadNums := (size + chunkSize - 1) / chunkSize
 	thread := min(int(uploadNums), d.UploadThread)
 	threadG, uploadCtx := errgroup.NewOrderedGroupWithContext(ctx, thread,
@@ -53,10 +59,6 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes
 		retry.Delay(time.Second),
 		retry.DelayType(retry.BackOffDelay))
 
-	ss, err := stream.NewStreamSectionReader(file, int(chunkSize))
-	if err != nil {
-		return err
-	}
 	for partIndex := range uploadNums {
 		if utils.IsCanceled(uploadCtx) {
 			break
@@ -68,6 +70,8 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes
 		var reader *stream.SectionReader
 		var rateLimitedRd io.Reader
 		sliceMD5 := ""
+		// 表单
+		b := bytes.NewBuffer(make([]byte, 0, 2048))
 		threadG.GoWithLifecycle(errgroup.Lifecycle{
 			Before: func(ctx context.Context) error {
 				if reader == nil {
@@ -82,7 +86,6 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes
 					if err != nil {
 						return err
 					}
-					rateLimitedRd = driver.NewLimitedUploadStream(ctx, reader)
 				}
 				return nil
 			},
@@ -90,9 +93,8 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes
 				// 重置分片reader位置，因为HashReader、上一次失败已经读取到分片EOF
 				reader.Seek(0, io.SeekStart)
 
-				// 创建表单数据
-				var b bytes.Buffer
-				w := multipart.NewWriter(&b)
+				b.Reset()
+				w := multipart.NewWriter(b)
 				// 添加表单字段
 				err = w.WriteField("preuploadID", createResp.Data.PreuploadID)
 				if err != nil {
@@ -107,21 +109,20 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes
 					return err
 				}
 				// 写入文件内容
-				fw, err := w.CreateFormFile("slice", fmt.Sprintf("%s.part%d", file.GetName(), partNumber))
-				if err != nil {
-					return err
-				}
-				_, err = utils.CopyWithBuffer(fw, rateLimitedRd)
+				_, err = w.CreateFormFile("slice", fmt.Sprintf("%s.part%d", file.GetName(), partNumber))
 				if err != nil {
 					return err
 				}
+				headSize := b.Len()
 				err = w.Close()
 				if err != nil {
 					return err
 				}
-
+				head := bytes.NewReader(b.Bytes()[:headSize])
+				tail := bytes.NewReader(b.Bytes()[headSize:])
+				rateLimitedRd = driver.NewLimitedUploadStream(ctx, io.MultiReader(head, reader, tail))
 				// 创建请求并设置header
-				req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadDomain+"/upload/v2/file/slice", &b)
+				req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadDomain+"/upload/v2/file/slice", rateLimitedRd)
 				if err != nil {
 					return err
 				}
@@ -157,7 +158,7 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes
 				return nil
 			},
 			After: func(err error) {
-				ss.RecycleSectionReader(reader)
+				ss.FreeSectionReader(reader)
 			},
 		})
 	}

drivers/139/driver.go

@@ -522,9 +522,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
 		var err error
 		fullHash := stream.GetHash().GetHash(utils.SHA256)
 		if len(fullHash) != utils.SHA256.Width {
-			cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
-			up = model.UpdateProgressWithRange(up, 50, 100)
-			_, fullHash, err = streamPkg.CacheFullInTempFileAndHash(stream, cacheFileProgress, utils.SHA256)
+			_, fullHash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA256)
 			if err != nil {
 				return err
 			}

drivers/189_tv/utils.go

@@ -5,17 +5,19 @@ import (
 	"encoding/base64"
 	"encoding/xml"
 	"fmt"
-	"github.com/skip2/go-qrcode"
 	"io"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/skip2/go-qrcode"
+
 	"github.com/OpenListTeam/OpenList/v4/drivers/base"
 	"github.com/OpenListTeam/OpenList/v4/internal/driver"
 	"github.com/OpenListTeam/OpenList/v4/internal/model"
 	"github.com/OpenListTeam/OpenList/v4/internal/op"
+	"github.com/OpenListTeam/OpenList/v4/internal/stream"
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
 
 	"github.com/go-resty/resty/v2"
@@ -311,11 +313,14 @@ func (y *Cloud189TV) RapidUpload(ctx context.Context, dstDir model.Obj, stream m
 
 // 旧版本上传，家庭云不支持覆盖
 func (y *Cloud189TV) OldUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) {
-	tempFile, err := file.CacheFullInTempFile()
-	if err != nil {
-		return nil, err
+	fileMd5 := file.GetHash().GetHash(utils.MD5)
+	var tempFile = file.GetFile()
+	var err error
+	if len(fileMd5) != utils.MD5.Width {
+		tempFile, fileMd5, err = stream.CacheFullAndHash(file, &up, utils.MD5)
+	} else if tempFile == nil {
+		tempFile, err = file.CacheFullAndWriter(&up, nil)
 	}
-	fileMd5, err := utils.HashFile(utils.MD5, tempFile)
 	if err != nil {
 		return nil, err
 	}
@@ -345,7 +350,7 @@ func (y *Cloud189TV) OldUpload(ctx context.Context, dstDir model.Obj, file model
 			header["Edrive-UploadFileId"] = fmt.Sprint(status.UploadFileId)
 		}
 
-		_, err := y.put(ctx, status.FileUploadUrl, header, true, io.NopCloser(tempFile), isFamily)
+		_, err := y.put(ctx, status.FileUploadUrl, header, true, tempFile, isFamily)
 		if err, ok := err.(*RespErr); ok && err.Code != "InputStreamReadError" {
 			return nil, err
 		}

drivers/189pc/utils.go

@@ -500,7 +500,8 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo
 	if err != nil {
 		return nil, err
 	}
-	ss, err := stream.NewStreamSectionReader(file, int(sliceSize))
+
+	ss, err := stream.NewStreamSectionReader(file, int(sliceSize), &up)
 	if err != nil {
 		return nil, err
 	}
@@ -581,7 +582,7 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo
 				return nil
 			},
 			After: func(err error) {
-				ss.RecycleSectionReader(reader)
+				ss.FreeSectionReader(reader)
 			},
 		},
 		)
@@ -857,9 +858,7 @@ func (y *Cloud189PC) GetMultiUploadUrls(ctx context.Context, isFamily bool, uplo
 
 // 旧版本上传，家庭云不支持覆盖
 func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) {
-	cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
-	up = model.UpdateProgressWithRange(up, 50, 100)
-	tempFile, fileMd5, err := stream.CacheFullInTempFileAndHash(file, cacheFileProgress, utils.MD5)
+	tempFile, fileMd5, err := stream.CacheFullAndHash(file, &up, utils.MD5)
 	if err != nil {
 		return nil, err
 	}

drivers/alias/driver.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/url"
 	stdpath "path"
 	"strings"
 
@@ -12,6 +13,7 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/internal/errs"
 	"github.com/OpenListTeam/OpenList/v4/internal/fs"
 	"github.com/OpenListTeam/OpenList/v4/internal/model"
+	"github.com/OpenListTeam/OpenList/v4/internal/op"
 	"github.com/OpenListTeam/OpenList/v4/internal/sign"
 	"github.com/OpenListTeam/OpenList/v4/internal/stream"
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
@@ -160,25 +162,18 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
 					sign.Sign(reqPath)),
 			}, nil
 		}
+
+		resultLink := *link
+		resultLink.SyncClosers = utils.NewSyncClosers(link)
 		if args.Redirect {
-			return link, nil
+			return &resultLink, nil
 		}
 
-		resultLink := &model.Link{
-			URL:           link.URL,
-			Header:        link.Header,
-			RangeReader:   link.RangeReader,
-			MFile:         link.MFile,
-			Concurrency:   link.Concurrency,
-			PartSize:      link.PartSize,
-			ContentLength: link.ContentLength,
-			SyncClosers:   utils.NewSyncClosers(link),
-		}
 		if resultLink.ContentLength == 0 {
 			resultLink.ContentLength = fi.GetSize()
 		}
 		if resultLink.MFile != nil {
-			return resultLink, nil
+			return &resultLink, nil
 		}
 		if d.DownloadConcurrency > 0 {
 			resultLink.Concurrency = d.DownloadConcurrency
@@ -186,7 +181,7 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
 		if d.DownloadPartSize > 0 {
 			resultLink.PartSize = d.DownloadPartSize * utils.KB
 		}
-		return resultLink, nil
+		return &resultLink, nil
 	}
 	return nil, errs.ObjectNotFound
 }
@@ -313,24 +308,29 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer,
 	reqPath, err := d.getReqPath(ctx, dstDir, true)
 	if err == nil {
 		if len(reqPath) == 1 {
-			return fs.PutDirectly(ctx, *reqPath[0], &stream.FileStream{
-				Obj:          s,
-				Mimetype:     s.GetMimetype(),
-				WebPutAsTask: s.NeedStore(),
-				Reader:       s,
-			})
-		} else {
-			file, err := s.CacheFullInTempFile()
+			storage, reqActualPath, err := op.GetStorageAndActualPath(*reqPath[0])
 			if err != nil {
 				return err
 			}
-			for _, path := range reqPath {
+			return op.Put(ctx, storage, reqActualPath, &stream.FileStream{
+				Obj:      s,
+				Mimetype: s.GetMimetype(),
+				Reader:   s,
+			}, up)
+		} else {
+			file, err := s.CacheFullAndWriter(nil, nil)
+			if err != nil {
+				return err
+			}
+			count := float64(len(reqPath) + 1)
+			up(100 / count)
+			for i, path := range reqPath {
 				err = errors.Join(err, fs.PutDirectly(ctx, *path, &stream.FileStream{
-					Obj:          s,
-					Mimetype:     s.GetMimetype(),
-					WebPutAsTask: s.NeedStore(),
-					Reader:       file,
+					Obj:      s,
+					Mimetype: s.GetMimetype(),
+					Reader:   file,
 				}))
+				up(float64(i+2) / float64(count) * 100)
 				_, e := file.Seek(0, io.SeekStart)
 				if e != nil {
 					return errors.Join(err, e)
@@ -402,10 +402,24 @@ func (d *Alias) Extract(ctx context.Context, obj model.Obj, args model.ArchiveIn
 		return nil, errs.ObjectNotFound
 	}
 	for _, dst := range dsts {
-		link, err := d.extract(ctx, dst, sub, args)
-		if err == nil {
-			return link, nil
+		reqPath := stdpath.Join(dst, sub)
+		link, err := d.extract(ctx, reqPath, args)
+		if err != nil {
+			continue
 		}
+		if link == nil {
+			return &model.Link{
+				URL: fmt.Sprintf("%s/ap%s?inner=%s&pass=%s&sign=%s",
+					common.GetApiUrl(ctx),
+					utils.EncodePath(reqPath, true),
+					utils.EncodePath(args.InnerPath, true),
+					url.QueryEscape(args.Password),
+					sign.SignArchive(reqPath)),
+			}, nil
+		}
+		resultLink := *link
+		resultLink.SyncClosers = utils.NewSyncClosers(link)
+		return &resultLink, nil
 	}
 	return nil, errs.NotImplement
 }

drivers/alias/util.go

@@ -2,8 +2,6 @@ package alias
 
 import (
 	"context"
-	"fmt"
-	"net/url"
 	stdpath "path"
 	"strings"
 
@@ -12,8 +10,6 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/internal/fs"
 	"github.com/OpenListTeam/OpenList/v4/internal/model"
 	"github.com/OpenListTeam/OpenList/v4/internal/op"
-	"github.com/OpenListTeam/OpenList/v4/internal/sign"
-	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
 	"github.com/OpenListTeam/OpenList/v4/server/common"
 )
 
@@ -140,8 +136,7 @@ func (d *Alias) listArchive(ctx context.Context, dst, sub string, args model.Arc
 	return nil, errs.NotImplement
 }
 
-func (d *Alias) extract(ctx context.Context, dst, sub string, args model.ArchiveInnerArgs) (*model.Link, error) {
-	reqPath := stdpath.Join(dst, sub)
+func (d *Alias) extract(ctx context.Context, reqPath string, args model.ArchiveInnerArgs) (*model.Link, error) {
 	storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath)
 	if err != nil {
 		return nil, err
@@ -149,20 +144,12 @@ func (d *Alias) extract(ctx context.Context, dst, sub string, args model.Archive
 	if _, ok := storage.(driver.ArchiveReader); !ok {
 		return nil, errs.NotImplement
 	}
-	if args.Redirect && common.ShouldProxy(storage, stdpath.Base(sub)) {
-		_, err = fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true})
-		if err != nil {
+	if args.Redirect && common.ShouldProxy(storage, stdpath.Base(reqPath)) {
+		_, err := fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true})
+		if err == nil {
 			return nil, err
 		}
-		link := &model.Link{
-			URL: fmt.Sprintf("%s/ap%s?inner=%s&pass=%s&sign=%s",
-				common.GetApiUrl(ctx),
-				utils.EncodePath(reqPath, true),
-				utils.EncodePath(args.InnerPath, true),
-				url.QueryEscape(args.Password),
-				sign.SignArchive(reqPath)),
-		}
-		return link, nil
+		return nil, nil
 	}
 	link, _, err := op.DriverExtract(ctx, storage, reqActualPath, args)
 	return link, err

drivers/aliyundrive_open/upload.go

@@ -191,9 +191,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m
 
 		hash := stream.GetHash().GetHash(utils.SHA1)
 		if len(hash) != utils.SHA1.Width {
-			cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
-			up = model.UpdateProgressWithRange(up, 50, 100)
-			_, hash, err = streamPkg.CacheFullInTempFileAndHash(stream, cacheFileProgress, utils.SHA1)
+			_, hash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1)
 			if err != nil {
 				return nil, err
 			}
@@ -218,14 +216,13 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m
 	if !createResp.RapidUpload {
 		// 2. normal upload
 		log.Debugf("[aliyundive_open] normal upload")
-
-		preTime := time.Now()
-		var offset, length int64 = 0, partSize
-		//var length
-		ss, err := streamPkg.NewStreamSectionReader(stream, int(partSize))
+		ss, err := streamPkg.NewStreamSectionReader(stream, int(partSize), &up)
 		if err != nil {
 			return nil, err
 		}
+
+		preTime := time.Now()
+		var offset, length int64 = 0, partSize
 		for i := 0; i < len(createResp.PartInfoList); i++ {
 			if utils.IsCanceled(ctx) {
 				return nil, ctx.Err()
@@ -253,7 +250,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m
 				retry.Attempts(3),
 				retry.DelayType(retry.BackOffDelay),
 				retry.Delay(time.Second))
-			ss.RecycleSectionReader(rd)
+			ss.FreeSectionReader(rd)
 			if err != nil {
 				return nil, err
 			}

drivers/cloudreve/util.go

@@ -237,15 +237,16 @@ func (d *Cloudreve) upLocal(ctx context.Context, stream model.FileStreamer, u Up
 }
 
 func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error {
+	DEFAULT := int64(u.ChunkSize)
+	ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT), &up)
+	if err != nil {
+		return err
+	}
+
 	uploadUrl := u.UploadURLs[0]
 	credential := u.Credential
 	var finish int64 = 0
 	var chunk int = 0
-	DEFAULT := int64(u.ChunkSize)
-	ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT))
-	if err != nil {
-		return err
-	}
 	for finish < stream.GetSize() {
 		if utils.IsCanceled(ctx) {
 			return ctx.Err()
@@ -294,7 +295,7 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U
 			retry.DelayType(retry.BackOffDelay),
 			retry.Delay(time.Second),
 		)
-		ss.RecycleSectionReader(rd)
+		ss.FreeSectionReader(rd)
 		if err != nil {
 			return err
 		}
@@ -306,13 +307,14 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U
 }
 
 func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error {
-	uploadUrl := u.UploadURLs[0]
-	var finish int64 = 0
 	DEFAULT := int64(u.ChunkSize)
-	ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT))
+	ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT), &up)
 	if err != nil {
 		return err
 	}
+
+	uploadUrl := u.UploadURLs[0]
+	var finish int64 = 0
 	for finish < stream.GetSize() {
 		if utils.IsCanceled(ctx) {
 			return ctx.Err()
@@ -353,7 +355,7 @@ func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u
 			retry.DelayType(retry.BackOffDelay),
 			retry.Delay(time.Second),
 		)
-		ss.RecycleSectionReader(rd)
+		ss.FreeSectionReader(rd)
 		if err != nil {
 			return err
 		}
@@ -367,14 +369,15 @@ func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u
 }
 
 func (d *Cloudreve) upS3(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error {
-	var finish int64 = 0
-	var chunk int = 0
-	var etags []string
 	DEFAULT := int64(u.ChunkSize)
-	ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT))
+	ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT), &up)
 	if err != nil {
 		return err
 	}
+
+	var finish int64 = 0
+	var chunk int = 0
+	var etags []string
 	for finish < stream.GetSize() {
 		if utils.IsCanceled(ctx) {
 			return ctx.Err()
@@ -415,7 +418,7 @@ func (d *Cloudreve) upS3(ctx context.Context, stream model.FileStreamer, u Uploa
 			retry.DelayType(retry.BackOffDelay),
 			retry.Delay(time.Second),
 		)
-		ss.RecycleSectionReader(rd)
+		ss.FreeSectionReader(rd)
 		if err != nil {
 			return err
 		}

drivers/cloudreve_v4/util.go

@@ -252,15 +252,16 @@ func (d *CloudreveV4) upLocal(ctx context.Context, file model.FileStreamer, u Fi
 }
 
 func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error {
+	DEFAULT := int64(u.ChunkSize)
+	ss, err := stream.NewStreamSectionReader(file, int(DEFAULT), &up)
+	if err != nil {
+		return err
+	}
+
 	uploadUrl := u.UploadUrls[0]
 	credential := u.Credential
 	var finish int64 = 0
 	var chunk int = 0
-	DEFAULT := int64(u.ChunkSize)
-	ss, err := stream.NewStreamSectionReader(file, int(DEFAULT))
-	if err != nil {
-		return err
-	}
 	for finish < file.GetSize() {
 		if utils.IsCanceled(ctx) {
 			return ctx.Err()
@@ -309,7 +310,7 @@ func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u F
 			retry.DelayType(retry.BackOffDelay),
 			retry.Delay(time.Second),
 		)
-		ss.RecycleSectionReader(rd)
+		ss.FreeSectionReader(rd)
 		if err != nil {
 			return err
 		}
@@ -321,13 +322,14 @@ func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u F
 }
 
 func (d *CloudreveV4) upOneDrive(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error {
-	uploadUrl := u.UploadUrls[0]
-	var finish int64 = 0
 	DEFAULT := int64(u.ChunkSize)
-	ss, err := stream.NewStreamSectionReader(file, int(DEFAULT))
+	ss, err := stream.NewStreamSectionReader(file, int(DEFAULT), &up)
 	if err != nil {
 		return err
 	}
+
+	uploadUrl := u.UploadUrls[0]
+	var finish int64 = 0
 	for finish < file.GetSize() {
 		if utils.IsCanceled(ctx) {
 			return ctx.Err()
@@ -369,7 +371,7 @@ func (d *CloudreveV4) upOneDrive(ctx context.Context, file model.FileStreamer, u
 			retry.DelayType(retry.BackOffDelay),
 			retry.Delay(time.Second),
 		)
-		ss.RecycleSectionReader(rd)
+		ss.FreeSectionReader(rd)
 		if err != nil {
 			return err
 		}
@@ -383,14 +385,15 @@ func (d *CloudreveV4) upOneDrive(ctx context.Context, file model.FileStreamer, u
 }
 
 func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error {
-	var finish int64 = 0
-	var chunk int = 0
-	var etags []string
 	DEFAULT := int64(u.ChunkSize)
-	ss, err := stream.NewStreamSectionReader(file, int(DEFAULT))
+	ss, err := stream.NewStreamSectionReader(file, int(DEFAULT), &up)
 	if err != nil {
 		return err
 	}
+
+	var finish int64 = 0
+	var chunk int = 0
+	var etags []string
 	for finish < file.GetSize() {
 		if utils.IsCanceled(ctx) {
 			return ctx.Err()
@@ -432,7 +435,7 @@ func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileU
 			retry.DelayType(retry.BackOffDelay),
 			retry.Delay(time.Second),
 		)
-		ss.RecycleSectionReader(rd)
+		ss.FreeSectionReader(rd)
 		if err != nil {
 			return err
 		}

drivers/crypt/driver.go

@@ -401,7 +401,6 @@ func (d *Crypt) Put(ctx context.Context, dstDir model.Obj, streamer model.FileSt
 		},
 		Reader:            wrappedIn,
 		Mimetype:          "application/octet-stream",
-		WebPutAsTask:      streamer.NeedStore(),
 		ForceStreamUpload: true,
 		Exist:             streamer.GetExist(),
 	}

drivers/doubao/util.go

@@ -449,10 +449,11 @@ func (d *Doubao) uploadNode(uploadConfig *UploadConfig, dir model.Obj, file mode
 
 // Upload 普通上传实现
 func (d *Doubao) Upload(ctx context.Context, config *UploadConfig, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, dataType string) (model.Obj, error) {
-	ss, err := stream.NewStreamSectionReader(file, int(file.GetSize()))
+	ss, err := stream.NewStreamSectionReader(file, int(file.GetSize()), &up)
 	if err != nil {
 		return nil, err
 	}
+
 	reader, err := ss.GetSectionReader(0, file.GetSize())
 	if err != nil {
 		return nil, err
@@ -503,7 +504,7 @@ func (d *Doubao) Upload(ctx context.Context, config *UploadConfig, dstDir model.
 		}
 		return nil
 	})
-	ss.RecycleSectionReader(reader)
+	ss.FreeSectionReader(reader)
 	if err != nil {
 		return nil, err
 	}
@@ -542,15 +543,15 @@ func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fi
 	if config.InnerUploadAddress.AdvanceOption.SliceSize > 0 {
 		chunkSize = int64(config.InnerUploadAddress.AdvanceOption.SliceSize)
 	}
+	ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up)
+	if err != nil {
+		return nil, err
+	}
+
 	totalParts := (fileSize + chunkSize - 1) / chunkSize
 	// 创建分片信息组
 	parts := make([]UploadPart, totalParts)
 
-	// 用 stream.NewStreamSectionReader 替代缓存临时文件
-	ss, err := stream.NewStreamSectionReader(file, int(chunkSize))
-	if err != nil {
-		return nil, err
-	}
 	up(10.0) // 更新进度
 	// 设置并行上传
 	thread := min(int(totalParts), d.uploadThread)
@@ -641,7 +642,7 @@ func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fi
 				return nil
 			},
 			After: func(err error) {
-				ss.RecycleSectionReader(reader)
+				ss.FreeSectionReader(reader)
 			},
 		})
 	}

drivers/dropbox/meta.go

@@ -13,7 +13,7 @@ type Addition struct {
 	ClientSecret    string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"`
 	AccessToken     string
 	RefreshToken    string `json:"refresh_token" required:"true"`
-	RootNamespaceId string
+	RootNamespaceId string `json:"RootNamespaceId" required:"false"`
 }
 
 var config = driver.Config{

drivers/dropbox/util.go

@@ -175,6 +175,13 @@ func (d *Dropbox) finishUploadSession(ctx context.Context, toPath string, offset
 	}
 	req.Header.Set("Content-Type", "application/octet-stream")
 	req.Header.Set("Authorization", "Bearer "+d.AccessToken)
+	if d.RootNamespaceId != "" {
+	apiPathRootJson, err := d.buildPathRootHeader()
+	if err != nil {
+	    return err
+	}
+	req.Header.Set("Dropbox-API-Path-Root", apiPathRootJson)
+}
 
 	uploadFinishArgs := UploadFinishArgs{
 		Commit: struct {
@@ -219,6 +226,13 @@ func (d *Dropbox) startUploadSession(ctx context.Context) (string, error) {
 	}
 	req.Header.Set("Content-Type", "application/octet-stream")
 	req.Header.Set("Authorization", "Bearer "+d.AccessToken)
+	if d.RootNamespaceId != "" {
+	apiPathRootJson, err := d.buildPathRootHeader()
+	if err != nil {
+	    return "", err
+	}
+	req.Header.Set("Dropbox-API-Path-Root", apiPathRootJson)
+}
 	req.Header.Set("Dropbox-API-Arg", "{\"close\":false}")
 
 	res, err := base.HttpClient.Do(req)
@@ -233,3 +247,11 @@ func (d *Dropbox) startUploadSession(ctx context.Context) (string, error) {
 	_ = res.Body.Close()
 	return sessionId, nil
 }
+
+func (d *Dropbox) buildPathRootHeader() (string, error) {
+    return utils.Json.MarshalToString(map[string]interface{}{
+        ".tag": "root",
+        "root": d.RootNamespaceId,
+    })
+}
+

drivers/google_drive/driver.go

@@ -162,7 +162,7 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi
 				SetBody(driver.NewLimitedUploadStream(ctx, stream))
 		}, nil)
 	} else {
-		err = d.chunkUpload(ctx, stream, putUrl)
+		err = d.chunkUpload(ctx, stream, putUrl, up)
 	}
 	return err
 }

drivers/google_drive/util.go

@@ -254,13 +254,14 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) {
 	return res, nil
 }
 
-func (d *GoogleDrive) chunkUpload(ctx context.Context, file model.FileStreamer, url string) error {
+func (d *GoogleDrive) chunkUpload(ctx context.Context, file model.FileStreamer, url string, up driver.UpdateProgress) error {
 	var defaultChunkSize = d.ChunkSize * 1024 * 1024
-	var offset int64 = 0
-	ss, err := stream.NewStreamSectionReader(file, int(defaultChunkSize))
+	ss, err := stream.NewStreamSectionReader(file, int(defaultChunkSize), &up)
 	if err != nil {
 		return err
 	}
+
+	var offset int64 = 0
 	url += "?includeItemsFromAllDrives=true&supportsAllDrives=true"
 	for offset < file.GetSize() {
 		if utils.IsCanceled(ctx) {
@@ -300,12 +301,13 @@ func (d *GoogleDrive) chunkUpload(ctx context.Context, file model.FileStreamer,
 				}
 				return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors)
 			}
+			up(float64(offset+chunkSize) / float64(file.GetSize()) * 100)
 			return nil
 		},
 			retry.Attempts(3),
 			retry.DelayType(retry.BackOffDelay),
 			retry.Delay(time.Second))
-		ss.RecycleSectionReader(reader)
+		ss.FreeSectionReader(reader)
 		if err != nil {
 			return err
 		}

drivers/ilanzou/driver.go

@@ -276,9 +276,7 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreame
 	etag := s.GetHash().GetHash(utils.MD5)
 	var err error
 	if len(etag) != utils.MD5.Width {
-		cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
-		up = model.UpdateProgressWithRange(up, 50, 100)
-		_, etag, err = stream.CacheFullInTempFileAndHash(s, cacheFileProgress, utils.MD5)
+		_, etag, err = stream.CacheFullAndHash(s, &up, utils.MD5)
 		if err != nil {
 			return nil, err
 		}

drivers/mediatrack/driver.go

@@ -180,7 +180,7 @@ func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, file model.FileS
 	if err != nil {
 		return err
 	}
-	tempFile, err := file.CacheFullInTempFile()
+	tempFile, err := file.CacheFullAndWriter(&up, nil)
 	if err != nil {
 		return err
 	}

drivers/mopan/driver.go

@@ -263,7 +263,7 @@ func (d *MoPan) Remove(ctx context.Context, obj model.Obj) error {
 }
 
 func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
-	file, err := stream.CacheFullInTempFile()
+	file, err := stream.CacheFullAndWriter(&up, nil)
 	if err != nil {
 		return nil, err
 	}

drivers/netease_music/util.go

@@ -223,7 +223,7 @@ func (d *NeteaseMusic) removeSongObj(file model.Obj) error {
 }
 
 func (d *NeteaseMusic) putSongStream(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress) error {
-	tmp, err := stream.CacheFullInTempFile()
+	tmp, err := stream.CacheFullAndWriter(&up, nil)
 	if err != nil {
 		return err
 	}

drivers/onedrive/util.go

@@ -238,13 +238,14 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil
 	if err != nil {
 		return err
 	}
-	uploadUrl := jsoniter.Get(res, "uploadUrl").ToString()
-	var finish int64 = 0
 	DEFAULT := d.ChunkSize * 1024 * 1024
-	ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT))
+	ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT), &up)
 	if err != nil {
 		return err
 	}
+
+	uploadUrl := jsoniter.Get(res, "uploadUrl").ToString()
+	var finish int64 = 0
 	for finish < stream.GetSize() {
 		if utils.IsCanceled(ctx) {
 			return ctx.Err()
@@ -285,7 +286,7 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil
 			retry.DelayType(retry.BackOffDelay),
 			retry.Delay(time.Second),
 		)
-		ss.RecycleSectionReader(rd)
+		ss.FreeSectionReader(rd)
 		if err != nil {
 			return err
 		}

drivers/onedrive_app/util.go

@@ -152,13 +152,14 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model.
 	if err != nil {
 		return err
 	}
-	uploadUrl := jsoniter.Get(res, "uploadUrl").ToString()
-	var finish int64 = 0
 	DEFAULT := d.ChunkSize * 1024 * 1024
-	ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT))
+	ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT), &up)
 	if err != nil {
 		return err
 	}
+
+	uploadUrl := jsoniter.Get(res, "uploadUrl").ToString()
+	var finish int64 = 0
 	for finish < stream.GetSize() {
 		if utils.IsCanceled(ctx) {
 			return ctx.Err()
@@ -199,7 +200,7 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model.
 			retry.DelayType(retry.BackOffDelay),
 			retry.Delay(time.Second),
 		)
-		ss.RecycleSectionReader(rd)
+		ss.FreeSectionReader(rd)
 		if err != nil {
 			return err
 		}

drivers/pikpak/driver.go

@@ -12,6 +12,7 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/internal/driver"
 	"github.com/OpenListTeam/OpenList/v4/internal/model"
 	"github.com/OpenListTeam/OpenList/v4/internal/op"
+	streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream"
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
 	hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
 	"github.com/go-resty/resty/v2"
@@ -212,15 +213,11 @@ func (d *PikPak) Remove(ctx context.Context, obj model.Obj) error {
 }
 
 func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
-	hi := stream.GetHash()
-	sha1Str := hi.GetHash(hash_extend.GCID)
-	if len(sha1Str) < hash_extend.GCID.Width {
-		tFile, err := stream.CacheFullInTempFile()
-		if err != nil {
-			return err
-		}
+	sha1Str := stream.GetHash().GetHash(hash_extend.GCID)
 
-		sha1Str, err = utils.HashFile(hash_extend.GCID, tFile, stream.GetSize())
+	if len(sha1Str) < hash_extend.GCID.Width {
+		var err error
+		_, sha1Str, err = streamPkg.CacheFullAndHash(stream, &up, hash_extend.GCID, stream.GetSize())
 		if err != nil {
 			return err
 		}

drivers/pikpak/util.go

@@ -438,20 +438,19 @@ func (d *PikPak) UploadByOSS(ctx context.Context, params *S3Params, s model.File
 }
 
 func (d *PikPak) UploadByMultipart(ctx context.Context, params *S3Params, fileSize int64, s model.FileStreamer, up driver.UpdateProgress) error {
+	tmpF, err := s.CacheFullAndWriter(&up, nil)
+	if err != nil {
+		return err
+	}
+
 	var (
 		chunks    []oss.FileChunk
 		parts     []oss.UploadPart
 		imur      oss.InitiateMultipartUploadResult
 		ossClient *oss.Client
 		bucket    *oss.Bucket
-		err       error
 	)
 
-	tmpF, err := s.CacheFullInTempFile()
-	if err != nil {
-		return err
-	}
-
 	if ossClient, err = oss.New(params.Endpoint, params.AccessKeyID, params.AccessKeySecret); err != nil {
 		return err
 	}

drivers/quark_open/driver.go

@@ -14,7 +14,6 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/internal/driver"
 	"github.com/OpenListTeam/OpenList/v4/internal/errs"
 	"github.com/OpenListTeam/OpenList/v4/internal/model"
-	streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream"
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
 	"github.com/go-resty/resty/v2"
 )
@@ -158,9 +157,7 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File
 	}
 
 	if len(writers) > 0 {
-		cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
-		up = model.UpdateProgressWithRange(up, 50, 100)
-		_, err := streamPkg.CacheFullInTempFileAndWriter(stream, cacheFileProgress, io.MultiWriter(writers...))
+		_, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...))
 		if err != nil {
 			return err
 		}

drivers/quark_uc/driver.go

@@ -13,7 +13,6 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/internal/driver"
 	"github.com/OpenListTeam/OpenList/v4/internal/errs"
 	"github.com/OpenListTeam/OpenList/v4/internal/model"
-	streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream"
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
 	"github.com/go-resty/resty/v2"
 	log "github.com/sirupsen/logrus"
@@ -144,9 +143,7 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File
 	}
 
 	if len(writers) > 0 {
-		cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
-		up = model.UpdateProgressWithRange(up, 50, 100)
-		_, err := streamPkg.CacheFullInTempFileAndWriter(stream, cacheFileProgress, io.MultiWriter(writers...))
+		_, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...))
 		if err != nil {
 			return err
 		}

drivers/strm/driver.go

@@ -173,8 +173,9 @@ func (d *Strm) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*
 		}, nil
 	}
 
-	// 没有修改link的字段，可直接返回
-	return link, nil
+	resultLink := *link
+	resultLink.SyncClosers = utils.NewSyncClosers(link)
+	return &resultLink, nil
 }
 
 var _ driver.Driver = (*Strm)(nil)

drivers/terabox/driver.go

@@ -132,7 +132,7 @@ func (d *Terabox) Remove(ctx context.Context, obj model.Obj) error {
 func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
 	resp, err := base.RestyClient.R().
 		SetContext(ctx).
-		Get("https://d.terabox.com/rest/2.0/pcs/file?method=locateupload")
+		Get("https://" + d.url_domain_prefix + "-data.terabox.com/rest/2.0/pcs/file?method=locateupload")
 	if err != nil {
 		return err
 	}
@@ -179,7 +179,7 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt
 	}
 
 	// upload chunks
-	tempFile, err := stream.CacheFullInTempFile()
+	tempFile, err := stream.CacheFullAndWriter(&up, nil)
 	if err != nil {
 		return err
 	}

drivers/thunder/driver.go

@@ -371,9 +371,7 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.Fi
 	gcid := file.GetHash().GetHash(hash_extend.GCID)
 	var err error
 	if len(gcid) < hash_extend.GCID.Width {
-		cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
-		up = model.UpdateProgressWithRange(up, 50, 100)
-		_, gcid, err = stream.CacheFullInTempFileAndHash(file, cacheFileProgress, hash_extend.GCID, file.GetSize())
+		_, gcid, err = stream.CacheFullAndHash(file, &up, hash_extend.GCID, file.GetSize())
 		if err != nil {
 			return err
 		}

drivers/thunder_browser/driver.go

@@ -491,9 +491,7 @@ func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream
 	gcid := stream.GetHash().GetHash(hash_extend.GCID)
 	var err error
 	if len(gcid) < hash_extend.GCID.Width {
-		cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
-		up = model.UpdateProgressWithRange(up, 50, 100)
-		_, gcid, err = streamPkg.CacheFullInTempFileAndHash(stream, cacheFileProgress, hash_extend.GCID, stream.GetSize())
+		_, gcid, err = streamPkg.CacheFullAndHash(stream, &up, hash_extend.GCID, stream.GetSize())
 		if err != nil {
 			return err
 		}

drivers/thunderx/driver.go

@@ -372,9 +372,7 @@ func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, file model.F
 	gcid := file.GetHash().GetHash(hash_extend.GCID)
 	var err error
 	if len(gcid) < hash_extend.GCID.Width {
-		cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
-		up = model.UpdateProgressWithRange(up, 50, 100)
-		_, gcid, err = stream.CacheFullInTempFileAndHash(file, cacheFileProgress, hash_extend.GCID, file.GetSize())
+		_, gcid, err = stream.CacheFullAndHash(file, &up, hash_extend.GCID, file.GetSize())
 		if err != nil {
 			return err
 		}

drivers/url_tree/meta.go

@@ -10,7 +10,7 @@ type Addition struct {
 	// driver.RootPath
 	// driver.RootID
 	// define other
-	UrlStructure string `json:"url_structure" type:"text" required:"true" default:"https://cdn.oplist.org/gh/OpenListTeam/OpenList/README.md\nhttps://cdn.oplist.org/gh/OpenListTeam/OpenList/README_cn.md\nfolder:\n  CONTRIBUTING.md:1635:https://cdn.oplist.org/gh/OpenListTeam/OpenList/CONTRIBUTING.md\n  CODE_OF_CONDUCT.md:2093:https://cdn.oplist.org/gh/OpenListTeam/OpenList/CODE_OF_CONDUCT.md" help:"structure:FolderName:\n  [FileName:][FileSize:][Modified:]Url"`
+	UrlStructure string `json:"url_structure" type:"text" required:"true" default:"https://raw.githubusercontent.com/OpenListTeam/OpenList/main/README.md\nhttps://raw.githubusercontent.com/OpenListTeam/OpenList/main/README_cn.md\nfolder:\n  CONTRIBUTING.md:1635:https://raw.githubusercontent.com/OpenListTeam/OpenList/main/CONTRIBUTING.md\n  CODE_OF_CONDUCT.md:2093:https://raw.githubusercontent.com/OpenListTeam/OpenList/main/CODE_OF_CONDUCT.md" help:"structure:FolderName:\n  [FileName:][FileSize:][Modified:]Url"`
 	HeadSize     bool   `json:"head_size" type:"bool" default:"false" help:"Use head method to get file size, but it may be failed."`
 	Writable     bool   `json:"writable" type:"bool" default:"false"`
 }

drivers/weiyun/driver.go

@@ -317,7 +317,7 @@ func (d *WeiYun) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
 	if folder, ok = dstDir.(*Folder); !ok {
 		return nil, errs.NotSupport
 	}
-	file, err := stream.CacheFullInTempFile()
+	file, err := stream.CacheFullAndWriter(&up, nil)
 	if err != nil {
 		return nil, err
 	}

entrypoint.sh

@@ -5,10 +5,21 @@ umask ${UMASK}
 if [ "$1" = "version" ]; then
   ./openlist version
 else
+  # Define the target directory path for aria2 service
+  ARIA2_DIR="/opt/service/start/aria2"
+  
   if [ "$RUN_ARIA2" = "true" ]; then
-   cp -a /opt/service/stop/aria2 /opt/service/start 2>/dev/null
+    # If aria2 should run and target directory doesn't exist, copy it
+    if [ ! -d "$ARIA2_DIR" ]; then
+      mkdir -p "$ARIA2_DIR"
+      cp -r /opt/service/stop/aria2/* "$ARIA2_DIR" 2>/dev/null
+    fi
+    runsvdir /opt/service/start &
+  else
+    # If aria2 should NOT run and target directory exists, remove it
+    if [ -d "$ARIA2_DIR" ]; then
+      rm -rf "$ARIA2_DIR"
+    fi
   fi
-
-  chown -R ${PUID}:${PGID} /opt
-  exec su-exec ${PUID}:${PGID} runsvdir /opt/service/start
-fi
+  exec ./openlist server --no-prefix
+fi

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/charmbracelet/bubbletea v1.3.6
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e
-	github.com/coreos/go-oidc v2.4.0+incompatible
+	github.com/coreos/go-oidc v2.3.0+incompatible
 	github.com/deckarep/golang-set/v2 v2.8.0
 	github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8
 	github.com/disintegration/imaging v1.6.2

go.sum

@@ -205,8 +205,6 @@ github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJ
 github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
 github.com/coreos/go-oidc v2.3.0+incompatible h1:+5vEsrgprdLjjQ9FzIKAzQz1wwPD+83hQRfUIPh7rO0=
 github.com/coreos/go-oidc v2.3.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
-github.com/coreos/go-oidc v2.4.0+incompatible h1:xjdlhLWXcINyUJgLQ9I76g7osgC2goiL6JDXS6Fegjk=
-github.com/coreos/go-oidc v2.4.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
 github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
 github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=

internal/bootstrap/config.go

@@ -77,6 +77,10 @@ func InitConfig() {
 			log.Fatalf("update config struct error: %+v", err)
 		}
 	}
+	if !conf.Conf.Force {
+		confFromEnv()
+	}
+
 	if conf.Conf.MaxConcurrency > 0 {
 		net.DefaultConcurrencyLimit = &net.ConcurrencyLimit{Limit: conf.Conf.MaxConcurrency}
 	}
@@ -91,26 +95,32 @@ func InitConfig() {
 	} else {
 		conf.MaxBufferLimit = conf.Conf.MaxBufferLimit * utils.MB
 	}
-	log.Infof("max buffer limit: %d", conf.MaxBufferLimit)
-	if !conf.Conf.Force {
-		confFromEnv()
+	log.Infof("max buffer limit: %dMB", conf.MaxBufferLimit/utils.MB)
+	if conf.Conf.MmapThreshold > 0 {
+		conf.MmapThreshold = conf.Conf.MmapThreshold * utils.MB
+	} else {
+		conf.MmapThreshold = 0
 	}
+	log.Infof("mmap threshold: %dMB", conf.Conf.MmapThreshold)
+
 	if len(conf.Conf.Log.Filter.Filters) == 0 {
 		conf.Conf.Log.Filter.Enable = false
 	}
 	// convert abs path
 	convertAbsPath := func(path *string) {
-		if !filepath.IsAbs(*path) {
+		if *path != "" && !filepath.IsAbs(*path) {
 			*path = filepath.Join(pwd, *path)
 		}
 	}
+	convertAbsPath(&conf.Conf.Database.DBFile)
+	convertAbsPath(&conf.Conf.Scheme.CertFile)
+	convertAbsPath(&conf.Conf.Scheme.KeyFile)
+	convertAbsPath(&conf.Conf.Scheme.UnixFile)
+	convertAbsPath(&conf.Conf.Log.Name)
 	convertAbsPath(&conf.Conf.TempDir)
 	convertAbsPath(&conf.Conf.BleveDir)
-	convertAbsPath(&conf.Conf.Log.Name)
-	convertAbsPath(&conf.Conf.Database.DBFile)
-	if conf.Conf.DistDir != "" {
-		convertAbsPath(&conf.Conf.DistDir)
-	}
+	convertAbsPath(&conf.Conf.DistDir)
+
 	err := os.MkdirAll(conf.Conf.TempDir, 0o777)
 	if err != nil {
 		log.Fatalf("create temp dir error: %+v", err)

internal/bootstrap/data/setting.go

@@ -107,8 +107,8 @@ func InitialSettings() []model.SettingItem {
 		{Key: conf.AllowMounted, Value: "true", Type: conf.TypeBool, Group: model.SITE},
 		{Key: conf.RobotsTxt, Value: "User-agent: *\nAllow: /", Type: conf.TypeText, Group: model.SITE},
 		// style settings
-		{Key: conf.Logo, Value: "https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg", Type: conf.TypeText, Group: model.STYLE},
-		{Key: conf.Favicon, Value: "https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg", Type: conf.TypeString, Group: model.STYLE},
+		{Key: conf.Logo, Value: "https://res.oplist.org/logo/logo.svg", MigrationValue: "https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg", Type: conf.TypeText, Group: model.STYLE},
+		{Key: conf.Favicon, Value: "https://res.oplist.org/logo/logo.svg", MigrationValue: "https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg", Type: conf.TypeString, Group: model.STYLE},
 		{Key: conf.MainColor, Value: "#1890ff", Type: conf.TypeString, Group: model.STYLE},
 		{Key: "home_icon", Value: "🏠", Type: conf.TypeString, Group: model.STYLE},
 		{Key: "home_container", Value: "max_980px", Type: conf.TypeSelect, Options: "max_980px,hope_container", Group: model.STYLE},
@@ -141,7 +141,7 @@ func InitialSettings() []model.SettingItem {
 		//		{Key: conf.PdfViewers, Value: `{
 		//	"pdf.js":"https://openlistteam.github.io/pdf.js/web/viewer.html?file=$url"
 		//}`, Type: conf.TypeText, Group: model.PREVIEW},
-		{Key: "audio_cover", Value: "https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg", Type: conf.TypeString, Group: model.PREVIEW},
+		{Key: "audio_cover", Value: "https://res.oplist.org/logo/logo.svg", MigrationValue: "https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg", Type: conf.TypeString, Group: model.PREVIEW},
 		{Key: conf.AudioAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW},
 		{Key: conf.VideoAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW},
 		{Key: conf.PreviewArchivesByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW},

internal/conf/config.go

@@ -120,6 +120,7 @@ type Config struct {
 	Log                   LogConfig   `json:"log" envPrefix:"LOG_"`
 	DelayedStart          int         `json:"delayed_start" env:"DELAYED_START"`
 	MaxBufferLimit        int         `json:"max_buffer_limitMB" env:"MAX_BUFFER_LIMIT_MB"`
+	MmapThreshold         int         `json:"mmap_thresholdMB" env:"MMAP_THRESHOLD_MB"`
 	MaxConnections        int         `json:"max_connections" env:"MAX_CONNECTIONS"`
 	MaxConcurrency        int         `json:"max_concurrency" env:"MAX_CONCURRENCY"`
 	TlsInsecureSkipVerify bool        `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"`
@@ -176,6 +177,7 @@ func DefaultConfig(dataDir string) *Config {
 			},
 		},
 		MaxBufferLimit:        -1,
+		MmapThreshold:         4,
 		MaxConnections:        0,
 		MaxConcurrency:        64,
 		TlsInsecureSkipVerify: true,

internal/conf/var.go

@@ -25,7 +25,10 @@ var PrivacyReg []*regexp.Regexp
 var (
 	// StoragesLoaded loaded success if empty
 	StoragesLoaded = false
-	MaxBufferLimit int
+	// 单个Buffer最大限制
+	MaxBufferLimit = 16 * 1024 * 1024
+	// 超过该阈值的Buffer将使用 mmap 分配，可主动释放内存
+	MmapThreshold = 4 * 1024 * 1024
 )
 var (
 	RawIndexHtml string

internal/fs/archive.go

@@ -70,25 +70,25 @@ func (t *ArchiveDownloadTask) RunWithoutPushUploadTask() (*ArchiveContentUploadT
 	}()
 	var decompressUp model.UpdateProgress
 	if t.CacheFull {
-		var total, cur int64 = 0, 0
+		total := int64(0)
 		for _, s := range ss {
 			total += s.GetSize()
 		}
 		t.SetTotalBytes(total)
 		t.Status = "getting src object"
-		for _, s := range ss {
-			if s.GetFile() == nil {
-				_, err = stream.CacheFullInTempFileAndWriter(s, func(p float64) {
-					t.SetProgress((float64(cur) + float64(s.GetSize())*p/100.0) / float64(total))
-				}, nil)
+		part := 100 / float64(len(ss)+1)
+		for i, s := range ss {
+			if s.GetFile() != nil {
+				continue
 			}
-			cur += s.GetSize()
+			_, err = s.CacheFullAndWriter(nil, nil)
 			if err != nil {
 				return nil, err
+			} else {
+				t.SetProgress(float64(i+1) * part)
 			}
 		}
-		t.SetProgress(100.0)
-		decompressUp = func(_ float64) {}
+		decompressUp = model.UpdateProgressWithRange(t.SetProgress, 100-part, 100)
 	} else {
 		decompressUp = t.SetProgress
 	}

internal/fs/put.go

@@ -69,7 +69,7 @@ func putAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer)
 		return nil, errors.WithStack(errs.UploadNotSupported)
 	}
 	if file.NeedStore() {
-		_, err := file.CacheFullInTempFile()
+		_, err := file.CacheFullAndWriter(nil, nil)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to create temp file")
 		}

internal/model/obj.go

@@ -2,7 +2,6 @@ package model
 
 import (
 	"io"
-	"os"
 	"sort"
 	"strings"
 	"time"
@@ -40,16 +39,17 @@ type FileStreamer interface {
 	utils.ClosersIF
 	Obj
 	GetMimetype() string
-	//SetReader(io.Reader)
 	NeedStore() bool
 	IsForceStreamUpload() bool
 	GetExist() Obj
 	SetExist(Obj)
-	//for a non-seekable Stream, RangeRead supports peeking some data, and CacheFullInTempFile still works
+	// for a non-seekable Stream, RangeRead supports peeking some data, and CacheFullAndWriter still works
 	RangeRead(http_range.Range) (io.Reader, error)
-	//for a non-seekable Stream, if Read is called, this function won't work
-	CacheFullInTempFile() (File, error)
-	SetTmpFile(r *os.File)
+	// for a non-seekable Stream, if Read is called, this function won't work.
+	// caches the full Stream and writes it to writer (if provided, even if the stream is already cached).
+	CacheFullAndWriter(up *UpdateProgress, writer io.Writer) (File, error)
+	SetTmpFile(file File)
+	// if the Stream is not a File and is not cached, returns nil.
 	GetFile() File
 }
 

internal/model/user.go

@@ -185,5 +185,5 @@ func (u *User) WebAuthnCredentials() []webauthn.Credential {
 }
 
 func (u *User) WebAuthnIcon() string {
-	return "https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg"
+	return "https://res.oplist.org/logo/logo.svg"
 }

internal/net/request.go

@@ -1,7 +1,6 @@
 package net
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -15,6 +14,7 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/internal/conf"
 	"github.com/OpenListTeam/OpenList/v4/internal/model"
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
+	"github.com/rclone/rclone/lib/mmap"
 
 	"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
 	"github.com/aws/aws-sdk-go/aws/awsutil"
@@ -255,7 +255,10 @@ func (d *downloader) sendChunkTask(newConcurrency bool) error {
 				finalSize += firstSize - minSize
 			}
 		}
-		buf.Reset(int(finalSize))
+		err := buf.Reset(int(finalSize))
+		if err != nil {
+			return err
+		}
 		ch := chunk{
 			start: d.pos,
 			size:  finalSize,
@@ -645,11 +648,13 @@ func (mr MultiReadCloser) Close() error {
 }
 
 type Buf struct {
-	buffer *bytes.Buffer
-	size   int //expected size
-	ctx    context.Context
-	off    int
-	rw     sync.Mutex
+	size int //expected size
+	ctx  context.Context
+	offR int
+	offW int
+	rw   sync.Mutex
+	buf  []byte
+	mmap bool
 
 	readSignal  chan struct{}
 	readPending bool
@@ -658,76 +663,100 @@ type Buf struct {
 // NewBuf is a buffer that can have 1 read & 1 write at the same time.
 // when read is faster write, immediately feed data to read after written
 func NewBuf(ctx context.Context, maxSize int) *Buf {
-	return &Buf{
-		ctx:    ctx,
-		buffer: bytes.NewBuffer(make([]byte, 0, maxSize)),
-		size:   maxSize,
-
+	br := &Buf{
+		ctx:        ctx,
+		size:       maxSize,
 		readSignal: make(chan struct{}, 1),
 	}
-}
-func (br *Buf) Reset(size int) {
-	br.rw.Lock()
-	defer br.rw.Unlock()
-	if br.buffer == nil {
-		return
+	if conf.MmapThreshold > 0 && maxSize >= conf.MmapThreshold {
+		m, err := mmap.Alloc(maxSize)
+		if err == nil {
+			br.buf = m
+			br.mmap = true
+			return br
+		}
 	}
-	br.buffer.Reset()
-	br.size = size
-	br.off = 0
+	br.buf = make([]byte, maxSize)
+	return br
 }
 
-func (br *Buf) Read(p []byte) (n int, err error) {
+func (br *Buf) Reset(size int) error {
+	br.rw.Lock()
+	defer br.rw.Unlock()
+	if br.buf == nil {
+		return io.ErrClosedPipe
+	}
+	if size > cap(br.buf) {
+		return fmt.Errorf("reset size %d exceeds max size %d", size, cap(br.buf))
+	}
+	br.size = size
+	br.offR = 0
+	br.offW = 0
+	return nil
+}
+
+func (br *Buf) Read(p []byte) (int, error) {
 	if err := br.ctx.Err(); err != nil {
 		return 0, err
 	}
 	if len(p) == 0 {
 		return 0, nil
 	}
-	if br.off >= br.size {
+	if br.offR >= br.size {
 		return 0, io.EOF
 	}
 	for {
 		br.rw.Lock()
-		if br.buffer != nil {
-			n, err = br.buffer.Read(p)
-		} else {
-			err = io.ErrClosedPipe
-		}
-		if err != nil && err != io.EOF {
+		if br.buf == nil {
 			br.rw.Unlock()
-			return
+			return 0, io.ErrClosedPipe
 		}
-		if n > 0 {
-			br.off += n
+
+		if br.offW < br.offR {
 			br.rw.Unlock()
-			return n, nil
+			return 0, io.ErrUnexpectedEOF
 		}
-		br.readPending = true
-		br.rw.Unlock()
-		// n==0, err==io.EOF
-		select {
-		case <-br.ctx.Done():
-			return 0, br.ctx.Err()
-		case _, ok := <-br.readSignal:
-			if !ok {
-				return 0, io.ErrClosedPipe
+		if br.offW == br.offR {
+			br.readPending = true
+			br.rw.Unlock()
+			select {
+			case <-br.ctx.Done():
+				return 0, br.ctx.Err()
+			case _, ok := <-br.readSignal:
+				if !ok {
+					return 0, io.ErrClosedPipe
+				}
+				continue
 			}
-			continue
 		}
+
+		n := copy(p, br.buf[br.offR:br.offW])
+		br.offR += n
+		br.rw.Unlock()
+		if n < len(p) && br.offR >= br.size {
+			return n, io.EOF
+		}
+		return n, nil
 	}
 }
 
-func (br *Buf) Write(p []byte) (n int, err error) {
+func (br *Buf) Write(p []byte) (int, error) {
 	if err := br.ctx.Err(); err != nil {
 		return 0, err
 	}
+	if len(p) == 0 {
+		return 0, nil
+	}
 	br.rw.Lock()
 	defer br.rw.Unlock()
-	if br.buffer == nil {
+	if br.buf == nil {
 		return 0, io.ErrClosedPipe
 	}
-	n, err = br.buffer.Write(p)
+	if br.offW >= br.size {
+		return 0, io.ErrShortWrite
+	}
+	n := copy(br.buf[br.offW:], p[:min(br.size-br.offW, len(p))])
+	br.offW += n
 	if br.readPending {
 		br.readPending = false
 		select {
@@ -735,12 +764,21 @@ func (br *Buf) Write(p []byte) (n int, err error) {
 		default:
 		}
 	}
-	return
+	if n < len(p) {
+		return n, io.ErrShortWrite
+	}
+	return n, nil
 }
 
-func (br *Buf) Close() {
+func (br *Buf) Close() error {
 	br.rw.Lock()
 	defer br.rw.Unlock()
-	br.buffer = nil
+	var err error
+	if br.mmap {
+		err = mmap.Free(br.buf)
+		br.mmap = false
+	}
+	br.buf = nil
 	close(br.readSignal)
+	return err
 }

internal/stream/stream.go

@@ -1,7 +1,6 @@
 package stream
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -13,8 +12,10 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/internal/conf"
 	"github.com/OpenListTeam/OpenList/v4/internal/errs"
 	"github.com/OpenListTeam/OpenList/v4/internal/model"
+	"github.com/OpenListTeam/OpenList/v4/pkg/buffer"
 	"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
+	"github.com/rclone/rclone/lib/mmap"
 	"go4.org/readerutil"
 )
 
@@ -27,13 +28,19 @@ type FileStream struct {
 	ForceStreamUpload bool
 	Exist             model.Obj //the file existed in the destination, we can reuse some info since we wil overwrite it
 	utils.Closers
-	tmpFile  *os.File //if present, tmpFile has full content, it will be deleted at last
-	peekBuff *bytes.Reader
+
+	tmpFile   model.File //if present, tmpFile has full content, it will be deleted at last
+	peekBuff  *buffer.Reader
+	size      int64
+	oriReader io.Reader // the original reader, used for caching
 }
 
 func (f *FileStream) GetSize() int64 {
-	if f.tmpFile != nil {
-		info, err := f.tmpFile.Stat()
+	if f.size > 0 {
+		return f.size
+	}
+	if file, ok := f.tmpFile.(*os.File); ok {
+		info, err := file.Stat()
 		if err == nil {
 			return info.Size()
 		}
@@ -54,16 +61,20 @@ func (f *FileStream) IsForceStreamUpload() bool {
 }
 
 func (f *FileStream) Close() error {
-	var err1, err2 error
+	if f.peekBuff != nil {
+		f.peekBuff.Reset()
+		f.peekBuff = nil
+	}
 
+	var err1, err2 error
 	err1 = f.Closers.Close()
 	if errors.Is(err1, os.ErrClosed) {
 		err1 = nil
 	}
-	if f.tmpFile != nil {
-		err2 = os.RemoveAll(f.tmpFile.Name())
+	if file, ok := f.tmpFile.(*os.File); ok {
+		err2 = os.RemoveAll(file.Name())
 		if err2 != nil {
-			err2 = errs.NewErr(err2, "failed to remove tmpFile [%s]", f.tmpFile.Name())
+			err2 = errs.NewErr(err2, "failed to remove tmpFile [%s]", file.Name())
 		} else {
 			f.tmpFile = nil
 		}
@@ -79,20 +90,55 @@ func (f *FileStream) SetExist(obj model.Obj) {
 	f.Exist = obj
 }
 
-// CacheFullInTempFile save all data into tmpFile. Not recommended since it wears disk,
-// and can't start upload until the file is written. It's not thread-safe!
-func (f *FileStream) CacheFullInTempFile() (model.File, error) {
-	if file := f.GetFile(); file != nil {
-		return file, nil
+// CacheFullAndWriter save all data into tmpFile or memory.
+// It's not thread-safe!
+func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writer) (model.File, error) {
+	if cache := f.GetFile(); cache != nil {
+		if writer == nil {
+			return cache, nil
+		}
+		_, err := cache.Seek(0, io.SeekStart)
+		if err == nil {
+			reader := f.Reader
+			if up != nil {
+				cacheProgress := model.UpdateProgressWithRange(*up, 0, 50)
+				*up = model.UpdateProgressWithRange(*up, 50, 100)
+				reader = &ReaderUpdatingProgress{
+					Reader: &SimpleReaderWithSize{
+						Reader: reader,
+						Size:   f.GetSize(),
+					},
+					UpdateProgress: cacheProgress,
+				}
+			}
+			_, err = utils.CopyWithBuffer(writer, reader)
+			if err == nil {
+				_, err = cache.Seek(0, io.SeekStart)
+			}
+		}
+		if err != nil {
+			return nil, err
+		}
+		return cache, nil
 	}
-	tmpF, err := utils.CreateTempFile(f.Reader, f.GetSize())
-	if err != nil {
-		return nil, err
+
+	reader := f.Reader
+	if up != nil {
+		cacheProgress := model.UpdateProgressWithRange(*up, 0, 50)
+		*up = model.UpdateProgressWithRange(*up, 50, 100)
+		reader = &ReaderUpdatingProgress{
+			Reader: &SimpleReaderWithSize{
+				Reader: reader,
+				Size:   f.GetSize(),
+			},
+			UpdateProgress: cacheProgress,
+		}
 	}
-	f.Add(tmpF)
-	f.tmpFile = tmpF
-	f.Reader = tmpF
-	return tmpF, nil
+	if writer != nil {
+		reader = io.TeeReader(reader, writer)
+	}
+	f.Reader = reader
+	return f.cache(f.GetSize())
 }
 
 func (f *FileStream) GetFile() model.File {
@@ -106,40 +152,80 @@ func (f *FileStream) GetFile() model.File {
 }
 
 // RangeRead have to cache all data first since only Reader is provided.
-// also support a peeking RangeRead at very start, but won't buffer more than conf.MaxBufferLimit data in memory
+// It's not thread-safe!
 func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) {
 	if httpRange.Length < 0 || httpRange.Start+httpRange.Length > f.GetSize() {
 		httpRange.Length = f.GetSize() - httpRange.Start
 	}
-	var cache io.ReaderAt = f.GetFile()
-	if cache != nil {
-		return io.NewSectionReader(cache, httpRange.Start, httpRange.Length), nil
+	if f.GetFile() != nil {
+		return io.NewSectionReader(f.GetFile(), httpRange.Start, httpRange.Length), nil
 	}
 
 	size := httpRange.Start + httpRange.Length
 	if f.peekBuff != nil && size <= int64(f.peekBuff.Len()) {
 		return io.NewSectionReader(f.peekBuff, httpRange.Start, httpRange.Length), nil
 	}
-	if size <= int64(conf.MaxBufferLimit) {
-		bufSize := min(size, f.GetSize())
-		// 使用bytes.Buffer作为io.CopyBuffer的写入对象，CopyBuffer会调用Buffer.ReadFrom
-		// 即使被写入的数据量与Buffer.Cap一致，Buffer也会扩大
-		buf := make([]byte, bufSize)
-		n, err := io.ReadFull(f.Reader, buf)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", bufSize, n, err)
-		}
-		f.peekBuff = bytes.NewReader(buf)
-		f.Reader = io.MultiReader(f.peekBuff, f.Reader)
-		cache = f.peekBuff
-	} else {
-		var err error
-		cache, err = f.CacheFullInTempFile()
+
+	cache, err := f.cache(size)
+	if err != nil {
+		return nil, err
+	}
+
+	return io.NewSectionReader(cache, httpRange.Start, httpRange.Length), nil
+}
+
+// *旧笔记
+// 使用bytes.Buffer作为io.CopyBuffer的写入对象，CopyBuffer会调用Buffer.ReadFrom
+// 即使被写入的数据量与Buffer.Cap一致，Buffer也会扩大
+
+func (f *FileStream) cache(maxCacheSize int64) (model.File, error) {
+	if maxCacheSize > int64(conf.MaxBufferLimit) {
+		tmpF, err := utils.CreateTempFile(f.Reader, f.GetSize())
 		if err != nil {
 			return nil, err
 		}
+		f.Add(tmpF)
+		f.tmpFile = tmpF
+		f.Reader = tmpF
+		return tmpF, nil
 	}
-	return io.NewSectionReader(cache, httpRange.Start, httpRange.Length), nil
+
+	if f.peekBuff == nil {
+		f.peekBuff = &buffer.Reader{}
+		f.oriReader = f.Reader
+	}
+	bufSize := maxCacheSize - int64(f.peekBuff.Len())
+	var buf []byte
+	if conf.MmapThreshold > 0 && bufSize >= int64(conf.MmapThreshold) {
+		m, err := mmap.Alloc(int(bufSize))
+		if err == nil {
+			f.Add(utils.CloseFunc(func() error {
+				return mmap.Free(m)
+			}))
+			buf = m
+		}
+	}
+	if buf == nil {
+		buf = make([]byte, bufSize)
+	}
+	n, err := io.ReadFull(f.oriReader, buf)
+	if bufSize != int64(n) {
+		return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", bufSize, n, err)
+	}
+	f.peekBuff.Append(buf)
+	if int64(f.peekBuff.Len()) >= f.GetSize() {
+		f.Reader = f.peekBuff
+		f.oriReader = nil
+	} else {
+		f.Reader = io.MultiReader(f.peekBuff, f.oriReader)
+	}
+	return f.peekBuff, nil
+}
+
+func (f *FileStream) SetTmpFile(file model.File) {
+	f.AddIfCloser(file)
+	f.tmpFile = file
+	f.Reader = file
 }
 
 var _ model.FileStreamer = (*SeekableStream)(nil)
@@ -156,7 +242,6 @@ type SeekableStream struct {
 	*FileStream
 	// should have one of belows to support rangeRead
 	rangeReadCloser model.RangeReadCloserIF
-	size            int64
 }
 
 func NewSeekableStream(fs *FileStream, link *model.Link) (*SeekableStream, error) {
@@ -178,38 +263,26 @@ func NewSeekableStream(fs *FileStream, link *model.Link) (*SeekableStream, error
 		if err != nil {
 			return nil, err
 		}
-		if _, ok := rr.(*model.FileRangeReader); ok {
-			fs.Reader, err = rr.RangeRead(fs.Ctx, http_range.Range{Length: -1})
-			if err != nil {
-				return nil, err
-			}
-			fs.Add(link)
-			return &SeekableStream{FileStream: fs, size: size}, nil
-		}
 		rrc := &model.RangeReadCloser{
 			RangeReader: rr,
 		}
+		if _, ok := rr.(*model.FileRangeReader); ok {
+			fs.Reader, err = rrc.RangeRead(fs.Ctx, http_range.Range{Length: -1})
+			if err != nil {
+				return nil, err
+			}
+		}
+		fs.size = size
 		fs.Add(link)
 		fs.Add(rrc)
-		return &SeekableStream{FileStream: fs, rangeReadCloser: rrc, size: size}, nil
+		return &SeekableStream{FileStream: fs, rangeReadCloser: rrc}, nil
 	}
 	return nil, fmt.Errorf("illegal seekableStream")
 }
 
-func (ss *SeekableStream) GetSize() int64 {
-	if ss.size > 0 {
-		return ss.size
-	}
-	return ss.FileStream.GetSize()
-}
-
-//func (ss *SeekableStream) Peek(length int) {
-//
-//}
-
 // RangeRead is not thread-safe, pls use it in single thread only.
 func (ss *SeekableStream) RangeRead(httpRange http_range.Range) (io.Reader, error) {
-	if ss.tmpFile == nil && ss.rangeReadCloser != nil {
+	if ss.GetFile() == nil && ss.rangeReadCloser != nil {
 		rc, err := ss.rangeReadCloser.RangeRead(ss.Ctx, httpRange)
 		if err != nil {
 			return nil, err
@@ -219,47 +292,37 @@ func (ss *SeekableStream) RangeRead(httpRange http_range.Range) (io.Reader, erro
 	return ss.FileStream.RangeRead(httpRange)
 }
 
-//func (f *FileStream) GetReader() io.Reader {
-//	return f.Reader
-//}
-
 // only provide Reader as full stream when it's demanded. in rapid-upload, we can skip this to save memory
 func (ss *SeekableStream) Read(p []byte) (n int, err error) {
+	if err := ss.generateReader(); err != nil {
+		return 0, err
+	}
+	return ss.FileStream.Read(p)
+}
+
+func (ss *SeekableStream) generateReader() error {
 	if ss.Reader == nil {
 		if ss.rangeReadCloser == nil {
-			return 0, fmt.Errorf("illegal seekableStream")
+			return fmt.Errorf("illegal seekableStream")
 		}
 		rc, err := ss.rangeReadCloser.RangeRead(ss.Ctx, http_range.Range{Length: -1})
 		if err != nil {
-			return 0, err
+			return err
 		}
 		ss.Reader = rc
 	}
-	return ss.Reader.Read(p)
+	return nil
 }
 
-func (ss *SeekableStream) CacheFullInTempFile() (model.File, error) {
-	if file := ss.GetFile(); file != nil {
-		return file, nil
-	}
-	tmpF, err := utils.CreateTempFile(ss, ss.GetSize())
-	if err != nil {
+func (ss *SeekableStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writer) (model.File, error) {
+	if err := ss.generateReader(); err != nil {
 		return nil, err
 	}
-	ss.Add(tmpF)
-	ss.tmpFile = tmpF
-	ss.Reader = tmpF
-	return tmpF, nil
-}
-
-func (f *FileStream) SetTmpFile(r *os.File) {
-	f.Add(r)
-	f.tmpFile = r
-	f.Reader = r
+	return ss.FileStream.CacheFullAndWriter(up, writer)
 }
 
 type ReaderWithSize interface {
-	io.ReadCloser
+	io.Reader
 	GetSize() int64
 }
 
@@ -293,7 +356,10 @@ func (r *ReaderUpdatingProgress) Read(p []byte) (n int, err error) {
 }
 
 func (r *ReaderUpdatingProgress) Close() error {
-	return r.Reader.Close()
+	if c, ok := r.Reader.(io.Closer); ok {
+		return c.Close()
+	}
+	return nil
 }
 
 type RangeReadReadAtSeeker struct {
@@ -311,19 +377,20 @@ type headCache struct {
 func (c *headCache) head(p []byte) (int, error) {
 	n := 0
 	for _, buf := range c.bufs {
-		if len(buf)+n >= len(p) {
-			n += copy(p[n:], buf[:len(p)-n])
+		n += copy(p[n:], buf)
+		if n == len(p) {
 			return n, nil
-		} else {
-			n += copy(p[n:], buf)
 		}
 	}
-	w, err := io.ReadAtLeast(c.reader, p[n:], 1)
-	if w > 0 {
-		buf := make([]byte, w)
-		copy(buf, p[n:n+w])
+	nn, err := io.ReadFull(c.reader, p[n:])
+	if nn > 0 {
+		buf := make([]byte, nn)
+		copy(buf, p[n:])
 		c.bufs = append(c.bufs, buf)
-		n += w
+		n += nn
+		if err == io.ErrUnexpectedEOF {
+			err = io.EOF
+		}
 	}
 	return n, err
 }
@@ -422,6 +489,9 @@ func (r *RangeReadReadAtSeeker) getReaderAtOffset(off int64) (io.Reader, error)
 }
 
 func (r *RangeReadReadAtSeeker) ReadAt(p []byte, off int64) (n int, err error) {
+	if off < 0 || off >= r.ss.GetSize() {
+		return 0, io.EOF
+	}
 	if off == 0 && r.headCache != nil {
 		return r.headCache.head(p)
 	}
@@ -430,12 +500,15 @@ func (r *RangeReadReadAtSeeker) ReadAt(p []byte, off int64) (n int, err error) {
 	if err != nil {
 		return 0, err
 	}
-	n, err = io.ReadAtLeast(rr, p, 1)
-	off += int64(n)
-	if err == nil {
-		r.readerMap.Store(int64(off), rr)
-	} else {
-		rr = nil
+	n, err = io.ReadFull(rr, p)
+	if n > 0 {
+		off += int64(n)
+		switch err {
+		case nil:
+			r.readerMap.Store(int64(off), rr)
+		case io.ErrUnexpectedEOF:
+			err = io.EOF
+		}
 	}
 	return n, err
 }
@@ -444,20 +517,14 @@ func (r *RangeReadReadAtSeeker) Seek(offset int64, whence int) (int64, error) {
 	switch whence {
 	case io.SeekStart:
 	case io.SeekCurrent:
-		if offset == 0 {
-			return r.masterOff, nil
-		}
 		offset += r.masterOff
 	case io.SeekEnd:
 		offset += r.ss.GetSize()
 	default:
-		return 0, errs.NotSupport
+		return 0, errors.New("Seek: invalid whence")
 	}
-	if offset < 0 {
-		return r.masterOff, errors.New("invalid seek: negative position")
-	}
-	if offset > r.ss.GetSize() {
-		offset = r.ss.GetSize()
+	if offset < 0 || offset > r.ss.GetSize() {
+		return 0, errors.New("Seek: invalid offset")
 	}
 	r.masterOff = offset
 	return offset, nil
@@ -465,6 +532,8 @@ func (r *RangeReadReadAtSeeker) Seek(offset int64, whence int) (int64, error) {
 
 func (r *RangeReadReadAtSeeker) Read(p []byte) (n int, err error) {
 	n, err = r.ReadAt(p, r.masterOff)
-	r.masterOff += int64(n)
+	if n > 0 {
+		r.masterOff += int64(n)
+	}
 	return n, err
 }

internal/stream/stream_test.go

@@ -0,0 +1,88 @@
+package stream
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/OpenListTeam/OpenList/v4/internal/conf"
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+	"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
+)
+
+func TestFileStream_RangeRead(t *testing.T) {
+	conf.MaxBufferLimit = 16 * 1024 * 1024
+	type args struct {
+		httpRange http_range.Range
+	}
+	buf := []byte("github.com/OpenListTeam/OpenList")
+	f := &FileStream{
+		Obj: &model.Object{
+			Size: int64(len(buf)),
+		},
+		Reader: io.NopCloser(bytes.NewReader(buf)),
+	}
+	tests := []struct {
+		name string
+		f    *FileStream
+		args args
+		want func(f *FileStream, got io.Reader, err error) error
+	}{
+		{
+			name: "range 11-12",
+			f:    f,
+			args: args{
+				httpRange: http_range.Range{Start: 11, Length: 12},
+			},
+			want: func(f *FileStream, got io.Reader, err error) error {
+				if f.GetFile() != nil {
+					return errors.New("cached")
+				}
+				b, _ := io.ReadAll(got)
+				if !bytes.Equal(buf[11:11+12], b) {
+					return fmt.Errorf("=%s ,want =%s", b, buf[11:11+12])
+				}
+				return nil
+			},
+		},
+		{
+			name: "range 11-21",
+			f:    f,
+			args: args{
+				httpRange: http_range.Range{Start: 11, Length: 21},
+			},
+			want: func(f *FileStream, got io.Reader, err error) error {
+				if f.GetFile() == nil {
+					return errors.New("not cached")
+				}
+				b, _ := io.ReadAll(got)
+				if !bytes.Equal(buf[11:11+21], b) {
+					return fmt.Errorf("=%s ,want =%s", b, buf[11:11+21])
+				}
+				return nil
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.f.RangeRead(tt.args.httpRange)
+			if err := tt.want(tt.f, got, err); err != nil {
+				t.Errorf("FileStream.RangeRead() %v", err)
+			}
+		})
+	}
+	t.Run("after", func(t *testing.T) {
+		if f.GetFile() == nil {
+			t.Error("not cached")
+		}
+		buf2 := make([]byte, len(buf))
+		if _, err := io.ReadFull(f, buf2); err != nil {
+			t.Errorf("FileStream.Read() error = %v", err)
+		}
+		if !bytes.Equal(buf, buf2) {
+			t.Errorf("FileStream.Read() = %s, want %s", buf2, buf)
+		}
+	})
+}

internal/stream/util.go

@@ -8,13 +8,14 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"sync"
 
 	"github.com/OpenListTeam/OpenList/v4/internal/conf"
 	"github.com/OpenListTeam/OpenList/v4/internal/model"
 	"github.com/OpenListTeam/OpenList/v4/internal/net"
 	"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
+	"github.com/OpenListTeam/OpenList/v4/pkg/pool"
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
+	"github.com/rclone/rclone/lib/mmap"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -141,81 +142,61 @@ func (r *ReaderWithCtx) Close() error {
 	return nil
 }
 
-func CacheFullInTempFileAndWriter(stream model.FileStreamer, up model.UpdateProgress, w io.Writer) (model.File, error) {
-	if cache := stream.GetFile(); cache != nil {
-		if w != nil {
-			_, err := cache.Seek(0, io.SeekStart)
-			if err == nil {
-				var reader io.Reader = stream
-				if up != nil {
-					reader = &ReaderUpdatingProgress{
-						Reader:         stream,
-						UpdateProgress: up,
-					}
-				}
-				_, err = utils.CopyWithBuffer(w, reader)
-				if err == nil {
-					_, err = cache.Seek(0, io.SeekStart)
-				}
-			}
-			return cache, err
-		}
-		if up != nil {
-			up(100)
-		}
-		return cache, nil
-	}
-
-	var reader io.Reader = stream
-	if up != nil {
-		reader = &ReaderUpdatingProgress{
-			Reader:         stream,
-			UpdateProgress: up,
-		}
-	}
-
-	if w != nil {
-		reader = io.TeeReader(reader, w)
-	}
-	tmpF, err := utils.CreateTempFile(reader, stream.GetSize())
-	if err == nil {
-		stream.SetTmpFile(tmpF)
-	}
-	return tmpF, err
-}
-
-func CacheFullInTempFileAndHash(stream model.FileStreamer, up model.UpdateProgress, hashType *utils.HashType, hashParams ...any) (model.File, string, error) {
+func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashType *utils.HashType, hashParams ...any) (model.File, string, error) {
 	h := hashType.NewFunc(hashParams...)
-	tmpF, err := CacheFullInTempFileAndWriter(stream, up, h)
+	tmpF, err := stream.CacheFullAndWriter(up, h)
 	if err != nil {
 		return nil, "", err
 	}
-	return tmpF, hex.EncodeToString(h.Sum(nil)), err
+	return tmpF, hex.EncodeToString(h.Sum(nil)), nil
 }
 
 type StreamSectionReader struct {
 	file    model.FileStreamer
 	off     int64
-	bufPool *sync.Pool
+	bufPool *pool.Pool[[]byte]
 }
 
-func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int) (*StreamSectionReader, error) {
+func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *model.UpdateProgress) (*StreamSectionReader, error) {
 	ss := &StreamSectionReader{file: file}
-	if file.GetFile() == nil {
-		maxBufferSize = min(maxBufferSize, int(file.GetSize()))
-		if maxBufferSize > conf.MaxBufferLimit {
-			_, err := file.CacheFullInTempFile()
-			if err != nil {
-				return nil, err
-			}
-		} else {
-			ss.bufPool = &sync.Pool{
-				New: func() any {
-					return make([]byte, maxBufferSize)
-				},
-			}
+	if file.GetFile() != nil {
+		return ss, nil
+	}
+
+	maxBufferSize = min(maxBufferSize, int(file.GetSize()))
+	if maxBufferSize > conf.MaxBufferLimit {
+		_, err := file.CacheFullAndWriter(up, nil)
+		if err != nil {
+			return nil, err
+		}
+		return ss, nil
+	}
+	if conf.MmapThreshold > 0 && maxBufferSize >= conf.MmapThreshold {
+		ss.bufPool = &pool.Pool[[]byte]{
+			New: func() []byte {
+				buf, err := mmap.Alloc(maxBufferSize)
+				if err == nil {
+					file.Add(utils.CloseFunc(func() error {
+						return mmap.Free(buf)
+					}))
+				} else {
+					buf = make([]byte, maxBufferSize)
+				}
+				return buf
+			},
+		}
+	} else {
+		ss.bufPool = &pool.Pool[[]byte]{
+			New: func() []byte {
+				return make([]byte, maxBufferSize)
+			},
 		}
 	}
+
+	file.Add(utils.CloseFunc(func() error {
+		ss.bufPool.Reset()
+		return nil
+	}))
 	return ss, nil
 }
 
@@ -227,7 +208,7 @@ func (ss *StreamSectionReader) GetSectionReader(off, length int64) (*SectionRead
 		if off != ss.off {
 			return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.off)
 		}
-		tempBuf := ss.bufPool.Get().([]byte)
+		tempBuf := ss.bufPool.Get()
 		buf = tempBuf[:length]
 		n, err := io.ReadFull(ss.file, buf)
 		if int64(n) != length {
@@ -240,7 +221,7 @@ func (ss *StreamSectionReader) GetSectionReader(off, length int64) (*SectionRead
 	return &SectionReader{io.NewSectionReader(cache, off, length), buf}, nil
 }
 
-func (ss *StreamSectionReader) RecycleSectionReader(sr *SectionReader) {
+func (ss *StreamSectionReader) FreeSectionReader(sr *SectionReader) {
 	if sr != nil {
 		if sr.buf != nil {
 			ss.bufPool.Put(sr.buf[0:cap(sr.buf)])

pkg/buffer/bytes.go

@@ -0,0 +1,92 @@
+package buffer
+
+import (
+	"errors"
+	"io"
+)
+
+// 用于存储不复用的[]byte
+type Reader struct {
+	bufs   [][]byte
+	length int
+	offset int
+}
+
+func (r *Reader) Len() int {
+	return r.length
+}
+
+func (r *Reader) Append(buf []byte) {
+	r.length += len(buf)
+	r.bufs = append(r.bufs, buf)
+}
+
+func (r *Reader) Read(p []byte) (int, error) {
+	n, err := r.ReadAt(p, int64(r.offset))
+	if n > 0 {
+		r.offset += n
+	}
+	return n, err
+}
+
+func (r *Reader) ReadAt(p []byte, off int64) (int, error) {
+	if off < 0 || off >= int64(r.length) {
+		return 0, io.EOF
+	}
+
+	n, length := 0, int64(0)
+	readFrom := false
+	for _, buf := range r.bufs {
+		newLength := length + int64(len(buf))
+		if readFrom {
+			w := copy(p[n:], buf)
+			n += w
+		} else if off < newLength {
+			readFrom = true
+			w := copy(p[n:], buf[int(off-length):])
+			n += w
+		}
+		if n == len(p) {
+			return n, nil
+		}
+		length = newLength
+	}
+
+	return n, io.EOF
+}
+
+func (r *Reader) Seek(offset int64, whence int) (int64, error) {
+	var abs int
+	switch whence {
+	case io.SeekStart:
+		abs = int(offset)
+	case io.SeekCurrent:
+		abs = r.offset + int(offset)
+	case io.SeekEnd:
+		abs = r.length + int(offset)
+	default:
+		return 0, errors.New("Seek: invalid whence")
+	}
+
+	if abs < 0 || abs > r.length {
+		return 0, errors.New("Seek: invalid offset")
+	}
+
+	r.offset = abs
+	return int64(abs), nil
+}
+
+func (r *Reader) Reset() {
+	clear(r.bufs)
+	r.bufs = nil
+	r.length = 0
+	r.offset = 0
+}
+
+func NewReader(buf ...[]byte) *Reader {
+	b := &Reader{}
+	for _, b1 := range buf {
+		b.Append(b1)
+	}
+	return b
+}

pkg/buffer/bytes_test.go

@@ -0,0 +1,95 @@
+package buffer
+
+import (
+	"errors"
+	"io"
+	"testing"
+)
+
+func TestReader_ReadAt(t *testing.T) {
+	type args struct {
+		p   []byte
+		off int64
+	}
+	bs := &Reader{}
+	bs.Append([]byte("github.com"))
+	bs.Append([]byte("/"))
+	bs.Append([]byte("OpenList"))
+	bs.Append([]byte("Team/"))
+	bs.Append([]byte("OpenList"))
+	tests := []struct {
+		name string
+		b    *Reader
+		args args
+		want func(a args, n int, err error) error
+	}{
+		{
+			name: "readAt len 10 offset 0",
+			b:    bs,
+			args: args{
+				p:   make([]byte, 10),
+				off: 0,
+			},
+			want: func(a args, n int, err error) error {
+				if n != len(a.p) {
+					return errors.New("read length not match")
+				}
+				if string(a.p) != "github.com" {
+					return errors.New("read content not match")
+				}
+				if err != nil {
+					return err
+				}
+				return nil
+			},
+		},
+		{
+			name: "readAt len 12 offset 11",
+			b:    bs,
+			args: args{
+				p:   make([]byte, 12),
+				off: 11,
+			},
+			want: func(a args, n int, err error) error {
+				if n != len(a.p) {
+					return errors.New("read length not match")
+				}
+				if string(a.p) != "OpenListTeam" {
+					return errors.New("read content not match")
+				}
+				if err != nil {
+					return err
+				}
+				return nil
+			},
+		},
+		{
+			name: "readAt len 50 offset 24",
+			b:    bs,
+			args: args{
+				p:   make([]byte, 50),
+				off: 24,
+			},
+			want: func(a args, n int, err error) error {
+				if n != bs.Len()-int(a.off) {
+					return errors.New("read length not match")
+				}
+				if string(a.p[:n]) != "OpenList" {
+					return errors.New("read content not match")
+				}
+				if err != io.EOF {
+					return errors.New("expect eof")
+				}
+				return nil
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.b.ReadAt(tt.args.p, tt.args.off)
+			if err := tt.want(tt.args, got, err); err != nil {
+				t.Errorf("Bytes.ReadAt() error = %v", err)
+			}
+		})
+	}
+}

pkg/errgroup/errgroup.go

@@ -53,11 +53,12 @@ func (g *Group) Go(do func(ctx context.Context) error) {
 }
 
 type Lifecycle struct {
-	// Before在OrderedGroup是线程安全的
+	// Before在OrderedGroup是线程安全的。
+	// 只会被调用一次
 	Before func(ctx context.Context) error
 	// 如果Before返回err就不调用Do
 	Do func(ctx context.Context) error
-	// 最后调用After
+	// 最后调用一次After
 	After func(err error)
 }
 

pkg/pool/pool.go

@@ -0,0 +1,37 @@
+package pool
+
+import "sync"
+
+type Pool[T any] struct {
+	New    func() T
+	MaxCap int
+
+	cache []T
+	mu    sync.Mutex
+}
+
+func (p *Pool[T]) Get() T {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if len(p.cache) == 0 {
+		return p.New()
+	}
+	item := p.cache[len(p.cache)-1]
+	p.cache = p.cache[:len(p.cache)-1]
+	return item
+}
+
+func (p *Pool[T]) Put(item T) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.MaxCap == 0 || len(p.cache) < int(p.MaxCap) {
+		p.cache = append(p.cache, item)
+	}
+}
+
+func (p *Pool[T]) Reset() {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	clear(p.cache)
+	p.cache = nil
+}

pkg/utils/io.go

@@ -194,32 +194,32 @@ type SyncClosersIF interface {
 
 type SyncClosers struct {
 	closers []io.Closer
-	ref     atomic.Int32
+	ref     int32
 }
 
 var _ SyncClosersIF = (*SyncClosers)(nil)
 
 func (c *SyncClosers) AcquireReference() bool {
-	ref := c.ref.Add(1)
+	ref := atomic.AddInt32(&c.ref, 1)
 	if ref > 0 {
 		// log.Debugf("SyncClosers.AcquireReference %p,ref=%d\n", c, ref)
 		return true
 	}
-	c.ref.Store(math.MinInt16)
+	atomic.StoreInt32(&c.ref, math.MinInt16)
 	return false
 }
 
 func (c *SyncClosers) Close() error {
-	ref := c.ref.Add(-1)
+	ref := atomic.AddInt32(&c.ref, -1)
 	if ref < -1 {
-		c.ref.Store(math.MinInt16)
+		atomic.StoreInt32(&c.ref, math.MinInt16)
 		return nil
 	}
 	// log.Debugf("SyncClosers.Close %p,ref=%d\n", c, ref+1)
 	if ref > 0 {
 		return nil
 	}
-	c.ref.Store(math.MinInt16)
+	atomic.StoreInt32(&c.ref, math.MinInt16)
 
 	var errs []error
 	for _, closer := range c.closers {
@@ -234,7 +234,7 @@ func (c *SyncClosers) Close() error {
 
 func (c *SyncClosers) Add(closer io.Closer) {
 	if closer != nil {
-		if c.ref.Load() < 0 {
+		if atomic.LoadInt32(&c.ref) < 0 {
 			panic("Not reusable")
 		}
 		c.closers = append(c.closers, closer)
@@ -243,7 +243,7 @@ func (c *SyncClosers) Add(closer io.Closer) {
 
 func (c *SyncClosers) AddIfCloser(a any) {
 	if closer, ok := a.(io.Closer); ok {
-		if c.ref.Load() < 0 {
+		if atomic.LoadInt32(&c.ref) < 0 {
 			panic("Not reusable")
 		}
 		c.closers = append(c.closers, closer)

server/static/static.go

@@ -116,10 +116,10 @@ func UpdateIndex() {
 	mainColor := setting.GetStr(conf.MainColor)
 	utils.Log.Debug("Applying replacements for default pages...")
 	replaceMap1 := map[string]string{
-		"https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg": favicon,
-		"https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.png": logo,
-		"Loading...":            title,
-		"main_color: undefined": fmt.Sprintf("main_color: '%s'", mainColor),
+		"https://res.oplist.org/logo/logo.svg": favicon,
+		"https://res.oplist.org/logo/logo.png": logo,
+		"Loading...":                           title,
+		"main_color: undefined":                fmt.Sprintf("main_color: '%s'", mainColor),
 	}
 	conf.ManageHtml = replaceStrings(conf.RawIndexHtml, replaceMap1)
 	utils.Log.Debug("Applying replacements for manage pages...")