Backups/OpenList
1
0
Fork 0
mirror of https://github.com/OpenListTeam/OpenList.git synced 2025-09-19 04:06:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

perf(link): optimize concurrent response (#641)

Browse Source 
* fix(crypt): bug caused by link cache

* perf(crypt,mega,halalcloud,quark,uc): optimize concurrent response link

* chore: 删除无用代码

* ftp

* 修复bug；资源释放

* 添加SyncClosers

* local,sftp,smb

* 重构，优化，增强

* Update internal/stream/util.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com>

* chore

* chore

* 优化，修复bug

* .

---------

Signed-off-by: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
j2rong4cn
2025-07-12 17:57:54 +08:00
committed by GitHub
parent e5fbe72581
commit cc01b410a4
83 changed files with 796 additions and 751 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
server/common/proxy.go
View File

@ -12,81 +12,63 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/net"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error {
if link.MFile != nil {
if clr, ok := link.MFile.(io.Closer); ok {
defer clr.Close()
}
attachHeader(w, file)
contentType := link.Header.Get("Content-Type")
if contentType != "" {
w.Header().Set("Content-Type", contentType)
}
attachHeader(w, file, link.Header)
http.ServeContent(w, r, file.GetName(), file.ModTime(), link.MFile)
return nil
} else if link.RangeReadCloser != nil {
attachHeader(w, file)
return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &stream.RateLimitRangeReadCloser{
RangeReadCloserIF: link.RangeReadCloser,
Limiter: stream.ServerDownloadLimit,
})
} else if link.Concurrency > 0 || link.PartSize > 0 {
attachHeader(w, file)
size := file.GetSize()
rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) {
requestHeader := ctx.Value("request_header")
if requestHeader == nil {
requestHeader = http.Header{}
}
header := net.ProcessHeader(requestHeader.(http.Header), link.Header)
down := net.NewDownloader(func(d *net.Downloader) {
d.Concurrency = link.Concurrency
d.PartSize = link.PartSize
})
req := &net.HttpRequestParams{
URL: link.URL,
Range: httpRange,
Size: size,
HeaderRef: header,
}
rc, err := down.Download(ctx, req)
return rc, err
}
return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &stream.RateLimitRangeReadCloser{
RangeReadCloserIF: &model.RangeReadCloser{RangeReader: rangeReader},
Limiter: stream.ServerDownloadLimit,
})
} else {
//transparent proxy
header := net.ProcessHeader(r.Header, link.Header)
res, err := net.RequestHttp(r.Context(), r.Method, header, link.URL)
if err != nil {
return err
}
defer res.Body.Close()
}
maps.Copy(w.Header(), res.Header)
w.WriteHeader(res.StatusCode)
if r.Method == http.MethodHead {
return nil
if link.Concurrency > 0 || link.PartSize > 0 {
attachHeader(w, file, link.Header)
rrf, _ := stream.GetRangeReaderFromLink(file.GetSize(), link)
if link.RangeReader == nil {
r = r.WithContext(context.WithValue(r.Context(), net.RequestHeaderKey{}, r.Header))
}
_, err = utils.CopyWithBuffer(w, &stream.RateLimitReader{
Reader: res.Body,
Limiter: stream.ServerDownloadLimit,
Ctx: r.Context(),
return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &model.RangeReadCloser{
RangeReader: rrf,
})
}
if link.RangeReader != nil {
attachHeader(w, file, link.Header)
return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &model.RangeReadCloser{
RangeReader: link.RangeReader,
})
}
//transparent proxy
header := net.ProcessHeader(r.Header, link.Header)
res, err := net.RequestHttp(r.Context(), r.Method, header, link.URL)
if err != nil {
return err
}
defer res.Body.Close()
maps.Copy(w.Header(), res.Header)
w.WriteHeader(res.StatusCode)
if r.Method == http.MethodHead {
return nil
}
_, err = utils.CopyWithBuffer(w, &stream.RateLimitReader{
Reader: res.Body,
Limiter: stream.ServerDownloadLimit,
Ctx: r.Context(),
})
return err
}
func attachHeader(w http.ResponseWriter, file model.Obj) {
func attachHeader(w http.ResponseWriter, file model.Obj, header http.Header) {
fileName := file.GetName()
w.Header().Set("Content-Disposition", utils.GenerateContentDisposition(fileName))
w.Header().Set("Content-Type", utils.GetMimeType(fileName))
w.Header().Set("Etag", GetEtag(file))
contentType := header.Get("Content-Type")
if len(contentType) > 0 {
w.Header().Set("Content-Type", contentType)
}
}
func GetEtag(file model.Obj) string {
hash := ""
@ -106,12 +88,12 @@ func ProxyRange(ctx context.Context, link *model.Link, size int64) {
if link.MFile != nil {
return
}
if link.RangeReadCloser == nil && !strings.HasPrefix(link.URL, GetApiUrl(ctx)+"/") {
var rrc, err = stream.GetRangeReadCloserFromLink(size, link)
if link.RangeReader == nil && !strings.HasPrefix(link.URL, GetApiUrl(ctx)+"/") {
rrf, err := stream.GetRangeReaderFromLink(size, link)
if err != nil {
return
}
link.RangeReadCloser = rrc
link.RangeReader = rrf
}
}

6
server/ftp/fsread.go
View File

@ -45,12 +45,12 @@ func OpenDownload(ctx context.Context, reqPath string, offset int64) (*FileDownl
if err != nil {
return nil, err
}
fileStream := stream.FileStream{
ss, err := stream.NewSeekableStream(&stream.FileStream{
Obj: obj,
Ctx: ctx,
}
ss, err := stream.NewSeekableStream(fileStream, link)
}, link)
if err != nil {
_ = link.Close()
return nil, err
}
reader, err := stream.NewReadAtSeeker(ss, offset)

4
server/handles/archive.go
View File

@ -321,7 +321,7 @@ func ArchiveDown(c *gin.Context) {
common.ErrorResp(c, err, 500)
return
}
down(c, link)
redirect(c, link)
}
}
@ -351,7 +351,7 @@ func ArchiveProxy(c *gin.Context) {
common.ErrorResp(c, err, 500)
return
}
localProxy(c, link, file, storage.GetStorage().ProxyRange)
proxy(c, link, file, storage.GetStorage().ProxyRange)
} else {
common.ErrorStrResp(c, "proxy not allowed", 403)
return

27
server/handles/down.go
View File

@ -2,8 +2,8 @@ package handles
import (
"bytes"
"errors"
"fmt"
"io"
stdpath "path"
"strconv"
"strings"
@ -12,6 +12,7 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/net"
"github.com/OpenListTeam/OpenList/v4/internal/setting"
"github.com/OpenListTeam/OpenList/v4/internal/sign"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
@ -44,7 +45,7 @@ func Down(c *gin.Context) {
common.ErrorResp(c, err, 500)
return
}
down(c, link)
redirect(c, link)
}
}
@ -77,22 +78,15 @@ func Proxy(c *gin.Context) {
common.ErrorResp(c, err, 500)
return
}
localProxy(c, link, file, storage.GetStorage().ProxyRange)
proxy(c, link, file, storage.GetStorage().ProxyRange)
} else {
common.ErrorStrResp(c, "proxy not allowed", 403)
return
}
}
func down(c *gin.Context, link *model.Link) {
if clr, ok := link.MFile.(io.Closer); ok {
defer func(clr io.Closer) {
err := clr.Close()
if err != nil {
log.Errorf("close link data error: %v", err)
}
}(clr)
}
func redirect(c *gin.Context, link *model.Link) {
defer link.Close()
var err error
c.Header("Referrer-Policy", "no-referrer")
c.Header("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate")
@ -110,7 +104,8 @@ func down(c *gin.Context, link *model.Link) {
c.Redirect(302, link.URL)
}
func localProxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange bool) {
func proxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange bool) {
defer link.Close()
var err error
if link.URL != "" && setting.GetBool(conf.ForwardDirectLinkParams) {
query := c.Request.URL.Query()
@ -161,7 +156,11 @@ func localProxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange boo
if Writer.IsWritten() {
log.Errorf("%s %s local proxy error: %+v", c.Request.Method, c.Request.URL.Path, err)
} else {
common.ErrorResp(c, err, 500, true)
if statusCode, ok := errors.Unwrap(err).(net.ErrorHttpStatusCode); ok {
common.ErrorResp(c, err, int(statusCode), true)
} else {
common.ErrorResp(c, err, 500, true)
}
}
}

15
server/handles/fsmanage.go
View File

@ -2,7 +2,6 @@ package handles
import (
"fmt"
"io"
stdpath "path"
"github.com/OpenListTeam/OpenList/v4/internal/task"
@ -17,7 +16,6 @@ import (
"github.com/OpenListTeam/OpenList/v4/server/common"
"github.com/gin-gonic/gin"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
type MkdirOrLinkReq struct {
@ -376,7 +374,7 @@ func Link(c *gin.Context) {
common.ErrorResp(c, err, 500)
return
}
if storage.Config().OnlyLocal {
if storage.Config().NoLinkURL || storage.Config().OnlyLinkMFile {
common.SuccessResp(c, model.Link{
URL: fmt.Sprintf("%s/p%s?d&sign=%s",
common.GetApiUrl(c),
@ -385,18 +383,11 @@ func Link(c *gin.Context) {
})
return
}
link, _, err := fs.Link(c, rawPath, model.LinkArgs{IP: c.ClientIP(), Header: c.Request.Header})
link, _, err := fs.Link(c, rawPath, model.LinkArgs{IP: c.ClientIP(), Header: c.Request.Header, Redirect: true})
if err != nil {
common.ErrorResp(c, err, 500)
return
}
if clr, ok := link.MFile.(io.Closer); ok {
defer func(clr io.Closer) {
err := clr.Close()
if err != nil {
log.Errorf("close link data error: %v", err)
}
}(clr)
}
defer link.Close()
common.SuccessResp(c, link)
}

1
server/handles/fsread.go
View File

@ -315,6 +315,7 @@ func FsGet(c *gin.Context) {
common.ErrorResp(c, err, 500)
return
}
defer link.Close()
rawURL = link.URL
}
}

24
server/handles/fsup.go
View File

@ -28,6 +28,12 @@ func getLastModified(c *gin.Context) time.Time {
}
func FsStream(c *gin.Context) {
defer func() {
if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 {
_, _ = utils.CopyWithBuffer(io.Discard, c.Request.Body)
}
_ = c.Request.Body.Close()
}()
path := c.GetHeader("File-Path")
path, err := url.PathUnescape(path)
if err != nil {
@ -44,7 +50,6 @@ func FsStream(c *gin.Context) {
}
if !overwrite {
if res, _ := fs.Get(c, path, &fs.GetArgs{NoLog: true}); res != nil {
_, _ = utils.CopyWithBuffer(io.Discard, c.Request.Body)
common.ErrorStrResp(c, "file exists", 403)
return
}
@ -90,15 +95,11 @@ func FsStream(c *gin.Context) {
} else {
err = fs.PutDirectly(c, dir, s, true)
}
defer c.Request.Body.Close()
if err != nil {
common.ErrorResp(c, err, 500)
return
}
if t == nil {
if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 {
_, _ = utils.CopyWithBuffer(io.Discard, c.Request.Body)
}
common.SuccessResp(c)
return
}
@ -108,6 +109,12 @@ func FsStream(c *gin.Context) {
}
func FsForm(c *gin.Context) {
defer func() {
if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 {
_, _ = utils.CopyWithBuffer(io.Discard, c.Request.Body)
}
_ = c.Request.Body.Close()
}()
path := c.GetHeader("File-Path")
path, err := url.PathUnescape(path)
if err != nil {
@ -124,7 +131,6 @@ func FsForm(c *gin.Context) {
}
if !overwrite {
if res, _ := fs.Get(c, path, &fs.GetArgs{NoLog: true}); res != nil {
_, _ = utils.CopyWithBuffer(io.Discard, c.Request.Body)
common.ErrorStrResp(c, "file exists", 403)
return
}
@ -164,7 +170,7 @@ func FsForm(c *gin.Context) {
if len(mimetype) == 0 {
mimetype = utils.GetMimeType(name)
}
s := stream.FileStream{
s := &stream.FileStream{
Obj: &model.Object{
Name: name,
Size: file.Size,
@ -180,9 +186,9 @@ func FsForm(c *gin.Context) {
s.Reader = struct {
io.Reader
}{f}
t, err = fs.PutAsTask(c, dir, &s)
t, err = fs.PutAsTask(c, dir, s)
} else {
err = fs.PutDirectly(c, dir, &s, true)
err = fs.PutDirectly(c, dir, s, true)
}
if err != nil {
common.ErrorResp(c, err, 500)

65
server/s3/backend.go
View File

@ -142,7 +142,7 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strin
}
// GetObject fetchs the object from the filesystem.
func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) {
func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (s3Obj *gofakes3.Object, err error) {
bucket, err := getBucketByName(bucketName)
if err != nil {
return nil, err
@ -164,6 +164,11 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
if err != nil {
return nil, err
}
defer func() {
if s3Obj == nil {
_ = link.Close()
}
}()
size := file.GetSize()
rnge, err := rangeRequest.Range(size)
@ -171,49 +176,19 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
return nil, err
}
if link.RangeReadCloser == nil && link.MFile == nil && len(link.URL) == 0 {
rrf, err := stream.GetRangeReaderFromLink(size, link)
if err != nil {
return nil, fmt.Errorf("the remote storage driver need to be enhanced to support s3")
}
var rdr io.ReadCloser
length := int64(-1)
start := int64(0)
var rd io.Reader
if rnge != nil {
start, length = rnge.Start, rnge.Length
}
// 参考 server/common/proxy.go
if link.MFile != nil {
_, err := link.MFile.Seek(start, io.SeekStart)
if err != nil {
return nil, err
}
if rdr2, ok := link.MFile.(io.ReadCloser); ok {
rdr = rdr2
} else {
rdr = io.NopCloser(link.MFile)
}
rd, err = rrf.RangeRead(ctx, http_range.Range(*rnge))
} else {
remoteFileSize := file.GetSize()
if length >= 0 && start+length >= remoteFileSize {
length = -1
}
rrc := link.RangeReadCloser
if len(link.URL) > 0 {
var converted, err = stream.GetRangeReadCloserFromLink(remoteFileSize, link)
if err != nil {
return nil, err
}
rrc = converted
}
if rrc != nil {
remoteReader, err := rrc.RangeRead(ctx, http_range.Range{Start: start, Length: length})
if err != nil {
return nil, err
}
rdr = utils.ReadCloser{Reader: remoteReader, Closer: rrc}
} else {
return nil, errs.NotSupport
}
rd, err = rrf.RangeRead(ctx, http_range.Range{Length: -1})
}
if err != nil {
return nil, err
}
meta := map[string]string{
@ -236,7 +211,7 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
Metadata: meta,
Size: size,
Range: rnge,
Contents: rdr,
Contents: utils.ReadCloser{Reader: rd, Closer: link},
}, nil
}
@ -318,11 +293,11 @@ func (b *s3Backend) PutObject(
return result, err
}
if err := stream.Close(); err != nil {
// remove file when close error occurred (FsPutErr)
_ = fs.Remove(ctx, fp)
return result, err
}
// if err := stream.Close(); err != nil {
// // remove file when close error occurred (FsPutErr)
// _ = fs.Remove(ctx, fp)
// return result, err
// }
b.meta.Store(fp, meta)

15
server/webdav/webdav.go
View File

@ -9,6 +9,7 @@ import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
@ -16,6 +17,7 @@ import (
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/net"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
@ -245,11 +247,15 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta
if err != nil {
return http.StatusInternalServerError, err
}
defer link.Close()
if storage.GetStorage().ProxyRange {
common.ProxyRange(ctx, link, fi.GetSize())
}
err = common.Proxy(w, r, link, fi)
if err != nil {
if statusCode, ok := errors.Unwrap(err).(net.ErrorHttpStatusCode); ok {
return int(statusCode), err
}
return http.StatusInternalServerError, fmt.Errorf("webdav proxy error: %+v", err)
}
} else if storage.GetStorage().WebdavProxy() && downProxyUrl != "" {
@ -264,6 +270,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta
if err != nil {
return http.StatusInternalServerError, err
}
defer link.Close()
http.Redirect(w, r, link.URL, http.StatusFound)
}
return 0, nil
@ -305,6 +312,12 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i
}
func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, err error) {
defer func() {
if n, _ := io.ReadFull(r.Body, []byte{0}); n == 1 {
_, _ = utils.CopyWithBuffer(io.Discard, r.Body)
}
_ = r.Body.Close()
}()
reqPath, status, err := h.stripPrefix(r.URL.Path)
if err != nil {
return status, err
@ -344,8 +357,6 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
return http.StatusNotFound, err
}
_ = r.Body.Close()
_ = fsStream.Close()
// TODO(rost): Returning 405 Method Not Allowed might not be appropriate.
if err != nil {
return http.StatusMethodNotAllowed, err