Backups/alist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(alias): add DownloadConcurrency and DownloadPartSize option (#7829)

Browse Source 
* fix(net): goroutine logic bug (AlistGo/alist#7215)

* Fix goroutine logic bug

* Fix bug

---------

Co-authored-by: hpy hs <hshpy.pengyu@gmail.com>

* perf(net): sequential and dynamic concurrency

* fix(net): incorrect error return

* feat(alias):  add `DownloadConcurrency` and `DownloadPartSize` option

* feat(net): add `ConcurrencyLimit`

* pref(net): create `chunk` on demand

* refactor

* refactor

* fix(net): `r.Closers.Add` has no effect

* refactor

---------

Co-authored-by: hpy hs <hshpy.pengyu@gmail.com>
This commit is contained in:
j2rong4cn
2025-01-27 20:08:39 +08:00
committed by GitHub
parent bdcf450203
commit 2be0c3d1a0
24 changed files with 396 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
drivers/alias/driver.go
View File

@ -110,6 +110,16 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
for _, dst := range dsts {
link, err := d.link(ctx, dst, sub, args)
if err == nil {
if !args.Redirect && len(link.URL) > 0 {
// 正常情况下 多并发 仅支持返回URL的驱动
// alias套娃alias 可以让crypt、mega等驱动(不返回URL的) 支持并发
if d.DownloadConcurrency > 0 {
link.Concurrency = d.DownloadConcurrency
}
if d.DownloadPartSize > 0 {
link.PartSize = d.DownloadPartSize * utils.KB
}
}
return link, nil
}
}

6
drivers/alias/meta.go
View File

@ -9,8 +9,10 @@ type Addition struct {
// Usually one of two
// driver.RootPath
// define other
Paths string `json:"paths" required:"true" type:"text"`
ProtectSameName bool `json:"protect_same_name" default:"true" required:"false" help:"Protects same-name files from Delete or Rename"`
Paths string `json:"paths" required:"true" type:"text"`
ProtectSameName bool `json:"protect_same_name" default:"true" required:"false" help:"Protects same-name files from Delete or Rename"`
DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"`
DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"`
}
var config = driver.Config{

10
drivers/alias/util.go
View File

@ -9,6 +9,7 @@ import (
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/fs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/internal/sign"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/alist-org/alist/v3/server/common"
@ -94,10 +95,15 @@ func (d *Alias) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([
func (d *Alias) link(ctx context.Context, dst, sub string, args model.LinkArgs) (*model.Link, error) {
reqPath := stdpath.Join(dst, sub)
storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{})
// 参考 crypt 驱动
storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath)
if err != nil {
return nil, err
}
if _, ok := storage.(*Alias); !ok && !args.Redirect {
link, _, err := op.Link(ctx, storage, reqActualPath, args)
return link, err
}
_, err = fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true})
if err != nil {
return nil, err
@ -114,7 +120,7 @@ func (d *Alias) link(ctx context.Context, dst, sub string, args model.LinkArgs)
}
return link, nil
}
link, _, err := fs.Link(ctx, reqPath, args)
link, _, err := op.Link(ctx, storage, reqActualPath, args)
return link, err
}

7
drivers/crypt/driver.go
View File

@ -275,7 +275,6 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
rrc = converted
}
if rrc != nil {
//remoteRangeReader, err :=
remoteReader, err := rrc.RangeRead(ctx, http_range.Range{Start: underlyingOffset, Length: length})
remoteClosers.AddClosers(rrc.GetClosers())
if err != nil {
@ -288,10 +287,8 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
if err != nil {
return nil, err
}
//remoteClosers.Add(remoteLink.MFile)
//keep reuse same MFile and close at last.
remoteClosers.Add(remoteLink.MFile)
return io.NopCloser(remoteLink.MFile), nil
// 可以直接返回，读取完也不会调用Close，直到连接断开Close
return remoteLink.MFile, nil
}
return nil, errs.NotSupport

15
drivers/github/driver.go
View File

@ -5,6 +5,13 @@ import (
"encoding/base64"
"errors"
"fmt"
"io"
"net/http"
stdpath "path"
"strings"
"sync"
"text/template"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/errs"
@ -12,12 +19,6 @@ import (
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/go-resty/resty/v2"
log "github.com/sirupsen/logrus"
"io"
"net/http"
stdpath "path"
"strings"
"sync"
"text/template"
)
type Github struct {
@ -656,7 +657,7 @@ func (d *Github) putBlob(ctx context.Context, stream model.FileStreamer, up driv
contentReader, contentWriter := io.Pipe()
go func() {
encoder := base64.NewEncoder(base64.StdEncoding, contentWriter)
if _, err := io.Copy(encoder, stream); err != nil {
if _, err := utils.CopyWithBuffer(encoder, stream); err != nil {
_ = contentWriter.CloseWithError(err)
return
}

16
drivers/halalcloud/driver.go
View File

@ -4,12 +4,17 @@ import (
"context"
"crypto/sha1"
"fmt"
"io"
"net/url"
"path"
"strconv"
"time"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/pkg/http_range"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
@ -19,11 +24,6 @@ import (
pubUserFile "github.com/city404/v6-public-rpc-proto/go/v6/userfile"
"github.com/rclone/rclone/lib/readers"
"github.com/zzzhr1990/go-common-entity/userfile"
"io"
"net/url"
"path"
"strconv"
"time"
)
type HalalCloud struct {
@ -251,7 +251,6 @@ func (d *HalalCloud) getLink(ctx context.Context, file model.Obj, args model.Lin
size := result.FileSize
chunks := getChunkSizes(result.Sizes)
var finalClosers utils.Closers
resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) {
length := httpRange.Length
if httpRange.Length >= 0 && httpRange.Start+httpRange.Length >= size {
@ -269,7 +268,6 @@ func (d *HalalCloud) getLink(ctx context.Context, file model.Obj, args model.Lin
sha: result.Sha1,
shaTemp: sha1.New(),
}
finalClosers.Add(oo)
return readers.NewLimitedReadCloser(oo, length), nil
}
@ -281,7 +279,7 @@ func (d *HalalCloud) getLink(ctx context.Context, file model.Obj, args model.Lin
duration = time.Until(time.Now().Add(time.Hour))
}
resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader, Closers: finalClosers}
resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader}
return &model.Link{
RangeReadCloser: resultRangeReadCloser,
Expiration: &duration,

4
drivers/mega/driver.go
View File

@ -84,7 +84,6 @@ func (d *Mega) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*
//}
size := file.GetSize()
var finalClosers utils.Closers
resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) {
length := httpRange.Length
if httpRange.Length >= 0 && httpRange.Start+httpRange.Length >= size {
@ -103,11 +102,10 @@ func (d *Mega) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*
d: down,
skip: httpRange.Start,
}
finalClosers.Add(oo)
return readers.NewLimitedReadCloser(oo, length), nil
}
resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader, Closers: finalClosers}
resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader}
resultLink := &model.Link{
RangeReadCloser: resultRangeReadCloser,
}

1
drivers/netease_music/types.go
View File

@ -64,7 +64,6 @@ func (lrc *LyricObj) getLyricLink() *model.Link {
sr := io.NewSectionReader(reader, httpRange.Start, httpRange.Length)
return io.NopCloser(sr), nil
},
Closers: utils.EmptyClosers(),
},
}
}

2
drivers/netease_music/upload.go
View File

@ -47,7 +47,7 @@ func (u *uploader) init(stream model.FileStreamer) error {
}
h := md5.New()
io.Copy(h, stream)
utils.CopyWithBuffer(h, stream)
u.md5 = hex.EncodeToString(h.Sum(nil))
_, err := u.file.Seek(0, io.SeekStart)
if err != nil {

4
drivers/quqi/util.go
View File

@ -300,9 +300,7 @@ func (d *Quqi) linkFromCDN(id string) (*model.Link, error) {
bufferReader := bufio.NewReader(decryptReader)
bufferReader.Discard(int(decryptedOffset))
return utils.NewReadCloser(bufferReader, func() error {
return nil
}), nil
return io.NopCloser(bufferReader), nil
}
return &model.Link{