mirror of
https://github.com/OpenListTeam/OpenList.git
synced 2025-07-18 17:38:07 +08:00
fix(115): optimize upload (close #364)
This commit is contained in:
@ -7,6 +7,7 @@ import (
|
|||||||
|
|
||||||
"github.com/OpenListTeam/OpenList/internal/driver"
|
"github.com/OpenListTeam/OpenList/internal/driver"
|
||||||
"github.com/OpenListTeam/OpenList/internal/model"
|
"github.com/OpenListTeam/OpenList/internal/model"
|
||||||
|
streamPkg "github.com/OpenListTeam/OpenList/internal/stream"
|
||||||
"github.com/OpenListTeam/OpenList/pkg/http_range"
|
"github.com/OpenListTeam/OpenList/pkg/http_range"
|
||||||
"github.com/OpenListTeam/OpenList/pkg/utils"
|
"github.com/OpenListTeam/OpenList/pkg/utils"
|
||||||
driver115 "github.com/SheltonZhu/115driver/pkg/driver"
|
driver115 "github.com/SheltonZhu/115driver/pkg/driver"
|
||||||
@ -184,12 +185,8 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
|
|||||||
}
|
}
|
||||||
preHash = strings.ToUpper(preHash)
|
preHash = strings.ToUpper(preHash)
|
||||||
fullHash := stream.GetHash().GetHash(utils.SHA1)
|
fullHash := stream.GetHash().GetHash(utils.SHA1)
|
||||||
if len(fullHash) <= 0 {
|
if len(fullHash) != utils.SHA1.Width {
|
||||||
tmpF, err := stream.CacheFullInTempFile()
|
_, fullHash, err = streamPkg.CacheFullInTempFileAndHash(stream, utils.SHA256)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
fullHash, err = utils.HashFile(utils.SHA1, tmpF)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -3,7 +3,6 @@ package _115_open
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@ -14,6 +13,8 @@ import (
|
|||||||
"github.com/OpenListTeam/OpenList/internal/driver"
|
"github.com/OpenListTeam/OpenList/internal/driver"
|
||||||
"github.com/OpenListTeam/OpenList/internal/model"
|
"github.com/OpenListTeam/OpenList/internal/model"
|
||||||
"github.com/OpenListTeam/OpenList/internal/op"
|
"github.com/OpenListTeam/OpenList/internal/op"
|
||||||
|
"github.com/OpenListTeam/OpenList/internal/stream"
|
||||||
|
"github.com/OpenListTeam/OpenList/pkg/http_range"
|
||||||
"github.com/OpenListTeam/OpenList/pkg/utils"
|
"github.com/OpenListTeam/OpenList/pkg/utils"
|
||||||
sdk "github.com/xhofe/115-sdk-go"
|
sdk "github.com/xhofe/115-sdk-go"
|
||||||
"golang.org/x/time/rate"
|
"golang.org/x/time/rate"
|
||||||
@ -215,28 +216,27 @@ func (d *Open115) Remove(ctx context.Context, obj model.Obj) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error {
|
func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error {
|
||||||
if err := d.WaitLimit(ctx); err != nil {
|
err := d.WaitLimit(ctx)
|
||||||
return err
|
|
||||||
}
|
|
||||||
tempF, err := file.CacheFullInTempFile()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// cal full sha1
|
sha1 := file.GetHash().GetHash(utils.SHA1)
|
||||||
sha1, err := utils.HashReader(utils.SHA1, tempF)
|
if len(sha1) != utils.SHA1.Width {
|
||||||
|
_, sha1, err = stream.CacheFullInTempFileAndHash(file, utils.SHA256)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const PreHashSize int64 = 128 * utils.KB
|
||||||
|
hashSize := PreHashSize
|
||||||
|
if file.GetSize() < PreHashSize {
|
||||||
|
hashSize = file.GetSize()
|
||||||
|
}
|
||||||
|
reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
_, err = tempF.Seek(0, io.SeekStart)
|
sha1128k, err := utils.HashReader(utils.SHA1, reader)
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// pre 128k sha1
|
|
||||||
sha1128k, err := utils.HashReader(utils.SHA1, io.LimitReader(tempF, 128*1024))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
_, err = tempF.Seek(0, io.SeekStart)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -265,15 +265,11 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
_, err = tempF.Seek(start, io.SeekStart)
|
reader, err = file.RangeRead(http_range.Range{Start: start, Length: end - start + 1})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
signVal, err := utils.HashReader(utils.SHA1, io.LimitReader(tempF, end-start+1))
|
signVal, err := utils.HashReader(utils.SHA1, reader)
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
_, err = tempF.Seek(0, io.SeekStart)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -299,7 +295,7 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// 4. upload
|
// 4. upload
|
||||||
err = d.multpartUpload(ctx, tempF, file, up, tokenResp, resp)
|
err = d.multpartUpload(ctx, file, up, tokenResp, resp)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -68,7 +68,7 @@ func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp
|
|||||||
// } `json:"data"`
|
// } `json:"data"`
|
||||||
// }
|
// }
|
||||||
|
|
||||||
func (d *Open115) multpartUpload(ctx context.Context, tempF model.File, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error {
|
func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error {
|
||||||
fileSize := stream.GetSize()
|
fileSize := stream.GetSize()
|
||||||
chunkSize := calPartSize(fileSize)
|
chunkSize := calPartSize(fileSize)
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user