Files
OpenList/drivers/139/util.go
MadDogOwner 87cf95f50b fix(139): refactor part upload logic (#1184)
* fix(139): refactor part upload logic

Signed-off-by: MadDogOwner <xiaoran@xrgzs.top>

* fix(139): handle upload errors

Signed-off-by: MadDogOwner <xiaoran@xrgzs.top>

* fix(139): sort upload parts by PartNumber before uploading

Signed-off-by: MadDogOwner <xiaoran@xrgzs.top>

* fix(139): improve error handling

Signed-off-by: MadDogOwner <xiaoran@xrgzs.top>

* fix(139): add validation for upload part index to prevent out of bounds errors

Signed-off-by: MadDogOwner <xiaoran@xrgzs.top>

---------

Signed-off-by: MadDogOwner <xiaoran@xrgzs.top>
2025-08-31 15:47:12 +08:00

673 lines
19 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package _139
import (
"context"
"encoding/base64"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
"sort"
"strconv"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/OpenList/v4/pkg/utils/random"
"github.com/go-resty/resty/v2"
jsoniter "github.com/json-iterator/go"
log "github.com/sirupsen/logrus"
)
// do others that not defined in Driver interface
func (d *Yun139) isFamily() bool {
return d.Type == "family"
}
func encodeURIComponent(str string) string {
r := url.QueryEscape(str)
r = strings.Replace(r, "+", "%20", -1)
r = strings.Replace(r, "%21", "!", -1)
r = strings.Replace(r, "%27", "'", -1)
r = strings.Replace(r, "%28", "(", -1)
r = strings.Replace(r, "%29", ")", -1)
r = strings.Replace(r, "%2A", "*", -1)
return r
}
func calSign(body, ts, randStr string) string {
body = encodeURIComponent(body)
strs := strings.Split(body, "")
sort.Strings(strs)
body = strings.Join(strs, "")
body = base64.StdEncoding.EncodeToString([]byte(body))
res := utils.GetMD5EncodeStr(body) + utils.GetMD5EncodeStr(ts+":"+randStr)
res = strings.ToUpper(utils.GetMD5EncodeStr(res))
return res
}
func getTime(t string) time.Time {
stamp, _ := time.ParseInLocation("20060102150405", t, utils.CNLoc)
return stamp
}
func (d *Yun139) refreshToken() error {
if d.ref != nil {
return d.ref.refreshToken()
}
decode, err := base64.StdEncoding.DecodeString(d.Authorization)
if err != nil {
return fmt.Errorf("authorization decode failed: %s", err)
}
decodeStr := string(decode)
splits := strings.Split(decodeStr, ":")
if len(splits) < 3 {
return fmt.Errorf("authorization is invalid, splits < 3")
}
d.Account = splits[1]
strs := strings.Split(splits[2], "|")
if len(strs) < 4 {
return fmt.Errorf("authorization is invalid, strs < 4")
}
expiration, err := strconv.ParseInt(strs[3], 10, 64)
if err != nil {
return fmt.Errorf("authorization is invalid")
}
expiration -= time.Now().UnixMilli()
if expiration > 1000*60*60*24*15 {
// Authorization有效期大于15天无需刷新
return nil
}
if expiration < 0 {
return fmt.Errorf("authorization has expired")
}
url := "https://aas.caiyun.feixin.10086.cn:443/tellin/authTokenRefresh.do"
var resp RefreshTokenResp
reqBody := "<root><token>" + splits[2] + "</token><account>" + splits[1] + "</account><clienttype>656</clienttype></root>"
_, err = base.RestyClient.R().
ForceContentType("application/xml").
SetBody(reqBody).
SetResult(&resp).
Post(url)
if err != nil {
return err
}
if resp.Return != "0" {
return fmt.Errorf("failed to refresh token: %s", resp.Desc)
}
d.Authorization = base64.StdEncoding.EncodeToString([]byte(splits[0] + ":" + splits[1] + ":" + resp.Token))
op.MustSaveDriverStorage(d)
return nil
}
func (d *Yun139) request(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) {
url := "https://yun.139.com" + pathname
req := base.RestyClient.R()
randStr := random.String(16)
ts := time.Now().Format("2006-01-02 15:04:05")
if callback != nil {
callback(req)
}
body, err := utils.Json.Marshal(req.Body)
if err != nil {
return nil, err
}
sign := calSign(string(body), ts, randStr)
svcType := "1"
if d.isFamily() {
svcType = "2"
}
req.SetHeaders(map[string]string{
"Accept": "application/json, text/plain, */*",
"CMS-DEVICE": "default",
"Authorization": "Basic " + d.getAuthorization(),
"mcloud-channel": "1000101",
"mcloud-client": "10701",
//"mcloud-route": "001",
"mcloud-sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign),
//"mcloud-skey":"",
"mcloud-version": "7.14.0",
"Origin": "https://yun.139.com",
"Referer": "https://yun.139.com/w/",
"x-DeviceInfo": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||",
"x-huawei-channelSrc": "10000034",
"x-inner-ntwk": "2",
"x-m4c-caller": "PC",
"x-m4c-src": "10002",
"x-SvcType": svcType,
"Inner-Hcy-Router-Https": "1",
})
var e BaseResp
req.SetResult(&e)
res, err := req.Execute(method, url)
log.Debugln(res.String())
if !e.Success {
return nil, errors.New(e.Message)
}
if resp != nil {
err = utils.Json.Unmarshal(res.Body(), resp)
if err != nil {
return nil, err
}
}
return res.Body(), nil
}
func (d *Yun139) requestRoute(data interface{}, resp interface{}) ([]byte, error) {
url := "https://user-njs.yun.139.com/user/route/qryRoutePolicy"
req := base.RestyClient.R()
randStr := random.String(16)
ts := time.Now().Format("2006-01-02 15:04:05")
callback := func(req *resty.Request) {
req.SetBody(data)
}
if callback != nil {
callback(req)
}
body, err := utils.Json.Marshal(req.Body)
if err != nil {
return nil, err
}
sign := calSign(string(body), ts, randStr)
svcType := "1"
if d.isFamily() {
svcType = "2"
}
req.SetHeaders(map[string]string{
"Accept": "application/json, text/plain, */*",
"CMS-DEVICE": "default",
"Authorization": "Basic " + d.getAuthorization(),
"mcloud-channel": "1000101",
"mcloud-client": "10701",
//"mcloud-route": "001",
"mcloud-sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign),
//"mcloud-skey":"",
"mcloud-version": "7.14.0",
"Origin": "https://yun.139.com",
"Referer": "https://yun.139.com/w/",
"x-DeviceInfo": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||",
"x-huawei-channelSrc": "10000034",
"x-inner-ntwk": "2",
"x-m4c-caller": "PC",
"x-m4c-src": "10002",
"x-SvcType": svcType,
"Inner-Hcy-Router-Https": "1",
})
var e BaseResp
req.SetResult(&e)
res, err := req.Execute(http.MethodPost, url)
log.Debugln(res.String())
if !e.Success {
return nil, errors.New(e.Message)
}
if resp != nil {
err = utils.Json.Unmarshal(res.Body(), resp)
if err != nil {
return nil, err
}
}
return res.Body(), nil
}
func (d *Yun139) post(pathname string, data interface{}, resp interface{}) ([]byte, error) {
return d.request(pathname, http.MethodPost, func(req *resty.Request) {
req.SetBody(data)
}, resp)
}
func (d *Yun139) getFiles(catalogID string) ([]model.Obj, error) {
start := 0
limit := 100
files := make([]model.Obj, 0)
for {
data := base.Json{
"catalogID": catalogID,
"sortDirection": 1,
"startNumber": start + 1,
"endNumber": start + limit,
"filterType": 0,
"catalogSortType": 0,
"contentSortType": 0,
"commonAccountInfo": base.Json{
"account": d.getAccount(),
"accountType": 1,
},
}
var resp GetDiskResp
_, err := d.post("/orchestration/personalCloud/catalog/v1.0/getDisk", data, &resp)
if err != nil {
return nil, err
}
for _, catalog := range resp.Data.GetDiskResult.CatalogList {
f := model.Object{
ID: catalog.CatalogID,
Name: catalog.CatalogName,
Size: 0,
Modified: getTime(catalog.UpdateTime),
Ctime: getTime(catalog.CreateTime),
IsFolder: true,
}
files = append(files, &f)
}
for _, content := range resp.Data.GetDiskResult.ContentList {
f := model.ObjThumb{
Object: model.Object{
ID: content.ContentID,
Name: content.ContentName,
Size: content.ContentSize,
Modified: getTime(content.UpdateTime),
HashInfo: utils.NewHashInfo(utils.MD5, content.Digest),
},
Thumbnail: model.Thumbnail{Thumbnail: content.ThumbnailURL},
//Thumbnail: content.BigthumbnailURL,
}
files = append(files, &f)
}
if start+limit >= resp.Data.GetDiskResult.NodeCount {
break
}
start += limit
}
return files, nil
}
func (d *Yun139) newJson(data map[string]interface{}) base.Json {
common := map[string]interface{}{
"catalogType": 3,
"cloudID": d.CloudID,
"cloudType": 1,
"commonAccountInfo": base.Json{
"account": d.getAccount(),
"accountType": 1,
},
}
return utils.MergeMap(data, common)
}
func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) {
pageNum := 1
files := make([]model.Obj, 0)
for {
data := d.newJson(base.Json{
"catalogID": catalogID,
"contentSortType": 0,
"pageInfo": base.Json{
"pageNum": pageNum,
"pageSize": 100,
},
"sortDirection": 1,
})
var resp QueryContentListResp
_, err := d.post("/orchestration/familyCloud-rebuild/content/v1.2/queryContentList", data, &resp)
if err != nil {
return nil, err
}
path := resp.Data.Path
for _, catalog := range resp.Data.CloudCatalogList {
f := model.Object{
ID: catalog.CatalogID,
Name: catalog.CatalogName,
Size: 0,
IsFolder: true,
Modified: getTime(catalog.LastUpdateTime),
Ctime: getTime(catalog.CreateTime),
Path: path, // 文件夹上一级的Path
}
files = append(files, &f)
}
for _, content := range resp.Data.CloudContentList {
f := model.ObjThumb{
Object: model.Object{
ID: content.ContentID,
Name: content.ContentName,
Size: content.ContentSize,
Modified: getTime(content.LastUpdateTime),
Ctime: getTime(content.CreateTime),
Path: path, // 文件所在目录的Path
},
Thumbnail: model.Thumbnail{Thumbnail: content.ThumbnailURL},
//Thumbnail: content.BigthumbnailURL,
}
files = append(files, &f)
}
if resp.Data.TotalCount == 0 {
break
}
pageNum++
}
return files, nil
}
func (d *Yun139) groupGetFiles(catalogID string) ([]model.Obj, error) {
pageNum := 1
files := make([]model.Obj, 0)
for {
data := d.newJson(base.Json{
"groupID": d.CloudID,
"catalogID": path.Base(catalogID),
"contentSortType": 0,
"sortDirection": 1,
"startNumber": pageNum,
"endNumber": pageNum + 99,
"path": path.Join(d.RootFolderID, catalogID),
})
var resp QueryGroupContentListResp
_, err := d.post("/orchestration/group-rebuild/content/v1.0/queryGroupContentList", data, &resp)
if err != nil {
return nil, err
}
path := resp.Data.GetGroupContentResult.ParentCatalogID
for _, catalog := range resp.Data.GetGroupContentResult.CatalogList {
f := model.Object{
ID: catalog.CatalogID,
Name: catalog.CatalogName,
Size: 0,
IsFolder: true,
Modified: getTime(catalog.UpdateTime),
Ctime: getTime(catalog.CreateTime),
Path: catalog.Path, // 文件夹的真实Path root:/开头
}
files = append(files, &f)
}
for _, content := range resp.Data.GetGroupContentResult.ContentList {
f := model.ObjThumb{
Object: model.Object{
ID: content.ContentID,
Name: content.ContentName,
Size: content.ContentSize,
Modified: getTime(content.UpdateTime),
Ctime: getTime(content.CreateTime),
Path: path, // 文件所在目录的Path
},
Thumbnail: model.Thumbnail{Thumbnail: content.ThumbnailURL},
//Thumbnail: content.BigthumbnailURL,
}
files = append(files, &f)
}
if (pageNum + 99) > resp.Data.GetGroupContentResult.NodeCount {
break
}
pageNum = pageNum + 100
}
return files, nil
}
func (d *Yun139) getLink(contentId string) (string, error) {
data := base.Json{
"appName": "",
"contentID": contentId,
"commonAccountInfo": base.Json{
"account": d.getAccount(),
"accountType": 1,
},
}
res, err := d.post("/orchestration/personalCloud/uploadAndDownload/v1.0/downloadRequest",
data, nil)
if err != nil {
return "", err
}
return jsoniter.Get(res, "data", "downloadURL").ToString(), nil
}
func (d *Yun139) familyGetLink(contentId string, path string) (string, error) {
data := d.newJson(base.Json{
"contentID": contentId,
"path": path,
})
res, err := d.post("/orchestration/familyCloud-rebuild/content/v1.0/getFileDownLoadURL",
data, nil)
if err != nil {
return "", err
}
return jsoniter.Get(res, "data", "downloadURL").ToString(), nil
}
func (d *Yun139) groupGetLink(contentId string, path string) (string, error) {
data := d.newJson(base.Json{
"contentID": contentId,
"groupID": d.CloudID,
"path": path,
})
res, err := d.post("/orchestration/group-rebuild/groupManage/v1.0/getGroupFileDownLoadURL",
data, nil)
if err != nil {
return "", err
}
return jsoniter.Get(res, "data", "downloadURL").ToString(), nil
}
func unicode(str string) string {
textQuoted := strconv.QuoteToASCII(str)
textUnquoted := textQuoted[1 : len(textQuoted)-1]
return textUnquoted
}
func (d *Yun139) personalRequest(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) {
url := d.getPersonalCloudHost() + pathname
req := base.RestyClient.R()
randStr := random.String(16)
ts := time.Now().Format("2006-01-02 15:04:05")
if callback != nil {
callback(req)
}
body, err := utils.Json.Marshal(req.Body)
if err != nil {
return nil, err
}
sign := calSign(string(body), ts, randStr)
svcType := "1"
if d.isFamily() {
svcType = "2"
}
req.SetHeaders(map[string]string{
"Accept": "application/json, text/plain, */*",
"Authorization": "Basic " + d.getAuthorization(),
"Caller": "web",
"Cms-Device": "default",
"Mcloud-Channel": "1000101",
"Mcloud-Client": "10701",
"Mcloud-Route": "001",
"Mcloud-Sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign),
"Mcloud-Version": "7.14.0",
"x-DeviceInfo": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||",
"x-huawei-channelSrc": "10000034",
"x-inner-ntwk": "2",
"x-m4c-caller": "PC",
"x-m4c-src": "10002",
"x-SvcType": svcType,
"X-Yun-Api-Version": "v1",
"X-Yun-App-Channel": "10000034",
"X-Yun-Channel-Source": "10000034",
"X-Yun-Client-Info": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||",
"X-Yun-Module-Type": "100",
"X-Yun-Svc-Type": "1",
})
var e BaseResp
req.SetResult(&e)
res, err := req.Execute(method, url)
if err != nil {
return nil, err
}
log.Debugln(res.String())
if !e.Success {
return nil, errors.New(e.Message)
}
if resp != nil {
err = utils.Json.Unmarshal(res.Body(), resp)
if err != nil {
return nil, err
}
}
return res.Body(), nil
}
func (d *Yun139) personalPost(pathname string, data interface{}, resp interface{}) ([]byte, error) {
return d.personalRequest(pathname, http.MethodPost, func(req *resty.Request) {
req.SetBody(data)
}, resp)
}
func getPersonalTime(t string) time.Time {
stamp, err := time.ParseInLocation("2006-01-02T15:04:05.999-07:00", t, utils.CNLoc)
if err != nil {
panic(err)
}
return stamp
}
func (d *Yun139) personalGetFiles(fileId string) ([]model.Obj, error) {
files := make([]model.Obj, 0)
nextPageCursor := ""
for {
data := base.Json{
"imageThumbnailStyleList": []string{"Small", "Large"},
"orderBy": "updated_at",
"orderDirection": "DESC",
"pageInfo": base.Json{
"pageCursor": nextPageCursor,
"pageSize": 100,
},
"parentFileId": fileId,
}
var resp PersonalListResp
_, err := d.personalPost("/file/list", data, &resp)
if err != nil {
return nil, err
}
nextPageCursor = resp.Data.NextPageCursor
for _, item := range resp.Data.Items {
var isFolder = (item.Type == "folder")
var f model.Obj
if isFolder {
f = &model.Object{
ID: item.FileId,
Name: item.Name,
Size: 0,
Modified: getPersonalTime(item.UpdatedAt),
Ctime: getPersonalTime(item.CreatedAt),
IsFolder: isFolder,
}
} else {
var Thumbnails = item.Thumbnails
var ThumbnailUrl string
if d.UseLargeThumbnail {
for _, thumb := range Thumbnails {
if strings.Contains(thumb.Style, "Large") {
ThumbnailUrl = thumb.Url
break
}
}
}
if ThumbnailUrl == "" && len(Thumbnails) > 0 {
ThumbnailUrl = Thumbnails[len(Thumbnails)-1].Url
}
f = &model.ObjThumb{
Object: model.Object{
ID: item.FileId,
Name: item.Name,
Size: item.Size,
Modified: getPersonalTime(item.UpdatedAt),
Ctime: getPersonalTime(item.CreatedAt),
IsFolder: isFolder,
},
Thumbnail: model.Thumbnail{Thumbnail: ThumbnailUrl},
}
}
files = append(files, f)
}
if len(nextPageCursor) == 0 {
break
}
}
return files, nil
}
func (d *Yun139) personalGetLink(fileId string) (string, error) {
data := base.Json{
"fileId": fileId,
}
res, err := d.personalPost("/file/getDownloadUrl",
data, nil)
if err != nil {
return "", err
}
var cdnUrl = jsoniter.Get(res, "data", "cdnUrl").ToString()
if cdnUrl != "" {
return cdnUrl, nil
} else {
return jsoniter.Get(res, "data", "url").ToString(), nil
}
}
func (d *Yun139) getAuthorization() string {
if d.ref != nil {
return d.ref.getAuthorization()
}
return d.Authorization
}
func (d *Yun139) getAccount() string {
if d.ref != nil {
return d.ref.getAccount()
}
return d.Account
}
func (d *Yun139) getPersonalCloudHost() string {
if d.ref != nil {
return d.ref.getPersonalCloudHost()
}
return d.PersonalCloudHost
}
func (d *Yun139) uploadPersonalParts(ctx context.Context, partInfos []PartInfo, uploadPartInfos []PersonalPartInfo, rateLimited *driver.RateLimitReader, p *driver.Progress) error {
// 确保数组以 PartNumber 从小到大排序
sort.Slice(uploadPartInfos, func(i, j int) bool {
return uploadPartInfos[i].PartNumber < uploadPartInfos[j].PartNumber
})
for _, uploadPartInfo := range uploadPartInfos {
index := uploadPartInfo.PartNumber - 1
if index < 0 || index >= len(partInfos) {
return fmt.Errorf("invalid PartNumber %d: index out of bounds (partInfos length: %d)", uploadPartInfo.PartNumber, len(partInfos))
}
partSize := partInfos[index].PartSize
log.Debugf("[139] uploading part %+v/%+v", index, len(partInfos))
limitReader := io.LimitReader(rateLimited, partSize)
r := io.TeeReader(limitReader, p)
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadPartInfo.UploadUrl, r)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Content-Length", fmt.Sprint(partSize))
req.Header.Set("Origin", "https://yun.139.com")
req.Header.Set("Referer", "https://yun.139.com/")
req.ContentLength = partSize
err = func() error {
res, err := base.HttpClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
log.Debugf("[139] uploaded: %+v", res)
if res.StatusCode != http.StatusOK {
body, _ := io.ReadAll(res.Body)
return fmt.Errorf("unexpected status code: %d, body: %s", res.StatusCode, string(body))
}
return nil
}()
if err != nil {
return err
}
}
return nil
}