diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index ef37811a..06b51319 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -20,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/cookie" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" ) @@ -240,68 +241,61 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U var finish int64 = 0 var chunk int = 0 DEFAULT := int64(u.ChunkSize) - retryCount := 0 - maxRetries := 3 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } left := stream.GetSize() - finish byteSize := min(left, DEFAULT) - utils.Log.Debugf("[Cloudreve-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) - byteData := make([]byte, byteSize) - n, err := io.ReadFull(stream, byteData) - utils.Log.Debug(err, n) + err := retry.Do( + func() error { + utils.Log.Debugf("[Cloudreve-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(stream, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"?chunk="+strconv.Itoa(chunk), + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + if err != nil { + return err + } + req.ContentLength = byteSize + req.Header.Set("Authorization", fmt.Sprint(credential)) + req.Header.Set("User-Agent", d.getUA()) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return fmt.Errorf("server error: %d", res.StatusCode) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + var up Resp + err = json.Unmarshal(body, &up) + if err != nil { + return err + } + if up.Code != 0 { + return errors.New(up.Msg) + } + return nil + }, + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) if err != nil { return err } - req, err := http.NewRequest("POST", uploadUrl+"?chunk="+strconv.Itoa(chunk), - driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) - if err != nil { - return err - } - req = req.WithContext(ctx) - req.ContentLength = byteSize - // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) - req.Header.Set("Authorization", fmt.Sprint(credential)) - req.Header.Set("User-Agent", d.getUA()) - err = func() error { - res, err := base.HttpClient.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode != 200 { - return errors.New(res.Status) - } - body, err := io.ReadAll(res.Body) - if err != nil { - return err - } - var up Resp - err = json.Unmarshal(body, &up) - if err != nil { - return err - } - if up.Code != 0 { - return errors.New(up.Msg) - } - return nil - }() - if err == nil { - retryCount = 0 - finish += byteSize - up(float64(finish) * 100 / float64(stream.GetSize())) - chunk++ - } else { - retryCount++ - if retryCount > maxRetries { - return fmt.Errorf("upload failed after %d retries due to server errors, error: %s", maxRetries, err) - } - backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + return fmt.Errorf("server error: %d", res.StatusCode) + case res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200: + data, _ := io.ReadAll(res.Body) + return errors.New(string(data)) + default: + return nil + } + }, retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) if err != nil { return err } - req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) - if err != nil { - return err - } - req = req.WithContext(ctx) - req.ContentLength = byteSize - // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) - req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) - req.Header.Set("User-Agent", d.getUA()) - res, err := base.HttpClient.Do(req) - if err != nil { - return err - } - // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession - switch { - case res.StatusCode >= 500 && res.StatusCode <= 504: - retryCount++ - if retryCount > maxRetries { - res.Body.Close() - return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) - } - backoff := time.Duration(1< maxRetries { - return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) - } - backoff := time.Duration(1<") - req, err := http.NewRequest( - "POST", + req, err := http.NewRequestWithContext(ctx, + http.MethodPost, u.CompleteURL, strings.NewReader(bodyBuilder.String()), ) diff --git a/drivers/cloudreve_v4/util.go b/drivers/cloudreve_v4/util.go index a3a687c4..5e7559e3 100644 --- a/drivers/cloudreve_v4/util.go +++ b/drivers/cloudreve_v4/util.go @@ -20,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" ) @@ -255,68 +256,61 @@ func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u F var finish int64 = 0 var chunk int = 0 DEFAULT := int64(u.ChunkSize) - retryCount := 0 - maxRetries := 3 for finish < file.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } left := file.GetSize() - finish byteSize := min(left, DEFAULT) - utils.Log.Debugf("[CloudreveV4-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) - byteData := make([]byte, byteSize) - n, err := io.ReadFull(file, byteData) - utils.Log.Debug(err, n) + err := retry.Do( + func() error { + utils.Log.Debugf("[CloudreveV4-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(file, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"?chunk="+strconv.Itoa(chunk), + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + if err != nil { + return err + } + + req.ContentLength = byteSize + req.Header.Set("Authorization", fmt.Sprint(credential)) + req.Header.Set("User-Agent", d.getUA()) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return fmt.Errorf("server error: %d", res.StatusCode) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + var up Resp + err = json.Unmarshal(body, &up) + if err != nil { + return err + } + if up.Code != 0 { + return errors.New(up.Msg) + } + return nil + }, retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) if err != nil { return err } - req, err := http.NewRequest("POST", uploadUrl+"?chunk="+strconv.Itoa(chunk), - driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) - if err != nil { - return err - } - req = req.WithContext(ctx) - req.ContentLength = byteSize - // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) - req.Header.Set("Authorization", fmt.Sprint(credential)) - req.Header.Set("User-Agent", d.getUA()) - err = func() error { - res, err := base.HttpClient.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode != 200 { - return errors.New(res.Status) - } - body, err := io.ReadAll(res.Body) - if err != nil { - return err - } - var up Resp - err = json.Unmarshal(body, &up) - if err != nil { - return err - } - if up.Code != 0 { - return errors.New(up.Msg) - } - return nil - }() - if err == nil { - retryCount = 0 - finish += byteSize - up(float64(finish) * 100 / float64(file.GetSize())) - chunk++ - } else { - retryCount++ - if retryCount > maxRetries { - return fmt.Errorf("upload failed after %d retries due to server errors, error: %s", maxRetries, err) - } - backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + return fmt.Errorf("server error: %d", res.StatusCode) + case res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200: + data, _ := io.ReadAll(res.Body) + return errors.New(string(data)) + default: + return nil + } + }, retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) if err != nil { return err } - req, err := http.NewRequest(http.MethodPut, uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) - if err != nil { - return err - } - req = req.WithContext(ctx) - req.ContentLength = byteSize - // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) - req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, file.GetSize())) - req.Header.Set("User-Agent", d.getUA()) - res, err := base.HttpClient.Do(req) - if err != nil { - return err - } - // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession - switch { - case res.StatusCode >= 500 && res.StatusCode <= 504: - retryCount++ - if retryCount > maxRetries { - res.Body.Close() - return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) - } - backoff := time.Duration(1< maxRetries { - return fmt.Errorf("upload failed after %d retries due to server errors", maxRetries) - } - backoff := time.Duration(1<") - req, err := http.NewRequest( - "POST", + req, err := http.NewRequestWithContext(ctx, + http.MethodPost, u.CompleteURL, strings.NewReader(bodyBuilder.String()), ) diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index 2093bffb..be86d50b 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -16,6 +16,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" ) @@ -230,7 +231,7 @@ func toAPIMetadata(stream model.FileStreamer) Metadata { func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { url := d.GetMetaUrl(false, stdpath.Join(dstDir.GetPath(), stream.GetName())) + "/createUploadSession" - metadata := map[string]interface{}{"item": toAPIMetadata(stream)} + metadata := map[string]any{"item": toAPIMetadata(stream)} res, err := d.Request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(metadata).SetContext(ctx) }, nil) @@ -240,54 +241,53 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() var finish int64 = 0 DEFAULT := d.ChunkSize * 1024 * 1024 - retryCount := 0 - maxRetries := 3 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } left := stream.GetSize() - finish byteSize := min(left, DEFAULT) - utils.Log.Debugf("[Onedrive] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) - byteData := make([]byte, byteSize) - n, err := io.ReadFull(stream, byteData) - utils.Log.Debug(err, n) + err = retry.Do( + func() error { + utils.Log.Debugf("[Onedrive] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(stream, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + if err != nil { + return err + } + req.ContentLength = byteSize + req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession + switch { + case res.StatusCode >= 500 && res.StatusCode <= 504: + return fmt.Errorf("server error: %d", res.StatusCode) + case res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200: + data, _ := io.ReadAll(res.Body) + return errors.New(string(data)) + default: + return nil + } + }, + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) if err != nil { return err } - req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) - if err != nil { - return err - } - req = req.WithContext(ctx) - req.ContentLength = byteSize - // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) - req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) - res, err := base.HttpClient.Do(req) - if err != nil { - return err - } - // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession - switch { - case res.StatusCode >= 500 && res.StatusCode <= 504: - retryCount++ - if retryCount > maxRetries { - res.Body.Close() - return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) - } - backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + return fmt.Errorf("server error: %d", res.StatusCode) + case res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200: + data, _ := io.ReadAll(res.Body) + return errors.New(string(data)) + default: + return nil + } + }, + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) if err != nil { return err } - req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) - if err != nil { - return err - } - req = req.WithContext(ctx) - req.ContentLength = byteSize - // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) - req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) - res, err := base.HttpClient.Do(req) - if err != nil { - return err - } - // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession - switch { - case res.StatusCode >= 500 && res.StatusCode <= 504: - retryCount++ - if retryCount > maxRetries { - res.Body.Close() - return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) - } - backoff := time.Duration(1<