Compare commits

..

2 Commits

Author SHA1 Message Date
86393d25e9 feat(ilanzou): enable local sorting 2025-07-25 17:32:16 +08:00
4df53c965c feat(cloudreve): enable local sorting 2025-07-25 17:32:00 +08:00
719 changed files with 4864 additions and 8616 deletions

View File

@ -61,7 +61,7 @@ jobs:
strategy: strategy:
matrix: matrix:
include: include:
- target: "!(*musl*|*windows-arm64*|*windows7-*|*android*|*freebsd*)" # xgo and loongarch - target: "!(*musl*|*windows-arm64*|*android*|*freebsd*)" # xgo and loongarch
hash: "md5" hash: "md5"
- target: "linux-!(arm*)-musl*" #musl-not-arm - target: "linux-!(arm*)-musl*" #musl-not-arm
hash: "md5-linux-musl" hash: "md5-linux-musl"
@ -69,8 +69,6 @@ jobs:
hash: "md5-linux-musl-arm" hash: "md5-linux-musl-arm"
- target: "windows-arm64" #win-arm64 - target: "windows-arm64" #win-arm64
hash: "md5-windows-arm64" hash: "md5-windows-arm64"
- target: "windows7-*" #win7
hash: "md5-windows7"
- target: "android-*" #android - target: "android-*" #android
hash: "md5-android" hash: "md5-android"
- target: "freebsd-*" #freebsd - target: "freebsd-*" #freebsd
@ -93,7 +91,6 @@ jobs:
run: bash build.sh dev web run: bash build.sh dev web
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
FRONTEND_REPO: ${{ vars.FRONTEND_REPO }}
- name: Build - name: Build
uses: OpenListTeam/cgo-actions@v1.2.2 uses: OpenListTeam/cgo-actions@v1.2.2
@ -106,10 +103,10 @@ jobs:
musl-base-url: "https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" musl-base-url: "https://github.com/OpenListTeam/musl-compilers/releases/latest/download/"
x-flags: | x-flags: |
github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$built_at github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$built_at
github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors <noreply@openlist.team> github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=OpenList
github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit
github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag
github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=dev
- name: Compress - name: Compress
run: | run: |

View File

@ -39,7 +39,6 @@ jobs:
run: bash build.sh dev web run: bash build.sh dev web
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
FRONTEND_REPO: ${{ vars.FRONTEND_REPO }}
- name: Build - name: Build
uses: OpenListTeam/cgo-actions@v1.2.2 uses: OpenListTeam/cgo-actions@v1.2.2
@ -50,10 +49,10 @@ jobs:
out-dir: build out-dir: build
x-flags: | x-flags: |
github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$built_at github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$built_at
github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors <noreply@openlist.team> github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=OpenList
github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit
github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag
github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=dev
output: openlist$ext output: openlist$ext
- name: Upload artifact - name: Upload artifact

View File

@ -66,7 +66,6 @@ jobs:
bash build.sh release ${{ matrix.build-type == 'lite' && 'lite' || '' }} ${{ matrix.target-platform }} bash build.sh release ${{ matrix.build-type == 'lite' && 'lite' || '' }} ${{ matrix.target-platform }}
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
FRONTEND_REPO: ${{ vars.FRONTEND_REPO }}
- name: Upload assets - name: Upload assets
uses: softprops/action-gh-release@v2 uses: softprops/action-gh-release@v2

View File

@ -31,7 +31,7 @@ env:
REGISTRY: ghcr.io REGISTRY: ghcr.io
ARTIFACT_NAME: 'binaries_docker_release' ARTIFACT_NAME: 'binaries_docker_release'
ARTIFACT_NAME_LITE: 'binaries_docker_release_lite' ARTIFACT_NAME_LITE: 'binaries_docker_release_lite'
RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons. RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64'
IMAGE_PUSH: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} IMAGE_PUSH: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
permissions: permissions:
@ -66,7 +66,6 @@ jobs:
run: bash build.sh release docker-multiplatform run: bash build.sh release docker-multiplatform
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
FRONTEND_REPO: ${{ vars.FRONTEND_REPO }}
- name: Upload artifacts - name: Upload artifacts
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
@ -106,7 +105,6 @@ jobs:
run: bash build.sh release lite docker-multiplatform run: bash build.sh release lite docker-multiplatform
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
FRONTEND_REPO: ${{ vars.FRONTEND_REPO }}
- name: Upload artifacts - name: Upload artifacts
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
@ -127,19 +125,15 @@ jobs:
image: ["latest", "ffmpeg", "aria2", "aio"] image: ["latest", "ffmpeg", "aria2", "aio"]
include: include:
- image: "latest" - image: "latest"
base_image_tag: "base"
build_arg: "" build_arg: ""
tag_favor: "" tag_favor: ""
- image: "ffmpeg" - image: "ffmpeg"
base_image_tag: "ffmpeg"
build_arg: INSTALL_FFMPEG=true build_arg: INSTALL_FFMPEG=true
tag_favor: "suffix=-ffmpeg,onlatest=true" tag_favor: "suffix=-ffmpeg,onlatest=true"
- image: "aria2" - image: "aria2"
base_image_tag: "aria2"
build_arg: INSTALL_ARIA2=true build_arg: INSTALL_ARIA2=true
tag_favor: "suffix=-aria2,onlatest=true" tag_favor: "suffix=-aria2,onlatest=true"
- image: "aio" - image: "aio"
base_image_tag: "aio"
build_arg: | build_arg: |
INSTALL_FFMPEG=true INSTALL_FFMPEG=true
INSTALL_ARIA2=true INSTALL_ARIA2=true
@ -195,9 +189,7 @@ jobs:
context: . context: .
file: Dockerfile.ci file: Dockerfile.ci
push: ${{ env.IMAGE_PUSH == 'true' }} push: ${{ env.IMAGE_PUSH == 'true' }}
build-args: | build-args: ${{ matrix.build_arg }}
BASE_IMAGE_TAG=${{ matrix.base_image_tag }}
${{ matrix.build_arg }}
tags: ${{ steps.meta.outputs.tags }} tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }} labels: ${{ steps.meta.outputs.labels }}
platforms: ${{ env.RELEASE_PLATFORMS }} platforms: ${{ env.RELEASE_PLATFORMS }}
@ -211,19 +203,15 @@ jobs:
image: ["latest", "ffmpeg", "aria2", "aio"] image: ["latest", "ffmpeg", "aria2", "aio"]
include: include:
- image: "latest" - image: "latest"
base_image_tag: "base"
build_arg: "" build_arg: ""
tag_favor: "suffix=-lite,onlatest=true" tag_favor: "suffix=-lite,onlatest=true"
- image: "ffmpeg" - image: "ffmpeg"
base_image_tag: "ffmpeg"
build_arg: INSTALL_FFMPEG=true build_arg: INSTALL_FFMPEG=true
tag_favor: "suffix=-lite-ffmpeg,onlatest=true" tag_favor: "suffix=-lite-ffmpeg,onlatest=true"
- image: "aria2" - image: "aria2"
base_image_tag: "aria2"
build_arg: INSTALL_ARIA2=true build_arg: INSTALL_ARIA2=true
tag_favor: "suffix=-lite-aria2,onlatest=true" tag_favor: "suffix=-lite-aria2,onlatest=true"
- image: "aio" - image: "aio"
base_image_tag: "aio"
build_arg: | build_arg: |
INSTALL_FFMPEG=true INSTALL_FFMPEG=true
INSTALL_ARIA2=true INSTALL_ARIA2=true
@ -279,9 +267,7 @@ jobs:
context: . context: .
file: Dockerfile.ci file: Dockerfile.ci
push: ${{ env.IMAGE_PUSH == 'true' }} push: ${{ env.IMAGE_PUSH == 'true' }}
build-args: | build-args: ${{ matrix.build_arg }}
BASE_IMAGE_TAG=${{ matrix.base_image_tag }}
${{ matrix.build_arg }}
tags: ${{ steps.meta.outputs.tags }} tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }} labels: ${{ steps.meta.outputs.labels }}
platforms: ${{ env.RELEASE_PLATFORMS }} platforms: ${{ env.RELEASE_PLATFORMS }}

View File

@ -20,7 +20,7 @@ env:
IMAGE_NAME_DOCKERHUB: openlist IMAGE_NAME_DOCKERHUB: openlist
REGISTRY: ghcr.io REGISTRY: ghcr.io
ARTIFACT_NAME: 'binaries_docker_release' ARTIFACT_NAME: 'binaries_docker_release'
RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons. RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64'
IMAGE_PUSH: ${{ github.event_name == 'push' }} IMAGE_PUSH: ${{ github.event_name == 'push' }}
IMAGE_TAGS_BETA: | IMAGE_TAGS_BETA: |
type=ref,event=pr type=ref,event=pr
@ -55,7 +55,6 @@ jobs:
run: bash build.sh beta docker-multiplatform run: bash build.sh beta docker-multiplatform
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
FRONTEND_REPO: ${{ vars.FRONTEND_REPO }}
- name: Upload artifacts - name: Upload artifacts
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
@ -78,19 +77,15 @@ jobs:
image: ["latest", "ffmpeg", "aria2", "aio"] image: ["latest", "ffmpeg", "aria2", "aio"]
include: include:
- image: "latest" - image: "latest"
base_image_tag: "base"
build_arg: "" build_arg: ""
tag_favor: "" tag_favor: ""
- image: "ffmpeg" - image: "ffmpeg"
base_image_tag: "ffmpeg"
build_arg: INSTALL_FFMPEG=true build_arg: INSTALL_FFMPEG=true
tag_favor: "suffix=-ffmpeg,onlatest=true" tag_favor: "suffix=-ffmpeg,onlatest=true"
- image: "aria2" - image: "aria2"
base_image_tag: "aria2"
build_arg: INSTALL_ARIA2=true build_arg: INSTALL_ARIA2=true
tag_favor: "suffix=-aria2,onlatest=true" tag_favor: "suffix=-aria2,onlatest=true"
- image: "aio" - image: "aio"
base_image_tag: "aio"
build_arg: | build_arg: |
INSTALL_FFMPEG=true INSTALL_FFMPEG=true
INSTALL_ARIA2=true INSTALL_ARIA2=true
@ -142,9 +137,7 @@ jobs:
context: . context: .
file: Dockerfile.ci file: Dockerfile.ci
push: ${{ env.IMAGE_PUSH == 'true' }} push: ${{ env.IMAGE_PUSH == 'true' }}
build-args: | build-args: ${{ matrix.build_arg }}
BASE_IMAGE_TAG=${{ matrix.base_image_tag }}
${{ matrix.build_arg }}
tags: ${{ steps.meta.outputs.tags }} tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }} labels: ${{ steps.meta.outputs.labels }}
platforms: ${{ env.RELEASE_PLATFORMS }} platforms: ${{ env.RELEASE_PLATFORMS }}

57
Dockerfile Normal file
View File

@ -0,0 +1,57 @@
FROM docker.io/library/alpine:edge AS builder
LABEL stage=go-builder
WORKDIR /app/
RUN apk add --no-cache bash curl jq gcc git go musl-dev
COPY go.mod go.sum ./
RUN go mod download
COPY ./ ./
RUN bash build.sh release docker
FROM alpine:edge
ARG INSTALL_FFMPEG=false
ARG INSTALL_ARIA2=false
LABEL MAINTAINER="OpenList"
WORKDIR /opt/openlist/
RUN apk update && \
apk upgrade --no-cache && \
apk add --no-cache bash ca-certificates su-exec tzdata runit; \
[ "$INSTALL_FFMPEG" = "true" ] && apk add --no-cache ffmpeg; \
[ "$INSTALL_ARIA2" = "true" ] && apk add --no-cache curl aria2 && \
mkdir -p /opt/aria2/.aria2 && \
wget https://github.com/P3TERX/aria2.conf/archive/refs/heads/master.tar.gz -O /tmp/aria-conf.tar.gz && \
tar -zxvf /tmp/aria-conf.tar.gz -C /opt/aria2/.aria2 --strip-components=1 && rm -f /tmp/aria-conf.tar.gz && \
sed -i 's|rpc-secret|#rpc-secret|g' /opt/aria2/.aria2/aria2.conf && \
sed -i 's|/root/.aria2|/opt/aria2/.aria2|g' /opt/aria2/.aria2/aria2.conf && \
sed -i 's|/root/.aria2|/opt/aria2/.aria2|g' /opt/aria2/.aria2/script.conf && \
sed -i 's|/root|/opt/aria2|g' /opt/aria2/.aria2/aria2.conf && \
sed -i 's|/root|/opt/aria2|g' /opt/aria2/.aria2/script.conf && \
mkdir -p /opt/service/stop/aria2/log && \
echo '#!/bin/sh' > /opt/service/stop/aria2/run && \
echo 'exec 2>&1' >> /opt/service/stop/aria2/run && \
echo 'exec aria2c --enable-rpc --rpc-allow-origin-all --conf-path=/opt/aria2/.aria2/aria2.conf' >> /opt/service/stop/aria2/run && \
echo '#!/bin/sh' > /opt/service/stop/aria2/log/run && \
echo 'mkdir -p /opt/openlist/data/log/aria2 2>/dev/null' >> /opt/service/stop/aria2/log/run && \
echo 'exec svlogd /opt/openlist/data/log/aria2' >> /opt/service/stop/aria2/log/run && \
chmod +x /opt/service/stop/aria2/run /opt/service/stop/aria2/log/run && \
touch /opt/aria2/.aria2/aria2.session && \
/opt/aria2/.aria2/tracker.sh ; \
rm -rf /var/cache/apk/*
RUN mkdir -p /opt/service/start/openlist && \
echo '#!/bin/sh' > /opt/service/start/openlist/run && \
echo 'exec 2>&1' >> /opt/service/start/openlist/run && \
echo 'cd /opt/openlist' >> /opt/service/start/openlist/run && \
echo 'exec ./openlist server --no-prefix' >> /opt/service/start/openlist/run && \
chmod +x /opt/service/start/openlist/run
COPY --chmod=755 --from=builder /app/bin/openlist ./
COPY --chmod=755 entrypoint.sh /entrypoint.sh
RUN /entrypoint.sh version
ENV PUID=0 PGID=0 UMASK=022 RUN_ARIA2=${INSTALL_ARIA2}
VOLUME /opt/openlist/data/
EXPOSE 5244 5245
CMD [ "/entrypoint.sh" ]

49
Dockerfile.ci Normal file
View File

@ -0,0 +1,49 @@
FROM docker.io/library/alpine:edge
ARG TARGETPLATFORM
ARG INSTALL_FFMPEG=false
ARG INSTALL_ARIA2=false
LABEL MAINTAINER="OpenList"
WORKDIR /opt/openlist/
RUN apk update && \
apk upgrade --no-cache && \
apk add --no-cache bash ca-certificates su-exec tzdata runit; \
[ "$INSTALL_FFMPEG" = "true" ] && apk add --no-cache ffmpeg; \
[ "$INSTALL_ARIA2" = "true" ] && apk add --no-cache curl aria2 && \
mkdir -p /opt/aria2/.aria2 && \
wget https://github.com/P3TERX/aria2.conf/archive/refs/heads/master.tar.gz -O /tmp/aria-conf.tar.gz && \
tar -zxvf /tmp/aria-conf.tar.gz -C /opt/aria2/.aria2 --strip-components=1 && rm -f /tmp/aria-conf.tar.gz && \
sed -i 's|rpc-secret|#rpc-secret|g' /opt/aria2/.aria2/aria2.conf && \
sed -i 's|/root/.aria2|/opt/aria2/.aria2|g' /opt/aria2/.aria2/aria2.conf && \
sed -i 's|/root/.aria2|/opt/aria2/.aria2|g' /opt/aria2/.aria2/script.conf && \
sed -i 's|/root|/opt/aria2|g' /opt/aria2/.aria2/aria2.conf && \
sed -i 's|/root|/opt/aria2|g' /opt/aria2/.aria2/script.conf && \
mkdir -p /opt/service/stop/aria2/log && \
echo '#!/bin/sh' > /opt/service/stop/aria2/run && \
echo 'exec 2>&1' >> /opt/service/stop/aria2/run && \
echo 'exec aria2c --enable-rpc --rpc-allow-origin-all --conf-path=/opt/aria2/.aria2/aria2.conf' >> /opt/service/stop/aria2/run && \
echo '#!/bin/sh' > /opt/service/stop/aria2/log/run && \
echo 'mkdir -p /opt/openlist/data/log/aria2 2>/dev/null' >> /opt/service/stop/aria2/log/run && \
echo 'exec svlogd /opt/openlist/data/log/aria2' >> /opt/service/stop/aria2/log/run && \
chmod +x /opt/service/stop/aria2/run /opt/service/stop/aria2/log/run && \
touch /opt/aria2/.aria2/aria2.session && \
/opt/aria2/.aria2/tracker.sh ; \
rm -rf /var/cache/apk/*
RUN mkdir -p /opt/service/start/openlist && \
echo '#!/bin/sh' > /opt/service/start/openlist/run && \
echo 'exec 2>&1' >> /opt/service/start/openlist/run && \
echo 'cd /opt/openlist' >> /opt/service/start/openlist/run && \
echo 'exec ./openlist server --no-prefix' >> /opt/service/start/openlist/run && \
chmod +x /opt/service/start/openlist/run
COPY --chmod=755 /build/${TARGETPLATFORM}/openlist ./
COPY --chmod=755 entrypoint.sh /entrypoint.sh
RUN /entrypoint.sh version
ENV PUID=0 PGID=0 UMASK=022 RUN_ARIA2=${INSTALL_ARIA2}
VOLUME /opt/openlist/data/
EXPOSE 5244 5245
CMD [ "/entrypoint.sh" ]

View File

@ -1,11 +0,0 @@
version: v1
plugins:
- plugin: buf.build/protocolbuffers/go:v1.36.7
out: .
opt:
- paths=source_relative
- plugin: buf.build/grpc/go:v1.5.1
out: .
opt:
- paths=source_relative
- require_unimplemented_servers=false

View File

@ -1 +0,0 @@
version: v1

View File

@ -4,9 +4,6 @@ builtAt="$(date +'%F %T %z')"
gitAuthor="The OpenList Projects Contributors <noreply@openlist.team>" gitAuthor="The OpenList Projects Contributors <noreply@openlist.team>"
gitCommit=$(git log --pretty=format:"%h" -1) gitCommit=$(git log --pretty=format:"%h" -1)
# Set frontend repository, default to OpenListTeam/OpenList-Frontend
frontendRepo="${FRONTEND_REPO:-OpenListTeam/OpenList-Frontend}"
githubAuthArgs="" githubAuthArgs=""
if [ -n "$GITHUB_TOKEN" ]; then if [ -n "$GITHUB_TOKEN" ]; then
githubAuthArgs="--header \"Authorization: Bearer $GITHUB_TOKEN\"" githubAuthArgs="--header \"Authorization: Bearer $GITHUB_TOKEN\""
@ -20,15 +17,15 @@ fi
if [ "$1" = "dev" ]; then if [ "$1" = "dev" ]; then
version="dev" version="dev"
webVersion="rolling" webVersion="dev"
elif [ "$1" = "beta" ]; then elif [ "$1" = "beta" ]; then
version="beta" version="beta"
webVersion="rolling" webVersion="dev"
else else
git tag -d beta || true git tag -d beta || true
# Always true if there's no tag # Always true if there's no tag
version=$(git describe --abbrev=0 --tags 2>/dev/null || echo "v0.0.0") version=$(git describe --abbrev=0 --tags 2>/dev/null || echo "v0.0.0")
webVersion=$(eval "curl -fsSL --max-time 2 $githubAuthArgs \"https://api.github.com/repos/$frontendRepo/releases/latest\"" | grep "tag_name" | head -n 1 | awk -F ":" '{print $2}' | sed 's/\"//g;s/,//g;s/ //g') webVersion=$(eval "curl -fsSL --max-time 2 $githubAuthArgs \"https://api.github.com/repos/OpenListTeam/OpenList-Frontend/releases/latest\"" | grep "tag_name" | head -n 1 | awk -F ":" '{print $2}' | sed 's/\"//g;s/,//g;s/ //g')
fi fi
echo "backend version: $version" echo "backend version: $version"
@ -48,21 +45,30 @@ ldflags="\
-X 'github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=$webVersion' \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=$webVersion' \
" "
FetchWebRolling() { FetchWebDev() {
pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/tags/rolling\"") pre_release_tag=$(eval "curl -fsSL --max-time 2 $githubAuthArgs https://api.github.com/repos/OpenListTeam/OpenList-Frontend/releases" | jq -r 'map(select(.prerelease)) | first | .tag_name')
if [ -z "$pre_release_tag" ] || [ "$pre_release_tag" == "null" ]; then
# fall back to latest release
pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/OpenListTeam/OpenList-Frontend/releases/latest\"")
else
pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/OpenListTeam/OpenList-Frontend/releases/tags/$pre_release_tag\"")
fi
pre_release_assets=$(echo "$pre_release_json" | jq -r '.assets[].browser_download_url') pre_release_assets=$(echo "$pre_release_json" | jq -r '.assets[].browser_download_url')
# There is no lite for rolling if [ "$useLite" = true ]; then
pre_release_tar_url=$(echo "$pre_release_assets" | grep "openlist-frontend-dist" | grep -v "lite" | grep "\.tar\.gz$") pre_release_tar_url=$(echo "$pre_release_assets" | grep "openlist-frontend-dist-lite" | grep "\.tar\.gz$")
else
pre_release_tar_url=$(echo "$pre_release_assets" | grep "openlist-frontend-dist" | grep -v "lite" | grep "\.tar\.gz$")
fi
curl -fsSL "$pre_release_tar_url" -o dist.tar.gz curl -fsSL "$pre_release_tar_url" -o web-dist-dev.tar.gz
rm -rf public/dist && mkdir -p public/dist rm -rf public/dist && mkdir -p public/dist
tar -zxvf dist.tar.gz -C public/dist tar -zxvf web-dist-dev.tar.gz -C public/dist
rm -rf dist.tar.gz rm -rf web-dist-dev.tar.gz
} }
FetchWebRelease() { FetchWebRelease() {
release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/latest\"") release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/OpenListTeam/OpenList-Frontend/releases/latest\"")
release_assets=$(echo "$release_json" | jq -r '.assets[].browser_download_url') release_assets=$(echo "$release_json" | jq -r '.assets[].browser_download_url')
if [ "$useLite" = true ]; then if [ "$useLite" = true ]; then
@ -89,45 +95,6 @@ BuildWinArm64() {
go build -o "$1" -ldflags="$ldflags" -tags=jsoniter . go build -o "$1" -ldflags="$ldflags" -tags=jsoniter .
} }
BuildWin7() {
# Setup Win7 Go compiler (patched version that supports Windows 7)
go_version=$(go version | grep -o 'go[0-9]\+\.[0-9]\+\.[0-9]\+' | sed 's/go//')
echo "Detected Go version: $go_version"
curl -fsSL --retry 3 -o go-win7.zip -H "Authorization: Bearer $GITHUB_TOKEN" \
"https://github.com/XTLS/go-win7/releases/download/patched-${go_version}/go-for-win7-linux-amd64.zip"
rm -rf go-win7
unzip go-win7.zip -d go-win7
rm go-win7.zip
# Set permissions for all wrapper files
chmod +x ./wrapper/zcc-win7
chmod +x ./wrapper/zcxx-win7
chmod +x ./wrapper/zcc-win7-386
chmod +x ./wrapper/zcxx-win7-386
# Build for both 386 and amd64 architectures
for arch in "386" "amd64"; do
echo "building for windows7-${arch}"
export GOOS=windows
export GOARCH=${arch}
export CGO_ENABLED=1
# Use architecture-specific wrapper files
if [ "$arch" = "386" ]; then
export CC=$(pwd)/wrapper/zcc-win7-386
export CXX=$(pwd)/wrapper/zcxx-win7-386
else
export CC=$(pwd)/wrapper/zcc-win7
export CXX=$(pwd)/wrapper/zcxx-win7
fi
# Use the patched Go compiler for Win7 compatibility
$(pwd)/go-win7/bin/go build -o "${1}-${arch}.exe" -ldflags="$ldflags" -tags=jsoniter .
done
}
BuildDev() { BuildDev() {
rm -rf .git/ rm -rf .git/
mkdir -p "dist" mkdir -p "dist"
@ -167,7 +134,7 @@ BuildDocker() {
PrepareBuildDockerMusl() { PrepareBuildDockerMusl() {
mkdir -p build/musl-libs mkdir -p build/musl-libs
BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/"
FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross i486-linux-musl-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross riscv64-linux-musl-cross powerpc64le-linux-musl-cross loongarch64-linux-musl-cross) ## Disable s390x-linux-musl-cross builds FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross i486-linux-musl-cross s390x-linux-musl-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross riscv64-linux-musl-cross powerpc64le-linux-musl-cross)
for i in "${FILES[@]}"; do for i in "${FILES[@]}"; do
url="${BASE}${i}.tgz" url="${BASE}${i}.tgz"
lib_tgz="build/${i}.tgz" lib_tgz="build/${i}.tgz"
@ -186,8 +153,8 @@ BuildDockerMultiplatform() {
docker_lflags="--extldflags '-static -fpic' $ldflags" docker_lflags="--extldflags '-static -fpic' $ldflags"
export CGO_ENABLED=1 export CGO_ENABLED=1
OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-riscv64 linux-ppc64le linux-loong64) ## Disable linux-s390x builds OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-s390x linux-riscv64 linux-ppc64le)
CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc loongarch64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc s390x-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc)
for i in "${!OS_ARCHES[@]}"; do for i in "${!OS_ARCHES[@]}"; do
os_arch=${OS_ARCHES[$i]} os_arch=${OS_ARCHES[$i]}
cgo_cc=${CGO_ARGS[$i]} cgo_cc=${CGO_ARGS[$i]}
@ -219,171 +186,12 @@ BuildRelease() {
rm -rf .git/ rm -rf .git/
mkdir -p "build" mkdir -p "build"
BuildWinArm64 ./build/"$appName"-windows-arm64.exe BuildWinArm64 ./build/"$appName"-windows-arm64.exe
BuildWin7 ./build/"$appName"-windows7
xgo -out "$appName" -ldflags="$ldflags" -tags=jsoniter . xgo -out "$appName" -ldflags="$ldflags" -tags=jsoniter .
# why? Because some target platforms seem to have issues with upx compression # why? Because some target platforms seem to have issues with upx compression
# upx -9 ./"$appName"-linux-amd64 # upx -9 ./"$appName"-linux-amd64
# cp ./"$appName"-windows-amd64.exe ./"$appName"-windows-amd64-upx.exe # cp ./"$appName"-windows-amd64.exe ./"$appName"-windows-amd64-upx.exe
# upx -9 ./"$appName"-windows-amd64-upx.exe # upx -9 ./"$appName"-windows-amd64-upx.exe
mv "$appName"-* build mv "$appName"-* build
# Build LoongArch with glibc (both old world abi1.0 and new world abi2.0)
# Separate from musl builds to avoid cache conflicts
BuildLoongGLIBC ./build/$appName-linux-loong64-abi1.0 abi1.0
BuildLoongGLIBC ./build/$appName-linux-loong64 abi2.0
}
BuildLoongGLIBC() {
local target_abi="$2"
local output_file="$1"
local oldWorldGoVersion="1.24.3"
if [ "$target_abi" = "abi1.0" ]; then
echo building for linux-loong64-abi1.0
else
echo building for linux-loong64-abi2.0
target_abi="abi2.0" # Default to abi2.0 if not specified
fi
# Note: No longer need global cache cleanup since ABI1.0 uses isolated cache directory
echo "Using optimized cache strategy: ABI1.0 has isolated cache, ABI2.0 uses standard cache"
if [ "$target_abi" = "abi1.0" ]; then
# Setup abi1.0 toolchain and patched Go compiler similar to cgo-action implementation
echo "Setting up Loongson old-world ABI1.0 toolchain and patched Go compiler..."
# Download and setup patched Go compiler for old-world
if ! curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \
"https://github.com/loong64/loong64-abi1.0-toolchains/releases/download/20250722/go${oldWorldGoVersion}.linux-amd64.tar.gz" \
-o go-loong64-abi1.0.tar.gz; then
echo "Error: Failed to download patched Go compiler for old-world ABI1.0"
if [ -n "$GITHUB_TOKEN" ]; then
echo "Error output from curl:"
curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \
"https://github.com/loong64/loong64-abi1.0-toolchains/releases/download/20250722/go${oldWorldGoVersion}.linux-amd64.tar.gz" \
-o go-loong64-abi1.0.tar.gz || true
fi
return 1
fi
rm -rf go-loong64-abi1.0
mkdir go-loong64-abi1.0
if ! tar -xzf go-loong64-abi1.0.tar.gz -C go-loong64-abi1.0 --strip-components=1; then
echo "Error: Failed to extract patched Go compiler"
return 1
fi
rm go-loong64-abi1.0.tar.gz
# Download and setup GCC toolchain for old-world
if ! curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \
"https://github.com/loong64/loong64-abi1.0-toolchains/releases/download/20250722/loongson-gnu-toolchain-8.3.novec-x86_64-loongarch64-linux-gnu-rc1.1.tar.xz" \
-o gcc8-loong64-abi1.0.tar.xz; then
echo "Error: Failed to download GCC toolchain for old-world ABI1.0"
if [ -n "$GITHUB_TOKEN" ]; then
echo "Error output from curl:"
curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \
"https://github.com/loong64/loong64-abi1.0-toolchains/releases/download/20250722/loongson-gnu-toolchain-8.3.novec-x86_64-loongarch64-linux-gnu-rc1.1.tar.xz" \
-o gcc8-loong64-abi1.0.tar.xz || true
fi
return 1
fi
rm -rf gcc8-loong64-abi1.0
mkdir gcc8-loong64-abi1.0
if ! tar -Jxf gcc8-loong64-abi1.0.tar.xz -C gcc8-loong64-abi1.0 --strip-components=1; then
echo "Error: Failed to extract GCC toolchain"
return 1
fi
rm gcc8-loong64-abi1.0.tar.xz
# Setup separate cache directory for ABI1.0 to avoid cache pollution
abi1_cache_dir="$(pwd)/go-loong64-abi1.0-cache"
mkdir -p "$abi1_cache_dir"
echo "Using separate cache directory for ABI1.0: $abi1_cache_dir"
# Use patched Go compiler for old-world build (critical for ABI1.0 compatibility)
echo "Building with patched Go compiler for old-world ABI1.0..."
echo "Using isolated cache directory: $abi1_cache_dir"
# Use env command to set environment variables locally without affecting global environment
if ! env GOOS=linux GOARCH=loong64 \
CC="$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-gcc" \
CXX="$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-g++" \
CGO_ENABLED=1 \
GOCACHE="$abi1_cache_dir" \
$(pwd)/go-loong64-abi1.0/bin/go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then
echo "Error: Build failed with patched Go compiler"
echo "Attempting retry with cache cleanup..."
env GOCACHE="$abi1_cache_dir" $(pwd)/go-loong64-abi1.0/bin/go clean -cache
if ! env GOOS=linux GOARCH=loong64 \
CC="$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-gcc" \
CXX="$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-g++" \
CGO_ENABLED=1 \
GOCACHE="$abi1_cache_dir" \
$(pwd)/go-loong64-abi1.0/bin/go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then
echo "Error: Build failed again after cache cleanup"
echo "Build environment details:"
echo "GOOS=linux"
echo "GOARCH=loong64"
echo "CC=$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-gcc"
echo "CXX=$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-g++"
echo "CGO_ENABLED=1"
echo "GOCACHE=$abi1_cache_dir"
echo "Go version: $($(pwd)/go-loong64-abi1.0/bin/go version)"
echo "GCC version: $($(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-gcc --version | head -1)"
return 1
fi
fi
else
# Setup abi2.0 toolchain for new world glibc build
echo "Setting up new-world ABI2.0 toolchain..."
if ! curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \
"https://github.com/loong64/cross-tools/releases/download/20250507/x86_64-cross-tools-loongarch64-unknown-linux-gnu-legacy.tar.xz" \
-o gcc12-loong64-abi2.0.tar.xz; then
echo "Error: Failed to download GCC toolchain for new-world ABI2.0"
if [ -n "$GITHUB_TOKEN" ]; then
echo "Error output from curl:"
curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \
"https://github.com/loong64/cross-tools/releases/download/20250507/x86_64-cross-tools-loongarch64-unknown-linux-gnu-legacy.tar.xz" \
-o gcc12-loong64-abi2.0.tar.xz || true
fi
return 1
fi
rm -rf gcc12-loong64-abi2.0
mkdir gcc12-loong64-abi2.0
if ! tar -Jxf gcc12-loong64-abi2.0.tar.xz -C gcc12-loong64-abi2.0 --strip-components=1; then
echo "Error: Failed to extract GCC toolchain"
return 1
fi
rm gcc12-loong64-abi2.0.tar.xz
export GOOS=linux
export GOARCH=loong64
export CC=$(pwd)/gcc12-loong64-abi2.0/bin/loongarch64-unknown-linux-gnu-gcc
export CXX=$(pwd)/gcc12-loong64-abi2.0/bin/loongarch64-unknown-linux-gnu-g++
export CGO_ENABLED=1
# Use standard Go compiler for new-world build
echo "Building with standard Go compiler for new-world ABI2.0..."
if ! go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then
echo "Error: Build failed with standard Go compiler"
echo "Attempting retry with cache cleanup..."
go clean -cache
if ! go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then
echo "Error: Build failed again after cache cleanup"
echo "Build environment details:"
echo "GOOS=$GOOS"
echo "GOARCH=$GOARCH"
echo "CC=$CC"
echo "CXX=$CXX"
echo "CGO_ENABLED=$CGO_ENABLED"
echo "Go version: $(go version)"
echo "GCC version: $($CC --version | head -1)"
return 1
fi
fi
fi
} }
BuildReleaseLinuxMusl() { BuildReleaseLinuxMusl() {
@ -441,7 +249,6 @@ BuildReleaseLinuxMuslArm() {
done done
} }
BuildReleaseAndroid() { BuildReleaseAndroid() {
rm -rf .git/ rm -rf .git/
mkdir -p "build" mkdir -p "build"
@ -537,7 +344,7 @@ MakeRelease() {
tar -czvf compress/"$i$liteSuffix".tar.gz "$appName" tar -czvf compress/"$i$liteSuffix".tar.gz "$appName"
rm -f "$appName" rm -f "$appName"
done done
for i in $(find . -type f \( -name "$appName-windows-*" -o -name "$appName-windows7-*" \)); do for i in $(find . -type f -name "$appName-windows-*"); do
cp "$i" "$appName".exe cp "$i" "$appName".exe
zip compress/$(echo $i | sed 's/\.[^.]*$//')$liteSuffix.zip "$appName".exe zip compress/$(echo $i | sed 's/\.[^.]*$//')$liteSuffix.zip "$appName".exe
rm -f "$appName".exe rm -f "$appName".exe
@ -584,7 +391,7 @@ for arg in "$@"; do
done done
if [ "$buildType" = "dev" ]; then if [ "$buildType" = "dev" ]; then
FetchWebRolling FetchWebDev
if [ "$dockerType" = "docker" ]; then if [ "$dockerType" = "docker" ]; then
BuildDocker BuildDocker
elif [ "$dockerType" = "docker-multiplatform" ]; then elif [ "$dockerType" = "docker-multiplatform" ]; then
@ -596,7 +403,7 @@ if [ "$buildType" = "dev" ]; then
fi fi
elif [ "$buildType" = "release" -o "$buildType" = "beta" ]; then elif [ "$buildType" = "release" -o "$buildType" = "beta" ]; then
if [ "$buildType" = "beta" ]; then if [ "$buildType" = "beta" ]; then
FetchWebRolling FetchWebDev
else else
FetchWebRelease FetchWebRelease
fi fi
@ -677,5 +484,4 @@ else
echo -e " $0 release" echo -e " $0 release"
echo -e " $0 release lite" echo -e " $0 release lite"
echo -e " $0 release docker lite" echo -e " $0 release docker lite"
echo -e " $0 release linux_musl"
fi fi

View File

@ -4,8 +4,6 @@ Copyright © 2022 NAME HERE <EMAIL ADDRESS>
package cmd package cmd
import ( import (
"fmt"
"github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/setting"
@ -26,11 +24,10 @@ var AdminCmd = &cobra.Command{
if err != nil { if err != nil {
utils.Log.Errorf("failed get admin user: %+v", err) utils.Log.Errorf("failed get admin user: %+v", err)
} else { } else {
utils.Log.Infof("get admin user from CLI") utils.Log.Infof("Admin user's username: %s", admin.Username)
fmt.Println("Admin user's username:", admin.Username) utils.Log.Infof("The password can only be output at the first startup, and then stored as a hash value, which cannot be reversed")
fmt.Println("The password can only be output at the first startup, and then stored as a hash value, which cannot be reversed") utils.Log.Infof("You can reset the password with a random string by running [openlist admin random]")
fmt.Println("You can reset the password with a random string by running [openlist admin random]") utils.Log.Infof("You can also set a new password by running [openlist admin set NEW_PASSWORD]")
fmt.Println("You can also set a new password by running [openlist admin set NEW_PASSWORD]")
} }
}, },
} }
@ -39,7 +36,6 @@ var RandomPasswordCmd = &cobra.Command{
Use: "random", Use: "random",
Short: "Reset admin user's password to a random string", Short: "Reset admin user's password to a random string",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
utils.Log.Infof("reset admin user's password to a random string from CLI")
newPwd := random.String(8) newPwd := random.String(8)
setAdminPassword(newPwd) setAdminPassword(newPwd)
}, },
@ -48,12 +44,12 @@ var RandomPasswordCmd = &cobra.Command{
var SetPasswordCmd = &cobra.Command{ var SetPasswordCmd = &cobra.Command{
Use: "set", Use: "set",
Short: "Set admin user's password", Short: "Set admin user's password",
RunE: func(cmd *cobra.Command, args []string) error { Run: func(cmd *cobra.Command, args []string) {
if len(args) == 0 { if len(args) == 0 {
return fmt.Errorf("Please enter the new password") utils.Log.Errorf("Please enter the new password")
return
} }
setAdminPassword(args[0]) setAdminPassword(args[0])
return nil
}, },
} }
@ -64,8 +60,7 @@ var ShowTokenCmd = &cobra.Command{
Init() Init()
defer Release() defer Release()
token := setting.GetStr(conf.Token) token := setting.GetStr(conf.Token)
utils.Log.Infof("show admin token from CLI") utils.Log.Infof("Admin token: %s", token)
fmt.Println("Admin token:", token)
}, },
} }
@ -82,10 +77,9 @@ func setAdminPassword(pwd string) {
utils.Log.Errorf("failed update admin user: %+v", err) utils.Log.Errorf("failed update admin user: %+v", err)
return return
} }
utils.Log.Infof("admin user has been update from CLI") utils.Log.Infof("admin user has been updated:")
fmt.Println("admin user has been updated:") utils.Log.Infof("username: %s", admin.Username)
fmt.Println("username:", admin.Username) utils.Log.Infof("password: %s", pwd)
fmt.Println("password:", pwd)
DelAdminCacheOnline() DelAdminCacheOnline()
} }

View File

@ -4,8 +4,6 @@ Copyright © 2022 NAME HERE <EMAIL ADDRESS>
package cmd package cmd
import ( import (
"fmt"
"github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@ -26,8 +24,7 @@ var Cancel2FACmd = &cobra.Command{
if err != nil { if err != nil {
utils.Log.Errorf("failed to cancel 2FA: %+v", err) utils.Log.Errorf("failed to cancel 2FA: %+v", err)
} else { } else {
utils.Log.Infof("2FA is canceled from CLI") utils.Log.Info("2FA canceled")
fmt.Println("2FA canceled")
DelAdminCacheOnline() DelAdminCacheOnline()
} }
} }

View File

@ -1,42 +1,51 @@
package cmd package cmd
import ( import (
"context" "os"
"path/filepath"
"strconv"
"github.com/OpenListTeam/OpenList/v5/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap"
"github.com/OpenListTeam/OpenList/v5/internal/bootstrap" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/data"
"github.com/sirupsen/logrus" "github.com/OpenListTeam/OpenList/v4/internal/db"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
log "github.com/sirupsen/logrus"
) )
func Init(ctx context.Context) { func Init() {
if flags.Dev {
flags.Debug = true
}
initLogrus()
bootstrap.InitConfig() bootstrap.InitConfig()
bootstrap.InitDriverPlugins() bootstrap.Log()
bootstrap.InitDB()
data.InitData()
bootstrap.InitStreamLimit()
bootstrap.InitIndex()
bootstrap.InitUpgradePatch()
} }
func Release() { func Release() {
db.Close()
} }
func initLog(l *logrus.Logger) { var pid = -1
if flags.Debug { var pidFile string
l.SetLevel(logrus.DebugLevel)
l.SetReportCaller(true) func initDaemon() {
} else { ex, err := os.Executable()
l.SetLevel(logrus.InfoLevel) if err != nil {
l.SetReportCaller(false) log.Fatal(err)
}
exPath := filepath.Dir(ex)
_ = os.MkdirAll(filepath.Join(exPath, "daemon"), 0700)
pidFile = filepath.Join(exPath, "daemon/pid")
if utils.Exists(pidFile) {
bytes, err := os.ReadFile(pidFile)
if err != nil {
log.Fatal("failed to read pid file", err)
}
id, err := strconv.Atoi(string(bytes))
if err != nil {
log.Fatal("failed to parse pid data", err)
}
pid = id
} }
} }
func initLogrus() {
formatter := logrus.TextFormatter{
ForceColors: true,
EnvironmentOverrideColors: true,
TimestampFormat: "2006-01-02 15:04:05",
FullTimestamp: true,
}
logrus.SetFormatter(&formatter)
initLog(logrus.StandardLogger())
}

View File

@ -1,40 +1,10 @@
package flags package flags
import (
"os"
"path/filepath"
"github.com/sirupsen/logrus"
)
var ( var (
ConfigFile string DataDir string
Debug bool Debug bool
NoPrefix bool NoPrefix bool
Dev bool Dev bool
ForceBinDir bool ForceBinDir bool
LogStd bool LogStd bool
pwd string
) )
// Program working directory
func PWD() string {
if pwd != "" {
return pwd
}
if ForceBinDir {
ex, err := os.Executable()
if err != nil {
logrus.Fatal(err)
}
pwd = filepath.Dir(ex)
return pwd
}
d, err := os.Getwd()
if err != nil {
logrus.Fatal(err)
}
pwd = d
return d
}

View File

@ -4,7 +4,10 @@ import (
"fmt" "fmt"
"os" "os"
"github.com/OpenListTeam/OpenList/v5/cmd/flags" "github.com/OpenListTeam/OpenList/v4/cmd/flags"
_ "github.com/OpenListTeam/OpenList/v4/drivers"
_ "github.com/OpenListTeam/OpenList/v4/internal/archive"
_ "github.com/OpenListTeam/OpenList/v4/internal/offline_download"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
@ -24,10 +27,10 @@ func Execute() {
} }
func init() { func init() {
RootCmd.PersistentFlags().StringVarP(&flags.ConfigFile, "config", "c", "data/config.json", "config file") RootCmd.PersistentFlags().StringVar(&flags.DataDir, "data", "data", "data folder")
RootCmd.PersistentFlags().BoolVar(&flags.Debug, "debug", false, "start with debug mode") RootCmd.PersistentFlags().BoolVar(&flags.Debug, "debug", false, "start with debug mode")
RootCmd.PersistentFlags().BoolVar(&flags.NoPrefix, "no-prefix", false, "disable env prefix") RootCmd.PersistentFlags().BoolVar(&flags.NoPrefix, "no-prefix", false, "disable env prefix")
RootCmd.PersistentFlags().BoolVar(&flags.Dev, "dev", false, "start with dev mode") RootCmd.PersistentFlags().BoolVar(&flags.Dev, "dev", false, "start with dev mode")
RootCmd.PersistentFlags().BoolVarP(&flags.ForceBinDir, "force-bin-dir", "f", false, "force to use the directory where the binary file is located as data directory") RootCmd.PersistentFlags().BoolVar(&flags.ForceBinDir, "force-bin-dir", false, "Force to use the directory where the binary file is located as data directory")
RootCmd.PersistentFlags().BoolVar(&flags.LogStd, "log-std", false, "force to log to std") RootCmd.PersistentFlags().BoolVar(&flags.LogStd, "log-std", false, "Force to log to std")
} }

View File

@ -13,9 +13,15 @@ import (
"syscall" "syscall"
"time" "time"
"github.com/OpenListTeam/OpenList/v5/cmd/flags" "github.com/OpenListTeam/OpenList/v4/cmd/flags"
"github.com/OpenListTeam/OpenList/v5/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap"
"github.com/OpenListTeam/OpenList/v5/server" "github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/OpenList/v4/server"
"github.com/OpenListTeam/OpenList/v4/server/middlewares"
"github.com/OpenListTeam/sftpd-openlist"
ftpserver "github.com/fclairamb/ftpserverlib"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@ -29,127 +35,214 @@ var ServerCmd = &cobra.Command{
Short: "Start the server at the specified address", Short: "Start the server at the specified address",
Long: `Start the server at the specified address Long: `Start the server at the specified address
the address is defined in config file`, the address is defined in config file`,
Run: func(_ *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
serverCtx, serverCancel := context.WithCancel(context.Background()) Init()
defer serverCancel() if conf.Conf.DelayedStart != 0 {
Init(serverCtx) utils.Log.Infof("delayed start for %d seconds", conf.Conf.DelayedStart)
time.Sleep(time.Duration(conf.Conf.DelayedStart) * time.Second)
if !flags.Debug { }
bootstrap.InitOfflineDownloadTools()
bootstrap.LoadStorages()
bootstrap.InitTaskManager()
if !flags.Debug && !flags.Dev {
gin.SetMode(gin.ReleaseMode) gin.SetMode(gin.ReleaseMode)
} }
r := gin.New() r := gin.New()
r.Use(gin.LoggerWithWriter(log.StandardLogger().Out))
r.Use(gin.RecoveryWithWriter(log.StandardLogger().Out))
server.Init(r)
// gin log
if conf.Conf.Log.Filter.Enable {
r.Use(middlewares.FilteredLogger())
} else {
r.Use(gin.LoggerWithWriter(log.StandardLogger().Out))
}
r.Use(gin.RecoveryWithWriter(log.StandardLogger().Out))
server.Init(r)
var httpHandler http.Handler = r var httpHandler http.Handler = r
if conf.Conf.Scheme.EnableH2c { if conf.Conf.Scheme.EnableH2c {
httpHandler = h2c.NewHandler(r, &http2.Server{}) httpHandler = h2c.NewHandler(r, &http2.Server{})
} }
var httpSrv, httpsSrv, unixSrv *http.Server var httpSrv, httpsSrv, unixSrv *http.Server
if conf.Conf.Scheme.HttpPort > 0 { if conf.Conf.Scheme.HttpPort != -1 {
httpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpPort) httpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpPort)
log.Infoln("start HTTP server", "@", httpBase) utils.Log.Infof("start HTTP server @ %s", httpBase)
httpSrv = &http.Server{Addr: httpBase, Handler: httpHandler} httpSrv = &http.Server{Addr: httpBase, Handler: httpHandler}
go func() { go func() {
err := httpSrv.ListenAndServe() err := httpSrv.ListenAndServe()
if err != nil && !errors.Is(err, http.ErrServerClosed) { if err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Errorln("start HTTP server", ":", err) utils.Log.Fatalf("failed to start http: %s", err.Error())
serverCancel()
} }
}() }()
} }
if conf.Conf.Scheme.HttpsPort > 0 { if conf.Conf.Scheme.HttpsPort != -1 {
httpsBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpsPort) httpsBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpsPort)
log.Infoln("start HTTPS server", "@", httpsBase) utils.Log.Infof("start HTTPS server @ %s", httpsBase)
httpsSrv = &http.Server{Addr: httpsBase, Handler: r} httpsSrv = &http.Server{Addr: httpsBase, Handler: r}
go func() { go func() {
err := httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) err := httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile)
if err != nil && !errors.Is(err, http.ErrServerClosed) { if err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Errorln("start HTTPS server", ":", err) utils.Log.Fatalf("failed to start https: %s", err.Error())
serverCancel()
} }
}() }()
} }
if conf.Conf.Scheme.UnixFile != "" { if conf.Conf.Scheme.UnixFile != "" {
log.Infoln("start Unix server", "@", conf.Conf.Scheme.UnixFile) utils.Log.Infof("start unix server @ %s", conf.Conf.Scheme.UnixFile)
unixSrv = &http.Server{Handler: httpHandler} unixSrv = &http.Server{Handler: httpHandler}
go func() { go func() {
listener, err := net.Listen("unix", conf.Conf.Scheme.UnixFile) listener, err := net.Listen("unix", conf.Conf.Scheme.UnixFile)
if err != nil { if err != nil {
log.Errorln("start Unix server", ":", err) utils.Log.Fatalf("failed to listen unix: %+v", err)
serverCancel()
return
} }
// set socket file permission
mode, err := strconv.ParseUint(conf.Conf.Scheme.UnixFilePerm, 8, 32) mode, err := strconv.ParseUint(conf.Conf.Scheme.UnixFilePerm, 8, 32)
if err != nil { if err != nil {
log.Errorln("parse unix_file_perm", ":", err) utils.Log.Errorf("failed to parse socket file permission: %+v", err)
} else { } else {
err = os.Chmod(conf.Conf.Scheme.UnixFile, os.FileMode(mode)) err = os.Chmod(conf.Conf.Scheme.UnixFile, os.FileMode(mode))
if err != nil { if err != nil {
log.Errorln("chmod socket file", ":", err) utils.Log.Errorf("failed to chmod socket file: %+v", err)
} }
} }
err = unixSrv.Serve(listener) err = unixSrv.Serve(listener)
if err != nil && !errors.Is(err, http.ErrServerClosed) { if err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Errorln("start Unix server", ":", err) utils.Log.Fatalf("failed to start unix: %s", err.Error())
serverCancel()
} }
}() }()
} }
if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable {
s3r := gin.New()
s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out))
server.InitS3(s3r)
s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port)
utils.Log.Infof("start S3 server @ %s", s3Base)
go func() {
var err error
if conf.Conf.S3.SSL {
httpsSrv = &http.Server{Addr: s3Base, Handler: s3r}
err = httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile)
}
if !conf.Conf.S3.SSL {
httpSrv = &http.Server{Addr: s3Base, Handler: s3r}
err = httpSrv.ListenAndServe()
}
if err != nil && !errors.Is(err, http.ErrServerClosed) {
utils.Log.Fatalf("failed to start s3 server: %s", err.Error())
}
}()
}
var ftpDriver *server.FtpMainDriver
var ftpServer *ftpserver.FtpServer
if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable {
var err error
ftpDriver, err = server.NewMainDriver()
if err != nil {
utils.Log.Fatalf("failed to start ftp driver: %s", err.Error())
} else {
utils.Log.Infof("start ftp server on %s", conf.Conf.FTP.Listen)
go func() {
ftpServer = ftpserver.NewFtpServer(ftpDriver)
err = ftpServer.ListenAndServe()
if err != nil {
utils.Log.Fatalf("problem ftp server listening: %s", err.Error())
}
}()
}
}
var sftpDriver *server.SftpDriver
var sftpServer *sftpd.SftpServer
if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable {
var err error
sftpDriver, err = server.NewSftpDriver()
if err != nil {
utils.Log.Fatalf("failed to start sftp driver: %s", err.Error())
} else {
utils.Log.Infof("start sftp server on %s", conf.Conf.SFTP.Listen)
go func() {
sftpServer = sftpd.NewSftpServer(sftpDriver)
err = sftpServer.RunServer()
if err != nil {
utils.Log.Fatalf("problem sftp server listening: %s", err.Error())
}
}()
}
}
// Wait for interrupt signal to gracefully shutdown the server with
// a timeout of 1 second.
quit := make(chan os.Signal, 1) quit := make(chan os.Signal, 1)
// kill (no param) default send syscanll.SIGTERM // kill (no param) default send syscanll.SIGTERM
// kill -2 is syscall.SIGINT // kill -2 is syscall.SIGINT
// kill -9 is syscall. SIGKILL but can"t be catch, so don't need add it // kill -9 is syscall. SIGKILL but can"t be catch, so don't need add it
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
select { <-quit
case <-quit: utils.Log.Println("Shutdown server...")
case <-serverCtx.Done(): fs.ArchiveContentUploadTaskManager.RemoveAll()
}
log.Println("shutdown server...")
Release() Release()
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
quitCtx, quitCancel := context.WithTimeout(context.Background(), time.Second) defer cancel()
defer quitCancel()
var wg sync.WaitGroup var wg sync.WaitGroup
if httpSrv != nil { if conf.Conf.Scheme.HttpPort != -1 {
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
if err := httpSrv.Shutdown(quitCtx); err != nil { if err := httpSrv.Shutdown(ctx); err != nil {
log.Errorln("shutdown HTTP server", ":", err) utils.Log.Fatal("HTTP server shutdown err: ", err)
} }
}() }()
} }
if httpsSrv != nil { if conf.Conf.Scheme.HttpsPort != -1 {
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
if err := httpsSrv.Shutdown(quitCtx); err != nil { if err := httpsSrv.Shutdown(ctx); err != nil {
log.Errorln("shutdown HTTPS server", ":", err) utils.Log.Fatal("HTTPS server shutdown err: ", err)
} }
}() }()
} }
if unixSrv != nil { if conf.Conf.Scheme.UnixFile != "" {
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
if err := unixSrv.Shutdown(quitCtx); err != nil { if err := unixSrv.Shutdown(ctx); err != nil {
log.Errorln("shutdown Unix server", ":", err) utils.Log.Fatal("Unix server shutdown err: ", err)
}
}()
}
if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable && ftpServer != nil && ftpDriver != nil {
wg.Add(1)
go func() {
defer wg.Done()
ftpDriver.Stop()
if err := ftpServer.Stop(); err != nil {
utils.Log.Fatal("FTP server shutdown err: ", err)
}
}()
}
if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable && sftpServer != nil && sftpDriver != nil {
wg.Add(1)
go func() {
defer wg.Done()
if err := sftpServer.Close(); err != nil {
utils.Log.Fatal("SFTP server shutdown err: ", err)
} }
}() }()
} }
wg.Wait() wg.Wait()
log.Println("server exit") utils.Log.Println("Server exit")
}, },
} }
func init() { func init() {
RootCmd.AddCommand(ServerCmd) RootCmd.AddCommand(ServerCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// serverCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// serverCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
} }
// OutOpenListInit 暴露用于外部启动server的函数 // OutOpenListInit 暴露用于外部启动server的函数

View File

@ -4,7 +4,6 @@ Copyright © 2023 NAME HERE <EMAIL ADDRESS>
package cmd package cmd
import ( import (
"fmt"
"os" "os"
"strconv" "strconv"
@ -23,61 +22,28 @@ var storageCmd = &cobra.Command{
} }
var disableStorageCmd = &cobra.Command{ var disableStorageCmd = &cobra.Command{
Use: "disable [mount path]", Use: "disable",
Short: "Disable a storage by mount path", Short: "Disable a storage",
RunE: func(cmd *cobra.Command, args []string) error { Run: func(cmd *cobra.Command, args []string) {
if len(args) < 1 { if len(args) < 1 {
return fmt.Errorf("mount path is required") utils.Log.Errorf("mount path is required")
return
} }
mountPath := args[0] mountPath := args[0]
Init() Init()
defer Release() defer Release()
storage, err := db.GetStorageByMountPath(mountPath) storage, err := db.GetStorageByMountPath(mountPath)
if err != nil { if err != nil {
return fmt.Errorf("failed to query storage: %+v", err) utils.Log.Errorf("failed to query storage: %+v", err)
} } else {
storage.Disabled = true storage.Disabled = true
err = db.UpdateStorage(storage) err = db.UpdateStorage(storage)
if err != nil { if err != nil {
return fmt.Errorf("failed to update storage: %+v", err) utils.Log.Errorf("failed to update storage: %+v", err)
} } else {
utils.Log.Infof("Storage with mount path [%s] has been disabled from CLI", mountPath) utils.Log.Infof("Storage with mount path [%s] have been disabled", mountPath)
fmt.Printf("Storage with mount path [%s] has been disabled\n", mountPath)
return nil
},
}
var deleteStorageCmd = &cobra.Command{
Use: "delete [id]",
Short: "Delete a storage by id",
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) < 1 {
return fmt.Errorf("id is required")
}
id, err := strconv.Atoi(args[0])
if err != nil {
return fmt.Errorf("id must be a number")
}
if force, _ := cmd.Flags().GetBool("force"); force {
fmt.Printf("Are you sure you want to delete storage with id [%d]? [y/N]: ", id)
var confirm string
fmt.Scanln(&confirm)
if confirm != "y" && confirm != "Y" {
fmt.Println("Delete operation cancelled.")
return nil
} }
} }
Init()
defer Release()
err = db.DeleteStorageById(uint(id))
if err != nil {
return fmt.Errorf("failed to delete storage by id: %+v", err)
}
utils.Log.Infof("Storage with id [%d] have been deleted from CLI", id)
fmt.Printf("Storage with id [%d] have been deleted\n", id)
return nil
}, },
} }
@ -122,14 +88,14 @@ var storageTableHeight int
var listStorageCmd = &cobra.Command{ var listStorageCmd = &cobra.Command{
Use: "list", Use: "list",
Short: "List all storages", Short: "List all storages",
RunE: func(cmd *cobra.Command, args []string) error { Run: func(cmd *cobra.Command, args []string) {
Init() Init()
defer Release() defer Release()
storages, _, err := db.GetStorages(1, -1) storages, _, err := db.GetStorages(1, -1)
if err != nil { if err != nil {
return fmt.Errorf("failed to query storages: %+v", err) utils.Log.Errorf("failed to query storages: %+v", err)
} else { } else {
fmt.Printf("Found %d storages\n", len(storages)) utils.Log.Infof("Found %d storages", len(storages))
columns := []table.Column{ columns := []table.Column{
{Title: "ID", Width: 4}, {Title: "ID", Width: 4},
{Title: "Driver", Width: 16}, {Title: "Driver", Width: 16},
@ -172,11 +138,10 @@ var listStorageCmd = &cobra.Command{
m := model{t} m := model{t}
if _, err := tea.NewProgram(m).Run(); err != nil { if _, err := tea.NewProgram(m).Run(); err != nil {
fmt.Printf("failed to run program: %+v\n", err) utils.Log.Errorf("failed to run program: %+v", err)
os.Exit(1) os.Exit(1)
} }
} }
return nil
}, },
} }
@ -186,8 +151,6 @@ func init() {
storageCmd.AddCommand(disableStorageCmd) storageCmd.AddCommand(disableStorageCmd)
storageCmd.AddCommand(listStorageCmd) storageCmd.AddCommand(listStorageCmd)
storageCmd.PersistentFlags().IntVarP(&storageTableHeight, "height", "H", 10, "Table height") storageCmd.PersistentFlags().IntVarP(&storageTableHeight, "height", "H", 10, "Table height")
storageCmd.AddCommand(deleteStorageCmd)
deleteStorageCmd.Flags().BoolP("force", "f", false, "Force delete without confirmation")
// Here you will define your flags and configuration settings. // Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command // Cobra supports Persistent Flags which will work for this command

View File

@ -186,7 +186,9 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
preHash = strings.ToUpper(preHash) preHash = strings.ToUpper(preHash)
fullHash := stream.GetHash().GetHash(utils.SHA1) fullHash := stream.GetHash().GetHash(utils.SHA1)
if len(fullHash) != utils.SHA1.Width { if len(fullHash) != utils.SHA1.Width {
_, fullHash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1) cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
up = model.UpdateProgressWithRange(up, 50, 100)
_, fullHash, err = streamPkg.CacheFullInTempFileAndHash(stream, cacheFileProgress, utils.SHA1)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -321,7 +321,7 @@ func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.Upload
err error err error
) )
tmpF, err := s.CacheFullAndWriter(&up, nil) tmpF, err := s.CacheFullInTempFile()
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -131,23 +131,6 @@ func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs)
}, nil }, nil
} }
func (d *Open115) GetObjInfo(ctx context.Context, path string) (model.Obj, error) {
if err := d.WaitLimit(ctx); err != nil {
return nil, err
}
resp, err := d.client.GetFolderInfoByPath(ctx, path)
if err != nil {
return nil, err
}
return &Obj{
Fid: resp.FileID,
Fn: resp.FileName,
Fc: resp.FileCategory,
Sha1: resp.Sha1,
Pc: resp.PickCode,
}, nil
}
func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) {
if err := d.WaitLimit(ctx); err != nil { if err := d.WaitLimit(ctx); err != nil {
return nil, err return nil, err
@ -239,7 +222,9 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre
} }
sha1 := file.GetHash().GetHash(utils.SHA1) sha1 := file.GetHash().GetHash(utils.SHA1)
if len(sha1) != utils.SHA1.Width { if len(sha1) != utils.SHA1.Width {
_, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
up = model.UpdateProgressWithRange(up, 50, 100)
_, sha1, err = stream.CacheFullInTempFileAndHash(file, cacheFileProgress, utils.SHA1)
if err != nil { if err != nil {
return err return err
} }

View File

@ -9,7 +9,6 @@ import (
sdk "github.com/OpenListTeam/115-sdk-go" sdk "github.com/OpenListTeam/115-sdk-go"
"github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/model"
streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/aliyun/aliyun-oss-go-sdk/oss"
"github.com/avast/retry-go" "github.com/avast/retry-go"
@ -70,6 +69,9 @@ func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp
// } // }
func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error {
fileSize := stream.GetSize()
chunkSize := calPartSize(fileSize)
ossClient, err := oss.New(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) ossClient, err := oss.New(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken))
if err != nil { if err != nil {
return err return err
@ -84,13 +86,6 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer,
return err return err
} }
fileSize := stream.GetSize()
chunkSize := calPartSize(fileSize)
ss, err := streamPkg.NewStreamSectionReader(stream, int(chunkSize), &up)
if err != nil {
return err
}
partNum := (stream.GetSize() + chunkSize - 1) / chunkSize partNum := (stream.GetSize() + chunkSize - 1) / chunkSize
parts := make([]oss.UploadPart, partNum) parts := make([]oss.UploadPart, partNum)
offset := int64(0) offset := int64(0)
@ -103,13 +98,10 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer,
if i == partNum { if i == partNum {
partSize = fileSize - (i-1)*chunkSize partSize = fileSize - (i-1)*chunkSize
} }
rd, err := ss.GetSectionReader(offset, partSize) rd := utils.NewMultiReadable(io.LimitReader(stream, partSize))
if err != nil {
return err
}
rateLimitedRd := driver.NewLimitedUploadStream(ctx, rd)
err = retry.Do(func() error { err = retry.Do(func() error {
rd.Seek(0, io.SeekStart) _ = rd.Reset()
rateLimitedRd := driver.NewLimitedUploadStream(ctx, rd)
part, err := bucket.UploadPart(imur, rateLimitedRd, partSize, int(i)) part, err := bucket.UploadPart(imur, rateLimitedRd, partSize, int(i))
if err != nil { if err != nil {
return err return err
@ -120,7 +112,6 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer,
retry.Attempts(3), retry.Attempts(3),
retry.DelayType(retry.BackOffDelay), retry.DelayType(retry.BackOffDelay),
retry.Delay(time.Second)) retry.Delay(time.Second))
ss.FreeSectionReader(rd)
if err != nil { if err != nil {
return err return err
} }

View File

@ -64,6 +64,14 @@ func (d *Pan123) List(ctx context.Context, dir model.Obj, args model.ListArgs) (
func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
if f, ok := file.(File); ok { if f, ok := file.(File); ok {
//var resp DownResp
var headers map[string]string
if !utils.IsLocalIPAddr(args.IP) {
headers = map[string]string{
//"X-Real-IP": "1.1.1.1",
"X-Forwarded-For": args.IP,
}
}
data := base.Json{ data := base.Json{
"driveId": 0, "driveId": 0,
"etag": f.Etag, "etag": f.Etag,
@ -75,27 +83,25 @@ func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs)
} }
resp, err := d.Request(DownloadInfo, http.MethodPost, func(req *resty.Request) { resp, err := d.Request(DownloadInfo, http.MethodPost, func(req *resty.Request) {
req.SetBody(data) req.SetBody(data).SetHeaders(headers)
}, nil) }, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
downloadUrl := utils.Json.Get(resp, "data", "DownloadUrl").ToString() downloadUrl := utils.Json.Get(resp, "data", "DownloadUrl").ToString()
ou, err := url.Parse(downloadUrl) u, err := url.Parse(downloadUrl)
if err != nil { if err != nil {
return nil, err return nil, err
} }
u_ := ou.String() nu := u.Query().Get("params")
nu := ou.Query().Get("params")
if nu != "" { if nu != "" {
du, _ := base64.StdEncoding.DecodeString(nu) du, _ := base64.StdEncoding.DecodeString(nu)
u, err := url.Parse(string(du)) u, err = url.Parse(string(du))
if err != nil { if err != nil {
return nil, err return nil, err
} }
u_ = u.String()
} }
u_ := u.String()
log.Debug("download url: ", u_) log.Debug("download url: ", u_)
res, err := base.NoRedirectClient.R().SetHeader("Referer", "https://www.123pan.com/").Get(u_) res, err := base.NoRedirectClient.R().SetHeader("Referer", "https://www.123pan.com/").Get(u_)
if err != nil { if err != nil {
@ -112,7 +118,7 @@ func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs)
link.URL = utils.Json.Get(res.Body(), "data", "redirect_url").ToString() link.URL = utils.Json.Get(res.Body(), "data", "redirect_url").ToString()
} }
link.Header = http.Header{ link.Header = http.Header{
"Referer": []string{fmt.Sprintf("%s://%s/", ou.Scheme, ou.Host)}, "Referer": []string{"https://www.123pan.com/"},
} }
return &link, nil return &link, nil
} else { } else {
@ -182,7 +188,9 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, file model.FileStrea
etag := file.GetHash().GetHash(utils.MD5) etag := file.GetHash().GetHash(utils.MD5)
var err error var err error
if len(etag) < utils.MD5.Width { if len(etag) < utils.MD5.Width {
_, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5) cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
up = model.UpdateProgressWithRange(up, 50, 100)
_, etag, err = stream.CacheFullInTempFileAndHash(file, cacheFileProgress, utils.MD5)
if err != nil { if err != nil {
return err return err
} }

View File

@ -11,8 +11,7 @@ type Addition struct {
driver.RootID driver.RootID
//OrderBy string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"` //OrderBy string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"`
//OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` //OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
AccessToken string AccessToken string
UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"`
} }
var config = driver.Config{ var config = driver.Config{
@ -23,11 +22,6 @@ var config = driver.Config{
func init() { func init() {
op.RegisterDriver(func() driver.Driver { op.RegisterDriver(func() driver.Driver {
// 新增默认选项 要在RegisterDriver初始化设置 才会对正在使用的用户生效 return &Pan123{}
return &Pan123{
Addition: Addition{
UploadThread: 3,
},
}
}) })
} }

162
drivers/123/upload.go Normal file
View File

@ -0,0 +1,162 @@
package _123
import (
"context"
"fmt"
"io"
"net/http"
"strconv"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/go-resty/resty/v2"
)
func (d *Pan123) getS3PreSignedUrls(ctx context.Context, upReq *UploadResp, start, end int) (*S3PreSignedURLs, error) {
data := base.Json{
"bucket": upReq.Data.Bucket,
"key": upReq.Data.Key,
"partNumberEnd": end,
"partNumberStart": start,
"uploadId": upReq.Data.UploadId,
"StorageNode": upReq.Data.StorageNode,
}
var s3PreSignedUrls S3PreSignedURLs
_, err := d.Request(S3PreSignedUrls, http.MethodPost, func(req *resty.Request) {
req.SetBody(data).SetContext(ctx)
}, &s3PreSignedUrls)
if err != nil {
return nil, err
}
return &s3PreSignedUrls, nil
}
func (d *Pan123) getS3Auth(ctx context.Context, upReq *UploadResp, start, end int) (*S3PreSignedURLs, error) {
data := base.Json{
"StorageNode": upReq.Data.StorageNode,
"bucket": upReq.Data.Bucket,
"key": upReq.Data.Key,
"partNumberEnd": end,
"partNumberStart": start,
"uploadId": upReq.Data.UploadId,
}
var s3PreSignedUrls S3PreSignedURLs
_, err := d.Request(S3Auth, http.MethodPost, func(req *resty.Request) {
req.SetBody(data).SetContext(ctx)
}, &s3PreSignedUrls)
if err != nil {
return nil, err
}
return &s3PreSignedUrls, nil
}
func (d *Pan123) completeS3(ctx context.Context, upReq *UploadResp, file model.FileStreamer, isMultipart bool) error {
data := base.Json{
"StorageNode": upReq.Data.StorageNode,
"bucket": upReq.Data.Bucket,
"fileId": upReq.Data.FileId,
"fileSize": file.GetSize(),
"isMultipart": isMultipart,
"key": upReq.Data.Key,
"uploadId": upReq.Data.UploadId,
}
_, err := d.Request(UploadCompleteV2, http.MethodPost, func(req *resty.Request) {
req.SetBody(data).SetContext(ctx)
}, nil)
return err
}
func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.FileStreamer, up driver.UpdateProgress) error {
tmpF, err := file.CacheFullInTempFile()
if err != nil {
return err
}
// fetch s3 pre signed urls
size := file.GetSize()
chunkSize := min(size, 16*utils.MB)
chunkCount := int(size / chunkSize)
lastChunkSize := size % chunkSize
if lastChunkSize > 0 {
chunkCount++
} else {
lastChunkSize = chunkSize
}
// only 1 batch is allowed
batchSize := 1
getS3UploadUrl := d.getS3Auth
if chunkCount > 1 {
batchSize = 10
getS3UploadUrl = d.getS3PreSignedUrls
}
for i := 1; i <= chunkCount; i += batchSize {
if utils.IsCanceled(ctx) {
return ctx.Err()
}
start := i
end := min(i+batchSize, chunkCount+1)
s3PreSignedUrls, err := getS3UploadUrl(ctx, upReq, start, end)
if err != nil {
return err
}
// upload each chunk
for j := start; j < end; j++ {
if utils.IsCanceled(ctx) {
return ctx.Err()
}
curSize := chunkSize
if j == chunkCount {
curSize = lastChunkSize
}
err = d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, j, end, io.NewSectionReader(tmpF, chunkSize*int64(j-1), curSize), curSize, false, getS3UploadUrl)
if err != nil {
return err
}
up(float64(j) * 100 / float64(chunkCount))
}
}
// complete s3 upload
return d.completeS3(ctx, upReq, file, chunkCount > 1)
}
func (d *Pan123) uploadS3Chunk(ctx context.Context, upReq *UploadResp, s3PreSignedUrls *S3PreSignedURLs, cur, end int, reader *io.SectionReader, curSize int64, retry bool, getS3UploadUrl func(ctx context.Context, upReq *UploadResp, start int, end int) (*S3PreSignedURLs, error)) error {
uploadUrl := s3PreSignedUrls.Data.PreSignedUrls[strconv.Itoa(cur)]
if uploadUrl == "" {
return fmt.Errorf("upload url is empty, s3PreSignedUrls: %+v", s3PreSignedUrls)
}
req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, reader))
if err != nil {
return err
}
req = req.WithContext(ctx)
req.ContentLength = curSize
//req.Header.Set("Content-Length", strconv.FormatInt(curSize, 10))
res, err := base.HttpClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode == http.StatusForbidden {
if retry {
return fmt.Errorf("upload s3 chunk %d failed, status code: %d", cur, res.StatusCode)
}
// refresh s3 pre signed urls
newS3PreSignedUrls, err := getS3UploadUrl(ctx, upReq, cur, end)
if err != nil {
return err
}
s3PreSignedUrls.Data.PreSignedUrls = newS3PreSignedUrls.Data.PreSignedUrls
// retry
reader.Seek(0, io.SeekStart)
return d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, cur, end, reader, curSize, true, getS3UploadUrl)
}
if res.StatusCode != http.StatusOK {
body, err := io.ReadAll(res.Body)
if err != nil {
return err
}
return fmt.Errorf("upload s3 chunk %d failed, status code: %d, body: %s", cur, res.StatusCode, body)
}
return nil
}

View File

@ -2,9 +2,7 @@ package _123_open
import ( import (
"context" "context"
"fmt"
"strconv" "strconv"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/errs"
@ -97,22 +95,6 @@ func (d *Open123) Rename(ctx context.Context, srcObj model.Obj, newName string)
} }
func (d *Open123) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { func (d *Open123) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
// 尝试使用上传+MD5秒传功能实现复制
// 1. 创建文件
// parentFileID 父目录id上传到根目录时填写 0
parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64)
if err != nil {
return fmt.Errorf("parse parentFileID error: %v", err)
}
etag := srcObj.(File).Etag
createResp, err := d.create(parentFileId, srcObj.GetName(), etag, srcObj.GetSize(), 2, false)
if err != nil {
return err
}
// 是否秒传
if createResp.Data.Reuse {
return nil
}
return errs.NotSupport return errs.NotSupport
} }
@ -122,64 +104,27 @@ func (d *Open123) Remove(ctx context.Context, obj model.Obj) error {
return d.trash(fileId) return d.trash(fileId)
} }
func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error {
// 1. 创建文件
// parentFileID 父目录id上传到根目录时填写 0
parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64)
if err != nil {
return nil, fmt.Errorf("parse parentFileID error: %v", err)
}
// etag 文件md5
etag := file.GetHash().GetHash(utils.MD5) etag := file.GetHash().GetHash(utils.MD5)
if len(etag) < utils.MD5.Width { if len(etag) < utils.MD5.Width {
_, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5) cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
up = model.UpdateProgressWithRange(up, 50, 100)
_, etag, err = stream.CacheFullInTempFileAndHash(file, cacheFileProgress, utils.MD5)
if err != nil { if err != nil {
return nil, err return err
} }
} }
createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false)
if err != nil { if err != nil {
return nil, err return err
} }
// 是否秒传
if createResp.Data.Reuse { if createResp.Data.Reuse {
// 秒传成功才会返回正确的 FileID否则为 0 return nil
if createResp.Data.FileID != 0 {
return File{
FileName: file.GetName(),
Size: file.GetSize(),
FileId: createResp.Data.FileID,
Type: 2,
Etag: etag,
}, nil
}
} }
// 2. 上传分片 return d.Upload(ctx, file, createResp, up)
err = d.Upload(ctx, file, createResp, up)
if err != nil {
return nil, err
}
// 3. 上传完毕
for range 60 {
uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID)
// 返回错误代码未知20103文档也没有具体说
if err == nil && uploadCompleteResp.Data.Completed && uploadCompleteResp.Data.FileID != 0 {
up(100)
return File{
FileName: file.GetName(),
Size: file.GetSize(),
FileId: uploadCompleteResp.Data.FileID,
Type: 2,
Etag: etag,
}, nil
}
// 若接口返回的completed为 false 时则需间隔1秒继续轮询此接口获取上传最终结果。
time.Sleep(time.Second)
}
return nil, fmt.Errorf("upload complete timeout")
} }
var _ driver.Driver = (*Open123)(nil) var _ driver.Driver = (*Open123)(nil)
var _ driver.PutResult = (*Open123)(nil)

View File

@ -73,9 +73,7 @@ func (f File) GetName() string {
} }
func (f File) CreateTime() time.Time { func (f File) CreateTime() time.Time {
// 返回的时间没有时区信息,默认 UTC+8 parsedTime, err := time.Parse("2006-01-02 15:04:05", f.CreateAt)
loc := time.FixedZone("UTC+8", 8*60*60)
parsedTime, err := time.ParseInLocation("2006-01-02 15:04:05", f.CreateAt, loc)
if err != nil { if err != nil {
return time.Now() return time.Now()
} }
@ -83,9 +81,7 @@ func (f File) CreateTime() time.Time {
} }
func (f File) ModTime() time.Time { func (f File) ModTime() time.Time {
// 返回的时间没有时区信息,默认 UTC+8 parsedTime, err := time.Parse("2006-01-02 15:04:05", f.UpdateAt)
loc := time.FixedZone("UTC+8", 8*60*60)
parsedTime, err := time.ParseInLocation("2006-01-02 15:04:05", f.UpdateAt, loc)
if err != nil { if err != nil {
return time.Now() return time.Now()
} }
@ -158,23 +154,52 @@ type DownloadInfoResp struct {
} `json:"data"` } `json:"data"`
} }
// 创建文件V2返回
type UploadCreateResp struct { type UploadCreateResp struct {
BaseResp BaseResp
Data struct { Data struct {
FileID int64 `json:"fileID"` FileID int64 `json:"fileID"`
PreuploadID string `json:"preuploadID"` PreuploadID string `json:"preuploadID"`
Reuse bool `json:"reuse"` Reuse bool `json:"reuse"`
SliceSize int64 `json:"sliceSize"` SliceSize int64 `json:"sliceSize"`
Servers []string `json:"servers"`
} `json:"data"` } `json:"data"`
} }
// 上传完毕V2返回 type UploadUrlResp struct {
BaseResp
Data struct {
PresignedURL string `json:"presignedURL"`
}
}
type UploadCompleteResp struct { type UploadCompleteResp struct {
BaseResp
Data struct {
Async bool `json:"async"`
Completed bool `json:"completed"`
FileID int64 `json:"fileID"`
} `json:"data"`
}
type UploadAsyncResp struct {
BaseResp BaseResp
Data struct { Data struct {
Completed bool `json:"completed"` Completed bool `json:"completed"`
FileID int64 `json:"fileID"` FileID int64 `json:"fileID"`
} `json:"data"` } `json:"data"`
} }
type UploadResp struct {
BaseResp
Data struct {
AccessKeyId string `json:"AccessKeyId"`
Bucket string `json:"Bucket"`
Key string `json:"Key"`
SecretAccessKey string `json:"SecretAccessKey"`
SessionToken string `json:"SessionToken"`
FileId int64 `json:"FileId"`
Reuse bool `json:"Reuse"`
EndPoint string `json:"EndPoint"`
StorageNode string `json:"StorageNode"`
UploadId string `json:"UploadId"`
} `json:"data"`
}

151
drivers/123_open/upload.go Normal file
View File

@ -0,0 +1,151 @@
package _123_open
import (
"context"
"net/http"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/errgroup"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/avast/retry-go"
"github.com/go-resty/resty/v2"
)
func (d *Open123) create(parentFileID int64, filename string, etag string, size int64, duplicate int, containDir bool) (*UploadCreateResp, error) {
var resp UploadCreateResp
_, err := d.Request(UploadCreate, http.MethodPost, func(req *resty.Request) {
req.SetBody(base.Json{
"parentFileId": parentFileID,
"filename": filename,
"etag": strings.ToLower(etag),
"size": size,
"duplicate": duplicate,
"containDir": containDir,
})
}, &resp)
if err != nil {
return nil, err
}
return &resp, nil
}
func (d *Open123) url(preuploadID string, sliceNo int64) (string, error) {
// get upload url
var resp UploadUrlResp
_, err := d.Request(UploadUrl, http.MethodPost, func(req *resty.Request) {
req.SetBody(base.Json{
"preuploadId": preuploadID,
"sliceNo": sliceNo,
})
}, &resp)
if err != nil {
return "", err
}
return resp.Data.PresignedURL, nil
}
func (d *Open123) complete(preuploadID string) (*UploadCompleteResp, error) {
var resp UploadCompleteResp
_, err := d.Request(UploadComplete, http.MethodPost, func(req *resty.Request) {
req.SetBody(base.Json{
"preuploadID": preuploadID,
})
}, &resp)
if err != nil {
return nil, err
}
return &resp, nil
}
func (d *Open123) async(preuploadID string) (*UploadAsyncResp, error) {
var resp UploadAsyncResp
_, err := d.Request(UploadAsync, http.MethodPost, func(req *resty.Request) {
req.SetBody(base.Json{
"preuploadID": preuploadID,
})
}, &resp)
if err != nil {
return nil, err
}
return &resp, nil
}
func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createResp *UploadCreateResp, up driver.UpdateProgress) error {
size := file.GetSize()
chunkSize := createResp.Data.SliceSize
uploadNums := (size + chunkSize - 1) / chunkSize
threadG, uploadCtx := errgroup.NewGroupWithContext(ctx, d.UploadThread,
retry.Attempts(3),
retry.Delay(time.Second),
retry.DelayType(retry.BackOffDelay))
for partIndex := int64(0); partIndex < uploadNums; partIndex++ {
if utils.IsCanceled(uploadCtx) {
return ctx.Err()
}
partIndex := partIndex
partNumber := partIndex + 1 // 分片号从1开始
offset := partIndex * chunkSize
size := min(chunkSize, size-offset)
limitedReader, err := file.RangeRead(http_range.Range{
Start: offset,
Length: size})
if err != nil {
return err
}
limitedReader = driver.NewLimitedUploadStream(ctx, limitedReader)
threadG.Go(func(ctx context.Context) error {
uploadPartUrl, err := d.url(createResp.Data.PreuploadID, partNumber)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, "PUT", uploadPartUrl, limitedReader)
if err != nil {
return err
}
req = req.WithContext(ctx)
req.ContentLength = size
res, err := base.HttpClient.Do(req)
if err != nil {
return err
}
_ = res.Body.Close()
progress := 10.0 + 85.0*float64(threadG.Success())/float64(uploadNums)
up(progress)
return nil
})
}
if err := threadG.Wait(); err != nil {
return err
}
uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID)
if err != nil {
return err
}
if uploadCompleteResp.Data.Async == false || uploadCompleteResp.Data.Completed {
return nil
}
for {
uploadAsyncResp, err := d.async(createResp.Data.PreuploadID)
if err != nil {
return err
}
if uploadAsyncResp.Data.Completed {
break
}
}
up(100)
return nil
}

View File

@ -19,14 +19,16 @@ var ( //不同情况下获取的AccessTokenQPS限制不同 如下模块化易于
AccessToken = InitApiInfo(Api+"/api/v1/access_token", 1) AccessToken = InitApiInfo(Api+"/api/v1/access_token", 1)
RefreshToken = InitApiInfo(Api+"/api/v1/oauth2/access_token", 1) RefreshToken = InitApiInfo(Api+"/api/v1/oauth2/access_token", 1)
UserInfo = InitApiInfo(Api+"/api/v1/user/info", 1) UserInfo = InitApiInfo(Api+"/api/v1/user/info", 1)
FileList = InitApiInfo(Api+"/api/v2/file/list", 3) FileList = InitApiInfo(Api+"/api/v2/file/list", 4)
DownloadInfo = InitApiInfo(Api+"/api/v1/file/download_info", 0) DownloadInfo = InitApiInfo(Api+"/api/v1/file/download_info", 0)
Mkdir = InitApiInfo(Api+"/upload/v1/file/mkdir", 2) Mkdir = InitApiInfo(Api+"/upload/v1/file/mkdir", 2)
Move = InitApiInfo(Api+"/api/v1/file/move", 1) Move = InitApiInfo(Api+"/api/v1/file/move", 1)
Rename = InitApiInfo(Api+"/api/v1/file/name", 1) Rename = InitApiInfo(Api+"/api/v1/file/name", 1)
Trash = InitApiInfo(Api+"/api/v1/file/trash", 2) Trash = InitApiInfo(Api+"/api/v1/file/trash", 2)
UploadCreate = InitApiInfo(Api+"/upload/v2/file/create", 2) UploadCreate = InitApiInfo(Api+"/upload/v1/file/create", 2)
UploadComplete = InitApiInfo(Api+"/upload/v2/file/upload_complete", 0) UploadUrl = InitApiInfo(Api+"/upload/v1/file/get_upload_url", 0)
UploadComplete = InitApiInfo(Api+"/upload/v1/file/upload_complete", 0)
UploadAsync = InitApiInfo(Api+"/upload/v1/file/upload_async_result", 1)
) )
func (d *Open123) Request(apiInfo *ApiInfo, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { func (d *Open123) Request(apiInfo *ApiInfo, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) {

View File

@ -70,6 +70,14 @@ func (d *Pan123Share) List(ctx context.Context, dir model.Obj, args model.ListAr
func (d *Pan123Share) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { func (d *Pan123Share) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
// TODO return link of file, required // TODO return link of file, required
if f, ok := file.(File); ok { if f, ok := file.(File); ok {
//var resp DownResp
var headers map[string]string
if !utils.IsLocalIPAddr(args.IP) {
headers = map[string]string{
//"X-Real-IP": "1.1.1.1",
"X-Forwarded-For": args.IP,
}
}
data := base.Json{ data := base.Json{
"shareKey": d.ShareKey, "shareKey": d.ShareKey,
"SharePwd": d.SharePwd, "SharePwd": d.SharePwd,
@ -79,27 +87,25 @@ func (d *Pan123Share) Link(ctx context.Context, file model.Obj, args model.LinkA
"size": f.Size, "size": f.Size,
} }
resp, err := d.request(DownloadInfo, http.MethodPost, func(req *resty.Request) { resp, err := d.request(DownloadInfo, http.MethodPost, func(req *resty.Request) {
req.SetBody(data) req.SetBody(data).SetHeaders(headers)
}, nil) }, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
downloadUrl := utils.Json.Get(resp, "data", "DownloadURL").ToString() downloadUrl := utils.Json.Get(resp, "data", "DownloadURL").ToString()
ou, err := url.Parse(downloadUrl) u, err := url.Parse(downloadUrl)
if err != nil { if err != nil {
return nil, err return nil, err
} }
u_ := ou.String() nu := u.Query().Get("params")
nu := ou.Query().Get("params")
if nu != "" { if nu != "" {
du, _ := base64.StdEncoding.DecodeString(nu) du, _ := base64.StdEncoding.DecodeString(nu)
u, err := url.Parse(string(du)) u, err = url.Parse(string(du))
if err != nil { if err != nil {
return nil, err return nil, err
} }
u_ = u.String()
} }
u_ := u.String()
log.Debug("download url: ", u_) log.Debug("download url: ", u_)
res, err := base.NoRedirectClient.R().SetHeader("Referer", "https://www.123pan.com/").Get(u_) res, err := base.NoRedirectClient.R().SetHeader("Referer", "https://www.123pan.com/").Get(u_)
if err != nil { if err != nil {
@ -116,7 +122,7 @@ func (d *Pan123Share) Link(ctx context.Context, file model.Obj, args model.LinkA
link.URL = utils.Json.Get(res.Body(), "data", "redirect_url").ToString() link.URL = utils.Json.Get(res.Body(), "data", "redirect_url").ToString()
} }
link.Header = http.Header{ link.Header = http.Header{
"Referer": []string{fmt.Sprintf("%s://%s/", ou.Scheme, ou.Host)}, "Referer": []string{"https://www.123pan.com/"},
} }
return &link, nil return &link, nil
} }

View File

@ -522,17 +522,21 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
var err error var err error
fullHash := stream.GetHash().GetHash(utils.SHA256) fullHash := stream.GetHash().GetHash(utils.SHA256)
if len(fullHash) != utils.SHA256.Width { if len(fullHash) != utils.SHA256.Width {
_, fullHash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA256) cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
up = model.UpdateProgressWithRange(up, 50, 100)
_, fullHash, err = streamPkg.CacheFullInTempFileAndHash(stream, cacheFileProgress, utils.SHA256)
if err != nil { if err != nil {
return err return err
} }
} }
size := stream.GetSize() size := stream.GetSize()
partSize := d.getPartSize(size) var partSize = d.getPartSize(size)
part := int64(1) part := size / partSize
if size > partSize { if size%partSize > 0 {
part = (size + partSize - 1) / partSize part++
} else if part == 0 {
part = 1
} }
partInfos := make([]PartInfo, 0, part) partInfos := make([]PartInfo, 0, part)
for i := int64(0); i < part; i++ { for i := int64(0); i < part; i++ {
@ -634,10 +638,11 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
// Update Progress // Update Progress
r := io.TeeReader(limitReader, p) r := io.TeeReader(limitReader, p)
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadPartInfo.UploadUrl, r) req, err := http.NewRequest("PUT", uploadPartInfo.UploadUrl, r)
if err != nil { if err != nil {
return err return err
} }
req = req.WithContext(ctx)
req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Content-Length", fmt.Sprint(partSize)) req.Header.Set("Content-Length", fmt.Sprint(partSize))
req.Header.Set("Origin", "https://yun.139.com") req.Header.Set("Origin", "https://yun.139.com")
@ -783,10 +788,12 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
size := stream.GetSize() size := stream.GetSize()
// Progress // Progress
p := driver.NewProgress(size, up) p := driver.NewProgress(size, up)
partSize := d.getPartSize(size) var partSize = d.getPartSize(size)
part := int64(1) part := size / partSize
if size > partSize { if size%partSize > 0 {
part = (size + partSize - 1) / partSize part++
} else if part == 0 {
part = 1
} }
rateLimited := driver.NewLimitedUploadStream(ctx, stream) rateLimited := driver.NewLimitedUploadStream(ctx, stream)
for i := int64(0); i < part; i++ { for i := int64(0); i < part; i++ {
@ -800,10 +807,12 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
limitReader := io.LimitReader(rateLimited, byteSize) limitReader := io.LimitReader(rateLimited, byteSize)
// Update Progress // Update Progress
r := io.TeeReader(limitReader, p) r := io.TeeReader(limitReader, p)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, resp.Data.UploadResult.RedirectionURL, r) req, err := http.NewRequest("POST", resp.Data.UploadResult.RedirectionURL, r)
if err != nil { if err != nil {
return err return err
} }
req = req.WithContext(ctx)
req.Header.Set("Content-Type", "text/plain;name="+unicode(stream.GetName())) req.Header.Set("Content-Type", "text/plain;name="+unicode(stream.GetName()))
req.Header.Set("contentSize", strconv.FormatInt(size, 10)) req.Header.Set("contentSize", strconv.FormatInt(size, 10))
req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1)) req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1))

View File

@ -365,10 +365,11 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F
log.Debugf("uploadData: %+v", uploadData) log.Debugf("uploadData: %+v", uploadData)
requestURL := uploadData.RequestURL requestURL := uploadData.RequestURL
uploadHeaders := strings.Split(decodeURIComponent(uploadData.RequestHeader), "&") uploadHeaders := strings.Split(decodeURIComponent(uploadData.RequestHeader), "&")
req, err := http.NewRequestWithContext(ctx, http.MethodPut, requestURL, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) req, err := http.NewRequest(http.MethodPut, requestURL, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData)))
if err != nil { if err != nil {
return err return err
} }
req = req.WithContext(ctx)
for _, v := range uploadHeaders { for _, v := range uploadHeaders {
i := strings.Index(v, "=") i := strings.Index(v, "=")
req.Header.Set(v[0:i], v[i+1:]) req.Header.Set(v[0:i], v[i+1:])

View File

@ -5,19 +5,17 @@ import (
"encoding/base64" "encoding/base64"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"github.com/skip2/go-qrcode"
"io" "io"
"net/http" "net/http"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"github.com/skip2/go-qrcode"
"github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/go-resty/resty/v2" "github.com/go-resty/resty/v2"
@ -313,14 +311,11 @@ func (y *Cloud189TV) RapidUpload(ctx context.Context, dstDir model.Obj, stream m
// 旧版本上传,家庭云不支持覆盖 // 旧版本上传,家庭云不支持覆盖
func (y *Cloud189TV) OldUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { func (y *Cloud189TV) OldUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) {
fileMd5 := file.GetHash().GetHash(utils.MD5) tempFile, err := file.CacheFullInTempFile()
var tempFile = file.GetFile() if err != nil {
var err error return nil, err
if len(fileMd5) != utils.MD5.Width {
tempFile, fileMd5, err = stream.CacheFullAndHash(file, &up, utils.MD5)
} else if tempFile == nil {
tempFile, err = file.CacheFullAndWriter(&up, nil)
} }
fileMd5, err := utils.HashFile(utils.MD5, tempFile)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -350,7 +345,7 @@ func (y *Cloud189TV) OldUpload(ctx context.Context, dstDir model.Obj, file model
header["Edrive-UploadFileId"] = fmt.Sprint(status.UploadFileId) header["Edrive-UploadFileId"] = fmt.Sprint(status.UploadFileId)
} }
_, err := y.put(ctx, status.FileUploadUrl, header, true, tempFile, isFamily) _, err := y.put(ctx, status.FileUploadUrl, header, true, io.NopCloser(tempFile), isFamily)
if err, ok := err.(*RespErr); ok && err.Code != "InputStreamReadError" { if err, ok := err.(*RespErr); ok && err.Code != "InputStreamReadError" {
return nil, err return nil, err
} }

View File

@ -7,7 +7,6 @@ import (
"encoding/hex" "encoding/hex"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"hash"
"io" "io"
"net/http" "net/http"
"net/http/cookiejar" "net/http/cookiejar"
@ -473,7 +472,7 @@ func (y *Cloud189PC) refreshSession() (err error) {
// 无法上传大小为0的文件 // 无法上传大小为0的文件
func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) {
size := file.GetSize() size := file.GetSize()
sliceSize := min(size, partSize(size)) sliceSize := partSize(size)
params := Params{ params := Params{
"parentFolderId": dstDir.GetID(), "parentFolderId": dstDir.GetID(),
@ -501,99 +500,64 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo
return nil, err return nil, err
} }
ss, err := stream.NewStreamSectionReader(file, int(sliceSize), &up) threadG, upCtx := errgroup.NewGroupWithContext(ctx, y.uploadThread,
if err != nil {
return nil, err
}
threadG, upCtx := errgroup.NewOrderedGroupWithContext(ctx, y.uploadThread,
retry.Attempts(3), retry.Attempts(3),
retry.Delay(time.Second), retry.Delay(time.Second),
retry.DelayType(retry.BackOffDelay)) retry.DelayType(retry.BackOffDelay))
count := 1 count := int(size / sliceSize)
if size > sliceSize {
count = int((size + sliceSize - 1) / sliceSize)
}
lastPartSize := size % sliceSize lastPartSize := size % sliceSize
if lastPartSize == 0 { if lastPartSize > 0 {
count++
} else {
lastPartSize = sliceSize lastPartSize = sliceSize
} }
fileMd5 := utils.MD5.NewFunc()
silceMd5Hexs := make([]string, 0, count)
silceMd5 := utils.MD5.NewFunc() silceMd5 := utils.MD5.NewFunc()
var writers io.Writer = silceMd5 silceMd5Hexs := make([]string, 0, count)
teeReader := io.TeeReader(file, io.MultiWriter(fileMd5, silceMd5))
fileMd5Hex := file.GetHash().GetHash(utils.MD5) byteSize := sliceSize
var fileMd5 hash.Hash
if len(fileMd5Hex) != utils.MD5.Width {
fileMd5 = utils.MD5.NewFunc()
writers = io.MultiWriter(silceMd5, fileMd5)
}
for i := 1; i <= count; i++ { for i := 1; i <= count; i++ {
if utils.IsCanceled(upCtx) { if utils.IsCanceled(upCtx) {
break break
} }
offset := int64((i)-1) * sliceSize
size := sliceSize
if i == count { if i == count {
size = lastPartSize byteSize = lastPartSize
}
byteData := make([]byte, byteSize)
// 读取块
silceMd5.Reset()
if _, err := io.ReadFull(teeReader, byteData); err != io.EOF && err != nil {
return nil, err
} }
partInfo := ""
var reader *stream.SectionReader
var rateLimitedRd io.Reader
threadG.GoWithLifecycle(errgroup.Lifecycle{
Before: func(ctx context.Context) error {
if reader == nil {
var err error
reader, err = ss.GetSectionReader(offset, size)
if err != nil {
return err
}
silceMd5.Reset()
w, err := utils.CopyWithBuffer(writers, reader)
if w != size {
return fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", size, w, err)
}
// 计算块md5并进行hex和base64编码
md5Bytes := silceMd5.Sum(nil)
silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Bytes)))
partInfo = fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes))
rateLimitedRd = driver.NewLimitedUploadStream(ctx, reader) // 计算块md5并进行hex和base64编码
} md5Bytes := silceMd5.Sum(nil)
return nil silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Bytes)))
}, partInfo := fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes))
Do: func(ctx context.Context) error {
reader.Seek(0, io.SeekStart)
uploadUrls, err := y.GetMultiUploadUrls(ctx, isFamily, initMultiUpload.Data.UploadFileID, partInfo)
if err != nil {
return err
}
// step.4 上传切片 threadG.Go(func(ctx context.Context) error {
uploadUrl := uploadUrls[0] uploadUrls, err := y.GetMultiUploadUrls(ctx, isFamily, initMultiUpload.Data.UploadFileID, partInfo)
_, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, if err != nil {
driver.NewLimitedUploadStream(ctx, rateLimitedRd), isFamily) return err
if err != nil { }
return err
} // step.4 上传切片
up(float64(threadG.Success()) * 100 / float64(count)) uploadUrl := uploadUrls[0]
return nil _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false,
}, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData)), isFamily)
After: func(err error) { if err != nil {
ss.FreeSectionReader(reader) return err
}, }
}, up(float64(threadG.Success()) * 100 / float64(count))
) return nil
})
} }
if err = threadG.Wait(); err != nil { if err = threadG.Wait(); err != nil {
return nil, err return nil, err
} }
if fileMd5 != nil { fileMd5Hex := strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil)))
fileMd5Hex = strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil)))
}
sliceMd5Hex := fileMd5Hex sliceMd5Hex := fileMd5Hex
if file.GetSize() > sliceSize { if file.GetSize() > sliceSize {
sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(silceMd5Hexs, "\n"))) sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(silceMd5Hexs, "\n")))
@ -656,12 +620,11 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode
cache = tmpF cache = tmpF
} }
sliceSize := partSize(size) sliceSize := partSize(size)
count := 1 count := int(size / sliceSize)
if size > sliceSize {
count = int((size + sliceSize - 1) / sliceSize)
}
lastSliceSize := size % sliceSize lastSliceSize := size % sliceSize
if lastSliceSize == 0 { if lastSliceSize > 0 {
count++
} else {
lastSliceSize = sliceSize lastSliceSize = sliceSize
} }
@ -775,8 +738,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode
} }
// step.4 上传切片 // step.4 上传切片
rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize)) _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, io.NewSectionReader(cache, offset, byteSize), isFamily)
_, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, rateLimitedRd, isFamily)
if err != nil { if err != nil {
return err return err
} }
@ -858,7 +820,9 @@ func (y *Cloud189PC) GetMultiUploadUrls(ctx context.Context, isFamily bool, uplo
// 旧版本上传,家庭云不支持覆盖 // 旧版本上传,家庭云不支持覆盖
func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) {
tempFile, fileMd5, err := stream.CacheFullAndHash(file, &up, utils.MD5) cacheFileProgress := model.UpdateProgressWithRange(up, 0, 50)
up = model.UpdateProgressWithRange(up, 50, 100)
tempFile, fileMd5, err := stream.CacheFullInTempFileAndHash(file, cacheFileProgress, utils.MD5)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -5,7 +5,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"net/url"
stdpath "path" stdpath "path"
"strings" "strings"
@ -13,7 +12,6 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/sign"
"github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils"
@ -80,18 +78,10 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) {
return nil, errs.ObjectNotFound return nil, errs.ObjectNotFound
} }
for _, dst := range dsts { for _, dst := range dsts {
obj, err := fs.Get(ctx, stdpath.Join(dst, sub), &fs.GetArgs{NoLog: true}) obj, err := d.get(ctx, path, dst, sub)
if err != nil { if err == nil {
continue return obj, nil
} }
return &model.Object{
Path: path,
Name: obj.GetName(),
Size: obj.GetSize(),
Modified: obj.ModTime(),
IsFolder: obj.IsDir(),
HashInfo: obj.GetHash(),
}, nil
} }
return nil, errs.ObjectNotFound return nil, errs.ObjectNotFound
} }
@ -109,27 +99,7 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([
var objs []model.Obj var objs []model.Obj
fsArgs := &fs.ListArgs{NoLog: true, Refresh: args.Refresh} fsArgs := &fs.ListArgs{NoLog: true, Refresh: args.Refresh}
for _, dst := range dsts { for _, dst := range dsts {
tmp, err := fs.List(ctx, stdpath.Join(dst, sub), fsArgs) tmp, err := d.list(ctx, dst, sub, fsArgs)
if err == nil {
tmp, err = utils.SliceConvert(tmp, func(obj model.Obj) (model.Obj, error) {
thumb, ok := model.GetThumb(obj)
objRes := model.Object{
Name: obj.GetName(),
Size: obj.GetSize(),
Modified: obj.ModTime(),
IsFolder: obj.IsDir(),
}
if !ok {
return &objRes, nil
}
return &model.ObjThumb{
Object: objRes,
Thumbnail: model.Thumbnail{
Thumbnail: thumb,
},
}, nil
})
}
if err == nil { if err == nil {
objs = append(objs, tmp...) objs = append(objs, tmp...)
} }
@ -143,45 +113,45 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
if !ok { if !ok {
return nil, errs.ObjectNotFound return nil, errs.ObjectNotFound
} }
// proxy || ftp,s3
if common.GetApiUrl(ctx) == "" {
args.Redirect = false
}
for _, dst := range dsts { for _, dst := range dsts {
reqPath := stdpath.Join(dst, sub) reqPath := stdpath.Join(dst, sub)
link, fi, err := d.link(ctx, reqPath, args) link, file, err := d.link(ctx, reqPath, args)
if err != nil { if err != nil {
continue continue
} }
if link == nil { var resultLink *model.Link
// 重定向且需要通过代理 if link != nil {
return &model.Link{ resultLink = &model.Link{
URL: link.URL,
Header: link.Header,
RangeReader: link.RangeReader,
SyncClosers: utils.NewSyncClosers(link),
ContentLength: link.ContentLength,
}
if link.MFile != nil {
resultLink.RangeReader = &model.FileRangeReader{
RangeReaderIF: stream.GetRangeReaderFromMFile(file.GetSize(), link.MFile),
}
}
} else {
resultLink = &model.Link{
URL: fmt.Sprintf("%s/p%s?sign=%s", URL: fmt.Sprintf("%s/p%s?sign=%s",
common.GetApiUrl(ctx), common.GetApiUrl(ctx),
utils.EncodePath(reqPath, true), utils.EncodePath(reqPath, true),
sign.Sign(reqPath)), sign.Sign(reqPath)),
}, nil }
}
resultLink := *link
resultLink.SyncClosers = utils.NewSyncClosers(link)
if args.Redirect {
return &resultLink, nil
} }
if !args.Redirect {
if resultLink.ContentLength == 0 { if d.DownloadConcurrency > 0 {
resultLink.ContentLength = fi.GetSize() resultLink.Concurrency = d.DownloadConcurrency
}
if d.DownloadPartSize > 0 {
resultLink.PartSize = d.DownloadPartSize * utils.KB
}
} }
if resultLink.MFile != nil { return resultLink, nil
return &resultLink, nil
}
if d.DownloadConcurrency > 0 {
resultLink.Concurrency = d.DownloadConcurrency
}
if d.DownloadPartSize > 0 {
resultLink.PartSize = d.DownloadPartSize * utils.KB
}
return &resultLink, nil
} }
return nil, errs.ObjectNotFound return nil, errs.ObjectNotFound
} }
@ -308,29 +278,24 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer,
reqPath, err := d.getReqPath(ctx, dstDir, true) reqPath, err := d.getReqPath(ctx, dstDir, true)
if err == nil { if err == nil {
if len(reqPath) == 1 { if len(reqPath) == 1 {
storage, reqActualPath, err := op.GetStorageAndActualPath(*reqPath[0]) return fs.PutDirectly(ctx, *reqPath[0], &stream.FileStream{
if err != nil { Obj: s,
return err Mimetype: s.GetMimetype(),
} WebPutAsTask: s.NeedStore(),
return op.Put(ctx, storage, reqActualPath, &stream.FileStream{ Reader: s,
Obj: s, })
Mimetype: s.GetMimetype(),
Reader: s,
}, up)
} else { } else {
file, err := s.CacheFullAndWriter(nil, nil) file, err := s.CacheFullInTempFile()
if err != nil { if err != nil {
return err return err
} }
count := float64(len(reqPath) + 1) for _, path := range reqPath {
up(100 / count)
for i, path := range reqPath {
err = errors.Join(err, fs.PutDirectly(ctx, *path, &stream.FileStream{ err = errors.Join(err, fs.PutDirectly(ctx, *path, &stream.FileStream{
Obj: s, Obj: s,
Mimetype: s.GetMimetype(), Mimetype: s.GetMimetype(),
Reader: file, WebPutAsTask: s.NeedStore(),
Reader: file,
})) }))
up(float64(i+2) / float64(count) * 100)
_, e := file.Seek(0, io.SeekStart) _, e := file.Seek(0, io.SeekStart)
if e != nil { if e != nil {
return errors.Join(err, e) return errors.Join(err, e)
@ -402,24 +367,10 @@ func (d *Alias) Extract(ctx context.Context, obj model.Obj, args model.ArchiveIn
return nil, errs.ObjectNotFound return nil, errs.ObjectNotFound
} }
for _, dst := range dsts { for _, dst := range dsts {
reqPath := stdpath.Join(dst, sub) link, err := d.extract(ctx, dst, sub, args)
link, err := d.extract(ctx, reqPath, args) if err == nil {
if err != nil { return link, nil
continue
} }
if link == nil {
return &model.Link{
URL: fmt.Sprintf("%s/ap%s?inner=%s&pass=%s&sign=%s",
common.GetApiUrl(ctx),
utils.EncodePath(reqPath, true),
utils.EncodePath(args.InnerPath, true),
url.QueryEscape(args.Password),
sign.SignArchive(reqPath)),
}, nil
}
resultLink := *link
resultLink.SyncClosers = utils.NewSyncClosers(link)
return &resultLink, nil
} }
return nil, errs.NotImplement return nil, errs.NotImplement
} }

View File

@ -2,6 +2,8 @@ package alias
import ( import (
"context" "context"
"fmt"
"net/url"
stdpath "path" stdpath "path"
"strings" "strings"
@ -10,6 +12,8 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/fs"
"github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/sign"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/common"
) )
@ -50,12 +54,55 @@ func (d *Alias) getRootAndPath(path string) (string, string) {
return parts[0], parts[1] return parts[0], parts[1]
} }
func (d *Alias) get(ctx context.Context, path string, dst, sub string) (model.Obj, error) {
obj, err := fs.Get(ctx, stdpath.Join(dst, sub), &fs.GetArgs{NoLog: true})
if err != nil {
return nil, err
}
return &model.Object{
Path: path,
Name: obj.GetName(),
Size: obj.GetSize(),
Modified: obj.ModTime(),
IsFolder: obj.IsDir(),
HashInfo: obj.GetHash(),
}, nil
}
func (d *Alias) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([]model.Obj, error) {
objs, err := fs.List(ctx, stdpath.Join(dst, sub), args)
// the obj must implement the model.SetPath interface
// return objs, err
if err != nil {
return nil, err
}
return utils.SliceConvert(objs, func(obj model.Obj) (model.Obj, error) {
thumb, ok := model.GetThumb(obj)
objRes := model.Object{
Name: obj.GetName(),
Size: obj.GetSize(),
Modified: obj.ModTime(),
IsFolder: obj.IsDir(),
}
if !ok {
return &objRes, nil
}
return &model.ObjThumb{
Object: objRes,
Thumbnail: model.Thumbnail{
Thumbnail: thumb,
},
}, nil
})
}
func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) (*model.Link, model.Obj, error) { func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) (*model.Link, model.Obj, error) {
storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
if !args.Redirect { // proxy || ftp,s3
if !args.Redirect || len(common.GetApiUrl(ctx)) == 0 {
return op.Link(ctx, storage, reqActualPath, args) return op.Link(ctx, storage, reqActualPath, args)
} }
obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true})
@ -136,7 +183,8 @@ func (d *Alias) listArchive(ctx context.Context, dst, sub string, args model.Arc
return nil, errs.NotImplement return nil, errs.NotImplement
} }
func (d *Alias) extract(ctx context.Context, reqPath string, args model.ArchiveInnerArgs) (*model.Link, error) { func (d *Alias) extract(ctx context.Context, dst, sub string, args model.ArchiveInnerArgs) (*model.Link, error) {
reqPath := stdpath.Join(dst, sub)
storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath)
if err != nil { if err != nil {
return nil, err return nil, err
@ -144,12 +192,20 @@ func (d *Alias) extract(ctx context.Context, reqPath string, args model.ArchiveI
if _, ok := storage.(driver.ArchiveReader); !ok { if _, ok := storage.(driver.ArchiveReader); !ok {
return nil, errs.NotImplement return nil, errs.NotImplement
} }
if args.Redirect && common.ShouldProxy(storage, stdpath.Base(reqPath)) { if args.Redirect && common.ShouldProxy(storage, stdpath.Base(sub)) {
_, err := fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) _, err = fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true})
if err == nil { if err != nil {
return nil, err return nil, err
} }
return nil, nil link := &model.Link{
URL: fmt.Sprintf("%s/ap%s?inner=%s&pass=%s&sign=%s",
common.GetApiUrl(ctx),
utils.EncodePath(reqPath, true),
utils.EncodePath(args.InnerPath, true),
url.QueryEscape(args.Password),
sign.SignArchive(reqPath)),
}
return link, nil
} }
link, _, err := op.DriverExtract(ctx, storage, reqActualPath, args) link, _, err := op.DriverExtract(ctx, storage, reqActualPath, args)
return link, err return link, err

Some files were not shown because too many files have changed in this diff Show More