From 9fd81e5ba4c000f0606d22e45e5842a441f02cb2 Mon Sep 17 00:00:00 2001 From: pikachuren Date: Tue, 10 Nov 2026 20:59:28 +0800 Subject: [PATCH] feat(upload): add optional force-overwrite before upload Drivers without overwrite semantics (Aliyun Drive via WebDAV, issue #100) silently create `xxx_(1).ext` copies instead of replacing the file. Add a global setting `upload_force_overwrite` (default off). When enabled, every upload first checks whether the target exists and deletes it, then writes: - /fs/put and /fs/form delete after the body has passed the size limit, so a rejected upload never destroys the existing file. - /fs/multipart/init only cleans up when a new session is created; resumed sessions are left alone. - WebDAV PUT and S3 PutObject follow the same order. - /fs/get_direct_upload_info deletes before signing the direct upload URL, but an explicit `Overwrite: false` header still wins. Directories are never deleted (uploading a file over a folder now fails with a readable error). Delete errors that mean "already gone" are ignored; other delete failures abort the upload instead of silently producing a copy. The setting's help text states that it costs one existence check and, when the file exists, one extra delete request per upload. --- src/backend/internal/model/db.ts | 17 +++ src/backend/internal/op/upload.ts | 103 ++++++++++++++++ src/backend/server/fs.ts | 20 +++ src/backend/server/s3.ts | 3 + src/backend/server/upload_overwrite.test.ts | 128 ++++++++++++++++++++ src/backend/server/webdav.ts | 4 + 6 files changed, 275 insertions(+) create mode 100644 src/backend/internal/op/upload.ts create mode 100644 src/backend/server/upload_overwrite.test.ts diff --git a/src/backend/internal/model/db.ts b/src/backend/internal/model/db.ts index e07a9c36..92e208cf 100644 --- a/src/backend/internal/model/db.ts +++ b/src/backend/internal/model/db.ts @@ -1007,6 +1007,23 @@ export const defaultDb = { group: 12, flag: 1, }, + // ── 上传行为 ── + // Issue #100：阿里云盘经 WebDAV 上传同名文件时会生成 xxx_(1) 副本而不是 + // 覆盖。驱动接口本身没有 overwrite 语义，只能在上传前主动删除旧文件。 + // 代价是每次上传多一次存在性检查（命中时再多一次删除），故默认关闭。 + { + key: "upload_force_overwrite", + value: "false", + type: "bool", + help: + "Before every upload, check whether the target file exists and delete it first. " + + "This makes uploads truly overwrite files on drivers that would otherwise create " + + "numbered copies (e.g. Aliyun Drive via WebDAV), but it adds one existence check " + + "and, when the file exists, one extra delete request per upload. " + + "Leave it off unless you need overwrite semantics.", + group: 4, + flag: 1, + }, ], storages: [], users: [ diff --git a/src/backend/internal/op/upload.ts b/src/backend/internal/op/upload.ts new file mode 100644 index 00000000..ad9bc409 --- /dev/null +++ b/src/backend/internal/op/upload.ts @@ -0,0 +1,103 @@ +/** + * 上传前「强制覆盖」处理。 + * + * 背景（Issue #100）：部分网盘驱动在上传同名文件时不会覆盖，而是自动生成 + * `xxx_(1).ext` 之类的副本（阿里云盘经 WebDAV 上传即如此）。驱动的上传接口 + * 本身不提供 overwrite 语义，因此只能在写入前主动删除旧文件。 + * + * 代价：开启后每次上传多 1 次存在性检查（文件存在时再多 1 次删除请求）， + * 因此默认关闭，并在设置项说明里明确提示。 + * + * 为什么放在上传入口而不是 `putItem` 内部： + * - 解压、种子转存等内部写路径会批量调用 `putItem`，逐个检查会成倍放大上游 + * 调用次数，且那些场景不需要「先删后写」； + * - 分片上传不走 `putItem`，需要单独在 init 阶段处理。 + */ +import { getSettings } from "../model/db" +import { getItem, removeItems } from "./storage" +import type { StorageRequestContext } from "./storage" + +const NOT_FOUND_RE = /not\s*found|no such|does not exist|不存在|404/i + +/** 判断错误是否表示「目标不存在」：这类错误在存在性检查里是正常结果 */ +export function isNotFoundError(err: any): boolean { + return NOT_FOUND_RE.test(String(err?.message || err || "")) +} + +/** + * 读取设置开关。 + * + * 读取失败时返回 false（保持旧行为），不让一次配置读取抖动导致所有上传失败。 + */ +export async function isForceOverwriteUploadEnabled(): Promise { + try { + const settings = await getSettings() + const value = String(settings.upload_force_overwrite ?? "") + .trim() + .toLowerCase() + return value === "true" || value === "1" + } catch (err) { + console.warn("[Upload] failed to read upload_force_overwrite setting:", err) + return false + } +} + +export interface UploadOverwriteDeps { + isEnabled?: () => Promise + getItemFn?: typeof getItem + removeItemsFn?: typeof removeItems +} + +/** + * 上传前准备：开启设置时，若目标文件已存在则先删除。 + * + * 行为约定： + * - 目标不存在：什么都不做（`getItem` 抛错是最常见路径）； + * - 目标是目录：直接报错，绝不删除目录（用文件覆盖目录没有合理语义）； + * - 删除失败且错误是「不存在」：视为已被并发删除，继续上传； + * - 其它删除失败：抛出可读错误，避免静默地又生成一个 `xxx_(1)` 副本。 + * + * 注意：调用方必须在**请求体已通过大小校验之后**、真正写入之前调用， + * 否则「上传被拒绝但旧文件已删」会造成数据丢失。 + */ +export async function prepareUploadTarget( + virtualPath: string, + requestContext?: StorageRequestContext, + deps: UploadOverwriteDeps = {}, +): Promise { + const enabled = await (deps.isEnabled || isForceOverwriteUploadEnabled)() + if (!enabled) return + + const path = String(virtualPath || "") + const segments = path.split("/").filter(Boolean) + const name = segments.pop() || "" + if (!name) return + const dir = `/${segments.join("/")}` + + const getItemFn = deps.getItemFn || getItem + const removeItemsFn = deps.removeItemsFn || removeItems + + let existing: any + try { + const result = await getItemFn(path, requestContext) + existing = result?.item + } catch { + // 不存在 / 无法读取：交给后续上传按原行为处理 + return + } + if (!existing) return + if (existing.is_dir) { + throw new Error( + `cannot upload over a directory: "${path}" already exists as a folder`, + ) + } + + try { + await removeItemsFn(dir, [name], requestContext) + } catch (err: any) { + if (isNotFoundError(err)) return + throw new Error( + `failed to delete the existing file before upload: ${err?.message || err}`, + ) + } +} diff --git a/src/backend/server/fs.ts b/src/backend/server/fs.ts index be7c751e..e310d6e8 100644 --- a/src/backend/server/fs.ts +++ b/src/backend/server/fs.ts @@ -13,6 +13,7 @@ import { } from "../internal/op/storage" import { resolveShare } from "../internal/op/share" import { resolvePath } from "../internal/model/db" +import { prepareUploadTarget } from "../internal/op/upload" import { getUserFromContext } from "./middlewares" import { can, @@ -978,6 +979,8 @@ fsRouter.put("/put", async (c) => { try { const body = await readBodyWithinLimit(c) if (!body.ok) return tooLargeResponse(c, body.limit) + // 先校验请求体再删旧文件：避免「请求被拒但原文件已丢」 + await prepareUploadTarget(reqPath, requestContext) await putItem(reqPath, body.buffer, requestContext) return c.json({ code: 200, message: "success", data: null }) } catch (e: any) { @@ -1018,6 +1021,8 @@ fsRouter.put("/form", async (c) => { // multipart 的 Content-Length 只覆盖整个表单，仍需按文件实际字节复核 const max = getUploadSizeLimit(c, false) if (buffer.length > max) return tooLargeResponse(c, max) + // 先校验体积再删旧文件：避免「请求被拒但原文件已丢」 + await prepareUploadTarget(reqPath, requestContext) await putItem(reqPath, buffer, requestContext) return c.json({ code: 200, message: "success", data: null }) } catch (e: any) { @@ -1392,6 +1397,15 @@ fsRouter.post("/get_direct_upload_info", async (c) => { } catch { // 目标不存在 / 无法读取：继续签发流程 } + } else { + // 全局「强制覆盖」：客户端直传时 Worker 不经手数据，只能在签发前删旧文件， + // 否则驱动会生成 xxx_(1) 副本（Issue #100）。 + // 显式 Overwrite:false 优先，保持「拒绝覆盖」的语义。 + try { + await prepareUploadTarget(dstPath, getStorageRequestContext(c)) + } catch (e: any) { + return c.json({ code: 500, message: safeErrorMessage(e), data: null }, 500) + } } try { const resolved = await resolvePath(dirPath) @@ -1498,6 +1512,12 @@ fsRouter.post("/multipart/init", async (c) => { session = existing resumed = true } else { + // 只在新会话时清理旧文件：续传会话的目标可能已经在首次 init 时删过， + // 再次删除会破坏正在进行的上传。 + await prepareUploadTarget( + `${actualDir}/${name}`.replace(/\/{2,}/g, "/"), + requestContext, + ) const info = await (driver as any).createUploadSession( actualDir, resolved.physical!, diff --git a/src/backend/server/s3.ts b/src/backend/server/s3.ts index d85e2f60..a122ba0f 100644 --- a/src/backend/server/s3.ts +++ b/src/backend/server/s3.ts @@ -6,6 +6,7 @@ import { putItem, removeItems, } from "../internal/op/storage" +import { prepareUploadTarget } from "../internal/op/upload" import { safeErrorMessage } from "../pkg/errs" import { appendDownloadSignToRawUrl } from "../pkg/sign" import { readBodyWithinLimit } from "../pkg/upload-limit" @@ -298,6 +299,8 @@ s3Router.put("/*", async (c) => { 400, ) } + // 体积校验通过后再清理旧文件（Issue #100：驱动会生成 xxx_(1) 副本） + await prepareUploadTarget(virtualPath, getCtx(c)) await putItem(virtualPath, body.buffer, getCtx(c)) // 不再返回伪造的 ETag。 // diff --git a/src/backend/server/upload_overwrite.test.ts b/src/backend/server/upload_overwrite.test.ts new file mode 100644 index 00000000..6680c512 --- /dev/null +++ b/src/backend/server/upload_overwrite.test.ts @@ -0,0 +1,128 @@ +import assert from "node:assert/strict" +import { test } from "node:test" + +import { + isNotFoundError, + prepareUploadTarget, + type UploadOverwriteDeps, +} from "../internal/op/upload" + +function makeDeps(options: { + enabled: boolean + existing?: { is_dir?: boolean; size?: number } | null + getError?: Error + removeError?: Error +}) { + const calls = { + get: [] as string[], + remove: [] as Array<{ dir: string; name: string }>, + } + const deps: UploadOverwriteDeps = { + isEnabled: async () => options.enabled, + getItemFn: (async (path: string) => { + calls.get.push(path) + if (options.getError) throw options.getError + return { + item: { + name: String(path).split("/").filter(Boolean).pop() || "", + size: options.existing?.size ?? 1, + is_dir: Boolean(options.existing?.is_dir), + modified: new Date().toISOString(), + sign: "", + type: 0, + }, + provider: "fake", + rawUrl: "", + } + }) as any, + removeItemsFn: (async (dir: string, names: string[]) => { + calls.remove.push({ dir, name: names[0] }) + if (options.removeError) throw options.removeError + }) as any, + } + return { deps, calls } +} + +test("does nothing when the setting is off", async () => { + const { deps, calls } = makeDeps({ enabled: false, existing: {} }) + await prepareUploadTarget("/dir/file.txt", undefined, deps) + assert.deepEqual(calls.get, []) + assert.deepEqual(calls.remove, []) +}) + +test("does nothing when the target does not exist", async () => { + const { deps, calls } = makeDeps({ + enabled: true, + getError: new Error("[Fake] Path 'file.txt' not found"), + }) + await prepareUploadTarget("/dir/file.txt", undefined, deps) + assert.equal(calls.get.length, 1) + assert.equal(calls.remove.length, 0) +}) + +test("deletes the existing file before uploading", async () => { + const { deps, calls } = makeDeps({ enabled: true, existing: { size: 10 } }) + await prepareUploadTarget("/dir/sub/file.txt", undefined, deps) + assert.deepEqual(calls.remove, [{ dir: "/dir/sub", name: "file.txt" }]) +}) + +test("root-level targets are deleted in the root directory", async () => { + const { deps, calls } = makeDeps({ enabled: true, existing: {} }) + await prepareUploadTarget("/file.txt", undefined, deps) + assert.deepEqual(calls.remove, [{ dir: "/", name: "file.txt" }]) +}) + +test("refuses to replace a directory with a file", async () => { + const { deps, calls } = makeDeps({ + enabled: true, + existing: { is_dir: true }, + }) + await assert.rejects( + () => prepareUploadTarget("/dir", undefined, deps), + /directory/i, + ) + assert.equal(calls.remove.length, 0, "must never delete a directory") +}) + +test("a concurrent delete (not found) is not an upload failure", async () => { + const { deps } = makeDeps({ + enabled: true, + existing: {}, + removeError: new Error("object not found"), + }) + await assert.doesNotReject(() => + prepareUploadTarget("/dir/file.txt", undefined, deps), + ) +}) + +test("other delete failures surface a readable error", async () => { + const { deps } = makeDeps({ + enabled: true, + existing: {}, + removeError: new Error("upstream 503"), + }) + await assert.rejects( + () => prepareUploadTarget("/dir/file.txt", undefined, deps), + /failed to delete the existing file before upload: upstream 503/, + ) +}) + +test("isNotFoundError only matches not-found style messages", () => { + assert.equal(isNotFoundError(new Error("Path 'a' not found")), true) + assert.equal(isNotFoundError(new Error("不存在")), true) + assert.equal(isNotFoundError(new Error("object does not exist")), true) + assert.equal(isNotFoundError(new Error("HTTP 404")), true) + assert.equal(isNotFoundError(new Error("HTTP 503")), false) + assert.equal(isNotFoundError(new Error("permission denied")), false) +}) + +test("the setting ships disabled by default", async () => { + const { defaultDb } = await import("../internal/model/db") + const setting = (defaultDb.settings as any[]).find( + (item) => item.key === "upload_force_overwrite", + ) + assert.ok(setting, "upload_force_overwrite must be declared") + assert.equal(setting.value, "false") + assert.equal(setting.type, "bool") + assert.match(setting.help, /extra delete request|adds one existence check/i) +}) diff --git a/src/backend/server/webdav.ts b/src/backend/server/webdav.ts index 1a774620..75ac387b 100644 --- a/src/backend/server/webdav.ts +++ b/src/backend/server/webdav.ts @@ -37,6 +37,7 @@ import { copyItems, } from "../internal/op/storage" import { buildWebDavPropfindResponse } from "../internal/webdav/webdav" +import { prepareUploadTarget } from "../internal/op/upload" import { safeErrorMessage } from "../pkg/errs" import { encodeDownloadPath } from "../pkg/path" import { appendDownloadSignToRawUrl } from "../pkg/sign" @@ -397,6 +398,9 @@ webdavRouter.all("/*", async (c) => { } // Uint8Array 而非 Buffer：ESA 边缘运行时没有全局 Buffer（Issue #169） const buffer = new Uint8Array(await c.req.arrayBuffer()) + // Issue #100：阿里云盘等驱动上传同名文件会生成 xxx_(1) 副本。 + // 开启 upload_force_overwrite 后，先删旧文件再写。 + await prepareUploadTarget(actualPath, ctx) await putItem(actualPath, buffer, ctx) return c.body(null, 201) }