Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions src/backend/internal/model/db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1007,6 +1007,23 @@ export const defaultDb = {
group: 12,
flag: 1,
},
// ── 上传行为 ──
// Issue #100:阿里云盘经 WebDAV 上传同名文件时会生成 xxx_(1) 副本而不是
// 覆盖。驱动接口本身没有 overwrite 语义,只能在上传前主动删除旧文件。
// 代价是每次上传多一次存在性检查(命中时再多一次删除),故默认关闭。
{
key: "upload_force_overwrite",
value: "false",
type: "bool",
help:
"Before every upload, check whether the target file exists and delete it first. " +
"This makes uploads truly overwrite files on drivers that would otherwise create " +
"numbered copies (e.g. Aliyun Drive via WebDAV), but it adds one existence check " +
"and, when the file exists, one extra delete request per upload. " +
"Leave it off unless you need overwrite semantics.",
group: 4,
flag: 1,
},
],
storages: [],
users: [
Expand Down
103 changes: 103 additions & 0 deletions src/backend/internal/op/upload.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
/**
* 上传前「强制覆盖」处理。
*
* 背景(Issue #100):部分网盘驱动在上传同名文件时不会覆盖,而是自动生成
* `xxx_(1).ext` 之类的副本(阿里云盘经 WebDAV 上传即如此)。驱动的上传接口
* 本身不提供 overwrite 语义,因此只能在写入前主动删除旧文件。
*
* 代价:开启后每次上传多 1 次存在性检查(文件存在时再多 1 次删除请求),
* 因此默认关闭,并在设置项说明里明确提示。
*
* 为什么放在上传入口而不是 `putItem` 内部:
* - 解压、种子转存等内部写路径会批量调用 `putItem`,逐个检查会成倍放大上游
* 调用次数,且那些场景不需要「先删后写」;
* - 分片上传不走 `putItem`,需要单独在 init 阶段处理。
*/
import { getSettings } from "../model/db"
import { getItem, removeItems } from "./storage"
import type { StorageRequestContext } from "./storage"

const NOT_FOUND_RE = /not\s*found|no such|does not exist|不存在|404/i

/** 判断错误是否表示「目标不存在」:这类错误在存在性检查里是正常结果 */
export function isNotFoundError(err: any): boolean {
return NOT_FOUND_RE.test(String(err?.message || err || ""))
}

/**
* 读取设置开关。
*
* 读取失败时返回 false(保持旧行为),不让一次配置读取抖动导致所有上传失败。
*/
export async function isForceOverwriteUploadEnabled(): Promise<boolean> {
try {
const settings = await getSettings()
const value = String(settings.upload_force_overwrite ?? "")
.trim()
.toLowerCase()
return value === "true" || value === "1"
} catch (err) {
console.warn("[Upload] failed to read upload_force_overwrite setting:", err)
return false
}
}

export interface UploadOverwriteDeps {
isEnabled?: () => Promise<boolean>
getItemFn?: typeof getItem
removeItemsFn?: typeof removeItems
}

/**
* 上传前准备:开启设置时,若目标文件已存在则先删除。
*
* 行为约定:
* - 目标不存在:什么都不做(`getItem` 抛错是最常见路径);
* - 目标是目录:直接报错,绝不删除目录(用文件覆盖目录没有合理语义);
* - 删除失败且错误是「不存在」:视为已被并发删除,继续上传;
* - 其它删除失败:抛出可读错误,避免静默地又生成一个 `xxx_(1)` 副本。
*
* 注意:调用方必须在**请求体已通过大小校验之后**、真正写入之前调用,
* 否则「上传被拒绝但旧文件已删」会造成数据丢失。
*/
export async function prepareUploadTarget(
virtualPath: string,
requestContext?: StorageRequestContext,
deps: UploadOverwriteDeps = {},
): Promise<void> {
const enabled = await (deps.isEnabled || isForceOverwriteUploadEnabled)()
if (!enabled) return

const path = String(virtualPath || "")
const segments = path.split("/").filter(Boolean)
const name = segments.pop() || ""
if (!name) return
const dir = `/${segments.join("/")}`

const getItemFn = deps.getItemFn || getItem
const removeItemsFn = deps.removeItemsFn || removeItems

let existing: any
try {
const result = await getItemFn(path, requestContext)
existing = result?.item
} catch {
// 不存在 / 无法读取:交给后续上传按原行为处理
return
}
if (!existing) return
if (existing.is_dir) {
throw new Error(
`cannot upload over a directory: "${path}" already exists as a folder`,
)
}

try {
await removeItemsFn(dir, [name], requestContext)
} catch (err: any) {
if (isNotFoundError(err)) return
throw new Error(
`failed to delete the existing file before upload: ${err?.message || err}`,
)
}
}
20 changes: 20 additions & 0 deletions src/backend/server/fs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
} from "../internal/op/storage"
import { resolveShare } from "../internal/op/share"
import { resolvePath } from "../internal/model/db"
import { prepareUploadTarget } from "../internal/op/upload"
import { getUserFromContext } from "./middlewares"
import {
can,
Expand Down Expand Up @@ -978,6 +979,8 @@ fsRouter.put("/put", async (c) => {
try {
const body = await readBodyWithinLimit(c)
if (!body.ok) return tooLargeResponse(c, body.limit)
// 先校验请求体再删旧文件:避免「请求被拒但原文件已丢」
await prepareUploadTarget(reqPath, requestContext)
await putItem(reqPath, body.buffer, requestContext)
return c.json({ code: 200, message: "success", data: null })
} catch (e: any) {
Expand Down Expand Up @@ -1018,6 +1021,8 @@ fsRouter.put("/form", async (c) => {
// multipart 的 Content-Length 只覆盖整个表单,仍需按文件实际字节复核
const max = getUploadSizeLimit(c, false)
if (buffer.length > max) return tooLargeResponse(c, max)
// 先校验体积再删旧文件:避免「请求被拒但原文件已丢」
await prepareUploadTarget(reqPath, requestContext)
await putItem(reqPath, buffer, requestContext)
return c.json({ code: 200, message: "success", data: null })
} catch (e: any) {
Expand Down Expand Up @@ -1392,6 +1397,15 @@ fsRouter.post("/get_direct_upload_info", async (c) => {
} catch {
// 目标不存在 / 无法读取:继续签发流程
}
} else {
// 全局「强制覆盖」:客户端直传时 Worker 不经手数据,只能在签发前删旧文件,
// 否则驱动会生成 xxx_(1) 副本(Issue #100)。
// 显式 Overwrite:false 优先,保持「拒绝覆盖」的语义。
try {
await prepareUploadTarget(dstPath, getStorageRequestContext(c))
} catch (e: any) {
return c.json({ code: 500, message: safeErrorMessage(e), data: null }, 500)
}
}
try {
const resolved = await resolvePath(dirPath)
Expand Down Expand Up @@ -1498,6 +1512,12 @@ fsRouter.post("/multipart/init", async (c) => {
session = existing
resumed = true
} else {
// 只在新会话时清理旧文件:续传会话的目标可能已经在首次 init 时删过,
// 再次删除会破坏正在进行的上传。
await prepareUploadTarget(
`${actualDir}/${name}`.replace(/\/{2,}/g, "/"),
requestContext,
)
const info = await (driver as any).createUploadSession(
actualDir,
resolved.physical!,
Expand Down
3 changes: 3 additions & 0 deletions src/backend/server/s3.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
putItem,
removeItems,
} from "../internal/op/storage"
import { prepareUploadTarget } from "../internal/op/upload"
import { safeErrorMessage } from "../pkg/errs"
import { appendDownloadSignToRawUrl } from "../pkg/sign"
import { readBodyWithinLimit } from "../pkg/upload-limit"
Expand Down Expand Up @@ -298,6 +299,8 @@ s3Router.put("/*", async (c) => {
400,
)
}
// 体积校验通过后再清理旧文件(Issue #100:驱动会生成 xxx_(1) 副本)
await prepareUploadTarget(virtualPath, getCtx(c))
await putItem(virtualPath, body.buffer, getCtx(c))
// 不再返回伪造的 ETag。
//
Expand Down
128 changes: 128 additions & 0 deletions src/backend/server/upload_overwrite.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
import assert from "node:assert/strict"
import { test } from "node:test"

import {
isNotFoundError,
prepareUploadTarget,
type UploadOverwriteDeps,
} from "../internal/op/upload"

function makeDeps(options: {
enabled: boolean
existing?: { is_dir?: boolean; size?: number } | null
getError?: Error
removeError?: Error
}) {
const calls = {
get: [] as string[],
remove: [] as Array<{ dir: string; name: string }>,
}
const deps: UploadOverwriteDeps = {
isEnabled: async () => options.enabled,
getItemFn: (async (path: string) => {
calls.get.push(path)
if (options.getError) throw options.getError
return {
item: {
name: String(path).split("/").filter(Boolean).pop() || "",
size: options.existing?.size ?? 1,
is_dir: Boolean(options.existing?.is_dir),
modified: new Date().toISOString(),
sign: "",
type: 0,
},
provider: "fake",
rawUrl: "",
}
}) as any,
removeItemsFn: (async (dir: string, names: string[]) => {
calls.remove.push({ dir, name: names[0] })
if (options.removeError) throw options.removeError
}) as any,
}
return { deps, calls }
}

test("does nothing when the setting is off", async () => {
const { deps, calls } = makeDeps({ enabled: false, existing: {} })
await prepareUploadTarget("/dir/file.txt", undefined, deps)
assert.deepEqual(calls.get, [])
assert.deepEqual(calls.remove, [])
})

test("does nothing when the target does not exist", async () => {
const { deps, calls } = makeDeps({
enabled: true,
getError: new Error("[Fake] Path 'file.txt' not found"),
})
await prepareUploadTarget("/dir/file.txt", undefined, deps)
assert.equal(calls.get.length, 1)
assert.equal(calls.remove.length, 0)
})

test("deletes the existing file before uploading", async () => {
const { deps, calls } = makeDeps({ enabled: true, existing: { size: 10 } })
await prepareUploadTarget("/dir/sub/file.txt", undefined, deps)
assert.deepEqual(calls.remove, [{ dir: "/dir/sub", name: "file.txt" }])
})

test("root-level targets are deleted in the root directory", async () => {
const { deps, calls } = makeDeps({ enabled: true, existing: {} })
await prepareUploadTarget("/file.txt", undefined, deps)
assert.deepEqual(calls.remove, [{ dir: "/", name: "file.txt" }])
})

test("refuses to replace a directory with a file", async () => {
const { deps, calls } = makeDeps({
enabled: true,
existing: { is_dir: true },
})
await assert.rejects(
() => prepareUploadTarget("/dir", undefined, deps),
/directory/i,
)
assert.equal(calls.remove.length, 0, "must never delete a directory")
})

test("a concurrent delete (not found) is not an upload failure", async () => {
const { deps } = makeDeps({
enabled: true,
existing: {},
removeError: new Error("object not found"),
})
await assert.doesNotReject(() =>
prepareUploadTarget("/dir/file.txt", undefined, deps),
)
})

test("other delete failures surface a readable error", async () => {
const { deps } = makeDeps({
enabled: true,
existing: {},
removeError: new Error("upstream 503"),
})
await assert.rejects(
() => prepareUploadTarget("/dir/file.txt", undefined, deps),
/failed to delete the existing file before upload: upstream 503/,
)
})

test("isNotFoundError only matches not-found style messages", () => {
assert.equal(isNotFoundError(new Error("Path 'a' not found")), true)
assert.equal(isNotFoundError(new Error("不存在")), true)
assert.equal(isNotFoundError(new Error("object does not exist")), true)
assert.equal(isNotFoundError(new Error("HTTP 404")), true)
assert.equal(isNotFoundError(new Error("HTTP 503")), false)
assert.equal(isNotFoundError(new Error("permission denied")), false)
})

test("the setting ships disabled by default", async () => {
const { defaultDb } = await import("../internal/model/db")
const setting = (defaultDb.settings as any[]).find(
(item) => item.key === "upload_force_overwrite",
)
assert.ok(setting, "upload_force_overwrite must be declared")
assert.equal(setting.value, "false")
assert.equal(setting.type, "bool")
assert.match(setting.help, /extra delete request|adds one existence check/i)
})
4 changes: 4 additions & 0 deletions src/backend/server/webdav.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import {
copyItems,
} from "../internal/op/storage"
import { buildWebDavPropfindResponse } from "../internal/webdav/webdav"
import { prepareUploadTarget } from "../internal/op/upload"
import { safeErrorMessage } from "../pkg/errs"
import { encodeDownloadPath } from "../pkg/path"
import { appendDownloadSignToRawUrl } from "../pkg/sign"
Expand Down Expand Up @@ -397,6 +398,9 @@ webdavRouter.all("/*", async (c) => {
}
// Uint8Array 而非 Buffer:ESA 边缘运行时没有全局 Buffer(Issue #169)
const buffer = new Uint8Array(await c.req.arrayBuffer())
// Issue #100:阿里云盘等驱动上传同名文件会生成 xxx_(1) 副本。
// 开启 upload_force_overwrite 后,先删旧文件再写。
await prepareUploadTarget(actualPath, ctx)
await putItem(actualPath, buffer, ctx)
return c.body(null, 201)
}
Expand Down
Loading