diff --git a/functions/webdav/protocol.ts b/functions/webdav/protocol.ts index 2067b81..0ac2735 100644 --- a/functions/webdav/protocol.ts +++ b/functions/webdav/protocol.ts @@ -15,6 +15,9 @@ const DAV_ENDPOINT = "/webdav"; const DAV_ENDPOINT_WITH_SLASH = "/webdav/"; const INTERNAL_PREFIX = "_$flaredrive$/"; const THUMBNAIL_PREFIX = "_$flaredrive$/thumbnails/"; +const THUMBNAIL_REFS_PREFIX = `${THUMBNAIL_PREFIX}refs/`; +// 缩略图摘要由客户端生成（hex 编码哈希），写入 R2 key 前先校验格式。 +const THUMBNAIL_DIGEST_RE = /^[a-f0-9]{16,128}$/; const DAV_CLASS = "1, 2"; const SUPPORT_METHODS = [ "OPTIONS", @@ -116,6 +119,57 @@ function escapeXml(value: string): string { .replaceAll("'", "'"); } +// —— 缩略图引用计数 ———————————————————————————————— +// 缩略图按内容摘要寻址（_$flaredrive$/thumbnails/.png）并被内容相同的 +// 文件共享，删除单个文件时不能直接删图。每个引用文件在 +// _$flaredrive$/thumbnails/refs// 保留一个不可变 marker 对象， +// 引用清零才回收缩略图。marker 是单次 PUT/DELETE 的独立对象，无读改写竞争： +// 并发删除最坏只会“多留”（无害泄漏），不会误删仍在使用的缩略图。 + +function isValidThumbnailDigest(digest: unknown): digest is string { + return typeof digest === "string" && THUMBNAIL_DIGEST_RE.test(digest); +} + +function thumbnailObjectKey(digest: string): string { + return `${THUMBNAIL_PREFIX}${digest}.png`; +} + +function thumbnailRefKey(digest: string, path: string): string { + return `${THUMBNAIL_REFS_PREFIX}${digest}/${path}`; +} + +async function addThumbnailRef( + bucket: R2Bucket, + digest: unknown, + path: string, +): Promise { + if (!isValidThumbnailDigest(digest)) return; + await bucket.put(thumbnailRefKey(digest, path), ""); +} + +/** 引用清零时回收缩略图。 */ +async function gcThumbnail(bucket: R2Bucket, digest: unknown): Promise { + if (!isValidThumbnailDigest(digest)) return; + const refs = await bucket.list({ + prefix: thumbnailRefKey(digest, ""), + limit: 1, + }); + if (refs.objects.length === 0) { + await bucket.delete(thumbnailObjectKey(digest)); + } +} + +/** 移除单个引用 marker，随后尝试回收缩略图。 */ +async function releaseThumbnailRef( + bucket: R2Bucket, + digest: unknown, + path: string, +): Promise { + if (!isValidThumbnailDigest(digest)) return; + await bucket.delete(thumbnailRefKey(digest, path)); + await gcThumbnail(bucket, digest); +} + function decodePathSegment(segment: string): string { try { return decodeURIComponent(segment); @@ -956,16 +1010,34 @@ async function deleteAll( excludeInternal: boolean = true, ): Promise { let cursor: string | undefined = undefined; + const releasedDigests = new Set(); do { - const objects = await bucket.list({ prefix, cursor }); + const objects = await bucket.list({ + prefix, + cursor, + include: ["customMetadata"], + }); const keys = objects.objects .map((object) => object.key) .filter((key) => !excludeInternal || !key.startsWith(INTERNAL_PREFIX)); if (keys.length > 0) { await bucket.delete(keys); } + // 先逐个移除引用 marker（internal 子树本身不含 marker，无需处理）， + // 全部分页结束后再按 digest 去重回收一次缩略图。 + for (const object of objects.objects) { + if (excludeInternal && object.key.startsWith(INTERNAL_PREFIX)) continue; + const { thumbnail } = object.customMetadata ?? {}; + if (isValidThumbnailDigest(thumbnail)) { + releasedDigests.add(thumbnail); + await bucket.delete(thumbnailRefKey(thumbnail, object.key)); + } + } cursor = objects.truncated ? objects.cursor : undefined; } while (cursor); + for (const digest of releasedDigests) { + await gcThumbnail(bucket, digest); + } } function calcContentRange(object: R2ObjectBody) { @@ -1200,10 +1272,22 @@ async function handlePut({ const existing = await bucket.head(path); const body = await request.arrayBuffer(); - const thumbnail = request.headers.get("fd-thumbnail"); + const rawThumbnail = request.headers.get("fd-thumbnail"); + const thumbnail = isValidThumbnailDigest(rawThumbnail) ? rawThumbnail : undefined; + // 覆盖写入会整体替换 customMetadata；previousThumbnail 是即将被替换掉/保留的旧引用。 + const previousThumbnail = existing?.customMetadata?.thumbnail; const preservedMetadata = getPreservedCustomMetadata(existing?.customMetadata); if (thumbnail) { preservedMetadata.thumbnail = thumbnail; + // 引用先行：并发删除最后一个同缩略图文件时不会把图收走。 + await addThumbnailRef(bucket, thumbnail, path); + // 客户端先传缩略图本体再传文件；若图在间隙中被并发回收，让客户端整体重试。 + if ((await bucket.head(thumbnailObjectKey(thumbnail))) === null) { + if (thumbnail !== previousThumbnail) { + await bucket.delete(thumbnailRefKey(thumbnail, path)); + } + return new Response("Thumbnail is missing", { status: 409 }); + } } const result = await bucket.put(path, body, { @@ -1212,9 +1296,16 @@ async function handlePut({ customMetadata: preservedMetadata, }); if (!result) { + if (thumbnail && thumbnail !== previousThumbnail) { + await bucket.delete(thumbnailRefKey(thumbnail, path)); + } return new Response("Preconditions failed", { status: 412 }); } + if (thumbnail && previousThumbnail !== thumbnail) { + await releaseThumbnailRef(bucket, previousThumbnail, path); + } + return existing === null ? createdResponse(path, false) : new Response(null, { status: 204 }); @@ -1276,6 +1367,8 @@ async function handleDelete({ if (path === "") { await deleteAll(bucket); + // deleteAll 会跳过 internal 子树；全删时把共享缩略图和 marker 一并清掉。 + await deleteAll(bucket, THUMBNAIL_PREFIX, false); return new Response(null, { status: 204 }); } @@ -1291,7 +1384,9 @@ async function handleDelete({ await bucket.delete(path); } } else { + const digest = resource?.customMetadata?.thumbnail; await bucket.delete(path); + await releaseThumbnailRef(bucket, digest, path); } return new Response(null, { status: 204 }); } @@ -1637,10 +1732,19 @@ async function handleCopy({ }); return; } + // 覆盖同名文件时目标旧缩略图引用需要释放（目录目标已由 deleteDestination 清理， + // 这里兜底处理文件对文件的直接覆盖）。 + const previous = await bucket.head(targetKey); + const previousThumbnail = previous?.customMetadata?.thumbnail; + const digest = source.customMetadata?.thumbnail; + await addThumbnailRef(bucket, digest, targetKey); await bucket.put(targetKey, source.body, { httpMetadata: source.httpMetadata, customMetadata: stripLockMetadata(source.customMetadata), }); + if (previousThumbnail !== digest) { + await releaseThumbnailRef(bucket, previousThumbnail, targetKey); + } }; if (isDirectory) { @@ -1785,12 +1889,16 @@ async function handleMove({ }); } } else { + const digest = source.customMetadata?.thumbnail; + // 先给目标补引用，再释放源引用，缩略图在移动全程都有引用覆盖。 + await addThumbnailRef(bucket, digest, target); await bucket.put(target, source.body, { httpMetadata: source.httpMetadata, customMetadata: getPreservedCustomMetadata(source.customMetadata), }); + await bucket.delete(object.key); + await releaseThumbnailRef(bucket, digest, object.key); } - await bucket.delete(object.key); }; if (isDirectory) { @@ -2016,8 +2124,13 @@ async function handlePostCreateMultipart({ path: string; request: Request; }): Promise { - const thumbnail = request.headers.get("fd-thumbnail"); + const rawThumbnail = request.headers.get("fd-thumbnail"); + const thumbnail = isValidThumbnailDigest(rawThumbnail) ? rawThumbnail : undefined; const customMetadata = thumbnail ? { thumbnail } : undefined; + // 引用先行：分片未完成期间，并发删除最后一个同缩略图文件不会把图收走。 + if (thumbnail) { + await addThumbnailRef(bucket, thumbnail, path); + } const multipartUpload = await bucket.createMultipartUpload(path, { httpMetadata: request.headers, customMetadata, diff --git a/src/app/__tests__/webdavProtocol.test.ts b/src/app/__tests__/webdavProtocol.test.ts index 5fcde4d..e7b362f 100644 --- a/src/app/__tests__/webdavProtocol.test.ts +++ b/src/app/__tests__/webdavProtocol.test.ts @@ -580,12 +580,14 @@ describe("webdav PUT", () => { test("overwrite existing file is 204 and preserves custom metadata", async () => { const bucket = new InMemoryBucket(); - bucket.seed([{ key: "a.txt", body: "old", customMetadata: { thumbnail: "t" } }]); + bucket.seed([{ key: "a.txt", body: "old", customMetadata: { thumbnail: "1".repeat(40) } }]); + // 真实客户端总是先上传缩略图本体，再带 fd-thumbnail 传文件 + bucket.seed([{ key: `_$flaredrive$/thumbnails/${"2".repeat(40)}.png`, body: "png" }]); const response = await call( req( "/webdav/a.txt", "PUT", - { Authorization: AUTH, "Content-Type": "text/plain", "fd-thumbnail": "new-t" }, + { Authorization: AUTH, "Content-Type": "text/plain", "fd-thumbnail": "2".repeat(40) }, "new" ), makeEnv(bucket) @@ -593,7 +595,7 @@ describe("webdav PUT", () => { expect(response.status).toBe(204); expect(bucket.rawText("a.txt")).toBe("new"); const head = await bucket.asBucket().head("a.txt"); - expect(head?.customMetadata?.thumbnail).toBe("new-t"); + expect(head?.customMetadata?.thumbnail).toBe("2".repeat(40)); expect(head?.httpMetadata?.contentType).toBe("text/plain"); }); diff --git a/src/app/__tests__/webdavThumbnailGc.test.ts b/src/app/__tests__/webdavThumbnailGc.test.ts new file mode 100644 index 0000000..ae869f0 --- /dev/null +++ b/src/app/__tests__/webdavThumbnailGc.test.ts @@ -0,0 +1,229 @@ +/** + * 缩略图引用计数（issue #16 移植修复）分支级直测。 + * + * 缩略图按内容摘要寻址并被内容相同的文件共享；这里覆盖引用 marker 的完整 + * 生命周期：上传建引用、重复共享、删除最后一个引用才回收、覆盖/复制/移动 + * 的引用同步、目录删除与全删的批量回收、非法 fd-thumbnail 头防护。 + */ +import { onRequest, type WebDavEnv } from "../../../functions/webdav/protocol"; +import { + InMemoryBucket, + basicAuthHeader, + makeContext, +} from "../testInMemoryBucket"; + +const AUTH = basicAuthHeader("user", "pass"); +const HOST = "http://drive.example.com"; + +const D1 = "1".repeat(40); +const D2 = "2".repeat(40); +const thumbKey = (digest: string) => `_$flaredrive$/thumbnails/${digest}.png`; +const refKey = (digest: string, path: string) => + `_$flaredrive$/thumbnails/refs/${digest}/${path}`; + +function makeEnv(bucket: InMemoryBucket): WebDavEnv { + return { + BUCKET: bucket.asBucket(), + WEBDAV_USERNAME: "user", + WEBDAV_PASSWORD: "pass", + }; +} + +function req( + path: string, + method: string, + headers?: Record, + body?: BodyInit +): Request { + return new Request(`${HOST}${path}`, { + method, + headers: { Authorization: AUTH, ...headers }, + body, + }); +} + +async function call(request: Request, env: WebDavEnv) { + return onRequest(makeContext(request, env)); +} + +async function seedDir(bucket: InMemoryBucket, path: string) { + await call( + req(`/webdav/${path === "" ? "" : path + "/"}`, "MKCOL"), + makeEnv(bucket) + ); +} + +/** 模拟客户端上传流程：先传缩略图本体，再带 fd-thumbnail 传文件。 */ +async function uploadWithThumbnail( + bucket: InMemoryBucket, + path: string, + digest: string +) { + await call( + req(`/webdav/_$flaredrive$/thumbnails/${digest}.png`, "PUT", {}, "png-blob"), + makeEnv(bucket) + ); + const res = await call( + req(`/webdav/${path}`, "PUT", { "fd-thumbnail": digest }, "file-body"), + makeEnv(bucket) + ); + expect([201, 204]).toContain(res.status); +} + +describe("webdav thumbnail reference counting", () => { + it("PUT with fd-thumbnail writes a marker and duplicate files share it", async () => { + const bucket = new InMemoryBucket(); + await uploadWithThumbnail(bucket, "a.png", D1); + await uploadWithThumbnail(bucket, "b.png", D1); + + expect(await bucket.asBucket().head(refKey(D1, "a.png"))).not.toBeNull(); + expect(await bucket.asBucket().head(refKey(D1, "b.png"))).not.toBeNull(); + expect(await bucket.asBucket().head(thumbKey(D1))).not.toBeNull(); + const head = await bucket.asBucket().head("a.png"); + expect(head?.customMetadata?.thumbnail).toBe(D1); + }); + + it("deleting one duplicate keeps the thumbnail; deleting the last one GCs it", async () => { + const bucket = new InMemoryBucket(); + await uploadWithThumbnail(bucket, "a.png", D1); + await uploadWithThumbnail(bucket, "b.png", D1); + + expect((await call(req("/webdav/a.png", "DELETE"), makeEnv(bucket))).status).toBe(204); + expect(await bucket.asBucket().head(refKey(D1, "a.png"))).toBeNull(); + expect(await bucket.asBucket().head(thumbKey(D1))).not.toBeNull(); + + expect((await call(req("/webdav/b.png", "DELETE"), makeEnv(bucket))).status).toBe(204); + expect(await bucket.asBucket().head(refKey(D1, "b.png"))).toBeNull(); + expect(await bucket.asBucket().head(thumbKey(D1))).toBeNull(); + }); + + it("overwriting with a different digest releases the old thumbnail", async () => { + const bucket = new InMemoryBucket(); + await uploadWithThumbnail(bucket, "a.png", D1); + await uploadWithThumbnail(bucket, "a.png", D2); + + expect(await bucket.asBucket().head(refKey(D1, "a.png"))).toBeNull(); + expect(await bucket.asBucket().head(refKey(D2, "a.png"))).not.toBeNull(); + expect(await bucket.asBucket().head(thumbKey(D1))).toBeNull(); + expect(await bucket.asBucket().head(thumbKey(D2))).not.toBeNull(); + }); + + it("overwriting without fd-thumbnail preserves the old reference", async () => { + const bucket = new InMemoryBucket(); + await uploadWithThumbnail(bucket, "a.png", D1); + + const res = await call( + req("/webdav/a.png", "PUT", {}, "replaced-body"), + makeEnv(bucket) + ); + expect([201, 204]).toContain(res.status); + // Davflare 语义：覆盖且未带 fd-thumbnail 时保留旧 customMetadata， + // 引用关系保持一致（marker 不动、缩略图不回收）。 + expect((await bucket.asBucket().head("a.png"))?.customMetadata?.thumbnail).toBe(D1); + expect(await bucket.asBucket().head(refKey(D1, "a.png"))).not.toBeNull(); + expect(await bucket.asBucket().head(thumbKey(D1))).not.toBeNull(); + }); + + it("directory delete releases every child marker and GCs shared digests once", async () => { + const bucket = new InMemoryBucket(); + await seedDir(bucket, "dir"); + await uploadWithThumbnail(bucket, "dir/x.png", D1); + await uploadWithThumbnail(bucket, "dir/y.png", D1); + await uploadWithThumbnail(bucket, "dir/z.png", D2); + + expect((await call(req("/webdav/dir/", "DELETE"), makeEnv(bucket))).status).toBe(204); + expect(await bucket.asBucket().head(refKey(D1, "dir/x.png"))).toBeNull(); + expect(await bucket.asBucket().head(refKey(D1, "dir/y.png"))).toBeNull(); + expect(await bucket.asBucket().head(refKey(D2, "dir/z.png"))).toBeNull(); + expect(await bucket.asBucket().head(thumbKey(D1))).toBeNull(); + expect(await bucket.asBucket().head(thumbKey(D2))).toBeNull(); + }); + + it("delete-all wipes the internal thumbnails subtree too", async () => { + const bucket = new InMemoryBucket(); + await uploadWithThumbnail(bucket, "a.png", D1); + await uploadWithThumbnail(bucket, "b.png", D1); + + expect((await call(req("/webdav/", "DELETE"), makeEnv(bucket))).status).toBe(204); + const leftovers = await bucket + .asBucket() + .list({ prefix: "_$flaredrive$/thumbnails/" }); + expect(leftovers.objects).toHaveLength(0); + expect(await bucket.asBucket().head("a.png")).toBeNull(); + }); + + it("COPY adds a marker for the destination and survives source deletion", async () => { + const bucket = new InMemoryBucket(); + await uploadWithThumbnail(bucket, "a.png", D1); + + const res = await call( + req("/webdav/a.png", "COPY", { Destination: `${HOST}/webdav/c.png` }), + makeEnv(bucket) + ); + expect([201, 204]).toContain(res.status); + expect(await bucket.asBucket().head(refKey(D1, "c.png"))).not.toBeNull(); + + await call(req("/webdav/a.png", "DELETE"), makeEnv(bucket)); + expect(await bucket.asBucket().head(thumbKey(D1))).not.toBeNull(); + await call(req("/webdav/c.png", "DELETE"), makeEnv(bucket)); + expect(await bucket.asBucket().head(thumbKey(D1))).toBeNull(); + }); + + it("COPY over a file with a different thumbnail releases the old one", async () => { + const bucket = new InMemoryBucket(); + await uploadWithThumbnail(bucket, "a.png", D1); + await uploadWithThumbnail(bucket, "c.png", D2); + + const res = await call( + req("/webdav/a.png", "COPY", { Destination: `${HOST}/webdav/c.png` }), + makeEnv(bucket) + ); + expect(res.status).toBe(204); + expect(await bucket.asBucket().head(refKey(D2, "c.png"))).toBeNull(); + expect(await bucket.asBucket().head(thumbKey(D2))).toBeNull(); + expect(await bucket.asBucket().head(refKey(D1, "c.png"))).not.toBeNull(); + expect((await bucket.asBucket().head("c.png"))?.customMetadata?.thumbnail).toBe(D1); + }); + + it("MOVE re-points the reference from source to destination", async () => { + const bucket = new InMemoryBucket(); + await uploadWithThumbnail(bucket, "a.png", D1); + + const res = await call( + req("/webdav/a.png", "MOVE", { Destination: `${HOST}/webdav/m.png` }), + makeEnv(bucket) + ); + expect([201, 204]).toContain(res.status); + expect(await bucket.asBucket().head(refKey(D1, "a.png"))).toBeNull(); + expect(await bucket.asBucket().head(refKey(D1, "m.png"))).not.toBeNull(); + expect(await bucket.asBucket().head(thumbKey(D1))).not.toBeNull(); + }); + + it("multipart create writes the marker before the object exists", async () => { + const bucket = new InMemoryBucket(); + await call( + req(`/webdav/_$flaredrive$/thumbnails/${D1}.png`, "PUT", {}, "png-blob"), + makeEnv(bucket) + ); + const res = await call( + req("/webdav/big.png?uploads", "POST", { "fd-thumbnail": D1 }), + makeEnv(bucket) + ); + expect(res.status).toBe(200); + expect(await bucket.asBucket().head(refKey(D1, "big.png"))).not.toBeNull(); + }); + + it("malformed fd-thumbnail header is rejected and writes no marker", async () => { + const bucket = new InMemoryBucket(); + const res = await call( + req("/webdav/evil.png", "PUT", { "fd-thumbnail": "../../evil" }, "x"), + makeEnv(bucket) + ); + expect([201, 204]).toContain(res.status); + const refs = await bucket.asBucket().list({ prefix: "_$flaredrive$/thumbnails/refs/" }); + expect(refs.objects).toHaveLength(0); + expect( + (await bucket.asBucket().head("evil.png"))?.customMetadata?.thumbnail + ).toBeUndefined(); + }); +});