From 48b795756d78e725247795704a4cb1c336ebf3a2 Mon Sep 17 00:00:00 2001 From: hesprs <190185753+hesprs@users.noreply.github.com> Date: Mon, 21 Sep 2026 04:04:34 +0800 Subject: [PATCH 1/4] refactor(request): cleaner request signature --- docs/src/pages/en/deep-dive/modules/s3.md | 2 +- docs/src/pages/en/deep-dive/request.md | 9 +- .../pages/en/development/debug-and-testing.md | 26 ++--- docs/src/pages/en/development/request.md | 13 +-- packages/gdrive/src/gdrive/auth.ts | 12 ++- .../gdrive/src/gdrive/check-connection.ts | 6 +- packages/gdrive/src/gdrive/fs.ts | 67 ++++++------- packages/gdrive/src/gdrive/upload.ts | 11 +-- packages/gdrive/src/setting.ts | 3 +- packages/gdrive/test/auth.test.ts | 5 +- packages/gdrive/test/fs-gdrive.test.ts | 39 ++++---- packages/plugin/dist/dev.spec.d.ts | 14 +-- ...B_s.spec.d.ts => index-C66-NLE8.spec.d.ts} | 19 ++-- packages/plugin/dist/index.spec.d.ts | 2 +- .../plugin/src/components/MigrationModal.ts | 4 +- .../src/fs/middlewares/custom-headers.ts | 7 +- packages/plugin/src/fs/middlewares/retry.ts | 8 +- packages/plugin/src/fs/vault/index.ts | 29 +++--- packages/plugin/src/fs/vault/request.ts | 17 ++-- .../plugin/src/fs/wrappers/cancellation.ts | 10 +- packages/plugin/src/modules/Extensibility.ts | 8 +- packages/plugin/src/modules/Observability.ts | 4 +- packages/plugin/src/modules/Registrar.ts | 19 ++-- packages/plugin/src/modules/Sync.ts | 6 +- packages/plugin/src/sdk/dev.ts | 2 +- packages/plugin/src/settings/head.ts | 4 +- packages/plugin/src/utils/pipe.ts | 4 +- packages/plugin/src/utils/to-error-message.ts | 3 - .../test/cancellation-middleware.test.ts | 4 +- .../test/custom-headers-middleware.test.ts | 5 +- packages/plugin/test/fs-vault.test.ts | 2 +- .../test/rate-limiter-middleware.test.ts | 12 +-- packages/plugin/test/retry-middleware.test.ts | 14 +-- packages/plugin/test/test-kit.ts | 22 ++--- packages/s3/src/index.ts | 12 +-- packages/s3/src/s3/check-connection.ts | 6 +- packages/s3/src/s3/fs.ts | 44 ++++----- packages/s3/src/s3/multipart.ts | 18 ++-- packages/s3/src/s3/sigv4.ts | 29 +++--- packages/s3/test/check-connection.test.ts | 3 +- packages/s3/test/fs-s3.test.ts | 98 +++++++++---------- packages/s3/test/helpers.ts | 3 +- packages/s3/test/sigv4-middleware.test.ts | 26 ++--- .../shared/src/{get-status.ts => error.ts} | 4 + .../webdav/src/webdav/check-connection.ts | 7 +- packages/webdav/src/webdav/chunked-upload.ts | 37 ++++--- packages/webdav/src/webdav/fs.ts | 34 +++---- packages/webdav/test/fs-webdav.test.ts | 88 ++++++++--------- 48 files changed, 389 insertions(+), 432 deletions(-) rename packages/plugin/dist/{index-DI2LCB_s.spec.d.ts => index-C66-NLE8.spec.d.ts} (99%) delete mode 100644 packages/plugin/src/utils/to-error-message.ts rename packages/shared/src/{get-status.ts => error.ts} (79%) diff --git a/docs/src/pages/en/deep-dive/modules/s3.md b/docs/src/pages/en/deep-dive/modules/s3.md index 6090de5f..626bf6fb 100644 --- a/docs/src/pages/en/deep-dive/modules/s3.md +++ b/docs/src/pages/en/deep-dive/modules/s3.md @@ -10,7 +10,7 @@ The module supports: - Cloudflare R2 - Backblaze B2 through its S3-compatible API - MinIO -- Garage +- RustFs - Aliyun / Tencent Cloud object storage - Wasabi - Ceph Object Gateway, DigitalOcean Spaces, and other S3-compatible services diff --git a/docs/src/pages/en/deep-dive/request.md b/docs/src/pages/en/deep-dive/request.md index 8d333385..d5303a9d 100644 --- a/docs/src/pages/en/deep-dive/request.md +++ b/docs/src/pages/en/deep-dive/request.md @@ -12,10 +12,10 @@ Both are function objects returning promises. They provide a small, middleware-f `Request` is defined in `packages/plugin/src/modules/Registrar.ts`: ```ts -type Request = (params: RequestParam | string) => Promise; +type Request = (url: string, params?: RequestParam) => Promise; ``` -`RequestParam` follows Obsidian's `RequestUrlParam`, except `body` uses the project's `Binary` (`Uint8Array`) instead of `ArrayBuffer` in Obsidian raw API. A string argument is treated as a `GET` toward this URL. +`RequestParam` follows Obsidian's `RequestUrlParam`, except `body` uses the project's `Binary` (`Uint8Array`) instead of `ArrayBuffer` in Obsidian raw API, and `url` moves from the parameters to the first argument. Omitting `params` performs a plain `GET` toward the URL. `RequestResponse` is an exported SDK type describing the response returned by `Request`. @@ -46,8 +46,9 @@ Remote file-system modules receive `getRequest()` in context, not the base funct `VaultRequest` is the local counterpart used by `VaultFs`, uses Obsidian vault cache smartly to improve performance. It is a discriminated operation function: ```ts -type VaultRequest = ( - params: T, +type VaultRequest = ( + key: string, + params?: T, ) => Promise; ``` diff --git a/docs/src/pages/en/development/debug-and-testing.md b/docs/src/pages/en/development/debug-and-testing.md index 0d3b076f..84b47f8a 100644 --- a/docs/src/pages/en/development/debug-and-testing.md +++ b/docs/src/pages/en/development/debug-and-testing.md @@ -40,7 +40,7 @@ type FsHarness = { }; type RequestHarness = { - calls: Array; + calls: Array; request: Request; }; ``` @@ -59,21 +59,23 @@ const testKit: { folder: (key: string) => FolderStat; flush: (turns?: number) => Promise; fs: (options?: FsOptions) => FsHarness; - request: (control: Request) => RequestHarness; + request: ( + control: (url: string, params: RequestParam) => MaybePromise>, + ) => RequestHarness; stream: (chunks?: Array) => ReadableStream; }; ``` -| Helper | Description | -| --------------------- | ---------------------------------------------------------------------------------------- | -| `bytes(value)` | Convert a string to `Binary`. | -| `deferred()` | Create a controlled promise. | -| `file(key, options?)` | Create a `FileStat`. | -| `folder(key)` | Create a `FolderStat`. | -| `flush(turns?)` | Wait for several microtask queues to finish (default 4). | -| `fs(options?)` | Create a stub filesystem. `control` overrides individual methods; `uid` sets `getUid()`. | -| `request(control)` | Wrap a request stub to record calls. | -| `stream(chunks?)` | Create a fake `ReadableStream` from an array. | +| Helper | Description | +| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bytes(value)` | Convert a string to `Binary`. | +| `deferred()` | Create a controlled promise. | +| `file(key, options?)` | Create a `FileStat`. | +| `folder(key)` | Create a `FolderStat`. | +| `flush(turns?)` | Wait for several microtask queues to finish (default 4). | +| `fs(options?)` | Create a stub filesystem. `control` overrides individual methods; `uid` sets `getUid()`. | +| `request(control)` | Wrap a response control to record calls. The control receives the same arguments as `Request` and returns response overrides; each recorded call is the url merged into its parameters. | +| `stream(chunks?)` | Create a fake `ReadableStream` from an array. | ### `fs()` Details diff --git a/docs/src/pages/en/development/request.md b/docs/src/pages/en/development/request.md index 3c800c03..c4d5d9bb 100644 --- a/docs/src/pages/en/development/request.md +++ b/docs/src/pages/en/development/request.md @@ -7,7 +7,7 @@ Sync Engine has two request systems: `Request` for remote HTTP calls and `VaultR Remote HTTP request function. Backends receive a composed `Request` instance in their constructor and must use it for all network calls. ```ts -type RequestParam = Omit & { +type RequestParam = Omit & { body?: string | Binary; ignoreCancellation?: boolean; }; @@ -20,10 +20,10 @@ type RequestResponse = { status: number; }; -type Request = (params: RequestParam | string) => Promise; +type Request = (url: string, params?: RequestParam) => Promise; ``` -`RequestParam` extends Obsidian's `RequestUrlParam` (minus `body`) with a `body` field accepting `string | Binary`. Passing a plain string instead of a `RequestParam` object uses it as the URL. +`RequestParam` extends Obsidian's `RequestUrlParam` (minus `body` and `url`) with a `body` field accepting `string | Binary`. The URL is always the first argument; omitting `params` performs a plain `GET`. `RequestResponse` is an exported SDK type for the response returned by `Request`. Set `ignoreCancellation` to `true` to let a request through after the sync has been cancelled. Reserve it for cleanup calls that release remote resources the backend already created, such as aborting an incomplete multipart upload. @@ -44,10 +44,11 @@ type VaultRequestParam = ( | { method: 'EXISTS' } | { method: 'STAT'; cached?: boolean } | { method: 'LIST'; cached?: boolean } -) & { key: string; ignoreCancellation?: boolean }; +) & { ignoreCancellation?: boolean }; -type VaultRequest = ( - params: T, +type VaultRequest = ( + key: string, + params?: T, ) => Promise; ``` diff --git a/packages/gdrive/src/gdrive/auth.ts b/packages/gdrive/src/gdrive/auth.ts index 39bfbdbe..60cec36e 100644 --- a/packages/gdrive/src/gdrive/auth.ts +++ b/packages/gdrive/src/gdrive/auth.ts @@ -1,5 +1,5 @@ -import type { Request, RequestParam } from '@hesprs/sync-engine-sdk'; -import { getStatus } from '@repo/shared/get-status'; +import type { Request } from '@hesprs/sync-engine-sdk'; +import { getStatus } from '@repo/shared/error'; import { Platform, requestUrl, SecretStorage } from 'obsidian'; import { buildUrl, @@ -243,10 +243,12 @@ export class TokenManager { /** Injects the bearer token into every remote request and retries once on 401. */ export function bearerMiddleware(request: Request, manager: TokenManager): Request { - return async (params) => { - const base: RequestParam = typeof params === 'string' ? { url: params } : params; + return async (url, params) => { const send = (token: string) => - request({ ...base, headers: { ...base.headers, Authorization: `Bearer ${token}` } }); + request(url, { + ...params, + headers: { ...params?.headers, Authorization: `Bearer ${token}` }, + }); try { return await send(await manager.getToken()); } catch (error: unknown) { diff --git a/packages/gdrive/src/gdrive/check-connection.ts b/packages/gdrive/src/gdrive/check-connection.ts index a279edf1..bae7ca6a 100644 --- a/packages/gdrive/src/gdrive/check-connection.ts +++ b/packages/gdrive/src/gdrive/check-connection.ts @@ -1,12 +1,12 @@ import type { CheckConnectionResult, Request } from '@hesprs/sync-engine-sdk'; +import { getMessage } from '@repo/shared/error'; import { DRIVE_API, buildUrl, parseDriveError } from './api'; export default async function checkConnection(request: Request): Promise { try { - const response = await request({ + const response = await request(buildUrl(DRIVE_API, '/about', { fields: 'storageQuota' }), { method: 'GET', throw: false, - url: buildUrl(DRIVE_API, '/about', { fields: 'storageQuota' }), }); if (response.status >= 200 && response.status < 300) return { success: true } as const; return { @@ -14,7 +14,7 @@ export default async function checkConnection(request: Request): Promise { - const response = await this.request(Object.assign(params, { throw: false })); + private async requestOrThrow(url: string, params: RequestParam = {}): Promise { + const response = await this.request(url, { ...params, throw: false }); if (response.status >= 200 && response.status < 300) return response; const error = new Error( parseDriveError(response) ?? - `Google Drive request failed: ${response.status} ${params.method} ${params.url}`, + `Google Drive request failed: ${response.status} ${params.method} ${url}`, ); (error as { status?: number }).status = response.status; throw error; @@ -109,14 +109,14 @@ export default class GdriveFs implements RootFs { const last = index === segments.length - 1; const childKey = `${prefix}${segment}${last && !isFolder(key) ? '' : '/'}`; const folder = !last || isFolder(key); - const response = await this.requestOrThrow({ - method: 'GET', - url: buildUrl(DRIVE_API, '/files', { + const response = await this.requestOrThrow( + buildUrl(DRIVE_API, '/files', { fields: 'files(id)', pageSize: '1', q: `'${parentId}' in parents and name = '${escapeQuery(segment)}' and mimeType ${folder ? '=' : '!='} '${FOLDER_MIME}' and trashed = false`, }), - }); + { method: 'GET' }, + ); const id = response.json().files?.[0]?.id; if (!id) return undefined; this.ids.set(childKey, id); @@ -159,10 +159,10 @@ export default class GdriveFs implements RootFs { async read(key: string): Promise { const id = this.resolveId(key); if (id === undefined) throw notFoundError(key); - const response = await this.requestOrThrow({ - method: 'GET', - url: buildUrl(DRIVE_API, `/files/${id}`, { alt: 'media' }), - }); + const response = await this.requestOrThrow( + buildUrl(DRIVE_API, `/files/${id}`, { alt: 'media' }), + { method: 'GET' }, + ); return response.bytes(); } @@ -174,10 +174,9 @@ export default class GdriveFs implements RootFs { chunkSize, concurrency, requestRange: async (start, endInclusive) => { - const response = await this.requestOrThrow({ + const response = await this.requestOrThrow(url, { headers: { Range: `bytes=${start}-${endInclusive}` }, method: 'GET', - url, }); return response.bytes(); }, @@ -214,19 +213,17 @@ export default class GdriveFs implements RootFs { async delete(key: string): Promise { const id = this.resolveId(key); if (id === undefined) return; + const trashed = this.options.useTrash; try { await this.requestOrThrow( - this.options.useTrash + buildUrl(DRIVE_API, `/files/${id}`, trashed ? { fields: 'id' } : {}), + trashed ? { body: textToUint8Array(JSON.stringify({ trashed: true })), headers: { 'Content-Type': 'application/json; charset=UTF-8' }, method: 'PATCH', - url: buildUrl(DRIVE_API, `/files/${id}`, { fields: 'id' }), } - : { - method: 'DELETE', - url: buildUrl(DRIVE_API, `/files/${id}`), - }, + : { method: 'DELETE' }, ); } catch (error) { if (getStatus(error) !== 404) throw error; @@ -245,11 +242,10 @@ export default class GdriveFs implements RootFs { query.addParents = newParentId; if (oldParentId !== undefined) query.removeParents = oldParentId; } - await this.requestOrThrow({ + await this.requestOrThrow(buildUrl(DRIVE_API, `/files/${id}`, query), { body: textToUint8Array(JSON.stringify({ name: basename(newKey) })), headers: { 'Content-Type': 'application/json; charset=UTF-8' }, method: 'PATCH', - url: buildUrl(DRIVE_API, `/files/${id}`, query), }); this.dropCache(oldKey); } @@ -263,14 +259,20 @@ export default class GdriveFs implements RootFs { parentId = this.resolveId(parent); } if (!parentId) throw new Error(`Parent is not created when creating "${key}"!`); - const response = await this.requestOrThrow({ - body: textToUint8Array( - JSON.stringify({ mimeType: FOLDER_MIME, name: basename(key), parents: [parentId] }), - ), - headers: { 'Content-Type': 'application/json; charset=UTF-8' }, - method: 'POST', - url: buildUrl(DRIVE_API, '/files', { fields: 'id' }), - }); + const response = await this.requestOrThrow( + buildUrl(DRIVE_API, '/files', { fields: 'id' }), + { + body: textToUint8Array( + JSON.stringify({ + mimeType: FOLDER_MIME, + name: basename(key), + parents: [parentId], + }), + ), + headers: { 'Content-Type': 'application/json; charset=UTF-8' }, + method: 'POST', + }, + ); const created = response.json(); if (!created.id) throw new Error('Google Drive did not return an id for a created folder!'); this.ids.set(key, created.id); @@ -285,7 +287,7 @@ export default class GdriveFs implements RootFs { pageSize: '1', q: `'${parentId}' in parents and name = '${escapeQuery(basename(key))}' and trashed = false`, }); - const response = await this.requestOrThrow({ method: 'GET', url }); + const response = await this.requestOrThrow(url, { method: 'GET' }); const entry = response.json().files?.[0]; if (!entry) throw notFoundError(key); return toFileStat(key, entry); @@ -312,9 +314,8 @@ export default class GdriveFs implements RootFs { q: 'trashed = false', }; if (pageToken) query.pageToken = pageToken; - const response = await this.requestOrThrow({ + const response = await this.requestOrThrow(buildUrl(DRIVE_API, '/files', query), { method: 'GET', - url: buildUrl(DRIVE_API, '/files', query), }); const parsed = response.json(); all.push(...(parsed.files ?? [])); diff --git a/packages/gdrive/src/gdrive/upload.ts b/packages/gdrive/src/gdrive/upload.ts index 4c00cf88..75c9ca1b 100644 --- a/packages/gdrive/src/gdrive/upload.ts +++ b/packages/gdrive/src/gdrive/upload.ts @@ -53,7 +53,7 @@ async function startSession({ size, url, }: SessionOptions): Promise<{ request: Request; location: string }> { - const response = await request({ + const response = await request(url, { body: textToUint8Array(JSON.stringify(metadata)), headers: { 'Content-Type': 'application/json; charset=UTF-8', @@ -61,7 +61,6 @@ async function startSession({ }, method, throw: false, - url, }); if (response.status < 200 || response.status >= 300) throw new Error( @@ -81,14 +80,13 @@ async function putChunk( total: number, ): Promise { const end = start + chunk.byteLength - 1; - const response = await request({ + const response = await request(location, { body: chunk, headers: { 'Content-Range': end < start ? `bytes */${total}` : `bytes ${start}-${end}/${total}`, }, method: 'PUT', throw: false, - url: location, }); if (response.status === 308) return; if (response.status >= 200 && response.status < 300) return response; @@ -106,12 +104,11 @@ export async function singleUpload(options: MultipartOptions, value: Binary): Pr value, textToUint8Array(`\r\n--${boundary}--`), ); - const response = await options.request({ + const response = await options.request(options.url, { body, headers: { 'Content-Type': `multipart/related; boundary=${boundary}` }, method: options.method, throw: false, - url: options.url, }); if (response.status < 200 || response.status >= 300) throw new Error( @@ -138,7 +135,7 @@ export async function resumableUpload( }).catch((error: unknown) => { // Best-effort session cancellation; Drive also expires sessions on its own. void options - .request({ ignoreCancellation: true, method: 'DELETE', url: session.location }) + .request(session.location, { ignoreCancellation: true, method: 'DELETE' }) .catch(() => {}); throw error; }); diff --git a/packages/gdrive/src/setting.ts b/packages/gdrive/src/setting.ts index d8cc0a00..ca632e7d 100644 --- a/packages/gdrive/src/setting.ts +++ b/packages/gdrive/src/setting.ts @@ -11,6 +11,7 @@ import type { } from '@hesprs/sync-engine-sdk'; import type { App, SettingGroupItem } from 'obsidian'; import { s } from '@hesprs/sync-engine-sdk'; +import { getMessage } from '@repo/shared/error'; import { normalizeBaseDir } from '@repo/shared/path'; import { Modal, Notice, Setting } from 'obsidian'; import type { TokenManager } from './gdrive/auth'; @@ -142,7 +143,7 @@ export default function gdriveSetting( } } catch (error) { if (cancelled) return; - const reason = error instanceof Error ? error.message : String(error); + const reason = getMessage(error); new Notice(translate('authorizationFailed', reason), 5); dispatch('errorGeneral', `Google Drive auth failed: \`${reason}\`.`); } finally { diff --git a/packages/gdrive/test/auth.test.ts b/packages/gdrive/test/auth.test.ts index 14cd2325..76b68e15 100644 --- a/packages/gdrive/test/auth.test.ts +++ b/packages/gdrive/test/auth.test.ts @@ -181,8 +181,7 @@ test('caches tokens and retries bearer requests after a 401', async () => { }; const manager = new TokenManager(storage as unknown as SecretStorage); const seen: Array = []; - const req = request((params) => { - if (typeof params === 'string') throw new Error('Unexpected string request'); + const req = request((url, params) => { seen.push(params.headers?.Authorization); if (seen.length === 1) { const error = new Error('Unauthorized') as Error & { status: number }; @@ -193,6 +192,6 @@ test('caches tokens and retries bearer requests after a 401', async () => { }); const wrapped = bearerMiddleware(req.request, manager); - expect((await wrapped({ method: 'GET', url: 'https://drive.test' })).status).toBe(200); + expect((await wrapped('https://drive.test')).status).toBe(200); expect(seen).toStrictEqual(['Bearer first', 'Bearer second']); }); diff --git a/packages/gdrive/test/fs-gdrive.test.ts b/packages/gdrive/test/fs-gdrive.test.ts index 536bc4b8..02d53705 100644 --- a/packages/gdrive/test/fs-gdrive.test.ts +++ b/packages/gdrive/test/fs-gdrive.test.ts @@ -1,5 +1,4 @@ -import type { Binary, RequestParam } from '@hesprs/sync-engine-sdk'; -import type { ResponseControl, ResponseOverrides } from '@hesprs/sync-engine-sdk/dev'; +import type { Binary, MaybePromise, RequestParam, RequestResponse } from '@hesprs/sync-engine-sdk'; import { testKit } from '@hesprs/sync-engine-sdk/dev'; import { beforeEach, expect, test } from 'bun:test'; import { openMemoryDB } from 'uni-kv'; @@ -12,27 +11,26 @@ const db: GdriveDB = openMemoryDB<{ gdriveIds: string }, { gdriveIdsMarker?: str 'gdrive-fs-test', ); -function response( - value: unknown = {}, - status = 200, - headers: Record = {}, -): ResponseOverrides { +type Control = (url: string, params: RequestParam) => MaybePromise>; + +function response(value: unknown = {}, status = 200, headers: Record = {}) { const body = new TextEncoder().encode(JSON.stringify(value)); return { bytes: () => body, headers, - json: () => value, + // oxlint-disable-next-line typescript/no-unnecessary-type-parameters + json: () => value as T, status, text: () => new TextDecoder().decode(body), }; } -function binaryResponse(value: Binary, status = 200): ResponseOverrides { +function binaryResponse(value: Binary, status = 200) { return { ...response({}, status), bytes: () => value }; } -function createFs(handler: ResponseControl) { - const harness = request(handler); +function createFs(handler: Control) { + const harness = request(handler); return { calls: harness.calls, fs: new GdriveFs(harness.request, { useTrash: true, userId: 'user-1' }, db), @@ -45,12 +43,11 @@ beforeEach(() => { }); test('writes and reads a file through Drive multipart upload', async () => { - const { calls, fs } = createFs((params) => { - if (params.url.startsWith(DRIVE_UPLOAD_API) && params.method === 'POST') + const { calls, fs } = createFs((url, params) => { + if (url.startsWith(DRIVE_UPLOAD_API) && params.method === 'POST') return response({ id: 'file-1', md5Checksum: 'drive-uid' }); - if (params.url === `${DRIVE_API}/files/file-1?alt=media`) - return binaryResponse(bytes('hello')); - throw new Error(`Unexpected request: ${params.method} ${params.url}`); + if (url === `${DRIVE_API}/files/file-1?alt=media`) return binaryResponse(bytes('hello')); + throw new Error(`Unexpected request: ${params.method} ${url}`); }); const stat = file('note.md', { mtime: 1_700_000_000_000, size: 5 }); @@ -65,8 +62,8 @@ test('writes and reads a file through Drive multipart upload', async () => { }); test('creates folders, lists visible descendants, and honors excluded subtrees', async () => { - const { calls, fs } = createFs((params) => { - if (params.method === 'POST' && params.url.startsWith(`${DRIVE_API}/files`)) + const { calls, fs } = createFs((url, params) => { + if (params.method === 'POST' && url.startsWith(`${DRIVE_API}/files`)) return response({ id: 'folder-1' }); return response({ files: [ @@ -94,11 +91,11 @@ test('creates folders, lists visible descendants, and honors excluded subtrees', }); test('moves a cached file with Drive native rename', async () => { - const { calls, fs } = createFs((params) => { - if (params.method === 'POST' && params.url.startsWith(DRIVE_UPLOAD_API)) + const { calls, fs } = createFs((url, params) => { + if (params.method === 'POST' && url.startsWith(DRIVE_UPLOAD_API)) return response({ id: 'file-1' }); if (params.method === 'PATCH') return response({ id: 'file-1' }); - throw new Error(`Unexpected request: ${params.method} ${params.url}`); + throw new Error(`Unexpected request: ${params.method} ${url}`); }); await fs.write('old.md', bytes('x'), file('old.md', { size: 1 })); diff --git a/packages/plugin/dist/dev.spec.d.ts b/packages/plugin/dist/dev.spec.d.ts index 5d6394f8..65ee48e3 100644 --- a/packages/plugin/dist/dev.spec.d.ts +++ b/packages/plugin/dist/dev.spec.d.ts @@ -1,4 +1,4 @@ -import { At as FileStat, Dt as RootFs, Ft as RecordStatsMap, It as Stat, Lt as StatsMap, Mt as MaybePromise, Ot as WrappedFs, Pt as RecordStat, Rt as Binary, d as RequestParam, f as RequestResponse, jt as FolderStat, lt as TaskNames, q as Decider, u as Request, yt as Fs } from "./index-DI2LCB_s.spec.js"; +import { At as FileStat, Dt as RootFs, Ft as RecordStatsMap, It as Stat, Lt as StatsMap, Mt as MaybePromise, Ot as WrappedFs, Pt as RecordStat, Rt as Binary, d as RequestParam, f as RequestResponse, jt as FolderStat, lt as TaskNames, q as Decider, u as Request, yt as Fs } from "./index-C66-NLE8.spec.js"; //#region src/sdk/debug-wrapper.d.ts declare function debugWrapper(original: Fs, log: (content: string) => void): WrappedFs; //#endregion @@ -24,8 +24,11 @@ type FsHarness = { control: Fs; fs: RootFs; }; -type RequestHarness = { - calls: Array; +type ResponseControl = (url: string, params: RequestParam) => MaybePromise; +type RequestHarness = { + calls: Array; request: Request; }; type ExtractedTask = { @@ -61,8 +64,7 @@ declare function flush(turns?: number): Promise; type ResponseOverrides = Partial> & { json?: () => unknown; }; -type ResponseControl = (params: T) => MaybePromise; -declare function request(control: ResponseControl): RequestHarness; +declare function request(control: ResponseControl): RequestHarness; declare function fs(options?: FsOptions): FsHarness; declare const testKit: { bytes: typeof bytes; @@ -84,4 +86,4 @@ declare const testKit: { //#region src/utils/sha-256.d.ts declare function sha256(input: string): Promise; //#endregion -export { type ResponseControl, type ResponseOverrides, debugWrapper, sha256, testKit }; \ No newline at end of file +export { debugWrapper, sha256, testKit }; \ No newline at end of file diff --git a/packages/plugin/dist/index-DI2LCB_s.spec.d.ts b/packages/plugin/dist/index-C66-NLE8.spec.d.ts similarity index 99% rename from packages/plugin/dist/index-DI2LCB_s.spec.d.ts rename to packages/plugin/dist/index-C66-NLE8.spec.d.ts index 56d37ff7..4e687a61 100644 --- a/packages/plugin/dist/index-DI2LCB_s.spec.d.ts +++ b/packages/plugin/dist/index-C66-NLE8.spec.d.ts @@ -1249,7 +1249,12 @@ type TriggerEntry = { priority: number; options?: () => SyncOptions; }; -type RequestParam = Omit & { +type Infras = { + localFs: Fs; + remoteFs: Fs; + record: RecordStore; +}; +type RequestParam = Omit & { body?: string | Binary; ignoreCancellation?: boolean; }; @@ -1260,12 +1265,7 @@ type RequestResponse = { headers: Record; status: number; }; -type Request = (params: RequestParam | string) => Promise; -type Infras = { - localFs: Fs; - remoteFs: Fs; - record: RecordStore; -}; +type Request = (url: string, params?: RequestParam) => Promise; declare class Registrar { private readonly ctx; private readonly cleanupCallbacks; @@ -1367,7 +1367,6 @@ type VaultRequestParam = ({ method: 'LIST'; cached?: boolean; }) & { - key: string; ignoreCancellation?: boolean; }; type VaultRequestResponseMap = { @@ -1382,7 +1381,9 @@ type VaultRequestResponseMap = { STAT: Stat; LIST: ListedFiles; }; -type VaultRequest = (params: T) => Promise; +type VaultRequest = (key: string, params?: T) => Promise; type TrashOption = 'local' | 'system' | 'permanent'; //#endregion export { RemoveRecord as $, setNeedMigration as A, FileStat as At, ObsidianLanguageCode as B, digOriginal as C, MoveAtom as Ct, readWithSize as D, RootFs as Dt, pipe as E, OutputAtom as Et, CallableOrObjectTree as F, RecordStatsMap as Ft, On as G, Translate as H, SettingEntry as I, Stat$1 as It, DeciderInput as J, CreateLocalDir as K, AugmentedModuleMeta as L, StatsMap as Lt, generateEditableList as M, MaybePromise as Mt, reactivelyValidate as N, Progress as Nt, writeWithValue as O, WrappedFs as Ot, s as P, RecordStat as Pt, RemoveRemote as Q, ModuleMeta as R, Binary as Rt, SelectFromContext as S, MkdirAtom as St, concurrency as T, OptimizerOutput as Tt, TranslationResource as U, Snippet as V, Dispatch as W, Upload as X, TaskFactory as Y, ResolveConflict as Z, Context as _, CustomAtom as _t, FsWrapperEntry as a, AddRecord as at, Translations as b, InputAtom as bt, RemoteFsEntry as c, ConflictResolverPayload as ct, RequestParam as d, DatabaseAsync as dt, RemoveLocal as et, RequestResponse as f, DatabaseSync as ft, SyncTerminateReason as g, BatchOptimizer as gt, SyncOptions as h, StoreSync as ht, DeciderEntry as i, CreateRemoteDir as it, LabelDefinition as j, FolderStat as jt, prefixWrapper as k, WriteAtom as kt, RemoteRequestMiddlewareEntry as l, TaskNames as lt, RemoteLister as m, StoreOperations as mt, CheckConnectionResult as n, MoveLocal as nt, LocalRequestMiddlewareEntry as o, BaseTask as ot, TriggerEntry as p, StoreAsync as pt, Decider as q, ConflictResolverEntry as r, Download as rt, OptimizerEntry as s, ConflictResolver as st, VaultRequest as t, MoveRemote as tt, Request as u, RecordStore as ut, Events as v, DeleteAtom as vt, chunkSize as w, OptimizerInput as wt, ExistingMemoryDB as x, ListReporter as xt, Settings as y, Fs as yt, Fragment as z }; \ No newline at end of file diff --git a/packages/plugin/dist/index.spec.d.ts b/packages/plugin/dist/index.spec.d.ts index 09e152c3..e3844332 100644 --- a/packages/plugin/dist/index.spec.d.ts +++ b/packages/plugin/dist/index.spec.d.ts @@ -1,2 +1,2 @@ -import { $ as RemoveRecord, A as setNeedMigration, At as FileStat, B as ObsidianLanguageCode, C as digOriginal, Ct as MoveAtom, D as readWithSize, Dt as RootFs, E as pipe, Et as OutputAtom, F as CallableOrObjectTree, Ft as RecordStatsMap, G as On, H as Translate, I as SettingEntry, It as Stat, J as DeciderInput, K as CreateLocalDir, L as AugmentedModuleMeta, Lt as StatsMap, M as generateEditableList, Mt as MaybePromise, N as reactivelyValidate, Nt as Progress, O as writeWithValue, Ot as WrappedFs, P as s, Pt as RecordStat, Q as RemoveRemote, R as ModuleMeta, Rt as Binary, S as SelectFromContext, St as MkdirAtom, T as concurrency, Tt as OptimizerOutput, U as TranslationResource, V as Snippet, W as Dispatch, X as Upload, Y as TaskFactory, Z as ResolveConflict, _ as Context, _t as CustomAtom, a as FsWrapperEntry, at as AddRecord, b as Translations, bt as InputAtom, c as RemoteFsEntry, ct as ConflictResolverPayload, d as RequestParam, dt as DatabaseAsync, et as RemoveLocal, f as RequestResponse, ft as DatabaseSync, g as SyncTerminateReason, gt as BatchOptimizer, h as SyncOptions, ht as StoreSync, i as DeciderEntry, it as CreateRemoteDir, j as LabelDefinition, jt as FolderStat, k as prefixWrapper, kt as WriteAtom, l as RemoteRequestMiddlewareEntry, lt as TaskNames, m as RemoteLister, mt as StoreOperations, n as CheckConnectionResult, nt as MoveLocal, o as LocalRequestMiddlewareEntry, ot as BaseTask, p as TriggerEntry, pt as StoreAsync, q as Decider, r as ConflictResolverEntry, rt as Download, s as OptimizerEntry, st as ConflictResolver, t as VaultRequest, tt as MoveRemote, u as Request, ut as RecordStore, v as Events, vt as DeleteAtom, w as chunkSize, wt as OptimizerInput, x as ExistingMemoryDB, xt as ListReporter, y as Settings, yt as Fs, z as Fragment } from "./index-DI2LCB_s.spec.js"; +import { $ as RemoveRecord, A as setNeedMigration, At as FileStat, B as ObsidianLanguageCode, C as digOriginal, Ct as MoveAtom, D as readWithSize, Dt as RootFs, E as pipe, Et as OutputAtom, F as CallableOrObjectTree, Ft as RecordStatsMap, G as On, H as Translate, I as SettingEntry, It as Stat, J as DeciderInput, K as CreateLocalDir, L as AugmentedModuleMeta, Lt as StatsMap, M as generateEditableList, Mt as MaybePromise, N as reactivelyValidate, Nt as Progress, O as writeWithValue, Ot as WrappedFs, P as s, Pt as RecordStat, Q as RemoveRemote, R as ModuleMeta, Rt as Binary, S as SelectFromContext, St as MkdirAtom, T as concurrency, Tt as OptimizerOutput, U as TranslationResource, V as Snippet, W as Dispatch, X as Upload, Y as TaskFactory, Z as ResolveConflict, _ as Context, _t as CustomAtom, a as FsWrapperEntry, at as AddRecord, b as Translations, bt as InputAtom, c as RemoteFsEntry, ct as ConflictResolverPayload, d as RequestParam, dt as DatabaseAsync, et as RemoveLocal, f as RequestResponse, ft as DatabaseSync, g as SyncTerminateReason, gt as BatchOptimizer, h as SyncOptions, ht as StoreSync, i as DeciderEntry, it as CreateRemoteDir, j as LabelDefinition, jt as FolderStat, k as prefixWrapper, kt as WriteAtom, l as RemoteRequestMiddlewareEntry, lt as TaskNames, m as RemoteLister, mt as StoreOperations, n as CheckConnectionResult, nt as MoveLocal, o as LocalRequestMiddlewareEntry, ot as BaseTask, p as TriggerEntry, pt as StoreAsync, q as Decider, r as ConflictResolverEntry, rt as Download, s as OptimizerEntry, st as ConflictResolver, t as VaultRequest, tt as MoveRemote, u as Request, ut as RecordStore, v as Events, vt as DeleteAtom, w as chunkSize, wt as OptimizerInput, x as ExistingMemoryDB, xt as ListReporter, y as Settings, yt as Fs, z as Fragment } from "./index-C66-NLE8.spec.js"; export { type AddRecord, type AugmentedModuleMeta, type BaseTask, type BatchOptimizer, type Binary, type CallableOrObjectTree, type CheckConnectionResult, type ConflictResolver, type ConflictResolverEntry, type ConflictResolverPayload, type Context, type CreateLocalDir, type CreateRemoteDir, type CustomAtom, type DatabaseAsync, type DatabaseSync, type Decider, type DeciderEntry, type DeciderInput, type DeleteAtom, type Dispatch, type Download, type Events, type ExistingMemoryDB, type FileStat, type FolderStat, type Fragment, type Fs, type FsWrapperEntry, type InputAtom, type LabelDefinition, type ListReporter, type LocalRequestMiddlewareEntry, type MaybePromise, type MkdirAtom, type ModuleMeta, type MoveAtom, type MoveLocal, type MoveRemote, type ObsidianLanguageCode, type On, type OptimizerEntry, type OptimizerInput, type OptimizerOutput, type OutputAtom, type Progress, type RecordStat, type RecordStatsMap, type RecordStore, type RemoteFsEntry, type RemoteLister, type RemoteRequestMiddlewareEntry, type RemoveLocal, type RemoveRecord, type RemoveRemote, type Request, type RequestParam, type RequestResponse, type ResolveConflict, type RootFs, SelectFromContext, type SettingEntry, type Settings, type Snippet, type Stat, type StatsMap, type StoreAsync, type StoreOperations, type StoreSync, type SyncOptions, type SyncTerminateReason, type TaskFactory, type TaskNames, type Translate, type TranslationResource, type Translations, type TriggerEntry, type Upload, type VaultRequest, type WrappedFs, type WriteAtom, chunkSize, concurrency, digOriginal, generateEditableList, pipe, prefixWrapper, reactivelyValidate, readWithSize, s, setNeedMigration, writeWithValue }; \ No newline at end of file diff --git a/packages/plugin/src/components/MigrationModal.ts b/packages/plugin/src/components/MigrationModal.ts index c534a709..ec937670 100644 --- a/packages/plugin/src/components/MigrationModal.ts +++ b/packages/plugin/src/components/MigrationModal.ts @@ -1,5 +1,6 @@ import type { Events } from '@'; import type { App, ToggleComponent } from 'obsidian'; +import { getMessage } from '@repo/shared/error'; import { Modal, Notice, Setting } from 'obsidian'; import { ref } from 'synthkernel'; import type { ExistingMemoryDB } from '@/modules/Bootstrap'; @@ -10,7 +11,6 @@ import type { SyncTerminateReason } from '@/modules/Sync'; import type { MaybePromise } from '@/types'; import renderProgress from '@/components/render-progress'; import roundPercent from '@/utils/round-percent'; -import toErrorMessage from '@/utils/to-error-message'; export type MigrationModalTranslations = { cancel: string; @@ -153,7 +153,7 @@ class MigrationModal extends Modal { .map((key) => remoteFs.delete(key)), ]); } catch (error) { - const message = toErrorMessage(error); + const message = getMessage(error); new Notice(`${translate('migrationFailed')}: ${message}`); return { reason: `Phase 2: ${message}`, success: false }; } diff --git a/packages/plugin/src/fs/middlewares/custom-headers.ts b/packages/plugin/src/fs/middlewares/custom-headers.ts index a79ffb52..422c190e 100644 --- a/packages/plugin/src/fs/middlewares/custom-headers.ts +++ b/packages/plugin/src/fs/middlewares/custom-headers.ts @@ -6,9 +6,6 @@ export default function customHeadersMiddleware( request: Request, options: CustomHeadersOptions, ): Request { - return (arg) => { - if (typeof arg === 'string') arg = { url: arg }; - arg.headers = { ...arg.headers, ...options }; - return request(arg); - }; + return (url, params) => + request(url, { ...params, headers: { ...params?.headers, ...options } }); } diff --git a/packages/plugin/src/fs/middlewares/retry.ts b/packages/plugin/src/fs/middlewares/retry.ts index 3b15cfab..fa1be855 100644 --- a/packages/plugin/src/fs/middlewares/retry.ts +++ b/packages/plugin/src/fs/middlewares/retry.ts @@ -1,5 +1,5 @@ -import type { ErrorLike } from '@repo/shared/get-status'; -import { getStatus } from '@repo/shared/get-status'; +import type { ErrorLike } from '@repo/shared/error'; +import { getStatus } from '@repo/shared/error'; import type { Request } from '@/modules/Registrar'; type RetryOptions = { @@ -15,10 +15,10 @@ const backoff = (count: number, baseMs = 1000, maxMs = 30_000): number => { export default function retryMiddleware(request: Request, options?: RetryOptions): Request { const { maxRetry = 4, isRetryable = isRetryableError, retryDelay = backoff } = options ?? {}; - return async (args) => { + return async (url, params) => { for (let i = 0; ; i++) try { - const response = await request(args); + const response = await request(url, params); if (RETRYABLE_STATUS_CODES.has(response.status) && i < maxRetry) { await sleep(retryDelay(i)); continue; diff --git a/packages/plugin/src/fs/vault/index.ts b/packages/plugin/src/fs/vault/index.ts index e88a9287..7ce4e5bb 100644 --- a/packages/plugin/src/fs/vault/index.ts +++ b/packages/plugin/src/fs/vault/index.ts @@ -33,11 +33,11 @@ export default class VaultFs implements RootFs { } read(key: string): Promise { - return this.request({ key, method: 'GET' }); + return this.request(key); } readStream(key: string, { size }: FileStat) { - return this.request({ key, method: 'GET_STREAM', size }); + return this.request(key, { method: 'GET_STREAM', size }); } async write(key: string, value: Binary): Promise { @@ -46,7 +46,7 @@ export default class VaultFs implements RootFs { let uid: string | undefined; let trial = 0; do { - await this.request({ key, method: 'PUT', value }); + await this.request(key, { key, method: 'PUT', value }); uid = await getFileUid(this, key, value.byteLength); trial++; } while (!uid && trial < MAX_WRITE_TRIAL); @@ -62,18 +62,17 @@ export default class VaultFs implements RootFs { while (true) { const result = await reader.read(); if (result.done) break; - await this.request({ key: tempPath, method: 'APPEND', value: result.value }); + await this.request(tempPath, { method: 'APPEND', value: result.value }); } if (await this.exists(key)) - await this.request({ key, method: 'DELETE', trash: 'permanent' }); + await this.request(key, { method: 'DELETE', trash: 'permanent' }); await this.move(tempPath, key); return await getFileUid(this, key); } catch (error) { await Promise.all([ reader.cancel(), - this.request({ + this.request(tempPath, { ignoreCancellation: true, - key: tempPath, method: 'DELETE', trash: 'permanent', }), @@ -85,19 +84,19 @@ export default class VaultFs implements RootFs { } delete(key: string): Promise { - return this.request({ key, method: 'DELETE' }); + return this.request(key, { method: 'DELETE' }); } move(oldKey: string, newKey: string): Promise { - return this.request({ destination: newKey, key: oldKey, method: 'MOVE' }); + return this.request(oldKey, { destination: newKey, method: 'MOVE' }); } mkdir(key: string): Promise { - return this.request({ key, method: 'MKDIR' }); + return this.request(key, { method: 'MKDIR' }); } exists(key: string) { - return this.request({ key, method: 'EXISTS' }); + return this.request(key, { method: 'EXISTS' }); } async list(key: string, reporter: ListReporter): Promise> { @@ -106,11 +105,7 @@ export default class VaultFs implements RootFs { let total = 1; const visit = async (dir: string) => { // https://github.com/hesprs/sync-engine/issues/222 - const { files, folders } = await this.request({ - cached: false, - key: dir, - method: 'LIST', - }); + const { files, folders } = await this.request(dir, { cached: false, method: 'LIST' }); completed++; total += files.length + folders.length; await Promise.all([ @@ -137,7 +132,7 @@ export default class VaultFs implements RootFs { } async stat(key: string): Promise { - const { type, mtime, size } = await this.request({ key, method: 'STAT' }); + const { type, mtime, size } = await this.request(key, { method: 'STAT' }); return type === 'file' ? { isDir: false, key, mtime, size, uid: `${mtime}~${size}` } : { isDir: true, key }; diff --git a/packages/plugin/src/fs/vault/request.ts b/packages/plugin/src/fs/vault/request.ts index 4868b729..c18d7887 100644 --- a/packages/plugin/src/fs/vault/request.ts +++ b/packages/plugin/src/fs/vault/request.ts @@ -21,7 +21,7 @@ type VaultRequestParam = ( | { method: 'EXISTS' } | { method: 'STAT'; cached?: boolean } | { method: 'LIST'; cached?: boolean } -) & { key: string; ignoreCancellation?: boolean }; +) & { ignoreCancellation?: boolean }; type VaultRequestResponseMap = { GET: Binary; @@ -36,8 +36,9 @@ type VaultRequestResponseMap = { LIST: ListedFiles; }; -export type VaultRequest = ( - params: T, +export type VaultRequest = ( + key: string, + params?: T, ) => Promise; // Capacitor ranged local file request only supports those extensions @@ -88,13 +89,15 @@ export default function createVaultRequest(app: App): VaultRequest { const canUseCache = () => workspace.layoutReady; return async ( - params: T, + key: string, + params?: T, ): Promise => { - const { method, key } = params; const path = toVaultPath(key); + const get = () => adapter.readBinary(path).then((buffer) => toUint8Array(buffer)) as never; + if (!params) return get(); + const { method } = params; - if (method === 'GET') - return adapter.readBinary(path).then((buffer) => toUint8Array(buffer)) as never; + if (method === 'GET') return get(); if (method === 'GET_STREAM') { let url = adapter.getResourcePath(path); // Local file fetch streaming isn't supported in iOS diff --git a/packages/plugin/src/fs/wrappers/cancellation.ts b/packages/plugin/src/fs/wrappers/cancellation.ts index d11c4e56..aa529895 100644 --- a/packages/plugin/src/fs/wrappers/cancellation.ts +++ b/packages/plugin/src/fs/wrappers/cancellation.ts @@ -72,13 +72,9 @@ export function cancellationMiddleware< T extends (...args: ReadonlyArray) => Promise, >(request: T, isCancelled: Ref): T { return ((...params: Parameters) => { - const payload = params[0]; - if ( - payload && - typeof payload === 'object' && - (payload as { ignoreCancellation?: boolean }).ignoreCancellation - ) - return request(...params); + // Both `Request` and `VaultRequest` take options as their second argument. + const options = params[1] as { ignoreCancellation?: boolean } | undefined; + if (options?.ignoreCancellation) return request(...params); assertNotCancelled(isCancelled); const promise = new Promise>>((resolve, reject) => { const unsub = isCancelled.subscribe((cancelled) => { diff --git a/packages/plugin/src/modules/Extensibility.ts b/packages/plugin/src/modules/Extensibility.ts index a1463628..368ecdaa 100644 --- a/packages/plugin/src/modules/Extensibility.ts +++ b/packages/plugin/src/modules/Extensibility.ts @@ -4,12 +4,12 @@ import type { Ref } from 'synthkernel'; import type { DatabaseAsync, StoreAsync, StoreOperations } from 'uni-kv'; import hash from '@repo/shared/crypto'; import { importCode } from '@repo/shared/e2e-utils.spec'; +import { getMessage } from '@repo/shared/error'; import obsidian, { Notice, requestUrl } from 'obsidian'; import { compare } from 'verkit'; import type { General } from '@/types'; import UntrustedModuleModal from '@/components/UntrustedModuleModal'; import sha256 from '@/utils/sha-256'; -import toErrorMessage from '@/utils/to-error-message'; import untilTrue from '@/utils/until-true'; import type { Dispatch } from './EventBus'; import type { Snippet, Translate } from './I18n'; @@ -189,7 +189,7 @@ export default class Extensibility { Object.assign(discoveredMeta, { enabled: false }), ); } - const message = toErrorMessage(error); + const message = getMessage(error); dispatch('errorGeneral', `Module \`${id}\` failed to load: ${message}`); new Notice(`${translate('failedToLoadModule', name)}: ${message}`); } @@ -234,7 +234,7 @@ export default class Extensibility { } await this.installModule(meta, module); } catch (error) { - const message = toErrorMessage(error); + const message = getMessage(error); dispatch('errorGeneral', `Failed to download module \`${id}\`: ${message}`); new Notice(`${translate('failedToDownloadModule', name)}: ${message}`); } @@ -262,7 +262,7 @@ export default class Extensibility { this.sourceCache.set(url, content); return content as Array; } catch (error) { - const message = toErrorMessage(error); + const message = getMessage(error); dispatch('errorGeneral', `Failed to fetch source from \`${url}\`: ${message}`); if (manual) new Notice(`${translate('failedToFetchSource', url)}: ${message}`); return []; diff --git a/packages/plugin/src/modules/Observability.ts b/packages/plugin/src/modules/Observability.ts index cdb21da7..f66d0c1f 100644 --- a/packages/plugin/src/modules/Observability.ts +++ b/packages/plugin/src/modules/Observability.ts @@ -1,11 +1,11 @@ import type { Events, Translations } from '@'; import type { App, Command, DataAdapter, IconName } from 'obsidian'; import type { Ref } from 'synthkernel'; +import { getMessage } from '@repo/shared/error'; import { Notice, Platform, setIcon } from 'obsidian'; import { computed, ref } from 'synthkernel'; import type { Progress } from '@/types'; import roundPercent from '@/utils/round-percent'; -import toErrorMessage from '@/utils/to-error-message'; import { formatTime } from '@/utils/unit-converter'; import type { Dispatch, On } from './EventBus'; import type { Translate } from './I18n'; @@ -298,7 +298,7 @@ export default class Observability { const file = await app.vault.create(filePath, log); await app.workspace.getLeaf().openFile(file); } catch (error) { - const message = toErrorMessage(error); + const message = getMessage(error); new Notice(`${translate('exportLogsFailed')}: ${message}`); dispatch('errorGeneral', `Export log failed: \`${message}\`.`); } diff --git a/packages/plugin/src/modules/Registrar.ts b/packages/plugin/src/modules/Registrar.ts index a7508a50..48c1d37c 100644 --- a/packages/plugin/src/modules/Registrar.ts +++ b/packages/plugin/src/modules/Registrar.ts @@ -33,7 +33,9 @@ export type OptimizerEntry = OrderedApplyEntry; export type TriggerEntry = { priority: number; options?: () => SyncOptions }; -export type RequestParam = Omit & { +export type Infras = { localFs: Fs; remoteFs: Fs; record: RecordStore }; + +export type RequestParam = Omit & { body?: string | Binary; ignoreCancellation?: boolean; }; @@ -45,14 +47,13 @@ export type RequestResponse = { headers: Record; status: number; }; -export type Request = (params: RequestParam | string) => Promise; - -export type Infras = { localFs: Fs; remoteFs: Fs; record: RecordStore }; - -const request: Request = async (params: RequestParam | string) => { - if (typeof params === 'object' && params.body instanceof Uint8Array) - (params as RequestUrlParam).body = toArrayBuffer(params.body); - const response = await requestUrl(params as RequestUrlParam); +export type Request = (url: string, params?: RequestParam) => Promise; +const request: Request = async (url: string, params?: RequestParam) => { + const body = params?.body; + if (body instanceof Uint8Array) (params as RequestUrlParam).body = toArrayBuffer(body); + const response = await requestUrl( + params ? (Object.assign(params, { url }) as RequestUrlParam) : url, + ); return { bytes: () => toUint8Array(response.arrayBuffer), headers: response.headers, diff --git a/packages/plugin/src/modules/Sync.ts b/packages/plugin/src/modules/Sync.ts index dfeacb01..38fc4d7f 100644 --- a/packages/plugin/src/modules/Sync.ts +++ b/packages/plugin/src/modules/Sync.ts @@ -1,5 +1,6 @@ import type { Events, Translations } from '@'; import type { Ref } from 'synthkernel'; +import { getMessage } from '@repo/shared/error'; import { isSub } from '@repo/shared/path'; import { ref } from 'synthkernel'; import type { Fs, ListReporter } from '@/fs'; @@ -31,7 +32,6 @@ import { taskMap, } from '@/sync'; import { prepareGlobMatch } from '@/utils/glob-match'; -import toErrorMessage from '@/utils/to-error-message'; import type { Dispatch, On } from './EventBus'; import type { Translate } from './I18n'; import type { DeleteConfirmReturn } from './ProgressModal'; @@ -265,7 +265,7 @@ export default class Sync { failedCount++; dispatch('taskFailed', { ...toTaskInfo(task), - error: toErrorMessage(error), + error: getMessage(error), }); } }), @@ -282,7 +282,7 @@ export default class Sync { } catch (error) { terminateReason = isCancelled() ? { result: 'cancelled' } - : ({ error: toErrorMessage(error), result: 'failed' } as const); + : ({ error: getMessage(error), result: 'failed' } as const); } finally { cleanup(); dispatch('syncTerminated', terminateReason); diff --git a/packages/plugin/src/sdk/dev.ts b/packages/plugin/src/sdk/dev.ts index f1430d65..66c1f5d7 100644 --- a/packages/plugin/src/sdk/dev.ts +++ b/packages/plugin/src/sdk/dev.ts @@ -1,3 +1,3 @@ export { default as debugWrapper } from './debug-wrapper'; -export { default as testKit, type ResponseControl, type ResponseOverrides } from '$/test-kit'; +export { default as testKit } from '$/test-kit'; export { default as sha256 } from '@/utils/sha-256'; diff --git a/packages/plugin/src/settings/head.ts b/packages/plugin/src/settings/head.ts index dc2cca2c..dffaace4 100644 --- a/packages/plugin/src/settings/head.ts +++ b/packages/plugin/src/settings/head.ts @@ -1,5 +1,6 @@ import type { Context, Events, Settings } from '@'; import type { DatabaseSync } from 'uni-kv'; +import { getMessage } from '@repo/shared/error'; import { ExtraButtonComponent, Notice, PluginSettingTab, setTooltip } from 'obsidian'; import type { ModuleCtor } from '@/modules/Extensibility'; import type { Fragment, Snippet, Translate } from '@/modules/I18n'; @@ -12,7 +13,6 @@ import type { import type { CallableOrObjectTree } from '@/modules/Setting'; import type { Dispatch } from '@/sdk'; import type { General, MaybePromise } from '@/types'; -import toErrorMessage from '@/utils/to-error-message'; import type { AugmentedSettingDefinitionItem, LabelDefinition } from './utils'; import ModuleManagement from './module-management'; import { s } from './utils'; @@ -239,7 +239,7 @@ function setupCheckConnection({ } } catch (error) { setError(); - const message = toErrorMessage(error); + const message = getMessage(error); log(`Check connection to \`${settings.remoteFs}\` failed: \`${message}\`.`); if (force) new Notice(`${translate('checkConnectionFailed')}: ${message}`); else scheduleCheckConnection(); diff --git a/packages/plugin/src/utils/pipe.ts b/packages/plugin/src/utils/pipe.ts index 97b8ff5f..1d0db9c1 100644 --- a/packages/plugin/src/utils/pipe.ts +++ b/packages/plugin/src/utils/pipe.ts @@ -1,5 +1,5 @@ -import type { ErrorLike } from '@repo/shared/get-status'; -import { getStatus } from '@repo/shared/get-status'; +import type { ErrorLike } from '@repo/shared/error'; +import { getStatus } from '@repo/shared/error'; import { Platform } from 'obsidian'; import type { Fs } from '@/fs'; import type { Binary, FileStat } from '@/types'; diff --git a/packages/plugin/src/utils/to-error-message.ts b/packages/plugin/src/utils/to-error-message.ts deleted file mode 100644 index 231212de..00000000 --- a/packages/plugin/src/utils/to-error-message.ts +++ /dev/null @@ -1,3 +0,0 @@ -export default function toErrorMessage(error: unknown) { - return error instanceof Error ? error.message : String(error); -} diff --git a/packages/plugin/test/cancellation-middleware.test.ts b/packages/plugin/test/cancellation-middleware.test.ts index 8f27f829..4621636a 100644 --- a/packages/plugin/test/cancellation-middleware.test.ts +++ b/packages/plugin/test/cancellation-middleware.test.ts @@ -10,7 +10,7 @@ test('cancellation middleware rejects before dispatch', () => { const harness = request(() => ({})); const wrapped = cancellationMiddleware(harness.request, ref(true)); - expect(() => wrapped({ url: 'note.md' })).toThrow('Sync cancelled by user.'); + expect(() => wrapped('note.md')).toThrow('Sync cancelled by user.'); expect(harness.calls).toStrictEqual([]); }); @@ -20,7 +20,7 @@ test('cancellation middleware rejects after in-flight response resolves when can const harness = request(() => responseDeferred.promise); const wrapped = cancellationMiddleware(harness.request, isCancelled); - const pending = wrapped({ url: 'note.md' }); + const pending = wrapped('note.md'); await flush(); isCancelled(true); responseDeferred.resolve({}); diff --git a/packages/plugin/test/custom-headers-middleware.test.ts b/packages/plugin/test/custom-headers-middleware.test.ts index 009b7d43..96cce23b 100644 --- a/packages/plugin/test/custom-headers-middleware.test.ts +++ b/packages/plugin/test/custom-headers-middleware.test.ts @@ -4,7 +4,7 @@ import { customHeadersMiddleware } from '@/fs'; const { request } = testKit; -test('custom headers middleware normalizes string input to request params', () => { +test('custom headers middleware adds headers to a bare url request', () => { const harness = request(() => ({})); const wrapped = customHeadersMiddleware(harness.request, { 'x-added': 'value' }); @@ -20,12 +20,11 @@ test('custom headers middleware merges supplied headers and overrides duplicates }); expect( - wrapped({ + wrapped('note.md', { headers: { 'x-keep': 'keep', 'x-override': 'old', }, - url: 'note.md', }), ).resolves.toMatchObject({ status: 200 }); expect(harness.calls).toStrictEqual([ diff --git a/packages/plugin/test/fs-vault.test.ts b/packages/plugin/test/fs-vault.test.ts index bae511aa..812e43b7 100644 --- a/packages/plugin/test/fs-vault.test.ts +++ b/packages/plugin/test/fs-vault.test.ts @@ -327,7 +327,7 @@ test('list should report hidden entries the file tree omits', async () => { test('LIST should keep using the file tree when the caller does not opt out', async () => { const vault = createVaultStub(HIDDEN_OPTIONS); - expect(await vault.request({ key: 'folder/', method: 'LIST' })).toStrictEqual({ + expect(await vault.request('folder/', { method: 'LIST' })).toStrictEqual({ files: ['folder/note.md'], folders: [], }); diff --git a/packages/plugin/test/rate-limiter-middleware.test.ts b/packages/plugin/test/rate-limiter-middleware.test.ts index 853b6330..d49b2937 100644 --- a/packages/plugin/test/rate-limiter-middleware.test.ts +++ b/packages/plugin/test/rate-limiter-middleware.test.ts @@ -7,15 +7,13 @@ const { deferred, flush, request } = testKit; test('rate limiter middleware queues second request until first resolves', async () => { const firstDeferred = deferred>(); - const harness = request((params) => { - const url = typeof params === 'string' ? params : params.url; - if (url === 'first.md') return firstDeferred.promise; - return { status: 202 }; - }); + const harness = request((url) => + url === 'first.md' ? firstDeferred.promise : { status: 202 }, + ); const wrapped = rateLimiterMiddleware(harness.request, { maxConcurrency: 1, minInterval: 0 }); - const firstPending = wrapped({ url: 'first.md' }); - const secondPending = wrapped({ url: 'second.md' }); + const firstPending = wrapped('first.md'); + const secondPending = wrapped('second.md'); await flush(); expect(harness.calls).toStrictEqual([{ url: 'first.md' }]); diff --git a/packages/plugin/test/retry-middleware.test.ts b/packages/plugin/test/retry-middleware.test.ts index cf907aba..93cd0c60 100644 --- a/packages/plugin/test/retry-middleware.test.ts +++ b/packages/plugin/test/retry-middleware.test.ts @@ -15,7 +15,7 @@ test('retry middleware retries retryable request and waits between attempts', () }); const wrapped = retryMiddleware(harness.request, { maxRetry: 2, retryDelay: () => 25 }); - expect(wrapped({ url: 'retry.md' })).resolves.toMatchObject({ status: 200 }); + expect(wrapped('retry.md')).resolves.toMatchObject({ status: 200 }); expect(harness.calls).toStrictEqual([ { url: 'retry.md' }, { url: 'retry.md' }, @@ -37,7 +37,7 @@ test('retry middleware stops on non-retryable error', () => { retryDelay: () => 25, }); - expect(wrapped({ url: 'missing.md' })).rejects.toStrictEqual({ res: { status: 404 } }); + expect(wrapped('missing.md')).rejects.toStrictEqual({ res: { status: 404 } }); expect(harness.calls).toStrictEqual([{ url: 'missing.md' }]); expect(sleepSpy).not.toHaveBeenCalled(); }); @@ -57,7 +57,7 @@ test('retry middleware retries iOS timeout error with localized message and nume }); const wrapped = retryMiddleware(harness.request, { maxRetry: 2, retryDelay: () => 25 }); - expect(wrapped({ url: 'timeout.md' })).resolves.toMatchObject({ status: 200 }); + expect(wrapped('timeout.md')).resolves.toMatchObject({ status: 200 }); expect(harness.calls).toStrictEqual([{ url: 'timeout.md' }, { url: 'timeout.md' }]); expect(sleepSpy).toHaveBeenCalledTimes(1); }); @@ -69,7 +69,7 @@ test('retry middleware retries Capacitor-bridged URLSession error with domain st }); const wrapped = retryMiddleware(harness.request, { maxRetry: 2, retryDelay: () => 25 }); - expect(wrapped({ url: 'capacitor.md' })).rejects.toStrictEqual({ + expect(wrapped('capacitor.md')).rejects.toStrictEqual({ code: 'NSURLErrorDomain', message: '请求超时。', }); @@ -88,7 +88,7 @@ test('retry middleware stops on non-retryable URLSession error code', () => { }); const wrapped = retryMiddleware(harness.request, { maxRetry: 3, retryDelay: () => 25 }); - expect(wrapped({ url: 'ssl.md' })).rejects.toStrictEqual({ + expect(wrapped('ssl.md')).rejects.toStrictEqual({ code: -1200, domain: 'NSURLErrorDomain', message: 'An SSL error has occurred.', @@ -106,7 +106,7 @@ test('retry middleware retries returned retryable status response', () => { }); const wrapped = retryMiddleware(harness.request, { maxRetry: 2, retryDelay: () => 25 }); - expect(wrapped({ throw: false, url: 'flaky.md' })).resolves.toMatchObject({ status: 200 }); + expect(wrapped('flaky.md', { throw: false })).resolves.toMatchObject({ status: 200 }); expect(harness.calls).toStrictEqual([ { throw: false, url: 'flaky.md' }, { throw: false, url: 'flaky.md' }, @@ -120,7 +120,7 @@ test('retry middleware returns retryable status response after exhausting retrie const harness = request(() => ({ status: 503 })); const wrapped = retryMiddleware(harness.request, { maxRetry: 2, retryDelay: () => 25 }); - expect(wrapped({ throw: false, url: 'down.md' })).resolves.toMatchObject({ status: 503 }); + expect(wrapped('down.md', { throw: false })).resolves.toMatchObject({ status: 503 }); expect(harness.calls).toHaveLength(3); expect(sleepSpy).toHaveBeenCalledTimes(2); }); diff --git a/packages/plugin/test/test-kit.ts b/packages/plugin/test/test-kit.ts index 68380928..03c389f8 100644 --- a/packages/plugin/test/test-kit.ts +++ b/packages/plugin/test/test-kit.ts @@ -37,8 +37,10 @@ type FsHarness = { fs: RootFs; }; -type RequestHarness = { - calls: Array; +type ResponseControl = (url: string, params: RequestParam) => MaybePromise; + +type RequestHarness = { + calls: Array; request: Request; }; @@ -194,7 +196,7 @@ const defaultBytes = () => bytes('ok'); const defaultText = () => 'ok'; const defaultJson = () => ({}); -export type ResponseOverrides = Partial> & { +type ResponseOverrides = Partial> & { json?: () => unknown; }; @@ -209,17 +211,13 @@ function response(overrides: ResponseOverrides = {}): RequestResponse { }; } -export type ResponseControl = (params: T) => MaybePromise; - -function request( - control: ResponseControl, -): RequestHarness { - const calls: Array = []; +function request(control: ResponseControl): RequestHarness { + const calls: Array = []; return { calls, - request: async (params: RequestParam | string) => { - calls.push(params as T); - return response(await control(params as T)); + request: async (url, params) => { + calls.push({ url, ...params }); + return response(await control(url, params ?? {})); }, }; } diff --git a/packages/s3/src/index.ts b/packages/s3/src/index.ts index df5167fd..50055850 100644 --- a/packages/s3/src/index.ts +++ b/packages/s3/src/index.ts @@ -144,12 +144,12 @@ export default class S3 { } catch { throw new Error('Please enter a valid S3 proxy URL!'); } - return (params) => { - const originalUrl = typeof params === 'string' ? params : params.url; - const original = new URL(originalUrl); - const rewritten = `${proxy.protocol}//${proxy.host}${original.pathname}${original.search}`; - if (typeof params === 'string') return request(rewritten); - return request({ ...params, url: rewritten }); + return (url, params) => { + const original = new URL(url); + return request( + `${proxy.protocol}//${proxy.host}${original.pathname}${original.search}`, + params, + ); }; }, priority: 303, diff --git a/packages/s3/src/s3/check-connection.ts b/packages/s3/src/s3/check-connection.ts index 4fac2572..9eec0cc0 100644 --- a/packages/s3/src/s3/check-connection.ts +++ b/packages/s3/src/s3/check-connection.ts @@ -1,4 +1,5 @@ import type { CheckConnectionResult, Request } from '@hesprs/sync-engine-sdk'; +import { getMessage } from '@repo/shared/error'; import type { UrlStyle } from './sigv4'; import { buildUrl } from './url'; @@ -20,14 +21,13 @@ export async function checkConnection( key: '/', urlStyle: options.urlStyle, }); - const response = await request({ method: 'HEAD', throw: false, url }); + const response = await request(url, { method: 'HEAD', throw: false }); if (response.status >= 200 && response.status < 300) return { success: true } as const; return { reason: `HTTP ${response.status}`, success: false, } as const; } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - return { reason: errorMessage, success: false } as const; + return { reason: getMessage(error), success: false } as const; } } diff --git a/packages/s3/src/s3/fs.ts b/packages/s3/src/s3/fs.ts index a5aa8517..e58316c7 100644 --- a/packages/s3/src/s3/fs.ts +++ b/packages/s3/src/s3/fs.ts @@ -10,7 +10,7 @@ import type { } from '@hesprs/sync-engine-sdk'; import { chunkSize, concurrency } from '@hesprs/sync-engine-sdk'; import { concatBinary, textToUint8Array } from '@repo/shared/binary'; -import { getStatus } from '@repo/shared/get-status'; +import { getStatus } from '@repo/shared/error'; import parseXML from '@repo/shared/parse-xml'; import { dirname, encodeUrl, isFolder } from '@repo/shared/path'; import createRangeReadStream from '@repo/shared/read-stream'; @@ -146,24 +146,24 @@ export default class S3Fs implements RootFs { }); } - private async requestOrThrow(params: RequestParam): Promise { - const response = await this.request(Object.assign(params, { throw: false })); + private readonly requestOrThrow = async ( + url: string, + params: RequestParam = {}, + ): Promise => { + const response = await this.request(url, { ...params, throw: false }); if (response.status >= 200 && response.status < 300) return response; const body = response.text(); const s3Error = parseS3Error(body); const error = new Error( - s3Error ?? `S3 request failed: ${response.status} ${params.method} ${params.url}`, + s3Error ?? `S3 request failed: ${response.status} ${params.method} ${url}`, ); (error as { status?: number }).status = response.status; throw error; - } + }; async read(key: string): Promise { - const response = await this.requestOrThrow({ - method: 'GET', - url: this.buildUrl(key), - }); + const response = await this.requestOrThrow(this.buildUrl(key), { method: 'GET' }); return response.bytes(); } @@ -173,10 +173,9 @@ export default class S3Fs implements RootFs { chunkSize, concurrency, requestRange: async (start, endInclusive) => { - const response = await this.requestOrThrow({ + const response = await this.requestOrThrow(url, { headers: { Range: `bytes=${start}-${endInclusive}` }, method: 'GET', - url, }); return response.bytes(); }, @@ -185,11 +184,10 @@ export default class S3Fs implements RootFs { } async write(key: string, value: Binary): Promise { - const response = await this.requestOrThrow({ + const response = await this.requestOrThrow(this.buildUrl(key), { body: value, headers: { 'Content-Type': 'application/octet-stream' }, method: 'PUT', - url: this.buildUrl(key), }); const etag = getHeader(response.headers, 'etag'); if (etag) return etag; @@ -205,7 +203,7 @@ export default class S3Fs implements RootFs { bucket: this.bucket, endpoint: this.endpoint, key, - request: (params) => this.requestOrThrow(params), + request: this.requestOrThrow, stat: (k) => this.stat(k), urlStyle: this.urlStyle, }, @@ -215,10 +213,7 @@ export default class S3Fs implements RootFs { async delete(key: string): Promise { try { - await this.requestOrThrow({ - method: 'DELETE', - url: this.buildUrl(key), - }); + await this.requestOrThrow(this.buildUrl(key), { method: 'DELETE' }); } catch (error) { if (getStatus(error) === 404) return; throw error; @@ -238,14 +233,13 @@ export default class S3Fs implements RootFs { { bucket: this.bucket, endpoint: this.endpoint, key: '/', urlStyle: this.urlStyle }, { delete: '' }, ); - const response = await this.requestOrThrow({ + const response = await this.requestOrThrow(url, { body: textToUint8Array(body), headers: { 'Content-MD5': await md5Base64(body), 'Content-Type': 'application/xml', }, method: 'POST', - url, }); Object.assign(result, parseBatchDeleteResponse(response.text(), batch)); } @@ -256,13 +250,12 @@ export default class S3Fs implements RootFs { // S3 has no native rename — copy then delete const copySource = `${this.bucket}/${encodeUrl(oldKey)}`; const destUrl = this.buildUrl(newKey); - await this.requestOrThrow({ + await this.requestOrThrow(destUrl, { headers: { 'Content-Type': 'application/octet-stream', 'x-amz-copy-source': copySource, }, method: 'PUT', - url: destUrl, }); await this.delete(oldKey); } @@ -273,11 +266,10 @@ export default class S3Fs implements RootFs { // S3 has no real folders — create a 0-byte placeholder object const url = this.buildUrl(dirKey); try { - await this.requestOrThrow({ + await this.requestOrThrow(url, { body: new Uint8Array(0), headers: { 'Content-Type': 'application/octet-stream' }, method: 'PUT', - url, }); } catch (error) { if (getStatus(error) === 409) continue; @@ -288,7 +280,7 @@ export default class S3Fs implements RootFs { async stat(key: string): Promise { if (isFolder(key)) return { isDir: true, key }; - const response = await this.requestOrThrow({ method: 'HEAD', url: this.buildUrl(key) }); + const response = await this.requestOrThrow(this.buildUrl(key), { method: 'HEAD' }); const etag = getHeader(response.headers, 'etag'); const contentLength = getHeader(response.headers, 'content-length'); const lastModified = getHeader(response.headers, 'last-modified'); @@ -323,7 +315,7 @@ export default class S3Fs implements RootFs { { bucket: this.bucket, endpoint: this.endpoint, key: '/', urlStyle: this.urlStyle }, query, ); - const response = await this.requestOrThrow({ method: 'GET', url }); + const response = await this.requestOrThrow(url, { method: 'GET' }); const { ListBucketResult: listing } = parseXML(response.text()); const contents = asArray(listing.Contents); await Promise.all( diff --git a/packages/s3/src/s3/multipart.ts b/packages/s3/src/s3/multipart.ts index 029d88f2..7d04e3df 100644 --- a/packages/s3/src/s3/multipart.ts +++ b/packages/s3/src/s3/multipart.ts @@ -1,4 +1,4 @@ -import type { Binary, RequestParam, Stat } from '@hesprs/sync-engine-sdk'; +import type { Binary, Request, Stat } from '@hesprs/sync-engine-sdk'; import { textToUint8Array } from '@repo/shared/binary'; import chunkedUpload from '@repo/shared/chunked-upload'; import parseXML from '@repo/shared/parse-xml'; @@ -25,10 +25,7 @@ export type MultipartUploadOptions = { bucket: string; urlStyle: UrlStyle; key: string; - request: (params: RequestParam) => Promise<{ - headers: Record; - text: () => string; - }>; + request: Request; stat: (key: string) => Promise; }; @@ -61,11 +58,10 @@ async function uploadPart( }, { partNumber: String(partNumber), uploadId }, ); - const response = await options.request({ + const response = await options.request(url, { body: chunk, headers: { 'Content-Type': 'application/octet-stream' }, method: 'PUT', - url, }); const etag = getHeader(response.headers, 'etag'); if (!etag) throw new Error(`S3 multipart: no ETag for part ${partNumber}`); @@ -82,7 +78,7 @@ function abortMultipart(options: MultipartUploadOptions, uploadId: string) { }, { uploadId }, ); - return options.request({ ignoreCancellation: true, method: 'DELETE', url }).catch(() => {}); + return options.request(url, { ignoreCancellation: true, method: 'DELETE' }).catch(() => {}); } export async function multipartUpload( @@ -98,10 +94,9 @@ export async function multipartUpload( }, { uploads: '' }, ); - const initiateResponse = await options.request({ + const initiateResponse = await options.request(initiateUrl, { headers: { 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }, method: 'POST', - url: initiateUrl, }); const uploadId = parseUploadId(initiateResponse.text()); @@ -123,11 +118,10 @@ export async function multipartUpload( }, { uploadId }, ); - const completeResponse = await options.request({ + const completeResponse = await options.request(completeUrl, { body: textToUint8Array(completeBody), headers: { 'Content-Type': 'application/xml' }, method: 'POST', - url: completeUrl, }); const etag = parseXML(completeResponse.text()) diff --git a/packages/s3/src/s3/sigv4.ts b/packages/s3/src/s3/sigv4.ts index 7af6dcbc..7d908334 100644 --- a/packages/s3/src/s3/sigv4.ts +++ b/packages/s3/src/s3/sigv4.ts @@ -14,13 +14,6 @@ export type SigV4Options = { service: string; }; -type InternalRequest = { - method: string; - url: string; - headers: Record; - body?: Binary | string; -}; - const encoder = new TextEncoder(); function toHex(bytes: Binary): string { @@ -103,12 +96,11 @@ export async function signRequest( method, url, headers: rawHeaders, - body, - }: RequestParam & { method: string; headers: Record }, + }: RequestParam & { method: string; url: string; headers: Record }, { sessionToken, secretAccessKey, region, service, accessKeyId }: SigV4Options, date: Date, db: S3DB, -): Promise { +): Promise> { const host = new URL(url).host; const headers: Record = { ...rawHeaders }; @@ -158,16 +150,19 @@ export async function signRequest( // Strips host from actually sent headers to prevent Electron throwing delete headers.host; - return { body, headers, method, url }; + return headers; } export function sigv4Middleware(request: Request, credentials: SigV4Options, db: S3DB): Request { - return async (params) => { - const input = - typeof params === 'string' - ? { headers: {}, method: 'GET', url: params } - : { ...params, headers: params.headers ?? {}, method: params.method ?? 'GET' }; - return request(await signRequest(input, credentials, new Date(), db)); + return async (url, params = {}) => { + const input = { ...params, method: params.method ?? 'GET' }; + const headers = await signRequest( + { ...input, headers: input.headers ?? {}, url }, + credentials, + new Date(), + db, + ); + return request(url, { ...input, headers }); }; } diff --git a/packages/s3/test/check-connection.test.ts b/packages/s3/test/check-connection.test.ts index e46af7bc..7d8cebbf 100644 --- a/packages/s3/test/check-connection.test.ts +++ b/packages/s3/test/check-connection.test.ts @@ -1,4 +1,3 @@ -import type { RequestParam } from '@hesprs/sync-engine-sdk'; import { testKit } from '@hesprs/sync-engine-sdk/dev'; import { expect, test } from 'bun:test'; import { checkConnection } from '@/s3/check-connection'; @@ -13,7 +12,7 @@ const connectionOptions = { }; test('checkConnection uses the request pipeline for a signed head bucket request', async () => { - const harness = testKit.request(() => response()); + const harness = testKit.request(() => response()); const request = sigv4Middleware(harness.request, defaultCredentials, memoryDB); expect(await checkConnection(connectionOptions, request)).toStrictEqual({ success: true }); diff --git a/packages/s3/test/fs-s3.test.ts b/packages/s3/test/fs-s3.test.ts index d02f206b..994bb594 100644 --- a/packages/s3/test/fs-s3.test.ts +++ b/packages/s3/test/fs-s3.test.ts @@ -1,5 +1,11 @@ -import type { Binary, InputAtom, OptimizerInput, RequestParam } from '@hesprs/sync-engine-sdk'; -import type { ResponseControl } from '@hesprs/sync-engine-sdk/dev'; +import type { + Binary, + InputAtom, + MaybePromise, + OptimizerInput, + RequestParam, + RequestResponse, +} from '@hesprs/sync-engine-sdk'; import { testKit } from '@hesprs/sync-engine-sdk/dev'; import { expect, mock, test } from 'bun:test'; import type { S3FsOptions } from '@/s3/fs'; @@ -16,25 +22,20 @@ void mock.module('@repo/shared/parse-xml', () => ({ default: () => parsedResponse, })); -type RequestHandler = ResponseControl; -type S3Harness = { - calls: Array; - fs: S3Fs; - setRequest: (handler: RequestHandler) => void; -}; +type Control = (url: string, params: RequestParam) => MaybePromise>; const defaultOptions = { ...defaultS3Options, } as const satisfies Omit; -function createS3Fs(options: Partial = {}): S3Harness { - let requestHandler: RequestHandler = () => response(); - const harness = testKit.request((params) => requestHandler(params)); +function createS3Fs(options: Partial = {}) { + let requestHandler: Control = () => response(); + const harness = testKit.request((url, params) => requestHandler(url, params)); const request = sigv4Middleware(harness.request, defaultCredentials, memoryDB); return { calls: harness.calls, fs: new S3Fs({ ...defaultOptions, ...options, request }), - setRequest: (handler) => { + setRequest: (handler: Control) => { requestHandler = handler; }, }; @@ -57,9 +58,9 @@ function textBody(params: RequestParam): string { test('read gets encoded object bytes and maps S3 XML errors', async () => { const s3 = createS3Fs(); - s3.setRequest((params) => { + s3.setRequest((url, params) => { assertSignedRequest(params, 'GET'); - expect(params.url).toBe('https://s3.example.com/vault/Notes/file%20A.md'); + expect(url).toBe('https://s3.example.com/vault/Notes/file%20A.md'); return response({ body: bytes('content') }); }); expect(await s3.fs.read('Notes/file A.md')).toStrictEqual(bytes('content')); @@ -80,9 +81,9 @@ test('read gets encoded object bytes and maps S3 XML errors', async () => { test('write sends binary PUT and uses ETag or HEAD metadata fallback', async () => { const s3 = createS3Fs(); - s3.setRequest((params) => { + s3.setRequest((url, params) => { assertSignedRequest(params, 'PUT'); - expect(params.url).toBe('https://s3.example.com/vault/Notes/file.md'); + expect(url).toBe('https://s3.example.com/vault/Notes/file.md'); expect(params.headers?.['Content-Type']).toBe('application/octet-stream'); expect(params.body).toStrictEqual(bytes('hello')); return response({ headers: { ETag: '"write-etag"' } }); @@ -90,7 +91,7 @@ test('write sends binary PUT and uses ETag or HEAD metadata fallback', async () expect(await s3.fs.write('Notes/file.md', bytes('hello'))).toBe('"write-etag"'); const fallback = createS3Fs(); - fallback.setRequest((params) => { + fallback.setRequest((_url, params) => { if (params.method === 'PUT') return response(); expect(params.method).toBe('HEAD'); return response({ @@ -107,7 +108,7 @@ test('write sends binary PUT and uses ETag or HEAD metadata fallback', async () test('writeStream buffers below-part-size input into one PUT', async () => { const s3 = createS3Fs(); - s3.setRequest((params) => { + s3.setRequest((_url, params) => { expect(params.method).toBe('PUT'); expect(params.body).toStrictEqual(bytes('hello world')); return response({ headers: { etag: 'buffered-etag' } }); @@ -122,9 +123,9 @@ test('writeStream buffers below-part-size input into one PUT', async () => { test('writeStream uploads exact multipart parts and completes with ETag', async () => { const partSize = 5 * 1024 * 1024; const s3 = createS3Fs(); - s3.setRequest((params) => { - const url = new URL(params.url); - if (params.method === 'POST' && url.searchParams.has('uploads')) { + s3.setRequest((url, params) => { + const address = new URL(url); + if (params.method === 'POST' && address.searchParams.has('uploads')) { expect(params.headers?.['x-amz-content-sha256']).toBe('UNSIGNED-PAYLOAD'); parsedResponse = { InitiateMultipartUploadResult: { UploadId: 'upload-1' } }; return response({ @@ -132,13 +133,13 @@ test('writeStream uploads exact multipart parts and completes with ETag', async }); } if (params.method === 'PUT') { - expect(url.searchParams.get('uploadId')).toBe('upload-1'); + expect(address.searchParams.get('uploadId')).toBe('upload-1'); expect(params.headers?.['Content-Type']).toBe('application/octet-stream'); - const partNumber = url.searchParams.get('partNumber'); + const partNumber = address.searchParams.get('partNumber'); return response({ headers: { etag: `part-${partNumber}` } }); } expect(params.method).toBe('POST'); - expect(url.searchParams.get('uploadId')).toBe('upload-1'); + expect(address.searchParams.get('uploadId')).toBe('upload-1'); expect(params.headers?.['Content-Type']).toBe('application/xml'); expect(textBody(params)).toContain('1part-1'); expect(textBody(params)).toContain('2part-2'); @@ -161,9 +162,9 @@ test('writeStream aborts multipart upload after part failure', async () => { const s3 = createS3Fs(); const uploadError = new Error('part failed'); const aborted = deferred(); - s3.setRequest((params) => { - const url = new URL(params.url); - if (params.method === 'POST' && url.searchParams.has('uploads')) { + s3.setRequest((url, params) => { + const address = new URL(url); + if (params.method === 'POST' && address.searchParams.has('uploads')) { parsedResponse = { InitiateMultipartUploadResult: { UploadId: 'upload-2' } }; return response({ text: 'upload-2', @@ -171,7 +172,7 @@ test('writeStream aborts multipart upload after part failure', async () => { } if (params.method === 'PUT') throw uploadError; expect(params.method).toBe('DELETE'); - expect(url.searchParams.get('uploadId')).toBe('upload-2'); + expect(address.searchParams.get('uploadId')).toBe('upload-2'); aborted.resolve(); return response({ status: 204 }); }); @@ -187,7 +188,7 @@ test('writeStream aborts multipart upload after part failure', async () => { test('delete and exists treat 404 as absent but propagate other statuses', async () => { const calls: Array = []; const s3 = createS3Fs(); - s3.setRequest((params) => { + s3.setRequest((_url, params) => { calls.push(params.method ?? ''); if (calls.length === 1 || calls.length === 3) return response({ status: 404 }); return response({ status: 500, text: 'InternalError' }); @@ -204,10 +205,10 @@ test('delete and exists treat 404 as absent but propagate other statuses', async test('batchDelete escapes keys, sends MD5 XML, and batches at 1000 keys', async () => { const s3 = createS3Fs(); const bodies: Array = []; - s3.setRequest((params) => { + s3.setRequest((url, params) => { expect(params.method).toBe('POST'); - const url = new URL(params.url); - expect(url.searchParams.has('delete')).toBe(true); + const address = new URL(url); + expect(address.searchParams.has('delete')).toBe(true); expect(params.headers?.['Content-Type']).toBe('application/xml'); expect(params.headers?.['Content-MD5']).toMatch(/^[A-Za-z0-9+/]{22}==$/u); parsedResponse = @@ -279,9 +280,9 @@ test('batch delete rejects only atoms with S3 partial failures', async () => { test('move copies encoded source before deleting old key', async () => { const s3 = createS3Fs(); - s3.setRequest((params) => { + s3.setRequest((url, params) => { if (params.method === 'PUT') { - expect(params.url).toBe('https://s3.example.com/vault/new%20folder/new.md'); + expect(url).toBe('https://s3.example.com/vault/new%20folder/new.md'); expect(params.headers).toMatchObject({ 'Content-Type': 'application/octet-stream', 'x-amz-copy-source': 'vault/old%20folder/old.md', @@ -289,7 +290,7 @@ test('move copies encoded source before deleting old key', async () => { return response(); } expect(params.method).toBe('DELETE'); - expect(params.url).toBe('https://s3.example.com/vault/old%20folder/old.md'); + expect(url).toBe('https://s3.example.com/vault/old%20folder/old.md'); return response(); }); await s3.fs.move('old folder/old.md', 'new folder/new.md'); @@ -298,11 +299,11 @@ test('move copies encoded source before deleting old key', async () => { test('mkdir recursively creates placeholders in ancestor order and ignores conflicts', async () => { const s3 = createS3Fs(); - s3.setRequest((params) => { + s3.setRequest((url, params) => { expect(params.method).toBe('PUT'); expect(params.headers?.['Content-Type']).toBe('application/octet-stream'); expect(params.body).toStrictEqual(new Uint8Array(0)); - if (params.url.endsWith('/Notes/A%20B/')) throw { res: { status: 409 } }; + if (url.endsWith('/Notes/A%20B/')) throw { res: { status: 409 } }; return response({ status: 201 }); }); await s3.fs.mkdir('Notes/A B/Child/', true); @@ -316,10 +317,9 @@ test('mkdir recursively creates placeholders in ancestor order and ignores confl test('stat returns root, folder placeholders, and file metadata with ETag fallback', async () => { const s3 = createS3Fs(); expect(await s3.fs.stat('/')).toStrictEqual({ isDir: true, key: '/' }); - s3.setRequest((params) => { + s3.setRequest((url, params) => { expect(params.method).toBe('HEAD'); - if (params.url.endsWith('/folder/')) - return response({ headers: { 'content-length': '0' } }); + if (url.endsWith('/folder/')) return response({ headers: { 'content-length': '0' } }); return response({ headers: { 'content-length': '12', @@ -340,14 +340,14 @@ test('stat returns root, folder placeholders, and file metadata with ETag fallba test('list returns files and prefixes, excludes queried key, reports exclusions, and paginates', async () => { const s3 = createS3Fs(); const progress: Array = []; - s3.setRequest((params) => { - const url = new URL(params.url); + s3.setRequest((url, params) => { + const address = new URL(url); expect(params.method).toBe('GET'); - expect(url.searchParams.get('list-type')).toBe('2'); - expect(url.searchParams.has('delimiter')).toBe(false); - expect(url.searchParams.get('prefix')).toBe('Notes/'); - if (url.searchParams.has('continuation-token')) { - expect(url.searchParams.get('continuation-token')).toBe('next-page'); + expect(address.searchParams.get('list-type')).toBe('2'); + expect(address.searchParams.has('delimiter')).toBe(false); + expect(address.searchParams.get('prefix')).toBe('Notes/'); + if (address.searchParams.has('continuation-token')) { + expect(address.searchParams.get('continuation-token')).toBe('next-page'); parsedResponse = { ListBucketResult: { Contents: { @@ -404,8 +404,8 @@ test('list returns files and prefixes, excludes queried key, reports exclusions, test('list maps unified root to an empty S3 prefix', async () => { const s3 = createS3Fs(); - s3.setRequest((params) => { - expect(new URL(params.url).searchParams.get('prefix')).toBe(''); + s3.setRequest((url, _params) => { + expect(new URL(url).searchParams.get('prefix')).toBe(''); parsedResponse = { ListBucketResult: { IsTruncated: 'false' } }; return response({ text: '' }); }); diff --git a/packages/s3/test/helpers.ts b/packages/s3/test/helpers.ts index 2ae3267d..ee0f0a33 100644 --- a/packages/s3/test/helpers.ts +++ b/packages/s3/test/helpers.ts @@ -1,5 +1,4 @@ import type { Binary } from '@hesprs/sync-engine-sdk'; -import type { ResponseOverrides } from '@hesprs/sync-engine-sdk/dev'; import { openMemoryDB } from 'uni-kv'; export const memoryDB = openMemoryDB< @@ -34,7 +33,7 @@ export function response( status?: number; text?: string; } = {}, -): ResponseOverrides { +) { return { bytes: () => options.body ?? emptyBinary, headers: options.headers ?? {}, diff --git a/packages/s3/test/sigv4-middleware.test.ts b/packages/s3/test/sigv4-middleware.test.ts index 0f6fd39f..513983a9 100644 --- a/packages/s3/test/sigv4-middleware.test.ts +++ b/packages/s3/test/sigv4-middleware.test.ts @@ -11,7 +11,7 @@ beforeEach(() => { }); function createTransport() { - const harness = testKit.request(() => response()); + const harness = testKit.request(() => response()); return { calls: harness.calls, transport: harness.request }; } @@ -29,11 +29,10 @@ test('middleware signs request parameters without changing body or URL', async ( const request = sigv4Middleware(transport, defaultCredentials, memoryDB); const body = new Uint8Array([1, 2, 3]); - await request({ + await request('https://s3.example.com/vault/file.bin', { body, headers: { 'Content-Type': 'application/octet-stream' }, method: 'PUT', - url: 'https://s3.example.com/vault/file.bin', }); const call = calls[0]; @@ -58,7 +57,7 @@ test('middleware signs temporary session credentials with the security token', a expect(call.headers?.authorization).toContain('x-amz-security-token'); }); -test('middleware treats string requests as GET requests', async () => { +test('middleware defaults to GET when no params are given', async () => { const { calls, transport } = createTransport(); const request = sigv4Middleware(transport, defaultCredentials, memoryDB); @@ -75,21 +74,16 @@ test('middleware signs custom headers before proxy rewrites the URL', async () = const { calls, transport } = createTransport(); const proxy = (request: Request): Request => - (params) => { - if (typeof params === 'string') return request(params); - const original = new URL(params.url); - return request({ - ...params, - url: `https://proxy.example.com${original.pathname}${original.search}`, - }); + (url, params) => { + const original = new URL(url); + return request( + `https://proxy.example.com${original.pathname}${original.search}`, + params, + ); }; const signed = sigv4Middleware(proxy(transport), defaultCredentials, memoryDB); - await signed({ - headers: { 'x-custom': 'value' }, - method: 'GET', - url: 'https://s3.example.com/vault/file.md', - }); + await signed('https://s3.example.com/vault/file.md', { headers: { 'x-custom': 'value' } }); const call = calls[0]; if (!call) throw new Error('Expected transport request'); diff --git a/packages/shared/src/get-status.ts b/packages/shared/src/error.ts similarity index 79% rename from packages/shared/src/get-status.ts rename to packages/shared/src/error.ts index a8eed3be..032ad8e2 100644 --- a/packages/shared/src/get-status.ts +++ b/packages/shared/src/error.ts @@ -18,3 +18,7 @@ export function getStatus(error: unknown): number | undefined { const candidates = [err.status, err.res?.status, err.response?.status]; for (const candidate of candidates) if (typeof candidate === 'number') return candidate; } + +export function getMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/webdav/src/webdav/check-connection.ts b/packages/webdav/src/webdav/check-connection.ts index eee50eaa..6e39411f 100644 --- a/packages/webdav/src/webdav/check-connection.ts +++ b/packages/webdav/src/webdav/check-connection.ts @@ -1,4 +1,5 @@ import type { CheckConnectionResult, Request } from '@hesprs/sync-engine-sdk'; +import { getMessage } from '@repo/shared/error'; import { normalizeUrl } from '@repo/shared/path'; import { buildUrl, getAuthorization, parseWebDAVError } from './utils'; @@ -19,13 +20,12 @@ export async function checkConnection( ): Promise { const Authorization = getAuthorization(options.username, options.password); try { - const response = await request({ + const response = await request(buildUrl(normalizeUrl(options.endpoint), '/'), { body: CHECK_CONNECTION_BODY, contentType: 'application/xml', headers: { Authorization, Depth: '0' }, method: 'PROPFIND', throw: false, - url: buildUrl(normalizeUrl(options.endpoint), '/'), }); if (response.status === 200 || response.status === 207) return { success: true } as const; return { @@ -33,7 +33,6 @@ export async function checkConnection( success: false, } as const; } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - return { reason: errorMessage, success: false } as const; + return { reason: getMessage(error), success: false } as const; } } diff --git a/packages/webdav/src/webdav/chunked-upload.ts b/packages/webdav/src/webdav/chunked-upload.ts index 084db651..ffc0480f 100644 --- a/packages/webdav/src/webdav/chunked-upload.ts +++ b/packages/webdav/src/webdav/chunked-upload.ts @@ -1,10 +1,8 @@ -import type { Binary, RequestParam, RequestResponse, Stat } from '@hesprs/sync-engine-sdk'; +import type { Binary, Request, Stat } from '@hesprs/sync-engine-sdk'; import chunkedUpload from '@repo/shared/chunked-upload'; import { encodeURIComponent3986 } from '@repo/shared/path'; import { buildUrl, getFileUid, getHeader } from './utils'; -type ThrowRequest = (params: RequestParam) => Promise; - // Nextcloud rejects non-final chunks below 5 MiB const NEXTCLOUD_CHUNK_SIZE = 5 * 1024 * 1024; const NEXTCLOUD_MAX_CONCURRENT = 3; @@ -12,7 +10,7 @@ const NEXTCLOUD_MAX_CONCURRENT = 3; type NextcloudChunkedUploadOptions = { auth: string; endpoint: string; - request: ThrowRequest; + request: Request; stat: (key: string) => Promise; username: string; }; @@ -26,12 +24,11 @@ function getUploadEndpoint(endpoint: string, username: string) { : `${endpoint.slice(0, filesMarkerIndex)}/uploads/${encodedUsername}`; } -function deleteChunkUpload(request: ThrowRequest, auth: string, uploadFolderUrl: string) { - return request({ +function deleteChunkUpload(request: Request, auth: string, uploadFolderUrl: string) { + return request(uploadFolderUrl, { headers: { Authorization: auth }, ignoreCancellation: true, method: 'DELETE', - url: uploadFolderUrl, }).catch(() => {}); } @@ -48,10 +45,9 @@ export default async function writeNextcloudChunkedUpload( const uploadFileUrl = buildUrl(uploadEndpoint, `${uploadId}/.file`); const destination = buildUrl(options.endpoint, key); - await options.request({ + await options.request(uploadFolderUrl, { headers: { Authorization: options.auth, Destination: destination }, method: 'MKCOL', - url: uploadFolderUrl, }); try { @@ -59,24 +55,25 @@ export default async function writeNextcloudChunkedUpload( chunkSize: NEXTCLOUD_CHUNK_SIZE, concurrency: NEXTCLOUD_MAX_CONCURRENT, uploadChunk: async (chunk, chunkNumber) => { - await options.request({ - body: chunk, - headers: { - Authorization: options.auth, - Destination: destination, - 'OC-Total-Length': String(size), + await options.request( + buildUrl(uploadEndpoint, `${uploadFolderKey}${chunkNumber}`), + { + body: chunk, + headers: { + Authorization: options.auth, + Destination: destination, + 'OC-Total-Length': String(size), + }, + method: 'PUT', }, - method: 'PUT', - url: buildUrl(uploadEndpoint, `${uploadFolderKey}${chunkNumber}`), - }); + ); }, value, }); - const response = await options.request({ + const response = await options.request(uploadFileUrl, { headers: { Authorization: options.auth, Destination: destination }, method: 'MOVE', - url: uploadFileUrl, }); const etag = getHeader(response.headers, 'etag') ?? getHeader(response.headers, 'oc-etag'); diff --git a/packages/webdav/src/webdav/fs.ts b/packages/webdav/src/webdav/fs.ts index d56af055..561d883f 100644 --- a/packages/webdav/src/webdav/fs.ts +++ b/packages/webdav/src/webdav/fs.ts @@ -11,7 +11,7 @@ import type { } from '@hesprs/sync-engine-sdk'; import { chunkSize, concurrency } from '@hesprs/sync-engine-sdk'; import { concatBinary } from '@repo/shared/binary'; -import { getStatus } from '@repo/shared/get-status'; +import { getStatus } from '@repo/shared/error'; import parseXML from '@repo/shared/parse-xml'; import { dirname, @@ -151,19 +151,18 @@ function extractNextLink(linkHeader: string): string | undefined { type PropfindPayload = { depth?: '0' | '1' | 'infinity'; - request: (params: RequestParam) => Promise; + request: Request; auth: string; } & ({ key: string; endpoint: string } | { url: string }); async function propfind(args: PropfindPayload) { const { request, depth = '0', auth } = args; const url = 'url' in args ? args.url : buildUrl(args.endpoint, args.key); - const response = await request({ + const response = await request(url, { body: PROPFIND_BODY, contentType: 'application/xml', headers: { Authorization: auth, Depth: depth }, method: 'PROPFIND', - url, }); const parsed = parseXML(response.text()); const items = asArray(parsed.multistatus.response); @@ -215,8 +214,11 @@ export default class WebdavFs implements RootFs { this.endpoint = normalizeUrl(options.endpoint); } - private readonly requestOrThrow = async (params: RequestParam): Promise => { - const response = await this.request(Object.assign(params, { throw: false })); + private readonly requestOrThrow = async ( + url: string, + params: RequestParam = {}, + ): Promise => { + const response = await this.request(url, { ...params, throw: false }); if (response.status >= 200 && response.status < 300) return response; const error = new Error( parseWebDAVError(response.text()) ?? @@ -231,10 +233,9 @@ export default class WebdavFs implements RootFs { } async read(key: string) { - const response = await this.requestOrThrow({ + const response = await this.requestOrThrow(buildUrl(this.endpoint, key), { headers: { Authorization: this.auth }, method: 'GET', - url: buildUrl(this.endpoint, key), }); return response.bytes(); } @@ -244,7 +245,7 @@ export default class WebdavFs implements RootFs { chunkSize, concurrency, requestRange: async (start, endInclusive) => { - const response = await this.requestOrThrow({ + const response = await this.requestOrThrow(buildUrl(this.endpoint, key), { headers: { // Prevents intermediaries and servers from content-encoding the body, which makes them ignore the Range header and return the whole file: https://github.com/hesprs/sync-engine/issues/263 'Accept-Encoding': 'identity', @@ -252,7 +253,6 @@ export default class WebdavFs implements RootFs { Range: `bytes=${start}-${endInclusive}`, }, method: 'GET', - url: buildUrl(this.endpoint, key), }); return response.bytes(); @@ -262,11 +262,10 @@ export default class WebdavFs implements RootFs { } async write(key: string, value: Binary) { - const response = await this.requestOrThrow({ + const response = await this.requestOrThrow(buildUrl(this.endpoint, key), { body: value, headers: { Authorization: this.auth }, method: 'PUT', - url: buildUrl(this.endpoint, key), }); const etag = getHeader(response.headers, 'etag'); return etag ? normalizeEtag(etag) : getFileUid(await this.stat(key), key); @@ -278,7 +277,7 @@ export default class WebdavFs implements RootFs { { auth: this.auth, endpoint: this.endpoint, - request: (params) => this.requestOrThrow(params), + request: this.requestOrThrow, stat: (targetKey) => this.stat(targetKey), username: this.options.username, }, @@ -291,10 +290,9 @@ export default class WebdavFs implements RootFs { async delete(key: string) { try { - await this.requestOrThrow({ + await this.requestOrThrow(buildUrl(this.endpoint, key), { headers: { Authorization: this.auth }, method: 'DELETE', - url: buildUrl(this.endpoint, key), }); } catch (error) { if (getStatus(error) === 404) return; @@ -303,10 +301,9 @@ export default class WebdavFs implements RootFs { } async move(oldKey: string, newKey: string) { - await this.requestOrThrow({ + await this.requestOrThrow(buildUrl(this.endpoint, oldKey), { headers: { Authorization: this.auth, Destination: buildUrl(this.endpoint, newKey) }, method: 'MOVE', - url: buildUrl(this.endpoint, oldKey), }); } @@ -314,10 +311,9 @@ export default class WebdavFs implements RootFs { const directoryKeys = recursive ? getRecursiveKeys(key) : [key]; for (const directoryKey of directoryKeys) try { - await this.requestOrThrow({ + await this.requestOrThrow(buildUrl(this.endpoint, directoryKey), { headers: { Authorization: this.auth }, method: 'MKCOL', - url: buildUrl(this.endpoint, directoryKey), }); } catch (error) { if (getStatus(error) === 405) continue; diff --git a/packages/webdav/test/fs-webdav.test.ts b/packages/webdav/test/fs-webdav.test.ts index 52b38305..9b9222e9 100644 --- a/packages/webdav/test/fs-webdav.test.ts +++ b/packages/webdav/test/fs-webdav.test.ts @@ -1,5 +1,10 @@ -import type { Binary, Progress, Request, RootFs } from '@hesprs/sync-engine-sdk'; -import type { ResponseControl, ResponseOverrides } from '@hesprs/sync-engine-sdk/dev'; +import type { + Binary, + MaybePromise, + Progress, + RequestParam, + RequestResponse, +} from '@hesprs/sync-engine-sdk'; import { chunkSize } from '@hesprs/sync-engine-sdk'; import { testKit } from '@hesprs/sync-engine-sdk/dev'; import { beforeEach, expect, mock, test } from 'bun:test'; @@ -10,21 +15,16 @@ import WebdavFs from '@/webdav/fs'; const { bytes, deferred, file, flush, request, stream: createStream } = testKit; const sharedDate = new Date('Mon, 01 Jan 2024 00:00:00 GMT').valueOf(); -type RequestParam = Exclude[0], string>; +type Control = (url: string, params: RequestParam) => MaybePromise>; type ParsedResponse = { multistatus: { response: Array } }; -type WebdavHarness = { - calls: Array; - fs: RootFs; - setRequest: (handler: ResponseControl) => void; -}; const emptyBinary: Binary = new Uint8Array(0); -const defaultResponse: ResponseOverrides = { +const defaultResponse = { bytes: () => emptyBinary, text: () => '', }; -let response: ResponseOverrides; +let response: Partial; let parsedResponse: ParsedResponse; const defaultOptions = { @@ -47,14 +47,14 @@ beforeEach(() => { }; }); -function createWebdavFs(options: Partial = {}): WebdavHarness { - let requestHandler: ResponseControl = () => response; - const harness = request((params) => requestHandler(params)); +function createWebdavFs(options: Partial = {}) { + let requestHandler: Control = () => response; + const harness = request((url, params) => requestHandler(url, params)); return { calls: harness.calls, fs: new WebdavFs({ ...defaultOptions, ...options, request: harness.request }), - setRequest: (handler: ResponseControl) => { + setRequest: (handler: Control) => { requestHandler = handler; }, }; @@ -97,7 +97,7 @@ async function collectStream(source: ReadableStream): Promise { } test('checkConnection returns success for a healthy endpoint', async () => { - const harness = request(() => defaultResponse); + const harness = request(() => defaultResponse); expect(await checkConnection(defaultOptions, harness.request)).toStrictEqual({ success: true }); expect(harness.calls[0]).toMatchObject({ @@ -151,9 +151,9 @@ test('stat parses dav fields and prefers etag for uid', async () => { test('writeStream buffers chunks into one put', async () => { const webdav = createWebdavFs(); - webdav.setRequest((params) => { + webdav.setRequest((url, params) => { expect(params.method).toBe('PUT'); - expect(params.url).toBe('https://dav.example.com/dav/Notes/file.md'); + expect(url).toBe('https://dav.example.com/dav/Notes/file.md'); expect(params.body).toStrictEqual(bytes('hello')); return { ...defaultResponse, headers: { etag: 'buffered-uid' } }; }); @@ -175,16 +175,16 @@ test('chunked writeStream uses exact Nextcloud urls and headers', async () => { }); let uploadFolderUrl = ''; const destination = 'https://dav.example.com/remote.php/dav/files/alice/Notes/file.md'; - webdav.setRequest((params) => { + webdav.setRequest((url, params) => { if (params.method === 'MKCOL') { - uploadFolderUrl = params.url; + uploadFolderUrl = url; expect(params.headers).toMatchObject({ Destination: destination, }); return { ...defaultResponse, status: 201 }; } if (params.method === 'PUT') { - expect(params.url).toBe(`${uploadFolderUrl}1`); + expect(url).toBe(`${uploadFolderUrl}1`); expect(params.headers).toMatchObject({ Destination: destination, 'OC-Total-Length': '7', @@ -192,7 +192,7 @@ test('chunked writeStream uses exact Nextcloud urls and headers', async () => { return { ...defaultResponse, status: 200 }; } if (params.method === 'MOVE') { - expect(params.url).toBe(`${uploadFolderUrl}.file`); + expect(url).toBe(`${uploadFolderUrl}.file`); expect(params.headers).toMatchObject({ Destination: destination, }); @@ -221,9 +221,9 @@ test('chunked writeStream uses exact Nextcloud urls and headers', async () => { test('empty chunked stream skips put and still mkcol move', async () => { const webdav = createWebdavFs({ chunkedUpload: true }); let uploadFolderUrl = ''; - webdav.setRequest((params) => { + webdav.setRequest((url, params) => { if (params.method === 'MKCOL') { - uploadFolderUrl = params.url; + uploadFolderUrl = url; return { ...defaultResponse, status: 201 }; } if (params.method === 'MOVE') return { ...defaultResponse, headers: { etag: 'empty-uid' } }; @@ -247,9 +247,9 @@ test('chunked upload error deletes temp folder and rethrows original error', () const webdav = createWebdavFs({ chunkedUpload: true }); let uploadFolderUrl = ''; const uploadError = new Error('upload failed'); - webdav.setRequest((params) => { + webdav.setRequest((url, params) => { if (params.method === 'MKCOL') { - uploadFolderUrl = params.url; + uploadFolderUrl = url; return { ...defaultResponse, status: 201 }; } if (params.method === 'PUT') throw uploadError; @@ -272,9 +272,9 @@ test('chunked finalization error deletes temp folder and rethrows original error const webdav = createWebdavFs({ chunkedUpload: true }); let uploadFolderUrl = ''; const moveError = new Error('move failed'); - webdav.setRequest((params) => { + webdav.setRequest((url, params) => { if (params.method === 'MKCOL') { - uploadFolderUrl = params.url; + uploadFolderUrl = url; return { ...defaultResponse, status: 201 }; } if (params.method === 'PUT') return { ...defaultResponse, status: 200 }; @@ -320,12 +320,12 @@ test('requestOrThrow throws parsed WebDAV error message with status', () => { test('mkdir recursively creates parent folders in order', async () => { const webdav = createWebdavFs({ endpoint: 'https://dav.example.com/dav' }); - webdav.setRequest((params) => { - if (params.url === 'https://dav.example.com/dav/Notes/') return response; - if (params.url === 'https://dav.example.com/dav/Notes/Folder%20A/') + webdav.setRequest((url, _params) => { + if (url === 'https://dav.example.com/dav/Notes/') return response; + if (url === 'https://dav.example.com/dav/Notes/Folder%20A/') return { ...defaultResponse, status: 405 }; - if (params.url === 'https://dav.example.com/dav/Notes/Folder%20A/Child/') return response; - throw new Error(`Unexpected URL: ${params.url}`); + if (url === 'https://dav.example.com/dav/Notes/Folder%20A/Child/') return response; + throw new Error(`Unexpected URL: ${url}`); }); await webdav.fs.mkdir('Notes/Folder A/Child/', true); @@ -427,16 +427,16 @@ test('list bfs updates progress when infinity is disabled', async () => { ]; const webdav = createWebdavFs({ endpoint: 'https://dav.example.com/dav' }); - webdav.setRequest((params) => { - if (params.url === 'https://dav.example.com/dav/Notes/') { + webdav.setRequest((url, _params) => { + if (url === 'https://dav.example.com/dav/Notes/') { setXmlResponse(rootItems); return response; } - if (params.url === 'https://dav.example.com/dav/Notes/Folder%20A/') { + if (url === 'https://dav.example.com/dav/Notes/Folder%20A/') { setXmlResponse(childItems); return response; } - throw new Error(`Unexpected URL: ${params.url}`); + throw new Error(`Unexpected URL: ${url}`); }); let storedProgress: Progress = { completed: 0, total: 0 }; @@ -492,12 +492,12 @@ test('list reporter can exclude entries and stop descent', async () => { ]; const webdav = createWebdavFs({ endpoint: 'https://dav.example.com/dav' }); - webdav.setRequest((params) => { - if (params.url === 'https://dav.example.com/dav/Notes/') { + webdav.setRequest((url, _params) => { + if (url === 'https://dav.example.com/dav/Notes/') { setXmlResponse(rootItems); return response; } - throw new Error(`Unexpected recursive request: ${params.url}`); + throw new Error(`Unexpected recursive request: ${url}`); }); const list = await webdav.fs.list('Notes/', ({ current }) => @@ -526,14 +526,14 @@ test('readStream requests SDK chunk size ranges from stat size', async () => { const ranges: Array = []; const encodings: Array = []; - const pending = new Map>>(); + const pending = new Map>>>(); const webdav = createWebdavFs({ endpoint: 'https://dav.example.com/dav' }); - webdav.setRequest((params) => { + webdav.setRequest((_url, params) => { if (params.method === 'PROPFIND') return response; const range = params.headers?.Range ?? ''; ranges.push(range); encodings.push(params.headers?.['Accept-Encoding']); - const wait = deferred(); + const wait = deferred>(); pending.set(range, wait); return wait.promise; }); @@ -544,13 +544,13 @@ test('readStream requests SDK chunk size ranges from stat size', async () => { }); const collected = collectStream( - await webdav.fs.readStream('Notes/file.bin', file('Notes/file.bin', { size })), + webdav.fs.readStream('Notes/file.bin', file('Notes/file.bin', { size })), ); await flush(); expect(ranges).toStrictEqual(expectedRanges); expect(encodings.every((encoding) => encoding === 'identity')).toBe(true); - const makeResponse = (byte: number): ResponseOverrides => ({ + const makeResponse = (byte: number): Partial => ({ bytes: () => new Uint8Array([byte]), status: 206, }); From d8e1b1a10e3f6649d5150352e7d6206122f3bf0d Mon Sep 17 00:00:00 2001 From: hesprs <190185753+hesprs@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:30:57 +0800 Subject: [PATCH 2/4] fix(sync): resolve double LocalFs instantiation race --- docs/src/pages/en/deep-dive/sync.md | 2 + docs/src/pages/en/development/events.md | 43 +++++++++++-------- packages/plugin/dist/dev.spec.d.ts | 2 +- ...LE8.spec.d.ts => index-DbHTJm2U.spec.d.ts} | 6 +++ packages/plugin/dist/index.spec.d.ts | 2 +- .../plugin/src/fs/wrappers/optimization.ts | 16 ++++++- packages/plugin/src/modules/Bootstrap.ts | 26 +++++------ packages/plugin/src/modules/Sync.ts | 2 + packages/plugin/src/settings/head.ts | 19 ++++---- packages/s3/src/s3/check-connection.ts | 21 +++++---- packages/s3/src/s3/fs.ts | 22 +--------- packages/s3/src/s3/url.ts | 19 ++++++++ packages/s3/test/check-connection.test.ts | 28 +++++++++--- 13 files changed, 126 insertions(+), 82 deletions(-) rename packages/plugin/dist/{index-C66-NLE8.spec.d.ts => index-DbHTJm2U.spec.d.ts} (99%) diff --git a/docs/src/pages/en/deep-dive/sync.md b/docs/src/pages/en/deep-dive/sync.md index a7e352a6..3ff3d6d7 100644 --- a/docs/src/pages/en/deep-dive/sync.md +++ b/docs/src/pages/en/deep-dive/sync.md @@ -20,6 +20,8 @@ Cancellation does not roll back completed operations. Task errors raised after c ## Traversal and Glob Matching +After infrastructure initialization, the routine dispatches `syncInitialized` with the run's `Infras` and the compiled matcher. + The routine compiles the configured matcher once, then starts local and remote discovery concurrently. Local traversal calls `localFs.list('/')` with the matcher. Full remote traversal receives a reporter that forwards progress and applies the matcher to each reported path. The default remote lister performs the full traversal. If the remote root does not exist, it recreates the root, clears records for the local/remote pair, and returns an empty list. diff --git a/docs/src/pages/en/development/events.md b/docs/src/pages/en/development/events.md index 86597735..58563707 100644 --- a/docs/src/pages/en/development/events.md +++ b/docs/src/pages/en/development/events.md @@ -39,28 +39,33 @@ unsubscribe(); `Events` is a merged event map contributed by all internal modules. Every event key and its payload type: -| Event | Payload | -| ---------------------- | ----------------------------------------------------------------------------------- | -| `logSync` | `string` sync log message | -| `logGeneral` | `string` general log message | -| `errorSync` | `string` sync error log message | -| `errorGeneral` | `string` general error log message | -| `moduleLoaded` | `string` module name | -| `moduleUnloaded` | `string` module name | -| `syncStarted` | `{ isCancelled: Ref; trigger: string }` | -| `remoteWalkProgress` | `Progress` | -| `syncTerminated` | `SyncTerminateReason` | -| `requestConfirmDelete` | `Array` pending local-remove tasks | -| `requestConfirmTasks` | `Array` | -| `syncCanceled` | `undefined` (no payload) | -| `taskCompleted` | `TaskInfo` (`{ name: TaskNames; key: string; prettyName: string; isDir: boolean }`) | -| `taskFailed` | `FailedTaskInfo` (`TaskInfo` & `{ error: string }`) | -| `executionStarted` | `Array` | -| `tasksConfirmed` | `Array` | -| `deleteConfirmed` | `{ delete: Array; reupload: Array }` | +| Event | Payload | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------- | +| `logSync` | `string` sync log message | +| `logGeneral` | `string` general log message | +| `errorSync` | `string` sync error log message | +| `errorGeneral` | `string` general error log message | +| `moduleLoaded` | `string` module name | +| `moduleUnloaded` | `string` module name | +| `syncStarted` | `{ isCancelled: Ref; trigger: string }` | +| `syncInitialized` | `Infras & { match: (path: string) => GlobMatchResult }` — the run's file systems, record store, and compiled matcher | +| `remoteWalkProgress` | `Progress` | +| `syncTerminated` | `SyncTerminateReason` | +| `requestConfirmDelete` | `Array` pending local-remove tasks | +| `requestConfirmTasks` | `Array` | +| `syncCanceled` | `undefined` (no payload) | +| `taskCompleted` | `TaskInfo` (`{ name: TaskNames; key: string; prettyName: string; isDir: boolean }`) | +| `taskFailed` | `FailedTaskInfo` (`TaskInfo` & `{ error: string }`) | +| `executionStarted` | `Array` | +| `tasksConfirmed` | `Array` | +| `deleteConfirmed` | `{ delete: Array; reupload: Array }` | ::: tip `syncStarted.isCancelled` is a SynthKernel `Ref` — call it as `isCancelled()` to read, or subscribe with `isCancelled.subscribe(...)`. ::: + +## Sync Lifecycle Events + +`syncStarted` fires before the file-system stacks exist; `syncInitialized` fires once per run after infrastructure initialization and before traversal, and is the only point where the sync's actual `localFs`, `remoteFs`, and `record` are published. Its `Infras` shape is `{ localFs: Fs; remoteFs: Fs; record: RecordStore }`, documented with the [remote lister](./sync#remote-lister); `match` is the compiled [inclusion/exclusion matcher](../usage/settings#inclusion-and-exclusion-rules). diff --git a/packages/plugin/dist/dev.spec.d.ts b/packages/plugin/dist/dev.spec.d.ts index 65ee48e3..9f16fd19 100644 --- a/packages/plugin/dist/dev.spec.d.ts +++ b/packages/plugin/dist/dev.spec.d.ts @@ -1,4 +1,4 @@ -import { At as FileStat, Dt as RootFs, Ft as RecordStatsMap, It as Stat, Lt as StatsMap, Mt as MaybePromise, Ot as WrappedFs, Pt as RecordStat, Rt as Binary, d as RequestParam, f as RequestResponse, jt as FolderStat, lt as TaskNames, q as Decider, u as Request, yt as Fs } from "./index-C66-NLE8.spec.js"; +import { At as FileStat, Dt as RootFs, Ft as RecordStatsMap, It as Stat, Lt as StatsMap, Mt as MaybePromise, Ot as WrappedFs, Pt as RecordStat, Rt as Binary, d as RequestParam, f as RequestResponse, jt as FolderStat, lt as TaskNames, q as Decider, u as Request, yt as Fs } from "./index-DbHTJm2U.spec.js"; //#region src/sdk/debug-wrapper.d.ts declare function debugWrapper(original: Fs, log: (content: string) => void): WrappedFs; //#endregion diff --git a/packages/plugin/dist/index-C66-NLE8.spec.d.ts b/packages/plugin/dist/index-DbHTJm2U.spec.d.ts similarity index 99% rename from packages/plugin/dist/index-C66-NLE8.spec.d.ts rename to packages/plugin/dist/index-DbHTJm2U.spec.d.ts index 4e687a61..7f2d6272 100644 --- a/packages/plugin/dist/index-C66-NLE8.spec.d.ts +++ b/packages/plugin/dist/index-DbHTJm2U.spec.d.ts @@ -1138,6 +1138,9 @@ type Events = MergeSingleKey; type Settings = MergeSingleKey; type Translations = MergeSingleKey; //#endregion +//#region src/utils/glob-match.d.ts +type GlobMatchResult = 'include' | 'exclude' | 'advance' | 'probe'; +//#endregion //#region src/modules/Sync.d.ts type SyncTerminateReason = { result: 'cancelled'; @@ -1186,6 +1189,9 @@ declare class Sync { isCancelled: Ref; trigger: string; }; + syncInitialized: Infras & { + match: (path: string) => GlobMatchResult; + }; remoteWalkProgress: Progress; syncTerminated: SyncTerminateReason; requestConfirmDelete: Array; diff --git a/packages/plugin/dist/index.spec.d.ts b/packages/plugin/dist/index.spec.d.ts index e3844332..a014556b 100644 --- a/packages/plugin/dist/index.spec.d.ts +++ b/packages/plugin/dist/index.spec.d.ts @@ -1,2 +1,2 @@ -import { $ as RemoveRecord, A as setNeedMigration, At as FileStat, B as ObsidianLanguageCode, C as digOriginal, Ct as MoveAtom, D as readWithSize, Dt as RootFs, E as pipe, Et as OutputAtom, F as CallableOrObjectTree, Ft as RecordStatsMap, G as On, H as Translate, I as SettingEntry, It as Stat, J as DeciderInput, K as CreateLocalDir, L as AugmentedModuleMeta, Lt as StatsMap, M as generateEditableList, Mt as MaybePromise, N as reactivelyValidate, Nt as Progress, O as writeWithValue, Ot as WrappedFs, P as s, Pt as RecordStat, Q as RemoveRemote, R as ModuleMeta, Rt as Binary, S as SelectFromContext, St as MkdirAtom, T as concurrency, Tt as OptimizerOutput, U as TranslationResource, V as Snippet, W as Dispatch, X as Upload, Y as TaskFactory, Z as ResolveConflict, _ as Context, _t as CustomAtom, a as FsWrapperEntry, at as AddRecord, b as Translations, bt as InputAtom, c as RemoteFsEntry, ct as ConflictResolverPayload, d as RequestParam, dt as DatabaseAsync, et as RemoveLocal, f as RequestResponse, ft as DatabaseSync, g as SyncTerminateReason, gt as BatchOptimizer, h as SyncOptions, ht as StoreSync, i as DeciderEntry, it as CreateRemoteDir, j as LabelDefinition, jt as FolderStat, k as prefixWrapper, kt as WriteAtom, l as RemoteRequestMiddlewareEntry, lt as TaskNames, m as RemoteLister, mt as StoreOperations, n as CheckConnectionResult, nt as MoveLocal, o as LocalRequestMiddlewareEntry, ot as BaseTask, p as TriggerEntry, pt as StoreAsync, q as Decider, r as ConflictResolverEntry, rt as Download, s as OptimizerEntry, st as ConflictResolver, t as VaultRequest, tt as MoveRemote, u as Request, ut as RecordStore, v as Events, vt as DeleteAtom, w as chunkSize, wt as OptimizerInput, x as ExistingMemoryDB, xt as ListReporter, y as Settings, yt as Fs, z as Fragment } from "./index-C66-NLE8.spec.js"; +import { $ as RemoveRecord, A as setNeedMigration, At as FileStat, B as ObsidianLanguageCode, C as digOriginal, Ct as MoveAtom, D as readWithSize, Dt as RootFs, E as pipe, Et as OutputAtom, F as CallableOrObjectTree, Ft as RecordStatsMap, G as On, H as Translate, I as SettingEntry, It as Stat, J as DeciderInput, K as CreateLocalDir, L as AugmentedModuleMeta, Lt as StatsMap, M as generateEditableList, Mt as MaybePromise, N as reactivelyValidate, Nt as Progress, O as writeWithValue, Ot as WrappedFs, P as s, Pt as RecordStat, Q as RemoveRemote, R as ModuleMeta, Rt as Binary, S as SelectFromContext, St as MkdirAtom, T as concurrency, Tt as OptimizerOutput, U as TranslationResource, V as Snippet, W as Dispatch, X as Upload, Y as TaskFactory, Z as ResolveConflict, _ as Context, _t as CustomAtom, a as FsWrapperEntry, at as AddRecord, b as Translations, bt as InputAtom, c as RemoteFsEntry, ct as ConflictResolverPayload, d as RequestParam, dt as DatabaseAsync, et as RemoveLocal, f as RequestResponse, ft as DatabaseSync, g as SyncTerminateReason, gt as BatchOptimizer, h as SyncOptions, ht as StoreSync, i as DeciderEntry, it as CreateRemoteDir, j as LabelDefinition, jt as FolderStat, k as prefixWrapper, kt as WriteAtom, l as RemoteRequestMiddlewareEntry, lt as TaskNames, m as RemoteLister, mt as StoreOperations, n as CheckConnectionResult, nt as MoveLocal, o as LocalRequestMiddlewareEntry, ot as BaseTask, p as TriggerEntry, pt as StoreAsync, q as Decider, r as ConflictResolverEntry, rt as Download, s as OptimizerEntry, st as ConflictResolver, t as VaultRequest, tt as MoveRemote, u as Request, ut as RecordStore, v as Events, vt as DeleteAtom, w as chunkSize, wt as OptimizerInput, x as ExistingMemoryDB, xt as ListReporter, y as Settings, yt as Fs, z as Fragment } from "./index-DbHTJm2U.spec.js"; export { type AddRecord, type AugmentedModuleMeta, type BaseTask, type BatchOptimizer, type Binary, type CallableOrObjectTree, type CheckConnectionResult, type ConflictResolver, type ConflictResolverEntry, type ConflictResolverPayload, type Context, type CreateLocalDir, type CreateRemoteDir, type CustomAtom, type DatabaseAsync, type DatabaseSync, type Decider, type DeciderEntry, type DeciderInput, type DeleteAtom, type Dispatch, type Download, type Events, type ExistingMemoryDB, type FileStat, type FolderStat, type Fragment, type Fs, type FsWrapperEntry, type InputAtom, type LabelDefinition, type ListReporter, type LocalRequestMiddlewareEntry, type MaybePromise, type MkdirAtom, type ModuleMeta, type MoveAtom, type MoveLocal, type MoveRemote, type ObsidianLanguageCode, type On, type OptimizerEntry, type OptimizerInput, type OptimizerOutput, type OutputAtom, type Progress, type RecordStat, type RecordStatsMap, type RecordStore, type RemoteFsEntry, type RemoteLister, type RemoteRequestMiddlewareEntry, type RemoveLocal, type RemoveRecord, type RemoveRemote, type Request, type RequestParam, type RequestResponse, type ResolveConflict, type RootFs, SelectFromContext, type SettingEntry, type Settings, type Snippet, type Stat, type StatsMap, type StoreAsync, type StoreOperations, type StoreSync, type SyncOptions, type SyncTerminateReason, type TaskFactory, type TaskNames, type Translate, type TranslationResource, type Translations, type TriggerEntry, type Upload, type VaultRequest, type WrappedFs, type WriteAtom, chunkSize, concurrency, digOriginal, generateEditableList, pipe, prefixWrapper, reactivelyValidate, readWithSize, s, setNeedMigration, writeWithValue }; \ No newline at end of file diff --git a/packages/plugin/src/fs/wrappers/optimization.ts b/packages/plugin/src/fs/wrappers/optimization.ts index d753e652..806012ae 100644 --- a/packages/plugin/src/fs/wrappers/optimization.ts +++ b/packages/plugin/src/fs/wrappers/optimization.ts @@ -211,23 +211,35 @@ class OptimizationFs implements WrappedFs { // Write operations race the flush timer, since the opposite side read gating them can complete before the timer fires. Companion wrapper observes reads and dispatches needle reads to the opposite side FS as an anticipation of write, and allows it to obtain ahead-of-time transformed write keys. Writes that arrive before the flush are held until it registers the anticipated write. class OptimizationCompanionFs implements WrappedFs { + private unwrapped?: Fs; + constructor( readonly original: Fs, private readonly options: OptimizationCompanionOptions, ) {} + private getThatFs() { + if (this.unwrapped) return this.unwrapped; + let original: Fs = this.options.getThatFs(); + while (!(original instanceof OptimizationCompanionFs) && 'original' in original) + original = original.original; + if ('original' in original) original = original.original as Fs; + this.unwrapped = original; + return original; + } + getUid() { return this.original.getUid(); } read(key: string, stat: FileStat) { this.options.thatPool.add(stat.key); // Dispatch a explore needle to opposite FS to observe the transformed key - attempt(() => this.options.getThatFs().read(key, stat)); + attempt(() => this.getThatFs().read(key, stat)); return this.original.read(key, stat); } readStream(key: string, stat: FileStat) { this.options.thatPool.add(stat.key); - attempt(() => this.options.getThatFs().read(key, stat)); + attempt(() => this.getThatFs().read(key, stat)); return this.original.readStream(key, stat); } write(key: string, value: Binary, stat: FileStat) { diff --git a/packages/plugin/src/modules/Bootstrap.ts b/packages/plugin/src/modules/Bootstrap.ts index 327cf686..90e4bb21 100644 --- a/packages/plugin/src/modules/Bootstrap.ts +++ b/packages/plugin/src/modules/Bootstrap.ts @@ -242,9 +242,8 @@ export default class Bootstrap { priority: 20_000, }); registerLocalFsWrapper({ - apply: (fs) => { - this.localFs = fs; - return optimizationCompanionWrapper(fs, { + apply: (fs) => + optimizationCompanionWrapper(fs, { getThatFs: () => { if (!this.remoteFs) throw new Error( @@ -253,8 +252,7 @@ export default class Bootstrap { return this.remoteFs; }, thatPool: this.remotePool, - }); - }, + }), priority: 21_000, }); @@ -312,9 +310,8 @@ export default class Bootstrap { priority: 20_000, }); registerRemoteFsWrapper({ - apply: (fs) => { - this.remoteFs = fs; - return optimizationCompanionWrapper(fs, { + apply: (fs) => + optimizationCompanionWrapper(fs, { getThatFs: () => { if (!this.localFs) throw new Error( @@ -323,8 +320,7 @@ export default class Bootstrap { return this.localFs; }, thatPool: this.localPool, - }); - }, + }), priority: 21_000, }); @@ -402,15 +398,19 @@ export default class Bootstrap { }); this.cleanupCallbacks.push( - on('syncStarted', ({ isCancelled }) => { - this.isCancelled = isCancelled; + on('syncStarted', ({ isCancelled }) => (this.isCancelled = isCancelled)), + on('syncInitialized', ({ localFs, remoteFs }) => { + this.localFs = localFs; + this.remoteFs = remoteFs; + }), + on('syncTerminated', () => { + this.isCancelled = undefined; this.memoryStates.hangingOperations.length = 0; this.localFs = undefined; this.remoteFs = undefined; this.localPool.clear(); this.remotePool.clear(); }), - on('syncTerminated', () => (this.isCancelled = undefined)), ); }; diff --git a/packages/plugin/src/modules/Sync.ts b/packages/plugin/src/modules/Sync.ts index 38fc4d7f..c7d1025e 100644 --- a/packages/plugin/src/modules/Sync.ts +++ b/packages/plugin/src/modules/Sync.ts @@ -71,6 +71,7 @@ export default class Sync { declare readonly events: { syncStarted: { isCancelled: Ref; trigger: string }; + syncInitialized: Infras & { match: (path: string) => GlobMatchResult }; remoteWalkProgress: Progress; syncTerminated: SyncTerminateReason; requestConfirmDelete: Array; @@ -181,6 +182,7 @@ export default class Sync { const match = prepareGlobMatch(inclusionRules, exclusionRules); const { reporter: localReporter, pruner: localPruner } = prepareReporter(match); const { reporter: remoteReporter, pruner: remotePruner } = prepareReporter(match); + dispatch('syncInitialized', { ...infras, match }); const [localList, remoteList] = await Promise.all([ localFs.list('/', localReporter), diff --git a/packages/plugin/src/settings/head.ts b/packages/plugin/src/settings/head.ts index dffaace4..acfafb11 100644 --- a/packages/plugin/src/settings/head.ts +++ b/packages/plugin/src/settings/head.ts @@ -223,6 +223,12 @@ function setupCheckConnection({ setError(); return; } + const onFailure = (message: string) => { + setError(); + log(`Check connection to \`${settings.remoteFs}\` failed: \`${message}\`.`); + if (force) new Notice(`${translate('checkConnectionFailed')}: ${message}`, 5000); + else scheduleCheckConnection(); + }; try { setChecking(); @@ -231,18 +237,9 @@ function setupCheckConnection({ memoryDB.setMeta('lastCheckedFs', settings.remoteFs); setSuccess(); if (force) new Notice(translate('checkConnectionSuccess')); - } else { - setError(); - log(`Check connection to \`${settings.remoteFs}\` failed: \`${result.reason}\`.`); - if (force) new Notice(`${translate('checkConnectionFailed')}: ${result.reason}`); - else scheduleCheckConnection(); - } + } else onFailure(result.reason); } catch (error) { - setError(); - const message = getMessage(error); - log(`Check connection to \`${settings.remoteFs}\` failed: \`${message}\`.`); - if (force) new Notice(`${translate('checkConnectionFailed')}: ${message}`); - else scheduleCheckConnection(); + onFailure(getMessage(error)); } }; diff --git a/packages/s3/src/s3/check-connection.ts b/packages/s3/src/s3/check-connection.ts index 9eec0cc0..91ce420e 100644 --- a/packages/s3/src/s3/check-connection.ts +++ b/packages/s3/src/s3/check-connection.ts @@ -1,7 +1,7 @@ import type { CheckConnectionResult, Request } from '@hesprs/sync-engine-sdk'; import { getMessage } from '@repo/shared/error'; import type { UrlStyle } from './sigv4'; -import { buildUrl } from './url'; +import { buildUrlWithQuery, parseS3Error } from './url'; export type S3ConnectionOptions = { endpoint: string; @@ -15,16 +15,19 @@ export async function checkConnection( request: Request, ): Promise { try { - const url = buildUrl({ - bucket: options.bucket, - endpoint: options.endpoint, - key: '/', - urlStyle: options.urlStyle, - }); - const response = await request(url, { method: 'HEAD', throw: false }); + const url = buildUrlWithQuery( + { + bucket: options.bucket, + endpoint: options.endpoint, + key: '/', + urlStyle: options.urlStyle, + }, + { 'list-type': '2', 'max-keys': '0' }, + ); + const response = await request(url, { method: 'GET', throw: false }); if (response.status >= 200 && response.status < 300) return { success: true } as const; return { - reason: `HTTP ${response.status}`, + reason: parseS3Error(response.text()) ?? `S3: HTTP ${response.status}`, success: false, } as const; } catch (error) { diff --git a/packages/s3/src/s3/fs.ts b/packages/s3/src/s3/fs.ts index e58316c7..4c5c4541 100644 --- a/packages/s3/src/s3/fs.ts +++ b/packages/s3/src/s3/fs.ts @@ -17,7 +17,7 @@ import createRangeReadStream from '@repo/shared/read-stream'; import type { UrlStyle } from './sigv4'; import { PART_SIZE, multipartUpload } from './multipart'; import { md5Base64 } from './sigv4'; -import { buildUrl, buildUrlWithQuery, getHeader } from './url'; +import { buildUrl, buildUrlWithQuery, formatS3Error, getHeader, parseS3Error } from './url'; export type S3FsOptions = { accessKeyId: string; @@ -30,13 +30,6 @@ export type S3FsOptions = { export const BATCH_DELETE_MAX_KEYS = 1000; -type S3ErrorResponse = { - Error?: { - Code?: string; - Message?: string; - }; -}; - type S3ListBucketResult = { ListBucketResult: { Contents?: S3Object | Array; @@ -81,19 +74,6 @@ function escapeXml(str: string): string { .replaceAll("'", '''); } -function parseS3Error(xml: string): string | undefined { - try { - const error = parseXML(xml).Error; - if (error?.Code) return formatS3Error(error.Code, error.Message); - } catch { - /* Ignore malformed S3 error XML and use the HTTP fallback. */ - } -} - -function formatS3Error(code: string, message?: string): string { - return `S3 ${code}: ${message ?? ''}`; -} - function asArray(value: T | Array | undefined): Array { return value === undefined ? [] : Array.isArray(value) ? value : [value]; } diff --git a/packages/s3/src/s3/url.ts b/packages/s3/src/s3/url.ts index 6ee38bd2..6d102a91 100644 --- a/packages/s3/src/s3/url.ts +++ b/packages/s3/src/s3/url.ts @@ -1,3 +1,4 @@ +import parseXML from '@repo/shared/parse-xml'; import { encodeUrl } from '@repo/shared/path'; import type { UrlStyle } from './sigv4'; @@ -8,6 +9,24 @@ export type UrlOptions = { urlStyle: UrlStyle; }; +type S3ErrorResponse = { + Error?: { + Code?: string; + Message?: string; + }; +}; +export function parseS3Error(xml: string): string | undefined { + try { + const error = parseXML(xml).Error; + if (error?.Code) return formatS3Error(error.Code, error.Message); + } catch { + /* Ignore malformed S3 error XML and use the HTTP fallback. */ + } +} +export function formatS3Error(code: string, message?: string): string { + return `S3 ${code}: ${message ?? ''}`; +} + export function buildUrl({ endpoint, bucket, key, urlStyle }: UrlOptions): string { const encodedPath = encodeUrl(key); if (urlStyle === 'virtualHosted') { diff --git a/packages/s3/test/check-connection.test.ts b/packages/s3/test/check-connection.test.ts index 7d8cebbf..1e8bea4b 100644 --- a/packages/s3/test/check-connection.test.ts +++ b/packages/s3/test/check-connection.test.ts @@ -1,9 +1,16 @@ import { testKit } from '@hesprs/sync-engine-sdk/dev'; -import { expect, test } from 'bun:test'; +import { expect, mock, test } from 'bun:test'; import { checkConnection } from '@/s3/check-connection'; import { sigv4Middleware } from '@/s3/sigv4'; import { defaultCredentials, defaultS3Options, memoryDB, response } from './helpers'; +void mock.module('@repo/shared/parse-xml', () => ({ + default: (xml: string) => { + if (!xml) throw new Error('empty XML'); + return { Error: { Code: 'AccessDenied', Message: 'Access Denied' } }; + }, +})); + const connectionOptions = { bucket: defaultS3Options.bucket, endpoint: defaultS3Options.endpoint, @@ -11,7 +18,7 @@ const connectionOptions = { urlStyle: defaultS3Options.urlStyle, }; -test('checkConnection uses the request pipeline for a signed head bucket request', async () => { +test('checkConnection uses the request pipeline for a signed empty list request', async () => { const harness = testKit.request(() => response()); const request = sigv4Middleware(harness.request, defaultCredentials, memoryDB); @@ -19,20 +26,31 @@ test('checkConnection uses the request pipeline for a signed head bucket request const call = harness.calls[0]; if (!call) throw new Error('Expected checkConnection request'); - expect(call.method).toBe('HEAD'); + expect(call.method).toBe('GET'); expect(call.headers?.['x-amz-content-sha256']).toBe('UNSIGNED-PAYLOAD'); expect(call.headers?.authorization).toMatch( /^AWS4-HMAC-SHA256 Credential=access-key\/\d{8}\/us-east-1\/s3\/aws4_request, SignedHeaders=.*?, Signature=[0-9a-f]{64}$/u, ); const url = new URL(call.url); expect(url.pathname).toBe('/vault/'); - expect(url.search).toBe(''); + expect(url.searchParams.get('list-type')).toBe('2'); + expect(url.searchParams.get('max-keys')).toBe('0'); +}); + +test('checkConnection surfaces the S3 error body on failure', async () => { + const denied = testKit.request(() => + response({ status: 403, text: '...' }), + ).request; + expect(await checkConnection(connectionOptions, denied)).toStrictEqual({ + reason: 'S3 AccessDenied: Access Denied', + success: false, + }); }); test('checkConnection returns HTTP and thrown request failures', async () => { const failed = testKit.request(() => response({ status: 403 })).request; expect(await checkConnection(connectionOptions, failed)).toStrictEqual({ - reason: 'HTTP 403', + reason: 'S3: HTTP 403', success: false, }); From 67e6580229928d458da5cf14142719c822102756 Mon Sep 17 00:00:00 2001 From: hesprs <190185753+hesprs@users.noreply.github.com> Date: Mon, 21 Sep 2026 15:22:02 +0800 Subject: [PATCH 3/4] fix(gdrive): fix error parsing and resumable upload --- .../pages/en/development/debug-and-testing.md | 20 ++++----- docs/src/pages/en/development/events.md | 40 +++++++++--------- packages/gdrive/src/gdrive/api.ts | 21 ++++++---- packages/gdrive/src/gdrive/upload.ts | 41 +++++++++++++------ packages/gdrive/test/fs-gdrive.test.ts | 40 ++++++++++++++++++ .../webdav/src/webdav/check-connection.ts | 6 +-- 6 files changed, 114 insertions(+), 54 deletions(-) diff --git a/docs/src/pages/en/development/debug-and-testing.md b/docs/src/pages/en/development/debug-and-testing.md index 84b47f8a..788b0221 100644 --- a/docs/src/pages/en/development/debug-and-testing.md +++ b/docs/src/pages/en/development/debug-and-testing.md @@ -66,16 +66,16 @@ const testKit: { }; ``` -| Helper | Description | -| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `bytes(value)` | Convert a string to `Binary`. | -| `deferred()` | Create a controlled promise. | -| `file(key, options?)` | Create a `FileStat`. | -| `folder(key)` | Create a `FolderStat`. | -| `flush(turns?)` | Wait for several microtask queues to finish (default 4). | -| `fs(options?)` | Create a stub filesystem. `control` overrides individual methods; `uid` sets `getUid()`. | -| `request(control)` | Wrap a response control to record calls. The control receives the same arguments as `Request` and returns response overrides; each recorded call is the url merged into its parameters. | -| `stream(chunks?)` | Create a fake `ReadableStream` from an array. | +| Helper | Description | +| --------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `bytes(value)` | Convert a string to `Binary`. | +| `deferred()` | Create a controlled promise. | +| `file(key, options?)` | Create a `FileStat`. | +| `folder(key)` | Create a `FolderStat`. | +| `flush(turns?)` | Wait for several microtask queues to finish (default 4). | +| `fs(options?)` | Create a stub filesystem. `control` overrides individual methods; `uid` sets `getUid()`. | +| `request(control)` | Wrap a response control to record calls. The control receives the same arguments as `Request` and returns response overrides. | +| `stream(chunks?)` | Create a fake `ReadableStream` from an array. | ### `fs()` Details diff --git a/docs/src/pages/en/development/events.md b/docs/src/pages/en/development/events.md index 58563707..8eb36214 100644 --- a/docs/src/pages/en/development/events.md +++ b/docs/src/pages/en/development/events.md @@ -39,26 +39,26 @@ unsubscribe(); `Events` is a merged event map contributed by all internal modules. Every event key and its payload type: -| Event | Payload | -| ---------------------- | -------------------------------------------------------------------------------------------------------------------- | -| `logSync` | `string` sync log message | -| `logGeneral` | `string` general log message | -| `errorSync` | `string` sync error log message | -| `errorGeneral` | `string` general error log message | -| `moduleLoaded` | `string` module name | -| `moduleUnloaded` | `string` module name | -| `syncStarted` | `{ isCancelled: Ref; trigger: string }` | -| `syncInitialized` | `Infras & { match: (path: string) => GlobMatchResult }` — the run's file systems, record store, and compiled matcher | -| `remoteWalkProgress` | `Progress` | -| `syncTerminated` | `SyncTerminateReason` | -| `requestConfirmDelete` | `Array` pending local-remove tasks | -| `requestConfirmTasks` | `Array` | -| `syncCanceled` | `undefined` (no payload) | -| `taskCompleted` | `TaskInfo` (`{ name: TaskNames; key: string; prettyName: string; isDir: boolean }`) | -| `taskFailed` | `FailedTaskInfo` (`TaskInfo` & `{ error: string }`) | -| `executionStarted` | `Array` | -| `tasksConfirmed` | `Array` | -| `deleteConfirmed` | `{ delete: Array; reupload: Array }` | +| Event | Payload | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `logSync` | `string` sync log message | +| `logGeneral` | `string` general log message | +| `errorSync` | `string` sync error log message | +| `errorGeneral` | `string` general error log message | +| `moduleLoaded` | `string` module name | +| `moduleUnloaded` | `string` module name | +| `syncStarted` | `{ isCancelled: Ref; trigger: string }` | +| `syncInitialized` | `Infras & { match: (path: string) => GlobMatchResult }`: the run's file systems, record store, and compiled matcher | +| `remoteWalkProgress` | `Progress` | +| `syncTerminated` | `SyncTerminateReason` | +| `requestConfirmDelete` | `Array` pending local-remove tasks | +| `requestConfirmTasks` | `Array` | +| `syncCanceled` | `undefined` (no payload) | +| `taskCompleted` | `TaskInfo` (`{ name: TaskNames; key: string; prettyName: string; isDir: boolean }`) | +| `taskFailed` | `FailedTaskInfo` (`TaskInfo` & `{ error: string }`) | +| `executionStarted` | `Array` | +| `tasksConfirmed` | `Array` | +| `deleteConfirmed` | `{ delete: Array; reupload: Array }` | ::: tip diff --git a/packages/gdrive/src/gdrive/api.ts b/packages/gdrive/src/gdrive/api.ts index c5dff975..cddfe59c 100644 --- a/packages/gdrive/src/gdrive/api.ts +++ b/packages/gdrive/src/gdrive/api.ts @@ -24,6 +24,11 @@ export type DriveFileList = { nextPageToken?: string; }; +type DriveError = { + error?: { code?: number; message?: string } | string; + error_description?: string; +}; + const mtimeMissing = new Error('Google Drive did not return the modified time for a file!'); /** Escapes a string literal used inside a Drive `q` search expression. */ @@ -48,14 +53,14 @@ export function getHeader( } export function parseDriveError(response: RequestResponse): string | undefined { - const parsed = response as { - error?: { code?: number; message?: string } | string; - error_description?: string; - }; - if (typeof parsed.error === 'string') - return `Google Drive ${parsed.error}: ${parsed.error_description ?? ''}`; - if (parsed.error?.message) - return `Google Drive ${parsed.error.code ?? response.status}: ${parsed.error.message}`; + try { + const { error, error_description } = response.json(); + if (typeof error === 'string') return `Google Drive ${error}: ${error_description ?? ''}`; + if (error?.message) + return `Google Drive ${error.code ?? response.status}: ${error.message}`; + } catch { + // Non-JSON error body (e.g. empty 503 responses). + } } export function toFileStat(key: string, file: DriveFile): FileStat { diff --git a/packages/gdrive/src/gdrive/upload.ts b/packages/gdrive/src/gdrive/upload.ts index 75c9ca1b..c1c2c180 100644 --- a/packages/gdrive/src/gdrive/upload.ts +++ b/packages/gdrive/src/gdrive/upload.ts @@ -1,10 +1,11 @@ import type { Binary, Request, RequestResponse } from '@hesprs/sync-engine-sdk'; -import { chunkSize, concurrency } from '@hesprs/sync-engine-sdk'; import { concatBinary, textToUint8Array } from '@repo/shared/binary'; -import chunkedUpload from '@repo/shared/chunked-upload'; import type { DriveFile } from './api'; import { getHeader, parseDriveError } from './api'; +// Resumable uploads must be sequential (Drive rejects chunks that skip ahead of the uploaded size), so the SDK's memory-tuned chunk size does not apply here. +const GDRIVE_CHUNK_SIZE = 8 * 1024 ** 2; + const MIME_BY_EXTENSION: Record = { base: 'application/json', canvas: 'application/json', @@ -122,23 +123,37 @@ export async function resumableUpload( value: ReadableStream, ): Promise { const session = await startSession(options); - const total = options.size; + const { size: total } = options; + const reader = value.getReader(); + let buffer: Binary = new Uint8Array(0); let final: RequestResponse | undefined; - await chunkedUpload({ - chunkSize, - concurrency, - onChunkResult: (response) => { - if (response) final = response; - }, - uploadChunk: (chunk, _index, offset) => putChunk(session, chunk, offset, total), - value, - }).catch((error: unknown) => { + let offset = 0; + // Sequential by necessity: a Drive session rejects any chunk whose offset + const upload = async (chunk: Binary) => { + const response = await putChunk(session, chunk, offset, total); + offset += chunk.byteLength; + if (response) final = response; + }; + try { + let done = false; + while (!done) { + const read = await reader.read(); + if (read.done) done = true; + else buffer = concatBinary(buffer, read.value); + while (buffer.byteLength >= GDRIVE_CHUNK_SIZE) { + const chunk = buffer.slice(0, GDRIVE_CHUNK_SIZE); + buffer = buffer.slice(GDRIVE_CHUNK_SIZE); + await upload(chunk); + } + } + if (buffer.byteLength > 0) await upload(buffer); + } catch (error) { // Best-effort session cancellation; Drive also expires sessions on its own. void options .request(session.location, { ignoreCancellation: true, method: 'DELETE' }) .catch(() => {}); throw error; - }); + } final ??= await putChunk(session, new Uint8Array(0), total, total); if (!final) throw new Error('Google Drive upload finished incomplete.'); return final.json(); diff --git a/packages/gdrive/test/fs-gdrive.test.ts b/packages/gdrive/test/fs-gdrive.test.ts index 02d53705..71f09bca 100644 --- a/packages/gdrive/test/fs-gdrive.test.ts +++ b/packages/gdrive/test/fs-gdrive.test.ts @@ -104,3 +104,43 @@ test('moves a cached file with Drive native rename', async () => { expect(move?.url).toContain('/files/file-1'); expect(new TextDecoder().decode(move?.body as Binary)).toBe('{"name":"new.md"}'); }); + +test('uploads streamed files in ascending contiguous chunks over one resumable session', async () => { + const chunkSize = 8 * 1024 ** 2; // Fixed upload chunk size, independent of SDK settings. + const location = 'https://upload.googleapis.com/session/1'; + let puts = 0; + const { calls, fs } = createFs((url, params) => { + if (params.method === 'POST') return response({}, 200, { location }); + if (params.method === 'PUT') + return ++puts < 3 + ? response({}, 308) + : response({ id: 'file-1', md5Checksum: 'drive-uid' }); + throw new Error(`Unexpected request: ${params.method} ${url}`); + }); + + const pieces = [ + bytes('a'.repeat(1000)), + new Uint8Array(chunkSize).fill(1), + new Uint8Array(chunkSize).fill(2), + bytes('b'.repeat(5)), + ]; + const total = pieces.reduce((sum, piece) => sum + piece.byteLength, 0); + const stream = new ReadableStream({ + start(controller) { + pieces.forEach((piece) => controller.enqueue(piece)); + controller.close(); + }, + }); + + expect( + await fs.writeStream('big.bin', stream, file('big.bin', { mtime: 1, size: total })), + ).toBe('drive-uid'); + const ranges = calls + .filter((call) => call.method === 'PUT') + .map((call) => call.headers?.['Content-Range']); + expect(ranges).toStrictEqual([ + `bytes 0-${chunkSize - 1}/${total}`, + `bytes ${chunkSize}-${chunkSize * 2 - 1}/${total}`, + `bytes ${chunkSize * 2}-${total - 1}/${total}`, + ]); +}); diff --git a/packages/webdav/src/webdav/check-connection.ts b/packages/webdav/src/webdav/check-connection.ts index 6e39411f..b9e06f79 100644 --- a/packages/webdav/src/webdav/check-connection.ts +++ b/packages/webdav/src/webdav/check-connection.ts @@ -15,12 +15,12 @@ const CHECK_CONNECTION_BODY = ` `; export async function checkConnection( - options: WebdavConnectionOptions, + { username, password, endpoint }: WebdavConnectionOptions, request: Request, ): Promise { - const Authorization = getAuthorization(options.username, options.password); + const Authorization = getAuthorization(username, password); try { - const response = await request(buildUrl(normalizeUrl(options.endpoint), '/'), { + const response = await request(buildUrl(normalizeUrl(endpoint), '/'), { body: CHECK_CONNECTION_BODY, contentType: 'application/xml', headers: { Authorization, Depth: '0' }, From 9d74bd795fcb9891acced8c02991dd2c6a8ed90c Mon Sep 17 00:00:00 2001 From: hesprs <190185753+hesprs@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:13:04 +0800 Subject: [PATCH 4/4] chore(ver): prepare release --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ manifest.json | 2 +- modules.json | 12 ++++++------ packages/plugin/package.json | 2 +- packages/shared/src/read-stream.ts | 7 ++++++- versions.json | 3 ++- 6 files changed, 45 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c02fc8d..f8230844 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,35 @@ All notable changes to this project will be documented in this file. +## Sync Engine v3.1.9 - 2026-09-21 + +### Core + +- Fixed local operation optimization fails to apply due to double local filesystem instantiation. +- Made operation failures retry on abnormal status codes and iOS/macOS specific errors. +- Improved iOS/iPadOS local file streaming to support all file formats. +- Ensured proper cleanup of local file read or write streaming on failure or cancellation. + +### Google Drive Module + +- Improved small file upload speed by uploading those files in a single multipart request. +- Fixed ineffective error message extraction. +- Ensured proper cleanup of resumable upload sessions on failure or cancellation. + +### S3 Module + +- Fixed ineffective error handling due to request middleware parameter rewrite. +- Made connection check return server message instead of generic error codes. +- Ensured proper cleanup of multipart upload sessions on failure or cancellation. + +### WebDAV Module + +- Ensured proper cleanup of Nextcloud-style chunked upload sessions on failure or cancellation. + +### Contributors + +@kuznetsov-m, @hesprs + ## Sync Engine v3.1.8 - 2026-09-19 - Improved iOS/iPadOS local file streaming by ranged requests on supported file formats, instead of relying on the already-broken single `fetch` streaming. diff --git a/manifest.json b/manifest.json index d12d03f1..a5e6c5b4 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "sync-engine", "name": "Sync Engine", - "version": "3.1.8", + "version": "3.1.9", "minAppVersion": "1.13.0", "authorUrl": "https://hesprs.github.io", "description": "The extensible vault synchronization engine: Fast · Free · Reliable. Supports WebDAV, S3, and Google Drive.", diff --git a/modules.json b/modules.json index d9c61559..523af44f 100644 --- a/modules.json +++ b/modules.json @@ -2,31 +2,31 @@ { "id": "webdav", "name": "WebDAV", - "version": "0.1.17", + "version": "0.1.18", "description": "WebDAV backend support.", "icon": "server", "main": "https://sync.consensia.cc/modules/webdav.js", - "minPluginVersion": "3.1.0", + "minPluginVersion": "3.1.9", "readme": "https://sync.consensia.cc/deep-dive/modules/webdav" }, { "id": "s3", "name": "S3", - "version": "0.1.5", + "version": "0.1.6", "description": "S3 and S3-compatible backend support.", "icon": "server", "main": "https://sync.consensia.cc/modules/s3.js", - "minPluginVersion": "3.1.5", + "minPluginVersion": "3.1.9", "readme": "https://sync.consensia.cc/deep-dive/modules/s3" }, { "id": "gdrive", "name": "Google Drive", - "version": "0.0.6", + "version": "0.1.0", "description": "Google Drive backend support.", "icon": "server", "main": "https://sync.consensia.cc/modules/gdrive.js", - "minPluginVersion": "3.1.5", + "minPluginVersion": "3.1.9", "readme": "https://sync.consensia.cc/deep-dive/modules/gdrive" }, { diff --git a/packages/plugin/package.json b/packages/plugin/package.json index 8d6205ff..167cb6f3 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@hesprs/sync-engine-sdk", - "version": "3.1.8", + "version": "3.1.9", "description": "Official SDK for developing modules targeting Sync Engine, the extensible Obsidian syncing plugin.", "keywords": [ "obsidian-plugin", diff --git a/packages/shared/src/read-stream.ts b/packages/shared/src/read-stream.ts index f27f8030..239b340e 100644 --- a/packages/shared/src/read-stream.ts +++ b/packages/shared/src/read-stream.ts @@ -21,7 +21,12 @@ export default function createRangeReadStream({ const runFinalize = () => { if (!finalize || finalized) return; finalized = true; - void finalize(); + try { + const result = finalize(); + if (result instanceof Promise) result.catch(() => {}); + } catch { + // Best-effort and silence errors + } }; if (totalChunks === 0) { runFinalize(); diff --git a/versions.json b/versions.json index 87e2657d..95d9633f 100644 --- a/versions.json +++ b/versions.json @@ -14,5 +14,6 @@ "3.1.5": "1.13.0", "3.1.6": "1.13.0", "3.1.7": "1.13.0", - "3.1.8": "1.13.0" + "3.1.8": "1.13.0", + "3.1.9": "1.13.0" }