diff --git a/modules.json b/modules.json index 523af44f..082cde95 100644 --- a/modules.json +++ b/modules.json @@ -2,7 +2,7 @@ { "id": "webdav", "name": "WebDAV", - "version": "0.1.18", + "version": "0.1.19", "description": "WebDAV backend support.", "icon": "server", "main": "https://sync.consensia.cc/modules/webdav.js", @@ -12,7 +12,7 @@ { "id": "s3", "name": "S3", - "version": "0.1.6", + "version": "0.1.7", "description": "S3 and S3-compatible backend support.", "icon": "server", "main": "https://sync.consensia.cc/modules/s3.js", diff --git a/packages/plugin/dist/dev.spec.d.ts b/packages/plugin/dist/dev.spec.d.ts index 9f16fd19..49f84432 100644 --- a/packages/plugin/dist/dev.spec.d.ts +++ b/packages/plugin/dist/dev.spec.d.ts @@ -1,4 +1,4 @@ -import { At as FileStat, Dt as RootFs, Ft as RecordStatsMap, It as Stat, Lt as StatsMap, Mt as MaybePromise, Ot as WrappedFs, Pt as RecordStat, Rt as Binary, d as RequestParam, f as RequestResponse, jt as FolderStat, lt as TaskNames, q as Decider, u as Request, yt as Fs } from "./index-DbHTJm2U.spec.js"; +import { At as FileStat, Dt as RootFs, Ft as RecordStatsMap, It as Stat, Lt as StatsMap, Mt as MaybePromise, Ot as WrappedFs, Pt as RecordStat, Rt as Binary, d as RequestParam, f as RequestResponse, jt as FolderStat, lt as TaskNames, q as Decider, u as Request, yt as Fs } from "./index-JdV6c5O1.spec.js"; //#region src/sdk/debug-wrapper.d.ts declare function debugWrapper(original: Fs, log: (content: string) => void): WrappedFs; //#endregion diff --git a/packages/plugin/dist/index-DbHTJm2U.spec.d.ts b/packages/plugin/dist/index-JdV6c5O1.spec.d.ts similarity index 99% rename from packages/plugin/dist/index-DbHTJm2U.spec.d.ts rename to packages/plugin/dist/index-JdV6c5O1.spec.d.ts index 7f2d6272..0a2f54bb 100644 --- a/packages/plugin/dist/index-DbHTJm2U.spec.d.ts +++ b/packages/plugin/dist/index-JdV6c5O1.spec.d.ts @@ -212,16 +212,7 @@ declare class Storage { readonly root: { clearRecordStores: () => Promise; deleteRecordStore: (namespace?: string) => MaybePromise; - getRecordStore: (namespace?: string) => { - get(key: string): Promise; - set(key: string, value: RecordStat): Promise; - delete(key: string): Promise; - clear(): Promise; - keys(): Promise; - values(): Promise; - entries(): Promise<[string, RecordStat][]>; - batch(operations: StoreOperations[]): Promise[]>; - }; + getRecordStore: (namespace?: string) => RecordStore; indexedDB: DatabaseAsync; memoryDB: { getStore(name: K): { diff --git a/packages/plugin/dist/index.spec.d.ts b/packages/plugin/dist/index.spec.d.ts index a014556b..d9a38e5a 100644 --- a/packages/plugin/dist/index.spec.d.ts +++ b/packages/plugin/dist/index.spec.d.ts @@ -1,2 +1,2 @@ -import { $ as RemoveRecord, A as setNeedMigration, At as FileStat, B as ObsidianLanguageCode, C as digOriginal, Ct as MoveAtom, D as readWithSize, Dt as RootFs, E as pipe, Et as OutputAtom, F as CallableOrObjectTree, Ft as RecordStatsMap, G as On, H as Translate, I as SettingEntry, It as Stat, J as DeciderInput, K as CreateLocalDir, L as AugmentedModuleMeta, Lt as StatsMap, M as generateEditableList, Mt as MaybePromise, N as reactivelyValidate, Nt as Progress, O as writeWithValue, Ot as WrappedFs, P as s, Pt as RecordStat, Q as RemoveRemote, R as ModuleMeta, Rt as Binary, S as SelectFromContext, St as MkdirAtom, T as concurrency, Tt as OptimizerOutput, U as TranslationResource, V as Snippet, W as Dispatch, X as Upload, Y as TaskFactory, Z as ResolveConflict, _ as Context, _t as CustomAtom, a as FsWrapperEntry, at as AddRecord, b as Translations, bt as InputAtom, c as RemoteFsEntry, ct as ConflictResolverPayload, d as RequestParam, dt as DatabaseAsync, et as RemoveLocal, f as RequestResponse, ft as DatabaseSync, g as SyncTerminateReason, gt as BatchOptimizer, h as SyncOptions, ht as StoreSync, i as DeciderEntry, it as CreateRemoteDir, j as LabelDefinition, jt as FolderStat, k as prefixWrapper, kt as WriteAtom, l as RemoteRequestMiddlewareEntry, lt as TaskNames, m as RemoteLister, mt as StoreOperations, n as CheckConnectionResult, nt as MoveLocal, o as LocalRequestMiddlewareEntry, ot as BaseTask, p as TriggerEntry, pt as StoreAsync, q as Decider, r as ConflictResolverEntry, rt as Download, s as OptimizerEntry, st as ConflictResolver, t as VaultRequest, tt as MoveRemote, u as Request, ut as RecordStore, v as Events, vt as DeleteAtom, w as chunkSize, wt as OptimizerInput, x as ExistingMemoryDB, xt as ListReporter, y as Settings, yt as Fs, z as Fragment } from "./index-DbHTJm2U.spec.js"; +import { $ as RemoveRecord, A as setNeedMigration, At as FileStat, B as ObsidianLanguageCode, C as digOriginal, Ct as MoveAtom, D as readWithSize, Dt as RootFs, E as pipe, Et as OutputAtom, F as CallableOrObjectTree, Ft as RecordStatsMap, G as On, H as Translate, I as SettingEntry, It as Stat, J as DeciderInput, K as CreateLocalDir, L as AugmentedModuleMeta, Lt as StatsMap, M as generateEditableList, Mt as MaybePromise, N as reactivelyValidate, Nt as Progress, O as writeWithValue, Ot as WrappedFs, P as s, Pt as RecordStat, Q as RemoveRemote, R as ModuleMeta, Rt as Binary, S as SelectFromContext, St as MkdirAtom, T as concurrency, Tt as OptimizerOutput, U as TranslationResource, V as Snippet, W as Dispatch, X as Upload, Y as TaskFactory, Z as ResolveConflict, _ as Context, _t as CustomAtom, a as FsWrapperEntry, at as AddRecord, b as Translations, bt as InputAtom, c as RemoteFsEntry, ct as ConflictResolverPayload, d as RequestParam, dt as DatabaseAsync, et as RemoveLocal, f as RequestResponse, ft as DatabaseSync, g as SyncTerminateReason, gt as BatchOptimizer, h as SyncOptions, ht as StoreSync, i as DeciderEntry, it as CreateRemoteDir, j as LabelDefinition, jt as FolderStat, k as prefixWrapper, kt as WriteAtom, l as RemoteRequestMiddlewareEntry, lt as TaskNames, m as RemoteLister, mt as StoreOperations, n as CheckConnectionResult, nt as MoveLocal, o as LocalRequestMiddlewareEntry, ot as BaseTask, p as TriggerEntry, pt as StoreAsync, q as Decider, r as ConflictResolverEntry, rt as Download, s as OptimizerEntry, st as ConflictResolver, t as VaultRequest, tt as MoveRemote, u as Request, ut as RecordStore, v as Events, vt as DeleteAtom, w as chunkSize, wt as OptimizerInput, x as ExistingMemoryDB, xt as ListReporter, y as Settings, yt as Fs, z as Fragment } from "./index-JdV6c5O1.spec.js"; export { type AddRecord, type AugmentedModuleMeta, type BaseTask, type BatchOptimizer, type Binary, type CallableOrObjectTree, type CheckConnectionResult, type ConflictResolver, type ConflictResolverEntry, type ConflictResolverPayload, type Context, type CreateLocalDir, type CreateRemoteDir, type CustomAtom, type DatabaseAsync, type DatabaseSync, type Decider, type DeciderEntry, type DeciderInput, type DeleteAtom, type Dispatch, type Download, type Events, type ExistingMemoryDB, type FileStat, type FolderStat, type Fragment, type Fs, type FsWrapperEntry, type InputAtom, type LabelDefinition, type ListReporter, type LocalRequestMiddlewareEntry, type MaybePromise, type MkdirAtom, type ModuleMeta, type MoveAtom, type MoveLocal, type MoveRemote, type ObsidianLanguageCode, type On, type OptimizerEntry, type OptimizerInput, type OptimizerOutput, type OutputAtom, type Progress, type RecordStat, type RecordStatsMap, type RecordStore, type RemoteFsEntry, type RemoteLister, type RemoteRequestMiddlewareEntry, type RemoveLocal, type RemoveRecord, type RemoveRemote, type Request, type RequestParam, type RequestResponse, type ResolveConflict, type RootFs, SelectFromContext, type SettingEntry, type Settings, type Snippet, type Stat, type StatsMap, type StoreAsync, type StoreOperations, type StoreSync, type SyncOptions, type SyncTerminateReason, type TaskFactory, type TaskNames, type Translate, type TranslationResource, type Translations, type TriggerEntry, type Upload, type VaultRequest, type WrappedFs, type WriteAtom, chunkSize, concurrency, digOriginal, generateEditableList, pipe, prefixWrapper, reactivelyValidate, readWithSize, s, setNeedMigration, writeWithValue }; \ No newline at end of file diff --git a/packages/plugin/src/modules/Storage.ts b/packages/plugin/src/modules/Storage.ts index d6d4b9db..1f84b1d7 100644 --- a/packages/plugin/src/modules/Storage.ts +++ b/packages/plugin/src/modules/Storage.ts @@ -14,7 +14,7 @@ export default class Storage { constructor(private readonly ctx: { getNamespace: () => string }) {} - private readonly getRecordStore = (namespace?: string) => + private readonly getRecordStore = (namespace?: string): RecordStore => this.indexedDB.getStore(namespace || this.ctx.getNamespace()); private readonly deleteRecordStore = (namespace?: string): MaybePromise => { diff --git a/packages/plugin/src/settings/head.ts b/packages/plugin/src/settings/head.ts index acfafb11..57576f25 100644 --- a/packages/plugin/src/settings/head.ts +++ b/packages/plugin/src/settings/head.ts @@ -213,6 +213,7 @@ function setupCheckConnection({ }; const scheduleCheckConnection = () => (timeout = window.setTimeout(() => void check(), CHECK_CONNECTION_INTERVAL)); + const cleanup = () => window.clearTimeout(timeout); const check = async (force = false) => { if (memoryDB.getMeta('lastCheckedFs') === settings.remoteFs && !force) { @@ -227,7 +228,8 @@ function setupCheckConnection({ setError(); log(`Check connection to \`${settings.remoteFs}\` failed: \`${message}\`.`); if (force) new Notice(`${translate('checkConnectionFailed')}: ${message}`, 5000); - else scheduleCheckConnection(); + cleanup(); + scheduleCheckConnection(); }; try { @@ -243,7 +245,7 @@ function setupCheckConnection({ } }; - return { check, cleanup: () => window.clearTimeout(timeout) }; + return { check, cleanup }; } function addLabel( diff --git a/packages/s3/src/index.ts b/packages/s3/src/index.ts index 50055850..3b611317 100644 --- a/packages/s3/src/index.ts +++ b/packages/s3/src/index.ts @@ -13,9 +13,14 @@ import type { Context, DatabaseSync, Binary, + RecordStore, + StoreOperations, + Stat, + RecordStat, } from '@hesprs/sync-engine-sdk'; import type { App } from 'obsidian'; import { digOriginal, prefixWrapper } from '@hesprs/sync-engine-sdk'; +import normalizeEtag from '@repo/shared/normalize-etag'; import type { UrlStyle } from '@/s3/sigv4'; import type { S3Translations } from '@/setting'; import { sigv4Middleware } from '@/s3/sigv4'; @@ -59,6 +64,7 @@ export default class S3 { registerRemoteOptimizer: (entry: OptimizerEntry) => () => void; registerRemoteRequestMiddleware: (entry: RemoteRequestMiddlewareEntry) => () => void; memoryDB: S3DB; + getRecordStore: (namespace?: string) => RecordStore; // TODO: remove after October 13 }>, ) { ctx.registerI18n('en', en); @@ -96,6 +102,7 @@ export default class S3 { registerRemoteOptimizer, registerRemoteRequestMiddleware, memoryDB, + getRecordStore, } = this.ctx; this.cleanup.push( registerRemoteFs('s3', { @@ -159,6 +166,8 @@ export default class S3 { priority: 604, }), ); + + if (this.settings.remoteFs === 's3') void migrateEtag(getRecordStore()).catch(() => {}); }; private readonly resolveConfig = () => { @@ -196,3 +205,16 @@ export default class S3 { this.cleanup.length = 0; }; } + +// TODO: remove after October 13 +async function migrateEtag(store: RecordStore) { + const changes: Array> = []; + for (const [key, stat] of await store.entries()) { + if (stat.isDir) return; + const remote = normalizeEtag(stat.remote); + if (remote !== stat.remote) + changes.push({ key, type: 'set', value: Object.assign(stat, { remote }) }); + } + if (!changes.length) return; + await store.batch(changes); +} diff --git a/packages/s3/src/s3/check-connection.ts b/packages/s3/src/s3/check-connection.ts index 91ce420e..9bc7b044 100644 --- a/packages/s3/src/s3/check-connection.ts +++ b/packages/s3/src/s3/check-connection.ts @@ -1,7 +1,8 @@ import type { CheckConnectionResult, Request } from '@hesprs/sync-engine-sdk'; import { getMessage } from '@repo/shared/error'; import type { UrlStyle } from './sigv4'; -import { buildUrlWithQuery, parseS3Error } from './url'; +import { buildUrlWithQuery } from './url'; +import { parseS3Error } from './utils'; export type S3ConnectionOptions = { endpoint: string; diff --git a/packages/s3/src/s3/fs.ts b/packages/s3/src/s3/fs.ts index 4c5c4541..fc0f1f8a 100644 --- a/packages/s3/src/s3/fs.ts +++ b/packages/s3/src/s3/fs.ts @@ -11,13 +11,15 @@ import type { import { chunkSize, concurrency } from '@hesprs/sync-engine-sdk'; import { concatBinary, textToUint8Array } from '@repo/shared/binary'; import { getStatus } from '@repo/shared/error'; +import normalizeEtag from '@repo/shared/normalize-etag'; import parseXML from '@repo/shared/parse-xml'; import { dirname, encodeUrl, isFolder } from '@repo/shared/path'; import createRangeReadStream from '@repo/shared/read-stream'; import type { UrlStyle } from './sigv4'; import { PART_SIZE, multipartUpload } from './multipart'; import { md5Base64 } from './sigv4'; -import { buildUrl, buildUrlWithQuery, formatS3Error, getHeader, parseS3Error } from './url'; +import { buildUrl, buildUrlWithQuery, getHeader } from './url'; +import { formatS3Error, getFileUid, parseS3Error } from './utils'; export type S3FsOptions = { accessKeyId: string; @@ -170,10 +172,7 @@ export default class S3Fs implements RootFs { method: 'PUT', }); const etag = getHeader(response.headers, 'etag'); - if (etag) return etag; - const stat = await this.stat(key); - if (!stat.isDir) return stat.uid; - throw new Error(`S3 write returned a folder stat for ${key}.`); + return etag ? normalizeEtag(etag) : getFileUid(await this.stat(key), key); } async writeStream(key: string, value: ReadableStream, stat: FileStat): Promise { @@ -268,7 +267,13 @@ export default class S3Fs implements RootFs { if (!contentLength) throw sizeMissing; const mtime = new Date(lastModified).valueOf(); const size = Number.parseInt(contentLength); - return { isDir: false, key, mtime, size, uid: etag ?? `${mtime}~${size}` }; + return { + isDir: false, + key, + mtime, + size, + uid: etag ? normalizeEtag(etag) : `${mtime}~${size}`, + }; } async exists(key: string): Promise { @@ -321,7 +326,7 @@ export default class S3Fs implements RootFs { key: Key, mtime, size, - uid: ETag ?? `${mtime}~${size}`, + uid: ETag ? normalizeEtag(ETag) : `${mtime}~${size}`, }); } }), diff --git a/packages/s3/src/s3/multipart.ts b/packages/s3/src/s3/multipart.ts index 7d04e3df..d4b66fa9 100644 --- a/packages/s3/src/s3/multipart.ts +++ b/packages/s3/src/s3/multipart.ts @@ -1,24 +1,17 @@ import type { Binary, Request, Stat } from '@hesprs/sync-engine-sdk'; import { textToUint8Array } from '@repo/shared/binary'; import chunkedUpload from '@repo/shared/chunked-upload'; +import normalizeEtag from '@repo/shared/normalize-etag'; import parseXML from '@repo/shared/parse-xml'; import type { UrlStyle } from './sigv4'; import { buildUrlWithQuery, getHeader } from './url'; +import { getFileUid } from './utils'; export const PART_SIZE = 5 * 1024 * 1024; // 5 MiB — S3 minimum part size const MAX_CONCURRENT = 3; -type InitiateMultipartUploadResponse = { - InitiateMultipartUploadResult?: { - UploadId?: string; - }; -}; - -type CompleteMultipartUploadResponse = { - CompleteMultipartUploadResult?: { - ETag?: string; - }; -}; +type InitiateMultipartUploadResponse = { InitiateMultipartUploadResult?: { UploadId?: string } }; +type CompleteMultipartUploadResponse = { CompleteMultipartUploadResult?: { ETag?: string } }; export type MultipartUploadOptions = { endpoint: string; @@ -49,15 +42,7 @@ async function uploadPart( partNumber: number, chunk: Binary, ): Promise<{ partNumber: number; etag: string }> { - const url = buildUrlWithQuery( - { - bucket: options.bucket, - endpoint: options.endpoint, - key: options.key, - urlStyle: options.urlStyle, - }, - { partNumber: String(partNumber), uploadId }, - ); + const url = buildUrlWithQuery(options, { partNumber: String(partNumber), uploadId }); const response = await options.request(url, { body: chunk, headers: { 'Content-Type': 'application/octet-stream' }, @@ -69,15 +54,7 @@ async function uploadPart( } function abortMultipart(options: MultipartUploadOptions, uploadId: string) { - const url = buildUrlWithQuery( - { - bucket: options.bucket, - endpoint: options.endpoint, - key: options.key, - urlStyle: options.urlStyle, - }, - { uploadId }, - ); + const url = buildUrlWithQuery(options, { uploadId }); return options.request(url, { ignoreCancellation: true, method: 'DELETE' }).catch(() => {}); } @@ -85,16 +62,9 @@ export async function multipartUpload( options: MultipartUploadOptions, value: ReadableStream, ): Promise { - const initiateUrl = buildUrlWithQuery( - { - bucket: options.bucket, - endpoint: options.endpoint, - key: options.key, - urlStyle: options.urlStyle, - }, - { uploads: '' }, - ); - const initiateResponse = await options.request(initiateUrl, { + const { key, request, stat } = options; + const initiateUrl = buildUrlWithQuery(options, { uploads: '' }); + const initiateResponse = await request(initiateUrl, { headers: { 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }, method: 'POST', }); @@ -109,16 +79,8 @@ export async function multipartUpload( }); const completeBody = buildCompleteMultipartXml(parts); - const completeUrl = buildUrlWithQuery( - { - bucket: options.bucket, - endpoint: options.endpoint, - key: options.key, - urlStyle: options.urlStyle, - }, - { uploadId }, - ); - const completeResponse = await options.request(completeUrl, { + const completeUrl = buildUrlWithQuery(options, { uploadId }); + const completeResponse = await request(completeUrl, { body: textToUint8Array(completeBody), headers: { 'Content-Type': 'application/xml' }, method: 'POST', @@ -126,11 +88,7 @@ export async function multipartUpload( const etag = parseXML(completeResponse.text()) .CompleteMultipartUploadResult?.ETag; - if (etag) return etag; - const stat = await options.stat(options.key); - if (stat.isDir) - throw new Error(`S3 multipart upload returned a folder stat for ${options.key}.`); - return stat.uid; + return etag ? normalizeEtag(etag) : getFileUid(await stat(key), key); } catch (error) { void abortMultipart(options, uploadId); throw error; diff --git a/packages/s3/src/s3/url.ts b/packages/s3/src/s3/url.ts index 6d102a91..6ee38bd2 100644 --- a/packages/s3/src/s3/url.ts +++ b/packages/s3/src/s3/url.ts @@ -1,4 +1,3 @@ -import parseXML from '@repo/shared/parse-xml'; import { encodeUrl } from '@repo/shared/path'; import type { UrlStyle } from './sigv4'; @@ -9,24 +8,6 @@ export type UrlOptions = { urlStyle: UrlStyle; }; -type S3ErrorResponse = { - Error?: { - Code?: string; - Message?: string; - }; -}; -export function parseS3Error(xml: string): string | undefined { - try { - const error = parseXML(xml).Error; - if (error?.Code) return formatS3Error(error.Code, error.Message); - } catch { - /* Ignore malformed S3 error XML and use the HTTP fallback. */ - } -} -export function formatS3Error(code: string, message?: string): string { - return `S3 ${code}: ${message ?? ''}`; -} - export function buildUrl({ endpoint, bucket, key, urlStyle }: UrlOptions): string { const encodedPath = encodeUrl(key); if (urlStyle === 'virtualHosted') { diff --git a/packages/s3/src/s3/utils.ts b/packages/s3/src/s3/utils.ts new file mode 100644 index 00000000..2987ef25 --- /dev/null +++ b/packages/s3/src/s3/utils.ts @@ -0,0 +1,25 @@ +import type { Stat } from '@hesprs/sync-engine-sdk'; +import parseXML from '@repo/shared/parse-xml'; + +type S3ErrorResponse = { + Error?: { + Code?: string; + Message?: string; + }; +}; +export function parseS3Error(xml: string): string | undefined { + try { + const error = parseXML(xml).Error; + if (error?.Code) return formatS3Error(error.Code, error.Message); + } catch { + /* Ignore malformed S3 error XML and use the HTTP fallback. */ + } +} +export function formatS3Error(code: string, message?: string): string { + return `S3 ${code}: ${message ?? ''}`; +} + +export function getFileUid(stat: Stat, key: string) { + if (stat.isDir) throw new Error(`WebDAV write returned a folder stat for ${key}.`); + return stat.uid; +} diff --git a/packages/s3/test/fs-s3.test.ts b/packages/s3/test/fs-s3.test.ts index 994bb594..b6ef042d 100644 --- a/packages/s3/test/fs-s3.test.ts +++ b/packages/s3/test/fs-s3.test.ts @@ -88,7 +88,7 @@ test('write sends binary PUT and uses ETag or HEAD metadata fallback', async () expect(params.body).toStrictEqual(bytes('hello')); return response({ headers: { ETag: '"write-etag"' } }); }); - expect(await s3.fs.write('Notes/file.md', bytes('hello'))).toBe('"write-etag"'); + expect(await s3.fs.write('Notes/file.md', bytes('hello'))).toBe('write-etag'); const fallback = createS3Fs(); fallback.setRequest((_url, params) => { diff --git a/packages/shared/src/normalize-etag.ts b/packages/shared/src/normalize-etag.ts new file mode 100644 index 00000000..a7cc94f8 --- /dev/null +++ b/packages/shared/src/normalize-etag.ts @@ -0,0 +1,5 @@ +export default function normalizeEtag(raw: string) { + raw = raw.trim(); + if (raw.startsWith('W/')) raw = raw.slice(2); + return raw.replaceAll(/^"|"$/gu, ''); +} diff --git a/packages/shared/src/read-stream.ts b/packages/shared/src/read-stream.ts index 239b340e..5884fc02 100644 --- a/packages/shared/src/read-stream.ts +++ b/packages/shared/src/read-stream.ts @@ -85,6 +85,8 @@ export default function createRangeReadStream({ void requestRange(start, endInclusive) .then((buffer) => { if (closed) return; + if (buffer.byteLength !== endInclusive - start + 1) + throw new Error('Ranged request does not return content with correct length!'); pending.set(currentIndex, buffer); pendingBytes += buffer.byteLength; inFlight--; diff --git a/packages/webdav/src/index.ts b/packages/webdav/src/index.ts index f88326f3..6c6c4d65 100644 --- a/packages/webdav/src/index.ts +++ b/packages/webdav/src/index.ts @@ -9,9 +9,13 @@ import type { TranslationResource, Settings, Context, + RecordStore, + RecordStat, + StoreOperations, } from '@hesprs/sync-engine-sdk'; import type { App } from 'obsidian'; import { digOriginal, prefixWrapper } from '@hesprs/sync-engine-sdk'; +import normalizeEtag from '@repo/shared/normalize-etag'; import type { WebdavTranslations } from './setting'; import { en, zh, zhTW, ru } from './i18n'; import webdavSetting from './setting'; @@ -38,6 +42,7 @@ export default class Webdav { registerRemoteFsWrapper: (entry: FsWrapperEntry) => () => void; registerSetting: (entry: SettingEntry) => () => void; registerI18n: (lang: ObsidianLanguageCode, translations: TranslationResource) => void; + getRecordStore: (namespace?: string) => RecordStore; // TODO: remove after October 13 }>, ) { if (!this.moduleSettings.baseDirectory) @@ -66,6 +71,7 @@ export default class Webdav { app: { secretStorage }, registerRemoteFsWrapper, registerSetting, + getRecordStore, } = this.ctx; const resolveConfig = () => { const { @@ -100,6 +106,8 @@ export default class Webdav { priority: 749, }), ); + + if (this.settings.remoteFs === 'webdav') void migrateEtag(getRecordStore()).catch(() => {}); }; readonly dispose = () => { @@ -107,3 +115,16 @@ export default class Webdav { this.cleanup.length = 0; }; } + +// TODO: remove after October 13 +async function migrateEtag(store: RecordStore) { + const changes: Array> = []; + for (const [key, stat] of await store.entries()) { + if (stat.isDir) return; + const remote = normalizeEtag(stat.remote); + if (remote !== stat.remote) + changes.push({ key, type: 'set', value: Object.assign(stat, { remote }) }); + } + if (!changes.length) return; + await store.batch(changes); +} diff --git a/packages/webdav/src/webdav/fs.ts b/packages/webdav/src/webdav/fs.ts index 561d883f..40933430 100644 --- a/packages/webdav/src/webdav/fs.ts +++ b/packages/webdav/src/webdav/fs.ts @@ -12,6 +12,7 @@ import type { import { chunkSize, concurrency } from '@hesprs/sync-engine-sdk'; import { concatBinary } from '@repo/shared/binary'; import { getStatus } from '@repo/shared/error'; +import normalizeEtag from '@repo/shared/normalize-etag'; import parseXML from '@repo/shared/parse-xml'; import { dirname, @@ -120,10 +121,6 @@ function toKey(href: string, endpoint: string, isDir: boolean) { return normalizeKey(normalizeChar(stripped), isDir); } -function normalizeEtag(raw: string) { - return raw.startsWith('W/') ? raw.slice(2) : raw; -} - function toStat(endpoint: string, { propstat, href }: WebDAVResponseItem): Stat | undefined { const propstats = propstat ? asArray(propstat) : []; const validPropstat = propstats.find(({ status, prop }) => isSuccessStatus(status) && prop); diff --git a/packages/webdav/test/fs-webdav.test.ts b/packages/webdav/test/fs-webdav.test.ts index 9b9222e9..84388777 100644 --- a/packages/webdav/test/fs-webdav.test.ts +++ b/packages/webdav/test/fs-webdav.test.ts @@ -145,7 +145,7 @@ test('stat parses dav fields and prefers etag for uid', async () => { key: 'Notes/file.md', mtime: sharedDate, size: 12, - uid: '"etag-123"', + uid: 'etag-123', }); }); @@ -540,27 +540,33 @@ test('readStream requests SDK chunk size ranges from stat size', async () => { const expectedRanges = Array.from({ length: Math.ceil(size / chunkSize) }, (_, index) => { const start = index * chunkSize; - return `bytes=${start}-${Math.min(start + chunkSize - 1, size - 1)}`; + return { + byte: index + 1, + length: Math.min(chunkSize, size - start), + range: `bytes=${start}-${Math.min(start + chunkSize - 1, size - 1)}`, + }; }); const collected = collectStream( webdav.fs.readStream('Notes/file.bin', file('Notes/file.bin', { size })), ); await flush(); - expect(ranges).toStrictEqual(expectedRanges); + expect(ranges).toStrictEqual(expectedRanges.map(({ range }) => range)); expect(encodings.every((encoding) => encoding === 'identity')).toBe(true); - const makeResponse = (byte: number): Partial => ({ - bytes: () => new Uint8Array([byte]), - status: 206, - }); - - for (let index = expectedRanges.length - 1; index >= 0; index--) - pending.get(expectedRanges[index])?.resolve(makeResponse(index + 1)); + for (const { byte, length, range } of [...expectedRanges].reverse()) + pending.get(range)?.resolve({ + bytes: () => new Uint8Array(length).fill(byte), + status: 206, + }); await flush(); - expect(ranges).toStrictEqual(expectedRanges); - expect(await collected).toStrictEqual( - new Uint8Array(expectedRanges.map((_, index) => index + 1)), - ); + expect(ranges).toStrictEqual(expectedRanges.map(({ range }) => range)); + const expected = new Uint8Array(size); + let offset = 0; + for (const { byte, length } of expectedRanges) { + expected.fill(byte, offset, offset + length); + offset += length; + } + expect(await collected).toStrictEqual(expected); });