fix(etag): normalize etag for webdav and s3

modules.json

@@ -2,7 +2,7 @@
 	{
 		"id": "webdav",
 		"name": "WebDAV",
-		"version": "0.1.18",
+		"version": "0.1.19",
 		"description": "WebDAV backend support.",
 		"icon": "server",
 		"main": "https://sync.consensia.cc/modules/webdav.js",
@@ -12,7 +12,7 @@
 	{
 		"id": "s3",
 		"name": "S3",
-		"version": "0.1.6",
+		"version": "0.1.7",
 		"description": "S3 and S3-compatible backend support.",
 		"icon": "server",
 		"main": "https://sync.consensia.cc/modules/s3.js",

packages/plugin/dist/dev.spec.d.ts

@@ -1,4 +1,4 @@
-import { At as FileStat, Dt as RootFs, Ft as RecordStatsMap, It as Stat, Lt as StatsMap, Mt as MaybePromise, Ot as WrappedFs, Pt as RecordStat, Rt as Binary, d as RequestParam, f as RequestResponse, jt as FolderStat, lt as TaskNames, q as Decider, u as Request, yt as Fs } from "./index-DbHTJm2U.spec.js";
+import { At as FileStat, Dt as RootFs, Ft as RecordStatsMap, It as Stat, Lt as StatsMap, Mt as MaybePromise, Ot as WrappedFs, Pt as RecordStat, Rt as Binary, d as RequestParam, f as RequestResponse, jt as FolderStat, lt as TaskNames, q as Decider, u as Request, yt as Fs } from "./index-JdV6c5O1.spec.js";
 //#region src/sdk/debug-wrapper.d.ts
 declare function debugWrapper(original: Fs, log: (content: string) => void): WrappedFs;
 //#endregion

packages/plugin/dist/index-DbHTJm2U.spec.d.ts → packages/plugin/dist/index-JdV6c5O1.spec.d.ts

@@ -212,16 +212,7 @@ declare class Storage {
   readonly root: {
     clearRecordStores: () => Promise<void>;
     deleteRecordStore: (namespace?: string) => MaybePromise<void>;
-    getRecordStore: (namespace?: string) => {
-      get(key: string): Promise<RecordStat | undefined>;
-      set(key: string, value: RecordStat): Promise<void>;
-      delete(key: string): Promise<void>;
-      clear(): Promise<void>;
-      keys(): Promise<string[]>;
-      values(): Promise<RecordStat[]>;
-      entries(): Promise<[string, RecordStat][]>;
-      batch(operations: StoreOperations<RecordStat>[]): Promise<GetResult<RecordStat>[]>;
-    };
+    getRecordStore: (namespace?: string) => RecordStore;
     indexedDB: DatabaseAsync<General$1, General$1>;
     memoryDB: {
       getStore<K extends string | number | symbol>(name: K): {

packages/plugin/dist/index.spec.d.ts

@@ -1,2 +1,2 @@
-import { $ as RemoveRecord, A as setNeedMigration, At as FileStat, B as ObsidianLanguageCode, C as digOriginal, Ct as MoveAtom, D as readWithSize, Dt as RootFs, E as pipe, Et as OutputAtom, F as CallableOrObjectTree, Ft as RecordStatsMap, G as On, H as Translate, I as SettingEntry, It as Stat, J as DeciderInput, K as CreateLocalDir, L as AugmentedModuleMeta, Lt as StatsMap, M as generateEditableList, Mt as MaybePromise, N as reactivelyValidate, Nt as Progress, O as writeWithValue, Ot as WrappedFs, P as s, Pt as RecordStat, Q as RemoveRemote, R as ModuleMeta, Rt as Binary, S as SelectFromContext, St as MkdirAtom, T as concurrency, Tt as OptimizerOutput, U as TranslationResource, V as Snippet, W as Dispatch, X as Upload, Y as TaskFactory, Z as ResolveConflict, _ as Context, _t as CustomAtom, a as FsWrapperEntry, at as AddRecord, b as Translations, bt as InputAtom, c as RemoteFsEntry, ct as ConflictResolverPayload, d as RequestParam, dt as DatabaseAsync, et as RemoveLocal, f as RequestResponse, ft as DatabaseSync, g as SyncTerminateReason, gt as BatchOptimizer, h as SyncOptions, ht as StoreSync, i as DeciderEntry, it as CreateRemoteDir, j as LabelDefinition, jt as FolderStat, k as prefixWrapper, kt as WriteAtom, l as RemoteRequestMiddlewareEntry, lt as TaskNames, m as RemoteLister, mt as StoreOperations, n as CheckConnectionResult, nt as MoveLocal, o as LocalRequestMiddlewareEntry, ot as BaseTask, p as TriggerEntry, pt as StoreAsync, q as Decider, r as ConflictResolverEntry, rt as Download, s as OptimizerEntry, st as ConflictResolver, t as VaultRequest, tt as MoveRemote, u as Request, ut as RecordStore, v as Events, vt as DeleteAtom, w as chunkSize, wt as OptimizerInput, x as ExistingMemoryDB, xt as ListReporter, y as Settings, yt as Fs, z as Fragment } from "./index-DbHTJm2U.spec.js";
+import { $ as RemoveRecord, A as setNeedMigration, At as FileStat, B as ObsidianLanguageCode, C as digOriginal, Ct as MoveAtom, D as readWithSize, Dt as RootFs, E as pipe, Et as OutputAtom, F as CallableOrObjectTree, Ft as RecordStatsMap, G as On, H as Translate, I as SettingEntry, It as Stat, J as DeciderInput, K as CreateLocalDir, L as AugmentedModuleMeta, Lt as StatsMap, M as generateEditableList, Mt as MaybePromise, N as reactivelyValidate, Nt as Progress, O as writeWithValue, Ot as WrappedFs, P as s, Pt as RecordStat, Q as RemoveRemote, R as ModuleMeta, Rt as Binary, S as SelectFromContext, St as MkdirAtom, T as concurrency, Tt as OptimizerOutput, U as TranslationResource, V as Snippet, W as Dispatch, X as Upload, Y as TaskFactory, Z as ResolveConflict, _ as Context, _t as CustomAtom, a as FsWrapperEntry, at as AddRecord, b as Translations, bt as InputAtom, c as RemoteFsEntry, ct as ConflictResolverPayload, d as RequestParam, dt as DatabaseAsync, et as RemoveLocal, f as RequestResponse, ft as DatabaseSync, g as SyncTerminateReason, gt as BatchOptimizer, h as SyncOptions, ht as StoreSync, i as DeciderEntry, it as CreateRemoteDir, j as LabelDefinition, jt as FolderStat, k as prefixWrapper, kt as WriteAtom, l as RemoteRequestMiddlewareEntry, lt as TaskNames, m as RemoteLister, mt as StoreOperations, n as CheckConnectionResult, nt as MoveLocal, o as LocalRequestMiddlewareEntry, ot as BaseTask, p as TriggerEntry, pt as StoreAsync, q as Decider, r as ConflictResolverEntry, rt as Download, s as OptimizerEntry, st as ConflictResolver, t as VaultRequest, tt as MoveRemote, u as Request, ut as RecordStore, v as Events, vt as DeleteAtom, w as chunkSize, wt as OptimizerInput, x as ExistingMemoryDB, xt as ListReporter, y as Settings, yt as Fs, z as Fragment } from "./index-JdV6c5O1.spec.js";
 export { type AddRecord, type AugmentedModuleMeta, type BaseTask, type BatchOptimizer, type Binary, type CallableOrObjectTree, type CheckConnectionResult, type ConflictResolver, type ConflictResolverEntry, type ConflictResolverPayload, type Context, type CreateLocalDir, type CreateRemoteDir, type CustomAtom, type DatabaseAsync, type DatabaseSync, type Decider, type DeciderEntry, type DeciderInput, type DeleteAtom, type Dispatch, type Download, type Events, type ExistingMemoryDB, type FileStat, type FolderStat, type Fragment, type Fs, type FsWrapperEntry, type InputAtom, type LabelDefinition, type ListReporter, type LocalRequestMiddlewareEntry, type MaybePromise, type MkdirAtom, type ModuleMeta, type MoveAtom, type MoveLocal, type MoveRemote, type ObsidianLanguageCode, type On, type OptimizerEntry, type OptimizerInput, type OptimizerOutput, type OutputAtom, type Progress, type RecordStat, type RecordStatsMap, type RecordStore, type RemoteFsEntry, type RemoteLister, type RemoteRequestMiddlewareEntry, type RemoveLocal, type RemoveRecord, type RemoveRemote, type Request, type RequestParam, type RequestResponse, type ResolveConflict, type RootFs, SelectFromContext, type SettingEntry, type Settings, type Snippet, type Stat, type StatsMap, type StoreAsync, type StoreOperations, type StoreSync, type SyncOptions, type SyncTerminateReason, type TaskFactory, type TaskNames, type Translate, type TranslationResource, type Translations, type TriggerEntry, type Upload, type VaultRequest, type WrappedFs, type WriteAtom, chunkSize, concurrency, digOriginal, generateEditableList, pipe, prefixWrapper, reactivelyValidate, readWithSize, s, setNeedMigration, writeWithValue };

packages/plugin/src/modules/Storage.ts

@@ -14,7 +14,7 @@ export default class Storage {
 
 	constructor(private readonly ctx: { getNamespace: () => string }) {}
 
-	private readonly getRecordStore = (namespace?: string) =>
+	private readonly getRecordStore = (namespace?: string): RecordStore =>
 		this.indexedDB.getStore(namespace || this.ctx.getNamespace());
 
 	private readonly deleteRecordStore = (namespace?: string): MaybePromise<void> => {

packages/plugin/src/settings/head.ts

@@ -213,6 +213,7 @@ function setupCheckConnection({
 	};
 	const scheduleCheckConnection = () =>
 		(timeout = window.setTimeout(() => void check(), CHECK_CONNECTION_INTERVAL));
+	const cleanup = () => window.clearTimeout(timeout);
 
 	const check = async (force = false) => {
 		if (memoryDB.getMeta('lastCheckedFs') === settings.remoteFs && !force) {
@@ -227,7 +228,8 @@ function setupCheckConnection({
 			setError();
 			log(`Check connection to \`${settings.remoteFs}\` failed: \`${message}\`.`);
 			if (force) new Notice(`${translate('checkConnectionFailed')}: ${message}`, 5000);
-			else scheduleCheckConnection();
+			cleanup();
+			scheduleCheckConnection();
 		};
 
 		try {
@@ -243,7 +245,7 @@ function setupCheckConnection({
 		}
 	};
 
-	return { check, cleanup: () => window.clearTimeout(timeout) };
+	return { check, cleanup };
 }
 
 function addLabel(

packages/s3/src/index.ts

@@ -13,9 +13,14 @@ import type {
 	Context,
 	DatabaseSync,
 	Binary,
+	RecordStore,
+	StoreOperations,
+	Stat,
+	RecordStat,
 } from '@hesprs/sync-engine-sdk';
 import type { App } from 'obsidian';
 import { digOriginal, prefixWrapper } from '@hesprs/sync-engine-sdk';
+import normalizeEtag from '@repo/shared/normalize-etag';
 import type { UrlStyle } from '@/s3/sigv4';
 import type { S3Translations } from '@/setting';
 import { sigv4Middleware } from '@/s3/sigv4';
@@ -59,6 +64,7 @@ export default class S3 {
 			registerRemoteOptimizer: (entry: OptimizerEntry) => () => void;
 			registerRemoteRequestMiddleware: (entry: RemoteRequestMiddlewareEntry) => () => void;
 			memoryDB: S3DB;
+			getRecordStore: (namespace?: string) => RecordStore; // TODO: remove after October 13
 		}>,
 	) {
 		ctx.registerI18n('en', en);
@@ -96,6 +102,7 @@ export default class S3 {
 			registerRemoteOptimizer,
 			registerRemoteRequestMiddleware,
 			memoryDB,
+			getRecordStore,
 		} = this.ctx;
 		this.cleanup.push(
 			registerRemoteFs('s3', {
@@ -159,6 +166,8 @@ export default class S3 {
 				priority: 604,
 			}),
 		);
+
+		if (this.settings.remoteFs === 's3') void migrateEtag(getRecordStore()).catch(() => {});
 	};
 
 	private readonly resolveConfig = () => {
@@ -196,3 +205,16 @@ export default class S3 {
 		this.cleanup.length = 0;
 	};
 }
+
+// TODO: remove after October 13
+async function migrateEtag(store: RecordStore) {
+	const changes: Array<StoreOperations<RecordStat>> = [];
+	for (const [key, stat] of await store.entries()) {
+		if (stat.isDir) return;
+		const remote = normalizeEtag(stat.remote);
+		if (remote !== stat.remote)
+			changes.push({ key, type: 'set', value: Object.assign(stat, { remote }) });
+	}
+	if (!changes.length) return;
+	await store.batch(changes);
+}

packages/s3/src/s3/check-connection.ts

@@ -1,7 +1,8 @@
 import type { CheckConnectionResult, Request } from '@hesprs/sync-engine-sdk';
 import { getMessage } from '@repo/shared/error';
 import type { UrlStyle } from './sigv4';
-import { buildUrlWithQuery, parseS3Error } from './url';
+import { buildUrlWithQuery } from './url';
+import { parseS3Error } from './utils';
 
 export type S3ConnectionOptions = {
 	endpoint: string;

packages/s3/src/s3/fs.ts

@@ -11,13 +11,15 @@ import type {
 import { chunkSize, concurrency } from '@hesprs/sync-engine-sdk';
 import { concatBinary, textToUint8Array } from '@repo/shared/binary';
 import { getStatus } from '@repo/shared/error';
+import normalizeEtag from '@repo/shared/normalize-etag';
 import parseXML from '@repo/shared/parse-xml';
 import { dirname, encodeUrl, isFolder } from '@repo/shared/path';
 import createRangeReadStream from '@repo/shared/read-stream';
 import type { UrlStyle } from './sigv4';
 import { PART_SIZE, multipartUpload } from './multipart';
 import { md5Base64 } from './sigv4';
-import { buildUrl, buildUrlWithQuery, formatS3Error, getHeader, parseS3Error } from './url';
+import { buildUrl, buildUrlWithQuery, getHeader } from './url';
+import { formatS3Error, getFileUid, parseS3Error } from './utils';
 
 export type S3FsOptions = {
 	accessKeyId: string;
@@ -170,10 +172,7 @@ export default class S3Fs implements RootFs {
 			method: 'PUT',
 		});
 		const etag = getHeader(response.headers, 'etag');
-		if (etag) return etag;
-		const stat = await this.stat(key);
-		if (!stat.isDir) return stat.uid;
-		throw new Error(`S3 write returned a folder stat for ${key}.`);
+		return etag ? normalizeEtag(etag) : getFileUid(await this.stat(key), key);
 	}
 
 	async writeStream(key: string, value: ReadableStream<Binary>, stat: FileStat): Promise<string> {
@@ -268,7 +267,13 @@ export default class S3Fs implements RootFs {
 		if (!contentLength) throw sizeMissing;
 		const mtime = new Date(lastModified).valueOf();
 		const size = Number.parseInt(contentLength);
-		return { isDir: false, key, mtime, size, uid: etag ?? `${mtime}~${size}` };
+		return {
+			isDir: false,
+			key,
+			mtime,
+			size,
+			uid: etag ? normalizeEtag(etag) : `${mtime}~${size}`,
+		};
 	}
 
 	async exists(key: string): Promise<boolean> {
@@ -321,7 +326,7 @@ export default class S3Fs implements RootFs {
 							key: Key,
 							mtime,
 							size,
-							uid: ETag ?? `${mtime}~${size}`,
+							uid: ETag ? normalizeEtag(ETag) : `${mtime}~${size}`,
 						});
 					}
 				}),

packages/s3/src/s3/multipart.ts

@@ -1,24 +1,17 @@
 import type { Binary, Request, Stat } from '@hesprs/sync-engine-sdk';
 import { textToUint8Array } from '@repo/shared/binary';
 import chunkedUpload from '@repo/shared/chunked-upload';
+import normalizeEtag from '@repo/shared/normalize-etag';
 import parseXML from '@repo/shared/parse-xml';
 import type { UrlStyle } from './sigv4';
 import { buildUrlWithQuery, getHeader } from './url';
+import { getFileUid } from './utils';
 
 export const PART_SIZE = 5 * 1024 * 1024; // 5 MiB — S3 minimum part size
 const MAX_CONCURRENT = 3;
 
-type InitiateMultipartUploadResponse = {
-	InitiateMultipartUploadResult?: {
-		UploadId?: string;
-	};
-};
-
-type CompleteMultipartUploadResponse = {
-	CompleteMultipartUploadResult?: {
-		ETag?: string;
-	};
-};
+type InitiateMultipartUploadResponse = { InitiateMultipartUploadResult?: { UploadId?: string } };
+type CompleteMultipartUploadResponse = { CompleteMultipartUploadResult?: { ETag?: string } };
 
 export type MultipartUploadOptions = {
 	endpoint: string;
@@ -49,15 +42,7 @@ async function uploadPart(
 	partNumber: number,
 	chunk: Binary,
 ): Promise<{ partNumber: number; etag: string }> {
-	const url = buildUrlWithQuery(
-		{
-			bucket: options.bucket,
-			endpoint: options.endpoint,
-			key: options.key,
-			urlStyle: options.urlStyle,
-		},
-		{ partNumber: String(partNumber), uploadId },
-	);
+	const url = buildUrlWithQuery(options, { partNumber: String(partNumber), uploadId });
 	const response = await options.request(url, {
 		body: chunk,
 		headers: { 'Content-Type': 'application/octet-stream' },
@@ -69,32 +54,17 @@ async function uploadPart(
 }
 
 function abortMultipart(options: MultipartUploadOptions, uploadId: string) {
-	const url = buildUrlWithQuery(
-		{
-			bucket: options.bucket,
-			endpoint: options.endpoint,
-			key: options.key,
-			urlStyle: options.urlStyle,
-		},
-		{ uploadId },
-	);
+	const url = buildUrlWithQuery(options, { uploadId });
 	return options.request(url, { ignoreCancellation: true, method: 'DELETE' }).catch(() => {});
 }
 
 export async function multipartUpload(
 	options: MultipartUploadOptions,
 	value: ReadableStream<Binary>,
 ): Promise<string> {
-	const initiateUrl = buildUrlWithQuery(
-		{
-			bucket: options.bucket,
-			endpoint: options.endpoint,
-			key: options.key,
-			urlStyle: options.urlStyle,
-		},
-		{ uploads: '' },
-	);
-	const initiateResponse = await options.request(initiateUrl, {
+	const { key, request, stat } = options;
+	const initiateUrl = buildUrlWithQuery(options, { uploads: '' });
+	const initiateResponse = await request(initiateUrl, {
 		headers: { 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' },
 		method: 'POST',
 	});
@@ -109,28 +79,16 @@ export async function multipartUpload(
 		});
 
 		const completeBody = buildCompleteMultipartXml(parts);
-		const completeUrl = buildUrlWithQuery(
-			{
-				bucket: options.bucket,
-				endpoint: options.endpoint,
-				key: options.key,
-				urlStyle: options.urlStyle,
-			},
-			{ uploadId },
-		);
-		const completeResponse = await options.request(completeUrl, {
+		const completeUrl = buildUrlWithQuery(options, { uploadId });
+		const completeResponse = await request(completeUrl, {
 			body: textToUint8Array(completeBody),
 			headers: { 'Content-Type': 'application/xml' },
 			method: 'POST',
 		});
 
 		const etag = parseXML<CompleteMultipartUploadResponse>(completeResponse.text())
 			.CompleteMultipartUploadResult?.ETag;
-		if (etag) return etag;
-		const stat = await options.stat(options.key);
-		if (stat.isDir)
-			throw new Error(`S3 multipart upload returned a folder stat for ${options.key}.`);
-		return stat.uid;
+		return etag ? normalizeEtag(etag) : getFileUid(await stat(key), key);
 	} catch (error) {
 		void abortMultipart(options, uploadId);
 		throw error;

packages/s3/src/s3/url.ts

@@ -1,4 +1,3 @@
-import parseXML from '@repo/shared/parse-xml';
 import { encodeUrl } from '@repo/shared/path';
 import type { UrlStyle } from './sigv4';
 
@@ -9,24 +8,6 @@ export type UrlOptions = {
 	urlStyle: UrlStyle;
 };
 
-type S3ErrorResponse = {
-	Error?: {
-		Code?: string;
-		Message?: string;
-	};
-};
-export function parseS3Error(xml: string): string | undefined {
-	try {
-		const error = parseXML<S3ErrorResponse>(xml).Error;
-		if (error?.Code) return formatS3Error(error.Code, error.Message);
-	} catch {
-		/* Ignore malformed S3 error XML and use the HTTP fallback. */
-	}
-}
-export function formatS3Error(code: string, message?: string): string {
-	return `S3 ${code}: ${message ?? ''}`;
-}
-
 export function buildUrl({ endpoint, bucket, key, urlStyle }: UrlOptions): string {
 	const encodedPath = encodeUrl(key);
 	if (urlStyle === 'virtualHosted') {

packages/s3/src/s3/utils.ts

@@ -0,0 +1,25 @@
+import type { Stat } from '@hesprs/sync-engine-sdk';
+import parseXML from '@repo/shared/parse-xml';
+
+type S3ErrorResponse = {
+	Error?: {
+		Code?: string;
+		Message?: string;
+	};
+};
+export function parseS3Error(xml: string): string | undefined {
+	try {
+		const error = parseXML<S3ErrorResponse>(xml).Error;
+		if (error?.Code) return formatS3Error(error.Code, error.Message);
+	} catch {
+		/* Ignore malformed S3 error XML and use the HTTP fallback. */
+	}
+}
+export function formatS3Error(code: string, message?: string): string {
+	return `S3 ${code}: ${message ?? ''}`;
+}
+
+export function getFileUid(stat: Stat, key: string) {
+	if (stat.isDir) throw new Error(`WebDAV write returned a folder stat for ${key}.`);
+	return stat.uid;
+}

packages/s3/test/fs-s3.test.ts

@@ -88,7 +88,7 @@ test('write sends binary PUT and uses ETag or HEAD metadata fallback', async ()
 		expect(params.body).toStrictEqual(bytes('hello'));
 		return response({ headers: { ETag: '"write-etag"' } });
 	});
-	expect(await s3.fs.write('Notes/file.md', bytes('hello'))).toBe('"write-etag"');
+	expect(await s3.fs.write('Notes/file.md', bytes('hello'))).toBe('write-etag');
 
 	const fallback = createS3Fs();
 	fallback.setRequest((_url, params) => {

packages/shared/src/normalize-etag.ts

@@ -0,0 +1,5 @@
+export default function normalizeEtag(raw: string) {
+	raw = raw.trim();
+	if (raw.startsWith('W/')) raw = raw.slice(2);
+	return raw.replaceAll(/^"|"$/gu, '');
+}

packages/shared/src/read-stream.ts

@@ -85,6 +85,8 @@ export default function createRangeReadStream({
 		void requestRange(start, endInclusive)
 			.then((buffer) => {
 				if (closed) return;
+				if (buffer.byteLength !== endInclusive - start + 1)
+					throw new Error('Ranged request does not return content with correct length!');
 				pending.set(currentIndex, buffer);
 				pendingBytes += buffer.byteLength;
 				inFlight--;