diff --git a/apps/buddy/electron/main/__tests__/mediaProtocol.spec.ts b/apps/buddy/electron/main/__tests__/mediaProtocol.spec.ts new file mode 100644 index 000000000..46dcf0eae --- /dev/null +++ b/apps/buddy/electron/main/__tests__/mediaProtocol.spec.ts @@ -0,0 +1,65 @@ +import { Buffer } from 'node:buffer' +import { mkdtemp, rm, symlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { serveMediaFile } from '../serveMediaFile' + +const cleanup: Array<() => Promise> = [] +afterEach(async () => { + for (const dispose of cleanup.splice(0)) await dispose() +}) +async function directory() { + const root = await mkdtemp(join(tmpdir(), 'buddy-media-')) + cleanup.push(() => rm(root, { recursive: true, force: true })) + return root +} + +describe('media protocol', () => { + it('streams the whole file with range support advertised', async () => { + const path = join(await directory(), 'clip.mp4') + await writeFile(path, Buffer.from('0123456789')) + const response = await serveMediaFile(path, null) + expect(response.status).toBe(200) + expect(response.headers.get('content-type')).toBe('video/mp4') + expect(response.headers.get('accept-ranges')).toBe('bytes') + expect(response.headers.get('content-length')).toBe('10') + expect(await response.text()).toBe('0123456789') + }) + + it('answers byte ranges with 206 and only the requested bytes', async () => { + const path = join(await directory(), 'song.mp3') + await writeFile(path, Buffer.from('0123456789')) + const middle = await serveMediaFile(path, 'bytes=2-5') + expect(middle.status).toBe(206) + expect(middle.headers.get('content-range')).toBe('bytes 2-5/10') + expect(await middle.text()).toBe('2345') + const tail = await serveMediaFile(path, 'bytes=-3') + expect(tail.headers.get('content-range')).toBe('bytes 7-9/10') + expect(await tail.text()).toBe('789') + }) + + it('rejects unsatisfiable ranges, unknown types and empty files cleanly', async () => { + const root = await directory() + await writeFile(join(root, 'clip.webm'), Buffer.from('abc')) + await writeFile(join(root, 'empty.ogg'), '') + await writeFile(join(root, 'notes.txt'), 'abc') + const unsatisfiable = await serveMediaFile(join(root, 'clip.webm'), 'bytes=10-') + expect(unsatisfiable.status).toBe(416) + expect(unsatisfiable.headers.get('content-range')).toBe('bytes */3') + expect((await serveMediaFile(join(root, 'notes.txt'), null)).status).toBe(415) + const empty = await serveMediaFile(join(root, 'empty.ogg'), null) + expect(empty.status).toBe(200) + expect(empty.headers.get('content-length')).toBe('0') + }) + + it('does not follow a link at the leaf', async () => { + const root = await directory() + await writeFile(join(root, 'secret.mp4'), 'secret') + try { + await symlink(join(root, 'secret.mp4'), join(root, 'link.mp4')) + } + catch { return } + expect((await serveMediaFile(join(root, 'link.mp4'), null)).status).toBe(404) + }) +}) diff --git a/apps/buddy/electron/main/app/DesktopRuntimeHost.ts b/apps/buddy/electron/main/app/DesktopRuntimeHost.ts index 412e68cae..3515fcb40 100644 --- a/apps/buddy/electron/main/app/DesktopRuntimeHost.ts +++ b/apps/buddy/electron/main/app/DesktopRuntimeHost.ts @@ -28,6 +28,7 @@ import { LexoraConfigStore } from '../config/LexoraConfigStore' import { ContextPanelHost } from '../context-panel/ContextPanelHost' import { DesktopPerformanceMonitor } from '../diagnostics/DesktopPerformanceMonitor' import { registerExtensionAuthoringRpc } from '../extensions/registerExtensionAuthoringRpc' +import { installMediaProtocol } from '../mediaProtocol' import { DesktopNetwork } from '../network/DesktopNetwork' import { registerWebHostRpc } from '../network/registerWebHostRpc' import { createDesktopFeatures } from '../platform/desktopFeatures' @@ -279,6 +280,7 @@ export class DesktopRuntimeHost { service.start() this.performance.start() this.#subscriptions.push(installAttachmentProtocol(service)) + this.#subscriptions.push(installMediaProtocol(service)) this.#subscriptions.push(installRendererProtocol()) } diff --git a/apps/buddy/electron/main/app/environment.ts b/apps/buddy/electron/main/app/environment.ts index b07f44a12..a8c280358 100644 --- a/apps/buddy/electron/main/app/environment.ts +++ b/apps/buddy/electron/main/app/environment.ts @@ -14,6 +14,7 @@ import { OPERATING_SYSTEM } from '../../../shared/platform/identifiers' import { attachmentSchemePrivileges } from '../attachmentProtocol' import { DesktopDiagnosticLogger } from '../desktopDiagnostics' import { extensionSchemePrivileges } from '../extensions/ExtensionProtocol' +import { mediaSchemePrivileges } from '../mediaProtocol' import { resolveBuddyRuntimePaths } from '../paths' import { desktopHosts } from '../platform/desktopHost' import { rendererSchemePrivileges } from '../rendererProtocol' @@ -130,7 +131,7 @@ export function initializeDesktopEnvironment(environment: DesktopEnvironment): v } } bootstrapStep('register_protocols', () => { - protocol.registerSchemesAsPrivileged([...attachmentSchemePrivileges, rendererSchemePrivileges, extensionSchemePrivileges, fileIconSchemePrivileges, themeSchemePrivileges]) + protocol.registerSchemesAsPrivileged([...attachmentSchemePrivileges, rendererSchemePrivileges, mediaSchemePrivileges, extensionSchemePrivileges, fileIconSchemePrivileges, themeSchemePrivileges]) }) bootstrapStep('desktop_identity', () => desktopHosts[currentPlatform.id].setIdentity?.(paths.desktopName)) } diff --git a/apps/buddy/electron/main/attachmentProtocol.ts b/apps/buddy/electron/main/attachmentProtocol.ts index 634b04235..86a3f13d1 100644 --- a/apps/buddy/electron/main/attachmentProtocol.ts +++ b/apps/buddy/electron/main/attachmentProtocol.ts @@ -8,6 +8,8 @@ import { resolveBuddyFileReader } from '../../platform/native/nativeHost' import { artifactsRequestSchemas, artifactsResponseSchemas } from '../../shared/artifacts/artifactApi' import { attachmentsRequestSchemas } from '../../shared/conversation/attachmentApi' import { composerResourcesRpc } from '../../shared/conversation/composerApi' +import { mediaKindOf } from '../../shared/files/mediaFiles' +import { serveMediaFile } from './serveMediaFile' const ATTACHMENT_PROTOCOL = 'lexora-attachment' const ARTIFACT_PROTOCOL = 'lexora-artifact' @@ -54,6 +56,9 @@ export function installAttachmentProtocol(runtime: BuddyServiceSupervisor): () = const preview = artifactsResponseSchemas.artifactPreview.parse( await runtime.request(method, input), ) + // Audio and video are streamed in ranges instead of being read whole, so a long recording can be seeked. + if (scheme === ARTIFACT_PROTOCOL && mediaKindOf(preview.path)) + return await serveMediaFile(preview.path, request.headers.get('range'), request.signal) const body = await readFile(preview.path) return new Response(body, { headers: { diff --git a/apps/buddy/electron/main/mediaProtocol.ts b/apps/buddy/electron/main/mediaProtocol.ts new file mode 100644 index 000000000..816726230 --- /dev/null +++ b/apps/buddy/electron/main/mediaProtocol.ts @@ -0,0 +1,31 @@ +import type { BuddyServiceSupervisor } from './runtime/BuddyServiceSupervisor' +import { protocol } from 'electron' +import { MEDIA_PROTOCOL, parseMediaUrl } from '../../shared/files/mediaFiles' +import { spaceFilesRpc } from '../../shared/spaces/spaceFileApi' +import { serveMediaFile } from './serveMediaFile' + +export const mediaSchemePrivileges: Electron.CustomScheme = { + scheme: MEDIA_PROTOCOL, + privileges: { secure: true, standard: true, stream: true, supportFetchAPI: true }, +} + +export function installMediaProtocol(runtime: Pick): () => void { + protocol.handle(MEDIA_PROTOCOL, async (request) => { + if (request.method !== 'GET') + return new Response(null, { status: 405 }) + const target = parseMediaUrl(request.url) + if (!target) + return new Response(null, { status: 400 }) + try { + // The service re-checks the space, directory grant and revision on every request, as it does for any file read. + const located = spaceFilesRpc.locate.response.parse(await runtime.request(spaceFilesRpc.locate.method, spaceFilesRpc.locate.input.parse(target))) + if (located.kind !== 'file') + return new Response(null, { status: 404 }) + return await serveMediaFile(located.path, request.headers.get('range'), request.signal) + } + catch { + return new Response(null, { status: 404 }) + } + }) + return () => protocol.unhandle(MEDIA_PROTOCOL) +} diff --git a/apps/buddy/electron/main/serveMediaFile.ts b/apps/buddy/electron/main/serveMediaFile.ts new file mode 100644 index 000000000..7c376c8f0 --- /dev/null +++ b/apps/buddy/electron/main/serveMediaFile.ts @@ -0,0 +1,53 @@ +import { lstat, open } from 'node:fs/promises' +import { Readable } from 'node:stream' +import { mediaMimeType, parseMediaRange } from '../../shared/files/mediaFiles' + +// Streams a local audio or video file with HTTP Range support, so media elements can seek without loading it whole. +export async function serveMediaFile(path: string, rangeHeader: string | null, signal?: AbortSignal): Promise { + const mimeType = mediaMimeType(path) + if (!mimeType) + return new Response(null, { status: 415 }) + // The path is canonical when located; refusing a leaf that has since become a link narrows the swap window. + const linked = await lstat(path) + if (!linked.isFile()) + return new Response(null, { status: 404 }) + const handle = await open(path, 'r') + try { + const metadata = await handle.stat() + if (!metadata.isFile() || metadata.size !== linked.size) { + await handle.close() + return new Response(null, { status: 404 }) + } + const headers: Record = { + 'accept-ranges': 'bytes', + 'cache-control': 'private, no-store', + 'content-type': mimeType, + 'x-content-type-options': 'nosniff', + } + const range = rangeHeader ? parseMediaRange(rangeHeader, metadata.size) : null + if (range === 'unsatisfiable') { + await handle.close() + return new Response(null, { status: 416, headers: { ...headers, 'content-range': `bytes */${metadata.size}` } }) + } + if (metadata.size === 0) { + await handle.close() + return new Response(null, { status: 200, headers: { ...headers, 'content-length': '0' } }) + } + const start = range?.start ?? 0 + const end = range?.end ?? metadata.size - 1 + const stream = handle.createReadStream({ start, end, autoClose: true }) + signal?.addEventListener('abort', () => stream.destroy(), { once: true }) + return new Response(Readable.toWeb(stream) as ReadableStream, { + status: range ? 206 : 200, + headers: { + ...headers, + 'content-length': String(end - start + 1), + ...(range ? { 'content-range': `bytes ${start}-${end}/${metadata.size}` } : {}), + }, + }) + } + catch (error) { + await handle.close().catch(() => {}) + throw error + } +} diff --git a/apps/buddy/index.html b/apps/buddy/index.html index dbf6c18ce..9e528e4f3 100644 --- a/apps/buddy/index.html +++ b/apps/buddy/index.html @@ -3,7 +3,7 @@ - + Lexora diff --git a/apps/buddy/service/src/artifacts/ArtifactService.ts b/apps/buddy/service/src/artifacts/ArtifactService.ts index f6c3a4e12..0f679b8cf 100644 --- a/apps/buddy/service/src/artifacts/ArtifactService.ts +++ b/apps/buddy/service/src/artifacts/ArtifactService.ts @@ -18,6 +18,7 @@ import { import { relativeCanonicalPath } from '../../../platform/filesystem/filePaths' import { Emitter } from '../../../shared/events/Emitter' import { copyEventSnapshot } from '../../../shared/events/eventSnapshot' +import { mediaMimeType } from '../../../shared/files/mediaFiles' import { resolveGrantedPath } from '../directories/resolveGrantedPath' export const BUDDY_ARTIFACT_COUNT_LIMIT = 512 @@ -355,7 +356,13 @@ export class ArtifactService { resolvePreview(id: string): { mimeType: string, path: string } { const artifact = this.#requireVisibleArtifact(id) - if (artifact.kind !== 'file' || !artifact.mimeType.startsWith('image/')) + if (artifact.kind !== 'file') + throw new ArtifactError('VALIDATION_FAILED') + // Audio and video are served in ranges by the protocol; their content type comes from the extension, which the artifact record does not map. + const mimeType = mediaMimeType(artifact.currentPath) + if (mimeType) + return { mimeType, path: artifact.currentPath } + if (!artifact.mimeType.startsWith('image/')) throw new ArtifactError('VALIDATION_FAILED') return { mimeType: artifact.mimeType, path: artifact.currentPath } } @@ -373,7 +380,10 @@ export class ArtifactService { ) { throw new ArtifactError('VALIDATION_FAILED') } - const bytes = await readFile(artifact.currentPath) + // The record keeps the path the file had when it was presented; it may have been moved or deleted since. + const bytes = await readFile(artifact.currentPath).catch((error: NodeJS.ErrnoException) => { + throw error.code === 'ENOENT' ? new ArtifactError('FILE_NOT_FOUND') : error + }) if (bytes.byteLength > BUDDY_ARTIFACT_TEXT_BYTES_LIMIT) throw new ArtifactError('ARTIFACT_SIZE_LIMIT') let text: string diff --git a/apps/buddy/service/src/artifacts/__tests__/ArtifactService.spec.ts b/apps/buddy/service/src/artifacts/__tests__/ArtifactService.spec.ts index 104c9e72e..80e5650e0 100644 --- a/apps/buddy/service/src/artifacts/__tests__/ArtifactService.spec.ts +++ b/apps/buddy/service/src/artifacts/__tests__/ArtifactService.spec.ts @@ -111,6 +111,25 @@ describe('artifactService', () => { expect(fixture.service.listConversationArtifacts('conversation-1')).toHaveLength(2) }) + it('resolves audio and video outputs for streaming but still refuses other binary files', async () => { + const fixture = await createFixture() + const paths = ['voice.MP3', 'clip.mp4', 'archive.zip'].map(name => join(fixture.workspace, name)) + await Promise.all(paths.map(path => writeFile(path, Buffer.from([1, 2, 3])))) + const [voice, clip, archive] = await fixture.service.presentOutputs({ conversationId: 'conversation-1', cwd: fixture.workspace, grants: fixture.grants, paths }) + expect(fixture.service.resolvePreview(voice!.id)).toEqual({ mimeType: 'audio/mpeg', path: voice!.currentPath }) + expect(fixture.service.resolvePreview(clip!.id)).toEqual({ mimeType: 'video/mp4', path: clip!.currentPath }) + expect(() => fixture.service.resolvePreview(archive!.id)).toThrow(/artifact operation failed/i) + }) + + it('reports an output whose file was moved or deleted as not found', async () => { + const fixture = await createFixture() + const path = join(fixture.workspace, 'report.md') + await writeFile(path, '# report\n') + const [artifact] = await fixture.service.presentOutputs({ conversationId: 'conversation-1', cwd: fixture.workspace, grants: fixture.grants, paths: [path] }) + await rm(path) + await expect(fixture.service.readText(artifact!.id)).rejects.toMatchObject({ code: 'FILE_NOT_FOUND' }) + }) + it.each(['report.md', 'REPORT.MD', 'report.markdown'])('reads %s as Markdown without changing the file', async (name) => { const fixture = await createFixture() const path = join(fixture.workspace, name) diff --git a/apps/buddy/service/src/rpc/runtimeRequest.ts b/apps/buddy/service/src/rpc/runtimeRequest.ts index e359a94b4..ccbdaa02b 100644 --- a/apps/buddy/service/src/rpc/runtimeRequest.ts +++ b/apps/buddy/service/src/rpc/runtimeRequest.ts @@ -36,6 +36,7 @@ export type BuddyServiceErrorCode | 'DIRECTORY_NOT_AUTHORIZED' | 'MODEL_INPUT_UNSUPPORTED' | 'MODEL_INPUT_TOO_LARGE' + | 'FILE_NOT_FOUND' | 'SPACE_UNAVAILABLE' | 'VALIDATION_FAILED' diff --git a/apps/buddy/service/src/spaces/SpaceFileService.ts b/apps/buddy/service/src/spaces/SpaceFileService.ts index a2440f5b6..e7d329b6c 100644 --- a/apps/buddy/service/src/spaces/SpaceFileService.ts +++ b/apps/buddy/service/src/spaces/SpaceFileService.ts @@ -5,6 +5,7 @@ import { Buffer } from 'node:buffer' import { stat } from 'node:fs/promises' import { isAbsolute, relative, resolve, sep } from 'node:path' import { Emitter } from '../../../shared/events/Emitter' +import { mediaKindOf } from '../../../shared/files/mediaFiles' import { resolveGrantedPath } from '../directories/resolveGrantedPath' import { readFilePreview } from '../files/readFilePreview' import { BuddyServiceError } from '../rpc/runtimeRequest' @@ -51,9 +52,18 @@ export class SpaceFileService { } async read(input: SpaceFileTarget): Promise { - const target = await this.resolve(input) + const target = await this.resolve(input).catch(rethrowMissingFile) this.requireDirectory(input) - const preview = await readFilePreview(target.root, target.path) + const media = mediaKindOf(target.path) + if (media) { + // Audio and video are streamed by the media protocol, so the size limits that apply to inline previews do not. + const metadata = await stat(target.path).catch(rethrowMissingFile) + if (!metadata.isFile()) + throw new BuddyServiceError('VALIDATION_FAILED') + this.requireDirectory(input) + return { kind: media, sizeBytes: metadata.size, text: null, imageUrl: null } + } + const preview = await readFilePreview(target.root, target.path).catch(rethrowMissingFile) this.requireDirectory(input) return preview } @@ -235,3 +245,13 @@ export class SpaceFileService { return { path: target.canonicalPath, root: directory.canonicalRoot } } } + +// A file moved or deleted after it was listed is reported on its own, so the preview can say so. +function rethrowMissingFile(error: unknown): never { + for (let current = error, depth = 0; current && typeof current === 'object' && depth < 5; current = (current as { cause?: unknown }).cause, depth++) { + const code = (current as { code?: unknown }).code + if (code === 'ENOENT' || code === 'PATH_NOT_FOUND') + throw new BuddyServiceError('FILE_NOT_FOUND') + } + throw error +} diff --git a/apps/buddy/service/src/spaces/__tests__/SpaceFileService.spec.ts b/apps/buddy/service/src/spaces/__tests__/SpaceFileService.spec.ts index 28747a832..6948f1772 100644 --- a/apps/buddy/service/src/spaces/__tests__/SpaceFileService.spec.ts +++ b/apps/buddy/service/src/spaces/__tests__/SpaceFileService.spec.ts @@ -74,6 +74,24 @@ describe('space file browsing', () => { expect(await f.files.read({ ...f.target, path: 'large.txt' })).toMatchObject({ kind: 'oversized', text: null }) }) + it('previews audio and video by type and size without reading their content', async () => { + const f = await fixture() + await writeFile(join(f.workspace, 'clip.MP4'), Buffer.alloc(2 * 1024 * 1024, 1)) + await writeFile(join(f.workspace, 'song.mp3'), Buffer.from([1, 2, 3])) + expect(await f.files.read({ ...f.target, path: 'clip.MP4' })).toEqual({ kind: 'video', sizeBytes: 2 * 1024 * 1024, text: null, imageUrl: null }) + expect(await f.files.read({ ...f.target, path: 'song.mp3' })).toEqual({ kind: 'audio', sizeBytes: 3, text: null, imageUrl: null }) + await rm(join(f.workspace, 'song.mp3')) + await expect(f.files.read({ ...f.target, path: 'song.mp3' })).rejects.toMatchObject({ code: 'FILE_NOT_FOUND' }) + }) + + it('reports a file that was moved or deleted as not found', async () => { + const f = await fixture() + await writeFile(join(f.workspace, 'note.md'), '# note\n') + await expect(f.files.read({ ...f.target, path: 'note.md' })).resolves.toMatchObject({ kind: 'text' }) + await rm(join(f.workspace, 'note.md')) + await expect(f.files.read({ ...f.target, path: 'note.md' })).rejects.toMatchObject({ code: 'FILE_NOT_FOUND' }) + }) + it('saves UTF-8 content without changing its BOM, rejects stale etags and rechecks grants', async () => { const f = await fixture() const path = join(f.workspace, 'note.md') diff --git a/apps/buddy/shared/files/__tests__/mediaFiles.spec.ts b/apps/buddy/shared/files/__tests__/mediaFiles.spec.ts new file mode 100644 index 000000000..ccfcd3833 --- /dev/null +++ b/apps/buddy/shared/files/__tests__/mediaFiles.spec.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { createMediaUrl, mediaKindOf, mediaMimeType, parseMediaRange, parseMediaUrl } from '../mediaFiles' + +const target = { spaceId: 'space-1', directoryId: 'directory-1', revision: 3, path: '媒体/演示 视频.mp4' } + +describe('media files', () => { + it('tells audio and video apart by extension, ignoring case and hidden names', () => { + expect(mediaKindOf('a/b/clip.MP4')).toBe('video') + expect(mediaKindOf('song.flac')).toBe('audio') + expect(mediaMimeType('clip.mov')).toBe('video/quicktime') + expect(mediaKindOf('notes.md')).toBeNull() + expect(mediaKindOf('.mp3')).toBeNull() + expect(mediaKindOf('constructor')).toBeNull() + expect(mediaMimeType('toString')).toBeNull() + }) + + it('round-trips a file target through the media URL, including non-ASCII paths', () => { + expect(parseMediaUrl(createMediaUrl(target))).toEqual(target) + expect(parseMediaUrl(createMediaUrl(target, 2))).toEqual(target) + }) + + it('rejects URLs of another scheme, host, shape or payload', () => { + const url = createMediaUrl(target) + expect(parseMediaUrl(url.replace('lexora-media:', 'lexora-app:'))).toBeNull() + expect(parseMediaUrl(url.replace('//file/', '//other/'))).toBeNull() + expect(parseMediaUrl(`${url}/extra`)).toBeNull() + expect(parseMediaUrl('lexora-media://file/not-json')).toBeNull() + expect(parseMediaUrl(`lexora-media://file/${btoa(JSON.stringify({ ...target, path: 'a.mp4', extra: true })).replaceAll('=', '')}`)).toBeNull() + }) + + it('parses single byte ranges and refuses everything else', () => { + expect(parseMediaRange('bytes=0-99', 1000)).toEqual({ start: 0, end: 99 }) + expect(parseMediaRange('bytes=900-', 1000)).toEqual({ start: 900, end: 999 }) + expect(parseMediaRange('bytes=-100', 1000)).toEqual({ start: 900, end: 999 }) + expect(parseMediaRange('bytes=-5000', 1000)).toEqual({ start: 0, end: 999 }) + expect(parseMediaRange('bytes=500-5000', 1000)).toEqual({ start: 500, end: 999 }) + expect(parseMediaRange('bytes=1000-', 1000)).toBe('unsatisfiable') + expect(parseMediaRange('bytes=10-5', 1000)).toBe('unsatisfiable') + expect(parseMediaRange('bytes=0-1,5-9', 1000)).toBe('unsatisfiable') + expect(parseMediaRange('bytes=-', 1000)).toBe('unsatisfiable') + expect(parseMediaRange('bytes=0-0', 0)).toBe('unsatisfiable') + }) +}) diff --git a/apps/buddy/shared/files/filePreview.ts b/apps/buddy/shared/files/filePreview.ts index 5a6aad0f2..cdcf1549e 100644 --- a/apps/buddy/shared/files/filePreview.ts +++ b/apps/buddy/shared/files/filePreview.ts @@ -15,7 +15,7 @@ export const directoryPageSchema = z.object({ }).strict() export const filePreviewSchema = z.object({ - kind: z.enum(['text', 'image', 'binary', 'oversized']), + kind: z.enum(['text', 'image', 'audio', 'video', 'binary', 'oversized']), sizeBytes: z.number().int().nonnegative(), text: z.string().max(1024 * 1024).nullable(), imageUrl: z.string().max(12 * 1024 * 1024).nullable(), diff --git a/apps/buddy/shared/files/mediaFiles.ts b/apps/buddy/shared/files/mediaFiles.ts new file mode 100644 index 000000000..c7c9a7146 --- /dev/null +++ b/apps/buddy/shared/files/mediaFiles.ts @@ -0,0 +1,104 @@ +import type { SpaceFileTarget } from '../spaces/spaceFileApi' +import { spaceFileTargetSchema } from '../spaces/spaceFileApi' + +export const MEDIA_PROTOCOL = 'lexora-media' + +export type MediaKind = 'audio' | 'video' + +// Only containers and codecs Chromium can decode on its own; there is no transcoding fallback. +const AUDIO_MIME_TYPES: Readonly> = { + mp3: 'audio/mpeg', + wav: 'audio/wav', + ogg: 'audio/ogg', + oga: 'audio/ogg', + opus: 'audio/ogg', + m4a: 'audio/mp4', + aac: 'audio/aac', + flac: 'audio/flac', + weba: 'audio/webm', +} + +const VIDEO_MIME_TYPES: Readonly> = { + mp4: 'video/mp4', + m4v: 'video/mp4', + webm: 'video/webm', + mov: 'video/quicktime', + ogv: 'video/ogg', +} + +function extensionOf(path: string): string { + const name = path.split(/\//).at(-1) ?? '' + const dot = name.lastIndexOf('.') + return dot > 0 ? name.slice(dot + 1).toLowerCase() : '' +} + +export function mediaKindOf(path: string): MediaKind | null { + const extension = extensionOf(path) + if (Object.hasOwn(AUDIO_MIME_TYPES, extension)) + return 'audio' + return Object.hasOwn(VIDEO_MIME_TYPES, extension) ? 'video' : null +} + +export function mediaMimeType(path: string): string | null { + const extension = extensionOf(path) + if (Object.hasOwn(AUDIO_MIME_TYPES, extension)) + return AUDIO_MIME_TYPES[extension]! + return Object.hasOwn(VIDEO_MIME_TYPES, extension) ? VIDEO_MIME_TYPES[extension]! : null +} + +function toBase64Url(value: string): string { + const bytes = new TextEncoder().encode(value) + let binary = '' + for (const byte of bytes) binary += String.fromCharCode(byte) + return btoa(binary).replaceAll('+', '-').replaceAll('/', '_').replace(/=+$/, '') +} + +function fromBase64Url(value: string): string { + const binary = atob(value.replaceAll('-', '+').replaceAll('_', '/')) + return new TextDecoder('utf-8', { fatal: true }).decode(Uint8Array.from(binary, character => character.charCodeAt(0))) +} + +// The URL carries the file target, never a filesystem path; the main process re-checks the grant on every request. +export function createMediaUrl(target: SpaceFileTarget, version = 0): string { + const { spaceId, directoryId, revision, path } = target + const encoded = toBase64Url(JSON.stringify({ spaceId, directoryId, revision, path })) + return `${MEDIA_PROTOCOL}://file/${encoded}${version ? `?v=${version}` : ''}` +} + +export function parseMediaUrl(value: string): SpaceFileTarget | null { + try { + const url = new URL(value) + if (url.protocol !== `${MEDIA_PROTOCOL}:` || url.hostname !== 'file') + return null + const segments = url.pathname.split('/').filter(Boolean) + if (segments.length !== 1) + return null + const parsed = spaceFileTargetSchema.safeParse(JSON.parse(fromBase64Url(segments[0]!))) + return parsed.success ? parsed.data : null + } + catch { + return null + } +} + +export type MediaByteRange = { start: number, end: number } | 'unsatisfiable' + +// Single `bytes=a-b`, `bytes=a-` and `bytes=-n` forms; multi-range requests are refused rather than answered in full. +export function parseMediaRange(header: string, size: number): MediaByteRange { + const match = /^bytes=(\d*)-(\d*)$/.exec(header.trim()) + if (!match || (!match[1] && !match[2]) || size <= 0) + return 'unsatisfiable' + let start: number + let end: number + if (!match[1]) { + start = Math.max(size - Number(match[2]), 0) + end = size - 1 + } + else { + start = Number(match[1]) + end = match[2] ? Number(match[2]) : size - 1 + } + if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start >= size || end < start) + return 'unsatisfiable' + return { start, end: Math.min(end, size - 1) } +} diff --git a/apps/buddy/shared/runtime/localChatError.ts b/apps/buddy/shared/runtime/localChatError.ts index 1e7e741d0..ee2f32eb8 100644 --- a/apps/buddy/shared/runtime/localChatError.ts +++ b/apps/buddy/shared/runtime/localChatError.ts @@ -34,6 +34,7 @@ export type LocalChatErrorCode | 'FILE_RECOVERY_REQUIRED' | 'FILE_OPERATION_CANCELLED' | 'FILE_PERMISSION_REVOKED' + | 'FILE_NOT_FOUND' | 'FILE_UNSUPPORTED' | 'FILE_ENGINE_INTERRUPTED' | 'LOCAL_CHAT_OPERATION_FAILED' @@ -107,6 +108,7 @@ const LOCAL_CHAT_ERROR_CODES = new Set([ 'FILE_RECOVERY_REQUIRED', 'FILE_OPERATION_CANCELLED', 'FILE_PERMISSION_REVOKED', + 'FILE_NOT_FOUND', 'FILE_UNSUPPORTED', 'FILE_ENGINE_INTERRUPTED', 'LOCAL_CHAT_OPERATION_FAILED', diff --git a/apps/buddy/src/app/workbench/useDesktopWorkbench.ts b/apps/buddy/src/app/workbench/useDesktopWorkbench.ts index 852cfb07f..71d09646d 100644 --- a/apps/buddy/src/app/workbench/useDesktopWorkbench.ts +++ b/apps/buddy/src/app/workbench/useDesktopWorkbench.ts @@ -12,6 +12,7 @@ import type { ChatReadingPositions } from '@/modules/tasks/ui' import type { CommandContext, DropPosition, ResourceRef, SplitDirection, WorkbenchView } from '@/workbench/common/workbench' import type { ViewCloseDecision } from '@/workbench/services/WorkbenchController' import { buddyUserContentToText, getBuddyUserContentResourceIds, hasBuddyUserContent } from '@buddy-shared/conversation/buddyUserContent' +import { mediaKindOf } from '@buddy-shared/files/mediaFiles' import { isSkillAvailable } from '@buddy-shared/skills/skillApi' import { spaceFileTargetSchema } from '@buddy-shared/spaces/spaceFileApi' import { NButton, useDialog } from 'naive-ui' @@ -386,11 +387,16 @@ export function useDesktopWorkbench(options: { buddyCatalog?: Readonly(null) const failed = shallowRef(false) -const modes = computed(() => fileDocumentModes({ preview: preview.value?.kind === 'image' || (preview.value?.kind === 'text' && isMarkdownFile(props.view.title)), source: preview.value?.kind === 'text', edit: false })) +const missing = shallowRef(false) +const modes = computed(() => fileDocumentModes({ preview: preview.value?.kind === 'image' || preview.value?.kind === 'audio' || preview.value?.kind === 'video' || (preview.value?.kind === 'text' && isMarkdownFile(props.view.title)), source: preview.value?.kind === 'text', edit: false })) const mode = computed({ get: () => resolveFileDocumentMode(props.view.state.mode, modes.value), set: value => controller.updateView(props.view.id, { state: { ...props.view.state, mode: value } }) }) const quoteSource = computed(() => ({ kind: 'file', @@ -38,19 +41,26 @@ const selection = useResourceSelectionQuote({ language: () => props.language, }) const attempt = shallowRef(0) +const media = computed(() => { + const kind = preview.value?.kind + return kind === 'audio' || kind === 'video' ? { kind, url: createMediaUrl(spaceFileTargetSchema.strip().parse(props.view.resource.data), attempt.value) } : null +}) watch([identity, attempt], async (_, __, onCleanup) => { let active = true onCleanup(() => active = false) preview.value = null failed.value = false + missing.value = false try { const value = await props.files.readFile(spaceFileTargetSchema.strip().parse(props.view.resource.data)) if (active) preview.value = value } - catch { - if (active) + catch (error) { + if (active) { failed.value = true + missing.value = error instanceof Error && parseLocalChatPublicError(error.message)?.code === 'FILE_NOT_FOUND' + } } }, { immediate: true }) const document = useTemplateRef>('document') @@ -80,7 +90,7 @@ function captureResource() { diff --git a/apps/buddy/src/modules/tasks/widgets/context-panel/__tests__/useResourcePreviews.spec.ts b/apps/buddy/src/modules/tasks/widgets/context-panel/__tests__/useResourcePreviews.spec.ts index 8b5ef2a49..cb63f10bd 100644 --- a/apps/buddy/src/modules/tasks/widgets/context-panel/__tests__/useResourcePreviews.spec.ts +++ b/apps/buddy/src/modules/tasks/widgets/context-panel/__tests__/useResourcePreviews.spec.ts @@ -225,6 +225,16 @@ describe('resource previews', () => { expect(state.textPreviewLoading.value).toBe(false) }) + it('tells a moved or deleted file apart from other read failures', async () => { + const artifact = { ...createArtifact('moved'), name: 'moved.md', mimeType: 'text/markdown' } + const missing = own(() => useArtifactPreview({ artifact: () => artifact, readText: () => failWith('Error invoking remote method: Error: LEXORA_LOCAL_CHAT_ERROR:FILE_NOT_FOUND:0') })) + const broken = own(() => useArtifactPreview({ artifact: () => artifact, readText: () => failWith('LEXORA_LOCAL_CHAT_ERROR:LOCAL_CHAT_OPERATION_FAILED:0') })) + await vi.waitFor(() => expect(missing.state.textPreviewFailed.value).toBe(true)) + await vi.waitFor(() => expect(broken.state.textPreviewFailed.value).toBe(true)) + expect(missing.state.textPreviewMissing.value).toBe(true) + expect(broken.state.textPreviewMissing.value).toBe(false) + }) + it('ignores late text success and failure after switching artifacts or closing the preview', async () => { const first = deferred() const second = deferred() @@ -302,6 +312,12 @@ describe('resource previews', () => { }) }) +function failWith(message: string) { + return async (): Promise => { + throw new Error(message) + } +} + function filesTab(): TaskFilesContextTab { return { id: 'files:first', scope: 'task:first', kind: 'files', rootName: 'fixture', target: { spaceId: 'space', directoryId: 'directory', revision: 1, path: '' } } } diff --git a/apps/buddy/src/modules/tasks/widgets/context-panel/useArtifactPreview.ts b/apps/buddy/src/modules/tasks/widgets/context-panel/useArtifactPreview.ts index 1daf09518..92122a6f8 100644 --- a/apps/buddy/src/modules/tasks/widgets/context-panel/useArtifactPreview.ts +++ b/apps/buddy/src/modules/tasks/widgets/context-panel/useArtifactPreview.ts @@ -1,4 +1,6 @@ import type { LocalArtifact, LocalArtifactText } from '@buddy-shared/artifacts/artifactApi' +import { mediaKindOf } from '@buddy-shared/files/mediaFiles' +import { parseLocalChatPublicError } from '@buddy-shared/runtime/localChatError' import { computed, readonly, shallowRef, watch } from 'vue' import { isMarkdownArtifact, isTextMimeType } from './artifactContextPresentation' @@ -14,6 +16,7 @@ export function useArtifactPreview(options: ArtifactPreviewOptions) { const previewOpen = shallowRef(false) const textPreview = shallowRef(null) const textPreviewFailed = shallowRef(false) + const textPreviewMissing = shallowRef(false) const textPreviewLoading = shallowRef(false) const imageUrl = computed(() => { const artifact = options.artifact() @@ -21,6 +24,12 @@ export function useArtifactPreview(options: ArtifactPreviewOptions) { ? `lexora-artifact://preview/${encodeURIComponent(artifact.artifactId)}?v=${encodeURIComponent(artifact.updatedAt)}` : null }) + // Audio and video play in place; the protocol serves them in ranges so seeking does not load the whole file. + const media = computed(() => { + const artifact = options.artifact() + const kind = artifact.kind === 'file' ? mediaKindOf(artifact.name) : null + return kind ? { kind, url: `lexora-artifact://preview/${encodeURIComponent(artifact.artifactId)}?v=${encodeURIComponent(artifact.updatedAt)}` } : null + }) const previewUrl = computed(() => previewFailed.value ? null : imageUrl.value) const previewSources = computed(() => previewUrl.value ? [previewUrl.value] : []) @@ -41,6 +50,7 @@ export function useArtifactPreview(options: ArtifactPreviewOptions) { previewOpen.value = false textPreview.value = null textPreviewFailed.value = false + textPreviewMissing.value = false textPreviewLoading.value = kind === 'file' && (isTextMimeType(mimeType) || isMarkdownArtifact(options.artifact())) if (!textPreviewLoading.value) return @@ -49,9 +59,11 @@ export function useArtifactPreview(options: ArtifactPreviewOptions) { if (active) textPreview.value = result } - catch { - if (active) + catch (error) { + if (active) { textPreviewFailed.value = true + textPreviewMissing.value = error instanceof Error && parseLocalChatPublicError(error.message)?.code === 'FILE_NOT_FOUND' + } } finally { if (active) @@ -72,6 +84,7 @@ export function useArtifactPreview(options: ArtifactPreviewOptions) { return { refreshText: () => revision.value++, failImage, + media, openPreview, previewIndex, previewOpen, @@ -79,6 +92,7 @@ export function useArtifactPreview(options: ArtifactPreviewOptions) { previewUrl, textPreview: readonly(textPreview), textPreviewFailed: readonly(textPreviewFailed), + textPreviewMissing: readonly(textPreviewMissing), textPreviewLoading: readonly(textPreviewLoading), } } diff --git a/apps/buddy/src/shared/lib/localChatError.ts b/apps/buddy/src/shared/lib/localChatError.ts index 7c590b844..92315780a 100644 --- a/apps/buddy/src/shared/lib/localChatError.ts +++ b/apps/buddy/src/shared/lib/localChatError.ts @@ -49,6 +49,7 @@ export function resolveLocalChatErrorMessage(error: unknown, language: BuddyLoca FILE_RECOVERY_REQUIRED: 'desktop.error.fileRecoveryRequired', FILE_OPERATION_CANCELLED: 'desktop.chat.unknownError', FILE_PERMISSION_REVOKED: 'desktop.error.directoryNotAuthorized', + FILE_NOT_FOUND: 'desktop.error.fileNotFound', FILE_UNSUPPORTED: 'desktop.chat.unknownError', FILE_ENGINE_INTERRUPTED: 'desktop.chat.unknownError', LOCAL_CHAT_OPERATION_FAILED: 'desktop.chat.unknownError', diff --git a/apps/buddy/src/shared/ui/files/DesktopDocumentContent.vue b/apps/buddy/src/shared/ui/files/DesktopDocumentContent.vue index 1bd5b6e29..d49a8d8e6 100644 --- a/apps/buddy/src/shared/ui/files/DesktopDocumentContent.vue +++ b/apps/buddy/src/shared/ui/files/DesktopDocumentContent.vue @@ -1,5 +1,6 @@ + + + + diff --git a/apps/buddy/src/shared/ui/files/__tests__/DesktopMediaContent.spec.ts b/apps/buddy/src/shared/ui/files/__tests__/DesktopMediaContent.spec.ts new file mode 100644 index 000000000..8ff7d4c15 --- /dev/null +++ b/apps/buddy/src/shared/ui/files/__tests__/DesktopMediaContent.spec.ts @@ -0,0 +1,42 @@ +// @vitest-environment jsdom +import { describe, expect, it } from 'vitest' +import { createApp, h, nextTick, shallowRef } from 'vue' +import DesktopMediaContent from '../DesktopMediaContent.vue' + +function mount(kind: 'audio' | 'video') { + const url = shallowRef('lexora-media://file/abc') + const root = document.createElement('div') + document.body.append(root) + const app = createApp({ render: () => h(DesktopMediaContent, { kind, url: url.value, name: 'clip', failedLabel: '无法播放' }) }) + app.mount(root) + const dispose = () => { + app.unmount() + root.remove() + } + return { root, url, dispose } +} + +describe('media content', () => { + it('renders a video player for video and an audio player for audio', () => { + const video = mount('video') + expect(video.root.querySelector('video')?.getAttribute('src')).toBe('lexora-media://file/abc') + expect(video.root.querySelector('audio')).toBeNull() + video.dispose() + const audio = mount('audio') + expect(audio.root.querySelector('audio')?.getAttribute('src')).toBe('lexora-media://file/abc') + expect(audio.root.querySelector('video')).toBeNull() + audio.dispose() + }) + + it('reports a playback error and clears it when the source changes', async () => { + const view = mount('video') + expect(view.root.querySelector('[role="alert"]')).toBeNull() + view.root.querySelector('video')!.dispatchEvent(new Event('error')) + await nextTick() + expect(view.root.querySelector('[role="alert"]')?.textContent?.trim()).toBe('无法播放') + view.url.value = 'lexora-media://file/def' + await nextTick() + expect(view.root.querySelector('[role="alert"]')).toBeNull() + view.dispose() + }) +}) diff --git a/apps/buddy/src/workbench/common/workbenchLabels.ts b/apps/buddy/src/workbench/common/workbenchLabels.ts index 4801467b5..773d67dc3 100644 --- a/apps/buddy/src/workbench/common/workbenchLabels.ts +++ b/apps/buddy/src/workbench/common/workbenchLabels.ts @@ -33,6 +33,8 @@ const zh = { discardDraft: '放弃内容', missing: '此视图暂不可用，资源和布局已保留。', failed: '视图加载失败。', + fileMissing: '文件已被移动或删除。', + mediaFailed: '无法播放此文件，可能是当前不支持的编码格式。', restoreFailed: '工作台恢复失败，请重试。', backupFailed: '未保存内容的备份失败。请保存文件或重试备份。', conflict: '文件已在外部修改。请比较内容后选择保留的版本。', @@ -96,6 +98,8 @@ const en: Labels = { discardDraft: 'Discard draft', missing: 'This view is unavailable. Its resource and layout have been retained.', failed: 'The view could not be loaded.', + fileMissing: 'This file has been moved or deleted.', + mediaFailed: 'This file cannot be played. Its encoding may not be supported.', restoreFailed: 'Could not restore the workbench. Please retry.', backupFailed: 'Could not back up unsaved changes. Save the file or retry the backup.', conflict: 'The file has changed on disk. Compare both versions before choosing which to keep.',