diff --git a/electron/mediaSandbox.ts b/electron/mediaSandbox.ts index c0ffe18b..d9b3d56f 100644 --- a/electron/mediaSandbox.ts +++ b/electron/mediaSandbox.ts @@ -7,6 +7,8 @@ import { fileURLToPath, pathToFileURL } from "url"; import { MAX_MEDIA_BYTES, type MediaJob, type MediaResult } from "./mediaSandboxFormat"; const JOB_TIMEOUT_MS = 120_000; +// A chat video has no ten-second cap, so it gets longer. +const CHAT_VIDEO_TIMEOUT_MS = 10 * 60_000; // Closed when idle, so it holds no memory between uploads and never keeps the app from quitting. const IDLE_CLOSE_MS = 30_000; const PARTITION = "gryt-media-sandbox"; @@ -111,7 +113,7 @@ async function runOne(job: MediaJob): Promise { const id = nextId++; return new Promise((resolve) => { // A job that runs out of time takes the renderer with it, since it may still be decoding. - const timer = setTimeout(() => discard("The media sandbox ran out of time"), JOB_TIMEOUT_MS); + const timer = setTimeout(() => discard("The media sandbox ran out of time"), job.kind === "chatvideo" ? CHAT_VIDEO_TIMEOUT_MS : JOB_TIMEOUT_MS); pending.set(id, { resolve, timer }); wc.send("media-sandbox:job", id, job); }); diff --git a/electron/mediaSandboxFormat.ts b/electron/mediaSandboxFormat.ts index ae6d7793..2380ffad 100644 --- a/electron/mediaSandboxFormat.ts +++ b/electron/mediaSandboxFormat.ts @@ -31,13 +31,24 @@ export const MAX_MEDIA_BYTES = 64 * 1024 * 1024; export const MAX_INPUT_PIXELS = 100_000_000; // The worker's cap too: a 1,500-frame GIF came out as a 7 MB avatar. export const MAX_ANIMATION_FRAMES = 300; +/** A chat video keeps its shape and sound, inside this box (GRYT-1669). */ +export const CHAT_VIDEO_MAX_SIDE = 1280; + +/** Even sides inside the box, never enlarged: what AV1 at 4:2:0 wants. */ +export function chatVideoSize(w: number, h: number): { width: number; height: number } { + const scale = Math.min(1, CHAT_VIDEO_MAX_SIDE / w, CHAT_VIDEO_MAX_SIDE / h); + const even = (n: number) => Math.max(2, Math.floor((n * scale) / 2) * 2); + return { width: even(w), height: even(h) }; +} + /** A chat video's still, as wide as the jail's poster. */ export const POSTER_WIDTH = 320; export type MediaJob = | { kind: "image"; use: MediaUse; bytes: Uint8Array } | { kind: "video"; use: VideoUse; bytes: Uint8Array } - | { kind: "poster"; bytes: Uint8Array }; + | { kind: "poster"; bytes: Uint8Array } + | { kind: "chatvideo"; bytes: Uint8Array }; export type MediaResult = | { @@ -55,6 +66,7 @@ export type MediaResult = } | { ok: true; kind: "video"; video: Uint8Array; poster: Uint8Array; width: number; height: number } | { ok: true; kind: "poster"; poster: Uint8Array; width: number; height: number } + | { ok: true; kind: "chatvideo"; video: Uint8Array; poster: Uint8Array; width: number; height: number } | { ok: false; reason: string }; const USES = new Set(["upload", "banner", "avatar", "emoji"]); @@ -73,6 +85,7 @@ export function parseMediaRequest(message: unknown): { id: number; job: MediaJob return { id: m.id, job: { kind: "video", use: job.use, bytes: job.bytes } }; } if (job.kind === "poster") return { id: m.id, job: { kind: "poster", bytes: job.bytes } }; + if (job.kind === "chatvideo") return { id: m.id, job: { kind: "chatvideo", bytes: job.bytes } }; return null; } diff --git a/electron/mediaSandboxPage.ts b/electron/mediaSandboxPage.ts index 1fcba3f1..170525a3 100644 --- a/electron/mediaSandboxPage.ts +++ b/electron/mediaSandboxPage.ts @@ -2,10 +2,11 @@ /* Runs in the media sandbox window: a sandboxed renderer with no Node, no network and no storage. Untrusted pictures and videos are decoded here, never in the main process. */ -import { BufferTarget, CanvasSource, Mp4OutputFormat, Output } from "mediabunny"; +import { BlobSource, BufferTarget, CanvasSource, Conversion, Input, MATROSKA, MP4, Mp4OutputFormat, Output, QTFF, WEBM } from "mediabunny"; import { buildAnimatedWebp, + chatVideoSize, dominantColour, IMAGE_PROFILES, imageBox, @@ -262,6 +263,36 @@ async function poster(bytes: Uint8Array): Promise { } } +/* A chat video written out again with its sound: AV1 and Opus in MP4, every frame drawn into + one size. The container list leaves out HLS, which would go looking for other files. */ +async function chatVideo(bytes: Uint8Array): Promise { + const input = new Input({ source: new BlobSource(new Blob([bytes as Uint8Array])), formats: [MP4, QTFF, MATROSKA, WEBM] }); + try { + const track = await input.getPrimaryVideoTrack(); + if (!track) return { ok: false, reason: "No picture in the video" }; + const { width, height } = chatVideoSize(track.displayWidth, track.displayHeight); + const target = new BufferTarget(); + const output = new Output({ format: new Mp4OutputFormat({ fastStart: "in-memory" }), target }); + const conversion = await Conversion.init({ + input, + output, + tracks: "primary", + video: { width, height, fit: "contain", codec: "av1", bitrate: 2_000_000, frameRate: 30, forceTranscode: true, allowRotationMetadata: false }, + audio: { codec: "opus", bitrate: 128_000, numberOfChannels: 2, sampleRate: 48_000, forceTranscode: true }, + }); + if (!conversion.isValid) return { ok: false, reason: "The video could not be decoded" }; + await conversion.execute(); + if (!target.buffer) return { ok: false, reason: "The encoder wrote nothing" }; + const video = new Uint8Array(target.buffer); + // The poster comes from what was written, not from the upload. + const still = await poster(video); + if (!still.ok || still.kind !== "poster") return { ok: false, reason: "No poster from the converted video" }; + return { ok: true, kind: "chatvideo", video, poster: still.poster, width, height }; + } finally { + input.dispose?.(); + } +} + async function run(job: MediaJob): Promise { if (!(job.bytes instanceof Uint8Array) || job.bytes.length === 0 || job.bytes.length > MAX_MEDIA_BYTES) { return { ok: false, reason: "File is empty or too large to process" }; @@ -269,6 +300,7 @@ async function run(job: MediaJob): Promise { if (job.kind === "image" && job.use in IMAGE_PROFILES) return image(job.use, job.bytes); if (job.kind === "video" && job.use in VIDEO_BOXES) return video(job.use, job.bytes); if (job.kind === "poster") return poster(job.bytes); + if (job.kind === "chatvideo") return chatVideo(job.bytes); return { ok: false, reason: "Unknown job" }; } diff --git a/scripts/check-media-sandbox.mjs b/scripts/check-media-sandbox.mjs index da287807..520953a9 100644 --- a/scripts/check-media-sandbox.mjs +++ b/scripts/check-media-sandbox.mjs @@ -8,6 +8,7 @@ import assert from "node:assert/strict"; import { buildAnimatedWebp, + chatVideoSize, dominantColour, imageBox, parseMediaRequest, @@ -88,4 +89,11 @@ const clear = [0, 255, 0, 0]; assert.equal(dominantColour(new Uint8Array([...red, ...red, ...blue, ...clear, ...clear, ...clear])), "#ff0000"); assert.equal(dominantColour(new Uint8Array(clear)), null, "a see-through picture has no colour"); +// A chat video keeps its shape inside 1280px, with even sides, and is never enlarged. +assert.deepEqual(chatVideoSize(1920, 1080), { width: 1280, height: 720 }); +assert.deepEqual(chatVideoSize(1080, 1920), { width: 720, height: 1280 }); +assert.deepEqual(chatVideoSize(641, 359), { width: 640, height: 358 }); +assert.deepEqual(chatVideoSize(8192, 8192), { width: 1280, height: 1280 }); +assert.deepEqual(parseMediaRequest({ type: "gryt-media-job", id: 5, job: { kind: "chatvideo", bytes: job.bytes } }), { id: 5, job: { kind: "chatvideo", bytes: job.bytes } }); + console.log("media sandbox: ok"); diff --git a/src/packages/socket/src/components/MessageAttachment.tsx b/src/packages/socket/src/components/MessageAttachment.tsx index 2a97f9b3..3bf45b6a 100644 --- a/src/packages/socket/src/components/MessageAttachment.tsx +++ b/src/packages/socket/src/components/MessageAttachment.tsx @@ -1,3 +1,5 @@ +import { Spinner } from "@gryt/ui"; + import { useSealedVideo } from "../hooks/useSealedVideo"; import { useStableFileUrl } from "../hooks/useStableFileUrl"; import { readBlobUrl } from "../utils/downloadFile"; @@ -34,6 +36,16 @@ export function MessageAttachment({ const openLocal = local ? () => readBlobUrl(local) : undefined; const fileName = meta?.original_name; + if (meta?.refused) { + return ( +
+ {fileName ? `${fileName} couldn't be processed, so it isn't shown.` : "This file couldn't be processed, so it isn't shown."} +
+ ); + } + // Still with the server's image worker: fetching it now would just wait (GRYT-1669). + if (meta?.processing) return ; + if (mime.startsWith("image/")) { const imgSrc = local || url; return ( @@ -133,3 +145,25 @@ function SealedVideoAttachment({ ); } + +/** A loader in the attachment's own shape, over the sender's local copy when there is one. */ +function ProcessingAttachment({ meta }: { meta: AttachmentMeta }) { + const w = meta.width && meta.height ? Math.min(320, meta.width) : 320; + const h = meta.width && meta.height ? Math.round((w * meta.height) / meta.width) : 180; + return ( +
+ {meta.local_url && meta.mime?.startsWith("image/") && ( + + )} +
+ + Processing… +
+
+ ); +} diff --git a/src/packages/socket/src/components/chatUtils.tsx b/src/packages/socket/src/components/chatUtils.tsx index 350eef9a..a849340e 100644 --- a/src/packages/socket/src/components/chatUtils.tsx +++ b/src/packages/socket/src/components/chatUtils.tsx @@ -16,6 +16,10 @@ export type AttachmentMeta = { width: number | null; height: number | null; has_thumbnail: boolean; + /** The server's image worker still has it: drawn as a loader, not fetched (GRYT-1669). */ + processing?: boolean; + /** The worker couldn't write it out, so the server never serves it. */ + refused?: boolean; /** Blob URL of the file itself, when it is decrypted or still being sent. */ local_url?: string; /** A sealed file not fetched yet. Calling it fetches and decrypts it. */ diff --git a/src/packages/socket/src/hooks/chatEventHandlers.ts b/src/packages/socket/src/hooks/chatEventHandlers.ts index d775879a..057acf36 100644 --- a/src/packages/socket/src/hooks/chatEventHandlers.ts +++ b/src/packages/socket/src/hooks/chatEventHandlers.ts @@ -163,6 +163,26 @@ export function handleMessageEdited( } } +// ── chat:attachments ───────────────────────────────────────────────── + +/** A message's attachments finished processing, or one was refused. Only the attachments + change; text and edit time stay as they are (GRYT-1669). */ +export function handleAttachmentsSettled( + payload: { conversation_id: string; message_id: string; enriched_attachments: ChatMessage["enriched_attachments"] }, + activeConversationId: string, + getCacheKey: CacheKeyFn, + setMessageCache: Dispatch>, + setChatMessages: Dispatch>, +): void { + if (!payload?.conversation_id || !payload.message_id || !Array.isArray(payload.enriched_attachments)) return; + const key = getCacheKey(payload.conversation_id); + if (!key) return; + const apply = (msg: ChatMessage) => + msg.message_id === payload.message_id ? { ...msg, enriched_attachments: payload.enriched_attachments } : msg; + setMessageCache((prev) => ({ ...prev, [key]: (prev[key] || []).map(apply) })); + if (payload.conversation_id === activeConversationId) setChatMessages((prev) => prev.map(apply)); +} + // ── chat:deleted ───────────────────────────────────────────────────── export function handleMessageDeleted( diff --git a/src/packages/socket/src/hooks/useChat.ts b/src/packages/socket/src/hooks/useChat.ts index 88ca7b6a..00973723 100644 --- a/src/packages/socket/src/hooks/useChat.ts +++ b/src/packages/socket/src/hooks/useChat.ts @@ -22,6 +22,7 @@ import { sealedNotificationBody } from "../utils/sealedNotification"; import { ChatErrorPayload, type ChatErrorRef, + handleAttachmentsSettled, handleChatErrorEvent, handleHistoryPayload, handleMessageDeleted, @@ -504,6 +505,8 @@ export function useChat({ const onEdited = (updatedMessage: ChatMessage) => handleMessageEdited(updatedMessage, activeConversationId, cacheKeyFor, setMessageCache, setChatMessages); + const onAttachments = (payload: Parameters[0]) => + handleAttachmentsSettled(payload, activeConversationId, cacheKeyFor, setMessageCache, setChatMessages); const onReportSubmitted = () => { toast.success("Report submitted"); @@ -582,6 +585,7 @@ export function useChat({ currentConnection.on("chat:reaction", onReaction); currentConnection.on("chat:deleted", onDeleted); currentConnection.on("chat:edited", onEdited); + currentConnection.on("chat:attachments", onAttachments); currentConnection.on("report:submitted", onReportSubmitted); currentConnection.on("report:already_reported", onAlreadyReported); currentConnection.on("chat:purge_user", onPurgeUser); @@ -594,6 +598,7 @@ export function useChat({ currentConnection.off("chat:reaction", onReaction); currentConnection.off("chat:deleted", onDeleted); currentConnection.off("chat:edited", onEdited); + currentConnection.off("chat:attachments", onAttachments); currentConnection.off("report:submitted", onReportSubmitted); currentConnection.off("report:already_reported", onAlreadyReported); currentConnection.off("chat:purge_user", onPurgeUser);