A loader on attachments the server is still processing, and the sandbox converts chat videos with sound (GRYT-1669)

electron/mediaSandbox.ts

@@ -7,6 +7,8 @@ import { fileURLToPath, pathToFileURL } from "url";
 import { MAX_MEDIA_BYTES, type MediaJob, type MediaResult } from "./mediaSandboxFormat";
 
 const JOB_TIMEOUT_MS = 120_000;
+// A chat video has no ten-second cap, so it gets longer.
+const CHAT_VIDEO_TIMEOUT_MS = 10 * 60_000;
 // Closed when idle, so it holds no memory between uploads and never keeps the app from quitting.
 const IDLE_CLOSE_MS = 30_000;
 const PARTITION = "gryt-media-sandbox";
@@ -111,7 +113,7 @@ async function runOne(job: MediaJob): Promise<MediaResult> {
   const id = nextId++;
   return new Promise<MediaResult>((resolve) => {
     // A job that runs out of time takes the renderer with it, since it may still be decoding.
-    const timer = setTimeout(() => discard("The media sandbox ran out of time"), JOB_TIMEOUT_MS);
+    const timer = setTimeout(() => discard("The media sandbox ran out of time"), job.kind === "chatvideo" ? CHAT_VIDEO_TIMEOUT_MS : JOB_TIMEOUT_MS);
     pending.set(id, { resolve, timer });
     wc.send("media-sandbox:job", id, job);
   });

electron/mediaSandboxFormat.ts

@@ -31,13 +31,24 @@ export const MAX_MEDIA_BYTES = 64 * 1024 * 1024;
 export const MAX_INPUT_PIXELS = 100_000_000;
 // The worker's cap too: a 1,500-frame GIF came out as a 7 MB avatar.
 export const MAX_ANIMATION_FRAMES = 300;
+/** A chat video keeps its shape and sound, inside this box (GRYT-1669). */
+export const CHAT_VIDEO_MAX_SIDE = 1280;
+
+/** Even sides inside the box, never enlarged: what AV1 at 4:2:0 wants. */
+export function chatVideoSize(w: number, h: number): { width: number; height: number } {
+  const scale = Math.min(1, CHAT_VIDEO_MAX_SIDE / w, CHAT_VIDEO_MAX_SIDE / h);
+  const even = (n: number) => Math.max(2, Math.floor((n * scale) / 2) * 2);
+  return { width: even(w), height: even(h) };
+}
+
 /** A chat video's still, as wide as the jail's poster. */
 export const POSTER_WIDTH = 320;
 
 export type MediaJob =
   | { kind: "image"; use: MediaUse; bytes: Uint8Array }
   | { kind: "video"; use: VideoUse; bytes: Uint8Array }
-  | { kind: "poster"; bytes: Uint8Array };
+  | { kind: "poster"; bytes: Uint8Array }
+  | { kind: "chatvideo"; bytes: Uint8Array };
 
 export type MediaResult =
   | {
@@ -55,6 +66,7 @@ export type MediaResult =
     }
   | { ok: true; kind: "video"; video: Uint8Array; poster: Uint8Array; width: number; height: number }
   | { ok: true; kind: "poster"; poster: Uint8Array; width: number; height: number }
+  | { ok: true; kind: "chatvideo"; video: Uint8Array; poster: Uint8Array; width: number; height: number }
   | { ok: false; reason: string };
 
 const USES = new Set<string>(["upload", "banner", "avatar", "emoji"]);
@@ -73,6 +85,7 @@ export function parseMediaRequest(message: unknown): { id: number; job: MediaJob
     return { id: m.id, job: { kind: "video", use: job.use, bytes: job.bytes } };
   }
   if (job.kind === "poster") return { id: m.id, job: { kind: "poster", bytes: job.bytes } };
+  if (job.kind === "chatvideo") return { id: m.id, job: { kind: "chatvideo", bytes: job.bytes } };
   return null;
 }
 

electron/mediaSandboxPage.ts

@@ -2,10 +2,11 @@
 
 /* Runs in the media sandbox window: a sandboxed renderer with no Node, no network and no
    storage. Untrusted pictures and videos are decoded here, never in the main process. */
-import { BufferTarget, CanvasSource, Mp4OutputFormat, Output } from "mediabunny";
+import { BlobSource, BufferTarget, CanvasSource, Conversion, Input, MATROSKA, MP4, Mp4OutputFormat, Output, QTFF, WEBM } from "mediabunny";
 
 import {
   buildAnimatedWebp,
+  chatVideoSize,
   dominantColour,
   IMAGE_PROFILES,
   imageBox,
@@ -262,13 +263,44 @@ async function poster(bytes: Uint8Array): Promise<MediaResult> {
   }
 }
 
+/* A chat video written out again with its sound: AV1 and Opus in MP4, every frame drawn into
+   one size. The container list leaves out HLS, which would go looking for other files. */
+async function chatVideo(bytes: Uint8Array): Promise<MediaResult> {
+  const input = new Input({ source: new BlobSource(new Blob([bytes as Uint8Array<ArrayBuffer>])), formats: [MP4, QTFF, MATROSKA, WEBM] });
+  try {
+    const track = await input.getPrimaryVideoTrack();
+    if (!track) return { ok: false, reason: "No picture in the video" };
+    const { width, height } = chatVideoSize(track.displayWidth, track.displayHeight);
+    const target = new BufferTarget();
+    const output = new Output({ format: new Mp4OutputFormat({ fastStart: "in-memory" }), target });
+    const conversion = await Conversion.init({
+      input,
+      output,
+      tracks: "primary",
+      video: { width, height, fit: "contain", codec: "av1", bitrate: 2_000_000, frameRate: 30, forceTranscode: true, allowRotationMetadata: false },
+      audio: { codec: "opus", bitrate: 128_000, numberOfChannels: 2, sampleRate: 48_000, forceTranscode: true },
+    });
+    if (!conversion.isValid) return { ok: false, reason: "The video could not be decoded" };
+    await conversion.execute();
+    if (!target.buffer) return { ok: false, reason: "The encoder wrote nothing" };
+    const video = new Uint8Array(target.buffer);
+    // The poster comes from what was written, not from the upload.
+    const still = await poster(video);
+    if (!still.ok || still.kind !== "poster") return { ok: false, reason: "No poster from the converted video" };
+    return { ok: true, kind: "chatvideo", video, poster: still.poster, width, height };
+  } finally {
+    input.dispose?.();
+  }
+}
+
 async function run(job: MediaJob): Promise<MediaResult> {
   if (!(job.bytes instanceof Uint8Array) || job.bytes.length === 0 || job.bytes.length > MAX_MEDIA_BYTES) {
     return { ok: false, reason: "File is empty or too large to process" };
   }
   if (job.kind === "image" && job.use in IMAGE_PROFILES) return image(job.use, job.bytes);
   if (job.kind === "video" && job.use in VIDEO_BOXES) return video(job.use, job.bytes);
   if (job.kind === "poster") return poster(job.bytes);
+  if (job.kind === "chatvideo") return chatVideo(job.bytes);
   return { ok: false, reason: "Unknown job" };
 }
 

scripts/check-media-sandbox.mjs

@@ -8,6 +8,7 @@ import assert from "node:assert/strict";
 
 import {
   buildAnimatedWebp,
+  chatVideoSize,
   dominantColour,
   imageBox,
   parseMediaRequest,
@@ -88,4 +89,11 @@ const clear = [0, 255, 0, 0];
 assert.equal(dominantColour(new Uint8Array([...red, ...red, ...blue, ...clear, ...clear, ...clear])), "#ff0000");
 assert.equal(dominantColour(new Uint8Array(clear)), null, "a see-through picture has no colour");
 
+// A chat video keeps its shape inside 1280px, with even sides, and is never enlarged.
+assert.deepEqual(chatVideoSize(1920, 1080), { width: 1280, height: 720 });
+assert.deepEqual(chatVideoSize(1080, 1920), { width: 720, height: 1280 });
+assert.deepEqual(chatVideoSize(641, 359), { width: 640, height: 358 });
+assert.deepEqual(chatVideoSize(8192, 8192), { width: 1280, height: 1280 });
+assert.deepEqual(parseMediaRequest({ type: "gryt-media-job", id: 5, job: { kind: "chatvideo", bytes: job.bytes } }), { id: 5, job: { kind: "chatvideo", bytes: job.bytes } });
+
 console.log("media sandbox: ok");

src/packages/socket/src/components/MessageAttachment.tsx

@@ -1,3 +1,5 @@
+import { Spinner } from "@gryt/ui";
+
 import { useSealedVideo } from "../hooks/useSealedVideo";
 import { useStableFileUrl } from "../hooks/useStableFileUrl";
 import { readBlobUrl } from "../utils/downloadFile";
@@ -34,6 +36,16 @@ export function MessageAttachment({
   const openLocal = local ? () => readBlobUrl(local) : undefined;
   const fileName = meta?.original_name;
 
+  if (meta?.refused) {
+    return (
+      <div className="text-xs text-gryt-muted" style={{ padding: "8px 0" }}>
+        {fileName ? `${fileName} couldn't be processed, so it isn't shown.` : "This file couldn't be processed, so it isn't shown."}
+      </div>
+    );
+  }
+  // Still with the server's image worker: fetching it now would just wait (GRYT-1669).
+  if (meta?.processing) return <ProcessingAttachment meta={meta} />;
+
   if (mime.startsWith("image/")) {
     const imgSrc = local || url;
     return (
@@ -133,3 +145,25 @@ function SealedVideoAttachment({
     </MessageContextMenu>
   );
 }
+
+/** A loader in the attachment's own shape, over the sender's local copy when there is one. */
+function ProcessingAttachment({ meta }: { meta: AttachmentMeta }) {
+  const w = meta.width && meta.height ? Math.min(320, meta.width) : 320;
+  const h = meta.width && meta.height ? Math.round((w * meta.height) / meta.width) : 180;
+  return (
+    <div
+      role="status"
+      aria-label="Processing attachment"
+      className="relative flex items-center justify-center overflow-hidden rounded-(--gryt-radius-md)"
+      style={{ width: w, height: Math.min(h, 360), background: "var(--gryt-neutral-3)" }}
+    >
+      {meta.local_url && meta.mime?.startsWith("image/") && (
+        <img src={meta.local_url} alt="" className="absolute inset-0 h-full w-full object-cover" style={{ opacity: 0.4 }} />
+      )}
+      <div className="relative flex items-center gap-2 text-xs">
+        <Spinner size={14} />
+        Processing…
+      </div>
+    </div>
+  );
+}

src/packages/socket/src/components/chatUtils.tsx

@@ -16,6 +16,10 @@ export type AttachmentMeta = {
   width: number | null;
   height: number | null;
   has_thumbnail: boolean;
+  /** The server's image worker still has it: drawn as a loader, not fetched (GRYT-1669). */
+  processing?: boolean;
+  /** The worker couldn't write it out, so the server never serves it. */
+  refused?: boolean;
   /** Blob URL of the file itself, when it is decrypted or still being sent. */
   local_url?: string;
   /** A sealed file not fetched yet. Calling it fetches and decrypts it. */

src/packages/socket/src/hooks/chatEventHandlers.ts

@@ -163,6 +163,26 @@ export function handleMessageEdited(
   }
 }
 
+// ── chat:attachments ─────────────────────────────────────────────────
+
+/** A message's attachments finished processing, or one was refused. Only the attachments
+    change; text and edit time stay as they are (GRYT-1669). */
+export function handleAttachmentsSettled(
+  payload: { conversation_id: string; message_id: string; enriched_attachments: ChatMessage["enriched_attachments"] },
+  activeConversationId: string,
+  getCacheKey: CacheKeyFn,
+  setMessageCache: Dispatch<SetStateAction<MessageCache>>,
+  setChatMessages: Dispatch<SetStateAction<ChatMessage[]>>,
+): void {
+  if (!payload?.conversation_id || !payload.message_id || !Array.isArray(payload.enriched_attachments)) return;
+  const key = getCacheKey(payload.conversation_id);
+  if (!key) return;
+  const apply = (msg: ChatMessage) =>
+    msg.message_id === payload.message_id ? { ...msg, enriched_attachments: payload.enriched_attachments } : msg;
+  setMessageCache((prev) => ({ ...prev, [key]: (prev[key] || []).map(apply) }));
+  if (payload.conversation_id === activeConversationId) setChatMessages((prev) => prev.map(apply));
+}
+
 // ── chat:deleted ─────────────────────────────────────────────────────
 
 export function handleMessageDeleted(

src/packages/socket/src/hooks/useChat.ts

@@ -22,6 +22,7 @@ import { sealedNotificationBody } from "../utils/sealedNotification";
 import {
   ChatErrorPayload,
   type ChatErrorRef,
+  handleAttachmentsSettled,
   handleChatErrorEvent,
   handleHistoryPayload,
   handleMessageDeleted,
@@ -504,6 +505,8 @@ export function useChat({
 
     const onEdited = (updatedMessage: ChatMessage) =>
       handleMessageEdited(updatedMessage, activeConversationId, cacheKeyFor, setMessageCache, setChatMessages);
+    const onAttachments = (payload: Parameters<typeof handleAttachmentsSettled>[0]) =>
+      handleAttachmentsSettled(payload, activeConversationId, cacheKeyFor, setMessageCache, setChatMessages);
 
     const onReportSubmitted = () => {
       toast.success("Report submitted");
@@ -582,6 +585,7 @@ export function useChat({
     currentConnection.on("chat:reaction", onReaction);
     currentConnection.on("chat:deleted", onDeleted);
     currentConnection.on("chat:edited", onEdited);
+    currentConnection.on("chat:attachments", onAttachments);
     currentConnection.on("report:submitted", onReportSubmitted);
     currentConnection.on("report:already_reported", onAlreadyReported);
     currentConnection.on("chat:purge_user", onPurgeUser);
@@ -594,6 +598,7 @@ export function useChat({
       currentConnection.off("chat:reaction", onReaction);
       currentConnection.off("chat:deleted", onDeleted);
       currentConnection.off("chat:edited", onEdited);
+      currentConnection.off("chat:attachments", onAttachments);
       currentConnection.off("report:submitted", onReportSubmitted);
       currentConnection.off("report:already_reported", onAlreadyReported);
       currentConnection.off("chat:purge_user", onPurgeUser);