Feat: HTTP worker cancellation upon client closing connection, fix for large orchestration jobs failing to run and compression inprovements

Services/Hosting/CoalescingCompressionProvider.cs

@@ -0,0 +1,28 @@
+using Microsoft.AspNetCore.ResponseCompression;
+
+namespace Craft.Hosting;
+
+/// <summary>
+/// Feeds an encoder in 64 KiB blocks however the response is written, while still streaming.
+/// </summary>
+/// <remarks>
+/// A string body (<c>Response.WriteAsync</c>) or a <c>CraftResult.Stream</c> writer reaches the encoder
+/// through the response pipe, which hands it one ~4 KiB segment per write. Brotli's fast qualities
+/// compress each write as an isolated fragment, so a 300 KB ListLogs body at Fastest came out 85 KB
+/// instead of 29 KB, and every encoder paid per-call overhead (Brotli Optimal ~10-15% CPU). Coalescing to
+/// 64 KiB recovers the ratio (36 KB; exact for bodies up to 64 KiB) and the CPU, and output still leaves
+/// every 64 KiB — the body is never materialised. Flushes pass straight through, so an explicit flush
+/// (SSE, progressive output) behaves exactly as before; writes of 64 KiB or more bypass the buffer.
+/// </remarks>
+public sealed class CoalescingCompressionProvider(ICompressionProvider inner) : ICompressionProvider
+{
+    // Under the 85 KB large-object threshold, so the per-response buffer is a cheap gen0 allocation.
+    private const int BlockSize = 64 * 1024;
+
+    public string EncodingName => inner.EncodingName;
+    public bool SupportsFlush => inner.SupportsFlush;
+
+    // ponytail: BufferedStream allocates its 64 KiB buffer per compressed response (gen0). Swap for an
+    // ArrayPool-backed buffer if allocation rate ever shows up in the GC diagnostics.
+    public Stream CreateStream(Stream outputStream) => new BufferedStream(inner.CreateStream(outputStream), BlockSize);
+}

Services/Hosting/CraftHostBuilderExtensions.cs

@@ -223,15 +223,19 @@ public static IServiceCollection AddCraftResponseCompression(
         services.AddResponseCompression(options =>
         {
             options.EnableForHttps = true;
-            options.Providers.Add<BrotliCompressionProvider>();
-            options.Providers.Add<GzipCompressionProvider>();
+            // Order is preference: equal-q ties go to the earliest provider, so br wins over gzip.
+            // Every provider is coalesced — see CoalescingCompressionProvider.
+            ICompressionProvider[] providers =
+            [
+                new BrotliCompressionProvider(Options.Create(new BrotliCompressionProviderOptions { Level = level })),
+                new GzipCompressionProvider(Options.Create(new GzipCompressionProviderOptions { Level = level })),
+            ];
+            foreach (var provider in providers)
+                options.Providers.Add(new CoalescingCompressionProvider(provider));
             options.MimeTypes = ResponseCompressionDefaults.MimeTypes.Concat(
                 second);
         });
 
-        services.Configure<BrotliCompressionProviderOptions>(o => o.Level = level);
-        services.Configure<GzipCompressionProviderOptions>(o => o.Level = level);
-
         return services;
     }
 

Services/Hosting/Endpoints/PowerShellDispatchEndpoint.cs

@@ -100,8 +100,19 @@ await ServeCachedAsync(context, cache, psRunner, orchestrator, logger,
                 var result = await psRunner.ExecuteHttpScript(endpoint, context);
 
                 // Writes can invalidate anything, so they clear the cache regardless of endpoint name.
+                // Do this even on a client abort: a write may have partially applied before the
+                // connection dropped, so dropping the possibly-stale cache is the safe choice.
                 if (context.Request.Method != "GET") InvalidateForWrite(context, cache);
 
+                // Client hung up mid-request (navigated away / hit Cancel). The pipeline was already
+                // stopped and the worker reclaimed; there is no live connection to write to, and a
+                // partial result must not be cached. Bail before the cache write and response write.
+                if (context.RequestAborted.IsCancellationRequested)
+                {
+                    logger.LogInformation("[HTTP] /API/{Endpoint} cancelled by client", endpoint);
+                    return;
+                }
+
                 if (useCache && cacheKey is not null && result.StatusCode is >= 200 and < 400)
                     await cache.Set(cacheKey, result);
 

Services/PowerShellHost/PowerShellRunnerService.cs

@@ -228,10 +228,17 @@ public Dictionary<string, string> DiscoverHttpEndpoints()
     /// </summary>
     public async Task<ScriptResult> ExecuteHttpScript(string route, HttpContext httpContext)
     {
+        // Only reads honour client-disconnect cancellation. A write (POST/PUT/DELETE/PATCH) runs to
+        // completion even if the client hangs up — a half-applied mutation with nobody listening is
+        // worse than one that finished. Reads mutate nothing, so aborting them mid-flight is safe.
+        var clientAborted = HttpMethods.IsGet(httpContext.Request.Method)
+            ? httpContext.RequestAborted
+            : CancellationToken.None;
+
         if (!DispatchProfiler.Enabled)
         {
             var req = await BuildRequestObject(httpContext);
-            return await ExecuteHttpScriptInternal(route, req, isHttp: true);
+            return await ExecuteHttpScriptInternal(route, req, isHttp: true, clientAborted: clientAborted);
         }
 
         // Profiling path: time request marshaling + the runner-side segments (checkout/invoke/extract).
@@ -241,7 +248,8 @@ public async Task<ScriptResult> ExecuteHttpScript(string route, HttpContext http
         var request = await BuildRequestObject(httpContext);
         var marshalTicks = Stopwatch.GetTimestamp() - mStart;
         var timing = new DispatchTiming();
-        var result = await ExecuteHttpScriptInternal(route, request, isHttp: true, timing);
+        var result = await ExecuteHttpScriptInternal(route, request, isHttp: true, timing,
+            clientAborted: clientAborted);
         DispatchProfiler.Record(marshalTicks, timing.CheckoutTicks, timing.InvokeTicks, timing.ExtractTicks,
             Stopwatch.GetTimestamp() - totalStart);
         return result;
@@ -258,7 +266,7 @@ public async Task<ScriptResult> ExecuteHttpScript(string route, Hashtable reques
     }
 
     private async Task<ScriptResult> ExecuteHttpScriptInternal(string route, Hashtable request, bool isHttp,
-        DispatchTiming? timing = null)
+        DispatchTiming? timing = null, CancellationToken clientAborted = default)
     {
         var sw = Stopwatch.StartNew();
         var entry = _repo.GetByRoute(route);
@@ -365,9 +373,16 @@ private async Task<ScriptResult> ExecuteHttpScriptInternal(string route, Hashtab
             worker.Streams.Verbose.DataAdded += onVerbose;
 
             var timeoutSeconds = isHttp ? _workerSettings.HttpTimeoutSeconds : _workerSettings.BgTimeoutSeconds;
-            using var cts = timeoutSeconds > 0
+            using var timeoutCts = timeoutSeconds > 0
                 ? new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))
                 : null;
+            // Link the client-disconnect token (present on live HTTP requests) with the timeout so
+            // either one stops the pipeline. When the client hangs up — navigated away or hit Cancel —
+            // RequestAborted fires, InvokeAsync calls PowerShell.Stop(), and the worker is freed instead
+            // of paging on for a response nobody is waiting for. Background cache refresh passes default
+            // (no live client), so only the timeout applies there.
+            using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(
+                timeoutCts?.Token ?? CancellationToken.None, clientAborted);
 
             // When Scripts.HttpHandler is set, ALL HTTP routes dispatch through that single
             // function instead of invoking the route's function directly. The endpoint name
@@ -378,7 +393,7 @@ private async Task<ScriptResult> ExecuteHttpScriptInternal(string route, Hashtab
                 ? _scriptsSettings.HttpHandler
                 : entry.FunctionName;
             var invokeStart = timing != null ? Stopwatch.GetTimestamp() : 0;
-            var results = await worker.InvokeAsync(targetFunction, parameters, cts?.Token ?? default);
+            var results = await worker.InvokeAsync(targetFunction, parameters, linkedCts.Token);
             if (timing != null) timing.InvokeTicks = Stopwatch.GetTimestamp() - invokeStart;
 
             var extractStart = timing != null ? Stopwatch.GetTimestamp() : 0;
@@ -397,6 +412,17 @@ private async Task<ScriptResult> ExecuteHttpScriptInternal(string route, Hashtab
         catch (OperationCanceledException) when (sw.ElapsedMilliseconds > 0)
         {
             sw.Stop();
+
+            // Client hung up (navigated away / hit Cancel) vs the request exceeding its time budget.
+            // A cancel is normal and expected — log it quietly and return 499 (never actually written;
+            // the connection is gone) so the dispatcher skips caching this partial result.
+            if (clientAborted.IsCancellationRequested)
+            {
+                _logger.LogInformation("[{Pool}] {Function} cancelled by client after {Ms}ms",
+                    poolLabel, entry?.FunctionName ?? route, sw.ElapsedMilliseconds);
+                return new ScriptResult { StatusCode = 499, Body = string.Empty };
+            }
+
             var timeoutSeconds = isHttp ? _workerSettings.HttpTimeoutSeconds : _workerSettings.BgTimeoutSeconds;
             _logger.LogWarning("[{Pool}] {Function} timed out after {Ms}ms (limit: {Limit}s)",
                 poolLabel, entry?.FunctionName ?? route, sw.ElapsedMilliseconds, timeoutSeconds);