feat: Log redaction, FIFO QOS for user vs API HTTP requests and http traffic accounting enrichment

Services/Bridges/LogBridge.cs

@@ -99,6 +99,9 @@ public static string[] ReadLog(int tail = 0, string? level = null, string? searc
             return Array.Empty<string>();
         }
 
+        for (var i = 0; i < allLines.Length; i++)
+            allLines[i] = LogRedactor.Reveal(allLines[i]);
+
         // Parse multi-level filter once
         string[]? levels = null;
         if (!string.IsNullOrEmpty(level))

Services/Configuration/FileLoggingSettings.cs

@@ -46,6 +46,15 @@ public class FileLoggingSettings
     /// </summary>
     public string LogLevel { get; set; } = "Information";
 
+    /// <summary>
+    /// Mask UPNs and customer domains in file and console output, keeping them readable at a glance
+    /// (see <c>LogRedactor</c>). <c>LogBridge</c> reveals them again. Env override: <c>CRAFT_LOG_REDACTION=false</c>.
+    /// </summary>
+    public bool Redact { get; set; } = true;
+
+    /// <summary>Extra domains (and their subdomains) left readable, e.g. the app's own.</summary>
+    public List<string> RedactAllowDomains { get; set; } = new();
+
     /// <summary>Resolved directory path, applying platform defaults when Directory is empty.</summary>
     internal string ResolvedDirectory => !string.IsNullOrEmpty(Directory)
         ? Directory

Services/Hosting/ApiEgressLimiterMiddleware.cs

@@ -41,10 +41,12 @@ public async Task InvokeAsync(HttpContext context)
     {
         ArgumentNullException.ThrowIfNull(context);
 
-        // UI and anonymous callers are never counted or capped — the feature is only about app-only
-        // automation. One header check and out, so interactive traffic pays essentially nothing.
+        // UI and anonymous callers are never capped — the cap is only about app-only automation. A
+        // signed-in user's traffic is still flagged so it is reported (never billed); anonymous is not.
         if (!CallerClassifier.IsApiClient(context))
         {
+            if (!string.IsNullOrEmpty(context.Request.Headers["x-ms-client-principal-name"].ToString()))
+                context.Items[ApiEgressWireCounterMiddleware.InteractiveItemKey] = true;
             await _next(context);
             return;
         }
@@ -59,7 +61,7 @@ public async Task InvokeAsync(HttpContext context)
         // concurrency cap, the overshoot is bounded to (concurrency × largest response).
         if (_ledger.ShouldReject())
         {
-            _ledger.RecordShed(appId);
+            _ledger.RecordShed(appId, context.GetRouteValue("endpoint") as string);
             await RejectAsync(context);
             return;
         }

Services/Hosting/ApiEgressWireCounterMiddleware.cs

@@ -34,6 +34,22 @@ public sealed class ApiEgressWireCounterMiddleware
     /// </summary>
     public const string ChargeItemKey = "Craft.Egress.Charge";
 
+    /// <summary>
+    /// Request item set by <see cref="ApiEgressLimiterMiddleware"/> on a request from an interactive
+    /// (signed-in user) caller: its bytes are recorded for reporting but never billed against the cap.
+    /// </summary>
+    public const string InteractiveItemKey = "Craft.Egress.Interactive";
+
+    /// <summary>Request item carrying the accounting label of a dispatched endpoint — see <see cref="TagEndpoint"/>.</summary>
+    public const string EndpointItemKey = "Craft.Egress.Endpoint";
+
+    /// <summary>
+    /// Optional response header a handler sets to sub-label its traffic (e.g. the Graph resource behind a
+    /// generic proxy endpoint, or the tool behind an MCP call). Consumed by <see cref="TagEndpoint"/> and
+    /// never sent to the client.
+    /// </summary>
+    public const string EndpointHeader = "X-Craft-Endpoint";
+
     private readonly RequestDelegate _next;
     private readonly EgressLedger _ledger;
 
@@ -62,7 +78,36 @@ public async Task InvokeAsync(HttpContext context)
         {
             context.Response.Body = original;
             if (context.Items.TryGetValue(ChargeItemKey, out var charge) && charge is string appId && appId.Length > 0)
-                _ledger.Record(counting.BytesWritten, appId);
+                _ledger.Record(counting.BytesWritten, appId, ResolveEndpoint(context), IsCacheHit(context), context.Response.StatusCode);
+            else if (context.Items.ContainsKey(InteractiveItemKey))
+                _ledger.RecordInteractive(counting.BytesWritten, ResolveEndpoint(context), IsCacheHit(context), context.Response.StatusCode);
         }
     }
+
+    /// <summary>
+    /// Labels a dispatched request for accounting: the endpoint name, suffixed with the handler's
+    /// <see cref="EndpointHeader"/> value when it set one (<c>ListGraphRequest:users</c>). The header is
+    /// removed so it never reaches the client. Call once the endpoint is known to exist and after the
+    /// handler's headers are applied — both the executed and the cache-hit paths.
+    /// </summary>
+    public static void TagEndpoint(HttpContext context, string endpoint)
+    {
+        ArgumentNullException.ThrowIfNull(context);
+        var tag = context.Response.Headers[EndpointHeader].ToString();
+        if (tag.Length > 0) context.Response.Headers.Remove(EndpointHeader);
+        context.Items[EndpointItemKey] = tag.Length > 0 ? $"{endpoint}:{tag}" : endpoint;
+    }
+
+    // Dispatched endpoints are tagged explicitly; any other matched /api route is a literal (native or
+    // built-in) path, labelled by its last segment. Unmatched requests stay unlabelled.
+    private static string? ResolveEndpoint(HttpContext context)
+    {
+        if (context.Items.TryGetValue(EndpointItemKey, out var tagged) && tagged is string label) return label;
+        if (context.GetEndpoint() is RouteEndpoint { RoutePattern: { Parameters.Count: 0, RawText: { } raw } })
+            return raw.TrimEnd('/').Split('/')[^1];
+        return null;
+    }
+
+    private static bool IsCacheHit(HttpContext context) =>
+        context.Response.Headers["X-Cache"].ToString().StartsWith("HIT", StringComparison.Ordinal);
 }

Services/Hosting/CraftHostBuilderExtensions.cs

@@ -160,6 +160,11 @@ public static LogLevel AddCraftLogging(this WebApplicationBuilder builder)
         builder.Configuration.GetSection("App:FileLogging").Bind(fileLoggingSettings);
         var level = fileLoggingSettings.ParsedLogLevel;
 
+        var redactEnv = Environment.GetEnvironmentVariable("CRAFT_LOG_REDACTION");
+        LogRedactor.Configure(
+            fileLoggingSettings.Redact && redactEnv is not ("0" or "false" or "False" or "FALSE"),
+            fileLoggingSettings.RedactAllowDomains);
+
         var fileLoggerProvider = new FileLoggerProvider(fileLoggingSettings, level);
         builder.Logging.AddProvider(fileLoggerProvider);
         LogBridge.Initialize(fileLoggerProvider);
@@ -169,10 +174,16 @@ public static LogLevel AddCraftLogging(this WebApplicationBuilder builder)
             options.TimestampFormat = "yyyy-MM-ddTHH:mm:ss.fffZ ";
             options.SingleLine = true;
         });
+        var services = builder.Logging.Services;
+        services.Remove(services.Single(d =>
+            d.ServiceType == typeof(ILoggerProvider) && d.ImplementationType == typeof(ConsoleLoggerProvider)));
+        services.AddSingleton<ConsoleLoggerProvider>();
+        services.AddSingleton<ILoggerProvider>(sp =>
+            new RedactingConsoleLoggerProvider(sp.GetRequiredService<ConsoleLoggerProvider>()));
 
         if (level > LogLevel.Debug)
         {
-            builder.Logging.AddFilter<ConsoleLoggerProvider>(l => l >= LogLevel.Information);
+            builder.Logging.AddFilter<RedactingConsoleLoggerProvider>(l => l >= LogLevel.Information);
 
             // Framework logging is noise at Information and above.
             builder.Logging.AddFilter("Microsoft.AspNetCore", LogLevel.Warning);