From c636ff9cb6bebc9146908e6ec69ecff3f9e6e8a8 Mon Sep 17 00:00:00 2001 From: "Jung Hyun, Nam" Date: Sun, 25 Oct 2026 22:09:48 +0900 Subject: [PATCH 1/2] =?UTF-8?q?AI=20=EC=9D=91=EB=8B=B5=EB=B3=84=20?= =?UTF-8?q?=EC=8A=A4=ED=82=AC=20=EB=B0=8F=20=EB=8F=84=EA=B5=AC=20=EC=B2=98?= =?UTF-8?q?=EB=A6=AC=20=EA=B8=B0=EB=A1=9D=20=ED=91=9C=EC=8B=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/SKILL_MARKETPLACE.ko.md | 10 +- docs/SKILL_MARKETPLACE.md | 10 +- .../ManagedAiResponseActivityView.cs | 118 ++++++++++ .../ManagedAi/ManagedAiWindow.cs | 50 +++- src/TableCloth.ManagedAi.Core/Contracts.cs | 10 +- .../OpenAiCodexProvider.cs | 6 +- .../Parsing/CodexJsonlParser.cs | 93 ++++++++ .../ActivityEvidenceTests.cs | 181 ++++++++++++++ .../ProviderTests.cs | 47 ++++ src/TableCloth.Test/ManagedAiWindowTests.cs | 221 +++++++++++++++++- 10 files changed, 729 insertions(+), 17 deletions(-) create mode 100644 src/TableCloth.App/ManagedAi/ManagedAiResponseActivityView.cs create mode 100644 src/TableCloth.ManagedAi.Test/ActivityEvidenceTests.cs diff --git a/docs/SKILL_MARKETPLACE.ko.md b/docs/SKILL_MARKETPLACE.ko.md index 63ebe6bf..bb1323e9 100644 --- a/docs/SKILL_MARKETPLACE.ko.md +++ b/docs/SKILL_MARKETPLACE.ko.md @@ -46,7 +46,13 @@ 스킬은 `%LOCALAPPDATA%\TableCloth\ManagedAi\profiles\openai-codex\codex-home\skills`에 보관합니다. 기존 가져오기 기능과 작업 잠금이 변경 작업을 제어합니다. 마켓플레이스는 이 프로필을 관리합니다. 사용자 전역 Codex 프로필과 스킬은 관리 범위에 포함하지 않습니다. [스킬 관리자](../src/TableCloth.ManagedAi.OpenAi/Skills/OpenAiCodexSkillManager.cs)와 [관리 경로](../src/TableCloth.ManagedAi.Core/ManagedAiPaths.cs)에서 저장 범위를 확인할 수 있습니다. -## 네이티브 Codex와 호스트 브리지의 실행 범위 +## 응답별 처리 기록과 실행 범위 + +각 응답 아래에 **스킬 및 도구 기록**을 표시합니다. 호스트 항목은 앱이 연동한 스킬 이름과 처리 완료, 실패 또는 완료 여부 미확인 상태를 표시합니다. MCP 항목은 Codex가 보고한 서버와 도구를 표시합니다. **MCP 응답 수신**은 도구 응답을 받았다는 뜻이며 스크립트의 성공이나 특정 스킬과의 연결을 입증하지 않습니다. [AI 창](../src/TableCloth.App/ManagedAi/ManagedAiWindow.cs)은 개별 응답에 해당 기록을 연결합니다. 메시지 복사 버튼은 응답의 마크다운 원문만 복사합니다. + +처리 기록으로 모델의 스킬 지침 참조 여부나 응답 반영 여부까지 확인하지는 못합니다. Codex 0.156.1의 [`exec --json` 항목 계약](https://github.com/openai/codex/blob/b412ff32c417f855c2b2d1581b77058eed87c84b/codex-rs/exec/src/exec_events.rs#L98)은 MCP 호출을 제공하지만 스킬 선택이나 지침 읽기 이벤트는 제공하지 않습니다. 따라서 기록이 없으면 확인된 스킬 실행 기록이 없다고 안내합니다. 스킬 미사용을 뜻하지는 않습니다. 설치 상태, 질문의 스킬 이름이나 모델이 작성한 사용 주장은 확인 근거에 포함하지 않습니다. 지침만 제공하는 스킬은 처리 기록 없이도 응답에 영향을 줄 수 있습니다. + +처리 메타데이터는 길이와 개수를 제한한 도구 식별자와 고정한 결과 구분만 보관합니다. 인수, 명령줄, 로컬 경로, 도구 결과와 오류 원문은 포함하지 않습니다. 일부 기록을 해석하거나 보관하지 못하면 기록이 불완전하다고 안내합니다. 앱은 현재 창의 메모리에서 기록을 관리하며 진단 로그나 다음 응답에 복사하지 않습니다. [JSONL 파서](../src/TableCloth.ManagedAi.OpenAi/Parsing/CodexJsonlParser.cs)에서 수집 범위를 확인할 수 있습니다. AI 사용 준비 과정에서는 대화와 검색에 사용하는 네이티브 Codex CLI를 설치합니다. Node.js와 Python은 설치하지 않습니다. 마켓플레이스에도 언어 런타임 설치, 버전 표시와 진단 기능을 제공하지 않습니다. [Codex 설치기](../src/TableCloth.ManagedAi.OpenAi/Distribution/OpenAiCodexPackageInstaller.cs)는 스킬 파일 설치기와 별도로 유지합니다. @@ -56,7 +62,7 @@ AI 사용 준비 과정에서는 대화와 검색에 사용하는 네이티브 C ## 자동 검증과 게시 경로의 구분 -자동 검증의 범위를 검토하겠습니다. 테스트는 검토 조건 없는 카탈로그 파싱, 다운로드 검증, 버전 추적, 교체 실패 시 복구, 제거와 뷰 모델 명령을 다룹니다. 백엔드 테스트는 합성 응답으로 실패와 복구 경로를 검사합니다. Avalonia 헤드리스 테스트는 언어, 테마와 창 크기를 달리해 마켓플레이스의 두 탭을 검사합니다. [스킬 테스트](../src/TableCloth.ManagedAi.Test/SkillMarketplaceTests.cs)와 [UI 테스트](../src/TableCloth.Test/SkillMarketplaceTests.cs)에 검증 항목을 기록했습니다. +자동 검증의 범위를 검토하겠습니다. 테스트는 검토 조건 없는 카탈로그 파싱, 다운로드 검증, 버전 추적, 교체 실패 시 복구, 제거와 뷰 모델 명령을 다룹니다. 백엔드 테스트는 합성 응답으로 실패와 복구 경로를 검사합니다. Avalonia 헤드리스 테스트는 언어, 테마와 창 크기를 달리해 마켓플레이스의 두 탭을 검사합니다. [스킬 테스트](../src/TableCloth.ManagedAi.Test/SkillMarketplaceTests.cs)와 [UI 테스트](../src/TableCloth.Test/SkillMarketplaceTests.cs)에 검증 항목을 기록했습니다. 처리 기록 검증은 MCP 이벤트 중복과 실패, 메타데이터 제한, 호스트 처리 결과, 응답 간 기록 분리와 실행 근거 없는 모델의 사용 주장을 다룹니다. [파서 테스트](../src/TableCloth.ManagedAi.Test)와 [대화 UI 테스트](../src/TableCloth.Test/ManagedAiWindowTests.cs)에서 사례를 확인할 수 있습니다. 테스트용 응답, 실제 저장소 다운로드와 네이티브 Codex 스킬 조회는 서로 다른 범위를 검증합니다. 테스트용 응답은 구현한 계약을 검사합니다. 실제 다운로드와 스킬 조회로는 설치 파일, 메타데이터와 활성 상태를 확인할 수 있습니다. 어느 결과도 모델의 스크립트 호출이나 외부 서비스, API 자격 증명, MCP 연결의 정상 동작을 입증하지 않습니다. 이 문서에서는 실제 서비스와 모델의 도구 실행을 검증한 것으로 안내하지 않습니다. 호스트 브리지도 로컬 CLI와 Windows 기능에 의존하므로 스킬 목록에 표시된 사실만으로 실제 호스트 작업의 성공을 판단하지 않습니다. diff --git a/docs/SKILL_MARKETPLACE.md b/docs/SKILL_MARKETPLACE.md index 1beb4abe..c4008362 100644 --- a/docs/SKILL_MARKETPLACE.md +++ b/docs/SKILL_MARKETPLACE.md @@ -46,7 +46,13 @@ A malformed `SKILL.md` remains visible with an invalid-manifest state. It can be Skills remain under `%LOCALAPPDATA%\TableCloth\ManagedAi\profiles\openai-codex\codex-home\skills`. TableCloth's existing importer and operation lock continue to control mutations. The marketplace manages this profile; the user's global Codex profile and skills remain outside its scope. See [the skill manager](../src/TableCloth.ManagedAi.OpenAi/Skills/OpenAiCodexSkillManager.cs) and [managed paths](../src/TableCloth.ManagedAi.Core/ManagedAiPaths.cs). -## Native Codex and host-bridge execution boundaries +## Response activity and execution boundaries + +Each assistant answer includes **Skill and tool activity** beneath its text. Host entries identify the skill whose application bridge ran and distinguish completed, failed, and unconfirmed operations. MCP entries identify the server and tool reported by Codex. **MCP response received** confirms a tool response, without claiming that a script succeeded or that the tool belongs to a particular skill. The [AI window](../src/TableCloth.App/ManagedAi/ManagedAiWindow.cs) keeps these records with the individual answer. Copying a message copies its original Markdown only. + +The footer also explains the limit of this evidence: it cannot confirm whether the model consulted skill instructions or reflected a result in its answer. Codex 0.156.1's [`exec --json` item contract](https://github.com/openai/codex/blob/b412ff32c417f855c2b2d1581b77058eed87c84b/codex-rs/exec/src/exec_events.rs#L98) exposes MCP calls but has no skill-selection or instruction-read event. An empty record therefore means no skill execution was confirmed, rather than proof that no skill was used. Installed state, a skill name in the question, and the model's own usage claim do not create evidence. Instruction-only skills can still affect an answer without producing an activity entry. + +Activity metadata contains bounded tool identities and fixed outcomes. It excludes arguments, command lines, local paths, tool results, and raw errors. Unsupported, conflicting, or excess MCP metadata produces an incomplete-evidence notice. Records remain in the current window's memory, do not enter the content-free diagnostic log, and do not carry over into the next answer. See the [JSONL parser](../src/TableCloth.ManagedAi.OpenAi/Parsing/CodexJsonlParser.cs). AI setup installs the native Codex CLI used for chat and search. It does not install Node.js or Python. The marketplace has no language-runtime installation, version, or diagnostic controls. The [Codex installer](../src/TableCloth.ManagedAi.OpenAi/Distribution/OpenAiCodexPackageInstaller.cs) remains separate from the skill-file installer. @@ -56,7 +62,7 @@ The certificate-expiry and Windows Sandbox integrations use application-owned ho ## Verification and publication boundaries -Automated checks cover catalog parsing without review gates, download validation, revision tracking, replacement rollback, removal, and view-model actions. Backend fixtures use synthetic responses to exercise failure and recovery paths. Headless Avalonia checks exercise the two marketplace tabs across languages, themes, and window sizes. See [skill tests](../src/TableCloth.ManagedAi.Test/SkillMarketplaceTests.cs) and [UI tests](../src/TableCloth.Test/SkillMarketplaceTests.cs). +Automated checks cover catalog parsing without review gates, download validation, revision tracking, replacement rollback, removal, and view-model actions. Backend fixtures use synthetic responses to exercise failure and recovery paths. Headless Avalonia checks exercise the two marketplace tabs across languages, themes, and window sizes. See [skill tests](../src/TableCloth.ManagedAi.Test/SkillMarketplaceTests.cs) and [UI tests](../src/TableCloth.Test/SkillMarketplaceTests.cs). Activity checks cover MCP lifecycle deduplication and failures, bounded metadata, host-operation outcomes, per-answer isolation, and model usage claims that have no execution evidence. See the [parser tests](../src/TableCloth.ManagedAi.Test) and [chat UI tests](../src/TableCloth.Test/ManagedAiWindowTests.cs). Fixture results, downloads from real repositories, and native Codex skill discovery establish different properties. Fixtures check the implemented contracts. Real downloads and discovery can confirm installed files, metadata, and enabled state. Neither establishes that the model invoked a script or that an external service, API credential, or MCP connection works. Actual service and model-tool execution remain unverified in this guide. The host bridges also depend on the local CLI and Windows capabilities; a skill listing alone does not establish successful host execution. diff --git a/src/TableCloth.App/ManagedAi/ManagedAiResponseActivityView.cs b/src/TableCloth.App/ManagedAi/ManagedAiResponseActivityView.cs new file mode 100644 index 00000000..dcce057a --- /dev/null +++ b/src/TableCloth.App/ManagedAi/ManagedAiResponseActivityView.cs @@ -0,0 +1,118 @@ +using Avalonia; +using Avalonia.Automation; +using Avalonia.Controls; +using Avalonia.Media; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; + +namespace TableCloth.ManagedAi; + +internal static class ManagedAiResponseActivityView +{ + public static Control Create(IReadOnlyList activities, bool evidenceIncomplete, bool responseCompleted) + { + var panel = new StackPanel { Spacing = 4, Margin = new Thickness(0, 6, 0, 0) }; + AutomationProperties.SetAutomationId(panel, "ManagedAiResponseActivity"); + panel.Children.Add(new TextBlock + { + Text = L("스킬 및 도구 기록", "Skill and tool activity"), FontWeight = FontWeight.SemiBold, + FontSize = 11, TextWrapping = TextWrapping.Wrap + }); + if (!responseCompleted) + panel.Children.Add(Text(L("응답을 완료하지 못했습니다. 아래 기록은 앱의 처리 내역만 나타냅니다.", + "The response did not complete. These records show app operations only."))); + if (activities.Count == 0) + panel.Children.Add(Text(L("확인된 스킬 실행 기록이 없습니다.", "No skill execution was confirmed."))); + else + { + var details = new StackPanel { Spacing = 3 }; + foreach (var activity in activities) + { + var name = Identifier(activity.Name); + var identity = activity.Kind == AiResponseActivityKind.McpTool + ? $"{Identifier(activity.Server)} / {name}" : name; + var row = Text($"{identity}: {OutcomeText(activity)}"); + AutomationProperties.SetAutomationId(row, "ManagedAiResponseActivityItem"); + details.Children.Add(row); + } + if (activities.Count > 3) + { + details.IsVisible = false; + // The application's theme does not provide an Expander template. + var disclosure = new Button + { + Content = L($"기록 {activities.Count}개 보기", $"View {activities.Count} activity records"), + FontSize = 11, Padding = new Thickness(6, 2), MinHeight = 24, + HorizontalAlignment = Avalonia.Layout.HorizontalAlignment.Left + }; + disclosure.Click += (_, _) => + { + details.IsVisible = !details.IsVisible; + disclosure.Content = details.IsVisible + ? L("기록 접기", "Hide activity records") + : L($"기록 {activities.Count}개 보기", $"View {activities.Count} activity records"); + }; + AutomationProperties.SetAutomationId(disclosure, "ManagedAiResponseActivityDetails"); + AutomationProperties.SetAutomationId(details, "ManagedAiResponseActivityItems"); + panel.Children.Add(disclosure); + panel.Children.Add(details); + } + else panel.Children.Add(details); + } + if (evidenceIncomplete) + panel.Children.Add(Text(L("일부 처리 기록을 표시하지 못했거나 완료 여부를 확인하지 못했습니다.", + "Some activity records could not be displayed or their completion confirmed."))); + var limit = L("스킬 지침 참조 여부와 응답 반영 여부는 확인할 수 없습니다.", + "Skill instruction use and its effect on the answer cannot be verified."); + if (activities.Any(activity => activity.Kind == AiResponseActivityKind.McpTool)) + limit += L(" MCP 기록만으로 스킬 사용을 확인할 수는 없습니다.", " MCP activity alone does not confirm skill use."); + panel.Children.Add(Text(limit)); + return panel; + } + + public static AiResponseActivityOutcome SandboxOutcome(SandboxCliRequest request, string report) + { + // Inspect only the app-owned status envelope. Output, paths, and identifiers never enter the activity UI. + try + { + using var json = JsonDocument.Parse(report, new JsonDocumentOptions { MaxDepth = 8 }); + var root = json.RootElement; + if (root.ValueKind != JsonValueKind.Object || + !root.TryGetProperty("Action", out var action) || action.ValueKind != JsonValueKind.String || + action.GetString() != request.Action.ToString().ToLowerInvariant() || + !root.TryGetProperty("Status", out var status) || status.ValueKind != JsonValueKind.String) + return AiResponseActivityOutcome.Incomplete; + return status.GetString() switch + { + "succeeded" when root.TryGetProperty("ExitCode", out var exitCode) && exitCode.TryGetInt32(out var code) && code == 0 + => AiResponseActivityOutcome.Completed, + "failed" or "unavailable" or "list-failed" => AiResponseActivityOutcome.Failed, + _ => AiResponseActivityOutcome.Incomplete + }; + } + catch (Exception ex) when (ex is JsonException or InvalidOperationException) + { return AiResponseActivityOutcome.Incomplete; } + } + + private static string OutcomeText(AiResponseActivity activity) => (activity.Kind, activity.Outcome) switch + { + (AiResponseActivityKind.HostSkill, AiResponseActivityOutcome.Completed) + => L("앱 처리 완료", "App operation completed"), + (AiResponseActivityKind.HostSkill, AiResponseActivityOutcome.Failed) + => L("앱 처리 실패", "App operation failed"), + (AiResponseActivityKind.McpTool, AiResponseActivityOutcome.Completed) + => L("MCP 응답 수신", "MCP response received"), + (AiResponseActivityKind.McpTool, AiResponseActivityOutcome.Failed) + => L("MCP 호출 실패", "MCP call failed"), + _ => L("완료 확인 불가", "Completion unconfirmed") + }; + + private static string Identifier(string? value) => !string.IsNullOrEmpty(value) && value.Length <= 128 && + Array.TrueForAll(value.ToCharArray(), c => char.IsAsciiLetterOrDigit(c) || c is '-' or '_' or '.') + ? value : L("식별 정보 없음", "Identity unavailable"); + + private static TextBlock Text(string value) => new() { Text = value, TextWrapping = TextWrapping.Wrap, FontSize = 11 }; + private static string L(string korean, string english) => ManagedAiText.Select(korean, english); +} diff --git a/src/TableCloth.App/ManagedAi/ManagedAiWindow.cs b/src/TableCloth.App/ManagedAi/ManagedAiWindow.cs index 1fdfe4df..81c958c0 100644 --- a/src/TableCloth.App/ManagedAi/ManagedAiWindow.cs +++ b/src/TableCloth.App/ManagedAi/ManagedAiWindow.cs @@ -454,7 +454,8 @@ private static Border CreatePrivacyNotice() return notice; } - private void AddMessage(string author, string content, bool user = false) + private void AddMessage(string author, string content, bool user = false, + AiChatResponse? response = null, IReadOnlyList? failedActivities = null) { var body = new StackPanel { Spacing = 8 }; var heading = new Grid { ColumnDefinitions = new ColumnDefinitions("*,Auto"), ColumnSpacing = 12 }; @@ -481,6 +482,10 @@ private void AddMessage(string author, string content, bool user = false) body.Children.Add(heading); body.Children.Add(new ChatMarkdownView(content, async link => await RunAsync(token => _session.OpenLinkAsync(link, token)))); + if (response is not null) + body.Children.Add(ManagedAiResponseActivityView.Create(response.Activities, response.ActivityEvidenceIncomplete, responseCompleted: true)); + else if (failedActivities is { Count: > 0 }) + body.Children.Add(ManagedAiResponseActivityView.Create(failedActivities, evidenceIncomplete: true, responseCompleted: false)); var bubble = new Border { Child = body, Padding = new Thickness(16, 12), CornerRadius = new CornerRadius(12), MaxWidth = 690, @@ -515,6 +520,7 @@ private async Task SendAsync() } var certificateLookup = _certificateSkillEnabled && CertificateExpiryIntent.Matches(text); var sandboxRequest = _sandboxSkillEnabled ? WindowsSandboxIntent.Parse(text) : null; + var localActivities = new List(); await RunAsync(async token => { if (certificateLookup && _messages.DisplayQuestion( @@ -539,14 +545,17 @@ await RunAsync(async token => { _status.Text = L("로컬 인증서 만료일을 확인하고 있습니다.", "Checking local certificate expiry dates."); if (_pendingStatus is not null) _pendingStatus.Text = _status.Text; - localReport = await RunBackendAsync(_certificateBridge.GetExpiryReportAsync, token); + localReport = await RunHostSkillAsync("tablecloth-certificate-expiry", _certificateBridge.GetExpiryReportAsync, + _ => AiResponseActivityOutcome.Completed, token); } if (sandboxRequest is not null) { _status.Text = sandboxRequest.SafeMode ? SafeModeText.Status : L("Windows Sandbox 명령을 실행하고 있습니다.", "Running the Windows Sandbox command."); if (_pendingStatus is not null) _pendingStatus.Text = _status.Text; - localSandboxReport = await RunBackendAsync(ct => _sandboxBridge.ExecuteAsync(sandboxRequest, ct), token); + localSandboxReport = await RunHostSkillAsync("tablecloth-windows-sandbox", + ct => _sandboxBridge.ExecuteAsync(sandboxRequest, ct), + report => ManagedAiResponseActivityView.SandboxOutcome(sandboxRequest, report), token); } var progress = Progress(); AiChatResponse response; @@ -561,12 +570,34 @@ await RunAsync(async token => throw; } RemovePending(); - AddMessage($"{L("식탁보", "TableCloth")} / {response.Model ?? model.Id}", response.Text); + response = response with { Activities = localActivities.Concat(response.Activities).ToArray() }; + AddMessage($"{L("식탁보", "TableCloth")} / {response.Model ?? model.Id}", response.Text, response: response); _status.Text = response.SearchCalls > 0 ? L($"응답을 완료했습니다. 웹 검색 {response.SearchCalls}회", $"Response complete. {response.SearchCalls} web searches.") : L("응답을 완료했습니다.", "Response complete."); - }, errorInChat: true); + }, errorInChat: true, failureActivities: () => localActivities); _input.Focus(); + + async Task RunHostSkillAsync(string id, Func> execute, + Func outcome, CancellationToken token) + { + try + { + var report = await RunBackendAsync(execute, token); + localActivities.Add(new(AiResponseActivityKind.HostSkill, id, outcome(report))); + return report; + } + catch (OperationCanceledException) + { + localActivities.Add(new(AiResponseActivityKind.HostSkill, id, AiResponseActivityOutcome.Incomplete)); + throw; + } + catch + { + localActivities.Add(new(AiResponseActivityKind.HostSkill, id, AiResponseActivityOutcome.Failed)); + throw; + } + } } private void SanitizeInput() @@ -606,7 +637,8 @@ private async Task SendWithBusyRetryAsync(Func action, bool errorInChat = false, bool keepSettingsOpen = false) + private async Task RunAsync(Func action, bool errorInChat = false, bool keepSettingsOpen = false, + Func>? failureActivities = null) { if (_operation is not null) return; if (!keepSettingsOpen) _settingsOverlay.IsVisible = false; @@ -645,7 +677,11 @@ private async Task RunAsync(Func action, bool errorInCh UpdateEnabled(); if (_closing) Close(); } - void ShowFailure(string message) { _status.Text = message; if (errorInChat) AddMessage(L("안내", "Notice"), message); } + void ShowFailure(string message) + { + _status.Text = message; + if (errorInChat) AddMessage(L("안내", "Notice"), message, failedActivities: failureActivities?.Invoke()); + } } // Backend methods can perform synchronous filesystem and process setup before their first await. diff --git a/src/TableCloth.ManagedAi.Core/Contracts.cs b/src/TableCloth.ManagedAi.Core/Contracts.cs index aa3c2bb7..528f27ba 100644 --- a/src/TableCloth.ManagedAi.Core/Contracts.cs +++ b/src/TableCloth.ManagedAi.Core/Contracts.cs @@ -49,7 +49,15 @@ public sealed record AiClientContext(Version? InstalledVersion, Uri OfficialHome public sealed record AiChatRequest(string Message, IReadOnlyList History, string? Model = null, string? LocalCertificateReport = null, string? LocalWindowsSandboxReport = null, AiClientContext? ClientContext = null); -public sealed record AiChatResponse(string Text, DateTimeOffset RetrievedAtUtc, int SearchCalls, string? Model = null); +public enum AiResponseActivityKind { HostSkill, McpTool } +public enum AiResponseActivityOutcome { Completed, Failed, Incomplete } +public sealed record AiResponseActivity(AiResponseActivityKind Kind, string Name, + AiResponseActivityOutcome Outcome, string? Server = null); +public sealed record AiChatResponse(string Text, DateTimeOffset RetrievedAtUtc, int SearchCalls, string? Model = null) +{ + public IReadOnlyList Activities { get; init; } = []; + public bool ActivityEvidenceIncomplete { get; init; } +} public sealed record AiModel(string Id, string DisplayName, bool IsDefault = false) { public AiModelTokenCost? TokenCost { get; init; } diff --git a/src/TableCloth.ManagedAi.OpenAi/OpenAiCodexProvider.cs b/src/TableCloth.ManagedAi.OpenAi/OpenAiCodexProvider.cs index e07e7eeb..734ad4d8 100644 --- a/src/TableCloth.ManagedAi.OpenAi/OpenAiCodexProvider.cs +++ b/src/TableCloth.ManagedAi.OpenAi/OpenAiCodexProvider.cs @@ -18,7 +18,11 @@ public async Task ChatAsync(AiChatRequest request, IProgress new AiChatResponse(parser.CompleteText(exitCode), DateTimeOffset.UtcNow, parser.SearchCalls, request.Model), progress, cancellationToken, request.Model); + (parser, exitCode) => new AiChatResponse(parser.CompleteText(exitCode), DateTimeOffset.UtcNow, parser.SearchCalls, request.Model) + { + Activities = parser.Activities, + ActivityEvidenceIncomplete = parser.ActivityEvidenceIncomplete + }, progress, cancellationToken, request.Model); } private async Task RunAsync(string prompt, bool structured, Func complete, diff --git a/src/TableCloth.ManagedAi.OpenAi/Parsing/CodexJsonlParser.cs b/src/TableCloth.ManagedAi.OpenAi/Parsing/CodexJsonlParser.cs index 6a9f16a3..0be2cb32 100644 --- a/src/TableCloth.ManagedAi.OpenAi/Parsing/CodexJsonlParser.cs +++ b/src/TableCloth.ManagedAi.OpenAi/Parsing/CodexJsonlParser.cs @@ -4,10 +4,16 @@ namespace TableCloth.ManagedAi.OpenAi; public sealed class CodexJsonlParser { + private const int MaxActivities = 128; + private const int MaxIdentityLength = 128; + private readonly List _activities = []; + private readonly Dictionary _activityIndexes = new(StringComparer.Ordinal); public int EventCount { get; private set; } public int UnknownEvents { get; private set; } public int SearchCalls { get; private set; } public string Stage { get; private set; } = "Thinking"; + public IReadOnlyList Activities => Array.AsReadOnly(_activities.ToArray()); + public bool ActivityEvidenceIncomplete { get; private set; } private bool _completed; private bool _failed; private string? _lastMessage; @@ -45,10 +51,14 @@ public void Accept(string line) !item.TryGetProperty("type", out var itemType)) break; if (itemType.GetString() == "web_search") { SearchCalls++; Stage = "Thinking"; } if (itemType.GetString() == "agent_message") { _lastMessage = item.GetProperty("text").GetString(); Stage = "Writing"; } + if (itemType.GetString() == "mcp_tool_call") ReadMcpActivity(item, completed: true); break; case "item.started": case "item.updated": if (root.TryGetProperty("item", out var active) && active.TryGetProperty("type", out var activeType)) + { Stage = activeType.GetString() switch { "web_search" => "Searching", "agent_message" => "Writing", _ => "Thinking" }; + if (activeType.GetString() == "mcp_tool_call") ReadMcpActivity(active, completed: false); + } break; case "thread.started": case "turn.started": break; default: UnknownEvents++; break; @@ -74,6 +84,89 @@ public string CompleteText(int exitCode) return _lastMessage; } + private void ReadMcpActivity(JsonElement item, bool completed) + { + // Provider event metadata is evidence of a tool exchange, not proof that an answer used a skill. + // Retain no arguments, output, errors, paths, or model-supplied attribution. + if (!TryReadIdentity(item, "id", out var id) || !TryReadIdentity(item, "server", out var server) || + !TryReadIdentity(item, "tool", out var tool)) + { + ActivityEvidenceIncomplete = true; + return; + } + + var outcome = ReadMcpOutcome(item, completed); + if (_activityIndexes.TryGetValue(id, out var index)) + { + var previous = _activities[index]; + if (previous.Name != tool || previous.Server != server) + { + ActivityEvidenceIncomplete = true; + return; + } + // A late start/update or an incomplete duplicate cannot erase a terminal result. + if (completed && previous.Outcome != AiResponseActivityOutcome.Incomplete && + outcome != AiResponseActivityOutcome.Incomplete && previous.Outcome != outcome) + ActivityEvidenceIncomplete = true; + if (previous.Outcome == AiResponseActivityOutcome.Failed || outcome == AiResponseActivityOutcome.Incomplete) return; + _activities[index] = previous with { Outcome = outcome }; + return; + } + if (_activities.Count >= MaxActivities) + { + ActivityEvidenceIncomplete = true; + return; + } + _activityIndexes.Add(id, _activities.Count); + _activities.Add(new(AiResponseActivityKind.McpTool, tool, outcome, server)); + } + + private AiResponseActivityOutcome ReadMcpOutcome(JsonElement item, bool completed) + { + var status = item.TryGetProperty("status", out var statusValue) && statusValue.ValueKind == JsonValueKind.String + ? statusValue.GetString() : null; + if (status == "failed" || item.TryGetProperty("error", out var error) && error.ValueKind != JsonValueKind.Null) + return AiResponseActivityOutcome.Failed; + + var hasResult = item.TryGetProperty("result", out var result) && result.ValueKind == JsonValueKind.Object; + if (hasResult) + { + // Codex 0.156.1 translates MCP isError into status=failed and omits it from the result. + // Also accept the protocol flag defensively if a future event variant retains it. + foreach (var field in new[] { "isError", "is_error" }) + { + if (!result.TryGetProperty(field, out var isError)) continue; + if (isError.ValueKind == JsonValueKind.True) return AiResponseActivityOutcome.Failed; + if (isError.ValueKind != JsonValueKind.False) + { + ActivityEvidenceIncomplete = true; + return AiResponseActivityOutcome.Incomplete; + } + } + } + if (!completed) + { + if (status != "in_progress") ActivityEvidenceIncomplete = true; + return AiResponseActivityOutcome.Incomplete; + } + if (status == "completed" && hasResult && result.TryGetProperty("content", out var content) && + content.ValueKind == JsonValueKind.Array) + return AiResponseActivityOutcome.Completed; + ActivityEvidenceIncomplete = true; + return AiResponseActivityOutcome.Incomplete; + } + + private static bool TryReadIdentity(JsonElement item, string field, out string value) + { + value = string.Empty; + if (!item.TryGetProperty(field, out var property) || property.ValueKind != JsonValueKind.String) return false; + var text = property.GetString(); + if (string.IsNullOrEmpty(text) || text.Length > MaxIdentityLength || + text.Any(c => !char.IsAsciiLetterOrDigit(c) && c is not '_' and not '-' and not '.')) return false; + value = text; + return true; + } + private void ReadFailure(JsonElement root) { if (root.TryGetProperty("message", out var message) && message.ValueKind == JsonValueKind.String) diff --git a/src/TableCloth.ManagedAi.Test/ActivityEvidenceTests.cs b/src/TableCloth.ManagedAi.Test/ActivityEvidenceTests.cs new file mode 100644 index 00000000..d403ecc3 --- /dev/null +++ b/src/TableCloth.ManagedAi.Test/ActivityEvidenceTests.cs @@ -0,0 +1,181 @@ +using System.Text.Json; +using TableCloth.ManagedAi.OpenAi; + +namespace TableCloth.ManagedAi.Test; + +[TestClass] +public sealed class ActivityEvidenceTests +{ + [TestMethod] + public void McpLifecycleKeepsOneRecordAndOnlySafeIdentityMetadata() + { + var parser = new CodexJsonlParser(); + parser.Accept(Mcp("item.started", "item_1", "in_progress")); + Assert.AreEqual(AiResponseActivityOutcome.Incomplete, parser.Activities.Single().Outcome); + parser.Accept(Mcp("item.updated", "item_1", "in_progress")); + parser.Accept(Mcp("item.completed", "item_1", "completed", new + { + content = new[] { new { type = "text", text = "private result C:\\private\\file.txt" } }, + structured_content = new { access_token = "private structured result" } + })); + parser.Accept(Mcp("item.started", "item_1", "in_progress")); + parser.Accept(Mcp("item.completed", "item_1", "completed", new { content = Array.Empty() })); + + var activity = parser.Activities.Single(); + Assert.AreEqual(new AiResponseActivity(AiResponseActivityKind.McpTool, "calculator-tool", + AiResponseActivityOutcome.Completed, "fixture_server"), activity); + Assert.IsFalse(parser.ActivityEvidenceIncomplete); + var retained = JsonSerializer.Serialize(parser.Activities); + Assert.DoesNotContain("private", retained); + Assert.DoesNotContain("file.txt", retained); + Assert.DoesNotContain("access_token", retained); + Assert.DoesNotContain("arguments", retained); + } + + [TestMethod] + public void McpFailuresAreDistinctFromCompletedExchangesAndDoNotRegress() + { + var parser = new CodexJsonlParser(); + parser.Accept(Mcp("item.completed", "item_failed", "failed", new { content = Array.Empty() })); + parser.Accept(Mcp("item.completed", "item_error", "completed", new { content = Array.Empty() }, + new { message = "private transport failure" })); + parser.Accept(Mcp("item.completed", "item_isError", "completed", new { content = Array.Empty(), isError = true })); + parser.Accept(Mcp("item.completed", "item_is_error", "completed", new { content = Array.Empty(), is_error = true })); + parser.Accept(Mcp("item.started", "item_failed", "in_progress")); + parser.Accept(Mcp("item.completed", "item_failed", "completed", new { content = Array.Empty() })); + + Assert.HasCount(4, parser.Activities); + Assert.IsTrue(parser.Activities.All(x => x.Outcome == AiResponseActivityOutcome.Failed)); + Assert.DoesNotContain("private", JsonSerializer.Serialize(parser.Activities)); + } + + [TestMethod] + public void ContradictoryIdentityOrTerminalStatusMarksEvidenceIncomplete() + { + var parser = new CodexJsonlParser(); + parser.Accept(Mcp("item.completed", "item_1", "completed", new { content = Array.Empty() })); + parser.Accept(Mcp("item.completed", "item_1", "failed", tool: "different-tool")); + Assert.AreEqual("calculator-tool", parser.Activities.Single().Name); + Assert.AreEqual(AiResponseActivityOutcome.Completed, parser.Activities.Single().Outcome); + Assert.IsTrue(parser.ActivityEvidenceIncomplete); + + parser.Accept(Mcp("item.completed", "item_1", "failed")); + Assert.AreEqual(AiResponseActivityOutcome.Failed, parser.Activities.Single().Outcome); + } + + [TestMethod] + [DataRow("completed", "failed")] + [DataRow("failed", "completed")] + public void ConflictingTerminalOutcomesKeepFailureAndMarkEvidenceIncomplete(string first, string second) + { + var parser = new CodexJsonlParser(); + parser.Accept(Mcp("item.completed", "item_1", first, new { content = Array.Empty() })); + Assert.IsFalse(parser.ActivityEvidenceIncomplete); + parser.Accept(Mcp("item.completed", "item_1", second, new { content = Array.Empty() })); + Assert.AreEqual(AiResponseActivityOutcome.Failed, parser.Activities.Single().Outcome); + Assert.IsTrue(parser.ActivityEvidenceIncomplete); + } + + [TestMethod] + public void StartedOrUpdatedWithoutTerminalEventRemainsIncomplete() + { + var parser = new CodexJsonlParser(); + parser.Accept(Mcp("item.started", "item_1", "in_progress")); + parser.Accept(Mcp("item.updated", "item_1", "in_progress")); + Assert.AreEqual(AiResponseActivityOutcome.Incomplete, parser.Activities.Single().Outcome); + Assert.IsFalse(parser.ActivityEvidenceIncomplete); + } + + [TestMethod] + [DataRow("{}")] + [DataRow("{\"content\":null}")] + [DataRow("{\"content\":{}}")] + [DataRow("{\"content\":[],\"isError\":\"true\"}")] + [DataRow("null")] + public void MissingOrMalformedResultCannotProveCompletedExchange(string result) + { + var parser = new CodexJsonlParser(); + parser.Accept(Mcp("item.completed", "item_1", "completed", JsonSerializer.Deserialize(result))); + Assert.AreEqual(AiResponseActivityOutcome.Incomplete, parser.Activities.Single().Outcome); + Assert.IsTrue(parser.ActivityEvidenceIncomplete); + } + + [TestMethod] + [DataRow(null)] + [DataRow("future_status")] + [DataRow("in_progress")] + public void MissingOrUnknownTerminalStatusCannotProveCompletedExchange(string? status) + { + var parser = new CodexJsonlParser(); + parser.Accept(Mcp("item.completed", "item_1", status, new { content = Array.Empty() })); + Assert.AreEqual(AiResponseActivityOutcome.Incomplete, parser.Activities.Single().Outcome); + Assert.IsTrue(parser.ActivityEvidenceIncomplete); + } + + [TestMethod] + [DataRow("tool", "C:\\private\\skill.py")] + [DataRow("server", "https://user:password@example.com")] + [DataRow("tool", "tool\nforged line")] + [DataRow("tool", "스킬")] + [DataRow("id", "")] + public void UnsafeOrMissingIdentitiesAreSkippedWithoutRejectingTheAnswer(string field, string value) + { + var item = new Dictionary + { + ["type"] = "mcp_tool_call", ["id"] = "item_1", ["server"] = "fixture_server", ["tool"] = "calculator-tool", + ["status"] = "completed", ["result"] = new { content = Array.Empty() } + }; + item[field] = value; + var parser = new CodexJsonlParser(); + parser.Accept(JsonSerializer.Serialize(new { type = "item.completed", item })); + CompleteAnswer(parser); + Assert.HasCount(0, parser.Activities); + Assert.IsTrue(parser.ActivityEvidenceIncomplete); + Assert.AreEqual("Fixture answer", parser.CompleteText(0)); + } + + [TestMethod] + public void ActivityCountAndIdentityLengthsAreBoundedAndSnapshotsStayStable() + { + var parser = new CodexJsonlParser(); + parser.Accept(Mcp("item.completed", "long_tool", "completed", new { content = Array.Empty() }, tool: new string('a', 129))); + Assert.HasCount(0, parser.Activities); + for (var i = 0; i < 129; i++) + parser.Accept(Mcp("item.started", $"item_{i}", "in_progress")); + var before = parser.Activities; + Assert.HasCount(128, before); + Assert.IsTrue(parser.ActivityEvidenceIncomplete); + parser.Accept(Mcp("item.completed", "item_0", "completed", new { content = Array.Empty() })); + Assert.AreEqual(AiResponseActivityOutcome.Incomplete, before[0].Outcome); + Assert.AreEqual(AiResponseActivityOutcome.Completed, parser.Activities[0].Outcome); + } + + [TestMethod] + public void ModelAssertionsInstalledSkillsAndUnrelatedToolsNeverBecomeSkillEvidence() + { + var parser = new CodexJsonlParser(); + parser.Accept("{\"type\":\"item.completed\",\"item\":{\"type\":\"command_execution\",\"command\":\"python skill.py\",\"exit_code\":0}}"); + parser.Accept("{\"type\":\"item.completed\",\"item\":{\"type\":\"web_search\"}}"); + parser.Accept("{\"type\":\"item.completed\",\"item\":{\"type\":\"agent_message\",\"text\":\"I used tablecloth-calculators. used_skills=[calculator]\"}}"); + parser.Accept("{\"type\":\"turn.completed\"}"); + Assert.Contains("used_skills", parser.CompleteText(0)); + Assert.HasCount(0, parser.Activities); + Assert.IsFalse(parser.ActivityEvidenceIncomplete); + Assert.HasCount(0, new CodexJsonlParser().Activities); + Assert.HasCount(0, new AiChatResponse("answer", DateTimeOffset.UtcNow, 0).Activities); + } + + private static string Mcp(string type, string id, string? status, object? result = null, object? error = null, + string tool = "calculator-tool") => JsonSerializer.Serialize(new + { + type, + item = new { type = "mcp_tool_call", id, server = "fixture_server", tool, status, + arguments = new { access_token = "private argument", path = "C:\\private\\file.txt" }, result, error } + }); + + private static void CompleteAnswer(CodexJsonlParser parser) + { + parser.Accept("{\"type\":\"item.completed\",\"item\":{\"type\":\"agent_message\",\"text\":\"Fixture answer\"}}"); + parser.Accept("{\"type\":\"turn.completed\"}"); + } +} diff --git a/src/TableCloth.ManagedAi.Test/ProviderTests.cs b/src/TableCloth.ManagedAi.Test/ProviderTests.cs index cfd3788f..d96e1ca9 100644 --- a/src/TableCloth.ManagedAi.Test/ProviderTests.cs +++ b/src/TableCloth.ManagedAi.Test/ProviderTests.cs @@ -86,9 +86,55 @@ public async Task NativeCodexRequestsLeaveFormerSkillToolInstallationsInactiveAn Assert.HasCount(1, Directory.GetFileSystemEntries(directory!)); } + [TestMethod] + public async Task ChatAttachesObservedActivitiesWithoutRetainingPayloadsOrLeakingAcrossTurns() + { + using var fixture = new InstallFixture(); + await fixture.Installer.InstallAsync(null, null, CancellationToken.None); + var runner = new ResponseRunner("Fixture answer", fixture.Paths) + { + ActivityEvents = + [ + JsonSerializer.Serialize(new { type = "item.completed", item = new + { + type = "mcp_tool_call", id = "item_1", server = "fixture_server", tool = "fixture_tool", status = "completed", + arguments = new { secret = "private argument" }, result = new { content = new[] { new { type = "text", text = "private result" } } }, error = (object?)null + } }), + JsonSerializer.Serialize(new { type = "item.completed", item = new + { + type = "mcp_tool_call", id = "item_2", server = "unsafe/server", tool = "fixture_tool", status = "failed" + } }) + ] + }; + var profile = new WindowsManagedAiProfile(fixture.Paths); + var auth = new OpenAiCodexAuthManager(fixture.Paths, fixture.Installer, profile, runner); + var skills = new OpenAiCodexSkillManager(fixture.Paths, fixture.Installer, profile, runner); + var provider = new OpenAiCodexProvider(fixture.Paths, fixture.Installer, auth, profile, runner, new(), new(fixture.Paths), skills); + + var first = await provider.ChatAsync(new("fixture", []), null, CancellationToken.None); + Assert.AreEqual(new AiResponseActivity(AiResponseActivityKind.McpTool, "fixture_tool", AiResponseActivityOutcome.Completed, + "fixture_server"), first.Activities.Single()); + Assert.IsTrue(first.ActivityEvidenceIncomplete); + Assert.DoesNotContain("private", JsonSerializer.Serialize(first)); + + runner.ActivityEvents = []; + var second = await provider.ChatAsync(new("followup", []), null, CancellationToken.None); + Assert.HasCount(0, second.Activities); + Assert.IsFalse(second.ActivityEvidenceIncomplete); + Assert.HasCount(1, first.Activities); + foreach (var log in Directory.GetFiles(fixture.Paths.Logs)) + { + var text = File.ReadAllText(log); + Assert.DoesNotContain("private", text); + Assert.DoesNotContain("fixture_tool", text); + Assert.DoesNotContain("fixture_server", text); + } + } + private sealed class ResponseRunner(string message, ManagedAiPaths paths) : IJsonlProcessRunner { public ProcessRunSpec? Invocation; + public IReadOnlyList ActivityEvents { get; set; } = []; public string ExternalManifest => Path.Combine(paths.Root, "external", "SKILL.md"); public Task RunAsync(ProcessRunSpec spec, Action stdout, Action? stderr, CancellationToken cancellationToken) { @@ -112,6 +158,7 @@ public Task RunAsync(ProcessRunSpec spec, Action stdout, else { Invocation = spec; + foreach (var activity in ActivityEvents) stdout(activity); stdout("{\"type\":\"item.completed\",\"item\":{\"type\":\"web_search\"}}"); stdout(JsonSerializer.Serialize(new { type = "item.completed", item = new { type = "agent_message", text = message } })); stdout("{\"type\":\"turn.completed\"}"); diff --git a/src/TableCloth.Test/ManagedAiWindowTests.cs b/src/TableCloth.Test/ManagedAiWindowTests.cs index c974455a..2af65f39 100644 --- a/src/TableCloth.Test/ManagedAiWindowTests.cs +++ b/src/TableCloth.Test/ManagedAiWindowTests.cs @@ -654,6 +654,7 @@ await headless.Dispatch(async () => await Ready(window); Assert.IsTrue(Find(window, "ManagedAiStarters").IsVisible); Assert.IsTrue(Find(window, "ManagedAiPrivacyNotice").IsEffectivelyVisible); + Assert.IsFalse(Controls(window).Any(x => AutomationProperties.GetAutomationId(x) == "ManagedAiResponseActivity")); Assert.HasCount(1, Find(window, "ManagedAiChatTranscript").Children); Assert.HasCount(3, Controls(window).OfType