From b306c69f7edf54cfa7c66595528c062d1ec93685 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 6 Nov 2026 19:16:56 -0500 Subject: [PATCH] Unify saved findings in the message Sources inspector Keep answers focused, disclose retained findings and evidence on demand, and add content-free debug publication correlation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../single_app/functions_appinsights.py | 12 +- .../functions_orchestration_execution.py | 20 +- .../src/components/chat/AnalysisResult.tsx | 176 +++++----- .../src/components/chat/MessageInspector.tsx | 69 +++- .../v2_ui/src/components/chat/MessageList.tsx | 25 +- application/v2_ui/src/lib/citations.ts | 7 + .../UNIFIED_SOURCES_SAVED_FINDINGS_FIX.md | 93 ++++++ docs/guides/analyze-results.md | 57 +++- docs/reference/chat-controls.md | 25 +- docs/reference/logging-tags.md | 21 ++ .../cosmos_response_boundaries.json | 22 +- ...rchestration_analysis_citation_delivery.py | 105 +++++- ui_tests/test_chat_saved_analysis.py | 303 ++++++++++++++---- 14 files changed, 738 insertions(+), 199 deletions(-) create mode 100644 docs/explanation/fixes/UNIFIED_SOURCES_SAVED_FINDINGS_FIX.md diff --git a/application/single_app/config.py b/application/single_app/config.py index bc46f8b51..d5b178b31 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -103,7 +103,7 @@ EXECUTOR_MAX_WORKERS = 30 CHAT_STREAM_EXECUTOR_MAX_WORKERS = max(8, int(os.getenv('SIMPLECHAT_CHAT_STREAM_WORKERS', '32') or '32')) SESSION_TYPE = 'filesystem' -VERSION = "0.261.418" +VERSION = "0.261.419" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_appinsights.py b/application/single_app/functions_appinsights.py index caa1392de..a44336fa3 100644 --- a/application/single_app/functions_appinsights.py +++ b/application/single_app/functions_appinsights.py @@ -134,23 +134,27 @@ }) LOGGER_WORKFLOW_HASH_KEYS = frozenset({ "conversationidhash", "turnidhash", "runidhash", "stepidhash", "producerstepidhash", - "documentidhash", "plannerbindinghash", "documentsethash", + "documentidhash", "plannerbindinghash", "documentsethash", "messageidhash", "producerrunidhash", }) # Application-owned snake_case codes; any other text under these keys is dropped. LOGGER_WORKFLOW_CODE_KEYS = frozenset({ "validationcode", "validationrule", "responsefailure", "executioncode", "outputcode", "durablestatus", "failurecode", "authorityreason", "capabilityid", "outputformat", "producercapabilityid", - "executionpath", "outcome", + "executionpath", "outcome", "analysisvalidationstatus", "analysissemantics", }) -def workflow_log_context(*, conversation_id=None, turn_id=None, run_id=None, step_id=None): +def workflow_log_context( + *, conversation_id=None, turn_id=None, run_id=None, step_id=None, + message_id=None, producer_run_id=None, producer_step_id=None, +): """Correlate workflow requests without recording their raw identifiers.""" return { f"{name}_hash": hashlib.sha256(value.encode("utf-8", errors="replace")).hexdigest() for name, value in ( ("conversation_id", conversation_id), ("turn_id", turn_id), ("run_id", run_id), - ("step_id", step_id), + ("step_id", step_id), ("message_id", message_id), + ("producer_run_id", producer_run_id), ("producer_step_id", producer_step_id), ) if isinstance(value, str) and value } diff --git a/application/single_app/functions_orchestration_execution.py b/application/single_app/functions_orchestration_execution.py index 88e321093..ace97bfd6 100644 --- a/application/single_app/functions_orchestration_execution.py +++ b/application/single_app/functions_orchestration_execution.py @@ -103,7 +103,7 @@ assert_current_request_sources_available, guard_model_callable, strict_source_authority, ) from content_screening.contracts import DocumentHeldError, ScreeningError -from functions_appinsights import log_chat_execution_outcome, log_event, workflow_log_context +from functions_appinsights import debug_print, log_chat_execution_outcome, log_event, workflow_log_context from functions_orchestration_adapters import resolve_context_source_manifest from functions_orchestration_checkpoints import ( CheckpointError, fingerprint, orchestration_answer_message_id, publication_documents_match, @@ -1838,6 +1838,24 @@ def _finalize(self, result, error, *, refreshes=0): document, {key: value for key, value in saved.items() if not key.startswith("_")}, ): raise HarnessExecutionError("message_not_saved") + for descriptor in (saved.get("metadata") or {}).get("saved_analyses") or []: + binding = descriptor.get("binding") or {} + debug_print( + "[SAVED_ANALYSIS] Findings published", + category="DOCUMENT_ANALYSIS", + **workflow_log_context( + conversation_id=self.record["conversation_id"], + turn_id=self.record.get("turn_id"), run_id=self.record["id"], + message_id=message_id, producer_run_id=binding.get("run_id"), + producer_step_id=binding.get("step_id"), + ), + descriptor_count=len(saved["metadata"]["saved_analyses"]), + record_count=descriptor.get("record_count"), + source_count=descriptor.get("source_count"), + available=descriptor.get("available", True), + analysis_validation_status=descriptor.get("validation_status"), + analysis_semantics=(descriptor.get("semantics") or {}).get("mode"), + ) # Publication preserves an administrator's earlier retraction of this reply. blocked = document.get("role") == "safety" if blocked: diff --git a/application/v2_ui/src/components/chat/AnalysisResult.tsx b/application/v2_ui/src/components/chat/AnalysisResult.tsx index 7c876800f..89e8e9de5 100644 --- a/application/v2_ui/src/components/chat/AnalysisResult.tsx +++ b/application/v2_ui/src/components/chat/AnalysisResult.tsx @@ -1,7 +1,8 @@ // AnalysisResult.tsx -// Lazy, complete-record pages supporting the readable answer above them. +// On-demand saved findings and passages within the message's Sources inspector. import { useCallback, useEffect, useState } from 'react'; +import { FileText, MessageSquare } from 'lucide-react'; import { fetchAnalysisEvidence, fetchAnalysisRecords } from '../../lib/endpoints'; import { analysisNotices, @@ -66,13 +67,13 @@ function Evidence({ return (
setOpen(event.currentTarget.open)} className="mt-2"> - Evidence for finding {record.record_id} + Supporting passages {open && (

{error || (evidence === null ? 'Loading saved evidence…' : - evidence.length ? `${evidence.length} saved passage(s).` : 'No supporting passages are available.')} + evidence.length ? `${evidence.length} supporting ${evidence.length === 1 ? 'passage' : 'passages'}.` : 'No supporting passages are available.')}

{error && setRetry((value) => value + 1)}>Retry evidence} {evidence?.map((item) => ( @@ -99,11 +100,12 @@ function Evidence({ export function AnalysisResult({ descriptor, onUnavailable, + reviewNumber, }: { descriptor: SavedAnalysisDescriptor; onUnavailable: (descriptor: SavedAnalysisDescriptor) => void; + reviewNumber?: number; }) { - const [open, setOpen] = useState(false); const [offset, setOffset] = useState(0); const [previousOffsets, setPreviousOffsets] = useState([]); const [page, setPage] = useState(null); @@ -126,7 +128,7 @@ export function AnalysisResult({ }, [descriptor, onUnavailable]); useEffect(() => { - if (!open || blocked) { + if (blocked) { return; } const controller = new AbortController(); @@ -149,7 +151,7 @@ export function AnalysisResult({ } }); return () => controller.abort(); - }, [descriptor, offset, open, blocked, retry, reportUnavailable]); + }, [descriptor, offset, blocked, retry, reportUnavailable]); const displayed = page?.records.length ?? 0; const displayedSources = new Set(page?.records.map((record) => record.document_id).filter(Boolean)).size; @@ -157,92 +159,108 @@ export function AnalysisResult({ const notices = page ? [...analysisNotices(page.validation.limitations), ...analysisNotices(page.validation.issues)] : []; return ( -
-

{descriptor.label || 'Saved analysis'}

+
+
+

+ {reviewNumber ? `Saved review ${reviewNumber}` : 'Saved findings'} +

+ + {descriptor.record_count} {descriptor.record_count === 1 ? 'finding' : 'findings'} · {descriptor.source_count} {descriptor.source_count === 1 ? 'source' : 'sources'} + +
+ {descriptor.semantics?.mode === 'source_observations' && ( +

Source observations, not reconciled values.

+ )} {blocked ? (

{unavailable || analysisUnavailableMessage()}

) : ( <> -

- {open && !page && !error ? 'Loading saved findings… ' : ''} - {error || (open && page - ? `Showing ${displayed ? page.offset + 1 : 0}–${page.offset + displayed} of ${page.total_records} records · ${displayedSources} of ${page.source_count} sources on this page.` - : `${descriptor.record_count} saved findings · ${descriptor.source_count} sources.`)} +

+ {!page && !error ? 'Loading saved findings… ' : ''} + {error || (page + ? `Showing ${displayed ? page.offset + 1 : 0}–${page.offset + displayed} of ${page.total_records} records · ${displayedSources} of ${page.source_count} ${page.source_count === 1 ? 'source' : 'sources'} on this page.` + : '')}

-

{analysisValidationNotice(status, descriptor.semantics)}

- { + {status !== 'valid' && ( +

{analysisValidationNotice(status)}

+ )} + { selectAnalysisResult(descriptor); document.getElementById('composer-input')?.focus(); }}> - Ask about this analysis + -
setOpen(event.currentTarget.open)} className="mt-2"> - - Findings and limitations - - {open && ( -
- {error && setRetry((value) => value + 1)}>Retry findings} - {notices.length > 0 && ( -
-

Limitations and validation issues

-
    - {notices.map((notice, index) =>
  • {notice}
    • )} -
-
- )} - {page?.records.map((record, index) => ( -
-

Finding {page.offset + index + 1}

-

- {record.source.file_name || record.document_id} · {record.record_id} -

-
- {Object.entries(record.values).map(([field, value]) => ( -
-
{field.replace(/_/g, ' ')}
-
{analysisValue(value)}
-
- ))} -
- -
- ))} - {page && page.total_records === 0 &&

No accepted findings were saved.

} -
- Advanced analysis details - - View diagnostics (JSON) - -
- -
- )} -
+

Uses the saved findings, not a new review of the sources.

+
+ {error && setRetry((value) => value + 1)}>Retry findings} + {page?.records.map((record, index) => ( +
+ + Finding {page.offset + index + 1}{' '} + + + +
+ {Object.entries(record.values).map(([field, value]) => ( +
+
{field.replace(/_/g, ' ')}
+
{analysisValue(value)}
+
+ ))} +
+ +
+ ))} + {page && page.total_records === 0 &&

No accepted findings were saved.

} +
+ Analysis limits +

{analysisValidationNotice(status, descriptor.semantics)}

+ {notices.length > 0 && ( +
    + {notices.map((notice, index) =>
  • {notice}
    • )} +
+ )} +
+ +
)}
); } + +export function AnalysisDiagnostics({ descriptor }: { descriptor: SavedAnalysisDescriptor }) { + return ( +
+

Saved review diagnostics

+

{descriptor.label || 'Saved analysis'}

+ + View diagnostics (JSON) + +
+ ); +} diff --git a/application/v2_ui/src/components/chat/MessageInspector.tsx b/application/v2_ui/src/components/chat/MessageInspector.tsx index 64c0c0718..2a1a5f109 100644 --- a/application/v2_ui/src/components/chat/MessageInspector.tsx +++ b/application/v2_ui/src/components/chat/MessageInspector.tsx @@ -4,7 +4,7 @@ // One panel with three sections rather than three separate panels: a user comparing what a // response cited against how it was generated should not have to close one to open another. -import { useEffect, useState } from 'react'; +import { useEffect, useMemo, useState } from 'react'; import { clsx } from 'clsx'; import { Braces, @@ -31,7 +31,7 @@ import { import { normalizeOrchestrationAttempt } from '../../lib/orchestration'; import { openOrchestrationRecovery } from '../../lib/orchestrationController'; import { m365ChipLabel, m365SecondaryLine } from '../../lib/m365Citations'; -import type { ChatMessage, Json, PersistedThought } from '../../lib/types'; +import type { ChatMessage, Json, PersistedThought, SavedAnalysisDescriptor } from '../../lib/types'; import { buildToolResultView, type RowMode } from '../../lib/agentCitationRows'; import { GlassButton } from '../ui/primitives'; import { M365KindIcon, M365OpenLink } from './M365CitationChip'; @@ -42,9 +42,15 @@ import { readEvidenceStatus, type EvidenceStatus } from '../../lib/evidenceStatu import { readMaskState } from '../../lib/masking'; import { WebProviderBadge } from './WebCitationCards'; import { safeWebSourceUrl } from '../../lib/webCitations'; +import { analysisUnavailableMessage, readSavedAnalyses } from '../../lib/savedAnalysis'; +import { AnalysisDiagnostics, AnalysisResult } from './AnalysisResult'; +import { CitationChip } from './CitationChip'; +import { documentCitationGroups } from '../../lib/citations'; export type InspectorSection = 'details' | 'sources' | 'reasoning'; +const ignoreUnavailableAnalysis = () => {}; + const SECTION_LABEL: Record = { details: 'Details', sources: 'Sources', @@ -170,15 +176,21 @@ function DetailsSection({ message }: { message: ChatMessage }) { ); } -function SourcesSection({ sources, evidence }: { sources: MessageSources; evidence: EvidenceStatus | null }) { +function SourcesSection({ sources, evidence, hasFindings = false }: { + sources: MessageSources; evidence: EvidenceStatus | null; hasFindings?: boolean; +}) { + const empty = hasFindings + ? 'No citations were attached to the answer. Saved findings may still include supporting passages.' + : 'This response did not cite any sources.'; + const documentGroups = documentCitationGroups(sources.documents); if (sources.total === 0 && !evidence) { - return This response did not cite any sources.; + return {empty}; } return (
- {sources.total === 0 && This response did not cite any sources.} + {sources.total === 0 && {empty}} {sources.conversations.length > 0 && (

@@ -190,14 +202,22 @@ function SourcesSection({ sources, evidence }: { sources: MessageSources; eviden {sources.documents.length > 0 && (

- Documents ({sources.documents.length}) + Document citations ({sources.documents.length})

    {sources.documents.map((citation, index) => { const described = describeDocumentCitation(citation); + const id = typeof citation.citation_id === 'string' + ? citation.citation_id.trim().replace(/^#/, '') : ''; + const group = documentGroups.get(id); + if (group && !described.isSummary) { + return
  • + +
    • ; + } return (
  • {described.isSummary ? ( @@ -454,15 +474,22 @@ export function MessageInspector({ section, onSection, onClose, + onAnalysisUnavailable = ignoreUnavailableAnalysis, + unavailableAnalysisKeys, }: { message: ChatMessage; section: InspectorSection; onSection: (next: InspectorSection) => void; onClose: () => void; + onAnalysisUnavailable?: (descriptor: SavedAnalysisDescriptor) => void; + unavailableAnalysisKeys?: ReadonlySet; }) { const sources = readSources(message as unknown as Json); const masks = readMaskState(message); const evidence = masks.hasAnyMask ? null : readEvidenceStatus(message.metadata?.evidence_status); + const savedAnalyses = useMemo(() => masks.hasAnyMask ? [] : readSavedAnalyses(message.metadata), + [masks.hasAnyMask, message.metadata]); + const hasFindings = savedAnalyses.length > 0; // An uploaded file cites nothing and has no reasoning. Its sources are what ingestion // extracted from it, which is the material any later citation of the file comes from. const upload = message.role === 'file'; @@ -509,11 +536,16 @@ export function MessageInspector({
-
+
{active === 'details' && ( <> + {savedAnalyses.map((descriptor, index) => descriptor && descriptor.available !== false + && !unavailableAnalysisKeys?.has(`${descriptor.conversation_id}:${descriptor.message_id}:${descriptor.result_sha256}`) ? ( + + ) : null)} )} {active === 'sources' && upload && ( @@ -527,7 +559,26 @@ export function MessageInspector({ This upload has no stored extraction to show. ) )} - {active === 'sources' && !upload && } + {active === 'sources' && !upload && ( +
+
+ {hasFindings &&

Cited in the answer

} + +
+ {hasFindings &&
+ {savedAnalyses.map((descriptor, index) => descriptor ? ( + 1 ? index + 1 : undefined} + /> + ) :

{analysisUnavailableMessage()}

)} +
} +
+ )} {active === 'reasoning' && }
diff --git a/application/v2_ui/src/components/chat/MessageList.tsx b/application/v2_ui/src/components/chat/MessageList.tsx index b8713b663..7432768d3 100644 --- a/application/v2_ui/src/components/chat/MessageList.tsx +++ b/application/v2_ui/src/components/chat/MessageList.tsx @@ -43,7 +43,6 @@ import { import { InlineMapCards } from './InlineMapCard'; import { ChatFilePreview } from './ChatFilePreview'; import { GeneratedArtifactCard } from './GeneratedArtifactCard'; -import { AnalysisResult } from './AnalysisResult'; import type { SavedAnalysisDescriptor } from '../../lib/types'; import { IconButton, MessageActions } from './MessageActions'; import { isCheckingFailureNotice, OrchestrationMessageRecovery } from './OrchestrationRecoveryNotice'; @@ -1262,16 +1261,17 @@ function MessageBubbleInner({ ) : null} {/* Inside the bubble, because a generated file belongs to the reply that produced it rather than sitting loose in the thread. */} - {hasSavedAnalysis && (masks.ranges.length > 0 ? ( + {hasSavedAnalysis && masks.ranges.length > 0 ? (

Saved findings are hidden while this message is masked.

- ) : savedAnalyses.map((descriptor, index) => descriptor ? ( - - ) : ( -

+ ) : analysisUnavailable ? ( +

{analysisUnavailableMessage()}

- )))} + ) : hasSavedAnalysis && savedAnalyses.some((descriptor) => descriptor?.validation_status !== 'valid') ? ( +

+ This review has unresolved checks or incomplete findings. Open Sources to inspect its limits. +

+ ) : null} {!analysisUnavailable && (!hasSavedAnalysis || masks.ranges.length === 0) && ( hasSavedAnalysis ? artifactCards.length > 0 && (
@@ -1319,7 +1319,12 @@ function MessageBubbleInner({ inspector || replayExpanded ? 'opacity-100' : 'opacity-0', )} > - {!analysisUnavailable && inspectMessage('sources')}> + + + ) : setEditing(true) : undefined} inspector={inspector} @@ -1354,6 +1359,8 @@ function MessageBubbleInner({ section={inspector} onSection={inspectMessage} onClose={() => setInspector(null)} + onAnalysisUnavailable={markAnalysisUnavailable} + unavailableAnalysisKeys={unavailableAnalysisKeys} /> )}

diff --git a/application/v2_ui/src/lib/citations.ts b/application/v2_ui/src/lib/citations.ts index a924034a2..d34148b07 100644 --- a/application/v2_ui/src/lib/citations.ts +++ b/application/v2_ui/src/lib/citations.ts @@ -133,6 +133,13 @@ function rawDocumentSources(records: readonly HybridCitation[]): Map { + return new Map([...rawDocumentSources(records)].map(([id, group]) => [ + id, group ? { ...group, id } : null, + ])); +} + function splitCitationId(rawId: string): ParsedCitation { const citationId = rawId.startsWith('#') ? rawId.slice(1) : rawId; const separator = citationId.lastIndexOf('_'); diff --git a/docs/explanation/fixes/UNIFIED_SOURCES_SAVED_FINDINGS_FIX.md b/docs/explanation/fixes/UNIFIED_SOURCES_SAVED_FINDINGS_FIX.md new file mode 100644 index 000000000..02b5ba5c5 --- /dev/null +++ b/docs/explanation/fixes/UNIFIED_SOURCES_SAVED_FINDINGS_FIX.md @@ -0,0 +1,93 @@ +# Unified Sources and saved findings + +Fixed/Implemented in version: **0.261.419**. +Application version: `application/single_app/config.py`. + +## Issue and root cause + +V2 rendered retained Analyze results automatically inside the assistant's answer +bubble. Labels came from internal execution-step titles, while counts, validation +terminology, arbitrary record fields, and follow-up controls competed with the +readable answer. Citations already had a separate per-message Sources inspector. +The saved result was useful; the duplicate, implementation-oriented presentation +was not. + +## Changes and impact + +`MessageList.tsx` no longer inserts the analysis reader in the answer bubble. +The existing Sources action opens `MessageInspector.tsx`, with answer citations +and saved findings alongside each other on desktop and stacked on narrow screens. +Citation counts, saved finding counts, and contributing-source counts remain +separate. No additional toolbar action, drawer mode, endpoint, or model pass was +introduced. Document citations reuse the existing citation viewer with recorded +identities rather than filename matching. + +`AnalysisResult.tsx` loads up to 25 complete records only while Sources is open. +Numbered findings disclose their full values; Supporting passages reads only +that finding's saved evidence references. Analysis limits retains validation +explanations and saved issues. Multiple reviews have neutral numbered labels. +Internal step labels and the bounded diagnostic JSON link belong to Details. +Incomplete or unavailable reviews retain brief answer-level notices. + +The existing conversation-bound authorization, masking, pagination, retries, +abort-on-navigation behavior, and stale-result handling remain in place. +Ask about this saved review still selects the whole saved result and disables +fresh source retrieval. It does not create record-scoped follow-up context. +Classic chat, conversation Documents/Plan, and the answer's generated prose are +unchanged. + +`functions_orchestration_execution.py` emits content-free publication diagnostics +only after saved-message read-back verification. `functions_appinsights.py` +preserves the approved message/producer hashes and analysis codes. The established +debug helper gates both console and configured App Insights output. Ordinary +execution-outcome logging remains unchanged. + +## Evidence interpretation + +Structural validation does not establish factual correctness or exhaustive +coverage. Source observations are not reconciled values. Saved findings are not +native comment records, and successful body analysis does not prove that native +comments were extracted. The UI does not infer unavailable comment details from +answer prose or absent metadata. + +The investigated conversation's telemetry established four saved body findings +and six exact passage matches, not four comments. The exact record bodies were +not exported during investigation, and the cause of the missing individual +comment records remains unproven. This fix changes presentation and correlation, +not native Word comment extraction. + +## Validation + +`ui_tests/test_chat_saved_analysis.py` exercises both interfaces, retained plural +reviews, safe text, pagination, supporting passages, downloads, saved-review +follow-ups, revocation, masking, stale pointers, retries, and late responses. +V2-specific scenarios cover an answer-first initial state, the single Sources +destination, Details-only diagnostics, keyboard interaction, identical source +filenames with distinct document identities, and 390/1280-pixel light/dark layouts. +The four viewport/theme scenarios also measure saved-finding text contrast against +the rendered background, requiring at least 4.5:1. Conflicting recorded citation +identities remain non-clickable rather than opening an ambiguous document. + +`functional_tests/test_orchestration_analysis_citation_delivery.py` exercises +real native producers and publication with deterministic external services. It +checks debug-off/on console and App Insights behavior, sanitized hashes/counts, +and absence of publication diagnostics when persistence fails. Related saved +analysis route, diagnostic, retained-publication, and telemetry tests protect the +existing contracts. + +Local validation passed: 74 saved-analysis browser cases, the final six +Sources-focused browser cases after the contrast checks, the production build, +and 161 network-blocked owner-contract cases covering all five storage owners +in the changed execution module. Documentation surface coverage passed all seven +checks; site quality passed all six checks. The repository-wide storage-contract +run was stopped in favor of these affected-owner checks; no full-suite pass is +claimed. + +These are local fixture checks, not a deployment or a rerun of the investigated +conversation. See [the user guide](../../guides/analyze-results.md) and +[logging reference](../../reference/logging-tags.md). + +The design detector's remaining warnings refer to the unchanged collaborative +typing-dot animation and a literal `` in an explanatory comment, not to the +new Sources controls. The image component itself has a real source and an error +handler; neither unrelated surface was changed. diff --git a/docs/guides/analyze-results.md b/docs/guides/analyze-results.md index b78a2cd79..88abf151b 100644 --- a/docs/guides/analyze-results.md +++ b/docs/guides/analyze-results.md @@ -4,15 +4,17 @@ title: "Read and discuss saved Analyze results" description: "Read the answer first, browse saved findings and evidence, and ask follow-up questions without requesting another source pass." section: "Guides" audience: user -version: "0.261.398" +version: "0.261.419" --- ## What this does Analyze reviews selected documents according to your question and saves the accepted findings for later review. Both classic chat and React V2 keep the -readable answer in the conversation, with a compact **Saved analysis** section -under it. A supporting file becoming ready does not replace that answer. +readable answer in the conversation. In V2, use the answer's existing **Sources** +action to inspect saved findings alongside citations. Classic chat retains its +inline **Saved analysis** section. A supporting file becoming ready does not +replace the answer. {% include media.html id="slide-analyze-and-compare" %} @@ -23,6 +25,7 @@ capability, see [Document Compare]({{ '/features/document-compare/' | relative_u Implemented in version: **0.261.109**. Application version alignment is tracked in `application/single_app/config.py`. +The unified V2 Sources experience was implemented in version: **0.261.419**. ## Start with the question, not a report schema @@ -72,26 +75,34 @@ The assistant's answer is the readable overview. It can explain the main themes without displaying every saved finding. Do not treat that overview, or a file preview, as the complete set of records. -1. Read the answer and the validation notice under **Saved analysis**. -2. Expand **Findings and limitations** to load a page of accepted findings. +1. Read the answer. Incomplete reviews retain a brief notice beneath it. +2. In V2, open **Sources** beneath that answer. Its existing inspector now holds + answer citations and saved findings, side by side on wide screens and stacked + on narrow screens. A page loads only when you open Sources. Expand a numbered + **Finding** to read its complete values. In classic chat, expand **Findings and + limitations** under **Saved analysis**. 3. Use **Next findings** and **Previous findings** to move through the saved set. Pages contain up to **25 complete records**; the last page may be shorter. 4. Compare the displayed count with the total. For example, `Showing 1–25 of 60 records` means that 35 records are on other pages. -A closed findings section reports zero records displayed, not zero records saved. -Source counts are separate: the sources represented on one page can be fewer than +A closed classic findings section reports zero records displayed, not zero records +saved. V2 does not load findings while its Sources inspector is closed. +Citation counts are not finding counts. Source counts are separate too: the +sources represented on one page can be fewer than the sources in the saved analysis. One source can contribute several findings. Neither count proves that every possible issue was discovered. Since **0.261.398**, a new orchestration reply can show several retained analyses -when a later step could not finish. Their headings identify the producing -steps; each has its own count, findings, evidence, and diagnostic link. Read +when a later step could not finish. V2 identifies these as numbered saved reviews; +classic headings identify the producing steps. Each has its own count, findings, +evidence, and diagnostic link. Read their validation notices separately. A usable source review does not mean the overall answer was delivered. These results are not selected automatically for the next turn. Use **Ask about -this analysis** on the particular result you want to discuss. Inspecting saved +this saved review** in V2, or **Ask about this analysis** in classic, on the +particular result you want to discuss. Inspecting saved work does not complete the failed step or make a partial result suitable for a step that requires complete input. Historical failed replies are not rewritten to add these sections. @@ -113,8 +124,9 @@ backdrop to return to the report. Wide tables scroll inside their own container. ## Inspect evidence and limitations -Expand **Evidence for finding …** beside a finding; the label includes that -finding's identifier. Saved passages can include a filename, page or page range, +In V2, expand a finding and then **Supporting passages**. In classic chat, expand +**Evidence for finding …** beside a finding; that label includes its identifier. +Saved passages can include a filename, page or page range, and chunk location when those details were saved. A finding without supporting passages says **No supporting passage saved for this finding.** @@ -123,10 +135,18 @@ include that table's markup. In the readable report, **Caveat:** lines under a finding qualify it without making the result partial, and **Analysis notes** record observations about a page or chunk range of a source. -The **Limitations and validation issues** section appears when the saved result -includes those notices. Read it alongside the findings, particularly when the +In V2, **Analysis limits** holds the review's interpretation and validation +notice plus any saved limitations and issues. Classic chat uses **Limitations +and validation issues** for the saved notices. Individual finding values retain +their caveats. Read the limits alongside the findings, particularly when the analysis is incomplete. +Answer citations are not automatically supporting evidence for every finding. +Only a finding's own saved evidence references supply its supporting passages. +Several findings can come from one document; none of these counts establishes +how many native Word or Excel comments were returned. Missing metadata stays +unknown rather than becoming a successful comment extraction. + | Notice | How to interpret it | | --- | --- | | **Structural checks passed.** | Structural checks completed successfully. This is not independent factual verification or proof that the findings are exhaustive. | @@ -139,14 +159,17 @@ Findings remain model judgments. Inspecting a saved quotation helps you understa its support; it does not reread the original source or independently verify the conclusion. -For an audit, **View diagnostics (JSON)** opens the separately saved processing +For an audit, use **Details** in the V2 message inspector, then **View diagnostics +(JSON)**. Classic chat keeps that link in the saved analysis. It opens the separately saved processing data in a new tab. It is not the final findings or a readable report. The response is bounded; `next_offset` identifies another byte page when the audit is larger than one response. Ordinary reading and follow-up questions do not use this data. ## Ask about the saved analysis -1. Select **Ask about this analysis** on the result you want to discuss. +1. Select **Ask about this saved review** in V2, or **Ask about this analysis** in + classic, on the result you want to discuss. This selects the whole saved + review, not only the finding currently expanded. 2. Check the composer notice: > Saved analysis selected. Explaining the saved analysis — not running a new pass over the original sources. @@ -194,7 +217,7 @@ An unavailable or stale notice is not an empty successful analysis: - **Saved analysis unavailable** can indicate that the conversation or its saved result was removed, that you lost access to the conversation, or that the saved - data failed an integrity check. Result reading, **Ask about this analysis**, and + data failed an integrity check. Result reading, the saved-review follow-up, and original result export controls are unavailable. A sign-in notice asks you to sign in again before checking access. - **Saved analysis is stale or has changed** asks you to reopen the conversation diff --git a/docs/reference/chat-controls.md b/docs/reference/chat-controls.md index 7574b5f7a..62af34157 100644 --- a/docs/reference/chat-controls.md +++ b/docs/reference/chat-controls.md @@ -4,7 +4,7 @@ title: "Chat interface controls" description: "Reference for every documented control in the SimpleChat chat interface." section: "Reference" audience: user -version: "0.261.418" +version: "0.261.419" --- ## How to use this reference @@ -488,7 +488,7 @@ conclusions are complete or correct. | Existing control | What it shows | Why you would use it | | --- | --- | --- | -| Show sources | The reply's recorded document, web, tool, and Microsoft 365 evidence; the accessible name includes a count when evidence exists. An empty panel says the response did not cite any sources. | Check whether an answer has supporting evidence rather than relying on plausible prose. | +| Show sources | The reply's recorded document, web, tool, and Microsoft 365 evidence, plus saved Analyze findings when present. Its count describes citations, not findings. Document citations use the existing source viewer. An answer with no citations can still have saved findings and supporting passages. | Inspect provenance and retained findings in one place rather than relying on plausible prose. | | Tool-call disclosure in Sources | Recorded operation label, arguments, result when available, execution outcome, and error when recorded. A failure without a result still shows its arguments and error. | Distinguish an executed operation's result from a failed request or the assistant's description of what it intended to do. | | Show reasoning | The reply's recorded reasoning activities, including failed tool activity when execution recorded a failure. | Understand the progression of work without treating reasoning text as proof of a tool result. | @@ -690,8 +690,10 @@ also depends on configured storage and current source access. Implemented in version **0.261.109**. These response-specific controls appear in classic chat and React V2 when an assistant message has a saved Analyze result. -They are rendered with the response rather than added to the always-visible -toolbar. See [Read and discuss saved Analyze results]({{ '/guides/analyze-results/' | relative_url }}). +In V2, since **0.261.419**, they live in the existing per-message **Sources** +inspector, not an automatic analysis block or another drawer. Classic retains +its inline saved-analysis section. Application version alignment is tracked in +`application/single_app/config.py`. See [Read and discuss saved Analyze results]({{ '/guides/analyze-results/' | relative_url }}). The answer remains the readable overview, even after a supporting export finishes. The overview and download previews are not the full findings set. Ordinary @@ -699,19 +701,20 @@ narrative requests such as “Explain the risks in these documents” do not req a report schema, scoring setup, or configuration interview. Since **0.261.398**, new orchestration replies may show several saved reviews -even when a later step failed. Each section is labelled by its producing step -and keeps its own validation notice and pages. Those plural results require +even when a later step failed. Each keeps its own validation notice and pages. +V2 numbers these saved reviews; internal producing-step labels remain in +**Details**. Classic continues to label the producing steps. Those plural results require explicit selection before a follow-up; inspecting them does not complete the failed answer or promote partial findings to complete input. | Control or notice | What it does | Why you would use it | Available when | | --- | --- | --- | --- | -| **Saved analysis** | Identifies the saved result and shows displayed-versus-total record counts, separate source counts, and its validation notice. | Distinguish a readable overview or one page from the full saved findings set. | The assistant message carries a saved result; an unavailable result shows a notice instead of result controls | -| **Findings and limitations** | Expands accepted findings and any **Limitations and validation issues**, loading up to 25 complete records at a time. | Inspect details without downloading a file or displaying the whole result at once. | The saved result is available | +| **Saved findings** / **Saved review** (V2), **Saved analysis** (classic) | Identifies the saved result and shows displayed-versus-total record counts and separate source counts. V2 loads pages only while Sources is open and keeps unresolved validation notices visible. | Distinguish a readable overview or one page from the full saved findings set. | The assistant message carries a saved result; an unavailable result shows a notice instead of result controls | +| **Finding** / **Analysis limits** (V2), **Findings and limitations** (classic) | In V2, numbered findings expand to complete saved values; Analysis limits holds the validation explanation and saved limitations/issues. Classic expands findings and limits together. Pages contain up to 25 complete records. | Read useful details on demand rather than a dump of internal result fields beneath the answer. | The saved result is available | | **Previous findings** / **Next findings** | Reads the preceding or following page of the same saved result. | Review records outside the currently displayed range. A page's source count may be smaller than the result's total source count. | A corresponding page exists and the current page has loaded | -| **Evidence for finding …** | Loads that finding's saved supporting passages and any saved filename, page, or chunk location. The label includes the finding's identifier. | Understand what supports a finding without treating it as independently verified. | The finding has saved evidence references | -| **Ask about this analysis** | Selects this saved result for the next message and focuses the composer. | Ask for an explanation of these findings instead of another original-source pass. | The saved result is available | -| **View diagnostics (JSON)** | Opens a separate, bounded diagnostic-data response in a new tab; `next_offset` identifies further byte pages. | Audit processing details without mixing them into findings, reports, or model input. | The saved result is available and access through its conversation is confirmed | +| **Supporting passages** (V2), **Evidence for finding …** (classic) | Loads that finding's explicitly referenced supporting passages and saved filename, page, or chunk location. A citation elsewhere in the answer is not substituted for missing finding evidence. | Understand what supports a finding without treating it as independently verified. | The finding has saved evidence references | +| **Ask about this saved review** (V2), **Ask about this analysis** (classic) | Selects the whole saved result for the next message and focuses the composer, not just the expanded record. | Explain retained findings instead of another original-source pass. | The saved result is available | +| **View diagnostics (JSON)** | In V2's existing **Details** section, opens a separate, bounded diagnostic-data response in a new tab; `next_offset` identifies further byte pages. Classic retains its existing inline link. | Audit processing details without mixing them into findings, reports, or model input. | The saved result is available and access through its conversation is confirmed | | **Saved analysis selected.** | Shows the composer notice: “Explaining the saved analysis — not running a new pass over the original sources.” A completed result can select this context automatically. | Confirm what the next ordinary follow-up will discuss. | Saved-analysis context is selected | | **Remove saved analysis context** | The accessible label of the notice's close button; removes the saved-result selection without deleting the result or draft. | Leave explanation mode before choosing a different task or source pass. | The saved-analysis composer notice is present | | **Downloads** | Expands the existing generated-file cards and their format-specific actions, such as **Download CSV**. | Obtain a supporting output after reviewing the answer and findings. | The available saved result has generated outputs; existing output readiness and approval rules still apply | diff --git a/docs/reference/logging-tags.md b/docs/reference/logging-tags.md index 2905bac41..f3c601ab9 100644 --- a/docs/reference/logging-tags.md +++ b/docs/reference/logging-tags.md @@ -430,6 +430,7 @@ for availability states and live-UI validation limits. - `[SAFETY_REMEDIATION]` - `[SAFETY_VIOLATIONS]` - `[SAFETY_WARNINGS]` +- `[SAVED_ANALYSIS]` - `[SAVE_CHUNKS]` - `[SAVE_CHUNKS_BATCH]` - `[SEARCH_CACHE_DEBUG]` @@ -794,12 +795,32 @@ findings, caveats, notes, prompts, file names, or model responses. | Event message | Severity | Properties | | --- | --- | --- | | `[DOCUMENT_ANALYSIS] Final findings validated` | Information when the result is `valid`; otherwise Warning | `sc_validation_code`, the record, window and evidence counts described below, and one `sc_issue__count` per validation issue code. | +| `[SAVED_ANALYSIS] Findings published` | Debug, only while debug logging is enabled | Confirmed saved-reply correlation and retained-result counts described below. | | `[ORCHESTRATION_EXECUTOR] A dependency-bound step could not complete.` | Warning | `sc_failure_code`, `sc_execution_code`, `sc_error_type`, `sc_authority_reason` when an access check refused the step (its `[ORCHESTRATION_EXTERNAL_SOURCES]` refusal event names the check), and, when the step refused a partial input, the `sc_producer_*` fields described below. | Analyze writes its event each time it validates newly assembled final findings. A step that resumes from an already saved final result does not write it again. The event carries `sc_conversation_id_hash` when the analysis runs for a conversation, but no run or step hash, so match it by time to the step events that follow it. +Since **0.261.419**, the debug-only publication event links a retained analysis to +its confirmed saved reply with `sc_conversation_id_hash`, `sc_turn_id_hash` when +present, `sc_run_id_hash`, `sc_message_id_hash`, `sc_producer_run_id_hash`, and +`sc_producer_step_id_hash`. Producer hashes describe the original producer, which +can differ from the publishing run after a retry. The event also includes +`sc_descriptor_count`, `sc_record_count`, `sc_source_count`, `sc_available`, +`sc_analysis_validation_status`, and `sc_analysis_semantics` when recorded. +One event is emitted per descriptor only after message persistence and read-back +verification. A model-free delivery refresh can emit it again. It reaches the +debug console and configured App Insights debug logger through `debug_print`; +`log_event(debug_only=True)` alone is console-only. With debug logging disabled, +neither sink emits this event. It contains no prompts, passages, filenames, +record values, raw identifiers, or raw result payloads. + +These are publication timestamps, not analysis durations or proof that a user's +full request was satisfied. Correlate by the hashes even when distributed +`OperationId` is missing. The application version is tracked in +`application/single_app/config.py`. + The executor event carries the run, conversation and step hashes and `sc_capability_id`, like the other executor events. diff --git a/functional_tests/cosmos_response_boundaries.json b/functional_tests/cosmos_response_boundaries.json index 24f701112..e061b2608 100644 --- a/functional_tests/cosmos_response_boundaries.json +++ b/functional_tests/cosmos_response_boundaries.json @@ -19971,7 +19971,7 @@ }, "application/single_app/functions_orchestration_execution.py::HarnessExecution._finalize": { "boundary": { - "sha256": "ce65fed43ee2cd34ff82f2c5c1edde3923f25418c836dd6f997c0d3a6e325d38", + "sha256": "d4f6a561a5e66e3327a65749761d1f6ef6a9ac1f1275e589aa9e938843e77e4e", "points": [ { "method": "read_item", @@ -19981,10 +19981,10 @@ ], "queries": [], "batches": [], - "module_sha256": "9a7de6fa7f15f0e382e67661b7d68d00f069fa71d364db940e367f8c3d510e93" + "module_sha256": "c82aaf7237a98c30cb5c1f4152e1ec0cfb29431100b4abb53463fa515f7d928b" }, "classification": "sdk-compatible", - "contract": "Read-only retained-result projection ends before screening and publication; its operation cache never contains message responses or conditional publication state. The published SDK point mapping is projected with items() into plain message JSON and compared with publication_documents_match, accepting only lossless integral-float/integer round trips. Stored guard fingerprints, all domain fields and Mapping-compatible mirroring remain unchanged; durable outcome telemetry reads finalized scalar fields. Review 0.261.417: Only review transport/checker handling changed in this module; this owner AST and SDK response consumers are unchanged. Existing owner-runtime evidence is retained.", + "contract": "Read-only retained-result projection ends before screening and publication; its operation cache never contains message responses or conditional publication state. The published SDK point mapping is projected with items() into plain message JSON and compared with publication_documents_match, accepting only lossless integral-float/integer round trips. Stored guard fingerprints, all domain fields and Mapping-compatible mirroring remain unchanged; durable outcome telemetry reads finalized scalar fields. Review 0.261.419: Saved-analysis debug publication reads only Mapping-compatible nested metadata after the existing plain-JSON read-back comparison; no SDK wrapper enters a strict contract. Native-producer tests cover publication, both debug sinks, and failed persistence.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_orchestration_continuation.py", @@ -20010,10 +20010,10 @@ ], "queries": [], "batches": [], - "module_sha256": "9a7de6fa7f15f0e382e67661b7d68d00f069fa71d364db940e367f8c3d510e93" + "module_sha256": "c82aaf7237a98c30cb5c1f4152e1ec0cfb29431100b4abb53463fa515f7d928b" }, "classification": "sdk-compatible", - "contract": "The previous assistant point mapping is checked for conversation/run ownership; model and reasoning fields are projected into plain dictionaries, without passing the SDK wrapper to an exact-dictionary contract. Read-operation reuse does not wrap this metadata lookup. Review 0.261.417: Only review transport/checker handling changed in this module; this owner AST and SDK response consumers are unchanged. Existing owner-runtime evidence is retained.", + "contract": "The previous assistant point mapping is checked for conversation/run ownership; model and reasoning fields are projected into plain dictionaries, without passing the SDK wrapper to an exact-dictionary contract. Read-operation reuse does not wrap this metadata lookup. Review 0.261.419: Module debug import and saved-analysis publication changed; this owner AST and SDK consumers are unchanged. Existing owner-runtime evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_orchestration_continuation.py", @@ -20033,10 +20033,10 @@ ], "queries": [], "batches": [], - "module_sha256": "9a7de6fa7f15f0e382e67661b7d68d00f069fa71d364db940e367f8c3d510e93" + "module_sha256": "c82aaf7237a98c30cb5c1f4152e1ec0cfb29431100b4abb53463fa515f7d928b" }, "classification": "sdk-compatible", - "contract": "The question point mapping supplies role and nested plain thread metadata; only a deepcopy of the validated nested thread dictionary is returned. Regeneration metadata is outside the retained-result read scope. Review 0.261.417: Only review transport/checker handling changed in this module; this owner AST and SDK response consumers are unchanged. Existing owner-runtime evidence is retained.", + "contract": "The question point mapping supplies role and nested plain thread metadata; only a deepcopy of the validated nested thread dictionary is returned. Regeneration metadata is outside the retained-result read scope. Review 0.261.419: Module debug import and saved-analysis publication changed; this owner AST and SDK consumers are unchanged. Existing owner-runtime evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_orchestration_continuation.py", @@ -20056,10 +20056,10 @@ ], "queries": [], "batches": [], - "module_sha256": "9a7de6fa7f15f0e382e67661b7d68d00f069fa71d364db940e367f8c3d510e93" + "module_sha256": "c82aaf7237a98c30cb5c1f4152e1ec0cfb29431100b4abb53463fa515f7d928b" }, "classification": "sdk-compatible", - "contract": "A normalized owned conversation retains its _etag for IfNotModified replacement; the returned SDK mapping is consumed only by the Mapping-compatible cache invalidator. Lost-CAS retries retain current ownership. This write remains outside read-operation reuse. Review 0.261.417: Only review transport/checker handling changed in this module; this owner AST and SDK response consumers are unchanged. Existing owner-runtime evidence is retained.", + "contract": "A normalized owned conversation retains its _etag for IfNotModified replacement; the returned SDK mapping is consumed only by the Mapping-compatible cache invalidator. Lost-CAS retries retain current ownership. This write remains outside read-operation reuse. Review 0.261.419: Module debug import and saved-analysis publication changed; this owner AST and SDK consumers are unchanged. Existing owner-runtime evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_orchestration_continuation.py", @@ -20084,10 +20084,10 @@ ], "queries": [], "batches": [], - "module_sha256": "9a7de6fa7f15f0e382e67661b7d68d00f069fa71d364db940e367f8c3d510e93" + "module_sha256": "c82aaf7237a98c30cb5c1f4152e1ec0cfb29431100b4abb53463fa515f7d928b" }, "classification": "sdk-compatible", - "contract": "Partitioned turn/source point mappings are reauthorized, checked for visibility and converted by normalize_history_message before strict saved-snapshot validation. Full source messages are resolved only after fingerprint checks; history disablement and revoked shared access cannot broaden reads. Result-operation snapshots never replace these message reads. Review 0.261.417: Only review transport/checker handling changed in this module; this owner AST and SDK response consumers are unchanged. Existing owner-runtime evidence is retained.", + "contract": "Partitioned turn/source point mappings are reauthorized, checked for visibility and converted by normalize_history_message before strict saved-snapshot validation. Full source messages are resolved only after fingerprint checks; history disablement and revoked shared access cannot broaden reads. Result-operation snapshots never replace these message reads. Review 0.261.419: Module debug import and saved-analysis publication changed; this owner AST and SDK consumers are unchanged. Existing owner-runtime evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_orchestration_continuation.py", diff --git a/functional_tests/test_orchestration_analysis_citation_delivery.py b/functional_tests/test_orchestration_analysis_citation_delivery.py index 23322a99b..ebdd1a6ab 100644 --- a/functional_tests/test_orchestration_analysis_citation_delivery.py +++ b/functional_tests/test_orchestration_analysis_citation_delivery.py @@ -1,16 +1,19 @@ # test_orchestration_analysis_citation_delivery.py """Native Analyze/Compare citations reach Compose, messages and streamed delivery. -Version: 0.261.398 +Version: 0.261.419 Implemented in: 0.261.356 Retained analyses after downstream failure covered in: 0.261.398 +Debug-only saved-findings publication correlation implemented in: 0.261.419 The actual headless execution, source producers, result store and message finalizer run with existing offline model/source/storage fixtures. No external harness is changed. """ +import hashlib import importlib import json +import logging import re import pytest @@ -32,14 +35,36 @@ ('document_compare', 'comparison'), ('document_compare', 'report'), ]) @pytest.mark.parametrize('delivery', ['execute', 'refresh']) -def test_native_citations_reach_model_saved_answer_and_done_frame(harness, narrative_io, capability, output, delivery): +@pytest.mark.parametrize('debug', [False, True]) +def test_native_citations_reach_model_saved_answer_and_done_frame( + harness, narrative_io, capability, output, delivery, debug, monkeypatch, caplog, capsys, +): + telemetry = importlib.import_module("functions_appinsights") + logger = logging.getLogger("test.saved_analysis.publication") + monkeypatch.setattr(telemetry, "get_appinsights_logger", lambda: logger) + monkeypatch.setattr(telemetry, "_load_logging_settings", lambda: {"enable_debug_logging": debug}) + monkeypatch.setattr(telemetry, "_azure_monitor_configured", True) + caplog.set_level(logging.DEBUG, logger=logger.name) + publications = [] + + def observe_publication(message, **properties): + persisted = harness.assistant_messages() + assert persisted, "Publication diagnostics must follow durable message persistence." + publications.append(properties) + telemetry.debug_print(message, **properties) + + monkeypatch.setattr(harness.execution, "debug_print", observe_publication) document_ids = ['document-1'] if capability == 'document_analyze' else ['document-1', 'target-1'] if capability == 'document_analyze': arguments = {'document_ids': document_ids, 'doc_scope': 'personal', 'analysis_prompt': 'Summarize with sources.'} + if output == "findings": + arguments["analysis_options"] = {"version": "analysis-options-v2", "mode": "source_observations"} outputs = [ {'name': 'findings', 'kind': 'records-v1'}, {'name': 'records', 'kind': 'records-v1'}, {'name': 'coverage', 'kind': 'structured-v1'}, {'name': 'report', 'kind': 'markdown-v1'}, ] + if output == "findings": + outputs = [item for item in outputs if item["name"] in {"findings", "coverage"}] else: arguments = {'left_document_id': 'document-1', 'right_document_ids': ['target-1'], 'doc_scope': 'personal', 'comparison_prompt': 'Compare with source passages.'} @@ -112,6 +137,38 @@ def reply(): assert descriptors[0]['message_id'] == message['id'] assert descriptors[0]['validation_status'] == 'valid' assert done['metadata']['saved_analyses'] == descriptors + assert len(publications) == 1 + properties = publications[0] + assert properties["record_count"] == descriptors[0]["record_count"] + assert properties["source_count"] == descriptors[0]["source_count"] + assert properties["descriptor_count"] == 1 + assert properties["available"] is True + for field, raw in ( + ("conversation_id_hash", "conversation-1"), ("run_id_hash", "run-1"), + ("message_id_hash", message["id"]), ("producer_run_id_hash", "run-1"), + ("producer_step_id_hash", "source"), + ): + assert properties[field] == hashlib.sha256(raw.encode()).hexdigest() + if output == "findings": + assert properties["analysis_semantics"] == "source_observations" + else: + assert not publications + traces = [record for record in caplog.records + if "[SAVED_ANALYSIS] Findings published" in getattr(record, "sc_message", "")] + expected_traces = int(debug and capability == "document_analyze") + assert len(traces) == expected_traces + console = capsys.readouterr().out + assert ("[SAVED_ANALYSIS] Findings published" in console) == bool(expected_traces) + if traces: + trace = traces[0] + for key, value in publications[0].items(): + if key != "category" and value is not None: + assert getattr(trace, f"sc_{key}") == value + safe = {key: value for key, value in trace.__dict__.items() if key.startswith("sc_")} + serialized = json.dumps(safe) + for forbidden in ("Summarize with sources", "document-1", "conversation-1", "run-1", + message["id"], "raw_analysis_items", "file_name", "quote", "prompt"): + assert forbidden not in serialized assert harness.blobs.file_uploads == 0 @@ -165,3 +222,47 @@ def reply(): assert len(page['records']) == descriptor['record_count'] assert page['validation']['status'] == 'valid' assert done['metadata']['saved_analyses'] == messages[0]['metadata']['saved_analyses'] + + +def test_failed_message_publication_emits_no_saved_findings_publication(harness, narrative_io, monkeypatch): + def reply(): + prompt = harness.model_calls[-1]["messages"][-1]["content"] + if "\n" in prompt and '"findings" list' in prompt: + return findings_from_original_source(prompt) + return "Summary of the retained findings." + + harness.create( + [ + {"step_id": "source", "capability_id": "document_analyze", + "arguments": {"document_ids": ["document-1"], "doc_scope": "personal", + "analysis_prompt": "Review the document."}, + "outputs": [{"name": "findings", "kind": "records-v1"}, + {"name": "coverage", "kind": "structured-v1"}]}, + compose_step(inputs={"evidence": { + "binding": input_binding("source", "findings"), "allow_partial": False, + }}), + ], + replies=[reply] * 30, final_response=input_binding("prepare"), + seeds={"document_ids": ["document-1"], "doc_scope": "personal"}, + original_seeds={"document_ids": ["document-1"], "doc_scope": "personal"}, + ) + execution = harness.prepare() + publications = [] + monkeypatch.setattr(harness.execution, "debug_print", lambda *args, **kwargs: publications.append(kwargs)) + + def fail_publication(document): + raise RuntimeError("PRIVATE_STORAGE_FAILURE") + + try: + result = harness.run_engine(execution) + assert result["status"] == "completed" + record = harness.read() + assert record["execution_steps"][0]["saved_analyses"] + monkeypatch.setattr(execution.lease, "publish_message", fail_publication) + with pytest.raises(RuntimeError, match="PRIVATE_STORAGE_FAILURE"): + execution._finalize(result, None) + assert not publications + messages = harness.assistant_messages() + assert not messages + finally: + execution.close() diff --git a/ui_tests/test_chat_saved_analysis.py b/ui_tests/test_chat_saved_analysis.py index 86c11875b..be901b9e9 100644 --- a/ui_tests/test_chat_saved_analysis.py +++ b/ui_tests/test_chat_saved_analysis.py @@ -1,9 +1,10 @@ # test_chat_saved_analysis.py """ Saved Analyze findings, evidence, and explanation context in both chat interfaces. -Version: 0.261.398 +Version: 0.261.419 Implemented in: 0.261.109 Retained analyses on failed orchestration replies covered in: 0.261.398 +Unified V2 Sources and saved findings implemented in: 0.261.419 Runs the real classic message/stream modules and React MessageList/Composer/store. Record/evidence requests cross the production Flask reader and serialized result @@ -26,6 +27,7 @@ import pytest import werkzeug +from PIL import Image from playwright.sync_api import expect REPO_ROOT = Path(__file__).resolve().parents[1] @@ -183,6 +185,13 @@ def handle(self, route): else: self.records(route) return + if path == "/api/get_citation" and request.method == "POST": + body = request.post_data_json + route.fulfill(json={ + "file_name": "Same name.txt", + "cited_text": f"Saved source passage from {body['document_id']}, page {body['page_number']}.", + }) + return if path == "/api/tabular/generated-output/runs/supporting-export": artifact = { **self.message["metadata"]["generated_analysis_artifacts"][0], @@ -215,6 +224,9 @@ def handle(self, route): if path.startswith("/api/conversations/") and path.endswith("/metadata"): route.fulfill(json={"conversation_id": path.split("/")[3], "title": "Saved review", "used_documents": []}) return + if path.startswith("/api/message/") and path.endswith("/metadata"): + route.fulfill(json={"metadata": self.message["metadata"]}) + return if path.endswith("/thoughts"): route.fulfill(json={"thoughts": []}) return @@ -230,9 +242,12 @@ def handle(self, route): if path == "/api/v2/orchestration/runs": route.fulfill(json={"runs": []}) return - if path == "/api/action-credentials/pending" and request.method == "GET": + if path in ("/api/action-credentials/pending", "/api/v2/user-input/pending") and request.method == "GET": route.fulfill(json={"requests": [], "csrf_token": "fixture-csrf"}) return + if path == "/api/v2/orchestration/questions" and request.method == "GET": + route.fulfill(json={"questions": [], "csrf_token": "fixture-csrf"}) + return if path == "/api/msgraph/pending-actions" and request.method == "GET": route.fulfill(json={"success": True, "pending_actions": [], "continuation_token": None}) return @@ -383,7 +398,58 @@ def mount(*, history=True, orchestration=False, shell=False, renderer_override=N def result(ui): - return ui.page.get_by_role("region", name="Saved analysis").first + open_sources(ui) + return ui.page.get_by_role("region", name="Saved findings" if ui.renderer == "v2" else "Saved analysis").first + + +def plural_review(ui, index): + name = f"Saved review {index + 1}" if ui.renderer == "v2" else ( + "Saved analysis: Deck claims" if index == 0 else "Saved analysis: Reviewer comments " + ) + return ui.page.get_by_role("region", name=name, exact=True) + + +def open_sources(ui): + """Use V2's existing Sources action; classic retains its inline result.""" + if ui.renderer != "v2": + return + inspector = ui.page.locator('[id^="message-inspector-"]').first + if not inspector.count() or inspector.get_by_role("button", name=re.compile(r"^Sources")).get_attribute("aria-pressed") != "true": + ui.page.get_by_role("button", name=re.compile(r"^Show sources")).first.click() + + +def load_findings(ui, region=None): + region = region if region is not None else result(ui) + if ui.renderer == "classic": + region.locator("summary").filter(has_text="Findings and limitations").click() + + +def open_evidence(ui, record_index=0): + region = result(ui) + if ui.renderer == "v2": + finding = region.locator("details").filter( + has=ui.page.get_by_text(f"Finding {record_index + 1}", exact=True), + ) + finding.locator(":scope > summary").click() + finding.locator("summary").filter(has_text="Supporting passages").click() + else: + region.locator("summary").filter(has_text=f"Evidence for finding record-{record_index}").click() + + +def diagnostics(ui, index=0): + if ui.renderer == "v2": + ui.page.get_by_role("button", name="Message details", exact=True).first.click() + return ui.page.get_by_role("region", name="Saved review diagnostics").nth(index).get_by_role( + "link", name="View diagnostics as JSON (opens a new tab)", + ) + return ui.page.get_by_role("region", name="Saved analysis").nth(index).get_by_role( + "link", name="View diagnostics as JSON (opens a new tab)", + ) + + +def ask_review(ui, region=None): + region = region if region is not None else result(ui) + return region.get_by_role("button", name=re.compile(r"Ask about this (?:analysis|saved review)"), exact=True) def context(ui): @@ -452,32 +518,34 @@ def test_failed_run_keeps_each_saved_analysis_and_requires_explicit_selection(an ui = analysis_ui descriptors = retain_failed_analyses(ui) ui.mount() - deck = ui.page.get_by_role("region", name="Saved analysis: Deck claims", exact=True) - reviewer = ui.page.get_by_role("region", name="Saved analysis: Reviewer comments ", exact=True) + assert not [item for item in ui.api.requests if item["path"] == "/api/analysis_results"] + open_sources(ui) + deck = plural_review(ui, 0) + reviewer = plural_review(ui, 1) expect(deck).to_be_visible() expect(reviewer).to_be_visible() expect(reviewer).to_contain_text("Partial analysis") - expect(reviewer).to_contain_text("Source observations, not reconciled values.") + expect(reviewer).to_contain_text("not reconciled values") initial = context(ui) assert initial is None - assert not [item for item in ui.api.requests if item["path"] == "/api/analysis_results"] - deck.locator("summary").filter(has_text="Findings and limitations").click() - reviewer.locator("summary").filter(has_text="Findings and limitations").click() + load_findings(ui, deck) + load_findings(ui, reviewer) expect(deck).to_contain_text("Showing 1–25 of 60 records") expect(reviewer).to_contain_text("Showing 1–1 of 1 records") expect(reviewer).to_contain_text("Review remains open.") expect(reviewer).not_to_contain_text("Control 0") assert reviewer.locator("script, img").count() == 0 expected = {key: descriptors[1][key] for key in ("conversation_id", "message_id", "result_sha256")} - reviewer.get_by_role("button", name="Ask about this analysis").click() + ask_review(ui, reviewer).click() selected = context(ui) assert selected == expected ui.page.evaluate("() => window.analysisUI.reload()") after_reload = context(ui) assert after_reload == expected - diagnostic_url = reviewer.get_by_role("link", name="View diagnostics as JSON (opens a new tab)").get_attribute("href") + diagnostic_url = diagnostics(ui, 1).get_attribute("href") assert parse_qs(urlsplit(diagnostic_url).query)["result_sha256"] == [descriptors[1]["result_sha256"]] ui.page.set_viewport_size({"width": 390, "height": 844}) + open_sources(ui) expect(deck).to_be_visible() expect(reviewer).to_be_visible() overflows = ui.page.evaluate("document.documentElement.scrollWidth > window.innerWidth + 1") @@ -490,10 +558,9 @@ def test_failed_live_completion_keeps_plural_analyses_without_auto_selecting(ana ui.api.messages = [] ui.mount(history=False) ui.page.evaluate("() => window.analysisUI.send()") - expect(ui.page.get_by_role("region", name="Saved analysis: Deck claims", exact=True)).to_be_visible() - expect(ui.page.get_by_role( - "region", name="Saved analysis: Reviewer comments ", exact=True, - )).to_be_visible() + open_sources(ui) + expect(plural_review(ui, 0)).to_be_visible() + expect(plural_review(ui, 1)).to_be_visible() selected = context(ui) assert selected is None @@ -504,8 +571,9 @@ def test_malformed_plural_entry_is_visible_without_hiding_valid_findings(analysi ui.api.message["metadata"]["saved_analyses"][1]["semantics"]["mode"] = "unknown-contract" ui.mount() expect(ui.page.get_by_text("Saved analysis unavailable.", exact=False).first).to_be_visible() - deck = ui.page.get_by_role("region", name="Saved analysis: Deck claims", exact=True) - deck.locator("summary").filter(has_text="Findings and limitations").click() + open_sources(ui) + deck = plural_review(ui, 0) + load_findings(ui, deck) expect(deck).to_contain_text("Control 0") selected = context(ui) assert selected is None @@ -515,16 +583,15 @@ def test_masking_a_plural_reply_hides_every_result_and_clears_explicit_selection ui = analysis_ui retain_failed_analyses(ui) ui.mount() - reviewer = ui.page.get_by_role( - "region", name="Saved analysis: Reviewer comments ", exact=True, - ) - reviewer.get_by_role("button", name="Ask about this analysis").click() + open_sources(ui) + reviewer = plural_review(ui, 1) + ask_review(ui, reviewer).click() before = context(ui) assert before is not None ui.api.message["metadata"]["masked"] = True ui.api.fixture["message"]["metadata"]["masked"] = True ui.page.evaluate("() => window.analysisUI.reload()") - expect(ui.page.get_by_role("button", name="Ask about this analysis")).to_have_count(0) + expect(ui.page.get_by_role("button", name=re.compile(r"Ask about this (?:analysis|saved review)"))).to_have_count(0) expect(ui.page.get_by_role("link", name="View diagnostics as JSON (opens a new tab)")).to_have_count(0) selected = context(ui) assert selected is None @@ -535,23 +602,24 @@ def test_saved_analysis_page_two_evidence_download_and_safe_text(analysis_ui): ui.api.unsafe_values = True ui.mount() expect(ui.page.get_by_text(ANSWER, exact=True).first).to_be_visible() - expect(result(ui)).to_contain_text("0 of 60 records displayed") assert context(ui) == expected_context(ui) assert not [item for item in ui.api.requests if item["path"] == "/api/analysis_results"] - diagnostics = result(ui).get_by_role("link", name="View diagnostics as JSON (opens a new tab)") - expect(diagnostics).to_be_visible() - diagnostic_url = urlsplit(diagnostics.get_attribute("href")) + if ui.renderer == "classic": + expect(result(ui)).to_contain_text("0 of 60 records displayed") + else: + expect(ui.page.get_by_role("region", name="Saved analysis")).to_have_count(0) + diagnostic_link = diagnostics(ui) + expect(diagnostic_link).to_be_visible() + diagnostic_url = urlsplit(diagnostic_link.get_attribute("href")) assert diagnostic_url.path == "/api/analysis_results" assert parse_qs(diagnostic_url.query) == { **{key: [value] for key, value in expected_context(ui).items()}, "representation": ["diagnostics"], } - expect(diagnostics).to_have_attribute("rel", "noopener noreferrer") - disclosure = result(ui).locator("summary").filter(has_text="Findings and limitations") - disclosure.focus() - disclosure.press("Enter") + expect(diagnostic_link).to_have_attribute("rel", "noopener noreferrer") + load_findings(ui) expect(result(ui)).to_contain_text("Showing 1–25 of 60 records") - expect(result(ui)).to_contain_text("1 of 1 sources on this page") + expect(result(ui)).to_contain_text("1 of 1 source") expect(result(ui)).to_contain_text("") assert result(ui).locator("img, script").count() == 0 assert not ui.page.evaluate("Boolean(window.analysisXss)") @@ -559,13 +627,13 @@ def test_saved_analysis_page_two_evidence_download_and_safe_text(analysis_ui): expect(result(ui)).to_contain_text("Showing 26–50 of 60 records") expect(result(ui)).to_contain_text("Control 25") expect(result(ui)).not_to_contain_text("Control 0") - result(ui).locator("summary").filter(has_text="Evidence for finding record-25").click() + open_evidence(ui, 25) expect(result(ui).get_by_role("blockquote")).to_contain_text("") expect(result(ui)).to_contain_text("page 26") reads = [item["query"] for item in ui.api.requests if item["path"] == "/api/analysis_results"] - assert reads[1]["offset"] == ["25"] - assert reads[1]["limit"] == ["25"] - assert reads[2]["record_id"] == ["record-25"] + assert reads[-2]["offset"] == ["25"] + assert reads[-2]["limit"] == ["25"] + assert reads[-1]["record_id"] == ["record-25"] assert all(query["result_sha256"] == [expected_context(ui)["result_sha256"]] for query in reads) expect(ui.page.get_by_text(ANSWER, exact=True).first).to_be_visible() expect(result(ui)).not_to_contain_text("RAW-NOTE-ONLY") @@ -646,9 +714,9 @@ def test_analysis_with_expanded_navigation_has_only_local_table_scrolling(analys dimensions = ui.page.evaluate("({viewport: innerWidth, document: document.documentElement.scrollWidth})") assert dimensions["document"] <= width + 1, dimensions close.click() - result(ui).locator("summary").filter(has_text="Findings and limitations").click() + load_findings(ui) expect(result(ui)).to_contain_text("Showing 1–25 of 60 records") - result(ui).locator("summary").filter(has_text="Evidence for finding record-0").click() + open_evidence(ui) expect(result(ui).get_by_role("blockquote")).to_be_visible() ui.page.locator("summary").filter(has_text="Downloads").click() button = download_button(ui) @@ -675,7 +743,7 @@ def test_download_prefers_the_server_utf8_filename(analysis_ui): def test_saved_explanation_request_bypasses_sources_and_orchestration(analysis_ui): ui = analysis_ui ui.mount(orchestration=True) - result(ui).get_by_role("button", name="Ask about this analysis", exact=True).click() + ask_review(ui).click() expect(ui.page.get_by_text("Explaining the saved analysis", exact=False)).to_be_visible() draft = ui.page.get_by_role("textbox", name="Message", exact=True) draft.fill("Explain the finding on the second page") @@ -689,7 +757,8 @@ def test_saved_explanation_request_bypasses_sources_and_orchestration(analysis_u assert body[key] is False assert not body["selected_document_ids"] assert "document_action" not in body and "analyze" not in body - assert not [item for item in ui.api.requests if item["path"].startswith("/api/v2/orchestration/")] + assert not [item for item in ui.api.requests + if item["method"] == "POST" and item["path"].startswith("/api/v2/orchestration/")] assert context(ui) == expected_context(ui), "Explanation messages keep the original result pointer." ui.page.get_by_role("button", name="Remove saved analysis context").click() assert context(ui) is None @@ -697,7 +766,7 @@ def test_saved_explanation_request_bypasses_sources_and_orchestration(analysis_u assert context(ui) is None, "History hydration must not undo the user's removal." ui.page.evaluate("() => window.analysisUI.open()") assert context(ui) is None, "Reselecting the same conversation must preserve explicit context removal." - result(ui).get_by_role("button", name="Ask about this analysis", exact=True).click() + ask_review(ui).click() choose_new_source(ui) assert context(ui) is None ui.page.evaluate("() => window.analysisUI.fresh()") @@ -716,9 +785,9 @@ def test_denied_missing_and_stale_results_are_visible_not_empty_success(analysis else: ui.api.message["metadata"]["saved_analysis"]["result_sha256"] = "0" * 64 ui.mount() - result(ui).locator("summary").filter(has_text="Findings and limitations").click() + load_findings(ui) expect(result(ui)).to_contain_text("stale" if status == 409 else "unavailable") - expect(result(ui).get_by_role("button", name="Ask about this analysis")).to_have_count(0) + expect(ask_review(ui)).to_have_count(0) expect(download_button(ui)).not_to_be_visible() expect(result(ui)).not_to_contain_text("No accepted findings") assert context(ui) is None @@ -736,10 +805,12 @@ def test_validation_state_is_not_execution_completion(analysis_ui, status, notic ui.api.message["metadata"]["saved_analysis"]["validation_status"] = status ui.mount() expect(result(ui)).to_contain_text(notice) - result(ui).locator("summary").filter(has_text="Findings and limitations").click() + load_findings(ui) expect(result(ui)).to_contain_text("Showing 1–25") expect(result(ui)).to_contain_text(notice) - expect(result(ui)).to_contain_text("Judgments are not independently verified.") + if ui.renderer == "v2": + result(ui).locator("summary").filter(has_text="Analysis limits").click() + expect(result(ui)).to_contain_text("not independent factual verification" if status == "valid" else notice) def test_unavailable_or_masked_history_exposes_no_saved_result_controls(analysis_ui): @@ -748,13 +819,13 @@ def test_unavailable_or_masked_history_exposes_no_saved_result_controls(analysis ui.api.message["metadata"]["saved_analysis"]["available"] = False ui.mount() expect(result(ui)).to_contain_text("unavailable") - expect(ui.page.get_by_role("button", name="Ask about this analysis")).to_have_count(0) + expect(ui.page.get_by_role("button", name=re.compile(r"Ask about this (?:analysis|saved review)"))).to_have_count(0) assert context(ui) is None assert not [item for item in ui.api.requests if item["path"] == "/api/analysis_results"] ui.api.message["metadata"]["saved_analysis"]["available"] = True ui.api.message["metadata"]["masked"] = True ui.page.evaluate("() => window.analysisUI.reload()") - expect(ui.page.get_by_role("button", name="Ask about this analysis")).not_to_be_visible() + expect(ui.page.get_by_role("button", name=re.compile(r"Ask about this (?:analysis|saved review)"))).not_to_be_visible() expect(download_button(ui)).not_to_be_visible() assert context(ui) is None @@ -781,7 +852,7 @@ def test_late_findings_response_cannot_follow_conversation_navigation(analysis_u ui.api.hold_page = True ui.mount() with ui.page.expect_request("**/api/analysis_results?*"): - result(ui).locator("summary").filter(has_text="Findings and limitations").click() + load_findings(ui) ui.page.evaluate("() => window.analysisUI.change('another-conversation')") assert context(ui) is None for pending in ui.api.pending_pages: @@ -806,10 +877,10 @@ def test_supporting_export_completion_never_replaces_the_answer(analysis_ui): def test_evidence_rechecks_access_and_removes_the_previously_loaded_page(analysis_ui): ui = analysis_ui ui.mount() - result(ui).locator("summary").filter(has_text="Findings and limitations").click() + load_findings(ui) expect(result(ui)).to_contain_text("Control 0") ui.api.fixture["state"]["conversation_allowed"] = False - result(ui).locator("summary").filter(has_text="Evidence for finding record-0").click() + open_evidence(ui) expect(result(ui)).to_contain_text("unavailable") expect(result(ui)).not_to_contain_text("Control 0") expect(download_button(ui)).not_to_be_visible() @@ -819,10 +890,10 @@ def test_evidence_rechecks_access_and_removes_the_previously_loaded_page(analysi def test_source_revocation_does_not_withdraw_conversation_owned_findings(analysis_ui): ui = analysis_ui ui.mount() - result(ui).locator("summary").filter(has_text="Findings and limitations").click() + load_findings(ui) expect(result(ui)).to_contain_text("Control 0") ui.api.fixture["state"]["source_allowed"] = False - result(ui).locator("summary").filter(has_text="Evidence for finding record-0").click() + open_evidence(ui) expect(result(ui).get_by_role("blockquote")).to_contain_text("Owner: unassigned") expect(result(ui)).to_contain_text("Control 0") selected = context(ui) @@ -833,13 +904,17 @@ def test_transient_page_failure_can_retry_without_exposing_diagnostics(analysis_ ui = analysis_ui ui.api.page_failures = 1 ui.mount() - result(ui).locator("summary").filter(has_text="Findings and limitations").click() + load_findings(ui) expect(result(ui).get_by_role("button", name="Retry findings")).to_be_visible() expect(result(ui)).not_to_contain_text("PRIVATE_DIAGNOSTIC") result(ui).get_by_role("button", name="Retry findings").click() expect(result(ui)).to_contain_text("Showing 1–25") - result(ui).locator("summary").filter(has_text="Findings and limitations").click() - expect(result(ui)).to_contain_text("0 of 60 records displayed") + if ui.renderer == "classic": + load_findings(ui) + expect(result(ui)).to_contain_text("0 of 60 records displayed") + else: + ui.page.get_by_role("button", name=re.compile(r"^Show sources")).first.click() + expect(ui.page.get_by_role("region", name="Saved analysis")).to_have_count(0) def test_late_stream_completion_cannot_reselect_another_conversations_analysis(analysis_ui): @@ -893,7 +968,9 @@ def test_saved_records_are_not_misinterpreted_as_answer_mask_offsets(analysis_ui }""" ui.page.get_by_text(ANSWER, exact=True).evaluate(select_text) expect(ui.page.get_by_role("button", name="Mask selection", exact=True)).to_be_visible() - result(ui).locator("summary").filter(has_text="Findings and limitations").click() + load_findings(ui) + if ui.renderer == "v2": + result(ui).get_by_text("Finding 1", exact=True).click() result(ui).get_by_text("Control 0", exact=True).evaluate(select_text) expect(ui.page.get_by_role("button", name="Mask selection", exact=True)).not_to_be_visible() @@ -933,3 +1010,119 @@ def test_shared_followup_uses_the_public_pointer_without_source_invocation(analy assert target["source_mode"] == "saved_analysis" assert body["hybrid_search"] is False and body["image_generation"] is False assert not body["selected_document_ids"] + + +@pytest.mark.parametrize("analysis_ui", ["v2"], indirect=True) +@pytest.mark.parametrize("width,theme", [(1280, "light"), (1280, "dark"), (390, "light"), (390, "dark")]) +def test_unified_sources_is_answer_first_and_keyboard_readable(analysis_ui, width, theme): + ui = analysis_ui + ui.page.set_viewport_size({"width": width, "height": 900}) + ui.api.message["metadata"]["saved_analysis"]["label"] = "Read document body for comment context" + ui.api.message["hybrid_citations"] = [ + {"citation_id": f"{document}_{page}", "document_id": document, "page_number": page, + "file_name": "Same name.txt"} + for document, page in (("document-1", 1), ("document-1", 2), ("document-2", 1)) + ] + ui.api.message["web_search_citations"] = [{"url": "https://example.com/review", "title": "Public reference"}] + ui.mount() + ui.page.evaluate("(dark) => document.documentElement.classList.toggle('dark', dark)", theme == "dark") + expect(ui.page.get_by_text(ANSWER, exact=True)).to_be_visible() + expect(ui.page.get_by_role("region", name="Saved findings", exact=True)).to_have_count(0) + expect(ui.page.get_by_text("Read document body for comment context", exact=True)).to_have_count(0) + assert not [item for item in ui.api.requests if item["path"] == "/api/analysis_results"] + + sources = ui.page.get_by_role("button", name="Show sources (4)", exact=True) + expect(sources).to_have_count(1) + sources.focus() + sources.press("Enter") + expect(ui.page.get_by_text(ANSWER, exact=True)).to_be_in_viewport() + findings = result(ui) + expect(findings).to_contain_text("60 findings") + expect(findings).to_contain_text("Showing 1–25 of 60 records") + expect(ui.page.get_by_role("heading", name="Document citations (3)", exact=True)).to_be_visible() + expect(ui.page.get_by_role("link", name="Public reference", exact=False)).to_be_visible() + expect(ui.page.get_by_role("link", name="View diagnostics as JSON (opens a new tab)")).to_have_count(0) + expect(findings.get_by_text("Control 0", exact=True)).not_to_be_visible() + expect(ui.page.get_by_text("Read document body for comment context", exact=True)).to_have_count(0) + first = findings.get_by_text("Finding 1", exact=True) + first.locator("..").focus() + first.locator("..").press("Enter") + expect(findings.get_by_text("Control 0", exact=True)).to_be_visible() + expect(findings.get_by_role("blockquote")).to_have_count(0) + passages = findings.locator("summary").filter(has_text="Supporting passages").first + passages.focus() + passages.press("Enter") + expect(findings.get_by_role("blockquote")).to_contain_text("Owner: unassigned") + expect(ask_review(ui)).to_contain_text("saved review") + assert ui.page.evaluate("document.documentElement.scrollWidth <= innerWidth + 1") + reads = [item["query"] for item in ui.api.requests if item["path"] == "/api/analysis_results"] + assert len(reads) == 2 and reads[0]["limit"] == ["25"] and reads[1]["record_id"] == ["record-0"] + assert not any(query.get("representation") == ["diagnostics"] for query in reads) + artifacts = REPO_ROOT / "ui_tests" / "artifacts" + artifacts.mkdir(exist_ok=True) + screenshot = artifacts / f"unified-sources-{width}-{theme}.png" + ui.page.screenshot(path=str(screenshot), full_page=True) + value = findings.get_by_text("Control 0", exact=True) + bounds = value.bounding_box() + assert bounds is not None + foreground = [int(channel) for channel in re.findall( + r"\d+", value.evaluate("(element) => getComputedStyle(element).color"), + )[:3]] + with Image.open(screenshot) as image: + background = image.convert("RGB").getpixel((int(bounds["x"] + 1), int(bounds["y"] + 1))) + + def luminance(color): + channels = [channel / 255 for channel in color[:3]] + linear = [channel / 12.92 if channel <= 0.04045 else ((channel + 0.055) / 1.055) ** 2.4 + for channel in channels] + return sum(channel * weight for channel, weight in zip(linear, (0.2126, 0.7152, 0.0722))) + + levels = [luminance(foreground), luminance(background)] + contrast = (max(levels) + 0.05) / (min(levels) + 0.05) + assert contrast >= 4.5, f"Saved finding contrast is {contrast:.2f}:1" + + # Identical filenames and page numbers must not substitute a different document's identity. + citations = ui.page.get_by_role("region", name="Answer citations") + citations.get_by_role("button", name="Same name.txt 1", exact=True).nth(1).click() + expect(ui.page.get_by_role("dialog", name="Citation", exact=True)).to_contain_text( + "Saved source passage from document-2, page 1.", + ) + request = next(item for item in reversed(ui.api.requests) if item["path"] == "/api/get_citation") + assert request["body"]["document_id"] == "document-2" + assert request["body"]["citation_id"] == "document-2_1" + ui.page.get_by_role("button", name="Close citation", exact=True).click() + diagnostic_link = diagnostics(ui) + expect(diagnostic_link).to_be_visible() + expect(ui.page.get_by_text("Read document body for comment context", exact=True)).to_be_visible() + + +@pytest.mark.parametrize("analysis_ui", ["v2"], indirect=True) +def test_citations_without_saved_findings_keep_the_existing_sources_destination(analysis_ui): + ui = analysis_ui + ui.api.message["metadata"] = {} + ui.api.message["hybrid_citations"] = [] + ui.mount() + open_sources(ui) + expect(ui.page.get_by_text("This response did not cite any sources.", exact=True)).to_be_visible() + expect(ui.page.get_by_role("region", name="Saved findings", exact=True)).to_have_count(0) + assert not [item for item in ui.api.requests if item["path"] == "/api/analysis_results"] + ui.api.message["web_search_citations"] = [{"url": "https://example.com/reference", "title": "Cited reference"}] + ui.page.evaluate("() => window.analysisUI.reload()") + expect(ui.page.get_by_role("link", name="Cited reference", exact=False)).to_be_visible() + expect(ui.page.get_by_role("region", name="Saved findings", exact=True)).to_have_count(0) + + +@pytest.mark.parametrize("analysis_ui", ["v2"], indirect=True) +def test_sources_does_not_link_conflicting_recorded_citation_identities(analysis_ui): + ui = analysis_ui + ui.api.message["metadata"] = {} + ui.api.message["hybrid_citations"] = [ + {"citation_id": "ambiguous_1", "document_id": document, "page_number": 1, + "file_name": "Same name.txt"} for document in ("document-1", "document-2") + ] + ui.mount() + open_sources(ui) + citations = ui.page.get_by_role("region", name="Answer citations") + expect(citations.get_by_text("Same name.txt", exact=True)).to_have_count(2) + expect(citations.get_by_role("button")).to_have_count(0) + assert not [item for item in ui.api.requests if item["path"] == "/api/get_citation"]