From 946fcc9918660ebc0ed7ae37a27e7ac8afcf9786 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Thu, 5 Nov 2026 19:34:13 -0500 Subject: [PATCH] Fix selected-document orchestration planning recovery Align Analyze option contracts, retain selected sources through bounded plan correction, and add privacy-safe debug diagnostics while preserving Analyze and pre-plan inspection. Update documentation and application version to 0.261.411. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../single_app/functions_appinsights.py | 1 + .../functions_document_analysis_results.py | 75 ++- .../functions_orchestration_planner.py | 257 ++++++-- .../functions_orchestration_registry.py | 31 +- .../functions_orchestration_schema.py | 38 +- docs/admin/orchestration.md | 10 + .../features/CHAT_ORCHESTRATION.md | 34 +- ...STRATION_DOCUMENT_PLANNING_RECOVERY_FIX.md | 153 +++++ docs/reference/logging-tags.md | 74 ++- ...rchestration_document_planning_recovery.py | 591 ++++++++++++++++++ 11 files changed, 1180 insertions(+), 86 deletions(-) create mode 100644 docs/explanation/fixes/ORCHESTRATION_DOCUMENT_PLANNING_RECOVERY_FIX.md create mode 100644 functional_tests/test_orchestration_document_planning_recovery.py diff --git a/application/single_app/config.py b/application/single_app/config.py index 23865fe67..d8c83bb3f 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -103,7 +103,7 @@ EXECUTOR_MAX_WORKERS = 30 CHAT_STREAM_EXECUTOR_MAX_WORKERS = max(8, int(os.getenv('SIMPLECHAT_CHAT_STREAM_WORKERS', '32') or '32')) SESSION_TYPE = 'filesystem' -VERSION = "0.261.410" +VERSION = "0.261.411" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_appinsights.py b/application/single_app/functions_appinsights.py index 8b75656bb..caa1392de 100644 --- a/application/single_app/functions_appinsights.py +++ b/application/single_app/functions_appinsights.py @@ -134,6 +134,7 @@ }) LOGGER_WORKFLOW_HASH_KEYS = frozenset({ "conversationidhash", "turnidhash", "runidhash", "stepidhash", "producerstepidhash", + "documentidhash", "plannerbindinghash", "documentsethash", }) # Application-owned snake_case codes; any other text under these keys is dropped. LOGGER_WORKFLOW_CODE_KEYS = frozenset({ diff --git a/application/single_app/functions_document_analysis_results.py b/application/single_app/functions_document_analysis_results.py index 342064d27..d0d4b02bc 100644 --- a/application/single_app/functions_document_analysis_results.py +++ b/application/single_app/functions_document_analysis_results.py @@ -60,41 +60,80 @@ }) +class AnalysisOptionsError(ValueError): + """An application-owned option rejection, without recording supplied values.""" + + def __init__(self, message, *, rule): + super().__init__(message) + self.rule = rule + + def normalize_analysis_options(analysis_options=None, transformation_spec=None): """Normalize explicit semantics while preserving unversioned record behavior.""" options = {} if analysis_options is None else deepcopy(analysis_options) - if not isinstance(options, dict) or set(options) - { - 'version', 'mode', 'required_fields', 'transformation_spec', - }: - raise ValueError('Unsupported analysis options. Declare required field names or supported calculation rules.') + if not isinstance(options, dict): + raise AnalysisOptionsError('Analysis options must be an object.', rule='options_object_required') + if set(options) - {'version', 'mode', 'required_fields', 'transformation_spec'}: + raise AnalysisOptionsError( + 'Unsupported analysis options. Declare required field names or supported calculation rules.', + rule='unsupported_option_fields', + ) version = options.get('version', ANALYSIS_OPTIONS_VERSION) if version not in (ANALYSIS_OPTIONS_VERSION, ANALYSIS_SEMANTICS_VERSION): - raise ValueError('Unsupported analysis options version.') + raise AnalysisOptionsError('Unsupported analysis options version.', rule='unsupported_options_version') mode = options.get('mode', 'narrative_findings' if version == ANALYSIS_OPTIONS_VERSION else None) if version == ANALYSIS_OPTIONS_VERSION and mode != 'narrative_findings': - raise ValueError('This Analyze path supports narrative findings, not a required row cardinality or custom schema.') + raise AnalysisOptionsError( + 'This Analyze path supports narrative findings, not a required row cardinality or custom schema.', + rule='version_mode_mismatch', + ) if version == ANALYSIS_SEMANTICS_VERSION and mode not in (SOURCE_OBSERVATIONS, RECONCILED_RECORDS): - raise ValueError('Declare supported Analyze semantics; no mode is inferred from source text.') + raise AnalysisOptionsError( + 'Declare supported Analyze semantics; no mode is inferred from source text.', + rule='supported_mode_required', + ) fields = options.get('required_fields', []) if ( not isinstance(fields, list) or len(fields) > 200 or any(not isinstance(field, str) or not field.strip() or len(field) > 128 for field in fields) or len({field.strip() for field in fields}) != len(fields) ): - raise ValueError('Required analysis fields must be a unique list of at most 200 nonempty field names.') + raise AnalysisOptionsError( + 'Required analysis fields must be a unique list of at most 200 nonempty field names.', + rule='required_fields_invalid', + ) supplied_spec = options.get('transformation_spec') if supplied_spec is not None and not isinstance(supplied_spec, dict): - raise ValueError('Explicit analysis calculation rules must be a supported rule object.') - if transformation_spec is not None: - if not isinstance(transformation_spec, dict): - raise ValueError('Explicit analysis calculation rules must be a supported rule object.') - normalized_explicit = normalize_tabular_transformation_spec(transformation_spec) - if supplied_spec is not None and normalize_tabular_transformation_spec(supplied_spec) != normalized_explicit: - raise ValueError('Conflicting explicit analysis calculation rules were supplied.') - supplied_spec = transformation_spec - normalized_spec = normalize_tabular_transformation_spec(supplied_spec) + raise AnalysisOptionsError( + 'Explicit analysis calculation rules must be a supported rule object.', + rule='calculation_rules_object_required', + ) + try: + if transformation_spec is not None: + if not isinstance(transformation_spec, dict): + raise AnalysisOptionsError( + 'Explicit analysis calculation rules must be a supported rule object.', + rule='calculation_rules_object_required', + ) + normalized_explicit = normalize_tabular_transformation_spec(transformation_spec) + if supplied_spec is not None and normalize_tabular_transformation_spec(supplied_spec) != normalized_explicit: + raise AnalysisOptionsError( + 'Conflicting explicit analysis calculation rules were supplied.', + rule='calculation_rules_conflict', + ) + supplied_spec = transformation_spec + normalized_spec = normalize_tabular_transformation_spec(supplied_spec) + except AnalysisOptionsError: + raise + except ValueError as exc: + raise AnalysisOptionsError( + 'Explicit analysis calculation rules are unsupported.', rule='calculation_rules_invalid', + ) from exc if mode == SOURCE_OBSERVATIONS and (fields or normalized_spec): - raise ValueError('Source observations cannot declare reconciled fields or calculations.') + raise AnalysisOptionsError( + 'Source observations cannot declare reconciled fields or calculations.', + rule='observation_requirements_conflict', + ) return { 'version': version, 'mode': mode, diff --git a/application/single_app/functions_orchestration_planner.py b/application/single_app/functions_orchestration_planner.py index b4b6b730a..b4e736b67 100644 --- a/application/single_app/functions_orchestration_planner.py +++ b/application/single_app/functions_orchestration_planner.py @@ -25,11 +25,13 @@ Every plan uses the Gather / Reason / Render contract. There is one planner prompt, ``PLANNER_SYSTEM_PROMPT``, and one validator. -Version: 0.261.367 +Version: 0.261.411 Gathered results must be read, and an action's chart reaches the answer, in: 0.261.293 Empty-history clarification continuation fixed in: 0.261.367 +Selected-document planning recovery and debug diagnostics added in: 0.261.411 """ +import hashlib import json import logging import re @@ -40,7 +42,7 @@ from azure.identity import DefaultAzureCredential, get_bearer_token_provider from config import cognitive_services_scope -from functions_appinsights import log_event, workflow_log_context +from functions_appinsights import is_debug_enabled, log_event, workflow_log_context from functions_document_analysis_results import narrative_analysis_options from functions_orchestration_context import conversation_reference_messages, resolve_elicitation_candidates from functions_orchestration_deliverables import build_deliverable_availability @@ -75,6 +77,7 @@ WORKFLOW_RUN_INVALID_CODE, PlanValidationError, available_information_plan, + effective_plan_document_ids, normalize_elicitation, normalize_plan, plan_document_ids, @@ -100,6 +103,7 @@ PLANNER_TEMPERATURE = 0.1 # Known declaration and source-binding failures share one correction round. PLAN_REPAIR_ATTEMPTS = 1 +PLANNER_DEBUG_MAX_ITEMS = 32 # A workflow proposal gets the same correction round; one that still fails is dropped from the # plan, and the rest of the plan runs without it. WORKFLOW_REPAIR_CODES = frozenset({WORKFLOW_BLUEPRINT_INVALID_CODE, WORKFLOW_PROPOSAL_NOT_CONSUMABLE_CODE}) @@ -119,6 +123,7 @@ WORKFLOW_CONTEXT_UNAVAILABLE_RULE = 'workflow_context_unavailable' REPAIRABLE_PLAN_CODES = frozenset({ 'deliverables_invalid', 'source_kind_invalid', 'source_binding_required', + 'analysis_options_invalid', 'selected_requirements_missing', 'conversation_retrieval_conflict', *WORKFLOW_REPAIR_CODES, *WORKFLOW_RUN_REPAIR_CODES, *WORKFLOW_RESULTS_REPAIR_CODES, }) @@ -133,6 +138,13 @@ SOURCE_BINDING_FAILURE_MESSAGE = ( 'The plan could not bind the selected documents. Please retry your request.' ) +ANALYSIS_OPTIONS_FAILURE_MESSAGE = ( + 'The document-analysis plan could not be validated. Please retry your request.' +) +SELECTED_REQUIREMENTS_FAILURE_MESSAGE = ( + 'The plan could not include all selected documents or operations. ' + 'Your selection is unchanged. Please retry your request.' +) WORKFLOW_FAILURE_MESSAGE = ( 'The plan could not include the workflow you asked for. Please retry, or create the ' 'workflow in Workflows.' @@ -1341,6 +1353,31 @@ def plan_repair_message(error, *, handoff_present=False): ``handoff_present`` says the rejected plan holds an enabled workflow_handoff step, so an answer the plan was told to write is steered away from the handed-off part. """ + if error.code == 'analysis_options_invalid': + return ( + f'The server rejected the Analyze options ({error.rule}). ' + 'For ordinary narrative analysis, omit analysis_options so the server supplies its defaults, ' + 'or use {"version":"analysis-options-v2","mode":"source_observations"} without required_fields ' + 'or calculation rules. For reconciled fields/calculations, use ' + '{"version":"analysis-options-v2","mode":"reconciled_records"} and supported rules. ' + 'Do not discard user-required fields or calculations to make observation mode validate. ' + 'Do not mix legacy narrative_findings with a v2 version or omit the version for a v2 mode. ' + 'Preserve every selected document, requested operation and deliverable, and bind the actual ' + 'findings to the answer. Return the complete corrected plan as one JSON object.' + ) + if error.code == 'selected_requirements_missing': + return ( + f'The plan omitted selected work ({error.rule}). Recheck required_capabilities and every ' + 'ID in user_selected.documents. Use candidate_documents and selected_sources for ' + 'server-verified identities and types, not permission to add unselected sources. ' + 'Retain every selected document through compatible, enabled source-reading work with its ' + 'exact authorized document ID, not merely a filename in an instruction. ' + 'Use narrative Analyze only for narrative sources; use offered spreadsheet operations for ' + 'tabular sources, including the available native inspection/evidence path when appropriate. ' + 'Do not widen retrieval, remove a selection, invent a source, or replace grounded work with ' + 'a general-knowledge answer. Discovery samples are not complete source evidence. ' + 'Preserve every requested deliverable and return the complete corrected plan as one JSON object.' + ) if getattr(error, 'code', None) == 'conversation_retrieval_conflict': return ( 'This request must use the selected conversation messages, not workspace or web retrieval. ' @@ -1660,8 +1697,11 @@ def plan_request( actions = context.get('actions') or [] correlation = workflow_log_context(conversation_id=conversation_id, turn_id=turn_id) + deployment, current_proposal = None, None + proposal_documents = {} def _failure(reason, error=None, *, stage=None, message=None, attempt=None): + _diagnose(stage or 'planning_failed', current_proposal, attempt=attempt, error=error) log_event( '[ORCHESTRATION_PLANNER] The request could not be planned.', level=logging.WARNING, extra={ @@ -1687,8 +1727,130 @@ def _failure(reason, error=None, *, stage=None, message=None, attempt=None): if isinstance(candidate, dict) and candidate.get('source_kind') } + def _diagnose(stage, proposal=None, *, attempt=None, error=None): + if not is_debug_enabled(): + return + raw_steps = proposal.get('steps') if isinstance(proposal, dict) else None + steps = [step for step in raw_steps if isinstance(step, dict)] if isinstance(raw_steps, list) else [] + readable = {'steps': [ + { + 'capability_id': step.get('capability_id'), + 'enabled': step.get('enabled', True), + 'arguments': step.get('arguments') if isinstance(step.get('arguments'), dict) else {}, + } for step in steps + ]} + selected = {value for value in (seeds or {}).get('document_ids') or [] if isinstance(value, str)} + authorized = {value for value in authorized_document_ids or [] if isinstance(value, str)} + used = set(effective_plan_document_ids(readable, seeds)) + missing = selected - used + if stage == 'plan_proposal': + proposal_documents[attempt] = used + binding = [ + deployment, + *[getattr(planner_model, name, None) for name in ( + 'source', 'model_id', 'endpoint_id', 'provider', 'behavior_name', + )], + ] + details = { + **correlation, 'stage': stage, 'attempt': attempt, + 'selected_document_count': len(selected), 'authorized_document_count': len(authorized), + 'authorization_scope_supplied': authorized_document_ids is not None, + 'used_document_count': len(used), 'matched_document_count': len(selected & used), + 'missing_document_count': len(missing), 'unselected_document_count': len(used - selected), + 'required_capability_count': len(required), + 'missing_capability_count': len(set(required) - { + step['capability_id'] for step in steps + if isinstance(step.get('capability_id'), str) and step.get('enabled', True) + }), + 'step_count': len(steps), 'omitted_step_count': max(0, len(steps) - PLANNER_DEBUG_MAX_ITEMS), + 'remaining_corrections': max(0, PLAN_REPAIR_ATTEMPTS + 1 - (attempt or 1)), + 'normalization_lost_document_count': ( + len(proposal_documents.get(attempt, set()) - used) + if stage in ('plan_normalized', 'selected_requirements', 'plan_validated') else 0 + ), + 'model_routing_auto': (seeds or {}).get('model_routing') == 'auto', + 'planner_binding_provided': planner_model is not None, + 'planner_binding_hash': hashlib.sha256(json.dumps([ + value if isinstance(value, str) else None for value in binding + ]).encode('utf-8')).hexdigest(), + 'document_set_hash': hashlib.sha256(json.dumps(sorted(used)).encode('utf-8')).hexdigest(), + } + if isinstance(error, PlanValidationError): + details.update(validation_code=error.code, validation_rule=error.rule) + source_ids = sorted(selected | used, key=lambda value: (value not in missing, value)) + details['omitted_document_count'] = max(0, len(source_ids) - PLANNER_DEBUG_MAX_ITEMS) + log_event('[ORCHESTRATION_PLANNER] Planning diagnostic.', extra=details) + candidates = { + source['document_id']: source + for source in [*(context.get('candidate_documents') or []), *(context.get('selected_sources') or [])] + if isinstance(source, dict) and isinstance(source.get('document_id'), str) + } + for document_id in source_ids[:PLANNER_DEBUG_MAX_ITEMS]: + source = candidates.get(document_id, {}) + inspection = source.get('inspection') + log_event('[ORCHESTRATION_PLANNER] Source binding diagnostic.', extra={ + **correlation, 'stage': stage, 'attempt': attempt, + 'document_id_hash': hashlib.sha256(document_id.encode('utf-8', errors='replace')).hexdigest(), + 'selected': document_id in selected, 'used': document_id in used, + 'authorized': document_id in authorized if authorized_document_ids is not None else None, + 'source_is_narrative': source.get('source_kind') == 'narrative', + 'source_is_tabular': source.get('source_kind') == 'tabular', + 'source_availability_known': source.get('availability') in ('available', 'unavailable'), + 'source_available': source.get('availability') == 'available', + 'inspection_limited': isinstance(inspection, dict) and inspection.get('status') == 'limited', + }) + for step in steps[:PLANNER_DEBUG_MAX_ITEMS]: + arguments = step.get('arguments') if isinstance(step.get('arguments'), dict) else {} + options = arguments.get('analysis_options') + option_fields = options if isinstance(options, dict) else {} + log_event('[ORCHESTRATION_PLANNER] Step binding diagnostic.', extra={ + **correlation, **workflow_log_context(step_id=step.get('step_id')), + 'stage': stage, 'attempt': attempt, + 'capability_id': ( + step['capability_id'] if isinstance(step.get('capability_id'), str) + and step['capability_id'] in available_ids else 'unknown' + ), + 'enabled': step.get('enabled', True) is not False, + 'document_ids_present': 'document_ids' in arguments, + 'reference_present': 'reference' in arguments, + 'source_input_present': isinstance(step.get('inputs'), dict) and 'sources' in step['inputs'], + 'analysis_options_present': 'analysis_options' in arguments, + 'analysis_options_object': isinstance(options, dict), + 'options_version_present': 'version' in option_fields, + 'options_version_v1': option_fields.get('version') == 'analysis-options-v1', + 'options_version_v2': option_fields.get('version') == 'analysis-options-v2', + 'options_mode_present': 'mode' in option_fields, + 'options_mode_observations': option_fields.get('mode') == 'source_observations', + 'options_mode_records': option_fields.get('mode') == 'reconciled_records', + 'options_mode_legacy': option_fields.get('mode') == 'narrative_findings', + 'required_field_count': ( + len(option_fields['required_fields']) + if isinstance(option_fields.get('required_fields'), list) else 0 + ), + 'calculation_rules_present': bool(option_fields.get('transformation_spec')), + 'unknown_option_field_count': len(set(option_fields) - { + 'version', 'mode', 'required_fields', 'transformation_spec', + }), + }) + + def _correction_messages(error, stage, attempt, reply, *, handoff_present=False): + _diagnose(stage, current_proposal, attempt=attempt, error=error) + log_event( + '[ORCHESTRATION_PLANNER] Asking the planner to correct a rejected plan.', + level=logging.INFO, extra={ + **correlation, 'reason': error.code, 'attempt': attempt, 'revision': revision, + 'stage': stage, 'validation_code': error.code, 'validation_rule': error.rule, + }, + ) + return [ + *messages, + {'role': 'assistant', 'content': reply}, + {'role': 'user', 'content': plan_repair_message(error, handoff_present=handoff_present)}, + ] + def _normalize(raw, capability_ids, availability, selected_image, **options): raw = _with_analysis_semantics(raw, edit_context) + _diagnose('analysis_defaults', raw, attempt=attempt) if conversation_only and any( step.get('enabled', True) and step.get('capability_id') in CONVERSATION_RETRIEVAL_CAPABILITIES for step in raw.get('steps') or [] if isinstance(step, dict) @@ -1772,6 +1934,7 @@ def _normalize(raw, capability_ids, availability, selected_image, **options): except (PlannerError, APIError, AzureError, ValueError) as exc: return _failure('model_configuration_failed', exc, stage='model_binding') + _diagnose('planner_input') messages = build_planner_messages( context, replan_hint=replan_hint, edit_context=edit_context, contract_version=contract_version, @@ -1794,7 +1957,7 @@ def _normalize(raw, capability_ids, availability, selected_image, **options): try: reply, usage = _call_planner(client, deployment, messages, require_complete_response=True) except (PlannerError, APIError, AzureError) as exc: - return _failure('model_request_failed', exc, stage='model_request') + return _failure('model_request_failed', exc, stage='model_request', attempt=attempt) if usage is not None: usage_seen = True for field in ('prompt_tokens', 'completion_tokens', 'total_tokens'): @@ -1804,7 +1967,9 @@ def _normalize(raw, capability_ids, availability, selected_image, **options): parsed = extract_planner_json(reply) if not parsed: - return _failure('unparseable_plan') + return _failure('unparseable_plan', stage='plan_parsing', attempt=attempt) + current_proposal = parsed + _diagnose('plan_proposal', parsed, attempt=attempt) # Only the server reports why a workflow was not started or read, and only the server # decides that a plan waits for a workflow run; a model cannot write either. parsed.pop('workflow_run_notes', None) @@ -1890,38 +2055,27 @@ def _normalize(raw, capability_ids, availability, selected_image, **options): elicitation['token_usage'] = dict(token_usage) return 'elicitation', elicitation except PlanValidationError as exc: - # A question we cannot render is worse than no question: the run would stall - # on a card that never appears. Planning again without the option is the only - # honest recovery. log_event( - f"[ORCHESTRATION_PLANNER] Discarding an unrenderable question set: {exc}", - level=logging.WARNING, - ) - return plan_request( - user_message, - planner_context, - conversation_id, - user_id, - settings=settings, - approval_mode=approval_mode, - authorized_document_ids=authorized_document_ids, - replan_hint=replan_hint, - revision=revision, - allow_elicitation=False, - turn_id=turn_id, - seeds=seeds, - document_labels=document_labels, - request_context=request_context, - planner_model=planner_model, - edit_context=edit_context, - contract_version=contract_version, - existing_results=existing_results, - composition_profiles=composition_profiles, - export_catalog=export_catalog, + '[ORCHESTRATION_PLANNER] Discarding an unrenderable question set.', + level=logging.WARNING, extra={ + **correlation, 'stage': 'plan_clarification', 'attempt': attempt, + 'validation_code': exc.code, 'validation_rule': exc.rule, + }, ) + if attempt > PLAN_REPAIR_ATTEMPTS: + return _failure('invalid_elicitation', exc, stage='plan_clarification', attempt=attempt) + allow_elicitation = False + messages = [ + *messages, {'role': 'assistant', 'content': reply}, + {'role': 'user', 'content': ( + 'The clarification could not be rendered. Return a complete corrected plan ' + 'for the unchanged request and authorized source selection, not another elicitation.' + )}, + ] + continue if kind == 'elicitation': - return _failure('repeated_elicitation') + return _failure('repeated_elicitation', stage='plan_clarification', attempt=attempt) raw_steps = parsed.get('steps') if not isinstance(raw_steps, list) or not raw_steps: @@ -1959,26 +2113,15 @@ def _normalize(raw, capability_ids, availability, selected_image, **options): if repairable and attempt <= PLAN_REPAIR_ATTEMPTS: # One correction round: the planner sees exactly why the server refused the # plan, such as a promised file no step renders, and answers the same request. - log_event( - '[ORCHESTRATION_PLANNER] Asking the planner to correct a rejected plan.', - level=logging.INFO, extra={ - **correlation, 'reason': exc.code, 'attempt': attempt, 'revision': revision, - 'stage': 'plan_normalization', 'validation_code': exc.code, - 'validation_rule': exc.rule, - }, - ) + handoff_present = False if handoff_offered: # Loaded only when the request may hand work off. from functions_orchestration_workflow_handoffs import plan_has_workflow_handoff - repair = plan_repair_message(exc, handoff_present=plan_has_workflow_handoff(pristine)) - else: - repair = plan_repair_message(exc) - messages = [ - *messages, - {'role': 'assistant', 'content': reply}, - {'role': 'user', 'content': repair}, - ] + handoff_present = plan_has_workflow_handoff(pristine) + messages = _correction_messages( + exc, 'plan_normalization', attempt, reply, handoff_present=handoff_present, + ) continue if not ((workflow_error or run_error or results_error or handoff_error) and edit_context is None): return _failure( @@ -1987,6 +2130,7 @@ def _normalize(raw, capability_ids, availability, selected_image, **options): 'deliverables_invalid': DELIVERABLES_FAILURE_MESSAGE, 'source_kind_invalid': SOURCE_KIND_FAILURE_MESSAGE, 'source_binding_required': SOURCE_BINDING_FAILURE_MESSAGE, + 'analysis_options_invalid': ANALYSIS_OPTIONS_FAILURE_MESSAGE, 'conversation_retrieval_conflict': ( 'This request needs the conversation content, not a workspace search. ' 'Select or paste the target text and create a new plan.' @@ -2149,11 +2293,20 @@ def _normalize(raw, capability_ids, availability, selected_image, **options): for text in handoff_drop['repairs']: if text not in repairs: repairs.append(text) + _diagnose('plan_normalized', plan, attempt=attempt) + current_proposal = plan + try: + validate_plan_requirements(plan, seeds, allow_changes=edit_context is not None) + except PlanValidationError as exc: + if exc.code == 'selected_requirements_missing' and attempt <= PLAN_REPAIR_ATTEMPTS: + messages = _correction_messages(exc, 'selected_requirements', attempt, reply) + continue + return _failure( + 'invalid_plan_or_missing_requirement', exc, stage='selected_requirements', + message=SELECTED_REQUIREMENTS_FAILURE_MESSAGE, attempt=attempt, + ) + _diagnose('plan_validated', plan, attempt=attempt) break - try: - validate_plan_requirements(plan, seeds, allow_changes=edit_context is not None) - except PlanValidationError as exc: - return _failure('invalid_plan_or_missing_requirement', exc, stage='selected_requirements') if ( edit_context is not None and image_selected and not any( diff --git a/application/single_app/functions_orchestration_registry.py b/application/single_app/functions_orchestration_registry.py index 03a6913b2..11fbc029d 100644 --- a/application/single_app/functions_orchestration_registry.py +++ b/application/single_app/functions_orchestration_registry.py @@ -847,7 +847,9 @@ def _document_transform_descriptor(capability_id, label, instruction): 'analysis_options': { 'type': 'object', 'description': ( - 'Explicit result semantics. Use analysis-options-v2 with source_observations ' + 'Omit for ordinary narrative analysis; the server supplies observation semantics. ' + 'When declaring options, pair the version with a compatible mode. ' + 'Use analysis-options-v2 with source_observations ' 'for independently cited source statements; use reconciled_records for values ' 'that must be reconciled or satisfy declared fields/calculations. Repeated ' 'subject labels are allowed for observations. Observations expose findings, ' @@ -865,6 +867,33 @@ def _document_transform_descriptor(capability_id, label, instruction): }, 'transformation_spec': {'type': 'object'}, }, + 'allOf': [ + { + 'if': { + 'required': ['version'], + 'properties': {'version': {'const': 'analysis-options-v2'}}, + }, + 'then': { + 'required': ['mode'], + 'properties': {'mode': {'enum': ['source_observations', 'reconciled_records']}}, + }, + 'else': {'properties': {'mode': {'const': 'narrative_findings'}}}, + }, + { + 'if': { + 'required': ['mode'], + 'properties': {'mode': {'const': 'source_observations'}}, + }, + 'then': {'properties': { + 'required_fields': {'maxItems': 0}, + 'transformation_spec': {'maxProperties': 0}, + }}, + }, + ], + 'examples': [ + {'version': 'analysis-options-v2', 'mode': 'source_observations'}, + {'version': 'analysis-options-v2', 'mode': 'reconciled_records'}, + ], 'additionalProperties': False, }, }, diff --git a/application/single_app/functions_orchestration_schema.py b/application/single_app/functions_orchestration_schema.py index b6cdc82c5..8cb4c14d2 100644 --- a/application/single_app/functions_orchestration_schema.py +++ b/application/single_app/functions_orchestration_schema.py @@ -54,7 +54,7 @@ from agent_execution_context import AgentDelegationTimeout from functions_m365_operations import M365_SOURCES from functions_model_catalog import ModelCatalogError -from functions_document_analysis_results import normalize_analysis_options +from functions_document_analysis_results import AnalysisOptionsError, normalize_analysis_options from functions_web_iq import WebIQError, get_web_search_provider, resolve_web_iq_types from functions_orchestration_registry import ( APPROVAL_FLOOR_MANUAL, @@ -1042,9 +1042,27 @@ def validate_dependency_plan( if type(arguments) is not dict: raise PlanValidationError('Step arguments must be an object.') canonical_bytes(arguments) + analysis_options = None + if capability_id == 'document_analyze' and 'analysis_options' in arguments: + try: + analysis_options = normalize_analysis_options(arguments['analysis_options']) + except ValueError as exc: + raise PlanValidationError( + 'The declared Analyze semantics or field requirements are unsupported.', + code='analysis_options_invalid', + rule=exc.rule if isinstance(exc, AnalysisOptionsError) else 'analysis_options_invalid', + ) from exc + option_schema = capability['inputs']['properties']['analysis_options'] + if not Draft202012Validator(option_schema).is_valid(arguments['analysis_options']): + raise PlanValidationError( + 'The declared Analyze options do not match their contract.', + code='analysis_options_invalid', rule='analysis_options_schema_invalid', + ) if not Draft202012Validator(capability['inputs']).is_valid(arguments): raise PlanValidationError('Step arguments do not match the capability contract.') arguments = deepcopy(arguments) + if analysis_options is not None: + arguments['analysis_options'] = analysis_options if capability_id in ('web_search', 'deep_research') and 'result_types' in arguments: try: if get_web_search_provider(settings) == 'web_iq': @@ -1055,14 +1073,6 @@ def validate_dependency_plan( raise WebIQError('result_type_disabled') except WebIQError as exc: raise PlanValidationError('A requested web discovery type is disabled.') from exc - if capability_id == 'document_analyze' and 'analysis_options' in arguments: - try: - arguments['analysis_options'] = normalize_analysis_options(arguments['analysis_options']) - except ValueError as exc: - raise PlanValidationError( - 'The declared Analyze semantics or field requirements are unsupported.', - code='analysis_options_invalid', - ) from exc if capability_id == CAPABILITY_TABULAR_ANALYZE: # Native query/transform validators run only for explicitly admitted native work. from functions_orchestration_native_results import validate_native_orchestration_arguments @@ -1380,7 +1390,7 @@ def plan_document_ids(plan, *, include_disabled=False): document_ids.extend(_string_list(arguments.get(field))) if step.get('capability_id') == 'document_inspect': reference = arguments.get('reference') or {} - if reference.get('kind') == 'document': + if isinstance(reference, dict) and reference.get('kind') == 'document': document_ids.extend(_string_list(reference.get('document_id'))) return list(dict.fromkeys(document_ids)) @@ -1472,7 +1482,13 @@ def validate_plan_requirements(plan, seeds=None, *, allow_changes=False): if selected_documents - used_documents: messages.append('The plan does not use all selected documents.') if messages and not allow_changes: - raise PlanValidationError(' '.join(messages)) + raise PlanValidationError( + ' '.join(messages), code='selected_requirements_missing', + rule=( + 'selected_documents_and_capabilities_missing' if missing and selected_documents - used_documents + else 'selected_capabilities_missing' if missing else 'selected_documents_missing' + ), + ) if messages: repairs = plan.setdefault('validation', {}).setdefault('repairs', []) for message in messages: diff --git a/docs/admin/orchestration.md b/docs/admin/orchestration.md index 017ec4f07..70f67de38 100644 --- a/docs/admin/orchestration.md +++ b/docs/admin/orchestration.md @@ -904,6 +904,15 @@ run ID to correlate its hashed identifier in The HTTP status alone is insufficient: a planning stream can return HTTP 200 and then report a rejected proposal. +Since **0.261.411**, invalid Analyze declarations and omitted selected documents +share the planner's one correction round. Selecting a file does not run Analyze: +upload extraction and bounded planning previews are separate from approved step +execution. For repeat failures, enable the existing debug logging setting and +compare the initial proposal, normalized source bindings, and specific rejection +rule in [selected-document planning diagnostics]({{ '/reference/logging-tags/#selected-document-planning-diagnostics' | relative_url }}). +Do not treat a limited preview as proof that the file is missing or ask for a +blanket re-upload. Turn debug logging off after collecting the needed evidence. + Since **0.261.291**, when a web search, linked-page, deep research, agent, action or memory step is refused, `[ORCHESTRATION_EXTERNAL_SOURCES] A step's source was refused.` names the check that failed in `sc_reason`. It shares `sc_run_id_hash` and @@ -914,6 +923,7 @@ names the check that failed in `sc_reason`. It shares `sc_run_id_hash` and | No orchestration control appears in chat | The setting is off, or the user is in the classic interface. | Confirm Enable Chat Orchestration is on, and that the user is on a V2 chat page. | | A saved run says it was created by an earlier orchestration version | The run predates the current plan architecture and is refused rather than interpreted. | Start a new request in the same conversation. Earlier-version runs are omitted from the run list and cannot be rerun. | | Plans never mention documents | No workspace capability is enabled, or nothing in the user's documents matched the question. | Confirm at least one workspace type is enabled, and that the user has documents that have finished processing. | +| Picking a document ends with a planning error before any steps run | The proposed plan omitted a selected source or used incompatible internal Analyze options. Before 0.261.411 these errors could stop without correction. | Upgrade to 0.261.411 or later. A repeated failure logs `selected_requirements_missing` or `analysis_options_invalid` with a specific rule and proposal attempt. Debug diagnostics show whether the binding was absent in the model proposal or changed during normalization; the message is not an instruction to change an Analyze setting. | | Every plan is a single answering step | Capabilities are narrowed to answering only, or the retrieval capabilities are disabled elsewhere. | Review Capabilities on this page, then confirm document search and web search are enabled in their own settings groups. | | A plan is smaller than expected | The step cap trimmed it. | Raise Maximum steps in a plan, or ask a narrower question. | | A follow-up loses its subject | The relevant message is outside the history window, masked, inactive, or truncated. | Check Conversation History Limit in Chat settings and whether the earlier turn is still eligible. Repeat the missing detail if it is outside the retained context. | diff --git a/docs/explanation/features/CHAT_ORCHESTRATION.md b/docs/explanation/features/CHAT_ORCHESTRATION.md index b6e383d93..f737af7c4 100644 --- a/docs/explanation/features/CHAT_ORCHESTRATION.md +++ b/docs/explanation/features/CHAT_ORCHESTRATION.md @@ -1,6 +1,6 @@ # Chat Orchestration -**Version: 0.261.402** (tracked in `application/single_app/config.py`) +**Version: 0.261.411** (tracked in `application/single_app/config.py`) **Implemented in version: 0.261.086** **Knowledge phase added in version: 0.261.089** @@ -29,6 +29,7 @@ **Current-message source interpretation and memory-aware request resolution fixed in version: 0.261.349** **Private conversation references (`~`), `conversation_search` / `conversation_read`, and queued `conversation_send` handoffs added in version: 0.261.381** **Search evidence citation harness expectations aligned in version: 0.261.399** +**Selected-document planning recovery and debug diagnostics fixed/implemented in version: 0.261.411** ## Overview @@ -68,6 +69,36 @@ and delivery checks. `enable_chat_orchestration_actions` switch. Existing action scope settings and governance continue to determine which actions the caller can use. +## Selected documents, planning, and Analyze + +Picking a document supplies a source requirement; it does not automatically run +Analyze. A new upload can undergo extraction/OCR and indexing before a chat plan +exists. Planning can also inspect bounded spreadsheet headers and samples to +choose compatible work. Neither is execution of an orchestration Analyze step. +Only a validated, approved plan reaches the execution adapters. Automatic approval +does not bypass validation. + +Since **0.261.411**, a proposal that omits a selected file or declares incompatible +Analyze options receives targeted correction feedback. It shares the existing +single correction with other repairable planning failures: a second, different +failure does not start another retry budget. The server keeps source access, +source-kind checks, every required selection, and the selected planner model. +An explicit later plan edit can still narrow the selection, with a review warning. + +Analyze's internal options describe independent source observations versus +reconciled records, not an administrator toggle. Ordinary new narrative steps +receive observation defaults when the model omits these options. Explicit +version/mode combinations are checked together; saved legacy meanings remain +unchanged. An uncorrectable proposal fails clearly rather than becoming an +ungrounded answer or silently excluding a file. + +The existing debug logging switch enables content-free proposal and source-binding +diagnostics in console and Application Insights. These distinguish a model omission +from a normalization change using hashes, counts, and rejection rules, without +recording document contents or the model reply. See the +[document planning recovery fix](../fixes/ORCHESTRATION_DOCUMENT_PLANNING_RECOVERY_FIX.md) +and [logging reference](../../reference/logging-tags.md#selected-document-planning-diagnostics). + ## Internal dependency contract (M4) Gather / Reason / Render treats **Gather**, **Reason**, and **Render** as server-owned @@ -1431,6 +1462,7 @@ to the front. | `functional_tests/test_v2_orchestration_deliverables.mjs` | Browser normalization of deliverables and `delivers`, deliverable states that follow their steps, labels, generated-image helpers, a terminal frame read through the real run stream client, and a retry's image grouped under every answer that lists it, in React V2 and the classic client | | `ui_tests/test_v2_orchestration_generated_images.py` | The live chat loading an answer's generated images after the run, planned image cards that never offer Approve or Approve all while their image loads, and a reused image shown under both the earlier and the retried answer | | `functional_tests/test_orchestration_context_picker.py` | Picked tags reach the seeds and both search paths under the parameter `hybrid_search` really takes; a tag scopes the probe rather than replacing it; a picked document reaches the planner and the approval card by name; a browser-supplied name cannot widen access; search citations carry the workspace a document came from; a step can read what an earlier step found, an unusable binding is refused, and a run-time document still respects the configured ceiling | +| `functional_tests/test_orchestration_document_planning_recovery.py` | Picked-document HTTP planning across five file formats, preserved previews and Analyze availability, strict option semantics, source/operation retention, one shared correction including malformed questions, editor compatibility, Auto routing, no execution during planning, and private-content-free debug diagnostics in both sinks | | `functional_tests/test_orchestration_conversation_context.py` | Message eligibility, bounds, snapshot validation, nullable unused clarifications, strict response validation, bounded repair, token accounting, provider/refusal handling, contextualized adapters, synthesis roles, and URL provenance | | `functional_tests/test_orchestration_conversation_context_routes.py` | New and existing conversations across HTTP/SSE planning and execution, all approval modes, null clarifications, bounded recovery, model selection and attribution, revocation, completion failures, stream cleanup, stale sources and legacy cutoffs | | `ui_tests/test_v2_orchestration_conversation_context.py` | Matching clarification/model transport, cancellation, original-turn continuity, all approval modes, visible answer model names and navigation | diff --git a/docs/explanation/fixes/ORCHESTRATION_DOCUMENT_PLANNING_RECOVERY_FIX.md b/docs/explanation/fixes/ORCHESTRATION_DOCUMENT_PLANNING_RECOVERY_FIX.md new file mode 100644 index 000000000..4eb30127c --- /dev/null +++ b/docs/explanation/fixes/ORCHESTRATION_DOCUMENT_PLANNING_RECOVERY_FIX.md @@ -0,0 +1,153 @@ +# Selected-document orchestration planning recovery (v0.261.411) + +Fixed/Implemented in version: **0.261.411**, recorded in +`application/single_app/config.py`. + +## Issue and root cause + +Document-backed turns could finish upload/source preparation but stop with +"The request could not be planned" before a plan or execution existed. Two +distinct boundaries were involved: + +- The Analyze contract advertised independent version and mode choices that its + semantic normalizer could reject in combination. The `analysis_options_invalid` + category was not included in the existing correction path. +- A normalized plan omitting selected documents or positive operations was checked + after leaving the correction loop. That guard correctly prevented silent source + loss, but its generic rejection could not be corrected and lacked a specific + rule or proposal attempt. + +The Analyze interface was introduced in +[#1874](https://github.com/microsoft/simplechat/pull/1874). Its observation-identity +fix remains necessary; the missing schema/recovery integration is corrected here. +The selected-source guard predates that change. +[#1875](https://github.com/microsoft/simplechat/pull/1875) improves later retained +result and narrative delivery, which cannot help a turn rejected before planning +finishes. This fix preserves both improvements rather than rolling them back. + +The deployed diagnostic category did not retain the raw model proposal. It is not +proof of which invalid option or omitted step the model produced. Local real-module +and authenticated-route reproductions distinguish the rejected proposal from a +normalization change; new debug diagnostics make that difference observable. + +## Extraction, planning, and execution are different + +Picking a file supplies a source requirement, not an unconditional Analyze step. +A new upload may undergo OCR/extraction, image enrichment, chunking, and indexing. +Before planning, a selected spreadsheet can have a bounded schema/sample preview. +The planner then proposes JSON and the server validates it. Only a valid, approved +plan reaches adapters such as `run_document_analyze`; automatic approval still +happens after validation. + +`analysis_options` describes result semantics, not an administrator switch. New +ordinary narrative steps receive observation defaults when options are omitted. +Explicit v2 observations require a compatible version/mode and cannot demand +reconciled fields/calculations. Legacy options and existing edited plans keep +their established meaning. + +## Technical changes + +`functions_orchestration_registry.py` now expresses version/mode dependencies and +observation restrictions in the advertised schema, with complete v2 examples. +`functions_document_analysis_results.py` preserves strict `ValueError` compatibility +while attaching application-owned rejection rules, without retaining supplied +values. `functions_orchestration_schema.py` preserves the dedicated Analyze error +across both schema and semantic validation and classifies source/operation +omissions separately. + +`functions_orchestration_planner.py` validates selected requirements inside the +existing correction loop. Targeted feedback preserves every selection and asks +for compatible, explicit source bindings. All repairable categories share +`PLAN_REPAIR_ATTEMPTS = 1`: a second different error does not start another budget. +An unrenderable clarification stays in that same loop rather than recursively +restarting the planner and resetting its allowance. +The same planner/model is used, and uncorrectable proposals return stable, specific +messages rather than an empty plan, an ungrounded answer, or a silently smaller +selection. Explicit later editor narrowing remains visible for review. + +Canonical source accounting, authorization, source-kind checks, native Excel +comment evidence, partial-information narrative behavior, and complete-data +requirements are not bypassed. A limited preview is not a missing file and does +not require a blanket re-upload. No new extraction engine, route, setting, +dependency, cloud service, or deployment is introduced. + +## Debug diagnostics + +The existing debug switch gates content-free input/proposal/default/normalized/ +validated snapshots in both console and Application Insights. They record hashed +source/step identity, effective source counts, normalization differences, binding +fingerprints, option-shape flags, exact owned rejection rules, and the shared +attempt budget. Initial invalid proposals remain observable before defaults. + +`functions_appinsights.py` narrowly preserves three validated SHA-256 properties +through real logger serialization: document ID, document set, and planner binding. +It does not broadly allow arbitrary diagnostic text. Detail snapshots have explicit +32-entry source/step bounds with omitted counts; totals remain complete. + +Raw IDs, document text, filenames, prompts, model replies, arbitrary option values, +settings, URLs, credentials, and provider exception text are excluded. Debug-off +behavior retains concise correction/failure events without detailed snapshots. +See [selected-document planning diagnostics](../../reference/logging-tags.md#selected-document-planning-diagnostics) +for event names, properties, and a bounded query. + +## Validation and before/after behavior + +`functional_tests/test_orchestration_document_planning_recovery.py` uses the real +request/context/planner/validator/logger boundaries with external I/O isolated. +It covers PNG, PDF, DOCX, CSV and XLSX selections, manual/automatic approval, +Auto routing, correctable and terminal option/source failures, mixed errors +sharing one correction, implicit scoped search, multiple/disabled sources, source-free +controls, existing-plan narrowing, legacy semantics, explicit requirement bounds, +provider failures, and debug privacy/gating/serialization. + +| Before | After | +| --- | --- | +| A selected source omitted by the proposal stopped planning immediately. | A specific source/operation rule requests correction within the existing budget. | +| Incompatible Analyze semantics were advertised as valid and then rejected without correction. | The schema expresses compatible declarations; strict rejection retains its rule and can be corrected. | +| A different validation failure could not be diagnosed from a generic message. | Debug snapshots identify source coverage and the rejecting owner without private payloads. | +| Upload extraction or preview reads could be confused with Analyze execution. | Regression coverage proves planning does not invoke an execution adapter, even with automatic approval. | + +Related existing suites cover observation identity, native analysis consumption, +selected-source preparation, debug logger privacy, and native Excel comment +evidence. The new tests are behavioral; they do not assert an exact current app +version or substitute schema-text checks for the actual planner boundary. + +These are offline contract and route reproductions, not a live model success-rate +claim. The original deployed proposal remains unknown until a separately +authorized instrumented reproduction is collected. Azure configuration and live +conversations are not modified by this implementation. + +### Executed checks + +The combined picked-document, observation-identity, internal-analysis, +selected-source-planning, and available-information selection passed **214 tests**: + +```powershell +python -m pytest -q -p no:cacheprovider --disable-warnings --tb=short --show-capture=no .\functional_tests\test_orchestration_document_planning_recovery.py .\functional_tests\test_document_analysis_observation_identity.py .\functional_tests\test_orchestration_internal_analysis.py .\functional_tests\test_orchestration_selected_source_planning.py .\functional_tests\test_orchestration_available_information.py +``` + +Additional selections passed: the existing bound-model/editor unrenderable-question +regression (**1**), native Excel evidence (**7**), logger initialization/recursion +(**1**), standalone privacy audit (**5**), documentation surface coverage (**7/7**), +and documentation site quality (**6/6**). The whitespace check passed. Selections +are not presented as an aggregate count. + +The new Auto-routing HTTP fixture replaces only external Graph profile-photo I/O; +the real routing and user-preference paths run with the offline network guard +still active. No blocked network attempt is suppressed or cleared. + +### Existing baseline limitations + +The broader existing Ask AI debug/initialization selection reported **84 passed +and 3 failed**. The failures are the conversation-reference operations +`bind_conversation_reference`, `set_task_conversation_references`, and +`unbind_conversation_reference`: their existing diagnostics omit +`sc_assist_operation`. All three reproduce with the unchanged `HEAD` serializer +loaded in memory. This change only adds strictly validated source/binding hash +properties; it does not change or weaken Ask AI's operation allowlist. + +Before implementation, the older AST-extracted planner-failure fixture also had +ten setup errors from undefined `STORAGE_FAILURE_CODE`. It was not reported as +passing. The new regressions use real modules/routes and cover their own failure, +provider-error, correlation, and privacy behavior without depending on that +fixture or disabling its checks. diff --git a/docs/reference/logging-tags.md b/docs/reference/logging-tags.md index 5c044f193..a4149f894 100644 --- a/docs/reference/logging-tags.md +++ b/docs/reference/logging-tags.md @@ -550,10 +550,10 @@ Insights `customDimensions` or Log Analytics `AppTraces.Properties`. | --- | --- | | `sc_conversation_id_hash`, `sc_turn_id_hash`, `sc_run_id_hash`, `sc_step_id_hash` | SHA-256 of the exact workflow identifier encoded as UTF-8. Only applicable identifiers are present; these hashes do not grant access to the underlying records. Step IDs repeat across runs, so narrow by the run hash before the step hash. | | `sc_stage` | The failing boundary, such as `plan_normalization`, `claim_validation`, `settings`, `context`, `identity`, `result_binding`, `model_binding`, `run_detail`, or `scheduler_item`. On `[ORCHESTRATION_EXTERNAL_SOURCES]` refusal events, `external_source_preflight` (before or during the step's acquisition), `external_source_admission` (saving the step's result), or `external_source_read` (reading a saved result again). | -| `sc_validation_code` | Validation category, including `deliverables_invalid`, `source_kind_invalid`, or `source_binding_required`. | +| `sc_validation_code` | Validation category, including `deliverables_invalid`, `source_kind_invalid`, `source_binding_required`, `analysis_options_invalid`, or `selected_requirements_missing`. | | `sc_validation_rule` | Specific application-owned rejection, such as `invalid_quantity`, `non_file_format`, `answer_producer_mismatch`, `missing_final_response`, `file_format_mismatch`, `narrative_source_required`, or `document_sources_required`. | | `sc_response_failure` | A missing, incomplete, or refused planner completion, when applicable. | -| `sc_attempt` | Planner proposal number: 1 for the initial proposal and 2 for its single correction. Present on deliverables correction/failure events. | +| `sc_attempt` | Planner proposal number: 1 for the initial proposal and 2 for its single shared correction. Since 0.261.411, also present on Analyze-option and selected-requirement correction/failure events. | | `sc_execution_code`, `sc_output_code` | Safe execution or output-store failure category, where available. | | `sc_failure_code` | On step and adapter events, a step failure category, such as `step_timeout`, `run_timeout`, or `context_unavailable`. On `[CONTENT_SCREENING]` events, the source-authority code, such as `source_authority_unverified`, `source_authority_unavailable`, or `screening_revision_conflict`. | | `sc_authority_reason` | Which check failed for a source. With `source_authority_unverified`, the check that could not verify the source's current authority: `document_record_invalid`, `screening_marker_invalid`, `screening_state_unknown`, `screening_marker_incomplete`, `document_scope_invalid`, `workspace_record_invalid`, `scan_record_invalid`, `provenance_mismatch`, `document_id_invalid`, `manifest_context_invalid`, `manifest_scope_missing`, `manifest_revision_invalid`, or `manifest_batch_invalid`. With `screening_revision_conflict`, what saved evidence no longer matches in the current document: `source_revision_changed` (the document was re-versioned), `screening_generation_changed` (it was re-screened or remediated), `source_scope_changed` (its id or workspace differs), `provenance_shape_invalid` (the saved provenance has a different set of fields), `screened_record_changed` (a saved copy of the screened record differs), `source_version_changed` (a legacy reference without provenance), or `cached_evidence_unproven` (cached evidence from before screening enrollment). On `[ORCHESTRATION_EXECUTOR]` step, saved-wait, finalization and content-preparation failure events (since 0.261.209), the stable code of a refused external source, such as `external_identity_session_unavailable` (the step ran in a background continuation with no signed-in session), `external_identity_role_required`, `external_identity_access_restricted`, `external_identity_conversation_unavailable`, `external_identity_settings_unavailable`, or `external_configuration_changed`. On `[ORCHESTRATION_EXTERNAL_SOURCES]` refusal events (since 0.261.291), the same refusal code, such as `result_external_source_unavailable` or `result_external_capability_unavailable`, with the check that failed in `sc_reason`. | @@ -611,6 +611,76 @@ AppTraces | take 200 ``` +### Selected-document planning diagnostics + +Implemented in version **0.261.411**, tracked in +`application/single_app/config.py`. The existing **Enable Debug Logging** setting +gates these three events in both App Service console logs and Application Insights. +They use the normal application logger after the debug gate; they do not enable +third-party SDK payload logging. Normal correction and terminal failure events +remain available with debug logging off. + +| Event message | Meaning | +| --- | --- | +| `[ORCHESTRATION_PLANNER] Planning diagnostic.` | Source-coverage summary, counts, model-binding fingerprint, and an owned rejection rule when applicable. | +| `[ORCHESTRATION_PLANNER] Source binding diagnostic.` | One bounded, hashed source entry: selected, used, authorized when an explicit authorization set exists, server-resolved kind, availability, and limited-preview state. | +| `[ORCHESTRATION_PLANNER] Step binding diagnostic.` | One bounded step entry: hashed step ID, known capability, enabled state, binding-field presence, and Analyze option shape/known-value flags. | + +Stages are `planner_input`, `plan_proposal` (before defaults), `analysis_defaults`, +`plan_normalized`, and `plan_validated`. A rejected attempt also uses its actual +boundary, such as `plan_normalization` or `selected_requirements`. + +| Property | Meaning | +| --- | --- | +| `sc_document_id_hash` | SHA-256 of a referenced source ID, never its raw ID or filename. | +| `sc_planner_binding_hash` | Fingerprint of the selected deployment and available binding source/model/endpoint IDs, provider, and behavior name; never a URL, key, or client object. | +| `sc_document_set_hash` | Fingerprint of the sorted effective document references for comparison between stages. | +| `sc_selected_document_count`, `sc_used_document_count`, `sc_matched_document_count`, `sc_missing_document_count`, `sc_unselected_document_count` | Source-coverage counts from the canonical document-reference/effective-search helpers. | +| `sc_authorization_scope_supplied`, `sc_authorized_document_count` | Whether an explicit authorization set was supplied and its size. An absent set is not reported as a verified denial. | +| `sc_source_availability_known`, `sc_source_available` | Whether server availability metadata exists and whether it declares the source available; absence is not a verified missing file. | +| `sc_required_capability_count`, `sc_missing_capability_count` | Positive-operation coverage, separate from document coverage. | +| `sc_normalization_lost_document_count` | References present in the raw proposal but absent after normalization. Zero missing references at proposal time followed by loss here differs from a model omission. | +| `sc_remaining_corrections` | Remaining shared proposal-correction allowance; never a new per-error budget. | +| `sc_model_routing_auto`, `sc_planner_binding_provided` | Auto-routing selection and whether planning uses a resolved binding rather than the legacy client resolver. | +| `sc_step_count`, `sc_omitted_step_count`, `sc_omitted_document_count` | Total steps and explicit diagnostic omissions. At most 32 source and 32 step detail events are emitted per snapshot; summaries still count all references. | +| `sc_options_version_present`, `sc_options_version_v1`, `sc_options_version_v2`, `sc_options_mode_present`, `sc_options_mode_observations`, `sc_options_mode_records`, `sc_options_mode_legacy` | Presence/known-value flags, not arbitrary supplied option strings. | +| `sc_required_field_count`, `sc_calculation_rules_present`, `sc_unknown_option_field_count` | Structural Analyze requirements without field names, rule bodies, or values. | + +`selected_requirements_missing` rules distinguish `selected_documents_missing`, +`selected_capabilities_missing`, and `selected_documents_and_capabilities_missing`. +Analyze rules include `options_object_required`, `unsupported_option_fields`, +`unsupported_options_version`, `version_mode_mismatch`, `supported_mode_required`, +`required_fields_invalid`, `calculation_rules_object_required`, +`calculation_rules_conflict`, `calculation_rules_invalid`, +`observation_requirements_conflict`, and `analysis_options_schema_invalid`. + +A picked document missing from both `plan_proposal` and `plan_normalized` indicates +an omitted recognized binding in the proposal, not a normalization loss. These +structural diagnostics do not establish extraction fidelity or prove that every +source was read at execution. Do not log the raw prompt, model reply, source text, +filenames, arbitrary option fields/values, settings, or credentials to investigate. + +```kusto +let conversationHash = hash_sha256(""); +traces +| where timestamp > ago(2h) +| where tostring(customDimensions.sc_conversation_id_hash) == conversationHash +| where tostring(customDimensions.sc_message) startswith "[ORCHESTRATION_PLANNER]" +| project timestamp, + event=tostring(customDimensions.sc_message), + stage=tostring(customDimensions.sc_stage), + attempt=toint(customDimensions.sc_attempt), + documentHash=tostring(customDimensions.sc_document_id_hash), + selected=customDimensions.sc_selected, + used=customDimensions.sc_used, + missing=toint(customDimensions.sc_missing_document_count), + normalizationLoss=toint(customDimensions.sc_normalization_lost_document_count), + rule=tostring(customDimensions.sc_validation_rule), + omittedDocuments=toint(customDimensions.sc_omitted_document_count) +| order by timestamp asc +| take 500 +``` + ### Step, render and content-preparation events Since **0.261.141**, these events explain a slow or failed step without logging diff --git a/functional_tests/test_orchestration_document_planning_recovery.py b/functional_tests/test_orchestration_document_planning_recovery.py new file mode 100644 index 000000000..63b4c954e --- /dev/null +++ b/functional_tests/test_orchestration_document_planning_recovery.py @@ -0,0 +1,591 @@ +# test_orchestration_document_planning_recovery.py +#!/usr/bin/env python3 +""" +Functional regressions for picked-document planning and debug diagnostics. +Version: 0.261.411 +Implemented in: 0.261.411 + +Real authenticated routes, source metadata, planner, validation and logger +serialization run with external model/storage I/O isolated. Diagnostics must +identify source loss and invalid Analyze declarations without private payloads. +""" + +import hashlib +import importlib +import io +import json +import logging +from copy import deepcopy +from unittest.mock import Mock + +import httpx +import pytest +from jsonschema import Draft202012Validator +from openai import APITimeoutError +from openpyxl import Workbook + +from test_orchestration_source_kind_planning import ( + LEFT, RIGHT, candidates, enrich, modules, real_http_harness, source_runtime, +) +from test_support.orchestration_harness_execution import compose_step, input_binding + + +__all__ = ["modules", "real_http_harness", "source_runtime"] +DIAGNOSTIC_MESSAGES = { + "[ORCHESTRATION_PLANNER] Planning diagnostic.", + "[ORCHESTRATION_PLANNER] Source binding diagnostic.", + "[ORCHESTRATION_PLANNER] Step binding diagnostic.", +} +PRIVATE = "PRIVATE_DOCUMENT_PROMPT_OR_PROVIDER_VALUE" +OPTIONS = {"version": "analysis-options-v2", "mode": "source_observations"} + + +@pytest.fixture +def planning(source_runtime, monkeypatch): + telemetry = importlib.import_module("functions_appinsights") + logger = logging.getLogger("test.orchestration.document_planning") + flags = {"enable_debug_logging": True} + monkeypatch.setattr(telemetry, "_load_logging_settings", lambda: flags) + monkeypatch.setattr(telemetry, "get_appinsights_logger", lambda: logger) + source_runtime.telemetry = telemetry + source_runtime.logger = logger + source_runtime.debug_flags = flags + return source_runtime + + +def proposal(document_ids=(), *, capability="document_search", options=None): + producer = { + "step_id": "read", "capability_id": capability, + "arguments": {"document_ids": list(document_ids)}, + } + if capability == "document_analyze": + producer["arguments"]["analysis_prompt"] = PRIVATE + if options is not None: + producer["arguments"]["analysis_options"] = deepcopy(options) + output = "findings" + elif capability == "tabular_inspect": + output = "inspection" + else: + producer["arguments"]["query"] = PRIVATE + output = "prepared" + answer = compose_step("answer", inputs={ + "source": {"binding": input_binding("read", output), "allow_partial": False}, + } if document_ids else {}) + answer["arguments"].update( + instruction=PRIVATE, knowledge_basis="sources" if document_ids else "general_knowledge", + ) + answer["delivers"] = ["answer"] + return { + "kind": "plan", + "deliverables": [{ + "id": "answer", "kind": "answer", "requested": "explicit", + "description": "Answer from the selected sources.", "status": "planned", + }], + "steps": [producer, answer] if document_ids else [answer], + "final_response": input_binding("answer"), + } + + +def run_plan( + runtime, replies, *, identifiers=(LEFT,), context_updates=None, seed_updates=None, + message=PRIVATE, **options, +): + harness = runtime.harness + selected, seeds = candidates(runtime, identifiers) + seeds.update(seed_updates or {}) + offered = enrich(runtime, selected, seeds) + context = runtime.context.build_planner_context(message, candidates=offered, seeds=seeds) + context.update(context_updates or {}) + harness.replies = [json.dumps(reply) if isinstance(reply, dict) else reply for reply in replies] + return harness.planner.plan_request( + message, context, "conversation-1", "owner", + settings=harness.settings, seeds=seeds, authorized_document_ids=list(identifiers), + turn_id="document-planning-turn", request_context=harness.services().capability_request_bindings(), + **options, + ) + + +def diagnostic_records(caplog, *, stage=None, message=None): + return [ + record for record in caplog.records + if getattr(record, "sc_message", None) in DIAGNOSTIC_MESSAGES + and (stage is None or getattr(record, "sc_stage", None) == stage) + and (message is None or record.sc_message == message) + ] + + +def serialized_records(records): + return json.dumps([ + {name: value for name, value in record.__dict__.items() if name.startswith("sc_")} + for record in records + ]) + + +@pytest.mark.parametrize("debug", [False, True]) +def test_debug_diagnostics_identify_model_omission_in_both_sinks(planning, caplog, capsys, debug): + planning.debug_flags["enable_debug_logging"] = debug + with caplog.at_level(logging.INFO, logger=planning.logger.name): + with pytest.raises(planning.harness.planner.PlannerError): + run_plan(planning, [proposal(), proposal()]) + records = diagnostic_records(caplog) + assert bool(records) is debug + if debug: + initial = diagnostic_records(caplog, stage="planner_input", message="[ORCHESTRATION_PLANNER] Planning diagnostic.")[0] + assert initial.sc_remaining_corrections == 1 + summaries = [ + record for record in diagnostic_records(caplog, stage="plan_normalized") + if record.sc_message == "[ORCHESTRATION_PLANNER] Planning diagnostic." + ] + assert summaries + assert all(record.sc_selected_document_count == record.sc_missing_document_count == 1 for record in summaries) + assert all(record.sc_used_document_count == record.sc_normalization_lost_document_count == 0 for record in summaries) + source = diagnostic_records(caplog, stage="plan_proposal", message="[ORCHESTRATION_PLANNER] Source binding diagnostic.")[0] + assert source.sc_document_id_hash == hashlib.sha256(LEFT.encode()).hexdigest() + assert source.sc_selected is True and source.sc_used is False + failure = next( + record for record in reversed(caplog.records) + if getattr(record, "sc_message", None) == "[ORCHESTRATION_PLANNER] The request could not be planned." + ) + assert failure.sc_validation_rule == "selected_documents_missing" + assert failure.sc_attempt >= 1 + else: + assert any( + getattr(record, "sc_message", None) == "[ORCHESTRATION_PLANNER] The request could not be planned." + for record in caplog.records + ) + console = capsys.readouterr().out + assert ("Source binding diagnostic." in console) is debug + assert PRIVATE not in console + serialized_records(records) + + +def test_debug_options_show_exact_owner_rule_before_defaulting(planning, caplog, capsys): + invalid = proposal([LEFT], capability="document_analyze", options={"mode": "source_observations"}) + with caplog.at_level(logging.INFO, logger=planning.logger.name): + with pytest.raises(planning.harness.planner.PlannerError): + run_plan(planning, [invalid, invalid]) + steps = diagnostic_records(caplog, stage="plan_proposal", message="[ORCHESTRATION_PLANNER] Step binding diagnostic.") + analysis = next(record for record in steps if record.sc_capability_id == "document_analyze") + assert analysis.sc_options_mode_observations is True + assert analysis.sc_options_version_present is False + rejected = diagnostic_records(caplog, stage="plan_normalization", message="[ORCHESTRATION_PLANNER] Planning diagnostic.") + assert rejected[-1].sc_validation_rule == "version_mode_mismatch" + console = capsys.readouterr().out + assert PRIVATE not in console + serialized_records(diagnostic_records(caplog)) + + +def test_diagnostic_hashes_survive_serialization_without_expanding_text_allowlists(planning): + fingerprint = hashlib.sha256(PRIVATE.encode()).hexdigest() + for key in ("document_id_hash", "planner_binding_hash", "document_set_hash"): + valid = planning.telemetry._build_logger_extra("Diagnostic", {key: fingerprint}) + invalid = planning.telemetry._build_logger_extra("Diagnostic", {key: PRIVATE}) + assert valid[f"sc_{key}"] == fingerprint + assert f"sc_{key}" not in invalid + assert PRIVATE not in json.dumps(invalid) + private = planning.telemetry._build_logger_extra("Diagnostic", { + "filename": PRIVATE, "analysis_options": {"mode": PRIVATE}, "documents": [PRIVATE], + }) + assert PRIVATE not in json.dumps(private) + + +def test_debug_snapshot_bounds_are_explicit(planning, caplog): + identifiers = tuple(f"document-{index}" for index in range(101)) + with caplog.at_level(logging.INFO, logger=planning.logger.name): + with pytest.raises(planning.harness.planner.PlannerError): + run_plan(planning, [proposal(), proposal()], identifiers=identifiers) + summary = diagnostic_records(caplog, stage="plan_proposal", message="[ORCHESTRATION_PLANNER] Planning diagnostic.")[0] + assert summary.sc_selected_document_count == summary.sc_missing_document_count == 101 + assert summary.sc_omitted_document_count == 101 - planning.harness.planner.PLANNER_DEBUG_MAX_ITEMS + sources = diagnostic_records(caplog, stage="plan_proposal", message="[ORCHESTRATION_PLANNER] Source binding diagnostic.") + first = [record for record in sources if record.sc_attempt == 1] + assert len(first) == planning.harness.planner.PLANNER_DEBUG_MAX_ITEMS + + +@pytest.mark.parametrize("options,rule", [ + ({"mode": "source_observations"}, "version_mode_mismatch"), + ({"version": "analysis-options-v2", "mode": "narrative_findings"}, "supported_mode_required"), + ({"version": "analysis-options-v2"}, "supported_mode_required"), + ({"version": PRIVATE}, "unsupported_options_version"), + ({**OPTIONS, "required_fields": [PRIVATE]}, "observation_requirements_conflict"), + ({**OPTIONS, "required_fields": [" "]}, "required_fields_invalid"), + ({**OPTIONS, PRIVATE: PRIVATE}, "unsupported_option_fields"), + ({**OPTIONS, "transformation_spec": PRIVATE}, "calculation_rules_object_required"), + ({**OPTIONS, "transformation_spec": {"fields": [PRIVATE]}}, "calculation_rules_invalid"), + ({**OPTIONS, "transformation_spec": None}, "analysis_options_schema_invalid"), + ([PRIVATE], "options_object_required"), +]) +def test_invalid_analyze_options_are_corrected_once_with_owner_rule(planning, caplog, options, rule): + invalid = proposal([LEFT], capability="document_analyze", options=options) + valid = proposal([LEFT], capability="document_analyze", options=OPTIONS) + with caplog.at_level(logging.INFO, logger=planning.logger.name): + kind, plan = run_plan(planning, [invalid, valid]) + assert kind == "plan" + assert len(planning.harness.model_calls) == 2 + assert {call["model"] for call in planning.harness.model_calls} == {"gpt-4o"} + assert plan["steps"][0]["arguments"]["document_ids"] == [LEFT] + assert plan["steps"][0]["arguments"]["analysis_options"]["mode"] == "source_observations" + rejected = next( + record for record in caplog.records + if getattr(record, "sc_message", None) == "[ORCHESTRATION_PLANNER] Asking the planner to correct a rejected plan." + ) + assert rejected.sc_validation_code == "analysis_options_invalid" + assert rejected.sc_validation_rule == rule + assert rejected.sc_attempt == 1 + assert "PRIVATE_" not in serialized_records(diagnostic_records(caplog)) + + +@pytest.mark.parametrize("options", [ + {}, {"version": "analysis-options-v1"}, {"mode": "narrative_findings"}, + {"version": "analysis-options-v1", "mode": "narrative_findings", "required_fields": ["amount"]}, + OPTIONS, {**OPTIONS, "required_fields": [], "transformation_spec": {}}, + {"version": "analysis-options-v2", "mode": "reconciled_records", "required_fields": ["amount"]}, +]) +def test_previously_valid_analyze_options_remain_valid(planning, options): + registry = importlib.import_module("functions_orchestration_registry") + normalizer = importlib.import_module("functions_document_analysis_results") + normalized = normalizer.normalize_analysis_options(options) + schema_valid = Draft202012Validator( + registry.get_capability("document_analyze")["inputs"]["properties"]["analysis_options"], + ).is_valid(options) + assert schema_valid is True + kind, plan = run_plan(planning, [proposal([LEFT], capability="document_analyze", options=options)]) + assert kind == "plan" and len(planning.harness.model_calls) == 1 + assert plan["steps"][0]["arguments"]["analysis_options"] == normalized + context = json.loads(planning.harness.model_calls[0]["messages"][1]["content"]) + assert "document_analyze" in {capability["id"] for capability in context["capabilities"]} + + +@pytest.mark.parametrize("options", [ + {"mode": "source_observations"}, + {"version": "analysis-options-v1", "mode": "reconciled_records"}, + {"version": "analysis-options-v2", "mode": "narrative_findings"}, + {"version": "analysis-options-v2"}, + {**OPTIONS, "required_fields": ["amount"]}, + {**OPTIONS, "transformation_spec": {"fields": []}}, +]) +def test_advertised_analyze_schema_rejects_incompatible_semantics(planning, options): + registry = importlib.import_module("functions_orchestration_registry") + schema = registry.get_capability("document_analyze")["inputs"]["properties"]["analysis_options"] + schema_valid = Draft202012Validator(schema).is_valid(options) + assert schema_valid is False + + +@pytest.mark.parametrize("extension", ["png", "pdf", "docx", "csv", "xlsx"]) +@pytest.mark.parametrize("approval", ["manual", "auto"]) +def test_http_picked_document_recovery_preserves_scope_and_does_not_execute( + planning, monkeypatch, caplog, extension, approval, +): + harness = planning.harness + documents = harness.config.cosmos_user_documents_container + record = documents.read_item(LEFT, LEFT) + record["file_name"] = f"selected.{extension}" + documents.upsert_item(record) + if extension == "xlsx": + book = Workbook() + book.active.append(["Code", "Amount"]) + book.active.append(["001", 25]) + content = io.BytesIO() + book.save(content) + book.close() + harness.blobs.records[("PRIVATE_CONTAINER", "PRIVATE_PATH")]["data"] = content.getvalue() + execute = Mock(side_effect=AssertionError("Planning must not execute a step.")) + monkeypatch.setattr(harness.execution, "prepare_harness_execution", execute) + adapters = importlib.import_module("functions_orchestration_adapters") + monkeypatch.setitem(adapters.ADAPTER_REGISTRY, "document_analyze", execute) + request = "Answer the question using only the selected file." + harness.replies = [ + json.dumps({ + "relationship": "new_topic", "resolved_message": request, "message_ids": [], + "requires_retrieval": True, "clarification": "", + }), + json.dumps(proposal()), + json.dumps(proposal([LEFT])), + ] + with caplog.at_level(logging.INFO, logger=planning.logger.name): + response = planning.client.post("/api/v2/orchestration/plan", json={ + "conversation_id": "conversation-1", "turn_id": "picked-document-turn", + "message": request, "selected_document_ids": [LEFT], + "context_documents": [{"id": LEFT, "label": "Not an authoritative file type"}], + "approval_mode": approval, + }, buffered=True) + frames = [ + json.loads(frame.partition("data:")[2]) + for frame in response.get_data(as_text=True).split("\n\n") if frame.startswith("data:") + ] + assert response.status_code == 200 and "plan" in frames[-1], frames + plan = frames[-1]["plan"] + calls = [ + call for call in harness.model_calls + if call["messages"][0]["content"].startswith("Plan bounded work") + ] + assert len(calls) == 2 + if extension in ("csv", "xlsx"): + context = json.loads(calls[0]["messages"][1]["content"]) + inspection = context["selected_sources"][0]["inspection"] + assert inspection["status"] == "inspected" + assert inspection["tables"][0]["columns"] == ["Code", "Amount"] + assert inspection["tables"][0]["sample_rows"] + assert inspection["content_complete"] is False + assert plan["approval"]["mode"] == approval + assert harness.schema.effective_plan_document_ids(plan, {"document_ids": [LEFT]}) == [LEFT] + assert plan["steps"][0]["capability_id"] == "document_search" + assert execute.call_count == 0 + assert harness.blobs.file_uploads == 0 and harness.assistant_messages() == [] + summary = diagnostic_records(caplog, stage="plan_validated", message="[ORCHESTRATION_PLANNER] Planning diagnostic.")[-1] + assert summary.sc_selected_document_count == summary.sc_used_document_count == 1 + assert summary.sc_missing_document_count == 0 + + +@pytest.mark.parametrize("first,second,stage", [ + ("options", "selection", "selected_requirements"), + ("selection", "options", "plan_normalization"), + ("selection", "selection", "selected_requirements"), + ("options", "options", "plan_normalization"), +]) +def test_every_failure_path_shares_exactly_one_correction(planning, caplog, first, second, stage): + bad = { + "options": proposal([LEFT], capability="document_analyze", options={"mode": "source_observations"}), + "selection": proposal(), + } + with caplog.at_level(logging.INFO, logger=planning.logger.name): + with pytest.raises(planning.harness.planner.PlannerError) as failure: + run_plan(planning, [bad[first], bad[second], proposal([LEFT])]) + assert len(planning.harness.model_calls) == 2 + assert len(planning.harness.replies) == 1 + assert "PRIVATE_" not in str(failure.value) + expected_message = "selected" if second == "selection" else "document-analysis" + assert expected_message in str(failure.value) + terminal = next( + record for record in reversed(caplog.records) + if getattr(record, "sc_message", None) == "[ORCHESTRATION_PLANNER] The request could not be planned." + ) + assert terminal.sc_attempt == 2 and terminal.sc_stage == stage + + +def test_unrelated_malformed_arguments_are_not_made_repairable(planning): + invalid = proposal([LEFT]) + invalid["steps"][0]["arguments"][PRIVATE] = PRIVATE + with pytest.raises(planning.harness.planner.PlannerError): + run_plan(planning, [invalid, proposal([LEFT])]) + assert len(planning.harness.model_calls) == 1 + assert len(planning.harness.replies) == 1 + + +def test_partial_mixed_source_omission_is_corrected_without_dropping_either_file(planning): + kind, plan = run_plan( + planning, [proposal([LEFT]), proposal([LEFT, RIGHT])], identifiers=(LEFT, RIGHT), + ) + assert kind == "plan" and len(planning.harness.model_calls) == 2 + assert set(planning.harness.schema.plan_document_ids(plan)) == {LEFT, RIGHT} + + +def test_implicit_search_scope_keeps_every_selected_document_without_a_retry(planning): + raw = proposal([LEFT]) + raw["steps"][0]["arguments"].pop("document_ids") + kind, plan = run_plan(planning, [raw], identifiers=(LEFT, RIGHT)) + assert kind == "plan" and len(planning.harness.model_calls) == 1 + assert planning.harness.schema.effective_plan_document_ids( + plan, {"document_ids": [LEFT, RIGHT]}, + ) == [LEFT, RIGHT] + + +def test_a_disabled_read_does_not_satisfy_a_selected_document(planning): + raw = proposal() + disabled = proposal([LEFT])["steps"][0] + disabled["enabled"] = False + raw["steps"].insert(0, disabled) + kind, plan = run_plan(planning, [raw, proposal([LEFT])]) + assert kind == "plan" and len(planning.harness.model_calls) == 2 + assert planning.harness.schema.plan_document_ids(plan) == [LEFT] + + +def test_source_free_request_does_not_gain_an_analyze_step(planning): + kind, plan = run_plan(planning, [proposal()], identifiers=()) + assert kind == "plan" and len(planning.harness.model_calls) == 1 + assert [step["capability_id"] for step in plan["steps"]] == ["compose"] + + +def test_selected_document_does_not_prevent_explicit_editor_narrowing(planning): + previous = proposal([LEFT]) + before = deepcopy(previous) + narrowed = proposal() + narrowed["revised_request"] = "Rewrite only the existing answer." + kind, plan = run_plan(planning, [narrowed], edit_context={"current_plan": previous}) + assert kind == "plan" and len(planning.harness.model_calls) == 1 + assert previous == before + assert any("selected documents" in warning for warning in plan["validation"]["repairs"]) + + +def test_edit_failure_preserves_previous_plan_and_legacy_options(planning): + previous = proposal([LEFT], capability="document_analyze", options={}) + before = deepcopy(previous) + invalid = proposal([LEFT], capability="document_analyze", options={"mode": "source_observations"}) + invalid["revised_request"] = "Describe the selected source." + with pytest.raises(planning.harness.planner.PlannerError) as failure: + run_plan(planning, [invalid, invalid], edit_context={"current_plan": previous}) + assert "previous plan is unchanged" in str(failure.value) + assert previous == before + omitted = proposal([LEFT], capability="document_analyze") + omitted["revised_request"] = "Describe the selected source." + kind, plan = run_plan(planning, [omitted], edit_context={"current_plan": previous}) + assert kind == "plan" + assert plan["steps"][0]["arguments"]["analysis_options"]["mode"] == "narrative_findings" + + +def test_limited_spreadsheet_preview_can_still_produce_an_inspection_plan(planning): + source = { + "document_id": RIGHT, "file_name": "workbook.xlsx", "source_kind": "tabular", + "availability": "available", + "inspection": {"status": "limited", "content_complete": False, "tables": []}, + } + raw = proposal([RIGHT], capability="tabular_inspect") + kind, plan = run_plan( + planning, [raw], identifiers=(RIGHT,), context_updates={"selected_sources": [source]}, + ) + assert kind == "plan" and len(planning.harness.model_calls) == 1 + assert plan["steps"][0]["capability_id"] == "tabular_inspect" + + +def test_http_auto_routing_repairs_analyze_options_without_running_analysis(planning, monkeypatch): + harness = planning.harness + authentication = importlib.import_module("functions_authentication") + # Creating the fixture user's preferences must not fetch a Graph profile photo. + monkeypatch.setattr(authentication, "get_user_profile_image", lambda: None) + execute = Mock(side_effect=AssertionError("An Analyze adapter ran while planning.")) + adapters = importlib.import_module("functions_orchestration_adapters") + monkeypatch.setitem(adapters.ADAPTER_REGISTRY, "document_analyze", execute) + request = "Read the text from the selected source." + harness.replies = [ + json.dumps({ + "relationship": "new_topic", "resolved_message": request, "message_ids": [], + "requires_retrieval": True, "clarification": "", + }), + json.dumps(proposal([LEFT], capability="document_analyze", options={"mode": "source_observations"})), + json.dumps(proposal([LEFT], capability="document_analyze")), + ] + response = planning.client.post("/api/v2/orchestration/plan", json={ + "conversation_id": "conversation-1", "turn_id": "auto-model-turn", + "message": request, "selected_document_ids": [LEFT], + "approval_mode": "manual", "model_routing": "auto", + }, buffered=True) + frames = [ + json.loads(frame.partition("data:")[2]) + for frame in response.get_data(as_text=True).split("\n\n") if frame.startswith("data:") + ] + assert response.status_code == 200 and "plan" in frames[-1], frames + plan = frames[-1]["plan"] + assert plan["steps"][0]["arguments"]["analysis_options"]["mode"] == "source_observations" + assert all(step.get("model_binding") for step in plan["steps"]) + assert execute.call_count == 0 + assert len(harness.model_calls) == 3 + + +def test_provider_timeout_is_not_retried_as_a_validation_problem(planning): + error = APITimeoutError(request=httpx.Request("POST", f"https://private.invalid/{PRIVATE}")) + with pytest.raises(planning.harness.planner.PlannerError) as failure: + run_plan(planning, [error, proposal([LEFT])]) + assert failure.value.reason == "model_request_failed" + assert len(planning.harness.model_calls) == 1 + assert len(planning.harness.replies) == 1 + assert PRIVATE not in str(failure.value) + + +@pytest.mark.parametrize("count", [200, 201]) +def test_required_analysis_fields_are_bounded_without_trimming(planning, count): + raw = proposal([LEFT], capability="document_analyze", options={ + "version": "analysis-options-v2", "mode": "reconciled_records", + "required_fields": [f"field_{index}" for index in range(count)], + }) + kind, plan = run_plan(planning, [raw, proposal([LEFT], capability="document_analyze")]) + assert kind == "plan" + if count == 200: + assert len(planning.harness.model_calls) == 1 + assert len(plan["steps"][0]["arguments"]["analysis_options"]["required_fields"]) == count + else: + assert len(planning.harness.model_calls) == 2 + assert plan["steps"][0]["arguments"]["analysis_options"]["mode"] == "source_observations" + + +@pytest.mark.parametrize("debug", [False, True]) +def test_debug_diagnostics_do_not_change_invalid_unicode_rejection(planning, debug): + planning.debug_flags["enable_debug_logging"] = debug + raw = proposal(["\ud800"]) + with pytest.raises(planning.harness.planner.PlannerError): + run_plan(planning, [raw]) + assert len(planning.harness.model_calls) == 1 + + +def test_correction_preserves_explicit_reconciled_field_requirements(planning): + invalid = proposal([LEFT], capability="document_analyze", options={ + **OPTIONS, "required_fields": ["amount"], + }) + corrected = proposal([LEFT], capability="document_analyze", options={ + "version": "analysis-options-v2", "mode": "reconciled_records", + "required_fields": ["amount"], + }) + kind, plan = run_plan( + planning, [invalid, corrected], + message="Extract reconciled records from the selected source; every record must include amount.", + ) + assert kind == "plan" + assert plan["steps"][0]["arguments"]["analysis_options"]["required_fields"] == ["amount"] + repair = planning.harness.model_calls[-1]["messages"][-1]["content"] + assert "Do not discard user-required fields or calculations" in repair + + +def test_correction_cannot_widen_the_authorized_document_selection(planning): + with pytest.raises(planning.harness.planner.PlannerError): + run_plan(planning, [proposal(), proposal([LEFT, RIGHT])]) + assert len(planning.harness.model_calls) == 2 + for call in planning.harness.model_calls: + context = json.loads(call["messages"][1]["content"]) + assert context["user_selected"]["documents"] == [LEFT] + + +@pytest.mark.parametrize("retain_source,rule", [ + (True, "selected_capabilities_missing"), + (False, "selected_documents_and_capabilities_missing"), +]) +def test_positive_operation_requirements_are_not_dropped(planning, caplog, retain_source, rule): + initial = proposal([LEFT]) if retain_source else proposal() + corrected = proposal([LEFT], capability="document_analyze") + with caplog.at_level(logging.INFO, logger=planning.logger.name): + kind, plan = run_plan( + planning, [initial, corrected], seed_updates={"required_capabilities": ["document_analyze"]}, + ) + assert kind == "plan" and len(planning.harness.model_calls) == 2 + assert plan["steps"][0]["capability_id"] == "document_analyze" + correction = next( + record for record in caplog.records + if getattr(record, "sc_message", None) == "[ORCHESTRATION_PLANNER] Asking the planner to correct a rejected plan." + ) + assert correction.sc_validation_rule == rule + + +@pytest.mark.parametrize("first", ["selection", "options", "question"]) +def test_unrenderable_question_cannot_reset_the_shared_correction_budget(planning, first): + question = { + "kind": "elicitation", "message": "Choose a business rule.", + "clarification_context": {"reason": "business_rule", "document_ids": [LEFT]}, + "requested_schema": {"type": "object", "properties": {"nested": {"type": "object"}}}, + } + initial = { + "selection": proposal(), + "options": proposal([LEFT], capability="document_analyze", options={"mode": "source_observations"}), + "question": question, + }[first] + with pytest.raises(planning.harness.planner.PlannerError): + run_plan(planning, [initial, question, proposal([LEFT])]) + assert len(planning.harness.model_calls) == 2 + assert len(planning.harness.replies) == 1 + + +def test_unrenderable_question_keeps_the_same_model_and_selected_sources(planning): + question = { + "kind": "elicitation", "message": "Choose a business rule.", + "clarification_context": {"reason": "business_rule", "document_ids": [LEFT]}, + "requested_schema": {"type": "object", "properties": {"nested": {"type": "object"}}}, + } + kind, plan = run_plan(planning, [question, proposal([LEFT])]) + assert kind == "plan" and len(planning.harness.model_calls) == 2 + assert len(planning.harness.clients) == 1 + assert plan["steps"][0]["arguments"]["document_ids"] == [LEFT]