Fix orchestration progress, workbook citations, and message cleanup

application/single_app/config.py

@@ -103,7 +103,7 @@
 EXECUTOR_MAX_WORKERS = 30
 CHAT_STREAM_EXECUTOR_MAX_WORKERS = max(8, int(os.getenv('SIMPLECHAT_CHAT_STREAM_WORKERS', '32') or '32'))
 SESSION_TYPE = 'filesystem'
-VERSION = "0.261.416"
+VERSION = "0.261.417"
 IS_DEVELOPMENT = is_development_env_enabled()
 
 # Opt-out for deployments where App Service Easy Auth is active but the platform

application/single_app/functions_citation_tracking.py

@@ -50,7 +50,7 @@
 )
 _INLINE_DOCUMENT_SOURCE_PATTERN = re.compile(
     r"\(Source:((?:(?!\(Source:).){1,256}?),"
-    r"\s{0,8}(Page(?:s)?|Sheet(?:s)?|Location):"
+    r"\s{0,8}(Page(?:s)?|Sheet(?:s)?|Location|Worksheet cell):"
     r"((?:(?!\(Source:).){0,256}?)\)"
     r"(?=\s{0,8}(?:\[#|$|[.;,!? \n]))",
     re.IGNORECASE,
@@ -356,6 +356,10 @@
             _source_reference_matches_citation(source_reference, citation)
             for source_reference in unmatched_source_references
         )
+        if citation.get("native_evidence") or ":native-comments:" in citation_id:
+            # A worksheet/cell can contain several roots and replies. Only an exact
+            # retained identity proves which reviewer text the answer cited.
+            matched_by_source = False
         if not matched_by_id and not matched_by_source:
             continue
         if matched_by_id:

application/single_app/functions_orchestration_adapters.py

@@ -876,7 +876,8 @@ def _run_excel_comments(step, context, *, settings, user_id, emit, cancel_reques
             citations.append({
                 'source_type': 'document', 'document_id': source['document_id'],
                 'file_name': source.get('file_name'), 'citation_id': results[-1]['id'],
-                'location_label': 'Native annotation coverage',
+                'evidence_kind': 'native_annotation_coverage',
+                'location_label': 'Location', 'location_value': 'Native annotation inventory',
             })
             for index, record in enumerate(records):
                 results.append({
@@ -889,9 +890,13 @@ def _run_excel_comments(step, context, *, settings, user_id, emit, cancel_reques
                     'source_type': 'document', 'document_id': source['document_id'],
                     'file_name': source.get('file_name'), 'sheet': record.get('sheet'),
                     'cell': record.get('cell'), 'annotation_id': record.get('id'),
+                    'evidence_kind': 'native_excel_annotation',
                     'citation_id': results[-1]['id'], 'sheet_name': record.get('sheet'),
-                    'location_label': 'Worksheet cell',
-                    'location_value': f"{record.get('sheet')}!{record.get('cell')}",
+                    'location_label': 'Location',
+                    'location_value': (
+                        f"{record['sheet']}!{record['cell']}"
+                        if record.get('sheet') and record.get('cell') else 'Workbook annotation'
+                    ),
                 })
         current_manifest = resolve_context_source_manifest(
             context, identifiers, settings=settings, user_id=user_id, cancel_requested=cancel_requested,

application/single_app/functions_orchestration_composition.py

@@ -432,6 +432,8 @@
             }
         if not citation_id or location in (None, ''):
             continue
+        # The browser's retained-result capability is transport metadata, not model input.
+        citation = {key: value for key, value in citation.items() if key != 'native_evidence'}
         citations.append({
             **citation,
             'citation': build_inline_citation_marker(

application/single_app/functions_orchestration_result_runtime.py

@@ -6,6 +6,7 @@
 
 from copy import deepcopy
 from dataclasses import replace
+import json
 
 from content_screening.contracts import DocumentHeldError, ScreeningError
 from functions_analysis_access import analysis_source_snapshot
@@ -266,6 +267,7 @@
         'citation_id', 'chunk_id', 'file_name', 'title', 'page_number', 'chunk_sequence',
         'sheet_name', 'location_label', 'location_value', 'classification', 'document_classification', 'score',
         'evidence_id',
+        'annotation_id', 'evidence_kind', 'cell', 'sheet',
     )
 
     def source_key(source):
@@ -306,6 +308,7 @@
             for parent in lineage['upstream']:
                 visit(ResultRef.from_dict(parent))
         retained_citations = []
+        native_reference = None
         if (
             producer.capability_id == 'document_search'
             and (current.output_name, current.kind) in (
@@ -334,6 +337,8 @@
                     _search_evidence_citations(reader, prepared)
                     if current.output_name == 'evidence' else prepared['citations']
                 )
+                if prepared['content_scope'] == 'native_excel_annotations':
+                    native_reference = prepared_reader.reference.to_dict()
                 for source in lineage['sources']:
                     search_counts.setdefault(source_key(source), set())
         elif producer.capability_id in ('document_analyze', 'document_compare'):
@@ -367,10 +372,19 @@
             if len(matches) != 1:
                 raise ResultContractError('result_citation_source_invalid')
             projected = project(matches[0], citation)
+            if native_reference is not None:
+                projected.update({
+                    'native_evidence': deepcopy(native_reference),
+                    'source_version': matches[0]['source_version'],
+                    'source_revision': matches[0].get('source_revision'),
+                })
             identity_key = source_key(matches[0])
             if isinstance(citation.get('file_name'), str) and citation['file_name'].strip():
                 source_names[identity_key] = citation['file_name'].strip()
-            if producer.capability_id == 'document_search' and identity_key in search_counts:
+            if (
+                producer.capability_id == 'document_search' and identity_key in search_counts
+                and citation.get('evidence_kind') != 'native_annotation_coverage'
+            ):
                 search_counts[identity_key].add((
                     producer, current.manifest_sha256,
                     canonical_digest({name: value for name, value in projected.items() if name != 'evidence_id'}),
@@ -404,6 +418,66 @@
     return projected
 
 
+def read_native_citation_evidence(service, reference, *, citation_id, document_id):
+    """Resolve an exact committed annotation, with current source access and version.
+
+    Unlike ordinary retained-result inspection, a source citation also authorizes the
+    workbook it offers to preview. Never search a current inventory by cell or decode
+    a document identity from the synthetic citation ID.
+    """
+    if (
+        type(reference) is not ResultRef or reference.producer.capability_id != 'document_search'
+        or reference.output_name != 'prepared' or reference.kind != 'structured-v1'
+    ):
+        raise ResultContractError('native_citation_reference_invalid')
+    reader = service.open_result(reference, allow_partial=True, require_current_sources=True)
+    prepared = read_complete_input(reader)
+    if (
+        type(prepared) is not dict or prepared.get('content_scope') != 'native_excel_annotations'
+        or type(prepared.get('citations')) is not list or type(prepared.get('search_results')) is not list
+    ):
+        raise ResultContractError('native_citation_reference_invalid')
+    citations = [
+        item for item in prepared['citations']
+        if type(item) is dict and item.get('citation_id') == citation_id and item.get('document_id') == document_id
+    ]
+    hits = [
+        item for item in prepared['search_results']
+        if type(item) is dict and item.get('id') == citation_id and item.get('document_id') == document_id
+    ]
+    manifest = service._load_manifest(reference)
+    sources = [item for item in manifest['lineage']['sources'] if item['document_id'] == document_id]
+    if len(citations) != 1 or len(hits) != 1 or len(sources) != 1:
+        raise ResultUnavailableError('native_citation_unavailable')
+    service.access.authorize_input_sources(sources, require_snapshot=True)
+    try:
+        value = json.loads(hits[0]['chunk_text'])
+    except (KeyError, TypeError, ValueError):
+        raise ResultContractError('native_citation_evidence_invalid') from None
+    if type(value) is not dict:
+        raise ResultContractError('native_citation_evidence_invalid')
+    annotation = value.get('excel_annotation')
+    coverage = value.get('native_coverage')
+    if type(annotation) is dict:
+        citation = citations[0]
+        if any(citation.get(field) != annotation.get(record_field) for field, record_field in (
+            ('annotation_id', 'id'), ('sheet', 'sheet'), ('cell', 'cell'),
+        )):
+            raise ResultContractError('native_citation_evidence_invalid')
+        response = {'native_annotation': deepcopy(annotation), 'cited_text': annotation.get('text') or ''}
+    elif type(coverage) is dict:
+        response = {'native_coverage': deepcopy(coverage), 'cited_text': ''}
+    else:
+        raise ResultContractError('native_citation_evidence_invalid')
+    reader.recheck()
+    service.access.authorize_input_sources(sources, require_snapshot=True)
+    return {
+        **response, 'citation_id': citation_id, 'document_id': document_id,
+        'file_name': hits[0].get('file_name') or '', 'source_version': sources[0]['source_version'],
+        'evidence_kind': 'native_excel_annotation' if annotation is not None else 'native_annotation_coverage',
+    }
+
+
 def _authorize_task_result(step, context, task, *, reused=False):
     if type(task) is not TaskResult:
         raise ResultContractError('result_contract_invalid')

application/single_app/functions_saved_analysis.py

@@ -2072,7 +2072,7 @@ def cleanup_chat_analysis_messages(
                 ) if value
             }
             for user_id in thought_owners:
-                delete_thoughts(conversation_id, message["id"], user_id)
+                delete_thoughts(message["id"], user_id)
         descriptors = list(metadata.get("saved_analyses") or [])
         if metadata.get("saved_analysis"):
             descriptors.append(metadata["saved_analysis"])

application/single_app/route_backend_documents.py

@@ -24,6 +24,10 @@
     build_synced_document_delete_guard,
 )
 from functions_notifications import create_notification, delete_notifications_by_metadata
+from functions_orchestration_result_contracts import ResultContractError, ResultRef
+from functions_orchestration_result_runtime import read_native_citation_evidence
+from functions_orchestration_results import ResultUnavailableError
+from functions_workflow_result_store import WorkflowResultIntegrityError
 from functions_document_access_index import (
     DOCUMENT_ACCESS_SCOPE_PERSONAL,
     is_document_access_shadow_validation_enabled,
@@ -1765,6 +1769,38 @@ def get_citation():
         if not citation_id:
             return jsonify({"error": "Missing citation_id"}), 400
 
+        if data.get("native_evidence") is not None or (
+            isinstance(citation_id, str) and ":native-comments:" in citation_id
+        ):
+            if not data.get("native_evidence") or not document_id or not data.get("conversation_id"):
+                return jsonify({"error": "Retained native citation evidence is unavailable."}), 404
+            try:
+                reference = ResultRef.from_dict(data["native_evidence"])
+                if (
+                    reference.producer.user_id != user_id
+                    or reference.producer.conversation_id != data["conversation_id"]
+                ):
+                    return jsonify({"error": "Unauthorized access to citation"}), 403
+                # Bootstrap depends on initialized config; load only for native citation reads.
+                from functions_orchestration_bootstrap import build_orchestration_services
+
+                services = build_orchestration_services(user_id, data["conversation_id"])
+                response = read_native_citation_evidence(
+                    services.results, reference, citation_id=citation_id, document_id=document_id,
+                )
+                return jsonify(response), 200
+            except ResultContractError:
+                return jsonify({"error": "Invalid native citation reference."}), 400
+            except (WorkflowResultIntegrityError, ResourceNotFoundError):
+                return jsonify({"error": "Retained native citation evidence is unavailable."}), 404
+            except ScreeningError as error:
+                return jsonify({"error": error.public_message, "error_code": error.code}), error.status_code
+            except (ResultUnavailableError, PermissionError):
+                return jsonify({"error": "The cited workbook or retained evidence is no longer available."}), 403
+            except Exception as error:
+                log_event("[CITATION] Native citation lookup failed.", extra={"error_type": type(error).__name__}, level=logging.WARNING)
+                return jsonify({"error": "Unable to load retained native citation evidence."}), 500
+
         def build_citation_response(chunk):
             return jsonify({
                 "cited_text": chunk.get("chunk_text", ""),
@@ -1812,7 +1848,8 @@ def get_citation_for_scope(search_client, scope_name):
             except ResourceNotFoundError:
                 continue
             except Exception as e:
-                return jsonify({"error": f"Unexpected error: {str(e)}"}), 500
+                log_event("[CITATION] Citation lookup failed.", extra={"error_type": type(e).__name__}, level=logging.WARNING)
+                return jsonify({"error": "Unable to load the cited passage."}), 500
 
             if citation_response:
                 return citation_response