From 0fb76cda58293bc0a61bf7683132b967ceb63977 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Thu, 5 Nov 2026 13:36:16 -0500 Subject: [PATCH 1/2] Fix Excel comment evidence and add table-aware inspection Deliver canonical native records with source-bound paging, share streaming table discovery across analysis and citation previews, and document saved-state and migration limits. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../single_app/admin_settings_fields.py | 6 +- application/single_app/config.py | 2 +- .../single_app/content_screening/access.py | 22 +- .../content_screening/extraction.py | 3 + .../content_screening/publication.py | 40 +- .../single_app/content_screening/service.py | 7 +- .../single_app/excel_workbook_reader.py | 1001 ++++++++++++++++ .../single_app/functions_accessibility.py | 28 +- .../functions_agent_document_citations.py | 1 + .../functions_document_citations.py | 2 + application/single_app/functions_documents.py | 185 ++- .../functions_excel_accessibility.py | 656 ++++++++-- .../single_app/functions_excel_inventory.py | 379 ++++++ .../functions_native_tabular_compute.py | 53 +- .../single_app/functions_office_media.py | 20 +- .../single_app/functions_search_service.py | 138 ++- .../functions_tabular_generated_exports.py | 227 +++- application/single_app/route_backend_chats.py | 98 +- .../single_app/route_enhanced_citations.py | 151 +-- .../document_search_plugin.py | 27 +- .../tabular_processing_plugin.py | 1050 ++++++++++++++--- .../templates/admin/_panes/extraction.html | 4 +- .../chat/EnhancedCitationViewer.tsx | 260 +++- application/v2_ui/src/lib/endpoints.ts | 46 +- docs/_data/features.yml | 6 +- docs/admin/knowledge.md | 8 +- ..._ACCESSIBILITY_AND_STRUCTURE_EXTRACTION.md | 15 +- .../features/EXCEL_NATIVE_INSPECTION.md | 131 ++ ...L_COMMENT_CONTENT_AND_TABLE_HEADERS_FIX.md | 68 ++ docs/explanation/release_notes.md | 32 + docs/guides/inspect-excel-workbooks.md | 76 ++ docs/reference/actions/document-search.md | 21 +- docs/reference/actions/tabular-processing.md | 36 + docs/reference/chat-controls.md | 19 +- .../cosmos_response_boundaries.json | 566 ++++----- ...excel_document_accessibility_extraction.py | 15 +- ...test_excel_document_delete_sdk_contract.py | 87 ++ .../test_excel_inventory_persistence.py | 223 ++++ .../test_excel_native_images_and_release.py | 169 +++ ...t_excel_native_inspection_chat_evidence.py | 231 ++++ functional_tests/test_excel_preview_route.py | 127 ++ .../test_excel_table_discovery_and_scale.py | 144 +++ ...st_mixed_source_chat_search_consistency.py | 10 +- .../test_support/document_analysis.py | 3 +- ...r_enhanced_citations_schema_summary_fix.py | 9 +- ...group_blob_context_and_sheet_whitespace.py | 40 +- .../test_tabular_large_result_pagination.py | 24 +- .../test_tabular_model_token_budgets.py | 13 +- ...t_tabular_preview_json_sanitization_fix.py | 6 +- .../test_tabular_row_orchestration_scale.py | 360 +++--- ...st_tabular_table_aware_analytics_replay.py | 952 +++++++++++++++ .../fixtures/orchestration/harness_entry.tsx | 3 + ui_tests/test_v2_excel_citation_inspection.py | 191 +++ 53 files changed, 6934 insertions(+), 1057 deletions(-) create mode 100644 application/single_app/excel_workbook_reader.py create mode 100644 application/single_app/functions_excel_inventory.py create mode 100644 docs/explanation/features/EXCEL_NATIVE_INSPECTION.md create mode 100644 docs/explanation/fixes/EXCEL_COMMENT_CONTENT_AND_TABLE_HEADERS_FIX.md create mode 100644 docs/guides/inspect-excel-workbooks.md create mode 100644 functional_tests/test_excel_document_delete_sdk_contract.py create mode 100644 functional_tests/test_excel_inventory_persistence.py create mode 100644 functional_tests/test_excel_native_images_and_release.py create mode 100644 functional_tests/test_excel_native_inspection_chat_evidence.py create mode 100644 functional_tests/test_excel_preview_route.py create mode 100644 functional_tests/test_excel_table_discovery_and_scale.py create mode 100644 functional_tests/test_tabular_table_aware_analytics_replay.py create mode 100644 ui_tests/test_v2_excel_citation_inspection.py diff --git a/application/single_app/admin_settings_fields.py b/application/single_app/admin_settings_fields.py index 9addf9403..88061c1c0 100644 --- a/application/single_app/admin_settings_fields.py +++ b/application/single_app/admin_settings_fields.py @@ -2546,10 +2546,10 @@ { "key": "enable_office_embedded_image_analysis", "type": "switch", - "label": "Analyze images embedded in DOCX and PPTX files", + "label": "Analyze images embedded in DOCX, PPTX, and Excel files", "help": ( - "Neither extraction engine describes figures inside Word and PowerPoint " - "files. With this on, embedded images are pulled out, analysed with " + "Native extraction does not describe image pixels inside Word, PowerPoint, " + "or Excel files. With this on, embedded images are pulled out, analysed with " "whichever engine backs the selected extraction mode, and indexed as " "their own citable chunks. Works with Standard extraction too." ), diff --git a/application/single_app/config.py b/application/single_app/config.py index 29e148d72..6ce8c4d2a 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -103,7 +103,7 @@ EXECUTOR_MAX_WORKERS = 30 CHAT_STREAM_EXECUTOR_MAX_WORKERS = max(8, int(os.getenv('SIMPLECHAT_CHAT_STREAM_WORKERS', '32') or '32')) SESSION_TYPE = 'filesystem' -VERSION = "0.261.404" +VERSION = "0.261.405" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/content_screening/access.py b/application/single_app/content_screening/access.py index f03685900..b2a538e12 100644 --- a/application/single_app/content_screening/access.py +++ b/application/single_app/content_screening/access.py @@ -93,6 +93,7 @@ "group_document_collaboration_operation", "document_share_details", "public_document_projection_writer", "public_document_collaboration_operation", + "excel_native_inventory", # Recomputed per request from current authorization, so a stored copy is # always stale. The group projection re-adds them after serialization. "document_actions", "document_collaboration_actions", @@ -1068,6 +1069,23 @@ def _with_origin_kind(payload, document): return payload +def _with_current_excel_metadata(payload, document): + metadata = payload.get("accessibility_metadata") + marker = document.get(SCREENING_FIELD) + if ( + not isinstance(metadata, Mapping) or metadata.get("format") not in {"xlsx", "xlsm", "xls"} + or not isinstance(marker, Mapping) + ): + return payload + active = marker.get("active_blob") + inventory = metadata.get("inventory_storage") + digest = inventory.get("source_sha256") if isinstance(inventory, Mapping) else None + if not isinstance(active, Mapping) or not digest or active.get("content_hash") != digest: + payload.pop("accessibility_metadata", None) + payload["native_inspection_status"] = "requires_active_source_inspection" + return payload + + def public_document_payload(document): if not isinstance(document, Mapping): return {} @@ -1080,7 +1098,7 @@ def public_document_payload(document): payload["status"] = resolve_metadata_completion_status( document.get("status"), document.get("metadata_extraction_status"), ) - return payload + return _with_current_excel_metadata(payload, document) try: _require_available_metadata(document) config = import_module("config") @@ -1116,7 +1134,7 @@ def public_document_payload(document): document.get("status"), document.get("metadata_extraction_status"), ) payload["num_chunks"] = chunk_count_for_response(document) - return payload + return _with_current_excel_metadata(payload, document) def public_documents_payload(documents, user_id=None, *, metadata_reader=None): diff --git a/application/single_app/content_screening/extraction.py b/application/single_app/content_screening/extraction.py index d70649cec..9298e63ee 100644 --- a/application/single_app/content_screening/extraction.py +++ b/application/single_app/content_screening/extraction.py @@ -9,6 +9,7 @@ from dataclasses import dataclass, field from datetime import date, datetime, time from pathlib import Path +from typing import Callable from content_screening.contracts import ( CONTENT_METADATA_FIELDS, @@ -43,6 +44,8 @@ class ExtractionCapture: completed: bool = False preloaded_text: bool = False failure_code: str | None = None + artifact_storage: object | None = field(default=None, repr=False, compare=False) + heartbeat: Callable[[], None] | None = field(default=None, repr=False, compare=False) def _append(self, target, unit): self.characters += len(unit.text) diff --git a/application/single_app/content_screening/publication.py b/application/single_app/content_screening/publication.py index af3f77b54..4e3468043 100644 --- a/application/single_app/content_screening/publication.py +++ b/application/single_app/content_screening/publication.py @@ -2,6 +2,7 @@ """Application-side projection of inspected canonical content.""" import hashlib +import io import json import mimetypes from urllib.parse import quote @@ -171,6 +172,34 @@ def _remove_previous_raw_projection(document, scan, projection, storage): ) +def _tabular_publication_chunks(units, file_name, max_characters, *, include_native, schema_text=None): + chunks = publication_chunks([ + ContentUnit("table-schema", table_schema_text(units) if schema_text is None else schema_text, {"kind": "schema"}), + ], max_characters) + if not include_native or not any(unit.locator.get("kind") == "excel_native" for unit in units): + return chunks + prefix = f"### Excel native inspection\nWorkbook: {file_name}\nSearch preview; complete records are in the native inventory.\n" + if len(prefix) >= max_characters: + prefix = "### Excel native inspection\n" + if len(prefix) >= max_characters: + raise ScreeningValidationError("The publication chunk size cannot fit the Excel native evidence marker.") + remaining = 1000000 + for unit in units: + if unit.locator.get("kind") != "excel_native" or remaining <= len(prefix): + continue + preview = ContentUnit(unit.unit_id, unit.text[:remaining], unit.locator) + for chunk in publication_chunks([preview], max_characters - len(prefix)): + if remaining <= len(prefix): + break + text = prefix + chunk["chunk_text"][:remaining - len(prefix)] + chunks.append({ + **chunk, "chunk_text": text, "chunk_sequence": len(chunks) + 1, + "end": chunk["start"] + len(text) - len(prefix), + }) + remaining -= len(text) + return chunks + + def publish_document(document, units, scan, actor_id, *, storage): helpers = _document_helpers() subject = subject_from_document(document) @@ -195,9 +224,14 @@ def publish_document(document, units, scan, actor_id, *, storage): max(1, helpers.get_embedding_usable_tokens(settings) // 4), ) if structured_sheets(units): - chunks = publication_chunks([ - ContentUnit("table-schema", table_schema_text(units), {"kind": "schema"}), - ], max_characters) + extension = "." + file_name.rsplit(".", 1)[-1].lower() + schema_text = None + if extension in {".xlsx", ".xlsm", ".xls"}: + schema_text = helpers._build_tabular_schema_summary(io.BytesIO(content), file_name, extension) + chunks = _tabular_publication_chunks( + units, file_name, max_characters, include_native=not scan.get("sanitized"), + schema_text=schema_text, + ) else: chunks = publication_chunks(units, max_characters) if not chunks: diff --git a/application/single_app/content_screening/service.py b/application/single_app/content_screening/service.py index 7cf8c93e9..f81577d58 100644 --- a/application/single_app/content_screening/service.py +++ b/application/single_app/content_screening/service.py @@ -472,7 +472,7 @@ def process_screened_upload(document_id, user_id, temp_file_path, original_filen def _extract_and_inspect(scan, actor_id, source_path, processor, *, repository, storage, extraction_mode_override=None): subject = Subject.from_dict(scan["subject"]) scan, owner = _claim_scan(repository, _read_scan(repository, scan["id"])) - capture = ExtractionCapture(subject, scan["id"], scan["original_file_name"]) + capture = ExtractionCapture(subject, scan["id"], scan["original_file_name"], artifact_storage=storage) last_heartbeat = _now() def heartbeat(): @@ -1049,6 +1049,11 @@ def reprocess_document(subject, actor_id, extraction_mode, *, repository=None, s def prepare_document_deletion(document, actor_id, *, repository=None, storage=None): """Revoke a screened document before removing private evidence and originals.""" if SCREENING_FIELD not in document: + if ( + "excel_native_inventory" in document + or str(document.get("file_name") or "").lower().endswith((".xlsx", ".xlsm", ".xls")) + ): + _storage(storage).delete_revision(subject_from_document(document)) return repository = _repository(repository) subject = subject_from_document(document) diff --git a/application/single_app/excel_workbook_reader.py b/application/single_app/excel_workbook_reader.py new file mode 100644 index 000000000..b9df0fad2 --- /dev/null +++ b/application/single_app/excel_workbook_reader.py @@ -0,0 +1,1001 @@ +# excel_workbook_reader.py +"""Read saved workbook cells and table boundaries without executing Excel.""" + +from collections import Counter, OrderedDict +from copy import deepcopy +import hashlib +import io +import json +import math +import os +import posixpath +import re +import zipfile +from threading import RLock + +from defusedxml.ElementTree import iterparse +from defusedxml.ElementTree import fromstring +from openpyxl.formula.translate import Translator, TranslatorError +from openpyxl.styles.numbers import BUILTIN_FORMATS, is_date_format +from openpyxl.utils.cell import get_column_letter, range_boundaries +from openpyxl.utils.datetime import CALENDAR_MAC_1904, CALENDAR_WINDOWS_1900, from_excel +import xlrd + +from functions_office_media import ( + OFFICE_ZIP_ALLOWED_COMPRESSION, + OFFICE_ZIP_MAX_ENTRIES, + _read_zip_entry_bounded, +) + + +EXCEL_WORKBOOK_SCHEMA_VERSION = "2.0" +EXCEL_XML_PART_MAX_BYTES = 16 * 1024 * 1024 +EXCEL_SHEET_MAX_BYTES = 256 * 1024 * 1024 +EXCEL_SCAN_MAX_BYTES = 512 * 1024 * 1024 +EXCEL_SHARED_STRINGS_MAX_CHARS = 64 * 1024 * 1024 +EXCEL_MAX_SHEETS = 100 +EXCEL_MAX_TABLES = 5000 +EXCEL_MAX_PREVIEW_ROWS = 500 +EXCEL_PREVIEW_MAX_BYTES = 1024 * 1024 +EXCEL_PREVIEW_MAX_COLUMNS = 128 +EXCEL_MAX_COLUMNS = 16384 +EXCEL_MAX_ROWS = 1048576 +_INTEGER = re.compile(r"^[+-]?\d+$") +_CELL = re.compile(r"^\$?([A-Z]{1,3})\$?([1-9]\d*)$") +_LAYOUT_ELEMENTS = { + "mergeCell", "col", "pane", "pageMargins", "pageSetup", "printOptions", + "sheetFormatPr", "sheetView", "autoFilter", "sheetPr", +} +_MANIFEST_CACHE = OrderedDict() +_MANIFEST_LOCK = RLock() +_MANIFEST_CACHE_MAX_BYTES = 16 * 1024 * 1024 + + +def local_name(tag): + return tag.rsplit("}", 1)[-1] + + +def local_attribute(element, name, default=""): + return next((value for key, value in element.attrib.items() if local_name(key) == name), default) + + +def first_element(element, name): + return next((child for child in element.iter() if local_name(child.tag) == name), None) + + +def element_text(element): + return "".join(element.itertext()) if element is not None else "" + + +def cell_coordinates(reference): + match = _CELL.fullmatch(str(reference or "").upper()) + if not match: + raise ValueError("Invalid Excel cell reference.") + column = 0 + for letter in match[1]: + column = column * 26 + ord(letter) - ord("A") + 1 + row = int(match[2]) + if column > EXCEL_MAX_COLUMNS or row > EXCEL_MAX_ROWS: + raise ValueError("Excel cell reference is outside worksheet bounds.") + return row, column + + +def bounded_range(reference): + try: + bounds = range_boundaries(str(reference or "").replace("$", "")) + except (TypeError, ValueError) as error: + raise ValueError("Invalid Excel table range.") from error + low_column, low_row, high_column, high_row = bounds + if ( + any(type(value) is not int for value in bounds) + or not 1 <= low_column <= high_column <= EXCEL_MAX_COLUMNS + or not 1 <= low_row <= high_row <= EXCEL_MAX_ROWS + ): + raise ValueError("Excel table range is outside worksheet bounds.") + return bounds + + +def _range_reference(low_column, low_row, high_column, high_row): + return f"{get_column_letter(low_column)}{low_row}:{get_column_letter(high_column)}{high_row}" + + +def _blank_cell(reference): + return { + "cell": reference, "value": None, "raw_value": None, "value_type": "blank", + "value_status": "blank", "formula": None, "formula_type": None, + "formula_reference": None, "cached_value_available": True, "style_id": 0, + } + + +class _BoundedXmlStream: + def __init__(self, stream, reader, limit): + self.stream = stream + self.reader = reader + self.limit = limit + self.bytes_read = 0 + + def read(self, size=-1): + size = min(65536, size) if size >= 0 else 65536 + data = self.stream.read(size) + self.bytes_read += len(data) + self.reader.scanned_xml_bytes += len(data) + if self.bytes_read > self.limit or self.reader.scanned_xml_bytes > EXCEL_SCAN_MAX_BYTES: + raise ValueError("Excel XML scan exceeded its supported byte limit.") + return data + + +class _Region: + """An inferred occupied region, retaining samples but never all its rows.""" + + def __init__(self, row, low_column, high_column): + self.low_row = row + self.high_row = row + self.low_column = low_column + self.high_column = high_column + self.row_count = 0 + self.samples = [] + self.header_signature = None + self.repeated_header_rows = [] + self.repeated_header_count = 0 + self.ambiguous = False + self._last_row = None + + def observe(self, row, cells): + if self._last_row == row: + return + self._last_row = row + self.high_row = row + self.row_count += 1 + occupied = { + column: cell for column, cell in cells.items() + if self.low_column <= column <= self.high_column + } + if len(self.samples) < 8: + self.samples.append((row, deepcopy(occupied))) + self._choose_header() + signature = self._signature(occupied) + if ( + self.header_signature is not None and signature == self.header_signature + and row != self.header_row + ): + self.repeated_header_count += 1 + if len(self.repeated_header_rows) < 100: + self.repeated_header_rows.append(row) + + def _signature(self, cells): + return tuple( + (cells.get(column) or {}).get("value") + for column in range(self.low_column, self.high_column + 1) + ) + + def _choose_header(self): + self.header_row = None + self.title_rows = [] + self.header_status = "absent" + if not self.samples: + return + index = 0 + if len(self.samples[0][1]) == 1 and self.high_column > self.low_column: + if len(self.samples) < 2: + return + self.title_rows.append(self.samples[0][0]) + index = 1 + row, cells = self.samples[index] + values = self._signature(cells) + nonblank = [value for value in values if value is not None and value != ""] + if not nonblank or not all(isinstance(value, str) for value in nonblank): + return + if len(nonblank) == 1: + later = self.samples[index + 1:] + if not any( + any(cell["value_type"] not in ("text", "blank") for cell in sample.values()) + for _, sample in later + ): + self.header_status = "ambiguous" + return + self.header_row = row + self.header_status = "ambiguous" if self.ambiguous else "inferred" + self.header_signature = values + + def descriptor(self, sheet): + self._choose_header() + header_cells = next((cells for row, cells in self.samples if row == self.header_row), {}) + labels = [] + for column in range(self.low_column, self.high_column + 1): + value = (header_cells.get(column) or {}).get("value") + labels.append(str(value) if value is not None and value != "" else get_column_letter(column)) + names = _unique_column_names(labels, self.low_column) + reference = _range_reference(self.low_column, self.low_row, self.high_column, self.high_row) + identity = hashlib.sha256(f"{sheet['part']}\0{reference}".encode()).hexdigest()[:20] + return { + "id": f"region-{identity}", "name": f"Data block {reference}", "sheet": sheet["name"], + "origin": "inferred", "range": reference, "min_row": self.low_row, + "max_row": self.high_row, "min_column": self.low_column, + "max_column": self.high_column, "header_row": self.header_row, + "header_row_count": int(self.header_row is not None), "header_status": self.header_status, + "title_rows": self.title_rows, "totals_row_count": 0, + "data_start_row": (self.header_row + 1) if self.header_row else self.low_row, + "data_end_row": self.high_row, + "row_count": self.row_count - len(self.title_rows) - int(self.header_row is not None) - self.repeated_header_count, + "repeated_header_rows": self.repeated_header_rows, + "repeated_header_count": self.repeated_header_count, + "header_signature": list(self.header_signature) if self.header_signature is not None else None, + "columns": [ + {"id": get_column_letter(column), "name": name, "column": column} + for column, name in zip(range(self.low_column, self.high_column + 1), names) + ], + } + + +def _unique_column_names(labels, low_column): + counts = Counter(labels) + return [ + f"{label} [{get_column_letter(low_column + index)}]" if counts[label] > 1 else label + for index, label in enumerate(labels) + ] + + +class _Regions: + def __init__(self): + self.active = [] + self.finished = [] + self.last_row = None + + def add(self, row, cells): + occupied = sorted( + column for column, cell in cells.items() + if cell["value_status"] != "blank" + ) + if not occupied: + self._close() + self.last_row = row + return + if self.last_row is not None and row > self.last_row + 1: + self._close() + runs = [] + start = end = occupied[0] + for column in occupied[1:]: + if column == end + 1: + end = column + else: + runs.append((start, end)) + start = end = column + runs.append((start, end)) + touched = [] + for low, high in runs: + matches = [region for region in self.active if low <= region.high_column and high >= region.low_column] + if len(matches) == 1: + region = matches[0] + region.low_column = min(region.low_column, low) + region.high_column = max(region.high_column, high) + elif not matches: + region = _Region(row, low, high) + self.active.append(region) + else: + region = matches[0] + region.ambiguous = True + for other in matches[1:]: + self.active.remove(other) + self.finished.append(other) + if region not in touched: + touched.append(region) + for region in list(self.active): + if region not in touched: + self.active.remove(region) + self.finished.append(region) + for region in touched: + region.observe(row, cells) + self.last_row = row + if len(self.active) + len(self.finished) > EXCEL_MAX_TABLES: + raise ValueError("Workbook contains more supported data regions than can be inspected.") + + def _close(self): + self.finished.extend(self.active) + self.active = [] + + def finish(self, sheet): + self._close() + return [region.descriptor(sheet) for region in self.finished] + + +class ExcelWorkbookReader: + """A bounded source reader shared by inspection, previews, and analytical replay.""" + + def __init__(self, source, extension=None): + self.source = source + self.extension = (extension or os.path.splitext(os.fspath(source))[1] if isinstance(source, (str, os.PathLike)) else extension or ".xlsx").lower() + self.archive = None + self.legacy = None + self._owns_archive = not isinstance(source, zipfile.ZipFile) + self._xml_cache = {} + self._shared_strings = None + self._number_formats = None + self._manifest = None + self._sheet_cache = {} + self.scanned_xml_bytes = 0 + self.epoch = CALENDAR_WINDOWS_1900 + self.coverage_notes = [] + if self.extension == ".xls": + if isinstance(source, (bytes, bytearray)): + self.legacy = xlrd.open_workbook(file_contents=bytes(source), on_demand=True, formatting_info=True, ragged_rows=True) + else: + self.legacy = xlrd.open_workbook(source, on_demand=True, formatting_info=True, ragged_rows=True) + self.sheets = [ + {"name": self.legacy.sheet_names()[index], "part": f"biff-sheet-{index}", "index": index, + "type": "worksheet", "state": self.legacy.sheet_by_index(index).visibility} + for index in range(min(self.legacy.nsheets, EXCEL_MAX_SHEETS)) + ] + self.workbook_relationships = {} + else: + self.archive = source if isinstance(source, zipfile.ZipFile) else zipfile.ZipFile( + io.BytesIO(bytes(source)) if isinstance(source, (bytes, bytearray)) else source + ) + names = self.archive.namelist() + if len(names) > OFFICE_ZIP_MAX_ENTRIES or len(names) != len(set(names)): + self.close() + raise ValueError("Invalid or oversized workbook package.") + relationships = self.relationships("") + workbook_part = next((value["part"] for value in relationships.values() if value["kind"] == "officeDocument" and not value["external"]), "xl/workbook.xml") + root = self.read_xml(workbook_part) + self.workbook_relationships = self.relationships(workbook_part) + properties = first_element(root, "workbookPr") + if properties is not None and properties.get("date1904") in ("1", "true"): + self.epoch = CALENDAR_MAC_1904 + self.sheets = [] + entries = first_element(root, "sheets") + entries = list(entries) if entries is not None else [] + if len(entries) > EXCEL_MAX_SHEETS: + self.coverage_notes.append(f"Only the first {EXCEL_MAX_SHEETS} sheets were inspected.") + for entry in entries[:EXCEL_MAX_SHEETS]: + relation = self.workbook_relationships.get(local_attribute(entry, "id")) + if not relation or relation["external"] or relation["kind"] not in ("worksheet", "chartsheet"): + self.coverage_notes.append("A sheet relationship was unavailable or unsupported.") + continue + if not entry.get("name") or len(entry.get("name")) > 128: + raise ValueError("A worksheet name exceeds its supported bound.") + self.sheets.append({ + "name": entry.get("name", ""), "part": relation["part"], + "type": relation["kind"], "state": entry.get("state", "visible"), + }) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def close(self): + if self.archive is not None and self._owns_archive: + self.archive.close() + if self.legacy is not None: + self.legacy.release_resources() + + @property + def sheet_names(self): + return [sheet["name"] for sheet in self.sheets] + + def read_xml(self, part, required=True): + if part in self._xml_cache: + return self._xml_cache[part] + if part not in self.archive.namelist(): + if not required: + return None + raise ValueError("A required workbook part is missing.") + data = _read_zip_entry_bounded(self.archive, part, EXCEL_XML_PART_MAX_BYTES) + if data is None: + raise ValueError("Workbook metadata XML exceeds its supported byte limit.") + self.scanned_xml_bytes += len(data) + if self.scanned_xml_bytes > EXCEL_SCAN_MAX_BYTES: + raise ValueError("Excel XML scan exceeded its supported byte limit.") + root = fromstring(data) + self._xml_cache[part] = root + return root + + def relationships(self, part): + path = posixpath.join(posixpath.dirname(part), "_rels", posixpath.basename(part) + ".rels") + root = self.read_xml(path, required=False) + result = {} + for element in root if root is not None else []: + target = local_attribute(element, "Target") + external = local_attribute(element, "TargetMode").lower() == "external" + resolved = None + if not external: + if any(character in target for character in ("\\", ":", "?", "#")): + self.coverage_notes.append("Invalid package relationship targets were not followed.") + continue + resolved = posixpath.normpath(target.lstrip("/") if target.startswith("/") else posixpath.join(posixpath.dirname(part), target)) + if resolved.startswith("../") or resolved in ("..", ".", ""): + self.coverage_notes.append("Out-of-package relationship targets were not followed.") + continue + result[local_attribute(element, "Id")] = { + "kind": local_attribute(element, "Type").rsplit("/", 1)[-1], + "part": resolved, "external": external, "target": target, + } + return result + + def resolve_sheet(self, name): + exact = next((sheet for sheet in self.sheets if sheet["name"] == name), None) + if exact: + return exact + matches = [sheet for sheet in self.sheets if sheet["name"].strip().casefold() == str(name or "").strip().casefold()] + if len(matches) != 1: + raise ValueError("The requested worksheet was not found or is ambiguous.") + return matches[0] + + def _strings(self): + if self._shared_strings is not None: + return self._shared_strings + self._shared_strings = [] + relation = next((value for value in self.workbook_relationships.values() if value["kind"] == "sharedStrings" and not value["external"]), None) + if relation is None: + return self._shared_strings + count = 0 + root = None + with self._xml_events(relation["part"], EXCEL_SHEET_MAX_BYTES) as events: + for event, element in events: + if root is None and event == "start": + root = element + if event == "end" and local_name(element.tag) == "si": + value = element_text(element) + count += len(value) + if count > EXCEL_SHARED_STRINGS_MAX_CHARS or len(self._shared_strings) >= 1000000: + raise ValueError("Workbook shared strings exceed their supported memory bound.") + self._shared_strings.append(value) + element.clear() + if root is not None: + root.clear() + return self._shared_strings + + def _formats(self): + if self._number_formats is not None: + return self._number_formats + self._number_formats = ["General"] + relation = next((value for value in self.workbook_relationships.values() if value["kind"] == "styles" and not value["external"]), None) + if relation is None: + return self._number_formats + root = self.read_xml(relation["part"]) + custom = {int(element.get("numFmtId")): element.get("formatCode", "") for element in root.iter() if local_name(element.tag) == "numFmt"} + xfs = first_element(root, "cellXfs") + if xfs is not None: + if len(xfs) > 20000: + raise ValueError("Workbook styles exceed their supported count.") + self._number_formats = [ + custom.get(int(xf.get("numFmtId", 0)), BUILTIN_FORMATS.get(int(xf.get("numFmtId", 0)), "General")) + for xf in xfs + ] or ["General"] + return self._number_formats + + def _xml_events(self, part, limit): + return _XmlEvents(self, part, limit) + + def _cell(self, element, row_style, column_styles, shared_formulas): + reference = element.get("r") + _, column = cell_coordinates(reference) + style = element.get("s") + if style is None: + style = row_style or next((value for low, high, value in column_styles if low <= column <= high), "0") + result = _blank_cell(reference) + result["style_id"] = int(style) + formats = self._formats() + result["number_format"] = formats[result["style_id"]] if 0 <= result["style_id"] < len(formats) else None + formula = next((child for child in element if local_name(child.tag) == "f"), None) + value_element = next((child for child in element if local_name(child.tag) == "v"), None) + raw = value_element.text if value_element is not None else None + kind = element.get("t", "n") + if formula is not None: + text = formula.text or "" + result["formula_type"] = formula.get("t", "normal") + result["formula_reference"] = formula.get("ref") + identifier = formula.get("si") + if result["formula_type"] == "shared" and identifier is not None: + if text: + shared_formulas[identifier] = (reference, "=" + text) + elif identifier in shared_formulas: + origin, original = shared_formulas[identifier] + try: + text = Translator(original, origin=origin).translate_formula(reference).lstrip("=") + except TranslatorError as error: + raise ValueError("A shared formula reference could not be inspected.") from error + else: + result["formula_type"] = "unresolved_shared" + result["formula"] = "=" + text if text else None + result["cached_value_available"] = raw is not None + if raw is None: + result["value_status"] = "cache_missing" + if kind == "inlineStr": + raw = element_text(first_element(element, "is")) + value, value_type = raw, "text" + elif kind == "s": + strings = self._strings() + index = int(raw) if raw is not None else -1 + if not 0 <= index < len(strings): + raise ValueError("A workbook shared string reference is invalid.") + value, value_type = strings[index], "text" + elif raw is None: + value, value_type = None, "blank" + elif kind == "b": + if raw not in ("0", "1"): + raise ValueError("A saved workbook Boolean is invalid.") + value, value_type = raw == "1", "boolean" + elif kind in ("str", "d", "e"): + value = raw + value_type = {"str": "text", "d": "date", "e": "error"}[kind] + else: + try: + value = int(raw) if _INTEGER.fullmatch(raw) else float(raw) + except ValueError as error: + raise ValueError("A saved workbook numeric value is invalid.") from error + if not math.isfinite(value): + raise ValueError("A saved workbook numeric value is not finite.") + value_type = "number" + if result["number_format"] and is_date_format(result["number_format"]): + try: + value = from_excel(value, self.epoch).isoformat() + except (OverflowError, ValueError) as error: + raise ValueError("A saved workbook date is invalid.") from error + value_type = "date" + result.update(value=value, raw_value=raw, value_type=value_type) + if result["value_status"] != "cache_missing": + result["value_status"] = "blank" if value is None or value == "" else "error" if value_type == "error" else "saved" + return result + + def iter_sheet_rows(self, sheet, metadata=None): + sheet = self.resolve_sheet(sheet) if isinstance(sheet, str) else sheet + if self.legacy is not None: + legacy = self.legacy.sheet_by_index(sheet["index"]) + for row_index in range(legacy.nrows): + cells = {} + for column in range(legacy.row_len(row_index)): + cell = legacy.cell(row_index, column) + reference = f"{get_column_letter(column + 1)}{row_index + 1}" + value = cell.value + result = _blank_cell(reference) + if cell.ctype not in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK): + value_type = {xlrd.XL_CELL_TEXT: "text", xlrd.XL_CELL_BOOLEAN: "boolean", xlrd.XL_CELL_DATE: "date", xlrd.XL_CELL_ERROR: "error"}.get(cell.ctype, "number") + if value_type == "boolean": + value = bool(value) + elif value_type == "date": + value = xlrd.xldate.xldate_as_datetime(value, self.legacy.datemode).isoformat() + result.update(value=value, raw_value=cell.value, value_type=value_type, value_status="saved") + cells[column + 1] = result + yield row_index + 1, cells, {} + self.legacy.unload_sheet(sheet["index"]) + return + metadata = metadata if metadata is not None else {} + column_styles, shared_formulas = [], {} + stack = [] + previous_row = 0 + with self._xml_events(sheet["part"], EXCEL_SHEET_MAX_BYTES) as events: + for event, element in events: + name = local_name(element.tag) + if event == "start": + stack.append(element) + if len(stack) > 64: + raise ValueError("Workbook XML nesting exceeds its supported depth.") + continue + if name == "row": + row = int(element.get("r")) + if not previous_row < row <= EXCEL_MAX_ROWS: + raise ValueError("A worksheet row is outside supported bounds.") + previous_row = row + cells = {} + for child in element: + if local_name(child.tag) == "c": + cell = self._cell(child, element.get("s"), column_styles, shared_formulas) + cell_row, column = cell_coordinates(cell["cell"]) + if cell_row != row or column in cells: + raise ValueError("Worksheet cell coordinates are inconsistent.") + cells[column] = cell + yield row, cells, dict(element.attrib) + if len(stack) > 1: + stack[-2].remove(element) + element.clear() + elif name == "col": + if element.get("style") is not None: + column_styles.append((int(element.get("min")), int(element.get("max")), element.get("style"))) + if name in _LAYOUT_ELEMENTS: + layout = metadata.setdefault("layout", []) + if len(layout) < 200: + layout.append({"type": name, **dict(element.attrib)}) + elif name == "headerFooter": + metadata["header_footer"] = { + "flags": dict(element.attrib), + "expressions": {local_name(child.tag): child.text or "" for child in element}, + } + elif name == "cfRule": + metadata["conditional_format_rules"] = metadata.get("conditional_format_rules", 0) + 1 + elif name == "hyperlink": + metadata.setdefault("hyperlinks", []).append(dict(element.attrib)) + elif name == "dataValidation": + metadata.setdefault("validations", []).append({ + **dict(element.attrib), + "formulas": {local_name(child.tag): child.text or "" for child in element}, + }) + elif name == "tablePart": + metadata.setdefault("table_relationship_ids", []).append(local_attribute(element, "id")) + elif name in ("oleObjects", "cellControl", "featurePropertyBag"): + metadata.setdefault("unsupported_controls", []).append(name) + stack.pop() + + def _named_tables(self, sheet, metadata): + if self.archive is None: + return [] + relationships = self.relationships(sheet["part"]) + tables = [] + for identifier in metadata.get("table_relationship_ids", []): + relation = relationships.get(identifier) + if not relation or relation["kind"] != "table" or relation["external"]: + raise ValueError("A worksheet table relationship is unavailable.") + root = self.read_xml(relation["part"]) + low_column, low_row, high_column, high_row = bounded_range(root.get("ref")) + header_count = int(root.get("headerRowCount", "1")) + totals_count = int(root.get("totalsRowCount", "0")) + if header_count not in (0, 1) or totals_count not in (0, 1) or low_row + header_count + totals_count > high_row + 1: + raise ValueError("A named Excel table has an invalid header or totals range.") + table_columns = first_element(root, "tableColumns") + labels = [item.get("name") or get_column_letter(low_column + index) for index, item in enumerate(table_columns if table_columns is not None else [])] + if len(labels) != high_column - low_column + 1: + raise ValueError("A named Excel table has inconsistent column definitions.") + names = _unique_column_names(labels, low_column) + identity = hashlib.sha256(f"{sheet['part']}\0{relation['part']}\0{root.get('id')}".encode()).hexdigest()[:20] + tables.append({ + "id": f"table-{identity}", "name": root.get("displayName") or root.get("name") or "Named table", + "sheet": sheet["name"], "origin": "named", "part": relation["part"], + "range": root.get("ref"), "min_row": low_row, "max_row": high_row, + "min_column": low_column, "max_column": high_column, + "header_row": low_row if header_count else None, "header_row_count": header_count, + "header_status": "declared" if header_count else "absent", + "data_start_row": low_row + header_count, "data_end_row": high_row - totals_count, + "totals_row_count": totals_count, "title_rows": [], + "row_count": high_row - low_row + 1 - header_count - totals_count, + "repeated_header_rows": [], "repeated_header_count": 0, + "header_signature": None, + "columns": [ + {"id": get_column_letter(column), "name": name, "column": column} + for column, name in zip(range(low_column, high_column + 1), names) + ], + }) + return tables + + def scan_sheet(self, sheet, row_callback=None, wanted_cells=None): + sheet = self.resolve_sheet(sheet) if isinstance(sheet, str) else sheet + if sheet["part"] in self._sheet_cache and row_callback is None and not wanted_cells: + return deepcopy(self._sheet_cache[sheet["part"]]), {} + metadata = {"layout": [], "header_footer": {}, "conditional_format_rules": 0} + regions = _Regions() + wanted = set(wanted_cells or []) + saved_cells = {reference: _blank_cell(reference) for reference in wanted} + scanned_cells = 0 + for row, cells, attributes in self.iter_sheet_rows(sheet, metadata): + scanned_cells += len(cells) + regions.add(row, cells) + for cell in cells.values(): + if cell["cell"] in wanted: + saved_cells[cell["cell"]] = cell + if row_callback: + row_callback(row, cells, attributes) + if any(key in attributes for key in ("ht", "hidden", "s")) and len(metadata["layout"]) < 200: + metadata["layout"].append({"type": "row", **attributes}) + named = self._named_tables(sheet, metadata) + inferred = [ + table for table in regions.finish(sheet) + if not any( + table["min_row"] <= declared["max_row"] and table["max_row"] >= declared["min_row"] + and table["min_column"] <= declared["max_column"] and table["max_column"] >= declared["min_column"] + for declared in named + ) + ] + tables = sorted(named + inferred, key=lambda item: (item["min_row"], item["min_column"], item["id"])) + if len(tables) > EXCEL_MAX_TABLES: + raise ValueError("Workbook table inventory exceeds its supported count.") + result = {**sheet, **metadata, "tables": tables, "scanned_cells": scanned_cells} + self._sheet_cache[sheet["part"]] = result + return deepcopy(result), saved_cells + + def build_manifest(self): + cache_key = None + if self._manifest is None and isinstance(self.source, (bytes, bytearray)): + cache_key = (hashlib.sha256(self.source).hexdigest(), self.extension, EXCEL_WORKBOOK_SCHEMA_VERSION) + with _MANIFEST_LOCK: + cached = _MANIFEST_CACHE.get(cache_key) + if cached is not None: + _MANIFEST_CACHE.move_to_end(cache_key) + self._manifest = deepcopy(cached[0]) + if self._manifest is None: + sheets = [self.scan_sheet(sheet)[0] for sheet in self.sheets] + self._manifest = { + "schema_version": EXCEL_WORKBOOK_SCHEMA_VERSION, + "format": self.extension.lstrip("."), "sheet_count": len(sheets), + "table_count": sum(len(sheet["tables"]) for sheet in sheets), + "coverage": "complete" if not self.coverage_notes else "partial", + "coverage_notes": list(self.coverage_notes), + "sheets": sheets, + } + if cache_key is not None: + self._manifest["source_sha256"] = cache_key[0] + size = len(json.dumps(self._manifest, ensure_ascii=True).encode("utf-8")) + if size <= _MANIFEST_CACHE_MAX_BYTES: + with _MANIFEST_LOCK: + while _MANIFEST_CACHE and ( + len(_MANIFEST_CACHE) >= 16 + or sum(item[1] for item in _MANIFEST_CACHE.values()) + size > _MANIFEST_CACHE_MAX_BYTES + ): + _MANIFEST_CACHE.popitem(last=False) + _MANIFEST_CACHE[cache_key] = deepcopy(self._manifest), size + return deepcopy(self._manifest) + + def use_manifest(self, manifest): + """Reuse only a structural scan bound to these exact source bytes.""" + if not isinstance(self.source, (bytes, bytearray)): + raise TypeError("Reusable workbook manifests require immutable source bytes.") + expected = hashlib.sha256(self.source).hexdigest() + if ( + not isinstance(manifest, dict) + or manifest.get("schema_version") != EXCEL_WORKBOOK_SCHEMA_VERSION + or manifest.get("source_sha256") != expected + or [(item.get("name"), item.get("part")) for item in manifest.get("sheets") or []] + != [(item["name"], item["part"]) for item in self.sheets] + ): + raise ValueError("The workbook manifest belongs to a different source.") + self._manifest = deepcopy(manifest) + + def resolve_table(self, table_id=None, sheet_name=None): + manifest = self.build_manifest() + sheets = [self.resolve_sheet(sheet_name)["name"]] if sheet_name else self.sheet_names + tables = [table for sheet in manifest["sheets"] if sheet["name"] in sheets for table in sheet["tables"]] + if table_id: + matches = [table for table in tables if table["id"] == table_id or table["name"] == table_id] + if len(matches) != 1: + raise ValueError("The requested table was not found or is ambiguous.") + return matches[0] + if len(tables) != 1: + raise ValueError("The worksheet has multiple tables or no data table; select a table.") + return tables[0] + + def select_table(self, table_id=None, sheet_name=None, cell_range=None, header_row=None): + """Resolve a declared/inferred table or an explicitly requested analytical range.""" + if header_row is not None: + if isinstance(header_row, bool) or not str(header_row).isascii() or not str(header_row).isdigit(): + raise ValueError("header_row must be an actual worksheet row number or zero.") + header_row = int(header_row) + if cell_range: + if not sheet_name: + if len(self.sheets) != 1: + raise ValueError("Select a worksheet for an explicit table range.") + sheet_name = self.sheets[0]["name"] + low_column, low_row, high_column, high_row = bounded_range(cell_range) + table = { + "id": "range-" + hashlib.sha256(f"{sheet_name}\0{cell_range}".encode()).hexdigest()[:20], + "name": f"Selected range {cell_range}", "sheet": self.resolve_sheet(sheet_name)["name"], + "origin": "range", "range": cell_range, "min_row": low_row, "max_row": high_row, + "min_column": low_column, "max_column": high_column, + "header_row": None, "header_row_count": 0, "header_status": "absent", + "title_rows": [], "totals_row_count": 0, "data_start_row": low_row, + "data_end_row": high_row, "row_count": high_row - low_row + 1, + "repeated_header_rows": [], "repeated_header_count": 0, "header_signature": None, + "columns": [ + {"id": get_column_letter(column), "name": get_column_letter(column), "column": column} + for column in range(low_column, high_column + 1) + ], + } + if header_row is None: + raise ValueError("Specify header_row for an explicit range; zero means no header.") + else: + table = deepcopy(self.resolve_table(table_id, sheet_name)) + if header_row is None: + return table + if header_row and not table["min_row"] <= header_row <= table["max_row"]: + raise ValueError("The selected header row is outside the table range.") + low_column, high_column = table["min_column"], table["max_column"] + labels = [get_column_letter(column) for column in range(low_column, high_column + 1)] + if header_row: + references = [f"{get_column_letter(column)}{header_row}" for column in range(low_column, high_column + 1)] + cells = self.read_cells(table["sheet"], references) + labels = [ + str(cells[reference]["value"]) if cells[reference]["value"] not in (None, "") else label + for reference, label in zip(references, labels) + ] + names = _unique_column_names(labels, low_column) + table.update({ + "header_row": header_row or None, "header_row_count": int(bool(header_row)), + "header_status": "explicit", "title_rows": [], + "data_start_row": header_row + 1 if header_row else table["min_row"], + "header_signature": labels if header_row else None, + "columns": [ + {"id": get_column_letter(column), "name": name, "column": column} + for column, name in zip(range(low_column, high_column + 1), names) + ], + }) + table["row_count"] = sum(1 for _ in self.iter_table_rows(table)) + return table + + def iter_table_rows(self, table): + sheet = self.resolve_sheet(table["sheet"]) + header = tuple(table["header_signature"]) if table.get("header_signature") is not None else None + low, high = table["min_column"], table["max_column"] + next_row = table["data_start_row"] + for row, cells, _ in self.iter_sheet_rows(sheet): + if row < table["data_start_row"]: + continue + if row > table["data_end_row"]: + break + if table["origin"] in ("named", "range"): + while next_row < row: + yield next_row, [_blank_cell(f"{get_column_letter(column)}{next_row}") for column in range(low, high + 1)] + next_row += 1 + values = [cells.get(column) or _blank_cell(f"{get_column_letter(column)}{row}") for column in range(low, high + 1)] + next_row = row + 1 + if table["origin"] == "inferred": + if not any(cell["value_status"] != "blank" for cell in values): + continue + if header is not None and tuple(cell["value"] for cell in values) == header: + continue + yield row, values + if table["origin"] in ("named", "range"): + while next_row <= table["data_end_row"]: + yield next_row, [_blank_cell(f"{get_column_letter(column)}{next_row}") for column in range(low, high + 1)] + next_row += 1 + + def read_table_page(self, table_id=None, sheet_name=None, start_row=0, max_rows=200, *, cell_range=None, header_row=None): + if type(start_row) is not int or start_row < 0: + raise ValueError("start_row must be a nonnegative integer.") + if type(max_rows) is not int or not 1 <= max_rows <= EXCEL_MAX_PREVIEW_ROWS: + raise ValueError("max_rows must be between 1 and 500.") + table = self.select_table(table_id, sheet_name, cell_range, header_row) + rows, row_numbers, text_truncated = [], [], False + size = 0 + for ordinal, (row, cells) in enumerate(self.iter_table_rows(table)): + if ordinal < start_row: + continue + if len(rows) >= max_rows: + break + values = [] + for cell in cells[:EXCEL_PREVIEW_MAX_COLUMNS]: + value = cell["value"] + if isinstance(value, str) and len(value) > 4000: + value = value[:4000] + " [preview shortened]" + text_truncated = True + values.append(value) + row_size = len(json.dumps(values, ensure_ascii=True, allow_nan=False).encode("utf-8")) + if rows and size + row_size > EXCEL_PREVIEW_MAX_BYTES: + break + if row_size > EXCEL_PREVIEW_MAX_BYTES: + raise ValueError("A worksheet row exceeds the supported preview byte budget; select a smaller range.") + size += row_size + rows.append(values) + row_numbers.append(row) + has_more = start_row + len(rows) < table["row_count"] + return { + "selected_sheet": table["sheet"], "sheet_names": self.sheet_names, + "sheet_count": len(self.sheets), "table_id": table["id"], "table_name": table["name"], + "table_range": table["range"], "header_row": table["header_row"], + "header_status": table["header_status"], "columns": [column["name"] for column in table["columns"][:EXCEL_PREVIEW_MAX_COLUMNS]], + "rows": rows, "row_numbers": row_numbers, "total_rows": table["row_count"], + "total_columns": len(table["columns"]), "start_row": start_row, "has_more": has_more, + "truncated": has_more, "next_start_row": start_row + len(rows) if has_more else None, + "columns_truncated": len(table["columns"]) > EXCEL_PREVIEW_MAX_COLUMNS, + "cell_text_truncated": text_truncated, + } + + def read_table_previews(self, tables, max_rows=3): + """Gather small previews of many tables in one sequential pass per sheet.""" + if type(max_rows) is not int or not 1 <= max_rows <= 20: + raise ValueError("The schema preview row limit is invalid.") + results = {table["id"]: [] for table in tables} + for sheet_name in dict.fromkeys(table["sheet"] for table in tables): + selected = [table for table in tables if table["sheet"] == sheet_name] + remaining = {table["id"] for table in selected if table["row_count"] > 0} + for row, cells, _ in self.iter_sheet_rows(self.resolve_sheet(sheet_name)): + for table in selected: + records = results[table["id"]] + if table["id"] not in remaining or not table["data_start_row"] <= row <= table["data_end_row"]: + continue + if table["origin"] in ("named", "range"): + next_row = table["data_start_row"] + len(records) + while next_row < row and len(records) < min(max_rows, table["row_count"]): + records.append({"row": next_row, "values": [None] * len(table["columns"])}) + next_row += 1 + values = [ + (cells.get(column) or {}).get("value") + for column in range(table["min_column"], table["max_column"] + 1) + ] + if table["origin"] == "inferred": + if all(value in (None, "") for value in values): + continue + if table.get("header_signature") is not None and values == table["header_signature"]: + continue + if len(records) < min(max_rows, table["row_count"]): + records.append({"row": row, "values": values}) + if len(records) >= min(max_rows, table["row_count"]): + remaining.discard(table["id"]) + if not remaining: + break + for table in selected: + if table["origin"] not in ("named", "range"): + continue + records = results[table["id"]] + while len(records) < min(max_rows, table["row_count"]): + records.append({ + "row": table["data_start_row"] + len(records), + "values": [None] * len(table["columns"]), + }) + return results + + def read_cells(self, sheet_name, references): + wanted = {str(reference).replace("$", "").upper() for reference in references} + if len(wanted) > 2000: + raise ValueError("A cell inspection request exceeds its supported page size.") + positions = {reference: cell_coordinates(reference)[0] for reference in wanted} + result = {reference: _blank_cell(reference) for reference in wanted} + if not wanted: + return result + for row, cells, _ in self.iter_sheet_rows(self.resolve_sheet(sheet_name)): + if row > max(positions.values()): + break + for cell in cells.values(): + if cell["cell"] in wanted: + result[cell["cell"]] = cell + return result + + +class _XmlEvents: + def __init__(self, reader, part, limit): + self.reader = reader + self.part = part + self.limit = limit + self.stream = None + + def __enter__(self): + info = self.reader.archive.getinfo(self.part) + if info.compress_type not in OFFICE_ZIP_ALLOWED_COMPRESSION or info.file_size > self.limit: + raise ValueError("Workbook XML exceeds its supported byte/compression policy.") + self.stream = self.reader.archive.open(self.part) + return iterparse( + _BoundedXmlStream(self.stream, self.reader, self.limit), + events=("start", "end"), forbid_dtd=True, forbid_entities=True, forbid_external=True, + ) + + def __exit__(self, exc_type, exc_value, traceback): + if self.stream is not None: + self.stream.close() + + +def build_excel_table_preview( + source_bytes, extension, *, sheet_name=None, sheet_index=None, table_id=None, + start_row=0, max_rows=200, cell_range=None, header_row=None, +): + """Choose a visible preview explicitly; analytical reads never default among tables.""" + with ExcelWorkbookReader(source_bytes, extension=extension) as reader: + manifest = reader.build_manifest() + names = reader.sheet_names + if not names: + raise ValueError("The workbook has no readable worksheets.") + if sheet_name: + selected = reader.resolve_sheet(sheet_name)["name"] + elif sheet_index is not None: + if isinstance(sheet_index, bool) or not str(sheet_index).isdigit() or int(sheet_index) >= len(names): + raise ValueError("The worksheet index is invalid.") + selected = names[int(sheet_index)] + else: + selected = names[0] + sheet = next(item for item in manifest["sheets"] if item["name"] == selected) + tables = sheet["tables"] + options = [ + {key: table[key] for key in ("id", "name", "range", "origin", "header_row", "header_status", "row_count")} + for table in tables + ] + if not tables and not cell_range: + if table_id: + raise ValueError("The requested table was not found.") + return { + "selected_sheet": selected, "sheet_names": names, "sheet_count": len(names), + "tables": [], "table_count": 0, "rows": [], "columns": [], "row_numbers": [], + "total_rows": 0, "total_columns": 0, "start_row": 0, "has_more": False, + "truncated": False, "next_start_row": None, + } + selected_table = table_id or (tables[0]["id"] if tables and not cell_range else None) + page = reader.read_table_page( + table_id=selected_table, sheet_name=selected, start_row=start_row, max_rows=max_rows, + cell_range=cell_range, header_row=header_row, + ) + return {**page, "tables": options, "table_count": len(tables), "coverage": manifest["coverage"]} diff --git a/application/single_app/functions_accessibility.py b/application/single_app/functions_accessibility.py index 8b5389205..5653fb2a9 100644 --- a/application/single_app/functions_accessibility.py +++ b/application/single_app/functions_accessibility.py @@ -30,7 +30,11 @@ from pypdf import PdfReader from functions_appinsights import log_event -from functions_excel_accessibility import extract_excel_accessibility_metadata +from functions_excel_accessibility import ( + extract_excel_accessibility_metadata, + format_excel_comment_for_chat, + normalize_excel_comments, +) from functions_document_style import ( build_legacy_doc_style_inventory, build_pptx_style_inventory, @@ -2191,11 +2195,11 @@ def _legacy_doc_style(): return cosmos_meta, formatting_events -def extract_office_accessibility_metadata(file_path, file_ext=None): +def extract_office_accessibility_metadata(file_path, file_ext=None, *, record_sink=None): """Dispatch native accessibility extraction for supported Word, PowerPoint and Excel files.""" ext = (file_ext or os.path.splitext(str(file_path or ''))[1]).lower().strip() if ext in ('.xlsx', '.xlsm', '.xls'): - return extract_excel_accessibility_metadata(file_path, ext) + return extract_excel_accessibility_metadata(file_path, ext, record_sink=record_sink) try: if ext in ('.docx', '.docm', '.dotx'): return _extract_word_ooxml_accessibility(file_path, ext) @@ -2266,10 +2270,26 @@ def format_accessibility_metadata_for_chat(accessibility_metadata): if fmt in ('XLSX', 'XLSM', 'XLS'): lines.append(f"- Saved workbook properties: {accessibility_metadata.get('properties') or {}}") + comments, normalized_counts = normalize_excel_comments(accessibility_metadata.get('comments') or []) + counts = accessibility_metadata.get('comment_summary') or normalized_counts lines.append( f"- Sheets: {accessibility_metadata.get('sheet_count', 'unknown')}; " f"observed native records: {accessibility_metadata.get('observed_counts') or {}}" ) + lines.append(f"- Canonical comment counts (compatibility copies are not extra notes): {counts}") + if comments: + lines.append("- Reviewer feedback is distinct from the workbook's saved data and must not overwrite it.") + lines.extend(format_excel_comment_for_chat(record) for record in comments) + inventory = accessibility_metadata.get('inventory_coverage') or {} + if inventory.get('comments', {}).get('complete') is False: + lines.append("- The parent comment inventory is a preview; retrieve the paged native inventory for all records/full text.") + for table in accessibility_metadata.get('tables') or []: + lines.append( + f"- Table {table.get('name')} on {table.get('sheet')}: " + f"range={table.get('range')}; header_row={table.get('header_row')}; " + f"header_status={table.get('header_status')}; data_rows={table.get('row_count')}; " + f"table_id={table.get('id')}" + ) for sheet in (accessibility_metadata.get('sheets') or [])[:10]: lines.append( f"- Sheet {sheet.get('name')}: state={sheet.get('state')}; " @@ -2278,7 +2298,7 @@ def format_accessibility_metadata_for_chat(accessibility_metadata): lines.append(f"- Inspection coverage: {accessibility_metadata.get('coverage', 'unknown')}") lines.extend(f" * {note}" for note in accessibility_metadata.get('coverage_notes', [])) lines.extend(format_style_inventory_for_chat(accessibility_metadata.get('style_inventory'))) - lines.append("- Location-specific comments, objects and formatting are indexed as Excel native inspection evidence.") + lines.append("- Further location-specific comments, objects and formatting are available as Excel native inspection evidence.") lines.append("- Native inspection is not an accessibility conformance result; unavailable evidence is not proof of absence.") return '\n'.join(lines) diff --git a/application/single_app/functions_agent_document_citations.py b/application/single_app/functions_agent_document_citations.py index d00cae378..514494419 100644 --- a/application/single_app/functions_agent_document_citations.py +++ b/application/single_app/functions_agent_document_citations.py @@ -270,6 +270,7 @@ def _build_citation_from_document_payload( document_id=document_id, citation_id=citation_id, page_number=_resolve_location_number(citation_chunk.get('page_number'), chunk_sequence), + chunk_text=citation_chunk.get('chunk_text'), chunk_id=citation_chunk.get('chunk_id'), chunk_sequence=chunk_sequence, group_id=document_item.get('group_id'), diff --git a/application/single_app/functions_document_citations.py b/application/single_app/functions_document_citations.py index 8217d457f..7f82d12e8 100644 --- a/application/single_app/functions_document_citations.py +++ b/application/single_app/functions_document_citations.py @@ -36,6 +36,8 @@ def resolve_citation_location( if sheet_name not in (None, ''): return 'Sheet', str(sheet_name) normalized_chunk_text = str(chunk_text or '').strip() + if is_tabular and normalized_chunk_text.startswith('### Excel native inspection'): + return 'Location', f"Native evidence segment {page_number}" if page_number not in (None, '') else 'Workbook native inspection' if is_tabular and ( normalized_chunk_text.startswith('Tabular workbook:') or normalized_chunk_text.startswith('Tabular data file:') diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index b9632f594..9852d6e75 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -39,6 +39,7 @@ current_extraction, is_publication, ) +from content_screening.storage import ScreeningStorage from content_screening.service import ( document_requires_screening, initial_document_marker, @@ -121,6 +122,12 @@ format_pptx_comment_block, ) from functions_excel_accessibility import build_excel_evidence_chunks +from excel_workbook_reader import ExcelWorkbookReader +from functions_excel_inventory import ( + EXCEL_INVENTORY_FIELD, + clear_excel_inventory_cache, + persist_excel_inventory, +) from functions_document_transform_sources import ( OFFICE_REFERENCE_MIME_TYPES, TEMPLATE_EXTENSIONS, @@ -682,6 +689,7 @@ def _build_office_embedded_image_chunks( file_path, settings, update_callback, + native_media_metadata=None, ): """Analyze images embedded in an Office file and return placeable content blocks. @@ -716,11 +724,13 @@ def _build_office_embedded_image_chunks( try: temp_image_dir = tempfile.mkdtemp(prefix='office_images_') + extra_arguments = {"native_media_metadata": native_media_metadata} if native_media_metadata is not None else {} embedded_images, image_diagnostics = extract_office_embedded_images_with_diagnostics( file_path, temp_image_dir, min_pixels=min_pixels, max_images=max_images, + **extra_arguments, ) candidate_count = image_diagnostics.get('candidates', 0) @@ -818,6 +828,10 @@ def _build_office_embedded_image_chunks( slide_number = embedded_image.get('slide_number') if slide_number: location_label = f" on slide {slide_number}" + elif embedded_image.get('sheet_name'): + location_label = f" on worksheet {embedded_image['sheet_name']}" + if embedded_image.get('cell'): + location_label += f" at {embedded_image['cell']}" heading = ( f"### Embedded image {image_index} of {total_images}: " @@ -829,6 +843,11 @@ def _build_office_embedded_image_chunks( 'word_offset': embedded_image.get('word_offset'), 'position_known': bool(embedded_image.get('position_known')), 'name': embedded_image.get('name'), + **({ + 'sheet_name': embedded_image.get('sheet_name'), 'cell': embedded_image.get('cell'), + 'object_id': embedded_image.get('object_id'), 'media_part': embedded_image.get('media_part'), + 'placements': embedded_image.get('placements') or [], + } if native_media_metadata is not None else {}), }) analyzed_count += 1 @@ -5954,7 +5973,7 @@ def get_document_record(user_id, document_id, group_id=None, public_workspace_id return _normalize_document_enhanced_citations(document_item) -def get_ordered_document_chunks(document_id, user_id, group_id=None, public_workspace_id=None, max_chunks=None): +def get_ordered_document_chunks(document_id, user_id, group_id=None, public_workspace_id=None, max_chunks=None, *, native_only=False): """Return ordered chunk records for a document after access has been verified.""" document_item = get_document_record( user_id=user_id, @@ -5991,18 +6010,24 @@ def get_ordered_document_chunks(document_id, user_id, group_id=None, public_work 'title', 'chunk_summary', ] + escaped_document_id = str(document_id).replace("'", "''") search_kwargs = { 'search_text': '*', - 'filter': f"document_id eq '{document_id}'", + 'filter': f"document_id eq '{escaped_document_id}'", 'select': ','.join(select_fields), } if max_chunks is not None: search_kwargs['top'] = max(1, int(max_chunks)) + if native_only: + search_kwargs.update(search_text='"Excel native inspection"', search_fields=['chunk_text']) try: results = list(search_client.search(**search_kwargs)) - except Exception as e: - print(f"Error retrieving chunks for document {document_id}: {e}") + except Exception as error: + log_event( + "[DOCUMENT_METADATA] Document evidence retrieval failed.", + level=logging.WARNING, extra={"document_id": document_id, "error_type": type(error).__name__}, + ) raise ordered_chunks = [] @@ -6037,7 +6062,7 @@ def get_ordered_document_chunks(document_id, user_id, group_id=None, public_work ) return assert_document_chunks_available( ordered_chunks, document_item, user_id, group_id, public_workspace_id, - expected_chunk_count=document_item.get("num_chunks") if max_chunks is None else None, + expected_chunk_count=document_item.get("num_chunks") if max_chunks is None and not native_only else None, ) def get_documents(user_id, group_id=None, public_workspace_id=None): @@ -6373,8 +6398,12 @@ def delete_document( raise PermissionError("Document access denied.") if strict and (not expected_etag or document_item.get("_etag") != expected_etag): raise ScreeningConflictError() - if SCREENING_FIELD in document_item: + if ( + SCREENING_FIELD in document_item or EXCEL_INVENTORY_FIELD in document_item + or str(document_item.get("file_name") or "").lower().endswith((".xlsx", ".xlsm", ".xls")) + ): prepare_document_deletion(document_item, user_id) + clear_excel_inventory_cache(subject_from_document(document_item)) document_item = cosmos_container.read_item(item=document_id, partition_key=document_id) # Log document deletion transaction before deletion @@ -10523,40 +10552,39 @@ def _build_tabular_schema_summary(temp_file_path, original_filename, file_ext): ) if file_ext in ('.xlsx', '.xls', '.xlsm'): - engine = 'openpyxl' if file_ext in ('.xlsx', '.xlsm') else 'xlrd' - excel_file = pandas.ExcelFile(temp_file_path, engine=engine) - visible_sheet_names = excel_file.sheet_names[:TABULAR_SCHEMA_SUMMARY_MAX_SHEETS] - omitted_sheet_count = max(len(excel_file.sheet_names) - TABULAR_SCHEMA_SUMMARY_MAX_SHEETS, 0) - workbook_sections = [] - - for sheet_name in visible_sheet_names: - df_preview = excel_file.parse( - sheet_name, - keep_default_na=False, - dtype=str, - nrows=TABULAR_SCHEMA_SUMMARY_MAX_PREVIEW_ROWS, - ) - compact_columns = _compact_tabular_columns(df_preview.columns.tolist()) - preview_rows = _build_compact_tabular_preview(df_preview) - workbook_sections.append( - f"Sheet: {_compact_tabular_schema_value(sheet_name, max_chars=80)}\n" - f"Columns ({len(df_preview.columns)}): {', '.join(compact_columns) if compact_columns else 'None'}\n" - f"Preview (first {min(len(df_preview), TABULAR_SCHEMA_SUMMARY_MAX_PREVIEW_ROWS)} rows):\n{preview_rows}" - ) - - sheet_summary = ", ".join( - _compact_tabular_schema_value(sheet_name, max_chars=80) - for sheet_name in visible_sheet_names - ) - if omitted_sheet_count: - sheet_summary += f", ... +{omitted_sheet_count} more sheets" - - return ( - f"Tabular workbook: {original_filename}\n" - f"Sheets ({len(excel_file.sheet_names)}): {sheet_summary if sheet_summary else 'None'}\n\n" - + "\n\n".join(workbook_sections) - + f"\n\n{plugin_note}" - ) + with ExcelWorkbookReader(temp_file_path, extension=file_ext) as reader: + manifest = reader.build_manifest() + sheets = manifest["sheets"][:TABULAR_SCHEMA_SUMMARY_MAX_SHEETS] + tables = [table for sheet in sheets for table in sheet["tables"]][:20] + previews = reader.read_table_previews(tables, TABULAR_SCHEMA_SUMMARY_MAX_PREVIEW_ROWS) + workbook_sections = [] + for table in tables: + columns = [column["name"] for column in table["columns"]] + df_preview = pandas.DataFrame( + [record["values"] for record in previews[table["id"]]], columns=columns, + ) + compact_columns = _compact_tabular_columns(columns) + preview_rows = _build_compact_tabular_preview(df_preview) + workbook_sections.append( + f"Sheet: {_compact_tabular_schema_value(table['sheet'], max_chars=80)}\n" + f"Table: {_compact_tabular_schema_value(table['name'], max_chars=100)}; " + f"table_id={table['id']}; range={table['range']}; " + f"header_row={table['header_row']}; header_status={table['header_status']}\n" + f"Columns ({len(columns)}): {', '.join(compact_columns) if compact_columns else 'None'}\n" + f"Preview (first {len(df_preview)} data rows; not the complete dataset):\n{preview_rows}" + ) + sheet_summary = ", ".join( + _compact_tabular_schema_value(sheet["name"], max_chars=80) for sheet in sheets + ) + omitted = manifest["table_count"] - len(tables) + return ( + f"Tabular workbook: {original_filename}\n" + f"Sheets ({manifest['sheet_count']}): {sheet_summary or 'None'}\n" + f"Tables ({manifest['table_count']}): independent table/range headers; never assume one header per sheet.\n\n" + + "\n\n".join(workbook_sections) + + (f"\n\nAdditional tables not shown in this bounded schema preview: {omitted}." if omitted else "") + + f"\n\n{plugin_note}" + ) raise ValueError(f"Unsupported tabular file type: {file_ext}") @@ -10643,6 +10671,75 @@ def process_single_tabular_sheet(df, document_id, user_id, file_name, update_cal return total_chunks_saved, total_embedding_tokens, embedding_model_name +def _enrich_excel_image_inventory(file_path, settings, update_callback, metadata, evidence, record_sink): + if not settings.get("enable_office_embedded_image_analysis", True): + metadata["image_analysis_summary"] = {"status": "disabled", "analyzed": 0} + return + blocks, count, engine, _ = _build_office_embedded_image_chunks( + file_path, settings, update_callback, native_media_metadata=metadata.get("image_locations") or {}, + ) + metadata["image_analysis_summary"] = { + "status": "generated" if count else "none_analyzed", "analyzed": count, "engine": engine, + "limitation": "Missing analysis is not proof of absent images; configured size/format/count limits still apply.", + } + remaining = max(0, 1000000 - sum(len(item["content"]) for item in evidence)) + for block in blocks: + record = { + "type": "image_analysis", "sheet": block.get("sheet_name") or "(workbook)", + "cell": block.get("cell"), "object_id": block.get("object_id"), + "media_part": block.get("media_part"), "placements": block.get("placements") or [], + "text": block["content"], "analysis_engine": engine, + "source_kind": "configured_image_analysis_not_saved_alt_text", + } + record_sink("image_analyses", record) + content = f"### Excel native inspection\n{block['content']}" + if len(content) <= remaining: + evidence.append({ + "content": content, "sheet_name": record["sheet"], "kind": "image_analyses", + "location": record.get("cell") or record.get("object_id") or "", + }) + remaining -= len(content) + else: + metadata["coverage_notes"].append("Image-analysis search previews reached their budget; complete generated records remain in the native inventory.") + + +def _extract_excel_native_inventory(document_id, user_id, temp_file_path, file_ext, group_id=None, public_workspace_id=None, update_callback=None): + """Keep full native records in revision-private storage, not the parent preview.""" + capture = current_extraction(document_id) + settings = get_settings() + + def enrich(metadata, evidence, sink): + _enrich_excel_image_inventory(temp_file_path, settings, update_callback, metadata, evidence, sink) + + if capture is not None: + if capture.artifact_storage is None: + raise ScreeningError(code="screening_configuration_required") + return persist_excel_inventory( + temp_file_path, file_ext, subject=capture.subject, + storage=capture.artifact_storage, capture=capture, + enrich=enrich, + ) + document = get_document_record( + user_id, document_id, group_id=group_id, public_workspace_id=public_workspace_id, + ) + if document is None: + raise LookupError("Document not found or access denied.") + client = CLIENTS.get("storage_account_office_docs_client") + if client is None: + metadata, evidence = extract_office_accessibility_metadata(temp_file_path, file_ext) + metadata["inventory_storage"] = {"status": "unavailable", "reason": "native_storage_not_configured"} + log_event( + "[DOCUMENT_METADATA] Full Excel inventory storage is not configured.", + level=logging.WARNING, extra={"document_id": document_id}, + ) + return metadata, evidence, None + return persist_excel_inventory( + temp_file_path, file_ext, subject=subject_from_document(document), + storage=ScreeningStorage(client=client, container_name=storage_account_content_screening_container_name), + enrich=enrich, + ) + + def process_tabular(document_id, user_id, temp_file_path, original_filename, file_ext, enable_enhanced_citations, update_callback, group_id=None, public_workspace_id=None, auto_extract_metadata=True): """Processes CSV, XLSX, or XLS files using pandas.""" is_group = group_id is not None @@ -10655,8 +10752,14 @@ def process_tabular(document_id, user_id, temp_file_path, original_filename, fil native_chunks = [] if file_ext in ('.xlsx', '.xlsm', '.xls'): - native_metadata, native_evidence = extract_office_accessibility_metadata(temp_file_path, file_ext) + native_metadata, native_evidence, inventory_reference = _extract_excel_native_inventory( + document_id, user_id, temp_file_path, file_ext, + group_id=group_id, public_workspace_id=public_workspace_id, + update_callback=update_callback, + ) native_fields = {'accessibility_metadata': native_metadata} + if inventory_reference is not None: + native_fields[EXCEL_INVENTORY_FIELD] = inventory_reference for field in ('title', 'authors', 'keywords', 'language'): if native_metadata.get(field): native_fields[field] = native_metadata[field] diff --git a/application/single_app/functions_excel_accessibility.py b/application/single_app/functions_excel_accessibility.py index cb7f9a6f4..05b55b5a3 100644 --- a/application/single_app/functions_excel_accessibility.py +++ b/application/single_app/functions_excel_accessibility.py @@ -19,8 +19,13 @@ from openpyxl.styles.numbers import BUILTIN_FORMATS from openpyxl.utils.cell import get_column_letter +from excel_workbook_reader import ( + EXCEL_WORKBOOK_SCHEMA_VERSION, + ExcelWorkbookReader, + cell_coordinates, +) from functions_appinsights import log_event -from functions_document_style import StyleAccumulator +from functions_document_style import StyleAccumulator, format_style_inventory_for_chat from functions_office_media import OFFICE_ZIP_MAX_ENTRIES, _read_zip_entry_bounded @@ -64,12 +69,20 @@ def _on(value): class _Inspection: - def __init__(self, extension): + def __init__(self, extension, record_sink=None, progress_callback=None): self.meta = { - "format": extension.lstrip("."), "schema_version": "1.0", + "format": extension.lstrip("."), "schema_version": EXCEL_WORKBOOK_SCHEMA_VERSION, "coverage": "partial", "coverage_notes": list(EXCEL_COVERAGE_NOTES), "properties": {}, "sheets": [], "comments": [], "objects": [], "issues": [], + "tables": [], "defined_names": [], "hyperlinks": [], "validations": [], + "property_records": [], "custom_properties": [], "extended_properties": [], + "comment_summary": { + "legacy_notes": 0, "thread_roots": 0, "replies": 0, + "thread_messages": 0, "compatibility_copies": 0, "raw_records": 0, + }, } + self.record_sink = record_sink + self.progress_callback = progress_callback self.evidence = [] self.evidence_chars = 0 self.bytes_read = 0 @@ -77,6 +90,8 @@ def __init__(self, extension): self.cells = 0 self.cache = {} self.parent_record_bytes = 0 + self.image_placement_count = 0 + self.image_map_bytes = 0 self.styles = StyleAccumulator("sheet_cell") self.counts = Counter() @@ -88,7 +103,7 @@ def note(self, message): inventory["coverage_notes"].append(message) def clean(self, value): - text = str(value or "") + text = str(value) if value is not None else "" if len(text) > EXCEL_MAX_TEXT: self.note("Long native strings were truncated to the per-field limit.") return text[:EXCEL_MAX_TEXT] + " [truncated]" @@ -148,18 +163,62 @@ def emit(self, sheet, kind, value): def record(self, collection, sheet, record): self.counts[collection] += 1 + record = {"sheet": sheet, **record} + if collection == "objects" and record.get("media_part"): + if self.image_placement_count < 100: + images = self.meta.setdefault("image_locations", {}) + existing = images.get(record["media_part"]) + image = dict(existing) if existing else { + "alt_text": str(record.get("alt_text") or "")[:1000], + "title_text": str(record.get("title") or "")[:1000], + "is_decorative": record.get("is_decorative"), "sheet_name": sheet, + "cell": record.get("cell"), "object_id": record.get("id"), "placements": [], + } + image["placements"] = list(image["placements"]) + [{ + "sheet_name": sheet, "cell": record.get("cell"), "object_id": record.get("id"), + }] + size = len(json.dumps(image, ensure_ascii=True).encode()) + previous = len(json.dumps(existing, ensure_ascii=True).encode()) if existing else 0 + if self.image_map_bytes + size - previous <= 65536: + images[record["media_part"]] = image + self.image_map_bytes += size - previous + self.image_placement_count += 1 + else: + self.note("The embedded-image placement preview reached its byte limit; full native object records are separate.") + else: + self.note("The embedded-image analysis placement map is limited to 100 anchors; the full object inventory is separate.") + if self.record_sink is not None: + self.record_sink(collection, record) + self.emit(sheet, collection, record) if self.records >= EXCEL_MAX_RECORDS: - self.note("Native comment/object records were truncated at the record limit.") + self.note("Parent native records were truncated at the record limit; full inventory requires its paged source.") return self.records += 1 - record = {"sheet": sheet, **record} - record_bytes = len(json.dumps(record, ensure_ascii=True).encode("utf-8")) + preview = self.preview(record) + record_bytes = len(json.dumps(preview, ensure_ascii=True).encode("utf-8")) if self.parent_record_bytes + record_bytes <= EXCEL_MAX_PARENT_BYTES // 2: - self.meta[collection].append(record) + self.meta.setdefault(collection, []).append(preview) self.parent_record_bytes += record_bytes else: - self.note("Parent comment/object inventory was capped; location-specific search evidence is separate.") - self.emit(sheet, collection, record) + self.note("Parent native inventory was capped; location-specific search evidence and full inventory are separate.") + + def preview(self, value): + if isinstance(value, dict): + result = {key: self.preview(item) for key, item in value.items()} + for key, item in value.items(): + if ( + isinstance(item, str) and len(item) > EXCEL_MAX_TEXT + or isinstance(item, list) and len(item) > 100 + ): + result[f"{key}_truncated"] = True + return result + if isinstance(value, list): + if len(value) > 100: + self.note("Parent native collection previews show at most 100 entries; use the full inventory for the remainder.") + return [self.preview(item) for item in value[:100]] + if isinstance(value, str): + return self.clean(value) + return value def finish(self): self.meta["observed_counts"] = dict(self.counts) @@ -201,9 +260,15 @@ def finish(self): "coverage": "partial; unsupported control types are not accessibility failures", } # Search evidence keeps the location-specific records; Cosmos remains a bounded inventory. - while len(json.dumps(self.meta, ensure_ascii=True).encode("utf-8")) > EXCEL_MAX_PARENT_BYTES - 1024: + reserve = min(8192, EXCEL_MAX_PARENT_BYTES // 6) + while len(json.dumps(self.meta, ensure_ascii=True).encode("utf-8")) > EXCEL_MAX_PARENT_BYTES - reserve: style = self.meta["style_inventory"] - candidates = [self.meta[key] for key in ("comments", "objects", "sheets") if self.meta[key]] + candidates = [ + self.meta[key] for key in ( + "comments", "objects", "sheets", "tables", "defined_names", "hyperlinks", + "validations", "property_records", "custom_properties", "extended_properties", + ) if self.meta[key] + ] candidates.extend(style[key] for key in ("outliers", "colors") if style[key]) candidates.extend(value["sheets"] for value in (style["layout"], style["page_setup"]) if value["sheets"]) if not candidates: @@ -222,12 +287,24 @@ def finish(self): style["outliers_truncated"] = len(style["outliers"]) < style["outlier_run_count"] self.note("Parent inventory was truncated at its serialized byte limit; search evidence is separate.") + self.meta["inventory_coverage"] = { + key: { + "observed": count, "retained": len(self.meta.get(key) or []), + "complete": count == len(self.meta.get(key) or []) and not any( + any(name.endswith("_truncated") and value is True for name, value in item.items()) + for item in self.meta.get(key) or [] + ), + } + for key, count in self.counts.items() + } + def coverage_content(): return f"{EXCEL_EVIDENCE_MARKER}\nWorkbook properties and inspection coverage:\n" + json.dumps({ "properties": self.meta["properties"], "coverage": self.meta["coverage"], "coverage_notes": self.meta["coverage_notes"], "observed_counts": self.meta["observed_counts"], + "comment_summary": self.meta["comment_summary"], }, ensure_ascii=True) while self.evidence and self.evidence_chars + len(coverage_content()) > EXCEL_MAX_EVIDENCE_CHARS: @@ -242,13 +319,41 @@ def _core_properties(state, root): if root is None: return names = {"title", "creator", "subject", "keywords", "description", "language", - "category", "created", "modified", "lastModifiedBy"} + "category", "created", "modified", "lastModifiedBy", "identifier", + "version", "revision", "lastPrinted", "contentStatus"} properties = {_local(item.tag): state.clean(item.text) for item in root if _local(item.tag) in names} state.meta["properties"] = properties for key in ("title", "subject", "description", "language", "category"): state.meta[key] = properties.get(key, "") state.meta["authors"] = [part.strip() for part in re.split(r"[;,]", properties.get("creator", "")) if part.strip()] state.meta["keywords"] = [part.strip() for part in re.split(r"[;,]", properties.get("keywords", "")) if part.strip()] + for element in root: + if _local(element.tag) in names: + state.record("property_records", "(workbook)", { + "kind": "core", "name": _local(element.tag), "value": element.text or "", + }) + + +def _additional_properties(state, archive, relationships): + for kind, fallback, collection in ( + ("extended-properties", "docProps/app.xml", "extended_properties"), + ("custom-properties", "docProps/custom.xml", "custom_properties"), + ): + part = next((path for relation, path in relationships.values() if relation == kind), fallback) + root = state.xml(archive, part) + if root is None: + continue + for element in root: + name = element.get("name") or _local(element.tag) + children = list(element) + values = [ + {"type": _local(child.tag), "value": _text(child)} + for child in children + ] + state.record(collection, "(workbook)", { + "name": name, "value": values if children else element.text or "", + "property_id": element.get("pid"), + }) def _color(element, theme, state, palette=COLOR_INDEX): @@ -372,7 +477,240 @@ def _header_sections(expression): return {key: value for key, value in sections.items() if value} +def normalize_excel_comments(records): + """Correlate only identifiable same-cell compatibility copies of real threads.""" + threaded, notes = [], [] + for item in records: + if not isinstance(item, dict): + raise ValueError("A stored Excel comment record is invalid.") + record = dict(item) + (threaded if record.get("kind") == "threaded_comment" else notes).append(record) + roots = { + (record.get("sheet"), str(record["id"]).casefold()): record + for record in threaded if record.get("id") and not record.get("parent_id") + } + summary = { + "legacy_notes": 0, "thread_roots": sum(not record.get("parent_id") for record in threaded), + "replies": sum(bool(record.get("parent_id")) for record in threaded), + "thread_messages": len(threaded), "compatibility_copies": 0, + "raw_records": len(threaded) + len(notes), + } + genuine_notes = [] + for note in notes: + author = str(note.get("author") or "") + root = roots.get((note.get("sheet"), author[3:].strip().casefold())) if author.lower().startswith("tc=") else None + text = str(note.get("text") or "") + compatibility = ( + root is not None and root.get("cell") == note.get("cell") + and text.lstrip().casefold().startswith("[threaded comment]") + and bool(str(root.get("text") or "").strip()) + and str(root["text"]).strip() in text + ) + if compatibility: + summary["compatibility_copies"] += 1 + else: + genuine_notes.append(note) + summary["legacy_notes"] = len(genuine_notes) + return threaded + genuine_notes, summary + + +def format_excel_comment_for_chat(record): + """Keep feedback, saved values, and unavailable fields separate in model evidence.""" + if record.get("record_fragment"): + return ( + f"### Partial native record at {record.get('sheet')}!{record.get('cell')}\n" + "This is an explicitly continued JSON record, not a complete comment. " + "Use the supplied continuation cursor to read the remainder.\n" + f"{record.get('record_json_fragment') or ''}" + ) + location = f"{record.get('sheet', '?')}!{record.get('cell', '?')}" + kind = "thread reply" if record.get("parent_id") else "thread root" if record.get("kind") == "threaded_comment" else "legacy note" + saved = { + "value": record.get("cell_value"), "value_type": record.get("cell_value_type"), + "status": record.get("cell_value_status", "not inspected"), + "formula": record.get("formula"), "cached_value_available": record.get("cached_value_available"), + } + details = { + key: record.get(key) for key in ("author", "date", "id", "parent_id", "resolved") + } + text = ( + f"### Reviewer feedback: {kind} at {location}\n" + f"Saved cell evidence (not the requested change): {json.dumps(saved, ensure_ascii=False)}\n" + f"Saved comment fields (null means unknown): {json.dumps(details, ensure_ascii=False)}\n" + f"Reviewer feedback text:\n{record.get('text') or ''}" + ) + if record.get("text_truncated"): + text += "\n[Parent text preview is shortened; retrieve the full native inventory text.]" + return text + + +def excel_native_question_sections(question): + text = str(question or "").lower() + sections = [] + for section, expression in ( + ("comments", r"\b(comments?|notes?|reviewer|feedback|replies|threaded)\b"), + ("custom_properties", r"\bcustom propert"), + ("extended_properties", r"\b(extended propert|application propert|company|manager)\b"), + ("property_records", r"\b(propert|metadata|authors?|keywords?|document title)"), + ("tables", r"\b(tables?|columns?|schema|named ranges?)\b"), + ("objects", r"\b(objects?|charts?|images?|pictures?|alt[- ]?text|decorative|shapes?)\b"), + ("image_analyses", r"\b(images?|pictures?|figures?)\b"), + ("hyperlinks", r"\b(hyperlinks?|links?)\b"), + ("validations", r"\bvalidations?\b"), + ("defined_names", r"\b(defined names?|named ranges?|print areas?|print titles?)\b"), + ): + if re.search(expression, text): + sections.append(section) + return sections or ["comments"] + + +def build_excel_native_chat_evidence(report, question, *, max_bytes=15000): + """Include whole requested records before optional layout/style summaries.""" + metadata = report.get("accessibility_metadata") or {} + inspection = report.get("excel_inspection") or {} + page = inspection.get("page") or {} + pages = inspection.get("pages") or ([page] if page else []) + comments, counts = normalize_excel_comments(metadata.get("comments") or []) + counts = metadata.get("comment_summary") or counts + header = ( + f"{EXCEL_EVIDENCE_MARKER}\n" + f"Workbook: {(report.get('document') or {}).get('file_name') or 'Selected workbook'}\n" + f"Canonical comment counts: {json.dumps(counts, ensure_ascii=True)}\n" + f"Native coverage: {metadata.get('coverage', 'unknown')}. " + "Saved native state is not rendered appearance or a conformance certificate. " + "Reviewer feedback is not the workbook's saved data. " + "No formulas, macros, or external links were executed.\n" + ) + parts, used, included, omitted = [header], len(header.encode("utf-8")), 0, 0 + + def append(value, record=False): + nonlocal used, included, omitted + size = len(value.encode("utf-8")) + 2 + if used + size > max_bytes - 1024: + if record: + omitted += 1 + return False + parts.append(value) + used += size + included += int(record) + return True + + requested = excel_native_question_sections(question) + for section in requested: + section_page = next((item for item in pages if item.get("section") == section), {}) + records = section_page.get("records") if section_page else metadata.get(section) + if section == "comments": + records = section_page.get("records") if section_page else comments + for record in records or []: + content = format_excel_comment_for_chat(record) if section == "comments" else ( + f"### Saved native {section}\n{json.dumps(record, ensure_ascii=False)}" + ) + append(content, record=True) + for cell in inspection.get("cells") or []: + append( + f"### Saved cell {cell.get('sheet')}!{cell.get('cell')}\n{json.dumps(cell, ensure_ascii=True)}", + record=True, + ) + append(f"Saved workbook properties: {json.dumps(metadata.get('properties') or {}, ensure_ascii=True)}") + if "image_analyses" in requested: + append( + "Image-analysis coverage (not saved alt text): " + + json.dumps(metadata.get("image_analysis_summary") or { + "status": "not_generated", "guidance": "Configured image analysis may require document reprocessing; saved native object metadata is separate.", + }, ensure_ascii=True) + ) + for sheet in metadata.get("sheets") or []: + append( + f"Sheet {sheet.get('name')}: state={sheet.get('state')}; " + f"headers/footers={json.dumps(sheet.get('header_footer') or {}, ensure_ascii=True)}" + ) + for note in metadata.get("coverage_notes") or []: + append(f"Coverage limitation: {note}") + if re.search(r"\b(font|format|style|size|colou?r)", str(question or ""), re.IGNORECASE): + for line in format_style_inventory_for_chat(report.get("style_inventory") or metadata.get("style_inventory")): + append(line) + incomplete = bool(any(item.get("has_more") for item in pages) or omitted) + if incomplete: + parts.append( + f"Returned {included} complete records/fragments in this bounded handoff; " + f"{omitted} additional selected records did not fit. Do not call this an exhaustive list. " + f"Native page continuations: {json.dumps({item['section']: item.get('next_cursor') for item in pages if item.get('has_more')}, ensure_ascii=True)}." + ) + return { + "summary": "\n\n".join(parts), "returned_records": included, + "records_omitted": omitted, "partial": incomplete, + "next_cursor": page.get("next_cursor"), "total_records": page.get("total_records"), + "worksheet_data_read": bool(inspection.get("cells")) or any("cell_value_status" in record for record in comments), + } + + +def has_excel_native_inspection_intent(question, file_names=None): + """Recognize native requirements independently from a simultaneous calculation.""" + text = re.sub( + r"\b(sum|average|median|aggregate|calculate|compute)\b|\btotal\s+(sales|revenue|tax|amount|cost)", + "", str(question or ""), flags=re.IGNORECASE, + ) + return is_excel_native_inspection_question(text, file_names=file_names) + + +def merge_excel_native_evidence(result, native_result, question): + """Keep both evidence kinds within the existing per-source summary budget.""" + if not native_result: + return result + result = dict(result) + native_by_source = { + envelope["document_id"]: envelope for envelope in native_result.get("evidence_envelopes") or [] + } + reports = { + citation.get("function_arguments", {}).get("document_id"): citation.get("function_result") + for citation in native_result.get("agent_citations") or [] + if citation.get("function_name") == "get_document_accessibility_report" + } + envelopes = [] + for existing in result.get("evidence_envelopes") or []: + envelope = dict(existing) + native = native_by_source.get(envelope.get("document_id")) + if native is None: + envelopes.append(envelope) + continue + summary = str(envelope.get("summary") or "") + computed_lines, computed_size = [], 0 + for line in summary.splitlines(keepends=True): + size = len(line.encode("utf-8")) + if computed_size + size > 10000: + break + computed_lines.append(line) + computed_size += size + computed = "".join(computed_lines) + computed_shortened = computed != summary + report = reports.get(envelope["document_id"]) + if isinstance(report, dict): + evidence = build_excel_native_chat_evidence( + report, question, max_bytes=max(2500, 15000 - computed_size), + ) + native_summary = evidence["summary"] + native_partial = evidence["partial"] + else: + native_summary = str(native.get("summary") or "Native inspection was unavailable.") + native_partial = True + envelope["summary"] = ( + f"{native_summary}\n\nComputed tabular evidence:\n{computed}" + + ("\n[Computed summary is shortened; the full tool result remains available.]" if computed_shortened else "") + ) + envelope["coverage"] = { + **dict(envelope.get("coverage") or {}), + "native_inspection_included": True, "native_records_partial": native_partial, + "native_status": native.get("status"), "computed_summary_shortened": computed_shortened, + } + envelope["citations"] = list(envelope.get("citations") or []) + list(native.get("citations") or []) + envelopes.append(envelope) + result["evidence_envelopes"] = envelopes + result["agent_citations"] = list(result.get("agent_citations") or []) + list(native_result.get("agent_citations") or []) + return result + + def _comments(state, archive, sheet, relationships, persons): + threaded, notes = [], [] for kind, path in relationships.values(): if kind not in ("comments", "threadedComment", "threadedComments"): continue @@ -388,14 +726,49 @@ def _comments(state, archive, sheet, relationships, persons): else: author = persons.get(item.get("personId")) text = "".join(_text(child) for child in item if _local(child.tag) == "text") - state.record("comments", sheet, { + record = { + "sheet": sheet, "kind": "note" if name == "comment" else "threaded_comment", - "cell": state.clean(item.get("ref")), "text": state.clean(text), + "cell": state.clean(item.get("ref")), "text": text, "author": author, "date": state.clean(item.get("dT")) or None, "id": state.clean(item.get("id")) or None, "parent_id": state.clean(item.get("parentId")) or None, "resolved": _on(item.get("done")) if "done" in item.attrib else None, - }) + "source_part": path, + } + (notes if name == "comment" else threaded).append(record) + records, summary = normalize_excel_comments(threaded + notes) + for key, count in summary.items(): + state.meta["comment_summary"][key] += count + return records + + +def _chart_content(state, archive, chart_part): + root = state.xml(archive, chart_part, required=True) + series = [] + for element in root.iter(): + if _local(element.tag) != "ser": + continue + values = [] + references = [] + for item in element.iter(): + if _local(item.tag) == "f": + references.append(item.text or "") + elif _local(item.tag) == "pt": + value = _first(item, "v") + values.append({"index": item.get("idx"), "value": _text(value)}) + series.append({ + "references": references, "cached_points": values, + "cached_point_count": len(values), + }) + return { + "chart_part": chart_part, + "chart_text": "\n".join( + item.text for item in root.iter() if _local(item.tag) == "t" and item.text + ), + "series": series, + "coverage": "Saved chart text, references and caches only; chart formulas are not evaluated.", + } def _objects(state, archive, sheet, relationships): @@ -425,12 +798,16 @@ def _objects(state, archive, sheet, relationships): state.record("objects", sheet, { "type": control_type or "vml_shape", "id": state.clean(shape.get("id")), "cell": f"{get_column_letter(int(column.text) + 1)}{int(row.text) + 1}" if row is not None and column is not None else None, - "alt_text": state.clean(shape.get("alt")), "title": state.clean(_attr(shape, "title")), + "alt_text": shape.get("alt") or "", "title": _attr(shape, "title") or "", + "saved_control_properties": { + _local(item.tag): _text(item) for item in client + } if client is not None else {}, "is_decorative": None, "alt_inspected": True, }) state.note("VML decorative status is unavailable; absent marking remains unknown.") continue parents = {child: parent for parent in root.iter() for child in parent} + drawing_relationships = state.rels(archive, path) for anchor in root: start = _first(anchor, "from") row = _first(start, "row") if start is not None else None @@ -447,103 +824,110 @@ def _objects(state, archive, sheet, relationships): "chart" if owner is not None and owner_type == "graphicFrame" and _first(owner, "chart") is not None else "group" if owner_type == "grpSp" else "shape" ) - state.record("objects", sheet, { + record = { "type": object_type, "id": state.clean(properties.get("id")), "name": state.clean(properties.get("name")), "cell": cell, - "alt_text": state.clean(properties.get("descr")), "title": state.clean(properties.get("title")), + "alt_text": properties.get("descr") or "", "title": properties.get("title") or "", "is_decorative": _on(decorative.get("val")) if decorative is not None else None, "alt_inspected": True, - }) + "text": "\n".join( + item.text for item in owner.iter() + if _local(item.tag) == "t" and item.text + ) if owner is not None else "", + "drawing_part": path, + } + chart = _first(owner, "chart") if owner is not None else None + if chart is not None: + relationship = drawing_relationships.get(_attr(chart, "id")) + if relationship and relationship[0] == "chart": + record.update(_chart_content(state, archive, relationship[1])) + else: + record["chart_coverage"] = "The saved chart part was unavailable." + state.note("A drawing referenced a chart without an available internal chart relationship.") + image = _first(owner, "blip") if owner is not None else None + if image is not None: + relationship = drawing_relationships.get(_attr(image, "embed")) + if relationship and relationship[0] == "image": + record["media_part"] = relationship[1] + state.record("objects", sheet, record) -def _sheet(state, archive, entry, path, kind, definitions, shared_strings, persons): +def _sheet(state, archive, entry, path, kind, definitions, reader, persons): name = state.clean(entry.get("name"))[:128] - root = state.xml(archive, path, required=True) - sheet = {"name": name, "state": entry.get("state", "visible"), "type": kind, "part": path} - state.meta["sheets"].append(sheet) - layout_names = {"mergeCell", "col", "pane", "pageMargins", "pageSetup", "printOptions", - "sheetFormatPr", "sheetView", "autoFilter", "sheetPr"} - layout = [] - for element in root.iter(): - local = _local(element.tag) - if local in layout_names or local == "row" and any(key in element.attrib for key in ("ht", "hidden", "s")): - if len(layout) < 200: - record = {"type": local, **{key: state.clean(value) for key, value in element.attrib.items()}} - style_id = element.get("style") if local == "col" else element.get("s") if local == "row" else None - if style_id is not None and 0 <= int(style_id) < len(definitions): - record["saved_style"] = definitions[int(style_id)] - layout.append(record) + relationships = state.rels(archive, path) + comments = _comments(state, archive, name, relationships, persons) + saved_styles = {} + + def inspect_styles(row, cells, attributes): + if state.progress_callback is not None and row % 1024 == 0: + state.progress_callback() + for cell in cells.values(): + if state.cells >= EXCEL_MAX_CELLS: + state.note(f"Cell formatting inspection sampled the first {EXCEL_MAX_CELLS} saved cells.") + return + state.cells += 1 + style_index = cell["style_id"] + spec = definitions[style_index] if 0 <= style_index < len(definitions) else {} + if not spec: + state.note("Some cell style references could not be resolved.") + reference = cell["cell"] + state.styles.add( + f"{name}!{reference}", state.clean(cell["value"])[:60], + **{key: spec.get(key) for key in ("font", "size_pt", "color", "bold", "italic", "underline")}, + ) + if style_index not in saved_styles: + if len(saved_styles) >= 100: + state.note("Per-sheet formatting evidence was capped at 100 distinct cell styles.") + continue + saved_styles[style_index] = {"style_id": style_index, "cells": [], "observed_cell_count": 0, **spec} + group = saved_styles[style_index] + group["observed_cell_count"] += 1 + if len(group["cells"]) < 30: + group["cells"].append(reference) else: - state.note("Sheet layout ranges were truncated to 200 records per sheet.") - sheet["layout"] = layout - sheet["conditional_format_rules"] = sum(_local(element.tag) == "cfRule" for element in root.iter()) - header_footer = _first(root, "headerFooter") - sheet["header_footer"] = {} - if header_footer is not None: - sheet["header_footer"] = { - "flags": dict(header_footer.attrib), - "expressions": {_local(child.tag): state.clean(child.text) for child in header_footer}, - "sections": {_local(child.tag): _header_sections(state.clean(child.text)) for child in header_footer}, + state.note("Style location lists show the first 30 cells per saved style per sheet.") + + sheet, cells = reader.scan_sheet( + name, row_callback=inspect_styles, wanted_cells={record["cell"] for record in comments}, + ) + expressions = sheet.get("header_footer", {}).get("expressions") or {} + if expressions: + sheet["header_footer"]["sections"] = { + key: _header_sections(value) for key, value in expressions.items() } + for record in sheet["layout"]: + style_id = record.get("style") if record["type"] == "col" else record.get("s") + if style_id is not None and 0 <= int(style_id) < len(definitions): + record["saved_style"] = definitions[int(style_id)] + state.meta["sheets"].append(state.preview(sheet)) + state.meta["scanned_sheet_count"] = state.meta.get("scanned_sheet_count", 0) + 1 + state.meta["scanned_cells"] = state.meta.get("scanned_cells", 0) + sheet["scanned_cells"] state.emit(name, "Sheet layout, headers and footers (raw Excel control codes; not evaluated)", sheet) - relationships = state.rels(archive, path) - _comments(state, archive, name, relationships, persons) + for table in sheet["tables"]: + state.record("tables", name, table) + for key in ("hyperlinks", "validations"): + for record in sheet.get(key) or []: + state.record(key, name, record) + for record in comments: + cell = cells[record["cell"]] + row, column = cell_coordinates(record["cell"]) + record.update({ + "cell_value": cell["value"], "cell_value_raw": cell["raw_value"], + "cell_value_type": cell["value_type"], "cell_value_status": cell["value_status"], + "formula": cell["formula"], "formula_type": cell["formula_type"], + "formula_reference": cell["formula_reference"], + "cached_value_available": cell["cached_value_available"], + "number_format": cell.get("number_format"), + "table_ids": [ + table["id"] for table in sheet["tables"] + if table["min_row"] <= row <= table["max_row"] + and table["min_column"] <= column <= table["max_column"] + ], + }) + state.record("comments", name, record) _objects(state, archive, name, relationships) - if _first(root, "oleObjects") is not None or any(kind in ("oleObject", "activeXControl", "control") for kind, _ in relationships.values()): + if sheet.get("unsupported_controls") or any(kind in ("oleObject", "activeXControl", "control") for kind, _ in relationships.values()): state.note("This workbook contains unsupported OLE/ActiveX controls; their alt text was not inspected.") - if any(_local(element.tag) in ("cellControl", "featurePropertyBag") for element in root.iter()): - state.note("Modern in-cell controls were detected; their accessibility metadata is unsupported.") - column_styles = [(int(column.get("min")), int(column.get("max")), column.get("style")) - for column in root.iter() if _local(column.tag) == "col" and column.get("style") is not None] - row_styles = {row.get("r"): row.get("s") for row in root.iter() - if _local(row.tag) == "row" and row.get("s") is not None} - saved_styles = {} - for element in root.iter(): - if _local(element.tag) != "c": - continue - if state.cells >= EXCEL_MAX_CELLS: - state.note(f"Cell formatting inspection sampled the first {EXCEL_MAX_CELLS} saved cells.") - break - state.cells += 1 - ref = state.clean(element.get("r"))[:32] - column_match = re.match(r"([A-Z]+)(\d+)$", ref) - inherited_style = None - if column_match: - column_index = 0 - for letter in column_match[1]: - column_index = column_index * 26 + ord(letter) - ord("A") + 1 - inherited_style = next((style for low, high, style in column_styles if low <= column_index <= high), None) - inherited_style = row_styles.get(column_match[2], inherited_style) - style_index = int(element.get("s", inherited_style or 0)) - spec = definitions[style_index] if 0 <= style_index < len(definitions) else {} - if not spec: - state.note("Some cell style references could not be resolved.") - value = _first(element, "v") - raw = _text(value) - if element.get("t") == "s": - index = int(raw) if raw else -1 - raw = shared_strings[index] if 0 <= index < len(shared_strings) else "[unresolved shared string]" - elif element.get("t") == "inlineStr": - raw = _text(_first(element, "is")) - # Text is used only for the bounded style accumulator, not copied to search evidence. - snippet = state.clean(raw)[:60] - if raw == "[unresolved shared string]": - state.note("Some shared-string references were outside the inspected string range.") - location = f"{name}!{ref}" - state.styles.add(location, snippet, **{key: spec.get(key) for key in ( - "font", "size_pt", "color", "bold", "italic", "underline" - )}) - if style_index not in saved_styles: - if len(saved_styles) >= 100: - state.note("Per-sheet formatting evidence was capped at 100 distinct cell styles.") - continue - saved_styles[style_index] = {"style_id": style_index, "cells": [], "observed_cell_count": 0, **spec} - group = saved_styles[style_index] - group["observed_cell_count"] += 1 - if len(group["cells"]) < 30: - group["cells"].append(ref) - else: - state.note("Style location lists show the first 30 cells per saved style per sheet.") for group in saved_styles.values(): state.emit(name, "Saved cell formatting (not worksheet data)", group) @@ -559,18 +943,14 @@ def _extract_ooxml(state, file_path): workbook_path = next((path for kind, path in relationships.values() if kind == "officeDocument"), "xl/workbook.xml") core_path = next((path for kind, path in relationships.values() if kind == "core-properties"), "docProps/core.xml") _core_properties(state, state.xml(archive, core_path)) + _additional_properties(state, archive, relationships) workbook = state.xml(archive, workbook_path, required=True) workbook_rels = state.rels(archive, workbook_path) definitions = _style_definitions(state, archive, workbook_rels) state.emit("(workbook)", "Saved default cell style", definitions[0]) - shared_strings, persons = [], {} + persons = {} for kind, path in workbook_rels.values(): - if kind == "sharedStrings": - root = state.xml(archive, path, required=True) - shared_strings = [state.clean(_text(item)) for item in islice(root, EXCEL_MAX_CELLS)] - if len(root) > EXCEL_MAX_CELLS: - state.note("Shared strings were truncated at the cell inspection limit.") - elif kind in ("person", "persons"): + if kind in ("person", "persons"): root = state.xml(archive, path, required=True) persons = {element.get("id"): state.clean(element.get("displayName")) for element in root.iter() if _local(element.tag) == "person"} @@ -579,26 +959,35 @@ def _extract_ooxml(state, file_path): state.meta["sheet_count"] = len(entries) if len(entries) > EXCEL_MAX_SHEETS: state.note(f"Only the first {EXCEL_MAX_SHEETS} sheets were inspected.") - for entry in entries[:EXCEL_MAX_SHEETS]: - relationship = workbook_rels.get(_attr(entry, "id")) - if not relationship or relationship[0] not in ("worksheet", "chartsheet"): - state.note("A sheet relationship is unsupported or unavailable.") - continue - try: - _sheet(state, archive, entry, relationship[1], relationship[0], definitions, shared_strings, persons) - except (ValueError, KeyError, OSError, ParseError, DefusedXmlException, zipfile.BadZipFile) as exc: - state.note(f"A sheet could not be completely inspected ({type(exc).__name__}).") - log_event("[DOCUMENT_METADATA] Excel sheet inspection incomplete.", level=30, - extra={"error_type": type(exc).__name__}) + with ExcelWorkbookReader(archive, extension="." + state.meta["format"]) as reader: + for entry in entries[:EXCEL_MAX_SHEETS]: + relationship = workbook_rels.get(_attr(entry, "id")) + if not relationship or relationship[0] not in ("worksheet", "chartsheet"): + state.note("A sheet relationship is unsupported or unavailable.") + continue + try: + _sheet(state, archive, entry, relationship[1], relationship[0], definitions, reader, persons) + except (ValueError, KeyError, OSError, ParseError, DefusedXmlException, zipfile.BadZipFile) as exc: + state.note(f"A sheet could not be completely inspected ({type(exc).__name__}).") + state.meta.setdefault("incomplete_sheets", []).append(state.clean(entry.get("name"))) + log_event("[DOCUMENT_METADATA] Excel sheet inspection incomplete.", level=30, + extra={"error_type": type(exc).__name__}) + state.meta["xml_scan_bytes"] = reader.scanned_xml_bytes + for note in reader.coverage_notes: + state.note(note) defined_names = [ {"name": state.clean(item.get("name")), "sheet_index": item.get("localSheetId"), - "expression": state.clean(item.text)} + "expression": item.text or "", "hidden": item.get("hidden") in ("1", "true")} for item in workbook.iter() if _local(item.tag) == "definedName" - and item.get("name") in ("_xlnm.Print_Titles", "_xlnm.Print_Area") + ] + for record in defined_names: + state.record("defined_names", "(workbook)", record) + state.meta["print_defined_names"] = [ + record for record in defined_names if record["name"] in ("_xlnm.Print_Titles", "_xlnm.Print_Area") ][:EXCEL_MAX_SHEETS * 2] - state.meta["print_defined_names"] = defined_names - if defined_names: - state.emit("(workbook)", "Print titles and print areas (not header/footer text)", defined_names) + if state.meta["print_defined_names"]: + state.emit("(workbook)", "Print titles and print areas (not header/footer text)", state.meta["print_defined_names"]) + state.meta["table_count"] = state.counts["tables"] def _extract_xls(state, file_path): @@ -642,10 +1031,15 @@ def _extract_xls(state, file_path): state.meta["sheets"].append(info) state.emit(name, "Legacy sheet layout", info) for (row, column), note in sheet.cell_note_map.items(): + value = sheet.cell_value(row, column) if row < sheet.nrows and column < sheet.row_len(row) else None + state.meta["comment_summary"]["raw_records"] += 1 + state.meta["comment_summary"]["legacy_notes"] += 1 state.record("comments", name, { "kind": "note", "cell": f"{get_column_letter(column + 1)}{row + 1}", - "text": state.clean(note.text), "author": state.clean(note.author), + "text": note.text, "author": state.clean(note.author), "date": None, "resolved": None, + "cell_value": value, "cell_value_status": "blank" if value is None or value == "" else "saved", + "formula": None, "cached_value_available": None, }) for row in range(sheet.nrows): if state.cells >= EXCEL_MAX_CELLS: @@ -671,9 +1065,9 @@ def _extract_xls(state, file_path): workbook.release_resources() -def extract_excel_accessibility_metadata(file_path, extension): +def extract_excel_accessibility_metadata(file_path, extension, *, record_sink=None, progress_callback=None): """Return bounded parent metadata and location-specific native evidence.""" - state = _Inspection(extension) + state = _Inspection(extension, record_sink=record_sink, progress_callback=progress_callback) try: if extension == ".xls": _extract_xls(state, file_path) @@ -735,5 +1129,7 @@ def is_excel_native_inspection_question(question, file_names=None): r"hidden (?:rows?|columns?|sheets?)|freeze panes?|conditional formatting|" r"document (?:title|properties)|workbook properties|keywords|" r"(?:cell |threaded )?(?:comments|notes)|author(?:s|ship)?|" - r"checkbox(?:es)?|check boxes|fill colo[u]?r|number formats?)\b", text, - )) + r"checkbox(?:es)?|check boxes|fill colo[u]?r|number formats?|" + r"custom properties|metadata|named ranges?|defined names?|hyperlinks?|data validation|" + r"cell values?|cell formulas?|saved formulas?|cached values?)\b", text, + ) or re.search(r"\bcells?\s+[a-z]{1,3}[1-9]\d*|!\$?[a-z]{1,3}\$?[1-9]\d*", text)) diff --git a/application/single_app/functions_excel_inventory.py b/application/single_app/functions_excel_inventory.py new file mode 100644 index 000000000..77c5905e2 --- /dev/null +++ b/application/single_app/functions_excel_inventory.py @@ -0,0 +1,379 @@ +# functions_excel_inventory.py +"""Version-bound, paged Excel inventories in the existing private artifact store.""" + +import base64 +from collections import OrderedDict +from copy import deepcopy +import hashlib +import json +import logging +import os +import tempfile +from threading import RLock +from uuid import uuid4 + +from azure.core.exceptions import AzureError, ResourceNotFoundError + +from content_screening.contracts import ScreeningError, Subject, subject_from_document +from functions_appinsights import log_event +from functions_excel_accessibility import extract_excel_accessibility_metadata +from excel_workbook_reader import EXCEL_WORKBOOK_SCHEMA_VERSION + + +EXCEL_INVENTORY_FIELD = "excel_native_inventory" +EXCEL_INVENTORY_SECTIONS = frozenset({ + "comments", "objects", "tables", "defined_names", "hyperlinks", "validations", + "property_records", "custom_properties", "extended_properties", + "image_analyses", +}) +EXCEL_INVENTORY_PART_BYTES = 256 * 1024 +EXCEL_INVENTORY_PART_RECORDS = 100 +EXCEL_INVENTORY_PAGE_BYTES = 14000 +EXCEL_INVENTORY_CACHE_BYTES = 16 * 1024 * 1024 +_CACHE = OrderedDict() +_CACHE_LOCK = RLock() + + +class ExcelInventoryPersistenceError(ScreeningError): + code = "excel_inventory_unavailable" + public_message = "The workbook inspection could not be stored. Retry the operation." + + +def _encoded(value): + return json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + + +def _source_digest(path): + digest = hashlib.sha256() + with open(path, "rb") as source: + for data in iter(lambda: source.read(65536), b""): + digest.update(data) + return digest.hexdigest() + + +def _cache_key(subject, source_digest, storage, cache_namespace=None): + namespace = cache_namespace if cache_namespace is not None else id(storage) + return subject.key, source_digest, EXCEL_WORKBOOK_SCHEMA_VERSION, namespace + + +def _remember(key, manifest, reference): + size = len(_encoded(manifest)) + if size > EXCEL_INVENTORY_CACHE_BYTES: + return + with _CACHE_LOCK: + _CACHE.pop(key, None) + while _CACHE and ( + len(_CACHE) >= 16 + or sum(entry[2] for entry in _CACHE.values()) + size > EXCEL_INVENTORY_CACHE_BYTES + ): + _CACHE.popitem(last=False) + _CACHE[key] = (deepcopy(manifest), deepcopy(reference), size) + + +def clear_excel_inventory_cache(subject=None): + with _CACHE_LOCK: + for key in list(_CACHE): + if subject is None or key[0] == subject.key: + del _CACHE[key] + + +class ExcelInventoryWriter: + """Write bounded immutable pages, then atomically publish their manifest.""" + + def __init__(self, storage, subject, source_digest, capture=None): + if not isinstance(subject, Subject): + raise TypeError("An authorized workbook subject is required.") + self.storage = storage + self.subject = subject + self.source_digest = source_digest + self.capture = capture + self.scan_id = f"excel-native:{EXCEL_WORKBOOK_SCHEMA_VERSION}:{source_digest}:{uuid4().hex}" + self.pages = [] + self.buffers = {} + self.sizes = {} + self.counts = {} + + def _write(self, name, value): + try: + return self.storage.write_json(self.subject, self.scan_id, name, value) + except (AzureError, ScreeningError, OSError, RuntimeError, ValueError, TypeError) as error: + log_event( + "[DOCUMENT_METADATA] Excel inventory persistence failed.", + level=logging.ERROR, + extra={"document_id": self.subject.document_id, "error_type": type(error).__name__}, + ) + raise ExcelInventoryPersistenceError() from error + + def add(self, collection, record): + if collection not in EXCEL_INVENTORY_SECTIONS or not isinstance(record, dict): + raise ValueError("An Excel inventory record is invalid.") + payload = _encoded(record) + if self.capture is not None: + self.capture.add_supplement(payload.decode("utf-8"), kind="excel_native") + records = self.buffers.setdefault(collection, []) + if records and ( + self.sizes.get(collection, 0) + len(payload) > EXCEL_INVENTORY_PART_BYTES + or len(records) >= EXCEL_INVENTORY_PART_RECORDS + ): + self._flush(collection) + records = self.buffers[collection] + records.append(record) + self.sizes[collection] = self.sizes.get(collection, 0) + len(payload) + self.counts[collection] = self.counts.get(collection, 0) + 1 + + def _flush(self, collection): + records = self.buffers.get(collection) or [] + if not records: + return + first = self.counts[collection] - len(records) + reference = self._write(f"excel-{collection}-{first}", records) + self.pages.append({ + "section": collection, "first": first, "count": len(records), "reference": reference, + }) + self.buffers[collection] = [] + self.sizes[collection] = 0 + + def finish(self, metadata): + metadata.pop("image_locations", None) + for collection in list(self.buffers): + self._flush(collection) + complete = ( + metadata.get("coverage") != "failed" + and not metadata.get("incomplete_sheets") + and metadata.get("sheet_count") == metadata.get("scanned_sheet_count", len(metadata.get("sheets") or [])) + ) + metadata["inventory_storage"] = { + "schema_version": EXCEL_WORKBOOK_SCHEMA_VERSION, "source_sha256": self.source_digest, + "status": "ready" if complete else "partial", + "counts": dict(self.counts), "page_count": len(self.pages), + } + manifest = { + "schema_version": EXCEL_WORKBOOK_SCHEMA_VERSION, "subject": self.subject.to_dict(), + "source_sha256": self.source_digest, "metadata": metadata, + "complete": complete, "counts": dict(self.counts), "pages": self.pages, + } + manifest_reference = self._write("excel-manifest", manifest) + reference = { + "schema_version": EXCEL_WORKBOOK_SCHEMA_VERSION, + "source_sha256": self.source_digest, "manifest": manifest_reference, + } + _remember(_cache_key(self.subject, self.source_digest, self.storage), manifest, reference) + return reference + + +def persist_excel_inventory(file_path, extension, *, subject, storage, capture=None, enrich=None): + digest = _source_digest(file_path) + writer = ExcelInventoryWriter(storage, subject, digest, capture=capture) + metadata, evidence = extract_excel_accessibility_metadata( + file_path, extension, record_sink=writer.add, + progress_callback=capture.heartbeat if capture is not None else None, + ) + if enrich is not None: + enrich(metadata, evidence, writer.add) + reference = writer.finish(metadata) + return metadata, evidence, reference + + +def _validate_manifest(manifest, subject, digest): + if ( + not isinstance(manifest, dict) + or manifest.get("schema_version") != EXCEL_WORKBOOK_SCHEMA_VERSION + or manifest.get("subject") != subject.to_dict() + or manifest.get("source_sha256") != digest + or not isinstance(manifest.get("metadata"), dict) + or not isinstance(manifest.get("counts"), dict) + or not isinstance(manifest.get("pages"), list) + or type(manifest.get("complete")) is not bool + ): + raise ValueError("The Excel inventory does not match the authorized source.") + counts = {} + for page in manifest["pages"]: + section = page.get("section") + if ( + section not in EXCEL_INVENTORY_SECTIONS + or type(page.get("first")) is not int or page["first"] < 0 + or type(page.get("count")) is not int or not 1 <= page["count"] <= EXCEL_INVENTORY_PART_RECORDS + or page["first"] != counts.get(section, 0) + or not isinstance(page.get("reference"), dict) + ): + raise ValueError("The Excel inventory page mapping is invalid.") + counts[section] = counts.get(section, 0) + page["count"] + if counts != manifest["counts"]: + raise ValueError("The Excel inventory counts are invalid.") + + +def open_excel_inventory(document, source_bytes, extension, *, storage, authorize, cache_namespace=None): + """The caller supplies freshly authorized, active-representation bytes.""" + if not callable(authorize) or not isinstance(source_bytes, bytes): + raise TypeError("Authorized source bytes and a source check are required.") + authorize() + subject = subject_from_document(document) + digest = hashlib.sha256(source_bytes).hexdigest() + key = _cache_key(subject, digest, storage, cache_namespace) + with _CACHE_LOCK: + cached = _CACHE.get(key) + if cached is not None: + _CACHE.move_to_end(key) + manifest, reference = deepcopy(cached[:2]) + else: + manifest = reference = None + if manifest is None: + stored = document.get(EXCEL_INVENTORY_FIELD) + if ( + isinstance(stored, dict) + and stored.get("schema_version") == EXCEL_WORKBOOK_SCHEMA_VERSION + and stored.get("source_sha256") == digest + and isinstance(stored.get("manifest"), dict) + and stored["manifest"].get("subject_key") == subject.key + ): + try: + manifest = storage.read_json(stored["manifest"], subject) + reference = stored + except ResourceNotFoundError: + log_event( + "[DOCUMENT_METADATA] Rebuilding an unavailable derived Excel inventory.", + level=logging.INFO, extra={"document_id": subject.document_id}, + ) + if manifest is None: + with tempfile.TemporaryDirectory(prefix="simplechat_excel_inventory_") as folder: + path = os.path.join(folder, "source" + extension) + with open(path, "wb") as source: + source.write(source_bytes) + _, _, reference = persist_excel_inventory(path, extension, subject=subject, storage=storage) + manifest = storage.read_json(reference["manifest"], subject) + _validate_manifest(manifest, subject, digest) + _remember(key, manifest, reference) + _validate_manifest(manifest, subject, digest) + authorize() + return ExcelInventoryReader(storage, subject, manifest, reference, authorize) + + +def open_stored_excel_inventory(document, *, storage, authorize): + """Read an immutable revision inventory without downloading its workbook again.""" + authorize() + reference = document.get(EXCEL_INVENTORY_FIELD) + if not isinstance(reference, dict) or reference.get("schema_version") != EXCEL_WORKBOOK_SCHEMA_VERSION: + return None + subject = subject_from_document(document) + digest = reference.get("source_sha256") + stored = reference.get("manifest") + if ( + not isinstance(digest, str) or len(digest) != 64 + or not isinstance(stored, dict) or stored.get("subject_key") != subject.key + ): + return None + active = (document.get("content_screening") or {}).get("active_blob") or {} + if active.get("content_hash") and active["content_hash"] != digest: + return None + manifest = storage.read_json(stored, subject) + _validate_manifest(manifest, subject, digest) + authorize() + return ExcelInventoryReader(storage, subject, manifest, reference, authorize) + + +class ExcelInventoryReader: + def __init__(self, storage, subject, manifest, reference, authorize): + self.storage = storage + self.subject = subject + self.manifest = manifest + self.reference = reference + self.authorize = authorize + + @property + def metadata(self): + self.authorize() + return deepcopy(self.manifest["metadata"]) + + def _cursor(self, section, offset, fragment=0): + return base64.urlsafe_b64encode(_encoded({ + "subject": self.subject.key, "source": self.manifest["source_sha256"], + "section": section, "offset": offset, "fragment": fragment, + })).decode("ascii").rstrip("=") + + def _parse_cursor(self, token, section): + if not isinstance(token, str) or not token or len(token) > 1024: + raise ValueError("The Excel inspection cursor is invalid.") + try: + value = json.loads(base64.b64decode(token + "=" * (-len(token) % 4), altchars=b"-_", validate=True)) + except (ValueError, UnicodeError) as error: + raise ValueError("The Excel inspection cursor is invalid.") from error + if ( + not isinstance(value, dict) or set(value) != {"subject", "source", "section", "offset", "fragment"} + or value["subject"] != self.subject.key or value["source"] != self.manifest["source_sha256"] + or value["section"] != section + or any(type(value[key]) is not int or value[key] < 0 for key in ("offset", "fragment")) + ): + raise ValueError("The Excel inspection cursor belongs to a different source or section.") + return value["offset"], value["fragment"] + + def page(self, section="comments", *, cursor=None, page_size=100, max_bytes=EXCEL_INVENTORY_PAGE_BYTES): + if section not in EXCEL_INVENTORY_SECTIONS: + raise ValueError("The requested Excel inventory section is invalid.") + if type(page_size) is not int or not 1 <= page_size <= 200: + raise ValueError("The Excel inventory page size must be between 1 and 200.") + if type(max_bytes) is not int or not 2048 <= max_bytes <= 65536: + raise ValueError("The Excel inventory response budget is invalid.") + self.authorize() + offset, fragment = self._parse_cursor(cursor, section) if cursor else (0, 0) + total = self.manifest["counts"].get(section, 0) + if offset > total or offset == total and fragment: + raise ValueError("The Excel inspection cursor is outside the inventory.") + records, used, next_offset, next_fragment = [], 0, offset, 0 + record_budget = max_bytes - 1024 + for page in self.manifest["pages"]: + if page["section"] != section or page["first"] + page["count"] <= offset: + continue + self.authorize() + values = self.storage.read_json(page["reference"], self.subject) + if not isinstance(values, list) or len(values) != page["count"] or any(not isinstance(value, dict) for value in values): + raise ValueError("The Excel inventory page failed its record check.") + self.authorize() + for index in range(max(offset - page["first"], 0), len(values)): + record = values[index] + encoded = _encoded(record) + if records and (len(records) >= page_size or used + len(encoded) > record_budget): + break + if not records and (len(encoded) > record_budget or fragment): + serialized = encoded.decode("utf-8") + if fragment >= len(serialized): + raise ValueError("The Excel record continuation is invalid.") + low, high = 1, min(len(serialized) - fragment, record_budget) + result = None + while low <= high: + middle = (low + high) // 2 + candidate = { + "sheet": record.get("sheet"), "cell": record.get("cell"), "id": record.get("id"), + "record_json_fragment": serialized[fragment:fragment + middle], + "fragment_offset": fragment, "record_characters": len(serialized), + "record_fragment": True, + } + if len(_encoded(candidate)) <= record_budget: + result = candidate + low = middle + 1 + else: + high = middle - 1 + if result is None: + raise ValueError("The Excel record cannot fit the requested response budget.") + piece = result["record_json_fragment"] + end = fragment + len(piece) + records.append(result) + if end < len(serialized): + next_offset, next_fragment = page["first"] + index, end + else: + next_offset = page["first"] + index + 1 + break + records.append(record) + used += len(encoded) + next_offset = page["first"] + index + 1 + if next_fragment or len(records) >= page_size or next_offset < page["first"] + page["count"]: + break + self.authorize() + has_more = next_offset < total or bool(next_fragment) + return { + "section": section, "records": records, "total_records": total, + "returned_records": len(records), "start_record": offset, + "complete": bool(self.manifest["complete"] and not has_more and offset == 0), + "inventory_complete": self.manifest["complete"], "has_more": has_more, + "next_cursor": self._cursor(section, next_offset, next_fragment) if has_more else None, + "source_sha256": self.manifest["source_sha256"], + } diff --git a/application/single_app/functions_native_tabular_compute.py b/application/single_app/functions_native_tabular_compute.py index 587469785..99d4b5b40 100644 --- a/application/single_app/functions_native_tabular_compute.py +++ b/application/single_app/functions_native_tabular_compute.py @@ -213,14 +213,51 @@ def _compute_request(plan, user_question): if operation == "transform" and task_type not in {"structured_export", "combined"}: raise NativeTabularComputeError("native_compute_operation_unsupported") query = validate_tabular_csv_query_expression(hints.get("query_expression") or "index == index") - return { + selected_table = hints.get("selected_table") if hints.get("selected_table") is not None else hints.get("table_id") + cell_range = hints.get("cell_range") if hints.get("cell_range") is not None else hints.get("selected_range") + header_row = hints.get("header_row") if hints.get("header_row") is not None else hints.get("selected_header_row") + selected_sheet = hints.get("selected_sheet") + if ( + (selected_sheet is not None and (not isinstance(selected_sheet, str) or len(selected_sheet) > 512)) + or any( + value is not None and (not isinstance(value, str) or not value.strip() or len(value) > 512) + for value in (selected_table, cell_range) + ) + or ( + header_row is not None + and (isinstance(header_row, bool) or not str(header_row).isascii() or not str(header_row).isdigit()) + ) + or (selected_table is not None and cell_range is not None) + or ( + hints.get("selected_table") is not None and hints.get("table_id") is not None + and hints["selected_table"] != hints["table_id"] + ) + or ( + hints.get("cell_range") is not None and hints.get("selected_range") is not None + and hints["cell_range"] != hints["selected_range"] + ) + or ( + hints.get("header_row") is not None and hints.get("selected_header_row") is not None + and str(hints["header_row"]) != str(hints["selected_header_row"]) + ) + ): + raise NativeTabularComputeError("native_compute_table_selection_invalid") + request = { "task_type": task_type, "operation": operation, "query_expression": query, - "selected_sheet": hints.get("selected_sheet"), + "selected_sheet": selected_sheet, "deliverable_contract": deepcopy(contract), "row_analysis_mode": plan.get("row_analysis_mode"), "row_analysis_questions": deepcopy(plan.get("row_analysis_questions") or []), "question_sha256": native_compute_digest(user_question), } + # Omitted selectors must not change fingerprints of existing CSV jobs. + request.update({ + key: value for key, value in { + "selected_table": selected_table, "cell_range": cell_range, + "header_row": int(header_row) if header_row is not None else None, + }.items() if value is not None + }) + return request def _prepare_native_source(engine, source, request, user_id): @@ -235,7 +272,12 @@ def _prepare_native_source(engine, source, request, user_id): plugin = engine.TabularProcessingPlugin(authorized_user_id=user_id) container, blob_path = locator["container"], locator["blob_path"] query = request["query_expression"] + selectors = plugin._table_selector_kwargs( + request.get("selected_table"), request.get("cell_range"), request.get("header_row"), + ) if source_format == "csv": + if selectors: + raise NativeTabularComputeError("native_compute_table_selection_invalid") result = plugin._query_csv_data_in_bounded_chunks( container, blob_path, filename, query, return_columns=None, start_row=0, max_rows=5, ) @@ -246,9 +288,10 @@ def _prepare_native_source(engine, source, request, user_id): else: selected_sheet = request["selected_sheet"] metadata = plugin._get_workbook_metadata(container, blob_path) - if selected_sheet: + if selected_sheet or selectors.get("table_id") not in (None, "all") or selectors.get("cell_range"): selected_sheet, _ = plugin._resolve_sheet_selection( container, blob_path, sheet_name=selected_sheet, require_explicit_sheet=True, + **selectors, ) sheets = [selected_sheet] else: @@ -259,6 +302,7 @@ def _prepare_native_source(engine, source, request, user_id): descriptor = plugin._build_generated_export_query_descriptor_from_location( container_name=container, blob_path=blob_path, filename=filename, query_expression=query, blob_version=version, selected_sheet=selected_sheet, sheet_names=sheets, + **selectors, ) count, samples = 0, [] for _, row in engine._iter_versioned_tabular_source_rows( @@ -279,6 +323,9 @@ def _prepare_native_source(engine, source, request, user_id): descriptor["document_id"] = source["document_id"] candidate = { "filename": filename, "selected_sheet": descriptor.get("selected_sheet"), + "selected_table": descriptor.get("selected_table"), + "cell_range": descriptor.get("cell_range"), "header_row": descriptor.get("header_row"), + "table_selections": deepcopy(descriptor.get("table_selections") or []), "rows": samples, "row_count": len(samples), "total_matches": count, "full_result_available": False, "source_authorization": descriptor, } diff --git a/application/single_app/functions_office_media.py b/application/single_app/functions_office_media.py index 25b820b4d..13713b355 100644 --- a/application/single_app/functions_office_media.py +++ b/application/single_app/functions_office_media.py @@ -548,7 +548,7 @@ def _inspect_containers(root_elem, rel_targets): return media_a11y_map -def extract_office_embedded_images_with_diagnostics(file_path, output_dir, min_pixels=150, max_images=25): +def extract_office_embedded_images_with_diagnostics(file_path, output_dir, min_pixels=150, max_images=25, native_media_metadata=None): """Extract embedded images and report what was skipped and why. Without the diagnostics a document containing only unsupported figures is indistinguishable @@ -586,6 +586,9 @@ def extract_office_embedded_images_with_diagnostics(file_path, output_dir, min_p media_slide_map = _build_pptx_media_slide_map(archive) media_word_offsets, total_body_words = build_docx_image_word_offsets(archive) media_a11y_map = _build_office_media_accessibility_map(archive) + if native_media_metadata is not None: + media_a11y_map.update(native_media_metadata) + native_seen_images = {} diagnostics['total_body_words'] = total_body_words candidate_names = [ @@ -626,6 +629,9 @@ def extract_office_embedded_images_with_diagnostics(file_path, output_dir, min_p digest = hashlib.sha256(image_bytes).hexdigest() if digest in seen_digests: + if native_media_metadata is not None and digest in native_seen_images: + placements = (media_a11y_map.get(media_name) or {}).get('placements') or [] + native_seen_images[digest]['placements'].extend(placements) _record_skip(diagnostics, 'duplicate_image') continue @@ -678,17 +684,24 @@ def extract_office_embedded_images_with_diagnostics(file_path, output_dir, min_p 'height': height, 'slide_number': media_slide_map.get(media_name), 'word_offset': media_word_offsets.get(media_name), - 'position_known': media_name in media_word_offsets or media_name in media_slide_map, + 'position_known': media_name in media_word_offsets or media_name in media_slide_map or bool(a11y_info.get('sheet_name')), 'source_format': source_extension.lstrip('.'), 'rasterized': is_vector, 'embedded_text': embedded_text, 'alt_text': a11y_info.get('alt_text', ''), 'title_text': a11y_info.get('title_text', ''), - 'is_decorative': bool(a11y_info.get('is_decorative', False)), + 'is_decorative': a11y_info.get('is_decorative') if a11y_info.get('sheet_name') else bool(a11y_info.get('is_decorative', False)), 'placement': a11y_info.get('placement', 'inline'), 'allow_overlap': bool(a11y_info.get('allow_overlap', False)), 'behind_doc': bool(a11y_info.get('behind_doc', False)), + **({ + 'sheet_name': a11y_info.get('sheet_name'), 'cell': a11y_info.get('cell'), + 'object_id': a11y_info.get('object_id'), 'media_part': media_name, + 'placements': list(a11y_info.get('placements') or []), + } if native_media_metadata is not None else {}), }) + if native_media_metadata is not None: + native_seen_images[digest] = extracted_images[-1] except (zipfile.BadZipFile, FileNotFoundError, OSError): return [], diagnostics @@ -1079,4 +1092,3 @@ def strip_appended_word_comments(text): if text.startswith(f"{WORD_COMMENTS_SECTION_HEADING}\n\n"): return '' return text - diff --git a/application/single_app/functions_search_service.py b/application/single_app/functions_search_service.py index 1ba12becb..4dc812229 100644 --- a/application/single_app/functions_search_service.py +++ b/application/single_app/functions_search_service.py @@ -11,6 +11,8 @@ from azure.identity import DefaultAzureCredential, get_bearer_token_provider from azure.cosmos.exceptions import CosmosResourceNotFoundError from openai import AzureOpenAI +from openpyxl.utils.cell import get_column_letter +from azure.core.exceptions import ResourceNotFoundError from content_screening import access as screening_access from content_screening.access import ( @@ -25,15 +27,23 @@ strict_source_authority_enabled, ) from content_screening.contracts import SourceAuthorityUnverifiedError +from content_screening.storage import ScreeningStorage from config import ( CLIENTS, cognitive_services_scope, cosmos_conversations_container, cosmos_messages_container, + storage_account_content_screening_container_name, ) +from excel_workbook_reader import ExcelWorkbookReader, bounded_range from functions_appinsights import log_event from functions_debug import debug_print from functions_documents import get_document_record, get_ordered_document_chunks +from functions_excel_inventory import ( + EXCEL_INVENTORY_SECTIONS, + open_excel_inventory, + open_stored_excel_inventory, +) from functions_group import get_user_groups from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers from model_endpoint_clients import build_openai_client_timeout_options @@ -1112,6 +1122,74 @@ def _normalize_accessibility_sections(sections): return requested or list(ACCESSIBILITY_REPORT_SECTIONS) +def _read_excel_inspection( + document_context, user_id, *, native_section="comments", native_cursor=None, + native_page_size=100, sheet_name=None, cell_range=None, + max_bytes=14000, +): + document = document_context["document"] + source_evidence = {PROVENANCE_FIELD: document_provenance(document)} + + def authorize(): + assert_evidence_available(source_evidence, user_id, cached=True) + + if native_section not in EXCEL_INVENTORY_SECTIONS: + raise ValueError("The Excel inspection section is invalid.") + if type(native_page_size) is not int or not 1 <= native_page_size <= 200: + raise ValueError("The Excel inspection page size must be between 1 and 200.") + storage = ScreeningStorage( + client=CLIENTS.get("storage_account_office_docs_client"), + container_name=storage_account_content_screening_container_name, + ) + try: + inventory = open_stored_excel_inventory(document, storage=storage, authorize=authorize) + except ResourceNotFoundError: + inventory = None + log_event( + "[DOCUMENT_METADATA] Rebuilding an unavailable derived Excel inventory.", + level=logging.INFO, extra={"document_id": document["id"]}, + ) + source_bytes = None + extension = "." + str(document.get("file_name") or "").rsplit(".", 1)[-1].lower() + if inventory is None or cell_range: + fresh_document, source_bytes = screening_access.read_available_document_bytes( + document, user_id=user_id, group_id=document_context.get("group_id"), + public_workspace_id=document_context.get("public_workspace_id"), purpose="accessibility", + ) + document_context["document"] = fresh_document + if inventory is None: + inventory = open_excel_inventory( + fresh_document, source_bytes, extension, storage=storage, authorize=authorize, + cache_namespace=( + storage_account_content_screening_container_name, + id(CLIENTS.get("storage_account_office_docs_client")), + ), + ) + metadata = inventory.metadata + page = inventory.page(native_section, cursor=native_cursor, page_size=native_page_size, max_bytes=max_bytes) + result = {"metadata": metadata, "page": page} + if cell_range: + low_column, low_row, high_column, high_row = bounded_range(cell_range) + if (high_column - low_column + 1) * (high_row - low_row + 1) > 2000: + raise ValueError("The Excel cell range exceeds its 2000-cell inspection limit.") + with ExcelWorkbookReader(source_bytes, extension=extension) as reader: + if not sheet_name: + if len(reader.sheet_names) != 1: + raise ValueError("Select a worksheet for cell inspection.") + sheet_name = reader.sheet_names[0] + references = [ + f"{get_column_letter(column)}{row}" + for row in range(low_row, high_row + 1) + for column in range(low_column, high_column + 1) + ] + cells = reader.read_cells(sheet_name, references) + result["cells"] = [ + {"sheet": sheet_name, **cells[reference]} for reference in references + ] + authorize() + return result + + def get_document_accessibility_payload( document_id, user_id, @@ -1121,6 +1199,12 @@ def get_document_accessibility_payload( conversation_id=None, sections=None, page_number=None, + native_section="comments", + native_cursor=None, + native_page_size=100, + sheet_name=None, + cell_range=None, + _native_max_bytes=14000, ): """Return the complete stored accessibility, style, and issue metadata for one document. @@ -1156,6 +1240,13 @@ def get_document_accessibility_payload( source_evidence = {PROVENANCE_FIELD: document_provenance(current_document)} stored_metadata = current_document.get("accessibility_metadata") + file_name = str(current_document.get("file_name") or "") + extension = "." + file_name.rsplit(".", 1)[-1].lower() if "." in file_name else "" + if not stored_metadata and extension in (".xlsx", ".xlsm", ".xls") and ( + current_document.get("source_file_available") or current_document.get("enhanced_citations") + or current_document.get("blob_path") or current_document.get("archived_blob_path") + ): + stored_metadata = {"format": extension.lstrip(".")} if not isinstance(stored_metadata, dict) or not stored_metadata: raise LookupError( "No accessibility metadata is stored for this document. It may have been uploaded before " @@ -1163,6 +1254,22 @@ def get_document_accessibility_payload( "re-upload the file to extract it." ) + excel_inspection = None + if stored_metadata.get("format") in ("xlsx", "xlsm", "xls") and ( + current_document.get("source_file_available") or current_document.get("enhanced_citations") + or current_document.get("blob_path") or current_document.get("archived_blob_path") + or current_document.get("excel_native_inventory") + ): + excel_inspection = _read_excel_inspection( + document_context, user_id, native_section=native_section, + native_cursor=native_cursor, native_page_size=native_page_size, + sheet_name=sheet_name, cell_range=cell_range, + max_bytes=_native_max_bytes, + ) + stored_metadata = excel_inspection["metadata"] + elif native_cursor or cell_range: + raise LookupError("The original workbook or full native inventory is unavailable for this inspection.") + requested_sections = _normalize_accessibility_sections(sections) report = {} if "accessibility" in requested_sections: @@ -1177,14 +1284,22 @@ def get_document_accessibility_payload( report["issues"] = stored_metadata.get(ACCESSIBILITY_REPORT_ISSUES_KEY) or [] page_chunks = None + chunks = [] if page_number not in (None, ""): - resolved_page = _coerce_positive_int(page_number, default_value=1) chunks = get_ordered_document_chunks( - document_id=document_id, - user_id=user_id, + document_id=document_id, user_id=user_id, group_id=document_context.get("group_id"), public_workspace_id=document_context.get("public_workspace_id"), ) + elif stored_metadata.get("format") in ("xlsx", "xlsm", "xls"): + chunks = get_ordered_document_chunks( + document_id=document_id, user_id=user_id, + group_id=document_context.get("group_id"), + public_workspace_id=document_context.get("public_workspace_id"), + max_chunks=1, native_only=True, + ) + if page_number not in (None, ""): + resolved_page = _coerce_positive_int(page_number, default_value=1) page_chunks = [ chunk for chunk in (chunks or []) if chunk.get("page_number") == resolved_page @@ -1202,6 +1317,23 @@ def get_document_accessibility_payload( if page_chunks is not None: payload["page_number"] = _coerce_positive_int(page_number, default_value=1) payload["page_chunks"] = page_chunks + if stored_metadata.get("format") in ("xlsx", "xlsm", "xls"): + native_chunk = next( + (chunk for chunk in chunks or [] if str(chunk.get("chunk_text") or "").startswith("### Excel native inspection")), + None, + ) + if native_chunk is not None: + payload["citation_chunk"] = native_chunk + if excel_inspection is not None: + payload["excel_inspection"] = { + key: value for key, value in excel_inspection.items() if key != "metadata" + } + page = excel_inspection["page"] + if native_section == "comments" and "accessibility_metadata" in payload: + payload["accessibility_metadata"]["comments"] = page["records"] + payload["accessibility_metadata"]["comment_page"] = { + key: value for key, value in page.items() if key != "records" + } return payload diff --git a/application/single_app/functions_tabular_generated_exports.py b/application/single_app/functions_tabular_generated_exports.py index 1f26415b6..3329a56bc 100644 --- a/application/single_app/functions_tabular_generated_exports.py +++ b/application/single_app/functions_tabular_generated_exports.py @@ -21,6 +21,7 @@ from datetime import datetime, timedelta, timezone from xml.sax.saxutils import escape as escape_xml_text +import pandas from azure.core import MatchConditions from azure.core.exceptions import ResourceExistsError from azure.cosmos.exceptions import CosmosResourceExistsError, CosmosResourceNotFoundError @@ -28,6 +29,7 @@ from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.azure_chat_prompt_execution_settings import AzureChatPromptExecutionSettings from semantic_kernel.contents.chat_history import ChatHistory as SKChatHistory from semantic_kernel_plugins.tabular_processing_plugin import TabularProcessingPlugin +from excel_workbook_reader import EXCEL_WORKBOOK_SCHEMA_VERSION, ExcelWorkbookReader from blob_container_readiness import ( BlobContainerUnavailableError, @@ -3165,7 +3167,7 @@ def _get_versioned_source_blob_client(source_descriptor): if isinstance(blob_properties, dict): current_etag = current_etag or str(blob_properties.get('etag') or '').strip() if current_etag != expected_etag: - raise ValueError('Source CSV changed after the export was queued') + raise ValueError('The tabular source changed after the computation was queued.') return blob_client @@ -3581,6 +3583,41 @@ def _checkpoint_source_input_batch(run, batch_rows, source_scan_row_count): return _replace_claimed_run(run) +def _iter_excel_table_source_chunks(reader, table, source_chunk_rows, source_row_offset=0): + """Keep only a bounded table batch, with physical worksheet provenance.""" + columns = [column['name'] for column in table['columns']] + rows, physical_rows, missing = [], [], {} + batch_start = source_row_offset + + def build_chunk(): + frame = pandas.DataFrame( + rows, columns=columns, index=range(batch_start, batch_start + len(rows)), + ) + frame.attrs.update({ + 'excel_table': deepcopy(table), 'excel_tables': [deepcopy(table)], + 'excel_row_numbers': tuple(physical_rows), + 'excel_row_table_ids': (table['id'],) * len(rows), + 'excel_table_ranges': {table['id']: table['range']}, + 'excel_missing_formula_caches': deepcopy(missing), + 'excel_index_offset': batch_start, + }) + return frame + + for physical_row, cells in reader.iter_table_rows(table): + ordinal = len(rows) + rows.append(['' if cell['value'] is None else str(cell['value']) for cell in cells]) + physical_rows.append(physical_row) + for column, cell in zip(columns, cells): + if cell['value_status'] == 'cache_missing': + missing.setdefault(column, set()).add(ordinal) + if len(rows) >= source_chunk_rows: + yield build_chunk() + batch_start += len(rows) + rows, physical_rows, missing = [], [], {} + if rows: + yield build_chunk() + + def _iter_versioned_tabular_source_rows( source_descriptor, source_format, @@ -3588,8 +3625,14 @@ def _iter_versioned_tabular_source_rows( resume_source_row, user_id=None, ): + if type(resume_source_row) is not int or resume_source_row < 0: + raise ValueError('The source replay cursor is invalid.') + source_chunk_rows = max(1, min(10000, int(source_chunk_rows or 1))) blob_path = str(source_descriptor.get('blob_path') or '').strip() + container_name = source_descriptor.get('container') tabular_plugin = TabularProcessingPlugin(authorized_user_id=user_id) + tabular_plugin._assert_blob_screening_access(container_name, blob_path) + assert_evidence_available(source_descriptor, user_id=user_id) if source_format == 'csv': source_blob_client = _get_versioned_source_blob_client(source_descriptor) @@ -3610,52 +3653,124 @@ def _iter_versioned_tabular_source_rows( tabular_plugin=tabular_plugin, start_source_row=resume_source_row, ) - yield from source_rows + for row_number, row in source_rows: + yield row_number, row + tabular_plugin._assert_blob_screening_access(container_name, blob_path) + _get_versioned_source_blob_client(source_descriptor) return - _get_versioned_source_blob_client(source_descriptor) - sheet_names = [ - str(sheet_name or '').strip() - for sheet_name in source_descriptor.get('sheet_names') or [] - if str(sheet_name or '').strip() - ] - if not sheet_names: - raise ValueError('Source-backed workbook replay requires an explicit worksheet scope') + if ( + type(source_descriptor.get('version')) is not int or source_descriptor.get('version') != 2 + or source_descriptor.get('selection_semantics') != 'excel-tables-v2' + or source_descriptor.get('workbook_schema_version') != EXCEL_WORKBOOK_SCHEMA_VERSION + ): + # A v1 job meant the entire worksheet with row 1 as its header. Table + # discovery is not an equivalent resume of that historical operation. + raise ValueError( + 'This queued workbook uses legacy worksheet-header semantics. Start a new computation to select its tables explicitly.' + ) + selections = source_descriptor.get('table_selections') + source_row_count = source_descriptor.get('source_row_count') + sheet_names = source_descriptor.get('sheet_names') + numeric_scope = source_descriptor.get('numeric_conversion_scope') + if ( + not isinstance(selections, list) or not isinstance(sheet_names, list) or not sheet_names + or any(not isinstance(name, str) or not name for name in sheet_names) + or len(set(sheet_names)) != len(sheet_names) + or ( + source_descriptor.get('selected_sheet') is not None + and source_descriptor.get('selected_sheet') not in sheet_names + ) + or type(source_row_count) is not int or source_row_count < 0 + or resume_source_row > source_row_count + or any(not isinstance(table, dict) for table in selections) + or numeric_scope not in ('table', 'sheet') + ): + raise ValueError('The saved workbook table selection or replay cursor is invalid.') + source_blob_client = _get_versioned_source_blob_client(source_descriptor) + source_bytes = source_blob_client.download_blob( + etag=source_descriptor.get('blob_etag'), match_condition=MatchConditions.IfNotModified, + ).readall() + if hashlib.sha256(source_bytes).hexdigest() != source_descriptor.get('source_sha256'): + raise ValueError('The saved workbook source identity no longer matches.') parsed_return_columns = tabular_plugin._parse_optional_column_list_argument( source_descriptor.get('return_columns') ) - source_row_offset = 0 - for sheet_name in sheet_names: - source_dataframe = tabular_plugin._read_tabular_blob_to_dataframe( - source_descriptor.get('container'), - blob_path, - sheet_name=sheet_name, - require_explicit_sheet=True, - ) - source_dataframe = tabular_plugin._try_numeric_conversion(source_dataframe) - source_dataframe.index = range( - source_row_offset, - source_row_offset + len(source_dataframe), - ) - source_row_offset += len(source_dataframe) - filtered_dataframe, _ = tabular_plugin._apply_query_expression_with_fallback( - source_dataframe, - query_expression=source_descriptor.get('query_expression'), - normalize_match=False, - ) - selected_columns = [ - column_name - for column_name in (parsed_return_columns or list(filtered_dataframe.columns)) - if column_name in filtered_dataframe.columns - ] - output_records = tabular_plugin._build_row_output_records( - filtered_dataframe, - selected_columns, - ) - for source_row_index, output_record in zip(filtered_dataframe.index, output_records): - source_row_number = int(source_row_index) + 1 - if source_row_number > resume_source_row: - yield source_row_number, output_record + query = validate_tabular_csv_query_expression(source_descriptor.get('query_expression')) + with ExcelWorkbookReader(source_bytes, extension=f'.{source_format}') as reader: + if any(name not in reader.sheet_names for name in sheet_names): + raise ValueError('The saved workbook worksheet selection is invalid.') + manifest = reader.build_manifest() + metadata = { + 'tables': [table for sheet in manifest['sheets'] for table in sheet['tables']], + 'tables_by_sheet': {sheet['name']: sheet['tables'] for sheet in manifest['sheets']}, + } + resolved_tables = [] + for sheet_name in sheet_names: + resolved_tables.extend(tabular_plugin._select_excel_tables( + reader, metadata, sheet_name, source_descriptor.get('table_id'), + source_descriptor.get('cell_range'), source_descriptor.get('header_row'), + allow_heterogeneous=True, + )) + selected_table = ( + resolved_tables[0]['id'] if len(resolved_tables) == 1 else source_descriptor.get('table_id') + ) + if ( + resolved_tables != selections or selected_table != source_descriptor.get('selected_table') + or sum(table['row_count'] for table in resolved_tables) != source_row_count + ): + raise ValueError('The saved workbook table, range, or header selection no longer matches.') + numeric_columns_by_scope = {} + for table in resolved_tables: + scope_key = table['sheet'] if numeric_scope == 'sheet' else table['id'] + columns = {column['name'] for column in table['columns']} + numeric_columns = numeric_columns_by_scope.setdefault(scope_key, set(columns)) + numeric_columns.intersection_update(columns) + actual_rows = 0 + for source_chunk in _iter_excel_table_source_chunks(reader, table, source_chunk_rows): + actual_rows += len(source_chunk) + for column in list(numeric_columns): + try: + pandas.to_numeric(source_chunk[column]) + except (ValueError, TypeError): + numeric_columns.discard(column) + if actual_rows != table['row_count']: + raise ValueError('The saved workbook table row count no longer matches.') + tabular_plugin._assert_blob_screening_access(container_name, blob_path) + _get_versioned_source_blob_client(source_descriptor) + source_row_offset = 0 + for table in resolved_tables: + scope_key = table['sheet'] if numeric_scope == 'sheet' else table['id'] + numeric_columns = numeric_columns_by_scope[scope_key] + actual_rows = table['row_count'] + if source_row_offset + actual_rows <= resume_source_row: + source_row_offset += actual_rows + continue + for source_chunk in _iter_excel_table_source_chunks( + reader, table, source_chunk_rows, source_row_offset, + ): + if source_chunk.index[-1] < resume_source_row: + continue + for column in numeric_columns: + source_chunk[column] = pandas.to_numeric(source_chunk[column]) + source_chunk = source_chunk.loc[source_chunk.index >= resume_source_row] + filtered_chunk, _ = tabular_plugin._apply_query_expression_with_fallback( + source_chunk, query_expression=query, normalize_match=False, + ) + selected_columns = [ + column for column in (parsed_return_columns or list(filtered_chunk.columns)) + if column in filtered_chunk.columns + ] + records = tabular_plugin._build_row_output_records(filtered_chunk, selected_columns) + tabular_plugin._assert_blob_screening_access(container_name, blob_path) + assert_evidence_available(source_descriptor, user_id=user_id) + _get_versioned_source_blob_client(source_descriptor) + for source_row_index, record in zip(filtered_chunk.index, records): + if source_descriptor.get('include_sheet_column') is True: + record['_sheet'] = table['sheet'] + yield int(source_row_index) + 1, record + source_row_offset += actual_rows + tabular_plugin._assert_blob_screening_access(container_name, blob_path) _get_versioned_source_blob_client(source_descriptor) @@ -3692,7 +3807,14 @@ def _stage_tabular_generated_output_source(run, settings): minimum=1, maximum=TABULAR_EXPORT_MAX_SOURCE_BATCH_CHARS, ) - resume_source_row = _safe_int(run.get('source_scan_row_count')) + resume_source_row = run.get('source_scan_row_count', 0) + if ( + type(resume_source_row) is not int or resume_source_row < 0 + or resume_source_row < _safe_int(run.get('source_staged_rows')) + ): + raise ValueError('The saved source staging cursor is invalid.') + if _safe_int(run.get('source_staged_rows')) > expected_row_count: + raise ValueError('The saved source staging row count is invalid.') pending_rows = [] pending_chars = 0 last_source_row_number = resume_source_row @@ -3705,6 +3827,11 @@ def _stage_tabular_generated_output_source(run, settings): ) for source_row_number, source_row in source_rows: + if ( + type(source_row_number) is not int or source_row_number <= last_source_row_number + or not isinstance(source_row, dict) + ): + raise ValueError('The source replay returned an invalid or repeated cursor.') source_row_text = _dump_generated_output_json(source_row) current_batch_row_limit = _get_tabular_source_batch_row_limit( source_descriptor, @@ -3760,7 +3887,21 @@ def _stage_tabular_generated_output_source(run, settings): )) if not isinstance(batch_rows, list): raise ValueError(f'Source input checkpoint {batch_number}/{staged_batch_count} was not a JSON array') + expected_rows = _prepare_tabular_source_rows( + batch_rows, start_row=sum(staged_chunk_row_counts), token_namespace=run.get('id'), + ) + for row, expected in zip(batch_rows, expected_rows): + if not isinstance(row, dict) or any( + row.get(field) != expected[field] for field in ( + TABULAR_EXPORT_INPUT_ROW_NUMBER_FIELD, + TABULAR_EXPORT_INPUT_ROW_IDENTITY_FIELD, + TABULAR_EXPORT_INPUT_ROW_TOKEN_FIELD, + ) + ): + raise ValueError('A source input checkpoint has invalid row identity or order.') staged_chunk_row_counts.append(len(batch_rows)) + if sum(staged_chunk_row_counts) != staged_row_count: + raise ValueError('The source input checkpoints do not match the staged row count.') chunk_manifest = _write_chunk_manifest_for_run( run.get('user_id'), diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index 34ee5ab1d..1b3cd705c 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -169,7 +169,14 @@ from functions_authentication import * from functions_search import * from functions_search_service import get_document_accessibility_payload, search_documents, search_relevant_tabular_candidates -from functions_excel_accessibility import EXCEL_EVIDENCE_MARKER, is_excel_native_inspection_question +from functions_excel_accessibility import ( + EXCEL_EVIDENCE_MARKER, + build_excel_native_chat_evidence, + excel_native_question_sections, + has_excel_native_inspection_intent, + is_excel_native_inspection_question, + merge_excel_native_evidence, +) from functions_accessibility import format_accessibility_metadata_for_chat from functions_public_workspaces import resolve_public_chat_workspace_ids from public_chat_scope_state import aggregate_public_workspace_ids @@ -13993,41 +14000,52 @@ def execute_source(source): 'active_group_ids': [source['group_id']] if source.get('group_id') else None, 'active_public_workspace_id': source.get('public_workspace_id'), } + sections = excel_native_question_sections(question) report = get_document_accessibility_payload( - source['document_id'], user_id, conversation_id=conversation_id, **scope, + source['document_id'], user_id, conversation_id=conversation_id, + native_section=sections[0], _native_max_bytes=10000, **scope, ) metadata = report.get('accessibility_metadata') or {} if metadata.get('format') not in ('xlsx', 'xlsm', 'xls'): raise ValueError('This source has no native Excel inspection metadata') - summary = format_accessibility_metadata_for_chat({ - **metadata, 'style_inventory': report.get('style_inventory') or {}, - }) - reference = annotate_document_search_payload( - {'document': report['document']}, 'get_document_accessibility_report', - ) - excerpts = search_documents( - query=f"Excel native inspection {question}", user_id=user_id, - document_ids=[source['document_id']], top_n=5, **scope, - ) - native_results = [ - item for item in excerpts.get('results', []) - if EXCEL_EVIDENCE_MARKER in str(item.get('chunk_text') or '') - ] - excerpts = annotate_document_search_payload({'results': native_results}, 'search_documents') - text = f"{summary[:6000]}\n{reference.get('citation', '')}" - if len(summary) > 6000: - text += "\n[Parent summary shortened; request the stored report for remaining details.]" - for item in native_results[:3]: - content = str(item.get('chunk_text') or '') - text += f"\n\n{content[:2500]}\n{item.get('citation', '')}" - if len(content) > 2500: - text += "\n[Search excerpt shortened; request this evidence segment for remaining details.]" - if not native_results: - text += "\nNo native evidence excerpts matched this query; only the stored parent summary is available." + pages = [] + if (report.get('excel_inspection') or {}).get('page'): + pages.append(report['excel_inspection']['page']) + for section in sections[1:]: + section_report = get_document_accessibility_payload( + source['document_id'], user_id, conversation_id=conversation_id, + native_section=section, _native_max_bytes=10000, **scope, + ) + section_page = (section_report.get('excel_inspection') or {}).get('page') + if section_page: + pages.append(section_page) + if pages: + report.setdefault('excel_inspection', {})['pages'] = pages + reference = annotate_document_search_payload(report, 'get_document_accessibility_report') + native_evidence = build_excel_native_chat_evidence(reference, question) + text = f"{native_evidence['summary']}\n{reference.get('citation', '')}" + native_results = [] + if not native_evidence['returned_records']: + excerpts = search_documents( + query=f"Excel native inspection {question}", user_id=user_id, + document_ids=[source['document_id']], top_n=5, **scope, + ) + native_results = [ + item for item in excerpts.get('results', []) + if EXCEL_EVIDENCE_MARKER in str(item.get('chunk_text') or '') + ] + excerpts = annotate_document_search_payload({'results': native_results}, 'search_documents') + for item in excerpts['results']: + content = str(item.get('chunk_text') or '') + addition = f"\n\n{content}\n{item.get('citation', '')}" + if len((text + addition).encode('utf-8')) > 16000: + native_evidence['partial'] = True + continue + text += addition source_citations = [{ 'tool_name': 'document_search.get_document_accessibility_report', 'plugin_name': 'document_search', 'function_name': 'get_document_accessibility_report', - 'function_arguments': {'document_id': source['document_id']}, + 'function_arguments': {'document_id': source['document_id'], 'native_section': sections[0]}, 'function_result': reference, 'success': True, }] if native_results: @@ -14045,7 +14063,11 @@ def execute_source(source): 'summary': text, 'citations': source_citations, 'coverage': { 'execution_mode': 'native_inspection', 'worksheet_data_computed': False, + 'worksheet_data_read': native_evidence['worksheet_data_read'], 'native_coverage': metadata.get('coverage'), + 'native_records_returned': native_evidence['returned_records'], + 'native_records_partial': native_evidence['partial'], + 'native_next_cursor': native_evidence['next_cursor'], 'matched_evidence_segments': len(native_results), }, } @@ -14088,6 +14110,18 @@ def _execute_mixed_source_tabular_evidence( cancel_requested=cancel_requested, request_correlation_id=request_correlation_id, ) + excel_native_sources = [ + source for source in tabular_sources + if str(source.get('file_name') or '').lower().endswith(('.xlsx', '.xlsm', '.xls')) + ] + native_result = None + if excel_native_sources and has_excel_native_inspection_intent( + user_question, file_names=[source.get('file_name') for source in excel_native_sources], + ): + native_result = _execute_excel_native_inspection_evidence( + excel_native_sources, user_question, user_id, conversation_id, selection_mode, + cancel_requested=cancel_requested, request_correlation_id=request_correlation_id, + ) source_contexts = build_tabular_file_contexts_from_manifest(tabular_sources) if has_request_context(): authorized_context = dict(getattr(g, 'authorized_chat_context', {}) or {}) @@ -14130,7 +14164,7 @@ def fail_unavailable_tabular_source(source): del source raise RuntimeError('Tabular processing is unavailable') - return { + return merge_excel_native_evidence({ 'evidence_envelopes': execute_tabular_evidence_sources( tabular_sources, fail_unavailable_tabular_source, @@ -14145,7 +14179,7 @@ def fail_unavailable_tabular_source(source): 'invocations': [], 'executed': False, 'token_usage': None, - } + }, native_result, user_question) execute_tabular = should_run_tabular_evidence( user_question, @@ -14359,7 +14393,7 @@ def execute_source(source): request_correlation_id=request_correlation_id, user_id=user_id, ) - return { + return merge_excel_native_evidence({ 'evidence_envelopes': evidence_envelopes, 'system_messages': system_messages, 'agent_citations': agent_citations, @@ -14367,7 +14401,7 @@ def execute_source(source): 'invocations': all_invocations, 'executed': execute_tabular, 'token_usage': token_usage if token_usage['request_count'] else None, - } + }, native_result, user_question) def _exclude_xsd_contract_sources_from_evidence(manifest, xsd_generation_contract): diff --git a/application/single_app/route_enhanced_citations.py b/application/single_app/route_enhanced_citations.py index e3d36cb82..8f5aad3c6 100644 --- a/application/single_app/route_enhanced_citations.py +++ b/application/single_app/route_enhanced_citations.py @@ -12,6 +12,7 @@ import mimetypes import io import uuid +import zipfile from urllib.parse import quote import pandas import fitz @@ -28,6 +29,7 @@ read_available_document_bytes, ) from content_screening.contracts import ScreeningError +from excel_workbook_reader import build_excel_table_preview from functions_authentication import login_required, user_required, get_current_user_id, get_current_user_info from functions_appinsights import log_event from functions_artifact_publication import publish_generated_chat_artifact_for_user @@ -144,6 +146,17 @@ def _serialize_tabular_preview_table(df_preview): return columns, rows +def _preview_integer(value, name, default, minimum=0, maximum=None): + if value is None: + return default + if not isinstance(value, str) or not value.isascii() or not value.isdigit(): + raise ValueError(f"{name} must be an integer.") + parsed = int(value) + if parsed < minimum or maximum is not None and parsed > maximum: + raise ValueError(f"{name} is outside the supported preview range.") + return parsed + + def _resolve_document_blob_reference(raw_doc): """Resolve the persisted blob container and path for the cited document.""" _, container_name, blob_name = get_available_blob_reference(raw_doc, purpose="citation") @@ -920,7 +933,7 @@ def get_enhanced_citation_tabular_preview(): doc_id = request.args.get("doc_id") sheet_name = request.args.get("sheet_name") sheet_index = request.args.get("sheet_index") - max_rows = min(request.args.get("max_rows", 200, type=int), 500) + table_id = request.args.get("table_id") if not doc_id: return jsonify({"error": "doc_id is required"}), 400 @@ -929,6 +942,9 @@ def get_enhanced_citation_tabular_preview(): return jsonify({"error": "User not authenticated"}), 401 try: + max_rows = _preview_integer(request.args.get("max_rows"), "max_rows", 200, 1, 500) + start_row = _preview_integer(request.args.get("start_row"), "start_row", 0) + header_row = _preview_integer(request.args.get("header_row"), "header_row", None, 0, 1048576) doc_response, status_code = get_document(user_id, doc_id) if status_code != 200: return doc_response, status_code @@ -952,99 +968,54 @@ def get_enhanced_citation_tabular_preview(): return jsonify({"error": "File is too large to preview"}), 400 raw_doc, data = read_available_document_bytes(raw_doc, user_id=user_id, purpose="preview") - # Read into DataFrame, limiting rows for preview efficiency - # Read max_rows + 1 so we can detect truncation without loading the full file - nrows_limit = max_rows + 1 - selected_sheet = None - sheet_names = [] if ext == 'csv': + # Keep the optional CSV decoder out of Excel-only preview requests. from functions_tabular_csv_query import read_tabular_csv - df = read_tabular_csv(io.BytesIO(data), keep_default_na=False, dtype=str, nrows=nrows_limit) - elif ext in ('xlsx', 'xlsm'): - excel_file = pandas.ExcelFile(io.BytesIO(data), engine='openpyxl') - sheet_names = list(excel_file.sheet_names) - if not sheet_names: - return jsonify({"error": "Workbook does not contain any readable sheets"}), 400 - - if sheet_name: - requested_sheet_name = sheet_name.strip() - matching_sheet_name = next( - (candidate for candidate in sheet_names if candidate.lower() == requested_sheet_name.lower()), - None, - ) - if not matching_sheet_name: - return jsonify({ - "error": f"Sheet '{requested_sheet_name}' was not found. Available sheets: {sheet_names}" - }), 400 - selected_sheet = matching_sheet_name - elif sheet_index not in (None, ''): - try: - resolved_sheet_index = int(sheet_index) - except ValueError: - return jsonify({"error": "sheet_index must be an integer"}), 400 - if resolved_sheet_index < 0 or resolved_sheet_index >= len(sheet_names): - return jsonify({ - "error": f"sheet_index {resolved_sheet_index} is out of range. Available sheets: {sheet_names}" - }), 400 - selected_sheet = sheet_names[resolved_sheet_index] - else: - selected_sheet = sheet_names[0] - - df = excel_file.parse(selected_sheet, keep_default_na=False, dtype=str, nrows=nrows_limit) - elif ext == 'xls': - excel_file = pandas.ExcelFile(io.BytesIO(data), engine='xlrd') - sheet_names = list(excel_file.sheet_names) - if not sheet_names: - return jsonify({"error": "Workbook does not contain any readable sheets"}), 400 - - if sheet_name: - requested_sheet_name = sheet_name.strip() - matching_sheet_name = next( - (candidate for candidate in sheet_names if candidate.lower() == requested_sheet_name.lower()), - None, - ) - if not matching_sheet_name: - return jsonify({ - "error": f"Sheet '{requested_sheet_name}' was not found. Available sheets: {sheet_names}" - }), 400 - selected_sheet = matching_sheet_name - elif sheet_index not in (None, ''): - try: - resolved_sheet_index = int(sheet_index) - except ValueError: - return jsonify({"error": "sheet_index must be an integer"}), 400 - if resolved_sheet_index < 0 or resolved_sheet_index >= len(sheet_names): - return jsonify({ - "error": f"sheet_index {resolved_sheet_index} is out of range. Available sheets: {sheet_names}" - }), 400 - selected_sheet = sheet_names[resolved_sheet_index] - else: - selected_sheet = sheet_names[0] - - df = excel_file.parse(selected_sheet, keep_default_na=False, dtype=str, nrows=nrows_limit) + frames, seen, selected = [], 0, 0 + columns = [] + with read_tabular_csv( + io.BytesIO(data), keep_default_na=False, dtype=str, chunksize=10000, + ) as chunks: + for frame in chunks: + columns = frame.columns.tolist() + if seen + len(frame) > start_row: + window = frame.iloc[max(0, start_row - seen):] + window = window.head(max_rows + 1 - selected) + frames.append(window) + selected += len(window) + seen += len(frame) + if selected > max_rows: + break + frame = pandas.concat(frames, ignore_index=True) if frames else pandas.DataFrame(columns=columns) + has_more = len(frame) > max_rows + frame = frame.head(max_rows) + names, rows = _serialize_tabular_preview_table(frame) + result = { + "selected_sheet": None, "sheet_names": [], "sheet_count": 0, + "tables": [], "table_count": 0, "total_rows": None if has_more else seen, + "total_columns": len(names), "columns": names, "rows": rows, + "row_numbers": list(range(start_row + 2, start_row + 2 + len(rows))), + "start_row": start_row, "has_more": has_more, "truncated": has_more, + "next_start_row": start_row + len(rows) if has_more else None, + } else: - return jsonify({"error": f"Unsupported file type: {ext}"}), 400 - - total_rows = len(df) - truncated = total_rows > max_rows - preview = df.head(max_rows) - columns, rows = _serialize_tabular_preview_table(preview) - - return jsonify({ - "filename": file_name, - "selected_sheet": selected_sheet, - "sheet_names": sheet_names, - "sheet_count": len(sheet_names), - "total_rows": total_rows if not truncated else None, - "total_columns": len(df.columns), - "columns": columns, - "rows": rows, - "truncated": truncated - }) + result = build_excel_table_preview( + data, "." + ext, sheet_name=sheet_name, sheet_index=sheet_index, + table_id=table_id, start_row=start_row, max_rows=max_rows, + cell_range=request.args.get("cell_range"), header_row=header_row, + ) + frame = pandas.DataFrame(result["rows"], columns=result["columns"]) + result["columns"], result["rows"] = _serialize_tabular_preview_table(frame) + return jsonify({"filename": file_name, **result}) - except Exception as e: - debug_print(f"Error generating tabular preview: {e}") - return jsonify({"error": str(e)}), 500 + except ScreeningError as error: + return jsonify({"error": error.public_message, "error_code": error.code}), error.status_code + except (ValueError, zipfile.BadZipFile, pandas.errors.EmptyDataError) as error: + _log_enhanced_citations_error("Spreadsheet preview selection failed.", error, document_id=doc_id) + return jsonify({"error": "The spreadsheet selection is invalid or unavailable. Choose a valid worksheet, table, and row window."}), 400 + except Exception as error: + _log_enhanced_citations_error("Spreadsheet preview failed.", error, document_id=doc_id) + return jsonify({"error": "Unable to load the spreadsheet preview."}), 500 @bp.route("/api/enhanced_citations/visio", methods=["GET"]) @swagger_route(security=get_auth_security()) diff --git a/application/single_app/semantic_kernel_plugins/document_search_plugin.py b/application/single_app/semantic_kernel_plugins/document_search_plugin.py index 1b9d7a304..0e65870fd 100644 --- a/application/single_app/semantic_kernel_plugins/document_search_plugin.py +++ b/application/single_app/semantic_kernel_plugins/document_search_plugin.py @@ -1,11 +1,13 @@ # document_search_plugin.py +import logging from typing import Annotated, Any, Dict, List, Optional, Union from semantic_kernel.functions import kernel_function from content_screening.contracts import ScreeningError from functions_authentication import get_current_user_id +from functions_appinsights import log_event from public_chat_scope_state import current_public_chat_scope from functions_agent_document_citations import annotate_document_search_payload from functions_search import ( @@ -154,6 +156,11 @@ def metadata(self) -> Dict[str, Any]: {'name': 'doc_scope', 'type': 'str', 'description': 'all, personal, group, or public.', 'required': False}, {'name': 'sections', 'type': 'str', 'description': 'Optional comma-separated sections: accessibility, style, issues.', 'required': False}, {'name': 'page_number', 'type': 'int', 'description': 'Optional page or slide number whose chunk text is also returned.', 'required': False}, + {'name': 'native_section', 'type': 'str', 'description': 'Excel inventory section: comments, tables, objects, image_analyses, property_records, custom_properties, extended_properties, defined_names, hyperlinks, validations.', 'required': False}, + {'name': 'native_cursor', 'type': 'str', 'description': 'Excel next_cursor returned by the previous inventory page.', 'required': False}, + {'name': 'native_page_size', 'type': 'int', 'description': 'Excel records per response, between 1 and 200.', 'required': False}, + {'name': 'sheet_name', 'type': 'str', 'description': 'Worksheet for an optional saved-cell range read.', 'required': False}, + {'name': 'cell_range', 'type': 'str', 'description': 'Optional Excel A1 range, at most 2000 cells, without executing formulas.', 'required': False}, ], 'returns': {'type': 'dict', 'description': 'Stored accessibility metadata, style inventory, and issues.'}, }, @@ -440,6 +447,10 @@ def retrieve_document_chunks( 'Use it to build custom style-guide or accessibility checks. This is extracted metadata, not a ' 'WCAG or PDF/UA conformance result. Use sections to limit the output and page_number to also ' 'return that page\'s stored chunk text. ' + 'For Excel, inspect native_section and follow excel_inspection.page.next_cursor until has_more ' + 'is false when the question requires every record. Compatibility notes are not extra comments. ' + 'Use sheet_name and cell_range for actual saved values and formula caches; do not infer them ' + 'from a schema preview. Existing stored originals are inspected without requiring re-upload. ' 'The payload carries a "citation" value; copy it verbatim into your answer when you use it.' ), ) @@ -451,6 +462,11 @@ def get_document_accessibility_report( page_number: Annotated[int, 'Optional 1-based stored page/slide or Excel evidence segment number, not an Excel printed page.'] = 0, active_group_ids: Annotated[str, 'Optional comma-separated group ids when resolving group content.'] = '', active_public_workspace_id: Annotated[str, 'Optional public workspace id when resolving public content.'] = '', + native_section: Annotated[str, 'Excel section: comments, tables, objects, image_analyses, property_records, custom_properties, extended_properties, defined_names, hyperlinks, validations.'] = 'comments', + native_cursor: Annotated[str, 'Excel inventory next_cursor from the previous page, including continued long records.'] = '', + native_page_size: Annotated[int, 'Excel inventory page size from 1 through 200; responses also have a byte budget.'] = 100, + sheet_name: Annotated[str, 'Worksheet name for optional direct saved-cell inspection.'] = '', + cell_range: Annotated[str, 'Optional A1 range with at most 2000 cells; reads saved values/formulas, never recalculates.'] = '', ) -> Annotated[dict, 'Stored accessibility, style inventory, and issue metadata for one document.']: try: return annotate_document_search_payload( @@ -463,6 +479,9 @@ def get_document_accessibility_report( active_public_workspace_id=self._resolve_allowed_public_workspace_ids(active_public_workspace_id), sections=sections, page_number=page_number if int(page_number or 0) > 0 else None, + native_section=native_section, native_cursor=native_cursor or None, + native_page_size=native_page_size, sheet_name=sheet_name or None, + cell_range=cell_range or None, ), 'get_document_accessibility_report', ) @@ -470,8 +489,12 @@ def get_document_accessibility_report( return {"error": error.public_message, "error_code": error.code, "status_code": error.status_code} except DocumentSearchScopeError as error: return {'error': str(error)} - except Exception as e: - return {'error': str(e)} + except Exception as error: + log_event( + "[DOCUMENT_METADATA] Document inspection action failed.", + level=logging.WARNING, extra={"error_type": type(error).__name__}, + ) + return {'error': 'Unable to inspect the requested document or workbook range.'} # bac-check: ignore - summarize_document_content delegates to get_document_chunks_payload's current-user scope check. @plugin_function_logger('DocumentSearchPlugin') diff --git a/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py b/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py index 878f70c9c..4eb859f1e 100644 --- a/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py +++ b/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py @@ -8,6 +8,7 @@ """ import asyncio import copy +import hashlib from datetime import date, datetime import io import json @@ -15,6 +16,7 @@ import math import re import tempfile +import tokenize import warnings import pandas from azure.core import MatchConditions @@ -22,6 +24,12 @@ from typing import Annotated, Dict, List, Optional, Set from urllib.parse import urlsplit, urlunsplit from semantic_kernel.functions import kernel_function +from excel_workbook_reader import ( + EXCEL_PREVIEW_MAX_BYTES, + EXCEL_PREVIEW_MAX_COLUMNS, + EXCEL_WORKBOOK_SCHEMA_VERSION, + ExcelWorkbookReader, +) from semantic_kernel_plugins.plugin_invocation_logger import PluginInvocationResult, plugin_function_logger from content_screening.access import ( PRIVATE_CONTAINER, @@ -51,6 +59,27 @@ ) +class TabularSelectionError(ValueError): + """A user-actionable selection failure, containing no storage error text.""" + + def __init__(self, message, *, code='table_selection_required', choices=None): + super().__init__(message) + self.public_message = message + self.code = code + self.choices = copy.deepcopy(choices or []) + + +class TabularSavedValueError(ValueError): + """The selected operation depends on formulas without saved results.""" + + def __init__(self): + self.public_message = ( + "Selected formula cells have no saved cached values. Recalculate and save " + "the workbook in Excel before computing these values." + ) + super().__init__(self.public_message) + + class TabularProcessingPlugin: """Provides data analysis functions on tabular files stored in blob storage.""" @@ -130,6 +159,7 @@ class TabularProcessingPlugin: ROW_OUTPUT_SAFE_CHAR_LIMIT = 50000 ROW_OUTPUT_PROTECTED_COLUMNS = ( '_sheet', + '_excel_source', '_matched_columns', '_matched_values', '_matched_on', @@ -138,14 +168,15 @@ class TabularProcessingPlugin: ) def __init__(self, *, authorized_user_id=None): - self._df_cache = {} # Per-instance cache: (container, blob_name, sheet_name) -> DataFrame - self._blob_data_cache = {} # Per-instance cache: (container, blob_name) -> raw bytes - self._workbook_metadata_cache = {} # Per-instance cache: (container, blob_name) -> workbook metadata + self._df_cache = {} # Source version and explicit selection -> DataFrame + self._blob_data_cache = {} # (container, blob_name, ETag) -> raw bytes + self._workbook_metadata_cache = {} # (container, blob_name, ETag) -> workbook metadata self._blob_version_cache = {} # Per-instance cache: (container, blob_name) -> immutable blob version metadata self._default_sheet_overrides = {} # (container, blob_name) -> default sheet name self._resolved_blob_location_overrides = {} # (source, filename) -> (container, blob_name) self._screening_source_versions = {} self._screening_user_id = authorized_user_id + self._last_table_reads = {} @classmethod def get_discovery_function_names(cls): @@ -419,11 +450,12 @@ def _is_authorized_blob_location(self, container_name: str, blob_path: str, auth def _evict_screened_blob_cache(self, container_name, blob_name): cache_key = (container_name, blob_name) - for cache_name in ("_blob_data_cache", "_workbook_metadata_cache", "_blob_version_cache"): - getattr(self, cache_name).pop(cache_key, None) - for key in list(self._df_cache): - if key[:2] == cache_key: - self._df_cache.pop(key, None) + for cache_name in ("_blob_data_cache", "_workbook_metadata_cache", "_df_cache", "_last_table_reads"): + cache = getattr(self, cache_name) + for key in list(cache): + if key[:2] == cache_key: + cache.pop(key, None) + self._blob_version_cache.pop(cache_key, None) self._screening_source_versions.pop(cache_key, None) def _assert_blob_screening_access(self, container_name, blob_name, *, purpose="native"): @@ -472,7 +504,8 @@ def _list_tabular_blobs(self, container_name: str, prefix: str) -> List[str]: def _download_tabular_blob_bytes(self, container_name: str, blob_name: str) -> bytes: """Download a blob once and reuse the raw bytes across sheet-aware operations.""" document, user_id = self._assert_blob_screening_access(container_name, blob_name) - cache_key = (container_name, blob_name) + blob_version = self._get_tabular_blob_version(container_name, blob_name, refresh=True) + cache_key = (container_name, blob_name, blob_version['blob_etag']) if cache_key in self._blob_data_cache: return self._blob_data_cache[cache_key] @@ -483,11 +516,6 @@ def _download_tabular_blob_bytes(self, container_name: str, blob_name: str) -> b client = self._get_blob_service_client() blob_client = client.get_blob_client(container=container_name, blob=blob_name) - blob_version = self._get_tabular_blob_version( - container_name, - blob_name, - refresh=True, - ) stream = blob_client.download_blob( etag=blob_version['blob_etag'], match_condition=MatchConditions.IfNotModified, @@ -508,7 +536,11 @@ def _get_tabular_blob_version(self, container_name: str, blob_name: str, refresh container=container_name, blob=blob_name, ) - blob_properties = blob_client.get_blob_properties() + try: + blob_properties = blob_client.get_blob_properties() + except Exception: + self._evict_screened_blob_cache(container_name, blob_name) + raise blob_etag = getattr(blob_properties, 'etag', None) blob_size = getattr(blob_properties, 'size', None) if isinstance(blob_properties, dict): @@ -527,6 +559,11 @@ def _get_tabular_blob_version(self, container_name: str, blob_name: str, refresh } if document is not None: blob_version[PROVENANCE_FIELD] = document_provenance(document) + previous = self._blob_version_cache.get(cache_key) + if previous is not None and previous != blob_version: + self._evict_screened_blob_cache(container_name, blob_name) + if document is not None: + self._screening_source_versions[cache_key] = document_provenance(document) self._blob_version_cache[cache_key] = blob_version return dict(blob_version) @@ -541,8 +578,8 @@ def _get_excel_engine(self, blob_name: str) -> Optional[str]: def _get_workbook_metadata(self, container_name: str, blob_name: str) -> dict: """Return workbook metadata including available sheet names for Excel files.""" - self._assert_blob_screening_access(container_name, blob_name) - cache_key = (container_name, blob_name) + blob_version = self._get_tabular_blob_version(container_name, blob_name, refresh=True) + cache_key = (container_name, blob_name, blob_version['blob_etag']) if cache_key in self._workbook_metadata_cache: return copy.deepcopy(self._workbook_metadata_cache[cache_key]) @@ -552,16 +589,26 @@ def _get_workbook_metadata(self, container_name: str, blob_name: str) -> dict: 'sheet_names': [], 'sheet_count': 0, 'default_sheet': None, + 'blob_etag': blob_version['blob_etag'], } if engine: data = self._download_tabular_blob_bytes(container_name, blob_name) - excel_file = pandas.ExcelFile(io.BytesIO(data), engine=engine) - sheet_names = list(excel_file.sheet_names) + extension = '.xls' if engine == 'xlrd' else '.xlsm' if blob_name.lower().endswith('.xlsm') else '.xlsx' + with ExcelWorkbookReader(data, extension=extension) as reader: + manifest = reader.build_manifest() + manifest['source_sha256'] = hashlib.sha256(data).hexdigest() + sheet_names = [sheet['name'] for sheet in manifest['sheets']] metadata.update({ 'sheet_names': sheet_names, 'sheet_count': len(sheet_names), 'default_sheet': sheet_names[0] if sheet_names else None, + 'inspection_schema_version': EXCEL_WORKBOOK_SCHEMA_VERSION, + 'table_count': manifest['table_count'], + 'tables': [table for sheet in manifest['sheets'] for table in sheet['tables']], + 'tables_by_sheet': {sheet['name']: sheet['tables'] for sheet in manifest['sheets']}, + 'coverage': manifest['coverage'], + 'workbook_manifest': manifest, }) self._assert_blob_screening_access(container_name, blob_name) @@ -575,8 +622,12 @@ def _resolve_sheet_selection( sheet_name: Optional[str] = None, sheet_index: Optional[str] = None, require_explicit_sheet: bool = False, + table_id: Optional[str] = None, + cell_range: Optional[str] = None, + header_row: Optional[int] = None, ) -> tuple: """Resolve a workbook sheet selection and enforce explicit choice when required.""" + table_id, cell_range, header_row = self._normalize_table_selectors(table_id, cell_range, header_row) workbook_metadata = self._get_workbook_metadata(container_name, blob_name) if not workbook_metadata.get('is_workbook'): return None, workbook_metadata @@ -584,6 +635,18 @@ def _resolve_sheet_selection( available_sheets = workbook_metadata.get('sheet_names', []) if not available_sheets: raise ValueError(f"Workbook '{blob_name}' does not contain any readable sheets.") + if table_id and table_id != 'all' and not sheet_name and sheet_index in (None, ''): + matches = [ + table for table in workbook_metadata.get('tables') or [] + if table['id'] == table_id or table['name'] == table_id + ] + if len(matches) != 1: + raise TabularSelectionError( + "The requested workbook table was not found or is ambiguous.", + code='table_selection_invalid', + choices=self._table_choices(workbook_metadata), + ) + sheet_name = matches[0]['sheet'] matched_sheet_name = self._match_workbook_sheet_name(sheet_name, available_sheets) if matched_sheet_name: @@ -591,9 +654,11 @@ def _resolve_sheet_selection( normalized_sheet_name = (sheet_name or '').strip() if normalized_sheet_name: - raise ValueError( + raise TabularSelectionError( f"Sheet '{normalized_sheet_name}' was not found in workbook '{blob_name}'. " - f"Available sheets: {available_sheets}." + f"Available sheets: {available_sheets}.", + code='sheet_selection_invalid', + choices=self._table_choices(workbook_metadata), ) normalized_sheet_index = None if sheet_index is None else str(sheet_index).strip() @@ -617,20 +682,59 @@ def _resolve_sheet_selection( # Use pre-selected default sheet if one was set by the orchestration layer override_key = (container_name, blob_name) - if override_key in self._default_sheet_overrides: + if not cell_range and override_key in self._default_sheet_overrides: override_sheet = self._default_sheet_overrides[override_key] matched_override_sheet = self._match_workbook_sheet_name(override_sheet, available_sheets) if matched_override_sheet: return matched_override_sheet, workbook_metadata if require_explicit_sheet: - raise ValueError( + raise TabularSelectionError( f"Workbook '{blob_name}' has multiple sheets: {available_sheets}. " - "Specify sheet_name or sheet_index on analytical calls." + "Specify sheet_name or sheet_index on analytical calls.", + code='sheet_selection_required', + choices=self._table_choices(workbook_metadata), ) return workbook_metadata.get('default_sheet'), workbook_metadata + @staticmethod + def _table_choices(workbook_metadata, sheet_name=None): + """Return compact choices rather than a guessed first table.""" + return [ + { + key: copy.deepcopy(table.get(key)) + for key in ('id', 'name', 'sheet', 'range', 'header_row', 'header_status', 'row_count', 'columns') + } + for table in workbook_metadata.get('tables') or [] + if sheet_name is None or table.get('sheet') == sheet_name + ] + + @staticmethod + def _tabular_error_payload(error): + """Keep validation actionable without returning SDK or provider messages.""" + log_event( + '[TABULAR_PROCESSING_PLUGIN] Tabular operation could not complete.', + level=logging.WARNING, + extra={'exception_type': type(error).__name__}, + ) + if isinstance(error, TabularSelectionError): + return { + 'error': error.public_message, 'error_code': error.code, + 'selection_required': True, 'table_choices': error.choices, + } + if isinstance(error, TabularSavedValueError): + return {'error': error.public_message, 'error_code': 'formula_cache_missing'} + if isinstance(error, (PermissionError, ScreeningError)): + return { + 'error': 'This tabular source is unavailable or is no longer authorized.', + 'error_code': 'tabular_source_unavailable', + } + return { + 'error': 'Unable to analyze this tabular source. Check the selection, column names, and query, then try again.', + 'error_code': 'tabular_analysis_failed', + } + def _match_workbook_sheet_name(self, requested_sheet_name: Optional[str], available_sheets: List[str]) -> Optional[str]: """Match a workbook sheet name while tolerating trailing whitespace and case drift.""" raw_sheet_name = None if requested_sheet_name is None else str(requested_sheet_name) @@ -698,7 +802,18 @@ def _build_row_output_records(self, row_frame, selected_columns): ] output_records = [] - for _, row in row_frame.iterrows(): + table = row_frame.attrs.get('excel_table') + if table is not None and '_excel_source' in row_frame.columns: + raise TabularSelectionError( + "The selected source uses the reserved Excel provenance field _excel_source.", + code='reserved_excel_column', + ) + self._require_saved_values(row_frame, selected_column_names) + source_rows = row_frame.attrs.get('excel_row_numbers') or () + source_tables = row_frame.attrs.get('excel_row_table_ids') or () + source_ranges = row_frame.attrs.get('excel_table_ranges') or {} + source_offset = row_frame.attrs.get('excel_index_offset', 0) + for row_index, row in row_frame.iterrows(): row_payload = { str(column_name): row.get(column_name) for column_name in selected_column_names @@ -715,6 +830,14 @@ def _build_row_output_records(self, row_frame, selected_columns): row_payload['_related_document_reference_values'] = merged_reference_values else: row_payload['_related_document_reference_values'] = related_document_reference_values + if table is not None: + ordinal = int(row_index) - source_offset + table_id = source_tables[ordinal] if 0 <= ordinal < len(source_tables) else table['id'] + row_payload['_excel_source'] = { + 'sheet': table['sheet'], 'table_id': table_id, + 'range': source_ranges.get(table_id, table['range']), + 'row': source_rows[ordinal] if 0 <= ordinal < len(source_rows) else None, + } output_records.append(row_payload) return output_records @@ -778,7 +901,7 @@ def _build_tabular_row_page_payload( safe_max_chars = int(max_chars or self.ROW_OUTPUT_SAFE_CHAR_LIMIT) requested_return_columns = self._parse_optional_column_list_argument(return_columns) return_columns_requested = return_columns is not None and str(return_columns).strip().casefold() not in {'', '*', 'all', 'all_columns', 'all columns'} - row_frame = pandas.DataFrame(list(rows or [])) + row_frame = rows.copy() if isinstance(rows, pandas.DataFrame) else pandas.DataFrame(list(rows or [])) total_count = max(0, int(total_row_count or 0)) if row_frame.empty: @@ -872,6 +995,9 @@ def _filter_rows_across_sheets( max_rows: int = 100, start_row: int = 0, return_columns=None, + table_id: Optional[str] = None, + cell_range: Optional[str] = None, + header_row: Optional[int] = None, ) -> Optional[str]: """Search for matching rows across all sheets that contain the requested column. @@ -887,6 +1013,7 @@ def _filter_rows_across_sheets( return None combined_results = [] + parsed_return_columns = self._parse_optional_column_list_argument(return_columns) sheets_searched = [] sheets_matched = [] total_matches = 0 @@ -899,6 +1026,8 @@ def _filter_rows_across_sheets( container_name, blob_name, sheet_name=sheet, + require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._try_numeric_conversion(df) @@ -916,6 +1045,8 @@ def _filter_rows_across_sheets( additional_filter_value=additional_filter_value, normalize_match=normalize_match, ) + except TabularSavedValueError: + raise except (KeyError, ValueError): continue @@ -936,7 +1067,7 @@ def _filter_rows_across_sheets( if remaining_capacity > 0: filtered = filtered_df.iloc[skip_remaining:skip_remaining + remaining_capacity] skip_remaining = 0 - for row in filtered.to_dict(orient='records'): + for row in self._build_row_output_records(filtered, parsed_return_columns or list(filtered.columns)): row['_sheet'] = sheet combined_results.append(row) else: @@ -992,6 +1123,9 @@ def _search_rows_across_sheets( normalize_match: bool = False, max_rows: int = 100, start_row: int = 0, + table_id: Optional[str] = None, + cell_range: Optional[str] = None, + header_row: Optional[int] = None, ) -> Optional[str]: """Search rows across worksheets when the relevant text column is unknown or broad.""" workbook_metadata = self._get_workbook_metadata(container_name, blob_name) @@ -1021,6 +1155,8 @@ def _search_rows_across_sheets( container_name, blob_name, sheet_name=sheet, + require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._try_numeric_conversion(df) @@ -1040,7 +1176,7 @@ def _search_rows_across_sheets( continue except Exception as query_error: return json.dumps({ - 'error': f"Query/filter error: {query_error}", + **self._tabular_error_payload(query_error), 'filename': filename, 'selected_sheet': 'ALL (cross-sheet search)', }, indent=2, default=str) @@ -1061,7 +1197,7 @@ def _search_rows_across_sheets( continue except ValueError as search_error: return json.dumps({ - 'error': str(search_error), + **self._tabular_error_payload(search_error), 'filename': filename, 'selected_sheet': 'ALL (cross-sheet search)', }, indent=2, default=str) @@ -1152,6 +1288,9 @@ def _lookup_value_across_sheets( normalize_match: bool = False, max_rows: int = 25, start_row: int = 0, + table_id: Optional[str] = None, + cell_range: Optional[str] = None, + header_row: Optional[int] = None, ) -> Optional[str]: """Look up matching rows across all sheets that contain the lookup column. @@ -1180,12 +1319,15 @@ def _lookup_value_across_sheets( container_name, blob_name, sheet_name=sheet, + require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._try_numeric_conversion(df) if lookup_column not in df.columns: continue + self._require_saved_values(df, [lookup_column]) sheets_searched.append(sheet) try: mask = self._build_series_match_mask( @@ -1206,6 +1348,8 @@ def _lookup_value_across_sheets( if sheet_matches == 0: continue + if target_column and target_column in df.columns: + self._require_saved_values(df[mask], [target_column]) sheets_matched.append(sheet) total_matches += sheet_matches if skip_remaining >= sheet_matches: @@ -1216,16 +1360,20 @@ def _lookup_value_across_sheets( if remaining_capacity > 0: matched_df = df[mask].iloc[skip_remaining:skip_remaining + remaining_capacity] skip_remaining = 0 + matched_records = self._build_row_output_records(matched_df, list(matched_df.columns)) if target_column and target_column in df.columns: - for _, row in matched_df.iterrows(): - combined_results.append({ + for row in matched_records: + result = { '_sheet': sheet, lookup_column: row[lookup_column], target_column: row[target_column], - '_full_row': {str(k): v for k, v in row.to_dict().items()}, - }) + '_full_row': row, + } + if '_excel_source' in row: + result['_excel_source'] = row['_excel_source'] + combined_results.append(result) else: - for row in matched_df.to_dict(orient='records'): + for row in matched_records: row['_sheet'] = sheet combined_results.append(row) else: @@ -1269,6 +1417,9 @@ def _query_tabular_data_across_sheets( max_rows: int = 100, start_row: int = 0, return_columns=None, + table_id: Optional[str] = None, + cell_range: Optional[str] = None, + header_row: Optional[int] = None, ) -> Optional[str]: """Execute a pandas query expression across all sheets of a multi-sheet workbook. @@ -1284,20 +1435,26 @@ def _query_tabular_data_across_sheets( return None combined_results = [] + parsed_return_columns = self._parse_optional_column_list_argument(return_columns) sheets_searched = [] sheets_matched = [] total_matches = 0 query_errors = [] skip_remaining = max(0, int(start_row or 0)) requested_max_rows = max(1, int(max_rows or 100)) + source_row_offset = 0 for sheet in available_sheets: df = self._read_tabular_blob_to_dataframe( container_name, blob_name, sheet_name=sheet, + require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._try_numeric_conversion(df) + df.index = range(source_row_offset, source_row_offset + len(df)) + df.attrs['excel_index_offset'] = source_row_offset try: result_df, _ = self._apply_query_expression_with_fallback( @@ -1305,13 +1462,16 @@ def _query_tabular_data_across_sheets( query_expression=query_expression, normalize_match=False, ) + except TabularSavedValueError: + raise except Exception as query_error: query_errors.append({ 'sheet_name': sheet, - 'error': str(query_error), + 'error': self._tabular_error_payload(query_error)['error'], }) continue + source_row_offset += len(df) sheets_searched.append(sheet) sheet_matches = len(result_df) if sheet_matches == 0: @@ -1325,7 +1485,8 @@ def _query_tabular_data_across_sheets( remaining_capacity = max(0, requested_max_rows - len(combined_results)) if remaining_capacity > 0: - for row in result_df.iloc[skip_remaining:skip_remaining + remaining_capacity].to_dict(orient='records'): + page = result_df.iloc[skip_remaining:skip_remaining + remaining_capacity] + for row in self._build_row_output_records(page, parsed_return_columns or list(page.columns)): row['_sheet'] = sheet combined_results.append(row) skip_remaining = 0 @@ -1383,7 +1544,17 @@ def _query_tabular_data_across_sheets( "total_matches": total_matches, } response_payload.update(row_page_payload) - return json.dumps(response_payload, indent=2, default=str) + return PluginInvocationResult( + json.dumps(response_payload, indent=2, default=str), + internal_metadata=self._build_tabular_generated_export_internal_metadata( + container_name=container_name, blob_path=blob_name, filename=filename, + source_function='query_tabular_data', query_expression=query_expression, + return_columns=return_columns, expected_row_count=total_matches, + sheet_names=sheets_searched, + include_sheet_column=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), + ), + ) def _count_rows_across_sheets( self, @@ -1398,6 +1569,9 @@ def _count_rows_across_sheets( additional_filter_value=None, query_expression: Optional[str] = None, normalize_match: bool = False, + table_id: Optional[str] = None, + cell_range: Optional[str] = None, + header_row: Optional[int] = None, ) -> Optional[str]: """Count rows across all worksheets that satisfy optional filters or queries.""" workbook_metadata = self._get_workbook_metadata(container_name, blob_name) @@ -1419,6 +1593,8 @@ def _count_rows_across_sheets( container_name, blob_name, sheet_name=sheet, + require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._try_numeric_conversion(df) @@ -1436,11 +1612,13 @@ def _count_rows_across_sheets( ) except KeyError: continue + except TabularSavedValueError: + raise except Exception as query_error: if query_expression: query_errors.append({ 'sheet_name': sheet, - 'error': str(query_error), + 'error': self._tabular_error_payload(query_error)['error'], }) continue @@ -1502,6 +1680,9 @@ def _get_distinct_values_across_sheets( url_path_segments: Optional[int] = None, normalize_match: bool = False, max_values: int = 100, + table_id: Optional[str] = None, + cell_range: Optional[str] = None, + header_row: Optional[int] = None, ) -> Optional[str]: """Return distinct values for a column across all worksheets that contain it.""" workbook_metadata = self._get_workbook_metadata(container_name, blob_name) @@ -1525,6 +1706,8 @@ def _get_distinct_values_across_sheets( container_name, blob_name, sheet_name=sheet, + require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) if column not in df.columns: continue @@ -1543,16 +1726,19 @@ def _get_distinct_values_across_sheets( ) except KeyError: continue + except TabularSavedValueError: + raise except Exception as query_error: if query_expression: query_errors.append({ 'sheet_name': sheet, - 'error': str(query_error), + 'error': self._tabular_error_payload(query_error)['error'], }) continue sheets_searched.append(sheet) applied_filters = sheet_filters or applied_filters + self._require_saved_values(filtered_df, [column]) sheet_distinct_values, sheet_matched_cells, sheet_extracted_matches = self._collect_distinct_display_values( filtered_df[column], normalize_match=normalize_match, @@ -1641,34 +1827,49 @@ def _evaluate_related_value_membership( max_rows: int = 100, start_row: int = 0, return_columns=None, + source_table_id: Optional[str] = None, + source_cell_range: Optional[str] = None, + source_header_row: Optional[int] = None, + target_table_id: Optional[str] = None, + target_cell_range: Optional[str] = None, + target_header_row: Optional[int] = None, + source_location=None, + target_location=None, + include_rows: bool = True, ) -> dict: """Evaluate a semi-join between a source cohort and a target fact worksheet.""" - source_sheet, workbook_metadata = self._resolve_sheet_selection( - container_name, - blob_name, + source_container, source_blob = source_location or (container_name, blob_name) + target_container, target_blob = target_location or (container_name, blob_name) + source_sheet, source_workbook_metadata = self._resolve_sheet_selection( + source_container, + source_blob, sheet_name=source_sheet_name, sheet_index=source_sheet_index, require_explicit_sheet=True, + **self._table_selector_kwargs(source_table_id, source_cell_range, source_header_row), ) target_sheet, workbook_metadata = self._resolve_sheet_selection( - container_name, - blob_name, + target_container, + target_blob, sheet_name=target_sheet_name, sheet_index=target_sheet_index, require_explicit_sheet=True, + **self._table_selector_kwargs(target_table_id, target_cell_range, target_header_row), ) source_df = self._read_tabular_blob_to_dataframe( - container_name, - blob_name, + source_container, + source_blob, sheet_name=source_sheet, require_explicit_sheet=True, + **self._table_selector_kwargs(source_table_id, source_cell_range, source_header_row), ) target_df = self._read_tabular_blob_to_dataframe( - container_name, - blob_name, + target_container, + target_blob, sheet_name=target_sheet, require_explicit_sheet=True, + **self._table_selector_kwargs(target_table_id, target_cell_range, target_header_row), ) source_df = self._try_numeric_conversion(source_df) @@ -1689,10 +1890,10 @@ def _evaluate_related_value_membership( for required_column in source_required_columns: if required_column not in source_df.columns: return self._build_missing_column_error_payload( - container_name, - blob_name, + source_container, + source_blob, filename, - workbook_metadata, + source_workbook_metadata, source_sheet, required_column, related_columns=[source_value_column, source_alias_column, source_filter_column], @@ -1702,8 +1903,8 @@ def _evaluate_related_value_membership( for required_column in target_required_columns: if required_column not in target_df.columns: return self._build_missing_column_error_payload( - container_name, - blob_name, + target_container, + target_blob, filename, workbook_metadata, target_sheet, @@ -1724,10 +1925,10 @@ def _evaluate_related_value_membership( except KeyError as missing_source_column_error: missing_source_column = str(missing_source_column_error).strip("'") return self._build_missing_column_error_payload( - container_name, - blob_name, + source_container, + source_blob, filename, - workbook_metadata, + source_workbook_metadata, source_sheet, missing_source_column, related_columns=[source_value_column, source_alias_column], @@ -1735,9 +1936,9 @@ def _evaluate_related_value_membership( ) except Exception as source_filter_error: return { - 'error': f"Source filter/query error on sheet '{source_sheet}': {source_filter_error}", + **self._tabular_error_payload(source_filter_error), 'filename': filename, - 'selected_sheet': source_sheet if workbook_metadata.get('is_workbook') else None, + 'selected_sheet': source_sheet if source_workbook_metadata.get('is_workbook') else None, } try: @@ -1752,8 +1953,8 @@ def _evaluate_related_value_membership( except KeyError as missing_target_column_error: missing_target_column = str(missing_target_column_error).strip("'") return self._build_missing_column_error_payload( - container_name, - blob_name, + target_container, + target_blob, filename, workbook_metadata, target_sheet, @@ -1763,11 +1964,17 @@ def _evaluate_related_value_membership( ) except Exception as target_filter_error: return { - 'error': f"Target filter/query error on sheet '{target_sheet}': {target_filter_error}", + **self._tabular_error_payload(target_filter_error), 'filename': filename, 'selected_sheet': target_sheet if workbook_metadata.get('is_workbook') else None, } + self._require_saved_values( + filtered_source_df, [column for column in (source_value_column, source_alias_column) if column], + ) + self._require_saved_values( + filtered_target_df, [column for column in (target_match_column, target_alias_column) if column], + ) source_member_map = {} variant_to_source_keys = {} for _, source_row in filtered_source_df.iterrows(): @@ -1809,7 +2016,7 @@ def _evaluate_related_value_membership( matched_target_rows = [] matched_target_value_variants = set() - for _, target_row in filtered_target_df.iterrows(): + for target_index, target_row in filtered_target_df.iterrows(): target_value_variants = set() for column_name in [target_match_column, target_alias_column]: if not column_name: @@ -1837,12 +2044,16 @@ def _evaluate_related_value_membership( for source_key in matched_source_keys: source_member_map[source_key]['matched_target_row_count'] += 1 - matched_row_payload = target_row.to_dict() - matched_row_payload['_matched_on'] = matched_variants[:3] - matched_row_payload['_matched_source_values'] = [ - source_member_map[source_key]['display_value'] - for source_key in matched_source_keys[:3] - ] + matched_row_payload = None + if include_rows: + row_frame = filtered_target_df.loc[[target_index]] + selected_columns = self._parse_optional_column_list_argument(return_columns) or list(row_frame.columns) + matched_row_payload = self._build_row_output_records(row_frame, selected_columns)[0] + matched_row_payload['_matched_on'] = matched_variants[:3] + matched_row_payload['_matched_source_values'] = [ + source_member_map[source_key]['display_value'] + for source_key in matched_source_keys[:3] + ] matched_target_rows.append(matched_row_payload) matched_source_values = [] @@ -1867,7 +2078,7 @@ def _evaluate_related_value_membership( matched_target_row_count = len(matched_target_rows) start, limit = self._parse_row_page_arguments(start_row, max_rows) - paged_matched_target_rows = matched_target_rows[start:start + limit] + paged_matched_target_rows = matched_target_rows[start:start + limit] if include_rows else [] row_page_payload = self._build_tabular_row_page_payload( paged_matched_target_rows, matched_target_row_count, @@ -1879,6 +2090,11 @@ def _evaluate_related_value_membership( response_payload = { 'filename': filename, 'selected_sheet': target_sheet if workbook_metadata.get('is_workbook') else None, + **self._table_result_metadata(target_df), + 'source_selection': self._table_result_metadata(source_df), + 'target_selection': self._table_result_metadata(target_df), + 'source_filename': source_blob.rsplit('/', 1)[-1], + 'target_filename': target_blob.rsplit('/', 1)[-1], 'relationship_type': 'set_membership', 'source_sheet': source_sheet, 'source_value_column': source_value_column, @@ -2347,6 +2563,7 @@ def _search_dataframe_rows( else: resolved_search_columns = list(df.columns) + self._require_saved_values(df, resolved_search_columns) resolved_return_columns = [ column_name for column_name in (requested_return_columns or []) if column_name in df.columns @@ -2377,7 +2594,10 @@ def _search_dataframe_rows( pass paged_matched_df = matched_df.iloc[normalized_start_row:normalized_start_row + normalized_max_rows] - for row_index, row in paged_matched_df.iterrows(): + output_records = self._build_row_output_records( + paged_matched_df, resolved_return_columns or list(paged_matched_df.columns), + ) + for (row_index, row), row_payload in zip(paged_matched_df.iterrows(), output_records): row_matched_columns = [] for column_name in resolved_search_columns: if not bool(column_masks[column_name].loc[row_index]): @@ -2388,22 +2608,6 @@ def _search_dataframe_rows( seen_matched_columns.add(lowered_column) matched_columns.append(column_name) - if resolved_return_columns: - row_payload = { - column_name: row.get(column_name) - for column_name in resolved_return_columns - } - related_document_reference_values = self._build_related_document_reference_values( - row, - excluded_columns=resolved_return_columns, - ) - if related_document_reference_values: - row_payload['_related_document_reference_values'] = related_document_reference_values - else: - row_payload = { - str(key): value for key, value in row.to_dict().items() - } - row_payload['_matched_columns'] = row_matched_columns row_payload['_matched_values'] = { column_name: row.get(column_name) @@ -2697,6 +2901,7 @@ def _apply_query_expression_with_fallback( if not query_expression: return df, False + self._require_saved_query_values(df, query_expression) try: return df.query(query_expression), False except Exception as query_error: @@ -2768,6 +2973,7 @@ def _apply_optional_dataframe_filters( ) current_filter_operator = filter_spec['operator'] or 'equals' + self._require_saved_values(filtered_df, [current_filter_column]) mask = self._build_series_match_mask( filtered_df[current_filter_column], current_filter_operator, @@ -3033,7 +3239,11 @@ def _build_sheet_schema_summary(self, df: pandas.DataFrame, sheet_name: Optional 'null_counts': df.isnull().sum().to_dict(), } - def _build_workbook_schema_summary(self, container_name: str, blob_name: str, filename: str, preview_rows: int = 3) -> dict: + def _build_workbook_schema_summary( + self, container_name: str, blob_name: str, filename: str, preview_rows: int = 3, + sheet_name: Optional[str] = None, table_id: Optional[str] = None, + cell_range: Optional[str] = None, header_row: Optional[int] = None, + ) -> dict: """Build a workbook-aware schema summary for prompt preload and file description.""" workbook_metadata = self._get_workbook_metadata(container_name, blob_name) if not workbook_metadata.get('is_workbook'): @@ -3046,6 +3256,85 @@ def _build_workbook_schema_summary(self, container_name: str, blob_name: str, fi 'sheet_count': 0, }) return summary + if workbook_metadata.get('workbook_manifest'): + data = self._download_tabular_blob_bytes(container_name, blob_name) + extension = '.xls' if blob_name.lower().endswith('.xls') else '.xlsm' if blob_name.lower().endswith('.xlsm') else '.xlsx' + per_sheet_schemas, samples = {}, {} + with ExcelWorkbookReader(data, extension=extension) as reader: + reader.use_manifest(workbook_metadata['workbook_manifest']) + for name in workbook_metadata['sheet_names']: + if sheet_name and name != sheet_name: + continue + table_schemas = [] + tables = workbook_metadata['tables_by_sheet'].get(name, []) + if table_id or cell_range or header_row is not None: + tables = self._select_excel_tables( + reader, workbook_metadata, name, table_id, cell_range, header_row, + require_header=False, allow_heterogeneous=True, + ) + previews = reader.read_table_previews(tables, max_rows=max(1, min(preview_rows, 20))) + for table in tables: + preview_records, preview_bytes, text_truncated = [], 0, False + for record in previews[table['id']]: + values = [] + for value in record['values'][:EXCEL_PREVIEW_MAX_COLUMNS]: + if isinstance(value, str) and len(value) > 4000: + value = value[:4000] + ' [preview shortened]' + text_truncated = True + values.append(value) + row_bytes = len(json.dumps(values, ensure_ascii=False, default=str).encode('utf-8')) + if preview_bytes + row_bytes > EXCEL_PREVIEW_MAX_BYTES: + break + preview_records.append({'row': record['row'], 'values': values}) + preview_bytes += row_bytes + columns = [column['name'] for column in table['columns'][:EXCEL_PREVIEW_MAX_COLUMNS]] + sample = pandas.DataFrame([record['values'] for record in preview_records], columns=columns) + schema = self._build_sheet_schema_summary(sample, name, preview_rows=preview_rows) + schema.update({ + 'row_count': table['row_count'], 'table_id': table['id'], + 'table_name': table['name'], 'table_range': table['range'], + 'header_row': table['header_row'], 'header_status': table['header_status'], + 'origin': table['origin'], 'null_counts_scope': 'preview', + 'selected_table': table['id'], + 'preview_row_numbers': [record['row'] for record in preview_records], + 'columns_truncated': len(table['columns']) > len(columns), + 'cell_text_truncated': text_truncated, + 'preview_limited': len(preview_records) < table['row_count'], + 'selection_required': table['header_status'] == 'ambiguous', + }) + table_schemas.append(schema) + if len(workbook_metadata['tables_by_sheet'].get(name, [])) == 1: + samples[name] = sample + per_sheet_schemas[name] = ( + {**table_schemas[0], 'table_count': 1, 'tables': table_schemas} + if len(table_schemas) == 1 else { + 'selected_sheet': name, 'table_count': len(table_schemas), + 'tables': table_schemas, 'columns': [], + 'selection_required': len(table_schemas) > 1 or any( + table['selection_required'] for table in table_schemas + ), + } + ) + relationship_metadata = self._build_workbook_relationship_metadata(samples) + summary = { + 'filename': filename, 'is_workbook': True, + 'sheet_names': workbook_metadata['sheet_names'], + 'sheet_count': workbook_metadata['sheet_count'], + 'table_count': workbook_metadata['table_count'], 'selected_sheet': sheet_name, + 'per_sheet_schemas': per_sheet_schemas, + 'sheet_role_hints': relationship_metadata.get('sheet_role_hints', {}), + 'relationship_hints': relationship_metadata.get('relationship_hints', []), + 'relationship_hints_scope': 'bounded table previews, not full-data validation', + 'table_choices': self._table_choices(workbook_metadata, sheet_name), + 'selection_required': any( + schema.get('selection_required') for schema in per_sheet_schemas.values() + ), + } + if len(per_sheet_schemas) == 1: + sheet_schema = next(iter(per_sheet_schemas.values())) + if sheet_schema.get('table_count') == 1: + summary.update(sheet_schema) + return summary per_sheet_schemas = {} sheet_dataframes = {} @@ -3105,12 +3394,17 @@ def _find_candidate_sheets_for_columns( if normalized_exclude_sheet and sheet_name.lower() == normalized_exclude_sheet: continue - dataframe = self._read_tabular_blob_to_dataframe( - container_name, - blob_name, - sheet_name=sheet_name, - ) - normalized_columns = {str(column).strip().lower() for column in dataframe.columns} + if workbook_metadata.get('workbook_manifest'): + normalized_columns = { + str(column['name']).strip().lower() + for table in workbook_metadata.get('tables_by_sheet', {}).get(sheet_name, []) + for column in table['columns'] + } + else: + dataframe = self._read_tabular_blob_to_dataframe( + container_name, blob_name, sheet_name=sheet_name, + ) + normalized_columns = {str(column).strip().lower() for column in dataframe.columns} matched_columns = [ target_column for target_column in normalized_targets if target_column in normalized_columns @@ -3142,6 +3436,11 @@ def _build_missing_column_error_payload( 'missing_column': missing_column, 'selected_sheet': selected_sheet if workbook_metadata.get('is_workbook') else None, } + if workbook_metadata.get('workbook_manifest'): + payload['candidate_tables'] = [ + table for table in self._table_choices(workbook_metadata) + if any(str(column['name']).casefold() == str(missing_column).casefold() for column in table['columns']) + ] if workbook_metadata.get('is_workbook') and workbook_metadata.get('sheet_count', 0) > 1: candidate_sheets = self._find_candidate_sheets_for_columns( @@ -3159,6 +3458,94 @@ def _build_missing_column_error_payload( return payload + @staticmethod + def _normalize_table_selectors(table_id=None, cell_range=None, header_row=None): + table_id = str(table_id or '').strip() or None + if table_id and table_id.casefold() == 'all': + table_id = 'all' + cell_range = str(cell_range or '').strip() or None + if table_id and cell_range: + raise TabularSelectionError( + "Specify table_id or cell_range, not both.", code='table_selection_invalid', + ) + if header_row is not None: + if isinstance(header_row, bool) or not str(header_row).isascii() or not str(header_row).isdigit(): + raise TabularSelectionError( + "header_row must be an actual worksheet row number or zero.", + code='header_selection_invalid', + ) + header_row = int(header_row) + if cell_range and header_row is None: + raise TabularSelectionError( + "Specify header_row for an explicit range; zero means no header.", + code='header_selection_required', + ) + return table_id, cell_range, header_row + + def _table_selector_kwargs(self, table_id=None, cell_range=None, header_row=None): + normalized = self._normalize_table_selectors(table_id, cell_range, header_row) + return { + key: value for key, value in zip(('table_id', 'cell_range', 'header_row'), normalized) + if value is not None + } + + def _select_excel_tables( + self, reader, workbook_metadata, sheet_name, table_id=None, cell_range=None, + header_row=None, *, allow_heterogeneous=False, require_header=True, + ): + """Resolve explicit choices identically for foreground analysis and replay.""" + table_id, cell_range, header_row = self._normalize_table_selectors(table_id, cell_range, header_row) + available = workbook_metadata.get('tables_by_sheet', {}).get(sheet_name, []) + choices = self._table_choices(workbook_metadata, sheet_name) + if not available and not cell_range and table_id is None: + return [] + if table_id == 'all': + if cell_range or header_row is not None: + raise TabularSelectionError( + "Select a single table or range for an explicit header override.", + code='table_selection_invalid', choices=choices, + ) + selected = [reader.select_table(table['id'], sheet_name) for table in available] + else: + if table_id is None and not cell_range and len(available) != 1: + raise TabularSelectionError( + "This worksheet contains multiple tables. Select table_id, or an explicit cell_range and header_row.", + choices=choices, + ) + try: + selected = [reader.select_table( + table_id=table_id, sheet_name=sheet_name, cell_range=cell_range, header_row=header_row, + )] + except ValueError as error: + raise TabularSelectionError( + "The table, worksheet, range, or header selection is invalid. Choose a discovered table or a valid range.", + code='table_selection_invalid', choices=choices, + ) from error + if require_header and any(table['header_status'] == 'ambiguous' for table in selected): + raise TabularSelectionError( + "The table header is ambiguous. Supply header_row using the actual worksheet row, or zero for no header.", + code='header_selection_required', choices=choices, + ) + if not allow_heterogeneous and selected: + columns = [column['name'] for column in selected[0]['columns']] + if any([column['name'] for column in table['columns']] != columns for table in selected): + raise TabularSelectionError( + "The selected tables have different schemas. Select one table for this analytical operation.", + code='table_schema_mismatch', choices=choices, + ) + return selected + + @staticmethod + def _table_result_metadata(dataframe): + table = dataframe.attrs.get('excel_table') + if table is None: + return {} + return { + 'selected_table': table['id'], 'table_id': table['id'], + 'table_name': table.get('name'), 'table_range': table['range'], + 'header_row': table.get('header_row'), 'header_status': table.get('header_status'), + } + def _read_tabular_blob_to_dataframe( self, container_name: str, @@ -3166,21 +3553,36 @@ def _read_tabular_blob_to_dataframe( sheet_name: Optional[str] = None, sheet_index: Optional[str] = None, require_explicit_sheet: bool = False, + table_id: Optional[str] = None, + cell_range: Optional[str] = None, + header_row: Optional[int] = None, ) -> pandas.DataFrame: """Download a blob and read it into a pandas DataFrame. Uses per-instance cache.""" self._assert_blob_screening_access(container_name, blob_name) + table_id, cell_range, header_row = self._normalize_table_selectors(table_id, cell_range, header_row) resolved_sheet_name, workbook_metadata = self._resolve_sheet_selection( container_name, blob_name, sheet_name=sheet_name, sheet_index=sheet_index, require_explicit_sheet=require_explicit_sheet, + table_id=table_id, ) sheet_cache_key = resolved_sheet_name or '__default__' - cache_key = (container_name, blob_name, sheet_cache_key) + if not workbook_metadata.get('is_workbook') and any( + value is not None for value in (table_id, cell_range, header_row) + ): + raise TabularSelectionError( + "Table, range, and header selectors apply only to Excel workbooks.", + code='table_selection_invalid', + ) + cache_key = ( + container_name, blob_name, workbook_metadata.get('blob_etag'), + sheet_cache_key, table_id, cell_range, header_row, + ) if cache_key in self._df_cache: log_event( - f"[TabularProcessingPlugin] Cache hit for {blob_name}" + f"[TABULAR_PROCESSING_PLUGIN] Cache hit for {blob_name}" + (f" [{resolved_sheet_name}]" if resolved_sheet_name else ''), level=logging.DEBUG, ) @@ -3191,22 +3593,42 @@ def _read_tabular_blob_to_dataframe( name_lower = blob_name.lower() if name_lower.endswith('.csv'): df = read_tabular_csv(io.BytesIO(data), keep_default_na=False, dtype=str) - elif name_lower.endswith('.xlsx') or name_lower.endswith('.xlsm'): - df = pandas.read_excel( - io.BytesIO(data), - engine='openpyxl', - keep_default_na=False, - dtype=str, - sheet_name=resolved_sheet_name, - ) - elif name_lower.endswith('.xls'): - df = pandas.read_excel( - io.BytesIO(data), - engine='xlrd', - keep_default_na=False, - dtype=str, - sheet_name=resolved_sheet_name, - ) + elif self._get_excel_engine(blob_name): + extension = '.xls' if name_lower.endswith('.xls') else '.xlsm' if name_lower.endswith('.xlsm') else '.xlsx' + with ExcelWorkbookReader(data, extension=extension) as reader: + reader.use_manifest(workbook_metadata['workbook_manifest']) + selected_tables = self._select_excel_tables( + reader, workbook_metadata, resolved_sheet_name, table_id, cell_range, header_row, + ) + if not selected_tables: + df = pandas.DataFrame() + else: + columns = [column['name'] for column in selected_tables[0]['columns']] + rows, source_rows, source_tables, missing = [], [], [], {} + for table in selected_tables: + for row_number, cells in reader.iter_table_rows(table): + ordinal = len(rows) + rows.append([ + '' if cell['value'] is None else str(cell['value']) + for cell in cells + ]) + source_rows.append(row_number) + source_tables.append(table['id']) + for column, cell in zip(columns, cells): + if cell['value_status'] == 'cache_missing': + missing.setdefault(column, set()).add(ordinal) + df = pandas.DataFrame(rows, columns=columns) + table_context = selected_tables[0] if len(selected_tables) == 1 else { + 'id': 'all', 'sheet': resolved_sheet_name, 'range': 'multiple selected tables', + } + df.attrs.update({ + 'excel_table': table_context, 'excel_row_numbers': tuple(source_rows), + 'excel_row_table_ids': tuple(source_tables), + 'excel_tables': copy.deepcopy(selected_tables), + 'excel_table_ranges': {table['id']: table['range'] for table in selected_tables}, + 'excel_missing_formula_caches': missing, + }) + self._last_table_reads[(container_name, blob_name, resolved_sheet_name)] = copy.deepcopy(selected_tables) else: raise ValueError(f"Unsupported tabular file type: {blob_name}") @@ -3214,13 +3636,42 @@ def _read_tabular_blob_to_dataframe( self._assert_blob_screening_access(container_name, blob_name) self._df_cache[cache_key] = df log_event( - f"[TabularProcessingPlugin] Cached DataFrame for {blob_name}" + f"[TABULAR_PROCESSING_PLUGIN] Cached DataFrame for {blob_name}" + (f" [{resolved_sheet_name}]" if resolved_sheet_name else '') + f" ({len(df)} rows)", level=logging.DEBUG, ) return df.copy() + @staticmethod + def _require_saved_values(dataframe, columns): + missing = dataframe.attrs.get('excel_missing_formula_caches') or {} + if not missing: + return + offset = dataframe.attrs.get('excel_index_offset', 0) + selected_indices = {int(index) - offset for index in dataframe.index} + for column in columns: + if selected_indices.intersection(missing.get(column, ())): + raise TabularSavedValueError() + + def _require_saved_query_values(self, dataframe, query_expression): + """Ignore quoted literals and check the columns actually read by the query.""" + if not query_expression or not dataframe.attrs.get('excel_missing_formula_caches'): + return + expression = str(query_expression) + referenced = set(re.findall(r'`([^`]+)`', expression)) + expression = re.sub(r'`[^`]+`', ' ', expression) + try: + referenced.update( + token.string for token in tokenize.generate_tokens(io.StringIO(expression).readline) + if token.type == tokenize.NAME + ) + except (tokenize.TokenError, IndentationError): + referenced.update(dataframe.columns) + self._require_saved_values( + dataframe, [column for column in dataframe.columns if str(column) in referenced], + ) + def _try_numeric_conversion(self, df: pandas.DataFrame) -> pandas.DataFrame: """Attempt to convert string columns to numeric where possible.""" for col in df.columns: @@ -3485,9 +3936,13 @@ def _resolve_blob_location_with_fallback(self, user_id: str, conversation_id: st override = self._get_resolved_blob_location_override(source, filename) if override and self._is_authorized_blob_location(override[0], override[1], authorized_context): - override = resolve_available_blob_location(*override, user_id=user_id) - self._assert_blob_screening_access(*override) - return override + try: + resolved_override = resolve_available_blob_location(*override, user_id=user_id) + self._assert_blob_screening_access(*resolved_override) + return resolved_override + except (ScreeningError, PermissionError): + self._evict_screened_blob_cache(*override) + raise attempts = [] @@ -3518,6 +3973,7 @@ def _resolve_blob_location_with_fallback(self, user_id: str, conversation_id: st log_event(f"[TABULAR_PROCESSING_PLUGIN] Found blob at {container}/{blob_path}", level=logging.DEBUG) return container, blob_path except (ScreeningError, PermissionError): + self._evict_screened_blob_cache(container, blob_path) raise except Exception: continue @@ -3538,8 +3994,13 @@ def build_generated_export_query_descriptor( group_id: Optional[str] = None, public_workspace_id: Optional[str] = None, expected_row_count: int = 0, + sheet_name: Optional[str] = None, + sheet_index: Optional[str] = None, + table_id: Optional[str] = None, + cell_range: Optional[str] = None, + header_row: Optional[int] = None, ) -> dict: - """Resolve an authorized, version-pinned CSV query for durable export replay.""" + """Resolve an authorized, version-pinned table query for durable export replay.""" container_name, blob_path = self._resolve_blob_location_with_fallback( user_id, conversation_id, @@ -3548,6 +4009,11 @@ def build_generated_export_query_descriptor( group_id=group_id, public_workspace_id=public_workspace_id, ) + selected_sheet, _ = self._resolve_sheet_selection( + container_name, blob_path, sheet_name=sheet_name, sheet_index=sheet_index, + require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), + ) return self._build_generated_export_query_descriptor_from_location( container_name=container_name, blob_path=blob_path, @@ -3555,6 +4021,9 @@ def build_generated_export_query_descriptor( query_expression=query_expression, return_columns=return_columns, expected_row_count=expected_row_count, + selected_sheet=selected_sheet, + sheet_names=[selected_sheet] if selected_sheet else None, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) def _build_generated_export_query_descriptor_from_location( @@ -3569,6 +4038,11 @@ def _build_generated_export_query_descriptor_from_location( source_function: str = 'query_tabular_data', selected_sheet: Optional[str] = None, sheet_names: Optional[List[str]] = None, + table_id: Optional[str] = None, + cell_range: Optional[str] = None, + header_row: Optional[int] = None, + include_sheet_column: bool = False, + numeric_conversion_scope: Optional[str] = None, ) -> dict: """Pin a durable query descriptor to an already-authorized blob location.""" document, _user_id = self._assert_blob_screening_access(container_name, blob_path) @@ -3582,7 +4056,11 @@ def _build_generated_export_query_descriptor_from_location( raise ValueError('Durable source-backed generated exports require a supported tabular source') normalized_query_expression = validate_tabular_csv_query_expression(query_expression) - blob_version = dict(blob_version or self._get_tabular_blob_version(container_name, blob_path)) + table_id, cell_range, header_row = self._normalize_table_selectors(table_id, cell_range, header_row) + current_version = self._get_tabular_blob_version(container_name, blob_path, refresh=True) + if blob_version and blob_version.get('blob_etag') != current_version.get('blob_etag'): + raise ScreeningConflictError() + blob_version = dict(blob_version or current_version) blob_etag = blob_version.get('blob_etag') blob_size = blob_version.get('blob_size') @@ -3617,10 +4095,71 @@ def _build_generated_export_query_descriptor_from_location( if document is not None: descriptor[PROVENANCE_FIELD] = document_provenance(document) if source_format != 'csv': + metadata = self._get_workbook_metadata(container_name, blob_path) + if metadata.get('blob_etag') != blob_etag: + raise ScreeningConflictError() if not normalized_sheet_names: - raise ValueError('Durable workbook replay requires an explicit worksheet scope') + resolved_sheet, _ = self._resolve_sheet_selection( + container_name, blob_path, sheet_name=selected_sheet, + require_explicit_sheet=True, table_id=table_id, + cell_range=cell_range, header_row=header_row, + ) + normalized_sheet_names = [resolved_sheet] + normalized_sheet_names = [ + self._resolve_sheet_selection( + container_name, blob_path, sheet_name=name, require_explicit_sheet=True, + )[0] for name in normalized_sheet_names + ] + if len(set(normalized_sheet_names)) != len(normalized_sheet_names): + raise TabularSelectionError( + "A worksheet may only appear once in a durable table selection.", + code='table_selection_invalid', + ) + data = self._download_tabular_blob_bytes(container_name, blob_path) + selections = [] + with ExcelWorkbookReader(data, extension=f'.{source_format}') as reader: + reader.use_manifest(metadata['workbook_manifest']) + for name in normalized_sheet_names: + selections.extend(self._select_excel_tables( + reader, metadata, name, table_id, cell_range, header_row, + allow_heterogeneous=True, + )) + if numeric_conversion_scope is None: + schemas_by_sheet = {} + for table in selections: + schemas_by_sheet.setdefault(table['sheet'], set()).add( + tuple(column['name'] for column in table['columns']) + ) + numeric_conversion_scope = ( + 'sheet' if table_id == 'all' and all(len(schemas) == 1 for schemas in schemas_by_sheet.values()) + else 'table' + ) + if numeric_conversion_scope not in ('table', 'sheet'): + raise ValueError('The durable numeric conversion scope is invalid.') + descriptor.update({ + 'version': 2, 'selection_semantics': 'excel-tables-v2', + 'workbook_schema_version': EXCEL_WORKBOOK_SCHEMA_VERSION, + 'source_sha256': hashlib.sha256(data).hexdigest(), + 'table_id': table_id, 'cell_range': cell_range, 'header_row': header_row, + 'selected_table': selections[0]['id'] if len(selections) == 1 else table_id, + 'table_selections': copy.deepcopy(selections), + 'source_row_count': sum(table['row_count'] for table in selections), + 'numeric_conversion_scope': numeric_conversion_scope, + }) descriptor['selected_sheet'] = normalized_selected_sheet descriptor['sheet_names'] = normalized_sheet_names + if normalized_selected_sheet is not None: + descriptor['selected_sheet'] = self._match_workbook_sheet_name( + normalized_selected_sheet, normalized_sheet_names, + ) + descriptor['include_sheet_column'] = bool(include_sheet_column) + if self._get_tabular_blob_version(container_name, blob_path, refresh=True)['blob_etag'] != blob_etag: + raise ScreeningConflictError() + elif any(value is not None for value in (table_id, cell_range, header_row)): + raise TabularSelectionError( + "Table, range, and header selectors apply only to Excel workbooks.", + code='table_selection_invalid', + ) return descriptor def _build_generated_export_column_reference(self, column_name: str) -> str: @@ -3788,6 +4327,12 @@ def _build_tabular_generated_export_internal_metadata( return_columns: Optional[str], expected_row_count: int, replay_error: Optional[str] = None, + selected_sheet: Optional[str] = None, + sheet_names: Optional[List[str]] = None, + table_id: Optional[str] = None, + cell_range: Optional[str] = None, + header_row: Optional[int] = None, + include_sheet_column: bool = False, ) -> dict: """Build source authorization plus either a replay descriptor or a safe error.""" blob_version = self._get_tabular_blob_version(container_name, blob_path) @@ -3799,11 +4344,13 @@ def _build_tabular_generated_export_internal_metadata( ), } if replay_error: - internal_metadata['tabular_generated_export_source_error'] = str(replay_error) + internal_metadata['tabular_generated_export_source_error'] = ( + 'This operation cannot be replayed as a bounded row-local query.' + ) return internal_metadata - internal_metadata['tabular_generated_export_source'] = ( - self._build_generated_export_query_descriptor_from_location( + try: + internal_metadata['tabular_generated_export_source'] = self._build_generated_export_query_descriptor_from_location( container_name=container_name, blob_path=blob_path, filename=filename, @@ -3812,8 +4359,22 @@ def _build_tabular_generated_export_internal_metadata( expected_row_count=expected_row_count, blob_version=blob_version, source_function=source_function, + selected_sheet=selected_sheet, sheet_names=sheet_names, + include_sheet_column=include_sheet_column, + numeric_conversion_scope='sheet' if table_id == 'all' else 'table', + **self._table_selector_kwargs(table_id, cell_range, header_row), + ) + except TabularSelectionError: + raise + except ValueError as error: + log_event( + '[TABULAR_PROCESSING_PLUGIN] Query is not available for bounded replay.', + level=logging.DEBUG, debug_only=True, + extra={'exception_type': type(error).__name__}, + ) + internal_metadata['tabular_generated_export_source_error'] = ( + 'This operation cannot be replayed as a bounded row-local query.' ) - ) return internal_metadata def _build_source_authorization_from_location( @@ -3963,7 +4524,7 @@ async def list_tabular_files( 'requested_public_workspace_id': public_workspace_id, }, ) - return json.dumps({"error": str(exc)}) + return json.dumps(self._tabular_error_payload(exc)) user_id = authorized_context['user_id'] conversation_id = authorized_context['conversation_id'] @@ -3990,6 +4551,8 @@ def _sync_work(): "container": storage_account_user_documents_container_name, "sheet_names": workbook_metadata.get('sheet_names', []), "sheet_count": workbook_metadata.get('sheet_count', 0), + "table_count": workbook_metadata.get('table_count', 0), + "table_choices": self._table_choices(workbook_metadata), }) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error listing workspace blobs: {e}", level=logging.WARNING) @@ -4012,6 +4575,8 @@ def _sync_work(): "container": storage_account_personal_chat_container_name, "sheet_names": workbook_metadata.get('sheet_names', []), "sheet_count": workbook_metadata.get('sheet_count', 0), + "table_count": workbook_metadata.get('table_count', 0), + "table_choices": self._table_choices(workbook_metadata), }) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error listing chat blobs: {e}", level=logging.WARNING) @@ -4035,6 +4600,8 @@ def _sync_work(): "container": storage_account_group_documents_container_name, "sheet_names": workbook_metadata.get('sheet_names', []), "sheet_count": workbook_metadata.get('sheet_count', 0), + "table_count": workbook_metadata.get('table_count', 0), + "table_choices": self._table_choices(workbook_metadata), }) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error listing group blobs: {e}", level=logging.WARNING) @@ -4058,6 +4625,8 @@ def _sync_work(): "container": storage_account_public_documents_container_name, "sheet_names": workbook_metadata.get('sheet_names', []), "sheet_count": workbook_metadata.get('sheet_count', 0), + "table_count": workbook_metadata.get('table_count', 0), + "table_choices": self._table_choices(workbook_metadata), }) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error listing public blobs: {e}", level=logging.WARNING) @@ -4083,6 +4652,9 @@ async def describe_tabular_file( source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, + table_id: Annotated[str, "Optional discovered Excel table ID or name. Omit to discover all tables; 'all' explicitly selects same-schema tables."] = None, + cell_range: Annotated[str, "Optional Excel range such as A3:D100. Use instead of table_id and supply header_row."] = None, + header_row: Annotated[str, "Actual worksheet header row; zero means no header. Required for an explicit range or ambiguous header."] = None, ) -> Annotated[str, "JSON summary of the tabular file"]: """Get schema and preview of a tabular file.""" def _sync_work(): @@ -4093,7 +4665,19 @@ def _sync_work(): ) workbook_metadata = self._get_workbook_metadata(container, blob_path) - if workbook_metadata.get('is_workbook') and workbook_metadata.get('sheet_count', 0) > 1 and not (sheet_name or sheet_index): + if workbook_metadata.get('workbook_manifest'): + selected_sheet = None + if sheet_name or sheet_index not in (None, '') or table_id not in (None, '', 'all') or cell_range or header_row is not None: + selected_sheet, _ = self._resolve_sheet_selection( + container, blob_path, sheet_name=sheet_name, sheet_index=sheet_index, + require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), + ) + summary = self._build_workbook_schema_summary( + container, blob_path, filename, preview_rows=5, sheet_name=selected_sheet, + **self._table_selector_kwargs(table_id, cell_range, header_row), + ) + elif workbook_metadata.get('is_workbook') and workbook_metadata.get('sheet_count', 0) > 1 and not (sheet_name or sheet_index): summary = self._build_workbook_schema_summary( container, blob_path, @@ -4107,12 +4691,14 @@ def _sync_work(): sheet_name=sheet_name, sheet_index=sheet_index, require_explicit_sheet=False, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._read_tabular_blob_to_dataframe( container, blob_path, sheet_name=selected_sheet, require_explicit_sheet=False, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) summary = self._build_sheet_schema_summary(df, selected_sheet, preview_rows=5) summary.update({ @@ -4125,7 +4711,7 @@ def _sync_work(): return json.dumps(summary, indent=2, default=str) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error describing file: {e}", level=logging.WARNING) - return json.dumps({"error": str(e)}) + return json.dumps(self._tabular_error_payload(e)) return await asyncio.to_thread(_sync_work) @kernel_function( @@ -4153,6 +4739,9 @@ async def lookup_value( max_rows: Annotated[str, "Maximum matching rows to return"] = "25", group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, + table_id: Annotated[str, "Optional discovered Excel table ID or name; 'all' explicitly selects same-schema tables. Required when a worksheet has multiple tables."] = None, + cell_range: Annotated[str, "Optional Excel range such as A3:D100. Use instead of table_id and supply header_row."] = None, + header_row: Annotated[str, "Actual worksheet header row; zero means no header. Required for an explicit range or ambiguous header."] = None, ) -> Annotated[str, "JSON result containing matching rows and target-column values"]: """Look up values from a target column for matching rows.""" def _sync_work(): @@ -4165,7 +4754,7 @@ def _sync_work(): # When no explicit sheet_name is given, try cross-sheet search first normalized_sheet = (sheet_name or '').strip() normalized_sheet_idx = None if sheet_index is None else str(sheet_index).strip() - if not normalized_sheet and normalized_sheet_idx in (None, ''): + if not normalized_sheet and normalized_sheet_idx in (None, '') and table_id in (None, '', 'all') and not cell_range and header_row is None: cross_sheet_result = self._lookup_value_across_sheets( container, blob_path, filename, lookup_column, lookup_value, target_column, @@ -4173,6 +4762,7 @@ def _sync_work(): normalize_match=normalize_match_flag, max_rows=int(max_rows), start_row=int(start_row), + **self._table_selector_kwargs(table_id, cell_range, header_row), ) if cross_sheet_result is not None: return cross_sheet_result @@ -4182,12 +4772,14 @@ def _sync_work(): sheet_name=sheet_name, sheet_index=sheet_index, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._read_tabular_blob_to_dataframe( container, blob_path, sheet_name=selected_sheet, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._try_numeric_conversion(df) @@ -4221,6 +4813,7 @@ def _sync_work(): operator = (match_operator or 'equals').strip().lower() normalized_lookup_value = str(lookup_value) + self._require_saved_values(df, [lookup_column]) try: mask = self._build_series_match_mask( df[lookup_column], @@ -4233,10 +4826,12 @@ def _sync_work(): start, limit = self._parse_row_page_arguments(start_row, max_rows, default_max_rows=25) matched_df = df[mask] + self._require_saved_values(matched_df, [target_column]) matches = matched_df.iloc[start:start + limit] response = { "filename": filename, "selected_sheet": selected_sheet if workbook_metadata.get('is_workbook') else None, + **self._table_result_metadata(df), "lookup_column": lookup_column, "lookup_value": lookup_value, "target_column": target_column, @@ -4245,7 +4840,7 @@ def _sync_work(): "total_matches": len(matched_df), } response.update(self._build_tabular_row_page_payload( - matches.to_dict(orient='records'), + matches, len(matched_df), start_row=start, max_rows=limit, @@ -4257,7 +4852,7 @@ def _sync_work(): return json.dumps(response, indent=2, default=str) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error looking up value: {e}", level=logging.WARNING) - return json.dumps({"error": str(e)}) + return json.dumps(self._tabular_error_payload(e)) return await asyncio.to_thread(_sync_work) @kernel_function( @@ -4291,6 +4886,9 @@ async def get_distinct_values( max_values: Annotated[str, "Maximum distinct values to return"] = "100", group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, + table_id: Annotated[str, "Optional discovered Excel table ID or name; 'all' explicitly selects same-schema tables. Required when a worksheet has multiple tables."] = None, + cell_range: Annotated[str, "Optional Excel range such as A3:D100. Use instead of table_id and supply header_row."] = None, + header_row: Annotated[str, "Actual worksheet header row; zero means no header. Required for an explicit range or ambiguous header."] = None, ) -> Annotated[str, "JSON result containing deterministic distinct values and counts"]: """Return deterministic distinct values from a worksheet or across worksheets.""" def _sync_work(): @@ -4308,7 +4906,7 @@ def _sync_work(): normalized_sheet = (sheet_name or '').strip() normalized_sheet_idx = None if sheet_index is None else str(sheet_index).strip() - if not normalized_sheet and normalized_sheet_idx in (None, ''): + if not normalized_sheet and normalized_sheet_idx in (None, '') and table_id in (None, '', 'all') and not cell_range and header_row is None: cross_sheet_result = self._get_distinct_values_across_sheets( container, blob_path, @@ -4326,6 +4924,7 @@ def _sync_work(): url_path_segments=parsed_url_path_segments, normalize_match=normalize_match_flag, max_values=int(max_values), + **self._table_selector_kwargs(table_id, cell_range, header_row), ) if cross_sheet_result is not None: return cross_sheet_result @@ -4336,12 +4935,14 @@ def _sync_work(): sheet_name=sheet_name, sheet_index=sheet_index, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._read_tabular_blob_to_dataframe( container, blob_path, sheet_name=selected_sheet, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) if column not in df.columns: @@ -4392,11 +4993,12 @@ def _sync_work(): ) except Exception as query_error: return json.dumps({ - 'error': f"Query/filter error: {query_error}", + **self._tabular_error_payload(query_error), 'filename': filename, 'selected_sheet': selected_sheet if workbook_metadata.get('is_workbook') else None, }) + self._require_saved_values(filtered_df, [column]) distinct_display_values, matched_cell_count, extracted_match_count = self._collect_distinct_display_values( filtered_df[column], normalize_match=normalize_match_flag, @@ -4410,6 +5012,7 @@ def _sync_work(): response_payload = { 'filename': filename, 'selected_sheet': selected_sheet if workbook_metadata.get('is_workbook') else None, + **self._table_result_metadata(df), 'column': column, 'filter_applied': applied_filters, 'normalize_match': normalize_match_flag, @@ -4429,7 +5032,7 @@ def _sync_work(): return json.dumps(response_payload, indent=2, default=str) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error getting distinct values: {e}", level=logging.WARNING) - return json.dumps({"error": str(e)}) + return json.dumps(self._tabular_error_payload(e)) return await asyncio.to_thread(_sync_work) @@ -4459,6 +5062,9 @@ async def count_rows( source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, + table_id: Annotated[str, "Optional discovered Excel table ID or name; 'all' explicitly selects same-schema tables. Required when a worksheet has multiple tables."] = None, + cell_range: Annotated[str, "Optional Excel range such as A3:D100. Use instead of table_id and supply header_row."] = None, + header_row: Annotated[str, "Actual worksheet header row; zero means no header. Required for an explicit range or ambiguous header."] = None, ) -> Annotated[str, "JSON result containing a deterministic row count"]: """Count rows deterministically after optional filters or queries.""" def _sync_work(): @@ -4471,7 +5077,7 @@ def _sync_work(): normalized_sheet = (sheet_name or '').strip() normalized_sheet_idx = None if sheet_index is None else str(sheet_index).strip() - if not normalized_sheet and normalized_sheet_idx in (None, ''): + if not normalized_sheet and normalized_sheet_idx in (None, '') and table_id in (None, '', 'all') and not cell_range and header_row is None: cross_sheet_result = self._count_rows_across_sheets( container, blob_path, @@ -4484,6 +5090,7 @@ def _sync_work(): additional_filter_value=additional_filter_value, query_expression=query_expression, normalize_match=normalize_match_flag, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) if cross_sheet_result is not None: return cross_sheet_result @@ -4494,12 +5101,14 @@ def _sync_work(): sheet_name=sheet_name, sheet_index=sheet_index, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._read_tabular_blob_to_dataframe( container, blob_path, sheet_name=selected_sheet, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._try_numeric_conversion(df) @@ -4534,14 +5143,16 @@ def _sync_work(): ) except Exception as query_error: return json.dumps({ - 'error': f"Query/filter error: {query_error}", + **self._tabular_error_payload(query_error), 'filename': filename, 'selected_sheet': selected_sheet if workbook_metadata.get('is_workbook') else None, + **self._table_result_metadata(df), }) return json.dumps({ 'filename': filename, 'selected_sheet': selected_sheet if workbook_metadata.get('is_workbook') else None, + **self._table_result_metadata(df), 'rows_scanned': len(df), 'row_count': len(filtered_df), 'filter_applied': applied_filters, @@ -4549,7 +5160,7 @@ def _sync_work(): }, indent=2, default=str) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error counting rows: {e}", level=logging.WARNING) - return json.dumps({"error": str(e)}) + return json.dumps(self._tabular_error_payload(e)) return await asyncio.to_thread(_sync_work) @@ -4573,6 +5184,9 @@ async def aggregate_column( source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, + table_id: Annotated[str, "Optional discovered Excel table ID or name; 'all' explicitly selects same-schema tables. Required when a worksheet has multiple tables."] = None, + cell_range: Annotated[str, "Optional Excel range such as A3:D100. Use instead of table_id and supply header_row."] = None, + header_row: Annotated[str, "Actual worksheet header row; zero means no header. Required for an explicit range or ambiguous header."] = None, ) -> Annotated[str, "JSON result of the aggregation"]: """Execute an aggregation operation on a column.""" def _sync_work(): @@ -4587,12 +5201,14 @@ def _sync_work(): sheet_name=sheet_name, sheet_index=sheet_index, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._read_tabular_blob_to_dataframe( container, blob_path, sheet_name=selected_sheet, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._try_numeric_conversion(df) @@ -4609,6 +5225,7 @@ def _sync_work(): ) ) + self._require_saved_values(df, [column]) series = df[column] op = operation.lower().strip() @@ -4636,13 +5253,14 @@ def _sync_work(): return json.dumps({ "filename": filename, "selected_sheet": selected_sheet if workbook_metadata.get('is_workbook') else None, + **self._table_result_metadata(df), "column": column, "operation": op, "result": result, }, indent=2, default=str) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error aggregating column: {e}", level=logging.WARNING) - return json.dumps({"error": str(e)}) + return json.dumps(self._tabular_error_payload(e)) return await asyncio.to_thread(_sync_work) @kernel_function( @@ -4674,6 +5292,9 @@ async def filter_rows( max_rows: Annotated[str, "Maximum rows to return per page"] = "100", group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, + table_id: Annotated[str, "Optional discovered Excel table ID or name; 'all' explicitly selects same-schema tables. Required when a worksheet has multiple tables."] = None, + cell_range: Annotated[str, "Optional Excel range such as A3:D100. Use instead of table_id and supply header_row."] = None, + header_row: Annotated[str, "Actual worksheet header row; zero means no header. Required for an explicit range or ambiguous header."] = None, ) -> Annotated[str, "JSON list of matching rows"]: """Filter rows based on a condition.""" def _sync_work(): @@ -4687,7 +5308,7 @@ def _sync_work(): # When no explicit sheet_name is given, try cross-sheet search first normalized_sheet = (sheet_name or '').strip() normalized_sheet_idx = None if sheet_index is None else str(sheet_index).strip() - if not normalized_sheet and normalized_sheet_idx in (None, ''): + if not normalized_sheet and normalized_sheet_idx in (None, '') and table_id in (None, '', 'all') and not cell_range and header_row is None: cross_sheet_result = self._filter_rows_across_sheets( container, blob_path, @@ -4702,6 +5323,7 @@ def _sync_work(): max_rows=int(max_rows), start_row=int(start_row), return_columns=parsed_return_columns, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) if cross_sheet_result is not None: return cross_sheet_result @@ -4711,12 +5333,14 @@ def _sync_work(): sheet_name=sheet_name, sheet_index=sheet_index, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._read_tabular_blob_to_dataframe( container, blob_path, sheet_name=selected_sheet, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._try_numeric_conversion(df) @@ -4760,28 +5384,26 @@ def _sync_work(): ) ) except ValueError as filter_error: - return json.dumps({"error": str(filter_error)}) + return json.dumps(self._tabular_error_payload(filter_error)) start, limit = self._parse_row_page_arguments(start_row, max_rows) filtered = filtered_df.iloc[start:start + limit] response_payload = { "filename": filename, "selected_sheet": selected_sheet if workbook_metadata.get('is_workbook') else None, + **self._table_result_metadata(df), "filter_applied": applied_filters, "normalize_match": normalize_match_flag, "total_matches": len(filtered_df), } response_payload.update(self._build_tabular_row_page_payload( - filtered.to_dict(orient='records'), + filtered, len(filtered_df), start_row=start, max_rows=limit, return_columns=parsed_return_columns, )) response_json = json.dumps(response_payload, indent=2, default=str) - if not str(blob_path or '').lower().endswith('.csv'): - return response_json - replay_expression = None replay_error = None try: @@ -4808,11 +5430,14 @@ def _sync_work(): return_columns=return_columns, expected_row_count=len(filtered_df), replay_error=replay_error, + selected_sheet=selected_sheet, + sheet_names=[selected_sheet] if selected_sheet else None, + **self._table_selector_kwargs(table_id, cell_range, header_row), ), ) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error filtering rows: {e}", level=logging.WARNING) - return json.dumps({"error": str(e)}) + return json.dumps(self._tabular_error_payload(e)) return await asyncio.to_thread(_sync_work) @kernel_function( @@ -4847,6 +5472,9 @@ async def search_rows( max_rows: Annotated[str, "Maximum matching rows to return per page"] = "100", group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, + table_id: Annotated[str, "Optional discovered Excel table ID or name; 'all' explicitly selects same-schema tables. Required when a worksheet has multiple tables."] = None, + cell_range: Annotated[str, "Optional Excel range such as A3:D100. Use instead of table_id and supply header_row."] = None, + header_row: Annotated[str, "Actual worksheet header row; zero means no header. Required for an explicit range or ambiguous header."] = None, ) -> Annotated[str, "JSON result containing matching rows, matched columns, and search metadata"]: """Search rows across one or more columns while preserving row context.""" def _sync_work(): @@ -4861,7 +5489,7 @@ def _sync_work(): normalized_sheet = (sheet_name or '').strip() normalized_sheet_idx = None if sheet_index is None else str(sheet_index).strip() - if not normalized_sheet and normalized_sheet_idx in (None, ''): + if not normalized_sheet and normalized_sheet_idx in (None, '') and table_id in (None, '', 'all') and not cell_range and header_row is None: cross_sheet_result = self._search_rows_across_sheets( container, blob_path, @@ -4880,6 +5508,7 @@ def _sync_work(): normalize_match=normalize_match_flag, max_rows=int(max_rows), start_row=int(start_row), + **self._table_selector_kwargs(table_id, cell_range, header_row), ) if cross_sheet_result is not None: return cross_sheet_result @@ -4890,12 +5519,14 @@ def _sync_work(): sheet_name=sheet_name, sheet_index=sheet_index, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._read_tabular_blob_to_dataframe( container, blob_path, sheet_name=selected_sheet, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._try_numeric_conversion(df) @@ -4936,7 +5567,7 @@ def _sync_work(): ) except Exception as query_error: return json.dumps({ - 'error': f"Query/filter error: {query_error}", + **self._tabular_error_payload(query_error), 'filename': filename, 'selected_sheet': selected_sheet if workbook_metadata.get('is_workbook') else None, }) @@ -4977,14 +5608,16 @@ def _sync_work(): ) except ValueError as search_error: return json.dumps({ - 'error': str(search_error), + **self._tabular_error_payload(search_error), 'filename': filename, 'selected_sheet': selected_sheet if workbook_metadata.get('is_workbook') else None, + **self._table_result_metadata(df), }) response_json = json.dumps({ 'filename': filename, 'selected_sheet': selected_sheet if workbook_metadata.get('is_workbook') else None, + **self._table_result_metadata(df), 'search_value': search_value, 'search_operator': search_operator, 'searched_columns': search_result['searched_columns'], @@ -5000,9 +5633,6 @@ def _sync_work(): 'next_start_row': search_result['next_start_row'], 'data': search_result['data'], }, indent=2, default=str) - if not str(blob_path or '').lower().endswith('.csv'): - return response_json - replay_expression = None replay_error = None try: @@ -5033,11 +5663,14 @@ def _sync_work(): return_columns=return_columns, expected_row_count=search_result['total_matches'], replay_error=replay_error, + selected_sheet=selected_sheet, + sheet_names=[selected_sheet] if selected_sheet else None, + **self._table_selector_kwargs(table_id, cell_range, header_row), ), ) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error searching rows: {e}", level=logging.WARNING) - return json.dumps({"error": str(e)}) + return json.dumps(self._tabular_error_payload(e)) return await asyncio.to_thread(_sync_work) @@ -5064,6 +5697,9 @@ async def query_tabular_data( max_rows: Annotated[str, "Maximum rows to return per page"] = "100", group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, + table_id: Annotated[str, "Optional discovered Excel table ID or name; 'all' explicitly selects same-schema tables. Required when a worksheet has multiple tables."] = None, + cell_range: Annotated[str, "Optional Excel range such as A3:D100. Use instead of table_id and supply header_row."] = None, + header_row: Annotated[str, "Actual worksheet header row; zero means no header. Required for an explicit range or ambiguous header."] = None, ) -> Annotated[str, "JSON result of the query"]: """Execute a pandas query expression against a tabular file.""" def _sync_work(): @@ -5074,6 +5710,11 @@ def _sync_work(): group_id=group_id, public_workspace_id=public_workspace_id ) if str(blob_path or '').lower().endswith('.csv'): + if self._table_selector_kwargs(table_id, cell_range, header_row): + raise TabularSelectionError( + "Table, range, and header selectors apply only to Excel workbooks.", + code='table_selection_invalid', + ) return self._query_csv_data_in_bounded_chunks( container_name=container, blob_path=blob_path, @@ -5086,12 +5727,13 @@ def _sync_work(): # When no explicit sheet_name is given, try cross-sheet query first normalized_sheet = (sheet_name or '').strip() normalized_sheet_idx = None if sheet_index is None else str(sheet_index).strip() - if not normalized_sheet and normalized_sheet_idx in (None, ''): + if not normalized_sheet and normalized_sheet_idx in (None, '') and table_id in (None, '', 'all') and not cell_range and header_row is None: cross_sheet_result = self._query_tabular_data_across_sheets( container, blob_path, filename, query_expression, max_rows=int(max_rows), start_row=int(start_row), return_columns=parsed_return_columns, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) if cross_sheet_result is not None: return cross_sheet_result @@ -5101,12 +5743,14 @@ def _sync_work(): sheet_name=sheet_name, sheet_index=sheet_index, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._read_tabular_blob_to_dataframe( container, blob_path, sheet_name=selected_sheet, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._try_numeric_conversion(df) @@ -5120,45 +5764,32 @@ def _sync_work(): response_payload = { "filename": filename, "selected_sheet": selected_sheet if workbook_metadata.get('is_workbook') else None, + **self._table_result_metadata(df), "query_expression": query_expression, "query_expression_fallback": used_reviewer_style_fallback, "total_matches": len(result_df), } response_payload.update(self._build_tabular_row_page_payload( - sliced_result_df.to_dict(orient='records'), + sliced_result_df, len(result_df), start_row=start, max_rows=limit, return_columns=parsed_return_columns, )) response_json = json.dumps(response_payload, indent=2, default=str) - source_blob_version = self._get_tabular_blob_version(container, blob_path) - internal_metadata = { - 'tabular_source_authorization': self._build_source_authorization_from_location( - container, - blob_path, - blob_version=source_blob_version, - ), - } - if str(blob_path or '').lower().endswith('.csv'): - internal_metadata['tabular_generated_export_source'] = ( - self._build_generated_export_query_descriptor_from_location( - container_name=container, - blob_path=blob_path, - filename=filename, - query_expression=query_expression, - return_columns=return_columns, - expected_row_count=len(result_df), - blob_version=source_blob_version, - ) - ) return PluginInvocationResult( response_json, - internal_metadata=internal_metadata, + internal_metadata=self._build_tabular_generated_export_internal_metadata( + container_name=container, blob_path=blob_path, filename=filename, + source_function='query_tabular_data', query_expression=query_expression, + return_columns=return_columns, expected_row_count=len(result_df), + selected_sheet=selected_sheet, sheet_names=[selected_sheet], + **self._table_selector_kwargs(table_id, cell_range, header_row), + ), ) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error querying data: {e}", level=logging.WARNING) - return json.dumps({"error": f"Query error: {str(e)}. Ensure column names and values are correct."}) + return json.dumps(self._tabular_error_payload(e)) return await asyncio.to_thread(_sync_work) @kernel_function( @@ -5197,6 +5828,16 @@ async def filter_rows_by_related_values( max_rows: Annotated[str, "Maximum related target rows to return per page"] = "100", group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, + source_table_id: Annotated[str, "Optional discovered Excel table ID or name for the source cohort."] = None, + source_cell_range: Annotated[str, "Optional source Excel range instead of source_table_id; requires source_header_row."] = None, + source_header_row: Annotated[str, "Actual source worksheet header row; zero means no header."] = None, + target_table_id: Annotated[str, "Optional discovered Excel table ID or name for the target rows."] = None, + target_cell_range: Annotated[str, "Optional target Excel range instead of target_table_id; requires target_header_row."] = None, + target_header_row: Annotated[str, "Actual target worksheet header row; zero means no header."] = None, + source_filename: Annotated[str, "Optional different filename for the source cohort; defaults to filename."] = None, + target_filename: Annotated[str, "Optional different filename for the target rows; defaults to filename."] = None, + source_source: Annotated[str, "Optional source cohort storage scope: workspace, chat, group, or public. Defaults to source."] = None, + target_source: Annotated[str, "Optional target storage scope: workspace, chat, group, or public. Defaults to source."] = None, ) -> Annotated[str, "JSON result containing explainable set-membership filtering output"]: """Filter target rows by membership in a source-sheet cohort.""" def _sync_work(): @@ -5206,6 +5847,14 @@ def _sync_work(): user_id, conversation_id, filename, source, group_id=group_id, public_workspace_id=public_workspace_id ) + source_location = self._resolve_blob_location_with_fallback( + user_id, conversation_id, source_filename or filename, source_source or source, + group_id=group_id, public_workspace_id=public_workspace_id, + ) + target_location = self._resolve_blob_location_with_fallback( + user_id, conversation_id, target_filename or filename, target_source or source, + group_id=group_id, public_workspace_id=public_workspace_id, + ) result_payload = self._evaluate_related_value_membership( container, blob_path, @@ -5230,11 +5879,15 @@ def _sync_work(): max_rows=int(max_rows), start_row=int(start_row), return_columns=return_columns, + source_table_id=source_table_id, source_cell_range=source_cell_range, + source_header_row=source_header_row, target_table_id=target_table_id, + target_cell_range=target_cell_range, target_header_row=target_header_row, + source_location=source_location, target_location=target_location, ) return json.dumps(result_payload, indent=2, default=str) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error filtering rows by related values: {e}", level=logging.WARNING) - return json.dumps({"error": str(e)}) + return json.dumps(self._tabular_error_payload(e)) return await asyncio.to_thread(_sync_work) @@ -5271,6 +5924,16 @@ async def count_rows_by_related_values( source: Annotated[str, "Source: 'workspace', 'chat', 'group', or 'public'"] = "chat", group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, + source_table_id: Annotated[str, "Optional discovered Excel table ID or name for the source cohort."] = None, + source_cell_range: Annotated[str, "Optional source Excel range instead of source_table_id; requires source_header_row."] = None, + source_header_row: Annotated[str, "Actual source worksheet header row; zero means no header."] = None, + target_table_id: Annotated[str, "Optional discovered Excel table ID or name for the target rows."] = None, + target_cell_range: Annotated[str, "Optional target Excel range instead of target_table_id; requires target_header_row."] = None, + target_header_row: Annotated[str, "Actual target worksheet header row; zero means no header."] = None, + source_filename: Annotated[str, "Optional different filename for the source cohort; defaults to filename."] = None, + target_filename: Annotated[str, "Optional different filename for the target rows; defaults to filename."] = None, + source_source: Annotated[str, "Optional source cohort storage scope: workspace, chat, group, or public. Defaults to source."] = None, + target_source: Annotated[str, "Optional target storage scope: workspace, chat, group, or public. Defaults to source."] = None, ) -> Annotated[str, "JSON result containing an explainable relational row count"]: """Count target rows by membership in a source-sheet cohort.""" def _sync_work(): @@ -5280,6 +5943,14 @@ def _sync_work(): user_id, conversation_id, filename, source, group_id=group_id, public_workspace_id=public_workspace_id ) + source_location = self._resolve_blob_location_with_fallback( + user_id, conversation_id, source_filename or filename, source_source or source, + group_id=group_id, public_workspace_id=public_workspace_id, + ) + target_location = self._resolve_blob_location_with_fallback( + user_id, conversation_id, target_filename or filename, target_source or source, + group_id=group_id, public_workspace_id=public_workspace_id, + ) result_payload = self._evaluate_related_value_membership( container, blob_path, @@ -5302,6 +5973,10 @@ def _sync_work(): target_alias_column=target_alias_column, normalize_match=normalize_match_flag, max_rows=50, + source_table_id=source_table_id, source_cell_range=source_cell_range, + source_header_row=source_header_row, target_table_id=target_table_id, + target_cell_range=target_cell_range, target_header_row=target_header_row, + source_location=source_location, target_location=target_location, include_rows=False, ) if 'error' not in result_payload: result_payload['row_count'] = result_payload.get('matched_target_row_count', 0) @@ -5311,7 +5986,7 @@ def _sync_work(): return json.dumps(result_payload, indent=2, default=str) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error counting rows by related values: {e}", level=logging.WARNING) - return json.dumps({"error": str(e)}) + return json.dumps(self._tabular_error_payload(e)) return await asyncio.to_thread(_sync_work) @@ -5340,6 +6015,9 @@ async def group_by_aggregate( sort_descending: Annotated[str, "Whether top_results should be sorted descending (true/false)"] = "true", group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, + table_id: Annotated[str, "Optional discovered Excel table ID or name; 'all' explicitly selects same-schema tables. Required when a worksheet has multiple tables."] = None, + cell_range: Annotated[str, "Optional Excel range such as A3:D100. Use instead of table_id and supply header_row."] = None, + header_row: Annotated[str, "Actual worksheet header row; zero means no header. Required for an explicit range or ambiguous header."] = None, ) -> Annotated[str, "JSON result of the group-by aggregation"]: """Group by one column and aggregate another.""" def _sync_work(): @@ -5354,12 +6032,14 @@ def _sync_work(): sheet_name=sheet_name, sheet_index=sheet_index, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._read_tabular_blob_to_dataframe( container, blob_path, sheet_name=selected_sheet, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._try_numeric_conversion(df) @@ -5386,6 +6066,7 @@ def _sync_work(): "error": "Unsupported operation. Use count, sum, mean, min, max, median, or std." }) + self._require_saved_values(df, [group_by_column, aggregate_column]) grouped = df.groupby(group_by_column)[aggregate_column].agg(op) grouped = grouped.dropna() if grouped.empty: @@ -5400,6 +6081,7 @@ def _sync_work(): return json.dumps({ "filename": filename, "selected_sheet": selected_sheet if workbook_metadata.get('is_workbook') else None, + **self._table_result_metadata(df), "group_by": group_by_column, "aggregate_column": aggregate_column, "operation": op, @@ -5410,7 +6092,7 @@ def _sync_work(): }, indent=2, default=str) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error in group-by: {e}", level=logging.WARNING) - return json.dumps({"error": str(e)}) + return json.dumps(self._tabular_error_payload(e)) return await asyncio.to_thread(_sync_work) @kernel_function( @@ -5441,6 +6123,9 @@ async def group_by_datetime_component( sort_descending: Annotated[str, "Whether top_results should be sorted descending (true/false)"] = "true", group_id: Annotated[str, "Group ID (for group workspace documents)"] = None, public_workspace_id: Annotated[str, "Public workspace ID (for public workspace documents)"] = None, + table_id: Annotated[str, "Optional discovered Excel table ID or name; 'all' explicitly selects same-schema tables. Required when a worksheet has multiple tables."] = None, + cell_range: Annotated[str, "Optional Excel range such as A3:D100. Use instead of table_id and supply header_row."] = None, + header_row: Annotated[str, "Actual worksheet header row; zero means no header. Required for an explicit range or ambiguous header."] = None, ) -> Annotated[str, "JSON result of the datetime component grouping analysis"]: """Group data by a datetime component and aggregate a metric.""" def _sync_work(): @@ -5459,22 +6144,22 @@ def _sync_work(): sheet_name=sheet_name, sheet_index=sheet_index, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._read_tabular_blob_to_dataframe( container, blob_path, sheet_name=selected_sheet, require_explicit_sheet=True, + **self._table_selector_kwargs(table_id, cell_range, header_row), ) df = self._try_numeric_conversion(df) if filter_expression: try: - df = df.query(filter_expression) + df, _ = self._apply_query_expression_with_fallback(df, filter_expression) except Exception as query_error: - return json.dumps({ - "error": f"Filter query error: {query_error}. Ensure column names and values are correct." - }) + return json.dumps(self._tabular_error_payload(query_error)) if datetime_column not in df.columns: related_columns = [aggregate_column] if aggregate_column else [] @@ -5491,6 +6176,7 @@ def _sync_work(): ) ) + self._require_saved_values(df, [datetime_column]) parsed_datetime = self._parse_datetime_like_series(df[datetime_column]) valid_mask = parsed_datetime.notna() if not valid_mask.any(): @@ -5535,6 +6221,7 @@ def _sync_work(): available_columns=list(filtered_df.columns), ) ) + self._require_saved_values(filtered_df, [aggregate_column_name]) grouped = filtered_df.groupby(component_column_name)[aggregate_column_name].agg(op) grouped = grouped.dropna() @@ -5552,6 +6239,7 @@ def _sync_work(): return json.dumps({ "filename": filename, "selected_sheet": selected_sheet if workbook_metadata.get('is_workbook') else None, + **self._table_result_metadata(df), "datetime_column": datetime_column, "datetime_component": self._normalize_datetime_component(datetime_component), "aggregate_column": aggregate_column_name or None, @@ -5566,5 +6254,5 @@ def _sync_work(): }, indent=2, default=str) except Exception as e: log_event(f"[TABULAR_PROCESSING_PLUGIN] Error in datetime component grouping: {e}", level=logging.WARNING) - return json.dumps({"error": str(e)}) + return json.dumps(self._tabular_error_payload(e)) return await asyncio.to_thread(_sync_work) diff --git a/application/single_app/templates/admin/_panes/extraction.html b/application/single_app/templates/admin/_panes/extraction.html index 857acb69f..e9c910fbe 100644 --- a/application/single_app/templates/admin/_panes/extraction.html +++ b/application/single_app/templates/admin/_panes/extraction.html @@ -248,7 +248,7 @@
Azure AI Content Understanding<
Images Inside Office Files

- Neither extraction engine describes figures inside Word and PowerPoint files. When this is on, + Native extraction does not describe the pixels of images inside Word, PowerPoint, or Excel files. When this is on, embedded images are pulled out of the file, analyzed with whichever engine backs the selected extraction mode, and indexed as their own citable chunks. This works with Standard extraction too, using Document Intelligence. @@ -262,7 +262,7 @@

Images Inside Office Files
diff --git a/application/v2_ui/src/components/chat/EnhancedCitationViewer.tsx b/application/v2_ui/src/components/chat/EnhancedCitationViewer.tsx index 9aa30eadd..282742458 100644 --- a/application/v2_ui/src/components/chat/EnhancedCitationViewer.tsx +++ b/application/v2_ui/src/components/chat/EnhancedCitationViewer.tsx @@ -5,7 +5,7 @@ // falls back to the plain text citation, because a citation that cannot render its source // is still more useful as a readable passage than as a dead click. -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { useCallback, useEffect, useId, useMemo, useRef, useState, type KeyboardEvent as ReactKeyboardEvent } from 'react'; import { clsx } from 'clsx'; import { ChevronLeft, @@ -159,21 +159,20 @@ function PdfViewer({ docId, location, onFail }: ViewerProps) { /* -------------------------------------------------------------------------- */ /** - * What ingestion extracted from a cited image. + * The original cited passage, independent of worksheet/table preview navigation. * - * An image has no page to scroll to, so the picture alone cannot show why it was cited. The - * cited passage is the text the assistant actually read -- any recognised text plus the - * image's description -- and is shown beneath the picture rather than instead of it. - * Nothing is drawn when the passage cannot be loaded; the image is still the source. + * The passage remains tied to the citation rather than the table currently being previewed. */ -function CitedImagePassage({ +function CitedSourcePassage({ citationId, docId, location, + sourceLabel = 'What was extracted from this image', }: { citationId: string; docId: string; location: string; + sourceLabel?: string; }) { const [passage, setPassage] = useState(null); const [failed, setFailed] = useState(false); @@ -214,7 +213,7 @@ function CitedImagePassage({ className="max-h-[40%] shrink-0 overflow-y-auto border-t border-edge px-5 py-3" > - What was extracted from this image + {sourceLabel} {passage === null ? (

@@ -252,7 +251,7 @@ function ImageViewer({ docId, location, metadata, onFail, citationId }: ViewerPr />

{citationId ? ( - + ) : null} ); @@ -311,34 +310,61 @@ function MediaViewer({ docId, location, metadata, onFail, kind }: ViewerProps & /* Tabular */ /* -------------------------------------------------------------------------- */ -function TabularViewer({ docId, location, onFail, citationId }: ViewerProps) { +function TabularViewer({ docId, location, metadata, onFail, citationId }: ViewerProps) { const [preview, setPreview] = useState(null); - const [sheet, setSheet] = useState( - // A sheet citation carries the sheet name where a page citation carries a number. - Number.isNaN(Number.parseInt(location, 10)) ? location || undefined : undefined, - ); + const [sheet, setSheet] = useState(); + const [tableId, setTableId] = useState(); + const [startRow, setStartRow] = useState(0); + const [previousStarts, setPreviousStarts] = useState([]); + const [jumpRecord, setJumpRecord] = useState('1'); const [loading, setLoading] = useState(true); + const [error, setError] = useState(''); + const [retry, setRetry] = useState(0); + const hasLoaded = useRef(false); + const initialLocationApplied = useRef(false); + const sheetButtons = useRef(new Map()); + const panelId = useId(); + const tableSelectId = useId(); + const recordInputId = useId(); useEffect(() => { let cancelled = false; + let redirected = false; + const controller = new AbortController(); setLoading(true); + setError(''); void (async () => { try { - const result = await fetchTabularPreview(docId, { sheetName: sheet }); + const result = await fetchTabularPreview( + docId, { sheetName: sheet, tableId, startRow }, controller.signal, + ); if (!cancelled) { + if (!initialLocationApplied.current) { + initialLocationApplied.current = true; + const requested = (result.sheet_names ?? []).find((name) => + name === location || `${name}!` === location.slice(0, name.length + 1), + ); + if (requested && requested !== result.selected_sheet) { + redirected = true; + setSheet(requested); + return; + } + } setPreview(result); + setJumpRecord(String((result.start_row ?? startRow) + 1)); + hasLoaded.current = true; } } catch (error) { if (!cancelled) { - onFail( - error instanceof Error - ? error.message - : 'Could not load the spreadsheet.', - ); + const message = error instanceof Error ? error.message : 'Could not load the spreadsheet.'; + setError(message); + if (!hasLoaded.current) { + onFail(message); + } } } finally { - if (!cancelled) { + if (!cancelled && !redirected) { setLoading(false); } } @@ -346,8 +372,9 @@ function TabularViewer({ docId, location, onFail, citationId }: ViewerProps) { return () => { cancelled = true; + controller.abort(); }; - }, [docId, sheet, onFail]); + }, [docId, location, sheet, tableId, startRow, retry, onFail]); if (loading && !preview) { return ; @@ -358,19 +385,66 @@ function TabularViewer({ docId, location, onFail, citationId }: ViewerProps) { } const sheets = preview.sheet_names ?? []; + const tables = preview.tables ?? []; + const selectedSheet = preview.selected_sheet ?? sheets[0]; + const currentStart = preview.start_row ?? 0; + const rowCount = preview.rows?.length ?? 0; + const hasMore = preview.has_more ?? preview.truncated ?? false; + const nextStart = preview.next_start_row ?? currentStart + rowCount; + + function changeSheet(name: string) { + if (name === selectedSheet && !loading && !error) { + return; + } + setLoading(true); + setSheet(name); + setTableId(undefined); + setStartRow(0); + setPreviousStarts([]); + } + + function moveSheet(event: ReactKeyboardEvent, index: number) { + let next: number; + if (event.key === 'ArrowRight') { + next = (index + 1) % sheets.length; + } else if (event.key === 'ArrowLeft') { + next = (index - 1 + sheets.length) % sheets.length; + } else if (event.key === 'Home') { + next = 0; + } else if (event.key === 'End') { + next = sheets.length - 1; + } else { + return; + } + event.preventDefault(); + changeSheet(sheets[next]); + sheetButtons.current.get(sheets[next])?.focus(); + } return (
{sheets.length > 1 && ( -
- {sheets.map((name) => ( +
+ {sheets.map((name, index) => (
)} -
- +
+

+ {selectedSheet ? `Worksheet: ${selectedSheet}` : 'CSV data'} + {preview.table_range ? ` · ${preview.table_range}` : ''} + {preview.header_row ? ` · Header row ${preview.header_row}` : ''} + {preview.header_status === 'inferred' ? ' (detected)' : ''} + {preview.header_status === 'ambiguous' ? ' · Header is unconfirmed; cell coordinates are retained.' : ''} +

+ {tables.length > 1 && ( +
+ + +
+ )} +
+ {error && ( +
+ {error} + +
+ )} +
+ {loading ? : rowCount === 0 ? ( +

No data rows in this selection.

+ ) :
+ + {(preview.columns ?? []).map((column, index) => ( {(preview.rows ?? []).map((row, rowIndex) => ( + {row.map((cell, cellIndex) => ( -
{preview.table_name || selectedSheet || 'CSV'} data preview
+ {sheets.length ? 'Worksheet row' : 'File row'} + {column} @@ -398,6 +517,9 @@ function TabularViewer({ docId, location, onFail, citationId }: ViewerProps) {
+ {preview.row_numbers?.[rowIndex] ?? currentStart + rowIndex + 2} + ))}
+ }
- {preview.truncated && ( -

- Showing the first {preview.rows?.length ?? 0} rows. Download the file for - the rest. +

+

+ {rowCount ? `Records ${currentStart + 1}–${currentStart + rowCount}` : '0 records'} + {preview.total_rows != null ? ` of ${preview.total_rows}` : ''} + {hasMore ? ' · More records available' : ''}

+ + +
{ + event.preventDefault(); + const record = Number(jumpRecord); + if (!Number.isSafeInteger(record) || record < 1 || (preview.total_rows != null && record > preview.total_rows)) { + setError('Choose a record number within this table.'); + return; + } + if (record - 1 === currentStart) { + if (error) { + setRetry((current) => current + 1); + } + return; + } + setLoading(true); + setPreviousStarts((starts) => [...starts, currentStart]); + setStartRow(record - 1); + }} + > + + setJumpRecord(event.target.value)} + className="min-h-11 w-24 rounded-lg border border-edge-strong bg-surface-1 px-2 text-base text-text-1 sm:text-sm" + /> + +
+
+ {(preview.columns_truncated || preview.cell_text_truncated) && ( +

Some preview columns or long cell text are shortened. Inspect a smaller cell range or download the original for full content.

)} {citationId ? ( - + ) : null}
); @@ -572,7 +756,9 @@ export function EnhancedCitationViewer({

{VIEWER_LABELS[type]} - {location ? ` · ${locationLabel} ${location}` : ''} + {location ? type === 'tabular' + ? ` · ${/^\d+$/.test(location) ? `Evidence segment ${location}` : locationLabel === 'Sheet' ? `Sheet ${location}` : location}` + : ` · ${locationLabel} ${location}` : ''}

@@ -600,7 +786,7 @@ export function EnhancedCitationViewer({ {type === 'image' && } {type === 'video' && } {type === 'audio' && } - {type === 'tabular' && } + {type === 'tabular' && } {type === 'visio' && } diff --git a/application/v2_ui/src/lib/endpoints.ts b/application/v2_ui/src/lib/endpoints.ts index a360b6039..df55327cb 100644 --- a/application/v2_ui/src/lib/endpoints.ts +++ b/application/v2_ui/src/lib/endpoints.ts @@ -1406,6 +1406,16 @@ export async function fetchEnhancedCitationPdf( }; } +export interface WorkbookPreviewTable { + id: string; + name: string; + range: string; + origin: 'named' | 'inferred' | 'range'; + header_row?: number | null; + header_status?: string; + row_count?: number; +} + export interface TabularPreview { filename?: string; selected_sheet?: string | null; @@ -1416,11 +1426,32 @@ export interface TabularPreview { columns?: string[]; rows?: string[][]; truncated?: boolean; + tables?: WorkbookPreviewTable[]; + table_count?: number; + table_id?: string; + table_name?: string; + table_range?: string; + header_row?: number | null; + header_status?: string; + row_numbers?: number[]; + start_row?: number; + has_more?: boolean; + next_start_row?: number | null; + columns_truncated?: boolean; + cell_text_truncated?: boolean; } export function fetchTabularPreview( docId: string, - options: { sheetName?: string; maxRows?: number } = {}, + options: { + sheetName?: string; + tableId?: string; + startRow?: number; + maxRows?: number; + cellRange?: string; + headerRow?: number; + } = {}, + signal?: AbortSignal, ) { const params = new URLSearchParams({ doc_id: docId }); if (options.sheetName) { @@ -1429,8 +1460,21 @@ export function fetchTabularPreview( if (options.maxRows) { params.set('max_rows', String(options.maxRows)); } + if (options.tableId) { + params.set('table_id', options.tableId); + } + if (options.startRow !== undefined) { + params.set('start_row', String(options.startRow)); + } + if (options.cellRange) { + params.set('cell_range', options.cellRange); + } + if (options.headerRow !== undefined) { + params.set('header_row', String(options.headerRow)); + } return api.get( `/api/enhanced_citations/tabular_preview?${params.toString()}`, + signal, ); } diff --git a/docs/_data/features.yml b/docs/_data/features.yml index 8470eea58..965450f74 100644 --- a/docs/_data/features.yml +++ b/docs/_data/features.yml @@ -180,6 +180,8 @@ enhanced-citations: reference: - title: "Feature reference" url: /reference/features/ + - title: "Inspect Excel workbooks" + url: /guides/inspect-excel-workbooks/ chat-memory: name: "Chat memory" area: "Chat and retrieval" @@ -533,7 +535,7 @@ tabular-analysis: name: "Tabular analysis" area: "Media and ingestion" audience: "End users" - summary: "Runs structured CSV and spreadsheet analysis with durable planning, batching, checkpoints, and response controls." + summary: "Runs structured CSV and spreadsheet analysis with table/range-aware headers, durable planning, batching, checkpoints, and bounded responses." why: "Use this when users need computed answers, grouped results, exports, or charts from spreadsheet data rather than prose guesses. The many rollout flags exist because large workbook analysis needs reliability controls for planning, retries, batching, and durable checkpoints. The Analyze All Documents target stays off and hidden because no chat or workflow screen offers it yet." visual_sections: - heading: "Compute from the intended rows, not a sample" @@ -541,7 +543,7 @@ tabular-analysis: Schema inspection helps plan a calculation; the intended underlying rows must supply its totals and grouped results. Distinguish a computed table from the model's explanation and from an actual exported file. Check - source coverage, worksheet selection, validation notices, and limitations + source coverage, worksheet and table/range selection, validation notices, and limitations before relying on the result. The illustrated data is synthetic, not a benchmark or a capacity promise. media_id: slide-tabular-analysis diff --git a/docs/admin/knowledge.md b/docs/admin/knowledge.md index 56aba8907..1baad263f 100644 --- a/docs/admin/knowledge.md +++ b/docs/admin/knowledge.md @@ -190,8 +190,8 @@ there is nothing further to configure. ### Images Inside Office Files {#office-embedded-image-section} -Neither extraction engine describes figures embedded in Word and PowerPoint files, so a -chart pasted into a slide deck is invisible to search. With this on, embedded images are +Native extraction does not describe the pixels of images embedded in Word, PowerPoint, +or Excel files. With this on, embedded images are extracted from the file, analysed with whichever engine backs the selected extraction mode, and indexed as their own citable chunks. It works with Standard extraction as well as Enhanced. @@ -312,8 +312,8 @@ already captures the surrounding structure. | Content Understanding Key | Provides the secret credential used when the selected authentication mode requires one. | Empty | `azure_content_understanding_key` | | API Version | Default: | Not specified in defaults | `azure_content_understanding_api_version` | | Document Analyzer | Default: | Not specified in defaults | `azure_content_understanding_analyzer_id` | -| Image Analyzer | Content Understanding analyzer that describes images embedded in DOCX and PPTX files, and standalone images when the document analyzer finds no text in them. Its `Summary` field is indexed as the image description. | prebuilt-imageSearch | `azure_content_understanding_image_analyzer_id` | -| Analyze images embedded in DOCX and PPTX files | Exposes the capability after required services, permissions, and rollout policy are ready. | On | `enable_office_embedded_image_analysis`; capability toggle | +| Image Analyzer | Content Understanding analyzer that describes images embedded in DOCX, PPTX, and OOXML Excel workbooks, and standalone images when the document analyzer finds no text in them. Its `Summary` field is indexed as the image description. Excel descriptions retain their worksheet/cell placements separately from saved alternative text. | prebuilt-imageSearch | `azure_content_understanding_image_analyzer_id` | +| Analyze images embedded in DOCX, PPTX, and Excel files | Adds configured image descriptions/OCR during ingestion or reprocessing, retaining available worksheet/object anchors separately from saved alternative text. Excel support was added in 0.261.405. | On | `enable_office_embedded_image_analysis`; capability toggle | | Minimum Image Size (pixels) | Images narrower or shorter than this are skipped as icons or spacers. | Not specified in defaults | `office_embedded_image_min_pixels` | | Maximum Images Per Document | Caps per-document cost. Duplicate images are analyzed once. | Not specified in defaults | `office_embedded_image_max_per_document` | | Use APIM instead of direct Document Intelligence endpoint | Routes Document Intelligence calls through APIM instead of calling the service endpoint directly. | Off | `enable_document_intelligence_apim`; capability toggle | diff --git a/docs/explanation/features/DOCUMENT_ACCESSIBILITY_AND_STRUCTURE_EXTRACTION.md b/docs/explanation/features/DOCUMENT_ACCESSIBILITY_AND_STRUCTURE_EXTRACTION.md index e7254b739..22a98f1ea 100644 --- a/docs/explanation/features/DOCUMENT_ACCESSIBILITY_AND_STRUCTURE_EXTRACTION.md +++ b/docs/explanation/features/DOCUMENT_ACCESSIBILITY_AND_STRUCTURE_EXTRACTION.md @@ -2,7 +2,8 @@ **Implemented in version: 0.261.391** (formatting, style inventory, and programmatic access added in **0.261.392**) **Excel implemented in version: 0.261.399** -**Related config.py version: `0.261.399`** +**Excel native content/table improvements implemented in version: 0.261.405** +**Related config.py version: `0.261.405`** ## 1. Overview and Purpose @@ -223,6 +224,16 @@ flowchart LR ### Excel native inspection (0.261.399) +The original extraction below was expanded in **0.261.405** with canonical +thread/reply handling, saved-cell context, streaming table discovery, and +revision-bound full-inventory paging. See +[Scalable Excel Native Inspection](EXCEL_NATIVE_INSPECTION.md) for current +storage/scale behavior and the +[comment/header fix](../fixes/EXCEL_COMMENT_CONTENT_AND_TABLE_HEADERS_FIX.md). +Existing retained Excel originals can be inspected without re-uploading. +The historical parent-record/style-sample limits below do not mean every +worksheet has one table or that complete native records must fit the parent. + Excel follows the same **parent metadata plus searchable native evidence** model as the other document formats without turning spreadsheet ingestion into full-data indexing. This lets users ask about the workbook itself, such as who authored it, which comments need review, whether an image has native alternative text, or what prints in a sheet's footer. `functions_excel_accessibility.py` reads modern workbook packages without saving or executing them. It follows workbook relationships to identify worksheets and chart sheets rather than assuming that a sheet's filename indicates its name or order. It also reads: @@ -251,7 +262,7 @@ Inspection is bounded to 5,000 ZIP entries, 16 MiB per XML part, 64 MiB total XM Legacy `.xls` is inspected with the existing OLE/BIFF dependencies. Its metadata and style coverage is best effort; unavailable object, thread and header/footer details are explicitly stated. No new cloud service or dependency is required. -To inspect an existing workbook, re-upload it to a personal, group or public workspace so native evidence is generated. Then ask, for example, "Show the cell notes and threaded comments", "Which objects have saved alt text?", or "What is in the first-page footer?" An agent with Document Search can request `get_document_accessibility_report` and retrieve the relevant native evidence segments for detailed findings. Chat-only uploads do not gain this workspace report. +From 0.261.405, existing active workbook originals can be inspected on demand without re-uploading. Reprocessing can add configured image-analysis enrichment; a new upload is needed only when required originals/inventories are unavailable. Ask, for example, "Show the cell notes and threaded comments", "Which objects have saved alt text?", or "What is in the first-page footer?" An agent with Document Search can request `get_document_accessibility_report` and follow native inventory continuations for detailed findings. Chat-only uploads do not gain workspace-report access. ### Programmatic Access: `get_document_accessibility_report` diff --git a/docs/explanation/features/EXCEL_NATIVE_INSPECTION.md b/docs/explanation/features/EXCEL_NATIVE_INSPECTION.md new file mode 100644 index 000000000..316227d9c --- /dev/null +++ b/docs/explanation/features/EXCEL_NATIVE_INSPECTION.md @@ -0,0 +1,131 @@ +# Scalable Excel Native Inspection + +Implemented in version: **0.261.405**. +Related application version: `application/single_app/config.py`. + +## Overview + +Excel inspection separates the workbook's saved data, reviewer feedback, and +document structure. A comment requesting a different value does not change the +saved cell value. A worksheet can contain several independent tables; its first +row is not automatically every table's header. + +The implementation uses the existing Open XML, openpyxl, xlrd, private artifact +storage, Document Search, and Tabular Processing dependencies. It introduces no +new cloud service. + +## Saved content and structure + +Modern `.xlsx` and `.xlsm` packages expose saved cell values and types, formula +text and available caches, genuine legacy notes, threaded messages and replies, +core/extended/custom properties, defined names, table definitions, hyperlinks, +validation expressions, layout, print information, and readable drawing/chart +text and caches. Drawing and comment locations retain worksheet/cell anchors. + +Threaded comments can have legacy compatibility representations. An identifiable +same-cell compatibility copy is not counted as another genuine note. Counts +distinguish legacy notes, thread roots, replies, and raw compatibility records. +Missing author/date/resolution fields remain unknown. A blank cell, zero, false, +and a formula without a saved result are different states. + +Images use the existing Office embedded-image analysis setting, configured +engine, size limits, and per-document cap during ingestion/reprocessing. +Generated image descriptions are separate from saved native alternative text. +Missing analysis does not prove that the workbook contains no pictures. + +## Tables and scale + +Named Excel table definitions provide authoritative ranges, columns, and +header/totals information. Ordinary occupied blocks are detected separately, +including horizontally or vertically separated regions, title rows, and +repeated headers. Inferred headers are labelled as inferred; ambiguous headers +require an explicit selection rather than a silent first-row assumption. + +Streaming worksheet reads support large workbooks without copying all rows into +metadata. Functional coverage includes 50,000- and 100,000-row workbooks, late-row +reads, and five independent named or ordinary tables on one worksheet. These +are correctness fixtures, not a capacity or performance guarantee for every file. + +Analytical operations use the selected table/range and preserve physical source +locations. Counts, filters, and aggregates must come from the underlying records, +not the displayed preview. CSV and ordinary single-table inputs retain their +existing analysis path. + +## Storage, access, and lifecycle + +The original retained workbook remains authoritative. Its table dataset is +identified by the source revision/hash, worksheet, range, headers, and columns; +worksheet rows are not placed in a large Cosmos metadata object. + +Parent Excel metadata remains bounded to 500,000 serialized bytes. Complete +native inventories use immutable, revision-bound pages in the existing private +artifact store, with the ready manifest written last. Long text has explicit +continuations rather than permanent loss at a parent-preview limit. Existing +durable tabular machinery handles large computed outputs. + +Every read, cached read, and continuation retains current source authorization +and revision checks. Private artifact references are not public document fields. +Metadata from an original file cannot be presented as the native metadata of a +different sanitized active representation. Revision cleanup removes its derived +private artifacts without deleting another document's artifacts. + +## Using existing files + +Existing stored comment records can be used without uploading the file again. +When the active original is retained, native inspection can generate the richer +inventory on demand. A retained current inventory can also be read without +downloading the workbook again. + +Image-analysis enrichment follows ingestion/reprocessing and its configured +cloud engine; an ordinary native metadata read does not silently run new vision +analysis. A re-upload is necessary only when required source bytes/inventories +are unavailable. A held or inaccessible document requires authorized resolution, +not re-uploading to bypass its policy. + +No automatic bulk reprocessing is started by this feature. + +## APIs and user workflow + +`Document Search.get_document_accessibility_report` preserves its +`accessibility`, `style`, and `issues` sections and adds bounded Excel native +section selection, continuation, and direct saved-cell range reads. Follow +`excel_inspection.page.next_cursor` while `has_more` is true when every record is +required. Explicit JSON record fragments must be continued before treating them +as a complete record. + +The V2 citation viewer has worksheet tabs, table selection, record paging, and a +record jump control. Its row headers show physical worksheet row numbers. +**Cited workbook evidence** remains the passage originally cited, independent +of the table currently previewed. Numeric native evidence segments are not +physical Excel print pages. + +See the [workbook inspection guide](../../guides/inspect-excel-workbooks.md) and +the [Document Search reference](../../reference/actions/document-search.md). + +## Validation and limitations + +Coverage includes real OOXML packages, canonical comment counts and contents, +saved/blank/uncached cell distinctions, final chat evidence, private-store +paging and authorization, table-aware previews, image anchors, and styled +desktop/mobile Playwright interaction. + +Relevant suites are `functional_tests/test_excel_*`, +`functional_tests/test_document_accessibility_report_action.py`, the tabular +analysis/replay suites, and `ui_tests/test_v2_excel_citation_inspection.py`. + +The investigated specimen's native model evidence contained the three threads +and one reply in 4,125 UTF-8 bytes; its metadata occupied 15,914 serialized bytes +and its original SHA-256 remained unchanged. A synthetic 50,000-row, +12-column worksheet expanded to 22,995,689 XML bytes, above the old 16 MiB +materialized-part ceiling. Its structural manifest occupied 1,746 bytes and a +reader-only CPython `tracemalloc` probe reported a 911,165-byte peak while +discovering the table and reading its last record. This measures traced Python +reader allocations, not total process RSS, analytical DataFrames, or a hosting +capacity guarantee. + +Inspection never recalculates Excel formulas, executes macros/ActiveX/OLE, +refreshes external links, or claims to reproduce Excel rendering. Conditional +formatting and other saved expressions are evidence, not evaluated appearance. +Legacy `.xls` is best effort for information its BIFF reader exposes. Input, +style-sampling, response-byte, and page limits remain explicit; partial evidence +must not be called an exhaustive whole-workbook result. diff --git a/docs/explanation/fixes/EXCEL_COMMENT_CONTENT_AND_TABLE_HEADERS_FIX.md b/docs/explanation/fixes/EXCEL_COMMENT_CONTENT_AND_TABLE_HEADERS_FIX.md new file mode 100644 index 000000000..65a20f70f --- /dev/null +++ b/docs/explanation/fixes/EXCEL_COMMENT_CONTENT_AND_TABLE_HEADERS_FIX.md @@ -0,0 +1,68 @@ +# Excel Comment Content and Table Headers Fix + +Fixed in version: **0.261.405**. +Application version is maintained in `application/single_app/config.py`. + +## Issue and root cause + +The Excel extractor could read comment text while the native chat handoff sent +only counts and a few shortened relevance excerpts. A request to list every +comment could therefore report that comment contents were unavailable even +though they had been extracted. + +Threaded comments' legacy compatibility copies were also classified as genuine +notes. One investigated specimen contained three threads and one reply plus +three compatibility copies, which appeared as seven independent comments. + +Preview/schema readers assumed the first worksheet row was its single header. +That misrepresented title rows and multiple tables. The V2 spreadsheet popup +also reused the image-specific extraction label. + +## Changes and impact + +Native evidence now prioritizes requested structured records and saved cell +context instead of depending on relevance hits. Compatibility correlation keeps +genuine notes and distinguishes roots, replies, and raw representations. +Long/native inventories remain available through revision-bound pages. +Reviewed publication preserves a native-evidence marker on every search +fragment and caps native search previews at one million characters, including +their markers. Complete inventories remain separately paged; sanitized +publication never republishes original native supplements. + +The shared streaming reader discovers named tables and ordinary regions, with +independent ranges and header locations. The preview and analytical selectors +use those boundaries rather than a worksheet-wide first-row header. +Released workbook schemas are generated from the active published bytes with +the same table-aware reader, including sanitized derivatives. + +The V2 popup labels workbook evidence accurately, identifies the selected sheet +and table/range, shows physical row coordinates, and pages records. Switching +between source sheets with identical values can legitimately leave the visible +values unchanged; this is not itself a switching defect. + +Primary components are `excel_workbook_reader.py`, +`functions_excel_accessibility.py`, `functions_excel_inventory.py`, +`functions_search_service.py`, `functions_documents.py`, the chat/tabular +adapters, and `EnhancedCitationViewer.tsx`. + +## Before and after + +| Before | After | +| --- | --- | +| Counts could reach the answer without comment text. | Requested comment records and available saved values are supplied together. | +| Compatibility notes inflated user-facing counts. | Genuine notes, threads, replies, and compatibility copies are distinguished. | +| A title could become the header for every worksheet table. | Each table/range has its own declared or explicitly inferred headers. | +| Large native records were lost at preview limits. | Parent previews are bounded; complete records have source-bound continuations. | +| Spreadsheet evidence was labelled as an image. | Workbook/CSV evidence has the appropriate label and stays tied to its citation. | + +## Validation + +Regression coverage uses real packages and inspects actual record contents and +final handoff text, including no relevance hits, blank cells, missing caches, +late rows, multiple tables, and unchanged original bytes. Private-store tests +cover restart/paging, revocation, source-bound cursors, failures, and cleanup. +UI tests exercise different and identical sheets, five tables, the 50,000th +record, error recovery, keyboard navigation, and desktop/mobile layout. + +See [Scalable Excel Native Inspection](../features/EXCEL_NATIVE_INSPECTION.md) +for the storage contract, test suites, and saved-state limitations. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 8d99cf09b..c8a1af536 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,38 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.405)** + +#### Bug Fixes + +* **Excel Comment Contents And Correct Thread Counts** + * Supplies actual native comment records and saved cell context to chat instead of only counts and shortened relevance excerpts. Reviewer requests remain separate from worksheet values. + * Distinguishes genuine legacy notes, threaded roots/replies, and identifiable compatibility copies so mirrored comments do not inflate counts. + * Existing retained originals can be inspected without re-uploading; missing formula caches remain unavailable rather than becoming zero. + * (Ref: `functions_excel_accessibility.py`, `functions_excel_inventory.py`, `functions_search_service.py`, `route_backend_chats.py`, `docs/explanation/fixes/EXCEL_COMMENT_CONTENT_AND_TABLE_HEADERS_FIX.md`) + +#### New Features + +* **Scalable, Table-Aware Saved Excel Inspection** + * Discovers independent named tables and ordinary cell blocks with their own ranges, title/header rows, and source coordinates across worksheets. + * Keeps compact document metadata while full native inventories use private, revision-bound pages. Analytical selection and durable replay retain table/range/header context. + * Adds saved properties, links/validation, readable chart/object content, and configured Office-image analysis with available Excel anchors. Formulas, macros, and external links are never executed/refreshed. + * (Ref: `excel_workbook_reader.py`, `tabular_processing_plugin.py`, `functions_tabular_generated_exports.py`, `docs/explanation/features/EXCEL_NATIVE_INSPECTION.md`) + +#### User Interface Enhancements + +* **Workbook Citation Tables And Bounded Record Navigation** + * Identifies the selected worksheet/table and actual header/row locations, with table selection, record paging, record jumps, keyboard navigation, and retryable selection errors. + * Uses workbook/CSV evidence labels instead of image wording and keeps the originally cited passage independent of preview navigation. + * (Ref: `EnhancedCitationViewer.tsx`, `route_enhanced_citations.py`, `docs/guides/inspect-excel-workbooks.md`) + +#### Breaking Changes + +* **Legacy Queued Workbook Replay Requires Restart** + * Pending workbook computations using v1 worksheet-header descriptors no longer resume under changed table/header semantics. + * **Migration**: Start a new computation to pin the intended worksheet/table/range and header. Existing completed exports are unchanged; CSV replay compatibility is retained. + * (Ref: `functions_tabular_generated_exports.py`, versioned workbook source descriptors) + ### **(v0.261.392)** #### Bug Fixes diff --git a/docs/guides/inspect-excel-workbooks.md b/docs/guides/inspect-excel-workbooks.md new file mode 100644 index 000000000..aeed1b9d7 --- /dev/null +++ b/docs/guides/inspect-excel-workbooks.md @@ -0,0 +1,76 @@ +--- +layout: page +title: "Inspect Excel data, comments, and structure" +description: "Read workbook feedback and metadata without confusing them with saved values or table calculations." +section: "Guides" +audience: user +version: "0.261.405" +--- + +Implemented in version: **0.261.405**, tracked in +`application/single_app/config.py`. + +## Ask about the file and its data separately + +Select the intended workspace workbook. To review feedback, ask: + +> List the genuine cell notes and threaded comments. Include worksheet/cell, +> saved value, author, text, replies, and available timestamps/status. +> Separate requested changes from the workbook's actual values. + +To compute business results, name the worksheet and table or range: + +> Describe the tables on Budget, then calculate total Amount for the Costs +> table. Do not combine other tables on that worksheet. + +An Excel cell can contain a saved formula without a cached result. SimpleChat +can report the formula, but it does not calculate Excel formulas or treat that +missing cache as zero. Recalculate/save in Excel when those results are needed. + +## Choose the actual table and headers + +A worksheet may contain title rows, five separate tables, or repeated printed +headers. Named Excel tables provide saved boundaries. Ordinary blocks have +detected boundaries and labelled inferred headers. When a header is ambiguous, +clarify the actual worksheet header row or that the range has no header. + +A schema preview is a sample. Use tabular analysis for exact counts, filters, +totals, and grouped results over all intended records. + +## Inspect a citation + +Open the workbook citation. The viewer identifies the selected worksheet, +table/range, and header row. Choose another **Table** when several regions are +present. Use **Previous records**, **Next records**, or **Go to record** for +large data. Record positions count data records; the left-hand row labels are +the original worksheet row numbers. + +**Cited workbook evidence** remains the passage supporting the answer. Switching +the preview does not rewrite it. Two different sheets may contain identical +values; use the visible worksheet identity and their metadata/comments rather +than assuming identical values mean the tab did not switch. + +## Large comment and metadata inventories + +An agent with Document Search can use +`get_document_accessibility_report(native_section="comments")` and follow each +`next_cursor` until `has_more` is false. Other sections include tables, objects, +properties, links, validation, and available image-analysis records. Direct cell +inspection accepts a worksheet and bounded A1 range. + +Returned pages, shortened parent previews, and explicit long-record fragments +are not exhaustive results. Check coverage and continuations before claiming +every comment or metadata record has been read. + +## Existing files and limitations + +Re-uploading is usually unnecessary when the active workbook is already stored. +Native inspection can generate new details from that retained original. +Configured image analysis may need reprocessing; ordinary metadata reads do not +silently run new vision analysis. If neither source bytes nor the needed +inventory remain, re-upload the file through the normal authorized workflow. + +Held/revoked sources require the appropriate access or review resolution. +Inspection never runs macros, ActiveX, embedded programs, or external refreshes, +and saved formatting is not a reproduction of Excel's rendered appearance. +Legacy `.xls` has narrower native coverage. diff --git a/docs/reference/actions/document-search.md b/docs/reference/actions/document-search.md index 98b124b94..7b37cfe08 100644 --- a/docs/reference/actions/document-search.md +++ b/docs/reference/actions/document-search.md @@ -52,11 +52,30 @@ From **0.261.399**, workspace Excel files also expose native workbook properties - Choose which parts to return with `sections` (`accessibility`, `style`, `issues`); all are returned by default. - Pass `page_number` to also get that page, slide or Excel indexed evidence segment's stored text. Excel segment numbers are not physical print-page numbers. -- Chat uploads are not supported; add the file to a workspace. Documents uploaded before this data was extracted need to be re-uploaded. +- The stored workspace report does not grant access to chat uploads. For an existing workspace Excel file, retained active source bytes can be inspected on demand without re-uploading; unavailable originals need the normal re-upload/reprocessing workflow. - The issues list is heuristic. It is not a formal WCAG, Section 508, or PDF/UA conformance check. To apply your own rules, write them into the agent's instructions and have it compare them with the returned values. See [Document accessibility, structure, and comment extraction](https://github.com/microsoft/simplechat/blob/13db876626a043e2c7b30f054354f07c9a1633b9/docs/explanation/features/DOCUMENT_ACCESSIBILITY_AND_STRUCTURE_EXTRACTION.md) for what is extracted for each file type. +### Paged Excel inspection + +Implemented in version: **0.261.405** (`application/single_app/config.py`). +Use `native_section` to request `comments`, `tables`, `objects`, +`property_records`, `custom_properties`, `extended_properties`, `defined_names`, +`hyperlinks`, `validations`, or available `image_analyses`. The default is +`comments`. `native_page_size` accepts 1–200 records, with a separate response +byte bound. Follow `excel_inspection.page.next_cursor` using `native_cursor` +while `has_more` is true. Explicit long JSON record fragments require their +continuation before the record is complete. + +`sheet_name` and `cell_range` read an A1 range of at most 2,000 saved cells, +including formula text/cache availability, without executing formulas. +Compatibility copies of threaded comments are not extra genuine notes. +Native evidence segment citations are not Excel print-page numbers. + +See [Inspect Excel workbooks]({{ '/guides/inspect-excel-workbooks/' | relative_url }}) +for table headers, saved values, and large-file workflows. + Shared wizard steps: [Common action setup steps](../#common-action-setup-steps). ## Related diff --git a/docs/reference/actions/tabular-processing.md b/docs/reference/actions/tabular-processing.md index 7c17ab569..2afeb5df0 100644 --- a/docs/reference/actions/tabular-processing.md +++ b/docs/reference/actions/tabular-processing.md @@ -46,6 +46,42 @@ Use Tabular Processing when users ask exact questions about spreadsheets or CSV - "Find rows where RenewalDate is in the next 30 days and Status is not Closed." - "Compare the Account sheet to the Tickets sheet and count open tickets by account owner." +## Tables, ranges, and saved formulas + +Implemented in version: **0.261.405** (`application/single_app/config.py`). +Describe a workbook before choosing a table when a worksheet contains several +regions. Named tables retain their saved definitions; ordinary blocks expose +their detected ranges and labelled inferred headers. A title row is not a +header for every table below it. + +Use the analytical operation's table/range/header selectors for the intended +dataset: `table_id` selects a discovered ID or named table; `cell_range` +selects an explicit Excel range and requires `header_row`. Specify the actual +worksheet header row when it is ambiguous; zero means no header. Related-source +operations expose the equivalent `source_` and `target_` selectors. An explicit +`table_id="all"` combines compatible same-schema tables, not unrelated schemas. +Do not silently combine different schemas. Explicit all-table +native jobs retain each selected table's identity through batching and replay. + +Physical worksheet/table/row provenance accompanies Excel row results. +Version-pinned exports preserve the selected ranges and headers instead of +reinterpreting the entire sheet on continuation. Large outputs still use the +existing bounded paging and durable export path. + +Queued workbook computations with legacy v1 worksheet-header descriptors must +be started again so the intended tables/headers can be pinned explicitly. +Previously completed export files are not rewritten. CSV descriptor compatibility +is retained. + +Formula text is inspectable, but analytical values require saved formula +caches. Missing results must not become zero. Recalculate/save the workbook in +Excel if those values are required; SimpleChat does not execute workbook +formulas, macros, or external refreshes. + +For reviewer feedback and workbook properties, use +[Document Search](../document-search/) rather than interpreting comments as data. +See [Inspect Excel workbooks]({{ '/guides/inspect-excel-workbooks/' | relative_url }}). + ## Troubleshooting | Symptom | Likely cause | Fix | diff --git a/docs/reference/chat-controls.md b/docs/reference/chat-controls.md index 86f497b10..bd6134021 100644 --- a/docs/reference/chat-controls.md +++ b/docs/reference/chat-controls.md @@ -4,7 +4,7 @@ title: "Chat interface controls" description: "Reference for every documented control in the SimpleChat chat interface." section: "Reference" audience: user -version: "0.261.398" +version: "0.261.405" --- ## How to use this reference @@ -43,6 +43,23 @@ for source selection, handoff review, and the private-conversation boundary. ## Conversation header and status +### Workbook citation preview + +Implemented in version: **0.261.405** (`application/single_app/config.py`). +Workbook citations show the selected worksheet and table/range independently +from their originally cited passage. + +| V2 control | Purpose | Availability | +| --- | --- | --- | +| Workbook worksheet tabs | Select a worksheet with mouse or Arrow/Home/End keys; the selected identity remains visible even when two sheets have identical values. | Enhanced workbook citation with multiple sheets | +| Table | Chooses one independently detected/named table instead of applying one header to the whole sheet. | Selected sheet has multiple tables | +| Previous records / Next records | Loads a bounded page rather than rendering the entire dataset. | Spreadsheet/CSV citation preview | +| Go to record | Navigates to a one-based data-record position; worksheet row labels preserve physical source coordinates. | Spreadsheet/CSV citation preview | +| Retry preview | Retries a failed selection without relabelling previously loaded data as the failed sheet/table. | Preview load failure | +| Cited workbook evidence / Cited CSV evidence | Shows the original cited extraction passage, not a new passage inferred from preview navigation. | Cited passage is available | + +See [Inspect Excel workbooks]({{ '/guides/inspect-excel-workbooks/' | relative_url }}). + ### Conversation details {% include media.html id="slide-conversation-details" %} diff --git a/functional_tests/cosmos_response_boundaries.json b/functional_tests/cosmos_response_boundaries.json index a3f628a88..eb4a4cf03 100644 --- a/functional_tests/cosmos_response_boundaries.json +++ b/functional_tests/cosmos_response_boundaries.json @@ -6036,51 +6036,51 @@ }, "functional_tests/test_tabular_row_orchestration_scale.py::_load_authorization_helper.ConversationContainer": { "boundary": { - "imports_sha256": "1b3486a678ce86b217c2d0984b5f1c5282175a7e46794a311f51e5d9ddd9abaf", - "inherited_bases": [], - "kind": "class", + "sha256": "354ad9c7632aaca0de50389f2097fc278368c12c68628e2cef378b6744363159", "methods": [ "read_item" ], - "sha256": "9e3f33f299a94a67c8d37b54e0b6a5d7832bc7dccb3a11a1f9e500d63b9a08f6" + "kind": "class", + "inherited_bases": [], + "imports_sha256": "d0ecc00fea7e0f993ed5b80616e34bd0e626d4b80854f46622aaddadef386fca" }, "classification": "sdk-default", - "contract": "Authorization fixture point reads return the installed SDK CosmosDict by default, preserving the existing owner fields and conversation partition assertion.", + "contract": "Installed-SDK CosmosDict point reads preserve ownership and ETags, isolate returned mutations, and reject a wrong partition with SDK 404.", "tests": [ - "functional_tests/test_export_evidence_cosmos_contract.py::test_changed_tabular_test_containers_now_return_sdk_points_by_default" + "functional_tests/test_tabular_row_orchestration_scale.py::test_sdk_default_authorization_conversation_points_preserve_owner_etag_and_partition" ] }, "functional_tests/test_tabular_row_orchestration_scale.py::_load_idempotent_artifact_helper.ConversationContainer": { "boundary": { - "imports_sha256": "1b3486a678ce86b217c2d0984b5f1c5282175a7e46794a311f51e5d9ddd9abaf", - "inherited_bases": [], - "kind": "class", + "sha256": "d6e6085c092ae42161772dedadf34e69ef73eaed809d917beed0b264838949ee", "methods": [ "read_item" ], - "sha256": "e7602052cff10d7c40e3d936520a8aef3e9f7d58082bb786bbfab8d7191a3315" + "kind": "class", + "inherited_bases": [], + "imports_sha256": "d0ecc00fea7e0f993ed5b80616e34bd0e626d4b80854f46622aaddadef386fca" }, "classification": "sdk-default", - "contract": "Artifact-upload fixture conversation reads return the installed SDK CosmosDict by default without changing the owner identity supplied to the real upload helper.", + "contract": "Installed-SDK CosmosDict conversation reads preserve ownership and ETags, isolate returned mutations, and reject wrong partitions with SDK 404.", "tests": [ - "functional_tests/test_export_evidence_cosmos_contract.py::test_changed_tabular_test_containers_now_return_sdk_points_by_default" + "functional_tests/test_tabular_row_orchestration_scale.py::test_sdk_default_artifact_conversation_points_preserve_owner_etag_and_partition" ] }, "functional_tests/test_tabular_row_orchestration_scale.py::_load_idempotent_artifact_helper.MessageContainer": { "boundary": { - "imports_sha256": "1b3486a678ce86b217c2d0984b5f1c5282175a7e46794a311f51e5d9ddd9abaf", - "inherited_bases": [], - "kind": "class", + "sha256": "dc02ff09ef6ba766d597d847dcd0d8f1f3072b897324821d574b21c526385678", "methods": [ "read_item", "upsert_item" ], - "sha256": "38864b1c2e1ceb96592fdcb4071ff480ed6d2f5030f8c1284393f885de82aa83" + "kind": "class", + "inherited_bases": [], + "imports_sha256": "d0ecc00fea7e0f993ed5b80616e34bd0e626d4b80854f46622aaddadef386fca" }, "classification": "sdk-default", - "contract": "Artifact fixture read/upsert results default to the installed SDK CosmosDict. Stored JSON fields, supplied ETags, not-found behavior and upsert counting remain unchanged.", + "contract": "Installed-SDK CosmosDict message reads and upserts preserve versioned ETags and partition routing; stale or missing conditional writes reject atomically with SDK 412.", "tests": [ - "functional_tests/test_export_evidence_cosmos_contract.py::test_changed_tabular_test_containers_now_return_sdk_points_by_default" + "functional_tests/test_tabular_row_orchestration_scale.py::test_sdk_default_artifact_message_points_preserve_versions_partitions_and_atomic_failures" ] }, "functional_tests/test_user_plugin_bulk_save_id_preservation.py::UserPluginBulkSaveTests.test_missing_legacy_graph_id_is_rejected_before_batch_mutations::point-seam-1": { @@ -7615,10 +7615,10 @@ ], "queries": [], "batches": [], - "module_sha256": "0371337955a69c822cbb5c5fadb0a8d858022a8ee6fe68deecc063d8c1a011fd" + "module_sha256": "039d3c3ae084862d761feb203f9f774377d60cc29b1ff7e31e0b0e6bc7322ba2" }, "classification": "sdk-compatible", - "contract": "_read_authorized_document: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Review 0.261.395: SDK-shaped current document/release-proof reads and plain blob/native query locators are executed through the real owners; canonical source scope is enforced before document consumption.", + "contract": "_read_authorized_document: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Review 0.261.395: SDK-shaped current document/release-proof reads and plain blob/native query locators are executed through the real owners; canonical source scope is enforced before document consumption. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_v2_document_source_runtime.py::test_real_source_owners_keep_sdk_points_release_proofs_and_plain_query_rows" @@ -7636,10 +7636,10 @@ ], "queries": [], "batches": [], - "module_sha256": "0371337955a69c822cbb5c5fadb0a8d858022a8ee6fe68deecc063d8c1a011fd" + "module_sha256": "039d3c3ae084862d761feb203f9f774377d60cc29b1ff7e31e0b0e6bc7322ba2" }, "classification": "sdk-compatible", - "contract": "_require_release_proof: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Review 0.261.395: SDK-shaped current document/release-proof reads and plain blob/native query locators are executed through the real owners; canonical source scope is enforced before document consumption.", + "contract": "_require_release_proof: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Review 0.261.395: SDK-shaped current document/release-proof reads and plain blob/native query locators are executed through the real owners; canonical source scope is enforced before document consumption. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_v2_document_source_runtime.py::test_real_source_owners_keep_sdk_points_release_proofs_and_plain_query_rows" @@ -7668,10 +7668,10 @@ } ], "batches": [], - "module_sha256": "0371337955a69c822cbb5c5fadb0a8d858022a8ee6fe68deecc063d8c1a011fd" + "module_sha256": "039d3c3ae084862d761feb203f9f774377d60cc29b1ff7e31e0b0e6bc7322ba2" }, "classification": "sdk-compatible", - "contract": "_resolve_blob_document: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Review 0.261.395: SDK-shaped current document/release-proof reads and plain blob/native query locators are executed through the real owners; canonical source scope is enforced before document consumption.", + "contract": "_resolve_blob_document: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Review 0.261.395: SDK-shaped current document/release-proof reads and plain blob/native query locators are executed through the real owners; canonical source scope is enforced before document consumption. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_v2_document_source_runtime.py::test_real_source_owners_keep_sdk_points_release_proofs_and_plain_query_rows" @@ -7689,10 +7689,10 @@ } ], "batches": [], - "module_sha256": "0371337955a69c822cbb5c5fadb0a8d858022a8ee6fe68deecc063d8c1a011fd" + "module_sha256": "039d3c3ae084862d761feb203f9f774377d60cc29b1ff7e31e0b0e6bc7322ba2" }, "classification": "sdk-compatible", - "contract": "_resolve_native_evidence: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Review 0.261.395: SDK-shaped current document/release-proof reads and plain blob/native query locators are executed through the real owners; canonical source scope is enforced before document consumption.", + "contract": "_resolve_native_evidence: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Review 0.261.395: SDK-shaped current document/release-proof reads and plain blob/native query locators are executed through the real owners; canonical source scope is enforced before document consumption. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_v2_document_source_runtime.py::test_real_source_owners_keep_sdk_points_release_proofs_and_plain_query_rows" @@ -14120,10 +14120,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "response-unused", - "contract": "_complete_document_search_sync_pass: point response values are discarded; query rows, if present, are separate plain SDK JSON. Existing write errors remain visible.", + "contract": "_complete_document_search_sync_pass: point response values are discarded; query rows, if present, are separate plain SDK JSON. Existing write errors remain visible. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "source-review", "tests": [] }, @@ -14139,10 +14139,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "response-unused", - "contract": "_delete_document_search_sync_record: point response values are discarded; query rows, if present, are separate plain SDK JSON. Existing write errors remain visible.", + "contract": "_delete_document_search_sync_record: point response values are discarded; query rows, if present, are separate plain SDK JSON. Existing write errors remain visible. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "source-review", "tests": [] }, @@ -14158,10 +14158,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "_get_document_family_items_from_document: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "_get_document_family_items_from_document: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14185,10 +14185,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "_process_document_upload_background_impl: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Bookkeeping review: final and partial counts use version/scoped Search keys plus acknowledged mutations; Cosmos activity-log points remain mapping consumers.", + "contract": "_process_document_upload_background_impl: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Bookkeeping review: final and partial counts use version/scoped Search keys plus acknowledged mutations; Cosmos activity-log points remain mapping consumers. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14209,10 +14209,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "_query_current_xsd_documents: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "_query_current_xsd_documents: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14231,10 +14231,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "_read_document_for_search_sync: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "_read_document_for_search_sync: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14253,10 +14253,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "_read_document_search_sync_record: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "_read_document_search_sync_record: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14275,10 +14275,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "_rebase_document_after_fenced_acl_projection: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "_rebase_document_after_fenced_acl_projection: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14297,10 +14297,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "_replace_document_search_sync_record: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "_replace_document_search_sync_record: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14334,10 +14334,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "_upsert_document_and_sync_access_index: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "_upsert_document_and_sync_access_index: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14356,10 +14356,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "_workspace_file_name_exists: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "_workspace_file_name_exists: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14378,10 +14378,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "create_document: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "create_document: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14390,7 +14390,7 @@ }, "application/single_app/functions_documents.py::delete_document": { "boundary": { - "sha256": "080105834da852cd36f85bc6ca2aecac5fe7519a943cb16e35338ff2bcb3bd38", + "sha256": "e951293cdfee47a6801b4a31b9c6eb74edbf834e93ee1ea0917f43e49f2b4147", "points": [ { "method": "read_item", @@ -14425,14 +14425,14 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "delete_document: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "Excel revision cleanup consumes installed-SDK CosmosDict point documents as mappings, checks owner/scope and ETag before native-artifact deletion, and preserves conditional document deletion. The real deletion owner is exercised for personal/group/public SDK responses and stale ETag failure; no strict top-level dict/JSON consumer was added.", "evidence_scope": "owner-runtime", "tests": [ - "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", - "functional_tests/test_content_screening_upload_only_intake.py::test_model_generated_metadata_applies_without_a_hold" + "functional_tests/test_excel_document_delete_sdk_contract.py::test_real_delete_owner_handles_sdk_points_and_scope_bound_native_cleanup", + "functional_tests/test_excel_document_delete_sdk_contract.py::test_stale_sdk_etag_prevents_private_inventory_deletion" ] }, "application/single_app/functions_documents.py::delete_document_revision": { @@ -14452,10 +14452,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "delete_document_revision: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "delete_document_revision: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14474,10 +14474,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "delete_from_blob_storage: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "delete_from_blob_storage: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14506,10 +14506,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "detect_doc_type: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "detect_doc_type: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14528,10 +14528,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "extract_document_metadata: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "extract_document_metadata: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14550,10 +14550,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "get_chat_upload_workspace_documents_for_collaboration: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "get_chat_upload_workspace_documents_for_collaboration: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14577,10 +14577,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "get_chat_upload_workspace_documents_for_conversation: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "get_chat_upload_workspace_documents_for_conversation: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14599,10 +14599,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "get_document_metadata: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "get_document_metadata: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14621,10 +14621,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "get_document_metadata_for_citations: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "get_document_metadata_for_citations: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14643,10 +14643,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "get_document_record: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "get_document_record: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14665,10 +14665,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "get_document_version: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "get_document_version: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14687,10 +14687,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "get_document_versions: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "get_document_versions: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14709,10 +14709,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "get_documents_shared_with_group: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "get_documents_shared_with_group: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14731,10 +14731,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "get_documents_shared_with_user: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "get_documents_shared_with_user: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14753,10 +14753,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "get_latest_version: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "get_latest_version: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14775,10 +14775,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "get_shared_groups_for_document: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "get_shared_groups_for_document: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14797,10 +14797,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "get_shared_users_for_document: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "get_shared_users_for_document: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14819,10 +14819,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "get_workspace_tags: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "get_workspace_tags: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14841,10 +14841,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "is_document_shared_with_group: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "is_document_shared_with_group: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14863,10 +14863,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "is_document_shared_with_user: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "is_document_shared_with_user: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14885,10 +14885,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "process_due_document_search_metadata_syncs: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "process_due_document_search_metadata_syncs: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14907,10 +14907,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "propagate_tags_to_blob_metadata: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "propagate_tags_to_blob_metadata: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14929,10 +14929,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "response-unused", - "contract": "queue_group_workspace_upload_from_temp_file: point response values are discarded; query rows, if present, are separate plain SDK JSON. Existing write errors remain visible.", + "contract": "queue_group_workspace_upload_from_temp_file: point response values are discarded; query rows, if present, are separate plain SDK JSON. Existing write errors remain visible. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "source-review", "tests": [] }, @@ -14948,10 +14948,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "response-unused", - "contract": "queue_personal_workspace_upload_from_temp_file: point response values are discarded; query rows, if present, are separate plain SDK JSON. Existing write errors remain visible.", + "contract": "queue_personal_workspace_upload_from_temp_file: point response values are discarded; query rows, if present, are separate plain SDK JSON. Existing write errors remain visible. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "source-review", "tests": [] }, @@ -14967,10 +14967,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "request_document_search_metadata_sync: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "request_document_search_metadata_sync: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -14989,10 +14989,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "share_document_with_group: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "share_document_with_group: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -15011,10 +15011,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "share_document_with_user: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "share_document_with_user: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -15038,10 +15038,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "sync_chat_upload_workspace_attachment_status: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "sync_chat_upload_workspace_attachment_status: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -15060,10 +15060,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "unshare_document_from_group: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "unshare_document_from_group: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -15082,10 +15082,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "unshare_document_from_user: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "unshare_document_from_user: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -15104,10 +15104,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "Workspace-scoped query rows are plain dictionaries. Status predicates only interpret strings; filenames cannot force success/failure or rewrite screening copy. Strict update returns the SDK saved mapping to mapping-compatible callers; read/write ETags, operation guards, access projection and screening holds are preserved. Runtime evidence exercises personal/group/public status writes with installed-SDK point responses and plain query rows. Bookkeeping review: only an exact legacy metadata warning is retired after a successful outcome; that transition uses the existing ETag-conditional storage helper. SDK mappings, query rows, scope and concurrent-state checks remain intact.", + "contract": "Workspace-scoped query rows are plain dictionaries. Status predicates only interpret strings; filenames cannot force success/failure or rewrite screening copy. Strict update returns the SDK saved mapping to mapping-compatible callers; read/write ETags, operation guards, access projection and screening holds are preserved. Runtime evidence exercises personal/group/public status writes with installed-SDK point responses and plain query rows. Bookkeeping review: only an exact legacy metadata warning is retired after a successful outcome; that transition uses the existing ETag-conditional storage helper. SDK mappings, query rows, scope and concurrent-state checks remain intact. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -15130,10 +15130,10 @@ } ], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "upgrade_legacy_documents: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "upgrade_legacy_documents: query rows/pages are plain JSON, not Cosmos point wrappers. Nested domain shape checks therefore need no point-response repair. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -15152,10 +15152,10 @@ ], "queries": [], "batches": [], - "module_sha256": "3069229d414393155e3ae4cfe19cfd699d353a5bf777d1c600f1fb95e20023e7" + "module_sha256": "4480c9650ded08edd11e42ebd5179ae5a94d829b4f3332b6642e362de8238502" }, "classification": "sdk-compatible", - "contract": "upload_to_blob: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation.", + "contract": "upload_to_blob: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Workspace outcome review: existing mapping/query consumers and conditional write contracts are retained; representative module runtime behavior is covered with installed-SDK point responses, including ETags and scope propagation. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workspace_metadata_outcomes.py::test_upload_dispatcher_preserves_ready_content_after_metadata_failure", @@ -23184,8 +23184,7 @@ }, "application/single_app/functions_search_service.py::_authorize_chat_upload_conversation": { "boundary": { - "batches": [], - "module_sha256": "30fd9ba3399ef45d3fe895558850c901e45d3a5895295d7db5f4e6c4968a33eb", + "sha256": "3c3492a4d44ec21724c0682922e0f6fbd3dc68894ec4bdacf20d82294159d42e", "points": [ { "method": "read_item", @@ -23194,10 +23193,11 @@ } ], "queries": [], - "sha256": "3c3492a4d44ec21724c0682922e0f6fbd3dc68894ec4bdacf20d82294159d42e" + "batches": [], + "module_sha256": "0afaa631fb35d52b9022fa8c7e777695aa84a450fc6fcbff2e7b6f1b04636a77" }, "classification": "sdk-compatible", - "contract": "_authorize_chat_upload_conversation: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained.", + "contract": "_authorize_chat_upload_conversation: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workflow_authoring_sdk_boundaries.py::test_search_upload_owners_consume_real_sdk_point_documents" @@ -23205,8 +23205,7 @@ }, "application/single_app/functions_search_service.py::_resolve_chat_upload_context": { "boundary": { - "batches": [], - "module_sha256": "30fd9ba3399ef45d3fe895558850c901e45d3a5895295d7db5f4e6c4968a33eb", + "sha256": "e795483e3a90794e3268e2833ba72694964ea9173814a9cd6a1f7e4b045e23ee", "points": [ { "method": "read_item", @@ -23221,10 +23220,11 @@ "response": "consumed" } ], - "sha256": "e795483e3a90794e3268e2833ba72694964ea9173814a9cd6a1f7e4b045e23ee" + "batches": [], + "module_sha256": "0afaa631fb35d52b9022fa8c7e777695aa84a450fc6fcbff2e7b6f1b04636a77" }, "classification": "sdk-compatible", - "contract": "_resolve_chat_upload_context: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained.", + "contract": "_resolve_chat_upload_context: point documents use Mapping field reads/JSON encoding/metadata projections; batch status/body lists are separate, never top-level-normalized as documents. Strict domain sinks remain with their storage owners. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_workflow_authoring_sdk_boundaries.py::test_search_upload_owners_consume_real_sdk_point_documents" @@ -24062,8 +24062,7 @@ }, "application/single_app/functions_tabular_generated_exports.py::_authorize_tabular_export_run_conversation": { "boundary": { - "batches": [], - "module_sha256": "0946bdc93ce63940e2e61ef2f3483c0e230e589a2ecafa0e3de6291401fc5893", + "sha256": "2db7ce9752384889b7e3c88a920b73086d376abad2b561cc916be2fcb67be600", "points": [ { "method": "read_item", @@ -24072,10 +24071,11 @@ } ], "queries": [], - "sha256": "2db7ce9752384889b7e3c88a920b73086d376abad2b561cc916be2fcb67be600" + "batches": [], + "module_sha256": "6c105b2f4f84d1fb3fdb0fce5f3a9d2d39b37c98dee01215ade1deaa1fb8b911" }, "classification": "sdk-compatible", - "contract": "The partitioned conversation CosmosDict is checked through its user_id and returned unchanged; missing conversations and ownership changes remain PermissionError refusals. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical.", + "contract": "The partitioned conversation CosmosDict is checked through its user_id and returned unchanged; missing conversations and ownership changes remain PermissionError refusals. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_v2_public_source_owner_sdk.py::test_tabular_export_owner_keeps_sdk_etags_and_public_scope_fences" @@ -24083,8 +24083,7 @@ }, "application/single_app/functions_tabular_generated_exports.py::_authorize_tabular_export_run_execution": { "boundary": { - "batches": [], - "module_sha256": "0946bdc93ce63940e2e61ef2f3483c0e230e589a2ecafa0e3de6291401fc5893", + "sha256": "7d4f6e9fff2a09cfdadff4a52acea40827b6c3b75fdbab423bc5ccf804ce7f72", "points": [ { "method": "read_item", @@ -24093,10 +24092,11 @@ } ], "queries": [], - "sha256": "7d4f6e9fff2a09cfdadff4a52acea40827b6c3b75fdbab423bc5ccf804ce7f72" + "batches": [], + "module_sha256": "6c105b2f4f84d1fb3fdb0fce5f3a9d2d39b37c98dee01215ade1deaa1fb8b911" }, "classification": "sdk-compatible", - "contract": "The conversation and optional producer point mappings feed field-based owner/native-compute checks. Public source ids and blob provenance are revalidated separately, and the authorized conversation mapping is returned. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical.", + "contract": "The conversation and optional producer point mappings feed field-based owner/native-compute checks. Public source ids and blob provenance are revalidated separately, and the authorized conversation mapping is returned. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_v2_public_source_owner_sdk.py::test_tabular_export_owner_keeps_sdk_etags_and_public_scope_fences" @@ -24104,8 +24104,7 @@ }, "application/single_app/functions_tabular_generated_exports.py::_query_scheduler_candidates_by_status": { "boundary": { - "batches": [], - "module_sha256": "0946bdc93ce63940e2e61ef2f3483c0e230e589a2ecafa0e3de6291401fc5893", + "sha256": "95dd3db2bdbcca82dcbad9eb846bcc46410a5043892925275d0b97976ef1241e", "points": [], "queries": [ { @@ -24114,10 +24113,11 @@ "response": "consumed" } ], - "sha256": "95dd3db2bdbcca82dcbad9eb846bcc46410a5043892925275d0b97976ef1241e" + "batches": [], + "module_sha256": "6c105b2f4f84d1fb3fdb0fce5f3a9d2d39b37c98dee01215ade1deaa1fb8b911" }, "classification": "sdk-compatible", - "contract": "Scheduler discovery iterates plain projected query rows, applies eligibility before the bounded result limit and returns those rows; query failures are logged and re-raised. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical.", + "contract": "Scheduler discovery iterates plain projected query rows, applies eligibility before the bounded result limit and returns those rows; query failures are logged and re-raised. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_v2_public_source_owner_sdk.py::test_tabular_export_owner_keeps_sdk_etags_and_public_scope_fences" @@ -24125,8 +24125,7 @@ }, "application/single_app/functions_tabular_generated_exports.py::_read_run": { "boundary": { - "batches": [], - "module_sha256": "0946bdc93ce63940e2e61ef2f3483c0e230e589a2ecafa0e3de6291401fc5893", + "sha256": "64e6ce341659ab2caf67081615f6fdcb6f5cf08a6a00adbb2c79bae62ebbbf00", "points": [ { "method": "read_item", @@ -24135,10 +24134,11 @@ } ], "queries": [], - "sha256": "64e6ce341659ab2caf67081615f6fdcb6f5cf08a6a00adbb2c79bae62ebbbf00" + "batches": [], + "module_sha256": "6c105b2f4f84d1fb3fdb0fce5f3a9d2d39b37c98dee01215ade1deaa1fb8b911" }, "classification": "sdk-compatible", - "contract": "The exact user/run partitioned read returns the SDK mapping unchanged to field-based run consumers, preserving nested source descriptors and ETags. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical.", + "contract": "The exact user/run partitioned read returns the SDK mapping unchanged to field-based run consumers, preserving nested source descriptors and ETags. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_v2_public_source_owner_sdk.py::test_tabular_export_owner_keeps_sdk_etags_and_public_scope_fences" @@ -24146,8 +24146,7 @@ }, "application/single_app/functions_tabular_generated_exports.py::_replace_run": { "boundary": { - "batches": [], - "module_sha256": "0946bdc93ce63940e2e61ef2f3483c0e230e589a2ecafa0e3de6291401fc5893", + "sha256": "c5bbea5e76c5ff60dfbb2bc32fc91949ec38988b5630578ea0cc1dafbc9c973a", "points": [ { "method": "replace_item", @@ -24156,10 +24155,11 @@ } ], "queries": [], - "sha256": "c5bbea5e76c5ff60dfbb2bc32fc91949ec38988b5630578ea0cc1dafbc9c973a" + "batches": [], + "module_sha256": "6c105b2f4f84d1fb3fdb0fce5f3a9d2d39b37c98dee01215ade1deaa1fb8b911" }, "classification": "sdk-compatible", - "contract": "The existing run mapping and its ETag are passed to IfNotModified replacement; the returned CosmosDict and fresh ETag remain available to the caller, and stale ETags still fail. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical.", + "contract": "The existing run mapping and its ETag are passed to IfNotModified replacement; the returned CosmosDict and fresh ETag remain available to the caller, and stale ETags still fail. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_v2_public_source_owner_sdk.py::test_tabular_export_owner_keeps_sdk_etags_and_public_scope_fences" @@ -24167,8 +24167,7 @@ }, "application/single_app/functions_tabular_generated_exports.py::get_tabular_generated_output_run_status": { "boundary": { - "batches": [], - "module_sha256": "0946bdc93ce63940e2e61ef2f3483c0e230e589a2ecafa0e3de6291401fc5893", + "sha256": "37033d74b534cb3d85ef135810b7eb61d429c05cf514899562a5e1ea9dc5a548", "points": [ { "method": "read_item", @@ -24177,10 +24176,11 @@ } ], "queries": [], - "sha256": "37033d74b534cb3d85ef135810b7eb61d429c05cf514899562a5e1ea9dc5a548" + "batches": [], + "module_sha256": "6c105b2f4f84d1fb3fdb0fce5f3a9d2d39b37c98dee01215ade1deaa1fb8b911" }, "classification": "sdk-compatible", - "contract": "The user-partitioned SDK run mapping feeds field-based screening and bounded completion repair; the status builder accepts dict subclasses and constructs an explicit plain public projection. Representative runtime evidence exercises this module's read/replace/authorization/query boundaries, not every status branch. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical.", + "contract": "The user-partitioned SDK run mapping feeds field-based screening and bounded completion repair; the status builder accepts dict subclasses and constructs an explicit plain public projection. Representative runtime evidence exercises this module's read/replace/authorization/query boundaries, not every status branch. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_v2_public_source_owner_sdk.py::test_tabular_export_owner_keeps_sdk_etags_and_public_scope_fences" @@ -24188,8 +24188,7 @@ }, "application/single_app/functions_tabular_generated_exports.py::queue_tabular_generated_output_run": { "boundary": { - "batches": [], - "module_sha256": "0946bdc93ce63940e2e61ef2f3483c0e230e589a2ecafa0e3de6291401fc5893", + "sha256": "eb5b3df6fef7c091dcea83e7295773d6f7d483cf18bff0fbba16fcaf42b10dd8", "points": [ { "method": "create_item", @@ -24203,10 +24202,11 @@ } ], "queries": [], - "sha256": "eb5b3df6fef7c091dcea83e7295773d6f7d483cf18bff0fbba16fcaf42b10dd8" + "batches": [], + "module_sha256": "6c105b2f4f84d1fb3fdb0fce5f3a9d2d39b37c98dee01215ade1deaa1fb8b911" }, "classification": "response-unused", - "contract": "Both queue create responses are discarded; the constructed plain run is returned for new work. The data-only replay/collision branches use _read_run mappings, compare exact compute/source fields and reauthorize before returning the existing SDK run. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical.", + "contract": "Both queue create responses are discarded; the constructed plain run is returned for new work. The data-only replay/collision branches use _read_run mappings, compare exact compute/source fields and reauthorize before returning the existing SDK run. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Only model protocol selection in the budget helper changed. Point/query/batch consumers, source authorization, ETags, publication ownership and conditional storage updates remain identical. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "source-review", "tests": [] }, @@ -26903,8 +26903,7 @@ }, "application/single_app/route_backend_chats.py::_build_tabular_related_document_catalog": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "82662957f069353aefbf2a550a0c79d4812d7390c33162431d2ca95d6607cba8", "points": [], "queries": [ { @@ -26913,10 +26912,11 @@ "response": "consumed" } ], - "sha256": "82662957f069353aefbf2a550a0c79d4812d7390c33162431d2ca95d6607cba8" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "Workspace document discovery consumes plain query rows and builds bounded related-document metadata; no Cosmos point wrapper enters its document catalog. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Workspace document discovery consumes plain query rows and builds bounded related-document metadata; no Cosmos point wrapper enters its document catalog. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -26926,8 +26926,7 @@ }, "application/single_app/route_backend_chats.py::_create_personal_conversation": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "fe575c723ec7aa2ecdc1e1fbe7eb10d77fe71508f0d629aa8836b974abdd9618", "points": [ { "method": "upsert_item", @@ -26941,10 +26940,11 @@ } ], "queries": [], - "sha256": "fe575c723ec7aa2ecdc1e1fbe7eb10d77fe71508f0d629aa8836b974abdd9618" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "response-unused", - "contract": "Both conversation upsert responses are discarded; the constructed plain conversation is logged and cache-invalidated, with existing persistence failures unchanged. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Both conversation upsert responses are discarded; the constructed plain conversation is logged and cache-invalidated, with existing persistence failures unchanged. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py" @@ -26952,8 +26952,7 @@ }, "application/single_app/route_backend_chats.py::_get_user_message_image_context": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "9e47c6d3efd9690037f39ffbb8a98eb7bfa2457a76d9f4810161a8bb8d2b83b4", "points": [ { "method": "read_item", @@ -26962,10 +26961,11 @@ } ], "queries": [], - "sha256": "9e47c6d3efd9690037f39ffbb8a98eb7bfa2457a76d9f4810161a8bb8d2b83b4" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "The user-message point mapping supplies nested plain metadata/thread fields for paired image attribution; only user/thread values escape the boundary. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "The user-message point mapping supplies nested plain metadata/thread fields for paired image attribution; only user/thread values escape the boundary. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -26975,8 +26975,7 @@ }, "application/single_app/route_backend_chats.py::_load_user_message_response_context": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "0c5fba39f955b382d6e3aaae020f036a56bdfc83993b061349c5a1790f1e482f", "points": [ { "method": "read_item", @@ -26985,10 +26984,11 @@ } ], "queries": [], - "sha256": "0c5fba39f955b382d6e3aaae020f036a56bdfc83993b061349c5a1790f1e482f" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "The user-message point mapping is read through get() into a plain response-context dictionary; nested thread/user metadata stays JSON and read failures retain existing fallback fields. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "The user-message point mapping is read through get() into a plain response-context dictionary; nested thread/user metadata stays JSON and read failures retain existing fallback fields. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -26998,8 +26998,7 @@ }, "application/single_app/route_backend_chats.py::_persist_screened_assistant": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "3dc11565c768ba2dba4022d2d2e79c54f0f949cdcdcd0d270ceab567a7adc7d1", "points": [ { "method": "delete_item", @@ -27008,10 +27007,11 @@ } ], "queries": [], - "sha256": "3dc11565c768ba2dba4022d2d2e79c54f0f949cdcdcd0d270ceab567a7adc7d1" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "response-unused", - "contract": "The local Cosmos delete response is discarded on source-screening rollback; its IfNotModified ETag comes from the persistence owner result. Assistant point normalization remains with persist_chat_reply rather than weakening screening. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "The local Cosmos delete response is discarded on source-screening rollback; its IfNotModified ETag comes from the persistence owner result. Assistant point normalization remains with persist_chat_reply rather than weakening screening. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py" @@ -27019,8 +27019,7 @@ }, "application/single_app/route_backend_chats.py::_read_agent_citation_artifact_payloads": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "6405a74b2bc9f5b3fec1e8dd855b4cf7286ce55f64019fa0ca3ee0e2b471f26e", "points": [], "queries": [ { @@ -27029,10 +27028,11 @@ "response": "consumed" } ], - "sha256": "6405a74b2bc9f5b3fec1e8dd855b4cf7286ce55f64019fa0ca3ee0e2b471f26e" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "Artifact and chunk discovery returns plain query rows to build_message_artifact_payload_map; it is not a point-response normalization boundary. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Artifact and chunk discovery returns plain query rows to build_message_artifact_payload_map; it is not a point-response normalization boundary. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27042,8 +27042,7 @@ }, "application/single_app/route_backend_chats.py::_read_recent_assistant_messages": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "046e824243430a02a5b186cd6e9d37d7e4b3b1c0388ad3620877f5594c0b75df", "points": [], "queries": [ { @@ -27052,10 +27051,11 @@ "response": "consumed" } ], - "sha256": "046e824243430a02a5b186cd6e9d37d7e4b3b1c0388ad3620877f5594c0b75df" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "Partitioned assistant query rows remain plain dictionaries through soft-delete filtering and saved-analysis history sanitization. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Partitioned assistant query rows remain plain dictionaries through soft-delete filtering and saved-analysis history sanitization. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27065,8 +27065,7 @@ }, "application/single_app/route_backend_chats.py::_reject_chat_submission": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "288e129d9b351e7154851236aa8dc9e729c3717bdd3f70eabb8b8ae115486315", "points": [ { "method": "upsert_item", @@ -27085,10 +27084,11 @@ } ], "queries": [], - "sha256": "288e129d9b351e7154851236aa8dc9e729c3717bdd3f70eabb8b8ae115486315" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "response-unused", - "contract": "Safety/user-message upsert and conversation replace responses are discarded; the browser payload is constructed from plain screened documents and explicit public fields. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Safety/user-message upsert and conversation replace responses are discarded; the browser payload is constructed from plain screened documents and explicit public fields. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py" @@ -27096,8 +27096,7 @@ }, "application/single_app/route_backend_chats.py::_resolve_authorized_conversation_context": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "fab1e43b2782cf73dbcb57a0054fce275c34a46f670d5cb3a1dbc341a40a69a3", "points": [ { "method": "read_item", @@ -27106,10 +27105,11 @@ } ], "queries": [], - "sha256": "fab1e43b2782cf73dbcb57a0054fce275c34a46f670d5cb3a1dbc341a40a69a3" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "The conversation point mapping feeds shared-backing and participation authorization using Mapping fields; no raw document reaches an exact-dictionary domain validator or bypasses access checks. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "The conversation point mapping feeds shared-backing and participation authorization using Mapping fields; no raw document reaches an exact-dictionary domain validator or bypasses access checks. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27119,8 +27119,7 @@ }, "application/single_app/route_backend_chats.py::_resolve_chat_selected_document_metadata": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "b74514b35b7026cc4e9b74ca6e636b73bdc2aeb5b43b9b831a1b889933ba86bc", "points": [], "queries": [ { @@ -27129,10 +27128,11 @@ "response": "consumed" } ], - "sha256": "b74514b35b7026cc4e9b74ca6e636b73bdc2aeb5b43b9b831a1b889933ba86bc" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "Workspace lookup consumes plain query rows, rechecks document availability and projects only id/name/title/workspace fields into a plain selection DTO. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Workspace lookup consumes plain query rows, rechecks document availability and projects only id/name/title/workspace fields into a plain selection DTO. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27142,8 +27142,7 @@ }, "application/single_app/route_backend_chats.py::_rollback_agent_citation_artifacts": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "f4b52dfb5a5a24a71a8b31c0839f7901ad49bc60e59be4104c6fbad5895e3ed8", "points": [ { "method": "delete_item", @@ -27163,10 +27162,11 @@ "response": "consumed" } ], - "sha256": "f4b52dfb5a5a24a71a8b31c0839f7901ad49bc60e59be4104c6fbad5895e3ed8" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "response-unused", - "contract": "Plain query rows provide chunk IDs; all chunk/artifact delete responses are discarded and rollback failure accounting remains explicit. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Plain query rows provide chunk IDs; all chunk/artifact delete responses are discarded and rollback failure accounting remains explicit. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py" @@ -27174,8 +27174,7 @@ }, "application/single_app/route_backend_chats.py::_save_chat_user_metadata": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "2eb3087c2648dc63a4e2153648111740ea6f09191e5cc4cf3e112692f13ec7f2", "points": [ { "method": "upsert_item", @@ -27184,10 +27183,11 @@ } ], "queries": [], - "sha256": "2eb3087c2648dc63a4e2153648111740ea6f09191e5cc4cf3e112692f13ec7f2" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "A new-message upsert may return the SDK mapping to Mapping-compatible chat persistence callers; retry metadata is delegated to patch_chat_message_metadata, which owns conditional update/normalization. No strict top-level DTO is synthesized from arbitrary iterables. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "A new-message upsert may return the SDK mapping to Mapping-compatible chat persistence callers; retry metadata is delegated to patch_chat_message_metadata, which owns conditional update/normalization. No strict top-level DTO is synthesized from arbitrary iterables. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27197,8 +27197,7 @@ }, "application/single_app/route_backend_chats.py::persist_agent_citation_artifacts": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "2ddf8a15d4535a230894dce969eec6547280a76d58c36b1517babbfa9cd59858", "points": [ { "method": "upsert_item", @@ -27212,10 +27211,11 @@ } ], "queries": [], - "sha256": "2ddf8a15d4535a230894dce969eec6547280a76d58c36b1517babbfa9cd59858" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "response-unused", - "contract": "Artifact upsert and cancellation-rollback delete responses are discarded; artifact identities are taken from constructed plain documents and cancellation remains visible. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Artifact upsert and cancellation-rollback delete responses are discarded; artifact identities are taken from constructed plain documents and cancellation remains visible. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py" @@ -27223,8 +27223,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats._answer_raced_block_submission": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "665b15455527190885836ef2693e580d7bb12f1095f5f852b3025e49300bf11f", "points": [ { "method": "read_item", @@ -27233,10 +27232,11 @@ } ], "queries": [], - "sha256": "665b15455527190885836ef2693e580d7bb12f1095f5f852b3025e49300bf11f" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "A reread SDK message is checked for conversation membership and inspected by Mapping-compatible block revision helpers; conflict/replay JSON consists of explicit revision fields, not the SDK wrapper. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "A reread SDK message is checked for conversation membership and inspected by Mapping-compatible block revision helpers; conflict/replay JSON consists of explicit revision fields, not the SDK wrapper. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27246,8 +27246,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats._find_originating_user_request": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "87cabd89ec423b1d136ca31f57302e7968915917aed47f813fde7509038cd88d", "points": [], "queries": [ { @@ -27256,10 +27255,11 @@ "response": "consumed" } ], - "sha256": "87cabd89ec423b1d136ca31f57302e7968915917aed47f813fde7509038cd88d" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "The partitioned prior-user query returns plain rows; only the first row content string is used as the originating request. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "The partitioned prior-user query returns plain rows; only the first row content string is used as the originating request. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27269,8 +27269,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats._get_latest_chat_thread_id": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "a038c79bedcf74e8e6845e86eb121bb297b509576060664c4f4665b4a194da0c", "points": [], "queries": [ { @@ -27279,10 +27278,11 @@ "response": "consumed" } ], - "sha256": "a038c79bedcf74e8e6845e86eb121bb297b509576060664c4f4665b4a194da0c" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "The TOP 1 thread projection is a plain query row; only thread_id is returned, with the existing read-failure fallback unchanged. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "The TOP 1 thread projection is a plain query row; only thread_id is returned, with the existing read-failure fallback unchanged. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27292,8 +27292,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats._load_block_revision_message": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "c123ca933146d12ce4abf1d4fe6908b5b85669bec2683a9596570165bfc55305", "points": [ { "method": "read_item", @@ -27302,10 +27301,11 @@ } ], "queries": [], - "sha256": "c123ca933146d12ce4abf1d4fe6908b5b85669bec2683a9596570165bfc55305" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "The point message mapping is checked against the authorized conversation and passed to Mapping-compatible revision readers/persistence; missing messages retain their 404 contract. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "The point message mapping is checked against the authorized conversation and passed to Mapping-compatible revision readers/persistence; missing messages retain their 404 contract. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27315,8 +27315,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats._load_or_create_analyze_conversation": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "bdca4fcc9d084df866c382d171029a7b50682ed4277ae576a76eb6242b6eba50", "points": [ { "method": "read_item", @@ -27330,10 +27329,11 @@ } ], "queries": [], - "sha256": "bdca4fcc9d084df866c382d171029a7b50682ed4277ae576a76eb6242b6eba50" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "Existing point mappings are checked for owner/deletion state; newly created SDK conversation mappings feed field-based activity logging and cache invalidation, not exact-dictionary result validation. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Existing point mappings are checked for owner/deletion state; newly created SDK conversation mappings feed field-based activity logging and cache invalidation, not exact-dictionary result validation. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27343,8 +27343,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats._resolve_document_action_selected_documents": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "25a6045d75aaf20cb18064c0862a5f129752ef8170f85a29e3bf068189cd9204", "points": [], "queries": [ { @@ -27353,10 +27352,11 @@ "response": "consumed" } ], - "sha256": "25a6045d75aaf20cb18064c0862a5f129752ef8170f85a29e3bf068189cd9204" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "Chat attachment resolution consumes plain partitioned query rows and emits display-name/context projections; workspace ownership remains with the document resolver. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Chat attachment resolution consumes plain partitioned query rows and emits display-name/context projections; workspace ownership remains with the document resolver. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27366,8 +27366,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats.add_message_image_revision_api": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "738633127c1ba804260536f901a7797409fd8d518e35920ec3ee14efd10786d5", "points": [ { "method": "read_item", @@ -27376,10 +27375,11 @@ } ], "queries": [], - "sha256": "738633127c1ba804260536f901a7797409fd8d518e35920ec3ee14efd10786d5" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "The reload callback returns a partitioned SDK message mapping to the Mapping-compatible image revision owner, which rechecks concurrent versions; the route emits only explicit revision/error fields. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "The reload callback returns a partitioned SDK message mapping to the Mapping-compatible image revision owner, which rechecks concurrent versions; the route emits only explicit revision/error fields. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27389,8 +27389,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats.chat_api": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "35213e61b04f93aa5598b53eaf9f24e89b00be17dc627ee3ae652826cf6e4fa1", "points": [ { "method": "read_item", @@ -27470,10 +27469,11 @@ "response": "consumed" } ], - "sha256": "35213e61b04f93aa5598b53eaf9f24e89b00be17dc627ee3ae652826cf6e4fa1" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "Retry/source point mappings supply role/content and nested model/thread metadata through Mapping access; history queries stay plain. Metadata persistence remains with its existing owner and local write responses are discarded. Action recovery now pauses before user-message persistence and model initialization; the original mapping-compatible Cosmos consumers and conditional persistence stay unchanged. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Added per-answer evidence metadata and stable model-configuration errors; existing point responses still use Mapping-compatible field access and the same save/authorization owners. No exact-dict sink or new storage call is introduced. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review 0.261.395: canonical source intent is nested plain request JSON; the existing SDK Mapping reads and unused message/conversation write responses are unchanged. Review 0.261.400: Excel native-inspection routing delegates to report/search readers; original retry/history Mapping consumers and unused writes are unchanged. Existing SDK route evidence and native routing/citation tests pass.", + "contract": "Retry/source point mappings supply role/content and nested model/thread metadata through Mapping access; history queries stay plain. Metadata persistence remains with its existing owner and local write responses are discarded. Action recovery now pauses before user-message persistence and model initialization; the original mapping-compatible Cosmos consumers and conditional persistence stay unchanged. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Added per-answer evidence metadata and stable model-configuration errors; existing point responses still use Mapping-compatible field access and the same save/authorization owners. No exact-dict sink or new storage call is introduced. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review 0.261.395: canonical source intent is nested plain request JSON; the existing SDK Mapping reads and unused message/conversation write responses are unchanged. Review 0.261.400: Excel native-inspection routing delegates to report/search readers; original retry/history Mapping consumers and unused writes are unchanged. Existing SDK route evidence and native routing/citation tests pass. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27486,8 +27486,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats.chat_stream_api.generate": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "1c761b170ff0f1f2128d58b6ec138d491e79f97d5a5a8cb7c26a60e012c01699", "points": [ { "method": "read_item", @@ -27547,10 +27546,11 @@ "response": "consumed" } ], - "sha256": "1c761b170ff0f1f2128d58b6ec138d491e79f97d5a5a8cb7c26a60e012c01699" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "Retry point mappings are checked for role/conversation and nested metadata through Mapping access; history queries remain plain, and local create/replace/delete return values are discarded. SSE telemetry does not alter storage contracts. Action recovery now pauses before user-message persistence and model initialization; the original mapping-compatible Cosmos consumers and conditional persistence stay unchanged. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Added per-answer evidence metadata and stable model-configuration errors; existing point responses still use Mapping-compatible field access and the same save/authorization owners. Streaming publication and cancellation fences remain unchanged. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review 0.261.395: canonical source intent is nested plain request JSON; the existing SDK Mapping reads and unused message/conversation write responses are unchanged. Review 0.261.400: Excel native-inspection routing delegates to report/search readers; original retry/history Mapping consumers and unused writes are unchanged. Existing SDK route evidence and native routing/citation tests pass.", + "contract": "Retry point mappings are checked for role/conversation and nested metadata through Mapping access; history queries remain plain, and local create/replace/delete return values are discarded. SSE telemetry does not alter storage contracts. Action recovery now pauses before user-message persistence and model initialization; the original mapping-compatible Cosmos consumers and conditional persistence stay unchanged. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Added per-answer evidence metadata and stable model-configuration errors; existing point responses still use Mapping-compatible field access and the same save/authorization owners. Streaming publication and cancellation fences remain unchanged. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review 0.261.395: canonical source intent is nested plain request JSON; the existing SDK Mapping reads and unused message/conversation write responses are unchanged. Review 0.261.400: Excel native-inspection routing delegates to report/search readers; original retry/history Mapping consumers and unused writes are unchanged. Existing SDK route evidence and native routing/citation tests pass. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27563,8 +27563,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats.chat_stream_api.generate.finalize_cancelled_stream_response": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "0f0c0fa4bb9109a639a9e7ef1f78a3a51005453fd15f926499832f328821f4f3", "points": [ { "method": "replace_item", @@ -27573,10 +27572,11 @@ } ], "queries": [], - "sha256": "0f0c0fa4bb9109a639a9e7ef1f78a3a51005453fd15f926499832f328821f4f3" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "response-unused", - "contract": "Cancellation refreshes the conversation with a discarded replace response; screened message persistence remains with its existing owner and the stream payload is assembled separately. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: The added evidence metadata is plain request-owned JSON. Direct Cosmos mutation responses remain unused; conditional cancellation/publication behavior is unchanged. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Cancellation refreshes the conversation with a discarded replace response; screened message persistence remains with its existing owner and the stream payload is assembled separately. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: The added evidence metadata is plain request-owned JSON. Direct Cosmos mutation responses remain unused; conditional cancellation/publication behavior is unchanged. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py" @@ -27584,8 +27584,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats.execute_document_action_chat_request": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "d58366b0356ee1583b35a2116430f06d1cb0304e1186f95ed19fff72b1714b06", "points": [ { "method": "read_item", @@ -27614,10 +27613,11 @@ } ], "queries": [], - "sha256": "d58366b0356ee1583b35a2116430f06d1cb0304e1186f95ed19fff72b1714b06" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "The original Analyze point mapping is checked for role/conversation/content and nested thread metadata; local message writes and cleanup deletes discard responses, while retained analysis contracts remain strict. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "The original Analyze point mapping is checked for role/conversation/content and nested thread metadata; local message writes and cleanup deletes discard responses, while retained analysis contracts remain strict. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27627,8 +27627,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats.execute_saved_analysis_chat_request": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "d05f7e80e44a9a9bebac52069256e6b2fe5a14d0b135ad25d8df2b925d464565", "points": [ { "method": "read_item", @@ -27668,10 +27667,11 @@ "response": "consumed" } ], - "sha256": "d05f7e80e44a9a9bebac52069256e6b2fe5a14d0b135ad25d8df2b925d464565" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "Retry input point mappings supply authorized role/content/thread fields; history queries are plain, and user/system writes and rollback deletes discard responses. Saved-analysis strict result admission remains with its storage owner. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Retry input point mappings supply authorized role/content/thread fields; history queries are plain, and user/system writes and rollback deletes discard responses. Saved-analysis strict result admission remains with its storage owner. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27681,8 +27681,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats.generate_image_from_proposal": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "69be4b24e818f4e852ff30fa1031f467c574c9693590231d7f5c858a19dfaa36", "points": [ { "method": "read_item", @@ -27696,10 +27695,11 @@ } ], "queries": [], - "sha256": "69be4b24e818f4e852ff30fa1031f467c574c9693590231d7f5c858a19dfaa36" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "The source assistant SDK mapping is authorized by conversation/role and used by Mapping-compatible planned-image lookup; conversation replacement discards its response and public image data is projected explicitly. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "The source assistant SDK mapping is authorized by conversation/role and used by Mapping-compatible planned-image lookup; conversation replacement discards its response and public image data is projected explicitly. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27709,8 +27709,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats.generate_image_from_proposal.read_conversation_image": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "7d6b5464e658524d26adfb6f6f7ff1e53a30f1bca4ef34bfc5db702897cc5377", "points": [ { "method": "read_item", @@ -27719,10 +27718,11 @@ } ], "queries": [], - "sha256": "7d6b5464e658524d26adfb6f6f7ff1e53a30f1bca4ef34bfc5db702897cc5377" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "The partitioned point mapping is returned to find_planned_proposal_image, which inspects message fields; a missing message returns None, without an empty-document or strict-type fallback. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "The partitioned point mapping is returned to find_planned_proposal_image, which inspects message fields; a missing message returns None, without an empty-document or strict-type fallback. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27732,8 +27732,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats.image_proposal_status": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "bf6e633a6029e92fb26ac8044b8e88a6b593fa8103188190d93797a3aabc91d2", "points": [], "queries": [ { @@ -27742,10 +27741,11 @@ "response": "consumed" } ], - "sha256": "bf6e633a6029e92fb26ac8044b8e88a6b593fa8103188190d93797a3aabc91d2" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "Proposal status iterates plain query rows, validates nested proposal dictionaries and emits explicit public image status fields. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Proposal status iterates plain query rows, validates nested proposal dictionaries and emits explicit public image status fields. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27755,8 +27755,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats.mask_message_api": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "4f023bbdec15c4ce11a324c31963c4689df6b465abdaf0d74b959c59aa099053", "points": [ { "method": "read_item", @@ -27776,10 +27775,11 @@ "response": "consumed" } ], - "sha256": "4f023bbdec15c4ce11a324c31963c4689df6b465abdaf0d74b959c59aa099053" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "Point message/conversation mappings and plain fallback query rows use field-based ownership and soft-delete checks; nested mask metadata goes to existing persistence, and public JSON projects mask fields only. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "Point message/conversation mappings and plain fallback query rows use field-based ownership and soft-delete checks; nested mask metadata goes to existing persistence, and public JSON projects mask fields only. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -27789,8 +27789,7 @@ }, "application/single_app/route_backend_chats.py::register_route_backend_chats.set_message_visual_style_api": { "boundary": { - "batches": [], - "module_sha256": "6869610ed801ff27baf7672a73bca8d82f3205dacc22c938b95317c035c57497", + "sha256": "ef785c631c95a57e1290f86dbf05a4c37972705387f146a543f7968ae5b7c7bf", "points": [ { "method": "read_item", @@ -27799,10 +27798,11 @@ } ], "queries": [], - "sha256": "ef785c631c95a57e1290f86dbf05a4c37972705387f146a543f7968ae5b7c7bf" + "batches": [], + "module_sha256": "8fa1302c970ed36dfbc70adee101d6ccdf4cc782e2981663bdc69ccf088ae0c2" }, "classification": "sdk-compatible", - "contract": "The point message mapping is checked for conversation identity and passed to Mapping-compatible visual-style/persistence owners; public JSON projects nested visual_styles rather than the SDK document. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged.", + "contract": "The point message mapping is checked for conversation identity and passed to Mapping-compatible visual-style/persistence owners; public JSON projects nested visual_styles rather than the SDK document. Storage function is unchanged; module-level recovery wiring is covered by representative real-owner SDK runtime evidence, not exhaustive coverage of this function. Combined V2 source-policy and stale-group recovery wiring preserves this storage function; both branches' representative owner-runtime evidence is retained. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.388: Function body and SDK point/query/batch operations are unchanged; module-only model-policy, source-presentation, or export-helper changes do not alter this Mapping consumer or its conditional-write contract. Review renewed for 0.261.389: Legacy and named model calls now use the shared explicit-API client factory. Redundant identity-header construction was removed; the factory retains the same identity context. SDK conversation/message consumers, object authorization, screening, cancellation, ETags and persistence operations are unchanged. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_action_credentials_bootstrap.py", @@ -32049,8 +32049,7 @@ }, "application/single_app/route_enhanced_citations.py::_get_authorized_chat_artifact_message": { "boundary": { - "batches": [], - "module_sha256": "49f426f48b0bba2b70834692911b667acb32888ba62e853e045cbe956e9af0ea", + "sha256": "a2c6beb6142f515b82306165e8d8001995a4fad0f7a776d9f1d43c70c84c7614", "points": [ { "method": "read_item", @@ -32064,10 +32063,11 @@ } ], "queries": [], - "sha256": "a2c6beb6142f515b82306165e8d8001995a4fad0f7a776d9f1d43c70c84c7614" + "batches": [], + "module_sha256": "bd86121e79c23e9b2456f10562a58f84d6d91b4ec6a3f0f4147fd401d22a5820" }, "classification": "sdk-compatible", - "contract": "Partitioned SDK conversation/message mappings feed participation, generated-artifact, publication and optional workspace screening checks through fields. The authorized message mapping is returned intact with nested plain metadata and ETag. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained.", + "contract": "Partitioned SDK conversation/message mappings feed participation, generated-artifact, publication and optional workspace screening checks through fields. The authorized message mapping is returned intact with nested plain metadata and ETag. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_v2_public_source_owner_sdk.py::test_enhanced_citation_owner_returns_sdk_artifact_and_uses_plain_upload_rows" @@ -32075,8 +32075,7 @@ }, "application/single_app/route_enhanced_citations.py::register_enhanced_citations_routes.get_enhanced_citation_tabular": { "boundary": { - "batches": [], - "module_sha256": "49f426f48b0bba2b70834692911b667acb32888ba62e853e045cbe956e9af0ea", + "sha256": "5cf97a0cec09c4811369dd7c311d5aaf3f358e0dc96e8c7e82803ebc89bbc365", "points": [ { "method": "read_item", @@ -32091,10 +32090,11 @@ "response": "consumed" } ], - "sha256": "5cf97a0cec09c4811369dd7c311d5aaf3f358e0dc96e8c7e82803ebc89bbc365" + "batches": [], + "module_sha256": "bd86121e79c23e9b2456f10562a58f84d6d91b4ec6a3f0f4147fd401d22a5820" }, "classification": "sdk-compatible", - "contract": "The SDK conversation owner check precedes file discovery. Partitioned file query rows remain plain dictionaries through approval/provenance checks and blob-field projection; the SDK conversation never enters strict JSON admission. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained.", + "contract": "The SDK conversation owner check precedes file discovery. Partitioned file query rows remain plain dictionaries through approval/provenance checks and blob-field projection; the SDK conversation never enters strict JSON admission. Public discovery now uses the bootstrap-free owner-configured boundary; this storage function and SDK response consumption are unchanged, with representative owner-runtime evidence retained. Review renewed for 0.261.405: exact owner AST, referenced import bindings, and SDK point/query/batch operations are unchanged. Module-only Excel helper/caller additions do not change this response consumer; existing specific runtime/source evidence is retained.", "evidence_scope": "owner-runtime", "tests": [ "functional_tests/test_v2_public_source_owner_sdk.py::test_enhanced_citation_owner_returns_sdk_artifact_and_uses_plain_upload_rows" diff --git a/functional_tests/test_excel_document_accessibility_extraction.py b/functional_tests/test_excel_document_accessibility_extraction.py index 070dadcdf..3ee12fe1e 100644 --- a/functional_tests/test_excel_document_accessibility_extraction.py +++ b/functional_tests/test_excel_document_accessibility_extraction.py @@ -1,7 +1,7 @@ # test_excel_document_accessibility_extraction.py """ Functional tests for native Excel inspection and workspace evidence persistence. -Version: 0.261.399 +Version: 0.261.405 Implemented in: 0.261.399 Real workbook packages exercise saved properties, styles, notes, threaded comments, @@ -172,7 +172,8 @@ def test_native_excel_properties_comments_objects_styles_and_printing(workbook_p assert any(item["color"] == "#FF0000" for item in style["colors"]) body = "\n".join(item["content"] for item in evidence) assert "Thread reply" in body and "$0.00" in body and "#FFFF00" in body - assert "Operations" not in body, "Native evidence must not become a copy of worksheet rows" + assert "Theme color" not in body, "Unannotated worksheet values must not become native row indexing" + assert any(item["cell"] == "A2" and item["cell_value"] == "Operations" for item in notes) assert "No worksheet rows" in body, "Blank sheets still have native evidence" assert any("External relationships" in note for note in metadata["coverage_notes"]) summary = format_accessibility_metadata_for_chat(metadata) @@ -399,6 +400,10 @@ def schema(*args): "pandas": pandas, "os": os, "logging": logging, "log_event": lambda *args, **kwargs: None, "extract_office_accessibility_metadata": extract_office_accessibility_metadata, + "_extract_excel_native_inventory": lambda document_id, user_id, path, extension, **scope: ( + *extract_office_accessibility_metadata(path, extension), None, + ), + "EXCEL_INVENTORY_FIELD": "excel_native_inventory", "build_excel_evidence_chunks": excel.build_excel_evidence_chunks, "get_embedding_safe_chunk_characters": lambda settings: 900, "get_settings": lambda: {"enable_extract_meta_data": True}, @@ -436,7 +441,7 @@ def test_ingestion_modes_scopes_native_properties_and_chunk_identity(workbook_pa assert native and all(len(item["page_text_content"]) <= 900 for item in native) if enhanced: assert chunks[0]["page_text_content"] == "schema preview" - assert all("Operations" not in item["page_text_content"] for item in native) + assert all("Theme color" not in item["page_text_content"] for item in native) assert any(item.get("num_chunks") == count for item in updates) assert any(item.get("title") == "AI-generated title" for item in updates) is auto_metadata @@ -562,6 +567,10 @@ def execute(sources, callback, selection_mode, **kwargs): "annotate_document_search_payload": annotate_document_search_payload, "format_accessibility_metadata_for_chat": format_accessibility_metadata_for_chat, "EXCEL_EVIDENCE_MARKER": excel.EXCEL_EVIDENCE_MARKER, + "build_excel_native_chat_evidence": excel.build_excel_native_chat_evidence, + "excel_native_question_sections": excel.excel_native_question_sections, + "has_excel_native_inspection_intent": excel.has_excel_native_inspection_intent, + "merge_excel_native_evidence": excel.merge_excel_native_evidence, "execute_tabular_evidence_sources": execute, } _load_functions("route_backend_chats.py", {"_execute_excel_native_inspection_evidence"}, namespace) diff --git a/functional_tests/test_excel_document_delete_sdk_contract.py b/functional_tests/test_excel_document_delete_sdk_contract.py new file mode 100644 index 000000000..893c4af69 --- /dev/null +++ b/functional_tests/test_excel_document_delete_sdk_contract.py @@ -0,0 +1,87 @@ +# test_excel_document_delete_sdk_contract.py +""" +Owner-runtime SDK contract for Excel document and private-inventory cleanup. +Version: 0.261.405 +Implemented in: 0.261.405 + +The actual deletion function consumes installed-SDK CosmosDict point responses. +External logging, Search, and blob side effects are isolated and scope checked. +""" + +import ast +from pathlib import Path +import os +import sys +from types import ModuleType, SimpleNamespace +from unittest.mock import patch + +import pytest +from azure.core import MatchConditions +from azure.cosmos._cosmos_responses import CosmosDict +from azure.cosmos.exceptions import CosmosResourceNotFoundError + +APP = Path(__file__).resolve().parents[1] / "application" / "single_app" +sys.path.insert(0, str(APP)) + +from content_screening.contracts import ScreeningConflictError, subject_from_document # noqa: E402 +from test_support.offline_bootstrap import OfflineContainer # noqa: E402 + + +def load_delete(container, calls): + source = ast.parse((APP / "functions_documents.py").read_text(encoding="utf-8")) + node = next(node for node in source.body if isinstance(node, ast.FunctionDef) and node.name == "delete_document") + namespace = { + "os": os, "cosmos_user_documents_container": container, + "cosmos_group_documents_container": container, "cosmos_public_documents_container": container, + "SCREENING_FIELD": "content_screening", "EXCEL_INVENTORY_FIELD": "excel_native_inventory", + "ScreeningConflictError": ScreeningConflictError, + "CosmosResourceNotFoundError": CosmosResourceNotFoundError, "MatchConditions": MatchConditions, + "subject_from_document": subject_from_document, + "prepare_document_deletion": lambda document, actor: calls.append(("private_cleanup", document, actor)), + "clear_excel_inventory_cache": lambda subject: calls.append(("cache_cleanup", subject)), + "delete_from_blob_storage": lambda document, **kwargs: calls.append(("source_cleanup", document, kwargs)), + "delete_document_access_index_for_document_fail_open": lambda document, **kwargs: {"success": True}, + "assert_group_document_source_writable": lambda document: None, + } + exec(compile(ast.Module(body=[node], type_ignores=[]), "functions_documents.py", "exec"), namespace) + return namespace["delete_document"] + + +@pytest.mark.parametrize("scope", ["personal", "group", "public"]) +def test_real_delete_owner_handles_sdk_points_and_scope_bound_native_cleanup(scope): + container, calls = OfflineContainer(), [] + document = {"id": "document", "user_id": "owner", "file_name": "review.xlsx", "version": 1} + kwargs = {} + if scope == "group": + document["group_id"] = "group" + kwargs["group_id"] = "group" + elif scope == "public": + document["public_workspace_id"] = "public" + kwargs["public_workspace_id"] = "public" + saved = container.upsert_item(document) + assert isinstance(saved, CosmosDict) + delete = load_delete(container, calls) + debug, activity = ModuleType("functions_debug"), ModuleType("functions_activity_logging") + debug.debug_print = lambda *args: None + activity.log_document_deletion_transaction = lambda **kwargs: calls.append(("activity", kwargs)) + with patch.dict(sys.modules, {"functions_debug": debug, "functions_activity_logging": activity}): + delete("owner", "document", strict=True, expected_etag=saved["_etag"], **kwargs) + assert "document" not in container.items + private = next(call for call in calls if call[0] == "private_cleanup") + assert isinstance(private[1], CosmosDict) + assert private[1]["id"] == "document" and private[2] == "owner" + assert next(call for call in calls if call[0] == "cache_cleanup")[1] == subject_from_document(document) + source = next(call for call in calls if call[0] == "source_cleanup") + assert source[2]["strict"] is True + + +def test_stale_sdk_etag_prevents_private_inventory_deletion(): + container, calls = OfflineContainer(), [] + container.upsert_item({"id": "document", "user_id": "owner", "file_name": "review.xlsx", "version": 1}) + delete = load_delete(container, calls) + debug = ModuleType("functions_debug") + debug.debug_print = lambda *args: None + with patch.dict(sys.modules, {"functions_debug": debug}): + with pytest.raises(ScreeningConflictError): + delete("owner", "document", strict=True, expected_etag="stale") + assert not calls and "document" in container.items diff --git a/functional_tests/test_excel_inventory_persistence.py b/functional_tests/test_excel_inventory_persistence.py new file mode 100644 index 000000000..f473796ee --- /dev/null +++ b/functional_tests/test_excel_inventory_persistence.py @@ -0,0 +1,223 @@ +# test_excel_inventory_persistence.py +""" +Functional tests for private, complete, revision-bound Excel inventories. +Version: 0.261.405 +Implemented in: 0.261.405 + +The real private artifact store runs against the existing Azure-like blob double. +Tests prove paging, full-text continuations, source binding, publication failures, +fresh authorization, screening capture, and revision-confined cleanup. +""" + +import json +import ast +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +from openpyxl import Workbook +from openpyxl.comments import Comment +from azure.core.exceptions import ResourceNotFoundError + +APP = Path(__file__).resolve().parents[1] / "application" / "single_app" +sys.path.insert(0, str(APP)) + +import functions_excel_accessibility as excel # noqa: E402 +from content_screening.contracts import Subject # noqa: E402 +from content_screening.extraction import ExtractionCapture # noqa: E402 +from content_screening.storage import ScreeningStorage, revision_prefix # noqa: E402 +from functions_excel_inventory import ( # noqa: E402 + EXCEL_INVENTORY_FIELD, + ExcelInventoryPersistenceError, + clear_excel_inventory_cache, + open_excel_inventory, + open_stored_excel_inventory, + persist_excel_inventory, +) +from test_content_screening_persistence import FakeBlobService, FakeSdkError # noqa: E402 +from test_excel_native_inspection_chat_evidence import review_workbook # noqa: E402 +from test_document_accessibility_report_action import _load_payload_namespace # noqa: E402 +from excel_workbook_reader import ExcelWorkbookReader, bounded_range # noqa: E402 +from openpyxl.utils.cell import get_column_letter # noqa: E402 + + +__all__ = ["review_workbook"] + + +@pytest.fixture(autouse=True) +def isolate_inventory_cache(): + clear_excel_inventory_cache() + yield + clear_excel_inventory_cache() + + +@pytest.fixture +def inventory_store(): + service = FakeBlobService() + return ScreeningStorage(service), service + + +def test_complete_inventory_survives_parent_record_limits_and_restart(review_workbook, inventory_store, monkeypatch): + store, service = inventory_store + subject = Subject("personal", "owner", "workbook", "1") + monkeypatch.setattr(excel, "EXCEL_MAX_RECORDS", 2) + metadata, _, reference = persist_excel_inventory(review_workbook, ".xlsx", subject=subject, storage=store) + assert metadata["comments"] == [] + assert len(json.dumps(metadata, ensure_ascii=True).encode()) <= 500000 + clear_excel_inventory_cache() + checks = [] + reader = open_stored_excel_inventory( + {"id": "workbook", "user_id": "owner", "version": 1, EXCEL_INVENTORY_FIELD: reference}, + storage=ScreeningStorage(service), authorize=lambda: checks.append("checked"), + ) + records, cursor = [], None + while True: + page = reader.page("comments", cursor=cursor, page_size=2) + records.extend(page["records"]) + cursor = page["next_cursor"] + if cursor is None: + break + assert len(records) == 5 + assert any(record["text"] == "Set cap to 19500." for record in records) + assert any(record["kind"] == "note" for record in records) + assert len(checks) >= 8 + assert reader.metadata["comment_summary"]["compatibility_copies"] == 3 + + +def test_long_unicode_text_continues_without_loss_or_oversized_responses(tmp_path, inventory_store): + store, _ = inventory_store + path = tmp_path / "long.xlsx" + workbook = Workbook() + workbook.active["A1"] = "Amount" + workbook.active["A2"] = 0 + text = ('review "\\\\ quote ' + chr(0x03B1) + "\n") * 1800 + "TAIL" + workbook.active["A2"].comment = Comment(text, "Reviewer") + workbook.save(path) + subject = Subject("personal", "owner", "long", "1") + _, _, reference = persist_excel_inventory(path, ".xlsx", subject=subject, storage=store) + reader = open_stored_excel_inventory( + {"id": "long", "user_id": "owner", EXCEL_INVENTORY_FIELD: reference}, + storage=store, authorize=lambda: None, + ) + fragments, cursor = [], None + while True: + page = reader.page("comments", cursor=cursor, max_bytes=4096) + assert len(json.dumps(page, ensure_ascii=False, separators=(",", ":")).encode()) <= 4096 + assert page["records"][0]["record_fragment"] is True + fragments.append(page["records"][0]["record_json_fragment"]) + cursor = page["next_cursor"] + if cursor is None: + break + record = json.loads("".join(fragments)) + assert record["text"] == text and record["cell_value"] == 0 + + +def test_source_and_section_bound_cursors_cannot_be_replayed(review_workbook, inventory_store): + store, _ = inventory_store + document = {"id": "workbook", "user_id": "owner", "version": 1} + original = review_workbook.read_bytes() + reader = open_excel_inventory(document, original, ".xlsx", storage=store, authorize=lambda: None) + page = reader.page("comments", page_size=1) + with pytest.raises(ValueError, match="different source or section"): + reader.page("objects", cursor=page["next_cursor"]) + updated = open_excel_inventory(document, original + b"new-revision", ".xlsx", storage=store, authorize=lambda: None) + with pytest.raises(ValueError, match="different source or section"): + updated.page("comments", cursor=page["next_cursor"]) + + +def test_every_page_rechecks_authorization(review_workbook, inventory_store): + store, _ = inventory_store + allowed = [True] + + def authorize(): + if not allowed[0]: + raise PermissionError("Source access revoked.") + + reader = open_excel_inventory( + {"id": "workbook", "user_id": "owner"}, + review_workbook.read_bytes(), ".xlsx", storage=store, authorize=authorize, + ) + first = reader.page("comments", page_size=1) + allowed[0] = False + with pytest.raises(PermissionError, match="revoked"): + reader.page("comments", cursor=first["next_cursor"]) + with pytest.raises(PermissionError, match="revoked"): + _ = reader.metadata + + +def test_native_text_is_included_in_private_screening_capture(review_workbook, inventory_store): + store, _ = inventory_store + subject = Subject("personal", "owner", "workbook", "1") + capture = ExtractionCapture(subject, "scan", "review.xlsx", artifact_storage=store) + capture.mode = "table" + persist_excel_inventory(review_workbook, ".xlsx", subject=subject, storage=store, capture=capture) + capture.completed = True + units = capture.finish({}) + text = "\n".join(unit.text for unit in units) + assert "Set cap to 19500." in text and "Hold approval." in text + assert all(unit.locator["kind"] == "excel_native" for unit in units) + + +def test_storage_failure_is_not_a_ready_inventory(review_workbook, inventory_store): + store, service = inventory_store + service.get_container_client("content-screening").fail_upload_name = "part-0000" + with pytest.raises(ExcelInventoryPersistenceError) as error: + persist_excel_inventory( + review_workbook, ".xlsx", subject=Subject("personal", "owner", "workbook", "1"), storage=store, + ) + assert "private-provider-error-canary" not in str(error.value) + + +def test_revision_cleanup_does_not_remove_another_workbook(review_workbook, inventory_store): + store, service = inventory_store + first, second = Subject("personal", "owner", "one", "1"), Subject("personal", "owner", "two", "1") + persist_excel_inventory(review_workbook, ".xlsx", subject=first, storage=store) + persist_excel_inventory(review_workbook, ".xlsx", subject=second, storage=store) + container = service.get_container_client("content-screening") + store.delete_revision(first) + assert not any(name.startswith(revision_prefix(first)) for name in container.blobs) + assert any(name.startswith(revision_prefix(second)) for name in container.blobs) + + +def test_real_report_backfills_existing_original_without_reupload(review_workbook, inventory_store): + store, service = inventory_store + document = {"id": "workbook", "user_id": "owner", "version": 1, + "file_name": "review.xlsx", "source_file_available": True} + namespace = _load_payload_namespace(document, chunks=[{ + "id": "workbook_2", "document_id": "workbook", "page_number": 2, + "chunk_text": excel.EXCEL_EVIDENCE_MARKER + "\nNative comments", + }]) + source_reads, checks = [], [] + + def available_bytes(doc, **kwargs): + source_reads.append(kwargs) + return document, review_workbook.read_bytes() + + namespace.update({ + "CLIENTS": {"storage_account_office_docs_client": service}, + "ScreeningStorage": lambda **kwargs: store, + "storage_account_content_screening_container_name": "content-screening", + "screening_access": SimpleNamespace(read_available_document_bytes=available_bytes), + "open_excel_inventory": open_excel_inventory, + "open_stored_excel_inventory": open_stored_excel_inventory, + "EXCEL_INVENTORY_SECTIONS": {"comments"}, + "ExcelWorkbookReader": ExcelWorkbookReader, "bounded_range": bounded_range, + "get_column_letter": get_column_letter, + "ResourceNotFoundError": ResourceNotFoundError, + "assert_evidence_available": lambda *args, **kwargs: checks.append("fresh"), + "_serialize_document": lambda doc, scope: {"id": doc["id"], "file_name": doc["file_name"]}, + }) + tree = ast.parse((APP / "functions_search_service.py").read_text(encoding="utf-8")) + node = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "_read_excel_inspection") + exec(compile(ast.Module(body=[node], type_ignores=[]), "functions_search_service.py", "exec"), namespace) + payload = namespace["get_document_accessibility_payload"]( + "workbook", "owner", sheet_name="Budget", cell_range="B4:B4", + ) + assert payload["schema_version"] == "2.0" + assert payload["accessibility_metadata"]["comment_summary"]["thread_roots"] == 3 + assert any(item["text"] == "Set cap to 19500." for item in payload["excel_inspection"]["page"]["records"]) + assert payload["excel_inspection"]["cells"][0]["value"] == 15000 + assert payload["citation_chunk"]["id"] == "workbook_2" + assert len(source_reads) == 1 and len(checks) >= 3 + assert "accessibility_metadata" not in document, "A read must not rewrite the original metadata/clearance." diff --git a/functional_tests/test_excel_native_images_and_release.py b/functional_tests/test_excel_native_images_and_release.py new file mode 100644 index 000000000..4fc2d288e --- /dev/null +++ b/functional_tests/test_excel_native_images_and_release.py @@ -0,0 +1,169 @@ +# test_excel_native_images_and_release.py +""" +Functional tests for Excel image anchors and release-safe metadata. +Version: 0.261.405 +Implemented in: 0.261.405 + +Real OOXML drawings retain worksheet/cell locations through existing image +extraction. Cleared/sanitized source hashes cannot borrow original metadata. +""" + +import ast +import io +from pathlib import Path +import random +import sys + +from openpyxl import Workbook +from openpyxl.drawing.image import Image as WorkbookImage +from PIL import Image + +APP = Path(__file__).resolve().parents[1] / "application" / "single_app" +sys.path.insert(0, str(APP)) + +from content_screening.access import _with_current_excel_metadata, is_public_document_field # noqa: E402 +from content_screening.contracts import ContentUnit, ScreeningValidationError # noqa: E402 +from content_screening.publication import _tabular_publication_chunks # noqa: E402 +from functions_excel_accessibility import extract_excel_accessibility_metadata # noqa: E402 +from functions_office_media import extract_office_embedded_images_with_diagnostics # noqa: E402 +from test_tabular_enhanced_citations_schema_summary_fix import load_tabular_schema_helpers # noqa: E402 + + +def test_real_excel_image_keeps_sheet_anchor_alt_and_analysis_location(tmp_path): + picture = tmp_path / "figure.png" + pixels = random.Random(7).randbytes(200 * 200 * 3) + Image.frombytes("RGB", (200, 200), pixels).save(picture) + path = tmp_path / "images.xlsx" + workbook = Workbook() + workbook.active.title = "Figures" + workbook.active.add_image(WorkbookImage(picture), "D9") + workbook.save(path) + metadata, _ = extract_excel_accessibility_metadata(path, ".xlsx") + locations = metadata["image_locations"] + assert len(locations) == 1 + output = tmp_path / "extracted" + output.mkdir() + images, diagnostics = extract_office_embedded_images_with_diagnostics( + path, str(output), min_pixels=100, max_images=5, native_media_metadata=locations, + ) + assert diagnostics["candidates"] == 1 + assert len(images) == 1 + assert images[0]["sheet_name"] == "Figures" and images[0]["cell"] == "D9" + assert images[0]["is_decorative"] is None + assert images[0]["position_known"] is True + assert images[0]["placements"][0]["cell"] == "D9" + + +def test_original_excel_metadata_is_not_exposed_after_active_source_sanitization(): + metadata = {"format": "xlsx", "comments": [{"text": "original-private-canary"}], + "inventory_storage": {"source_sha256": "a" * 64}} + original = {"content_screening": {"active_blob": {"content_hash": "a" * 64}}} + matching = _with_current_excel_metadata({"accessibility_metadata": metadata}, original) + assert matching["accessibility_metadata"] == metadata + sanitized = {"content_screening": {"active_blob": {"content_hash": "b" * 64}}} + released = _with_current_excel_metadata({"accessibility_metadata": metadata}, sanitized) + assert "accessibility_metadata" not in released + assert released["native_inspection_status"] == "requires_active_source_inspection" + assert not is_public_document_field("excel_native_inventory") + + +def test_duplicate_image_analysis_retains_each_excel_placement(tmp_path): + picture = tmp_path / "duplicate.png" + Image.frombytes("RGB", (200, 200), random.Random(11).randbytes(200 * 200 * 3)).save(picture) + path = tmp_path / "duplicates.xlsx" + workbook = Workbook() + workbook.active.title = "Figures" + workbook.active.add_image(WorkbookImage(picture), "D9") + workbook.active.add_image(WorkbookImage(picture), "F12") + workbook.save(path) + metadata, _ = extract_excel_accessibility_metadata(path, ".xlsx") + output = tmp_path / "extracted" + output.mkdir() + images, diagnostics = extract_office_embedded_images_with_diagnostics( + path, str(output), min_pixels=100, max_images=5, native_media_metadata=metadata["image_locations"], + ) + assert len(images) == 1 and diagnostics["skipped_reasons"]["duplicate_image"] == 1 + assert {item["cell"] for item in images[0]["placements"]} == {"D9", "F12"} + + +def test_image_analysis_uses_existing_gate_and_keeps_generated_content_separate(): + tree = ast.parse((APP / "functions_documents.py").read_text(encoding="utf-8")) + node = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "_enrich_excel_image_inventory") + calls = [] + + def build(*args, **kwargs): + calls.append(kwargs) + return ([{"content": "The figure contains a delivery schedule.", "sheet_name": "Figures", + "cell": "D9", "media_part": "xl/media/image1.png", "placements": [{"sheet_name": "Figures", "cell": "D9"}]}], + 1, "configured-engine", 0) + + namespace = {"_build_office_embedded_image_chunks": build} + exec(compile(ast.Module(body=[node], type_ignores=[]), "excel_image_enrichment", "exec"), namespace) + metadata, evidence, records = {"coverage_notes": [], "image_locations": {"xl/media/image1.png": {"sheet_name": "Figures"}}}, [], [] + enrich = namespace[node.name] + enrich("file.xlsx", {"enable_office_embedded_image_analysis": False}, lambda **kwargs: None, metadata, evidence, + lambda section, record: records.append((section, record))) + assert not calls and metadata["image_analysis_summary"]["status"] == "disabled" + enrich("file.xlsx", {"enable_office_embedded_image_analysis": True}, lambda **kwargs: None, metadata, evidence, + lambda section, record: records.append((section, record))) + assert len(calls) == 1 and calls[0]["native_media_metadata"] == metadata["image_locations"] + assert records[0][0] == "image_analyses" + assert records[0][1]["source_kind"] == "configured_image_analysis_not_saved_alt_text" + assert records[0][1]["cell"] == "D9" + assert "delivery schedule" in evidence[0]["content"] + + +def test_reviewed_native_publication_is_bounded_and_every_fragment_is_findable(): + units = [ + ContentUnit("sheet", "Data", {"kind": "table_sheet", "sheet_index": 1}), + ContentUnit("native-1", "reviewer feedback " * 80000, {"kind": "excel_native"}), + ContentUnit("native-2", "additional feedback " * 80000, {"kind": "excel_native"}), + ] + chunks = _tabular_publication_chunks( + units, "review.xlsx", 1024, include_native=True, + schema_text="Table: Data!A2:D9; header_row=2; Columns: Site, Cap, Delivered, Accepted", + ) + native = [chunk for chunk in chunks if chunk["unit_id"].startswith("native-")] + assert len(native) > 1 + assert sum(len(chunk["chunk_text"]) for chunk in native) <= 1000000 + assert all(chunk["chunk_text"].startswith("### Excel native inspection\n") for chunk in native) + assert all(len(chunk["chunk_text"]) <= 1024 for chunk in chunks) + assert [chunk["chunk_sequence"] for chunk in chunks] == list(range(1, len(chunks) + 1)) + assert chunks[0]["chunk_text"].startswith("Table: Data!A2:D9; header_row=2;") + assert units[1].text == "reviewer feedback " * 80000 + + sanitized = _tabular_publication_chunks(units, "review.xlsx", 1024, include_native=False) + assert all(chunk["unit_id"] == "table-schema" for chunk in sanitized) + + +def test_native_publication_rejects_chunk_sizes_that_cannot_identify_evidence(): + units = [ContentUnit("native", "feedback", {"kind": "excel_native"})] + try: + _tabular_publication_chunks(units, "review.xlsx", 8, include_native=True) + except ScreeningValidationError: + return + raise AssertionError("Native publication must not emit unidentifiable evidence fragments.") + + +def test_reviewed_workbook_schema_uses_active_bytes_and_actual_table_header(): + workbook = Workbook() + workbook.active.title = "Budget" + workbook.active.append(["Delivery review"]) + workbook.active.append(["Site", "Cap"]) + workbook.active.append(["South", 15000]) + source = io.BytesIO() + workbook.save(source) + source.seek(0) + helpers, _ = load_tabular_schema_helpers() + schema = helpers["_build_tabular_schema_summary"](source, "active-source.xlsx", ".xlsx") + units = [ + ContentUnit("sheet", "Budget", {"kind": "table_sheet", "sheet_index": 1}), + ContentUnit("original-native", "original-private-canary", {"kind": "excel_native"}), + ] + chunks = _tabular_publication_chunks( + units, "active-source.xlsx", 1024, include_native=False, schema_text=schema, + ) + text = "\n".join(chunk["chunk_text"] for chunk in chunks) + assert "header_row=2;" in text + assert "Columns (2): Site, Cap" in text + assert "original-private-canary" not in text diff --git a/functional_tests/test_excel_native_inspection_chat_evidence.py b/functional_tests/test_excel_native_inspection_chat_evidence.py new file mode 100644 index 000000000..fac62664e --- /dev/null +++ b/functional_tests/test_excel_native_inspection_chat_evidence.py @@ -0,0 +1,231 @@ +# test_excel_native_inspection_chat_evidence.py +""" +Functional tests for complete, source-backed Excel native evidence. +Version: 0.261.405 +Implemented in: 0.261.405 + +Real packages distinguish threaded compatibility copies from genuine notes, +preserve saved cell values and unavailable caches, and retain comment content. +No model or cloud service is called. +""" + +import hashlib +import json +import sys +import zipfile +from pathlib import Path +from xml.etree import ElementTree as ET + +import pytest +from openpyxl import Workbook +from openpyxl.comments import Comment + +APP = Path(__file__).resolve().parents[1] / "application" / "single_app" +sys.path.insert(0, str(APP)) + +from functions_accessibility import format_accessibility_metadata_for_chat # noqa: E402 +from functions_excel_accessibility import extract_excel_accessibility_metadata # noqa: E402 +import functions_excel_accessibility as excel # noqa: E402 +from functions_agent_document_citations import annotate_document_search_payload # noqa: E402 +from test_excel_document_accessibility_extraction import _load_functions # noqa: E402 + + +REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" +SHEET_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" + + +def add_relationship(parts, path, identifier, kind, target): + root = ET.fromstring(parts.get(path, f'')) + ET.SubElement(root, f"{{{REL_NS}}}Relationship", { + "Id": identifier, + "Type": f"http://schemas.microsoft.com/office/2017/10/relationships/{kind}", + "Target": target, + }) + parts[path] = ET.tostring(root) + + +@pytest.fixture +def review_workbook(tmp_path): + path = tmp_path / "review.xlsx" + workbook = Workbook() + workbook.active.title = "Budget" + workbook.create_sheet("Delivery") + for sheet in workbook: + sheet.append(["Delivery review"]) + sheet.append(["Site", "Cap", "Delivered", "Accepted"]) + sheet.append(["North", 12000, 18, True]) + sheet.append(["South", 15000, 12, False]) + sheet.append(["East", 9000, 9, True]) + workbook["Budget"]["B4"].comment = Comment( + "[Threaded comment]\nSet cap to 19500.\nHold approval.", + "tc={root-budget}", + ) + workbook["Budget"]["D12"].comment = Comment( + "[Threaded comment]\nArrange a review.", "tc={root-blank}", + ) + workbook["Budget"]["A3"].comment = Comment("A genuine legacy note.", "Legacy reviewer") + workbook["Delivery"]["B4"].comment = Comment( + "[Threaded comment]\nKeep cap at 15000.", "tc={root-delivery}", + ) + workbook.save(path) + with zipfile.ZipFile(path) as archive: + parts = {name: archive.read(name) for name in archive.namelist()} + add_relationship(parts, "xl/_rels/workbook.xml.rels", "people", "person", "persons/person.xml") + parts["xl/persons/person.xml"] = b'' + for index, comments in enumerate(( + """Set cap to 19500. + Hold approval. + Arrange a review.""", + """Keep cap at 15000.""", + ), start=1): + add_relationship( + parts, f"xl/worksheets/_rels/sheet{index}.xml.rels", + "thread", "threadedComment", f"../threadedComments/comments{index}.xml", + ) + parts[f"xl/threadedComments/comments{index}.xml"] = f"{comments}".encode() + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in parts.items(): + archive.writestr(name, data) + return path + + +def test_threaded_compatibility_copies_do_not_become_legacy_notes(review_workbook): + before = hashlib.sha256(review_workbook.read_bytes()).hexdigest() + metadata, evidence = extract_excel_accessibility_metadata(review_workbook, ".xlsx") + summary = metadata["comment_summary"] + assert summary == { + "legacy_notes": 1, "thread_roots": 3, "replies": 1, + "thread_messages": 4, "compatibility_copies": 3, "raw_records": 8, + } + assert len(metadata["comments"]) == 5 + assert metadata["observed_counts"]["comments"] == 5 + assert all(not str(item.get("author")).startswith("tc=") for item in metadata["comments"]) + assert any(item["kind"] == "note" and item["text"] == "A genuine legacy note." for item in metadata["comments"]) + text = "\n".join(item["content"] for item in evidence) + assert "[Threaded comment]" not in text + assert hashlib.sha256(review_workbook.read_bytes()).hexdigest() == before + + +def test_native_records_include_saved_cell_values_and_blank_cells(review_workbook): + metadata, _ = extract_excel_accessibility_metadata(review_workbook, ".xlsx") + comments = {item["id"]: item for item in metadata["comments"] if item.get("id")} + budget = comments["{root-budget}"] + delivery = comments["{root-delivery}"] + blank = comments["{root-blank}"] + assert budget["sheet"] == "Budget" and budget["cell"] == "B4" + assert budget["cell_value"] == 15000 and budget["cell_value_status"] == "saved" + assert delivery["cell_value"] == 15000 and delivery["resolved"] is True + assert blank["cell_value"] is None and blank["cell_value_status"] == "blank" + assert comments["{reply-budget}"]["date"] is None + assert comments["{reply-budget}"]["resolved"] is None + assert comments["{reply-budget}"]["parent_id"] == "{root-budget}" + + +def test_chat_formatter_supplies_comment_text_not_just_counts(review_workbook): + metadata, _ = extract_excel_accessibility_metadata(review_workbook, ".xlsx") + rendered = format_accessibility_metadata_for_chat(metadata) + assert "Set cap to 19500." in rendered + assert "Hold approval." in rendered + assert "Keep cap at 15000." in rendered + assert "Budget!B4" in rendered and "Delivery!B4" in rendered + assert "15000" in rendered + assert "reviewer feedback" in rendered.lower() + + +def test_comment_inventory_sink_preserves_text_beyond_parent_preview(tmp_path): + path = tmp_path / "long-note.xlsx" + workbook = Workbook() + workbook.active["A1"] = "Value" + workbook.active["A2"] = 0 + text = "Start " + "long review text " * 1000 + "END-OF-COMMENT" + workbook.active["A2"].comment = Comment(text, "Reviewer") + workbook.save(path) + records = [] + metadata, _ = extract_excel_accessibility_metadata( + path, ".xlsx", record_sink=lambda kind, record: records.append((kind, record)), + ) + note = next(record for kind, record in records if kind == "comments") + assert note["text"] == text + assert note["cell_value"] == 0 + assert len(json.dumps(metadata, ensure_ascii=True).encode()) <= 500000 + assert metadata["comments"][0]["text_truncated"] is True + + +def test_formula_without_cache_is_not_blank_or_zero(tmp_path): + path = tmp_path / "formula.xlsx" + workbook = Workbook() + workbook.active["A1"] = "Amount" + workbook.active["A2"] = "=1+2" + workbook.active["A2"].comment = Comment("Confirm calculated amount.", "Reviewer") + workbook.save(path) + metadata, _ = extract_excel_accessibility_metadata(path, ".xlsx") + note = metadata["comments"][0] + assert note["formula"] == "=1+2" + assert note["cell_value"] is None + assert note["cell_value_status"] == "cache_missing" + assert note["cached_value_available"] is False + + +def test_actual_chat_handoff_uses_known_records_when_relevance_has_no_matches(review_workbook): + metadata, _ = extract_excel_accessibility_metadata(review_workbook, ".xlsx") + metadata["properties"]["long_context"] = "irrelevant prefix " * 1200 + metadata["style_inventory"]["long_context"] = "layout prefix " * 1500 + report = { + "document": {"id": "doc", "file_name": "review.xlsx"}, + "accessibility_metadata": metadata, + "style_inventory": metadata["style_inventory"], + "citation_chunk": {"id": "doc_2", "document_id": "doc", "file_name": "review.xlsx", + "page_number": 2, "chunk_text": excel.EXCEL_EVIDENCE_MARKER + "\nNative comments"}, + } + calls = [] + + def get_report(document_id, user_id, **kwargs): + calls.append((document_id, user_id, kwargs)) + return report + + def unexpected_search(**kwargs): + raise AssertionError("A known complete comment inventory must not depend on relevance search.") + + namespace = { + "get_document_accessibility_payload": get_report, "search_documents": unexpected_search, + "annotate_document_search_payload": annotate_document_search_payload, + "build_excel_native_chat_evidence": excel.build_excel_native_chat_evidence, + "excel_native_question_sections": excel.excel_native_question_sections, + "EXCEL_EVIDENCE_MARKER": excel.EXCEL_EVIDENCE_MARKER, + "execute_tabular_evidence_sources": lambda sources, callback, mode, **kwargs: [callback(source) for source in sources], + } + _load_functions("route_backend_chats.py", {"_execute_excel_native_inspection_evidence"}, namespace) + result = namespace["_execute_excel_native_inspection_evidence"]( + [{"document_id": "doc", "scope": "group", "group_id": "group"}], + "List every cell note and threaded comment and the saved cell values", + "owner", "conversation", "selected", + ) + text = result["evidence_envelopes"][0]["summary"] + assert "Set cap to 19500." in text and "Hold approval." in text + assert "Keep cap at 15000." in text and "Arrange a review." in text + assert "Budget!B4" in text and "Delivery!B4" in text + assert "Native evidence segment 2" in text and "[#doc_2]" in text + assert len(text.encode("utf-8")) <= 16384 + assert calls[0][2]["active_group_ids"] == ["group"] + assert result["evidence_envelopes"][0]["coverage"]["worksheet_data_computed"] is False + + +def test_mixed_calculation_handoff_preserves_native_and_computed_evidence(review_workbook): + metadata, _ = extract_excel_accessibility_metadata(review_workbook, ".xlsx") + report = {"document": {"id": "doc", "file_name": "review.xlsx"}, "accessibility_metadata": metadata} + native = { + "evidence_envelopes": [{"document_id": "doc", "status": "completed", "summary": "native", "citations": []}], + "agent_citations": [{"function_name": "get_document_accessibility_report", + "function_arguments": {"document_id": "doc"}, "function_result": report}], + } + computed = { + "evidence_envelopes": [{"document_id": "doc", "status": "completed", + "summary": "Computed total cap = 36000.", "coverage": {}, "citations": []}], + "agent_citations": [], + } + merged = excel.merge_excel_native_evidence(computed, native, "Calculate total cap and list comments") + text = merged["evidence_envelopes"][0]["summary"] + assert "Set cap to 19500." in text + assert "Computed total cap = 36000." in text + assert len(text.encode("utf-8")) <= 16384 + assert computed["evidence_envelopes"][0]["summary"] == "Computed total cap = 36000." diff --git a/functional_tests/test_excel_preview_route.py b/functional_tests/test_excel_preview_route.py new file mode 100644 index 000000000..057bc8c82 --- /dev/null +++ b/functional_tests/test_excel_preview_route.py @@ -0,0 +1,127 @@ +# test_excel_preview_route.py +""" +Functional tests for the real bounded Excel/CSV preview handler. +Version: 0.261.405 +Implemented in: 0.261.405 + +The route body and shared reader execute against real workbook packages. +Only authenticated-document lookup and external blob I/O are isolated. +""" + +import ast +import io +from pathlib import Path +import sys +from types import SimpleNamespace +import zipfile + +from flask import Flask, jsonify, request +import pandas +import pytest + +APP = Path(__file__).resolve().parents[1] / "application" / "single_app" +sys.path.insert(0, str(APP)) + +from content_screening.contracts import ScreeningError # noqa: E402 +from excel_workbook_reader import ExcelWorkbookReader, build_excel_table_preview # noqa: E402 +from test_excel_table_discovery_and_scale import five_tables # noqa: E402 + + +__all__ = ["five_tables"] + + +def preview_app(content, filename="tables.xlsx", allowed=True, failure=None): + source = (APP / "route_enhanced_citations.py").read_text(encoding="utf-8") + tree = ast.parse(source) + calls = {"reads": 0, "lookups": 0, "errors": []} + app = Flask(__name__) + document = {"id": "workbook", "file_name": filename, "user_id": "owner"} + + def get_document(user_id, document_id): + calls["lookups"] += 1 + assert user_id == "owner" and document_id == "workbook" + return (jsonify(document), 200) if allowed else (jsonify({"error": "Access denied."}), 403) + + def read_bytes(doc, **kwargs): + calls["reads"] += 1 + if failure is not None: + raise failure + return document, content + + namespace = { + "pandas": pandas, "io": io, "zipfile": zipfile, "jsonify": jsonify, "request": request, + "get_document": get_document, "get_current_user_id": lambda: "owner", + "get_settings": lambda: {}, "ScreeningError": ScreeningError, + "build_excel_table_preview": build_excel_table_preview, + "read_available_document_bytes": read_bytes, + "_resolve_document_blob_reference": lambda doc: ("container", "source"), + "_log_enhanced_citations_error": lambda *args, **kwargs: calls["errors"].append(args), + "CLIENTS": {"storage_account_office_docs_client": SimpleNamespace( + get_blob_client=lambda **kwargs: SimpleNamespace( + get_blob_properties=lambda: SimpleNamespace(size=len(content)), + ), + )}, + } + names = {"_sanitize_tabular_preview_value", "_serialize_tabular_preview_table", "_preview_integer"} + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name in names: + exec(compile(ast.Module(body=[node], type_ignores=[]), "preview_helpers", "exec"), namespace) + registrar = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "register_enhanced_citations_routes") + handler = next(node for node in registrar.body if isinstance(node, ast.FunctionDef) and node.name == "get_enhanced_citation_tabular_preview") + handler.decorator_list = [] + exec(compile(ast.Module(body=[handler], type_ignores=[]), "preview_route", "exec"), namespace) + app.add_url_rule("/preview", view_func=namespace[handler.name]) + return app.test_client(), calls + + +def test_real_route_selects_each_table_and_retains_physical_rows(five_tables): + path, _, origins = five_tables + with ExcelWorkbookReader(path) as reader: + tables = reader.build_manifest()["sheets"][0]["tables"] + client, calls = preview_app(path.read_bytes()) + for index, (table, (header_row, _)) in enumerate(zip(tables, origins), start=1): + response = client.get("/preview", query_string={ + "doc_id": "workbook", "sheet_name": "Five tables", "table_id": table["id"], + "start_row": "2", "max_rows": "1", + }) + assert response.status_code == 200 + payload = response.get_json() + assert payload["columns"] == ["Case", "Amount"] + assert payload["rows"] == [[f"T{index}-3", str(index * 100 + 3)]] + assert payload["row_numbers"] == [header_row + 3] + assert payload["table_count"] == 5 and payload["has_more"] is False + assert calls["reads"] == 5 + + +def test_csv_paging_keeps_headers_and_reaches_late_records(): + content = ("Case,Amount\n" + "".join(f"C{index},{index}\n" for index in range(1, 15001))).encode() + client, _ = preview_app(content, "data.csv") + response = client.get("/preview", query_string={"doc_id": "workbook", "start_row": "14999", "max_rows": "2"}) + assert response.status_code == 200 + payload = response.get_json() + assert payload["columns"] == ["Case", "Amount"] + assert payload["rows"] == [["C15000", "15000"]] + assert payload["total_rows"] == 15000 and payload["row_numbers"] == [15001] + + +@pytest.mark.parametrize("query", [{"max_rows": "0"}, {"max_rows": "501"}, {"start_row": "-1"}, {"header_row": "bad"}]) +def test_invalid_windows_are_explicit_errors(query): + client, calls = preview_app(b"Case,Amount\nA,1\n", "data.csv") + response = client.get("/preview", query_string={"doc_id": "workbook", **query}) + assert response.status_code == 400 + assert calls["reads"] == 0 and calls["errors"] + + +def test_permission_denial_precedes_blob_access(): + client, calls = preview_app(b"private", allowed=False) + response = client.get("/preview", query_string={"doc_id": "workbook"}) + assert response.status_code == 403 + assert calls["reads"] == 0 + + +def test_provider_failure_never_reaches_browser(): + client, calls = preview_app(b"private", failure=RuntimeError("private-provider-secret-canary")) + response = client.get("/preview", query_string={"doc_id": "workbook"}) + assert response.status_code == 500 + assert "private-provider-secret-canary" not in response.get_data(as_text=True) + assert calls["errors"] diff --git a/functional_tests/test_excel_table_discovery_and_scale.py b/functional_tests/test_excel_table_discovery_and_scale.py new file mode 100644 index 000000000..5364a6949 --- /dev/null +++ b/functional_tests/test_excel_table_discovery_and_scale.py @@ -0,0 +1,144 @@ +# test_excel_table_discovery_and_scale.py +""" +Functional tests for streaming, table-aware saved Excel inspection. +Version: 0.261.405 +Implemented in: 0.261.405 + +Named and ordinary regions retain their own headers and cell coordinates. +Large fixtures are generated incrementally and neither metadata nor previews +materialize the complete dataset. +""" + +import hashlib +import json +import sys +from pathlib import Path + +import pytest +from openpyxl import Workbook +from openpyxl.worksheet.table import Table + +APP = Path(__file__).resolve().parents[1] / "application" / "single_app" +sys.path.insert(0, str(APP)) + + +def workbook_reader(path): + from excel_workbook_reader import ExcelWorkbookReader + return ExcelWorkbookReader(path) + + +@pytest.fixture(params=[False, True], ids=["ordinary-blocks", "named-tables"]) +def five_tables(request, tmp_path): + path = tmp_path / "five-tables.xlsx" + workbook = Workbook() + sheet = workbook.active + sheet.title = "Five tables" + origins = [(3, 1), (3, 5), (10, 1), (10, 5), (17, 1)] + for index, (row, column) in enumerate(origins, start=1): + sheet.cell(row - 1, column, f"Review table {index}") + sheet.cell(row, column, "Case") + sheet.cell(row, column + 1, "Amount") + for offset in range(1, 4): + sheet.cell(row + offset, column, f"T{index}-{offset}") + sheet.cell(row + offset, column + 1, index * 100 + offset) + if request.param: + from openpyxl.utils.cell import get_column_letter + ref = f"{get_column_letter(column)}{row}:{get_column_letter(column + 1)}{row + 3}" + sheet.add_table(Table(displayName=f"Review{index}", ref=ref)) + workbook.save(path) + return path, request.param, origins + + +def test_five_tables_have_independent_headers_ranges_and_values(five_tables): + path, named, origins = five_tables + before = hashlib.sha256(path.read_bytes()).digest() + with workbook_reader(path) as reader: + manifest = reader.build_manifest() + tables = manifest["sheets"][0]["tables"] + assert len(tables) == 5 + assert len({table["id"] for table in tables}) == 5 + for index, (table, (row, column)) in enumerate(zip(tables, origins), start=1): + assert table["header_row"] == row + assert table["min_column"] == column + assert table["row_count"] == 3 + assert table["origin"] == ("named" if named else "inferred") + assert [item["name"] for item in table["columns"]] == ["Case", "Amount"] + page = reader.read_table_page(table["id"], max_rows=2) + assert page["rows"] == [[f"T{index}-1", index * 100 + 1], [f"T{index}-2", index * 100 + 2]] + assert page["row_numbers"] == [row + 1, row + 2] + assert page["has_more"] is True and page["next_start_row"] == 2 + last = reader.read_table_page(table["id"], start_row=2, max_rows=2) + assert last["rows"] == [[f"T{index}-3", index * 100 + 3]] + assert last["has_more"] is False + assert hashlib.sha256(path.read_bytes()).digest() == before + + +@pytest.mark.parametrize("row_count", [50000, 100000]) +def test_large_workbook_last_row_and_compact_manifest(tmp_path, row_count): + path = tmp_path / f"large-{row_count}.xlsx" + workbook = Workbook(write_only=True) + sheet = workbook.create_sheet("Large data") + sheet.append(["Dataset review"]) + sheet.append(["Case", "Amount"]) + for index in range(1, row_count + 1): + sheet.append([f"CASE-{index:06d}", index]) + workbook.save(path) + with workbook_reader(path) as reader: + manifest = reader.build_manifest() + table = manifest["sheets"][0]["tables"][0] + assert table["header_row"] == 2 and table["row_count"] == row_count + assert len(json.dumps(manifest, ensure_ascii=True).encode()) < 20000 + page = reader.read_table_page(table["id"], start_row=row_count - 2, max_rows=2) + assert page["rows"] == [[f"CASE-{row_count - 1:06d}", row_count - 1], [f"CASE-{row_count:06d}", row_count]] + assert page["row_numbers"] == [row_count + 1, row_count + 2] + assert page["has_more"] is False + cells = reader.read_cells("Large data", [f"B{row_count + 2}"]) + assert cells[f"B{row_count + 2}"]["value"] == row_count + + +def test_repeated_headers_do_not_become_business_rows(tmp_path): + path = tmp_path / "repeated.xlsx" + workbook = Workbook() + sheet = workbook.active + for row in [ + ["Case", "Amount"], ["A", 10], ["B", 20], + ["Case", "Amount"], ["C", 30], + ]: + sheet.append(row) + workbook.save(path) + with workbook_reader(path) as reader: + table = reader.build_manifest()["sheets"][0]["tables"][0] + assert table["row_count"] == 3 + page = reader.read_table_page(table["id"]) + assert page["rows"] == [["A", 10], ["B", 20], ["C", 30]] + assert page["row_numbers"] == [2, 3, 5] + + +def test_empty_and_unlabelled_regions_do_not_invent_headers(tmp_path): + path = tmp_path / "unlabelled.xlsx" + workbook = Workbook() + workbook.active.title = "Values" + workbook.active.append([10, False]) + workbook.active.append([20, True]) + workbook.create_sheet("Empty") + workbook.save(path) + with workbook_reader(path) as reader: + manifest = reader.build_manifest() + table = manifest["sheets"][0]["tables"][0] + assert table["header_row"] is None + assert table["header_status"] == "absent" + assert table["row_count"] == 2 + assert manifest["sheets"][1]["tables"] == [] + page = reader.read_table_page(table["id"]) + assert page["columns"] == ["A", "B"] + assert page["rows"] == [[10, False], [20, True]] + + +def test_invalid_table_and_oversized_preview_are_rejected(five_tables): + path, _, _ = five_tables + with workbook_reader(path) as reader: + table = reader.build_manifest()["sheets"][0]["tables"][0] + with pytest.raises(ValueError, match="table"): + reader.read_table_page("../../another-document") + with pytest.raises(ValueError, match="max_rows"): + reader.read_table_page(table["id"], max_rows=501) diff --git a/functional_tests/test_mixed_source_chat_search_consistency.py b/functional_tests/test_mixed_source_chat_search_consistency.py index dd6af150f..27b596c36 100644 --- a/functional_tests/test_mixed_source_chat_search_consistency.py +++ b/functional_tests/test_mixed_source_chat_search_consistency.py @@ -2,7 +2,7 @@ # test_mixed_source_chat_search_consistency.py """ Functional test for mixed-source Chat and Search consistency. -Version: 0.261.399 +Version: 0.261.405 Implemented in: 0.250.064; additive tabular evidence gating updated in 0.260.025; derived from Enhanced Citations in 0.261.266 @@ -35,7 +35,11 @@ sys.path.insert(0, str(APP_ROOT)) import functions_mixed_source_orchestration as orchestration # pyright: ignore[reportMissingImports] -from functions_excel_accessibility import is_excel_native_inspection_question +from functions_excel_accessibility import ( + has_excel_native_inspection_intent, + is_excel_native_inspection_question, + merge_excel_native_evidence, +) def _read(path): @@ -121,6 +125,8 @@ async def no_generated_output(**kwargs): namespace = { "asyncio": asyncio, "is_excel_native_inspection_question": is_excel_native_inspection_question, + "has_excel_native_inspection_intent": has_excel_native_inspection_intent, + "merge_excel_native_evidence": merge_excel_native_evidence, "has_request_context": lambda: False, "build_tabular_file_contexts_from_manifest": orchestration.build_tabular_file_contexts_from_manifest, "is_tabular_processing_enabled": lambda settings: settings.get("tabular", True), diff --git a/functional_tests/test_support/document_analysis.py b/functional_tests/test_support/document_analysis.py index 6e8a54b4f..fcbb5afd8 100644 --- a/functional_tests/test_support/document_analysis.py +++ b/functional_tests/test_support/document_analysis.py @@ -1,7 +1,7 @@ # document_analysis.py """Original-source fixtures and isolated SDK seams for the real Analyze producer. -Version: 0.261.122 +Version: 0.261.405 Implemented in: 0.261.109 """ @@ -133,6 +133,7 @@ def assert_group_role(user_id, group_id, **kwargs): cosmos_group_documents_container=screening_container('group'), cosmos_public_documents_container=screening_container('public'), cosmos_content_screening_container=None, + storage_account_content_screening_container_name='fixture-screening', ): targets = ('functions_search', 'functions_search_service', 'functions_document_analysis', 'functions_debug') originals = {name: sys.modules.get(name) for name in (*stubs, *targets)} diff --git a/functional_tests/test_tabular_enhanced_citations_schema_summary_fix.py b/functional_tests/test_tabular_enhanced_citations_schema_summary_fix.py index 06cd9a83a..93af03183 100644 --- a/functional_tests/test_tabular_enhanced_citations_schema_summary_fix.py +++ b/functional_tests/test_tabular_enhanced_citations_schema_summary_fix.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ Functional test for enhanced-citation tabular schema summary fallback fix. -Version: 0.240.023 +Version: 0.261.405 Implemented in: 0.240.023 This test ensures enhanced-citation tabular uploads stay in schema-summary mode, @@ -20,6 +20,9 @@ sys.path.append(ROOT_DIR) sys.path.append(os.path.join(ROOT_DIR, 'application', 'single_app')) +from excel_workbook_reader import ExcelWorkbookReader # noqa: E402 +from functions_tabular_csv_query import read_tabular_csv # noqa: E402 +from test_support.versioning import assert_app_version_at_least # noqa: E402 FUNCTIONS_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'functions_documents.py') CONFIG_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'config.py') TARGET_FUNCTIONS = { @@ -58,6 +61,8 @@ def load_tabular_schema_helpers(): namespace = { 'os': os, 'pandas': pandas, + 'ExcelWorkbookReader': ExcelWorkbookReader, + 'read_tabular_csv': read_tabular_csv, } exec(compile(module, FUNCTIONS_FILE, 'exec'), namespace) return namespace, source @@ -137,7 +142,7 @@ def test_version_bump_applied(): try: version = read_config_version() - assert version == '0.240.023', version + assert_app_version_at_least("0.240.023") print('✅ Config version bump passed') return True diff --git a/functional_tests/test_tabular_group_blob_context_and_sheet_whitespace.py b/functional_tests/test_tabular_group_blob_context_and_sheet_whitespace.py index 321ce9caf..c6e7e82e2 100644 --- a/functional_tests/test_tabular_group_blob_context_and_sheet_whitespace.py +++ b/functional_tests/test_tabular_group_blob_context_and_sheet_whitespace.py @@ -2,8 +2,9 @@ # test_tabular_group_blob_context_and_sheet_whitespace.py """ Functional test for tabular group blob context and sheet whitespace handling. -Version: 0.240.031 +Version: 0.261.405 Implemented in: 0.240.031 +Request-scoped authorization fixture updated in: 0.261.405 This test ensures workbook tabs with trailing whitespace still resolve correctly and that tabular analysis can reuse pre-resolved group blob @@ -14,7 +15,9 @@ import importlib.util import os import sys +from unittest.mock import patch +from flask import Flask, g import pandas as pd @@ -148,12 +151,35 @@ def test_resolved_group_blob_override_survives_missing_group_id(): 'group-123/CCO-Legal File Plan 2025_Final Approved.xlsx', ) - resolved_container, resolved_blob = plugin._resolve_blob_location_with_fallback( - 'test-user', - 'test-conversation', - 'CCO-Legal File Plan 2025_Final Approved.xlsx', - 'group', - ) + location = ('group-documents', 'group-123/CCO-Legal File Plan 2025_Final Approved.xlsx') + + def authorize_group(user_id, group_id, **kwargs): + assert (user_id, group_id) == ('test-user', 'group-123') + + def authorize_blob(container, blob, *, purpose='native'): + assert (container, blob) == location + return None, 'test-user' + + app = Flask(__name__) + with app.test_request_context('/'), patch.object( + PLUGIN_MODULE, 'get_current_user_id', return_value='test-user', + ), patch.object( + PLUGIN_MODULE, 'assert_group_role', side_effect=authorize_group, + ) as membership_check, patch.object( + PLUGIN_MODULE, 'resolve_available_blob_location', + side_effect=lambda container, blob, **kwargs: (container, blob), + ), patch.object(plugin, '_assert_blob_screening_access', side_effect=authorize_blob): + g.authorized_chat_context = { + 'user_id': 'test-user', 'conversation_id': 'test-conversation', + 'active_group_ids': ['group-123'], 'active_group_id': 'group-123', + } + resolved_container, resolved_blob = plugin._resolve_blob_location_with_fallback( + 'test-user', + 'test-conversation', + 'CCO-Legal File Plan 2025_Final Approved.xlsx', + 'group', + ) + assert membership_check.call_count == 1 assert resolved_container == 'group-documents', (resolved_container, resolved_blob) assert resolved_blob == 'group-123/CCO-Legal File Plan 2025_Final Approved.xlsx', (resolved_container, resolved_blob) diff --git a/functional_tests/test_tabular_large_result_pagination.py b/functional_tests/test_tabular_large_result_pagination.py index c142b54fd..49e5fe22f 100644 --- a/functional_tests/test_tabular_large_result_pagination.py +++ b/functional_tests/test_tabular_large_result_pagination.py @@ -2,8 +2,9 @@ # test_tabular_large_result_pagination.py """ Functional test for tabular SK large-result pagination and output trimming. -Version: 0.250.155 +Version: 0.261.405 Implemented in: 0.242.067; bounded CSV query path in 0.250.060; source descriptor generalization in 0.250.127; serialized row-size estimation in 0.250.152; contains replay semantics in 0.250.155 +Explicit offline source authorization fixtures updated in: 0.261.405 This test ensures row-returning tabular processing tools support start_row/max_rows pagination, avoid skipped rows after auto-trimming oversized output, honor @@ -41,6 +42,15 @@ TabularProcessingPlugin = PLUGIN_MODULE.TabularProcessingPlugin +def authorize_fixture_source(plugin, container_name, blob_path): + """Keep row-shaping unit fixtures restricted to their one mocked source.""" + def authorize(container, blob, *, purpose="native"): + assert (container, blob) == (container_name, blob_path) + return None, 'test-user' + + plugin._assert_blob_screening_access = authorize + + class MockCsvBlobClient: """Minimal blob client for bounded CSV query tests.""" @@ -103,6 +113,11 @@ def read_dataframe(container, blob, sheet_name=None, sheet_index=None, require_e return workbook_frames[selected_sheet].copy() plugin._read_tabular_blob_to_dataframe = read_dataframe + # These row-shaping fixtures have frames, not an original workbook package. + # Real workbook replay and authorization are covered by the table-aware suite. + plugin._build_tabular_generated_export_internal_metadata = lambda **kwargs: { + 'tabular_generated_export_source_error': 'This in-memory fixture has no version-pinned original.', + } return plugin @@ -363,6 +378,7 @@ def test_query_tabular_csv_uses_bounded_shared_engine_and_exact_descriptor(): 'nested/version-7/large-results.csv', ) plugin._get_blob_service_client = lambda: MockCsvBlobServiceClient(csv_content) + authorize_fixture_source(plugin, 'mock-container', 'nested/version-7/large-results.csv') plugin._read_tabular_blob_to_dataframe = lambda *args, **kwargs: (_ for _ in ()).throw( AssertionError('CSV query must not use the whole-DataFrame reader') ) @@ -419,6 +435,7 @@ def test_filter_rows_csv_attaches_replayable_descriptor_for_full_cohort(): 'nested/version-7/large-results.csv', ) plugin._get_blob_service_client = lambda: MockCsvBlobServiceClient(csv_content) + authorize_fixture_source(plugin, 'mock-container', 'nested/version-7/large-results.csv') plugin._read_tabular_blob_to_dataframe = lambda *args, **kwargs: source_frame.copy() result = asyncio.run(plugin.filter_rows( @@ -477,6 +494,7 @@ def test_filter_rows_csv_contains_replay_uses_literal_semantics(): 'nested/version-7/large-results.csv', ) plugin._get_blob_service_client = lambda: MockCsvBlobServiceClient(csv_content) + authorize_fixture_source(plugin, 'mock-container', 'nested/version-7/large-results.csv') plugin._read_tabular_blob_to_dataframe = lambda *args, **kwargs: source_frame.copy() result = asyncio.run(plugin.filter_rows( @@ -530,6 +548,7 @@ def test_filter_rows_csv_rejects_non_replayable_normalized_matching(): 'nested/version-7/large-results.csv', ) plugin._get_blob_service_client = lambda: MockCsvBlobServiceClient(csv_content) + authorize_fixture_source(plugin, 'mock-container', 'nested/version-7/large-results.csv') plugin._read_tabular_blob_to_dataframe = lambda *args, **kwargs: source_frame.copy() result = asyncio.run(plugin.filter_rows( @@ -547,7 +566,7 @@ def test_filter_rows_csv_rejects_non_replayable_normalized_matching(): assert payload['total_matches'] == 2, payload assert 'tabular_generated_export_source' not in metadata, metadata - assert 'normalize_match=true' in metadata['tabular_generated_export_source_error'], metadata + assert 'bounded row-local query' in metadata['tabular_generated_export_source_error'], metadata print('✅ Normalized filter_rows replay rejection passed') return True @@ -579,6 +598,7 @@ def test_search_rows_csv_attaches_replayable_multi_column_descriptor(): 'nested/version-7/large-results.csv', ) plugin._get_blob_service_client = lambda: MockCsvBlobServiceClient(csv_content) + authorize_fixture_source(plugin, 'mock-container', 'nested/version-7/large-results.csv') plugin._read_tabular_blob_to_dataframe = lambda *args, **kwargs: source_frame.copy() result = asyncio.run(plugin.search_rows( diff --git a/functional_tests/test_tabular_model_token_budgets.py b/functional_tests/test_tabular_model_token_budgets.py index cc72a788a..45dcc3bf6 100644 --- a/functional_tests/test_tabular_model_token_budgets.py +++ b/functional_tests/test_tabular_model_token_budgets.py @@ -1,8 +1,9 @@ # test_tabular_model_token_budgets.py """ Functional tests for scoped tabular model budgets and generation ceilings. -Version: 0.261.035 +Version: 0.261.405 Implemented in: 0.261.035 +Source checkpoint identity fixture updated in: 0.261.405 The principal regressions use the shipped catalog and real shared resolver. Only endpoint retrieval and model I/O are replaced with in-memory boundaries; @@ -35,7 +36,7 @@ ) from semantic_kernel.contents.chat_history import ChatHistory as SKChatHistory -from test_tabular_row_orchestration_scale import _load_performance_helpers +from test_tabular_row_orchestration_scale import _load_contract_helpers, _load_performance_helpers # The shared standalone harness establishes the application import path first. from functions_model_budget_runtime import prepare_model_execution_settings @@ -743,11 +744,15 @@ def test_large_context_threshold_and_small_input_boundary_are_preserved(): def test_small_character_budget_survives_source_staging(): helpers, _ = _load_budget_helpers() + contracts = _load_contract_helpers() + helpers['_prepare_tabular_source_rows'] = contracts['_prepare_tabular_source_rows'] source_rows = [{'value': 'x' * 1700} for _ in range(3)] checkpoints = [] def checkpoint(run, rows, source_scan_row_count): - checkpoints.append(list(rows)) + checkpoints.append(contracts['_prepare_tabular_source_rows']( + rows, start_row=run.get('source_staged_rows', 0), token_namespace=run['id'], + )) run['source_staged_rows'] = run.get('source_staged_rows', 0) + len(rows) run['source_staged_batches'] = len(checkpoints) run['source_scan_row_count'] = source_scan_row_count @@ -755,7 +760,7 @@ def checkpoint(run, rows, source_scan_row_count): helpers.update({ 'TABULAR_EXTENSIONS': {'csv'}, - '_iter_versioned_tabular_source_rows': lambda *args: enumerate(source_rows, start=1), + '_iter_versioned_tabular_source_rows': lambda *args, **kwargs: enumerate(source_rows, start=1), '_dump_generated_output_json': json.dumps, '_checkpoint_source_input_batch': checkpoint, '_input_blob_path': lambda *args: args[-1], diff --git a/functional_tests/test_tabular_preview_json_sanitization_fix.py b/functional_tests/test_tabular_preview_json_sanitization_fix.py index 2af713e32..7eaf153b4 100644 --- a/functional_tests/test_tabular_preview_json_sanitization_fix.py +++ b/functional_tests/test_tabular_preview_json_sanitization_fix.py @@ -2,7 +2,7 @@ # test_tabular_preview_json_sanitization_fix.py """ Functional test for tabular preview JSON sanitization. -Version: 0.240.030 +Version: 0.261.405 Implemented in: 0.240.030 This test ensures the enhanced citation tabular preview converts pandas null-like @@ -131,8 +131,8 @@ def test_route_uses_serialized_preview_output_and_version_bump(): _, source = load_preview_helpers() required_snippets = [ - 'columns, rows = _serialize_tabular_preview_table(preview)', - '"columns": columns,', + 'names, rows = _serialize_tabular_preview_table(frame)', + 'result["columns"], result["rows"] = _serialize_tabular_preview_table(frame)', '"rows": rows,', ] missing = [snippet for snippet in required_snippets if snippet not in source] diff --git a/functional_tests/test_tabular_row_orchestration_scale.py b/functional_tests/test_tabular_row_orchestration_scale.py index a8fdc94b4..9e9e7ad85 100644 --- a/functional_tests/test_tabular_row_orchestration_scale.py +++ b/functional_tests/test_tabular_row_orchestration_scale.py @@ -1,8 +1,10 @@ # test_tabular_row_orchestration_scale.py """ Functional test for scalable per-row tabular orchestration. -Version: 0.261.396 +Version: 0.261.405 Updated in: 0.261.035 for shared model-budget resolution and scoped batching. +Table-aware descriptors and real-package replay fixtures updated in: 0.261.405 +SDK-default conversation/message fixture contracts updated in: 0.261.405 Implemented in: 0.250.060; generated CSV formula safety in 0.250.065; generated file export routing in 0.250.072; source descriptor generalization in 0.250.127; unified durable run contract in 0.250.128; hierarchical analysis in 0.250.129; combined analysis and export in 0.250.130; scale validation in 0.250.132; direct source-backed exhaustive queueing in 0.250.133; direct queue call-site hardening in 0.250.134; model-validation auto retry in 0.250.135; model-aware parallel throughput in 0.250.136; Phase 1 acceleration contracts and observability in 0.250.137; Phase 2 truthful background handoff in 0.250.138; Phase 3 durable LLM generation planning in 0.250.139; Phase 4 compact row response protocol in 0.250.140; Phase 5 completion-driven checkpointing in 0.250.141; Phase 6 rolling worker pool in 0.250.142; Phase 7 independent batch retries in 0.250.143; Phase 8 scale, chaos, and rollout in 0.250.144; background metadata streaming fix in 0.250.145; source-token echo recovery in 0.250.146; fixed-window stale heartbeat fix in 0.250.147; nested CSV output recovery in 0.250.148; generic tabular artifact routing and fast startup in 0.250.149; balanced concurrency waves and default completion checkpoints in 0.250.152; Search shared preflight adapter in 0.250.159; aggregate route-helper harness coverage in 0.250.166; Analyze artifact Phase 7A harness compatibility updated in 0.250.178; reviewed correctness planning and semantic validation updated in 0.250.179; artifact publication lifecycle updated in 0.250.180; safe failure helper compatibility updated in 0.250.199; exhaustive Markdown scale compatibility updated in 0.250.200 This test ensures generated exports preserve source identity and row order while @@ -12,6 +14,7 @@ import ast import asyncio import csv +from copy import deepcopy import heapq import hashlib import io @@ -21,6 +24,7 @@ import math import os import pandas +import pytest import re import sys import time @@ -31,7 +35,9 @@ from collections import Counter from typing import Any, Dict, Optional +from azure.core import MatchConditions from azure.cosmos._cosmos_responses import CosmosDict +from azure.cosmos.exceptions import CosmosHttpResponseError, CosmosResourceNotFoundError REPO_ROOT = Path(__file__).resolve().parents[1] @@ -127,7 +133,10 @@ 'iter_tabular_csv_query_rows', 'validate_tabular_csv_query_expression', } -AUTHORIZATION_FUNCTIONS = {'_authorize_tabular_export_run_execution'} +AUTHORIZATION_FUNCTIONS = { + '_authorize_tabular_export_run_conversation', + '_authorize_tabular_export_run_execution', +} CANCELLATION_FUNCTIONS = { '_is_artifact_publication_recoverable', '_can_cancel_run', @@ -640,55 +649,6 @@ def _load_tabular_request_intent_helpers(): return namespace -def _load_tabular_descriptor_builder(): - """Load the plugin's version-pinned tabular descriptor builder in isolation.""" - module_tree = ast.parse( - TABULAR_PLUGIN_MODULE.read_text(encoding='utf-8'), - filename=str(TABULAR_PLUGIN_MODULE), - ) - plugin_class = next( - node - for node in module_tree.body - if isinstance(node, ast.ClassDef) and node.name == 'TabularProcessingPlugin' - ) - function_node = next( - node - for node in plugin_class.body - if isinstance(node, ast.FunctionDef) - and node.name == '_build_generated_export_query_descriptor_from_location' - ) - namespace = { - 'List': list, - 'Optional': Optional, - 'validate_tabular_csv_query_expression': _load_source_reader_helpers()[ - 'validate_tabular_csv_query_expression' - ], - } - exec( - compile(ast.Module(body=[function_node], type_ignores=[]), str(TABULAR_PLUGIN_MODULE), 'exec'), - namespace, - ) - return namespace['_build_generated_export_query_descriptor_from_location'] - - -def _load_tabular_source_replay_helper(fake_plugin, version_checks): - """Load durable tabular replay with a fake workbook parser and version check.""" - function_node = _get_function_node('_iter_versioned_tabular_source_rows') - namespace = { - 'MatchConditions': SimpleNamespace(IfNotModified='if-not-modified'), - 'TABULAR_EXPORT_FINAL_SPOOL_MAX_MEMORY_BYTES': 1024 * 1024, - 'TabularProcessingPlugin': lambda: fake_plugin, - '_get_versioned_source_blob_client': lambda descriptor: version_checks.append(descriptor) or object(), - 'iter_tabular_csv_query_rows': lambda **kwargs: (), - 'tempfile': __import__('tempfile'), - } - exec( - compile(ast.Module(body=[function_node], type_ignores=[]), str(EXPORT_MODULE), 'exec'), - namespace, - ) - return namespace['_iter_versioned_tabular_source_rows'] - - def _build_query_invocation(start_row, row_count, total_matches=300, source_etag='etag-source-7'): class InvocationPayload(dict): pass @@ -953,13 +913,15 @@ def upload_json_blob(path, payload, metadata=None): def _load_authorization_helper(conversation_owner, visible_public_ids=None, group_authorizer=None): - class CosmosResourceNotFoundError(Exception): - pass - class ConversationContainer: def read_item(self, item, partition_key): - assert item == partition_key - return CosmosDict({'id': item, 'user_id': conversation_owner}, response_headers={}) + if item != partition_key: + raise CosmosResourceNotFoundError(status_code=404, message='Conversation partition not found.') + etag = f'"conversation-{item}-1"' + return CosmosDict( + {'id': item, 'user_id': conversation_owner, '_etag': etag}, + response_headers={'etag': etag}, + ) module_tree = ast.parse(EXPORT_MODULE.read_text(encoding='utf-8'), filename=str(EXPORT_MODULE)) selected_nodes = [ @@ -1376,12 +1338,15 @@ def _assert_concise_background_handoff_contract(text, expected_row_count, expect def _load_idempotent_artifact_helper(): - class CosmosResourceNotFoundError(Exception): - pass - class ConversationContainer: def read_item(self, item, partition_key): - return CosmosDict({'id': item, 'user_id': 'user-1'}, response_headers={}) + if item != partition_key: + raise CosmosResourceNotFoundError(status_code=404, message='Conversation partition not found.') + etag = f'"conversation-{item}-1"' + return CosmosDict( + {'id': item, 'user_id': 'user-1', '_etag': etag}, + response_headers={'etag': etag}, + ) class MessageContainer: def __init__(self): @@ -1389,14 +1354,26 @@ def __init__(self): self.upsert_count = 0 def read_item(self, item, partition_key): - if item not in self.items: - raise CosmosResourceNotFoundError() - return CosmosDict(dict(self.items[item]), response_headers={}) - - def upsert_item(self, item): - self.items[item['id']] = dict(item) + key = (partition_key, item) + if key not in self.items: + raise CosmosResourceNotFoundError(status_code=404, message='Message partition not found.') + saved = deepcopy(self.items[key]) + return CosmosDict(saved, response_headers={'etag': saved['_etag']}) + + def upsert_item(self, body, *, etag=None, match_condition=None): + if not isinstance(body, dict) or not body.get('id') or not body.get('conversation_id'): + raise ValueError('A message id and conversation partition are required.') + key = (body['conversation_id'], body['id']) + current = self.items.get(key) + if match_condition == MatchConditions.IfNotModified and ( + current is None or current['_etag'] != etag + ): + raise CosmosHttpResponseError(status_code=412, message='Message ETag changed.') + saved = deepcopy(dict(body)) + saved['_etag'] = f'"message-{self.upsert_count + 1}"' + self.items[key] = saved self.upsert_count += 1 - return CosmosDict(dict(item), response_headers={}) + return CosmosDict(deepcopy(saved), response_headers={'etag': saved['_etag']}) class BlobClient: def __init__(self): @@ -4849,102 +4826,79 @@ def test_downloadable_complete_csv_request_uses_shared_artifact_intent(): assert helpers['question_requests_tabular_generated_output'](user_question) is True -def test_durable_tabular_descriptor_supports_all_configured_formats(): - """Version-pinned replay descriptors cover CSV and every supported workbook type.""" - descriptor_builder = _load_tabular_descriptor_builder() - - class FakeDescriptorPlugin: - SUPPORTED_EXTENSIONS = ('.csv', '.xls', '.xlsm', '.xlsx') - - @staticmethod - def _infer_source_from_container(container_name): - assert container_name == 'user-documents' - return 'workspace' +def test_durable_tabular_descriptor_supports_all_configured_formats(monkeypatch): + """Build real version-pinned descriptors without replacing authorization or readers.""" + # Import the runtime harness at execution time so its I/O seams cannot leak + # into the other AST-only tests in this module. + from test_native_tabular_compute_service import USER, native_runtime + from test_tabular_table_aware_analytics_replay import _legacy_workbook_bytes, _one_table, _set_source - @staticmethod - def _get_tabular_blob_version(*args, **kwargs): - raise AssertionError('The supplied version pin must be used') - - plugin = FakeDescriptorPlugin() for source_format in ('csv', 'xlsx', 'xls', 'xlsm'): is_workbook = source_format != 'csv' - descriptor = descriptor_builder( - plugin, - container_name='user-documents', - blob_path=f'user-1/source.{source_format}', - filename=f'source.{source_format}', - query_expression='index == index', - expected_row_count=12, - blob_version={'blob_etag': 'etag-12', 'blob_size': 2048}, - sheet_names=['First', 'Second'] if is_workbook else None, - ) - - assert descriptor['source_format'] == source_format - assert descriptor['expected_row_count'] == 12 - assert descriptor['blob_etag'] == 'etag-12' - if is_workbook: - assert descriptor['sheet_names'] == ['First', 'Second'] - else: - assert 'sheet_names' not in descriptor - - -def test_durable_workbook_replay_preserves_sheet_order_and_resume_position(): - """Workbook rows replay in sheet order and resume after the last staged physical row.""" - class FakeWorkbookPlugin: - def __init__(self): - self.frames = { - 'First': pandas.DataFrame([ - {'Item_ID': 'A-1', 'Value': 'one'}, - {'Item_ID': 'A-2', 'Value': 'two'}, - ]), - 'Second': pandas.DataFrame([ - {'Item_ID': 'B-1', 'Value': 'three'}, - ]), - } - - @staticmethod - def _parse_optional_column_list_argument(value): - assert value is None - return None - - def _read_tabular_blob_to_dataframe(self, container, blob_path, sheet_name, require_explicit_sheet): - assert container == 'user-documents' - assert blob_path == 'user-1/source.xlsx' - assert require_explicit_sheet is True - return self.frames[sheet_name].copy() - - @staticmethod - def _try_numeric_conversion(dataframe): - return dataframe + count = 1 if source_format == 'xls' else 12 + with native_runtime(monkeypatch, row_count=12, filename=f'source.{source_format}') as runtime: + if is_workbook: + _set_source(runtime, _legacy_workbook_bytes() if source_format == 'xls' else _one_table(12)) + plugin = runtime.engine.TabularProcessingPlugin(authorized_user_id=USER) + metadata = plugin._get_workbook_metadata('user-documents', f'{USER}/source.{source_format}') + version = plugin._get_tabular_blob_version('user-documents', f'{USER}/source.{source_format}', refresh=True) + descriptor = plugin._build_generated_export_query_descriptor_from_location( + container_name='user-documents', blob_path=f'{USER}/source.{source_format}', + filename=f'source.{source_format}', query_expression='index == index', + expected_row_count=count, blob_version=version, + sheet_names=metadata['sheet_names'] if is_workbook else None, + ) + replayed = list(runtime.engine._iter_versioned_tabular_source_rows( + descriptor, source_format, 3, 0, user_id=USER, + )) + assert descriptor['source_format'] == source_format + assert descriptor['expected_row_count'] == len(replayed) == count + assert descriptor['blob_etag'] == 'source-blob-1' + if is_workbook: + assert descriptor['version'] == 2 and descriptor['source_row_count'] == count + assert len(descriptor['table_selections']) == 1 + assert replayed[-1][1]['_excel_source']['row'] == (1 if source_format == 'xls' else 14) + else: + assert descriptor['version'] == 1 and 'sheet_names' not in descriptor - @staticmethod - def _apply_query_expression_with_fallback(dataframe, query_expression, normalize_match): - assert query_expression == 'index == index' - assert normalize_match is False - return dataframe.query(query_expression), False - @staticmethod - def _build_row_output_records(dataframe, selected_columns): - return dataframe[selected_columns].to_dict(orient='records') +def test_durable_workbook_replay_preserves_sheet_order_and_resume_position(monkeypatch): + """Replay real selected tables; keep physical rows separate from resume cursors.""" + # Keep application imports and external-I/O seams scoped to this runtime. + from test_native_tabular_compute_service import USER, native_runtime + from test_tabular_table_aware_analytics_replay import _set_source, _workbook_bytes - version_checks = [] - replay_rows = _load_tabular_source_replay_helper(FakeWorkbookPlugin(), version_checks) - descriptor = { - 'container': 'user-documents', - 'blob_path': 'user-1/source.xlsx', - 'blob_etag': 'etag-source', - 'query_expression': 'index == index', - 'return_columns': None, - 'sheet_names': ['First', 'Second'], - } - - all_rows = list(replay_rows(descriptor, 'xlsx', 1000, 0)) - resumed_rows = list(replay_rows(descriptor, 'xlsx', 1000, 2)) - - assert [source_row_number for source_row_number, _ in all_rows] == [1, 2, 3] - assert [row['Item_ID'] for _, row in all_rows] == ['A-1', 'A-2', 'B-1'] - assert resumed_rows == [(3, {'Item_ID': 'B-1', 'Value': 'three'})] - assert len(version_checks) == 4 + with native_runtime(monkeypatch, filename='source.xlsx') as runtime: + _set_source(runtime, _workbook_bytes([ + { + 'name': 'First', 'rows': { + 2: {1: 'Item_ID', 2: 'Value'}, 3: {1: 'A-1', 2: 'one'}, 4: {1: 'A-2', 2: 'two'}, + }, + }, + { + 'name': 'Second', 'rows': {7: {1: 'Item_ID', 2: 'Value'}, 8: {1: 'B-1', 2: 'three'}}, + }, + ])) + plugin = runtime.engine.TabularProcessingPlugin(authorized_user_id=USER) + descriptor = plugin._build_generated_export_query_descriptor_from_location( + container_name='user-documents', blob_path=f'{USER}/source.xlsx', filename='source.xlsx', + query_expression='index == index', sheet_names=['First', 'Second'], expected_row_count=3, + ) + all_rows = list(runtime.engine._iter_versioned_tabular_source_rows( + descriptor, 'xlsx', 1, 0, user_id=USER, + )) + resumed_rows = list(runtime.engine._iter_versioned_tabular_source_rows( + descriptor, 'xlsx', 1, 2, user_id=USER, + )) + assert [cursor for cursor, _ in all_rows] == [1, 2, 3] + assert [row['Item_ID'] for _, row in all_rows] == ['A-1', 'A-2', 'B-1'] + assert [row['_excel_source']['row'] for _, row in all_rows] == [3, 4, 8] + assert resumed_rows == all_rows[-1:] + descriptor['version'] = 1 + with pytest.raises(ValueError, match='legacy worksheet-header'): + list(runtime.engine._iter_versioned_tabular_source_rows( + descriptor, 'xlsx', 1, 0, user_id=USER, + )) def test_direct_source_backed_queue_supports_all_workbook_formats(): @@ -5875,6 +5829,32 @@ def authorize_group(user_id, group_id, allowed_roles): authorize_public(public_run) +def test_sdk_default_authorization_conversation_points_preserve_owner_etag_and_partition(): + """Execute the actual fixture and its conversation-ownership consumer with SDK points.""" + authorize = _load_authorization_helper('user-1') + container = authorize.__globals__['cosmos_conversations_container'] + first = container.read_item(item='conversation-1', partition_key='conversation-1') + assert type(first) is CosmosDict + assert first['id'] == 'conversation-1' and first['user_id'] == 'user-1' + assert first['_etag'] + first['user_id'] = 'client-mutation' + again = container.read_item(item='conversation-1', partition_key='conversation-1') + authorized = authorize({ + 'id': 'sdk-conversation-contract', 'user_id': 'user-1', 'conversation_id': 'conversation-1', + }) + assert type(again) is type(authorized) is CosmosDict + assert again['user_id'] == authorized['user_id'] == 'user-1' + assert again['_etag'] == authorized['_etag'] == first['_etag'] + with pytest.raises(CosmosResourceNotFoundError) as missing: + container.read_item(item='conversation-1', partition_key='another-conversation') + assert missing.value.status_code == 404 + wrong_owner = _load_authorization_helper('another-owner') + with pytest.raises(PermissionError, match='ownership'): + wrong_owner({ + 'id': 'sdk-conversation-contract', 'user_id': 'user-1', 'conversation_id': 'conversation-1', + }) + + def test_durable_cancellation_is_idempotent_and_terminal(): """Cancel persists a terminal state that cannot be resumed or canceled again.""" helpers, stored_run = _load_cancellation_helpers({ @@ -6252,6 +6232,72 @@ def test_final_artifact_publication_is_retry_idempotent(): assert message_container.upsert_count == 1 +def test_sdk_default_artifact_conversation_points_preserve_owner_etag_and_partition(): + """The publication fixture's point reads are real SDK mappings, not plain substitutes.""" + upload, _ = _load_idempotent_artifact_helper() + container = upload.__globals__['cosmos_conversations_container'] + first = container.read_item(item='conversation-1', partition_key='conversation-1') + assert type(first) is CosmosDict + assert first == { + 'id': 'conversation-1', 'user_id': 'user-1', '_etag': first['_etag'], + } + first['user_id'] = 'client-mutation' + again = container.read_item(item='conversation-1', partition_key='conversation-1') + assert type(again) is CosmosDict + assert again['user_id'] == 'user-1' and again['_etag'] == first['_etag'] + with pytest.raises(CosmosResourceNotFoundError) as missing: + container.read_item(item='conversation-1', partition_key='another-conversation') + assert missing.value.status_code == 404 + + +def test_sdk_default_artifact_message_points_preserve_versions_partitions_and_atomic_failures(): + """Exercise SDK writes/reads, nested isolation, partitions, and conditional failure atomicity.""" + _, container = _load_idempotent_artifact_helper() + body = { + 'id': 'same-message', 'conversation_id': 'conversation-1', 'metadata': {'value': 'original'}, + } + created = container.upsert_item(body=body) + other = container.upsert_item(body={ + **body, 'conversation_id': 'conversation-2', 'metadata': {'value': 'other-partition'}, + }) + body['metadata']['value'] = 'mutated-input' + created['metadata']['value'] = 'mutated-response' + first_read = container.read_item(item='same-message', partition_key='conversation-1') + other_read = container.read_item(item='same-message', partition_key='conversation-2') + assert all(type(document) is CosmosDict for document in (created, other, first_read, other_read)) + assert type(first_read['metadata']) is dict + assert first_read['metadata'] == {'value': 'original'} + assert other_read['metadata'] == {'value': 'other-partition'} + assert first_read['_etag'] == created['_etag'] != other_read['_etag'] == other['_etag'] + with pytest.raises(CosmosResourceNotFoundError) as missing: + container.read_item(item='same-message', partition_key='unrelated-conversation') + assert missing.value.status_code == 404 + updated = container.upsert_item( + body={**first_read, 'metadata': {'value': 'updated'}}, + etag=first_read['_etag'], match_condition=MatchConditions.IfNotModified, + ) + saved = container.read_item(item='same-message', partition_key='conversation-1') + assert type(updated) is type(saved) is CosmosDict + assert saved['metadata'] == {'value': 'updated'} and saved['_etag'] != first_read['_etag'] + before = deepcopy(container.items) + writes_before = container.upsert_count + with pytest.raises(CosmosHttpResponseError) as stale: + container.upsert_item( + body={**first_read, 'metadata': {'value': 'stale-write'}}, + etag=first_read['_etag'], match_condition=MatchConditions.IfNotModified, + ) + assert stale.value.status_code == 412 + with pytest.raises(CosmosHttpResponseError) as absent: + container.upsert_item( + body={'id': 'missing', 'conversation_id': 'conversation-1'}, + etag='missing-etag', match_condition=MatchConditions.IfNotModified, + ) + assert absent.value.status_code == 412 + with pytest.raises(ValueError, match='partition'): + container.upsert_item(body={'id': 'without-partition'}) + assert container.items == before and container.upsert_count == writes_before + + def test_scheduler_filters_before_limiting_candidates(): """Ineligible rows before the scan limit cannot hide an older recoverable run.""" runs = [ diff --git a/functional_tests/test_tabular_table_aware_analytics_replay.py b/functional_tests/test_tabular_table_aware_analytics_replay.py new file mode 100644 index 000000000..c50c1cb2e --- /dev/null +++ b/functional_tests/test_tabular_table_aware_analytics_replay.py @@ -0,0 +1,952 @@ +# test_tabular_table_aware_analytics_replay.py +""" +Real-package tests for table-aware analytics and version-pinned durable replay. +Version: 0.261.405 +Implemented in: 0.261.405 + +The existing offline native runtime replaces external I/O, not the workbook +reader, public analytical tools, source staging, checkpoints, or result reader. +OOXML packages are built in memory without Excel execution or temporary files. +""" + +import asyncio +from copy import deepcopy +import importlib +import io +import json +import struct +from xml.sax.saxutils import escape, quoteattr +import zipfile + +from flask import Flask, g +import pytest + +from test_native_tabular_compute_service import ( + CONVERSATION, + USER, + compute_plan, + native_runtime, + open_result, + submit, +) + + +MAIN_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" +REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" +PACKAGE_NS = "http://schemas.openxmlformats.org/package/2006/relationships" +CONTENT_NS = "http://schemas.openxmlformats.org/package/2006/content-types" + + +def _column_name(column): + result = "" + while column: + column, remainder = divmod(column - 1, 26) + result = chr(65 + remainder) + result + return result + + +def _workbook_bytes(sheets): + """Build a real spreadsheet package, including independently named tables.""" + contents = io.BytesIO() + overrides = [ + '', + ] + workbook_sheets, workbook_relationships = [], [] + table_number = 0 + with zipfile.ZipFile(contents, "w", compression=zipfile.ZIP_DEFLATED) as package: + package.writestr( + "_rels/.rels", + f'', + ) + for sheet_number, sheet in enumerate(sheets, start=1): + name, rows, tables = sheet["name"], sheet["rows"], sheet.get("tables") or [] + workbook_sheets.append( + f'' + ) + workbook_relationships.append( + f'' + ) + overrides.append( + f'' + ) + table_parts, table_relationships = [], [] + for name, reference, columns in tables: + table_number += 1 + table_parts.append(f'') + table_relationships.append( + f'' + ) + table_columns = "".join( + f'' + for index, column in enumerate(columns, start=1) + ) + package.writestr( + f"xl/tables/table{table_number}.xml", + f'' + f'' + f'{table_columns}
', + ) + overrides.append( + f'' + ) + worksheet_rows = [] + for row_number, cells in sorted(rows.items()): + serialized_cells = [] + for column, value in sorted(cells.items()): + reference = f"{_column_name(column)}{row_number}" + if isinstance(value, tuple) and value[0] == "formula": + cached = f"{escape(str(value[2]))}" if value[2] is not None else "" + body = f"{escape(value[1])}{cached}" + serialized_cells.append(f'{body}') + elif isinstance(value, bool): + serialized_cells.append(f'{int(value)}') + elif isinstance(value, (int, float)): + serialized_cells.append(f'{value}') + elif value is not None: + serialized_cells.append( + f'' + f'{escape(str(value))}' + ) + worksheet_rows.append(f'{"".join(serialized_cells)}') + table_xml = ( + f'{"".join(table_parts)}' + if table_parts else "" + ) + package.writestr( + f"xl/worksheets/sheet{sheet_number}.xml", + f'' + f'{"".join(worksheet_rows)}{table_xml}', + ) + if table_relationships: + package.writestr( + f"xl/worksheets/_rels/sheet{sheet_number}.xml.rels", + f'{"".join(table_relationships)}', + ) + package.writestr( + "xl/workbook.xml", + f'' + f'{"".join(workbook_sheets)}', + ) + package.writestr( + "xl/_rels/workbook.xml.rels", + f'{"".join(workbook_relationships)}', + ) + package.writestr( + "[Content_Types].xml", + f'' + f'{"".join(overrides)}', + ) + return contents.getvalue() + + +def _five_tables(named): + rows, tables = {}, [] + origins = [(3, 1), (3, 5), (10, 1), (10, 5), (17, 1)] + for index, (header, column) in enumerate(origins, start=1): + rows.setdefault(header - 1, {})[column] = f"Review table {index}" + rows.setdefault(header, {}).update({column: "Case", column + 1: "Amount"}) + for offset in range(1, 4): + rows.setdefault(header + offset, {}).update({ + column: f"T{index}-{offset}", column + 1: index * 100 + offset, + }) + if named: + reference = f"{_column_name(column)}{header}:{_column_name(column + 1)}{header + 3}" + tables.append((f"Review{index}", reference, ["Case", "Amount"])) + return _workbook_bytes([{"name": "Five tables", "rows": rows, "tables": tables}]), origins + + +def _one_table(count=3, *, header=2, table_name=None, column="amount"): + rows = {header - 1: {1: "Saved dataset"}, header: {1: "Item_ID", 2: column}} + rows.update({ + header + index: {1: f"item-{index:06d}", 2: index} + for index in range(1, count + 1) + }) + tables = [(table_name, f"A{header}:B{header + count}", ["Item_ID", column])] if table_name else [] + return _workbook_bytes([{"name": "Data", "rows": rows, "tables": tables}]) + + +def _legacy_workbook_bytes(): + """A real BIFF8/OLE workbook with one saved numeric cell and no header.""" + def record(code, payload=b""): + return struct.pack("=", "filter_value": "0"}), + ("query_tabular_data", {"query_expression": "amount >= 0"}), + ("lookup_value", {"lookup_column": "Item_ID", "lookup_value": "missing", "target_column": "amount"}), + ]: + payload, _ = _call(runtime, plugin, operation, table_id="Facts", **arguments) + assert payload["error_code"] == "formula_cache_missing", payload + assert "result" not in payload and "value" not in payload and "row_count" not in payload + saved, _ = _call( + runtime, plugin, "lookup_value", table_id="Facts", lookup_column="Item_ID", + lookup_value="saved", target_column="amount", + ) + assert saved["value"] == 30 + irrelevant, invocation = _call( + runtime, plugin, "query_tabular_data", table_id="Facts", + query_expression='Item_ID == "missing"', return_columns="Item_ID", + ) + assert irrelevant["total_matches"] == 1 + descriptor = invocation.internal_metadata["tabular_generated_export_source"] + assert len(list(_replay(runtime, descriptor))) == 1 + descriptor["return_columns"] = None + with pytest.raises(ValueError, match="cached"): + list(_replay(runtime, descriptor)) + counted, _ = _call( + runtime, plugin, "count_rows", query_expression='Item_ID == "amount"', + table_id="Facts", + ) + assert counted["row_count"] == 0 + + +def test_explicit_all_native_tables_keep_heterogeneous_identities(monkeypatch): + with native_runtime(monkeypatch, filename="source.xlsx") as runtime: + _set_source(runtime, _workbook_bytes([{ + "name": "Data", + "rows": { + 3: {1: "Item_ID", 2: "amount"}, 4: {1: "A", 2: 10}, 5: {1: "B", 2: 20}, + 10: {1: "Region", 2: "Units", 3: "Flag"}, + 11: {1: "East", 2: 4, 3: True}, 12: {1: "West", 2: 8, 3: False}, + }, + "tables": [ + ("Facts", "A3:B5", ["Item_ID", "amount"]), + ("Regions", "A10:C12", ["Region", "Units", "Flag"]), + ], + }])) + descriptor = _descriptor(runtime, table_id="all", expected_row_count=4) + assert len(descriptor["table_selections"]) == 2 + rows = list(_replay(runtime, descriptor)) + assert [cursor for cursor, _ in rows] == [1, 2, 3, 4] + assert set(rows[0][1]) == {"Item_ID", "amount", "_excel_source"} + assert set(rows[2][1]) == {"Region", "Units", "Flag", "_excel_source"} + assert [row["_excel_source"]["row"] for _, row in rows] == [4, 5, 11, 12] + assert len({row["_excel_source"]["table_id"] for _, row in rows}) == 2 + plan = compute_plan(runtime, durable=True) + plan["requested_output_hints"]["selected_table"] = "all" + request = runtime.service._compute_request(plan, "Compute doubled amounts.") + candidate, pinned = runtime.service._prepare_native_source( + runtime.engine, runtime.source_manifest[0], request, USER, + ) + assert pinned["expected_row_count"] == candidate["total_matches"] == 4 + assert len(candidate["table_selections"]) == 2 + assert candidate["full_result_available"] is False + foreground, _ = _call( + runtime, _plugin(runtime), "aggregate_column", table_id="all", column="amount", operation="sum", + ) + assert foreground["error_code"] == "table_schema_mismatch" + + +def test_related_tables_use_independent_ranges_and_physical_rows(monkeypatch): + with native_runtime(monkeypatch, filename="source.xlsx") as runtime: + _set_source(runtime, _workbook_bytes([{ + "name": "Data", + "rows": { + 3: {1: "Owner", 2: "Team"}, 4: {1: "Ada", 2: "Blue"}, + 5: {1: "Bea", 2: "Green"}, 10: {1: "Assignee", 2: "amount"}, + 11: {1: "Ada", 2: ("formula", "1+2", None)}, 12: {1: "Bea", 2: 20}, + }, + "tables": [ + ("People", "A3:B5", ["Owner", "Team"]), + ("Work", "A10:B12", ["Assignee", "amount"]), + ], + }])) + arguments = { + "source_sheet_name": "Data", "source_value_column": "Owner", + "target_sheet_name": "Data", "target_match_column": "Assignee", + "source_table_id": "People", "target_cell_range": "A10:B12", "target_header_row": "10", + "source_filter_column": "Team", "source_filter_value": "Blue", + } + count, _ = _call(runtime, _plugin(runtime), "count_rows_by_related_values", **arguments) + assert count["row_count"] == 1 + rows, _ = _call( + runtime, _plugin(runtime), "filter_rows_by_related_values", + return_columns="Assignee", **arguments, + ) + assert rows["matched_target_row_count"] == 1 + assert rows["data"][0]["Assignee"] == "Ada" + assert rows["data"][0]["_excel_source"]["row"] == 11 + assert rows["source_selection"]["header_row"] == 3 + assert rows["target_selection"]["header_row"] == 10 + + +def test_csv_and_single_table_compatibility(monkeypatch): + with native_runtime(monkeypatch, row_count=3) as runtime: + plugin = _plugin(runtime) + aggregate, _ = _call(runtime, plugin, "aggregate_column", column="amount", operation="sum") + assert int(aggregate["result"]) == 6 and aggregate["selected_sheet"] is None + queried, invocation = _call(runtime, plugin, "query_tabular_data", query_expression="amount >= 2") + assert queried["total_matches"] == 2 + descriptor = invocation.internal_metadata["tabular_generated_export_source"] + assert descriptor["version"] == 1 and "table_selections" not in descriptor + invalid, _ = _call( + runtime, plugin, "query_tabular_data", query_expression="index == index", header_row="0", + ) + assert invalid["error_code"] == "table_selection_invalid" + + +def test_cross_sheet_tools_and_replay_preserve_table_scope_and_global_cursor(monkeypatch): + with native_runtime(monkeypatch, filename="source.xlsx") as runtime: + _set_source(runtime, _workbook_bytes([ + { + "name": "First ", + "rows": {2: {1: "Item_ID", 2: "amount"}, 3: {1: "A", 2: 10}, 4: {1: "B", 2: 20}}, + "tables": [("FirstFacts", "A2:B4", ["Item_ID", "amount"])], + }, + { + "name": "Second", + "rows": {7: {1: "Item_ID", 2: "amount"}, 8: {1: "C", 2: 30}, 9: {1: "D", 2: 40}}, + "tables": [("SecondFacts", "A7:B9", ["Item_ID", "amount"])], + }, + ])) + plugin = _plugin(runtime) + query, invocation = _call( + runtime, plugin, "query_tabular_data", query_expression="index >= 1", return_columns="Item_ID", + ) + assert query["total_matches"] == 3 + assert [row["Item_ID"] for row in query["data"]] == ["B", "C", "D"] + assert [row["_excel_source"]["row"] for row in query["data"]] == [4, 8, 9] + descriptor = invocation.internal_metadata["tabular_generated_export_source"] + assert descriptor["sheet_names"] == ["First ", "Second"] + replayed = list(_replay(runtime, descriptor, chunk_rows=1)) + assert [cursor for cursor, _ in replayed] == [2, 3, 4] + assert [row for _, row in replayed] == query["data"] + count, _ = _call(runtime, plugin, "count_rows", filter_column="amount", filter_operator=">", filter_value="15") + assert count["row_count"] == 3 + lookup, _ = _call( + runtime, plugin, "lookup_value", lookup_column="Item_ID", lookup_value="C", target_column="amount", + ) + assert lookup["total_matches"] == 1 + assert lookup["data"][0]["_excel_source"]["row"] == 8 + mismatch, _ = _call( + runtime, plugin, "aggregate_column", column="amount", operation="sum", + sheet_name="Second", table_id="FirstFacts", + ) + assert mismatch["error_code"] == "table_selection_invalid" + explicit, _ = _call( + runtime, plugin, "count_rows", table_id="FirstFacts", + ) + assert explicit["selected_sheet"] == "First " and explicit["header_row"] == 2 + + +def test_grouped_and_datetime_operations_respect_nonfirst_table_headers(monkeypatch): + with native_runtime(monkeypatch, filename="source.xlsx") as runtime: + _set_source(runtime, _workbook_bytes([{ + "name": "Data", + "rows": { + 2: {1: "Category", 2: "amount"}, 3: {1: "unrelated", 2: 9999}, + 7: {1: "Category", 2: "amount", 3: "When"}, + 8: {1: "Blue", 2: 10, 3: "2026-01-05"}, + 9: {1: "Blue", 2: 20, 3: "2026-01-06"}, + 10: {1: "Green", 2: 40, 3: "2026-01-12"}, + }, + "tables": [ + ("Unrelated", "A2:B3", ["Category", "amount"]), + ("Events", "A7:C10", ["Category", "amount", "When"]), + ], + }])) + plugin = _plugin(runtime) + grouped, _ = _call( + runtime, plugin, "group_by_aggregate", group_by_column="Category", + aggregate_column="amount", operation="sum", table_id="Events", + ) + assert grouped["result"] == {"Blue": 30, "Green": 40} + assert grouped["header_row"] == 7 + dated, _ = _call( + runtime, plugin, "group_by_datetime_component", datetime_column="When", + datetime_component="day_name", aggregate_column="amount", operation="sum", + cell_range="A7:C10", header_row="7", + ) + expected = dict.fromkeys(plugin.DAY_NAME_ORDER, 0) + expected.update({"Monday": 50, "Tuesday": 20}) + assert dated["result"] == expected + assert dated["parsed_rows"] == 3 and dated["table_range"] == "A7:C10" + + +def test_separate_file_relationships_authorize_each_explicit_table(monkeypatch): + with native_runtime(monkeypatch, filename="source.xlsx") as runtime: + source_bytes = _workbook_bytes([{ + "name": "Data", "rows": {3: {1: "Owner", 2: "Team"}, 4: {1: "Ada", 2: "Blue"}}, + "tables": [("People", "A3:B4", ["Owner", "Team"])], + }]) + target_bytes = _workbook_bytes([{ + "name": "Data", + "rows": {7: {1: "Assignee", 2: "amount"}, 8: {1: "Ada", 2: 15}, 9: {1: "Bea", 2: 20}}, + "tables": [("Work", "A7:B9", ["Assignee", "amount"])], + }]) + _set_source(runtime, source_bytes) + second = { + **deepcopy(runtime.document), "id": "target-1", "file_name": "target.xlsx", + "blob_path": f"{USER}/target.xlsx", + } + runtime.blobs.values[("user-documents", second["blob_path"])] = { + "bytes": target_bytes, "etag": "target-blob-1", "metadata": {}, + } + reads = [] + + def resolve_document(container, blob_path, user_id): + reads.append(blob_path) + if user_id != USER or container != "user-documents" or not runtime.state["allowed"]: + raise PermissionError("Fixture relationship source denied.") + if blob_path == second["blob_path"]: + return deepcopy(second) + if blob_path == runtime.document["blob_path"]: + return deepcopy(runtime.document) + raise PermissionError("Fixture relationship source denied.") + + runtime.patcher.setattr(runtime.screening, "_resolve_blob_document", resolve_document) + + def read_document(document_id, user_id, **kwargs): + if document_id == second["id"]: + return resolve_document("user-documents", second["blob_path"], user_id) + if document_id == runtime.document["id"]: + return resolve_document("user-documents", runtime.document["blob_path"], user_id) + raise PermissionError("Fixture relationship source denied.") + + runtime.patcher.setattr(runtime.screening, "_read_authorized_document", read_document) + plugin = _plugin(runtime) + arguments = { + "source_sheet_name": "Data", "source_value_column": "Owner", "source_table_id": "People", + "target_sheet_name": "Data", "target_match_column": "Assignee", "target_table_id": "Work", + "target_filename": "target.xlsx", + } + payload, _ = _call(runtime, plugin, "filter_rows_by_related_values", **arguments) + assert payload["matched_target_row_count"] == 1 + assert payload["source_filename"] == "source.xlsx" + assert payload["target_filename"] == "target.xlsx" + assert payload["data"][0]["_excel_source"]["row"] == 8 + assert set(reads) == {runtime.document["blob_path"], second["blob_path"]} + runtime.state["allowed"] = False + denied, _ = _call(runtime, plugin, "count_rows_by_related_values", **arguments) + assert denied["error_code"] == "tabular_source_unavailable" + assert "row_count" not in denied + + +def test_streamed_numeric_conversion_checks_late_rows_before_querying(monkeypatch): + with native_runtime(monkeypatch, filename="source.xlsx") as runtime: + rows = {2: {1: "Item_ID", 2: "amount"}} + rows.update({index + 2: {1: f"row-{index}", 2: index} for index in range(1, 1201)}) + rows[1202][2] = "unknown" + _set_source(runtime, _workbook_bytes([{"name": "Data", "rows": rows}])) + query, invocation = _call( + runtime, _plugin(runtime), "query_tabular_data", query_expression='amount == "2"', + ) + assert query["total_matches"] == 1 + descriptor = invocation.internal_metadata["tabular_generated_export_source"] + replayed = list(_replay(runtime, descriptor, chunk_rows=100)) + assert [row for _, row in replayed] == query["data"] + assert replayed[0][0] == 2 and replayed[0][1]["amount"] == "2" + + +def test_native_range_hints_pin_actual_headers_and_selected_ranges(monkeypatch): + with native_runtime(monkeypatch, filename="source.xlsx") as runtime: + _set_source(runtime, _one_table(3)) + plan = compute_plan(runtime, durable=True) + plan["requested_output_hints"].update({ + "selected_sheet": "", "selected_range": "A2:B5", "selected_header_row": "2", + }) + request = runtime.service._compute_request(plan, "Compute doubled amounts.") + candidate, descriptor = runtime.service._prepare_native_source( + runtime.engine, runtime.source_manifest[0], request, USER, + ) + assert candidate["cell_range"] == descriptor["cell_range"] == "A2:B5" + assert candidate["header_row"] == descriptor["header_row"] == 2 + assert candidate["total_matches"] == descriptor["expected_row_count"] == 3 + assert descriptor["table_selections"][0]["origin"] == "range" + assert [row["_excel_source"]["row"] for row in candidate["rows"]] == [3, 4, 5] + + +def test_same_schema_all_tables_keep_full_data_numeric_semantics_on_replay(monkeypatch): + with native_runtime(monkeypatch, filename="source.xlsx") as runtime: + _set_source(runtime, _workbook_bytes([{ + "name": "Data", + "rows": { + 2: {1: "Item_ID", 2: "amount"}, 3: {1: "A", 2: 1}, 4: {1: "B", 2: 2}, + 8: {1: "Item_ID", 2: "amount"}, 9: {1: "C", 2: 3}, 10: {1: "D", 2: "unknown"}, + }, + "tables": [ + ("FirstFacts", "A2:B4", ["Item_ID", "amount"]), + ("SecondFacts", "A8:B10", ["Item_ID", "amount"]), + ], + }])) + payload, invocation = _call( + runtime, _plugin(runtime), "query_tabular_data", + table_id="all", query_expression='amount == "2"', + ) + descriptor = invocation.internal_metadata["tabular_generated_export_source"] + replayed = list(_replay(runtime, descriptor, chunk_rows=1)) + assert payload["total_matches"] == len(replayed) == 1 + assert descriptor["numeric_conversion_scope"] == "sheet" + assert [row for _, row in replayed] == payload["data"] + plan = compute_plan(runtime, durable=True, query='amount == "2"') + plan["requested_output_hints"]["selected_table"] = "all" + request = runtime.service._compute_request(plan, "Compute doubled amounts.") + candidate, pinned = runtime.service._prepare_native_source( + runtime.engine, runtime.source_manifest[0], request, USER, + ) + assert pinned["numeric_conversion_scope"] == "sheet" + assert candidate["total_matches"] == 1 and candidate["rows"] == payload["data"] + + +def test_repeated_headers_keep_physical_rows_separate_from_replay_cursors(monkeypatch): + with native_runtime(monkeypatch, filename="source.xlsx") as runtime: + _set_source(runtime, _workbook_bytes([{ + "name": "Data", + "rows": { + 2: {1: "Item_ID", 2: "amount"}, 3: {1: "A", 2: 10}, + 4: {1: "Item_ID", 2: "amount"}, 5: {1: "B", 2: 20}, + }, + }])) + descriptor = _descriptor(runtime, expected_row_count=2) + assert descriptor["source_row_count"] == 2 + rows = list(_replay(runtime, descriptor, chunk_rows=1)) + assert [cursor for cursor, _ in rows] == [1, 2] + assert [row["_excel_source"]["row"] for _, row in rows] == [3, 5] + assert list(_replay(runtime, descriptor, resume=1, chunk_rows=1)) == rows[-1:] + + +def test_source_columns_cannot_replace_protected_excel_provenance(monkeypatch): + with native_runtime(monkeypatch, filename="source.xlsx") as runtime: + _set_source(runtime, _workbook_bytes([{ + "name": "Data", "rows": {2: {1: "Item_ID", 2: "_excel_source"}, 3: {1: "A", 2: "spoofed"}}, + "tables": [("Facts", "A2:B3", ["Item_ID", "_excel_source"])], + }])) + payload, _ = _call( + runtime, _plugin(runtime), "query_tabular_data", + query_expression="index == index", return_columns="Item_ID", + ) + assert payload["error_code"] == "reserved_excel_column" + assert "data" not in payload + descriptor = _descriptor(runtime, expected_row_count=1) + with pytest.raises(ValueError, match="reserved"): + list(_replay(runtime, descriptor)) + + +def test_existing_csv_request_fingerprints_and_durable_jobs_remain_replayable(monkeypatch): + with native_runtime(monkeypatch, row_count=3) as runtime: + plan = compute_plan(runtime, durable=True) + pending = submit(runtime, plan) + job_id = pending["handle"]["job_id"] + stored = runtime.engine._read_run(USER, job_id) + legacy_request = runtime.service._compute_request(plan, "Compute doubled amounts.") + for field in ("selected_table", "cell_range", "header_row"): + legacy_request.pop(field, None) + legacy_fingerprint = runtime.service.native_compute_digest({ + "request": legacy_request, "sources": stored["compute_context"]["sources"], + }) + stored["compute_context"]["request_fingerprint"] = legacy_fingerprint + runtime.jobs.replace_item(job_id, stored) + reopened = submit(runtime, plan) + assert reopened["status"] == "pending" and reopened["handle"]["job_id"] == job_id + assert reopened["handle"]["request_fingerprint"] == legacy_fingerprint + finished = runtime.engine.process_tabular_generated_output_run(job_id, USER) + assert finished["status"] == "completed", finished.get("last_error") + ready = open_result(runtime, reopened["handle"]) + rows = list(ready["reader"].iter_records()) + assert rows == [ + {"Item_ID": "item-000001", "doubled": 2}, + {"Item_ID": "item-000002", "doubled": 4}, + {"Item_ID": "item-000003", "doubled": 6}, + ] + assert runtime.jobs.created == 1 + + +def test_provider_exceptions_never_reach_public_tool_payloads(monkeypatch): + with native_runtime(monkeypatch, filename="source.xlsx") as runtime: + plugin = _plugin(runtime) + runtime.patcher.setattr( + plugin, "_get_workbook_metadata", + lambda *args, **kwargs: (_ for _ in ()).throw( + RuntimeError("Provider failure with private endpoint and credential details."), + ), + ) + payload, _ = _call(runtime, plugin, "describe_tabular_file") + assert payload["error_code"] == "tabular_analysis_failed" + assert "Provider" not in payload["error"] and "credential" not in payload["error"] + + +@pytest.mark.parametrize("corruption", ["duplicate_cursor", "expected_count", "checkpoint_identity"]) +def test_staging_rejects_repeated_cursors_and_inconsistent_checkpoint_counts(monkeypatch, corruption): + with native_runtime(monkeypatch, filename="source.xlsx") as runtime: + _set_source(runtime, _one_table(3, table_name="Facts")) + plan = compute_plan(runtime, durable=True) + plan["requested_output_hints"]["selected_table"] = "Facts" + pending = submit(runtime, plan) + job_id = pending["handle"]["job_id"] + run = runtime.engine._read_run(USER, job_id) + if corruption == "expected_count": + run["source_descriptor"]["expected_row_count"] = 4 + runtime.jobs.replace_item(job_id, run) + claimed = runtime.engine._try_claim_run(USER, job_id, runtime.settings) + if corruption == "duplicate_cursor": + runtime.patcher.setattr( + runtime.engine, "_iter_versioned_tabular_source_rows", + lambda *args, **kwargs: iter([(1, {"Item_ID": "A"}), (1, {"Item_ID": "A"})]), + ) + elif corruption == "checkpoint_identity": + original = runtime.engine._download_json_blob + + def corrupt_checkpoint(path): + result = original(path) + if isinstance(result, list) and result: + result[0][runtime.engine.TABULAR_EXPORT_INPUT_ROW_NUMBER_FIELD] = 9 + return result + + runtime.patcher.setattr(runtime.engine, "_download_json_blob", corrupt_checkpoint) + with pytest.raises(ValueError): + runtime.engine._stage_tabular_generated_output_source(claimed, runtime.settings) + saved = runtime.engine._read_run(USER, job_id) + assert saved["source_staging_complete"] is False + assert saved["status"] != "completed" + assert runtime.publications == [] + + +def test_durable_checkpoint_resume_preserves_exact_selected_table_count(monkeypatch): + with native_runtime(monkeypatch, filename="source.xlsx") as runtime: + _set_source(runtime, _one_table(37, header=7, table_name="Facts")) + plan = compute_plan(runtime, durable=True) + plan["requested_output_hints"].update({"selected_table": "Facts", "header_row": "7"}) + pending = submit(runtime, plan) + assert pending["status"] == "pending" and pending["reader"] is None + job_id = pending["handle"]["job_id"] + run = runtime.engine._read_run(USER, job_id) + assert run["source_descriptor"]["expected_row_count"] == 37 + assert run["source_descriptor"]["header_row"] == 7 + run["source_descriptor"].update({"batch_max_rows": 5, "schema_probe_rows": 5}) + runtime.jobs.replace_item(job_id, run) + claimed = runtime.engine._try_claim_run(USER, job_id, runtime.settings) + original = runtime.engine._checkpoint_source_input_batch + calls = [] + + def interrupt_after_checkpoint(*args, **kwargs): + saved = original(*args, **kwargs) + calls.append(saved["source_staged_rows"]) + if len(calls) == 2: + raise RuntimeError("Injected worker interruption after a durable checkpoint.") + return saved + + with monkeypatch.context() as patcher: + patcher.setattr(runtime.engine, "_checkpoint_source_input_batch", interrupt_after_checkpoint) + with pytest.raises(RuntimeError, match="Injected worker"): + runtime.engine._stage_tabular_generated_output_source(claimed, runtime.settings) + interrupted = runtime.engine._read_run(USER, job_id) + assert interrupted["source_staged_rows"] == 10 + assert interrupted["source_scan_row_count"] == 10 + assert interrupted["source_staging_complete"] is False + interrupted["last_heartbeat_at"] = "2000-01-01T00:00:00+00:00" + runtime.jobs.replace_item(job_id, interrupted) + finished = runtime.engine.process_tabular_generated_output_run(job_id, USER) + assert finished["status"] == "completed", finished.get("last_error") + ready = open_result(runtime, pending["handle"]) + rows = list(ready["reader"].iter_records()) + assert len(rows) == 37 + assert sum(row["doubled"] for row in rows) == 37 * 38 + assert rows[-1] == {"Item_ID": "item-000037", "doubled": 74} + stored = runtime.engine._read_run(USER, job_id) + checkpoint_rows = [] + for number in range(1, stored["source_staged_batches"] + 1): + checkpoint_rows.extend(runtime.engine._download_json_blob( + runtime.engine._input_blob_path(USER, CONVERSATION, job_id, number), + )) + assert len(checkpoint_rows) == stored["source_staged_rows"] == 37 + assert [row["_excel_source"]["row"] for row in checkpoint_rows] == list(range(8, 45)) + token_field = runtime.engine.TABULAR_EXPORT_INPUT_ROW_TOKEN_FIELD + assert len({row[token_field] for row in checkpoint_rows}) == 37 + assert runtime.publications == [] diff --git a/ui_tests/fixtures/orchestration/harness_entry.tsx b/ui_tests/fixtures/orchestration/harness_entry.tsx index 1264aad02..0aa74d379 100644 --- a/ui_tests/fixtures/orchestration/harness_entry.tsx +++ b/ui_tests/fixtures/orchestration/harness_entry.tsx @@ -44,6 +44,7 @@ import { OrchestrationRunView } from '../../../application/v2_ui/src/components/ import { OrchestrationMapView } from '../../../application/v2_ui/src/components/chat/OrchestrationMapView'; import { OrchestrationPlanEditorHost } from '../../../application/v2_ui/src/components/chat/OrchestrationPlanEditor'; import { MessageList } from '../../../application/v2_ui/src/components/chat/MessageList'; +import { EnhancedCitationViewer } from '../../../application/v2_ui/src/components/chat/EnhancedCitationViewer'; import { ConversationDrawer } from '../../../application/v2_ui/src/components/chat/ConversationDrawer'; import { Composer } from '../../../application/v2_ui/src/components/chat/Composer'; import { ComposerEditor } from '../../../application/v2_ui/src/components/chat/ComposerEditor'; @@ -246,6 +247,7 @@ type ComponentName = | 'WorkflowEditorDialog' | 'WorkflowRunHistory' | 'M365Cards' + | 'EnhancedCitationViewer' | 'Toaster'; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -260,6 +262,7 @@ const components: Record ReactElement | null> = { OrchestrationRunView, OrchestrationMapView, MessageList, + EnhancedCitationViewer, ConversationDrawer, Composer, PromptExperience, diff --git a/ui_tests/test_v2_excel_citation_inspection.py b/ui_tests/test_v2_excel_citation_inspection.py new file mode 100644 index 000000000..ee6349a75 --- /dev/null +++ b/ui_tests/test_v2_excel_citation_inspection.py @@ -0,0 +1,191 @@ +# test_v2_excel_citation_inspection.py +""" +UI tests for table-aware Excel citation previews and stable source evidence. +Version: 0.261.405 +Implemented in: 0.261.405 + +The real V2 viewer runs through real HTTP fetches against synthetic API transport. +The shared Azure Playwright connection fixture uses Entra authentication when +configured and the existing local browser otherwise. +""" + +import json +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +import pytest +from playwright.sync_api import expect + +from ui_tests.fixtures.orchestration import harness_build +from ui_tests.fixtures.playwright_connection import connect_options + + +__all__ = ["connect_options"] +PASSAGE = "Saved reviewer evidence: Budget!B4 contains 15000; the reviewer requests 19500." +ROOT = Path(__file__).resolve().parents[1] + + +def load_harness(page, origin): + styles = list((ROOT / "application" / "single_app" / "static" / "v2" / "assets").glob("*.css")) + assert len(styles) == 1, "Build the V2 UI before running styled citation tests." + response = page.goto(f"{origin}/{harness_build.HARNESS_HTML_REL}") + assert response is not None and response.ok + page.add_style_tag(url=f"{origin}/application/single_app/static/v2/assets/{styles[0].name}") + + +def install_transport(page, *, identical=False): + requests, failures, errors = [], {"Delivery": False}, [] + page.on("pageerror", lambda error: errors.append(str(error))) + + def transport(route): + parsed = urlparse(route.request.url) + if parsed.path == "/api/get_citation": + route.fulfill(status=200, content_type="application/json", body=json.dumps({"cited_text": PASSAGE})) + return + if parsed.path != "/api/enhanced_citations/tabular_preview": + route.fulfill(status=200, content_type="application/json", body="{}") + return + query = parse_qs(parsed.query) + requests.append(query) + sheet = query.get("sheet_name", ["Budget"])[0] + if failures.get(sheet): + route.fulfill(status=500, content_type="application/json", body=json.dumps({"error": "Preview temporarily unavailable."})) + return + table_id = query.get("table_id", [f"{sheet.lower()}-1"])[0] + table_number = int(table_id.rsplit("-", 1)[-1]) + start = int(query.get("start_row", ["0"])[0]) + count = min(200, 50000 - start) + tables = [ + {"id": f"{sheet.lower()}-{index}", "name": f"Review {index}", "range": f"A{index * 10}:B{50000 + index * 10}", + "origin": "inferred", "header_row": 2, "header_status": "inferred", "row_count": 50000} + for index in range(1, 6 if sheet == "Budget" else 2) + ] + rows = [ + ["North", "15000"] if identical else [f"{sheet}-{table_number}-{index + 1}", str(index + 1)] + for index in range(start, start + count) + ] + payload = { + "filename": "review.xlsx", "selected_sheet": sheet, "sheet_names": ["Budget", "Delivery"], + "sheet_count": 2, "tables": tables, "table_count": len(tables), + "table_id": table_id, "table_name": f"Review {table_number}", "table_range": tables[table_number - 1]["range"], + "header_row": 2, "header_status": "inferred", "total_rows": 50000, + "columns": ["Site", "Amount"], "rows": rows, + "row_numbers": list(range(start + 3, start + 3 + count)), "start_row": start, + "has_more": start + count < 50000, "truncated": start + count < 50000, + "next_start_row": start + count if start + count < 50000 else None, + } + route.fulfill(status=200, content_type="application/json", body=json.dumps(payload)) + + page.route("**/api/**", transport) + return requests, failures, errors + + +def mount_viewer(page): + page.wait_for_function("() => Boolean(window.OrchHarness)") + page.evaluate("""() => { + const H = window.OrchHarness; + H.reset(); + window.viewerFallbacks = []; + H.mount('mount-a', 'EnhancedCitationViewer', { + type: 'tabular', docId: 'workbook', location: '1', locationLabel: 'Page', + metadata: {file_name: 'review.xlsx', enhanced_citations: true}, + citationId: 'workbook_2', onClose: () => H.unmount('mount-a'), + onFallback: (message) => window.viewerFallbacks.push(message), + }); + }""") + dialog = page.get_by_role("dialog", name="Cited source", exact=True) + expect(dialog.get_by_role("tab", name="Budget", exact=True)).to_have_attribute("aria-selected", "true") + expect(dialog.get_by_role("tabpanel")).to_have_attribute("aria-busy", "false") + return dialog + + +@pytest.mark.ui +@pytest.mark.parametrize("viewport", [{"width": 1440, "height": 900}, {"width": 390, "height": 844}], ids=["desktop", "mobile"]) +def test_real_sheets_five_tables_and_original_evidence(page, viewport): + harness_build.ensure_bundle() + page.set_viewport_size(viewport) + requests, _, errors = install_transport(page) + with harness_build.start_static_server() as origin: + load_harness(page, origin) + dialog = mount_viewer(page) + expect(dialog.get_by_role("columnheader", name="Site", exact=True)).to_be_visible() + expect(dialog.get_by_role("cell", name="Budget-1-1", exact=True)).to_be_visible() + expect(dialog.get_by_text("Cited workbook evidence", exact=True)).to_be_visible() + expect(dialog.get_by_text("What was extracted from this image", exact=True)).to_have_count(0) + expect(dialog.get_by_text(PASSAGE, exact=True)).to_be_visible() + dialog.get_by_label("Table", exact=True).select_option("budget-5") + expect(dialog.get_by_role("cell", name="Budget-5-1", exact=True)).to_be_visible() + expect(dialog.get_by_role("cell", name="Budget-1-1", exact=True)).to_have_count(0) + dialog.get_by_role("tab", name="Delivery", exact=True).click() + expect(dialog.get_by_role("tab", name="Delivery", exact=True)).to_have_attribute("aria-selected", "true") + expect(dialog.get_by_role("cell", name="Delivery-1-1", exact=True)).to_be_visible() + expect(dialog.get_by_text(PASSAGE, exact=True)).to_be_visible() + assert any(request.get("table_id") == ["budget-5"] for request in requests) + assert any(request.get("sheet_name") == ["Delivery"] for request in requests) + assert dialog.evaluate("(element) => element.scrollWidth <= element.clientWidth") + assert errors == [] + + +@pytest.mark.ui +def test_identical_source_sheets_still_have_correct_selected_identity_and_keyboard_navigation(page): + harness_build.ensure_bundle() + requests, _, errors = install_transport(page, identical=True) + with harness_build.start_static_server() as origin: + load_harness(page, origin) + dialog = mount_viewer(page) + budget = dialog.get_by_role("tab", name="Budget", exact=True) + budget.focus() + budget.press("ArrowRight") + delivery = dialog.get_by_role("tab", name="Delivery", exact=True) + expect(delivery).to_have_attribute("aria-selected", "true") + expect(delivery).to_be_focused() + expect(dialog.get_by_role("tabpanel")).to_have_attribute("aria-label", "Delivery data preview") + expect(dialog.get_by_role("cell", name="North", exact=True).first).to_be_visible() + delivery.press("Home") + expect(budget).to_have_attribute("aria-selected", "true") + budget.click() + expect(dialog.get_by_role("tabpanel")).to_have_attribute("aria-busy", "false") + assert any(request.get("sheet_name") == ["Delivery"] for request in requests) + assert errors == [] + + +@pytest.mark.ui +def test_fifty_thousandth_record_is_loaded_without_rendering_fifty_thousand_rows(page): + harness_build.ensure_bundle() + requests, _, errors = install_transport(page) + with harness_build.start_static_server() as origin: + load_harness(page, origin) + dialog = mount_viewer(page) + dialog.get_by_label("Go to record", exact=True).fill("50000") + dialog.get_by_role("button", name="Go", exact=True).click() + expect(dialog.get_by_role("cell", name="Budget-1-50000", exact=True)).to_be_visible() + expect(dialog.get_by_role("rowheader", name="50002", exact=True)).to_be_visible() + expect(dialog.get_by_text("Records 50000–50000 of 50000", exact=True)).to_be_visible() + assert len(dialog.get_by_role("row").all()) <= 201 + assert any(request.get("start_row") == ["49999"] for request in requests) + dialog.get_by_role("button", name="Previous records", exact=True).click() + expect(dialog.get_by_role("cell", name="Budget-1-1", exact=True)).to_be_visible() + dialog.get_by_label("Go to record", exact=True).fill("1") + dialog.get_by_role("button", name="Go", exact=True).click() + expect(dialog.get_by_role("tabpanel")).to_have_attribute("aria-busy", "false") + assert errors == [] + + +@pytest.mark.ui +def test_failed_switch_does_not_mislabel_previous_data_and_can_retry(page): + harness_build.ensure_bundle() + _, failures, errors = install_transport(page) + with harness_build.start_static_server() as origin: + load_harness(page, origin) + dialog = mount_viewer(page) + failures["Delivery"] = True + dialog.get_by_role("tab", name="Delivery", exact=True).click() + expect(dialog.get_by_role("alert")).to_contain_text("Preview temporarily unavailable.") + expect(dialog.get_by_role("tab", name="Budget", exact=True)).to_have_attribute("aria-selected", "true") + expect(dialog.get_by_role("cell", name="Budget-1-1", exact=True)).to_be_visible() + failures["Delivery"] = False + dialog.get_by_role("button", name="Retry preview", exact=True).click() + expect(dialog.get_by_role("tab", name="Delivery", exact=True)).to_have_attribute("aria-selected", "true") + expect(dialog.get_by_role("cell", name="Delivery-1-1", exact=True)).to_be_visible() + assert page.evaluate("window.viewerFallbacks") == [] + assert errors == [] From 07cc12b4847e27746dff40b0fd088edbcb6f3cd7 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Thu, 5 Nov 2026 14:43:01 -0500 Subject: [PATCH 2/2] Fix Excel inspection import bindings Resolve the UUID factory at inventory generation time and use a single Excel module import in native evidence tests. Add real-store UUID replacement regression coverage, document validation, and bump the application patch version to 0.261.409. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../single_app/functions_excel_inventory.py | 4 +-- ...L_COMMENT_CONTENT_AND_TABLE_HEADERS_FIX.md | 23 ++++++++++++ .../test_excel_inventory_persistence.py | 36 +++++++++++++++++-- ...t_excel_native_inspection_chat_evidence.py | 17 +++++---- 5 files changed, 68 insertions(+), 14 deletions(-) diff --git a/application/single_app/config.py b/application/single_app/config.py index 81ed1b018..3dde6e2f0 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -103,7 +103,7 @@ EXECUTOR_MAX_WORKERS = 30 CHAT_STREAM_EXECUTOR_MAX_WORKERS = max(8, int(os.getenv('SIMPLECHAT_CHAT_STREAM_WORKERS', '32') or '32')) SESSION_TYPE = 'filesystem' -VERSION = "0.261.408" +VERSION = "0.261.409" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_excel_inventory.py b/application/single_app/functions_excel_inventory.py index 77c5905e2..882920fef 100644 --- a/application/single_app/functions_excel_inventory.py +++ b/application/single_app/functions_excel_inventory.py @@ -10,7 +10,7 @@ import os import tempfile from threading import RLock -from uuid import uuid4 +import uuid from azure.core.exceptions import AzureError, ResourceNotFoundError @@ -87,7 +87,7 @@ def __init__(self, storage, subject, source_digest, capture=None): self.subject = subject self.source_digest = source_digest self.capture = capture - self.scan_id = f"excel-native:{EXCEL_WORKBOOK_SCHEMA_VERSION}:{source_digest}:{uuid4().hex}" + self.scan_id = f"excel-native:{EXCEL_WORKBOOK_SCHEMA_VERSION}:{source_digest}:{uuid.uuid4().hex}" self.pages = [] self.buffers = {} self.sizes = {} diff --git a/docs/explanation/fixes/EXCEL_COMMENT_CONTENT_AND_TABLE_HEADERS_FIX.md b/docs/explanation/fixes/EXCEL_COMMENT_CONTENT_AND_TABLE_HEADERS_FIX.md index 4dd16ec5d..fea48505b 100644 --- a/docs/explanation/fixes/EXCEL_COMMENT_CONTENT_AND_TABLE_HEADERS_FIX.md +++ b/docs/explanation/fixes/EXCEL_COMMENT_CONTENT_AND_TABLE_HEADERS_FIX.md @@ -2,6 +2,7 @@ Fixed in version: **0.261.408**. Application version is maintained in `application/single_app/config.py`. +Import binding follow-up fixed in version: **0.261.409**. ## Issue and root cause @@ -64,5 +65,27 @@ cover restart/paging, revocation, source-bound cursors, failures, and cleanup. UI tests exercise different and identical sheets, five tables, the 50,000th record, error recovery, keyboard navigation, and desktop/mobile layout. +### Import binding follow-up + +PR #1891's CodeQL alerts 5132 and 5136 identified a captured UUID factory and +mixed imports in the native-evidence test. `functions_excel_inventory.py` now +resolves `uuid.uuid4()` when creating each generation, so a factory replacement +after import is observed. The native-evidence test consistently uses its existing +`excel` module import. Neither change moves deferred imports or alters bootstrap, +authorization, source binding, storage format, or the inspection interface. + +`test_excel_inventory_persistence.py` replaces the UUID factory twice after the +inventory module is loaded, verifies the expected immutable generation keys, +reads both generations' complete comment records, and checks factory restoration. +The regression fails with the former imported function binding. Both updated +tests record version **0.261.409**, matching the single patch increment in +`application/single_app/config.py`. + +Focused offline validation passes all **16** inventory/native-evidence tests +under normal Python and all **16** under optimized Python. Documentation coverage +passes **7/7** checks and site quality **6/6**. The Cosmos response-boundary +inventory remains unchanged at **1,085 owners**, **425 doubles**, and **zero +review gaps**, so no storage-boundary reclassification is needed. + See [Scalable Excel Native Inspection](../features/EXCEL_NATIVE_INSPECTION.md) for the storage contract, test suites, and saved-state limitations. diff --git a/functional_tests/test_excel_inventory_persistence.py b/functional_tests/test_excel_inventory_persistence.py index 9988143f2..b316d4948 100644 --- a/functional_tests/test_excel_inventory_persistence.py +++ b/functional_tests/test_excel_inventory_persistence.py @@ -1,8 +1,9 @@ # test_excel_inventory_persistence.py """ Functional tests for private, complete, revision-bound Excel inventories. -Version: 0.261.408 +Version: 0.261.409 Implemented in: 0.261.408 +UUID import binding regression implemented in: 0.261.409 The real private artifact store runs against the existing Azure-like blob double. Tests prove paging, full-text continuations, source binding, publication failures, @@ -12,6 +13,7 @@ import json import ast import sys +import uuid from pathlib import Path from types import SimpleNamespace @@ -24,7 +26,7 @@ sys.path.insert(0, str(APP)) import functions_excel_accessibility as excel # noqa: E402 -from content_screening.contracts import Subject # noqa: E402 +from content_screening.contracts import Subject, hash_payload # noqa: E402 from content_screening.extraction import ExtractionCapture # noqa: E402 from content_screening.storage import ScreeningStorage, revision_prefix # noqa: E402 from functions_excel_inventory import ( # noqa: E402 @@ -58,6 +60,36 @@ def inventory_store(): return ScreeningStorage(service), service +def test_inventory_generations_use_current_uuid_factory(review_workbook, inventory_store, monkeypatch): + store, _ = inventory_store + subject = Subject("personal", "owner", "workbook", "1") + original_factory = uuid.uuid4 + references, expected_scan_keys = [], [] + with monkeypatch.context() as scoped: + for generation in (uuid.UUID(int=1), uuid.UUID(int=2)): + scoped.setattr(uuid, "uuid4", lambda current=generation: current) + _, _, reference = persist_excel_inventory(review_workbook, ".xlsx", subject=subject, storage=store) + scan_id = ( + f"excel-native:{excel.EXCEL_WORKBOOK_SCHEMA_VERSION}:" + f"{reference['source_sha256']}:{generation.hex}" + ) + references.append(reference) + expected_scan_keys.append(hash_payload(scan_id)) + actual_scan_keys = [reference["manifest"]["scan_key"] for reference in references] + assert uuid.uuid4 is original_factory + assert actual_scan_keys == expected_scan_keys + assert actual_scan_keys[0] != actual_scan_keys[1] + for reference in references: + clear_excel_inventory_cache() + reader = open_stored_excel_inventory( + {"id": "workbook", "user_id": "owner", "version": 1, EXCEL_INVENTORY_FIELD: reference}, + storage=store, authorize=lambda: None, + ) + records = reader.page("comments")["records"] + assert len(records) == 5 + assert any(record["text"] == "Set cap to 19500." for record in records) + + def test_complete_inventory_survives_parent_record_limits_and_restart(review_workbook, inventory_store, monkeypatch): store, service = inventory_store subject = Subject("personal", "owner", "workbook", "1") diff --git a/functional_tests/test_excel_native_inspection_chat_evidence.py b/functional_tests/test_excel_native_inspection_chat_evidence.py index c8f3f293a..a92f48fc2 100644 --- a/functional_tests/test_excel_native_inspection_chat_evidence.py +++ b/functional_tests/test_excel_native_inspection_chat_evidence.py @@ -1,7 +1,7 @@ # test_excel_native_inspection_chat_evidence.py """ Functional tests for complete, source-backed Excel native evidence. -Version: 0.261.408 +Version: 0.261.409 Implemented in: 0.261.408 Real packages distinguish threaded compatibility copies from genuine notes, @@ -24,7 +24,6 @@ sys.path.insert(0, str(APP)) from functions_accessibility import format_accessibility_metadata_for_chat # noqa: E402 -from functions_excel_accessibility import extract_excel_accessibility_metadata # noqa: E402 import functions_excel_accessibility as excel # noqa: E402 from functions_agent_document_citations import annotate_document_search_payload # noqa: E402 from test_excel_document_accessibility_extraction import _load_functions # noqa: E402 @@ -91,7 +90,7 @@ def review_workbook(tmp_path): def test_threaded_compatibility_copies_do_not_become_legacy_notes(review_workbook): before = hashlib.sha256(review_workbook.read_bytes()).hexdigest() - metadata, evidence = extract_excel_accessibility_metadata(review_workbook, ".xlsx") + metadata, evidence = excel.extract_excel_accessibility_metadata(review_workbook, ".xlsx") summary = metadata["comment_summary"] assert summary == { "legacy_notes": 1, "thread_roots": 3, "replies": 1, @@ -107,7 +106,7 @@ def test_threaded_compatibility_copies_do_not_become_legacy_notes(review_workboo def test_native_records_include_saved_cell_values_and_blank_cells(review_workbook): - metadata, _ = extract_excel_accessibility_metadata(review_workbook, ".xlsx") + metadata, _ = excel.extract_excel_accessibility_metadata(review_workbook, ".xlsx") comments = {item["id"]: item for item in metadata["comments"] if item.get("id")} budget = comments["{root-budget}"] delivery = comments["{root-delivery}"] @@ -122,7 +121,7 @@ def test_native_records_include_saved_cell_values_and_blank_cells(review_workboo def test_chat_formatter_supplies_comment_text_not_just_counts(review_workbook): - metadata, _ = extract_excel_accessibility_metadata(review_workbook, ".xlsx") + metadata, _ = excel.extract_excel_accessibility_metadata(review_workbook, ".xlsx") rendered = format_accessibility_metadata_for_chat(metadata) assert "Set cap to 19500." in rendered assert "Hold approval." in rendered @@ -141,7 +140,7 @@ def test_comment_inventory_sink_preserves_text_beyond_parent_preview(tmp_path): workbook.active["A2"].comment = Comment(text, "Reviewer") workbook.save(path) records = [] - metadata, _ = extract_excel_accessibility_metadata( + metadata, _ = excel.extract_excel_accessibility_metadata( path, ".xlsx", record_sink=lambda kind, record: records.append((kind, record)), ) note = next(record for kind, record in records if kind == "comments") @@ -158,7 +157,7 @@ def test_formula_without_cache_is_not_blank_or_zero(tmp_path): workbook.active["A2"] = "=1+2" workbook.active["A2"].comment = Comment("Confirm calculated amount.", "Reviewer") workbook.save(path) - metadata, _ = extract_excel_accessibility_metadata(path, ".xlsx") + metadata, _ = excel.extract_excel_accessibility_metadata(path, ".xlsx") note = metadata["comments"][0] assert note["formula"] == "=1+2" assert note["cell_value"] is None @@ -167,7 +166,7 @@ def test_formula_without_cache_is_not_blank_or_zero(tmp_path): def test_actual_chat_handoff_uses_known_records_when_relevance_has_no_matches(review_workbook): - metadata, _ = extract_excel_accessibility_metadata(review_workbook, ".xlsx") + metadata, _ = excel.extract_excel_accessibility_metadata(review_workbook, ".xlsx") metadata["properties"]["long_context"] = "irrelevant prefix " * 1200 metadata["style_inventory"]["long_context"] = "layout prefix " * 1500 report = { @@ -211,7 +210,7 @@ def unexpected_search(**kwargs): def test_mixed_calculation_handoff_preserves_native_and_computed_evidence(review_workbook): - metadata, _ = extract_excel_accessibility_metadata(review_workbook, ".xlsx") + metadata, _ = excel.extract_excel_accessibility_metadata(review_workbook, ".xlsx") report = {"document": {"id": "doc", "file_name": "review.xlsx"}, "accessibility_metadata": metadata} native = { "evidence_envelopes": [{"document_id": "doc", "status": "completed", "summary": "native", "citations": []}],