Fix Excel comment content and add scalable table-aware inspection

application/single_app/admin_settings_fields.py

@@ -2557,10 +2557,10 @@
         {
             "key": "enable_office_embedded_image_analysis",
             "type": "switch",
-            "label": "Analyze images embedded in DOCX and PPTX files",
+            "label": "Analyze images embedded in DOCX, PPTX, and Excel files",
             "help": (
-                "Neither extraction engine describes figures inside Word and PowerPoint "
-                "files. With this on, embedded images are pulled out, analysed with "
+                "Native extraction does not describe image pixels inside Word, PowerPoint, "
+                "or Excel files. With this on, embedded images are pulled out, analysed with "
                 "whichever engine backs the selected extraction mode, and indexed as "
                 "their own citable chunks. Works with Standard extraction too."
             ),

application/single_app/config.py

@@ -103,7 +103,7 @@
 EXECUTOR_MAX_WORKERS = 30
 CHAT_STREAM_EXECUTOR_MAX_WORKERS = max(8, int(os.getenv('SIMPLECHAT_CHAT_STREAM_WORKERS', '32') or '32'))
 SESSION_TYPE = 'filesystem'
-VERSION = "0.261.407"
+VERSION = "0.261.409"
 IS_DEVELOPMENT = is_development_env_enabled()
 
 # Opt-out for deployments where App Service Easy Auth is active but the platform

application/single_app/content_screening/access.py

@@ -93,6 +93,7 @@
     "group_document_collaboration_operation", "document_share_details",
     "public_document_projection_writer",
     "public_document_collaboration_operation",
+    "excel_native_inventory",
     # Recomputed per request from current authorization, so a stored copy is
     # always stale. The group projection re-adds them after serialization.
     "document_actions", "document_collaboration_actions",
@@ -1068,6 +1069,23 @@
     return payload
 
 
+def _with_current_excel_metadata(payload, document):
+    metadata = payload.get("accessibility_metadata")
+    marker = document.get(SCREENING_FIELD)
+    if (
+        not isinstance(metadata, Mapping) or metadata.get("format") not in {"xlsx", "xlsm", "xls"}
+        or not isinstance(marker, Mapping)
+    ):
+        return payload
+    active = marker.get("active_blob")
+    inventory = metadata.get("inventory_storage")
+    digest = inventory.get("source_sha256") if isinstance(inventory, Mapping) else None
+    if not isinstance(active, Mapping) or not digest or active.get("content_hash") != digest:
+        payload.pop("accessibility_metadata", None)
+        payload["native_inspection_status"] = "requires_active_source_inspection"
+    return payload
+
+
 def public_document_payload(document):
     if not isinstance(document, Mapping):
         return {}
@@ -1080,7 +1098,7 @@
         payload["status"] = resolve_metadata_completion_status(
             document.get("status"), document.get("metadata_extraction_status"),
         )
-        return payload
+        return _with_current_excel_metadata(payload, document)
     try:
         _require_available_metadata(document)
         config = import_module("config")
@@ -1116,7 +1134,7 @@
             document.get("status"), document.get("metadata_extraction_status"),
         )
     payload["num_chunks"] = chunk_count_for_response(document)
-    return payload
+    return _with_current_excel_metadata(payload, document)
 
 
 def public_documents_payload(documents, user_id=None, *, metadata_reader=None):

application/single_app/content_screening/extraction.py

@@ -9,6 +9,7 @@
 from dataclasses import dataclass, field
 from datetime import date, datetime, time
 from pathlib import Path
+from typing import Callable
 
 from content_screening.contracts import (
     CONTENT_METADATA_FIELDS,
@@ -43,6 +44,8 @@
     completed: bool = False
     preloaded_text: bool = False
     failure_code: str | None = None
+    artifact_storage: object | None = field(default=None, repr=False, compare=False)
+    heartbeat: Callable[[], None] | None = field(default=None, repr=False, compare=False)
 
     def _append(self, target, unit):
         self.characters += len(unit.text)

application/single_app/content_screening/publication.py

@@ -2,6 +2,7 @@
 """Application-side projection of inspected canonical content."""
 
 import hashlib
+import io
 import json
 import mimetypes
 from urllib.parse import quote
@@ -171,6 +172,34 @@
     )
 
 
+def _tabular_publication_chunks(units, file_name, max_characters, *, include_native, schema_text=None):
+    chunks = publication_chunks([
+        ContentUnit("table-schema", table_schema_text(units) if schema_text is None else schema_text, {"kind": "schema"}),
+    ], max_characters)
+    if not include_native or not any(unit.locator.get("kind") == "excel_native" for unit in units):
+        return chunks
+    prefix = f"### Excel native inspection\nWorkbook: {file_name}\nSearch preview; complete records are in the native inventory.\n"
+    if len(prefix) >= max_characters:
+        prefix = "### Excel native inspection\n"
+    if len(prefix) >= max_characters:
+        raise ScreeningValidationError("The publication chunk size cannot fit the Excel native evidence marker.")
+    remaining = 1000000
+    for unit in units:
+        if unit.locator.get("kind") != "excel_native" or remaining <= len(prefix):
+            continue
+        preview = ContentUnit(unit.unit_id, unit.text[:remaining], unit.locator)
+        for chunk in publication_chunks([preview], max_characters - len(prefix)):
+            if remaining <= len(prefix):
+                break
+            text = prefix + chunk["chunk_text"][:remaining - len(prefix)]
+            chunks.append({
+                **chunk, "chunk_text": text, "chunk_sequence": len(chunks) + 1,
+                "end": chunk["start"] + len(text) - len(prefix),
+            })
+            remaining -= len(text)
+    return chunks
+
+
 def publish_document(document, units, scan, actor_id, *, storage):
     helpers = _document_helpers()
     subject = subject_from_document(document)
@@ -195,9 +224,14 @@
         max(1, helpers.get_embedding_usable_tokens(settings) // 4),
     )
     if structured_sheets(units):
-        chunks = publication_chunks([
-            ContentUnit("table-schema", table_schema_text(units), {"kind": "schema"}),
-        ], max_characters)
+        extension = "." + file_name.rsplit(".", 1)[-1].lower()
+        schema_text = None
+        if extension in {".xlsx", ".xlsm", ".xls"}:
+            schema_text = helpers._build_tabular_schema_summary(io.BytesIO(content), file_name, extension)
+        chunks = _tabular_publication_chunks(
+            units, file_name, max_characters, include_native=not scan.get("sanitized"),
+            schema_text=schema_text,
+        )
     else:
         chunks = publication_chunks(units, max_characters)
     if not chunks:

application/single_app/content_screening/service.py

@@ -472,7 +472,7 @@
 def _extract_and_inspect(scan, actor_id, source_path, processor, *, repository, storage, extraction_mode_override=None):
     subject = Subject.from_dict(scan["subject"])
     scan, owner = _claim_scan(repository, _read_scan(repository, scan["id"]))
-    capture = ExtractionCapture(subject, scan["id"], scan["original_file_name"])
+    capture = ExtractionCapture(subject, scan["id"], scan["original_file_name"], artifact_storage=storage)
     last_heartbeat = _now()
 
     def heartbeat():
@@ -1049,6 +1049,11 @@
 def prepare_document_deletion(document, actor_id, *, repository=None, storage=None):
     """Revoke a screened document before removing private evidence and originals."""
     if SCREENING_FIELD not in document:
+        if (
+            "excel_native_inventory" in document
+            or str(document.get("file_name") or "").lower().endswith((".xlsx", ".xlsm", ".xls"))
+        ):
+            _storage(storage).delete_revision(subject_from_document(document))
         return
     repository = _repository(repository)
     subject = subject_from_document(document)