Feature/table download

README.md

@@ -32,6 +32,14 @@ It supports:
 
 ## 🏗 Architecture overview
 
+### Global system architecture
+
+![Global system architecture](docs/images/summarization-tool-global-architecture.png)
+
+*Architecture overview. The platform connects document ingestion, the React user interface, FastAPI orchestration, parser outputs, LLM workflows, evaluation, collaboration, persistence, authentication, and observability.*
+
+Detailed backend diagrams: [Backend visual workflow map](docs/backend/README.md).
+
 The application is organized around a small set of core services:
 
 | Service | Port | Purpose |
@@ -244,9 +252,14 @@ SummarizationTool-dev/
 
 ## 📚 Additional documentation
 
+- [Documentation index](docs/INDEX.md) — **start here** — full navigation map for all backend, frontend, and deployment docs
+- [Frontend technical design docs](docs/frontend/README.md) — frontend architecture, page-by-page docs, component index, hooks, and TypeScript interfaces
+- [Glossary](docs/glossary.md) — definitions for all Azure services, tools, and project-specific terms
 - [Backend README](backend/README.md) — backend setup and processing details
-- [Migration guide](docs/migration-guide.md) — architecture migration and platform transition notes
-- [GitHub auth setup](docs/setup-github-auth.md) — Better Auth GitHub OAuth configuration
+- [Backend technical design docs](docs/backend/README.md) — backend architecture, workflows, diagrams, data models, schemas, and appendices
+- [Backend class reference](docs/backend/appendices/class-reference.md) — field-level reference for backend ORM models, schemas, dataclasses, service attributes, and provider classes
+- [Migration guide](docs/superpowers/migration-guide.md) — architecture migration and platform transition notes
+- [GitHub auth setup](docs/superpowers/setup-github-auth.md) — Better Auth GitHub OAuth configuration
 - [Dockerize & deploy to Azure](docs/superpowers/plans/dockerize-and-deploy.md) — deployment architecture and implementation notes
 
 ---

auth-service/src/index.ts

@@ -16,14 +16,29 @@ import "dotenv/config";
 import express from "express";
 import cors from "cors";
 import { betterAuth } from "better-auth";
+import { getMigrations } from "better-auth/db/migration";
 import { toNodeHandler } from "better-auth/node";
 import { Pool } from "pg";
 
 // ---------- Shared DB Pool ----------
 
+function databaseRequiresSSL(url: string | undefined): boolean {
+  if (!url) return false;
+  try {
+    const parsed = new URL(url);
+    const host = parsed.hostname.toLowerCase();
+    if (host === "azure.com" || host.endsWith(".azure.com")) return true;
+    if (parsed.searchParams.get("sslmode") === "require") return true;
+  } catch {
+    // Non-standard connection string — fall back to explicit sslmode check only
+    if (/[?&]sslmode=require(&|$)/.test(url)) return true;
+  }
+  return false;
+}
+
 const dbPool = new Pool({
   connectionString: process.env.DATABASE_URL,
-  ssl: { rejectUnauthorized: false },
+  ssl: databaseRequiresSSL(process.env.DATABASE_URL) ? { rejectUnauthorized: false } : false,
 });
 
 function getAllowedEmails(): Set<string> {
@@ -155,10 +170,23 @@ app.get("/api/auth/validate", async (req, res) => {
 });
 
 const PORT = process.env.PORT || 3001;
-app.listen(PORT, () => {
-  console.log(`✅ Better Auth sidecar running on http://localhost:${PORT}`);
-  console.log(`   Auth endpoints: http://localhost:${PORT}/api/auth/*`);
-  console.log(
-    `   GitHub OAuth: ${process.env.GITHUB_CLIENT_ID ? "ENABLED" : "DISABLED (no credentials)"}`
-  );
+
+async function start() {
+  console.log("[Migration] Running Better Auth database migrations...");
+  const { runMigrations } = await getMigrations((auth as any).options);
+  await runMigrations();
+  console.log("[Migration] Done.");
+
+  app.listen(PORT, () => {
+    console.log(`✅ Better Auth sidecar running on http://localhost:${PORT}`);
+    console.log(`   Auth endpoints: http://localhost:${PORT}/api/auth/*`);
+    console.log(
+      `   GitHub OAuth: ${process.env.GITHUB_CLIENT_ID ? "ENABLED" : "DISABLED (no credentials)"}`
+    );
+  });
+}
+
+start().catch((err) => {
+  console.error("[Startup] Fatal error:", err);
+  process.exit(1);
 });

backend/api/documents/router.py

@@ -1351,6 +1351,81 @@ async def extract_figure_content(
     return await generate_figure_summary(document_id, figure_id, request, http_request)
 
 
+@router.get("/{document_id}/tables/download", dependencies=[Depends(get_current_user)])
+async def download_all_tables_zip(document_id: str):
+    """
+    Download all extracted tables as a single ZIP archive.
+
+    Resolves the processor once, fetches all table HTML files concurrently via
+    asyncio.gather(), packages them into an in-memory ZIP, and returns it as
+    application/zip.
+
+    IMPORTANT: must be registered before /{document_id}/tables/{table_filename}
+    so FastAPI does not route the literal segment "download" to get_table_html.
+    """
+    import asyncio
+    import io
+    import zipfile
+    from fastapi.responses import Response as _Response
+
+    try:
+        resolved_processor = await file_service.resolve_processed_processor(document_id)
+        if not resolved_processor:
+            raise HTTPException(status_code=404, detail="Processed document not found")
+
+        metadata = await file_service.get_processed_metadata(
+            document_id, resolved_processor
+        )
+        tables_count = (metadata or {}).get("tables_found", 0)
+
+        if not tables_count:
+            blobs = await file_service._blob.list_blobs_with_prefix(
+                f"global/{document_id}/processed/{resolved_processor}/tables/",
+                limit=500,
+            )
+            tables_count = len([b for b in blobs if b.lower().endswith(".html")])
+
+        if not tables_count:
+            raise HTTPException(
+                status_code=404, detail="No tables found for this document"
+            )
+
+        print(f"[TABLE-ZIP] Building ZIP for {document_id}: {tables_count} tables")
+
+        async def fetch_one(i: int):
+            return i, await file_service.get_processing_file_bytes(
+                document_id, resolved_processor, f"tables/table-{i}.html"
+            )
+
+        results = await asyncio.gather(
+            *[fetch_one(i) for i in range(1, tables_count + 1)]
+        )
+
+        buf = io.BytesIO()
+        with zipfile.ZipFile(buf, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
+            for i, data in results:
+                if data:
+                    zf.writestr(f"table-{i}.html", data)
+        buf.seek(0)
+
+        print(f"[TABLE-ZIP] ✅ ZIP ready for {document_id}")
+        return _Response(
+            content=buf.read(),
+            media_type="application/zip",
+            headers={
+                "Content-Disposition": f'attachment; filename="tables-{document_id[:8]}.zip"'
+            },
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        print(f"[TABLE-ZIP] Error: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error creating tables ZIP: {str(e)}"
+        )
+
+
 @router.get(
     "/{document_id}/tables/{table_filename}", dependencies=[Depends(get_current_user)]
 )

backend/services/document/organized_file_service.py

@@ -13,6 +13,7 @@
 path, and as a read cache to avoid redundant blob downloads within one container lifetime.
 """
 
+import asyncio
 import json
 import os
 import hashlib
@@ -187,14 +188,14 @@ async def get_processing_file_bytes(
             / proc_str
             / Path(norm_path)
         )
-        if tmp_file.exists():
-            return tmp_file.read_bytes()
+        if await asyncio.to_thread(tmp_file.exists):
+            return await asyncio.to_thread(tmp_file.read_bytes)
 
         blob_path = f"global/{file_hash}/processed/{proc_str}/{norm_path}"
         data = await self._blob.download_bytes(blob_path)
         if data:
-            tmp_file.parent.mkdir(parents=True, exist_ok=True)
-            tmp_file.write_bytes(data)
+            await asyncio.to_thread(tmp_file.parent.mkdir, parents=True, exist_ok=True)
+            await asyncio.to_thread(tmp_file.write_bytes, data)
         return data
 
     async def processing_file_exists(

docs/INDEX.md

@@ -0,0 +1,84 @@
+# Science-GPT Documentation Index
+
+Start here. Every document in this repository is listed below with its audience and a short description of what it covers.
+
+**New to the project?** Read the [Product Overview](README.md) first, then the [Glossary](glossary.md), then the entry point for the area you're working in (backend or frontend).
+
+---
+
+## Start here
+
+| Document | Audience | What it covers |
+|---|---|---|
+| [Product Overview](README.md) | Everyone | What the tool does, who uses it (PMRA reviewers), how it was tested, SME evaluation results |
+| [Glossary](glossary.md) | Everyone | Plain-language definitions for all Azure services, tools, and project-specific terms |
+
+---
+
+## Backend
+
+| Document | Audience | What it covers |
+|---|---|---|
+| [Backend TDD — Entry Point](backend/README.md) | Engineers | Master index for the full backend technical design; start here for anything backend |
+| [01 — Architecture](backend/01-architecture.md) | Engineers | Service boundaries, package responsibilities, five major data flows, dependency direction |
+| [02 — API Surface](backend/02-api-surface.md) | Engineers | All 14 routers documented with every endpoint, request/response shapes, auth, and exceptions |
+| [03 — Data Models](backend/03-data-models.md) | Engineers | All 13 ORM models with field-level types, constraints, indexes, and migration notes |
+| [04 — Schemas](backend/04-schemas.md) | Engineers | All Pydantic request/response schemas with design notes |
+| [05 — Document Processing](backend/05-document-processing.md) | Engineers | Upload, SHA-256 deduplication, parser selection, Azure DI and Docling pipelines, blob storage, bounding boxes |
+| [06 — LLM Layer](backend/06-llm-layer.md) | Engineers | All 7 provider clients, dispatch table, timeout budgets, structured output handling, cost tracking |
+| [07 — Extraction Flow](backend/07-extraction-flow.md) | Engineers | Entity extraction end-to-end: concurrency model, provider dispatch, reference/bbox matching, session persistence |
+| [08 — Evaluation Flow](backend/08-evaluation-flow.md) | Engineers | G-Eval scoring, combined JSON parsing, background job lifecycle, cancellation, cost tracking |
+| [09 — Sessions, Sharing & Groups](backend/09-session-sharing-groups.md) | Engineers | Session lifecycle, restore-view construction, group membership rules, shared session read path |
+| [10 — Template System](backend/10-template-system.md) | Engineers | Template CRUD, version snapshots, fork, scope change, access-control algorithms, folder operations |
+| [11 — Auth, Security & Observability](backend/11-auth-security-observability.md) | Engineers | Better Auth session validation, auth proxy, CORS, secrets loading, structlog, Prometheus, OpenTelemetry, CostTracker |
+
+### Backend appendices
+
+| Document | What it covers |
+|---|---|
+| [API Endpoint Index](backend/appendices/api-endpoint-index.md) | Compact table of every route — method, path, purpose |
+| [Class Index](backend/appendices/class-index.md) | All backend classes organised by package |
+| [Class Reference](backend/appendices/class-reference.md) | Field-level reference for ORM models, Pydantic schemas, and service classes |
+| [Data Flow Diagrams](backend/appendices/data-flow-diagrams.md) | 15 text-format diagrams covering every major request flow |
+| [Risks, Assumptions & Testing](backend/appendices/risks-assumptions-testing.md) | Runtime/data/provider assumptions, risk table with mitigations, 13-category test strategy, 12-step smoke test |
+
+---
+
+## Frontend
+
+| Document | Audience | What it covers |
+|---|---|---|
+| [Frontend TDD — Entry Point](frontend/README.md) | Engineers | Tech stack, page map, workflow diagram, architecture overview |
+| [01 — App Shell](frontend/01-app-shell.md) | Engineers | `App.tsx` — `DocumentData` interface, step routing, `onComplete()` pattern, session persistence, navigation guards |
+| [02 — Auth](frontend/02-auth.md) | Engineers | `LoginPage`, `AuthCallback`, `authUtils.ts` — OAuth flow, token lifecycle, `authenticatedFetch()`, visibility refresh |
+| [03 — Upload](frontend/03-upload.md) | Engineers | Workflow step 1 — file upload, SHA-256 deduplication, auto-processing, parser selection |
+| [04 — Processing](frontend/04-processing.md) | Engineers | Workflow step 2 — parsed content inspection, re-processing, PDF bounding box viewer |
+| [05 — Study Config](frontend/05-study-config.md) | Engineers | Workflow step 3 — study type, entity editor, template loading, model selection |
+| [06 — Extraction](frontend/06-extraction.md) | Engineers | Workflow step 4 — concurrent entity extraction, PDF reference highlighting, multi-model comparison, in-place editing |
+| [07 — Evaluation](frontend/07-evaluation.md) | Engineers | Workflow step 5 — G-Eval metrics, background jobs, human score overrides, Excel export |
+| [08 — Simplified Flow](frontend/08-simplified-flow.md) | Engineers | One-click pipeline, `useSimplifiedPipeline` hook, batched extraction, stage progression |
+| [09 — Chat](frontend/09-chat.md) | Engineers | Freeform document Q&A, multi-document context, message ratings |
+| [10 — Session History](frontend/10-session-history.md) | Engineers | Browse, restore, share, and delete sessions; shared sessions (read-only) |
+| [11 — Templates](frontend/11-templates.md) | Engineers | Template CRUD, version history, fork, scope change, folder organisation |
+| [12 — Groups](frontend/12-groups.md) | Engineers | Group lifecycle, member roles, add/remove members, user search |
+| [13 — Executive Mode](frontend/13-executive-mode.md) | Engineers | Standalone summary generation without structured entity review |
+| [14 — Batch Results](frontend/14-batch-results.md) | Engineers | Cross-file results table, fuzzy search, column visibility, Excel export |
+
+### Frontend appendices
+
+| Document | What it covers |
+|---|---|
+| [Component Index](frontend/appendices/component-index.md) | All shared components with props and usage |
+| [Hooks & Contexts](frontend/appendices/hooks-contexts.md) | All custom hooks and `ThemeContext` with exported APIs |
+| [Types & Interfaces](frontend/appendices/types-interfaces.md) | Key TypeScript interfaces: `DocumentData`, `Entity`, `Template`, `Group`, and more |
+
+---
+
+## Deployment & operations
+
+| Document | Audience | What it covers |
+|---|---|---|
+| [GitHub Auth Setup](superpowers/setup-github-auth.md) | DevOps | 10-step guide to configuring GitHub Enterprise Cloud OAuth |
+| [Migration Guide](superpowers/migration-guide.md) | Engineers | Supabase → Azure Postgres + Better Auth migration history |
+| [Deployment Plan](superpowers/plans/dockerize-and-deploy.md) | DevOps | Azure Container Apps architecture, CI/CD pipeline, provisioning record |
+| [Logging Stack](../logging/README.md) | DevOps | LGTM stack setup: Grafana, Loki, Tempo, Prometheus; NSG firewall rules |