diff --git a/app/lifecycle.py b/app/lifecycle.py index f4cb27d..d578b8b 100644 --- a/app/lifecycle.py +++ b/app/lifecycle.py @@ -1,6 +1,7 @@ """Recoverable deletion and append-only voiding for the trusted workspace.""" import json +import hashlib from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Annotated, Literal @@ -27,6 +28,12 @@ def valid_text(cls, value): return value +class PurgeDeletedRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$") + confirmation: Literal["DELETE ALL"] + + LIFECYCLE_SCHEMA = ( "ALTER TABLE receipts ADD COLUMN lifecycle_state TEXT NOT NULL DEFAULT 'ACTIVE' CHECK(lifecycle_state IN ('ACTIVE','DELETED','VOIDED'))", "ALTER TABLE receipts ADD COLUMN lifecycle_version INTEGER NOT NULL DEFAULT 0", @@ -94,22 +101,55 @@ def apply_lifecycle(store, receipt_id: str, request: LifecycleRequest) -> dict: return event +_PURGE_SAFE = "NOT EXISTS (SELECT 1 FROM receipt_amendments a WHERE a.receipt_id=r.receipt_id) AND NOT EXISTS (SELECT 1 FROM receipt_reviews v WHERE v.receipt_id=r.receipt_id AND json_extract(v.result_json,'$.decision')='APPROVED') AND NOT EXISTS (SELECT 1 FROM classifications c WHERE c.receipt_id=r.receipt_id AND c.decision='AUTO_FILED')" + + +def _fingerprint(rows) -> str: + return hashlib.sha256(json.dumps([(row['receipt_id'], row['lifecycle_version']) for row in rows], separators=(',', ':')).encode()).hexdigest() + + +def deleted_purge_preview(store) -> dict: + with store.connect() as db: + rows = db.execute("SELECT receipt_id, lifecycle_version FROM receipts WHERE lifecycle_state='DELETED' ORDER BY receipt_id").fetchall() + return {'count': len(rows), 'fingerprint': _fingerprint(rows)} + + +def _erase_rows(db, rows, upload_dir: Path) -> int: + removed = 0 + for row in rows: + receipt_id = str(UUID(row['receipt_id'])) + # Use only generated filenames; never trust an image_path from the DB. + try: + for extension in ('.jpg', '.png', '.pdf', '.preview.png'): + (upload_dir / (receipt_id + extension)).unlink(missing_ok=True) + except OSError: + continue + for table in ('receipt_payment_events', 'receipt_reprocessing', 'review_audit', 'receipt_reviews', 'line_items', 'classifications', 'lifecycle_events'): + db.execute(f"DELETE FROM {table} WHERE receipt_id=?", (receipt_id,)) + db.execute("DELETE FROM receipts WHERE receipt_id=?", (receipt_id,)) + removed += 1 + return removed + + +def purge_deleted_now(store, upload_dir: Path, request: PurgeDeletedRequest) -> dict: + """Erase the confirmed snapshot of deleted receipts, including unexpired ones.""" + with store.connect() as db: + db.execute('BEGIN IMMEDIATE') + rows = db.execute("SELECT receipt_id, lifecycle_version FROM receipts WHERE lifecycle_state='DELETED' ORDER BY receipt_id").fetchall() + if not rows or request.fingerprint != _fingerprint(rows): + raise ReviewConflict('Deleted receipts changed. Reload and confirm again') + safe = db.execute(f"SELECT count(*) FROM receipts r WHERE lifecycle_state='DELETED' AND {_PURGE_SAFE}").fetchone()[0] + if safe != len(rows): + raise ReviewConflict('Some deleted receipts contain protected decisions and cannot be erased') + # Existing audit triggers permit deletion only after purge_after has expired. + db.execute("UPDATE receipts SET purge_after=datetime('now','-1 second') WHERE lifecycle_state='DELETED'") + removed = _erase_rows(db, rows, upload_dir) + return {'removed': removed, 'remaining': len(rows) - removed} + + def purge_expired(store, upload_dir: Path) -> int: """Bounded, retryable cleanup. Never follow a DB-supplied path or delete a void.""" - removed = 0 with store.connect() as db: db.execute("BEGIN IMMEDIATE") - rows = db.execute("SELECT receipt_id FROM receipts r WHERE lifecycle_state='DELETED' AND julianday(purge_after)<=julianday('now') AND NOT EXISTS (SELECT 1 FROM receipt_amendments a WHERE a.receipt_id=r.receipt_id) AND NOT EXISTS (SELECT 1 FROM receipt_reviews v WHERE v.receipt_id=r.receipt_id AND json_extract(v.result_json,'$.decision')='APPROVED') AND NOT EXISTS (SELECT 1 FROM classifications c WHERE c.receipt_id=r.receipt_id AND c.decision='AUTO_FILED') LIMIT 100").fetchall() - for row in rows: - receipt_id = str(UUID(row[0])) - # Keep the DB row on a file failure so the next run can retry safely. - try: - for extension in ('.jpg', '.png', '.pdf', '.preview.png'): - (upload_dir / (receipt_id + extension)).unlink(missing_ok=True) - except OSError: - continue - for table in ('receipt_payment_events', 'receipt_reprocessing', 'review_audit', 'receipt_reviews', 'line_items', 'classifications', 'lifecycle_events'): - db.execute(f"DELETE FROM {table} WHERE receipt_id=?", (receipt_id,)) - db.execute("DELETE FROM receipts WHERE receipt_id=?", (receipt_id,)) - removed += 1 - return removed + rows = db.execute(f"SELECT receipt_id FROM receipts r WHERE lifecycle_state='DELETED' AND julianday(purge_after)<=julianday('now') AND {_PURGE_SAFE} LIMIT 100").fetchall() + return _erase_rows(db, rows, upload_dir) diff --git a/app/main.py b/app/main.py index 3d8e307..baf1e7f 100644 --- a/app/main.py +++ b/app/main.py @@ -30,7 +30,8 @@ ) from app.images import receipt_image, receipt_preview from app.agents.router import build_agent_router -from app.lifecycle import LifecycleRequest, apply_lifecycle, purge_expired +from app.lifecycle import (LifecycleRequest, PurgeDeletedRequest, apply_lifecycle, + deleted_purge_preview, purge_deleted_now, purge_expired) from app.reprocessing import ReprocessRequest, reprocess from app.statements import ( MonthlyExportRequest, @@ -642,6 +643,17 @@ async def lifecycle(receipt_id: UUID, body: LifecycleRequest, settings: Annotated[Settings, Depends(require_api_key)]) -> dict: return await run_in_threadpool(apply_lifecycle, ReceiptStore(settings.database_path), str(receipt_id), body) + @api.get("/receipts/deleted/purge-preview", tags=["receipts"], + summary="Preview the current deleted receipts before permanent erasure") + async def preview_deleted_purge(settings: Annotated[Settings, Depends(require_api_key)]) -> dict: + return await run_in_threadpool(deleted_purge_preview, ReceiptStore(settings.database_path)) + + @api.post("/receipts/deleted/purge", tags=["receipts"], + summary="Permanently erase the confirmed set of deleted receipts") + async def purge_deleted(body: PurgeDeletedRequest, + settings: Annotated[Settings, Depends(require_api_key)]) -> dict: + return await run_in_threadpool(purge_deleted_now, ReceiptStore(settings.database_path), settings.upload_dir, body) + @api.get("/reviews", tags=["reviews"], summary="1. List receipts awaiting human review", description="Read only. Copy a receipt_id, then use GET /receipts/{receipt_id}. Empty items means no pending receipts on THIS server. Local port 8000 and the AWS tunnel port 18000 use separate databases. Finalized, AUTO_FILED, FAILED and PROCESSING receipts are excluded.") async def reviews(settings: Annotated[Settings, Depends(require_api_key)], diff --git a/docs/receipt-lifecycle.md b/docs/receipt-lifecycle.md index 61bdf12..39e3148 100644 --- a/docs/receipt-lifecycle.md +++ b/docs/receipt-lifecycle.md @@ -28,6 +28,8 @@ The restore deadline is exactly 30 days after deletion, in UTC; the UI displays Cleanup removes only server-generated UUID filenames within the configured upload directory, never stored arbitrary paths. It removes the receipt's dependent rows, including rejection/lifecycle evidence, only after expiry. Approved and amendment audit protections remain. Voided receipts have no purge deadline. A generic server warning indicates a failed cleanup pass without logging receipt contents. +**Empty Deleted receipts** allows the authenticated bookkeeper to erase the entire Deleted receipts tab before the normal deadline. The dialog requires typing `DELETE ALL` and the API verifies a snapshot of deleted receipt IDs and versions before erasing them. A changed snapshot causes a conflict instead of deleting newly added records. The operation preserves the same audit and filename safeguards as automatic cleanup and reports any files that could not be erased for automatic retry. Restoring the erased receipts through the app is impossible. + This is application retention, not deletion from independent backups. Backups retain their own lifetime. Before deploying, back up the database and uploads together. Migration from schema versions 1–3 to 4 preserves existing evidence and marks every current record ACTIVE. No existing record is automatically deleted or voided on migration. `POST /receipts/{id}/lifecycle` accepts `request_id`, `action`, `expected_version` (lifecycle_version), `expected_record_version`, `reviewer`, and `reason`. Reasons require 10–2000 characters. Use the same UUID and payload after an uncertain response. Actions run inside a write transaction; stale versions, processing receipts, invalid transitions, and restore collisions return 409. Inspect the current record before retrying a conflict. diff --git a/docs/user-guide.md b/docs/user-guide.md index 844ede1..3fc462f 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -176,6 +176,8 @@ Choose **Deleted receipts** in the top navigation to inspect recoverable records **Restore receipt** is available before the 30-day deadline, subject to the app’s duplicate-file checks. The lifecycle history preserves the recorded actions and self-reported reviewer labels. In this image an earlier restore appears in the history, but the receipt currently shown is **Deleted**; the button has not been used in this pictured state. Processing receipts must finish processing before deletion; accepted receipts use **Void receipt** and do not enter Deleted receipts. See [receipt lifecycle](receipt-lifecycle.md) for eligibility, retention and conflict handling. +To clear this tab immediately, select **Empty Deleted receipts**, check the count in the confirmation dialog, type `DELETE ALL`, and select **Erase deleted receipts**. This permanently removes all currently deleted receipts and their saved files, including receipts still within their restore window. Active and voided receipts remain. If the list changes while the dialog is open, reload and confirm the new count. Independent backups follow their own retention policy. + ## Monthly close ### Find the period diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 56f5076..77636e5 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -18,6 +18,9 @@ import { import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Textarea } from "@/components/ui/textarea"; +import { + Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, +} from "@/components/ui/dialog"; import { Table, TableBody, @@ -301,7 +304,13 @@ function Workspace({ [filters, setFilters] = useState(emptyFilters), [checked, setChecked] = useState>(new Set()), [exportBusy, setExportBusy] = useState(false), - [monthlyReturn, setMonthlyReturn] = useState(null); + [monthlyReturn, setMonthlyReturn] = useState(null), + [purgeOpen, setPurgeOpen] = useState(false), + [purgePreview, setPurgePreview] = useState<{ count: number; fingerprint: string } | null>(null), + [purgePhrase, setPurgePhrase] = useState(""), + [purgeBusy, setPurgeBusy] = useState(false), + [purgeError, setPurgeError] = useState(""), + [purgeResult, setPurgeResult] = useState(""); const now = useClock(); const heading = useRef(null); useEffect(() => { @@ -391,6 +400,40 @@ function Workspace({ setExportBusy(false); } } + async function openPurge() { + setPurgeOpen(true); + setPurgePreview(null); + setPurgePhrase(""); + setPurgeError(""); + try { + setPurgePreview(await request<{ count: number; fingerprint: string }>( + "/receipts/deleted/purge-preview", token, + )); + } catch (cause) { + setPurgeError(message(cause)); + } + } + async function emptyDeleted() { + if (!purgePreview || !purgePreview.count || purgePhrase !== "DELETE ALL") return; + setPurgeBusy(true); + setPurgeError(""); + try { + const result = await request<{ removed: number; remaining: number }>( + "/receipts/deleted/purge", token, + { method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ fingerprint: purgePreview.fingerprint, confirmation: purgePhrase }) }, + ); + setPurgeResult(`${result.removed} deleted receipt${result.removed === 1 ? "" : "s"} permanently erased.${result.remaining ? ` ${result.remaining} could not be erased and will be retried automatically.` : ""}`); + setPurgeOpen(false); + setOffset(0); + setRefresh((n) => n + 1); + } catch (cause) { + setPurgeError(message(cause)); + if (cause instanceof ApiError && cause.status === 409) setRefresh((n) => n + 1); + } finally { + setPurgeBusy(false); + } + } return ( <> @@ -537,6 +580,9 @@ function Workspace({ ? "Restore receipts within 30 days. Expired receipts and their files are automatically erased; finalized receipts never enter this area." : "Human decisions take precedence. Voided receipts remain visible as evidence and are excluded from totals and exports."}

+ {view === "deleted" && purgeResult && ( + {purgeResult} + )} {view === "history" && (
+ {view === "deleted" && ( +
+

{page.total} deleted receipt{page.total === 1 ? "" : "s"}

+ +
+ )} {view === "history" && (

@@ -864,6 +918,30 @@ function Workspace({ )}

+ { if (!purgeBusy) setPurgeOpen(open); }}> + + + Empty Deleted receipts? + + Permanently erase {purgePreview?.count ?? "…"} deleted receipt{purgePreview?.count === 1 ? "" : "s"} and their saved files now. You cannot restore them afterward. Active and voided receipts are unaffected. + + + {!purgePreview && !purgeError &&

Loading deleted receipts…

} + {purgePreview && purgePreview.count > 0 && ( +
+ + setPurgePhrase(event.target.value)} /> +
+ )} + {purgeError && {purgeError}} + + + + + + ); } diff --git a/tests/test_lifecycle.py b/tests/test_lifecycle.py index 6595756..6154c90 100644 --- a/tests/test_lifecycle.py +++ b/tests/test_lifecycle.py @@ -112,6 +112,52 @@ def test_purge_only_expired_deleted_files_and_rows(monkeypatch, tmp_path, reject assert purge_expired(store, tmp_path) == 0 +def test_empty_deleted_requires_current_confirmation_and_preserves_active(monkeypatch, tmp_path): + client, first, review, store = setup_review(monkeypatch, tmp_path) + first_id = first['receipt_id'] + review.update(decision='REJECTED', corrected_data=None, category=None) + assert client.post(f'/receipts/{first_id}/review', headers=HEADERS, json=review).status_code == 200 + assert change(client, first_id, body('DELETE', record_version=1)).status_code == 200 + preview = client.get('/receipts/deleted/purge-preview', headers=HEADERS).json() + assert preview['count'] == 1 + assert client.get('/receipts/deleted/purge-preview').status_code == 401 + assert client.post('/receipts/deleted/purge', json={**preview, 'confirmation': 'DELETE ALL'}).status_code == 401 + second = client.post('/receipts/upload', headers=HEADERS, + files={'receipt': ('second.jpg', b'\xff\xd8\xffsecond', 'image/jpeg')}).json() + second_id = second['receipt_id'] + assert change(client, second_id, body('DELETE')).status_code == 200 + stale = client.post('/receipts/deleted/purge', headers=HEADERS, + json={'fingerprint': preview['fingerprint'], 'confirmation': 'DELETE ALL'}) + assert stale.status_code == 409 + assert store.get(first_id) is not None and store.get(second_id) is not None + active = client.post('/receipts/upload', headers=HEADERS, + files={'receipt': ('active.jpg', b'\xff\xd8\xffactive', 'image/jpeg')}).json() + active_id = active['receipt_id'] + voided = client.post('/receipts/upload', headers=HEADERS, + files={'receipt': ('voided.jpg', b'\xff\xd8\xffvoided', 'image/jpeg')}).json() + voided_id = voided['receipt_id'] + with store.connect() as db: + db.execute("UPDATE classifications SET decision='AUTO_FILED' WHERE receipt_id=?", (voided_id,)) + assert change(client, voided_id, body('VOID')).status_code == 200 + current = client.get('/receipts/deleted/purge-preview', headers=HEADERS).json() + assert current['count'] == 2 + invalid = client.post('/receipts/deleted/purge', headers=HEADERS, + json={'fingerprint': current['fingerprint'], 'confirmation': 'yes'}) + assert invalid.status_code == 422 + result = client.post('/receipts/deleted/purge', headers=HEADERS, + json={'fingerprint': current['fingerprint'], 'confirmation': 'DELETE ALL'}) + assert result.status_code == 200, result.text + assert result.json() == {'removed': 2, 'remaining': 0} + for receipt_id in (first_id, second_id): + assert store.get(receipt_id) is None + assert not (tmp_path / f'{receipt_id}.jpg').exists() + assert store.get(active_id) is not None + assert (tmp_path / f'{active_id}.jpg').exists() + assert store.get(voided_id)['lifecycle_state'] == 'VOIDED' + assert (tmp_path / f'{voided_id}.jpg').exists() + assert client.get('/receipts?state=DELETED', headers=HEADERS).json()['total'] == 0 + + def test_auth_validation_processing_and_stale_review(monkeypatch, tmp_path): client, receipt, review, store = setup_review(monkeypatch, tmp_path) rid = receipt['receipt_id']