From 51b79c33db97de1bae60fc53e727d3b14d5a502f Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Wed, 16 Sep 2026 23:30:48 -0700 Subject: [PATCH 01/20] fix(import): scope known-series recovery and preserve safety evidence --- docs/development/IMPORT_REVIEW_RECOVERY.md | 22 +- .../services/import_completed_cleanup.py | 104 ++++-- .../services/import_known_series_recovery.py | 60 ++-- src/pullbox/services/import_review_actions.py | 13 +- .../services/import_safety_diagnostics.py | 9 +- tests/unit/test_import_completed_cleanup.py | 100 ++++++ .../unit/test_import_known_series_recovery.py | 321 +++++++++++++++++- tests/unit/test_import_safety_diagnostics.py | 27 ++ 8 files changed, 595 insertions(+), 61 deletions(-) diff --git a/docs/development/IMPORT_REVIEW_RECOVERY.md b/docs/development/IMPORT_REVIEW_RECOVERY.md index a011dc08..5c1de10d 100644 --- a/docs/development/IMPORT_REVIEW_RECOVERY.md +++ b/docs/development/IMPORT_REVIEW_RECOVERY.md @@ -97,8 +97,10 @@ ineligible. - **Skip unusable files** excludes empty, unsupported, and page-less files, including those confirmed unusable by a later source recheck. - **Allow oversized files once** retries only decompression-size blocks marked - overrideable. It does not change the global archive safety policy or approve - dangerous archive content. + overrideable, including failed files whose latest source recheck recorded an + overrideable size limit rather than a scan-time safety block. The original + recheck evidence is retained alongside the one-job exception. It does not + change the global archive safety policy or approve dangerous archive content. - **Retry source inspection** rechecks files that were unreadable, changed, or temporarily could not be inspected, then resumes only work that now passes. - **Recognize already-owned issues** clears conflicts whose issue already has a @@ -118,9 +120,14 @@ ineligible. only files whose saved series and issue identities agree. Existing catalog ownership, issue numbers, per-file conflicts, manual decisions, skips, and safety blocks are checked before an actor-bound preview is issued. Ambiguous - duplicate candidates and a series containing an unpreviewed ready file are - excluded. The confirmed scope runs through normal background Step 4 source - validation and import rules. Successful files and source paths are untouched; + duplicate candidates remain excluded, including another ready file claiming + the same issue or a manually chosen keeper. Explicit file exclusions and + safety-review decisions remain protected. Eligible files are isolated into + new recovery groups with links to their original groups; an unresolved + sibling no longer blocks otherwise proven files. The actor-bound preview + authorizes only those groups, not other ready files or Story Arcs in the job. + The confirmed scope runs through normal background Step 4 source validation + and the original copy or keep-in-place rules. Successful files and source paths are untouched; unresolved files remain in Follow-up. This is not a blanket repair of stale IDs or a replacement for manual review when trusted evidence disagrees. - **Recheck deferred files** checks the remaining unmatched files in a resumable @@ -174,7 +181,10 @@ status-only correction does not launch another import. A completed source recheck reports a file ready only after both archive safety and saved target identity checks pass. Missing, empty, or otherwise blocked sources are counted as blocked even when the archive-level inspection itself -completed successfully. +completed successfully. A `source_identity_changed` result means the current +series or issue identity disagrees with the reviewed match, not that the file +was necessarily modified on disk. Its message directs the user to Follow-up; +it is neither automatically retryable nor overrideable. Completed-import source rechecks inspect one bounded page before writing its refreshed evidence, then commit that page before reading more archives. This keeps slow archive I/O outside SQLite's single-writer window, bounds memory, and diff --git a/src/pullbox/services/import_completed_cleanup.py b/src/pullbox/services/import_completed_cleanup.py index ed444d36..b8e37617 100644 --- a/src/pullbox/services/import_completed_cleanup.py +++ b/src/pullbox/services/import_completed_cleanup.py @@ -34,7 +34,10 @@ from pullbox.models.series import Series from pullbox.services.audit_service import AuditService from pullbox.services.import_counters import recompute_file_counters, recompute_series_counters -from pullbox.services.import_deferred_recovery import load_empty_stale_series +from pullbox.services.import_deferred_recovery import ( + load_empty_stale_series, + refresh_recovered_groups, +) from pullbox.services.import_known_series_recovery import load_known_series_recovery from pullbox.services.import_review_actions import apply_safety_allow_once_to_file from pullbox.services.import_review_recheck import retryable_failed_source_filters @@ -289,12 +292,23 @@ def _file_filters(job_id: int, action: CompletedImportCleanupAction) -> tuple[An ) ) elif action is CompletedImportCleanupAction.ALLOW_OVERSIZED_FILES: - filters.extend( - [ - ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED, - _safety_filter(ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT), - _overrideable_expression().is_(True), - ] + filters.append( + or_( + and_( + ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED, + _safety_filter(ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT), + _overrideable_expression().is_(True), + ), + and_( + ImportedFile.status == ImportedFileStatus.FAILED, + ImportedFile.diagnostics["safety_block"].as_string().is_(None), + _source_revalidation_category_expression() + == ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT.value, + ImportedFile.diagnostics["source_revalidation"]["overrideable"] + .as_boolean() + .is_(True), + ), + ) ) elif action is CompletedImportCleanupAction.RETRY_SOURCE_INSPECTION: retryable_categories = [ @@ -1520,7 +1534,8 @@ async def _prepare_series_for_retry( async def _apply_known_series_recovery(session: AsyncSession, job: ImportJob) -> set[int]: plans = await load_known_series_recovery(session, job.id) - affected: set[int] = set() + sources: set[int] = set() + targets: dict[int, ImportedSeries] = {} for batch in batched(plans, 400): items = { item.id: item @@ -1547,38 +1562,53 @@ async def _apply_known_series_recovery(session: AsyncSession, job: ImportJob) -> file = files.get(plan.file_id) if item is None or file is None: raise ValidationError("Recovery evidence disappeared. Preview the action again.") - _apply_known_series_file(item, file, plan, affected) + sources.add(item.id) + target = targets.get(item.id) + if target is None: + candidate = dict(item.diagnostics or {}).get("selected_candidate") + candidate = candidate if isinstance(candidate, dict) else {} + target = ImportedSeries( + import_job_id=job.id, + raw_series_name=item.raw_series_name, + raw_year=item.raw_year, + raw_publisher=item.raw_publisher, + source_folder=item.source_folder, + cv_id=plan.cv_id, + cv_title=item.raw_series_name, + cv_year=item.raw_year, + cv_issue_count=candidate.get("issue_count"), + cv_match_method=plan.match_method, + cv_match_score=1.0, + status=ImportSeriesStatus.CONFIRMED, + selected_for_import=True, + has_files=True, + diagnostics={ + "kind": "known_series_recovery", + "source_import_series_id": item.id, + "source_preserved": True, + "file_identity_review_required": True, + }, + ) + session.add(target) + await session.flush() + targets[item.id] = target + file.import_series_id = target.id + _apply_known_series_file(file, plan) await refresh_story_arc_entries_for_import_files( session, import_job_id=job.id, import_file_ids=list(files), ) await session.flush() - return affected + target_ids = {item.id for item in targets.values()} + await refresh_recovered_groups(session, job, sources | target_ids) + return target_ids def _apply_known_series_file( - item: ImportedSeries, file: ImportedFile, plan: KnownSeriesRecovery, - affected: set[int], ) -> None: - if item.id not in affected: - candidate = dict(item.diagnostics or {}).get("selected_candidate") - candidate = candidate if isinstance(candidate, dict) else {} - item.cv_id = plan.cv_id - item.cv_match_method = plan.match_method - item.cv_match_score = 1.0 - item.cv_title = item.raw_series_name - item.cv_year = item.raw_year - item.cv_issue_count = candidate.get("issue_count") - item.diagnostics = { - **dict(item.diagnostics or {}), - "previous_reason": "trusted_source_identity_conflict", - "reason": "known_series_recovered", - "file_identity_review_required": True, - } - affected.add(item.id) file.status = ImportedFileStatus.CONFIRMED file.include_in_import = True file.matched_issue_cv_id = int(plan.summary["provider_id"]) @@ -1590,6 +1620,7 @@ def _apply_known_series_file( "target_issue_summary": plan.summary, "completed_import_cleanup": { "action": CompletedImportCleanupAction.RECOVER_KNOWN_SERIES.value, + "source_import_series_id": plan.series_id, "evidence_digest": plan.evidence_digest, "source_preserved": True, "resolved_at": datetime.now(UTC).isoformat(), @@ -1644,9 +1675,26 @@ async def apply_completed_import_cleanup( affected_file_ids: tuple[int, ...] = () requires_import_retry = True elif action is CompletedImportCleanupAction.RECOVER_KNOWN_SERIES: + from pullbox.services.import_retry_helpers import require_retained_import_destination + + require_retained_import_destination(job) affected_series_ids = await _apply_known_series_recovery(session, job) affected_file_ids = () requires_import_retry = await _prepare_series_for_retry(session, job, affected_series_ids) + job.progress_snapshot = { + **dict(job.progress_snapshot or {}), + "deferred_recovery": { + "state": "prepared", + "run_id": uuid4().hex, + "series_ids": sorted(affected_series_ids), + "actor_id": actor_id, + "action": action.value, + }, + "mode": "import", + "phase": "deferred_recovery", + "progress": 0, + "message": "Queued recovery of verified files...", + } elif action is CompletedImportCleanupAction.ACCEPT_RECOMMENDED_CONFLICTS: affected_series_ids = await _apply_recommended_conflicts(session, job) affected_file_ids = () diff --git a/src/pullbox/services/import_known_series_recovery.py b/src/pullbox/services/import_known_series_recovery.py index d077924d..f81188e0 100644 --- a/src/pullbox/services/import_known_series_recovery.py +++ b/src/pullbox/services/import_known_series_recovery.py @@ -3,13 +3,13 @@ from __future__ import annotations import json -from collections import Counter +from collections import Counter, defaultdict from dataclasses import asdict, dataclass from hashlib import sha256 from itertools import batched from typing import TYPE_CHECKING, Any -from sqlalchemy import exists, select +from sqlalchemy import exists, or_, select from pullbox.core.issue_numbers import parse_issue_number_text from pullbox.core.name_matcher import NameMatcher @@ -26,6 +26,7 @@ from pullbox.models.series import Series from pullbox.providers.base import IssueSummary from pullbox.services.import_file_match_targets import trusted_source_issue_identity_matches_target +from pullbox.services.import_file_selection import not_excluded_from_review from pullbox.services.import_source_metadata import ( build_import_metadata_conflict, source_metadata_for_import_file, @@ -118,6 +119,8 @@ def _file_plan( }: return None diagnostics = dict(file.diagnostics or {}) + if diagnostics.get("review_selection") is False: + return None if diagnostics.get("kind") in { "metadata_conflict", "source_scope_review", @@ -129,7 +132,8 @@ def _file_plan( if (file.match_method or "").startswith(("manual", "orphan_recovery")): return None if any( - diagnostics.get(key) for key in ("safety_block", "source_revalidation", "safety_exception") + diagnostics.get(key) + for key in ("safety_block", "source_revalidation", "safety_exception", "safety_review") ): return None if ( @@ -218,7 +222,6 @@ async def load_known_series_recovery( return () plans: list[KnownSeriesRecovery] = [] matched_local_ids: dict[int, int | None] = {} - ready_files_by_series: dict[int, set[int]] = {} cursor = 0 while True: items = list( @@ -257,11 +260,6 @@ async def load_known_series_recovery( identity = _known_identity(item, files, job.source_type) if identity is None: continue - ready_files_by_series[item.id] = { - file.id - for file in files - if file.status in {ImportedFileStatus.MATCHED, ImportedFileStatus.CONFIRMED} - } for file in files: plan = _file_plan(item, file, *identity) if plan is not None: @@ -269,7 +267,34 @@ async def load_known_series_recovery( matched_local_ids[file.id] = file.matched_issue_id issue_ids = sorted({int(plan.summary["provider_id"]) for plan in plans}) local_targets = {} - for ids in batched(issue_ids, 400): + ready_claims: dict[int, set[int]] = defaultdict(set) + for ids in batched(issue_ids, 300): + for file_id, source_cv_id, target_cv_id, local_cv_id in ( + await session.execute( + select( + ImportedFile.id, + ImportedFile.comicvine_issue_id, + ImportedFile.matched_issue_cv_id, + Issue.comicvine_id, + ) + .outerjoin(Issue, Issue.id == ImportedFile.matched_issue_id) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.status.in_( + (ImportedFileStatus.MATCHED, ImportedFileStatus.CONFIRMED) + ), + not_excluded_from_review(), + or_( + ImportedFile.comicvine_issue_id.in_(ids), + ImportedFile.matched_issue_cv_id.in_(ids), + Issue.comicvine_id.in_(ids), + ), + ) + ) + ).all(): + for claimed_id in (source_cv_id, target_cv_id, local_cv_id): + if claimed_id is not None: + ready_claims[claimed_id].add(file_id) for issue, series_cv_id, owned in ( await session.execute( select(Issue, Series.comicvine_id, exists().where(LibraryFile.issue_id == Issue.id)) @@ -282,7 +307,8 @@ async def load_known_series_recovery( counts = Counter(int(plan.summary["provider_id"]) for plan in plans) result = [] for plan in plans: - if counts[int(plan.summary["provider_id"])] != 1: + issue_cv_id = int(plan.summary["provider_id"]) + if counts[issue_cv_id] != 1 or ready_claims[issue_cv_id] - {plan.file_id}: continue local = local_targets.get(int(plan.summary["provider_id"])) if local is not None: @@ -297,12 +323,6 @@ async def load_known_series_recovery( elif matched_local_ids[plan.file_id] is not None: continue result.append(plan) - eligible_ids = {plan.file_id for plan in result} - # Step 4 consumes all ready files in a series. Never revive a parent if that - # would implicitly authorize an unpreviewed ready file or manual decision. - return tuple( - sorted( - (plan for plan in result if ready_files_by_series[plan.series_id] <= eligible_ids), - key=lambda plan: plan.file_id, - ) - ) + # Apply isolates these files into new scoped groups. Unpreviewed siblings + # stay in their original parent and are never authorized by this recovery. + return tuple(sorted(result, key=lambda plan: plan.file_id)) diff --git a/src/pullbox/services/import_review_actions.py b/src/pullbox/services/import_review_actions.py index 2f87d0e3..66068ead 100644 --- a/src/pullbox/services/import_review_actions.py +++ b/src/pullbox/services/import_review_actions.py @@ -20,7 +20,10 @@ ) from pullbox.services.import_duplicates import duplicate_merge_is_actionable, is_duplicate_series from pullbox.services.import_file_selection import set_review_file_selection -from pullbox.services.import_safety_diagnostics import normalize_import_safety_diagnostics +from pullbox.services.import_safety_diagnostics import ( + ImportSafetyCategory, + normalize_import_safety_diagnostics, +) from pullbox.services.import_story_arc_resolution import ( refresh_story_arc_entries_for_import_files, ) @@ -59,6 +62,14 @@ def apply_safety_allow_once_to_file( """Apply the canonical one-job safety exception payload to one staged file.""" diagnostics = dict(imp_file.diagnostics or {}) previous_block = diagnostics.pop("safety_block", None) + if previous_block is None and retry_import and imp_file.status is ImportedFileStatus.FAILED: + revalidation = diagnostics.get("source_revalidation") + if ( + isinstance(revalidation, Mapping) + and revalidation.get("category") == ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT + and revalidation.get("overrideable") is True + ): + previous_block = revalidation if not isinstance(previous_block, Mapping): raise ValidationError("This safety block cannot be overridden.") normalized_previous_block = normalize_import_safety_diagnostics(previous_block) diff --git a/src/pullbox/services/import_safety_diagnostics.py b/src/pullbox/services/import_safety_diagnostics.py index 1bbb100e..fc263ca8 100644 --- a/src/pullbox/services/import_safety_diagnostics.py +++ b/src/pullbox/services/import_safety_diagnostics.py @@ -327,8 +327,13 @@ def classify_import_safety_failure( return ImportSafetyClassification( category=category, code=stable_code, - sanitized_reason=_SANITIZED_REASONS[category], - retryable=category in _RETRYABLE_CATEGORIES, + sanitized_reason=( + "The file's series or issue identity disagrees with the saved import match. " + "Review the file and its match in Follow-up before retrying." + if stable_code == "source_identity_changed" + else _SANITIZED_REASONS[category] + ), + retryable=category in _RETRYABLE_CATEGORIES and stable_code != "source_identity_changed", overrideable=overrideable, ) diff --git a/tests/unit/test_import_completed_cleanup.py b/tests/unit/test_import_completed_cleanup.py index 1eff7fa0..a742440b 100644 --- a/tests/unit/test_import_completed_cleanup.py +++ b/tests/unit/test_import_completed_cleanup.py @@ -2,6 +2,7 @@ from __future__ import annotations +from copy import deepcopy from datetime import UTC, datetime from typing import TYPE_CHECKING @@ -318,6 +319,105 @@ async def test_allow_oversized_files_requeues_only_overrideable_rows( assert not_approved.status is ImportedFileStatus.SAFETY_BLOCKED +@pytest.mark.parametrize("source_type", list(ImportSourceType)) +async def test_allow_oversized_files_includes_failed_source_rechecks( + db_session: AsyncSession, source_type: ImportSourceType +) -> None: + job, imported_series = await _seed_job(db_session) + job.source_type = source_type + failed = _blocked_file( + job, + imported_series, + name="large-rechecked.cbz", + category=ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT, + overrideable=True, + ) + evidence = { + **failed.diagnostics["safety_block"], + "kind": "source_revalidation", + "source": "completed_import_recheck", + "code": "archive_decompressed_size_limit", + "retryable": False, + } + failed.status = ImportedFileStatus.FAILED + failed.diagnostics = {"source_revalidation": evidence} + db_session.add(failed) + await db_session.commit() + original_path = failed.file_path + action = CompletedImportCleanupAction.ALLOW_OVERSIZED_FILES + + summary = await summarize_completed_import_cleanup_scope(db_session, job.id, action) + assert summary.affected_file_count == 1 + preview = await preview_completed_import_cleanup(db_session, job.id, action, actor_id=42) + + assert preview.affected_file_count == 1 + assert failed.status is ImportedFileStatus.FAILED + assert failed.diagnostics == {"source_revalidation": evidence} + page = await list_completed_import_cleanup_files(db_session, job.id, action) + assert [item.id for item in page.items] == [failed.id] + result = await apply_completed_import_cleanup( + db_session, + job.id, + action, + actor_id=42, + preview_token=preview.preview_token, + ) + assert result.requires_import_retry is True + assert result.affected_file_count == 1 + assert failed.status is ImportedFileStatus.CONFIRMED + assert failed.include_in_import is True + assert failed.file_path == original_path + assert failed.diagnostics["safety_exception"]["allowed_once"] is True + assert failed.diagnostics["safety_exception"]["previous_block"]["overrideable"] is True + assert failed.diagnostics["source_revalidation"] == evidence + + +@pytest.mark.parametrize( + "protection", ["dangerous", "identity", "not_overrideable", "active_block", "skipped"] +) +async def test_failed_size_approval_does_not_admit_other_blocked_files( + db_session: AsyncSession, protection: str +) -> None: + job, imported_series = await _seed_job(db_session) + failed = _blocked_file( + job, + imported_series, + name="protected.cbz", + category=ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT, + overrideable=True, + ) + evidence = dict(failed.diagnostics["safety_block"]) + failed.status = ImportedFileStatus.FAILED + failed.diagnostics = {"source_revalidation": evidence} + if protection == "dangerous": + evidence["category"] = ImportSafetyCategory.DANGEROUS_PATH_OR_PAYLOAD.value + evidence["code"] = "dangerous_archive_path" + elif protection == "identity": + evidence["category"] = ImportSafetyCategory.SOURCE_CHANGED.value + evidence["code"] = "source_identity_changed" + elif protection == "not_overrideable": + evidence["overrideable"] = False + elif protection == "active_block": + failed.diagnostics["safety_block"] = build_import_safety_diagnostics( + "dangerous_archive_path", + code="dangerous_archive_path", + ) + else: + failed.status = ImportedFileStatus.SKIPPED + db_session.add(failed) + await db_session.commit() + before = deepcopy(failed.diagnostics) + + summary = await summarize_completed_import_cleanup_scope( + db_session, + job.id, + CompletedImportCleanupAction.ALLOW_OVERSIZED_FILES, + ) + + assert summary.affected_file_count == 0 + assert failed.diagnostics == before + + @pytest.mark.asyncio async def test_accept_recommended_conflicts_requires_one_high_confidence_choice( db_session: AsyncSession, diff --git a/tests/unit/test_import_known_series_recovery.py b/tests/unit/test_import_known_series_recovery.py index 6aa6d880..630defd4 100644 --- a/tests/unit/test_import_known_series_recovery.py +++ b/tests/unit/test_import_known_series_recovery.py @@ -2,6 +2,7 @@ from copy import deepcopy from datetime import UTC, datetime +from unittest.mock import AsyncMock import pytest @@ -10,6 +11,7 @@ ImportedFile, ImportedFileStatus, ImportedSeries, + ImportFileHandlingMode, ImportJob, ImportJobStatus, ImportSeriesStatus, @@ -22,6 +24,7 @@ preview_completed_import_cleanup, ) from pullbox.services.import_known_series_recovery import load_known_series_recovery +from pullbox.services.import_workflow_state import deferred_recovery_scope async def seed_recovery(session, *, source_type=ImportSourceType.MYLAR3, method="mylar3_cv_id"): @@ -104,7 +107,17 @@ async def test_known_series_recovery_keeps_good_files_and_does_not_mutate_previe @pytest.mark.parametrize( "protection", - ["skip", "imported", "safety", "identity", "manual", "wrong_series", "wrong_issue"], + [ + "skip", + "imported", + "safety", + "safety_review", + "excluded", + "identity", + "manual", + "wrong_series", + "wrong_issue", + ], ) async def test_known_series_recovery_preserves_protected_or_conflicting_files( db_session, protection @@ -119,6 +132,10 @@ async def test_known_series_recovery_preserves_protected_or_conflicting_files( **file.diagnostics, "safety_block": {"category": "dangerous_path_or_payload"}, } + elif protection == "safety_review": + file.diagnostics = {**file.diagnostics, "safety_review": {"action": "allow_once"}} + elif protection == "excluded": + file.diagnostics = {**file.diagnostics, "review_selection": False} elif protection == "identity": file.diagnostics = { **file.diagnostics, @@ -254,8 +271,13 @@ async def test_known_series_action_queues_only_previewed_files_and_preserves_con ) assert result.requires_import_retry assert job.status is ImportJobStatus.IMPORTING - assert series.cv_id == 796 - assert series.status is ImportSeriesStatus.CONFIRMED + target = await db_session.get(ImportedSeries, file.import_series_id) + assert target.id != series.id + assert target.cv_id == 796 + assert target.status is ImportSeriesStatus.CONFIRMED + assert deferred_recovery_scope(job) == (target.id,) + assert series.cv_id is None + assert series.status is ImportSeriesStatus.NO_MATCH assert file.status is ImportedFileStatus.CONFIRMED assert conflict.status is ImportedFileStatus.NO_MATCH assert not conflict.include_in_import @@ -333,10 +355,301 @@ async def test_known_series_recovery_never_implicitly_imports_unpreviewed_ready_ ) ) await db_session.commit() - assert await load_known_series_recovery(db_session, job.id) == () + plans = await load_known_series_recovery(db_session, job.id) + assert [plan.file_id for plan in plans] == [file.id] assert file.status is ImportedFileStatus.MATCHED +@pytest.mark.parametrize( + "source_type,method", + [(ImportSourceType.MYLAR3, "mylar3_cv_id"), (ImportSourceType.FILESYSTEM, "comicinfo_cv_id")], +) +async def test_known_series_recovery_isolates_proven_files_from_unresolved_siblings( + db_session, source_type, method +): + job, parent, good = await seed_recovery(db_session, source_type=source_type, method=method) + db_session.add(User(id=42, username="operator", password_hash="unused")) + job.file_handling_mode = ImportFileHandlingMode.IN_PLACE + job.move_to_library = False + job.source_preserved = True + parent.source_folder = "/comics/Batman" + unresolved = ImportedFile( + import_job_id=job.id, + import_series_id=parent.id, + file_path="/comics/Batman/002.cbz", + file_name="Batman 002.cbz", + file_size=1024, + file_format="cbz", + status=ImportedFileStatus.CONFIRMED, + include_in_import=True, + matched_issue_cv_id=999, + match_method="manual", + diagnostics={"kind": "metadata_conflict"}, + ) + unrelated = ImportedSeries( + import_job_id=job.id, + raw_series_name="Unrelated ready series", + status=ImportSeriesStatus.CONFIRMED, + selected_for_import=True, + cv_id=999, + ) + db_session.add_all([unresolved, unrelated]) + await db_session.commit() + before = deepcopy(unresolved.diagnostics) + action = CompletedImportCleanupAction.RECOVER_KNOWN_SERIES + + plans = await load_known_series_recovery(db_session, job.id) + assert [plan.file_id for plan in plans] == [good.id] + assert good.import_series_id == parent.id + assert not db_session.dirty + preview = await preview_completed_import_cleanup(db_session, job.id, action, actor_id=42) + await apply_completed_import_cleanup( + db_session, + job.id, + action, + actor_id=42, + preview_token=preview.preview_token, + ) + + assert good.import_series_id != parent.id + target = await db_session.get(ImportedSeries, good.import_series_id) + assert target.cv_id == 796 + assert target.status is ImportSeriesStatus.CONFIRMED + assert target.selected_for_import + assert target.files_total == 1 + assert target.source_folder == parent.source_folder + assert target.diagnostics["source_import_series_id"] == parent.id + assert good.diagnostics["completed_import_cleanup"]["source_import_series_id"] == parent.id + assert good.file_path == "/comics/Batman/001.cbz" + assert parent.status is ImportSeriesStatus.NO_MATCH + assert parent.cv_id is None + assert parent.files_total == 1 + assert unresolved.import_series_id == parent.id + assert unresolved.status is ImportedFileStatus.CONFIRMED + assert unresolved.include_in_import + assert unresolved.diagnostics == before + assert deferred_recovery_scope(job) == (target.id,) + assert unrelated.id not in deferred_recovery_scope(job) + assert job.file_handling_mode is ImportFileHandlingMode.IN_PLACE + assert not job.move_to_library + assert job.source_preserved + + +@pytest.mark.parametrize("claim", ["source", "target", "local"]) +async def test_known_series_recovery_does_not_compete_with_a_manual_keeper(db_session, claim): + job, parent, good = await seed_recovery(db_session) + keeper = ImportedFile( + import_job_id=job.id, + import_series_id=parent.id, + file_path="/comics/Batman/manual-keeper.cbz", + file_name="manual-keeper.cbz", + file_size=1024, + file_format="cbz", + status=ImportedFileStatus.CONFIRMED, + include_in_import=True, + matched_issue_cv_id=good.matched_issue_cv_id, + match_method="manual", + diagnostics={"review_selection": True}, + ) + if claim == "source": + keeper.comicvine_issue_id = good.matched_issue_cv_id + keeper.matched_issue_cv_id = None + elif claim == "local": + from pullbox.models.issue import Issue + from pullbox.models.series import Series + + catalog = Series(title="Batman", sort_title="batman", comicvine_id=796) + db_session.add(catalog) + await db_session.flush() + issue = Issue(series_id=catalog.id, comicvine_id=1001, issue_number=1) + db_session.add(issue) + await db_session.flush() + keeper.matched_issue_id = issue.id + keeper.matched_issue_cv_id = None + db_session.add(keeper) + await db_session.commit() + + assert await load_known_series_recovery(db_session, job.id) == () + + +async def test_known_series_recovery_revalidates_new_manual_claim_after_preview(db_session): + job, parent, good = await seed_recovery(db_session) + action = CompletedImportCleanupAction.RECOVER_KNOWN_SERIES + preview = await preview_completed_import_cleanup(db_session, job.id, action, actor_id=42) + other = ImportedSeries( + import_job_id=job.id, + raw_series_name="Another review group", + status=ImportSeriesStatus.CONFIRMED, + ) + db_session.add(other) + await db_session.flush() + db_session.add( + ImportedFile( + import_job_id=job.id, + import_series_id=other.id, + file_path="/comics/manual-keeper.cbz", + file_name="manual-keeper.cbz", + file_format="cbz", + status=ImportedFileStatus.CONFIRMED, + matched_issue_cv_id=good.matched_issue_cv_id, + match_method="manual", + include_in_import=True, + ) + ) + await db_session.commit() + + with pytest.raises(ValidationError, match="scope changed"): + await apply_completed_import_cleanup( + db_session, + job.id, + action, + actor_id=42, + preview_token=preview.preview_token, + ) + assert good.import_series_id == parent.id + assert job.status is ImportJobStatus.COMPLETED + + +async def test_known_series_recovery_keeps_one_group_across_apply_batches(db_session): + from sqlalchemy import select + + job, parent, first = await seed_recovery(db_session) + db_session.add(User(id=42, username="operator", password_hash="unused")) + for number in range(2, 403): + diagnostics = deepcopy(first.diagnostics) + diagnostics["source_metadata"]["comicinfo"]["number"] = str(number) + diagnostics["target_issue_summary"].update( + provider_id=str(1000 + number), + issue_number=number, + issue_number_text=str(number), + ) + db_session.add( + ImportedFile( + import_job_id=job.id, + import_series_id=parent.id, + file_path=f"/comics/Batman/{number}.cbz", + file_name=f"Batman {number}.cbz", + file_format="cbz", + file_size=1024, + status=ImportedFileStatus.MATCHED, + comicvine_issue_id=1000 + number, + matched_issue_cv_id=1000 + number, + parsed_series="Batman", + parsed_issue_number=number, + diagnostics=diagnostics, + ) + ) + await db_session.commit() + action = CompletedImportCleanupAction.RECOVER_KNOWN_SERIES + preview = await preview_completed_import_cleanup(db_session, job.id, action, actor_id=42) + assert preview.affected_file_count == 402 + + await apply_completed_import_cleanup( + db_session, + job.id, + action, + actor_id=42, + preview_token=preview.preview_token, + ) + + scope = deferred_recovery_scope(job) + assert scope is not None and len(scope) == 1 + target = await db_session.get(ImportedSeries, scope[0]) + assert target.files_total == 402 + assert target.files_matched == 402 + assert parent.status is ImportSeriesStatus.SKIPPED + assert parent.files_total == 0 + assert set(await db_session.scalars(select(ImportedFile.import_series_id))) == {target.id} + + +@pytest.mark.parametrize("changed_source", [False, True]) +async def test_known_series_recovery_runs_scoped_import_and_checks_source_signature( + db_session, tmp_path, monkeypatch, changed_source +): + from pullbox.core.library_file_ownership import build_file_identity_signature + from pullbox.models.issue import Issue + from pullbox.models.library import LibraryFileStorageMode, LibraryRoot + from pullbox.models.series import Series + from pullbox.services import import_job_execution as execution + from pullbox.services.import_referenced_sources import MYLAR_REFERENCE_ROOT_ID_SIGNATURE_KEY + from pullbox.services.import_service import ImportService + from scripts.mylar3_import_fixture import create_minimal_cbz + from tests.unit.test_import_file_execution import _mock_register_library_file + + job, parent, good = await seed_recovery(db_session) + root = LibraryRoot(name="Original library", path=str(tmp_path), enabled=True) + catalog = Series(title="Batman", sort_title="batman", year_start=1940, comicvine_id=796) + db_session.add_all([root, catalog, User(id=42, username="operator", password_hash="unused")]) + await db_session.flush() + db_session.add(Issue(series_id=catalog.id, comicvine_id=1001, issue_number=1)) + comic = tmp_path / "Batman" / "Batman 001.cbz" + create_minimal_cbz(comic) + signature = build_file_identity_signature(comic) + signature[MYLAR_REFERENCE_ROOT_ID_SIGNATURE_KEY] = root.id + good.file_path, good.file_size, good.source_signature = ( + str(comic), + comic.stat().st_size, + signature, + ) + parent.source_folder = str(comic.parent) + job.file_handling_mode = ImportFileHandlingMode.IN_PLACE + job.move_to_library = False + job.source_preserved = True + job.effective_transfer_method = "leave_in_place" + unrelated = ImportedSeries( + import_job_id=job.id, + raw_series_name="Unrelated", + cv_id=999, + status=ImportSeriesStatus.CONFIRMED, + selected_for_import=True, + ) + db_session.add(unrelated) + await db_session.commit() + action = CompletedImportCleanupAction.RECOVER_KNOWN_SERIES + preview = await preview_completed_import_cleanup(db_session, job.id, action, actor_id=42) + await apply_completed_import_cleanup( + db_session, + job.id, + action, + actor_id=42, + preview_token=preview.preview_token, + ) + await db_session.commit() + if changed_source: + comic.write_bytes(b"changed after the recovery preview") + before = comic.read_bytes(), comic.stat().st_mtime_ns + register = _mock_register_library_file() + monkeypatch.setattr("pullbox.services.import_service.register_library_file", register) + arcs = AsyncMock(side_effect=AssertionError("Unrelated Story Arcs must not execute")) + monkeypatch.setattr(execution, "_execute_story_arc_materialization", arcs) + series_service = AsyncMock() + series_service.add_from_comicvine.return_value = catalog + service = ImportService( + series_service=series_service, metadata_service=AsyncMock(), event_bus=AsyncMock() + ) + + await service.run_import(db_session, job.id) + + await db_session.refresh(good) + await db_session.refresh(unrelated) + assert job.status is ImportJobStatus.COMPLETED + assert job.progress_snapshot["deferred_recovery"]["state"] == "completed" + assert unrelated.status is ImportSeriesStatus.CONFIRMED + arcs.assert_not_awaited() + assert (comic.read_bytes(), comic.stat().st_mtime_ns) == before + if changed_source: + register.assert_not_awaited() + assert good.status is ImportedFileStatus.FAILED + assert good.diagnostics["source_revalidation"]["code"] == "source_changed" + else: + register.assert_awaited_once() + assert good.status is ImportedFileStatus.IMPORTED + assert register.await_args.kwargs["storage_mode"] is LibraryFileStorageMode.REFERENCED + assert register.await_args.kwargs["move_to_library"] is False + assert register.await_args.kwargs["library_root_id"] == root.id + assert register.await_args.kwargs["expected_source_signature"] == signature + + @pytest.mark.parametrize("local_state", ["different_series", "owned", "different_number"]) async def test_known_series_recovery_respects_current_catalog_and_ownership( db_session, local_state diff --git a/tests/unit/test_import_safety_diagnostics.py b/tests/unit/test_import_safety_diagnostics.py index 48b0e678..6f67743a 100644 --- a/tests/unit/test_import_safety_diagnostics.py +++ b/tests/unit/test_import_safety_diagnostics.py @@ -10,6 +10,7 @@ ImportSafetyCategory, build_import_safety_diagnostics, classify_import_safety_failure, + normalize_import_safety_diagnostics, summarize_import_safety_failures, ) @@ -17,6 +18,32 @@ from collections.abc import Mapping +def test_identity_disagreement_is_not_reported_as_physical_source_change() -> None: + result = build_import_safety_diagnostics( + "The current source identity conflicts with /private/library/file.cbz", + kind="source_revalidation", + code="source_identity_changed", + overrideable_hint=True, + ) + + assert "identity" in result["reason"] + assert "Follow-up" in result["reason"] + assert "source changed" not in result["reason"].lower() + assert "/private" not in result["reason"] + assert result["retryable"] is False + assert result["overrideable"] is False + legacy = normalize_import_safety_diagnostics( + { + "category": "source_changed", + "code": "source_identity_changed", + "reason": "The source changed or became unavailable after scanning.", + "retryable": True, + } + ) + assert legacy["reason"] == result["reason"] + assert legacy["retryable"] is False + + @pytest.mark.parametrize( ( "reason", From fd158e55df0b2bcdca23f1064729cf8faf6a46dc Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Wed, 16 Sep 2026 23:56:39 -0700 Subject: [PATCH 02/20] fix(scheduler): bound exclusive admission and retry busy maintenance Release reservations on cancellation, reject duplicate waiting runs, and keep deferred maintenance retries separate from recurring jobs. Keep database persistence outside the admission lock. TDD: reproduced admission and lost-retry failures; 78 focused tests and 7856 unit tests pass. --- src/pullbox/core/scheduler.py | 162 ++++++++++++------ src/pullbox/core/scheduler_error_helpers.py | 6 +- .../test_scheduler_exclusive_admission.py | 140 +++++++++++++++ 3 files changed, 249 insertions(+), 59 deletions(-) create mode 100644 tests/unit/test_scheduler_exclusive_admission.py diff --git a/src/pullbox/core/scheduler.py b/src/pullbox/core/scheduler.py index 2fc7d007..fbe82766 100644 --- a/src/pullbox/core/scheduler.py +++ b/src/pullbox/core/scheduler.py @@ -10,9 +10,9 @@ import asyncio import time from collections import deque -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from functools import wraps -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal import structlog from apscheduler.events import ( # type: ignore[import-untyped] @@ -102,6 +102,8 @@ _MANUAL_QUEUE_DEFERRED_TASK_IDS = {"run_health_checks"} _HOT_TASK_INTERVAL_SECONDS = 300 _HOT_TASK_PERSIST_WINDOW_SECONDS = 300.0 +_EXCLUSIVE_WAIT_SECONDS = 5.0 +_EXCLUSIVE_RETRY_SECONDS = 60 # ── Scheduler ───────────────────────────────────────────────── @@ -382,7 +384,11 @@ def delay_task_next_run(self, task_id: str, *, run_at: datetime) -> bool: def _visible_jobs(self) -> list[Any]: """Return user-facing jobs without internal continuation plumbing.""" - return [job for job in self._scheduler.get_jobs() if not job.id.endswith("__continuation")] + return [ + job + for job in self._scheduler.get_jobs() + if not job.id.endswith(("__continuation", "__exclusive_retry")) + ] @staticmethod def _continuation_job_id(task_id: str) -> str: @@ -405,7 +411,6 @@ def _wrap_task( @wraps(func) async def wrapper() -> None: log = logger.bind(task_id=task_id) - reserved_exclusive = False if trigger_type in {"scheduled", "manual"} and await scheduler._defer_task_for_import( task_id, log, @@ -413,52 +418,20 @@ async def wrapper() -> None: ): return - async with scheduler._execution_admission_lock: - active_exclusive = scheduler._exclusive_active_task_id - if active_exclusive is not None and active_exclusive != task_id: - stats = scheduler._task_stats.setdefault(task_id, TaskStats()) - stats.last_exclusive_block_at = datetime.now(UTC).isoformat() - stats.exclusive_block_count += 1 - await scheduler._persist_task_stat( - task_id, - stats, - trigger_type=trigger_type, - reason="exclusive_block", - ) - log.debug( - "task_skipped_exclusive", - trigger_type=trigger_type, - exclusive_task_id=active_exclusive, - ) - return - - if scheduler._running_counts.get(task_id, 0) > 0: - stats = scheduler._task_stats.setdefault(task_id, TaskStats()) - stats.last_overlap_at = datetime.now(UTC).isoformat() - stats.overlap_count += 1 - await scheduler._persist_task_stat( - task_id, - stats, - trigger_type=trigger_type, - reason="overlap", - ) - scheduler._log_task_overlap(task_id, trigger_type=trigger_type) - return - - if exclusive: - scheduler._exclusive_active_task_id = task_id - reserved_exclusive = True - - if exclusive: - while any( - running_task_id != task_id and count > 0 - for running_task_id, count in scheduler._running_counts.items() - ): - await asyncio.sleep(0.05) + reservation = await scheduler._reserve_execution( + task_id, log, trigger_type=trigger_type, exclusive=exclusive + ) + if reservation != "reserved": + if exclusive and reservation == "exclusive_block": + scheduler._schedule_exclusive_retry(task_id, wrapper) + return + if exclusive and not await scheduler._wait_for_exclusive_execution( + task_id, wrapper, log, trigger_type=trigger_type + ): + return log.debug("task_started", trigger_type=trigger_type) start = time.monotonic() - scheduler._running_counts[task_id] = scheduler._running_counts.get(task_id, 0) + 1 run_context = bind_scheduler_run_context( task_id=task_id, trigger_type=trigger_type, @@ -540,18 +513,95 @@ async def wrapper() -> None: ) finally: reset_scheduler_run_context(run_context) - current = scheduler._running_counts.get(task_id, 0) - if current <= 1: - scheduler._running_counts.pop(task_id, None) - else: - scheduler._running_counts[task_id] = current - 1 - if reserved_exclusive: - async with scheduler._execution_admission_lock: - if scheduler._exclusive_active_task_id == task_id: - scheduler._exclusive_active_task_id = None + scheduler._release_execution(task_id) return wrapper + async def _reserve_execution( + self, task_id: str, log: Any, *, trigger_type: str, exclusive: bool + ) -> Literal["reserved", "exclusive_block", "overlap"]: + """Reserve atomically, without holding admission behind database writes.""" + reason: Literal["exclusive_block", "overlap"] + async with self._execution_admission_lock: + active_exclusive = self._exclusive_active_task_id + stats = self._task_stats.setdefault(task_id, TaskStats()) + if active_exclusive is not None and active_exclusive != task_id: + stats.last_exclusive_block_at = datetime.now(UTC).isoformat() + stats.exclusive_block_count += 1 + reason = "exclusive_block" + elif self._running_counts.get(task_id, 0) > 0: + stats.last_overlap_at = datetime.now(UTC).isoformat() + stats.overlap_count += 1 + reason = "overlap" + else: + self._running_counts[task_id] = 1 + if exclusive: + self._exclusive_active_task_id = task_id + return "reserved" + + await self._persist_task_stat(task_id, stats, trigger_type=trigger_type, reason=reason) + if reason == "overlap": + self._log_task_overlap(task_id, trigger_type=trigger_type) + else: + log.debug( + "task_skipped_exclusive", + trigger_type=trigger_type, + exclusive_task_id=active_exclusive, + ) + return reason + + def _release_execution(self, task_id: str) -> None: + """Release in-memory admission synchronously, including during cancellation.""" + self._running_counts.pop(task_id, None) + if self._exclusive_active_task_id == task_id: + self._exclusive_active_task_id = None + + async def _wait_for_exclusive_execution( + self, task_id: str, wrapped: Callable[[], Any], log: Any, *, trigger_type: str + ) -> bool: + """Bound the drain window; a busy application gets a later maintenance retry.""" + retry_id = f"{task_id}__exclusive_retry" + try: + async with asyncio.timeout(_EXCLUSIVE_WAIT_SECONDS): + while any( + other_id != task_id and count > 0 + for other_id, count in self._running_counts.items() + ): + await asyncio.sleep(0.05) + if self._scheduler.get_job(retry_id) is not None: + self._scheduler.remove_job(retry_id) + return True + except TimeoutError: + self._release_execution(task_id) + except BaseException: + self._release_execution(task_id) + raise + + retry_at = self._schedule_exclusive_retry(task_id, wrapped) + stats = self._task_stats.setdefault(task_id, TaskStats()) + stats.last_status = "deferred" + stats.last_execution = datetime.now(UTC).isoformat() + stats.last_duration_seconds = 0.0 + stats.running_since = None + log.info( + "task_deferred_exclusive_busy", + trigger_type=trigger_type, + retry_at=retry_at.isoformat(), + ) + await self._persist_task_stat(task_id, stats, trigger_type=trigger_type, reason="deferred") + return False + + def _schedule_exclusive_retry(self, task_id: str, wrapped: Callable[[], Any]) -> datetime: + retry_at = datetime.now(UTC) + timedelta(seconds=_EXCLUSIVE_RETRY_SECONDS) + self._scheduler.add_job( + wrapped, + "date", + id=f"{task_id}__exclusive_retry", + run_date=retry_at, + replace_existing=True, + ) + return retry_at + async def _defer_task_for_import(self, task_id: str, log: Any, *, trigger_type: str) -> bool: """Skip scheduler-managed background work while an import owns the runtime.""" from pullbox.database import database_maintenance_reason diff --git a/src/pullbox/core/scheduler_error_helpers.py b/src/pullbox/core/scheduler_error_helpers.py index 597a43c1..b31d8bea 100644 --- a/src/pullbox/core/scheduler_error_helpers.py +++ b/src/pullbox/core/scheduler_error_helpers.py @@ -29,7 +29,7 @@ def is_unusable_persist_error(exc: BaseException) -> bool: def logical_task_id(job_id: str) -> str: """Normalize scheduler one-shot job IDs to their logical task IDs.""" - suffix = "_manual" - if job_id.endswith(suffix): - return job_id[: -len(suffix)] + for suffix in ("_manual", "__continuation", "__exclusive_retry"): + if job_id.endswith(suffix): + return job_id[: -len(suffix)] return job_id diff --git a/tests/unit/test_scheduler_exclusive_admission.py b/tests/unit/test_scheduler_exclusive_admission.py new file mode 100644 index 00000000..ec48be3b --- /dev/null +++ b/tests/unit/test_scheduler_exclusive_admission.py @@ -0,0 +1,140 @@ +"""Exclusive maintenance must not indefinitely reserve the scheduler while waiting.""" + +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock + +import pytest + +from pullbox.core.scheduler import PullboxScheduler, get_current_task_trigger_type + + +@pytest.fixture +def scheduler(monkeypatch: pytest.MonkeyPatch) -> PullboxScheduler: + monkeypatch.setattr( + "pullbox.core.scheduler.has_active_import_scheduler_protection", + AsyncMock(return_value=False), + ) + monkeypatch.setattr("pullbox.core.scheduler._EXCLUSIVE_WAIT_SECONDS", 0.01, raising=False) + result = PullboxScheduler() + result._persist_task_stat = AsyncMock() + return result + + +async def test_busy_exclusive_task_defers_without_blocking_other_jobs( + scheduler: PullboxScheduler, +) -> None: + scheduler._running_counts["sync_new_issues"] = 1 + backup = AsyncMock() + task = asyncio.create_task(scheduler._wrap_task(backup, "run_backups", exclusive=True)()) + try: + done, _ = await asyncio.wait({task}, timeout=1) + assert task in done, "Busy backup must yield instead of reserving the scheduler forever" + await task + backup.assert_not_awaited() + assert scheduler._exclusive_active_task_id is None + assert "run_backups" not in scheduler._running_counts + assert scheduler._task_stats["run_backups"].last_status == "deferred" + retry = scheduler._scheduler.get_job("run_backups__exclusive_retry") + assert retry is not None + assert retry not in scheduler._visible_jobs() + + ordinary = AsyncMock() + await scheduler._wrap_task(ordinary, "monitor_downloads")() + ordinary.assert_awaited_once() + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_cancelling_waiting_exclusive_task_releases_reservation( + scheduler: PullboxScheduler, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("pullbox.core.scheduler._EXCLUSIVE_WAIT_SECONDS", 60, raising=False) + scheduler._running_counts["sync_new_issues"] = 1 + backup = AsyncMock() + task = asyncio.create_task(scheduler._wrap_task(backup, "run_backups", exclusive=True)()) + await asyncio.sleep(0) + assert scheduler._exclusive_active_task_id == "run_backups" + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert scheduler._exclusive_active_task_id is None + assert "run_backups" not in scheduler._running_counts + assert scheduler._running_counts["sync_new_issues"] == 1 + assert scheduler._scheduler.get_job("run_backups__exclusive_retry") is None + backup.assert_not_awaited() + + +async def test_exclusive_retry_keeps_manual_intent_and_runs_after_work_drains( + scheduler: PullboxScheduler, +) -> None: + scheduler._running_counts["sync_new_issues"] = 1 + triggers: list[str] = [] + + async def backup() -> None: + triggers.append(get_current_task_trigger_type()) + + task = asyncio.create_task( + scheduler._wrap_task(backup, "run_backups", exclusive=True, trigger_type="manual")() + ) + try: + done, _ = await asyncio.wait({task}, timeout=1) + assert task in done, "Manual backup must be rescheduled instead of waiting indefinitely" + await task + retry = scheduler._scheduler.get_job("run_backups__exclusive_retry") + assert retry is not None + scheduler._running_counts.pop("sync_new_issues") + await retry.func() + assert triggers == ["manual"] + assert scheduler._task_stats["run_backups"].last_status == "completed" + assert scheduler._exclusive_active_task_id is None + assert scheduler._scheduler.get_job("run_backups__exclusive_retry") is None + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_duplicate_exclusive_run_cannot_enter_while_first_is_waiting( + scheduler: PullboxScheduler, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("pullbox.core.scheduler._EXCLUSIVE_WAIT_SECONDS", 60, raising=False) + scheduler._running_counts["sync_new_issues"] = 1 + backup = AsyncMock() + wrapped = scheduler._wrap_task(backup, "run_backups", exclusive=True) + first = asyncio.create_task(wrapped()) + second: asyncio.Task[None] | None = None + try: + await asyncio.sleep(0) + second = asyncio.create_task(wrapped()) + done, _ = await asyncio.wait({second}, timeout=0.2) + assert second in done, "An exclusive task waiting for admission is already reserved" + await second + assert scheduler._exclusive_active_task_id == "run_backups" + assert scheduler._task_stats["run_backups"].overlap_count == 1 + scheduler._running_counts.pop("sync_new_issues") + await asyncio.wait_for(first, timeout=1) + backup.assert_awaited_once() + finally: + tasks = [first] + ([second] if second is not None else []) + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +async def test_exclusive_retry_is_not_lost_when_another_maintenance_task_is_active( + scheduler: PullboxScheduler, +) -> None: + scheduler._exclusive_active_task_id = "maintain_database" + scheduler._running_counts["maintain_database"] = 1 + backup = AsyncMock() + await scheduler._wrap_task(backup, "run_backups", exclusive=True)() + retry = scheduler._scheduler.get_job("run_backups__exclusive_retry") + assert retry is not None, "Exclusive contention must retain a later retry" + assert scheduler._exclusive_active_task_id == "maintain_database" + assert "run_backups" not in scheduler._running_counts + backup.assert_not_awaited() From 5454bc43b8d53c096bf5fb017992572c718782e1 Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Wed, 16 Sep 2026 23:58:23 -0700 Subject: [PATCH 03/20] fix(airdcpp): tolerate transient queue telemetry overshoot Preserve strict identities and explicit completion; normalize unknown ETA and clamp display only. All 191 AirDC++ unit tests pass. --- src/pullbox/providers/airdcpp/contracts.py | 21 +++++----- .../services/airdcpp_reconciliation.py | 2 + tests/unit/test_airdcpp_queue_contracts.py | 39 +++++++++++++++++++ tests/unit/test_airdcpp_reconciliation.py | 24 +++++++++++- 4 files changed, 75 insertions(+), 11 deletions(-) diff --git a/src/pullbox/providers/airdcpp/contracts.py b/src/pullbox/providers/airdcpp/contracts.py index 751e78b6..1ccd2763 100644 --- a/src/pullbox/providers/airdcpp/contracts.py +++ b/src/pullbox/providers/airdcpp/contracts.py @@ -37,6 +37,14 @@ def _normalize_whole_number(value: object) -> object: return value +def _normalize_queue_eta(value: object) -> object: + """AirDC++ derives ETA from counters that can briefly exceed the size.""" + value = _normalize_whole_number(value) + if type(value) is int and -(2**63) <= value < 0: + return None + return value + + PositiveInt = Annotated[StrictInt, Field(gt=0)] NonNegativeInt = Annotated[StrictInt, Field(ge=0)] Port = Annotated[StrictInt, BeforeValidator(_normalize_port), Field(ge=0, le=65535)] @@ -56,6 +64,7 @@ def _normalize_whole_number(value: object) -> object: BeforeValidator(_normalize_whole_number), Field(gt=0, le=2**63 - 1), ] +QueueEta = Annotated[WholeNonNegativeInt | None, BeforeValidator(_normalize_queue_eta)] class AirDcppWireModel(BaseModel): @@ -163,16 +172,10 @@ class AirDcppQueueBundle(AirDcppWireModel): time_added: WholeNonNegativeInt time_finished: WholeNonNegativeInt speed: WholeNonNegativeInt - seconds_left: WholeNonNegativeInt + seconds_left: QueueEta sources: AirDcppQueueSourceInfo status: AirDcppQueueStatus - @model_validator(mode="after") - def validate_progress(self) -> AirDcppQueueBundle: - if self.downloaded_bytes > self.size: - raise ValueError("downloaded bytes cannot exceed bundle size") - return self - class AirDcppQueueBundleAddInfo(AirDcppWireModel): """Stable identity returned for both new and merged queue bundles.""" @@ -208,15 +211,13 @@ class AirDcppQueueFile(AirDcppWireModel): time_added: WholeNonNegativeInt time_finished: WholeNonNegativeInt speed: WholeNonNegativeInt - seconds_left: WholeNonNegativeInt + seconds_left: QueueEta sources: AirDcppQueueSourceInfo status: AirDcppQueueStatus tth: Annotated[StrictStr, Field(pattern=r"^[A-Z2-7]{39}$")] @model_validator(mode="after") def validate_progress(self) -> AirDcppQueueFile: - if self.downloaded_bytes > self.size: - raise ValueError("downloaded bytes cannot exceed file size") if self.type.id != "file": raise ValueError("AirDC++ queue recovery supports file items only") return self diff --git a/src/pullbox/services/airdcpp_reconciliation.py b/src/pullbox/services/airdcpp_reconciliation.py index 732ae4be..bb1cde76 100644 --- a/src/pullbox/services/airdcpp_reconciliation.py +++ b/src/pullbox/services/airdcpp_reconciliation.py @@ -713,6 +713,8 @@ def _shared_progress_snapshot(acquisition: AirDcppAcquisition) -> dict[str, obje transferred_bytes = ( int(transferred) if isinstance(transferred, int | float) and transferred >= 0 else None ) + if transferred_bytes is not None and size_bytes is not None: + transferred_bytes = min(transferred_bytes, size_bytes) progress = ( min(transferred_bytes / size_bytes, 1.0) if transferred_bytes is not None and size_bytes is not None diff --git a/tests/unit/test_airdcpp_queue_contracts.py b/tests/unit/test_airdcpp_queue_contracts.py index 63f48f23..970d996b 100644 --- a/tests/unit/test_airdcpp_queue_contracts.py +++ b/tests/unit/test_airdcpp_queue_contracts.py @@ -2,11 +2,15 @@ from __future__ import annotations +from contextlib import suppress + import httpx import pytest +from pydantic import ValidationError from pullbox.providers.airdcpp.api_client import AirDcppApiClient from pullbox.providers.airdcpp.contracts import ( + AirDcppQueueBundle, AirDcppQueueBundleAddInfo, AirDcppQueueFile, AirDcppSearchDownloadResponse, @@ -56,6 +60,41 @@ def test_queue_mutation_contracts_require_typed_bundle_and_file_identity() -> No assert queue_file.target.get_secret_value().startswith("/Downloads/") +@pytest.mark.parametrize("model", [AirDcppQueueBundle, AirDcppQueueFile]) +@pytest.mark.parametrize("eta", [-2, -4.0]) +def test_queue_telemetry_accepts_overshoot_without_claiming_completion(model, eta) -> None: + payload = _queue_file_payload() + payload.update(downloaded_bytes=100_000_001.0, seconds_left=eta) + parsed = None + with suppress(ValidationError): + parsed = model.model_validate(payload) + + assert parsed is not None, "Transient AirDC++ counters must not discard the queue" + assert parsed.downloaded_bytes == 100_000_001 + assert parsed.seconds_left is None + assert not parsed.status.completed + + +@pytest.mark.parametrize("model", [AirDcppQueueBundle, AirDcppQueueFile]) +@pytest.mark.parametrize("eta", [True, "-2", -1.5, float("inf"), float("nan"), -(2**64)]) +def test_queue_telemetry_still_rejects_malformed_eta(model, eta) -> None: + payload = _queue_file_payload() + payload["seconds_left"] = eta + with pytest.raises(ValidationError): + model.model_validate(payload) + + +@pytest.mark.parametrize( + "field,value", [("id", 0), ("bundle", 0), ("tth", "invalid"), ("type", {"id": "directory"})] +) +def test_queue_overshoot_does_not_weaken_file_identity(field, value) -> None: + payload = _queue_file_payload() + payload.update(downloaded_bytes=100_000_001.0, seconds_left=-2.0) + payload[field] = value + with pytest.raises(ValidationError): + AirDcppQueueFile.model_validate(payload) + + @pytest.mark.asyncio async def test_queue_client_uses_exact_bounded_endpoints_and_safe_payloads() -> None: requests: list[httpx.Request] = [] diff --git a/tests/unit/test_airdcpp_reconciliation.py b/tests/unit/test_airdcpp_reconciliation.py index 08a823b4..ed8271e1 100644 --- a/tests/unit/test_airdcpp_reconciliation.py +++ b/tests/unit/test_airdcpp_reconciliation.py @@ -24,7 +24,11 @@ AirDcppQueueFile, ) from pullbox.providers.airdcpp.errors import AirDcppUnavailableError -from pullbox.services.airdcpp_reconciliation import AirDcppReconciler, apply_airdcpp_bundle +from pullbox.services.airdcpp_reconciliation import ( + AirDcppReconciler, + _shared_progress_snapshot, + apply_airdcpp_bundle, +) from pullbox.services.airdcpp_search_cooldown import AirDcppCooldownReservation if TYPE_CHECKING: @@ -33,6 +37,24 @@ _TTH = "CUO74LMZUQMQCBR5UKTIFJPO32LVUH5VZBOL54Y" +def test_overshoot_keeps_download_active_and_clamps_display_only() -> None: + history = DownloadHistory(state=DownloadState.SENT) + acquisition = AirDcppAcquisition(download_history=history) + # Construct the telemetry independently so this regression tests projection, + # not the separately covered wire validation. + bundle = _bundle().model_copy(update={"downloaded_bytes": 100_000_001, "seconds_left": None}) + + assert apply_airdcpp_bundle(acquisition, bundle, at=datetime.now(UTC)) + assert history.state == DownloadState.DOWNLOADING + assert history.completed_at is None + assert history.downloaded_path is None + assert acquisition.route_snapshot["queue"]["downloaded_bytes"] == 100_000_001 + progress = _shared_progress_snapshot(acquisition) + assert progress["bytes_transferred"] == progress["size_bytes"] == 100_000_000 + assert progress["progress"] == 1.0 + assert progress["eta_seconds"] is None + + @pytest.fixture async def db_factory() -> AsyncGenerator[async_sessionmaker[AsyncSession], None]: engine = create_async_engine("sqlite+aiosqlite:///:memory:") From f6d53a240aea6fd2805a1224758d7108e5215ddc Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 00:06:27 -0700 Subject: [PATCH 04/20] fix(database): drain maintenance safely and bound health probes Drain existing transactions before maintenance, defer busy jobs, and retain fencing until cancelled worker threads stop. Bound SQLite integrity probes in the engine and purge search history in short retryable batches. TDD: reproduced active-writer admission, cancellation fencing, unbounded health SQL and retention batching failures. 227 focused tests passed, plus the final maintenance and retention regressions. --- src/pullbox/core/scheduler.py | 12 +++ src/pullbox/database.py | 56 ++++++++++- .../services/backup_runtime_service.py | 10 +- .../services/database_optimization_service.py | 30 ++++-- .../services/health_database_checks.py | 49 +++++++++- src/pullbox/tasks/search_task.py | 33 ++++++- .../test_database_maintenance_admission.py | 94 +++++++++++++++++++ .../test_database_optimization_service.py | 21 +++++ tests/unit/test_health_checks.py | 6 +- tests/unit/test_health_integrity_budget.py | 51 ++++++++++ .../test_maintenance_worker_cancellation.py | 57 +++++++++++ .../test_scheduler_exclusive_admission.py | 9 ++ tests/unit/test_search_log.py | 39 ++++++++ 13 files changed, 445 insertions(+), 22 deletions(-) create mode 100644 tests/unit/test_database_maintenance_admission.py create mode 100644 tests/unit/test_health_integrity_budget.py create mode 100644 tests/unit/test_maintenance_worker_cancellation.py diff --git a/src/pullbox/core/scheduler.py b/src/pullbox/core/scheduler.py index fbe82766..1078f962 100644 --- a/src/pullbox/core/scheduler.py +++ b/src/pullbox/core/scheduler.py @@ -74,6 +74,7 @@ build_scheduled_task_views, ) from pullbox.core.sqlite_lock import is_sqlite_locked_error +from pullbox.database import DatabaseMaintenanceBusyError from pullbox.services.import_activity import ( has_active_import_scheduler_protection, is_missing_import_jobs_table_error, @@ -467,6 +468,17 @@ async def wrapper() -> None: duration_seconds=round(elapsed, 2), logical_status=completed_status, ) + except DatabaseMaintenanceBusyError: + scheduler._release_execution(task_id) + retry_at = scheduler._schedule_exclusive_retry(task_id, wrapper) + stats.last_execution = datetime.now(UTC).isoformat() + stats.last_duration_seconds = round(time.monotonic() - start, 2) + stats.last_status = "deferred" + stats.running_since = None + log.info("task_deferred_database_busy", retry_at=retry_at.isoformat()) + await scheduler._persist_task_stat( + task_id, stats, trigger_type=trigger_type, reason="deferred" + ) except asyncio.CancelledError: elapsed = time.monotonic() - start ended_dt = datetime.now(UTC) diff --git a/src/pullbox/database.py b/src/pullbox/database.py index 52b9536e..f322e913 100644 --- a/src/pullbox/database.py +++ b/src/pullbox/database.py @@ -11,11 +11,12 @@ import shutil import sqlite3 import stat -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager +from collections.abc import AsyncGenerator, Awaitable +from contextlib import asynccontextmanager, suppress from datetime import UTC, datetime from pathlib import Path from typing import Any +from weakref import WeakSet import structlog from sqlalchemy import event @@ -23,6 +24,7 @@ from sqlalchemy.ext.asyncio import ( AsyncEngine, AsyncSession, + AsyncSessionTransaction, async_sessionmaker, create_async_engine, ) @@ -37,6 +39,9 @@ _maintenance_gate.set() _maintenance_lock = asyncio.Lock() _maintenance_reason: str | None = None +_MAINTENANCE_DRAIN_SECONDS = 5.0 +_active_sessions: WeakSet["GateAwareAsyncSession"] = WeakSet() +_draining_transactions: dict["GateAwareAsyncSession", AsyncSessionTransaction] = {} _SQLITE_BUSY_TIMEOUT_MS = 15000 _SQLITE_BUSY_TIMEOUT_PRAGMA = "PRAGMA busy_timeout=15000" _SQLITE_ALLOWED_JOURNAL_MODES = frozenset({"WAL", "DELETE"}) @@ -46,10 +51,21 @@ } +class DatabaseMaintenanceBusyError(RuntimeError): + """Existing transactions could not drain; maintenance may be retried later.""" + + class GateAwareAsyncSession(AsyncSession): """AsyncSession that pauses database I/O while maintenance is active.""" + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + _active_sessions.add(self) + async def _wait_for_ready(self) -> None: + transaction = _draining_transactions.get(self) + if transaction is not None and self.get_transaction() is transaction: + return await wait_for_database_ready() async def connection(self, *args: Any, **kwargs: Any) -> Any: @@ -385,6 +401,24 @@ async def dispose_engine() -> None: logger.debug("database_engine_disposed") +async def await_maintenance_worker[T](operation: Awaitable[T]) -> T: + """Keep the maintenance fence until a non-cancellable worker actually stops.""" + worker = asyncio.ensure_future(operation) + try: + return await asyncio.shield(worker) + except asyncio.CancelledError: + while not worker.done(): + try: + await asyncio.shield(worker) + except asyncio.CancelledError: + continue + except Exception: + break + with suppress(BaseException): + worker.result() + raise + + @asynccontextmanager async def database_maintenance_window(*, reason: str) -> AsyncGenerator[None, None]: """Temporarily pause new DB sessions for exclusive maintenance work.""" @@ -394,10 +428,28 @@ async def database_maintenance_window(*, reason: str) -> AsyncGenerator[None, No _maintenance_reason = reason _maintenance_gate.clear() try: + _draining_transactions.update( + (session, transaction) + for session in tuple(_active_sessions) + if (transaction := session.get_transaction()) is not None + ) + try: + async with asyncio.timeout(_MAINTENANCE_DRAIN_SECONDS): + while any( + session.get_transaction() is transaction + for session, transaction in _draining_transactions.items() + ): + await asyncio.sleep(0.01) + except TimeoutError as exc: + raise DatabaseMaintenanceBusyError( + "Database is busy; retry maintenance after current work finishes." + ) from exc + _draining_transactions.clear() await dispose_engine() logger.info("database_maintenance_started", reason=reason) yield finally: + _draining_transactions.clear() _maintenance_gate.set() _maintenance_reason = None logger.info("database_maintenance_finished", reason=reason) diff --git a/src/pullbox/services/backup_runtime_service.py b/src/pullbox/services/backup_runtime_service.py index 5f9acf97..63a53d29 100644 --- a/src/pullbox/services/backup_runtime_service.py +++ b/src/pullbox/services/backup_runtime_service.py @@ -5,7 +5,7 @@ import asyncio from typing import TYPE_CHECKING -from pullbox.database import database_maintenance_window +from pullbox.database import await_maintenance_worker, database_maintenance_window from pullbox.services.backup_service import BackupInfo, BackupService if TYPE_CHECKING: @@ -26,12 +26,16 @@ def service(self) -> BackupService: async def create_backup(self, *, backup_type: str) -> BackupInfo: """Create a backup while the database is in a maintenance window.""" async with database_maintenance_window(reason="backup"): - return await asyncio.to_thread(self._service.create_backup, backup_type=backup_type) + return await await_maintenance_worker( + asyncio.to_thread(self._service.create_backup, backup_type=backup_type) + ) async def restore_backup(self, filename: str) -> bool: """Restore a backup while the database is paused for maintenance.""" async with database_maintenance_window(reason="restore_backup"): - return await asyncio.to_thread(self._service.restore_backup, filename) + return await await_maintenance_worker( + asyncio.to_thread(self._service.restore_backup, filename) + ) async def cleanup_old_backups(self, *, retention_days: int) -> int: """Run retention cleanup off the event loop.""" diff --git a/src/pullbox/services/database_optimization_service.py b/src/pullbox/services/database_optimization_service.py index 7578cc20..7f549317 100644 --- a/src/pullbox/services/database_optimization_service.py +++ b/src/pullbox/services/database_optimization_service.py @@ -8,7 +8,12 @@ from dataclasses import dataclass from pathlib import Path -from pullbox.database import database_maintenance_window +from pullbox.core.sqlite_lock import is_sqlite_locked_error +from pullbox.database import ( + DatabaseMaintenanceBusyError, + await_maintenance_worker, + database_maintenance_window, +) _BUSY_TIMEOUT_MS = 30_000 @@ -17,6 +22,10 @@ class DatabaseOptimizationError(RuntimeError): """Raised when SQLite database optimization cannot run safely.""" +class DatabaseOptimizationBusyError(DatabaseOptimizationError, DatabaseMaintenanceBusyError): + """Database contention is safe to retry without calling it corruption.""" + + @dataclass(frozen=True, slots=True) class DatabaseOptimizationPreview: """Current SQLite storage state and the capacity needed to compact it.""" @@ -76,7 +85,7 @@ def optimize(self) -> DatabaseOptimizationResult: with self._connect(read_only=False) as connection: checkpoint = connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() if checkpoint is not None and int(checkpoint[0]) != 0: - raise DatabaseOptimizationError( + raise DatabaseOptimizationBusyError( "SQLite could not checkpoint the write-ahead log because the database is busy." ) connection.execute("VACUUM") @@ -94,7 +103,7 @@ def maintain(self) -> DatabaseMaintenanceResult: with self._connect(read_only=False) as connection: checkpoint = connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() if checkpoint is not None and int(checkpoint[0]) != 0: - raise DatabaseOptimizationError( + raise DatabaseOptimizationBusyError( "SQLite could not checkpoint the write-ahead log because the database is busy." ) connection.execute("REINDEX") @@ -106,7 +115,7 @@ def maintain(self) -> DatabaseMaintenanceResult: connection.execute("PRAGMA optimize=0x10002") checkpoint = connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() if checkpoint is not None and int(checkpoint[0]) != 0: - raise DatabaseOptimizationError( + raise DatabaseOptimizationBusyError( "SQLite could not checkpoint the write-ahead log after maintenance." ) integrity_row = connection.execute("PRAGMA quick_check").fetchone() @@ -174,9 +183,16 @@ def service(self) -> DatabaseOptimizationService: async def optimize(self) -> DatabaseOptimizationResult: """Compact the database outside the event loop under the maintenance gate.""" async with database_maintenance_window(reason="database_optimize"): - return await asyncio.to_thread(self._service.optimize) + return await await_maintenance_worker(asyncio.to_thread(self._service.optimize)) async def maintain(self) -> DatabaseMaintenanceResult: """Run recurring maintenance outside the event loop under the shared gate.""" - async with database_maintenance_window(reason="nightly_database_maintenance"): - return await asyncio.to_thread(self._service.maintain) + try: + async with database_maintenance_window(reason="nightly_database_maintenance"): + return await await_maintenance_worker(asyncio.to_thread(self._service.maintain)) + except sqlite3.OperationalError as exc: + if not is_sqlite_locked_error(exc): + raise + raise DatabaseOptimizationBusyError( + "Database is busy; maintenance will retry." + ) from exc diff --git a/src/pullbox/services/health_database_checks.py b/src/pullbox/services/health_database_checks.py index d6a343e1..09e4961f 100644 --- a/src/pullbox/services/health_database_checks.py +++ b/src/pullbox/services/health_database_checks.py @@ -3,7 +3,11 @@ from __future__ import annotations import asyncio +import sqlite3 +import time from collections.abc import Awaitable, Callable +from contextlib import closing +from typing import TYPE_CHECKING from sqlalchemy import select, text from sqlalchemy.ext.asyncio import AsyncSession @@ -20,6 +24,9 @@ ) from pullbox.services.health_types import CheckOutcome, SubCheckOutcome +if TYPE_CHECKING: + from pathlib import Path + _DB_CONNECTION_DEGRADED_MS = 250.0 _DB_CONNECTION_UNHEALTHY_MS = 1000.0 _DB_QUERY_DEGRADED_MS = 500.0 @@ -28,6 +35,7 @@ _DB_BLOAT_UNHEALTHY_RATIO = 0.3 _DB_BLOAT_DEGRADED_MB = 50.0 _DB_BLOAT_UNHEALTHY_MB = 250.0 +_INTEGRITY_BUDGET_SECONDS = 5.0 PerfCounter = Callable[[], float] RequiredDatabaseCheck = Callable[[AsyncSession], Awaitable[SubCheckOutcome]] @@ -90,11 +98,16 @@ async def check_database( "SQLite free-list bloat is high. Vacuuming the database should reclaim " "unused pages." ) - elif check.check_name == "integrity_check" and check.status != HealthStatus.HEALTHY: + elif check.check_name == "integrity_check" and check.status == HealthStatus.UNHEALTHY: guidance_parts.append( "SQLite quick_check reported an integrity issue. Stop background work " "and inspect the database." ) + elif check.check_name == "integrity_check" and check.status == HealthStatus.DEGRADED: + guidance_parts.append( + "Integrity verification did not finish within the health-check budget. " + "Use Database Maintenance for a full check when background activity is quiet." + ) if any( check.check_name == "integrity_check" and check.status == HealthStatus.UNHEALTHY @@ -243,13 +256,39 @@ async def check_db_size(session: AsyncSession) -> SubCheckOutcome | None: ) +def _bounded_integrity_check(path: Path) -> str: + """Interrupt expensive SQLite work inside SQLite, not only its awaiter.""" + deadline = time.monotonic() + _INTEGRITY_BUDGET_SECONDS + with closing( + sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True, timeout=1) + ) as connection: + connection.set_progress_handler(lambda: int(time.monotonic() >= deadline), 1000) + if time.monotonic() >= deadline: + raise TimeoutError("Integrity check budget exhausted") + rows = connection.execute("PRAGMA quick_check").fetchall() + return "; ".join(str(row[0]) for row in rows) + + async def check_db_integrity(session: AsyncSession) -> SubCheckOutcome | None: - """Run SQLite quick_check when supported.""" - if _sqlite_database_path(session) is None: + """Run a bounded read-only integrity probe without occupying the shared session.""" + path = _sqlite_database_path(session) + if path is None: return None - result = await session.execute(text("PRAGMA quick_check")) - status_text = str(result.scalar_one_or_none() or "").strip() + try: + status_text = await asyncio.to_thread(_bounded_integrity_check, path) + except (TimeoutError, sqlite3.OperationalError) as exc: + if isinstance(exc, sqlite3.OperationalError) and getattr( + exc, "sqlite_errorcode", None + ) not in {sqlite3.SQLITE_INTERRUPT, sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED}: + raise + return SubCheckOutcome( + check_name="integrity_check", + name="Integrity check", + status=HealthStatus.DEGRADED, + message="Integrity check deferred: database busy or health-check time budget exhausted", + details={"verification": "incomplete"}, + ) if not status_text: return None if status_text.lower() == "ok": diff --git a/src/pullbox/tasks/search_task.py b/src/pullbox/tasks/search_task.py index 987b8e41..b8aaf43f 100644 --- a/src/pullbox/tasks/search_task.py +++ b/src/pullbox/tasks/search_task.py @@ -1606,12 +1606,17 @@ async def _sync_wanted_sweep_schedule( ) +_SEARCH_LOG_PURGE_BATCH_SIZE = 500 + + async def purge_search_logs() -> None: """Delete search log entries older than the configured retention period.""" from datetime import UTC, datetime, timedelta from sqlalchemy import delete, select + from pullbox.core.sqlite_lock import run_sqlite_transaction_with_retry + factory = get_session_factory() async with factory() as session: @@ -1619,11 +1624,31 @@ async def purge_search_logs() -> None: retention_days = await _load_search_log_retention_days(session) cutoff = datetime.now(UTC) - timedelta(days=retention_days) - old_search_log_ids = select(SearchLog.id).where(SearchLog.created_at < cutoff) - await prune_unstarted_direct_discoveries(session, old_search_log_ids) - result = await session.execute(delete(SearchLog).where(SearchLog.created_at < cutoff)) - pruned = result.rowcount # type: ignore[attr-defined] await session.commit() + pruned = 0 + + async def purge_batch() -> int: + ids = list( + ( + await session.scalars( + select(SearchLog.id) + .where(SearchLog.created_at < cutoff) + .order_by(SearchLog.id) + .limit(_SEARCH_LOG_PURGE_BATCH_SIZE) + ) + ).all() + ) + if not ids: + return 0 + await prune_unstarted_direct_discoveries(session, ids) + await session.execute(delete(SearchLog).where(SearchLog.id.in_(ids))) + return len(ids) + + while count := await run_sqlite_transaction_with_retry( + session, purge_batch, event_name="purge_search_logs", logger=logger + ): + pruned += count + await asyncio.sleep(0) if pruned: logger.info( diff --git a/tests/unit/test_database_maintenance_admission.py b/tests/unit/test_database_maintenance_admission.py new file mode 100644 index 00000000..b06b635c --- /dev/null +++ b/tests/unit/test_database_maintenance_admission.py @@ -0,0 +1,94 @@ +"""Maintenance must drain existing transactions without trapping their commits.""" + +import asyncio +from contextlib import suppress +from unittest.mock import AsyncMock + +import pytest +from sqlalchemy import text +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +import pullbox.database as database + + +@pytest.fixture +async def factory(tmp_path, monkeypatch): + monkeypatch.setattr(database, "_maintenance_gate", asyncio.Event()) + database._maintenance_gate.set() + monkeypatch.setattr(database, "_maintenance_lock", asyncio.Lock()) + monkeypatch.setattr(database, "dispose_engine", AsyncMock()) + engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'maintenance.db'}") + async with engine.begin() as connection: + await connection.execute(text("CREATE TABLE sample (id INTEGER PRIMARY KEY)")) + yield async_sessionmaker(engine, class_=database.GateAwareAsyncSession) + await engine.dispose() + + +@pytest.mark.asyncio +async def test_existing_writer_can_commit_before_maintenance_but_new_reads_wait(factory): + entered = asyncio.Event() + release = asyncio.Event() + + async def maintain(): + async with database.database_maintenance_window(reason="test"): + entered.set() + await release.wait() + + async with factory() as writer, factory() as reader: + await writer.execute(text("INSERT INTO sample VALUES (1)")) + task = asyncio.create_task(maintain()) + while database._maintenance_gate.is_set(): + await asyncio.sleep(0) + read_task = asyncio.create_task(reader.execute(text("SELECT count(*) FROM sample"))) + try: + assert not entered.is_set(), "Maintenance started over an active writer" + await asyncio.wait_for(writer.commit(), 1) + await asyncio.wait_for(entered.wait(), 1) + assert not read_task.done() + release.set() + await task + assert (await read_task).scalar() == 1 + finally: + release.set() + task.cancel() + with suppress(asyncio.CancelledError): + await task + await read_task + + +@pytest.mark.asyncio +async def test_busy_maintenance_defers_and_reopens_database(factory, monkeypatch): + monkeypatch.setattr(database, "_MAINTENANCE_DRAIN_SECONDS", 0.01, raising=False) + error = None + async with factory() as writer: + await writer.execute(text("INSERT INTO sample VALUES (1)")) + try: + async with database.database_maintenance_window(reason="test"): + pass + except RuntimeError as exc: + error = exc + assert error is not None, "Maintenance must defer rather than run over an active writer" + assert "busy" in str(error).lower() + assert database._maintenance_gate.is_set() + assert database.database_maintenance_reason() is None + database.dispose_engine.assert_not_awaited() + await writer.commit() + + +@pytest.mark.asyncio +async def test_cancelled_maintenance_drain_reopens_database(factory): + async with factory() as writer: + await writer.execute(text("INSERT INTO sample VALUES (1)")) + + async def maintain(): + async with database.database_maintenance_window(reason="test"): + await asyncio.Event().wait() + + task = asyncio.create_task(maintain()) + while database._maintenance_gate.is_set(): + await asyncio.sleep(0) + task.cancel() + with suppress(asyncio.CancelledError): + await task + assert database._maintenance_gate.is_set() + await asyncio.wait_for(writer.commit(), 1) diff --git a/tests/unit/test_database_optimization_service.py b/tests/unit/test_database_optimization_service.py index 89fb8c0c..4b8374c0 100644 --- a/tests/unit/test_database_optimization_service.py +++ b/tests/unit/test_database_optimization_service.py @@ -44,6 +44,27 @@ def test_preview_reports_reclaimable_free_pages(tmp_path: Path) -> None: assert preview.integrity_result == "ok" +@pytest.mark.asyncio +async def test_busy_nightly_maintenance_is_retryable(tmp_path, monkeypatch): + from contextlib import suppress + + from pullbox.database import DatabaseMaintenanceBusyError + + runtime = DatabaseOptimizationRuntimeService(tmp_path / "pullbox.db") + + def busy(): + raise sqlite3.OperationalError("database is locked") + + monkeypatch.setattr(runtime.service, "maintain", busy) + retryable = False + with suppress(sqlite3.OperationalError): + try: + await runtime.maintain() + except DatabaseMaintenanceBusyError: + retryable = True + assert retryable, "A busy database must defer nightly maintenance, not permanently fail it" + + def test_optimize_checkpoints_and_vacuums_free_pages(tmp_path: Path) -> None: db_path = tmp_path / "pullbox.db" _create_fragmented_database(db_path) diff --git a/tests/unit/test_health_checks.py b/tests/unit/test_health_checks.py index e3b46f28..fd2466d6 100644 --- a/tests/unit/test_health_checks.py +++ b/tests/unit/test_health_checks.py @@ -404,7 +404,11 @@ async def test_integrity_check_flags_unhealthy_result(self, settings: MagicMock) ) service = _make_service(settings) - result = await service._check_db_integrity(session) + with patch( + "pullbox.services.health_database_checks._bounded_integrity_check", + return_value="row 17 missing from index ix_series_title_year", + ): + result = await service._check_db_integrity(session) assert result is not None assert result.status == HealthStatus.UNHEALTHY diff --git a/tests/unit/test_health_integrity_budget.py b/tests/unit/test_health_integrity_budget.py new file mode 100644 index 00000000..0a81f09c --- /dev/null +++ b/tests/unit/test_health_integrity_budget.py @@ -0,0 +1,51 @@ +"""SQLite integrity health checks must have a database-enforced time budget.""" + +import sqlite3 +from contextlib import suppress +from unittest.mock import AsyncMock + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine + +from pullbox.models.health import HealthStatus +from pullbox.services import health_database_checks as checks + + +@pytest.mark.asyncio +async def test_integrity_does_not_run_unbounded_sql_on_shared_health_session(tmp_path): + path = tmp_path / "health.db" + with sqlite3.connect(path) as connection: + connection.execute("CREATE TABLE sample (id INTEGER)") + engine = create_async_engine(f"sqlite+aiosqlite:///{path}") + try: + async with AsyncSession(engine) as session: + session.execute = AsyncMock(side_effect=AssertionError("Unbounded health SQL")) + result = None + with suppress(AssertionError): + result = await checks.check_db_integrity(session) + assert result is not None, ( + "Integrity checking must use its own bounded read-only connection" + ) + assert result.status == HealthStatus.HEALTHY + session.execute.assert_not_awaited() + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_integrity_budget_exhaustion_is_not_reported_as_corruption(tmp_path, monkeypatch): + path = tmp_path / "health.db" + with sqlite3.connect(path) as connection: + connection.execute("CREATE TABLE sample (id INTEGER)") + connection.executemany("INSERT INTO sample VALUES (?)", [(i,) for i in range(2000)]) + engine = create_async_engine(f"sqlite+aiosqlite:///{path}") + monkeypatch.setattr(checks, "_INTEGRITY_BUDGET_SECONDS", 0.0, raising=False) + try: + async with AsyncSession(engine) as session: + result = await checks.check_db_integrity(session) + assert result is not None + assert result.status == HealthStatus.DEGRADED + assert "budget" in result.message.lower() + assert "corrupt" not in result.message.lower() + finally: + await engine.dispose() diff --git a/tests/unit/test_maintenance_worker_cancellation.py b/tests/unit/test_maintenance_worker_cancellation.py new file mode 100644 index 00000000..cf02af6e --- /dev/null +++ b/tests/unit/test_maintenance_worker_cancellation.py @@ -0,0 +1,57 @@ +"""Cancellation cannot reopen the database while a maintenance thread still runs.""" + +import asyncio +import threading +from contextlib import suppress +from unittest.mock import AsyncMock + +import pytest + +import pullbox.database as database +from pullbox.services.backup_runtime_service import BackupRuntimeService +from pullbox.services.database_optimization_service import DatabaseOptimizationRuntimeService + + +@pytest.mark.parametrize("operation", ["backup", "restore", "optimize", "maintain"]) +async def test_cancellation_keeps_gate_closed_until_thread_finishes( + tmp_path, monkeypatch, operation +): + monkeypatch.setattr(database, "_maintenance_gate", asyncio.Event()) + database._maintenance_gate.set() + monkeypatch.setattr(database, "_maintenance_lock", asyncio.Lock()) + monkeypatch.setattr(database, "dispose_engine", AsyncMock()) + started = threading.Event() + release = threading.Event() + + def worker(*args, **kwargs): + started.set() + release.wait(3) + + if operation in {"backup", "restore"}: + runtime = BackupRuntimeService(tmp_path / "backups", tmp_path / "db") + name = f"{operation}_backup" if operation == "restore" else "create_backup" + monkeypatch.setattr(runtime.service, name, worker) + coro = ( + runtime.restore_backup("test.zip") + if operation == "restore" + else runtime.create_backup(backup_type="manual") + ) + else: + runtime = DatabaseOptimizationRuntimeService(tmp_path / "db") + monkeypatch.setattr(runtime.service, operation, worker) + coro = getattr(runtime, operation)() + task = asyncio.create_task(coro) + try: + while not started.is_set(): + await asyncio.sleep(0.001) + task.cancel() + await asyncio.sleep(0.01) + assert not database._maintenance_gate.is_set(), ( + "A live SQLite worker lost its maintenance fence" + ) + assert not task.done() + finally: + release.set() + with suppress(asyncio.CancelledError): + await task + assert database._maintenance_gate.is_set() diff --git a/tests/unit/test_scheduler_exclusive_admission.py b/tests/unit/test_scheduler_exclusive_admission.py index ec48be3b..d74aca28 100644 --- a/tests/unit/test_scheduler_exclusive_admission.py +++ b/tests/unit/test_scheduler_exclusive_admission.py @@ -8,6 +8,7 @@ import pytest from pullbox.core.scheduler import PullboxScheduler, get_current_task_trigger_type +from pullbox.database import DatabaseMaintenanceBusyError @pytest.fixture @@ -49,6 +50,14 @@ async def test_busy_exclusive_task_defers_without_blocking_other_jobs( await asyncio.gather(task, return_exceptions=True) +async def test_maintenance_blocked_by_request_transaction_is_retried(scheduler) -> None: + maintenance = AsyncMock(side_effect=DatabaseMaintenanceBusyError("Database is busy")) + await scheduler._wrap_task(maintenance, "maintain_database", exclusive=True)() + assert scheduler._task_stats["maintain_database"].last_status == "deferred" + assert scheduler._scheduler.get_job("maintain_database__exclusive_retry") is not None + assert scheduler._exclusive_active_task_id is None + + async def test_cancelling_waiting_exclusive_task_releases_reservation( scheduler: PullboxScheduler, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/test_search_log.py b/tests/unit/test_search_log.py index 3f18fe0a..3faace5d 100644 --- a/tests/unit/test_search_log.py +++ b/tests/unit/test_search_log.py @@ -599,3 +599,42 @@ async def test_purge_search_logs(self, db_factory: async_sessionmaker[AsyncSessi remaining = (await session.execute(select(SearchLog))).scalars().all() assert len(remaining) == 1 assert remaining[0].id == recent_log.id + + +async def test_scheduled_log_purge_commits_small_batches(db_factory, monkeypatch): + from datetime import UTC, datetime, timedelta + from unittest.mock import AsyncMock + + from pullbox.tasks import search_task + + async with db_factory() as session: + issue = await _seed_issue(session) + for _ in range(5): + session.add( + SearchLog( + issue_id=issue.id, + series_title="Batman", + issue_number=1, + search_type=SearchType.AUTOMATED, + created_at=datetime.now(UTC) - timedelta(days=30), + ) + ) + await session.commit() + + commits = [] + + class TrackingSession(AsyncSession): + async def commit(self): + await super().commit() + commits.append(True) + + factory = async_sessionmaker( + db_factory.kw["bind"], class_=TrackingSession, expire_on_commit=False + ) + monkeypatch.setattr(search_task, "get_session_factory", lambda: factory) + monkeypatch.setattr(search_task, "_load_search_log_retention_days", AsyncMock(return_value=7)) + monkeypatch.setattr(search_task, "_SEARCH_LOG_PURGE_BATCH_SIZE", 2, raising=False) + await search_task.purge_search_logs() + assert len(commits) >= 3, "Log retention must release the writer between bounded batches" + async with db_factory() as session: + assert list((await session.scalars(select(SearchLog))).all()) == [] From 97c32e7ba5bfeec18412cbcae7ce7b1b7294ed25 Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 00:18:16 -0700 Subject: [PATCH 05/20] fix(metadata): resume bounded sweeps and respect provider cooldowns Persist cursors and retry times, restore interrupted sweeps, commit metadata before provider waits, and share ComicVine throttle state across clients. TDD regressions and 130 metadata/provider/task tests pass. --- src/pullbox/app.py | 7 + src/pullbox/core/provider_cooldown.py | 52 +++ src/pullbox/providers/metadata/comicvine.py | 39 +- src/pullbox/services/metadata_service.py | 26 +- src/pullbox/tasks/metadata_scheduler_task.py | 10 +- src/pullbox/tasks/metadata_sweep_state.py | 95 ++++ src/pullbox/tasks/metadata_task.py | 435 ++++++++++--------- tests/tasks/test_metadata_search_trigger.py | 100 +++++ tests/tasks/test_scheduler_wrappers.py | 2 +- tests/unit/test_comicvine_cooldown.py | 37 ++ 10 files changed, 577 insertions(+), 226 deletions(-) create mode 100644 src/pullbox/core/provider_cooldown.py create mode 100644 src/pullbox/tasks/metadata_sweep_state.py create mode 100644 tests/unit/test_comicvine_cooldown.py diff --git a/src/pullbox/app.py b/src/pullbox/app.py index 460628a9..3f086aca 100644 --- a/src/pullbox/app.py +++ b/src/pullbox/app.py @@ -669,6 +669,13 @@ def _cleanup_import_metadata_recovery_task(task: asyncio.Task[object]) -> None: except Exception: logger.warning("search_wanted_sweep_recovery_failed", exc_info=True) + try: + from pullbox.tasks.metadata_sweep_state import recover_metadata_sweep_schedules + + await recover_metadata_sweep_schedules() + except Exception: + logger.warning("metadata_sweep_recovery_failed", exc_info=True) + search_on_add_recovery_task = asyncio.create_task(recover_recent_search_on_add_misses()) _startup_background_tasks.add(search_on_add_recovery_task) diff --git a/src/pullbox/core/provider_cooldown.py b/src/pullbox/core/provider_cooldown.py new file mode 100644 index 00000000..7e3babad --- /dev/null +++ b/src/pullbox/core/provider_cooldown.py @@ -0,0 +1,52 @@ +"""Process-local account cooldowns shared by short-lived provider clients.""" + +from __future__ import annotations + +import asyncio +import hashlib +import math +import time +import weakref +from datetime import UTC, datetime +from email.utils import parsedate_to_datetime + + +def retry_after_seconds(header: str | None, *, default: float) -> float: + """Accept seconds or an HTTP date; reject malformed and unbounded values.""" + try: + delay = float(header or "") + except ValueError: + try: + when = parsedate_to_datetime(header or "") + if when.tzinfo is None: + when = when.replace(tzinfo=UTC) + delay = (when - datetime.now(UTC)).total_seconds() + except (TypeError, ValueError, OverflowError): + return default + return min(delay, 7 * 86400) if math.isfinite(delay) and delay > 0 else default + + +class ProviderCooldown: + def __init__(self) -> None: + self.until = 0.0 + self.request_lock = asyncio.Lock() + self.last_request_time = 0.0 + + @property + def remaining_seconds(self) -> int: + return max(0, math.ceil(self.until - time.monotonic())) + + def defer(self, seconds: float) -> None: + self.until = max(self.until, time.monotonic() + seconds) + + +_STATES: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, ProviderCooldown]] = ( + weakref.WeakKeyDictionary() +) + + +def provider_cooldown(namespace: str, identity: str) -> ProviderCooldown: + """Keep credentials out of registry keys, logs, and durable state.""" + key = hashlib.sha256(f"{namespace}\0{identity}".encode()).hexdigest() + states = _STATES.setdefault(asyncio.get_running_loop(), {}) + return states.setdefault(key, ProviderCooldown()) diff --git a/src/pullbox/providers/metadata/comicvine.py b/src/pullbox/providers/metadata/comicvine.py index 75596c40..c1d00281 100644 --- a/src/pullbox/providers/metadata/comicvine.py +++ b/src/pullbox/providers/metadata/comicvine.py @@ -27,6 +27,7 @@ parse_issue_number_text, ) from pullbox.core.naming import detect_issue_type +from pullbox.core.provider_cooldown import provider_cooldown, retry_after_seconds from pullbox.providers.base import ( IssueMetadata, IssueSummary, @@ -388,9 +389,17 @@ def _extract_story_arcs(arc_credits: list[dict[str, Any]] | None) -> list[dict[s class ComicVineError(Exception): """Raised when the ComicVine API returns a non-OK status.""" - def __init__(self, status_code: int, message: str, *, retryable: bool = False) -> None: + def __init__( + self, + status_code: int, + message: str, + *, + retryable: bool = False, + retry_after_seconds: int | None = None, + ) -> None: self.status_code = status_code self.retryable = retryable + self.retry_after_seconds = retry_after_seconds super().__init__(message) @@ -438,6 +447,14 @@ async def _request( params: dict[str, Any] | None = None, ) -> dict[str, Any]: """Make a rate-limited GET request to the ComicVine API.""" + cooldown = provider_cooldown("comicvine", self._api_key) + if cooldown.remaining_seconds: + raise ComicVineError( + 429, + "ComicVine cooldown active", + retryable=True, + retry_after_seconds=cooldown.remaining_seconds, + ) if self._rate_coordinator is None: self._rate_coordinator = _rate_coordinator_for( api_key=self._api_key, @@ -448,6 +465,13 @@ async def _request( _resource_rate_key(endpoint), requests_per_second=self._requests_per_second, ) + if cooldown.remaining_seconds: + raise ComicVineError( + 429, + "ComicVine cooldown active", + retryable=True, + retry_after_seconds=cooldown.remaining_seconds, + ) request_params: dict[str, Any] = { "api_key": self._api_key, @@ -478,10 +502,15 @@ async def _request( log.warning("comicvine_not_found") raise ComicVineError(_STATUS_NOT_FOUND, f"Resource not found: {endpoint}") from None log.error("comicvine_http_error", status=http_status) + if http_status in {420, 429}: + cooldown.defer( + retry_after_seconds(exc.response.headers.get("Retry-After"), default=3600) + ) raise ComicVineError( http_status, f"HTTP {http_status}: {endpoint}", retryable=http_status in {408, 420, 429} or http_status >= 500, + retry_after_seconds=cooldown.remaining_seconds or None, ) from None except httpx.HTTPError as exc: log.error("comicvine_request_failed", error=str(exc)) @@ -498,7 +527,13 @@ async def _request( raise ComicVineError(status_code, f"Resource not found: {endpoint}") if status_code == _STATUS_RATE_LIMITED: log.warning("comicvine_rate_limited") - raise ComicVineError(status_code, "Rate limit exceeded", retryable=True) + cooldown.defer(retry_after_seconds(response.headers.get("Retry-After"), default=3600)) + raise ComicVineError( + status_code, + "Rate limit exceeded", + retryable=True, + retry_after_seconds=cooldown.remaining_seconds, + ) if status_code != _STATUS_OK: error_msg = data.get("error", "Unknown error") log.error("comicvine_api_error", status_code=status_code, error=error_msg) diff --git a/src/pullbox/services/metadata_service.py b/src/pullbox/services/metadata_service.py index 6bc8aa82..5fa7d15b 100644 --- a/src/pullbox/services/metadata_service.py +++ b/src/pullbox/services/metadata_service.py @@ -54,7 +54,15 @@ def _provider_error_from_comicvine(exc: ComicVineError) -> ProviderError: return ProviderError( "comicvine", str(exc), - details={"status_code": exc.status_code, "retryable": exc.retryable}, + details={ + "status_code": exc.status_code, + "retryable": exc.retryable, + **( + {"retry_after_seconds": exc.retry_after_seconds} + if exc.retry_after_seconds is not None + else {} + ), + }, ) @@ -972,6 +980,7 @@ async def refresh_series( series_id: int, *, force: bool = False, + commit_before_provider_wait: bool = False, ) -> Series: """Refresh metadata for a series if stale (past refresh interval). @@ -998,6 +1007,7 @@ async def refresh_series( if not series.comicvine_id: raise ProviderError("comicvine", "Series has no ComicVine ID") + comicvine_id = series.comicvine_id if force: refresh_series = getattr(type(self._provider), "refresh_series", None) @@ -1011,7 +1021,15 @@ async def refresh_series( raise _provider_error_from_comicvine(exc) from exc # noinspection PyTypeChecker - series = await self.fetch_series(session, series.comicvine_id) + if commit_before_provider_wait: + series = await self.fetch_series(session, comicvine_id, download_cover=False) + else: + series = await self.fetch_series(session, comicvine_id) + if commit_before_provider_wait: + await session.commit() + if series.cover_url: + await self.download_series_cover(series, series.cover_url) + await session.commit() await self.fetch_issues_for_series(session, series.id) await self.infer_series_status(session, series) synced_at = datetime.now(UTC) @@ -1021,6 +1039,10 @@ async def refresh_series( series.issue_catalog_error = None return series + async def close(self) -> None: + """Release provider connections after bounded background batches.""" + await self._provider.close() + async def download_cover(self, url: str, destination: Path) -> None: """Download and save a cover image.""" log = logger.bind(url=url, destination=str(destination)) diff --git a/src/pullbox/tasks/metadata_scheduler_task.py b/src/pullbox/tasks/metadata_scheduler_task.py index c2224e9f..e22df178 100644 --- a/src/pullbox/tasks/metadata_scheduler_task.py +++ b/src/pullbox/tasks/metadata_scheduler_task.py @@ -2,7 +2,7 @@ from __future__ import annotations -from pullbox.core.scheduler import scheduled_task +from pullbox.core.scheduler import TaskExecutionResult, scheduled_task from pullbox.tasks.metadata_task import refresh_metadata, sync_new_issues @@ -13,9 +13,9 @@ hour=1, minute=0, ) -async def scheduled_sync_new_issues() -> None: +async def scheduled_sync_new_issues() -> TaskExecutionResult: """Run the monitored-series issue sync on its configured cadence.""" - await sync_new_issues() + return await sync_new_issues() @scheduled_task( @@ -25,6 +25,6 @@ async def scheduled_sync_new_issues() -> None: hour=3, minute=15, ) -async def scheduled_refresh_metadata() -> None: +async def scheduled_refresh_metadata() -> TaskExecutionResult: """Run the stale-series metadata refresh on its nightly cadence.""" - await refresh_metadata() + return await refresh_metadata() diff --git a/src/pullbox/tasks/metadata_sweep_state.py b/src/pullbox/tasks/metadata_sweep_state.py new file mode 100644 index 00000000..1398f9fb --- /dev/null +++ b/src/pullbox/tasks/metadata_sweep_state.py @@ -0,0 +1,95 @@ +"""Durable, bounded nightly metadata sweep checkpoints (no schema migration).""" + +from __future__ import annotations + +import json +import math +from dataclasses import asdict, dataclass +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING + +from sqlalchemy import func, select + +from pullbox.core.scheduler import get_scheduler +from pullbox.models.config import SystemConfig +from pullbox.models.series import Series + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + +TASK_IDS = ("sync_new_issues", "refresh_metadata") + + +@dataclass +class MetadataSweep: + cursor: int = 0 + upper_bound: int = 0 + retry_at: float = 0.0 + active: bool = False + + +async def load_sweep(session: AsyncSession, task_id: str) -> MetadataSweep: + row = await session.get(SystemConfig, f"metadata_sweep_{task_id}") + try: + data = json.loads(row.value) if row else {} + state = MetadataSweep(**data) + if ( + type(state.cursor) is int + and type(state.upper_bound) is int + and type(state.active) is bool + and isinstance(state.retry_at, int | float) + and math.isfinite(state.retry_at) + and state.retry_at >= 0 + and 0 <= state.cursor <= state.upper_bound + ): + return state + except (TypeError, ValueError): + pass + return MetadataSweep() + + +async def start_sweep(session: AsyncSession, task_id: str) -> MetadataSweep: + state = await load_sweep(session, task_id) + if not state.active: + upper = await session.scalar( + select(func.max(Series.id)).where(Series.comicvine_id.isnot(None)) + ) + state = MetadataSweep(upper_bound=upper or 0, active=True) + await save_sweep(session, task_id, state) + await session.commit() + return state + + +async def save_sweep(session: AsyncSession, task_id: str, state: MetadataSweep) -> None: + key = f"metadata_sweep_{task_id}" + row = await session.get(SystemConfig, key) + value = json.dumps(asdict(state)) + if row is None: + session.add(SystemConfig(key=key, value=value, value_type="string")) + else: + row.value = value + + +def schedule_sweep(task_id: str, state: MetadataSweep) -> None: + if state.active: + get_scheduler().schedule_task_continuation( + task_id, + run_at=max( + datetime.now(UTC) + timedelta(seconds=60), + datetime.fromtimestamp(state.retry_at, UTC), + ), + interval_seconds=60, + ) + else: + get_scheduler().clear_task_continuation(task_id) + + +async def recover_metadata_sweep_schedules() -> None: + """Restore interrupted batches without waiting for tomorrow's cron.""" + from pullbox.database import get_session_factory + + async with get_session_factory()() as session: + for task_id in TASK_IDS: + state = await load_sweep(session, task_id) + if state.active: + schedule_sweep(task_id, state) diff --git a/src/pullbox/tasks/metadata_task.py b/src/pullbox/tasks/metadata_task.py index 7d851db8..eb480927 100644 --- a/src/pullbox/tasks/metadata_task.py +++ b/src/pullbox/tasks/metadata_task.py @@ -11,6 +11,7 @@ from __future__ import annotations +import asyncio import time from dataclasses import dataclass from datetime import UTC, datetime, timedelta @@ -20,22 +21,28 @@ from sqlalchemy import func, or_, select from pullbox.config import PullboxSettings, get_settings +from pullbox.core.exceptions import ProviderError from pullbox.core.log_deduper import log_deduped_warning +from pullbox.core.sqlite_lock import is_sqlite_locked_error if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession from pullbox.core.comicvine_key import get_comicvine_api_key -from pullbox.core.scheduler import get_scheduler +from pullbox.core.scheduler import TaskExecutionResult, get_scheduler from pullbox.database import get_session_factory from pullbox.models.issue import Issue, IssueStatus from pullbox.models.series import IssueCatalogState, Series, SeriesStatus -from pullbox.providers.metadata.comicvine import ComicVineProvider +from pullbox.providers.metadata.comicvine import ComicVineError, ComicVineProvider from pullbox.services.metadata_service import MetadataService +from pullbox.tasks.metadata_sweep_state import save_sweep, schedule_sweep, start_sweep logger = structlog.get_logger(__name__) _RECENT_ISSUE_SYNC_LIMIT = 100 +_METADATA_BATCH_SIZE = 25 +_METADATA_BATCH_SECONDS = 120.0 +_METADATA_SERIES_SECONDS = 900.0 _STANDARD_ISSUE_CHECK_INTERVAL = timedelta(hours=24) _ENDED_MONITORED_ISSUE_CHECK_INTERVAL = timedelta(days=14) _ENDED_UNMONITORED_ISSUE_CHECK_INTERVAL = timedelta(days=30) @@ -217,241 +224,237 @@ async def _sync_issue_catalog_for_series( return created, mode -async def sync_new_issues() -> None: - """Fetch issue lists from ComicVine for all ComicVine-backed series. +async def _sync_one_series( + metadata_svc: MetadataService, + session: AsyncSession, + series: Series, + *, + refresh_days: int, + local_issue_count: int, +) -> tuple[list[Issue], str, bool, bool]: + before = _take_snapshot(series) + log = logger.bind(series_id=series.id, title=series.title) + status_changed = metadata_changed = False + if series.comicvine_id and _metadata_refresh_due(series, refresh_days): + await metadata_svc.fetch_series(session, series.comicvine_id, download_cover=False) + # Provider waits must never retain the publisher/series writer lock. + await session.commit() + await session.refresh(series) + status_changed, metadata_changed = _detect_changes(series, before, log) + if series.cover_url: + await metadata_svc.download_series_cover(series, series.cover_url) + await session.commit() + + if not _issue_catalog_check_due(series): + return [], "skipped", status_changed, metadata_changed + created, mode = await _sync_issue_catalog_for_series( + metadata_svc, + session, + series, + full_refresh_days=refresh_days, + local_issue_count=local_issue_count, + ) + return created, mode, status_changed, metadata_changed + + +def _provider_pause_seconds(exc: Exception) -> float | None: + if is_sqlite_locked_error(exc): + return 60 + if isinstance(exc, TimeoutError): + return 300 + details = exc.details or {} if isinstance(exc, ProviderError) else {} + status = exc.status_code if isinstance(exc, ComicVineError) else details.get("status_code") + retryable = exc.retryable if isinstance(exc, ComicVineError) else details.get("retryable") + if status in {100, 107, 401, 403, 420, 429}: + retry = ( + exc.retry_after_seconds + if isinstance(exc, ComicVineError) + else details.get("retry_after_seconds") + ) + return float(retry) if retry else 3600 + return 300 if retryable else None + - Also refreshes stale series metadata and detects status/field changes. - """ +async def _run_metadata_sweep(task_id: str) -> TaskExecutionResult: settings = get_settings() factory = get_session_factory() - + series_to_search: list[int] = [] + started = time.monotonic() async with factory() as session: api_key = await get_comicvine_api_key(session) if not api_key: log_deduped_warning( logger, - "sync_new_issues_missing_comicvine_key", - key="sync_new_issues_missing_comicvine_key", - action_required="Configure a ComicVine API key to enable issue sync.", + f"{task_id}_missing_comicvine_key", + key=f"{task_id}_missing_comicvine_key", + action_required="Configure a ComicVine API key to enable metadata sync.", ) - return - + return TaskExecutionResult(status="completed") + + state = await start_sweep(session, task_id) + schedule_sweep(task_id, state) + if state.retry_at > datetime.now(UTC).timestamp(): + schedule_sweep(task_id, state) + return TaskExecutionResult(status="waiting") + + refresh_days = _metadata_refresh_days(settings) + predicates = [ + Series.comicvine_id.isnot(None), + Series.id > state.cursor, + Series.id <= state.upper_bound, + ] + if task_id == "refresh_metadata": + predicates.extend( + [ + Series.monitored.is_(True), + or_( + Series.metadata_last_refreshed.is_(None), + Series.metadata_last_refreshed + < datetime.now(UTC) - timedelta(days=refresh_days), + ), + ] + ) + ids = list( + ( + await session.scalars( + select(Series.id) + .where(*predicates) + .order_by(Series.id) + .limit(_METADATA_BATCH_SIZE + 1) + ) + ).all() + ) + if not ids: + state.active = False + state.retry_at = 0 + await save_sweep(session, task_id, state) + await session.commit() + schedule_sweep(task_id, state) + return TaskExecutionResult(status="completed") + + counts = { + int(series_id): int(count) + for series_id, count in ( + await session.execute( + select(Issue.series_id, func.count(Issue.id)) + .where(Issue.series_id.in_(ids[:_METADATA_BATCH_SIZE])) + .group_by(Issue.series_id) + ) + ).all() + } metadata_svc = await _create_metadata_service(api_key, settings, session) - metadata_refresh_days = _metadata_refresh_days(settings) + await session.commit() + processed = failed = new_issues = 0 + paused = False try: - # Process ALL series — sync_new_issues is monitoring-flag-independent. - # Load stable IDs up front so we can commit per-series without relying - # on long-lived ORM instances that would otherwise keep one write - # transaction open for the entire run. - result = await session.execute(select(Series.id).where(Series.comicvine_id.isnot(None))) - series_ids = list(result.scalars().all()) - - if not series_ids: - logger.debug("sync_new_issues_skip", reason="no series") - return - - local_counts_result = await session.execute( - select(Issue.series_id, func.count(Issue.id)) - .where(Issue.series_id.in_(series_ids)) - .group_by(Issue.series_id) - ) - local_issue_counts = { - int(series_id): int(count) - for series_id, count in local_counts_result.all() - if series_id is not None - } - - new_issues = 0 - status_changes = 0 - metadata_updates = 0 - failed = 0 - full_issue_syncs = 0 - recent_issue_syncs = 0 - skipped_issue_syncs = 0 - series_to_search: list[int] = [] - - for series_id in series_ids: + for series_id in ids[:_METADATA_BATCH_SIZE]: + if processed and time.monotonic() - started >= _METADATA_BATCH_SECONDS: + break series = await session.get(Series, series_id) if series is None: + state.cursor = series_id + await save_sweep(session, task_id, state) + await session.commit() + processed += 1 continue - - log = logger.bind(series_id=series.id, title=series.title) try: - # Snapshot current metadata before refresh - before = _take_snapshot(series) - - # Refresh series metadata only when stale. Issue-list sync - # below still runs every pass so new issue discovery is - # unchanged. - if series.comicvine_id and _metadata_refresh_due( - series, - metadata_refresh_days, - ): - await metadata_svc.fetch_series(session, series.comicvine_id) - # Re-fetch the series to see updated fields - await session.refresh(series) - - sc, mc = _detect_changes(series, before, log) - if sc: - status_changes += 1 - if mc: - metadata_updates += 1 - - # Fetch new issues. Complete catalogs use a one-page recent - # sync between periodic full refreshes; incomplete/stale - # catalogs still fetch the full list. - if not _issue_catalog_check_due(series): - skipped_issue_syncs += 1 - log.debug( - "sync_new_issues_issue_check_skipped", - issue_catalog_last_checked_at=( - series.issue_catalog_last_checked_at.isoformat() - if series.issue_catalog_last_checked_at - else None - ), - issue_check_interval_seconds=( - _issue_check_interval_for_series(series).total_seconds() - ), - ) - await session.commit() - continue - - created, issue_sync_mode = await _sync_issue_catalog_for_series( - metadata_svc, - session, - series, - full_refresh_days=metadata_refresh_days, - local_issue_count=local_issue_counts.get(series.id, 0), - ) - if issue_sync_mode == "full": - full_issue_syncs += 1 - else: - recent_issue_syncs += 1 - new_issues += len(created) - - # New issues on monitored series → WANTED, unmonitored → SKIPPED (default) - if created and series.monitored: - new_wanted_ids: list[int] = [] - for issue in created: - if issue.status == IssueStatus.SKIPPED: - issue.status = IssueStatus.WANTED - new_wanted_ids.append(issue.id) - - if new_wanted_ids: - series_to_search.append(series.id) - log.debug( - "new_issues_marked_wanted", - new_wanted=len(new_wanted_ids), + previous_cursor = state.cursor + search_after_commit = False + async with asyncio.timeout(_METADATA_SERIES_SECONDS): + if task_id == "refresh_metadata": + await metadata_svc.refresh_series( + session, + series_id, + commit_before_provider_wait=True, ) - - # Release SQLite's writer lock after each series so other - # background tasks, including scheduler stat persistence, - # are not blocked behind one long metadata sync. + else: + created, _mode, _sc, _mc = await _sync_one_series( + metadata_svc, + session, + series, + refresh_days=refresh_days, + local_issue_count=counts.get(series_id, 0), + ) + new_issues += len(created) + if created and series.monitored: + wanted = False + for issue in created: + if issue.status == IssueStatus.SKIPPED: + issue.status = IssueStatus.WANTED + wanted = True + if wanted: + search_after_commit = True + state.cursor = series_id + state.retry_at = 0 + await save_sweep(session, task_id, state) await session.commit() - - except Exception: + if search_after_commit: + series_to_search.append(series_id) + processed += 1 + except Exception as exc: await session.rollback() + state.cursor = previous_cursor + # Rollback expires ORM objects; use the stable ID, not their fields. + pause_seconds = _provider_pause_seconds(exc) + if pause_seconds is not None: + state.retry_at = ( + datetime.now(UTC) + timedelta(seconds=pause_seconds) + ).timestamp() + await save_sweep(session, task_id, state) + await session.commit() + paused = True + logger.warning( + "metadata_sweep_paused", + task_id=task_id, + series_id=series_id, + retry_seconds=pause_seconds, + failure_type=type(exc).__name__, + ) + break failed += 1 - log.exception("sync_new_issues_series_failed") - logger.info( - "sync_new_issues_complete", - new_issues=new_issues, - status_changes=status_changes, - metadata_updates=metadata_updates, - series_checked=len(series_ids), - full_issue_syncs=full_issue_syncs, - recent_issue_syncs=recent_issue_syncs, - skipped_issue_syncs=skipped_issue_syncs, - failed=failed, - ) - - # Schedule one-shot searches for series with new wanted issues - if series_to_search: - from pullbox.tasks.search_task import search_series_issues - - scheduler = get_scheduler() - for sid in series_to_search: - job_id = f"search_new_{sid}_{int(time.time())}" - scheduler._scheduler.add_job( - search_series_issues, - trigger="date", - args=[sid], - id=job_id, - misfire_grace_time=300, - ) - logger.info( - "scheduled_search_for_new_issues", - series_count=len(series_to_search), - series_ids=series_to_search, - ) - except Exception: - await session.rollback() - raise - - -async def refresh_metadata() -> None: - """Re-fetch metadata for series that are stale or have never been refreshed.""" - settings = get_settings() - factory = get_session_factory() - - async with factory() as session: - api_key = await get_comicvine_api_key(session) - if not api_key: - log_deduped_warning( - logger, - "refresh_metadata_missing_comicvine_key", - key="refresh_metadata_missing_comicvine_key", - action_required="Configure a ComicVine API key to enable metadata refresh.", + processed += 1 + state.cursor = series_id + await save_sweep(session, task_id, state) + await session.commit() + logger.exception(f"{task_id}_series_failed", series_id=series_id) + + state.active = paused or processed < len(ids) + await save_sweep(session, task_id, state) + await session.commit() + finally: + close = getattr(metadata_svc, "close", None) + if close is not None: + await close() + + if series_to_search: + from pullbox.tasks.search_task import search_series_issues + + for sid in series_to_search: + get_scheduler()._scheduler.add_job( + search_series_issues, + trigger="date", + args=[sid], + id=f"search_new_{sid}_{int(time.time())}", + misfire_grace_time=300, ) - return + schedule_sweep(task_id, state) + logger.info( + f"{task_id}_batch_complete", + series_checked=processed, + new_issues=new_issues, + failed=failed, + cursor=state.cursor, + upper_bound=state.upper_bound, + waiting=state.active, + ) + return TaskExecutionResult(status="waiting" if state.active else "completed") - metadata_svc = await _create_metadata_service(api_key, settings, session) - try: - cutoff = datetime.now(UTC) - timedelta(days=settings.metadata_refresh_days) - result = await session.execute( - select(Series.id).where( - Series.monitored.is_(True), - or_( - Series.metadata_last_refreshed.is_(None), - Series.metadata_last_refreshed < cutoff, - ), - ) - ) - stale_ids = list(result.scalars().all()) +async def sync_new_issues() -> TaskExecutionResult: + """Resume a bounded all-series issue sweep without monopolizing the scheduler.""" + return await _run_metadata_sweep("sync_new_issues") - if not stale_ids: - logger.debug("refresh_metadata_skip", reason="no stale series") - return - refreshed = 0 - failed = 0 - for series_id in stale_ids: - series = await session.get(Series, series_id) - if series is None: - continue - series_title = series.title - try: - await metadata_svc.refresh_series(session, series.id) - refreshed += 1 - # Release SQLite's writer lock after each series refresh so - # this nightly job doesn't monopolize the DB for the entire batch. - await session.commit() - except Exception: - await session.rollback() - failed += 1 - logger.exception( - "refresh_metadata_series_failed", - series_id=series_id, - title=series_title, - ) - - logger.info( - "refresh_metadata_complete", - refreshed=refreshed, - failed=failed, - total=len(stale_ids), - ) - except Exception: - await session.rollback() - raise +async def refresh_metadata() -> TaskExecutionResult: + """Resume a bounded sweep of stale monitored-series metadata.""" + return await _run_metadata_sweep("refresh_metadata") diff --git a/tests/tasks/test_metadata_search_trigger.py b/tests/tasks/test_metadata_search_trigger.py index b040fb4b..17d6b629 100644 --- a/tests/tasks/test_metadata_search_trigger.py +++ b/tests/tasks/test_metadata_search_trigger.py @@ -12,6 +12,7 @@ from __future__ import annotations +import asyncio import contextlib import os import sys @@ -36,6 +37,104 @@ _MOD = "pullbox.tasks.metadata_task" +async def test_interrupted_sync_restarts_at_first_uncommitted_series(db_factory): + from pullbox.tasks import metadata_task + + ids = [await _create_series(db_factory, comicvine_id=i) for i in (91001, 91002, 91003)] + blocked = asyncio.Event() + svc = _make_metadata_svc([]) + + async def fetch(session, series_id): + if series_id == ids[1]: + blocked.set() + await asyncio.Event().wait() + return [] + + svc.fetch_issues_for_series.side_effect = fetch + with _sync_patches(db_factory, svc, _make_scheduler()): + task = asyncio.create_task(metadata_task.sync_new_issues()) + await asyncio.wait_for(blocked.wait(), 2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + svc.fetch_issues_for_series.side_effect = None + svc.fetch_issues_for_series.return_value = [] + await metadata_task.sync_new_issues() + assert [call.args[1] for call in svc.fetch_series.await_args_list] == [ + 91001, + 91002, + 91002, + 91003, + ] + + +async def test_sync_resumes_bounded_batches_from_durable_cursor(db_factory, monkeypatch): + from pullbox.tasks import metadata_task + + for identifier in (91001, 91002, 91003): + await _create_series(db_factory, comicvine_id=identifier) + svc = _make_metadata_svc([]) + scheduler = _make_scheduler() + monkeypatch.setattr(metadata_task, "_METADATA_BATCH_SIZE", 2, raising=False) + with _sync_patches(db_factory, svc, scheduler): + result = await metadata_task.sync_new_issues() + assert svc.fetch_series.await_count == 2, "A run must yield after a bounded batch" + assert result.status == "waiting" + await metadata_task.sync_new_issues() + assert [call.args[1] for call in svc.fetch_series.await_args_list] == [91001, 91002, 91003] + scheduler.schedule_task_continuation.assert_called() + + +async def test_sync_pauses_whole_sweep_after_provider_throttle(db_factory): + from pullbox.core.exceptions import ProviderError + from pullbox.tasks import metadata_task + + for identifier in (91001, 91002, 91003): + await _create_series(db_factory, comicvine_id=identifier) + svc = _make_metadata_svc([]) + svc.fetch_series.side_effect = ProviderError( + "comicvine", "HTTP 420", details={"status_code": 420, "retryable": True} + ) + with _sync_patches(db_factory, svc, _make_scheduler()): + result = await metadata_task.sync_new_issues() + assert svc.fetch_series.await_count == 1, ( + "One throttle must pause the provider, not fail every series" + ) + assert result.status == "waiting" + await metadata_task.sync_new_issues() + assert svc.fetch_series.await_count == 1, "Persisted cooldown must survive a fresh batch" + + +async def test_sync_commits_metadata_before_waiting_for_issue_provider(db_factory): + from pullbox.tasks import metadata_task + + sid = await _create_series(db_factory, comicvine_id=91001) + svc = _make_metadata_svc([]) + tracking = _TrackingFactory(db_factory) + commits_after_metadata = [] + + async def write_series(session, identifier, **kwargs): + series = await session.get(Series, sid) + series.description = "Refreshed metadata" + await session.flush() + commits_after_metadata.append(tracking.commit_calls) + + async def check_issue_fetch(session, series_id): + assert tracking.commit_calls > commits_after_metadata[-1], ( + "Provider wait still holds metadata's write transaction" + ) + return [] + + svc.fetch_series.side_effect = write_series + svc.fetch_issues_for_series.side_effect = check_issue_fetch + with _sync_patches(tracking, svc, _make_scheduler()): + await metadata_task.sync_new_issues() + assert svc.fetch_series.await_args.kwargs.get("download_cover") is False + async with db_factory() as session: + series = await session.get(Series, sid) + assert series.issue_catalog_last_checked_at is not None + + # ── Fixtures ─────────────────────────────────────────────────────────── @@ -102,6 +201,7 @@ def _sync_patches( return_value=metadata_svc, ), patch(f"{_MOD}.get_scheduler", return_value=scheduler), + patch("pullbox.tasks.metadata_sweep_state.get_scheduler", return_value=scheduler), ): mock_settings.return_value = MagicMock() yield mock_settings diff --git a/tests/tasks/test_scheduler_wrappers.py b/tests/tasks/test_scheduler_wrappers.py index 04869f95..aa90d98a 100644 --- a/tests/tasks/test_scheduler_wrappers.py +++ b/tests/tasks/test_scheduler_wrappers.py @@ -376,7 +376,7 @@ async def test_refresh_metadata_commits_each_series_and_continues_after_failures failing_id = series_two.id fresh_id = fresh_series.id - async def _refresh_series(session: AsyncSession, series_id: int) -> None: + async def _refresh_series(session: AsyncSession, series_id: int, **kwargs: object) -> None: series = await session.get(Series, series_id) assert series is not None if series_id == failing_id: diff --git a/tests/unit/test_comicvine_cooldown.py b/tests/unit/test_comicvine_cooldown.py new file mode 100644 index 00000000..614f830e --- /dev/null +++ b/tests/unit/test_comicvine_cooldown.py @@ -0,0 +1,37 @@ +"""Provider throttling must apply across ComicVine clients and resource types.""" + +import httpx +import pytest + +from pullbox.providers.metadata.comicvine import ComicVineError, ComicVineProvider + + +@pytest.mark.parametrize("status,body", [(420, {}), (429, {}), (200, {"status_code": 107})]) +async def test_throttle_stops_other_clients_and_other_resources(status, body): + requests = [] + + def respond(request): + requests.append(request) + return httpx.Response(status, json=body, headers={"Retry-After": "7200"}) + + first = ComicVineProvider("same-account") + second = ComicVineProvider("same-account", rate_limit=100) + await first._client.aclose() + await second._client.aclose() + first._client = httpx.AsyncClient( + base_url="https://comicvine.test", transport=httpx.MockTransport(respond) + ) + second._client = httpx.AsyncClient( + base_url="https://comicvine.test", transport=httpx.MockTransport(respond) + ) + try: + with pytest.raises(ComicVineError): + await first._request("/issues/") + with pytest.raises(ComicVineError) as caught: + await second._request("/volume/4050-1/") + assert len(requests) == 1, "A second client ignored the account's throttle" + assert caught.value.retryable + assert caught.value.retry_after_seconds >= 7190 + finally: + await first.close() + await second.close() From 2b84de1d5225f5fd0b4d85e09f093c76e7624470 Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 00:20:25 -0700 Subject: [PATCH 06/20] fix(indexers): share throttle cooldowns and serialize provider requests Honor HTTP and XML rate limits, back off failed requests, preserve account/provider isolation, and prevent concurrent clients bypassing request pacing. TDD regressions and 27 focused provider tests pass. --- src/pullbox/core/provider_cooldown.py | 2 + src/pullbox/providers/indexer/newznab.py | 68 ++++++++++++-- tests/providers/test_newznab_cooldown.py | 111 +++++++++++++++++++++++ 3 files changed, 175 insertions(+), 6 deletions(-) create mode 100644 tests/providers/test_newznab_cooldown.py diff --git a/src/pullbox/core/provider_cooldown.py b/src/pullbox/core/provider_cooldown.py index 7e3babad..9ab81283 100644 --- a/src/pullbox/core/provider_cooldown.py +++ b/src/pullbox/core/provider_cooldown.py @@ -13,6 +13,8 @@ def retry_after_seconds(header: str | None, *, default: float) -> float: """Accept seconds or an HTTP date; reject malformed and unbounded values.""" + if not isinstance(header, str): + return default try: delay = float(header or "") except ValueError: diff --git a/src/pullbox/providers/indexer/newznab.py b/src/pullbox/providers/indexer/newznab.py index 3e2876fb..def9c9dc 100644 --- a/src/pullbox/providers/indexer/newznab.py +++ b/src/pullbox/providers/indexer/newznab.py @@ -4,7 +4,7 @@ Provides the shared base logic (XML parsing, capabilities, search) that the Torznab indexer also builds on. -Newznab API spec: https://newznab.readthedocs.io/en/latest/misc/api/ +Newznab API spec: https://newznab.readthedocs.io/en/latest/misc/api.html """ from __future__ import annotations @@ -22,6 +22,7 @@ from pullbox.core.acquisition import AcquisitionProtocol from pullbox.core.issue_numbers import format_issue_number +from pullbox.core.provider_cooldown import ProviderCooldown, provider_cooldown, retry_after_seconds from pullbox.providers.base import ( IndexerCapabilities, ProviderHealthResult, @@ -43,6 +44,17 @@ class NewznabError(Exception): """Raised when a Newznab API request fails.""" + def __init__( + self, + message: str, + *, + status_code: str | None = None, + retry_after_seconds: int | None = None, + ) -> None: + super().__init__(message) + self.status_code = status_code + self.retry_after_seconds = retry_after_seconds + class NewznabIndexer: """Newznab implementation of the Indexer protocol. @@ -94,17 +106,37 @@ async def _wait_for_rate_limit(self) -> None: """Simple rate limiter: enforce minimum interval between requests.""" import asyncio + cooldown = self._cooldown() now = time.monotonic() - elapsed = now - self._last_request_time + elapsed = now - max(self._last_request_time, cooldown.last_request_time) if elapsed < self._min_interval: await asyncio.sleep(self._min_interval - elapsed) self._last_request_time = time.monotonic() + cooldown.last_request_time = self._last_request_time # -- internal request plumbing ------------------------------------------ async def _request(self, params: dict[str, Any]) -> str: """Make a rate-limited GET request, returning raw XML text.""" + cooldown = self._cooldown() + self._check_cooldown(cooldown) + async with cooldown.request_lock: + self._check_cooldown(cooldown) + return await self._request_serialized(params, cooldown) + + def _cooldown(self) -> ProviderCooldown: + return provider_cooldown("newznab", f"{self._base_url}\0{self._api_key}") + + def _check_cooldown(self, cooldown: ProviderCooldown) -> None: + if cooldown.remaining_seconds: + raise NewznabError( + f"{self._name}: provider cooling down; retry in {cooldown.remaining_seconds}s", + retry_after_seconds=cooldown.remaining_seconds, + ) + + async def _request_serialized(self, params: dict[str, Any], cooldown: ProviderCooldown) -> str: await self._wait_for_rate_limit() + self._check_cooldown(cooldown) request_params: dict[str, Any] = ( {"apikey": self._api_key, **params} if self._api_key else dict(params) @@ -118,19 +150,43 @@ async def _request(self, params: dict[str, Any]) -> str: response = await self._client.get(url, params=request_params) response.raise_for_status() except httpx.TimeoutException: + cooldown.defer(60) log.error("newznab_timeout") - raise NewznabError(f"Request timed out: {self._name}") from None + raise NewznabError( + f"Request timed out: {self._name}", retry_after_seconds=cooldown.remaining_seconds + ) from None except httpx.HTTPStatusError as exc: log.error("newznab_http_error", status=exc.response.status_code) - raise NewznabError(f"HTTP {exc.response.status_code}") from None + status = exc.response.status_code + if status == 429 or status >= 500: + cooldown.defer( + retry_after_seconds( + exc.response.headers.get("Retry-After"), + default=900 if status == 429 else 60, + ) + ) + raise NewznabError( + f"HTTP {status}", + status_code=str(status), + retry_after_seconds=cooldown.remaining_seconds or None, + ) from None except httpx.HTTPError as exc: + cooldown.defer(60) log.error("newznab_request_failed", error=str(exc)) raise NewznabError(f"Request failed: {exc}") from None # Check for Newznab XML error responses text = response.text if " None: if error_el is not None: code = error_el.get("code", "?") description = error_el.get("description", "Unknown error") - raise NewznabError(f"{indexer_name}: error {code} — {description}") + raise NewznabError(f"{indexer_name}: error {code} — {description}", status_code=code) except (ElementTree.ParseError, DefusedXmlException): pass diff --git a/tests/providers/test_newznab_cooldown.py b/tests/providers/test_newznab_cooldown.py new file mode 100644 index 00000000..a2e45dee --- /dev/null +++ b/tests/providers/test_newznab_cooldown.py @@ -0,0 +1,111 @@ +"""Search retries must respect provider cooldowns across concurrent clients.""" + +import asyncio + +import httpx +import pytest + +from pullbox.core.provider_cooldown import retry_after_seconds +from pullbox.providers.indexer.newznab import NewznabError, NewznabIndexer + + +async def client(handler, *, url="https://indexer.test", key="account"): + result = NewznabIndexer("Test", url, key, rate_limit_per_minute=6000) + await result._client.aclose() + result._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + return result + + +@pytest.mark.parametrize("failure", ["429", "timeout", "xml500"]) +async def test_failed_provider_is_not_hammered_by_next_client(failure): + requests = [] + + def respond(request): + requests.append(request) + if failure == "timeout": + raise httpx.ReadTimeout("slow", request=request) + if failure == "xml500": + return httpx.Response( + 200, + text='', + headers={"Retry-After": "900"}, + ) + return httpx.Response(429, headers={"Retry-After": "900"}) + + first, second = await client(respond), await client(respond) + try: + with pytest.raises(NewznabError): + await first._request({"t": "search"}) + with pytest.raises(NewznabError) as caught: + await second._request({"t": "search"}) + assert len(requests) == 1, "A fresh client ignored the provider's cooldown" + assert caught.value.retry_after_seconds > 0 + finally: + await first.close() + await second.close() + + +async def test_concurrent_clients_share_one_provider_request_slot(): + requests = [] + entered = asyncio.Event() + release = asyncio.Event() + + async def respond(request): + requests.append(request) + entered.set() + await release.wait() + return httpx.Response(200, text="") + + first, second = await client(respond), await client(respond) + tasks = [] + try: + tasks.append(asyncio.create_task(first._request({"t": "search"}))) + await entered.wait() + tasks.append(asyncio.create_task(second._request({"t": "search"}))) + await asyncio.sleep(0.02) + assert len(requests) == 1, "Concurrent searches bypassed shared provider pacing" + finally: + release.set() + await asyncio.gather(*tasks) + await first.close() + await second.close() + + +async def test_throttled_indexer_does_not_block_another_provider(): + first = await client(lambda request: httpx.Response(429)) + second = await client( + lambda request: httpx.Response(200, text=""), url="https://other.test" + ) + try: + with pytest.raises(NewznabError): + await first._request({"t": "search"}) + assert await second._request({"t": "search"}) == "" + finally: + await first.close() + await second.close() + + +async def test_expired_cooldown_allows_provider_recovery(): + responses = [httpx.Response(429), httpx.Response(200, text="")] + indexer = await client(lambda request: responses.pop(0)) + try: + with pytest.raises(NewznabError): + await indexer._request({"t": "search"}) + indexer._cooldown().until = 0 + assert await indexer._request({"t": "search"}) == "" + finally: + await indexer.close() + + +@pytest.mark.parametrize("header", [None, "garbage", "-1", "nan", "inf"]) +def test_malformed_retry_after_uses_safe_fallback(header): + assert retry_after_seconds(header, default=60) == 60 + + +def test_retry_after_accepts_seconds_and_http_dates(): + from datetime import UTC, datetime, timedelta + from email.utils import format_datetime + + assert retry_after_seconds("120", default=900) == 120 + date = format_datetime(datetime.now(UTC) + timedelta(hours=2), usegmt=True) + assert 7190 <= retry_after_seconds(date, default=60) <= 7200 From 0a281d8bf6e25b1cb331779062b6032e00e664fb Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 00:22:09 -0700 Subject: [PATCH 07/20] fix(backup): release request transactions before maintenance Keep manual backup and restore compatible with bounded maintenance admission and return actionable busy responses. TDD regression asserts commit and close occur before the worker starts. --- src/pullbox/api/v1/health.py | 3 ++- src/pullbox/api/v1/system.py | 20 +++++++++++++++-- tests/unit/test_system_backup_routes.py | 30 ++++++++++++++++++++++--- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/pullbox/api/v1/health.py b/src/pullbox/api/v1/health.py index 1767b6f6..da31d37b 100644 --- a/src/pullbox/api/v1/health.py +++ b/src/pullbox/api/v1/health.py @@ -25,6 +25,7 @@ import pullbox from pullbox.api.deps import DbSession, InteractiveOperatorUser # noqa: TC001 from pullbox.core.exceptions import ValidationError +from pullbox.database import DatabaseMaintenanceBusyError from pullbox.models.health import HealthCurrentStatus as HealthCurrentStatusModel from pullbox.models.health import HealthStatus from pullbox.models.import_job import ImportJob, ImportJobStatus @@ -430,7 +431,7 @@ async def optimize_database( try: result = await DatabaseOptimizationRuntimeService(db_path).optimize() - except DatabaseOptimizationError as exc: + except (DatabaseOptimizationError, DatabaseMaintenanceBusyError) as exc: raise ValidationError(str(exc)) from exc await run_health_refresh(component="database") diff --git a/src/pullbox/api/v1/system.py b/src/pullbox/api/v1/system.py index 10119529..9bfb3bca 100644 --- a/src/pullbox/api/v1/system.py +++ b/src/pullbox/api/v1/system.py @@ -49,6 +49,7 @@ from pullbox.core.config_resolver import load_system_config_values from pullbox.core.shutdown import shutdown_manager from pullbox.core.sqlite_lock import is_sqlite_locked_error +from pullbox.database import DatabaseMaintenanceBusyError from pullbox.models.config import DEFAULT_SYSTEM_CONFIG, SystemConfig from pullbox.services.backup_runtime_service import BackupRuntimeService from pullbox.services.backup_service import BackupService @@ -279,7 +280,14 @@ async def create_backup( ) -> BackupCreatedResponse: """Create a manual backup of the Pullbox database.""" svc = await _get_backup_runtime_service(session) - info = await svc.create_backup(backup_type="manual") + await session.commit() + await session.close() + try: + info = await svc.create_backup(backup_type="manual") + except DatabaseMaintenanceBusyError as exc: + raise HTTPException( + status_code=503, detail=str(exc), headers={"Retry-After": "60"} + ) from exc return BackupCreatedResponse( message=f"Backup created: {info.filename}", backup=BackupResponse( @@ -383,7 +391,15 @@ async def restore_backup( raise ValidationError(f"Invalid backup filename: {filename}") svc = await _get_backup_runtime_service(session) - if not await svc.restore_backup(filename): + await session.commit() + await session.close() + try: + restored = await svc.restore_backup(filename) + except DatabaseMaintenanceBusyError as exc: + raise HTTPException( + status_code=503, detail=str(exc), headers={"Retry-After": "60"} + ) from exc + if not restored: from pullbox.core.exceptions import NotFoundError raise NotFoundError("Backup", filename) diff --git a/tests/unit/test_system_backup_routes.py b/tests/unit/test_system_backup_routes.py index dd9c8349..3e619e97 100644 --- a/tests/unit/test_system_backup_routes.py +++ b/tests/unit/test_system_backup_routes.py @@ -4,6 +4,7 @@ from pathlib import Path from types import SimpleNamespace +from unittest.mock import AsyncMock import pytest @@ -34,6 +35,29 @@ async def restore_backup(self, filename: str) -> bool: class TestSystemBackupRoutes: + @pytest.mark.parametrize("operation", ["create", "restore"]) + async def test_manual_maintenance_releases_request_transaction(self, monkeypatch, operation): + from pullbox.api.v1 import system + + session = AsyncMock() + runtime = _StubRuntimeService() + original_create = runtime.create_backup + + async def check_released(*args, **kwargs): + session.commit.assert_awaited_once() + session.close.assert_awaited_once() + return await original_create(backup_type="manual") if operation == "create" else False + + runtime.create_backup = check_released + runtime.restore_backup = check_released + monkeypatch.setattr(system, "_get_backup_runtime_service", AsyncMock(return_value=runtime)) + if operation == "restore": + with pytest.raises(NotFoundError): + await system.restore_backup("backup.zip", object(), session) + else: + response = await system.create_backup(object(), session) + assert response.backup.backup_type == "manual" + @pytest.mark.asyncio async def test_get_backup_runtime_service_uses_runtime_paths( self, @@ -67,7 +91,7 @@ async def _fake_get_runtime_service(_session: object) -> _StubRuntimeService: monkeypatch.setattr(system, "_get_backup_runtime_service", _fake_get_runtime_service) - response = await system.create_backup(object(), object()) + response = await system.create_backup(object(), AsyncMock()) assert runtime.create_calls == ["manual"] assert response.message == "Backup created: pullbox_backup_20260502_120000.zip" @@ -97,7 +121,7 @@ async def _fake_get_runtime_service(_session: object) -> _StubRuntimeService: response = await system.restore_backup( "pullbox_backup_20260502_120000.zip", object(), - object(), + AsyncMock(), ) assert runtime.restore_calls == ["pullbox_backup_20260502_120000.zip"] @@ -122,7 +146,7 @@ async def _fake_get_runtime_service(_session: object) -> _StubRuntimeService: monkeypatch.setattr(system, "_get_backup_runtime_service", _fake_get_runtime_service) with pytest.raises(NotFoundError): - await system.restore_backup("pullbox_backup_20260502_120000.zip", object(), object()) + await system.restore_backup("pullbox_backup_20260502_120000.zip", object(), AsyncMock()) @pytest.mark.asyncio async def test_restore_backup_route_rejects_unsafe_filename_before_service_lookup( From 22023bbd49b503c688a1428a6186f5de5f4cae93 Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 00:44:20 -0700 Subject: [PATCH 08/20] docs: record background maintenance and provider retry contracts Document bounded maintenance admission, cancellation-safe worker fencing, resumable metadata sweeps, and shared provider cooldowns. Full make ci-full passed: 11576 regression tests (91% coverage), 25 accessibility checks, 575 Chromium and 575 Firefox checks, container security, and 7 Docker smoke tests. Existing documented security exceptions remain unchanged. --- docs/development/ARCHITECTURE_OVERVIEW.md | 18 ++++++++++++++++++ docs/development/DATABASE_STANDARDS.md | 12 +++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/development/ARCHITECTURE_OVERVIEW.md b/docs/development/ARCHITECTURE_OVERVIEW.md index b9949ba7..b81cb26b 100644 --- a/docs/development/ARCHITECTURE_OVERVIEW.md +++ b/docs/development/ARCHITECTURE_OVERVIEW.md @@ -553,6 +553,24 @@ task registry. The scheduler runs recurring jobs for search, downloads, metadata refresh, health checks, imports, backups, dashboard refresh, blocklist cleanup, and related operational work. +Exclusive scheduled maintenance reserves admission in memory, outside database +stats writes. It waits at most five seconds for other tasks, then releases its +reservation and schedules a retry after 60 seconds if work is still active. +Cancellation also releases the reservation. + +Nightly issue and metadata sweeps checkpoint their last completed series and +initial upper bound in `SystemConfig`. Each batch handles at most 25 series and +checks a two-minute budget between series; an individual series has a separate +15-minute timeout. Pending batches resume through hidden continuations, including +after restart. Provider throttling pauses the sweep at its saved position instead +of repeatedly failing every remaining series. Series metadata writes are committed +before subsequent cover or issue-provider waits. + +ComicVine and Newznab clients share process-local account cooldowns, so creating a +new client does not bypass a throttle response. Newznab also serializes request +pacing per provider/account; unrelated accounts remain independent. Sweep retry +deadlines are durable, while the shared client cooldown registry itself is not. + The event bus is intentionally small and in-process. It supports domain side effects such as: diff --git a/docs/development/DATABASE_STANDARDS.md b/docs/development/DATABASE_STANDARDS.md index b011280e..ca5091e2 100644 --- a/docs/development/DATABASE_STANDARDS.md +++ b/docs/development/DATABASE_STANDARDS.md @@ -544,13 +544,23 @@ not suppress failures in those checks. - Database maintenance windows coordinate app traffic through the shared maintenance gate. -- Gate-aware sessions pause before database-touching operations. +- Gate-aware sessions pause new transactions while existing transactions have + up to five seconds to finish. If they cannot drain, maintenance yields with + a retryable busy result rather than holding the gate indefinitely. +- Manual backup, restore, and optimization requests release their own request + transaction before entering maintenance. Cancellation does not reopen the + gate until an already-running filesystem or SQLite worker has stopped. - An exclusive nightly task runs SQLite `REINDEX` and `PRAGMA optimize=0x10002` at 04:30, then verifies the database with `PRAGMA quick_check`. The all-tables mask is intentional because the maintenance connection has no prior query history. - Full SQLite `VACUUM` compaction remains an explicit operator action because it rewrites the database and can require substantial temporary disk space. +- Routine integrity health probes use a separate read-only SQLite connection + with a five-second execution budget. An interrupted or busy probe reports + incomplete verification, not confirmed database corruption. +- Search-log retention deletes at most 500 rows per transaction and releases + the writer between batches. **Required standard** From 267375fb8260171656c3c326c437b90990308dcd Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 11:29:06 -0700 Subject: [PATCH 09/20] fix(import): safely revalidate recovery sources and reviewed identities --- docs/development/IMPORT_REVIEW_RECOVERY.md | 21 +++ src/pullbox/services/import_file_execution.py | 7 +- src/pullbox/services/import_orphans.py | 3 + .../services/import_recovery_source.py | 143 ++++++++++++++++ src/pullbox/services/import_review_recheck.py | 72 ++++++++ tests/unit/test_import_file_execution.py | 162 ++++++++++++++++++ tests/unit/test_import_orphans.py | 17 ++ tests/unit/test_import_review_recheck.py | 60 +++++++ 8 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 src/pullbox/services/import_recovery_source.py diff --git a/docs/development/IMPORT_REVIEW_RECOVERY.md b/docs/development/IMPORT_REVIEW_RECOVERY.md index 5c1de10d..76c4fde2 100644 --- a/docs/development/IMPORT_REVIEW_RECOVERY.md +++ b/docs/development/IMPORT_REVIEW_RECOVERY.md @@ -190,6 +190,27 @@ refreshed evidence, then commit that page before reading more archives. This keeps slow archive I/O outside SQLite's single-writer window, bounds memory, and leaves completed pages durable if a later source needs another attempt. +An identity-conflicted failed file remains assignable in Follow-up; archive +safety failures do not become assignable through that exception. A manual +issue assignment may supersede a disagreement between `series.json` and +`cvinfo` only when freshly inspected ComicInfo proves the assigned issue, its +series identity does not contradict the reviewed series, and issue-number +checks pass. The original folder conflict is retained as resolved evidence; +other metadata conflicts retain their specific IDs in the failure diagnostics. +Step 4 also verifies that the assigned issue belongs to the chosen local series. + +Known-series, deferred, mixed-folder, and manually assigned recovery can +revalidate a device-number-only change after a container remount. The path, +inode, size, timestamp, and signature version must still agree. An archive +inspection under the approved source roots and an exact embedded issue-ID +check are required before refreshing the saved signature. The pre-inspection +and post-inspection identities must agree, and normal registration revalidates +the root and refreshed signature again. Existing approved size or single-page +exceptions remain scoped to that file; dangerous archive checks still run. +Ordinary signature validation, source files, Mylar metadata, and already-owned +library files are unchanged. These recovery rules apply to Mylar and folder +imports, in both copy and keep-in-place modes. + Recovery queries must not expand an entire library into SQL bind parameters. Mixed-folder lookups join existing references and discard exact same-title rows before loading archive diagnostics; the final shared identity rules still diff --git a/src/pullbox/services/import_file_execution.py b/src/pullbox/services/import_file_execution.py index 38240b72..64bb966c 100644 --- a/src/pullbox/services/import_file_execution.py +++ b/src/pullbox/services/import_file_execution.py @@ -58,6 +58,7 @@ from pullbox.services.import_placement_recovery import ( has_completed_direct_move_placement_record, ) +from pullbox.services.import_recovery_source import refresh_recovery_source from pullbox.services.import_referenced_sources import revalidate_mylar_in_place_file_root from pullbox.services.import_safety_diagnostics import build_import_safety_diagnostics from pullbox.utilities.settings import restore_file_from_utility_trash @@ -1045,6 +1046,7 @@ async def process_one_file(imp_file_id: int) -> tuple[int, int]: registration_library_root_id = None if in_place else target_library_root_id try: await raise_if_cancelled(session, job_id) + await refresh_recovery_source(session, current_job, item, imp_file) if in_place and current_job.source_type == ImportSourceType.MYLAR3: registration_library_root_id = await revalidate_mylar_in_place_file_root( session, @@ -1621,6 +1623,7 @@ async def report_current_file( continue if isinstance(exc, ReferencedFileValidationError): diagnostics = dict(imp_file.diagnostics or {}) + previous_block = diagnostics.get("source_revalidation") diagnostics["source_revalidation"] = build_import_safety_diagnostics( str(exc), kind="source_revalidation", @@ -1628,6 +1631,8 @@ async def report_current_file( source="source_revalidation", overrideable_hint=False, ) + if isinstance(previous_block, dict) and previous_block.get("code") == exc.reason: + diagnostics["source_revalidation"].update(previous_block) imp_file.status = ImportedFileStatus.FAILED imp_file.include_in_import = False imp_file.error_message = str(exc) @@ -1638,7 +1643,7 @@ async def report_current_file( job_id, "WARNING", "import_file_source_revalidation_failed", - message=f"Source changed after scan; rescan before retry: {imp_file_name}", + message=f"Source validation requires review: {imp_file_name}", source_path=imp_file_path, reason=exc.reason, ) diff --git a/src/pullbox/services/import_orphans.py b/src/pullbox/services/import_orphans.py index 3098c9f9..1191ecd0 100644 --- a/src/pullbox/services/import_orphans.py +++ b/src/pullbox/services/import_orphans.py @@ -99,6 +99,9 @@ async def add_from_comicvine( def requires_orphan_issue_decision(file: ImportedFile) -> bool: """Return whether Follow-up should ask for an issue assignment or skip.""" + if file.status is ImportedFileStatus.FAILED: + block = dict(file.diagnostics or {}).get("source_revalidation") + return isinstance(block, dict) and block.get("code") == "source_identity_changed" return file.status in { ImportedFileStatus.PENDING, ImportedFileStatus.MATCHED, diff --git a/src/pullbox/services/import_recovery_source.py b/src/pullbox/services/import_recovery_source.py new file mode 100644 index 00000000..88e4fb84 --- /dev/null +++ b/src/pullbox/services/import_recovery_source.py @@ -0,0 +1,143 @@ +"""Reinspect explicitly recovered files without relaxing ordinary source guards.""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +from sqlalchemy import select + +from pullbox.core.file_safety import ( + get_archive_size_limit_bytes, + is_dangerous_file_blocking_enabled, +) +from pullbox.core.library_file_ownership import ( + ReferencedFileValidationError, + build_file_identity_signature, + validate_file_identity_signature, +) +from pullbox.models.issue import Issue +from pullbox.models.series import Series +from pullbox.services.import_review_recheck import ( + _apply_completed_file_recheck, + _retry_source_roots, + inspect_review_source, +) +from pullbox.services.import_source_metadata import source_metadata_for_import_file + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.models.import_job import ImportedFile, ImportedSeries, ImportJob + + +async def refresh_recovery_source( + session: AsyncSession, job: ImportJob, item: ImportedSeries, file: ImportedFile +) -> None: + """Refresh a remounted source only after archive and exact issue checks pass.""" + manual = file.match_method == "orphan_recovery" + diagnostics = dict(file.diagnostics or {}) + source_metadata = diagnostics.get("source_metadata") + folder_conflict = isinstance(source_metadata, dict) and bool( + source_metadata.get("identity_conflicts") + ) + if not manual and dict(item.diagnostics or {}).get("kind") not in { + "known_series_recovery", + "deferred_recovery", + "completed_import_mixed_folder_recovery", + }: + return + previous = dict(file.source_signature or {}) + path = Path(file.file_path) + current = await asyncio.to_thread(build_file_identity_signature, path) + device_changed = previous.get("device") != current["device"] + # Only device renumbering is eligible, never a different path, inode, + # size or timestamp. The global signature validator remains strict. + expected = {**previous, "device": current["device"]} if device_changed else previous + validate_file_identity_signature(expected, current) + if device_changed and not isinstance(previous.get("device"), int): + raise ReferencedFileValidationError("source_signature_missing", "Missing source identity.") + if not folder_conflict and not device_changed: + return + + target = await session.scalar( + select(Issue) + .join(Series, Series.id == Issue.series_id) + .where( + Issue.id == file.matched_issue_id + if file.matched_issue_id + else Issue.comicvine_id == file.matched_issue_cv_id, + Series.id == item.series_id, + Series.comicvine_id == item.cv_id, + ) + ) + if target is None or target.comicvine_id is None: + raise ReferencedFileValidationError( + "source_identity_changed", "The recovery issue does not belong to the reviewed series." + ) + if file.matched_issue_cv_id not in (None, target.comicvine_id): + raise ReferencedFileValidationError( + "source_identity_changed", "Recovery issue IDs disagree." + ) + roots = await _retry_source_roots(session, job, file_ids=[file.id]) + pairs = [(root, await asyncio.to_thread(root.resolve, strict=True)) for root in roots] + block_dangerous = await is_dangerous_file_blocking_enabled(session) + max_size = await get_archive_size_limit_bytes(session) + exception = diagnostics.get("safety_exception") + previous_block = exception.get("previous_block") if isinstance(exception, dict) else None + approved_code = ( + previous_block.get("code") + if isinstance(exception, dict) + and exception.get("allowed_once") is True + and isinstance(previous_block, dict) + and previous_block.get("overrideable") is True + else None + ) + if approved_code == "archive_decompressed_size_limit": + max_size = sys.maxsize + metadata, content, signature = await asyncio.to_thread( + inspect_review_source, + path, + source_metadata_for_import_file(item, file), + expected, + roots=pairs, + block_dangerous=block_dangerous, + max_archive_size=max_size, + accept_replaced_files=False, + sidecars={}, + ) + content_block = content.get("file_safety") + if ( + approved_code == "single_page_comic" + and isinstance(content_block, dict) + and content_block.get("code") == approved_code + ): + content = {key: value for key, value in content.items() if key != "file_safety"} + embedded = metadata.diagnostics.get("embedded_identity") + if "file_safety" not in content and ( + not isinstance(embedded, dict) or embedded.get("issue_id") != target.comicvine_id + ): + raise ReferencedFileValidationError( + "source_identity_changed", + "Fresh ComicInfo does not prove the selected recovery issue. Review it in Follow-up.", + ) + ready = _apply_completed_file_recheck( + file, + metadata, + content, + signature, + reviewed_series_cv_id=item.cv_id, + ) + if not ready: + block = file.diagnostics["source_revalidation"] + raise ReferencedFileValidationError(str(block["code"]), str(block["reason"])) + file.diagnostics = { + **file.diagnostics, + "source_recheck": { + **file.diagnostics["source_recheck"], + "reason": "device_renumbered" if device_changed else "reviewed_issue_assignment", + "previous_signature": previous, + }, + } diff --git a/src/pullbox/services/import_review_recheck.py b/src/pullbox/services/import_review_recheck.py index 0fa09a57..d9b26cb5 100644 --- a/src/pullbox/services/import_review_recheck.py +++ b/src/pullbox/services/import_review_recheck.py @@ -177,6 +177,30 @@ def inspect_review_source( archive_member_evidence=evidence, ), ) + embedded = extractor.from_path( + path, + sidecar_data={ + **sidecars[folder], + "series_id": None, + "issue_id": None, + "booktype": None, + "identity_conflicts": [], + }, + archive_member_evidence=evidence, + ) + fresh = fresh.model_copy( + update={ + "diagnostics": { + **fresh.diagnostics, + "embedded_identity": { + "series_id": embedded.comicvine_series_id, + "issue_id": embedded.comicvine_issue_id, + "issue_number": embedded.issue_number, + "issue_number_text": embedded.issue_number_text, + }, + } + } + ) # Do not accept evidence from a file replaced during the inspection. validate_file_identity_signature( dict(current_signature), build_file_identity_signature(path) @@ -248,6 +272,7 @@ def _apply_completed_file_recheck( """Refresh source evidence without changing the saved import decision.""" diagnostics = dict(file.diagnostics or {}) previous_signature = dict(file.source_signature or {}) + metadata = _reviewed_folder_identity(file, metadata, reviewed_series_cv_id) source = {**metadata.diagnostics, **content} block = _completed_file_recheck_block( file, @@ -312,6 +337,8 @@ def _completed_file_recheck_block( reviewed_series_cv_id: int | None, ) -> dict[str, Any] | None: """Return the final safety block after archive and identity checks.""" + metadata = _reviewed_folder_identity(file, metadata, reviewed_series_cv_id) + source = {**source, "identity_conflicts": metadata.diagnostics.get("identity_conflicts")} raw_block = source.get("file_safety") block = dict(raw_block) if isinstance(raw_block, dict) else None identity_conflicts = source.get("identity_conflicts") @@ -328,6 +355,7 @@ def _completed_file_recheck_block( source="completed_import_recheck", overrideable_hint=False, ) + block["identity_conflicts"] = [*identity_conflicts, *saved_target_conflicts] if block is None and saved_target_conflicts: block = build_import_safety_diagnostics( "The replacement source does not match the issue reviewed during import.", @@ -340,6 +368,50 @@ def _completed_file_recheck_block( return block +def _reviewed_folder_identity( + file: ImportedFile, + metadata: SourceMetadata, + reviewed_series_cv_id: int | None, +) -> SourceMetadata: + """Honor an exact issue assignment without ignoring embedded ID conflicts.""" + embedded = metadata.diagnostics.get("embedded_identity") + conflicts = metadata.diagnostics.get("identity_conflicts") + if ( + file.match_method != "orphan_recovery" + or not reviewed_series_cv_id + or not file.matched_issue_cv_id + or not isinstance(embedded, dict) + or embedded.get("issue_id") != file.matched_issue_cv_id + or embedded.get("series_id") not in (None, reviewed_series_cv_id) + or not isinstance(conflicts, list) + or not conflicts + ): + return metadata + # Only a disagreement between the two folder sidecars is superseded. + # Mylar/ComicInfo issue conflicts and unknown evidence remain blocking. + if not all( + isinstance(conflict, dict) + and set(conflict) == {"field", "series.json", "cvinfo"} + and conflict["field"] == "comicvine_series_id" + and reviewed_series_cv_id in (conflict["series.json"], conflict["cvinfo"]) + for conflict in conflicts + ): + return metadata + diagnostics = dict(metadata.diagnostics) + diagnostics.pop("identity_conflicts", None) + diagnostics["reviewed_folder_identity"] = { + "series_id": reviewed_series_cv_id, + "issue_id": file.matched_issue_cv_id, + "conflicts": conflicts, + } + return metadata.model_copy( + update={ + "comicvine_series_id": reviewed_series_cv_id, + "diagnostics": diagnostics, + } + ) + + def _saved_target_identity_conflicts( file: ImportedFile, metadata: SourceMetadata, diff --git a/tests/unit/test_import_file_execution.py b/tests/unit/test_import_file_execution.py index 6ff33865..73587c13 100644 --- a/tests/unit/test_import_file_execution.py +++ b/tests/unit/test_import_file_execution.py @@ -76,6 +76,168 @@ def _make_service( ) +@pytest.mark.parametrize("source_type", list(ImportSourceType)) +@pytest.mark.parametrize("in_place", [True, False]) +@pytest.mark.parametrize("manual", [True, False]) +@pytest.mark.parametrize("approval", [None, "archive_decompressed_size_limit", "single_page_comic"]) +async def test_recovery_rechecks_device_renumbering_without_changing_sources( + db_session, monkeypatch, source_type, in_place, manual, approval +): + job, item, files, series, _issues = await _setup_full_scenario(db_session, num_issues=1) + file = files[0] + path = Path(file.file_path) + with zipfile.ZipFile(path, "w") as archive: + archive.writestr( + "ComicInfo.xml", + ( + "Batman1" + "https://comicvine.gamespot.com/issue/4000-100001/" + ), + ) + archive.writestr("1.jpg", b"image") + if approval != "single_page_comic": + archive.writestr("2.jpg", b"image") + root = LibraryRoot(name="Source", path=str(path.parent), enabled=True) + db_session.add(root) + await db_session.flush() + signature = build_file_identity_signature(path) + if source_type is ImportSourceType.MYLAR3: + signature[MYLAR_REFERENCE_ROOT_ID_SIGNATURE_KEY] = root.id + file.source_signature = {**signature, "device": int(signature["device"]) + 1} + file.file_size = path.stat().st_size + file.comicvine_issue_id = 100001 + file.matched_issue_cv_id = 100001 + file.diagnostics = {"target_issue_summary": {"provider_id": "100001", "issue_number": 1.0}} + if approval: + file.diagnostics["safety_exception"] = { + "allowed_once": True, + "previous_block": {"code": approval, "overrideable": True}, + } + if approval == "archive_decompressed_size_limit": + monkeypatch.setattr( + "pullbox.services.import_recovery_source.get_archive_size_limit_bytes", + AsyncMock(return_value=1), + ) + if manual: + file.match_method = "orphan_recovery" + item.user_selected_cv_id = item.cv_id + (path.parent / "series.json").write_text('{"metadata":{"comicid":97508}}') + (path.parent / "cvinfo").write_text("https://comicvine.gamespot.com/other/4050-53301/") + file.diagnostics["source_metadata"] = { + "identity_conflicts": [ + {"field": "comicvine_series_id", "series.json": 97508, "cvinfo": 53301} + ] + } + item.diagnostics = {"kind": "known_series_recovery"} + item.source_folder = str(path.parent) + job.source_type = source_type + job.file_handling_mode = ( + ImportFileHandlingMode.IN_PLACE if in_place else ImportFileHandlingMode.MANAGED_COPY + ) + job.move_to_library = not in_place + job.effective_transfer_method = "leave_in_place" if in_place else "copy" + job.convert_to_preferred_format = False + job.update_embedded_comicinfo_from_match = False + before = path.read_bytes(), path.stat().st_mtime_ns + await db_session.flush() + register = _mock_register_library_file() + series_service = AsyncMock() + series_service.add_from_comicvine.return_value = series + service = _make_service(series_service=series_service) + + with patch("pullbox.services.import_service.register_library_file", register): + await service.run_import(db_session, job.id) + + assert register.call_count == 1 + assert file.source_signature == signature + assert file.diagnostics["source_recheck"]["reason"] == "device_renumbered" + assert file.status is ImportedFileStatus.IMPORTED + if manual: + assert file.diagnostics["source_metadata"]["reviewed_folder_identity"]["issue_id"] == 100001 + assert (path.read_bytes(), path.stat().st_mtime_ns) == before + + +@pytest.mark.parametrize("source_type", list(ImportSourceType)) +@pytest.mark.parametrize( + "problem", + [ + "mtime", + "inode", + "size", + "path", + "wrong_issue", + "wrong_series", + "no_embedded_id", + "unsafe_archive", + "lost_root", + "changed_during_inspection", + ], +) +async def test_recovery_device_refresh_keeps_source_safety_guards( + db_session, monkeypatch, source_type, problem +): + from pullbox.core.library_file_ownership import ReferencedFileValidationError + from pullbox.services import import_review_recheck + from pullbox.services.import_recovery_source import refresh_recovery_source + + job, item, files, _series, _issues = await _setup_full_scenario(db_session, num_issues=1) + file = files[0] + path = Path(file.file_path) + issue_id = 999999 if problem == "wrong_issue" else 100001 + web = ( + "" + if problem == "no_embedded_id" + else f"https://comicvine.gamespot.com/issue/4000-{issue_id}/" + ) + if problem == "wrong_series": + web += " https://comicvine.gamespot.com/series/4050-999999/" + with zipfile.ZipFile(path, "w") as archive: + archive.writestr( + "ComicInfo.xml", + f"Batman1{web}", + ) + archive.writestr("1.jpg", b"image") + archive.writestr("2.jpg", b"image") + if problem == "unsafe_archive": + archive.writestr("../escaped.jpg", b"image") + root = LibraryRoot(name="Source", path=str(path.parent), enabled=problem != "lost_root") + db_session.add(root) + await db_session.flush() + signature = build_file_identity_signature(path) + signature["device"] = int(signature["device"]) + 1 + if source_type is ImportSourceType.MYLAR3: + signature[MYLAR_REFERENCE_ROOT_ID_SIGNATURE_KEY] = root.id + for field, mutation in {"mtime": "mtime_ns", "size": "size", "inode": "inode"}.items(): + if problem == field: + signature[mutation] = int(signature[mutation]) + 1 + if problem == "path": + signature["resolved_path"] = str(path.parent / "different.cbz") + file.source_signature = signature + file.matched_issue_cv_id = 100001 + file.diagnostics = {"target_issue_summary": {"provider_id": "100001", "issue_number": 1.0}} + item.diagnostics = {"kind": "known_series_recovery"} + job.source_type = source_type + job.file_handling_mode = ImportFileHandlingMode.IN_PLACE + if problem == "lost_root" and source_type is ImportSourceType.FILESYSTEM: + job.source_path = str(path.parent / "unavailable") + before = dict(signature) + if problem == "changed_during_inspection": + inspect = import_review_recheck.run_safety_checks + + def replacing_inspection(path, **kwargs): + result = inspect(path, **kwargs) + path.write_bytes(b"replacement during inspection") + return result + + monkeypatch.setattr(import_review_recheck, "run_safety_checks", replacing_inspection) + await db_session.flush() + + with pytest.raises(ReferencedFileValidationError): + await refresh_recovery_source(db_session, job, item, file) + + assert file.source_signature == before + + def test_placeholder_issue_target_preserves_only_base_compatible_exact_text() -> None: from pullbox.services.import_file_execution import ( _placeholder_issue_target_from_diagnostics, diff --git a/tests/unit/test_import_orphans.py b/tests/unit/test_import_orphans.py index 2478c586..e956b29d 100644 --- a/tests/unit/test_import_orphans.py +++ b/tests/unit/test_import_orphans.py @@ -109,6 +109,23 @@ async def _create_series_with_issue( await session.flush() +@pytest.mark.parametrize( + "code", + ["source_identity_changed", "source_changed", "dangerous_archive", "archive_inspection_failed"], +) +def test_failed_identity_conflict_can_be_resolved_in_follow_up(code): + from pullbox.services.import_orphans import requires_orphan_issue_decision + + file = ImportedFile( + status=ImportedFileStatus.FAILED, + diagnostics={ + "source_revalidation": {"code": code}, + }, + ) + + assert requires_orphan_issue_decision(file) is (code == "source_identity_changed") + + def test_apply_orphan_recovery_decisions_assigns_issue() -> None: from pullbox.services.import_orphans import apply_orphan_recovery_decisions diff --git a/tests/unit/test_import_review_recheck.py b/tests/unit/test_import_review_recheck.py index cc7ca6d9..c0c73647 100644 --- a/tests/unit/test_import_review_recheck.py +++ b/tests/unit/test_import_review_recheck.py @@ -886,3 +886,63 @@ async def test_retry_failed_rejects_replacement_for_different_saved_issue( assert changed.include_in_import is False assert changed.matched_issue_cv_id == 100008 assert changed.diagnostics["source_revalidation"]["code"] == "source_identity_changed" + + +@pytest.mark.parametrize("source_type", list(ImportSourceType)) +@pytest.mark.parametrize( + ("manual", "embedded_id", "number", "ready"), + [ + (True, 100008, "8", True), + (False, 100008, "8", False), + (True, 999999, "8", False), + (True, 100008, "9", False), + ], +) +async def test_completed_recheck_respects_proven_issue_assignment_over_folder_conflict( + db_session, tmp_path, source_type, manual, embedded_id, number, ready +): + job, item, files = await _fixture(db_session, tmp_path, source_type) + job.status = ImportJobStatus.COMPLETED + item.cv_id = 115251 + item.user_selected_cv_id = 115251 if manual else None + file = files[1] + file.status = ImportedFileStatus.FAILED + file.matched_issue_cv_id = 100008 + file.match_method = "orphan_recovery" if manual else "comicvine_issue_id" + file.diagnostics = { + **file.diagnostics, + "target_issue_summary": {"provider_id": "100008", "issue_number": 8.0}, + "source_revalidation": {"code": "source_changed", "retryable": True}, + } + (tmp_path / "Firefly (2018)" / "cvinfo").write_text( + "https://comicvine.gamespot.com/other/4050-123456/" + ) + path = Path(file.file_path) + with zipfile.ZipFile(path, "w") as archive: + archive.writestr( + "ComicInfo.xml", + ( + f"Firefly{number}" + f"https://comicvine.gamespot.com/issue/4000-{embedded_id}/" + ), + ) + archive.writestr("1.jpg", b"image") + archive.writestr("2.jpg", b"image") + before = {p: p.read_bytes() for p in path.parent.iterdir() if p.is_file()} + await db_session.flush() + + report = await prepare_completed_import_file_recheck( + db_session, job.id, source_roots=[tmp_path], apply=True, accept_replaced_files=True + ) + + assert report["files_prepared"] == int(ready) + assert report["blocked_files"] == int(not ready) + if ready: + evidence = file.diagnostics["source_metadata"]["reviewed_folder_identity"] + assert evidence["series_id"] == 115251 + assert evidence["issue_id"] == 100008 + assert evidence["conflicts"] + assert not file.diagnostics["source_metadata"].get("identity_conflicts") + else: + assert file.diagnostics["source_revalidation"]["identity_conflicts"] + assert before == {p: p.read_bytes() for p in before} From c5988698ba83c3316cdf155a21512407792d7283 Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 11:49:21 -0700 Subject: [PATCH 10/20] fix(import): reconcile provisional targets before conflict grouping --- docs/development/IMPORT_REVIEW_RECOVERY.md | 10 + .../services/import_file_match_outcomes.py | 1 + src/pullbox/services/import_file_matching.py | 14 + .../services/import_provisional_targets.py | 215 ++++++++++ tests/unit/test_import_provisional_targets.py | 382 ++++++++++++++++++ 5 files changed, 622 insertions(+) create mode 100644 src/pullbox/services/import_provisional_targets.py create mode 100644 tests/unit/test_import_provisional_targets.py diff --git a/docs/development/IMPORT_REVIEW_RECOVERY.md b/docs/development/IMPORT_REVIEW_RECOVERY.md index 76c4fde2..1e59044a 100644 --- a/docs/development/IMPORT_REVIEW_RECOVERY.md +++ b/docs/development/IMPORT_REVIEW_RECOVERY.md @@ -62,6 +62,16 @@ The shared checks cover fresh Mylar discovery and saved-review reconciliation. They do not modify Mylar, rename source files, or change completed-import recovery and ownership rules. No metadata-provider requests are introduced. +Before copy/conflict grouping, matching reconciles automatic provisional issue +targets against exact identities discovered in the same import-series row. +This runs after all file pages, so an untagged CBR and a later tagged CBZ cannot +escape copy review merely because their identities were learned at different +times. Only an unambiguous target with the same exact issue designation and +issue type is reused; annuals, lettered issues, conflicting metadata, manual +decisions, skips, and safety blocks are not overridden. Reads are paged, source +signatures and files are unchanged, and no provider requests are added. The +shared finalization applies to Mylar and folder imports in either handling mode. + ## Import Follow-up The Follow-up tab groups actionable work by import job rather than rendering diff --git a/src/pullbox/services/import_file_match_outcomes.py b/src/pullbox/services/import_file_match_outcomes.py index 01290136..35226493 100644 --- a/src/pullbox/services/import_file_match_outcomes.py +++ b/src/pullbox/services/import_file_match_outcomes.py @@ -54,6 +54,7 @@ class FileMatchLogEvent: "source_metadata", "mylar3_cross_folder_reconciliation", "mylar3_path_reconciliation", + "provisional_target_reconciliation", "review_selection", "review_source_action", "review_source_previous", diff --git a/src/pullbox/services/import_file_matching.py b/src/pullbox/services/import_file_matching.py index a363a482..ada422be 100644 --- a/src/pullbox/services/import_file_matching.py +++ b/src/pullbox/services/import_file_matching.py @@ -75,6 +75,7 @@ scan_review_file_target_weight, scan_review_progress_pct, ) +from pullbox.services.import_provisional_targets import reconcile_provisional_targets from pullbox.services.import_source_metadata import ( corroborated_import_title_conflict, import_file_has_deferred_archive_metadata, @@ -721,6 +722,19 @@ async def _finalize_import_series_file_groups( log_event: LogEventFunc, raise_if_cancelled: RaiseIfCancelledFunc, ) -> tuple[int, int]: + reconciled = await reconcile_provisional_targets( + session, job, imp_series, raise_if_cancelled=raise_if_cancelled + ) + if reconciled: + await log_event( + session, + job.id, + "DEBUG", + "import_provisional_targets_reconciled", + message=f"Resolved {reconciled} provisional issue targets before copy review.", + series=imp_series.raw_series_name, + files_reconciled=reconciled, + ) for kind in _FILE_TARGET_COHORT_KINDS: async for cohorts in _iter_file_target_cohort_batches( session, diff --git a/src/pullbox/services/import_provisional_targets.py b/src/pullbox/services/import_provisional_targets.py new file mode 100644 index 00000000..1572a9d2 --- /dev/null +++ b/src/pullbox/services/import_provisional_targets.py @@ -0,0 +1,215 @@ +"""Reconcile late-discovered exact identities before import copy grouping.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, NamedTuple + +from sqlalchemy import select + +from pullbox.core.issue_numbers import format_issue_number, normalize_issue_number_text +from pullbox.models.import_job import ImportedFile, ImportedFileStatus +from pullbox.models.issue import IssueType +from pullbox.services.import_file_issue_signals import ( + candidate_issue_number, + candidate_issue_number_text, +) +from pullbox.services.import_file_match_targets import ( + PROVIDER_MISSING_ISSUE_PLACEHOLDER_KIND, + PROVIDER_MISSING_ISSUE_PLACEHOLDER_METHOD, +) + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.models.import_job import ImportedSeries, ImportJob + +_PAGE_SIZE = 250 +_IssueKey = tuple[str, IssueType] + + +class _ExactTarget(NamedTuple): + file_id: int + issue_id: int | None + issue_cv_id: int + summary: dict[str, Any] + + +def _source_key(file: ImportedFile, series_cv_id: int) -> _IssueKey | None: + diagnostics = file.diagnostics or {} + metadata = diagnostics.get("source_metadata") or {} + if not isinstance(metadata, dict): + return None + if ( + diagnostics.get("safety_block") + or diagnostics.get("review_deferred") + or metadata.get("identity_conflicts") + or metadata.get("mylar3_folder_scope_conflict") + or diagnostics.get("comicvine_series_id") not in (None, series_cv_id) + ): + return None + number = candidate_issue_number(file) + if number is None: + return None + text = candidate_issue_number_text(file) + if file.issue_number_raw and text is None: + return None + try: + issue_type = IssueType(diagnostics.get("source_issue_type")) + return text or format_issue_number(number), issue_type + except (TypeError, ValueError): + return None + + +def _provisional_key(file: ImportedFile, series_cv_id: int) -> _IssueKey | None: + diagnostics = file.diagnostics or {} + key = _source_key(file, series_cv_id) + if ( + key is None + or diagnostics.get("kind") != PROVIDER_MISSING_ISSUE_PLACEHOLDER_KIND + or diagnostics.get("target_state") != "provisional_issue_target" + or diagnostics.get("target_series_cv_id") != series_cv_id + or diagnostics.get("target_issue_number") != candidate_issue_number(file) + or diagnostics.get("target_issue_type") != key[1].value + or diagnostics.get("resolution") + ): + return None + return key + + +def _exact_target(file: ImportedFile, series_cv_id: int) -> tuple[_IssueKey, _ExactTarget] | None: + key = _source_key(file, series_cv_id) + summary = (file.diagnostics or {}).get("target_issue_summary") + if key is None or not isinstance(summary, dict) or file.matched_issue_cv_id is None: + return None + if ( + str(summary.get("provider_id")) != str(file.matched_issue_cv_id) + or summary.get("issue_type") != key[1].value + or summary.get("issue_number") != candidate_issue_number(file) + ): + return None + try: + text = normalize_issue_number_text( + summary.get("issue_number_text") or summary["issue_number"] + ) + except (TypeError, ValueError): + return None + if text != key[0]: + return None + return key, _ExactTarget( + file.id, file.matched_issue_id, file.matched_issue_cv_id, dict(summary) + ) + + +async def reconcile_provisional_targets( + session: AsyncSession, + job: ImportJob, + series: ImportedSeries, + *, + raise_if_cancelled: Callable[[AsyncSession, int], Awaitable[None]], +) -> int: + """Join only unambiguous same-series, same-designation, same-type targets. + + Run after every file page has been evaluated so archive order cannot hide a + competing copy. Reads remain bounded and never access source files/providers. + """ + series_cv_id = series.user_selected_cv_id or series.cv_id + if series_cv_id is None: + return 0 + scope = ( + ImportedFile.import_job_id == job.id, + ImportedFile.import_series_id == series.id, + ImportedFile.status == ImportedFileStatus.MATCHED, + ) + after_id = 0 + reconciled = 0 + while True: + await raise_if_cancelled(session, job.id) + result = await session.execute( + select(ImportedFile) + .where( + *scope, + ImportedFile.id > after_id, + ImportedFile.match_method == PROVIDER_MISSING_ISSUE_PLACEHOLDER_METHOD, + ImportedFile.matched_issue_id.is_(None), + ImportedFile.matched_issue_cv_id.is_(None), + ImportedFile.comicvine_issue_id.is_(None), + ) + .order_by(ImportedFile.id) + .limit(_PAGE_SIZE) + ) + page = list(result.scalars()) + if not page: + break + after_id = page[-1].id + keys = {file.id: _provisional_key(file, series_cv_id) for file in page} + requested = {key for key in keys.values() if key is not None} + numbers = {file.parsed_issue_number for file in page if keys[file.id] is not None} + if not requested: + continue + targets: dict[_IssueKey, _ExactTarget | None] = {} + target_after_id = 0 + while True: + await raise_if_cancelled(session, job.id) + result = await session.execute( + select(ImportedFile) + .where( + *scope, + ImportedFile.id > target_after_id, + ImportedFile.matched_issue_cv_id > 0, + ImportedFile.parsed_issue_number.in_(numbers), + ) + .order_by(ImportedFile.id) + .limit(_PAGE_SIZE) + ) + candidates = list(result.scalars()) + if not candidates: + break + target_after_id = candidates[-1].id + for candidate in candidates: + target = _exact_target(candidate, series_cv_id) + if target is None or target[0] not in requested: + continue + key, exact = target + if key not in targets: + targets[key] = exact + else: + previous = targets[key] + if previous is None or (previous.issue_id, previous.issue_cv_id) != ( + exact.issue_id, + exact.issue_cv_id, + ): + targets[key] = None + await raise_if_cancelled(session, job.id) + for file in page: + key = keys[file.id] + exact = targets.get(key) if key is not None else None + if exact is None: + continue + diagnostics = dict(file.diagnostics or {}) + for field in ( + "kind", + "target_state", + "target_series_cv_id", + "target_series_title", + "target_series_issue_count", + "target_issue_number", + "target_issue_type", + "target_issue_title", + "rejection_reason", + ): + diagnostics.pop(field, None) + diagnostics["target_issue_summary"] = dict(exact.summary) + diagnostics["provisional_target_reconciliation"] = { + "evidence_file_id": exact.file_id, + "matched_issue_cv_id": exact.issue_cv_id, + "previous_match_method": file.match_method, + } + file.matched_issue_id = exact.issue_id + file.matched_issue_cv_id = exact.issue_cv_id + file.match_method = "issue_number" + file.diagnostics = diagnostics + reconciled += 1 + await session.flush() + return reconciled diff --git a/tests/unit/test_import_provisional_targets.py b/tests/unit/test_import_provisional_targets.py new file mode 100644 index 00000000..53e6a8f7 --- /dev/null +++ b/tests/unit/test_import_provisional_targets.py @@ -0,0 +1,382 @@ +"""Provisional targets must join proven issue identities before copy review.""" + +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest + +from pullbox.core.exceptions import JobCancelledError +from pullbox.core.source_metadata import MetadataSignal, SourceMetadata +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportFileHandlingMode, + ImportJob, + ImportJobStatus, + ImportSeriesStatus, + ImportSourceType, +) +from pullbox.services import import_file_matching, import_provisional_targets +from pullbox.services.import_file_conflicts import detect_conflicts +from pullbox.services.import_file_match_targets import PROVIDER_MISSING_ISSUE_PLACEHOLDER_METHOD +from pullbox.services.import_service import ImportService + + +async def _pair(session, *, source_type, mode, reverse=False, number=1.0, text="1"): + job = ImportJob( + source_path="/fixtures", + source_type=source_type, + status=ImportJobStatus.FILE_MATCHING, + file_handling_mode=mode, + ) + session.add(job) + await session.flush() + series = ImportedSeries( + import_job_id=job.id, + raw_series_name="X-Force", + cv_title="X-Force", + cv_id=71614, + cv_match_method=( + "mylar3_cv_id" if source_type == ImportSourceType.MYLAR3 else "folder_cv_id" + ), + cv_match_score=1.0, + status=ImportSeriesStatus.MATCHED, + file_count=2, + ) + session.add(series) + await session.flush() + common = { + "import_job_id": job.id, + "import_series_id": series.id, + "status": ImportedFileStatus.MATCHED, + "parsed_series": "X-Force", + "parsed_year": 2014, + "parsed_issue_number": number, + "issue_number_raw": text, + "match_confidence": "high", + } + evidence = { + "comicvine_series_id": 71614, + "source_issue_type": "issue", + "source_metadata": {"has_comicinfo": False}, + } + provisional = ImportedFile( + **common, + file_path=f"/fixtures/X-Force {text} (2014).cbr", + file_name=f"X-Force {text} (2014).cbr", + file_size=37522498, + file_format="cbr", + has_comicinfo=False, + match_method=PROVIDER_MISSING_ISSUE_PLACEHOLDER_METHOD, + diagnostics={ + **deepcopy(evidence), + "kind": "provider_missing_issue_placeholder", + "target_state": "provisional_issue_target", + "target_series_cv_id": 71614, + "target_issue_number": number, + "target_issue_type": "issue", + "rejection_reason": "Metadata will hydrate after import.", + "review_selection": True, + }, + source_signature={"size": 37522498, "inode": 100}, + ) + exact = ImportedFile( + **common, + file_path=f"/fixtures/X-Force {text} (2014).cbz", + file_name=f"X-Force {text} (2014).cbz", + file_size=39155934, + file_format="cbz", + has_comicinfo=True, + comicvine_issue_id=445188, + matched_issue_cv_id=445188, + match_method="comicvine_id", + diagnostics={ + **deepcopy(evidence), + "target_issue_summary": { + "provider_id": "445188", + "issue_number": number, + "issue_number_text": text, + "issue_type": "issue", + "title": "Offensive Acts", + "release_date": None, + "cover_url": None, + }, + }, + ) + session.add_all([exact, provisional] if reverse else [provisional, exact]) + await session.flush() + return job, series, provisional, exact + + +async def _finalize(session, job, series): + return await import_file_matching._finalize_import_series_file_groups( + session, + job, + series, + duplicate_group_counter=0, + conflict_group_counter=0, + detect_duplicate_copies=AsyncMock(return_value=(0, 0, [])), + detect_conflicts=detect_conflicts, + log_event=AsyncMock(), + raise_if_cancelled=AsyncMock(), + ) + + +@pytest.mark.parametrize("source_type", [ImportSourceType.MYLAR3, ImportSourceType.FILESYSTEM]) +@pytest.mark.parametrize( + "mode", [ImportFileHandlingMode.IN_PLACE, ImportFileHandlingMode.MANAGED_COPY] +) +@pytest.mark.parametrize("reverse", [False, True]) +@pytest.mark.parametrize("number,text", [(1.0, "1"), (13.0, "13a"), (0.5, "0.5"), (-1.0, "-1")]) +async def test_finalization_groups_provisional_and_exact_copies_across_pages( + db_session, monkeypatch, source_type, mode, reverse, number, text +): + job, series, provisional, exact = await _pair( + db_session, source_type=source_type, mode=mode, reverse=reverse, number=number, text=text + ) + monkeypatch.setattr(import_provisional_targets, "_PAGE_SIZE", 1) + signature = deepcopy(provisional.source_signature) + evidence = deepcopy(provisional.diagnostics["source_metadata"]) + + _duplicates, conflicts = await _finalize(db_session, job, series) + + assert conflicts == 1 + assert provisional.status == exact.status == ImportedFileStatus.CONFLICT + assert provisional.conflict_group_id == exact.conflict_group_id + assert provisional.matched_issue_cv_id == exact.matched_issue_cv_id == 445188 + assert provisional.match_method == "issue_number" + assert exact.is_preferred is True + assert provisional.include_in_import is False + previous = provisional.diagnostics["previous_diagnostics"] + assert ( + previous["target_issue_summary"] + == exact.diagnostics["previous_diagnostics"]["target_issue_summary"] + ) + assert previous["provisional_target_reconciliation"]["evidence_file_id"] == exact.id + assert "target_state" not in previous + assert previous["source_metadata"] == evidence + assert provisional.source_signature == signature + assert provisional.comicvine_issue_id is None + + +@pytest.mark.parametrize( + "case", + [ + "annual", + "letter", + "unknown_type", + "conflicting_identity", + "other_series", + "other_job", + "ambiguous", + "blocked", + "skipped", + "manual", + "manual_provisional", + "wrong_summary", + "changed_number", + "source_issue_id", + "unsafe_target", + "unmatched_target", + ], +) +async def test_provisional_reconciliation_preserves_uncertain_or_reviewed_files(db_session, case): + job, series, provisional, exact = await _pair( + db_session, source_type=ImportSourceType.MYLAR3, mode=ImportFileHandlingMode.IN_PLACE + ) + if case in {"annual", "unknown_type"}: + diagnostics = deepcopy(exact.diagnostics) + diagnostics["target_issue_summary"]["issue_type"] = ( + "annual" if case == "annual" else "unknown" + ) + diagnostics["source_issue_type"] = diagnostics["target_issue_summary"]["issue_type"] + exact.diagnostics = diagnostics + elif case == "letter": + exact.issue_number_raw = "1a" + diagnostics = deepcopy(exact.diagnostics) + diagnostics["target_issue_summary"]["issue_number_text"] = "1a" + exact.diagnostics = diagnostics + elif case == "conflicting_identity": + provisional.diagnostics = { + **provisional.diagnostics, + "source_metadata": {"identity_conflicts": [{"field": "comicvine_issue_id"}]}, + } + elif case in {"other_series", "other_job"}: + other_job, other_series, _other_provisional, _other_exact = await _pair( + db_session, source_type=ImportSourceType.MYLAR3, mode=ImportFileHandlingMode.IN_PLACE + ) + if case == "other_series": + exact.import_series_id = other_series.id + else: + exact.import_job_id = other_job.id + elif case == "ambiguous": + competing = ImportedFile( + import_job_id=job.id, + import_series_id=series.id, + file_path="/fixtures/competing.cbz", + file_name="X-Force 1 (2014) variant.cbz", + file_size=1000, + file_format="cbz", + parsed_issue_number=1, + issue_number_raw="1", + matched_issue_cv_id=999999, + status=ImportedFileStatus.MATCHED, + diagnostics={ + **deepcopy(exact.diagnostics), + "target_issue_summary": { + **exact.diagnostics["target_issue_summary"], + "provider_id": "999999", + }, + }, + ) + db_session.add(competing) + elif case in {"blocked", "skipped"}: + provisional.status = ( + ImportedFileStatus.SAFETY_BLOCKED if case == "blocked" else ImportedFileStatus.SKIPPED + ) + elif case == "manual": + provisional.match_method = "orphan_recovery" + elif case == "manual_provisional": + provisional.diagnostics = { + **provisional.diagnostics, + "resolution": "provisional_issue_created", + } + elif case == "wrong_summary": + exact.diagnostics = { + **exact.diagnostics, + "target_issue_summary": { + **exact.diagnostics["target_issue_summary"], + "provider_id": "999999", + }, + } + elif case == "changed_number": + provisional.diagnostics = {**provisional.diagnostics, "target_issue_number": 2} + elif case == "source_issue_id": + provisional.comicvine_issue_id = 999999 + elif case == "unsafe_target": + exact.diagnostics = {**exact.diagnostics, "safety_block": {"code": "unsafe_archive"}} + elif case == "unmatched_target": + exact.status = ImportedFileStatus.NO_MATCH + await db_session.flush() + before = deepcopy(provisional.diagnostics) + + count = await import_provisional_targets.reconcile_provisional_targets( + db_session, job, series, raise_if_cancelled=AsyncMock() + ) + + assert count == 0 + assert provisional.matched_issue_cv_id is None + assert provisional.diagnostics == before + + +async def test_reconciliation_is_idempotent_and_cancellable(db_session): + job, series, provisional, exact = await _pair( + db_session, source_type=ImportSourceType.FILESYSTEM, mode=ImportFileHandlingMode.IN_PLACE + ) + cancelled = AsyncMock(side_effect=JobCancelledError("stop")) + with pytest.raises(JobCancelledError, match="stop"): + await import_provisional_targets.reconcile_provisional_targets( + db_session, job, series, raise_if_cancelled=cancelled + ) + assert provisional.matched_issue_cv_id is None + first = await import_provisional_targets.reconcile_provisional_targets( + db_session, job, series, raise_if_cancelled=AsyncMock() + ) + second = await import_provisional_targets.reconcile_provisional_targets( + db_session, job, series, raise_if_cancelled=AsyncMock() + ) + assert first == 1 + assert second == 0 + assert provisional.matched_issue_cv_id == exact.matched_issue_cv_id + + +@pytest.mark.parametrize("source_type", [ImportSourceType.MYLAR3, ImportSourceType.FILESYSTEM]) +@pytest.mark.parametrize("reverse", [False, True]) +@pytest.mark.parametrize("page_size", [1, 2]) +async def test_matching_pipeline_groups_late_identity_without_provider_or_file_io( + db_session, monkeypatch, source_type, reverse, page_size +): + job, series, provisional, exact = await _pair( + db_session, + source_type=source_type, + mode=ImportFileHandlingMode.IN_PLACE, + reverse=reverse, + ) + for file in [provisional, exact]: + file.status = ImportedFileStatus.PENDING + file.matched_issue_cv_id = None + file.match_method = None + file.diagnostics = { + "source_issue_type": "issue", + "comicvine_series_id": series.cv_id, + "metadata_signals": { + "comicvine_series_id": ( + "mylar3" if source_type == ImportSourceType.MYLAR3 else "sidecar" + ), + }, + "source_metadata": { + "archive_metadata_deferred": source_type == ImportSourceType.MYLAR3, + "archive_entry_issue_hint_checked": True, + }, + } + if source_type == ImportSourceType.MYLAR3: + exact.comicvine_issue_id = None + else: + exact.diagnostics = { + **exact.diagnostics, + "metadata_signals": { + **exact.diagnostics["metadata_signals"], + "comicvine_issue_id": "comicinfo", + }, + } + await db_session.commit() + provider = Mock() + provider.cache_metrics.return_value = {} + for name in [ + "get_series", "get_issue", "get_issues_for_series", "get_issues_for_series_by_numbers" + ]: + setattr(provider, name, AsyncMock(side_effect=AssertionError("unexpected provider call"))) + service = ImportService( + series_service=AsyncMock(), + metadata_service=SimpleNamespace(_provider=provider), + event_bus=AsyncMock(), + ) + + async def local_metadata(item, file): + tagged = file.id == exact.id + return SourceMetadata( + original_title=file.file_name, + series_name=item.raw_series_name, + issue_number=1, + issue_number_text="1", + year=2014, + comicvine_series_id=71614, + comicvine_issue_id=445188 if tagged else None, + signals={ + "comicvine_series_id": MetadataSignal.MYLAR3, + **({"comicvine_issue_id": MetadataSignal.COMICINFO} if tagged else {}), + }, + diagnostics={ + "has_comicinfo": tagged, + "archive_metadata_loaded": True, + "archive_metadata_deferred": False, + "archive_entry_issue_hint_checked": True, + }, + ) + + loader = AsyncMock(side_effect=local_metadata) + monkeypatch.setattr(service, "_load_deferred_source_metadata_for_import_file", loader) + monkeypatch.setattr(service, "_detect_duplicate_copies", AsyncMock(return_value=(0, 0, []))) + monkeypatch.setattr(import_file_matching, "_FILE_PAGE_SIZE", page_size) + await service._run_file_matching(db_session, job) + + assert provisional.status == exact.status == ImportedFileStatus.CONFLICT + assert provisional.conflict_group_id == exact.conflict_group_id + assert provisional.matched_issue_cv_id == exact.matched_issue_cv_id == 445188 + assert job.total_files_conflict == 2 + assert job.total_files_matched == 0 + assert series.status == ImportSeriesStatus.MATCHED + assert not [call for call in provider.mock_calls if "get_" in call[0]] + assert loader.await_count == (2 if source_type == ImportSourceType.MYLAR3 else 0) From 4f0ad872bb4d97ff8e217541d57da18d03e1dfcb Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 12:09:05 -0700 Subject: [PATCH 11/20] fix: preserve hyphenated issue suffixes across metadata and imports --- docs/development/IMPORT_REVIEW_RECOVERY.md | 8 + src/pullbox/core/issue_numbers.py | 3 +- src/pullbox/core/release_parser.py | 25 +-- .../services/import_provisional_targets.py | 21 +- tests/unit/test_hyphenated_issue_numbers.py | 189 ++++++++++++++++++ .../unit/test_import_matching_performance.py | 3 +- tests/unit/test_import_provisional_targets.py | 5 +- .../test_letter_suffixed_issue_numbers.py | 25 ++- tests/unit/test_mylar3_reader.py | 8 +- 9 files changed, 252 insertions(+), 35 deletions(-) create mode 100644 tests/unit/test_hyphenated_issue_numbers.py diff --git a/docs/development/IMPORT_REVIEW_RECOVERY.md b/docs/development/IMPORT_REVIEW_RECOVERY.md index 1e59044a..e2fc72a3 100644 --- a/docs/development/IMPORT_REVIEW_RECOVERY.md +++ b/docs/development/IMPORT_REVIEW_RECOVERY.md @@ -72,6 +72,14 @@ decisions, skips, and safety blocks are not overridden. Reads are paged, source signatures and files are unchanged, and no provider requests are added. The shared finalization applies to Mylar and folder imports in either handling mode. +Exact issue designations retain letter suffixes and their hyphens, including +`13A`, `50-X`, and `50-O`. ComicVine metadata, local catalog reads, Mylar and +folder discovery, and release validation share this identity policy. A dashed +letter suffix is not a numeric range or a plain issue number; sibling letters +must not collapse into the same target. Refreshing live metadata can correct +an older zero-number fallback using its existing ComicVine issue ID without +discarding ownership. No schema migration or source-file rename is required. + ## Import Follow-up The Follow-up tab groups actionable work by import job rather than rendering diff --git a/src/pullbox/core/issue_numbers.py b/src/pullbox/core/issue_numbers.py index c954b0f5..15c8abf4 100644 --- a/src/pullbox/core/issue_numbers.py +++ b/src/pullbox/core/issue_numbers.py @@ -11,7 +11,8 @@ from collections.abc import Iterable _MAX_ISSUE_NUMBER_TEXT_LENGTH = 320 -_NUMERIC_SUFFIX_PATTERN = re.compile(r"^([+-]?(?:\d+(?:\.\d*)?|\.\d+))([A-Za-z]+)$") +# A suffix separator belongs to the exact designation, not the numeric value. +_NUMERIC_SUFFIX_PATTERN = re.compile(r"^([+-]?(?:\d+(?:\.\d*)?|\.\d+))(-?[A-Za-z]+)$") def _format_decimal(value: Decimal) -> str: diff --git a/src/pullbox/core/release_parser.py b/src/pullbox/core/release_parser.py index 07fed693..384379db 100644 --- a/src/pullbox/core/release_parser.py +++ b/src/pullbox/core/release_parser.py @@ -137,7 +137,7 @@ _YEAR_PAREN_RE = re.compile(r"\((\d{4})\)") # Issue number with hash prefix: #045, #5, #5.1 -_ISSUE_HASH_RE = re.compile(r"#([+-]?\d+(?:\.\d+)?[A-Za-z]*)") +_ISSUE_HASH_RE = re.compile(r"#([+-]?\d+(?:\.\d+)?(?:-?[A-Za-z]+)?)") # DC's One Million event used the literal issue number 1,000,000 across # multiple ongoing titles. Keep this exact exception narrow so arbitrary long @@ -148,14 +148,14 @@ _PROG_ISSUE_RE = re.compile(r"\bProg(?:ramme)?\.?\s*#?\s*(\d+(?:\.\d+)?)\b", re.IGNORECASE) # Bare four-digit issues are unambiguous only at the end of the stripped title. -_LONG_POSITIONAL_ISSUE_RE = re.compile(r"(?<=\s)(\d{4}(?:\.\d+)?[A-Za-z]*)\s*$") +_LONG_POSITIONAL_ISSUE_RE = re.compile(r"(?<=\s)(\d{4}(?:\.\d+)?(?:-?[A-Za-z]+)?)\s*$") _RANGE_OR_COUNT_PREFIX_RE = re.compile(r"(?:\d\s*[-\u2013\u2014]|\bof)\s*$", re.IGNORECASE) _VOLUME_YEAR_RE = re.compile(r"\b(?:v|vol(?:ume)?\.?)\s*((?:19|20)\d{2})\b", re.IGNORECASE) # Limited series marker: (of 05) _LIMITED_SERIES_RE = re.compile(r"\(of\s+\d+\)", re.IGNORECASE) _INLINE_LIMITED_SERIES_RE = re.compile( - r"(?[a-z][a-z0-9]{1,15})-" r"(?P[A-Z].*?[._\s]+(?i:No)\.?[._\s]*\d{1,5})\s*$" @@ -226,7 +226,9 @@ # Minimal separator-only filenames such as ``dc_connect_72.pdf`` are common # enough to support, but this path stays narrower than full dot release parsing. -_MINIMAL_SEPARATOR_ISSUE_RE = re.compile(r"^(?P<series>.+?)[._](?P<issue>0*\d{1,3}(?:\.\d+)?)$") +_MINIMAL_SEPARATOR_ISSUE_RE = re.compile( + r"^(?P<series>.+?)[._](?P<issue>0*\d{1,3}(?:\.\d+)?(?:-?[A-Za-z]+)?)$" +) _GENERIC_MINIMAL_SERIES_TOKENS = frozenset( {"scan", "scans", "page", "pages", "img", "image", "images", "cover", "covers", "comic"} ) @@ -295,7 +297,7 @@ def normalize_issue_number(raw: str | float | None) -> float | None: text = text[1:] limited_series_match = re.fullmatch( - r"(\d+(?:\.\d+)?)\s*(?:[\[(]\s*of\s+\d+\s*[\])]|\s+of\s+\d+)", + r"(\d+(?:\.\d+)?(?:-?[A-Za-z]+)?)\s*(?:[\[(]\s*of\s+\d+\s*[\])]|\s+of\s+\d+)", text, re.IGNORECASE, ) @@ -314,13 +316,8 @@ def normalize_issue_number(raw: str | float | None) -> float | None: if denom != 0: return num / denom - # Strip alpha suffixes: "5a" → "5", "12AU" → "12" - cleaned = re.sub(r"[a-zA-Z]+$", "", text).strip() - if not cleaned: - return None - try: - return float(cleaned) + return parse_issue_number_text(text)[0] except ValueError: return None @@ -820,7 +817,7 @@ def _extract_issue_number( # but NOT part of an alphanumeric word like "D4VE2" or "Spider-Man 2099" positional_matches = list( re.finditer( - r"(?<=\s)(\d{2,3}(?:\.\d+)?[A-Za-z]*)(?=\s|$)", + r"(?<=\s)(\d{2,3}(?:\.\d+)?(?:-?[A-Za-z]+)?)(?=\s|$)", clean, ) ) @@ -842,7 +839,7 @@ def _extract_issue_number( # Priority 8: Single digit number at word boundary after text # Must NOT be followed by a word (e.g. "4 Covers" is a count, not issue #4) - m = re.search(r"(?<=\s)(\d[A-Za-z]*)(?=\s|$)", clean) + m = re.search(r"(?<=\s)(\d(?:-?[A-Za-z]+)?)(?=\s|$)", clean) if m: # Check the word after the digit — if it's alphabetic, this is likely # a count or descriptor (e.g. "4 Covers", "3 Stories"), not an issue number diff --git a/src/pullbox/services/import_provisional_targets.py b/src/pullbox/services/import_provisional_targets.py index 1572a9d2..0d2454b6 100644 --- a/src/pullbox/services/import_provisional_targets.py +++ b/src/pullbox/services/import_provisional_targets.py @@ -55,8 +55,11 @@ def _source_key(file: ImportedFile, series_cv_id: int) -> _IssueKey | None: text = candidate_issue_number_text(file) if file.issue_number_raw and text is None: return None + source_issue_type = diagnostics.get("source_issue_type") + if not isinstance(source_issue_type, str): + return None try: - issue_type = IssueType(diagnostics.get("source_issue_type")) + issue_type = IssueType(source_issue_type) return text or format_issue_number(number), issue_type except (TypeError, ValueError): return None @@ -183,9 +186,9 @@ async def reconcile_provisional_targets( targets[key] = None await raise_if_cancelled(session, job.id) for file in page: - key = keys[file.id] - exact = targets.get(key) if key is not None else None - if exact is None: + provisional_key = keys[file.id] + resolved = targets.get(provisional_key) if provisional_key is not None else None + if resolved is None: continue diagnostics = dict(file.diagnostics or {}) for field in ( @@ -200,14 +203,14 @@ async def reconcile_provisional_targets( "rejection_reason", ): diagnostics.pop(field, None) - diagnostics["target_issue_summary"] = dict(exact.summary) + diagnostics["target_issue_summary"] = dict(resolved.summary) diagnostics["provisional_target_reconciliation"] = { - "evidence_file_id": exact.file_id, - "matched_issue_cv_id": exact.issue_cv_id, + "evidence_file_id": resolved.file_id, + "matched_issue_cv_id": resolved.issue_cv_id, "previous_match_method": file.match_method, } - file.matched_issue_id = exact.issue_id - file.matched_issue_cv_id = exact.issue_cv_id + file.matched_issue_id = resolved.issue_id + file.matched_issue_cv_id = resolved.issue_cv_id file.match_method = "issue_number" file.diagnostics = diagnostics reconciled += 1 diff --git a/tests/unit/test_hyphenated_issue_numbers.py b/tests/unit/test_hyphenated_issue_numbers.py new file mode 100644 index 00000000..48cec75b --- /dev/null +++ b/tests/unit/test_hyphenated_issue_numbers.py @@ -0,0 +1,189 @@ +"""Dashed issue suffixes remain exact identities across metadata and discovery.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import TYPE_CHECKING +from unittest.mock import AsyncMock + +import pytest +from sqlalchemy import select + +from pullbox.core.issue_numbers import normalize_issue_number_queries, parse_issue_number_text +from pullbox.core.release_parser import normalize_issue_number, parse_release_title +from pullbox.models.issue import Issue, IssueStatus +from pullbox.models.series import Series +from pullbox.providers.metadata.comicvine import ComicVineProvider +from pullbox.services.catalog.reader import CatalogReader +from pullbox.services.metadata_service import MetadataService + +if TYPE_CHECKING: + from pathlib import Path + + from sqlalchemy.ext.asyncio import AsyncSession + + +@pytest.mark.parametrize("number", ["50-x", "50-o"]) +@pytest.mark.parametrize("source", ["comicvine", "catalog"]) +async def test_metadata_keeps_hyphenated_issue_identity(number: str, source: str) -> None: + if source == "catalog": + summary = CatalogReader._summary( + (501, 10, number, number, "50", "Special issue", None, None, None), + datetime(2026, 9, 17, tzinfo=UTC), + ) + else: + provider = ComicVineProvider(api_key="fixture", rate_limit=999_999) + provider._request = AsyncMock( + return_value={ + "results": [{"id": 501, "issue_number": number}], + "number_of_total_results": 1, + } + ) + try: + summary = (await provider.get_issues_for_series("10"))[0] + finally: + await provider._client.aclose() + assert summary.issue_number == 50.0 + assert summary.issue_number_text == number.upper() + + +@pytest.mark.parametrize("batch", [False, True]) +async def test_comicvine_issue_details_keep_hyphenated_identity(batch: bool) -> None: + item = {"id": 501, "volume": {"id": 10}, "issue_number": "50-x"} + provider = ComicVineProvider(api_key="fixture", rate_limit=999_999) + provider._request = AsyncMock(return_value={"results": [item] if batch else item}) + try: + issue = ( + (await provider.get_issue_batch(["501"]))["501"] + if batch + else await provider.get_issue("501") + ) + finally: + await provider._client.aclose() + assert issue.issue_number == 50.0 + assert issue.issue_number_text == "50-X" + + +async def test_comicvine_targeted_lookup_preserves_dash_and_deduplicates() -> None: + provider = ComicVineProvider(api_key="fixture", rate_limit=999_999) + request = AsyncMock( + side_effect=[ + {"results": [{"id": 501, "issue_number": "50-o"}]}, + {"results": [{"id": 502, "issue_number": "50-x"}]}, + ] + ) + provider._request = request + try: + summaries = await provider.get_issues_for_series_by_numbers("10", ["050-x", "50-X", "50-o"]) + finally: + await provider._client.aclose() + assert [item.issue_number_text for item in summaries] == ["50-O", "50-X"] + assert [call.args[1]["filter"] for call in request.await_args_list] == [ + "volume:10,issue_number:50-O", + "volume:10,issue_number:50-X", + ] + + +@pytest.mark.parametrize("existing_zero", [False, True]) +async def test_refresh_keeps_suffix_siblings_and_repairs_old_zero( + tmp_path: Path, db_session: AsyncSession, existing_zero: bool +) -> None: + series = Series(title="X-O Manowar", sort_title="X-O Manowar", comicvine_id=10) + db_session.add(series) + await db_session.flush() + previous = None + if existing_zero: + # Older parsing flattened the first suffix to #0. Keep ownership and its row ID. + previous = Issue( + series_id=series.id, + comicvine_id=501, + issue_number=0, + issue_number_text="0", + status=IssueStatus.OWNED, + ) + db_session.add(previous) + await db_session.flush() + provider = ComicVineProvider(api_key="fixture", rate_limit=999_999) + provider._request = AsyncMock( + return_value={ + "number_of_total_results": 3, + "results": [ + {"id": 501, "issue_number": "50-x"}, + {"id": 502, "issue_number": "50-o"}, + {"id": 503, "issue_number": "50"}, + ], + } + ) + service = MetadataService(provider, tmp_path) + try: + await service.fetch_issues_for_series(db_session, series.id) + await service.fetch_issues_for_series(db_session, series.id) + finally: + await provider._client.aclose() + issues = list( + (await db_session.scalars(select(Issue).where(Issue.series_id == series.id))).all() + ) + assert {issue.comicvine_id: issue.effective_issue_number_text for issue in issues} == { + 501: "50-X", + 502: "50-O", + 503: "50", + } + if previous is not None: + assert next(issue for issue in issues if issue.comicvine_id == 501).id == previous.id + assert previous.status == IssueStatus.OWNED + + +@pytest.mark.parametrize( + "raw, number, exact", + [ + ("50-x", 50, "50-X"), + ("050-X", 50, "50-X"), + ("50-o", 50, "50-O"), + ("-1-x", -1, "-1-X"), + ("0.5-o", 0.5, "0.5-O"), + ], +) +def test_numeric_compatibility_supports_hyphenated_suffix( + raw: str, number: float, exact: str +) -> None: + assert normalize_issue_number(raw) == number + assert parse_issue_number_text(raw) == (number, exact) + + +def test_lookup_keys_keep_suffixes_distinct_from_ranges_and_plain_numbers() -> None: + assert normalize_issue_number_queries([50, "50-x", "050-X", "50-o", "50X"]) == [ + 50.0, + "50-O", + "50-X", + "50X", + ] + for invalid in ["50-51", "50-", "50--x", "50-x-o"]: + with pytest.raises(ValueError): + parse_issue_number_text(invalid) + + +@pytest.mark.parametrize( + "title", + [ + "X-O Manowar #050-x (1996).cbz", + "X-O Manowar 050-x (1996).cbz", + "X-O.Manowar.050-x.1996.cbz", + "X-O_Manowar_050-x.cbz", + "X-O Manowar No.50-x (1996).cbz", + "X-O Manowar 50-x of 68 (1996).cbz", + ], +) +def test_release_parser_preserves_suffix_without_mangling_series(title: str) -> None: + parsed = parse_release_title(title) + assert parsed is not None + assert parsed.issue_number == 50.0 + assert parsed.issue_number_text == "50-X" + assert parsed.series_name == "X-O Manowar" + assert not parsed.is_pack + + +def test_numeric_range_is_still_a_pack_not_a_hyphenated_issue() -> None: + parsed = parse_release_title("X-O Manowar 050-051 (1996).cbz") + assert parsed is not None + assert parsed.is_pack + assert parsed.pack_range == "50-51" diff --git a/tests/unit/test_import_matching_performance.py b/tests/unit/test_import_matching_performance.py index c155c9ba..43fe28f0 100644 --- a/tests/unit/test_import_matching_performance.py +++ b/tests/unit/test_import_matching_performance.py @@ -3,6 +3,7 @@ from types import SimpleNamespace from unittest.mock import AsyncMock +from pullbox.models.import_job import ImportedSeries from pullbox.services import import_file_matching as matching @@ -56,7 +57,7 @@ async def cohorts(*args, **kwargs): result = await matching._finalize_import_series_file_groups( session, SimpleNamespace(id=1), - SimpleNamespace(id=2, raw_series_name="Test"), + ImportedSeries(id=2, raw_series_name="Test"), duplicate_group_counter=3, conflict_group_counter=4, detect_duplicate_copies=detect, diff --git a/tests/unit/test_import_provisional_targets.py b/tests/unit/test_import_provisional_targets.py index 53e6a8f7..e93ed686 100644 --- a/tests/unit/test_import_provisional_targets.py +++ b/tests/unit/test_import_provisional_targets.py @@ -335,7 +335,10 @@ async def test_matching_pipeline_groups_late_identity_without_provider_or_file_i provider = Mock() provider.cache_metrics.return_value = {} for name in [ - "get_series", "get_issue", "get_issues_for_series", "get_issues_for_series_by_numbers" + "get_series", + "get_issue", + "get_issues_for_series", + "get_issues_for_series_by_numbers", ]: setattr(provider, name, AsyncMock(side_effect=AssertionError("unexpected provider call"))) service = ImportService( diff --git a/tests/unit/test_letter_suffixed_issue_numbers.py b/tests/unit/test_letter_suffixed_issue_numbers.py index 7bde65fe..8c7d6e3c 100644 --- a/tests/unit/test_letter_suffixed_issue_numbers.py +++ b/tests/unit/test_letter_suffixed_issue_numbers.py @@ -44,6 +44,11 @@ ("X-Manowar", "50X", "50x", True), ("X-Manowar", "50O", "50o", True), ("X-Manowar", "50X", "50O", False), + ("X-Manowar", "50-X", "050-x", True), + ("X-Manowar", "50-O", "50-o", True), + ("X-Manowar", "50-X", "50-O", False), + ("X-Manowar", "50-X", "50", False), + ("X-Manowar", "50", "50-X", False), ("Gen13", "0.5", "0.5", True), ("Gen13", "-1", "-1", True), ], @@ -106,6 +111,9 @@ def test_direct_search_validates_the_exact_requested_issue( ("13A", "13B", False), ("13A", "13", False), ("50O", "50X", False), + ("50-X", "050-x", True), + ("50-O", "50-X", False), + ("50-X", "50", False), ("0.5", "00.50", True), ("-1", "-01", True), ], @@ -116,7 +124,7 @@ def test_artifact_coverage_does_not_alias_lettered_issues( assert _coverage_numbers_match(wanted, offered) is expected -@pytest.mark.parametrize("number", ["13A", "13B", "13C", "50X", "50O", "0.5", "-1"]) +@pytest.mark.parametrize("number", ["13A", "13B", "13C", "50X", "50O", "50-X", "50-O", "0.5", "-1"]) async def test_folder_discovery_preserves_exact_issue_designation( tmp_path: Path, number: str ) -> None: @@ -134,26 +142,31 @@ async def test_folder_discovery_preserves_exact_issue_designation( assert getattr(parsed, "issue_number_text", None) == number -async def test_new_series_provider_targets_do_not_collapse_suffix_siblings() -> None: +@pytest.mark.parametrize( + "numbers, numeric", [(["13A", "13B", "13C"], 13), (["50-X", "50-O", "50X"], 50)] +) +async def test_new_series_provider_targets_do_not_collapse_suffix_siblings( + numbers: list[str], numeric: int +) -> None: provider = AsyncMock() provider.get_issues_for_series.return_value = [ IssueSummary( provider_id=str(100 + index), - issue_number=13, + issue_number=numeric, issue_number_text=number, title=None, release_date=None, cover_url=None, issue_type="issue", ) - for index, number in enumerate(["13A", "13B", "13C"]) + for index, number in enumerate(numbers) ] series = ImportedSeries(raw_series_name="Gen13", cv_id=123, cv_issue_count=3) files = [ ImportedFile( - file_name=f"Gen13 #{number}.cbz", parsed_issue_number=13, issue_number_raw=number + file_name=f"Gen13 #{number}.cbz", parsed_issue_number=numeric, issue_number_raw=number ) - for number in ["13A", "13B", "13C", "13"] + for number in [*numbers, str(numeric)] ] targets = await load_file_match_target_index( AsyncMock(), series, duplicate_series=False, metadata_provider=provider, files=files diff --git a/tests/unit/test_mylar3_reader.py b/tests/unit/test_mylar3_reader.py index a70f76d5..c5327579 100644 --- a/tests/unit/test_mylar3_reader.py +++ b/tests/unit/test_mylar3_reader.py @@ -33,10 +33,12 @@ def _create_mylar_db( @pytest.mark.parametrize("recorded", [False, True]) -async def test_mylar_discovery_keeps_lettered_issue_numbers(tmp_path: Path, recorded: bool) -> None: +@pytest.mark.parametrize("numbers", [["13A", "13B", "13C"], ["50-O", "50-X"]]) +async def test_mylar_discovery_keeps_lettered_issue_numbers( + tmp_path: Path, recorded: bool, numbers: list[str] +) -> None: db = tmp_path / "mylar.db" folder = tmp_path / "comics" / "Gen13" - numbers = ["13A", "13B", "13C"] for number in numbers: create_minimal_cbz(folder / f"Gen13 #{number} (1996).cbz") _create_mylar_db( @@ -47,7 +49,7 @@ async def test_mylar_discovery_keeps_lettered_issue_numbers(tmp_path: Path, reco "ComicName": "Gen13", "ComicYear": "1996", "ComicLocation": str(folder), - "Total": 3, + "Total": len(numbers), } ], issues=[ From 736f8ade7003083b8937436b7205cc52e2abd2ab Mon Sep 17 00:00:00 2001 From: Adam Hernandez <adam@Hernandez-Partners.local> Date: Thu, 17 Sep 2026 13:11:57 -0700 Subject: [PATCH 12/20] fix(import): preserve control requests during worker handoff --- src/pullbox/tasks/import_task.py | 41 +++++++++- tests/unit/test_import_runner_handoff.py | 98 ++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_import_runner_handoff.py diff --git a/src/pullbox/tasks/import_task.py b/src/pullbox/tasks/import_task.py index 780bed07..2705ab22 100644 --- a/src/pullbox/tasks/import_task.py +++ b/src/pullbox/tasks/import_task.py @@ -427,6 +427,7 @@ def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: self._lock = asyncio.Lock() self._worker_task: asyncio.Task[None] | None = None self._active_job_id: int | None = None + self._pending_wakeup_job_id: int | None = None self._dispatch_recovered_requested = False async def recover_and_dispatch(self) -> int: @@ -516,7 +517,9 @@ async def _resume_next_recovered_job(self) -> int | None: async def _start_if_idle(self, job_id: int) -> None: async with self._lock: if self._worker_task is not None and not self._worker_task.done(): - if self._active_job_id != job_id: + if self._active_job_id == job_id: + self._pending_wakeup_job_id = job_id + else: logger.info( "import_runner_already_active", active_job_id=self._active_job_id, @@ -526,6 +529,29 @@ async def _start_if_idle(self, job_id: int) -> None: self._start_worker_locked(job_id) + async def _dispatch_pending_wakeup(self, job_id: int, *, recover_requested: bool) -> None: + """Recheck a control request accepted while the prior worker was exiting.""" + async with self._lock: + if self._worker_task is not None and not self._worker_task.done(): + self._dispatch_recovered_requested |= recover_requested + return + async with self._session_factory() as session: + job = await session.get(ImportJob, job_id) + runnable = job is not None and ( + job.status in _SCAN_STATES + or job.status == ImportJobStatus.ROLLING_BACK + or ( + job.status == ImportJobStatus.IMPORTING + and dict(job.progress_snapshot or {}).get("phase") != "story_arc_placements" + ) + ) + if runnable: + self._dispatch_recovered_requested |= recover_requested + self._start_worker_locked(job_id) + return + if recover_requested: + await self._dispatch_recovered_job() + def _start_worker_locked(self, job_id: int) -> None: """Start one serial worker while the runner lock is held.""" self._active_job_id = job_id @@ -554,10 +580,19 @@ def _on_worker_done(self, task: asyncio.Task[None]) -> None: logger.exception("import_runner_worker_failed") finally: if self._worker_task is task: + pending_job_id = self._pending_wakeup_job_id + self._pending_wakeup_job_id = None self._worker_task = None self._active_job_id = None - if self._dispatch_recovered_requested: - self._dispatch_recovered_requested = False + recover_requested = self._dispatch_recovered_requested + self._dispatch_recovered_requested = False + if pending_job_id is not None: + _fire_and_forget( + self._dispatch_pending_wakeup( + pending_job_id, recover_requested=recover_requested + ) + ) + elif recover_requested: _fire_and_forget(self._dispatch_recovered_job()) async def _mark_paused(self, session: AsyncSession, job_id: int) -> None: diff --git a/tests/unit/test_import_runner_handoff.py b/tests/unit/test_import_runner_handoff.py new file mode 100644 index 00000000..ea154513 --- /dev/null +++ b/tests/unit/test_import_runner_handoff.py @@ -0,0 +1,98 @@ +"""Import controls must not lose a wakeup while the previous worker exits.""" + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime + +import pytest +from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker + +from pullbox.models.import_job import ImportJob, ImportJobStatus, ImportSourceType +from pullbox.tasks import import_task + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("final_status", "phase", "expected_runs"), + [ + (ImportJobStatus.IMPORTING, "queued", 2), + (ImportJobStatus.FILE_MATCHING, "file_matching", 2), + (ImportJobStatus.ROLLING_BACK, "rollback", 2), + (ImportJobStatus.PAUSED, "importing", 1), + (ImportJobStatus.STALLED, "importing", 1), + (ImportJobStatus.REVIEW, "review", 1), + (ImportJobStatus.COMPLETED, "done", 1), + (ImportJobStatus.FAILED, "failed", 1), + (ImportJobStatus.CANCELLED, "cancelled", 1), + (ImportJobStatus.ROLLED_BACK, "done", 1), + (ImportJobStatus.IMPORTING, "story_arc_placements", 1), + (None, "deleted", 1), + ], +) +async def test_control_wakeup_survives_worker_handoff_without_replaying_finished_work( + async_engine: AsyncEngine, + final_status: ImportJobStatus | None, + phase: str, + expected_runs: int, +) -> None: + factory = async_sessionmaker(async_engine, expire_on_commit=False) + async with factory() as session: + job = ImportJob( + source_path="/imports/comics", + source_type=ImportSourceType.FILESYSTEM, + status=ImportJobStatus.IMPORTING, + import_started_at=datetime.now(UTC), + ) + session.add(job) + await session.commit() + job_id = int(job.id) + + runner = import_task.ImportRunner(factory) + started = asyncio.Event() + release = asyncio.Event() + calls: list[int] = [] + running = 0 + maximum_running = 0 + + async def run_job(requested_id: int) -> None: + nonlocal running, maximum_running + running += 1 + maximum_running = max(maximum_running, running) + calls.append(requested_id) + try: + if len(calls) == 1: + started.set() + await release.wait() + async with factory() as session: + current = await session.get(ImportJob, requested_id) + assert current is not None + if final_status is None: + await session.delete(current) + else: + current.status = final_status if len(calls) == 1 else ImportJobStatus.COMPLETED + current.progress_snapshot = {"phase": phase} + await session.commit() + finally: + running -= 1 + + runner._run_job = run_job # type: ignore[method-assign] + await runner.request_execute(job_id) + await asyncio.wait_for(started.wait(), timeout=1) + worker = runner._worker_task + assert worker is not None + # The API has accepted a control action while the old worker still owns + # the runner. Repeated notifications must coalesce, not run concurrently. + await runner.request_resume(job_id) + await runner.request_execute(job_id) + await runner.request_resume(job_id) + release.set() + await asyncio.wait_for(worker, timeout=2) + await asyncio.sleep(0) + if import_task._background_tasks: + await asyncio.wait_for(asyncio.gather(*import_task._background_tasks), timeout=2) + if runner._worker_task is not None: + await asyncio.wait_for(runner._worker_task, timeout=2) + + assert calls == [job_id] * expected_runs + assert maximum_running == 1 From c4131dd6fceb1fb37f4b7854b638dca0f4e2effa Mon Sep 17 00:00:00 2001 From: Adam Hernandez <adam@Hernandez-Partners.local> Date: Thu, 17 Sep 2026 15:03:17 -0700 Subject: [PATCH 13/20] fix(import): reconcile local volume and known annual evidence --- docs/development/IMPORT_REVIEW_RECOVERY.md | 15 + .../services/import_file_match_candidates.py | 11 + .../services/import_file_match_outcomes.py | 4 + src/pullbox/services/import_file_matching.py | 36 +- src/pullbox/services/import_known_annuals.py | 169 +++++++ .../unit/test_import_local_match_evidence.py | 413 ++++++++++++++++++ 6 files changed, 645 insertions(+), 3 deletions(-) create mode 100644 src/pullbox/services/import_known_annuals.py create mode 100644 tests/unit/test_import_local_match_evidence.py diff --git a/docs/development/IMPORT_REVIEW_RECOVERY.md b/docs/development/IMPORT_REVIEW_RECOVERY.md index e2fc72a3..2c663057 100644 --- a/docs/development/IMPORT_REVIEW_RECOVERY.md +++ b/docs/development/IMPORT_REVIEW_RECOVERY.md @@ -72,6 +72,21 @@ decisions, skips, and safety blocks are not overridden. Reads are paged, source signatures and files are unchanged, and no provider requests are added. The shared finalization applies to Mylar and folder imports in either handling mode. +Provisional targets use each file's inspected issue type, not the type of the +first same-number file in a matching page. This preserves numbered collected +volumes when deferred ComicInfo changes an initial ordinary-issue classification +and keeps regular issues separate from same-number annuals. + +An unrecorded annual can join an already identified annual series in the same +source folder when its filename and ComicInfo agree on the annual title and +exact issue designation, its publication year matches, and exactly one trusted +series is eligible. Conflicting IDs, dates, competing series identities, safety +blocks, and manual decisions remain untouched. This only changes the import +review grouping: it never moves source files, borrows a missing reference's +issue ID, or dismisses that reference. Normal issue matching and duplicate review +still run afterward. The rule applies to Mylar and folder imports, for both +copy and keep-in-place modes, without metadata-provider requests. + Exact issue designations retain letter suffixes and their hyphens, including `13A`, `50-X`, and `50-O`. ComicVine metadata, local catalog reads, Mylar and folder discovery, and release validation share this identity policy. A dashed diff --git a/src/pullbox/services/import_file_match_candidates.py b/src/pullbox/services/import_file_match_candidates.py index 82632d2e..b0a3ca20 100644 --- a/src/pullbox/services/import_file_match_candidates.py +++ b/src/pullbox/services/import_file_match_candidates.py @@ -153,6 +153,11 @@ def _target_issue_type( return IssueType.ISSUE +def _provisional_file_type(file: ImportedFile, fallback: IssueType | None) -> IssueType | None: + source_type = _coerce_issue_type((file.diagnostics or {}).get("source_issue_type")) + return source_type or _file_source_issue_type(file) or fallback + + def select_file_match_candidate( imp_file: ImportedFile, target_index: FileMatchTargetIndex, @@ -201,6 +206,10 @@ def select_file_match_candidate( if target_issue_number is None: return None provisional_type = target_index.provisional_exact_types.get(exact_issue_number or "") + if provisional_type is not None: + # A provisional slot has no catalog identity; its type belongs to + # this file, not whichever same-number file populated the page first. + provisional_type = _provisional_file_type(imp_file, provisional_type) return FileMatchCandidate( matched_issue_id=matched_issue_id, matched_issue_cv_id=matched_issue_cv_id, @@ -221,6 +230,8 @@ def select_file_match_candidate( target_index.number_map[issue_number] ) synthetic_issue_type = target_index.synthetic_issue_types.get(issue_number) + if issue_number in target_index.provisional_issue_numbers: + synthetic_issue_type = _provisional_file_type(imp_file, synthetic_issue_type) return FileMatchCandidate( matched_issue_id=matched_issue_id, matched_issue_cv_id=matched_issue_cv_id, diff --git a/src/pullbox/services/import_file_match_outcomes.py b/src/pullbox/services/import_file_match_outcomes.py index 35226493..72529a27 100644 --- a/src/pullbox/services/import_file_match_outcomes.py +++ b/src/pullbox/services/import_file_match_outcomes.py @@ -81,6 +81,10 @@ def apply_matched_file_outcome( imp_file.matched_issue_cv_id = match_candidate.matched_issue_cv_id imp_file.match_confidence = match_candidate.confidence imp_file.match_method = match_candidate.method + if imp_file.parsed_issue_number is None: + # Volume designations can be discovered after the initial filename scan. + # Persist the accepted number so paged duplicate/conflict grouping sees it. + imp_file.parsed_issue_number = match_candidate.target_issue_number if duplicate_series and match_candidate.matched_issue is not None: matched_issue = match_candidate.matched_issue diff --git a/src/pullbox/services/import_file_matching.py b/src/pullbox/services/import_file_matching.py index ada422be..f3f4aeee 100644 --- a/src/pullbox/services/import_file_matching.py +++ b/src/pullbox/services/import_file_matching.py @@ -22,6 +22,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker from pullbox.core.exceptions import ImportProviderDegradedError, JobPausedError +from pullbox.core.release_parser import parse_release_title from pullbox.models.import_job import ( ImportedFile, ImportedFileStatus, @@ -66,6 +67,7 @@ from pullbox.services.import_file_split_series import ( split_explicit_issue_series_mismatches as _split_explicit_issue_series_mismatches, ) +from pullbox.services.import_known_annuals import reassign_to_known_annual_series from pullbox.services.import_progress_runtime import ( ScanReviewFileMatchProfile, ScanReviewProgressPlan, @@ -1402,9 +1404,15 @@ async def process_file_page( reset_file_match_state(imp_file) file_metadata = source_metadata_for_import_file(imp_series, imp_file) if ( - imp_series.cv_match_method == "mylar3_cv_id" - and load_deferred_source_metadata_for_import_file is not None + load_deferred_source_metadata_for_import_file is not None and _file_has_deferred_archive_metadata(imp_file) + and ( + imp_series.cv_match_method == "mylar3_cv_id" + or ( + (release := parse_release_title(imp_file.file_name)) is not None + and release.issue_type.value in {"annual", "volume"} + ) + ) ): deferred_metadata_loads += 1 deferred_metadata_started_at = time.monotonic() @@ -1428,6 +1436,26 @@ async def process_file_page( duplicate_series=duplicate_series, metadata_provider=metadata_provider, ) + annual_series = await reassign_to_known_annual_series( + session, imp_series, imp_file, file_metadata + ) + if annual_series is not None: + created_split_series_ids.append(annual_series.id) + await log_event( + session, + job.id, + "INFO", + "import_file_assigned_to_known_annual", + message=( + f"Assigned {imp_file.file_name} to known annual series " + f"{annual_series.cv_title}; issue matching will follow." + ), + file_name=imp_file.file_name, + source_import_series_id=imp_series.id, + target_import_series_id=annual_series.id, + target_series_cv_id=annual_series.cv_id, + ) + continue file_evaluation_started_at = time.monotonic() match_candidate, metadata_conflict = _evaluate_file_match_candidate( imp_series=imp_series, @@ -1544,7 +1572,9 @@ async def process_file_page( live_only=True, ) - return files, created_split_series_ids + return [ + file for file in files if file.import_series_id == imp_series.id + ], created_split_series_ids async def process_split_series_batch(batch_ids: list[int]) -> None: nonlocal conflict_group_counter diff --git a/src/pullbox/services/import_known_annuals.py b/src/pullbox/services/import_known_annuals.py new file mode 100644 index 00000000..1eddb2e8 --- /dev/null +++ b/src/pullbox/services/import_known_annuals.py @@ -0,0 +1,169 @@ +"""Place corroborated annuals in an existing local review series without path repair.""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import TYPE_CHECKING + +from sqlalchemy import select + +from pullbox.core.issue_numbers import normalize_issue_number_text +from pullbox.core.name_matcher import NameMatcher +from pullbox.core.source_metadata import SourceMetadataExtractor +from pullbox.models.import_job import ImportedFileStatus, ImportedSeries, ImportSeriesStatus +from pullbox.models.issue import IssueType + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.core.source_metadata import SourceMetadata + from pullbox.models.import_job import ImportedFile + +_YEAR_QUALIFIER = re.compile(r"\((\d{4})(?:\s*-\s*(\d{4})?)?\)") +_KNOWN_METHODS = ("mylar3_cv_id", "comicinfo_cv_id", "folder_cv_id") + + +def _annual_title(series: str, year: int) -> str | None: + for match in _YEAR_QUALIFIER.finditer(series): + start = int(match[1]) + if "-" not in match[0]: + if start != year: + return None + elif start > year or (match[2] and int(match[2]) < year): + return None + title = NameMatcher.normalize(_YEAR_QUALIFIER.sub("", series)) + return title if title.endswith(" annual") else None + + +async def reassign_to_known_annual_series( + session: AsyncSession, + parent: ImportedSeries, + file: ImportedFile, + metadata: SourceMetadata, +) -> ImportedSeries | None: + """Require agreeing filename/ComicInfo and one same-folder, same-year known series. + + This does not identify a replacement for a missing Mylar path or borrow its + issue ID. The moved review row still passes normal issue matching afterward. + """ + diagnostics = dict(file.diagnostics or {}) + source = metadata.diagnostics + comicinfo = source.get("comicinfo") + path = Path(file.file_path) + if ( + file.status != ImportedFileStatus.PENDING + or file.comicvine_issue_id is not None + or metadata.comicvine_issue_id is not None + or file.library_file_id is not None + or file.include_in_import + or file.match_method + or parent.user_selected_cv_id is not None + or parent.selected_for_import + or metadata.issue_type != IssueType.ANNUAL + or not isinstance(comicinfo, dict) + or not path.is_absolute() + or ".." in path.parts + or any( + diagnostics.get(key) + for key in ("safety_block", "resolution", "review_selection", "identity_conflicts") + ) + or any(source.get(key) for key in ("identity_conflicts", "mylar3_issue", "file_safety")) + ): + return None + if metadata.signals.get("comicvine_series_id") in ("comicinfo", "sidecar"): + return None + release = SourceMetadataExtractor().from_release_title(file.file_name) + if ( + release.issue_type != IssueType.ANNUAL + or release.year is None + or release.issue_number is None + or not release.series_name + or not comicinfo.get("series") + or not comicinfo.get("number") + or comicinfo.get("year") not in (None, release.year) + ): + return None + try: + number = normalize_issue_number_text(comicinfo["number"]) + if number != normalize_issue_number_text(release.issue_number_text or release.issue_number): + return None + except (TypeError, ValueError): + return None + filename_title = NameMatcher.normalize(release.series_name) + if not filename_title.endswith(" annual"): + filename_title += " annual" + if _annual_title(str(comicinfo["series"]), release.year) != filename_title: + return None + if ( + NameMatcher.normalize(parent.cv_title or "") == filename_title + and parent.cv_year == release.year + ): + return None + candidates = list( + await session.scalars( + select(ImportedSeries) + .where( + ImportedSeries.import_job_id == parent.import_job_id, + ImportedSeries.source_folder == str(path.parent), + ImportedSeries.cv_year == release.year, + ImportedSeries.cv_id.is_not(None), + ) + .order_by(ImportedSeries.id) + .limit(26) + ) + ) + if len(candidates) > 25: + return None + candidates = [ + candidate + for candidate in candidates + if NameMatcher.normalize(candidate.cv_title or "") == filename_title + ] + if len(candidates) != 1 or candidates[0].cv_id == parent.cv_id: + return None + target = candidates[0] + if ( + target.cv_match_method not in _KNOWN_METHODS + or (target.cv_match_score or 0) < 0.99 + or target.user_selected_cv_id is not None + or target.selected_for_import + or target.status not in {ImportSeriesStatus.MATCHED, ImportSeriesStatus.DUPLICATE} + ): + return None + evidence = { + "source_import_series_id": parent.id, + "target_import_series_id": target.id, + "target_series_cv_id": target.cv_id, + "filename": file.file_name, + "comicinfo_series": comicinfo["series"], + "issue_number_text": number, + "publication_year": release.year, + "method": "same_folder_known_annual_with_corroborated_title", + } + source = dict(source) + source.pop("mylar3_folder_scope_conflict", None) + source.pop("mylar3_unrecorded_file", None) + source["known_annual_series"] = evidence + diagnostics.update( + { + "source_metadata": source, + "source_issue_type": "annual", + "comicvine_series_id": target.cv_id, + "known_annual_series": evidence, + } + ) + file.diagnostics = diagnostics + file.import_series_id = target.id + file.parsed_series = target.cv_title + file.parsed_year = release.year + file.parsed_issue_number = release.issue_number + file.issue_number_raw = number + parent.file_count = max(0, int(parent.file_count or 0) - 1) + parent.sample_paths = [ + value for value in (parent.sample_paths or []) if value != file.file_path + ] + target.file_count = int(target.file_count or 0) + 1 + target.has_files = True + target.sample_paths = list(dict.fromkeys([*(target.sample_paths or [])[:4], file.file_path])) + return target diff --git a/tests/unit/test_import_local_match_evidence.py b/tests/unit/test_import_local_match_evidence.py new file mode 100644 index 00000000..9b28eb9e --- /dev/null +++ b/tests/unit/test_import_local_match_evidence.py @@ -0,0 +1,413 @@ +"""Local metadata must resolve volumes and annuals without borrowing stale issue IDs.""" + +import zipfile +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest + +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportFileHandlingMode, + ImportJob, + ImportJobStatus, + ImportSeriesStatus, + ImportSourceType, +) +from pullbox.services import import_file_matching +from pullbox.services.import_known_annuals import reassign_to_known_annual_series +from pullbox.services.import_service import ImportService +from pullbox.services.import_source_metadata import load_deferred_source_metadata_for_import_file + + +async def _job_series(session, path, source_type, mode, *, title, year, cv_id): + job = ImportJob( + source_path=str(path), + source_type=source_type, + file_handling_mode=mode, + status=ImportJobStatus.FILE_MATCHING, + ) + session.add(job) + await session.flush() + series = ImportedSeries( + import_job_id=job.id, + raw_series_name=title, + raw_year=year, + source_folder=str(path), + cv_title=title, + cv_year=year, + cv_id=cv_id, + cv_match_method="mylar3_cv_id" + if source_type == ImportSourceType.MYLAR3 + else "folder_cv_id", + cv_match_score=1, + status=ImportSeriesStatus.MATCHED, + ) + session.add(series) + await session.flush() + return job, series + + +def _file(job, series, path, *, number=None, issue_type="issue", xml=None): + if xml is not None: + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("ComicInfo.xml", xml) + archive.writestr("001.jpg", b"page one") + archive.writestr("002.jpg", b"page two") + return ImportedFile( + import_job_id=job.id, + import_series_id=series.id, + file_path=str(path), + file_name=path.name, + file_format="cbz", + file_size=path.stat().st_size if path.exists() else 0, + parsed_series=series.raw_series_name, + parsed_year=series.raw_year, + parsed_issue_number=number, + issue_number_raw=str(number) if number is not None else None, + status=ImportedFileStatus.PENDING, + diagnostics={ + "source_issue_type": issue_type, + "comicvine_series_id": series.cv_id, + "metadata_signals": { + "comicvine_series_id": "mylar3" + if job.source_type == ImportSourceType.MYLAR3 + else "sidecar" + }, + "source_metadata": { + "archive_metadata_deferred": True, + "mylar3_unrecorded_file": {"expected_series": series.raw_series_name}, + }, + }, + ) + + +async def _match(session, job): + await session.commit() + provider = Mock() + provider.cache_metrics.return_value = {} + for name in ( + "get_series", + "get_issue", + "get_issues_for_series", + "get_issues_for_series_by_numbers", + ): + setattr(provider, name, AsyncMock(side_effect=AssertionError("Unexpected provider call"))) + service = ImportService( + series_service=AsyncMock(), + metadata_service=SimpleNamespace(_provider=provider), + event_bus=AsyncMock(), + ) + await service._run_file_matching(session, job) + assert not [call for call in provider.mock_calls if "get_" in call[0]] + + +@pytest.mark.parametrize("source_type", list(ImportSourceType)) +@pytest.mark.parametrize("mode", list(ImportFileHandlingMode)) +@pytest.mark.parametrize("number,year", [(9, 2020), (15, 2021), (16, 2021)]) +@pytest.mark.parametrize("copies", [1, 2]) +async def test_deferred_volume_evidence_matches_without_stale_mylar_identity( + db_session, tmp_path, source_type, mode, number, year, copies +): + job, series = await _job_series( + db_session, tmp_path, source_type, mode, title="Dawn of X", year=2020, cv_id=124996 + ) + file = _file( + job, + series, + tmp_path / f"Dawn of X v{number:02d} ({year}).cbz", + xml=f"<ComicInfo><Series>Dawn Of X</Series><Title>v{number:02d}" + f"{year}", + ) + db_session.add(file) + before = (tmp_path / file.file_name).read_bytes() + duplicate = None + if copies == 2: + duplicate = _file( + job, + series, + tmp_path / f"Dawn of X v{number:02d} ({year}) (Digital).cbz", + xml=f"Dawn Of XAlternate cover" + f"{year}", + ) + db_session.add(duplicate) + + await _match(db_session, job) + + assert file.status == ( + ImportedFileStatus.MATCHED if copies == 1 else ImportedFileStatus.CONFLICT + ) + assert file.matched_issue_cv_id is None + assert file.parsed_issue_number == number + details = file.diagnostics.get("previous_diagnostics", file.diagnostics) + assert details["target_issue_number"] == number + assert details["target_issue_type"] == "volume" + if duplicate is not None: + assert duplicate.status == ImportedFileStatus.CONFLICT + assert duplicate.conflict_group_id == file.conflict_group_id + assert (tmp_path / file.file_name).read_bytes() == before + + +@pytest.mark.parametrize("source_type", list(ImportSourceType)) +@pytest.mark.parametrize("mode", list(ImportFileHandlingMode)) +@pytest.mark.parametrize("page_size", [1, 250]) +@pytest.mark.parametrize("copies", [1, 2]) +@pytest.mark.parametrize( + "title,parent_year,year,parent_cv,annual_cv,embedded_title", + [ + ("X-Men", 2021, 2023, 137402, 146988, "X-Men (2021-) Annual"), + ("Marauders", 2022, 2022, 142135, 141459, "Marauders Annual (2022)"), + ], +) +async def test_corroborated_annual_uses_known_annual_series_without_consuming_missing_reference( + db_session, + tmp_path, + source_type, + mode, + title, + parent_year, + year, + parent_cv, + annual_cv, + embedded_title, + monkeypatch, + page_size, + copies, +): + job, parent = await _job_series( + db_session, tmp_path, source_type, mode, title=title, year=parent_year, cv_id=parent_cv + ) + annual = ImportedSeries( + import_job_id=job.id, + raw_series_name=f"{title} Annual", + raw_year=year, + source_folder=str(tmp_path), + cv_title=f"{title} Annual", + cv_year=year, + cv_id=annual_cv, + cv_match_method=parent.cv_match_method, + cv_match_score=1, + status=ImportSeriesStatus.MATCHED, + diagnostics={"source_issue_type": "annual"}, + ) + db_session.add(annual) + await db_session.flush() + actual = _file( + job, + parent, + tmp_path / f"{title} Annual 001 ({year}).cbz", + number=1, + issue_type="annual", + xml=f"{embedded_title}1", + ) + missing = _file( + job, + annual, + tmp_path / f"{title} Annual 001 ({year}) (Digital) (Group).cbz", + number=1, + issue_type="annual", + ) + missing.status = ImportedFileStatus.SAFETY_BLOCKED + missing.comicvine_issue_id = 999999 + missing.diagnostics = {"safety_block": {"code": "source_missing", "category": "source_missing"}} + db_session.add_all([actual, missing]) + regular = _file( + job, + parent, + tmp_path / f"{title} 001 ({parent_year}).cbz", + number=1, + xml=f"{title}1", + ) + db_session.add(regular) + duplicate = None + if copies == 2: + duplicate = _file( + job, + parent, + tmp_path / f"{title} Annual 01 ({year}).cbz", + number=1, + issue_type="annual", + xml=f"{embedded_title}1" + "Alternate cover", + ) + db_session.add(duplicate) + before = (tmp_path / actual.file_name).read_bytes() + monkeypatch.setattr(import_file_matching, "_FILE_PAGE_SIZE", page_size) + + await _match(db_session, job) + + assert actual.import_series_id == annual.id + expected_status = ImportedFileStatus.MATCHED if copies == 1 else ImportedFileStatus.CONFLICT + assert actual.status == expected_status + assert actual.matched_issue_cv_id is None + assert actual.comicvine_issue_id is None + details = actual.diagnostics.get("previous_diagnostics", actual.diagnostics) + assert details["target_series_cv_id"] == annual_cv + assert details["target_issue_type"] == "annual" + assert details["source_metadata"]["known_annual_series"]["target_series_cv_id"] == annual_cv + if duplicate is not None: + assert duplicate.import_series_id == annual.id + assert duplicate.status == ImportedFileStatus.CONFLICT + assert duplicate.conflict_group_id == actual.conflict_group_id + assert missing.status == ImportedFileStatus.SAFETY_BLOCKED + assert missing.comicvine_issue_id == 999999 + assert regular.import_series_id == parent.id + assert regular.status == ImportedFileStatus.MATCHED + assert regular.conflict_group_id is None + assert regular.diagnostics["target_issue_type"] == "issue" + assert job.total_files_matched == (2 if copies == 1 else 1) + assert job.total_files_conflict == (0 if copies == 1 else 2) + assert (tmp_path / actual.file_name).read_bytes() == before + + +@pytest.mark.parametrize( + "case", + [ + "wrong_title", + "wrong_number", + "wrong_year", + "wrong_title_year", + "other_type", + "no_comicinfo", + "source_issue_id", + "source_series_id", + "identity_conflict", + "blocked", + "approved", + "manual_file", + "selected_file", + "registered_file", + "manual_parent", + "selected_parent", + "manual_target", + "selected_target", + "skipped_target", + "uncertain_target", + "other_folder", + "other_job", + "ambiguous_target", + "no_target_year", + ], +) +async def test_known_annual_reassignment_preserves_uncertain_or_reviewed_sources( + db_session, tmp_path, case +): + job, parent = await _job_series( + db_session, + tmp_path, + ImportSourceType.MYLAR3, + ImportFileHandlingMode.IN_PLACE, + title="Marauders", + year=2022, + cv_id=142135, + ) + target = ImportedSeries( + import_job_id=job.id, + raw_series_name="Marauders Annual", + raw_year=2022, + source_folder=str(tmp_path), + cv_title="Marauders Annual", + cv_year=2022, + cv_id=141459, + cv_match_method="mylar3_cv_id", + cv_match_score=1, + status=ImportSeriesStatus.MATCHED, + ) + db_session.add(target) + await db_session.flush() + actual = _file( + job, + parent, + tmp_path / "Marauders Annual 001 (2022).cbz", + number=1, + issue_type="annual", + xml="Marauders Annual (2022)1", + ) + metadata = await load_deferred_source_metadata_for_import_file(parent, actual) + source = deepcopy(metadata.diagnostics) + if case in {"wrong_title", "wrong_number", "wrong_year", "wrong_title_year"}: + field, value = { + "wrong_title": ("series", "X-Men Annual"), + "wrong_number": ("number", "2"), + "wrong_year": ("year", 2023), + "wrong_title_year": ("series", "Marauders Annual (2023)"), + }[case] + source["comicinfo"][field] = value + elif case == "other_type": + from pullbox.models.issue import IssueType + + metadata = metadata.model_copy(update={"issue_type": IssueType.ISSUE}) + elif case == "no_comicinfo": + source.pop("comicinfo") + elif case == "source_issue_id": + actual.comicvine_issue_id = 555 + elif case == "source_series_id": + from pullbox.core.source_metadata import MetadataSignal + + metadata = metadata.model_copy( + update={ + "signals": {**metadata.signals, "comicvine_series_id": MetadataSignal.COMICINFO} + } + ) + elif case == "identity_conflict": + source["identity_conflicts"] = [{"field": "comicvine_issue_id"}] + elif case in {"blocked", "approved"}: + actual.status = ( + ImportedFileStatus.SAFETY_BLOCKED + if case == "blocked" + else ImportedFileStatus.SAFETY_APPROVED + ) + elif case == "manual_file": + actual.match_method = "manual" + elif case == "selected_file": + actual.include_in_import = True + elif case == "registered_file": + actual.library_file_id = 42 + elif case == "manual_parent": + parent.user_selected_cv_id = parent.cv_id + elif case == "selected_parent": + parent.selected_for_import = True + elif case == "manual_target": + target.user_selected_cv_id = target.cv_id + elif case == "selected_target": + target.selected_for_import = True + elif case == "skipped_target": + target.status = ImportSeriesStatus.SKIPPED + elif case == "uncertain_target": + target.cv_match_method = "search" + elif case == "other_folder": + target.source_folder = str(tmp_path / "other") + elif case == "other_job": + other_job = ImportJob(source_path="/elsewhere", source_type=ImportSourceType.MYLAR3) + db_session.add(other_job) + await db_session.flush() + target.import_job_id = other_job.id + elif case == "ambiguous_target": + db_session.add( + ImportedSeries( + import_job_id=job.id, + raw_series_name="Marauders Annual", + raw_year=2022, + cv_title="Marauders Annual", + cv_year=2022, + source_folder=str(tmp_path), + cv_id=999, + cv_match_method="mylar3_cv_id", + cv_match_score=1, + status=ImportSeriesStatus.MATCHED, + ) + ) + elif case == "no_target_year": + target.cv_year = None + await db_session.flush() + metadata = metadata.model_copy(update={"diagnostics": source}) + before = deepcopy(actual.diagnostics) + + result = await reassign_to_known_annual_series(db_session, parent, actual, metadata) + + assert result is None + assert actual.import_series_id == parent.id + assert actual.diagnostics == before From 3eca920955db2914bd8b904a816fb82277e1ce50 Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 15:32:00 -0700 Subject: [PATCH 14/20] feat(import): recognize source volume folders and record Mylar provenance --- docs/development/IMPORT_REVIEW_RECOVERY.md | 27 ++ src/pullbox/core/collection_scanner.py | 108 ++++++- src/pullbox/core/mylar3_reader.py | 22 ++ src/pullbox/core/source_volume_layout.py | 102 ++++++ .../services/import_layout_analysis.py | 30 ++ .../services/import_scan_materialization.py | 53 +++- src/pullbox/services/import_scan_pipeline.py | 17 + .../import_review_workspace_detail.html | 2 +- .../test_import_collection_shell_ui_routes.py | 25 ++ tests/unit/test_import_series_overrides.py | 8 + tests/unit/test_import_volume_leaf.py | 299 ++++++++++++++++++ tests/unit/test_mylar_source_provenance.py | 79 +++++ 12 files changed, 765 insertions(+), 7 deletions(-) create mode 100644 src/pullbox/core/source_volume_layout.py create mode 100644 tests/unit/test_import_volume_leaf.py create mode 100644 tests/unit/test_mylar_source_provenance.py diff --git a/docs/development/IMPORT_REVIEW_RECOVERY.md b/docs/development/IMPORT_REVIEW_RECOVERY.md index 2c663057..cafbe582 100644 --- a/docs/development/IMPORT_REVIEW_RECOVERY.md +++ b/docs/development/IMPORT_REVIEW_RECOVERY.md @@ -95,6 +95,33 @@ must not collapse into the same target. Refreshing live metadata can correct an older zero-number fallback using its existing ComicVine issue ID without discarding ownership. No schema migration or source-file rename is required. +## Source Volume Folders And Mylar Provenance + +Automatic folder discovery recognizes `Series/v2017`, `Publisher/Series/v2017`, +`Series/v2`, and `Publisher/Series/v2`. The parent supplies a series-name hint; +`v2017` supplies a series-year hint, while `v2` is an ordinal volume, never an +issue number or a year. Filename identity with an issue designation or agreeing +local sidecar/ComicInfo evidence must corroborate the parent. Layout recognition +is not a confirmed ComicVine match. An ordinal folder without an explicit +series-year or series ID remains in the existing series-review workflow. + +Step 1 remains filename-only and does not open archives or call providers. +The scan reuses its existing local metadata reads for corroboration. Weak or +contradictory evidence stays reviewable per file, while proven per-file identities +continue through existing mixed-folder recovery. A legitimate series named `V2` +is not replaced by its parent. Custom source layouts remain authoritative, and +parent sidecars are not inherited across releases. No source folder or file is +renamed, moved, or rewritten by this recognition, and it never selects a future +managed-library layout. Existing copy/in-place policies remain independent. + +Mylar scans log `mylar3_source_provenance`, including optional +`mylar_info.DatabaseVersion`, series count, Story Arc/reading-list presence, +reading-list count, and supported configuration key names. Missing or malformed +version data becomes `unknown`; future numeric versions remain diagnostic data, +not admission gates. `config.ini` remains optional and no configuration values +or credentials are logged. Both paged and full snapshot readers remain read-only. +This introduces no database migration, monitoring change, or new import card. + ## Import Follow-up The Follow-up tab groups actionable work by import job rather than rendering diff --git a/src/pullbox/core/collection_scanner.py b/src/pullbox/core/collection_scanner.py index 7b780e7a..55e73afa 100644 --- a/src/pullbox/core/collection_scanner.py +++ b/src/pullbox/core/collection_scanner.py @@ -51,6 +51,7 @@ from pullbox.core.naming_type_detection import detect_issue_type from pullbox.core.release_parser import normalize_issue_number from pullbox.core.source_metadata import MetadataSignal, SourceMetadata, SourceMetadataExtractor +from pullbox.core.source_volume_layout import assess_volume_leaf, volume_leaf_from_path from pullbox.models.issue import IssueType logger = structlog.get_logger(__name__) @@ -1308,6 +1309,54 @@ def apply_layout_value( if conflicts: metadata_diagnostics["source_layout_conflicts"] = conflicts + if root is not None and ( + self._source_layout.mode == ImportLayoutMode.AUTO + or (layout_match is None and self._source_layout.fallback_to_auto) + ): + try: + leaf = volume_leaf_from_path(fpath.relative_to(root).as_posix()) + except ValueError: + leaf = None + if leaf is not None: + comicinfo = metadata_diagnostics.get("comicinfo") + local_names = [ + sidecar_data.get("series_name") if sidecar_data else None, + comicinfo.get("series") if isinstance(comicinfo, dict) else None, + ] + comicinfo_volume = ( + comicinfo.get("volume") if isinstance(comicinfo, dict) else None + ) + metadata_series_year = ( + int(comicinfo_volume) + if isinstance(comicinfo_volume, str) + and re.fullmatch(r"[0-9]{4}", comicinfo_volume) + else sidecar_data.get("year") + if sidecar_data + else None + ) + literal, confirmed, review = assess_volume_leaf( + leaf, + file_name, + metadata_names=tuple( + name for name in local_names if isinstance(name, str) and name + ), + proven_file_identity=( + metadata.signals.get("comicvine_series_id") == MetadataSignal.COMICINFO + or metadata.comicvine_issue_id is not None + ), + identity_conflict=bool(metadata_diagnostics.get("identity_conflicts")), + metadata_series_year=metadata_series_year, + ) + if not literal: + metadata_diagnostics["volume_leaf"] = { + **leaf.evidence(confirmed=confirmed, review_required=review), + "metadata_series_year": metadata_series_year, + } + if confirmed: + parsed_series = leaf.series + if parsed_publisher is None: + parsed_publisher = leaf.publisher + issue_number_raw: str | None = None if layout_issue_number_raw is not None: issue_number_raw = layout_issue_number_raw @@ -1504,6 +1553,18 @@ def _build_series_candidates( tuple[str, int | None, str | None], tuple[str, int | None, str | None], ] = {} + volume_hint = next( + ( + file.metadata_diagnostics["volume_leaf"] + for file in discovered_files + if isinstance(file.metadata_diagnostics.get("volume_leaf"), dict) + ), + None, + ) + if isinstance(volume_hint, dict): + folder_name = str(volume_hint["series"]) + folder_year = cast("int | None", volume_hint.get("series_year_hint")) + folder_publisher = cast("str | None", volume_hint.get("publisher")) folder_identity = _normalize_series_identity(folder_name) folder_is_series_boundary = root is None or source_dir != root @@ -1599,7 +1660,12 @@ def _build_series_candidates( discovered_file.parsed_series = folder_name if discovered_file.parsed_year is None: discovered_file.parsed_year = folder_year - identity = (folder_identity, discovered_file.parsed_year or folder_year) + identity = ( + folder_identity, + folder_year + if volume_hint is not None + else discovered_file.parsed_year or folder_year, + ) identity_key = (identity[0], identity[1], type_discriminator) @@ -1623,7 +1689,9 @@ def _build_series_candidates( identity_key, ( label_name, - discovered_file.parsed_year or folder_year, + folder_year + if volume_hint is not None and collapse_to_folder + else discovered_file.parsed_year or folder_year, discovered_file.parsed_publisher or folder_publisher, ), ) @@ -1683,6 +1751,41 @@ def _build_series_candidates( comicinfo_source = discovered_file.file_path break source_issue_type = _source_issue_type_for_group(files) + volume_evidence = [ + evidence + for file in files + if isinstance(evidence := file.metadata_diagnostics.get("volume_leaf"), dict) + ] + volume_confirmed = len(volume_evidence) == len(files) and all( + evidence.get("confirmed") for evidence in volume_evidence + ) + if volume_hint is not None and volume_confirmed: + metadata_years = { + year + for evidence in volume_evidence + if isinstance(year := evidence.get("metadata_series_year"), int) + } + raw_year = folder_year or ( + next(iter(metadata_years)) if len(metadata_years) == 1 else None + ) + + volume_diagnostics = {} + if isinstance(volume_hint, dict): + volume_diagnostics = { + "volume_leaf": { + **volume_hint, + "confirmed": volume_confirmed, + "review_required": any( + evidence.get("review_required") for evidence in volume_evidence + ), + "series_confirmation_required": bool( + volume_hint.get("volume_hint") is not None + and raw_year is None + and comicinfo_cv_id is None + and folder_cv_id is None + ), + } + } for discovered_file in files: if series_status is None and discovered_file.series_status: series_status = discovered_file.series_status @@ -1709,6 +1812,7 @@ def _build_series_candidates( ), "series_status": series_status, "issue_count_hint": issue_count_hint, + **volume_diagnostics, **( { "folder_identity": { diff --git a/src/pullbox/core/mylar3_reader.py b/src/pullbox/core/mylar3_reader.py index bf090d7b..5e3bfa6b 100644 --- a/src/pullbox/core/mylar3_reader.py +++ b/src/pullbox/core/mylar3_reader.py @@ -135,6 +135,7 @@ class Mylar3CollectionSnapshot: readlist_present: bool readlist_count: int arc_settings: Mylar3ArcSettingsSnapshot + database_version: str = "unknown" @dataclass(frozen=True, slots=True) @@ -146,6 +147,7 @@ class Mylar3ImportMetadataSnapshot: readlist_count: int arc_settings: Mylar3ArcSettingsSnapshot series_count: int = 0 + database_version: str = "unknown" @dataclass(frozen=True, slots=True) @@ -460,6 +462,7 @@ def _read_snapshot_sync(self) -> Mylar3CollectionSnapshot: issue_records = self._read_issue_records(conn) story_arc_rows, storyarcs_present = self._read_story_arc_rows(conn) readlist_present, readlist_count = self._read_readlist_count(conn) + database_version = self._read_database_version(conn) except sqlite3.DatabaseError as exc: msg = f"Could not read Mylar3 database: {exc}" raise MylarReadError(msg) from exc @@ -473,6 +476,7 @@ def _read_snapshot_sync(self) -> Mylar3CollectionSnapshot: readlist_present=readlist_present, readlist_count=readlist_count, arc_settings=self._read_arc_settings(), + database_version=database_version, ) def _require_database(self) -> None: @@ -504,6 +508,7 @@ def _read_import_metadata_sync(self) -> Mylar3ImportMetadataSnapshot: storyarcs_present = self._table_exists(conn, "storyarcs") readlist_present, readlist_count = self._read_readlist_count(conn) series_count = int(conn.execute("SELECT COUNT(*) FROM comics").fetchone()[0]) + database_version = self._read_database_version(conn) except sqlite3.DatabaseError as exc: msg = f"Could not read Mylar3 database: {exc}" raise MylarReadError(msg) from exc @@ -516,8 +521,24 @@ def _read_import_metadata_sync(self) -> Mylar3ImportMetadataSnapshot: readlist_count=readlist_count, arc_settings=self._read_arc_settings(), series_count=series_count, + database_version=database_version, ) + def _read_database_version(self, conn: sqlite3.Connection) -> str: + """Read optional schema provenance without rejecting older/future sources.""" + try: + columns = self._table_columns(conn, "mylar_info") + if "databaseversion" not in {column.casefold() for column in columns}: + return "unknown" + rows = conn.execute("SELECT DatabaseVersion FROM mylar_info LIMIT 2").fetchall() + except sqlite3.DatabaseError: + logger.warning("mylar3_database_version_unavailable") + return "unknown" + if len(rows) != 1 or not isinstance(rows[0][0], str | int): + return "unknown" + value = str(rows[0][0]).strip() + return value if re.fullmatch(r"[0-9]{1,9}(?:\.[0-9]{1,9}){0,2}", value) else "unknown" + def _read_import_series_page_sync( self, after_rowid: int, @@ -2536,6 +2557,7 @@ def _table_columns(self, conn: sqlite3.Connection, table_name: str) -> set[str]: "annuals": "PRAGMA table_info(annuals)", "storyarcs": "PRAGMA table_info(storyarcs)", "readlist": "PRAGMA table_info(readlist)", + "mylar_info": "PRAGMA table_info(mylar_info)", } query = queries.get(table_name) if query is None: diff --git a/src/pullbox/core/source_volume_layout.py b/src/pullbox/core/source_volume_layout.py new file mode 100644 index 00000000..6cd0a918 --- /dev/null +++ b/src/pullbox/core/source_volume_layout.py @@ -0,0 +1,102 @@ +"""Evidence-aware source folder hints; never a managed destination layout.""" + +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass +from pathlib import PurePosixPath + +from pullbox.core.library_layout import split_series_year +from pullbox.core.source_metadata import SourceMetadataExtractor + +_MARKER = re.compile(r"v([0-9]{1,4})", re.IGNORECASE) +_GENERIC = frozenset({"comics", "collection", "books", "downloads", "imports", "media", "library"}) + + +def normalized_title(value: str) -> str: + """Compare names without discarding non-ASCII letters or numeric titles.""" + return "".join(c for c in unicodedata.normalize("NFKC", value).casefold() if c.isalnum()) + + +@dataclass(frozen=True, slots=True) +class VolumeLeaf: + series: str + publisher: str | None + marker: str + year: int | None + volume: int | None + parent_year_conflict: bool = False + + def evidence(self, *, confirmed: bool, review_required: bool) -> dict[str, object]: + return { + "series": self.series, + "publisher": self.publisher, + "marker": self.marker, + "series_year_hint": self.year, + "volume_hint": self.volume, + "confirmed": confirmed, + "review_required": review_required, + } + + +def volume_leaf_from_path(relative_path: str) -> VolumeLeaf | None: + """Recognize only supported root-relative shapes without traversing ancestors.""" + path = PurePosixPath(relative_path.replace("\\", "/")) + parts = path.parts + if path.is_absolute() or ".." in parts or len(parts) not in (3, 4): + return None + marker = _MARKER.fullmatch(parts[-2]) + if marker is None or int(marker[1]) < 1: + return None + parent = re.sub(r"\s+\[(?:cv-)?\d+\]$", "", parts[-3], flags=re.IGNORECASE) + series, parent_year = split_series_year(parent) + if not series or normalized_title(series) in _GENERIC: + return None + value = int(marker[1]) + year = value if len(marker[1]) == 4 and 1800 <= value <= 2099 else None + publisher = parts[0] if len(parts) == 4 and parts[0].casefold() not in _GENERIC else None + return VolumeLeaf( + series=series, + publisher=publisher, + marker=parts[-2], + year=year or parent_year, + volume=value if year is None else None, + parent_year_conflict=year is not None and parent_year is not None and year != parent_year, + ) + + +def assess_volume_leaf( + leaf: VolumeLeaf, + file_name: str, + *, + metadata_names: tuple[str, ...] = (), + proven_file_identity: bool = False, + identity_conflict: bool = False, + metadata_series_year: int | None = None, +) -> tuple[bool, bool, bool]: + """Return (literal series title, corroborated layout, needs identity review). + + Exact per-file identity survives a wrong folder. A sidecar alone cannot + silently overrule a conflicting filename or another metadata identity. + """ + parsed = SourceMetadataExtractor().from_release_title(file_name) + if re.match(rf"^{re.escape(leaf.marker)}[ ._-]+#?\d", file_name, re.IGNORECASE): + return True, False, False + filename_title = parsed.series_name if parsed.issue_number is not None else None + names = [name for name in (*metadata_names, filename_title) if name] + if any(normalized_title(name) == normalized_title(leaf.marker) for name in names): + return True, False, False + expected = normalized_title(leaf.series) + confirmed = bool(names) and all(normalized_title(name) == expected for name in names) + if identity_conflict: + return False, False, True + if leaf.parent_year_conflict or ( + metadata_series_year is not None + and leaf.year is not None + and metadata_series_year != leaf.year + ): + return False, False, not proven_file_identity + if confirmed: + return False, True, False + return False, False, not proven_file_identity diff --git a/src/pullbox/services/import_layout_analysis.py b/src/pullbox/services/import_layout_analysis.py index b58b73da..60cc47da 100644 --- a/src/pullbox/services/import_layout_analysis.py +++ b/src/pullbox/services/import_layout_analysis.py @@ -23,6 +23,7 @@ resolve_source_layout_spec, ) from pullbox.core.source_metadata import SourceMetadataExtractor +from pullbox.core.source_volume_layout import assess_volume_leaf, volume_leaf_from_path _GENERIC_OR_TYPE_CONTAINERS = frozenset( { @@ -334,6 +335,35 @@ def _analyze_path( return self._auto_analyze_path(relative_path) def _auto_analyze_path(self, relative_path: str) -> _AnalyzedPath: + leaf = volume_leaf_from_path(relative_path) + if leaf is not None: + literal, confirmed, _review = assess_volume_leaf( + leaf, PurePosixPath(relative_path).name + ) + if not literal: + if not confirmed: + return self._needs_review_path( + relative_path, "volume_leaf_identity_unconfirmed" + ) + metadata = self._metadata_extractor.from_release_title( + PurePosixPath(relative_path).name + ) + return _AnalyzedPath( + cluster_key="auto:volume_leaf", + classification=LayoutClassification.NORMAL_LIBRARY, + confidence="high", + match=SourceLayoutMatch( + relative_path=relative_path, + series=leaf.series, + publisher=leaf.publisher, + year=leaf.year, + issue_number=metadata.issue_number_text, + ), + proposed_series_path_template=None, + proposed_issue_filename_template=None, + evidence=["volume_leaf", "filename_parent_agreement"], + warnings=[], + ) parts = PurePosixPath(relative_path).parts depth = len(parts) - 1 if depth == 1: diff --git a/src/pullbox/services/import_scan_materialization.py b/src/pullbox/services/import_scan_materialization.py index 3e8c9013..261d7323 100644 --- a/src/pullbox/services/import_scan_materialization.py +++ b/src/pullbox/services/import_scan_materialization.py @@ -30,6 +30,16 @@ _MYLAR_FOLDER_SCOPE_REVIEW_MESSAGE = ( "This unrecorded file appears to belong to another series in the selected Mylar folder." ) +_VOLUME_LEAF_REVIEW_REASON = "volume_leaf_identity_unconfirmed" +_VOLUME_LEAF_REVIEW_MESSAGE = ( + "The volume folder does not provide enough agreeing evidence for this file. " + "Confirm its series using the filename or embedded metadata before importing." +) + + +def _requires_volume_leaf_review(metadata_diagnostics: dict[str, object]) -> bool: + evidence = metadata_diagnostics.get("volume_leaf") + return isinstance(evidence, dict) and evidence.get("review_required") is True def _requires_source_layout_review(metadata_diagnostics: dict[str, object]) -> bool: @@ -62,20 +72,45 @@ async def materialize_discovered_scan_results( ) layout_review_count = sum( _requires_source_layout_review(dict(discovered_file.metadata_diagnostics)) + or _requires_volume_leaf_review(dict(discovered_file.metadata_diagnostics)) for discovered_file in discovered.files ) all_files_require_layout_review = bool(discovered.files) and layout_review_count == len( discovered.files ) series_diagnostics = dict(discovered.diagnostics) + volume_leaf = series_diagnostics.get("volume_leaf") + release_review = ( + isinstance(volume_leaf, dict) + and volume_leaf.get("series_confirmation_required") is True + ) + if release_review: + series_diagnostics.update( + { + "kind": "source_layout_review", + "reason": "volume_leaf_release_unconfirmed", + "rejection_reason": ( + "The volume folder does not identify a unique series release. " + "Confirm the series match before importing." + ), + } + ) if layout_review_count: series_diagnostics["source_layout_review_files"] = layout_review_count if all_files_require_layout_review: + volume_review = any( + _requires_volume_leaf_review(dict(file.metadata_diagnostics)) + for file in discovered.files + ) series_diagnostics.update( { "kind": "source_layout_review", - "reason": _SOURCE_LAYOUT_REVIEW_REASON, - "rejection_reason": _SOURCE_LAYOUT_REVIEW_MESSAGE, + "reason": _VOLUME_LEAF_REVIEW_REASON + if volume_review + else _SOURCE_LAYOUT_REVIEW_REASON, + "rejection_reason": _VOLUME_LEAF_REVIEW_MESSAGE + if volume_review + else _SOURCE_LAYOUT_REVIEW_MESSAGE, } ) item = ImportedSeries( @@ -89,7 +124,7 @@ async def materialize_discovered_scan_results( has_files=discovered.has_files, status=( ImportSeriesStatus.NO_MATCH - if all_files_require_layout_review or mylar_path_incompatible + if all_files_require_layout_review or mylar_path_incompatible or release_review else ImportSeriesStatus.PENDING ), diagnostics=series_diagnostics, @@ -118,10 +153,11 @@ async def materialize_discovered_scan_results( metadata_diagnostics = dict(df.metadata_diagnostics) safety_block = metadata_diagnostics.pop("file_safety", None) source_layout_review = _requires_source_layout_review(metadata_diagnostics) + volume_leaf_review = _requires_volume_leaf_review(metadata_diagnostics) mylar_folder_scope_review = _requires_mylar_folder_scope_review(metadata_diagnostics) if isinstance(safety_block, dict): file_status = ImportedFileStatus.SAFETY_BLOCKED - elif source_layout_review or mylar_folder_scope_review: + elif source_layout_review or mylar_folder_scope_review or volume_leaf_review: file_status = ImportedFileStatus.NO_MATCH else: file_status = ImportedFileStatus.PENDING @@ -150,6 +186,15 @@ async def materialize_discovered_scan_results( "rejection_reason": _SOURCE_LAYOUT_REVIEW_MESSAGE, } ) + elif volume_leaf_review: + diagnostics.update( + { + "kind": "source_scope_review", + "reason": _VOLUME_LEAF_REVIEW_REASON, + "rejection_reason": _VOLUME_LEAF_REVIEW_MESSAGE, + "preserve_series_match": True, + } + ) elif mylar_folder_scope_review: diagnostics.update( { diff --git a/src/pullbox/services/import_scan_pipeline.py b/src/pullbox/services/import_scan_pipeline.py index 7bc189d5..41493afe 100644 --- a/src/pullbox/services/import_scan_pipeline.py +++ b/src/pullbox/services/import_scan_pipeline.py @@ -677,8 +677,25 @@ async def check_mylar_staging_cancellation() -> None: readlist_count=legacy_snapshot.readlist_count, arc_settings=legacy_snapshot.arc_settings, series_count=len(legacy_snapshot.series), + database_version=legacy_snapshot.database_version, ) + await log_event( + session, + job_id, + "INFO", + "mylar3_source_provenance", + message="Read Mylar source database provenance", + source_kind="mylar3", + database_version=metadata.database_version, + series_count=metadata.series_count, + storyarcs_present=metadata.storyarcs_present, + readlist_present=metadata.readlist_present, + readlist_count=metadata.readlist_count, + config_present=metadata.arc_settings.present, + supported_config_keys=[value.key for value in metadata.arc_settings.values], + ) + story_arc_staging = StoryArcStagingResult( readlist_present=metadata.readlist_present, readlist_count=metadata.readlist_count, diff --git a/src/pullbox/ui/templates/partials/import_review_workspace_detail.html b/src/pullbox/ui/templates/partials/import_review_workspace_detail.html index 92d9fd91..6e121c34 100644 --- a/src/pullbox/ui/templates/partials/import_review_workspace_detail.html +++ b/src/pullbox/ui/templates/partials/import_review_workspace_detail.html @@ -53,7 +53,7 @@

{% endif %} - {% if series_diagnostics.get('kind') == 'source_layout_review' %}

Layout review: {{ series_diagnostics.get('source_layout_review_files', 1) }} {{ 'files do' if series_diagnostics.get('source_layout_review_files', 1) != 1 else 'file does' }} not fit the selected source layout and will not be matched automatically. {{ series_diagnostics.get('rejection_reason', '') }}

{% elif series_diagnostics.get('kind') == 'mylar3_path_incompatible' %}

Mylar path review: {{ series_diagnostics.get('rejection_reason', '') }} Correct the Mylar path mapping and retry this import.

{% endif %} + {% if series_diagnostics.get('kind') == 'source_layout_review' %}

{% if series_diagnostics.get('reason') not in ('volume_leaf_identity_unconfirmed', 'volume_leaf_release_unconfirmed') %}Layout review: {{ series_diagnostics.get('source_layout_review_files', 1) }} {{ 'files do' if series_diagnostics.get('source_layout_review_files', 1) != 1 else 'file does' }} not fit the selected source layout and will not be matched automatically. {% endif %}{{ series_diagnostics.get('rejection_reason', '') }}

{% elif series_diagnostics.get('kind') == 'mylar3_path_incompatible' %}

Mylar path review: {{ series_diagnostics.get('rejection_reason', '') }} Correct the Mylar path mapping and retry this import.

{% endif %} {% if reason_key != 'needs_series' %} {% include "partials/import_review_copy_choices.html" %} {% for file in safety_blocked_files_by_series_id.get(item.id, []) %} diff --git a/tests/ui/test_import_collection_shell_ui_routes.py b/tests/ui/test_import_collection_shell_ui_routes.py index 857c4f38..6f291a61 100644 --- a/tests/ui/test_import_collection_shell_ui_routes.py +++ b/tests/ui/test_import_collection_shell_ui_routes.py @@ -1845,6 +1845,31 @@ async def test_import_review_partial_explains_selected_layout_review( assert "1 file does not fit the selected source layout" in response.text assert "will not be matched automatically" in response.text + async def test_import_review_partial_explains_uncertain_volume_without_custom_layout( + self, + authenticated_client, + sec_db, + ) -> None: # type: ignore[no-untyped-def] + from pullbox.models.import_job import ImportedSeries + + job_id = await _seed_import_review_job(sec_db) + message = "The volume folder does not identify a unique series release." + async with sec_db() as session: + series = await session.get(ImportedSeries, 11) + assert series is not None + series.diagnostics = { + "kind": "source_layout_review", + "reason": "volume_leaf_release_unconfirmed", + "rejection_reason": message, + } + await session.commit() + + response = await authenticated_client.get(f"/import/{job_id}/review-partial") + + assert response.status_code == 200 + assert message in response.text + assert "not fit the selected source layout" not in response.text + async def test_import_review_partial_explains_incompatible_mylar_path( self, authenticated_client, diff --git a/tests/unit/test_import_series_overrides.py b/tests/unit/test_import_series_overrides.py index c0b19079..00f9129f 100644 --- a/tests/unit/test_import_series_overrides.py +++ b/tests/unit/test_import_series_overrides.py @@ -53,11 +53,18 @@ def _series_metadata(cv_id: int) -> SeriesMetadata: ) +@pytest.mark.parametrize( + "review_reason", + [None, "volume_leaf_identity_unconfirmed", "volume_leaf_release_unconfirmed"], +) async def test_override_cv_id_sets_metadata_and_reruns_matching( db_session: AsyncSession, + review_reason: str | None, ) -> None: job = await _create_job_row(db_session) series = await _create_series_row(db_session, job) + if review_reason: + series.diagnostics = {"kind": "source_layout_review", "reason": review_reason} calls: list[tuple[str, list[int] | None]] = [] async def fetch_series_metadata(cv_id: int) -> SeriesMetadata: @@ -134,6 +141,7 @@ async def run_file_matching( assert updated.cv_title == "Batman" assert updated.cv_publisher == "DC Comics" assert updated.cv_match_method == "user_override" + assert updated.diagnostics == {} assert calls == [ ("reclassify", [series.id]), ("reset", [series.id]), diff --git a/tests/unit/test_import_volume_leaf.py b/tests/unit/test_import_volume_leaf.py new file mode 100644 index 00000000..0ffa5a21 --- /dev/null +++ b/tests/unit/test_import_volume_leaf.py @@ -0,0 +1,299 @@ +"""Volume directories are source hints, never issue identities or target policies.""" + +import json +import zipfile + +import pytest +from sqlalchemy import select + +from pullbox.core.collection_scanner import CollectionScanner +from pullbox.core.library_layout import ImportLayoutMode, SourceLayoutSpec +from pullbox.core.source_volume_layout import assess_volume_leaf, volume_leaf_from_path +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportJob, + ImportSeriesStatus, + ImportSourceType, +) +from pullbox.services.import_layout_analysis import ImportLayoutAnalyzer +from pullbox.services.import_scan_materialization import materialize_discovered_scan_results + + +def _comic(path, xml=None): + path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("001.jpg", b"one") + archive.writestr("002.jpg", b"two") + if xml: + archive.writestr("ComicInfo.xml", xml) + return path + + +@pytest.mark.parametrize("publisher", [None, "DC Comics"]) +@pytest.mark.parametrize("marker,year", [("v2017", 2017), ("v2", None)]) +async def test_volume_leaf_layout_and_scan_agree(tmp_path, publisher, marker, year): + folder = (tmp_path / publisher if publisher else tmp_path) / "Mister Miracle" / marker + path = _comic(folder / "Mister Miracle 001.cbz") + before = path.read_bytes() + analysis = await ImportLayoutAnalyzer().analyze(tmp_path) + assert analysis.files_fitting == 1 + example = analysis.clusters[0].examples[0] + assert (example.series, example.publisher, example.year) == ("Mister Miracle", publisher, year) + assert analysis.can_apply_future_policy is False + assert analysis.archive_probes == 0 + results = [series async for series in CollectionScanner().scan(tmp_path)] + assert len(results) == 1 + assert (results[0].raw_series_name, results[0].raw_publisher, results[0].raw_year) == ( + "Mister Miracle", + publisher, + year, + ) + assert results[0].source_folder == str(folder) + assert results[0].files[0].parsed_issue_number == 1 + assert results[0].files[0].metadata_diagnostics["volume_leaf"]["marker"] == marker + assert path.read_bytes() == before + assert await ImportLayoutAnalyzer().analyze(tmp_path) == analysis + + +@pytest.mark.parametrize("kind", ["series.json", "cvinfo", "comicinfo"]) +async def test_volume_leaf_corroborates_existing_local_metadata(tmp_path, kind): + folder = tmp_path / "Mister Miracle" / "v2017" + xml = ( + "Mister Miracle1" + "https://comicvine.gamespot.com/mister-miracle/4050-103699/" + if kind == "comicinfo" + else None + ) + _comic(folder / "001.cbz", xml) + if kind == "series.json": + (folder / kind).write_text( + json.dumps({"metadata": {"name": "Mister Miracle", "comicid": 103699}}) + ) + elif kind == "cvinfo": + (folder / kind).write_text( + "https://comicvine.gamespot.com/4050-103699/\nname: Mister Miracle" + ) + results = [series async for series in CollectionScanner().scan(tmp_path)] + assert len(results) == 1 + assert results[0].raw_series_name == "Mister Miracle" + assert results[0].raw_year == 2017 + assert results[0].comicinfo_cv_id == 103699 + assert results[0].files[0].metadata_diagnostics["volume_leaf"]["review_required"] is False + + +@pytest.mark.parametrize("name", ["unknown.cbz", "Other Series 001.cbz"]) +@pytest.mark.parametrize("mode", ["managed_copy", "in_place"]) +async def test_uncertain_volume_folder_enters_review_without_touching_sources( + db_session, tmp_path, name, mode +): + path = _comic(tmp_path / "Mister Miracle" / "v2017" / name) + before = path.read_bytes() + analysis = await ImportLayoutAnalyzer().analyze(tmp_path) + assert analysis.files_ambiguous == 1 + results = [series async for series in CollectionScanner().scan(tmp_path)] + job = ImportJob( + source_path=str(tmp_path), source_type=ImportSourceType.FILESYSTEM, file_handling_mode=mode + ) + db_session.add(job) + await db_session.flush() + await materialize_discovered_scan_results(db_session, job, results) + file = await db_session.scalar(select(ImportedFile).where(ImportedFile.import_job_id == job.id)) + assert file.status == ImportedFileStatus.NO_MATCH + assert file.diagnostics["reason"] == "volume_leaf_identity_unconfirmed" + assert path.read_bytes() == before + + +async def test_volume_leaf_does_not_undo_proven_mixed_folder_recovery(tmp_path): + folder = tmp_path / "Crossed" / "v2012" + _comic( + folder / "Absolute Batman 001.cbz", + "Absolute Batman12024https://comicvine.gamespot.com/absolute-batman/4050-160294/", + ) + results = [series async for series in CollectionScanner().scan(tmp_path)] + assert len(results) == 1 + assert results[0].comicinfo_cv_id == 160294 + assert results[0].raw_series_name == "Absolute Batman" + assert results[0].raw_year == 2024 + assert ( + not results[0].files[0].metadata_diagnostics.get("source_layout", {}).get("review_required") + ) + + +async def test_real_v2_title_is_not_replaced_with_publisher(tmp_path): + _comic( + tmp_path / "Publisher" / "V2" / "V2 001.cbz", + "V21", + ) + results = [series async for series in CollectionScanner().scan(tmp_path)] + assert results[0].raw_series_name == "V2" + analysis = await ImportLayoutAnalyzer().analyze(tmp_path) + assert analysis.clusters[0].examples[0].series == "V2" + + +async def test_volume_folder_hint_does_not_change_publication_year_or_merge_releases(tmp_path): + for marker in ("v2017", "v2020"): + _comic(tmp_path / "Mister Miracle" / marker / "Mister Miracle 001 (2021).cbz") + results = [series async for series in CollectionScanner().scan(tmp_path)] + assert {series.raw_year for series in results} == {2017, 2020} + assert all(series.files[0].parsed_year == 2021 for series in results) + assert len({series.source_folder for series in results}) == 2 + + +async def test_custom_layout_remains_authoritative(tmp_path): + _comic(tmp_path / "Mister Miracle" / "v2017" / "Mister Miracle 001.cbz") + spec = SourceLayoutSpec( + mode=ImportLayoutMode.CUSTOM, + series_path_template="{Publisher}/{Series}", + fallback_to_auto=False, + ) + analysis = await ImportLayoutAnalyzer().analyze(tmp_path, spec=spec) + assert analysis.clusters[0].examples[0].evidence == ["selected_layout_match"] + results = [series async for series in CollectionScanner(source_layout=spec).scan(tmp_path)] + assert "volume_leaf" not in results[0].files[0].metadata_diagnostics + + +@pytest.mark.parametrize("separator", ["/", "\\"]) +def test_volume_leaf_keeps_unicode_and_path_separator_evidence(separator): + path = separator.join(["\u00c9ditions", "\u00c9toile", "v2017", "\u00c9toile 13A.cbz"]) + leaf = volume_leaf_from_path(path) + assert leaf.series == "\u00c9toile" + assert leaf.publisher == "\u00c9ditions" + assert assess_volume_leaf(leaf, "\u00c9toile 13A.cbz") == (False, True, False) + + +@pytest.mark.parametrize("title", ["\u00c9toile", "\u65e5\u672c\u8a9e"]) +async def test_scanner_preserves_unicode_volume_series(tmp_path, title): + _comic(tmp_path / title / "v2017" / f"{title} 001 (2018).cbz") + results = [series async for series in CollectionScanner().scan(tmp_path)] + assert results[0].raw_series_name == title + assert results[0].raw_year == 2017 + assert results[0].files[0].parsed_year == 2018 + + +@pytest.mark.parametrize( + "evidence", ["sidecar_title", "comicinfo_title", "conflicting_ids", "year"] +) +async def test_volume_leaf_conflicting_evidence_remains_reviewable(tmp_path, evidence): + folder = tmp_path / "Mister Miracle" / "v2017" + xml = ( + "Other Series1" + if evidence == "comicinfo_title" + else None + ) + if evidence == "year": + xml = ( + "Mister Miracle1" + "2020" + ) + _comic(folder / "Mister Miracle 001.cbz", xml) + if evidence == "sidecar_title": + (folder / "series.json").write_text(json.dumps({"name": "Other Series", "comicid": 123})) + if evidence == "conflicting_ids": + (folder / "series.json").write_text(json.dumps({"name": "Mister Miracle", "comicid": 123})) + (folder / "cvinfo").write_text("https://comicvine.gamespot.com/4050-456/") + results = [series async for series in CollectionScanner().scan(tmp_path)] + assert results[0].files[0].metadata_diagnostics["volume_leaf"]["review_required"] is True + + +async def test_volume_leaf_does_not_borrow_parent_sidecar_across_releases(tmp_path): + for marker in ("v2", "v3"): + _comic(tmp_path / "Mister Miracle" / marker / "Mister Miracle 001.cbz") + (tmp_path / "Mister Miracle" / "series.json").write_text( + json.dumps({"name": "Mister Miracle", "comicid": 103699}) + ) + results = [series async for series in CollectionScanner().scan(tmp_path)] + assert len(results) == 2 + assert len({series.source_folder for series in results}) == 2 + assert all(series.comicinfo_cv_id is None for series in results) + assert { + series.files[0].metadata_diagnostics["volume_leaf"]["volume_hint"] for series in results + } == {2, 3} + + +async def test_volume_leaf_does_not_inspect_above_selected_source_root(tmp_path): + folder = tmp_path / "Mister Miracle" / "v2017" + _comic(folder / "001.cbz") + results = [series async for series in CollectionScanner().scan(folder)] + assert "volume_leaf" not in results[0].files[0].metadata_diagnostics + + +async def test_volume_leaf_mixed_bucket_preserves_good_file(db_session, tmp_path): + folder = tmp_path / "Mister Miracle" / "v2017" + _comic(folder / "Mister Miracle 001.cbz") + _comic(folder / "Other Series 002.cbz") + results = [series async for series in CollectionScanner().scan(tmp_path)] + evidence = {item.raw_series_name: item.diagnostics["volume_leaf"] for item in results} + assert evidence["Mister Miracle"]["confirmed"] is True + assert evidence["Other Series"]["confirmed"] is False + assert evidence["Other Series"]["review_required"] is True + job = ImportJob(source_path=str(tmp_path), source_type=ImportSourceType.FILESYSTEM) + db_session.add(job) + await db_session.flush() + await materialize_discovered_scan_results(db_session, job, results) + files = list( + await db_session.scalars(select(ImportedFile).where(ImportedFile.import_job_id == job.id)) + ) + assert {file.file_name: file.status for file in files} == { + "Mister Miracle 001.cbz": ImportedFileStatus.PENDING, + "Other Series 002.cbz": ImportedFileStatus.NO_MATCH, + } + + +async def test_ordinal_only_releases_require_series_confirmation(db_session, tmp_path): + for marker in ("v2", "v3"): + _comic(tmp_path / "Mister Miracle" / marker / "Mister Miracle 001.cbz") + results = [series async for series in CollectionScanner().scan(tmp_path)] + job = ImportJob(source_path=str(tmp_path), source_type=ImportSourceType.FILESYSTEM) + db_session.add(job) + await db_session.flush() + await materialize_discovered_scan_results(db_session, job, results) + series = list( + await db_session.scalars( + select(ImportedSeries).where(ImportedSeries.import_job_id == job.id) + ) + ) + assert len(series) == 2 + assert all(item.status == ImportSeriesStatus.NO_MATCH for item in series) + assert all(item.diagnostics["reason"] == "volume_leaf_release_unconfirmed" for item in series) + expected = {item.id: dict(item.diagnostics) for item in series} + await db_session.flush() + for item in series: + await db_session.refresh(item) + assert item.diagnostics == expected[item.id] + + +async def test_ordinal_volume_preserves_explicit_metadata_series_year(tmp_path): + _comic( + tmp_path / "Mister Miracle" / "v2" / "Mister Miracle 001.cbz", + "Mister Miracle1" + "20172018", + ) + results = [series async for series in CollectionScanner().scan(tmp_path)] + assert results[0].raw_year == 2017 + assert not results[0].diagnostics["volume_leaf"]["series_confirmation_required"] + + +@pytest.mark.parametrize("mode", ["managed_copy", "in_place"]) +async def test_confirmed_volume_scan_keeps_source_and_target_policy(db_session, tmp_path, mode): + path = _comic(tmp_path / "Mister Miracle" / "v2017" / "Mister Miracle 13A.cbz") + original = (path.read_bytes(), path.stat().st_mtime_ns) + results = [series async for series in CollectionScanner().scan(tmp_path)] + policy = {"series_folder_template": "{Series} ({Year})", "post_processing_method": "copy"} + job = ImportJob( + source_path=str(tmp_path), + source_type=ImportSourceType.FILESYSTEM, + file_handling_mode=mode, + ingest_policy_snapshot=dict(policy), + ) + db_session.add(job) + await db_session.flush() + await materialize_discovered_scan_results(db_session, job, results) + await db_session.refresh(job) + file = await db_session.scalar(select(ImportedFile).where(ImportedFile.import_job_id == job.id)) + assert file.status == ImportedFileStatus.PENDING + assert file.file_path == str(path) + assert file.issue_number_raw == "13A" + assert job.ingest_policy_snapshot == policy + assert (path.read_bytes(), path.stat().st_mtime_ns) == original diff --git a/tests/unit/test_mylar_source_provenance.py b/tests/unit/test_mylar_source_provenance.py new file mode 100644 index 00000000..92cf4502 --- /dev/null +++ b/tests/unit/test_mylar_source_provenance.py @@ -0,0 +1,79 @@ +"""Optional Mylar schema version is diagnostic evidence, not an admission gate.""" + +import sqlite3 + +import pytest + +from pullbox.core.mylar3_reader import Mylar3Reader +from pullbox.models.import_job import ImportJob, ImportJobStatus, ImportSourceType +from pullbox.services.import_scan_pipeline import _load_mylar3_discovered_series +from scripts.mylar3_import_fixture import create_mylar3_db + + +@pytest.mark.parametrize( + "variant,value,expected", + [ + ("value", 11, "11"), + ("value", "9999", "9999"), + ("missing", None, "unknown"), + ("column_missing", None, "unknown"), + ("value", None, "unknown"), + ("value", "", "unknown"), + ("value", "not-a-version", "unknown"), + ("value", "secret\nvalue", "unknown"), + ], +) +async def test_optional_mylar_version_is_read_only_and_capability_based( + tmp_path, variant, value, expected +): + source = tmp_path / "mylar.db" + create_mylar3_db(source) + with sqlite3.connect(source) as conn: + if variant == "value": + conn.execute("CREATE TABLE mylar_info (DatabaseVersion)") + conn.execute("INSERT INTO mylar_info VALUES (?)", (value,)) + elif variant == "column_missing": + conn.execute("CREATE TABLE mylar_info (something_else)") + before = source.read_bytes() + reader = Mylar3Reader(source) + metadata = await reader.read_import_metadata() + snapshot = await reader.read_snapshot() + assert getattr(metadata, "database_version", None) == expected + assert getattr(snapshot, "database_version", None) == expected + assert metadata.series_count == 0 + assert snapshot.series == () + assert source.read_bytes() == before + + +async def test_mylar_source_version_is_in_durable_import_diagnostics(db_session, tmp_path): + source = tmp_path / "mylar.db" + create_mylar3_db(source) + with sqlite3.connect(source) as conn: + conn.execute("CREATE TABLE mylar_info (DatabaseVersion)") + conn.execute("INSERT INTO mylar_info VALUES (11)") + job = ImportJob( + source_path=str(source), + source_type=ImportSourceType.MYLAR3, + status=ImportJobStatus.SCANNING, + mylar3_path_map_confirmed=True, + ) + db_session.add(job) + await db_session.flush() + events = [] + + async def log_event(_session, _job_id, _level, event, **details): + events.append((event, details)) + + await _load_mylar3_discovered_series( + db_session, + job, + job_id=job.id, + mylar3_reader_cls=Mylar3Reader, + auto_detect_mylar3_path_map=lambda _path: None, + log_event=log_event, + ) + provenance = [details for event, details in events if event == "mylar3_source_provenance"] + assert len(provenance) == 1 + assert provenance[0]["database_version"] == "11" + assert provenance[0]["series_count"] == 0 + assert "config_values" not in provenance[0] From 42ba085316547920a61e0a904807fe2fa4d0ec1e Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 17:50:19 -0700 Subject: [PATCH 15/20] fix(import): recover exact catalog targets and guard mixed folders Preserve exact issue designations, reject corroborated foreign titles, and recover unique local catalog title-and-issue targets with durable checkpoints. Keep ambiguous titles and saved provider identity conflicts in review. Allow filename-based mixed-folder corrections only when a unique exact local title-and-issue target exists. Duplicate-title tests now include competing issue targets because title duplication alone does not establish ambiguity. --- src/pullbox/core/release_parser.py | 26 +- .../import_alternate_release_matching.py | 36 ++ .../services/import_completed_cleanup.py | 192 +++++--- .../services/import_deferred_recovery.py | 1 + .../import_deferred_recovery_execution.py | 326 +++++++++++++- .../services/import_source_metadata.py | 37 +- src/pullbox/services/metadata_service.py | 22 +- tests/unit/test_catalog_integration.py | 13 + .../test_import_alternate_release_matching.py | 156 +++++++ tests/unit/test_import_completed_cleanup.py | 66 ++- .../unit/test_import_corroborated_identity.py | 136 ++++++ ...test_import_deferred_recovery_execution.py | 412 +++++++++++++++++- tests/unit/test_import_source_metadata.py | 10 +- tests/unit/test_release_parser.py | 24 + 14 files changed, 1368 insertions(+), 89 deletions(-) diff --git a/src/pullbox/core/release_parser.py b/src/pullbox/core/release_parser.py index 384379db..b6f4f6e6 100644 --- a/src/pullbox/core/release_parser.py +++ b/src/pullbox/core/release_parser.py @@ -229,6 +229,13 @@ _MINIMAL_SEPARATOR_ISSUE_RE = re.compile( r"^(?P.+?)[._](?P0*\d{1,3}(?:\.\d+)?(?:-?[A-Za-z]+)?)$" ) +_DOTTED_LOCAL_ISSUE_RE = re.compile( + r"(?<=\D)\.(?P[+-]?\d+(?:\.\d+)?(?:-?[A-Za-z]+)?)\.(?=\s|\()" +) +_SPECIAL_POSITIONAL_ISSUE_RE = re.compile( + r"(?<=\s)(?P-[0-9]+(?:\.[0-9]+)?(?:-?[A-Za-z]+)?|" + r"0\.[0-9]+(?:-?[A-Za-z]+)?)(?=\s|$)" +) _GENERIC_MINIMAL_SERIES_TOKENS = frozenset( {"scan", "scans", "page", "pages", "img", "image", "images", "cover", "covers", "comic"} ) @@ -391,6 +398,11 @@ def parse_release_title( scan_group = scan_group or scene_match.group("group") working = scene_match.group("title") + # Some local libraries use one dot on each side of the issue designation, + # followed by a parenthesized year. Normalize only that bounded shape; a + # general dot replacement would damage decimal issues and dotted titles. + working = _DOTTED_LOCAL_ISSUE_RE.sub(r" \g ", working) + # Step c: Normalize separators (dots → spaces for dot-separated format) is_dot_separated = _is_dot_format(working) or _is_minimal_separator_issue_format(working) if is_dot_separated: @@ -789,7 +801,15 @@ def _extract_issue_number( remaining = clean[: m.start()] + clean[m.end() :] return num, remaining.strip(), exact_text - # Priority 6: Long-running series may omit "Prog" or "#" before issue 2487. + # Priority 6: Decimal issues below one and negative issue designations are + # valid catalog identities but fall outside the ordinary positional rules. + m = _SPECIAL_POSITIONAL_ISSUE_RE.search(clean) + if m: + num, exact_text = parse_issue_number_text(m.group("issue")) + remaining = clean[: m.start()] + clean[m.end() :] + return num, remaining.strip(), exact_text + + # Priority 7: Long-running series may omit "Prog" or "#" before issue 2487. # Require a known series prefix: a title such as Marvel 1602 is not issue 1602. m = _LONG_POSITIONAL_ISSUE_RE.search(clean) if m and expected_series: @@ -811,7 +831,7 @@ def _extract_issue_number( num, exact_text = parse_issue_number_text(token) return num, prefix, exact_text - # Priority 7: Positional number — a 2-3 digit number that sits between + # Priority 8: Positional number — a 2-3 digit number that sits between # the series name and metadata (year/brackets) # Match a number preceded by space (or after series-name text) # but NOT part of an alphanumeric word like "D4VE2" or "Spider-Man 2099" @@ -837,7 +857,7 @@ def _extract_issue_number( remaining = clean[: m.start()] + clean[m.end() :] return num, remaining.strip(), exact_text - # Priority 8: Single digit number at word boundary after text + # Priority 9: Single digit number at word boundary after text # Must NOT be followed by a word (e.g. "4 Covers" is a count, not issue #4) m = re.search(r"(?<=\s)(\d(?:-?[A-Za-z]+)?)(?=\s|$)", clean) if m: diff --git a/src/pullbox/services/import_alternate_release_matching.py b/src/pullbox/services/import_alternate_release_matching.py index eddbd395..d6c4efa9 100644 --- a/src/pullbox/services/import_alternate_release_matching.py +++ b/src/pullbox/services/import_alternate_release_matching.py @@ -16,6 +16,7 @@ from pullbox.services.import_cv_search import search_with_retry from pullbox.services.import_known_cv_match import ComicVineMatchEvaluation from pullbox.services.import_match_candidates import ( + build_ambiguous_series_conflict_diagnostics, build_candidate_diagnostics, build_signal_conflict_series_diagnostics, candidate_has_exact_title_match, @@ -299,6 +300,41 @@ async def evaluate_alternate_release_candidates( ), reverse=True, ) + competing_exact_candidate = next( + ( + candidate + for candidate in candidate_diagnostics + if int(candidate.get("cv_id") or 0) != int(best_result.provider_id) + and candidate_has_exact_title_match(best_candidate) + and candidate_has_exact_title_match(candidate) + and candidate.get("normalized_title") == best_candidate.get("normalized_title") + and candidate.get("year") == best_candidate.get("year") + and float(candidate.get("score", 0.0)) >= match_threshold + and abs(float(candidate.get("score", 0.0)) - best_score) + <= _AMBIGUOUS_SERIES_MATCH_DELTA + ), + None, + ) + if competing_exact_candidate is not None: + return ComicVineMatchEvaluation( + match=None, + diagnostics=build_ambiguous_series_conflict_diagnostics( + raw_name=raw_name, + raw_year=raw_year, + match_threshold=match_threshold, + selected_candidate={ + **best_candidate, + "match_method": "alternate_release_candidate", + "alternate_series_name": alternate_series, + }, + competing_candidate={ + **competing_exact_candidate, + "match_method": "alternate_release_candidate", + "alternate_series_name": alternate_series, + }, + top_candidates=_dedupe_and_sort_top_candidates(top_candidates)[:3], + ), + ) evaluation = ComicVineMatchEvaluation( match={ "cv_id": int(best_result.provider_id), diff --git a/src/pullbox/services/import_completed_cleanup.py b/src/pullbox/services/import_completed_cleanup.py index b8e37617..fd8f5479 100644 --- a/src/pullbox/services/import_completed_cleanup.py +++ b/src/pullbox/services/import_completed_cleanup.py @@ -3,6 +3,7 @@ from __future__ import annotations import enum +import re from collections.abc import Mapping from dataclasses import dataclass from datetime import UTC, datetime @@ -19,6 +20,8 @@ from pullbox.core.exceptions import NotFoundError, ValidationError from pullbox.core.issue_numbers import parse_issue_number_text from pullbox.core.name_matcher import NameMatcher +from pullbox.core.release_parser import parse_release_title +from pullbox.core.story_arc_ordering import extract_story_arc_order_prefix from pullbox.models.audit_log import AuditEventType from pullbox.models.import_job import ( ImportedFile, @@ -29,7 +32,7 @@ ImportJobStatus, ImportSeriesStatus, ) -from pullbox.models.issue import Issue, IssueStatus +from pullbox.models.issue import Issue, IssueStatus, IssueType from pullbox.models.library import LibraryFile, LibraryFileStorageMode from pullbox.models.series import Series from pullbox.services.audit_service import AuditService @@ -374,38 +377,72 @@ def _mixed_folder_source_identity( series_signal = str(signals.get("series_name") or "") issue_signal = str(signals.get("issue_number") or "") trusted_signals = {"comicinfo", "sidecar"} - if series_signal not in trusted_signals or issue_signal not in trusted_signals: - return None - source_metadata = _mapping(diagnostics.get("source_metadata")) comicinfo = _mapping(source_metadata.get("comicinfo")) - if series_signal == "comicinfo": - source_series_name = str(comicinfo.get("series") or "").strip() - else: - source_series_name = str(imported_file.parsed_series or "").strip() - if issue_signal == "comicinfo": - source_issue_number = comicinfo.get("number") - else: - source_issue_number = imported_file.issue_number_raw or imported_file.parsed_issue_number - if not source_series_name or not isinstance(source_issue_number, str | float | int): + if series_signal in trusted_signals and issue_signal in trusted_signals: + if series_signal == "comicinfo": + source_series_name = str(comicinfo.get("series") or "").strip() + else: + source_series_name = str(imported_file.parsed_series or "").strip() + if issue_signal == "comicinfo": + source_issue_number = comicinfo.get("number") + else: + source_issue_number = ( + imported_file.issue_number_raw or imported_file.parsed_issue_number + ) + if not source_series_name or not isinstance(source_issue_number, str | float | int): + return None + + trusted_series_cv_id = ( + _safe_int(diagnostics.get("comicvine_series_id")) + if str(signals.get("comicvine_series_id") or "") in trusted_signals + else None + ) + trusted_issue_cv_id = ( + imported_file.comicvine_issue_id + if str(signals.get("comicvine_issue_id") or "") in trusted_signals + else None + ) + return ( + source_series_name, + source_issue_number, + trusted_series_cv_id, + trusted_issue_cv_id, + series_signal, + ) + + # Filename evidence may propose a review correction only when no trusted + # issue identity would be overridden. The later local lookup still requires + # a unique exact title-and-issue target before this becomes actionable. + if ( + imported_file.comicvine_issue_id is not None + and str(signals.get("comicvine_issue_id") or "") in trusted_signals + ): + return None + identity_conflicts = source_metadata.get("identity_conflicts") + if isinstance(identity_conflicts, list) and identity_conflicts: return None - trusted_series_cv_id = ( - _safe_int(diagnostics.get("comicvine_series_id")) - if str(signals.get("comicvine_series_id") or "") in trusted_signals - else None - ) - trusted_issue_cv_id = ( - imported_file.comicvine_issue_id - if str(signals.get("comicvine_issue_id") or "") in trusted_signals - else None - ) + file_name = imported_file.file_name + order_prefix = extract_story_arc_order_prefix(file_name) + if order_prefix is not None: + file_name = order_prefix.residual_file_name + file_name = re.sub(r"\s*\(converted\)\s*", " ", file_name, flags=re.IGNORECASE) + parsed = parse_release_title(file_name) + if ( + parsed is None + or parsed.is_pack + or parsed.issue_type is not IssueType.ISSUE + or not parsed.series_name + or parsed.issue_number is None + ): + return None return ( - source_series_name, - source_issue_number, - trusted_series_cv_id, - trusted_issue_cv_id, - series_signal, + parsed.series_name, + parsed.issue_number_text or parsed.issue_number, + None, + None, + "filename_parse", ) @@ -423,6 +460,26 @@ async def _load_mixed_folder_resolutions( ), else_=ImportedFile.parsed_series, ) + trusted_identity_filter = and_( + series_signal.in_(("comicinfo", "sidecar")), + issue_signal.in_(("comicinfo", "sidecar")), + func.lower(func.trim(source_title_expression)) + != func.lower( + func.trim( + func.coalesce( + func.nullif(ImportedSeries.cv_title, ""), ImportedSeries.raw_series_name + ) + ) + ), + ) + filename_review_filter = and_( + ImportedSeries.files_no_match > 0, + or_( + issue_signal == "release_title", + ImportedFile.diagnostics["conflict_type"].as_string() + == "corroborated_file_series_mismatch", + ), + ) source_rows = ( await session.execute( select(ImportedFile, ImportedSeries, LibraryFile) @@ -431,16 +488,7 @@ async def _load_mixed_folder_resolutions( .where( ImportedFile.import_job_id == job_id, ImportedFile.status.in_((ImportedFileStatus.NO_MATCH, ImportedFileStatus.IMPORTED)), - series_signal.in_(("comicinfo", "sidecar")), - issue_signal.in_(("comicinfo", "sidecar")), - func.lower(func.trim(source_title_expression)) - != func.lower( - func.trim( - func.coalesce( - func.nullif(ImportedSeries.cv_title, ""), ImportedSeries.raw_series_name - ) - ) - ), + or_(trusted_identity_filter, filename_review_filter), ) .order_by(ImportedFile.id) ) @@ -501,20 +549,26 @@ async def _load_mixed_folder_resolutions( if not source_candidates: return () - local_series = list((await session.scalars(select(Series))).all()) - series_by_cv_id = { - int(series.comicvine_id): series - for series in local_series - if series.comicvine_id is not None and int(series.comicvine_id) in trusted_series_cv_ids - } + series_by_cv_id: dict[int, Series] = {} series_by_title: dict[str, list[Series]] = {} - for series in local_series: - normalized = NameMatcher.normalize(series.title) - if normalized in normalized_titles: - series_by_title.setdefault(normalized, []).append(series) + series_stream = await session.stream_scalars( + select(Series).order_by(Series.id).execution_options(yield_per=1_000) + ) + try: + async for series in series_stream: + if ( + series.comicvine_id is not None + and int(series.comicvine_id) in trusted_series_cv_ids + ): + series_by_cv_id[int(series.comicvine_id)] = series + normalized = NameMatcher.normalize(series.title) + if normalized in normalized_titles: + series_by_title.setdefault(normalized, []).append(series) + finally: + await series_stream.close() candidate_targets: list[ - tuple[ImportedFile, ImportedSeries, str, str, str, Series, int | None] + tuple[ImportedFile, ImportedSeries, str, str, str, tuple[Series, ...], int | None] ] = [] target_series_ids: set[int] = set() for ( @@ -527,12 +581,17 @@ async def _load_mixed_folder_resolutions( evidence_source, ) in source_candidates: target_series = series_by_cv_id.get(series_cv_id) if series_cv_id is not None else None - if target_series is None: - title_matches = series_by_title.get(NameMatcher.normalize(source_title), []) - if len(title_matches) != 1: - continue - target_series = title_matches[0] - if imported_series.series_id == target_series.id: + target_series_options = ( + (target_series,) + if target_series is not None + else tuple(series_by_title.get(NameMatcher.normalize(source_title), [])) + ) + if not target_series_options: + continue + target_series_options = tuple( + series for series in target_series_options if imported_series.series_id != series.id + ) + if not target_series_options: continue candidate_targets.append( ( @@ -541,11 +600,11 @@ async def _load_mixed_folder_resolutions( source_title, exact_number, evidence_source, - target_series, + target_series_options, issue_cv_id, ) ) - target_series_ids.add(int(target_series.id)) + target_series_ids.update(int(series.id) for series in target_series_options) if not candidate_targets: return () @@ -557,6 +616,11 @@ async def _load_mixed_folder_resolutions( issues_by_cv_id = { int(issue.comicvine_id): issue for issue in target_issues if issue.comicvine_id is not None } + target_series_by_id = { + int(series.id): series + for _file, _parent, _title, _number, _source, options, _issue_id in candidate_targets + for series in options + } issues_by_number: dict[tuple[int, str], list[Issue]] = {} for issue in target_issues: issues_by_number.setdefault( @@ -570,19 +634,27 @@ async def _load_mixed_folder_resolutions( source_title, exact_number, evidence_source, - target_series, + target_series_options, issue_cv_id, ) in candidate_targets: + target_series_option_ids = {int(series.id) for series in target_series_options} target_issue: Issue | None = ( issues_by_cv_id.get(issue_cv_id) if issue_cv_id is not None else None ) - if target_issue is not None and target_issue.series_id != target_series.id: + if target_issue is not None and target_issue.series_id not in target_series_option_ids: target_issue = None if target_issue is None: - number_matches = issues_by_number.get((int(target_series.id), exact_number), []) + number_matches = [ + issue + for series_id in target_series_option_ids + for issue in issues_by_number.get((series_id, exact_number), []) + ] if len(number_matches) != 1: continue target_issue = number_matches[0] + target_series = target_series_by_id.get(int(target_issue.series_id)) + if target_series is None: + continue resolved_targets.append( ( imported_file, diff --git a/src/pullbox/services/import_deferred_recovery.py b/src/pullbox/services/import_deferred_recovery.py index 3c3fdb77..de086bfb 100644 --- a/src/pullbox/services/import_deferred_recovery.py +++ b/src/pullbox/services/import_deferred_recovery.py @@ -112,6 +112,7 @@ def protected_file(file: ImportedFile, item: ImportedSeries) -> bool: "safety_review", ) ) + or (diagnostics.get("kind") == "metadata_conflict" and bool(provider_ids(file))) or _unresolved_identity_conflicts(file) or ( file.conflict_group_id is not None diff --git a/src/pullbox/services/import_deferred_recovery_execution.py b/src/pullbox/services/import_deferred_recovery_execution.py index f5137dbc..29175903 100644 --- a/src/pullbox/services/import_deferred_recovery_execution.py +++ b/src/pullbox/services/import_deferred_recovery_execution.py @@ -10,6 +10,7 @@ from sqlalchemy import select from pullbox.core.exceptions import JobPausedError, NotFoundError, ProviderError, ValidationError +from pullbox.core.issue_numbers import normalize_issue_number_text from pullbox.core.name_matcher import NameMatcher from pullbox.models.import_job import ( ImportControlRequest, @@ -23,6 +24,7 @@ ) from pullbox.models.issue import Issue from pullbox.schemas.import_job import ImportProgressEvent +from pullbox.services.catalog.contract import CatalogError from pullbox.services.import_counters import recompute_file_counters, recompute_series_counters from pullbox.services.import_deferred_recovery import ( apply_deferred_recovery, @@ -67,6 +69,150 @@ def _catalog_summary_payload(summary: IssueSummary) -> dict[str, Any]: return payload +def _filename_catalog_identity( + file: ImportedFile, + item: ImportedSeries, +) -> dict[str, str | int] | None: + """Return bounded filename identity eligible for local-catalog recovery.""" + if protected_file(file, item) or provider_ids(file): + return None + diagnostics = dict(file.diagnostics or {}) + source = source_metadata_for_import_file(item, file).diagnostics + if source.get("identity_conflicts"): + return None + parsed = source.get("filename_parse") + parsed = parsed if isinstance(parsed, dict) else {} + source_title = str(parsed.get("series_name") or "").strip() + raw_number = parsed.get("issue_number_text") or parsed.get("issue_number") + if not source_title or raw_number is None: + return None + try: + issue_number = normalize_issue_number_text(str(raw_number)) + except ValueError: + return None + issue_type = str(parsed.get("issue_type") or diagnostics.get("source_issue_type") or "issue") + if issue_type in {"annual", "special"}: + label = issue_type.title() + if not NameMatcher.normalize(source_title).endswith(f" {NameMatcher.normalize(label)}"): + source_title = f"{source_title} {label}" + normalized_title = NameMatcher.normalize(source_title) + parent_title = NameMatcher.normalize(item.cv_title or item.raw_series_name) + if not normalized_title or normalized_title == parent_title: + return None + title_match = NameMatcher().match(source_title, item.cv_title or item.raw_series_name) + corroborated = diagnostics.get("conflict_type") == "corroborated_file_series_mismatch" + if title_match.is_match and not corroborated and issue_type not in {"annual", "special"}: + return None + year = parsed.get("year") or file.parsed_year + return { + "key": normalized_title, + "query": source_title, + "issue_number": issue_number, + "year": int(year) if isinstance(year, int | float) else 0, + "issue_type": issue_type, + } + + +async def _catalog_title_candidates( + session: AsyncSession, + job: ImportJob, +) -> dict[str, dict[str, Any]]: + """Group deterministic filename searches so each catalog title is queried once.""" + candidates: dict[str, dict[str, Any]] = {} + for file, item in await load_deferred_rows(session, job.id): + identity = _filename_catalog_identity(file, item) + if identity is None: + continue + key = str(identity["key"]) + candidate = candidates.setdefault( + key, + { + "query": str(identity["query"]), + "issue_numbers": set(), + "years": set(), + "years_by_issue": defaultdict(set), + }, + ) + issue_number = str(identity["issue_number"]) + candidate["issue_numbers"].add(issue_number) + if int(identity["year"]): + year = int(identity["year"]) + candidate["years"].add(year) + candidate["years_by_issue"][issue_number].add(year) + return { + key: { + "query": value["query"], + "issue_numbers": sorted(value["issue_numbers"]), + "years": sorted(value["years"]), + "years_by_issue": { + number: sorted(years) for number, years in sorted(value["years_by_issue"].items()) + }, + } + for key, value in candidates.items() + } + + +def _summary_year(summary: IssueSummary) -> int | None: + release_date = summary.release_date + if release_date is None: + return None + if isinstance(release_date, str): + try: + return int(release_date[:4]) + except ValueError: + return None + return release_date.year + + +async def _search_exact_title_catalog( + metadata_service: MetadataService, + candidate: dict[str, Any], +) -> dict[str, list[dict[str, Any]]]: + """Return exact local-catalog issue targets for one normalized title.""" + query = str(candidate["query"]) + normalized_query = NameMatcher.normalize(query) + results = await metadata_service.search_catalog_series(query, limit=1_000) + exact_results = [ + result for result in results if NameMatcher.normalize(result.title) == normalized_query + ] + if not exact_results or len(exact_results) > 24: + return {} + needed_numbers = {str(value) for value in candidate.get("issue_numbers", [])} + fallback_years = {int(value) for value in candidate.get("years", [])} + raw_years_by_issue = candidate.get("years_by_issue") + years_by_issue = raw_years_by_issue if isinstance(raw_years_by_issue, dict) else {} + matches: dict[str, list[dict[str, Any]]] = defaultdict(list) + for result in exact_results: + summaries = await metadata_service.get_catalog_issue_summaries_for_series( + int(result.provider_id) + ) + for summary in summaries: + exact_number = normalize_issue_number_text( + summary.issue_number_text or summary.issue_number + ) + if exact_number not in needed_numbers: + continue + summary_year = _summary_year(summary) + expected_years = { + int(value) for value in years_by_issue.get(exact_number, fallback_years) + } + if ( + summary_year is not None + and expected_years + and all(abs(summary_year - year) > 1 for year in expected_years) + ): + continue + matches[exact_number].append( + { + "cv_id": int(result.provider_id), + "title": result.title, + "year": result.year_start, + "summary": _catalog_summary_payload(summary), + } + ) + return dict(matches) + + async def cancel_deferred_preparation(session: AsyncSession, job: ImportJob) -> bool: """Stop this recovery pass, retaining the original and any completed imports.""" state = recovery_state(job) @@ -261,6 +407,124 @@ async def _prepare_catalog_targets(session: AsyncSession, job: ImportJob) -> int return ready +async def _prepare_title_catalog_targets(session: AsyncSession, job: ImportJob) -> int: + """Stage only unique exact title-and-issue catalog identities.""" + state = recovery_state(job) + title_matches = state.get("title_matches", {}) + rows = await load_deferred_rows(session, job.id) + eligible: list[tuple[ImportedFile, ImportedSeries, dict[str, Any]]] = [] + for file, item in rows: + identity = _filename_catalog_identity(file, item) + if identity is None: + continue + options_by_number = title_matches.get(str(identity["key"]), {}) + options = options_by_number.get(str(identity["issue_number"]), []) + if len(options) != 1: + continue + eligible.append((file, item, options[0])) + + counts = Counter(str(target["summary"]["provider_id"]) for _, _, target in eligible) + issue_cv_ids = { + issue_cv_id + for _file, _item, target in eligible + if (issue_cv_id := positive_id(target["summary"].get("provider_id"))) is not None + } + existing_issue_ids: set[int] = set() + from itertools import batched + + for ids in batched(sorted(issue_cv_ids), 300): + existing_issue_ids.update( + int(value) + for value in await session.scalars( + select(Issue.comicvine_id).where(Issue.comicvine_id.in_(ids)) + ) + if value is not None + ) + + targets: dict[int, ImportedSeries] = {} + affected: set[int] = set() + ready = 0 + for file, original, target in eligible: + issue_cv_id = positive_id(target["summary"].get("provider_id")) + target_cv_id = positive_id(target.get("cv_id")) + if ( + issue_cv_id is None + or target_cv_id is None + or counts[str(issue_cv_id)] != 1 + or issue_cv_id in existing_issue_ids + ): + continue + target_item = targets.get(target_cv_id) + if target_item is None: + target_item = ImportedSeries( + import_job_id=job.id, + raw_series_name=str(target["title"]), + raw_year=target.get("year"), + cv_id=target_cv_id, + cv_title=str(target["title"]), + cv_year=target.get("year"), + cv_match_score=1.0, + cv_match_method="deferred_catalog_title_identity", + status=ImportSeriesStatus.CONFIRMED, + selected_for_import=True, + has_files=True, + diagnostics={ + "kind": "deferred_recovery", + "reason": "unique_catalog_title_and_issue", + "source_preserved": True, + }, + ) + session.add(target_item) + await session.flush() + targets[target_cv_id] = target_item + affected.update((original.id, target_item.id)) + file.import_series_id = target_item.id + file.status = ImportedFileStatus.CONFIRMED + file.matched_issue_cv_id = issue_cv_id + file.include_in_import = True + file.match_confidence = "high" + file.match_method = "completed_import_catalog_title_target" + file.error_message = None + file.conflict_group_id = None + apply_proven_identity( + file, + issue_cv_id=issue_cv_id, + series_cv_id=target_cv_id, + summary=target["summary"], + ) + file.diagnostics = { + **file.diagnostics, + "target_issue_summary": target["summary"], + "deferred_recovery": { + "action": "catalog_title_identity", + "source_import_series_id": original.id, + "target_series_cv_id": target_cv_id, + "source_preserved": True, + "resolved_at": datetime.now(UTC).isoformat(), + }, + } + ready += 1 + if affected: + from pullbox.services.import_story_arc_resolution import ( + refresh_story_arc_entries_for_import_files, + ) + + await refresh_recovered_groups(session, job, affected) + await refresh_story_arc_entries_for_import_files( + session, + import_job_id=job.id, + import_file_ids=[ + file.id for file, _, _ in eligible if file.status is ImportedFileStatus.CONFIRMED + ], + ) + state["series_ids"] = sorted( + set(state.get("series_ids", [])) | {item.id for item in targets.values()} + ) + save_recovery_state(job, state) + await session.flush() + return ready + + async def prepare_deferred_recovery( session: AsyncSession, job_id: int, @@ -333,10 +597,18 @@ async def report( candidates=await _catalog_candidates(session, job), completed=[], matches={}, + title_candidates=await _catalog_title_candidates(session, job), + title_completed=[], + title_matches={}, ) save_recovery_state(job, state) await session.commit() + title_state_needs_initialization = "title_candidates" not in state + pending_title_candidates = ( + await _catalog_title_candidates(session, job) if title_state_needs_initialization else {} + ) + candidates = state["candidates"] completed = set(state.get("completed", [])) for cv_id_text, needed_ids in sorted(candidates.items()): @@ -356,7 +628,7 @@ async def report( except NotFoundError: series = None summaries = [] - except ProviderError as exc: + except (ProviderError, CatalogError) as exc: job.error_message = ( "Metadata is temporarily unavailable. Resume recovery when it is available." ) @@ -387,10 +659,58 @@ async def report( check_control=False, ) + if title_state_needs_initialization: + state = recovery_state(job) + state["title_candidates"] = pending_title_candidates + state["title_completed"] = [] + state["title_matches"] = {} + save_recovery_state(job, state) + await session.commit() + + title_candidates = state.get("title_candidates", {}) + title_completed = set(state.get("title_completed", [])) + for key, candidate in sorted(title_candidates.items()): + if key in title_completed: + continue + await report( + len(title_completed), + len(title_candidates), + f"Checking exact title catalog {len(title_completed) + 1} " + f"of {len(title_candidates)}...", + durable=False, + ) + try: + state["title_matches"][key] = await _search_exact_title_catalog( + metadata_service, + candidate, + ) + except (ProviderError, CatalogError) as exc: + job.error_message = ( + "The local metadata catalog is temporarily unavailable. " + "Resume recovery after repairing or updating it." + ) + await report(len(title_completed), len(title_candidates), job.error_message) + raise JobPausedError(job.error_message) from exc + await raise_if_job_cancelled(session, job_id) + title_completed.add(key) + state["title_completed"] = sorted(title_completed) + save_recovery_state(job, state) + await report( + len(title_completed), + len(title_candidates), + f"Checked exact title catalog {len(title_completed)} of {len(title_candidates)}.", + check_control=False, + ) + await report(len(completed), max(len(candidates), 1), "Preparing verified files for import...") catalog_count = await _prepare_catalog_targets(session, job) + title_catalog_count = await _prepare_title_catalog_targets(session, job) state = recovery_state(job) - state.update(state="prepared", catalog_files_prepared=catalog_count) + state.update( + state="prepared", + catalog_files_prepared=catalog_count, + title_catalog_files_prepared=title_catalog_count, + ) job.error_message = None if not state.get("series_ids"): state["state"] = "completed" @@ -413,6 +733,8 @@ async def report( "local_counts": state.get("local_counts", {}), "catalogs_checked": len(completed), "catalog_files_prepared": catalog_count, + "title_catalogs_checked": len(title_completed), + "title_catalog_files_prepared": title_catalog_count, "source_preserved": True, }, ) diff --git a/src/pullbox/services/import_source_metadata.py b/src/pullbox/services/import_source_metadata.py index 942de31e..fa19b8f9 100644 --- a/src/pullbox/services/import_source_metadata.py +++ b/src/pullbox/services/import_source_metadata.py @@ -19,6 +19,7 @@ SourceMetadataExtractor, volume_subtitle_hint_from_filename, ) +from pullbox.core.story_arc_ordering import extract_story_arc_order_prefix from pullbox.core.type_semantics import issue_type_family from pullbox.models.import_job import ImportedFile, ImportedFileStatus, ImportedSeries from pullbox.models.issue import IssueType @@ -287,6 +288,15 @@ def source_metadata_for_import_file( if not isinstance(source_diagnostics, dict): source_diagnostics = {} source_diagnostics = dict(source_diagnostics) + if "filename_parse" not in source_diagnostics and parsed_filename is not None: + source_diagnostics["filename_parse"] = { + "series_name": parsed_filename.series_name, + "issue_number": parsed_filename.issue_number, + "issue_number_text": parsed_filename.issue_number_text, + "year": parsed_filename.year, + "volume": parsed_filename.volume, + "issue_type": parsed_filename.issue_type.value, + } volume_hint = _volume_subtitle_hint(imp_file.file_name, diagnostics) series_name = imp_file.parsed_series or imp_series.raw_series_name issue_number = imp_file.parsed_issue_number @@ -1011,7 +1021,7 @@ async def source_metadata_for_matching_series( def corroborated_import_title_conflict( metadata: SourceMetadata, target_series_title: str ) -> dict[str, Any] | None: - """Protect extended comic titles from a shorter parent-ID or number match.""" + """Protect corroborated foreign titles from a parent-ID or number match.""" if metadata.issue_type in _TYPE_QUALIFIED_SERIES_HINT_TYPES: return None filename = metadata.diagnostics.get("filename_parse") @@ -1020,12 +1030,17 @@ def corroborated_import_title_conflict( source_title = str(filename.get("series_name") or "").strip() exact_types = {"exact", "alternate", "token_set"} target_name = NameMatcher.normalize(target_series_title) - # Limit this guard to corroborated title extensions. Alternate catalog names - # and type-qualified annual/special buckets still use the established matcher. + title_match = _matcher.match(source_title, target_series_title) + normalized_source_title = NameMatcher.normalize(source_title) + is_corroborated_extension = normalized_source_title.startswith(f"{target_name} ") + is_clearly_unrelated = not title_match.is_match + # Preserve established alternate-title matching while rejecting either a + # corroborated title extension or a title the shared matcher cannot relate + # to the inherited parent at all. if ( not target_name - or not NameMatcher.normalize(source_title).startswith(f"{target_name} ") - or _matcher.match(source_title, target_series_title).match_type in exact_types + or title_match.match_type in exact_types + or not (is_corroborated_extension or is_clearly_unrelated) ): return None corroboration: dict[str, str] = {} @@ -1040,6 +1055,14 @@ def corroborated_import_title_conflict( for signal, title in corroboration.items() if title and _matcher.match(source_title, title).match_type in exact_types ] + # A wholly unrelated title is trustworthy by itself only when the filename + # has an explicit reading-order prefix. Free-form names and variant suffixes + # are too noisy to override the parent folder without ComicInfo or archive + # page corroboration. This guard blocks a coincidental parent issue-number + # match but never reassigns the file automatically. + reading_order_prefix = extract_story_arc_order_prefix(metadata.original_title or "") + if is_clearly_unrelated and reading_order_prefix is not None and not agreeing: + agreeing.append("filename_parse") if not agreeing: return None return { @@ -1050,8 +1073,8 @@ def corroborated_import_title_conflict( "target_series": target_series_title, "corroborating_signals": agreeing, "rejection_reason": ( - f"The filename and local file metadata identify {source_title}, " - f"not {target_series_title}. Choose the correct series and issue for this file." + f"Local file evidence identifies {source_title}, not {target_series_title}. " + "Choose the correct series and issue for this file." ), } diff --git a/src/pullbox/services/metadata_service.py b/src/pullbox/services/metadata_service.py index 5fa7d15b..3ce9db72 100644 --- a/src/pullbox/services/metadata_service.py +++ b/src/pullbox/services/metadata_service.py @@ -42,7 +42,7 @@ if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession - from pullbox.providers.base import IssueMetadata, IssueSummary + from pullbox.providers.base import IssueMetadata, IssueSummary, SeriesSearchResult from pullbox.providers.metadata.comicvine import ComicVineProvider from pullbox.services.catalog.reader import CatalogReader @@ -308,6 +308,26 @@ async def get_series_metadata_batch( except ComicVineError as exc: raise _provider_error_from_comicvine(exc) from exc + async def search_catalog_series( + self, + query: str, + *, + limit: int = 1_000, + ) -> list[SeriesSearchResult]: + """Search only the installed local catalog for recovery candidates.""" + if self._catalog is None or not self._catalog.available: + return [] + return await self._catalog.search(query, limit=min(max(1, limit), 1_000)) + + async def get_catalog_issue_summaries_for_series( + self, + comicvine_id: int, + ) -> list[IssueSummary]: + """Read issue identities only from the installed local catalog.""" + if self._catalog is None or not self._catalog.available: + return [] + return await self._catalog.issues(comicvine_id) + async def get_cached_series_metadata( self, comicvine_id: int, diff --git a/tests/unit/test_catalog_integration.py b/tests/unit/test_catalog_integration.py index 7798ef6c..f65e8bb3 100644 --- a/tests/unit/test_catalog_integration.py +++ b/tests/unit/test_catalog_integration.py @@ -28,6 +28,19 @@ async def test_metadata_hydration_uses_catalog_without_provider_calls(tmp_path): live.get_issues_for_series.assert_not_awaited() +async def test_recovery_title_search_is_local_catalog_only(tmp_path): + reader = installed_reader(tmp_path) + live = AsyncMock() + service = MetadataService(live, tmp_path, catalog=reader) + + results = await service.search_catalog_series("Batman", limit=5_000) + issues = await service.get_catalog_issue_summaries_for_series(10) + + assert [result.provider_id for result in results] == ["10"] + assert [issue.provider_id for issue in issues] == ["100"] + assert live.mock_calls == [] + + @pytest.mark.parametrize("ids", [[999, 10, 10], [10, 999]]) async def test_catalog_profile_batch_preserves_matches_when_a_series_is_missing(tmp_path, ids): live = AsyncMock() diff --git a/tests/unit/test_import_alternate_release_matching.py b/tests/unit/test_import_alternate_release_matching.py index e28faf54..52ce9779 100644 --- a/tests/unit/test_import_alternate_release_matching.py +++ b/tests/unit/test_import_alternate_release_matching.py @@ -95,6 +95,162 @@ def _search_side_effect(query: str, year: int | None = None, **_: object): ) +@pytest.mark.asyncio +async def test_duplicate_exact_catalog_titles_require_review() -> None: + """Two equally exact local-catalog volumes require review instead of a guessed winner.""" + provider = AsyncMock() + provider.is_local_catalog = True + provider.search_series_globally.return_value = ( + [ + _make_search_result( + provider_id="150001", + title="X-Men Annual", + year_start=2023, + publisher="Marvel", + ), + _make_search_result( + provider_id="150002", + title="X-Men Annual", + year_start=2023, + publisher="Marvel", + ), + ], + 2, + ) + source_metadata = SourceMetadata( + original_title="X-Men Annual 001 (2023).cbz", + series_name="X-Men", + issue_number=1.0, + year=2023, + issue_type=IssueType.ANNUAL, + diagnostics={ + "alternate_release_candidates": [ + { + "series_name": "X-Men Annual", + "year": 2023, + "file_name": "X-Men Annual 001 (2023).cbz", + "signal": MetadataSignal.RELEASE_TITLE.value, + "issue_type": IssueType.ANNUAL.value, + "issue_type_qualified": True, + } + ] + }, + ) + + evaluation = await evaluate_alternate_release_candidates( + provider=provider, + source_metadata=source_metadata, + raw_name="X-Men", + raw_year=2023, + semantic_engine=SemanticMatchEngine(policy=ImportPolicy()), + match_threshold=0.70, + existing_top_candidates=[], + ) + + assert evaluation is not None + assert evaluation.match is None + assert evaluation.diagnostics["kind"] == "series_conflict" + assert evaluation.diagnostics["reason"] == "ambiguous_candidates" + assert { + evaluation.diagnostics["selected_candidate"]["cv_id"], + evaluation.diagnostics["competing_candidate"]["cv_id"], + } == {150001, 150002} + provider.search_series_globally.assert_awaited_once_with( + "X-Men Annual", + max_results=1000, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("alternate_title", "catalog_title", "provider_id", "year", "issue_type"), + [ + ("Uncanny X-Men Special", "Uncanny X-Men Special", "74730", 2014, IssueType.SPECIAL), + ( + "New Avengers - Ultron Forever", + "New Avengers: Ultron Forever", + "81602", + 2015, + IssueType.ISSUE, + ), + ("New Avengers Finale", "New Avengers Finale", "33091", 2010, IssueType.ISSUE), + ( + "Ultimate Fantastic Four-Ultimate X-Men Annual", + "Ultimate Fantastic Four/Ultimate X-Men Annual", + "23137", + 2008, + IssueType.ANNUAL, + ), + ], +) +async def test_evaluate_alternate_release_candidates_recalls_unique_exact_catalog_series( + alternate_title: str, + catalog_title: str, + provider_id: str, + year: int, + issue_type: IssueType, +) -> None: + """The known omitted exact volumes remain reachable through bounded catalog search.""" + provider = AsyncMock() + provider.is_local_catalog = True + provider.search_series_globally.return_value = ( + [ + _make_search_result( + provider_id=provider_id, + title=catalog_title, + year_start=year, + publisher="Marvel", + ) + ], + 1, + ) + type_qualified = issue_type in {IssueType.ANNUAL, IssueType.SPECIAL} + source_metadata = SourceMetadata( + original_title=f"{alternate_title} 001 ({year}).cbz", + series_name="Parent series", + issue_number=1.0, + year=year, + issue_type=issue_type, + diagnostics={ + "alternate_release_candidates": [ + { + "series_name": alternate_title, + "year": year, + "file_name": f"{alternate_title} 001 ({year}).cbz", + "signal": MetadataSignal.RELEASE_TITLE.value, + **( + { + "issue_type": issue_type.value, + "issue_type_qualified": True, + } + if type_qualified + else {} + ), + } + ] + }, + ) + + evaluation = await evaluate_alternate_release_candidates( + provider=provider, + source_metadata=source_metadata, + raw_name="Parent series", + raw_year=year, + semantic_engine=SemanticMatchEngine(policy=ImportPolicy()), + match_threshold=0.70, + existing_top_candidates=[], + ) + + assert evaluation is not None + assert evaluation.match is not None + assert evaluation.match["cv_id"] == int(provider_id) + assert evaluation.match["cv_match_method"] == "alternate_release_candidate" + provider.search_series_globally.assert_awaited_once_with( + alternate_title, + max_results=1000, + ) + + @pytest.mark.asyncio async def test_evaluate_alternate_signal_conflict_returns_review_diagnostics() -> None: """The extracted helper keeps ComicInfo-vs-release-title conflict behavior.""" diff --git a/tests/unit/test_import_completed_cleanup.py b/tests/unit/test_import_completed_cleanup.py index a742440b..957f85be 100644 --- a/tests/unit/test_import_completed_cleanup.py +++ b/tests/unit/test_import_completed_cleanup.py @@ -1086,6 +1086,15 @@ async def test_mixed_folder_cleanup_leaves_ambiguous_series_and_files_untouched( monitored=True, ) db_session.add(duplicate_title) + await db_session.flush() + db_session.add( + Issue( + series_id=duplicate_title.id, + issue_number=1002, + issue_number_text="1002", + comicvine_id=8001002, + ) + ) await db_session.commit() with pytest.raises(ValidationError, match="No files are eligible"): @@ -1100,26 +1109,57 @@ async def test_mixed_folder_cleanup_leaves_ambiguous_series_and_files_untouched( @pytest.mark.asyncio -async def test_mixed_folder_cleanup_does_not_trust_filename_only_identity( +async def test_mixed_folder_cleanup_uses_unique_filename_title_and_issue_identity( db_session: AsyncSession, ) -> None: ( job, - _source_series, - _target_import_series, + source_series, + target_import_series, mixed_file, - _issue, - ) = await _seed_mixed_folder_candidate(db_session, source_signal="release_title") - - with pytest.raises(ValidationError, match="No files are eligible"): - await preview_completed_import_cleanup( - db_session, - job.id, - CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES, - actor_id=42, + issue, + ) = await _seed_mixed_folder_candidate( + db_session, + target_title="Thunderbolts", + target_year=2006, + source_signal="release_title", + ) + source_series.files_no_match = 1 + mixed_file.file_path = "/comics/Fritzi Ritz (1953)/042 - Thunderbolts 105 (converted).cbz" + mixed_file.file_name = "042 - Thunderbolts 105 (converted).cbz" + mixed_file.parsed_issue_number = 105 + mixed_file.issue_number_raw = "105" + issue.issue_number = 105 + issue.issue_number_text = "105" + db_session.add( + Series( + title="Thunderbolts", + sort_title="thunderbolts", + year_start=2016, + monitored=True, ) + ) + await db_session.commit() + + preview = await preview_completed_import_cleanup( + db_session, + job.id, + CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES, + actor_id=42, + ) + result = await apply_completed_import_cleanup( + db_session, + job.id, + CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES, + actor_id=42, + preview_token=preview.preview_token, + ) + await db_session.refresh(mixed_file) - assert mixed_file.status is ImportedFileStatus.NO_MATCH + assert result.requires_import_retry is True + assert mixed_file.import_series_id == target_import_series.id + assert mixed_file.matched_issue_id == issue.id + assert mixed_file.status is ImportedFileStatus.CONFIRMED @pytest.mark.asyncio diff --git a/tests/unit/test_import_corroborated_identity.py b/tests/unit/test_import_corroborated_identity.py index d63f7dd1..60653233 100644 --- a/tests/unit/test_import_corroborated_identity.py +++ b/tests/unit/test_import_corroborated_identity.py @@ -12,6 +12,7 @@ from pullbox.services.import_source_metadata import ( build_import_metadata_conflict, corroborated_import_title_conflict, + source_metadata_for_import_file, ) from pullbox.services.semantic_matching import ImportPolicy, SemanticMatchEngine @@ -103,6 +104,141 @@ def test_uncontradicted_mylar_identity_keeps_local_fast_path(): engine.match_against_issue.assert_not_called() +@pytest.mark.parametrize( + ("file_title", "issue_number", "issue_cv_id"), + [ + ("Action Comics", 969, 566668), + ("Thunderbolts", 105, 234803), + ], +) +def test_corroborated_unrelated_title_blocks_parent_issue_number_inheritance( + file_title: str, issue_number: int, issue_cv_id: int +) -> None: + parent = ImportedSeries( + raw_series_name="Fritzi Ritz", + cv_title="Fritzi Ritz", + cv_id=31895, + cv_match_method="mylar3_cv_id", + status=ImportSeriesStatus.MATCHED, + ) + file = ImportedFile( + file_name=f"{file_title} {issue_number} (2017).cbz", + parsed_series="Fritzi Ritz", + parsed_issue_number=issue_number, + diagnostics={ + "comicvine_series_id": 31895, + "metadata_signals": {"comicvine_series_id": "mylar3"}, + }, + ) + metadata = SourceMetadata( + original_title=file.file_name, + series_name="Fritzi Ritz", + issue_number=issue_number, + issue_type=IssueType.ISSUE, + diagnostics={ + "filename_parse": { + "series_name": file_title, + "issue_number": issue_number, + "year": 2017, + }, + "comicinfo": {"series": file_title, "number": str(issue_number)}, + }, + ) + entry = (None, issue_cv_id, False, None, "Existing parent issue") + + candidate, conflict = _evaluate_file_match_candidate( + imp_series=parent, + imp_file=file, + target_index=FileMatchTargetIndex(number_map={issue_number: entry}), + target_series=None, + file_metadata=metadata, + semantic_match_engine=SemanticMatchEngine(policy=ImportPolicy()), + build_import_metadata_conflict=build_import_metadata_conflict, + series_high_confidence=True, + ) + + assert candidate is None + assert conflict is not None + assert conflict["conflict_type"] == "corroborated_file_series_mismatch" + assert conflict["source_series"] == file_title + assert conflict["target_series"] == "Fritzi Ritz" + assert conflict["preserve_series_match"] is True + + +def test_unrelated_reading_order_filename_keeps_parent_issue_match_review_only() -> None: + """A clear foreign title cannot inherit a parent issue merely by sharing its number.""" + parent = ImportedSeries( + raw_series_name="Fritzi Ritz", + cv_title="Fritzi Ritz", + cv_id=31895, + cv_match_method="mylar3_cv_id", + status=ImportSeriesStatus.MATCHED, + ) + file = ImportedFile( + file_name="042 - Thunderbolts 105 (converted).cbz", + parsed_series="Fritzi Ritz", + parsed_issue_number=105, + diagnostics={ + "source_issue_type": IssueType.ISSUE.value, + "comicvine_series_id": 31895, + "metadata_signals": { + "comicvine_series_id": "mylar3", + "issue_number": "release_title", + }, + "source_metadata": { + "archive_metadata_loaded": True, + "archive_entry_issue_hint_checked": True, + }, + }, + ) + metadata = source_metadata_for_import_file(parent, file) + entry = (None, None, False, None, "Coincidental parent issue") + + assert metadata.diagnostics["filename_parse"]["series_name"] == ( + "042 - Thunderbolts (converted)" + ) + + candidate, conflict = _evaluate_file_match_candidate( + imp_series=parent, + imp_file=file, + target_index=FileMatchTargetIndex(number_map={105: entry}), + target_series=None, + file_metadata=metadata, + semantic_match_engine=SemanticMatchEngine(policy=ImportPolicy()), + build_import_metadata_conflict=build_import_metadata_conflict, + series_high_confidence=True, + ) + + assert candidate is None + assert conflict is not None + assert conflict["conflict_type"] == "corroborated_file_series_mismatch" + assert conflict["corroborating_signals"] == ["filename_parse"] + assert conflict["preserve_series_match"] is True + + +@pytest.mark.parametrize( + "file_name", + [ + "original issue name 001.cbz", + "Saga 001 dup.cbz", + ], +) +def test_uncorroborated_free_form_filename_does_not_override_parent(file_name: str) -> None: + metadata = SourceMetadata( + original_title=file_name, + series_name="Batman", + issue_number=1, + diagnostics={ + "filename_parse": { + "series_name": file_name.rsplit(" ", 1)[0], + "issue_number": 1, + } + }, + ) + + assert corroborated_import_title_conflict(metadata, "Batman") is None + + @pytest.mark.parametrize("corroboration", ["comicinfo", "archive_entry_issue_hint"]) def test_local_title_guard_is_shared_by_source_types(corroboration): metadata = SourceMetadata( diff --git a/tests/unit/test_import_deferred_recovery_execution.py b/tests/unit/test_import_deferred_recovery_execution.py index 64dba9fd..718072ac 100644 --- a/tests/unit/test_import_deferred_recovery_execution.py +++ b/tests/unit/test_import_deferred_recovery_execution.py @@ -13,13 +13,16 @@ ImportJobStatus, ImportSeriesStatus, ) -from pullbox.providers.base import IssueSummary, SeriesMetadata +from pullbox.providers.base import IssueSummary, SeriesMetadata, SeriesSearchResult from pullbox.services.catalog.reader import CatalogIssueSummary from pullbox.services.import_deferred_recovery import ( apply_deferred_recovery, plan_deferred_recovery, ) -from pullbox.services.import_deferred_recovery_execution import prepare_deferred_recovery +from pullbox.services.import_deferred_recovery_execution import ( + _search_exact_title_catalog, + prepare_deferred_recovery, +) from tests.unit.test_import_deferred_recovery import add_file, register, seed @@ -150,6 +153,411 @@ async def test_background_recovery_fetches_each_candidate_catalog_once_and_resum assert provider.get_issue_summaries_for_series.await_count == 1 +async def test_background_recovery_uses_unique_exact_local_catalog_title_and_issue( + db_session, +): + job, item, _, _, _ = await seed(db_session) + item.raw_series_name = "New Avengers" + item.raw_year = 2004 + item.cv_id = 11497 + item.series_id = None + file = await add_file( + db_session, + job, + item, + file_path="/comics/New Avengers/New Avengers Finale 01 (2010).cbr", + file_name="New Avengers Finale 01 (2010).cbr", + parsed_series="New Avengers", + parsed_issue_number=1, + parsed_year=2010, + comicvine_issue_id=None, + diagnostics={ + "source_issue_type": "issue", + "metadata_signals": {"issue_number": "release_title"}, + "source_metadata": { + "filename_parse": { + "series_name": "New Avengers Finale", + "issue_number": 1, + "issue_number_text": "1", + "year": 2010, + "volume": None, + "issue_type": "issue", + }, + "archive_entry_issue_hint": { + "series_name": "New Avengers Finale", + "issue_number": 1, + "confidence": "strong", + }, + }, + "kind": "metadata_conflict", + "conflict_type": "corroborated_file_series_mismatch", + }, + ) + job.status = ImportJobStatus.IMPORTING + job.progress_snapshot = {"deferred_recovery": {"state": "queued"}} + metadata = AsyncMock() + metadata.search_catalog_series.return_value = [ + SeriesSearchResult( + provider_id="33091", + title="New Avengers Finale", + year_start=2010, + publisher="Marvel", + issue_count=1, + status="Ended", + cover_url=None, + description=None, + ) + ] + metadata.get_catalog_issue_summaries_for_series.return_value = [ + IssueSummary( + provider_id="247986", + issue_number=1, + issue_number_text="1", + title="Finale", + release_date="2010-06-01", + cover_url=None, + issue_type="issue", + ) + ] + + assert await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + + assert file.status is ImportedFileStatus.CONFIRMED + assert file.matched_issue_cv_id == 247986 + target = await db_session.get(ImportedSeries, file.import_series_id) + assert target.cv_id == 33091 + assert target.cv_title == "New Avengers Finale" + assert file.diagnostics["deferred_recovery"]["action"] == "catalog_title_identity" + metadata.search_catalog_series.assert_awaited_once_with("New Avengers Finale", limit=1000) + metadata.get_catalog_issue_summaries_for_series.assert_awaited_once_with(33091) + assert not await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + assert metadata.search_catalog_series.await_count == 1 + + +async def test_catalog_title_years_are_scoped_to_each_issue_designation() -> None: + metadata = AsyncMock() + metadata.search_catalog_series.return_value = [ + SeriesSearchResult( + provider_id="123", + title="Example Annual", + year_start=2000, + publisher="Example", + issue_count=2, + status="Ended", + cover_url=None, + description=None, + ) + ] + metadata.get_catalog_issue_summaries_for_series.return_value = [ + IssueSummary( + provider_id="1001", + issue_number=1, + issue_number_text="1", + title=None, + release_date="2010-01-01", + cover_url=None, + issue_type="issue", + ), + IssueSummary( + provider_id="1002", + issue_number=2, + issue_number_text="2", + title=None, + release_date="2000-01-01", + cover_url=None, + issue_type="issue", + ), + ] + + matches = await _search_exact_title_catalog( + metadata, + { + "query": "Example Annual", + "issue_numbers": ["1", "2"], + "years": [2000, 2010], + "years_by_issue": {"1": [2000], "2": [2010]}, + }, + ) + + assert matches == {} + + +@pytest.mark.parametrize( + ( + "parent_title", + "file_name", + "source_issue_type", + "catalog_title", + "catalog_series_id", + "catalog_issue_id", + "issue_number", + "year", + ), + [ + ( + "Fantastic Four", + "Fantastic Four Annual 032 (2010).cbz", + "annual", + "Fantastic Four Annual", + 2129, + 220032, + 32, + 2010, + ), + ( + "Uncanny X-Men", + "Uncanny X-Men Special 001 (2009).cbz", + "special", + "Uncanny X-Men Special", + 74730, + 747301, + 1, + 2009, + ), + ( + "Ultimate Fantastic Four", + "Ultimate Fantastic Four - Ultimate X-Men Annual 001 (2006).cbz", + "annual", + "Ultimate Fantastic Four - Ultimate X-Men Annual", + 23137, + 231371, + 1, + 2006, + ), + ], +) +async def test_background_recovery_reparses_older_rows_without_saved_filename_parse( + db_session, + parent_title, + file_name, + source_issue_type, + catalog_title, + catalog_series_id, + catalog_issue_id, + issue_number, + year, +): + job, item, _, _, _ = await seed(db_session) + item.raw_series_name = parent_title + item.raw_year = year + item.cv_id = 90000 + item.series_id = None + file = await add_file( + db_session, + job, + item, + file_name=file_name, + parsed_series=parent_title, + parsed_issue_number=issue_number, + parsed_year=year, + comicvine_issue_id=None, + diagnostics={ + "source_issue_type": source_issue_type, + "metadata_signals": {"issue_number": "release_title"}, + # Older completed imports did not persist filename_parse here. + "source_metadata": {}, + }, + ) + job.status = ImportJobStatus.IMPORTING + job.progress_snapshot = {"deferred_recovery": {"state": "queued"}} + metadata = AsyncMock() + metadata.search_catalog_series.return_value = [ + SeriesSearchResult( + provider_id=str(catalog_series_id), + title=catalog_title, + year_start=year, + publisher="Marvel", + issue_count=1, + status="Ended", + cover_url=None, + description=None, + ) + ] + metadata.get_catalog_issue_summaries_for_series.return_value = [ + IssueSummary( + provider_id=str(catalog_issue_id), + issue_number=issue_number, + issue_number_text=str(issue_number), + title=None, + release_date=f"{year}-06-01", + cover_url=None, + issue_type="issue", + ) + ] + + assert await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + + assert file.status is ImportedFileStatus.CONFIRMED + assert file.matched_issue_cv_id == catalog_issue_id + target = await db_session.get(ImportedSeries, file.import_series_id) + assert target.cv_id == catalog_series_id + assert target.cv_title == catalog_title + metadata.search_catalog_series.assert_awaited_once_with(catalog_title, limit=1000) + + +async def test_background_recovery_keeps_duplicate_exact_catalog_titles_in_review(db_session): + job, item, _, _, _ = await seed(db_session) + item.raw_series_name = "X-Men" + item.raw_year = 2021 + item.cv_id = 137402 + item.series_id = None + file = await add_file( + db_session, + job, + item, + file_name="X-Men Annual 001 (2023).cbr", + parsed_series="X-Men", + parsed_issue_number=1, + parsed_year=2023, + comicvine_issue_id=None, + diagnostics={ + "source_issue_type": "annual", + "metadata_signals": {"issue_number": "release_title"}, + "source_metadata": { + "filename_parse": { + "series_name": "X-Men", + "issue_number": 1, + "issue_number_text": "1", + "year": 2023, + "issue_type": "annual", + } + }, + }, + ) + job.status = ImportJobStatus.IMPORTING + job.progress_snapshot = {"deferred_recovery": {"state": "queued"}} + metadata = AsyncMock() + metadata.search_catalog_series.return_value = [ + SeriesSearchResult( + provider_id="146988", + title="X-Men Annual", + year_start=2023, + publisher="Marvel", + issue_count=1, + status="Ended", + cover_url=None, + description=None, + ), + SeriesSearchResult( + provider_id="146999", + title="X-Men Annual", + year_start=2023, + publisher="Marvel", + issue_count=1, + status="Ended", + cover_url=None, + description=None, + ), + ] + metadata.get_catalog_issue_summaries_for_series.side_effect = [ + [ + IssueSummary( + provider_id="10001", + issue_number=1, + issue_number_text="1", + title=None, + release_date="2023-06-01", + cover_url=None, + issue_type="issue", + ) + ], + [ + IssueSummary( + provider_id="10002", + issue_number=1, + issue_number_text="1", + title=None, + release_date="2023-08-01", + cover_url=None, + issue_type="issue", + ) + ], + ] + + assert await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + + assert file.status is ImportedFileStatus.NO_MATCH + assert file.import_series_id == item.id + assert job.status is ImportJobStatus.COMPLETED + + +async def test_background_recovery_never_overrides_conflicting_saved_provider_issue( + db_session, +): + job, item, _, _, _ = await seed(db_session) + item.raw_series_name = "New Avengers" + item.raw_year = 2013 + item.cv_id = 55330 + item.series_id = None + file = await add_file( + db_session, + job, + item, + file_name="New Avengers - Ultron Forever 001 (2015).cbr", + parsed_series="New Avengers", + parsed_issue_number=1, + parsed_year=2015, + comicvine_issue_id=376665, + diagnostics={ + "source_issue_type": "issue", + "comicvine_series_id": 55330, + "metadata_signals": { + "comicvine_series_id": "mylar3", + "comicvine_issue_id": "mylar3", + }, + "source_metadata": { + "filename_parse": { + "series_name": "New Avengers - Ultron Forever", + "issue_number": 1, + "issue_number_text": "1", + "year": 2015, + "issue_type": "issue", + }, + "archive_entry_issue_hint": { + "series_name": "New Avengers - Ultron Forever", + "issue_number": 1, + "confidence": "strong", + }, + }, + "kind": "metadata_conflict", + "conflict_type": "corroborated_file_series_mismatch", + }, + ) + job.status = ImportJobStatus.IMPORTING + job.progress_snapshot = {"deferred_recovery": {"state": "queued"}} + metadata = AsyncMock() + metadata.get_series_metadata.return_value = SeriesMetadata( + provider_id="55330", + title="New Avengers", + sort_title="new avengers", + year_start=2013, + year_end=None, + status="Ended", + publisher="Marvel", + description=None, + cover_url=None, + issue_count=34, + comicvine_url=None, + ) + metadata.get_issue_summaries_for_series.return_value = [ + IssueSummary( + provider_id="376665", + issue_number=1, + issue_number_text="1", + title=None, + release_date="2013-01-01", + cover_url=None, + issue_type="issue", + ) + ] + + assert await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + + assert file.status is ImportedFileStatus.NO_MATCH + assert file.import_series_id == item.id + metadata.search_catalog_series.assert_not_awaited() + + async def test_catalog_checkpoint_serializes_local_catalog_cutoff(db_session): job, item, _, _, _ = await seed(db_session) item.series_id = None diff --git a/tests/unit/test_import_source_metadata.py b/tests/unit/test_import_source_metadata.py index 5fecc928..734481e9 100644 --- a/tests/unit/test_import_source_metadata.py +++ b/tests/unit/test_import_source_metadata.py @@ -60,7 +60,15 @@ def test_source_metadata_for_import_file_restores_persisted_signals() -> None: assert metadata.issue_number == 4.0 assert metadata.issue_type == IssueType.ISSUE assert metadata.signals == {"series_name": MetadataSignal.COMICINFO} - assert metadata.diagnostics == {"has_comicinfo": True} + assert metadata.diagnostics["has_comicinfo"] is True + assert metadata.diagnostics["filename_parse"] == { + "series_name": "Chicken Devil", + "issue_number": 4.0, + "issue_number_text": "4", + "year": 2022, + "volume": None, + "issue_type": IssueType.ISSUE.value, + } def test_source_metadata_for_import_file_uses_persisted_filename_issue_fallback() -> None: diff --git a/tests/unit/test_release_parser.py b/tests/unit/test_release_parser.py index cade3e85..2e0d78fa 100644 --- a/tests/unit/test_release_parser.py +++ b/tests/unit/test_release_parser.py @@ -40,6 +40,30 @@ def test_dc_one_million_issue_number_without_hash_is_exact(self) -> None: assert r.issue_number == 1_000_000.0 assert r.year == 1998 + @pytest.mark.parametrize( + ("filename", "expected_series", "expected_issue", "expected_year"), + [ + ("batman.104. (2021).cbz", "batman", 104.0, 2021), + ("robin.1000000.(1998).cbz", "robin", 1_000_000.0, 1998), + ("Series 0.5.cbz", "Series", 0.5, None), + ("Series -1.cbz", "Series", -1.0, None), + ], + ) + def test_local_filename_issue_designations_are_preserved( + self, + filename: str, + expected_series: str, + expected_issue: float, + expected_year: int | None, + ) -> None: + result = parse_release_title(filename, expected_series=(expected_series,)) + + assert result is not None + assert result.series_name == expected_series + assert result.issue_number == expected_issue + assert result.issue_number_text == str(expected_issue).removesuffix(".0") + assert result.year == expected_year + def test_html_entity_in_name(self) -> None: r = parse_release_title( "Spider-Man & Wolverine 003 [2025] [4 covers] [Digital] [dekabro-Empire]" From 51a4c7712fbb2e4fd284f0d7f46263de7b534abb Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 20:24:35 -0700 Subject: [PATCH 16/20] fix(import): recover misplaced referenced files using catalog targets --- docs/development/IMPORT_REVIEW_RECOVERY.md | 24 +- .../services/import_completed_cleanup.py | 18 +- .../import_deferred_recovery_execution.py | 60 +- .../services/import_reference_recovery.py | 403 +++++++++++++ src/pullbox/ui/import_results_context.py | 3 +- tests/unit/test_import_reference_recovery.py | 532 ++++++++++++++++++ 6 files changed, 1028 insertions(+), 12 deletions(-) create mode 100644 src/pullbox/services/import_reference_recovery.py create mode 100644 tests/unit/test_import_reference_recovery.py diff --git a/docs/development/IMPORT_REVIEW_RECOVERY.md b/docs/development/IMPORT_REVIEW_RECOVERY.md index cafbe582..1d6fa998 100644 --- a/docs/development/IMPORT_REVIEW_RECOVERY.md +++ b/docs/development/IMPORT_REVIEW_RECOVERY.md @@ -199,6 +199,28 @@ ineligible. them again. A different file for an owned issue remains a review decision; it is never automatically substituted for the owned copy. +The same recheck can repair already-imported, referenced comics in mixed folders, +even when their correct series or issue is not in Pullbox yet. It groups exact +title lookups against the local catalog and requires a unique issue number, type, +and agreeing per-file publication year (within one year for dated files). +Filename-only evidence without a publication year remains review-only. Trusted +embedded IDs must agree. Manual choices, safety decisions, managed artifacts, +duplicate candidates, and targets with another owned file are not overwritten. +Before changing an assignment, the worker rechecks the source path, size, and +timestamp inside an enabled reference-capable root. This also works with a +read-only root; no source bytes, filenames, or directories are changed. + +Missing metadata targets are registered as partial catalogs, without creating a +series folder or running file import/conversion. The existing LibraryFile row is +retained and assigned to the verified issue; old and new ownership counters and +import Story Arc links are refreshed. Reading history remains unchanged. +Empty provisional issues left under the wrong series are kept for audit but +marked skipped rather than becoming new wanted downloads. +Each repair records its previous assignment and commits with its recovery +checkpoint. Resume preserves completed repairs, and repeated runs do not create +another file registration. This is logical library repair, not authorization to +reorganize the user's filesystem. + The deferred pass uses complete local catalogs first. Exact issue identity may correct stale Mylar ownership only when the file's title, issue number, type, and embedded identity agree with the target. Conflicting embedded IDs remain @@ -218,7 +240,7 @@ are stored. Live provider progress does not rewrite the full durable recovery snapshot; each completed catalog produces one durable checkpoint, so a worker restart resumes after the last completed catalog without replaying it. -Recovered files run through normal Step 4 safety, current-source validation, +Previously unimported recovered files run through normal Step 4 safety, current-source validation, ownership checks, and the original copy or keep-in-place settings. Only newly prepared recovery groups execute, not unrelated ready files or Story Arcs. Cancellation stops this pass without rolling back the original import or diff --git a/src/pullbox/services/import_completed_cleanup.py b/src/pullbox/services/import_completed_cleanup.py index fd8f5479..f75b17f6 100644 --- a/src/pullbox/services/import_completed_cleanup.py +++ b/src/pullbox/services/import_completed_cleanup.py @@ -252,7 +252,14 @@ def _eligible_conflict_groups(job_id: int) -> Any: def _file_filters(job_id: int, action: CompletedImportCleanupAction) -> tuple[Any, ...]: filters: list[Any] = [ImportedFile.import_job_id == job_id] if action is CompletedImportCleanupAction.RECHECK_DEFERRED_FILES: - filters.append(ImportedFile.status == ImportedFileStatus.NO_MATCH) + from pullbox.services.import_reference_recovery import reference_candidate_ids + + filters.append( + or_( + ImportedFile.status == ImportedFileStatus.NO_MATCH, + ImportedFile.id.in_(reference_candidate_ids(job_id)), + ) + ) elif action is CompletedImportCleanupAction.DISMISS_MISSING_REFERENCES: filters.append( or_( @@ -1340,9 +1347,12 @@ async def _apply_recommended_conflicts( async def _apply_mixed_folder_resolutions( session: AsyncSession, job: ImportJob, + *, + resolutions: tuple[_MixedFolderResolution, ...] | None = None, ) -> tuple[set[int], set[int]]: """Rebucket exact embedded identities while preserving every source artifact.""" - resolutions = await _load_mixed_folder_resolutions(session, int(job.id)) + if resolutions is None: + resolutions = await _load_mixed_folder_resolutions(session, int(job.id)) if not resolutions: return set(), set() @@ -1477,6 +1487,8 @@ async def _apply_mixed_folder_resolutions( "evidence_source": resolution.evidence_source, "source_import_series_id": resolution.source_import_series_id, "source_import_series_name": resolution.source_import_series_name, + "source_issue_id": resolution.source_issue_id, + "source_library_file_id": resolution.source_library_file_id, "source_series_name": resolution.source_series_name, "target_import_series_id": target_import_series.id, "target_series_id": resolution.target_series_id, @@ -1723,6 +1735,7 @@ async def apply_completed_import_cleanup( raise ValidationError("The cleanup scope changed. Preview the action again.") if action is CompletedImportCleanupAction.RECHECK_DEFERRED_FILES: + from pullbox.services.import_reference_recovery import reference_candidates from pullbox.services.import_retry_helpers import require_retained_import_destination require_retained_import_destination(job) @@ -1734,6 +1747,7 @@ async def apply_completed_import_cleanup( "run_id": uuid4().hex, "series_ids": [], "stale_series_ids": stale_series_ids, + "reference_candidates": await reference_candidates(session, job_id), "actor_id": actor_id, }, "mode": "import", diff --git a/src/pullbox/services/import_deferred_recovery_execution.py b/src/pullbox/services/import_deferred_recovery_execution.py index 29175903..e14761c6 100644 --- a/src/pullbox/services/import_deferred_recovery_execution.py +++ b/src/pullbox/services/import_deferred_recovery_execution.py @@ -36,6 +36,10 @@ provider_ids, refresh_recovered_groups, ) +from pullbox.services.import_reference_recovery import ( + reference_candidates, + repair_catalog_references, +) from pullbox.services.import_source_metadata import source_metadata_for_import_file from pullbox.services.import_workflow_state import ( emit_live_progress, @@ -116,13 +120,17 @@ def _filename_catalog_identity( async def _catalog_title_candidates( session: AsyncSession, job: ImportJob, + references: dict[str, dict[str, Any]] | None = None, ) -> dict[str, dict[str, Any]]: """Group deterministic filename searches so each catalog title is queried once.""" candidates: dict[str, dict[str, Any]] = {} - for file, item in await load_deferred_rows(session, job.id): - identity = _filename_catalog_identity(file, item) - if identity is None: - continue + identities = [ + identity + for file, item in await load_deferred_rows(session, job.id) + if (identity := _filename_catalog_identity(file, item)) is not None + ] + identities.extend((references or {}).values()) + for identity in identities: key = str(identity["key"]) candidate = candidates.setdefault( key, @@ -207,6 +215,10 @@ async def _search_exact_title_catalog( "cv_id": int(result.provider_id), "title": result.title, "year": result.year_start, + "issue_count": result.issue_count, + "publisher": result.publisher, + "cover_url": result.cover_url, + "comicvine_url": result.comicvine_url, "summary": _catalog_summary_payload(summary), } ) @@ -544,7 +556,12 @@ async def prepare_deferred_recovery( revision_state = {"value": int(job.progress_revision or 0)} - def progress_event(current: int, total: int, message: str) -> ImportProgressEvent: + def progress_event( + current: int, + total: int, + message: str, + unit: str, + ) -> ImportProgressEvent: return ImportProgressEvent( job_id=job_id, status=ImportJobStatus.IMPORTING, @@ -556,7 +573,7 @@ def progress_event(current: int, total: int, message: str) -> ImportProgressEven current_file_progress_current=current, current_file_progress_total=total, current_file_progress_pct=round(100 * current / max(total, 1)), - current_file_progress_unit="catalogs", + current_file_progress_unit=unit, ) async def report( @@ -566,10 +583,11 @@ async def report( *, durable: bool = True, check_control: bool = True, + unit: str = "catalogs", ) -> None: if check_control: await raise_if_job_cancelled(session, job_id) - event = progress_event(current, total, message) + event = progress_event(current, total, message, unit) if durable: event.progress_revision = revision_state["value"] + 1 await emit_progress(session, job, event, progress_callback) @@ -591,13 +609,18 @@ async def report( await report(0, 1, "Reconciling deferred files with the existing library...") local_counts = await apply_deferred_recovery(session, job, running=True) state = recovery_state(job) + references = state.get("reference_candidates") + if references is None: + references = await reference_candidates(session, job.id) state.update( state="catalogs", local_counts=local_counts, candidates=await _catalog_candidates(session, job), completed=[], matches={}, - title_candidates=await _catalog_title_candidates(session, job), + reference_candidates=references, + reference_files_repaired=0, + title_candidates=await _catalog_title_candidates(session, job, references), title_completed=[], title_matches={}, ) @@ -705,11 +728,31 @@ async def report( await report(len(completed), max(len(candidates), 1), "Preparing verified files for import...") catalog_count = await _prepare_catalog_targets(session, job) title_catalog_count = await _prepare_title_catalog_targets(session, job) + await session.commit() + + async def reference_progress(current: int, total: int) -> None: + await report( + current, + total, + f"Checking misplaced references {current} of {total}...", + durable=False, + unit="files", + ) + + reference_count = await repair_catalog_references( + session, + job, + metadata_service, + state.get("reference_candidates", {}), + state.get("title_matches", {}), + progress=reference_progress, + ) state = recovery_state(job) state.update( state="prepared", catalog_files_prepared=catalog_count, title_catalog_files_prepared=title_catalog_count, + reference_files_repaired=reference_count, ) job.error_message = None if not state.get("series_ids"): @@ -735,6 +778,7 @@ async def report( "catalog_files_prepared": catalog_count, "title_catalogs_checked": len(title_completed), "title_catalog_files_prepared": title_catalog_count, + "reference_files_repaired": reference_count, "source_preserved": True, }, ) diff --git a/src/pullbox/services/import_reference_recovery.py b/src/pullbox/services/import_reference_recovery.py new file mode 100644 index 00000000..ba4474de --- /dev/null +++ b/src/pullbox/services/import_reference_recovery.py @@ -0,0 +1,403 @@ +"""Conservative catalog repair of already-imported, user-owned references.""" + +from __future__ import annotations + +import asyncio +import json +from collections import Counter +from datetime import datetime +from hashlib import sha256 +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from sqlalchemy import and_, or_, select + +from pullbox.core.exceptions import ConfigurationError +from pullbox.core.issue_numbers import normalize_issue_number_text +from pullbox.core.library_file_ownership import build_file_identity_signature +from pullbox.core.name_matcher import NameMatcher +from pullbox.core.release_parser import parse_release_title +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportSeriesStatus, +) +from pullbox.models.issue import Issue, IssueStatus +from pullbox.models.library import LibraryFile, LibraryFileStorageMode, LibraryRoot +from pullbox.models.series import IssueCatalogState, Series +from pullbox.providers.base import IssueSummary +from pullbox.services.catalog.reader import CatalogIssueSummary, CatalogSeriesMetadata +from pullbox.services.import_deferred_recovery import ( + apply_proven_identity, + positive_id, + provider_ids, + refresh_recovered_groups, + same_source, +) + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql import Select + + from pullbox.models.import_job import ImportJob + from pullbox.services.metadata_service import MetadataService + + +def reference_candidate_ids(job_id: int) -> Select[tuple[int]]: + """Bound the existing recheck preview to potentially misplaced references.""" + return ( + select(ImportedFile.id) + .join(ImportedSeries, ImportedSeries.id == ImportedFile.import_series_id) + .join(LibraryFile, LibraryFile.id == ImportedFile.library_file_id) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.status == ImportedFileStatus.IMPORTED, + LibraryFile.storage_mode == LibraryFileStorageMode.REFERENCED, + LibraryFile.file_path == ImportedFile.file_path, + LibraryFile.issue_id == ImportedFile.matched_issue_id, + ImportedSeries.user_selected_cv_id.is_(None), + or_( + ImportedFile.match_method.is_(None), + and_( + ~ImportedFile.match_method.startswith("manual"), + ~ImportedFile.match_method.startswith("orphan_recovery"), + ImportedFile.match_method != "completed_import_metadata_reassignment", + ), + ), + or_( + ImportedFile.parsed_series != ImportedSeries.raw_series_name, + and_( + ImportedSeries.files_no_match > 0, + ImportedFile.diagnostics["metadata_signals"]["issue_number"].as_string() + == "release_title", + ), + ), + ) + ) + + +def _stamp(file: ImportedFile, item: ImportedSeries, library: LibraryFile) -> str: + payload = [ + file.updated_at.isoformat(), + library.updated_at.isoformat(), + item.user_selected_cv_id, + item.cv_id, + item.cv_title, + item.raw_series_name, + item.series_id, + file.file_path, + file.file_name, + file.file_size, + file.status.value, + file.match_method, + file.import_series_id, + file.library_file_id, + file.matched_issue_id, + file.matched_issue_cv_id, + file.comicvine_issue_id, + file.conflict_group_id, + file.duplicate_group_id, + file.diagnostics, + file.source_signature, + library.source_signature, + library.issue_id, + library.storage_mode.value, + library.library_root_id, + ] + return sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() + + +async def reference_candidates(session: AsyncSession, job_id: int) -> dict[str, dict[str, Any]]: + """Freeze exact identity evidence without inspecting or modifying source files.""" + from pullbox.services.import_completed_cleanup import _mixed_folder_source_identity + + result: dict[str, dict[str, Any]] = {} + rows = await session.stream( + select(ImportedFile, ImportedSeries, LibraryFile, Issue) + .join(ImportedSeries, ImportedSeries.id == ImportedFile.import_series_id) + .join(LibraryFile, LibraryFile.id == ImportedFile.library_file_id) + .join(Issue, Issue.id == LibraryFile.issue_id) + .where(ImportedFile.id.in_(reference_candidate_ids(job_id))) + .execution_options(yield_per=500) + ) + try: + async for file, item, library, old_issue in rows: + diagnostics = dict(file.diagnostics or {}) + raw_source = diagnostics.get("source_metadata") + source = raw_source if isinstance(raw_source, dict) else {} + if ( + file.conflict_group_id is not None + or file.duplicate_group_id is not None + or not same_source(file, library) + or any( + diagnostics.get(key) + for key in ( + "safety_block", + "safety_exception", + "safety_review", + "source_revalidation", + "identity_conflicts", + ) + ) + or source.get("identity_conflicts") + ): + continue + identity = _mixed_folder_source_identity(file) + if identity is None: + continue + title, raw_number, series_cv_id, issue_cv_id, evidence = identity + # A derived old match is not source evidence. Any other saved ID must + # agree with the trusted embedded identity, or remain for review. + allowed_ids = {value for value in (old_issue.comicvine_id, issue_cv_id) if value} + if ( + not provider_ids(file).issubset(allowed_ids) + or (file.comicvine_issue_id is not None and issue_cv_id is None) + or (source.get("comicinfo") and evidence == "filename_parse" and provider_ids(file)) + ): + continue + key = NameMatcher.normalize(title) + if not key or key == NameMatcher.normalize(item.cv_title or item.raw_series_name): + continue + try: + number = normalize_issue_number_text(raw_number) + except ValueError: + continue + parsed = parse_release_title(file.file_name) + year = parsed.year if parsed is not None else file.parsed_year + issue_type = str(diagnostics.get("source_issue_type") or "issue") + if evidence == "filename_parse" and parsed is not None: + issue_type = parsed.issue_type.value + result[str(file.id)] = { + "key": key, + "query": title, + "issue_number": number, + "year": year or 0, + "issue_type": issue_type, + "series_cv_id": series_cv_id, + "issue_cv_id": issue_cv_id, + "evidence": evidence, + "stamp": _stamp(file, item, library), + } + finally: + await rows.close() + return result + + +def _target_agrees(identity: dict[str, Any], target: dict[str, Any]) -> bool: + summary = target["summary"] + if ( + NameMatcher.normalize(str(target["title"])) != identity["key"] + or str(summary.get("issue_type") or "issue") != identity["issue_type"] + or (identity["series_cv_id"] and identity["series_cv_id"] != target["cv_id"]) + or ( + identity["issue_cv_id"] + and identity["issue_cv_id"] != positive_id(summary["provider_id"]) + ) + ): + return False + release_date = summary.get("release_date") + if identity["year"] and release_date: + try: + return abs(int(str(release_date)[:4]) - int(identity["year"])) <= 1 + except ValueError: + return False + # Filename-only evidence cannot distinguish same-name reboots without a date. + return bool(identity["evidence"] != "filename_parse") + + +def _unchanged_source(path: str, signature: dict[str, Any], root_path: str) -> bool: + try: + current = build_file_identity_signature(Path(path)) + if not Path(str(current["resolved_path"])).is_relative_to( + Path(root_path).resolve(strict=True) + ): + return False + except (OSError, RuntimeError, ValueError, ConfigurationError): + return False + # Device/inode can change after a container upgrade; size, mtime and path + # remain portable. Recovery never writes to the artifact being inspected. + size = signature.get("size", signature.get("size_bytes")) + return bool( + size == current["size"] + and signature.get("mtime_ns") == current["mtime_ns"] + and signature.get("resolved_path", path) == current["resolved_path"] + ) + + +async def repair_catalog_references( + session: AsyncSession, + job: ImportJob, + metadata: MetadataService, + saved: dict[str, dict[str, Any]], + matches: dict[str, Any], + progress: Callable[[int, int], Awaitable[None]] | None = None, +) -> int: + """Create missing metadata targets and reuse ownership repair, never file import.""" + from pullbox.services.import_completed_cleanup import ( + _apply_mixed_folder_resolutions, + _MixedFolderResolution, + ) + from pullbox.services.import_workflow_state import raise_if_job_cancelled + + current = await reference_candidates(session, job.id) + plans: list[tuple[int, dict[str, Any], dict[str, Any]]] = [] + for file_id_text, identity in saved.items(): + if current.get(file_id_text) != identity: + continue + options = matches.get(identity["key"], {}).get(identity["issue_number"], []) + options = [option for option in options if _target_agrees(identity, option)] + if len(options) == 1: + plans.append((int(file_id_text), identity, options[0])) + counts = Counter(str(target["summary"]["provider_id"]) for _, _, target in plans) + state = dict(job.progress_snapshot.get("deferred_recovery") or {}) + repaired = int(state.get("reference_files_repaired", 0)) + for position, (file_id, identity, target) in enumerate(plans): + if progress is not None: + await progress(position, len(plans)) + if counts[str(target["summary"]["provider_id"])] != 1: + continue + await raise_if_job_cancelled(session, job.id) + file = await session.get(ImportedFile, file_id) + if file is None or file.library_file_id is None: + continue + library = await session.get(LibraryFile, file.library_file_id) + item = await session.get(ImportedSeries, file.import_series_id) + if library is None or item is None: + continue + root = await session.get(LibraryRoot, library.library_root_id) + if root is None or not root.enabled or not root.allow_referenced_registrations: + continue + if _stamp(file, item, library) != identity["stamp"]: + continue + inspected_root_path = root.path + await session.commit() + if not await asyncio.to_thread( + _unchanged_source, file.file_path, file.source_signature, root.path + ): + continue + # Reload ownership after filesystem I/O, before any catalog row is written. + await raise_if_job_cancelled(session, job.id) + await session.refresh(file) + await session.refresh(library) + await session.refresh(item) + await session.refresh(root) + if ( + _stamp(file, item, library) != identity["stamp"] + or not root.enabled + or not root.allow_referenced_registrations + or root.path != inspected_root_path + ): + continue + cv_id = int(target["cv_id"]) + issue_cv_id = int(target["summary"]["provider_id"]) + series = await session.scalar(select(Series).where(Series.comicvine_id == cv_id)) + created_series = series is None + issue = await session.scalar(select(Issue).where(Issue.comicvine_id == issue_cv_id)) + if issue is not None and (series is None or issue.series_id != series.id): + continue + if issue is not None and ( + issue.effective_issue_number_text != identity["issue_number"] + or issue.issue_type.value != identity["issue_type"] + ): + continue + if issue is not None and await session.scalar( + select(LibraryFile.id).where(LibraryFile.issue_id == issue.id) + ): + continue + if await session.scalar( + select(ImportedFile.id).where( + ImportedFile.matched_issue_cv_id == issue_cv_id, + ImportedFile.include_in_import.is_(True), + ImportedFile.status.in_((ImportedFileStatus.MATCHED, ImportedFileStatus.CONFIRMED)), + ) + ): + continue + if series is not None and issue is None: + existing = list( + await session.scalars(select(Issue).where(Issue.series_id == series.id)) + ) + if any(row.effective_issue_number_text == identity["issue_number"] for row in existing): + continue + if series is None: + series = await metadata.upsert_series_metadata( + session, + cv_id, + CatalogSeriesMetadata( + provider_id=str(cv_id), + title=str(target["title"]), + sort_title=str(target["title"]), + year_start=target.get("year"), + year_end=None, + status=None, + publisher=target.get("publisher"), + description=None, + cover_url=target.get("cover_url"), + issue_count=target.get("issue_count"), + comicvine_url=target.get("comicvine_url"), + ), + ) + series.monitored = False + series.issue_catalog_state = IssueCatalogState.PARTIAL + if issue is None: + payload = target["summary"] + cutoff = payload.get("source_cutoff_at") + summary = CatalogIssueSummary( + source_cutoff_at=datetime.fromisoformat(cutoff) if cutoff else None, + **{ + key: payload[key] for key in IssueSummary.__dataclass_fields__ if key in payload + }, + ) + await metadata.upsert_issue_summaries(session, series, [summary]) + issue = await session.scalar(select(Issue).where(Issue.comicvine_id == issue_cv_id)) + if issue is None or issue.series_id != series.id: + continue + resolution = _MixedFolderResolution( + file_id=file.id, + source_import_series_id=item.id, + source_import_series_name=item.raw_series_name, + target_series_id=series.id, + target_series_title=series.title, + target_issue_id=issue.id, + target_issue_cv_id=issue.comicvine_id, + target_issue_number=issue.issue_number, + target_issue_number_text=issue.effective_issue_number_text, + target_library_file_id=None, + source_library_file_id=library.id, + source_issue_id=library.issue_id, + source_library_updated_at=library.updated_at.isoformat(), + evidence_source=identity["evidence"], + source_series_name=identity["query"], + source_updated_at=file.updated_at.isoformat(), + ) + affected, _ = await _apply_mixed_folder_resolutions(session, job, resolutions=(resolution,)) + previous_issue = await session.get(Issue, resolution.source_issue_id) + if ( + previous_issue is not None + and previous_issue.comicvine_id is None + and previous_issue.metadata_source in {"provisional_import", "import_placeholder"} + and not await session.scalar( + select(LibraryFile.id).where(LibraryFile.issue_id == previous_issue.id) + ) + ): + # Keep the audit/reader row, but do not search for an issue invented + # from a misplaced filename under the wrong series. + previous_issue.status = IssueStatus.SKIPPED + if created_series: + target_group = await session.get(ImportedSeries, file.import_series_id) + assert target_group is not None + target_group.status = ImportSeriesStatus.IMPORTED + apply_proven_identity( + file, issue_cv_id=issue_cv_id, series_cv_id=cv_id, summary=target["summary"] + ) + await refresh_recovered_groups(session, job, affected) + repaired += 1 + state = dict(job.progress_snapshot.get("deferred_recovery") or {}) + state["reference_files_repaired"] = repaired + job.progress_snapshot = {**job.progress_snapshot, "deferred_recovery": state} + await session.commit() + if progress is not None and plans: + await progress(len(plans), len(plans)) + return repaired diff --git a/src/pullbox/ui/import_results_context.py b/src/pullbox/ui/import_results_context.py index 617243c9..b1c9efba 100644 --- a/src/pullbox/ui/import_results_context.py +++ b/src/pullbox/ui/import_results_context.py @@ -329,7 +329,8 @@ async def _load_files_for_status( "label": "Recheck deferred files", "description": ( "Group repeated file records, recognize completed imports, and recover exact issue " - "matches. Missing series catalogs are checked in the background. " + "matches, including misplaced files already kept in place. Missing series catalogs " + "are checked in the background. Source files are not moved or changed. " "Files that still need a decision remain here." ), "button_label": "Recheck files", diff --git a/tests/unit/test_import_reference_recovery.py b/tests/unit/test_import_reference_recovery.py new file mode 100644 index 00000000..14640000 --- /dev/null +++ b/tests/unit/test_import_reference_recovery.py @@ -0,0 +1,532 @@ +"""Catalog recovery repairs assignments, never the user's source files.""" + +from unittest.mock import AsyncMock + +import pytest +from sqlalchemy import func, select + +from pullbox.core.library_file_ownership import build_file_identity_signature +from pullbox.models.import_job import ( + ImportedFileStatus, + ImportedSeries, + ImportJobStatus, + ImportSourceType, +) +from pullbox.models.issue import Issue, IssueStatus +from pullbox.models.library import LibraryFile, LibraryFileStorageMode +from pullbox.models.series import Series +from pullbox.providers.base import IssueSummary, SeriesSearchResult +from pullbox.services.import_completed_cleanup import ( + CompletedImportCleanupAction, + apply_completed_import_cleanup, + count_completed_import_cleanup_scope, + preview_completed_import_cleanup, +) +from pullbox.services.import_deferred_recovery_execution import prepare_deferred_recovery +from pullbox.services.metadata_service import MetadataService +from tests.unit.test_import_deferred_recovery import add_file, register, seed + + +async def reference_case(session, tmp_path, source_type=ImportSourceType.MYLAR3): + job, item, series, issue, root = await seed(session, source_type=source_type) + series.title = item.raw_series_name = "Fritzi Ritz" + issue.status = IssueStatus.OWNED + root.path = str(tmp_path) + path = tmp_path / "Thunderbolts 104 (2021).cbz" + path.write_bytes(b"unchanged comic content") + file = await add_file( + session, + job, + item, + file_path=str(path), + file_name=path.name, + file_size=path.stat().st_size, + status=ImportedFileStatus.IMPORTED, + parsed_series="Thunderbolts", + comicvine_issue_id=None, + matched_issue_id=issue.id, + matched_issue_cv_id=issue.comicvine_id, + source_signature=build_file_identity_signature(path), + diagnostics={"metadata_signals": {"issue_number": "release_title"}}, + ) + library = await register(session, file, issue, root) + library.storage_mode = LibraryFileStorageMode.REFERENCED + file.library_file_id = library.id + item.files_no_match = 1 + metadata = MetadataService(AsyncMock(), tmp_path / "covers") + metadata.search_catalog_series = AsyncMock( + return_value=[ + SeriesSearchResult( + provider_id="700", + title="Thunderbolts", + year_start=2016, + publisher=None, + issue_count=130, + status=None, + cover_url=None, + description=None, + ) + ] + ) + metadata.get_catalog_issue_summaries_for_series = AsyncMock( + return_value=[ + IssueSummary( + provider_id="7001", + issue_number=104, + issue_number_text="104", + title=None, + release_date="2021-01-01", + cover_url=None, + issue_type="issue", + ) + ] + ) + job.status = ImportJobStatus.IMPORTING + job.progress_snapshot = {"deferred_recovery": {"state": "queued"}} + await session.commit() + return job, file, library, issue, metadata, path + + +@pytest.mark.parametrize("source_type", list(ImportSourceType)) +async def test_recovery_creates_missing_target_and_reassigns_reference_in_place( + db_session, + tmp_path, + source_type, +): + job, file, library, wrong_issue, metadata, path = await reference_case( + db_session, + tmp_path, + source_type, + ) + before = path.read_bytes(), path.stat().st_mtime_ns + original_library_id = library.id + + await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + + target = await db_session.scalar(select(Issue).where(Issue.comicvine_id == 7001)) + assert target is not None, "Missing catalog targets must not strand already-imported comics" + assert library.issue_id == target.id == file.matched_issue_id + assert library.id == original_library_id == file.library_file_id + assert file.status is ImportedFileStatus.IMPORTED + assert target.status is IssueStatus.OWNED + assert wrong_issue.status is not IssueStatus.OWNED + assert (path.read_bytes(), path.stat().st_mtime_ns) == before + assert library.file_path == file.file_path == str(path) + assert (await db_session.get(Series, target.series_id)).path is None + assert await db_session.scalar(select(func.count(LibraryFile.id))) == 1 + assert file.diagnostics["completed_import_cleanup"]["source_preserved"] + assert file.diagnostics["completed_import_cleanup"]["source_issue_id"] == wrong_issue.id + group = await db_session.get(ImportedSeries, file.import_series_id) + assert group.status.value == "imported" + + job.status = ImportJobStatus.IMPORTING + job.progress_snapshot = {"deferred_recovery": {"state": "queued"}} + await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + assert await db_session.scalar(select(func.count(LibraryFile.id))) == 1 + assert await db_session.scalar(select(func.count(Issue.id))) == 2 + + +async def test_reference_candidates_make_existing_recheck_action_available(db_session, tmp_path): + job, *_ = await reference_case(db_session, tmp_path) + counts = await count_completed_import_cleanup_scope( + db_session, + job.id, + CompletedImportCleanupAction.RECHECK_DEFERRED_FILES, + ) + assert counts == (1, 1), "Recovery must remain available even without NO_MATCH rows" + + +@pytest.mark.parametrize( + "protection", + [ + "manual", + "parent_manual", + "managed", + "changed", + "missing", + "safety", + "ambiguous", + "wrong_year", + "wrong_type", + "conflicting_id", + "changed_registration", + ], +) +async def test_reference_recovery_preserves_unsafe_or_ambiguous_assignments( + db_session, + tmp_path, + protection, +): + job, file, library, issue, metadata, path = await reference_case(db_session, tmp_path) + if protection == "manual": + file.match_method = "manual_issue" + elif protection == "parent_manual": + from pullbox.models.import_job import ImportedSeries + + (await db_session.get(ImportedSeries, file.import_series_id)).user_selected_cv_id = 100 + elif protection == "managed": + library.storage_mode = LibraryFileStorageMode.MANAGED + elif protection == "changed": + path.write_bytes(b"a different comic") + elif protection == "missing": + path.unlink() + elif protection == "safety": + file.diagnostics = {**file.diagnostics, "safety_exception": {"approved": True}} + elif protection == "ambiguous": + first = metadata.search_catalog_series.return_value[0] + from dataclasses import replace + + metadata.search_catalog_series.return_value = [first, replace(first, provider_id="701")] + elif protection in {"wrong_year", "wrong_type"}: + from dataclasses import replace + + first = metadata.get_catalog_issue_summaries_for_series.return_value[0] + updates = ( + {"release_date": "1990-01-01"} if protection == "wrong_year" else {"issue_type": "tpb"} + ) + metadata.get_catalog_issue_summaries_for_series.return_value = [replace(first, **updates)] + elif protection == "conflicting_id": + file.comicvine_issue_id = 9001 + else: + library.source_signature = {**library.source_signature, "mtime_ns": 1} + await db_session.commit() + + await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + + assert library.issue_id == issue.id == file.matched_issue_id + assert await db_session.scalar(select(Issue).where(Issue.comicvine_id == 7001)) is None + + +async def test_recheck_preview_queues_reference_repair_without_changing_files(db_session, tmp_path): + from pullbox.models.import_job import ImportFileHandlingMode + from pullbox.models.user import User + + job, file, library, issue, metadata, path = await reference_case(db_session, tmp_path) + db_session.add(User(id=42, username="recovery-test", password_hash="unused")) + job.status = ImportJobStatus.COMPLETED + job.progress_snapshot = {} + job.file_handling_mode = ImportFileHandlingMode.IN_PLACE + job.move_to_library = False + await db_session.commit() + action = CompletedImportCleanupAction.RECHECK_DEFERRED_FILES + preview = await preview_completed_import_cleanup(db_session, job.id, action, actor_id=42) + assert preview.affected_file_count == 1 + result = await apply_completed_import_cleanup( + db_session, + job.id, + action, + actor_id=42, + preview_token=preview.preview_token, + ) + assert result.requires_import_retry + assert file.matched_issue_id == library.issue_id == issue.id + metadata.search_catalog_series.assert_not_awaited() + assert path.exists() + await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + assert file.matched_issue_id != issue.id + + +@pytest.mark.parametrize( + "existing", ["series_only", "unowned_issue", "owned_issue", "wrong_number"] +) +async def test_recovery_respects_existing_catalog_and_ownership(db_session, tmp_path, existing): + job, file, library, wrong_issue, metadata, _ = await reference_case(db_session, tmp_path) + target_series = Series( + title="Thunderbolts", + sort_title="thunderbolts", + comicvine_id=700, + description="Keep full metadata", + ) + db_session.add(target_series) + await db_session.flush() + target = None + if existing != "series_only": + target = Issue( + series_id=target_series.id, + comicvine_id=7001, + issue_number=105 if existing == "wrong_number" else 104, + issue_number_text="105" if existing == "wrong_number" else "104", + ) + db_session.add(target) + await db_session.flush() + if existing == "owned_issue": + from pullbox.models.library import LibraryRoot + + item = await db_session.get(ImportedSeries, file.import_series_id) + other = await add_file(db_session, job, item, file_path="/comics/owned.cbz") + other.status = ImportedFileStatus.IMPORTED + root = await db_session.get(LibraryRoot, library.library_root_id) + await register(db_session, other, target, root) + await db_session.commit() + await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + assert target_series.description == "Keep full metadata" + if existing in {"owned_issue", "wrong_number"}: + assert library.issue_id == wrong_issue.id + else: + target = await db_session.scalar(select(Issue).where(Issue.comicvine_id == 7001)) + assert library.issue_id == target.id + + +async def add_second_reference(session, job, file, library, tmp_path, *, number=105): + from pullbox.models.import_job import ImportedSeries + from pullbox.models.library import LibraryRoot + + directory = tmp_path / "another source folder" + directory.mkdir(exist_ok=True) + path = directory / f"Thunderbolts {number} (2021).cbz" + path.write_bytes(b"second comic") + item = await session.get(ImportedSeries, file.import_series_id) + root = await session.get(LibraryRoot, library.library_root_id) + issue = await session.get(Issue, library.issue_id) + other = await add_file( + session, + job, + item, + file_name=path.name, + file_path=str(path), + file_size=path.stat().st_size, + status=ImportedFileStatus.IMPORTED, + parsed_series="Thunderbolts", + parsed_issue_number=number, + comicvine_issue_id=None, + matched_issue_id=issue.id, + source_signature=build_file_identity_signature(path), + diagnostics=file.diagnostics, + ) + other_library = await register(session, other, issue, root) + other_library.storage_mode = LibraryFileStorageMode.REFERENCED + other.library_file_id = other_library.id + await session.commit() + return other, other_library + + +async def test_two_physical_files_for_one_catalog_target_remain_for_review(db_session, tmp_path): + job, file, library, wrong, metadata, _ = await reference_case(db_session, tmp_path) + other, other_library = await add_second_reference( + db_session, + job, + file, + library, + tmp_path, + number=104, + ) + await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + assert file.matched_issue_id == other.matched_issue_id == wrong.id + assert library.issue_id == other_library.issue_id == wrong.id + assert await db_session.scalar(select(func.count(LibraryFile.id))) == 2 + + +async def test_interrupted_repairs_resume_without_repeating_or_losing_completed_work( + db_session, + tmp_path, + monkeypatch, +): + from dataclasses import replace + + from pullbox.models.import_job import ImportJob + from pullbox.services import import_completed_cleanup as cleanup + + job, file, library, wrong, metadata, _ = await reference_case(db_session, tmp_path) + _other, other_library = await add_second_reference(db_session, job, file, library, tmp_path) + first = metadata.get_catalog_issue_summaries_for_series.return_value[0] + metadata.get_catalog_issue_summaries_for_series.return_value.append( + replace( + first, + provider_id="7002", + issue_number=105, + issue_number_text="105", + ) + ) + original = cleanup._apply_mixed_folder_resolutions + calls = 0 + + async def interrupted(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise RuntimeError("Simulated worker interruption") + return await original(*args, **kwargs) + + monkeypatch.setattr(cleanup, "_apply_mixed_folder_resolutions", interrupted) + job_id = job.id + with pytest.raises(RuntimeError, match="Simulated"): + await prepare_deferred_recovery(db_session, job_id, metadata_service=metadata) + await db_session.rollback() + job = await db_session.get(ImportJob, job_id) + assert job.progress_snapshot["deferred_recovery"]["reference_files_repaired"] == 1 + monkeypatch.setattr(cleanup, "_apply_mixed_folder_resolutions", original) + await prepare_deferred_recovery(db_session, job_id, metadata_service=metadata) + await db_session.refresh(library) + await db_session.refresh(other_library) + await db_session.refresh(wrong) + assert library.issue_id != wrong.id != other_library.issue_id + assert library.issue_id != other_library.issue_id + assert job.progress_snapshot["deferred_recovery"]["reference_files_repaired"] == 2 + metadata.search_catalog_series.assert_awaited_once() + assert job.status is ImportJobStatus.COMPLETED + + +async def test_catalog_pause_does_not_modify_source_and_resume_checks_current_evidence( + db_session, + tmp_path, +): + from pullbox.core.exceptions import JobPausedError, ProviderError + + job, file, library, wrong, metadata, path = await reference_case(db_session, tmp_path) + metadata.search_catalog_series.side_effect = ProviderError("catalog", "unavailable") + with pytest.raises(JobPausedError): + await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + assert file.matched_issue_id == library.issue_id == wrong.id + path.write_bytes(b"replaced while paused") + metadata.search_catalog_series.side_effect = None + await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + assert file.matched_issue_id == library.issue_id == wrong.id + + +async def test_catalog_lookup_does_not_hold_database_transaction(db_session, tmp_path): + job, _, _, _, metadata, _ = await reference_case(db_session, tmp_path) + results = metadata.search_catalog_series.return_value + + async def search(*args, **kwargs): + assert not db_session.in_transaction() + return results + + metadata.search_catalog_series.side_effect = search + events = [] + + async def progress(event): + events.append(event) + + await prepare_deferred_recovery( + db_session, + job.id, + metadata_service=metadata, + progress_callback=progress, + ) + assert any(event.current_file_progress_unit == "files" for event in events) + assert job.progress_snapshot["progress"] == 100 + + +async def test_recovery_accepts_reference_only_root_without_enabling_writes(db_session, tmp_path): + from pullbox.models.library import LibraryRoot + + job, file, library, wrong, metadata, path = await reference_case(db_session, tmp_path) + root = await db_session.get(LibraryRoot, library.library_root_id) + root.allow_managed_writes = False + path.chmod(0o444) + await db_session.commit() + await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + assert file.matched_issue_id != wrong.id + assert not root.allow_managed_writes + assert path.stat().st_mode & 0o777 == 0o444 + + +async def test_manual_decision_during_catalog_lookup_is_not_overridden(db_session, tmp_path): + job, file, library, wrong, metadata, _ = await reference_case(db_session, tmp_path) + results = metadata.search_catalog_series.return_value + + async def search(*args, **kwargs): + file.match_method = "manual_issue" + await db_session.commit() + return results + + metadata.search_catalog_series.side_effect = search + await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + assert file.matched_issue_id == library.issue_id == wrong.id + assert file.match_method == "manual_issue" + + +async def test_cancel_between_repairs_keeps_completed_reference_and_other_files( + db_session, + tmp_path, + monkeypatch, +): + from dataclasses import replace + + from pullbox.core.exceptions import JobCancelledError + from pullbox.services import import_completed_cleanup as cleanup + from pullbox.services.import_deferred_recovery_execution import cancel_deferred_preparation + + job, file, library, wrong, metadata, _ = await reference_case(db_session, tmp_path) + other, other_library = await add_second_reference(db_session, job, file, library, tmp_path) + first = metadata.get_catalog_issue_summaries_for_series.return_value[0] + metadata.get_catalog_issue_summaries_for_series.return_value.append( + replace( + first, + provider_id="7002", + issue_number=105, + issue_number_text="105", + ) + ) + original = cleanup._apply_mixed_folder_resolutions + calls = 0 + + async def cancel_on_second(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise JobCancelledError("cancelled") + return await original(*args, **kwargs) + + monkeypatch.setattr(cleanup, "_apply_mixed_folder_resolutions", cancel_on_second) + with pytest.raises(JobCancelledError): + await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + # The worker rolls back the interrupted unit, not earlier committed repairs. + await db_session.rollback() + for row in (job, file, library, wrong, other, other_library): + await db_session.refresh(row) + job.status = ImportJobStatus.CANCELLING + assert await cancel_deferred_preparation(db_session, job) + assert library.issue_id != wrong.id + assert other_library.issue_id == wrong.id + assert file.status is other.status is ImportedFileStatus.IMPORTED + assert job.status is ImportJobStatus.COMPLETED + + +async def test_recovery_does_not_turn_wrong_provisional_issue_into_wanted_download( + db_session, + tmp_path, +): + job, file, library, wrong, metadata, _ = await reference_case(db_session, tmp_path) + wrong.comicvine_id = None + wrong.metadata_source = "provisional_import" + file.matched_issue_cv_id = None + series = await db_session.get(Series, wrong.series_id) + series.monitored = True + await db_session.commit() + await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + assert library.issue_id != wrong.id + assert wrong.status is IssueStatus.SKIPPED + + +async def test_same_timestamp_manual_change_during_file_check_is_preserved( + db_session, + tmp_path, + monkeypatch, +): + from sqlalchemy import update + + from pullbox.models.import_job import ImportedFile + from pullbox.services import import_reference_recovery as recovery + + job, file, library, wrong, metadata, _ = await reference_case(db_session, tmp_path) + original_to_thread = recovery.asyncio.to_thread + + async def checked(func, *args, **kwargs): + result = await original_to_thread(func, *args, **kwargs) + if func is recovery._unchanged_source: + await db_session.execute( + update(ImportedFile) + .where(ImportedFile.id == file.id) + .values( + match_method="manual_issue", + updated_at=file.updated_at, + ) + ) + await db_session.commit() + return result + + monkeypatch.setattr(recovery.asyncio, "to_thread", checked) + await prepare_deferred_recovery(db_session, job.id, metadata_service=metadata) + assert file.match_method == "manual_issue" + assert file.matched_issue_id == library.issue_id == wrong.id From 628adb1bd204db249016f21282dee3f308b813df Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 20:50:05 -0700 Subject: [PATCH 17/20] chore(security): scope approved zlib exception to DHI refresh --- .grype.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.grype.yaml b/.grype.yaml index 0ef4aabf..77274d24 100644 --- a/.grype.yaml +++ b/.grype.yaml @@ -542,6 +542,12 @@ ignore: name: zlib1g version: 1:1.3.dfsg+really1.3.1-1+dhi3 type: deb + # Re-reviewed 2026-09-17 for the DHI package refresh; same exposure and deadline. + - vulnerability: CVE-2026-85091 + package: + name: zlib1g + version: 1:1.3.dfsg+really1.3.1-1+dhi4 + type: deb - vulnerability: CVE-2026-85091 package: name: zlib1g-dev From 6b26e93f6568d50a0fdc897617a1cddf44672bee Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 20:54:19 -0700 Subject: [PATCH 18/20] test(security): enforce the approved zlib package exception --- tests/unit/test_github_actions_security_contracts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/test_github_actions_security_contracts.py b/tests/unit/test_github_actions_security_contracts.py index 26126d57..4692ef6c 100644 --- a/tests/unit/test_github_actions_security_contracts.py +++ b/tests/unit/test_github_actions_security_contracts.py @@ -1062,6 +1062,8 @@ def test_grype_current_dhi_zlib_and_libuuid_exceptions_are_exact_and_expiring() "deb", ) for package in ("zlib1g", "zlib1g-dev") + } | { + ("CVE-2026-85091", "zlib1g", "1:1.3.dfsg+really1.3.1-1+dhi4", "deb"), } | { (cve, "libuuid1", version, "deb") for cve in reviewed_cves - {"CVE-2026-85091"} From ff6062eb43fba6b001a1b0ffef0b55fc5f7de227 Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 21:25:31 -0700 Subject: [PATCH 19/20] fix(metadata): finish restore sweeps and stop keyless continuations --- docs/development/ARCHITECTURE_OVERVIEW.md | 4 + .../services/restore_recovery_service.py | 26 ++++- src/pullbox/tasks/metadata_task.py | 9 +- tests/tasks/test_metadata_search_trigger.py | 100 ++++++++++++++++++ 4 files changed, 136 insertions(+), 3 deletions(-) diff --git a/docs/development/ARCHITECTURE_OVERVIEW.md b/docs/development/ARCHITECTURE_OVERVIEW.md index b81cb26b..a8c13d9b 100644 --- a/docs/development/ARCHITECTURE_OVERVIEW.md +++ b/docs/development/ARCHITECTURE_OVERVIEW.md @@ -565,6 +565,10 @@ checks a two-minute budget between series; an individual series has a separate after restart. Provider throttling pauses the sweep at its saved position instead of repeatedly failing every remaining series. Series metadata writes are committed before subsequent cover or issue-provider waits. +Removing the ComicVine key stops the active sweep and clears its continuation. +Post-restore aftercare keeps its recovery marker while continuation batches remain +active; it observes completion without retaining a database transaction between checks. +Cancellation leaves the marker and sweep checkpoint available for the next startup. ComicVine and Newznab clients share process-local account cooldowns, so creating a new client does not bypass a throttle response. Newznab also serializes request diff --git a/src/pullbox/services/restore_recovery_service.py b/src/pullbox/services/restore_recovery_service.py index c1013bde..6a42a2da 100644 --- a/src/pullbox/services/restore_recovery_service.py +++ b/src/pullbox/services/restore_recovery_service.py @@ -8,6 +8,7 @@ from __future__ import annotations +import asyncio import json from datetime import UTC, datetime from typing import TYPE_CHECKING, Any @@ -23,6 +24,7 @@ RESTORE_RECOVERY_MARKER_FILENAME = "restore_recovery_pending.json" RESTORE_RECOVERY_STATUS_FILENAME = "restore_recovery_status.json" +_SWEEP_POLL_SECONDS = 5.0 _STEP_DEFINITIONS = ( ("cover_backfill", "Backfill series cover cache"), @@ -153,17 +155,37 @@ async def _run_cover_backfill_step() -> str: async def _run_issue_sync_step() -> str: from pullbox.tasks.metadata_task import sync_new_issues - await sync_new_issues() + result = await sync_new_issues() + if result.status == "waiting": + await _wait_for_metadata_sweep("sync_new_issues") return "ComicVine issue catalog sync completed." async def _run_metadata_refresh_step() -> str: from pullbox.tasks.metadata_task import refresh_metadata - await refresh_metadata() + result = await refresh_metadata() + if result.status == "waiting": + await _wait_for_metadata_sweep("refresh_metadata") return "Series metadata refresh completed." +async def _wait_for_metadata_sweep(task_id: str) -> None: + """Observe scheduled continuation batches without holding a database transaction.""" + from pullbox.database import get_session_factory + from pullbox.tasks.metadata_sweep_state import load_sweep + + factory = get_session_factory() + while True: + async with factory() as session: + state = await load_sweep(session, task_id) + if not state.active: + return + # The scheduler owns remaining work and provider cooldowns. Cancellation + # leaves the restore marker and durable sweep checkpoint for startup. + await asyncio.sleep(_SWEEP_POLL_SECONDS) + + async def run_restore_recovery_if_pending( *, data_dir: Path | None = None, diff --git a/src/pullbox/tasks/metadata_task.py b/src/pullbox/tasks/metadata_task.py index eb480927..bc47378b 100644 --- a/src/pullbox/tasks/metadata_task.py +++ b/src/pullbox/tasks/metadata_task.py @@ -35,7 +35,7 @@ from pullbox.models.series import IssueCatalogState, Series, SeriesStatus from pullbox.providers.metadata.comicvine import ComicVineError, ComicVineProvider from pullbox.services.metadata_service import MetadataService -from pullbox.tasks.metadata_sweep_state import save_sweep, schedule_sweep, start_sweep +from pullbox.tasks.metadata_sweep_state import load_sweep, save_sweep, schedule_sweep, start_sweep logger = structlog.get_logger(__name__) @@ -283,6 +283,13 @@ async def _run_metadata_sweep(task_id: str) -> TaskExecutionResult: async with factory() as session: api_key = await get_comicvine_api_key(session) if not api_key: + state = await load_sweep(session, task_id) + if state.active: + state.active = False + state.retry_at = 0 + await save_sweep(session, task_id, state) + await session.commit() + schedule_sweep(task_id, state) log_deduped_warning( logger, f"{task_id}_missing_comicvine_key", diff --git a/tests/tasks/test_metadata_search_trigger.py b/tests/tasks/test_metadata_search_trigger.py index 17d6b629..bba4877d 100644 --- a/tests/tasks/test_metadata_search_trigger.py +++ b/tests/tasks/test_metadata_search_trigger.py @@ -37,6 +37,96 @@ _MOD = "pullbox.tasks.metadata_task" +@pytest.mark.parametrize("task_id", ["sync_new_issues", "refresh_metadata"]) +async def test_missing_key_stops_active_metadata_continuation(db_factory, task_id): + from pullbox.tasks import metadata_task + from pullbox.tasks.metadata_sweep_state import MetadataSweep, load_sweep, save_sweep + + async with db_factory() as session: + await save_sweep( + session, task_id, MetadataSweep(cursor=1, upper_bound=3, retry_at=100, active=True) + ) + await session.commit() + scheduler = _make_scheduler() + svc = _make_metadata_svc([]) + with ( + _sync_patches(db_factory, svc, scheduler), + patch(f"{_MOD}.get_comicvine_api_key", new_callable=AsyncMock, return_value=None), + ): + await getattr(metadata_task, task_id)() + async with db_factory() as session: + state = await load_sweep(session, task_id) + assert state.active is False, "A missing key must not leave a minute-by-minute continuation" + assert state.retry_at == 0 + scheduler.clear_task_continuation.assert_called_once_with(task_id) + scheduler.schedule_task_continuation.assert_not_called() + svc.fetch_series.assert_not_awaited() + + +@pytest.mark.parametrize("task_id", ["sync_new_issues", "refresh_metadata"]) +@pytest.mark.parametrize("restart", [False, True]) +async def test_restore_waits_for_remaining_metadata_batches( + restore_db_factory, tmp_path, monkeypatch, task_id, restart +): + from pullbox.services import restore_recovery_service as service + from pullbox.tasks import metadata_task + from pullbox.tasks.metadata_sweep_state import load_sweep + + db_factory = restore_db_factory + for identifier in (91001, 91002, 91003): + await _create_series(db_factory, comicvine_id=identifier) + monkeypatch.setattr(metadata_task, "_METADATA_BATCH_SIZE", 2) + monkeypatch.setattr( + service, "_run_cover_backfill_step", AsyncMock(return_value="Covers checked") + ) + other_step = ( + "_run_metadata_refresh_step" if task_id == "sync_new_issues" else "_run_issue_sync_step" + ) + monkeypatch.setattr(service, other_step, AsyncMock(return_value="Other step checked")) + monkeypatch.setattr("pullbox.database.get_session_factory", lambda: db_factory) + # The observer must await the scheduler's next batch, not run a second sweep itself. + monkeypatch.setattr(service, "_SWEEP_POLL_SECONDS", 0.01, raising=False) + svc = _make_metadata_svc([]) + service.mark_restore_recovery_pending("restore.zip", data_dir=tmp_path) + restore_task = None + try: + with _sync_patches(db_factory, svc, _make_scheduler()): + restore_task = asyncio.create_task( + service.run_restore_recovery_if_pending(data_dir=tmp_path) + ) + async with asyncio.timeout(3): + while True: + async with db_factory() as session: + state = await load_sweep(session, task_id) + if state.active and state.cursor == 2: + break + await asyncio.sleep(0.01) + await asyncio.sleep(0.03) + assert not restore_task.done(), "Restore must not finish with a pending metadata batch" + assert service.has_pending_restore_recovery(data_dir=tmp_path) + assert service.get_restore_recovery_status(data_dir=tmp_path)["status"] == "running" + if restart: + restore_task.cancel() + with pytest.raises(asyncio.CancelledError): + await restore_task + assert service.has_pending_restore_recovery(data_dir=tmp_path) + restore_task = asyncio.create_task( + service.run_restore_recovery_if_pending(data_dir=tmp_path) + ) + else: + await getattr(metadata_task, task_id)() + result = await asyncio.wait_for(restore_task, 3) + assert result["status"] == "completed" + assert not service.has_pending_restore_recovery(data_dir=tmp_path) + calls = svc.fetch_series if task_id == "sync_new_issues" else svc.refresh_series + assert calls.await_count == 3 + finally: + if restore_task is not None and not restore_task.done(): + restore_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await restore_task + + async def test_interrupted_sync_restarts_at_first_uncommitted_series(db_factory): from pullbox.tasks import metadata_task @@ -138,6 +228,16 @@ async def check_issue_fetch(session, series_id): # ── Fixtures ─────────────────────────────────────────────────────────── +@pytest.fixture +async def restore_db_factory(tmp_path): + # Cancellation may discard a connection; persisted restore state must survive it. + engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'restore.db'}") + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield async_sessionmaker(engine, expire_on_commit=False) + await engine.dispose() + + @pytest.fixture async def db_factory() -> AsyncGenerator[async_sessionmaker[AsyncSession], None]: engine = create_async_engine("sqlite+aiosqlite:///:memory:", echo=False) From 7e959bdf20d860a37698d599d4917430a8e1d9c6 Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Thu, 17 Sep 2026 21:42:22 -0700 Subject: [PATCH 20/20] test: tolerate CI startup latency in job progress regression --- tests/utilities/test_job_queue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/utilities/test_job_queue.py b/tests/utilities/test_job_queue.py index 9bd02553..47f860e0 100644 --- a/tests/utilities/test_job_queue.py +++ b/tests/utilities/test_job_queue.py @@ -1960,7 +1960,9 @@ def shutdown(self) -> None: dispatch_task = asyncio.create_task(mgr.dispatch_next()) try: - await asyncio.wait_for(first_result_persisted.wait(), timeout=0.5) + # The event barrier proves incremental progress, not a subsecond startup SLA. + await asyncio.wait_for(first_result_persisted.wait(), timeout=5.0) + assert not dispatch_task.done() await db_session.refresh(job) assert job.state == JobState.RUNNING