From 19f24505605d181081b621a45329861dd745bd6e Mon Sep 17 00:00:00 2001 From: ttlequals0 Date: Sat, 10 Oct 2026 04:35:16 -0400 Subject: [PATCH 1/5] Add duplicate filters and counts, fix scan reporting for 2.10.0 --- CHANGELOG.MD | 16 +++ README.md | 1 + app.py | 4 + celery_worker.py | 2 + docker-compose.yml | 4 +- docs/api.md | 15 +++ openapi.yaml | 23 +++++ package-lock.json | 6 +- pixelprobe/api/scan_routes.py | 18 +++- pixelprobe/api/stats_routes.py | 3 + pixelprobe/media_checker.py | 6 +- pixelprobe/services/duplicate_service.py | 56 +++++++++++ pixelprobe/services/stats_service.py | 7 +- pixelprobe/tasks_parallel.py | 7 +- pixelprobe/version.py | 2 +- static/css/desktop.css | 18 ++++ static/js/app.js | 37 +++++++ templates/index.html | 19 +++- tests/frontend_dom_test.js | 36 ++++++- .../test_production_security_matrix.py | 29 +++++- tests/test_duplicates.py | 98 +++++++++++++++++++ tests/test_read_timeout.py | 24 +++++ tests/test_script_startup.py | 49 ++++++++++ tests/unit/test_stats_service.py | 23 ++++- tests/unit/test_tasks_parallel.py | 2 + tests/unit/test_worker_logging.py | 36 +++++++ 26 files changed, 520 insertions(+), 21 deletions(-) create mode 100644 pixelprobe/services/duplicate_service.py create mode 100644 tests/test_duplicates.py create mode 100644 tests/test_script_startup.py create mode 100644 tests/unit/test_worker_logging.py diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 3753c40..3b5a302 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0). +## [2.10.0] - 2026-10-10 + +### Added + +- Duplicate file counts and a Duplicates filter for recorded SHA-256 matches or same-filename candidates ([#88](https://github.com/ttlequals0/PixelProbe/issues/88)). Counts include every group member across inventory paths, excluding known missing files. Desktop rows and mobile cards show group sizes. Content matching excludes files without hashes and does not verify changes since the recorded baseline. + +### Fixed + +- Storage read failures, including stale file handles wrapped by the reader, are recorded as unreadable rather than generic scanner errors. Completed worker chunks report unverified file counts separately from corrupted files. +- Application and worker logs use stdout so successful INFO messages are not labeled as errors merely because they were written to stderr. +- Starting with `python app.py` reuses the running application module instead of importing it again through route dependencies and creating duplicate application state. + +### Security + +- Updated the frontend build dependency source-map-js to 1.2.2 for GHSA-68fv-2mgg-jv7q, which fixes denial of service from malicious indexed source maps. + ## [2.9.5] - 2026-10-05 ### Fixed diff --git a/README.md b/README.md index 8dc47f4..0c0072e 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Detection - FFmpeg validation for video and audio, plus PIL and ImageMagick decoding for images - Warning and corruption verdicts kept separate - Rolling hashes that flag content changes without a matching modification-time change +- Duplicate file counts and filters for recorded SHA-256 matches or same-filename candidates. Filename matches may have different content; hashes do not verify changes since the recorded baseline. - Exclusions and per-finding mark-as-good overrides Scanning and recovery diff --git a/app.py b/app.py index 392c7e6..b1029a5 100644 --- a/app.py +++ b/app.py @@ -7,6 +7,9 @@ import sys import atexit import logging +if __name__ == '__main__': + sys.modules['app'] = sys.modules[__name__] + from datetime import datetime, timezone from flask import Flask, jsonify, send_file, render_template, request, redirect, url_for from flask_sqlalchemy import SQLAlchemy @@ -57,6 +60,7 @@ # Configure logging logging.basicConfig( level=logging.INFO, + stream=sys.stdout, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) logger = logging.getLogger(__name__) diff --git a/celery_worker.py b/celery_worker.py index 283abc1..add8229 100644 --- a/celery_worker.py +++ b/celery_worker.py @@ -30,6 +30,7 @@ # Configure logging logging.basicConfig( level=logging.INFO, + stream=sys.stdout, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) logger = logging.getLogger(__name__) @@ -67,6 +68,7 @@ def main(): rc = celery_app.worker_main([ 'worker', '--loglevel', log_level.lower(), + '--logfile', '/dev/stdout', '--concurrency', str(concurrency), '--queues', 'pixelprobe', '--hostname', f'pixelprobe-worker@%h', diff --git a/docker-compose.yml b/docker-compose.yml index e263a68..5fc2bb0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,7 @@ services: # PixelProbe application pixelprobe: - image: ttlequals0/pixelprobe:${PIXELPROBE_VERSION:-2.9.5} + image: ttlequals0/pixelprobe:${PIXELPROBE_VERSION:-2.10.0} container_name: pixelprobe-app environment: # Security @@ -167,7 +167,7 @@ services: # P1 Celery Worker for distributed task processing celery-worker: - image: ttlequals0/pixelprobe:${PIXELPROBE_VERSION:-2.9.5} + image: ttlequals0/pixelprobe:${PIXELPROBE_VERSION:-2.10.0} container_name: pixelprobe-celery-worker command: python celery_worker.py environment: diff --git a/docs/api.md b/docs/api.md index 5668fc2..302d0be 100644 --- a/docs/api.md +++ b/docs/api.md @@ -204,6 +204,7 @@ Get paginated scan results with optional filters. - `is_corrupted` (string): Filter by corruption: `all`, `true`, `false` - `has_warnings` (string): Filter by warning flag: `all`, `true`, `false` (`true` excludes corrupted and marked-as-good files) - `bitrot_suspected` (string): Filter by suspected bitrot: `all`, `true`, `false` +- `duplicate_mode` (string): `hash` for recorded SHA-256 matches, `name` for same-filename candidates, or `all` to disable (default). Invalid values return HTTP 400. - `search` (string): Case-insensitive substring match on file path - `path` (string): Restrict results to one configured scan path (must exactly match a configured path) - `sort_field` (string): Field to sort by (default: `scan_date`). Valid values: `scan_date`, `file_path`, `file_size`, `file_type`, `scan_status`, `status`, `is_corrupted`, `marked_as_good`, `scan_tool`, `corruption_details`, `discovered_date`, `last_modified`. `status` sorts by corruption status. Unknown values fall back to `scan_date` descending. @@ -242,6 +243,7 @@ Get paginated scan results with optional filters. Notes: - `media_info` is a JSON-encoded **string**, not a nested object - parse it client-side (e.g. `json.loads(result["media_info"])`). - Items also carry warning, bitrot, and integrity fields: `has_warnings`, `warning_details`, `bitrot_suspected`, `last_integrity_check_date`, and related hash/baseline columns. +- Duplicate filtering adds `duplicate_mode` and `duplicate_group_size` to each result. Groups span the inventory before path, search, and status filters apply. Only files recorded as existing are eligible. Content matching requires equal recorded hashes and sizes; suspected bitrot and failed or unreadable observations are excluded. Filename matching compares exact case-sensitive basenames. Matches use recorded baselines without verifying current contents; filename candidates may contain different content. #### Get single scan result ```http @@ -442,6 +444,19 @@ GET /api/stats Get overall statistics about scanned files. +Duplicate statistics use the same inventory groups as `duplicate_mode` filtering: + +| Field | Meaning | +|---|---| +| `duplicate_files` | All files in matching recorded SHA-256 groups | +| `duplicate_groups` | Number of matching content groups | +| `duplicate_extra_files` | Files beyond one per content group | +| `filename_duplicate_files` | All files in same-filename candidate groups | +| `filename_duplicate_groups` | Number of same-filename candidate groups | +| `filename_duplicate_extra_files` | Files beyond one per filename group | + +Counts include only files recorded as existing. Content counts exclude absent hashes or sizes, suspected bitrot, and failed or unreadable observations. Filename counts do not prove matching contents. No fresh filesystem verification runs for these queries. + **Response:** ```json { diff --git a/openapi.yaml b/openapi.yaml index 16fcfbf..f787b46 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -841,6 +841,20 @@ paths: in: query schema: type: string + - name: duplicate_mode + in: query + description: >- + Hash groups require files recorded as existing, nonempty recorded + SHA-256 hashes, and equal sizes. Suspected bitrot and failed or + unreadable observations are excluded. Name groups use exact + case-sensitive basenames and are candidates only. Groups span + the inventory before search and path filters. Each matching row + includes duplicate_mode and duplicate_group_size. Hashes describe + recorded baselines without verifying current contents. + schema: + type: string + enum: [all, hash, name] + default: all - name: sort_field in: query schema: @@ -1460,6 +1474,15 @@ paths: never-checked count, the oldest check date (every checked file has been verified at least once since then), and the current bitrot-suspected count. + `duplicate_files` counts all files in recorded hash-and-size groups; + `duplicate_groups` counts those groups and `duplicate_extra_files` + counts copies beyond one per group. `filename_duplicate_files`, + `filename_duplicate_groups`, and `filename_duplicate_extra_files` + use exact case-sensitive basenames and describe candidates, not + confirmed content matches. Missing files are excluded. Hash matching + also excludes suspected bitrot, failed or unreadable observations, + absent hashes, and absent sizes. Hashes describe recorded baselines, + not fresh filesystem verification. **Example:** ```bash diff --git a/package-lock.json b/package-lock.json index f0b7522..9f4be8a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2790,9 +2790,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", "dev": true, "license": "BSD-3-Clause", "engines": { diff --git a/pixelprobe/api/scan_routes.py b/pixelprobe/api/scan_routes.py index efdb07a..4105fc1 100644 --- a/pixelprobe/api/scan_routes.py +++ b/pixelprobe/api/scan_routes.py @@ -19,6 +19,7 @@ from pixelprobe.utils.helpers import get_configured_scan_paths from pixelprobe.utils.paths import like_prefix from pixelprobe.api.scan_launch import launch_directory_scan +from pixelprobe.services.duplicate_service import duplicate_members # Remove direct limiter imports as we'll use decorators logger = logging.getLogger(__name__) @@ -186,9 +187,15 @@ def get_scan_results(): sort_field = request.args.get('sort_field', 'scan_date') sort_order = request.args.get('sort_order', 'desc') path_filter = request.args.get('path', '').strip() + duplicate_mode = request.args.get('duplicate_mode', 'all') + if duplicate_mode not in ('all', 'hash', 'name'): + return {'error': 'duplicate_mode must be all, hash, or name'}, 400 # Build query query = ScanResult.query + if duplicate_mode != 'all': + members = duplicate_members(duplicate_mode) + query = query.join(members, ScanResult.id == members.c.file_id).add_columns(members.c.group_size) # Apply path filter -- validate against configured paths to prevent probing if path_filter: @@ -205,11 +212,11 @@ def get_scan_results(): # Apply status filter if scan_status != 'all': - query = query.filter_by(scan_status=scan_status) + query = query.filter(ScanResult.scan_status == scan_status) # Apply corruption filter if is_corrupted == 'true': - query = query.filter_by(is_corrupted=True).filter_by(marked_as_good=False) + query = query.filter(ScanResult.is_corrupted.is_(True), ScanResult.marked_as_good.is_(False)) elif is_corrupted == 'false': query = query.filter( (ScanResult.is_corrupted == False) | @@ -261,6 +268,7 @@ def get_scan_results(): else: # Default sorting query = query.order_by(ScanResult.scan_date.desc()) + query = query.order_by(ScanResult.id.asc()) # Paginate - handle -1 as "show all" if per_page == -1: @@ -278,8 +286,12 @@ def __init__(self, items, total): # Build response results = [] - for result in pagination.items: + for item in pagination.items: + result, group_size = item if duplicate_mode != 'all' else (item, None) result_dict = result.to_dict() + if duplicate_mode != 'all': + result_dict['duplicate_mode'] = duplicate_mode + result_dict['duplicate_group_size'] = group_size # Convert timestamps to configured timezone for display if result.scan_date: diff --git a/pixelprobe/api/stats_routes.py b/pixelprobe/api/stats_routes.py index f3bff17..001d42a 100644 --- a/pixelprobe/api/stats_routes.py +++ b/pixelprobe/api/stats_routes.py @@ -9,6 +9,7 @@ from pixelprobe.version import __version__ from pixelprobe.utils.timezone import from_utc_to_configured, get_configured_timezone_name from pixelprobe.services.stats_service import StatsService +from pixelprobe.services.duplicate_service import duplicate_statistics from pixelprobe.auth import auth_required logger = logging.getLogger(__name__) @@ -54,6 +55,7 @@ def get_stats(): # Rolling-integrity coverage: answers "how much of the library has # been verified over time" - no single budgeted run report can result['integrity'] = StatsService().get_integrity_coverage() + result.update(duplicate_statistics()) return result @@ -117,6 +119,7 @@ def get_stats(): 'healthy_files': healthy_files, 'marked_as_good': marked_as_good, 'warning_files': warning_files, + **duplicate_statistics(), 'integrity': StatsService().get_integrity_coverage() } except Exception as e2: diff --git a/pixelprobe/media_checker.py b/pixelprobe/media_checker.py index b81efbf..1b6487e 100644 --- a/pixelprobe/media_checker.py +++ b/pixelprobe/media_checker.py @@ -1408,8 +1408,12 @@ def scan_file(self, file_path, force_rescan=False): file_path, None, scan_start_time, 'unreadable', str(e)) except Exception as e: logger.error(f"Error scanning file {file_path}: {str(e)}") + cause = e + while cause.__cause__ is not None: + cause = cause.__cause__ return self._save_unverifiable_result( - file_path, None, scan_start_time, 'error', str(e)) + file_path, None, scan_start_time, + 'unreadable' if isinstance(cause, OSError) else 'error', str(e)) finally: if 'media_file' in locals() and media_file is not None: media_file.close() diff --git a/pixelprobe/services/duplicate_service.py b/pixelprobe/services/duplicate_service.py new file mode 100644 index 0000000..28a24c1 --- /dev/null +++ b/pixelprobe/services/duplicate_service.py @@ -0,0 +1,56 @@ +from sqlalchemy import func, and_, or_ + +from pixelprobe.models import db, ScanResult + + +def _duplicate_groups(mode): + path = ScanResult.file_path + name = (func.split_part(path, '/', -1) if db.engine.dialect.name == 'postgresql' + else func.substr(path, func.length(func.rtrim(path, func.replace(path, '/', ''))) + 1)) + keys = [ScanResult.file_hash, ScanResult.file_size] if mode == 'hash' else [name] + eligible = [ScanResult.file_exists.is_(True)] + if mode == 'hash': + eligible.extend([ + ScanResult.file_hash.isnot(None), + func.trim(ScanResult.file_hash) != '', + ScanResult.file_size.isnot(None), + ScanResult.bitrot_suspected.is_(False), + or_(ScanResult.scan_status.is_(None), + ScanResult.scan_status.notin_(['error', 'failed', 'unreadable'])), + or_(ScanResult.scan_tool.is_(None), ScanResult.scan_tool != 'error'), + or_(ScanResult.last_integrity_outcome.is_(None), + ScanResult.last_integrity_outcome.notin_(['error', 'unreadable'])), + ]) + else: + eligible.append(name != '') + groups = db.session.query( + *(key.label(f'key_{index}') for index, key in enumerate(keys)), + func.count().label('group_size'), + func.min(ScanResult.id).label('group_id'), + ).filter(*eligible).group_by(*keys).having(func.count() > 1).subquery() + return groups, keys, eligible + + +def duplicate_members(mode): + groups, keys, eligible = _duplicate_groups(mode) + return db.session.query( + ScanResult.id.label('file_id'), groups.c.group_size, groups.c.group_id, + ).join(groups, and_( + *(key == groups.c[f'key_{index}'] for index, key in enumerate(keys)) + )).filter(*eligible).subquery() + + +def duplicate_statistics(): + result = {} + for mode, prefix in [('hash', 'duplicate'), ('name', 'filename_duplicate')]: + groups, _, _ = _duplicate_groups(mode) + files, group_count = db.session.query( + func.coalesce(func.sum(groups.c.group_size), 0), func.count(), + ).select_from(groups).one() + files = int(files) + result.update({ + f'{prefix}_files': files, + f'{prefix}_groups': group_count, + f'{prefix}_extra_files': files - group_count, + }) + return result diff --git a/pixelprobe/services/stats_service.py b/pixelprobe/services/stats_service.py index 1a7a5c0..a99d519 100644 --- a/pixelprobe/services/stats_service.py +++ b/pixelprobe/services/stats_service.py @@ -11,6 +11,7 @@ from pixelprobe.models import db, ScanResult, ScanReport from pixelprobe.utils.timezone import from_utc_to_configured, get_configured_timezone from pixelprobe.version import __version__ +from pixelprobe.services.duplicate_service import duplicate_statistics logger = logging.getLogger(__name__) @@ -49,7 +50,8 @@ def get_file_statistics(self) -> Dict: 'corrupted_files': stats[5] or 0, 'healthy_files': stats[6] or 0, 'marked_as_good': stats[7] or 0, - 'warning_files': stats[8] or 0 + 'warning_files': stats[8] or 0, + **duplicate_statistics(), } except Exception as e: @@ -230,7 +232,8 @@ def _get_stats_fallback(self) -> Dict: 'corrupted_files': corrupted_files, 'healthy_files': healthy_files, 'marked_as_good': marked_as_good, - 'warning_files': warning_files + 'warning_files': warning_files, + **duplicate_statistics(), } except Exception as e: logger.error(f"Fallback stats query also failed: {e}") diff --git a/pixelprobe/tasks_parallel.py b/pixelprobe/tasks_parallel.py index 4e677aa..a51a5d4 100644 --- a/pixelprobe/tasks_parallel.py +++ b/pixelprobe/tasks_parallel.py @@ -562,6 +562,7 @@ def claim_pending(): files_processed = 0 files_corrupted = 0 + files_unverified = 0 last_progress_write = time.time() for batch_ids in batch_process(claimed_ids, _CHUNK_COMMIT_BATCH): @@ -691,10 +692,13 @@ def claim_pending(): # Commit each post-decode result while the persistence # fence is held. Never keep that lock across the next IO. db.session.commit() + if not scan_result or result_outcome != 'completed': + files_unverified += 1 files_processed += 1 current_file = file_path except Exception as e: + files_unverified += 1 logger.error(f"Error scanning {file_path} in chunk {chunk_db_id}: {e}") member = _lock_chunk_result_write( scan_id, chunk_db_id, self.request.id, db_result.id) @@ -765,7 +769,7 @@ def claim_pending(): files_scanned=base_scanned + files_processed) logger.info(f"Chunk {chunk_db_id} completed: {files_processed} files " f"this attempt ({base_scanned + files_processed} total), " - f"{files_corrupted} corrupted") + f"{files_corrupted} corrupted, {files_unverified} unverified") maybe_finalize_scan(scan_id) _mark_task_finished(scan_id, self.request.id) @@ -774,6 +778,7 @@ def claim_pending(): 'chunk_id': chunk_db_id, 'files_processed': files_processed, 'files_corrupted': files_corrupted, + 'files_unverified': files_unverified, 'completed_at': datetime.now(timezone.utc).isoformat() } diff --git a/pixelprobe/version.py b/pixelprobe/version.py index 88fac26..6b77d2b 100644 --- a/pixelprobe/version.py +++ b/pixelprobe/version.py @@ -4,7 +4,7 @@ # Default version - this is the single source of truth -_DEFAULT_VERSION = '2.9.5' +_DEFAULT_VERSION = '2.10.0' # Allow override via environment variable for CI/CD, but default to the hardcoded version diff --git a/static/css/desktop.css b/static/css/desktop.css index aed693d..738cc0f 100644 --- a/static/css/desktop.css +++ b/static/css/desktop.css @@ -346,6 +346,24 @@ body { } /* Stats grid */ +#duplicate-controls { + margin-bottom: 1rem; +} + +#duplicate-mode { + width: 100%; + max-width: 22rem; + margin: 0.5rem 0; +} + +.duplicate-label { + display: block; + color: var(--text-secondary); + font-size: 0.8rem; + white-space: normal; + margin-top: 0.35rem; +} + .stats-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); diff --git a/static/js/app.js b/static/js/app.js index 195f7fc..144aaec 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -430,6 +430,11 @@ class StatsDashboard { this.updateStatCard('warning-files', stats.warning_files || 0); this.updateStatCard('bitrot-files', (stats.integrity && stats.integrity.bitrot_suspected) || 0); this.updateStatCard('pending-files', stats.pending_files); + this.updateStatCard('duplicate-files', stats.duplicate_files || 0); + const duplicateDetails = document.querySelector('#duplicate-details'); + const duplicateGroups = stats.duplicate_groups || 0; + const extraFiles = stats.duplicate_extra_files || 0; + if (duplicateDetails) duplicateDetails.textContent = `${duplicateGroups} ${duplicateGroups === 1 ? 'group' : 'groups'}; ${extraFiles} extra ${extraFiles === 1 ? 'file' : 'files'}`; this.updateStatCard('scanning-files', stats.scanning_files); this.lastIntegrity = stats.integrity || null; this.renderIntegrityCoverage(stats.integrity); @@ -1349,6 +1354,7 @@ class TableManager { this.sortField = 'scan_date'; this.sortOrder = 'desc'; this.filter = 'all'; + this.duplicateMode = 'hash'; this.searchQuery = ''; this.pathFilter = ''; this.selectedFiles = new Set(); @@ -1423,11 +1429,30 @@ class TableManager { this.updateSelectionUI(); this.filter = e.target.dataset.filter; + const duplicateControls = document.querySelector('#duplicate-controls'); + if (duplicateControls) duplicateControls.hidden = this.filter !== 'duplicates'; this.currentPage = 1; this.loadData(); }); }); + const duplicateMode = document.querySelector('#duplicate-mode'); + if (duplicateMode) { + duplicateMode.addEventListener('change', (event) => { + this.duplicateMode = event.target.value; + const help = document.querySelector('#duplicate-help'); + if (help) help.textContent = this.duplicateMode === 'name' + ? 'Matching filenames are candidates only; their contents may differ. Groups span all inventory paths.' + : 'Matches use recorded hash baselines. Files without hashes are excluded; changes since hashing are not verified.'; + this.selectedFiles.clear(); + const selectAll = document.querySelector('#select-all'); + if (selectAll) selectAll.checked = false; + this.updateSelectionUI(); + this.currentPage = 1; + this.loadData(); + }); + } + // Path filter const pathFilter = document.querySelector('#path-filter'); if (pathFilter) { @@ -1503,6 +1528,9 @@ class TableManager { // Map frontend filter values to backend parameters if (this.filter) { switch (this.filter) { + case 'duplicates': + params.duplicate_mode = this.duplicateMode; + break; case 'corrupted': params.is_corrupted = 'true'; break; @@ -1598,6 +1626,7 @@ class TableManager { details.className = 'file-details'; this.appendFileDetails(details, file); card.append(status, path, info, details, this.createFileActions(file, false), this.createFileCheckbox(file)); + if (file.duplicate_group_size > 1) path.appendChild(this.createDuplicateLabel(file)); return card; } @@ -1614,12 +1643,20 @@ class TableManager { else { cell.textContent = value; if (index === 2 || index === 6) { cell.title = value; cell.className = index === 2 ? 'file-path-cell' : 'text-truncate'; } } row.appendChild(cell); }); + if (file.duplicate_group_size > 1) row.children[2].appendChild(this.createDuplicateLabel(file)); const actions = document.createElement('td'); actions.appendChild(this.createFileActions(file, true)); row.appendChild(actions); return row; } + createDuplicateLabel(file) { + const label = document.createElement('span'); + label.className = 'duplicate-label'; + label.textContent = `${file.duplicate_group_size} files: ${file.duplicate_mode === 'name' ? 'same filename candidates' : 'same recorded content'}`; + return label; + } + createFileCheckbox(file) { const control = document.createElement('label'); control.className = 'file-checkbox-control'; diff --git a/templates/index.html b/templates/index.html index 8984fe6..cdf0d7a 100644 --- a/templates/index.html +++ b/templates/index.html @@ -178,7 +178,7 @@

Scan Progress

+ corrupted, warnings, bitrot, healthy, pending, duplicates -->
0
@@ -204,6 +204,12 @@

Scan Progress

0
Pending Files
+
+
0
+
Duplicate Files
+
+
All files in matching recorded SHA-256 groups.
+
-
Integrity Rechecks
@@ -244,6 +250,9 @@

Scan Progress

+
@@ -258,6 +267,14 @@

Scan Progress

+
diff --git a/tests/frontend_dom_test.js b/tests/frontend_dom_test.js index ce8a437..8a287f0 100644 --- a/tests/frontend_dom_test.js +++ b/tests/frontend_dom_test.js @@ -37,6 +37,36 @@ const statusCases = [ [{ scan_status: 'completed', has_warnings: true }, 'warning', 'Warning'], [{ scan_status: 'completed', is_corrupted: true }, 'danger', 'Corrupted'] ]; +for (const mode of ['hash', 'name']) { + const file = { id: 101, file_path: payload, scan_status: 'completed', duplicate_group_size: 3, duplicate_mode: mode }; + const expected = `3 files: ${mode === 'name' ? 'same filename candidates' : 'same recorded content'}`; + assert.equal(table.renderRow(file).querySelector('.duplicate-label').textContent, expected); + assert.equal(table.renderMobileCard(file).querySelector('.duplicate-label').textContent, expected); + assert.equal(table.renderMobileCard(file).querySelector('img'), null); +} +assert.equal(row.querySelector('.duplicate-label'), null); + +const duplicateControls = window.document.createElement('div'); +duplicateControls.innerHTML = ''; +window.document.body.appendChild(duplicateControls); +const requestedDuplicateModes = []; +const duplicateTable = new window.__TableManager({ getScanResults: async params => { + requestedDuplicateModes.push(params.duplicate_mode); + return { results: [], total: 0 }; +} }); +duplicateTable.bindEvents(); +window.document.querySelector('[data-filter="duplicates"]').click(); +assert.equal(window.document.querySelector('#duplicate-controls').hidden, false); +assert.equal(requestedDuplicateModes.at(-1), 'hash'); +const duplicateSelect = window.document.querySelector('#duplicate-mode'); +duplicateSelect.value = 'name'; +duplicateSelect.dispatchEvent(new window.Event('change')); +assert.equal(requestedDuplicateModes.at(-1), 'name'); +assert.match(window.document.querySelector('#duplicate-help').textContent, /contents may differ/); +window.document.querySelector('[data-filter="all"]').click(); +assert.equal(window.document.querySelector('#duplicate-controls').hidden, true); +assert.equal(requestedDuplicateModes.at(-1), undefined); + for (const [statusFile, statusClass, statusText] of statusCases) { const file = { id: 100, file_path: payload, file_size: 0, ...statusFile }; const desktopStatus = table.renderRow(file).children[1].querySelector('.badge'); @@ -194,6 +224,7 @@ async function verifyViewer() {
+
`, { runScripts: 'outside-only' }); @@ -217,7 +248,8 @@ async function verifyViewer() { getStats: async () => { if (shouldFail) throw new Error('network'); return { total_files: 8, completed_files: 5, healthy_files: 3, corrupted_files: 1, warning_files: 0, - pending_files: 0, scanning_files: 0, integrity: { total_files: 5, checked_percent: 40, + pending_files: 0, scanning_files: 0, duplicate_files: 2, duplicate_groups: 1, duplicate_extra_files: 1, + integrity: { total_files: 5, checked_percent: 40, attempted_files: 3, checked_files: 2, integrity_error_files: 1, integrity_unavailable_files: 1, never_attempted: 2, bitrot_suspected: 0 } }; } @@ -228,6 +260,8 @@ async function verifyViewer() { assert.equal(statsWindow.document.querySelector('#integrity-detail-panel').hidden, false); assert.equal(statsWindow.document.querySelector('#integrity-details-toggle').getAttribute('aria-expanded'), 'true'); assert.equal(statsWindow.document.querySelector('#total-files').textContent, '8'); + assert.equal(statsWindow.document.querySelector('#duplicate-files').textContent, '2'); + assert.equal(statsWindow.document.querySelector('#duplicate-details').textContent, '1 group; 1 extra file'); assert.match(statsWindow.document.querySelector('#integrity-details').textContent, /Successful integrity rechecks 2/); assert.match(statsWindow.document.querySelector('#integrity-details').textContent, /with no recorded recheck attempt 2/); assert.match(statsWindow.document.querySelector('#integrity-details').textContent, /Legacy integrity outcomes were not recorded/); diff --git a/tests/integration/test_production_security_matrix.py b/tests/integration/test_production_security_matrix.py index a839a4f..950a1d3 100644 --- a/tests/integration/test_production_security_matrix.py +++ b/tests/integration/test_production_security_matrix.py @@ -23,10 +23,12 @@ def test_production_security_matrix(): root = Path(__file__).parents[2] script = r''' +import atexit import os import re import subprocess import sys +import threading import app as app_module from pixelprobe.models import APIToken, SecurityAuditEvent, User, db @@ -56,8 +58,25 @@ def test_production_security_matrix(): child_env = os.environ | {'MATRIX_ADMIN_USERNAME': admin_username} child = subprocess.Popen([sys.executable, '-c', child_rate_test], env=child_env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, text=True) -assert child.stdout.readline().strip() == 'READY' + stderr=subprocess.STDOUT, text=True) +child_ready = threading.Event() +child_output = [] + +def drain_child_logs(): + for line in child.stdout: + child_output.append(line) + if line.strip() == 'READY': + child_ready.set() + +def cleanup_child(): + if child.poll() is None: + child.kill() + child.wait(timeout=10) + +atexit.register(cleanup_child) +child_reader = threading.Thread(target=drain_child_logs, daemon=True) +child_reader.start() +assert child_ready.wait(timeout=20), ''.join(child_output) with app.app_context(): admin = User(username=admin_username, email=f'{admin_username}@example.test', is_admin=True) @@ -241,8 +260,10 @@ def csrf(client, remote=None): assert response.status_code == 401 child.stdin.write('run\n') child.stdin.flush() -child_stdout, child_stderr = child.communicate(timeout=30) -assert child.returncode == 0, child_stdout + child_stderr +child.stdin.close() +child.wait(timeout=30) +child_reader.join(timeout=5) +assert child.returncode == 0, ''.join(child_output) client = app.test_client() response = client.post('/api/auth/login', base_url='https://localhost', json={'username': admin_username, 'password': 'wrong'}, diff --git a/tests/test_duplicates.py b/tests/test_duplicates.py new file mode 100644 index 0000000..1cce5fd --- /dev/null +++ b/tests/test_duplicates.py @@ -0,0 +1,98 @@ +import os +from uuid import uuid4 + +import pytest +from flask import Flask +from sqlalchemy import create_engine, text + +from pixelprobe.models import db as model_db, ScanResult +from pixelprobe.services.duplicate_service import duplicate_members, duplicate_statistics +from pixelprobe.services.stats_service import StatsService + + +def seed_duplicates(session): + rows = [ + ScanResult(file_path='/media/a/same.mp4', file_hash='a' * 64, file_size=10), + ScanResult(file_path='/media/b/same.mp4', file_hash='a' * 64, file_size=10), + ScanResult(file_path='/other/copy.mp4', file_hash='a' * 64, file_size=10), + ScanResult(file_path='/media/different-size.mp4', file_hash='a' * 64, file_size=11), + ScanResult(file_path='/media/missing/same.mp4', file_hash='a' * 64, file_size=10, file_exists=False), + ScanResult(file_path='/media/suspect.mp4', file_hash='a' * 64, file_size=10, bitrot_suspected=True), + ScanResult(file_path='/media/error.mp4', file_hash='a' * 64, file_size=10, last_integrity_outcome='error'), + ScanResult(file_path='/media/unreadable.mp4', file_hash='a' * 64, file_size=10, last_integrity_outcome='unreadable'), + ScanResult(file_path='/media/failed-scan.mp4', file_hash='a' * 64, file_size=10, scan_status='failed'), + ScanResult(file_path='/media/legacy-error.mp4', file_hash='a' * 64, file_size=10, scan_status='completed', scan_tool='error'), + ScanResult(file_path='/media/blank.mp4', file_hash=' ', file_size=10), + ScanResult(file_path='/media/no-hash/same.mp4', file_size=10), + ScanResult(file_path='/media/Same.mp4', file_size=10), + ] + session.add_all(rows) + session.commit() + return rows + + +def assert_grouping(session): + seed_duplicates(session) + stats = duplicate_statistics() + assert stats == { + 'duplicate_files': 3, 'duplicate_groups': 1, 'duplicate_extra_files': 2, + 'filename_duplicate_files': 3, 'filename_duplicate_groups': 1, + 'filename_duplicate_extra_files': 2, + } + for mode in ('hash', 'name'): + members = duplicate_members(mode) + assert session.query(members).count() == 3 + assert {row.group_size for row in session.query(members)} == {3} + + +def test_duplicate_eligibility_and_count_semantics(db): + assert_grouping(db.session) + assert StatsService().get_file_statistics()['duplicate_files'] == 3 + + +def test_duplicate_api_global_groups_pagination_and_stats(db, authenticated_client, monkeypatch): + seed_duplicates(db.session) + monkeypatch.setattr('pixelprobe.api.scan_routes.get_configured_scan_paths', lambda: ['/media']) + response = authenticated_client.get('/api/scan-results?duplicate_mode=hash&path=/media&per_page=1&sort_field=file_path&sort_order=asc') + assert response.status_code == 200 + data = response.get_json() + assert data['total'] == 2 + assert data['pages'] == 2 + assert data['results'][0]['duplicate_group_size'] == 3 + assert data['results'][0]['duplicate_mode'] == 'hash' + second = authenticated_client.get('/api/scan-results?duplicate_mode=hash&path=/media&per_page=1&page=2&sort_field=file_path&sort_order=asc').get_json() + assert second['results'][0]['id'] != data['results'][0]['id'] + named = authenticated_client.get('/api/scan-results?duplicate_mode=name&search=no-hash').get_json() + assert named['total'] == 1 + assert named['results'][0]['duplicate_group_size'] == 3 + filtered = authenticated_client.get('/api/scan-results?duplicate_mode=hash&scan_status=pending&is_corrupted=true') + assert filtered.status_code == 200 + assert filtered.get_json()['total'] == 0 + stats = authenticated_client.get('/api/stats').get_json() + assert stats['duplicate_files'] == 3 + assert stats['filename_duplicate_files'] == 3 + assert authenticated_client.get('/api/scan-results?duplicate_mode=visual').status_code == 400 + + +@pytest.mark.postgres +@pytest.mark.skipif(not os.environ.get('PIXELPROBE_TEST_POSTGRES_URI'), reason='PIXELPROBE_TEST_POSTGRES_URI not set') +def test_postgres_duplicate_basename_and_grouping(): + uri = os.environ['PIXELPROBE_TEST_POSTGRES_URI'] + schema = f'duplicates_{uuid4().hex[:12]}' + engine = create_engine(uri) + with engine.begin() as connection: + connection.execute(text(f'CREATE SCHEMA {schema}')) + app = Flask(__name__) + app.config.update(SQLALCHEMY_DATABASE_URI=uri, + SQLALCHEMY_ENGINE_OPTIONS={'connect_args': {'options': f'-csearch_path={schema}'}}) + model_db.init_app(app) + try: + with app.app_context(): + model_db.create_all() + assert_grouping(model_db.session) + model_db.session.remove() + model_db.engine.dispose() + finally: + with engine.begin() as connection: + connection.execute(text(f'DROP SCHEMA {schema} CASCADE')) + engine.dispose() diff --git a/tests/test_read_timeout.py b/tests/test_read_timeout.py index e36264f..96f5e25 100644 --- a/tests/test_read_timeout.py +++ b/tests/test_read_timeout.py @@ -3,6 +3,7 @@ pure-Python reads with no subprocess timeout to save them, so a watchdog deadline skips the file and lets the scan continue.""" import hashlib +import errno import os import threading import time @@ -126,6 +127,29 @@ def test_descriptor_path_reports_real_mime_and_keeps_offset(self, tmp_path): class TestScanFileSkipsUnreadableFile: + @pytest.mark.parametrize('wrapped', [False, True]) + def test_stale_handle_is_unreadable_without_retry(self, tmp_path, monkeypatch, wrapped): + path = tmp_path / 'stale.mkv' + path.write_bytes(b'content') + checker = PixelProbe(database_path=None) + calls = [] + + def stale_identity(_path): + calls.append(_path) + try: + raise OSError(errno.ESTALE, 'Stale file handle') + except OSError as exc: + if wrapped: + raise ValueError('File changed or is unavailable') from exc + raise + + monkeypatch.setattr(checker, '_file_identity', stale_identity) + result = checker.scan_file(str(path)) + assert result['outcome'] == 'unreadable' + assert result['is_corrupted'] is None + assert result['file_hash'] is None + assert len(calls) == 1 + def test_read_timeout_marks_file_and_continues(self, tmp_path, monkeypatch): f = tmp_path / 'poison.mkv' f.write_bytes(b'x' * 1024) diff --git a/tests/test_script_startup.py b/tests/test_script_startup.py new file mode 100644 index 0000000..56af260 --- /dev/null +++ b/tests/test_script_startup.py @@ -0,0 +1,49 @@ +import os +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +import pytest +from sqlalchemy.engine import make_url + + +@pytest.mark.skipif(not os.environ.get('PIXELPROBE_TEST_POSTGRES_URI'), + reason='PIXELPROBE_TEST_POSTGRES_URI not set') +def test_documented_script_starts_with_postgres(tmp_path): + database = make_url(os.environ['PIXELPROBE_TEST_POSTGRES_URI']) + with socket.socket() as listener: + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + env = dict(os.environ, + POSTGRES_HOST=database.host, + POSTGRES_PORT=str(database.port or 5432), + POSTGRES_DB=database.database, + POSTGRES_USER=database.username, + POSTGRES_PASSWORD=database.password or '', + DATABASE_URL=os.environ['PIXELPROBE_TEST_POSTGRES_URI'], + FLASK_ENV='production', + SECRET_KEY='script-startup-test', + SCHEDULER_ENABLED='false', DEBUG='false', PORT=str(port)) + root = Path(__file__).resolve().parents[1] + log_path = tmp_path / 'startup.log' + with log_path.open('w') as output: + process = subprocess.Popen([sys.executable, 'app.py'], cwd=root, + env=env, stdout=output, stderr=output) + try: + deadline = time.monotonic() + 30 + while time.monotonic() < deadline and process.poll() is None: + try: + with urllib.request.urlopen(f'http://127.0.0.1:{port}/', timeout=1) as response: + assert response.status == 200 + return + except urllib.error.URLError: + time.sleep(0.1) + pytest.fail(log_path.read_text()) + finally: + if process.poll() is None: + process.terminate() + process.wait(timeout=10) diff --git a/tests/unit/test_stats_service.py b/tests/unit/test_stats_service.py index 65acdaa..a00e102 100644 --- a/tests/unit/test_stats_service.py +++ b/tests/unit/test_stats_service.py @@ -15,9 +15,22 @@ class TestStatsService: def stats_service(self): """Create a stats service instance""" return StatsService() + + @pytest.fixture + def duplicate_stats(self): + counts = { + 'duplicate_files': 6, + 'duplicate_groups': 2, + 'duplicate_extra_files': 4, + 'filename_duplicate_files': 9, + 'filename_duplicate_groups': 3, + 'filename_duplicate_extra_files': 6, + } + with patch('pixelprobe.services.stats_service.duplicate_statistics', return_value=counts) as mocked: + yield counts, mocked @patch('pixelprobe.services.stats_service.db') - def test_get_file_statistics_success(self, mock_db, stats_service): + def test_get_file_statistics_success(self, mock_db, stats_service, duplicate_stats): """Test successful file statistics retrieval""" # Mock database result mock_result = MagicMock() @@ -36,10 +49,13 @@ def test_get_file_statistics_success(self, mock_db, stats_service): assert stats['healthy_files'] == 85 assert stats['marked_as_good'] == 3 assert stats['warning_files'] == 2 + counts, mocked = duplicate_stats + assert {key: stats[key] for key in counts} == counts + mocked.assert_called_once_with() @patch('pixelprobe.services.stats_service.db') @patch('pixelprobe.services.stats_service.ScanResult') - def test_get_file_statistics_fallback(self, mock_scan_result, mock_db, stats_service): + def test_get_file_statistics_fallback(self, mock_scan_result, mock_db, stats_service, duplicate_stats): """Test fallback when optimized query fails""" # Make optimized query fail mock_db.session.execute.side_effect = Exception("DB Error") @@ -53,6 +69,9 @@ def test_get_file_statistics_fallback(self, mock_scan_result, mock_db, stats_ser assert stats['total_files'] == 100 assert stats['completed_files'] == 80 + counts, mocked = duplicate_stats + assert {key: stats[key] for key in counts} == counts + mocked.assert_called_once_with() @patch('pixelprobe.services.stats_service.db') def test_get_system_info(self, mock_db, stats_service): diff --git a/tests/unit/test_tasks_parallel.py b/tests/unit/test_tasks_parallel.py index 55e9c95..1e9701d 100644 --- a/tests/unit/test_tasks_parallel.py +++ b/tests/unit/test_tasks_parallel.py @@ -346,6 +346,8 @@ def test_unreadable_result_is_not_media_corruption(self, tp, app, db, tmp_path): result = tp.process_chunk_task.apply( args=(chunk.id, 'scan-unreadable')).get() assert result['status'] == 'SUCCESS' + assert result['files_unverified'] == 1 + assert result['files_corrupted'] == 0 row = ScanResult.query.filter_by(file_path=file_path).first() member = ScanRunFile.query.filter_by( scan_id='scan-unreadable', file_path=file_path).first() diff --git a/tests/unit/test_worker_logging.py b/tests/unit/test_worker_logging.py new file mode 100644 index 0000000..715dffe --- /dev/null +++ b/tests/unit/test_worker_logging.py @@ -0,0 +1,36 @@ +import logging +import runpy +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from celery.app.log import Logging + + +def test_worker_logs_use_stdout(monkeypatch, capfd): + worker = Mock() + worker.worker_main.return_value = 0 + monkeypatch.setitem(sys.modules, 'app', SimpleNamespace(celery=worker)) + configure = Mock() + monkeypatch.setattr(logging, 'basicConfig', configure) + entrypoint = Path(__file__).resolve().parents[2] / 'celery_worker.py' + namespace = runpy.run_path(str(entrypoint)) + with pytest.raises(SystemExit) as stopped: + namespace['main']() + assert stopped.value.code == 0 + + assert configure.call_args.kwargs['stream'] is sys.stdout + argv = worker.worker_main.call_args.args[0] + destination = argv[argv.index('--logfile') + 1] + assert destination == '/dev/stdout' + handler = Logging._detect_handler(None, destination) + try: + handler.emit(logging.LogRecord('worker', logging.INFO, '', 0, + 'stdout regression', (), None)) + captured = capfd.readouterr() + assert 'stdout regression' in captured.out + assert 'stdout regression' not in captured.err + finally: + handler.close() From 552115791e318d6183c363a96da8a884037dd0b8 Mon Sep 17 00:00:00 2001 From: ttlequals0 Date: Sat, 10 Oct 2026 05:00:41 -0400 Subject: [PATCH 2/5] Update Werkzeug security fix for 2.10.0 --- CHANGELOG.MD | 1 + requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 3b5a302..fac68a8 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security - Updated the frontend build dependency source-map-js to 1.2.2 for GHSA-68fv-2mgg-jv7q, which fixes denial of service from malicious indexed source maps. +- Updated Werkzeug to 3.1.9 for CVE-2026-102598. On Windows, `safe_join` rejects NTFS special-device names with empty alternate data stream markers. ## [2.9.5] - 2026-10-05 diff --git a/requirements.txt b/requirements.txt index 238feaa..e11f6eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ python-magic==0.4.27 ffmpeg-python==0.2.0 SQLAlchemy==2.0.41 python-dotenv==1.2.2 -Werkzeug==3.1.6 +Werkzeug==3.1.9 gunicorn==23.0.0 pytz==2023.3 APScheduler==3.11.0 From ee7004b631fd2dfdf8c375fc66e531e124450cf1 Mon Sep 17 00:00:00 2001 From: ttlequals0 Date: Sat, 10 Oct 2026 08:21:26 -0400 Subject: [PATCH 3/5] Persist duplicate summaries and responsive results for 2.10.1 --- CHANGELOG.MD | 9 + docs/api.md | 30 ++ openapi.yaml | 88 +++- pixelprobe/api/scan_routes.py | 23 +- pixelprobe/api/stats_routes.py | 10 +- pixelprobe/migrations/duplicates.py | 174 ++++++++ pixelprobe/migrations/startup.py | 4 + pixelprobe/models.py | 44 ++ pixelprobe/scheduler.py | 27 +- pixelprobe/services/duplicate_service.py | 369 +++++++++++++++-- pixelprobe/services/stats_service.py | 13 +- pixelprobe/version.py | 2 +- static/css/desktop.css | 8 + static/js/app.js | 276 ++++++++++--- templates/index.html | 5 +- tests/frontend_dom_test.js | 233 ++++++++++- tests/test_duplicate_index_postgres.py | 493 +++++++++++++++++++++++ tests/test_duplicates.py | 66 ++- tests/test_scheduler.py | 5 + tests/unit/test_stats_service.py | 18 +- 20 files changed, 1795 insertions(+), 102 deletions(-) create mode 100644 pixelprobe/migrations/duplicates.py create mode 100644 tests/test_duplicate_index_postgres.py diff --git a/CHANGELOG.MD b/CHANGELOG.MD index fac68a8..555f8ee 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0). +## [Unreleased] + +## [2.10.1] - 2026-10-10 + +### Changed + +- Duplicate lists and statistics use persisted summaries. Until initialization completes, `/api/stats` returns null duplicate counts and duplicate-mode lists return HTTP 503 with `Retry-After: 30`. The API serves stale summaries with a status flag. Refresh targets 60 seconds under normal operation. Backlogs or outages can delay refresh; the stale flag reports delays over 60 seconds or failures. +- Resizing across the mobile breakpoint redraws accepted results without another request. Stats loading no longer blocks table results, filters, or operation detection. + ## [2.10.0] - 2026-10-10 ### Added diff --git a/docs/api.md b/docs/api.md index 302d0be..21274f1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -245,6 +245,23 @@ Notes: - Items also carry warning, bitrot, and integrity fields: `has_warnings`, `warning_details`, `bitrot_suspected`, `last_integrity_check_date`, and related hash/baseline columns. - Duplicate filtering adds `duplicate_mode` and `duplicate_group_size` to each result. Groups span the inventory before path, search, and status filters apply. Only files recorded as existing are eligible. Content matching requires equal recorded hashes and sizes; suspected bitrot and failed or unreadable observations are excluded. Filename matching compares exact case-sensitive basenames. Matches use recorded baselines without verifying current contents; filename candidates may contain different content. +Duplicate-mode responses include `duplicate_status` with `ready`, `updated_at`, and `stale`. `updated_at` records the UTC time of the last completed summary refresh. `ready: false` means no completed summary is available. `stale` is independent of `ready`. It becomes true after a refresh failure or when queued inventory changes have waited over 60 seconds. A stale summary is served, and its groups may differ from current eligible membership until refresh completes. + +While the summary is initializing, a request with `duplicate_mode=hash` or `duplicate_mode=name` returns HTTP 503 and `Retry-After: 30`: + +```json +{ + "error": "Duplicate index is initializing", + "duplicate_status": { + "ready": false, + "updated_at": null, + "stale": false + } +} +``` + +The refresh target is 60 seconds under normal operation. A backlog or database or worker outage can delay refreshes; `stale` reports when changes have waited over 60 seconds or refresh has failed. Requests without a duplicate mode remain available while the summary initializes. + #### Get single scan result ```http GET /api/scan-results/{result_id} @@ -457,6 +474,8 @@ Duplicate statistics use the same inventory groups as `duplicate_mode` filtering Counts include only files recorded as existing. Content counts exclude absent hashes or sizes, suspected bitrot, and failed or unreadable observations. Filename counts do not prove matching contents. No fresh filesystem verification runs for these queries. +The six duplicate count fields are numbers when a summary is ready and `null` while it initializes. `duplicate_status` reports readiness, the UTC refresh time, and whether refresh has failed or inventory changes have waited over 60 seconds. `stale` can be false while `ready` is false if the first refresh is still within that interval. A stale summary remains available, but its group counts may differ from current eligible membership until refresh completes. The refresh target is 60 seconds under normal operation. Backlog or an outage can delay it, with delayed or failed refresh reported by `stale`. The statistics endpoint remains available during initialization. + **Response:** ```json { @@ -469,6 +488,17 @@ Counts include only files recorded as existing. Content counts exclude absent ha "healthy_files": 940, "marked_as_good": 3, "warning_files": 7, + "duplicate_files": null, + "duplicate_groups": null, + "duplicate_extra_files": null, + "filename_duplicate_files": null, + "filename_duplicate_groups": null, + "filename_duplicate_extra_files": null, + "duplicate_status": { + "ready": false, + "updated_at": null, + "stale": false + }, "integrity": { "total_files": 1000, "checked_files": 800, diff --git a/openapi.yaml b/openapi.yaml index f787b46..c8e3190 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -175,6 +175,21 @@ components: type: integer schemas: + DuplicateStatus: + type: object + required: [ready, updated_at, stale] + properties: + ready: + type: boolean + description: Whether a completed duplicate summary is available. + updated_at: + type: string + format: date-time + nullable: true + description: UTC time of the last completed summary refresh, or null before the first refresh. + stale: + type: boolean + description: True when refresh has failed or inventory changes have waited over 60 seconds for processing. This is independent of ready. NotificationProviderInput: type: object required: [name, provider_type, configuration] @@ -873,7 +888,52 @@ paths: type: string responses: '200': - description: Scan results retrieved successfully + description: Scan results retrieved successfully. Duplicate-mode responses include duplicate_status. + content: + application/json: + schema: + type: object + properties: + results: + type: array + items: + type: object + properties: + duplicate_mode: + type: string + enum: [hash, name] + duplicate_group_size: + type: integer + total: + type: integer + page: + type: integer + per_page: + type: integer + pages: + type: integer + duplicate_status: + $ref: '#/components/schemas/DuplicateStatus' + additionalProperties: true + '503': + description: Duplicate summary is initializing. Returned only when duplicate_mode is hash or name and no completed summary is ready. + headers: + Retry-After: + description: Seconds before retrying. + schema: + type: string + example: '30' + content: + application/json: + schema: + type: object + required: [error, duplicate_status] + properties: + error: + type: string + example: Duplicate index is initializing + duplicate_status: + $ref: '#/components/schemas/DuplicateStatus' /scan-results/{result_id}: get: @@ -1493,6 +1553,32 @@ paths: responses: '200': description: Statistics retrieved successfully + content: + application/json: + schema: + type: object + properties: + duplicate_files: + type: integer + nullable: true + duplicate_groups: + type: integer + nullable: true + duplicate_extra_files: + type: integer + nullable: true + filename_duplicate_files: + type: integer + nullable: true + filename_duplicate_groups: + type: integer + nullable: true + filename_duplicate_extra_files: + type: integer + nullable: true + duplicate_status: + $ref: '#/components/schemas/DuplicateStatus' + additionalProperties: true /stats/trends: get: diff --git a/pixelprobe/api/scan_routes.py b/pixelprobe/api/scan_routes.py index 4105fc1..e1a3b4a 100644 --- a/pixelprobe/api/scan_routes.py +++ b/pixelprobe/api/scan_routes.py @@ -19,7 +19,7 @@ from pixelprobe.utils.helpers import get_configured_scan_paths from pixelprobe.utils.paths import like_prefix from pixelprobe.api.scan_launch import launch_directory_scan -from pixelprobe.services.duplicate_service import duplicate_members +from pixelprobe.services.duplicate_service import duplicate_index_status, duplicate_members # Remove direct limiter imports as we'll use decorators logger = logging.getLogger(__name__) @@ -191,6 +191,14 @@ def get_scan_results(): if duplicate_mode not in ('all', 'hash', 'name'): return {'error': 'duplicate_mode must be all, hash, or name'}, 400 + duplicate_status = None + if duplicate_mode != 'all': + duplicate_status = duplicate_index_status() + if not duplicate_status['ready']: + return ({'error': 'Duplicate index is initializing', + 'duplicate_status': duplicate_status}, + 503, {'Retry-After': '30'}) + # Build query query = ScanResult.query if duplicate_mode != 'all': @@ -204,7 +212,13 @@ def get_scan_results(): query = query.filter(ScanResult.file_path.like(like_prefix(path_filter), escape='\\')) else: # Invalid path -- return empty results - return {'results': [], 'total': 0, 'page': page, 'per_page': per_page, 'pages': 0} + empty_response = { + 'results': [], 'total': 0, 'page': page, + 'per_page': per_page, 'pages': 0, + } + if duplicate_status is not None: + empty_response['duplicate_status'] = duplicate_status + return empty_response # Apply search filter if search_query: @@ -315,13 +329,16 @@ def __init__(self, items, total): results.append(result_dict) - return { + response = { 'results': results, 'total': pagination.total, 'page': page, 'per_page': per_page, 'pages': pagination.pages } + if duplicate_status is not None: + response['duplicate_status'] = duplicate_status + return response @scan_bp.route('/scan-results/') @auth_required diff --git a/pixelprobe/api/stats_routes.py b/pixelprobe/api/stats_routes.py index 001d42a..ecd9afa 100644 --- a/pixelprobe/api/stats_routes.py +++ b/pixelprobe/api/stats_routes.py @@ -9,7 +9,7 @@ from pixelprobe.version import __version__ from pixelprobe.utils.timezone import from_utc_to_configured, get_configured_timezone_name from pixelprobe.services.stats_service import StatsService -from pixelprobe.services.duplicate_service import duplicate_statistics +from pixelprobe.services.duplicate_service import duplicate_index_status, duplicate_statistics from pixelprobe.auth import auth_required logger = logging.getLogger(__name__) @@ -55,7 +55,9 @@ def get_stats(): # Rolling-integrity coverage: answers "how much of the library has # been verified over time" - no single budgeted run report can result['integrity'] = StatsService().get_integrity_coverage() - result.update(duplicate_statistics()) + duplicate_status = duplicate_index_status() + result.update(duplicate_statistics(duplicate_status)) + result['duplicate_status'] = duplicate_status return result @@ -63,6 +65,7 @@ def get_stats(): logger.error(f"Error getting stats: {str(e)}", exc_info=True) # Fallback to individual queries if the optimized query fails try: + duplicate_status = duplicate_index_status() total_files = ScanResult.query.count() completed_files = ScanResult.query.filter_by(scan_status='completed').count() pending_files = ScanResult.query.filter( @@ -119,7 +122,8 @@ def get_stats(): 'healthy_files': healthy_files, 'marked_as_good': marked_as_good, 'warning_files': warning_files, - **duplicate_statistics(), + **duplicate_statistics(duplicate_status), + 'duplicate_status': duplicate_status, 'integrity': StatsService().get_integrity_coverage() } except Exception as e2: diff --git a/pixelprobe/migrations/duplicates.py b/pixelprobe/migrations/duplicates.py new file mode 100644 index 0000000..12ff3d3 --- /dev/null +++ b/pixelprobe/migrations/duplicates.py @@ -0,0 +1,174 @@ +"""Database objects that keep duplicate group keys queued for refresh.""" + +import logging + +from sqlalchemy import text + +from pixelprobe.migrations.startup import migration_connection + +logger = logging.getLogger(__name__) + +_HASH_ELIGIBLE = """file_exists IS TRUE + AND file_hash IS NOT NULL AND btrim(file_hash) <> '' + AND file_size IS NOT NULL AND bitrot_suspected IS FALSE + AND (scan_status IS NULL OR scan_status NOT IN ('error', 'failed', 'unreadable')) + AND (scan_tool IS NULL OR scan_tool <> 'error') + AND (last_integrity_outcome IS NULL + OR last_integrity_outcome NOT IN ('error', 'unreadable'))""" + + +def _hash_eligible(alias): + return f"""{alias}.file_exists IS TRUE + AND {alias}.file_hash IS NOT NULL AND btrim({alias}.file_hash) <> '' + AND {alias}.file_size IS NOT NULL AND {alias}.bitrot_suspected IS FALSE + AND ({alias}.scan_status IS NULL + OR {alias}.scan_status NOT IN ('error', 'failed', 'unreadable')) + AND ({alias}.scan_tool IS NULL OR {alias}.scan_tool <> 'error') + AND ({alias}.last_integrity_outcome IS NULL + OR {alias}.last_integrity_outcome NOT IN ('error', 'unreadable'))""" + + +def _enqueue_function(name, rows_sql): + return f"""CREATE OR REPLACE FUNCTION {name}() RETURNS trigger AS $$ +BEGIN + WITH changed_rows AS ( + {rows_sql} + ), dirty AS ( + SELECT DISTINCT 'hash'::varchar(8) AS mode, file_hash AS key_text, + file_size AS key_size + FROM changed_rows + WHERE hash_changed IS TRUE AND {_HASH_ELIGIBLE} + UNION + SELECT DISTINCT 'name'::varchar(8), split_part(file_path, '/', -1), -1::bigint + FROM changed_rows + WHERE name_changed IS TRUE AND file_exists IS TRUE + AND split_part(file_path, '/', -1) <> '' + ) + INSERT INTO duplicate_dirty_keys (mode, key_text, key_size, enqueued_at) + SELECT mode, key_text, key_size, clock_timestamp() + FROM dirty + ORDER BY mode, key_text, key_size + ON CONFLICT (mode, key_text, key_size) DO UPDATE + SET enqueued_at = LEAST(duplicate_dirty_keys.enqueued_at, EXCLUDED.enqueued_at); + RETURN NULL; +END; +$$ LANGUAGE plpgsql""" + + +_ROW_COLUMNS_OLD = """o.id, o.file_path, o.file_hash, o.file_size, o.file_exists, + o.bitrot_suspected, o.scan_status, o.scan_tool, o.last_integrity_outcome""" +_ROW_COLUMNS_NEW = """n.id, n.file_path, n.file_hash, n.file_size, n.file_exists, + n.bitrot_suspected, n.scan_status, n.scan_tool, n.last_integrity_outcome""" +_ROW_COLUMNS = """id, file_path, file_hash, file_size, file_exists, bitrot_suspected, + scan_status, scan_tool, last_integrity_outcome""" +_INSERT_ROWS = f"SELECT {_ROW_COLUMNS}, TRUE AS hash_changed, TRUE AS name_changed FROM new_rows" +_DELETE_ROWS = f"SELECT {_ROW_COLUMNS}, TRUE AS hash_changed, TRUE AS name_changed FROM old_rows" +_UPDATE_ROWS = f"""SELECT {_ROW_COLUMNS_OLD} + , (n.id IS NULL OR o.id IS DISTINCT FROM n.id + OR o.file_hash IS DISTINCT FROM n.file_hash + OR o.file_size IS DISTINCT FROM n.file_size + OR ({_hash_eligible('o')}) IS DISTINCT FROM ({_hash_eligible('n')})) AS hash_changed + , (n.id IS NULL OR o.id IS DISTINCT FROM n.id + OR split_part(o.file_path, '/', -1) IS DISTINCT FROM split_part(n.file_path, '/', -1) + OR (o.file_exists IS TRUE AND split_part(o.file_path, '/', -1) <> '') + IS DISTINCT FROM + (n.file_exists IS TRUE AND split_part(n.file_path, '/', -1) <> '')) AS name_changed + FROM old_rows o LEFT JOIN new_rows n ON n.id = o.id + WHERE n.id IS NULL OR o.file_hash IS DISTINCT FROM n.file_hash + OR o.file_size IS DISTINCT FROM n.file_size + OR ({_hash_eligible('o')}) IS DISTINCT FROM ({_hash_eligible('n')}) + OR split_part(o.file_path, '/', -1) IS DISTINCT FROM split_part(n.file_path, '/', -1) + OR (o.file_exists IS TRUE AND split_part(o.file_path, '/', -1) <> '') + IS DISTINCT FROM + (n.file_exists IS TRUE AND split_part(n.file_path, '/', -1) <> '') + UNION ALL + SELECT {_ROW_COLUMNS_NEW} + , (o.id IS NULL OR o.id IS DISTINCT FROM n.id + OR o.file_hash IS DISTINCT FROM n.file_hash + OR o.file_size IS DISTINCT FROM n.file_size + OR ({_hash_eligible('o')}) IS DISTINCT FROM ({_hash_eligible('n')})) AS hash_changed + , (o.id IS NULL OR o.id IS DISTINCT FROM n.id + OR split_part(o.file_path, '/', -1) IS DISTINCT FROM split_part(n.file_path, '/', -1) + OR (o.file_exists IS TRUE AND split_part(o.file_path, '/', -1) <> '') + IS DISTINCT FROM + (n.file_exists IS TRUE AND split_part(n.file_path, '/', -1) <> '')) AS name_changed + FROM new_rows n LEFT JOIN old_rows o ON o.id = n.id + WHERE o.id IS NULL OR o.file_hash IS DISTINCT FROM n.file_hash + OR o.file_size IS DISTINCT FROM n.file_size + OR ({_hash_eligible('o')}) IS DISTINCT FROM ({_hash_eligible('n')}) + OR split_part(o.file_path, '/', -1) IS DISTINCT FROM split_part(n.file_path, '/', -1) + OR (o.file_exists IS TRUE AND split_part(o.file_path, '/', -1) <> '') + IS DISTINCT FROM + (n.file_exists IS TRUE AND split_part(n.file_path, '/', -1) <> '')""" + + +_FUNCTIONS = ( + _enqueue_function('pixelprobe_enqueue_duplicates_insert', _INSERT_ROWS), + _enqueue_function('pixelprobe_enqueue_duplicates_update', _UPDATE_ROWS), + _enqueue_function('pixelprobe_enqueue_duplicates_delete', _DELETE_ROWS), +) + +_TRIGGERS = ( + ("scan_results_duplicate_insert", """CREATE TRIGGER scan_results_duplicate_insert + AFTER INSERT ON scan_results REFERENCING NEW TABLE AS new_rows + FOR EACH STATEMENT EXECUTE FUNCTION pixelprobe_enqueue_duplicates_insert()"""), + ("scan_results_duplicate_update", """CREATE TRIGGER scan_results_duplicate_update + AFTER UPDATE ON scan_results + REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows + FOR EACH STATEMENT EXECUTE FUNCTION pixelprobe_enqueue_duplicates_update()"""), + ("scan_results_duplicate_delete", """CREATE TRIGGER scan_results_duplicate_delete + AFTER DELETE ON scan_results REFERENCING OLD TABLE AS old_rows + FOR EACH STATEMENT EXECUTE FUNCTION pixelprobe_enqueue_duplicates_delete()"""), +) + + +def run_duplicate_index_migrations(db): + """Create the duplicate summary state and statement-level invalidation triggers.""" + statements = ( + """CREATE TABLE IF NOT EXISTS duplicate_group_summaries ( + mode VARCHAR(8) NOT NULL, + key_text VARCHAR(500) NOT NULL, + key_size BIGINT NOT NULL DEFAULT -1, + group_size INTEGER NOT NULL, + group_id INTEGER NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (mode, key_text, key_size), + CONSTRAINT ck_duplicate_summary_mode CHECK (mode IN ('hash', 'name')), + CONSTRAINT ck_duplicate_summary_size CHECK (group_size > 1) + )""", + """CREATE TABLE IF NOT EXISTS duplicate_dirty_keys ( + mode VARCHAR(8) NOT NULL, + key_text VARCHAR(500) NOT NULL, + key_size BIGINT NOT NULL DEFAULT -1, + enqueued_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (mode, key_text, key_size), + CONSTRAINT ck_duplicate_dirty_mode CHECK (mode IN ('hash', 'name')) + )""", + """CREATE INDEX IF NOT EXISTS idx_duplicate_dirty_enqueued_at + ON duplicate_dirty_keys (enqueued_at, mode, key_text, key_size)""", + """CREATE TABLE IF NOT EXISTS duplicate_index_state ( + id INTEGER PRIMARY KEY, + ready BOOLEAN NOT NULL DEFAULT FALSE, + updated_at TIMESTAMP WITH TIME ZONE, + refresh_failed BOOLEAN NOT NULL DEFAULT FALSE, + CONSTRAINT ck_duplicate_state_singleton CHECK (id = 1) + )""", + """INSERT INTO duplicate_index_state (id, ready, refresh_failed) + VALUES (1, FALSE, FALSE) ON CONFLICT (id) DO NOTHING""", + *_FUNCTIONS, + ) + try: + with migration_connection(db) as conn: + for statement in statements: + conn.execute(text(statement)) + existing_triggers = set(conn.execute(text(""" + SELECT tgname FROM pg_trigger + WHERE tgrelid = to_regclass('scan_results') AND NOT tgisinternal + """)).scalars()) + for trigger_name, trigger_sql in _TRIGGERS: + if trigger_name not in existing_triggers: + conn.execute(text(trigger_sql)) + conn.commit() + except Exception: + logger.exception('Duplicate index schema migration failed') + raise diff --git a/pixelprobe/migrations/startup.py b/pixelprobe/migrations/startup.py index 3091ee2..1d527c9 100644 --- a/pixelprobe/migrations/startup.py +++ b/pixelprobe/migrations/startup.py @@ -804,6 +804,10 @@ def _run_all_migrations(db, connection): logger.info("Running v2.9.5 file_type repair...") run_v2_9_5_migrations(db) + from pixelprobe.migrations.duplicates import run_duplicate_index_migrations + logger.info("Creating duplicate index state and invalidation triggers...") + run_duplicate_index_migrations(db) + logger.info("Creating performance indexes...") create_performance_indexes(db) logger.info("Performance indexes created successfully") diff --git a/pixelprobe/models.py b/pixelprobe/models.py index c04cff1..559ce61 100644 --- a/pixelprobe/models.py +++ b/pixelprobe/models.py @@ -188,6 +188,50 @@ def append_output(self, new_output): def __repr__(self): return f'' + +class DuplicateGroupSummary(db.Model): + __tablename__ = 'duplicate_group_summaries' + + mode = db.Column(db.String(8), primary_key=True) + key_text = db.Column(db.String(500), primary_key=True) + key_size = db.Column(db.BigInteger, primary_key=True, default=-1, server_default='-1') + group_size = db.Column(db.Integer, nullable=False) + group_id = db.Column(db.Integer, nullable=False) + updated_at = db.Column(db.DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc)) + + __table_args__ = ( + db.CheckConstraint("mode IN ('hash', 'name')", name='ck_duplicate_summary_mode'), + db.CheckConstraint('group_size > 1', name='ck_duplicate_summary_size'), + ) + + +class DuplicateDirtyKey(db.Model): + __tablename__ = 'duplicate_dirty_keys' + + mode = db.Column(db.String(8), primary_key=True) + key_text = db.Column(db.String(500), primary_key=True) + key_size = db.Column(db.BigInteger, primary_key=True, default=-1, server_default='-1') + enqueued_at = db.Column(db.DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc)) + + __table_args__ = ( + db.CheckConstraint("mode IN ('hash', 'name')", name='ck_duplicate_dirty_mode'), + db.Index('idx_duplicate_dirty_enqueued_at', 'enqueued_at', 'mode', 'key_text', 'key_size'), + ) + + +class DuplicateIndexState(db.Model): + __tablename__ = 'duplicate_index_state' + + id = db.Column(db.Integer, primary_key=True) + ready = db.Column(db.Boolean, nullable=False, default=False, server_default='false') + updated_at = db.Column(db.DateTime(timezone=True), nullable=True) + refresh_failed = db.Column(db.Boolean, nullable=False, default=False, + server_default='false') + + __table_args__ = (db.CheckConstraint('id = 1', name='ck_duplicate_state_singleton'),) + class IgnoredErrorPattern(db.Model): __tablename__ = 'ignored_error_patterns' diff --git a/pixelprobe/scheduler.py b/pixelprobe/scheduler.py index 789e8f7..b834549 100644 --- a/pixelprobe/scheduler.py +++ b/pixelprobe/scheduler.py @@ -65,7 +65,9 @@ class MediaScheduler: PROGRESS_STALE_AFTER = timedelta(minutes=30) def __init__(self, app=None): - self.scheduler = BackgroundScheduler() + self.scheduler = BackgroundScheduler( + executors={'duplicate_index': {'type': 'threadpool', 'max_workers': 1}} + ) self.app = app self._shutdown = False self.scan_lock = threading.Lock() @@ -378,6 +380,19 @@ def init_app(self, app): ) logger.info("Scheduled database schedule sync every 60 seconds") + self.scheduler.add_job( + func=self._refresh_duplicate_index, + trigger="interval", + seconds=30, + id="duplicate_index_refresh", + name="Refresh duplicate group summaries", + misfire_grace_time=30, + coalesce=True, + max_instances=1, + executor='duplicate_index', + ) + logger.info("Scheduled duplicate index refresh every 30 seconds") + # Load saved schedules from database with app.app_context(): self._load_saved_schedules() @@ -1190,6 +1205,16 @@ def _run_retention_cleanup(self): except Exception as e: logger.error(f"Failed to enqueue data retention cleanup: {e}") + def _refresh_duplicate_index(self): + if not self._can_dispatch(): + return + try: + with self.app.app_context(): + from pixelprobe.services.duplicate_service import refresh_duplicate_index + refresh_duplicate_index(can_dispatch=self._can_dispatch) + except Exception as e: + logger.error(f"Failed to refresh duplicate index: {e}", exc_info=True) + def shutdown(self): """Stop lease renewal and scheduler executors without waiting on jobs.""" if self._shutdown: diff --git a/pixelprobe/services/duplicate_service.py b/pixelprobe/services/duplicate_service.py index 28a24c1..ddfc760 100644 --- a/pixelprobe/services/duplicate_service.py +++ b/pixelprobe/services/duplicate_service.py @@ -1,13 +1,27 @@ -from sqlalchemy import func, and_, or_ +"""Read and refresh persisted duplicate group summaries.""" -from pixelprobe.models import db, ScanResult +from datetime import datetime, timedelta, timezone +import time +from sqlalchemy import and_, func, or_, text -def _duplicate_groups(mode): - path = ScanResult.file_path - name = (func.split_part(path, '/', -1) if db.engine.dialect.name == 'postgresql' - else func.substr(path, func.length(func.rtrim(path, func.replace(path, '/', ''))) + 1)) - keys = [ScanResult.file_hash, ScanResult.file_size] if mode == 'hash' else [name] +from pixelprobe.models import (db, DuplicateDirtyKey, DuplicateGroupSummary, + DuplicateIndexState, ScanResult) + +_DUPLICATE_REFRESH_LOCK_ID = 7283945191 +_DIRTY_KEY_BATCH_SIZE = 1000 +_REFRESH_TIME_BUDGET_SECS = 20 + + +def _basename_expression(): + return (func.split_part(ScanResult.file_path, '/', -1) + if db.engine.dialect.name == 'postgresql' + else func.substr(ScanResult.file_path, + func.length(func.rtrim(ScanResult.file_path, + func.replace(ScanResult.file_path, '/', ''))) + 1)) + + +def _eligible(mode): eligible = [ScanResult.file_exists.is_(True)] if mode == 'hash': eligible.extend([ @@ -22,35 +36,338 @@ def _duplicate_groups(mode): ScanResult.last_integrity_outcome.notin_(['error', 'unreadable'])), ]) else: - eligible.append(name != '') + eligible.append(_basename_expression() != '') + return eligible + + +def _duplicate_groups(mode): + """Reference grouping query for tests and cache reconciliation checks.""" + name = _basename_expression() + keys = [ScanResult.file_hash, ScanResult.file_size] if mode == 'hash' else [name] groups = db.session.query( *(key.label(f'key_{index}') for index, key in enumerate(keys)), func.count().label('group_size'), func.min(ScanResult.id).label('group_id'), - ).filter(*eligible).group_by(*keys).having(func.count() > 1).subquery() - return groups, keys, eligible + ).filter(*_eligible(mode)).group_by(*keys).having(func.count() > 1).subquery() + return groups, keys -def duplicate_members(mode): - groups, keys, eligible = _duplicate_groups(mode) - return db.session.query( - ScanResult.id.label('file_id'), groups.c.group_size, groups.c.group_id, - ).join(groups, and_( - *(key == groups.c[f'key_{index}'] for index, key in enumerate(keys)) - )).filter(*eligible).subquery() +def duplicate_index_status(): + state = db.session.get(DuplicateIndexState, 1) + if state is None: + return {'ready': False, 'updated_at': None, 'stale': False} + + updated_at = state.updated_at + if updated_at is not None: + if updated_at.tzinfo is None: + updated_at = updated_at.replace(tzinfo=timezone.utc) + updated_at = updated_at.astimezone(timezone.utc).isoformat() + stale_pending = db.session.query(DuplicateDirtyKey.mode).filter( + DuplicateDirtyKey.enqueued_at <= datetime.now(timezone.utc) - timedelta(seconds=60) + ).order_by(DuplicateDirtyKey.enqueued_at, DuplicateDirtyKey.mode, + DuplicateDirtyKey.key_text, DuplicateDirtyKey.key_size).first() is not None + return { + 'ready': bool(state.ready), + 'updated_at': updated_at, + 'stale': bool(state.refresh_failed or stale_pending), + } + +def duplicate_statistics(status=None): + status = status or duplicate_index_status() + prefixes = {'hash': 'duplicate', 'name': 'filename_duplicate'} + if not status['ready']: + return { + f'{prefix}_{suffix}': None + for prefix in prefixes.values() + for suffix in ('files', 'groups', 'extra_files') + } -def duplicate_statistics(): + groups = db.session.query( + DuplicateGroupSummary.mode, + func.sum(DuplicateGroupSummary.group_size), + func.count(), + ).group_by(DuplicateGroupSummary.mode).all() + totals = {mode: (int(files or 0), int(count or 0)) + for mode, files, count in groups} result = {} - for mode, prefix in [('hash', 'duplicate'), ('name', 'filename_duplicate')]: - groups, _, _ = _duplicate_groups(mode) - files, group_count = db.session.query( - func.coalesce(func.sum(groups.c.group_size), 0), func.count(), - ).select_from(groups).one() - files = int(files) + for mode, prefix in prefixes.items(): + files, count = totals.get(mode, (0, 0)) result.update({ f'{prefix}_files': files, - f'{prefix}_groups': group_count, - f'{prefix}_extra_files': files - group_count, + f'{prefix}_groups': count, + f'{prefix}_extra_files': files - count, }) return result + + +def duplicate_members(mode): + if mode not in ('hash', 'name'): + raise ValueError('Duplicate mode must be hash or name') + summary = DuplicateGroupSummary + if mode == 'hash': + key_match = and_(summary.key_text == ScanResult.file_hash, + summary.key_size == ScanResult.file_size) + else: + key_match = and_(summary.key_text == _basename_expression(), summary.key_size == -1) + return db.session.query( + ScanResult.id.label('file_id'), summary.group_size, summary.group_id, + ).join(summary, and_(summary.mode == mode, key_match)).filter( + *_eligible(mode) + ).subquery() + + +def _ensure_duplicate_indexes(can_dispatch=None): + if db.engine.dialect.name != 'postgresql': + return + indexes = ( + ('idx_scan_results_duplicate_hash_key', + "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_scan_results_duplicate_hash_key " + "ON scan_results (file_hash, file_size) WHERE file_exists IS TRUE " + "AND file_hash IS NOT NULL AND btrim(file_hash) <> '' AND file_size IS NOT NULL " + "AND bitrot_suspected IS FALSE " + "AND (scan_status IS NULL OR scan_status NOT IN ('error', 'failed', 'unreadable')) " + "AND (scan_tool IS NULL OR scan_tool <> 'error') " + "AND (last_integrity_outcome IS NULL " + "OR last_integrity_outcome NOT IN ('error', 'unreadable'))"), + ('idx_scan_results_duplicate_name_key', + "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_scan_results_duplicate_name_key " + "ON scan_results ((split_part(file_path, '/', -1))) " + "WHERE file_exists IS TRUE AND split_part(file_path, '/', -1) <> ''"), + ) + with db.engine.connect().execution_options(isolation_level='AUTOCOMMIT') as conn: + conn.execute(text('SET statement_timeout = 300000')) + try: + for name, create_sql in indexes: + if can_dispatch is not None and not can_dispatch(): + return False + valid = conn.execute(text(""" + SELECT i.indisvalid + FROM pg_index i + JOIN pg_class idx ON idx.oid = i.indexrelid + JOIN pg_namespace ns ON ns.oid = idx.relnamespace + WHERE idx.relname = :name AND ns.nspname = current_schema() + """), {'name': name}).scalar_one_or_none() + if valid is True: + continue + if valid is False: + conn.execute(text(f'DROP INDEX CONCURRENTLY IF EXISTS {name}')) + conn.execute(text(create_sql)) + return True + finally: + try: + conn.execute(text('RESET statement_timeout')) + except Exception: + conn.invalidate() + + +def _initial_grouping(conn): + conn.execute(text('DELETE FROM duplicate_group_summaries')) + conn.execute(text(f""" + INSERT INTO duplicate_group_summaries + (mode, key_text, key_size, group_size, group_id, updated_at) + SELECT 'hash', file_hash, file_size, count(*), min(id), clock_timestamp() + FROM scan_results + WHERE file_exists IS TRUE AND file_hash IS NOT NULL AND btrim(file_hash) <> '' + AND file_size IS NOT NULL AND bitrot_suspected IS FALSE + AND (scan_status IS NULL OR scan_status NOT IN ('error', 'failed', 'unreadable')) + AND (scan_tool IS NULL OR scan_tool <> 'error') + AND (last_integrity_outcome IS NULL + OR last_integrity_outcome NOT IN ('error', 'unreadable')) + GROUP BY file_hash, file_size + HAVING count(*) > 1 + """)) + conn.execute(text(""" + INSERT INTO duplicate_group_summaries + (mode, key_text, key_size, group_size, group_id, updated_at) + SELECT 'name', split_part(file_path, '/', -1), -1, count(*), min(id), clock_timestamp() + FROM scan_results + WHERE file_exists IS TRUE AND split_part(file_path, '/', -1) <> '' + GROUP BY split_part(file_path, '/', -1) + HAVING count(*) > 1 + """)) + + +def _values_clause(keys): + values = [] + params = {} + for index, (mode, key_text, key_size) in enumerate(keys): + values.append(f"(CAST(:mode_{index} AS varchar(8)), " + f"CAST(:text_{index} AS varchar(500)), " + f"CAST(:size_{index} AS bigint))") + params[f'mode_{index}'] = mode + params[f'text_{index}'] = key_text + params[f'size_{index}'] = key_size + return ', '.join(values), params + + +def _bounded_execute(conn, deadline, sql, params=None): + remaining_ms = max(1, int((deadline - time.monotonic()) * 1000)) + conn.execute(text("SELECT set_config('statement_timeout', :timeout, true)"), + {'timeout': f'{remaining_ms}ms'}) + return conn.execute(text(sql), params or {}) + + +def _refresh_claimed_keys(conn, keys, deadline): + all_values_sql, all_params = _values_clause(keys) + _bounded_execute(conn, deadline, f""" + DELETE FROM duplicate_group_summaries AS summary + USING (VALUES {all_values_sql}) AS wanted(mode, key_text, key_size) + WHERE summary.mode = wanted.mode AND summary.key_text = wanted.key_text + AND summary.key_size = wanted.key_size + """, all_params) + + for mode in ('hash', 'name'): + mode_keys = [key for key in keys if key[0] == mode] + if not mode_keys: + continue + values_sql, params = _values_clause(mode_keys) + if mode == 'hash': + match = """scan.file_hash = wanted.key_text AND scan.file_size = wanted.key_size + AND scan.file_exists IS TRUE AND scan.file_hash IS NOT NULL + AND btrim(scan.file_hash) <> '' AND scan.file_size IS NOT NULL + AND scan.bitrot_suspected IS FALSE + AND (scan.scan_status IS NULL OR scan.scan_status NOT IN ('error', 'failed', 'unreadable')) + AND (scan.scan_tool IS NULL OR scan.scan_tool <> 'error') + AND (scan.last_integrity_outcome IS NULL + OR scan.last_integrity_outcome NOT IN ('error', 'unreadable'))""" + else: + match = """split_part(scan.file_path, '/', -1) = wanted.key_text + AND scan.file_exists IS TRUE + AND split_part(scan.file_path, '/', -1) <> ''""" + _bounded_execute(conn, deadline, f""" + INSERT INTO duplicate_group_summaries + (mode, key_text, key_size, group_size, group_id, updated_at) + WITH wanted(mode, key_text, key_size) AS (VALUES {values_sql}), grouped AS ( + SELECT wanted.key_text, wanted.key_size, count(*) AS group_size, + min(scan.id) AS group_id + FROM wanted + JOIN scan_results AS scan ON wanted.mode = '{mode}' AND {match} + GROUP BY wanted.key_text, wanted.key_size + HAVING count(*) > 1 + ) + SELECT '{mode}', key_text, key_size, group_size, group_id, clock_timestamp() + FROM grouped + ON CONFLICT (mode, key_text, key_size) DO UPDATE SET + group_size = EXCLUDED.group_size, + group_id = EXCLUDED.group_id, + updated_at = EXCLUDED.updated_at + """, params) + + _bounded_execute(conn, deadline, f""" + DELETE FROM duplicate_dirty_keys AS dirty + USING (VALUES {all_values_sql}) AS wanted(mode, key_text, key_size) + WHERE dirty.mode = wanted.mode AND dirty.key_text = wanted.key_text + AND dirty.key_size = wanted.key_size + """, all_params) + + +def refresh_duplicate_index(batch_size=_DIRTY_KEY_BATCH_SIZE, can_dispatch=None): + """Create lookup indexes, backfill once, then reconcile queued keys.""" + if db.engine.dialect.name != 'postgresql': + return False + if can_dispatch is not None and not can_dispatch(): + return False + lock_conn = db.engine.connect() + lock_acquired = False + lock_attempted = False + lock_result_known = False + try: + lock_attempted = True + lock_acquired = bool(lock_conn.execute(text( + 'SELECT pg_try_advisory_lock(:lock_id)'), + {'lock_id': _DUPLICATE_REFRESH_LOCK_ID}, + ).scalar()) + lock_conn.commit() + lock_result_known = True + if not lock_acquired: + return False + + if not _ensure_duplicate_indexes(can_dispatch): + return False + if can_dispatch is not None and not can_dispatch(): + return False + with db.engine.begin() as conn: + conn.execute(text('SET LOCAL lock_timeout = 5000')) + + state = conn.execute(text( + 'SELECT ready FROM duplicate_index_state WHERE id = 1 FOR UPDATE' + )).scalar_one_or_none() + if state is None: + conn.execute(text(""" + INSERT INTO duplicate_index_state (id, ready, refresh_failed) + VALUES (1, FALSE, FALSE) ON CONFLICT (id) DO NOTHING + """)) + state = False + + if not state: + conn.execute(text('SET LOCAL statement_timeout = 300000')) + _initial_grouping(conn) + conn.execute(text(""" + UPDATE duplicate_index_state + SET ready = TRUE, updated_at = clock_timestamp(), refresh_failed = FALSE + WHERE id = 1 + """)) + return True + + deadline = time.monotonic() + _REFRESH_TIME_BUDGET_SECS + while time.monotonic() < deadline: + if can_dispatch is not None and not can_dispatch(): + return False + with db.engine.begin() as conn: + conn.execute(text('SET LOCAL lock_timeout = 5000')) + batch_deadline = min(deadline, time.monotonic() + 8) + rows = _bounded_execute(conn, batch_deadline, """ + WITH oldest AS ( + SELECT mode, key_text, key_size + FROM duplicate_dirty_keys + ORDER BY enqueued_at, mode, key_text, key_size + LIMIT :batch_size + ) + SELECT dirty.mode, dirty.key_text, dirty.key_size + FROM duplicate_dirty_keys AS dirty + JOIN oldest USING (mode, key_text, key_size) + ORDER BY dirty.mode, dirty.key_text, dirty.key_size + FOR UPDATE OF dirty SKIP LOCKED + """, {'batch_size': max(1, int(batch_size))}).all() + keys = [(row.mode, row.key_text, row.key_size) for row in rows] + if not keys: + _bounded_execute(conn, batch_deadline, """ + UPDATE duplicate_index_state + SET updated_at = clock_timestamp(), refresh_failed = FALSE + WHERE id = 1 + """) + break + _refresh_claimed_keys(conn, keys, batch_deadline) + _bounded_execute(conn, batch_deadline, """ + UPDATE duplicate_index_state + SET updated_at = clock_timestamp(), refresh_failed = FALSE + WHERE id = 1 + """) + return True + except Exception: + try: + with db.engine.begin() as conn: + conn.execute(text(""" + UPDATE duplicate_index_state + SET refresh_failed = TRUE + WHERE id = 1 + """)) + except Exception: + pass + raise + finally: + if lock_acquired: + try: + unlocked = lock_conn.execute(text( + 'SELECT pg_advisory_unlock(:lock_id)'), + {'lock_id': _DUPLICATE_REFRESH_LOCK_ID}, + ).scalar() + lock_conn.commit() + if not unlocked: + lock_conn.invalidate() + except Exception: + lock_conn.invalidate() + elif lock_attempted and not lock_result_known: + lock_conn.invalidate() + lock_conn.close() diff --git a/pixelprobe/services/stats_service.py b/pixelprobe/services/stats_service.py index a99d519..1a3abc4 100644 --- a/pixelprobe/services/stats_service.py +++ b/pixelprobe/services/stats_service.py @@ -11,7 +11,7 @@ from pixelprobe.models import db, ScanResult, ScanReport from pixelprobe.utils.timezone import from_utc_to_configured, get_configured_timezone from pixelprobe.version import __version__ -from pixelprobe.services.duplicate_service import duplicate_statistics +from pixelprobe.services.duplicate_service import duplicate_index_status, duplicate_statistics logger = logging.getLogger(__name__) @@ -40,7 +40,8 @@ def get_file_statistics(self) -> Dict: FROM scan_results """) ).fetchone() - + + duplicate_status = duplicate_index_status() return { 'total_files': stats[0] or 0, 'completed_files': stats[1] or 0, @@ -51,7 +52,8 @@ def get_file_statistics(self) -> Dict: 'healthy_files': stats[6] or 0, 'marked_as_good': stats[7] or 0, 'warning_files': stats[8] or 0, - **duplicate_statistics(), + **duplicate_statistics(duplicate_status), + 'duplicate_status': duplicate_status, } except Exception as e: @@ -222,6 +224,8 @@ def _get_stats_fallback(self) -> Dict: (ScanResult.scan_tool == None) | ScanResult.scan_tool.notin_(['error', 'unsupported']) ).count() + + duplicate_status = duplicate_index_status() return { 'total_files': total_files, @@ -233,7 +237,8 @@ def _get_stats_fallback(self) -> Dict: 'healthy_files': healthy_files, 'marked_as_good': marked_as_good, 'warning_files': warning_files, - **duplicate_statistics(), + **duplicate_statistics(duplicate_status), + 'duplicate_status': duplicate_status, } except Exception as e: logger.error(f"Fallback stats query also failed: {e}") diff --git a/pixelprobe/version.py b/pixelprobe/version.py index 6b77d2b..ab14855 100644 --- a/pixelprobe/version.py +++ b/pixelprobe/version.py @@ -4,7 +4,7 @@ # Default version - this is the single source of truth -_DEFAULT_VERSION = '2.10.0' +_DEFAULT_VERSION = '2.10.1' # Allow override via environment variable for CI/CD, but default to the hardcoded version diff --git a/static/css/desktop.css b/static/css/desktop.css index 738cc0f..6f4a66f 100644 --- a/static/css/desktop.css +++ b/static/css/desktop.css @@ -409,6 +409,14 @@ body { color: var(--warning-color, #b7791f); } +#duplicate-index-status { + margin-bottom: 0.75rem; +} + +#duplicate-index-status.is-stale { + color: var(--warning-color, #b7791f); +} + .integrity-details-toggle { display: inline-flex; align-items: center; diff --git a/static/js/app.js b/static/js/app.js index 144aaec..2fdf0f9 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -225,25 +225,23 @@ class APIClient { } if (!response.ok) { - // Surface the server's reason (e.g. "Celery workers not available") - let detail = ''; + let data = null; try { - const data = await response.json(); - if (data && data.error) { - detail = `: ${data.error}`; - } + data = await response.json(); } catch (parseError) { // Non-JSON error body } - throw new Error(`HTTP error! status: ${response.status}${detail}`); + const detail = data && data.error ? `: ${data.error}` : ''; + const error = new Error(`HTTP error! status: ${response.status}${detail}`); + error.status = response.status; + error.data = data; + error.retryAfter = response.headers && response.headers.get('Retry-After'); + throw error; } return await response.json(); } catch (error) { - // Handle network/connection errors silently for stats updates - if (endpoint === '/stats' || endpoint === '/system-info') { - return null; - } + if (endpoint === '/system-info') return null; throw error; } } @@ -384,6 +382,9 @@ class StatsDashboard { this.refreshInterval = null; this.lastIntegrity = null; this.lastSuccessfulRefresh = null; + this.lastDuplicateStats = null; + this.statsRequest = null; + this.statsPollInFlight = false; this.setupIntegrityDetailsToggle(); } @@ -410,15 +411,20 @@ class StatsDashboard { } async updateStats() { - try { + if (this.statsRequest) return this.statsRequest; + this.statsRequest = (async () => { const stats = await this.api.getStats(); if (stats) { this.renderStats(stats); this.lastSuccessfulRefresh = new Date(); this.renderRefreshStatus(); } - } catch (error) { - throw error; + return stats; + })(); + try { + return await this.statsRequest; + } finally { + this.statsRequest = null; } } @@ -430,16 +436,48 @@ class StatsDashboard { this.updateStatCard('warning-files', stats.warning_files || 0); this.updateStatCard('bitrot-files', (stats.integrity && stats.integrity.bitrot_suspected) || 0); this.updateStatCard('pending-files', stats.pending_files); - this.updateStatCard('duplicate-files', stats.duplicate_files || 0); - const duplicateDetails = document.querySelector('#duplicate-details'); - const duplicateGroups = stats.duplicate_groups || 0; - const extraFiles = stats.duplicate_extra_files || 0; - if (duplicateDetails) duplicateDetails.textContent = `${duplicateGroups} ${duplicateGroups === 1 ? 'group' : 'groups'}; ${extraFiles} extra ${extraFiles === 1 ? 'file' : 'files'}`; + this.renderDuplicateStats(stats); this.updateStatCard('scanning-files', stats.scanning_files); this.lastIntegrity = stats.integrity || null; this.renderIntegrityCoverage(stats.integrity); } + renderDuplicateStats(stats) { + const status = stats.duplicate_status || { ready: true, stale: false }; + const fileCount = stats.duplicate_files; + const groupCount = stats.duplicate_groups; + const extraCount = stats.duplicate_extra_files; + if (status.ready && [fileCount, groupCount, extraCount].every(Number.isFinite)) { + this.lastDuplicateStats = { fileCount, groupCount, extraCount, updatedAt: status.updated_at || null }; + this.updateStatCard('duplicate-files', fileCount); + const details = document.querySelector('#duplicate-details'); + if (details) { + const lastUpdated = status.updated_at ? ` Last updated ${new Date(status.updated_at).toLocaleString()}.` : ''; + const refresh = status.stale ? '. Refresh delayed.' : ''; + details.textContent = `${groupCount} ${groupCount === 1 ? 'group' : 'groups'}; ${extraCount} extra ${extraCount === 1 ? 'file' : 'files'}${refresh}${lastUpdated}`; + details.classList.toggle('is-stale', Boolean(status.stale)); + } + return; + } + + if (this.lastDuplicateStats) { + this.updateStatCard('duplicate-files', this.lastDuplicateStats.fileCount); + const details = document.querySelector('#duplicate-details'); + if (details) { + details.textContent = 'Refresh delayed; showing the last available duplicate counts.'; + details.classList.add('is-stale'); + } + } else { + const count = document.querySelector('#duplicate-files'); + if (count) count.textContent = 'Preparing...'; + const details = document.querySelector('#duplicate-details'); + if (details) { + details.textContent = 'Preparing duplicate counts.'; + details.classList.remove('is-stale'); + } + } + } + renderIntegrityCoverage(integrity, stale = false) { const element = document.querySelector('#integrity-checked'); if (!element) return; @@ -490,6 +528,7 @@ class StatsDashboard { if (warning) warning.hidden = !error; if (error) { this.renderIntegrityCoverage(this.lastIntegrity, true); + this.renderDuplicateRefreshFailure(); const previous = this.lastSuccessfulRefresh ? ` Showing data from ${this.lastSuccessfulRefresh.toLocaleString()}.` : ' No current data is available.'; @@ -500,6 +539,22 @@ class StatsDashboard { `Last refreshed ${this.lastSuccessfulRefresh.toLocaleString()}.` : ''; } + renderDuplicateRefreshFailure() { + const details = document.querySelector('#duplicate-details'); + if (!details) return; + if (this.lastDuplicateStats) { + const updated = this.lastDuplicateStats.updatedAt ? + ` Last updated ${new Date(this.lastDuplicateStats.updatedAt).toLocaleString()}.` : ''; + details.textContent = `Refresh delayed; showing the last available duplicate counts.${updated}`; + details.classList.add('is-stale'); + } else { + const count = document.querySelector('#duplicate-files'); + if (count) count.textContent = 'Preparing...'; + details.textContent = 'Preparing duplicate counts. Refresh will retry automatically.'; + details.classList.remove('is-stale'); + } + } + updateStatCard(id, value) { const element = document.querySelector(`#${id}`); if (element) { @@ -514,17 +569,21 @@ class StatsDashboard { this.statsPollDelay = 30000; const poll = async () => { if (generation !== this._statsPollGeneration) return; + if (this.statsPollInFlight) return; this.refreshInterval = null; if (document.hidden) { this.refreshInterval = setTimeout(poll, 30000); return; } + this.statsPollInFlight = true; try { await this.updateStats(); this.statsPollDelay = 30000; } catch (error) { this.statsPollDelay = Math.min(this.statsPollDelay * 2, 300000); this.renderRefreshStatus(error); + } finally { + this.statsPollInFlight = false; } if (generation === this._statsPollGeneration) { this.refreshInterval = setTimeout(poll, this.statsPollDelay); @@ -1359,6 +1418,15 @@ class TableManager { this.pathFilter = ''; this.selectedFiles = new Set(); this.lastClickedCheckbox = null; // Track last clicked checkbox for shift-select + this.inFlightRequests = new Map(); + this.requestGeneration = 0; + this.lastAcceptedData = null; + this.lastAcceptedKey = null; + this.resultsHidden = false; + this.lastLayoutIsMobile = window.innerWidth <= 768; + this.duplicateRetryTimer = null; + this.duplicateIndexNotReady = false; + this.resizeHandler = null; } async init() { @@ -1370,12 +1438,17 @@ class TableManager { } this.bindEvents(); + this.resizeHandler = () => { + const isMobile = window.innerWidth <= 768; + if (isMobile === this.lastLayoutIsMobile) return; + this.lastLayoutIsMobile = isMobile; + if (this.lastAcceptedData && !this.resultsHidden) { + this.renderTable(this.lastAcceptedData); + this.updatePagination(this.lastAcceptedData); + } + }; + window.addEventListener('resize', this.resizeHandler); await this.loadData(); - - // Handle window resize - window.addEventListener('resize', () => { - this.loadData(); - }); } bindEvents() { @@ -1429,6 +1502,11 @@ class TableManager { this.updateSelectionUI(); this.filter = e.target.dataset.filter; + if (this.filter !== 'duplicates') { + this.duplicateIndexNotReady = false; + this.clearDuplicateRetry(); + this.renderDuplicateIndexStatus('', false); + } const duplicateControls = document.querySelector('#duplicate-controls'); if (duplicateControls) duplicateControls.hidden = this.filter !== 'duplicates'; this.currentPage = 1; @@ -1491,7 +1569,10 @@ class TableManager { const selectAll = document.querySelector('#select-all'); if (selectAll) { selectAll.addEventListener('change', (e) => { - const checkboxes = document.querySelectorAll('.file-checkbox'); + const container = window.innerWidth <= 768 + ? document.querySelector('.mobile-results') + : document.querySelector('.table-container'); + const checkboxes = container ? container.querySelectorAll('.file-checkbox') : []; checkboxes.forEach(cb => { cb.checked = e.target.checked; if (e.target.checked) { @@ -1507,6 +1588,10 @@ class TableManager { } async loadData() { + const generation = ++this.requestGeneration; + this.clearDuplicateRetry(); + if (this.filter !== 'duplicates') this.duplicateIndexNotReady = false; + let requestParams = null; try { const params = { page: this.currentPage, @@ -1555,20 +1640,116 @@ class TableManager { } } - const data = await this.api.getScanResults(params); - this.renderTable(data); - this.updatePagination(data); + requestParams = params; + const key = JSON.stringify(params); + if (params.duplicate_mode && key !== this.lastAcceptedKey) { + this.setResultsVisible(false); + this.renderDuplicateIndexStatus('Preparing duplicate results.', false); + } + let request = this.inFlightRequests.get(key); + if (!request) { + request = this.api.getScanResults(params); + this.inFlightRequests.set(key, request); + request.finally(() => { + if (this.inFlightRequests.get(key) === request) this.inFlightRequests.delete(key); + }).catch(() => {}); + } + const data = await request; + if (!data || !Array.isArray(data.results)) return; + if (params.duplicate_mode && data.duplicate_status && data.duplicate_status.ready === false) { + if (generation === this.requestGeneration) this.showDuplicateIndexInitializing(params); + return; + } + if (generation === this.requestGeneration) this.acceptResults(data, params); } catch (error) { + if (generation === this.requestGeneration && this.filter === 'duplicates') { + const duplicateStatus = error && error.data && error.data.duplicate_status; + if (duplicateStatus && duplicateStatus.ready === false) { + this.showDuplicateIndexInitializing(requestParams); + } else { + const hasMatchingData = requestParams && JSON.stringify(requestParams) === this.lastAcceptedKey; + this.renderDuplicateIndexStatus(hasMatchingData + ? 'Duplicate results could not be refreshed. Showing the last available results.' + : 'Duplicate results are unavailable. Refresh will retry.', Boolean(hasMatchingData)); + } + } + } + } + + acceptResults(data, params) { + this.lastAcceptedData = data; + this.lastAcceptedKey = JSON.stringify(params); + this.setResultsVisible(true); + this.renderTable(data); + this.updatePagination(data); + const duplicateStatus = data.duplicate_status; + if (params.duplicate_mode && duplicateStatus && duplicateStatus.stale) { + this.duplicateIndexNotReady = false; + this.renderDuplicateIndexStatus('Duplicate results may be out of date. Refresh is delayed.', true); + } else { + this.duplicateIndexNotReady = false; + this.renderDuplicateIndexStatus('', false); + } + } + + showDuplicateIndexInitializing(params = null) { + this.duplicateIndexNotReady = true; + if (params && JSON.stringify(params) !== this.lastAcceptedKey) this.setResultsVisible(false); + this.renderDuplicateIndexStatus('Preparing duplicate results. Refresh will retry automatically.', false); + this.duplicateRetryTimer = setTimeout(() => { + this.duplicateRetryTimer = null; + if (this.filter === 'duplicates' && this.duplicateIndexNotReady) this.loadData(); + }, 30000); + } + + clearDuplicateRetry() { + if (this.duplicateRetryTimer) { + clearTimeout(this.duplicateRetryTimer); + this.duplicateRetryTimer = null; + } + } + + dispose() { + this.clearDuplicateRetry(); + if (this.resizeHandler) window.removeEventListener('resize', this.resizeHandler); + } + + setResultsVisible(visible) { + this.resultsHidden = !visible; + if (!visible) { + document.querySelector('#results-tbody')?.replaceChildren(); + document.querySelector('.mobile-results')?.replaceChildren(); + const pagination = document.querySelector('.pagination'); + if (pagination) pagination.replaceChildren(); + this.selectedFiles.clear(); + this.lastClickedCheckbox = null; + const selectAll = document.querySelector('#select-all'); + if (selectAll) selectAll.checked = false; + this.updateSelectionUI(); + } + for (const selector of ['.table-container', '.mobile-results', '.pagination']) { + const element = document.querySelector(selector); + if (element) element.style.display = visible ? '' : 'none'; } } + renderDuplicateIndexStatus(message, stale) { + const status = document.querySelector('#duplicate-index-status'); + if (!status) return; + status.textContent = message; + status.hidden = !message; + status.classList.toggle('is-stale', Boolean(stale && message)); + } + renderTable(data) { // Check if mobile const isMobile = window.innerWidth <= 768; if (isMobile) { + document.querySelector('#results-tbody')?.replaceChildren(); this.renderMobileCards(data); } else { + document.querySelector('.mobile-results')?.replaceChildren(); const tbody = document.querySelector('#results-tbody'); if (!tbody) return; @@ -1813,7 +1994,12 @@ class TableManager { // If shift key is pressed and we have a last clicked checkbox, select range if (event.shiftKey && this.lastClickedCheckbox !== null) { // Get all checkboxes currently in the DOM - const allCheckboxes = Array.from(document.querySelectorAll('.file-checkbox')); + const activeContainer = window.innerWidth <= 768 + ? document.querySelector('.mobile-results') + : document.querySelector('.table-container'); + const allCheckboxes = activeContainer + ? Array.from(activeContainer.querySelectorAll('.file-checkbox')) + : []; // Find indices of current and last clicked checkboxes const currentIndex = allCheckboxes.findIndex(cb => parseInt(cb.value) === fileId); @@ -2269,48 +2455,38 @@ class PixelProbeApp { } async init() { - // Initialize components - await this.stats.init(); - await this.table.init(); - - // Populate path filter dropdown + this.stats.init().catch(error => this.stats.renderRefreshStatus(error)); + this.table.init().catch(() => {}); this.loadScanPaths(); - - // Check for ongoing operations + this.detectOngoingOperations(); + } + + async detectOngoingOperations() { try { - // Check for ongoing scan const scanStatus = await this.api.getScanStatus(); - - // Check for active scan (remove stuck detection on page load - it will be handled by monitoring) if (scanStatus.is_scanning || scanStatus.is_running || (scanStatus.phase === 'scanning' && scanStatus.current > 0)) { this.progress.operationType = 'scan'; this.progress.startMonitoring('scan'); - return; // Only monitor one operation at a time + return; } - - // Check for ongoing cleanup + const cleanupStatus = await this.api.getCleanupStatus(); if (cleanupStatus.is_running) { this.progress.operationType = 'cleanup'; - // Attaching to a run in progress: it may be a scheduled one, - // whose kept records are not this person's to confirm. this.progress.startedHere = false; this.progress.startMonitoring('cleanup'); - return; // Only monitor one operation at a time + return; } - - // Check for ongoing file changes check + const fileChangesStatus = await this.api.getFileChangesStatus(); if (fileChangesStatus.is_running) { this.progress.operationType = 'file-changes'; this.progress.startMonitoring('file-changes'); - return; // Only monitor one operation at a time + return; } } catch (error) { } - - // Start background detection for scheduled scans this.startBackgroundScanDetection(); } diff --git a/templates/index.html b/templates/index.html index cdf0d7a..cb15b43 100644 --- a/templates/index.html +++ b/templates/index.html @@ -205,9 +205,9 @@

Scan Progress

Pending Files
-
0
+
Preparing...
Duplicate Files
-
+
Preparing duplicate counts.
All files in matching recorded SHA-256 groups.
@@ -275,6 +275,7 @@

Scan Progress

Matches use recorded hash baselines. Files without hashes are excluded; changes since hashing are not verified.

+
diff --git a/tests/frontend_dom_test.js b/tests/frontend_dom_test.js index 8a287f0..dd561a2 100644 --- a/tests/frontend_dom_test.js +++ b/tests/frontend_dom_test.js @@ -8,7 +8,7 @@ window.fetch = async () => ({ ok: true, json: async () => ({}) }); window.Chart = function () {}; let source = fs.readFileSync('static/js/app.js', 'utf8'); source = source.replace('document.addEventListener(\'DOMContentLoaded\'', '/* test suppresses bootstrap */ document.addEventListener(\'testDOMContentLoaded\''); -source += '\nwindow.__TableManager = TableManager; window.__PixelProbeApp = PixelProbeApp; window.__StatsDashboard = StatsDashboard; window.__ProgressManager = ProgressManager;'; +source += '\nwindow.__TableManager = TableManager; window.__PixelProbeApp = PixelProbeApp; window.__StatsDashboard = StatsDashboard; window.__ProgressManager = ProgressManager; window.__APIClient = APIClient;'; window.eval(source); const table = new window.__TableManager({ getScanResults: async () => ({}) }); @@ -67,6 +67,209 @@ window.document.querySelector('[data-filter="all"]').click(); assert.equal(window.document.querySelector('#duplicate-controls').hidden, true); assert.equal(requestedDuplicateModes.at(-1), undefined); +function makeTableWindow(width = 900) { + const tableDom = new JSDOM(` + +

+ + + + +
+ `, { runScripts: 'outside-only', url: 'http://localhost' }); + const tableWindow = tableDom.window; + Object.defineProperty(tableWindow, 'innerWidth', { value: width, writable: true, configurable: true }); + tableWindow.Chart = function () {}; + tableWindow.app = { viewFile() {}, rescanFile() {}, orphanCheckFile() {}, changeCheckFile() {}, acceptBitrot() {}, viewScanOutput() {}, downloadFile() {}, markFileAsGood() {} }; + let tableSource = fs.readFileSync('static/js/app.js', 'utf8'); + tableSource = tableSource.replace('document.addEventListener(\'DOMContentLoaded\'', 'document.addEventListener(\'testDOMContentLoaded\''); + tableSource += '\nwindow.__TableManager = TableManager; window.__PixelProbeApp = PixelProbeApp; window.__APIClient = APIClient;'; + tableWindow.eval(tableSource); + return tableWindow; +} + +(async () => { + const tableWindow = makeTableWindow(); + let calls = 0; + const table = new tableWindow.__TableManager({ + getScanResults: async () => { + calls++; + return { results: [{ id: 1, file_path: '/media/one.mp4', scan_status: 'completed' }], total: 1 }; + } + }); + await table.init(); + const row = tableWindow.document.querySelector('#results-tbody tr'); + row.querySelector('.file-checkbox').click(); + tableWindow.dispatchEvent(new tableWindow.Event('resize')); + await Promise.resolve(); + assert.equal(calls, 1); + assert.equal(tableWindow.document.querySelector('#results-tbody tr'), row); + + tableWindow.innerWidth = 700; + tableWindow.dispatchEvent(new tableWindow.Event('resize')); + const mobileSelection = tableWindow.document.querySelector('.mobile-results .file-checkbox'); + assert.ok(mobileSelection); + assert.equal(mobileSelection.checked, true); + tableWindow.innerWidth = 900; + tableWindow.dispatchEvent(new tableWindow.Event('resize')); + assert.equal(tableWindow.document.querySelector('#results-tbody .file-checkbox').checked, true); + assert.equal(calls, 1); + table.dispose(); +})(); + +(async () => { + const tableWindow = makeTableWindow(); + const pending = []; + const calls = []; + const table = new tableWindow.__TableManager({ + getScanResults: params => { + calls.push(params.search || ''); + return new Promise(resolve => pending.push({ search: params.search || '', resolve })); + } + }); + table.searchQuery = 'A'; + const firstA = table.loadData(); + table.searchQuery = 'B'; + const onlyB = table.loadData(); + table.searchQuery = 'A'; + const latestA = table.loadData(); + assert.deepEqual(calls, ['A', 'B']); + pending.find(item => item.search === 'A').resolve({ results: [{ id: 3, file_path: 'A.mp4' }], total: 100 }); + await Promise.all([firstA, latestA]); + assert.equal(tableWindow.document.querySelector('#results-tbody .file-path-cell').textContent, 'A.mp4'); + const latestPagination = tableWindow.document.querySelector('.pagination').innerHTML; + pending.find(item => item.search === 'B').resolve({ results: [{ id: 2, file_path: 'B.mp4' }], total: 20 }); + await onlyB; + assert.equal(tableWindow.document.querySelector('#results-tbody .file-path-cell').textContent, 'A.mp4'); + assert.equal(tableWindow.document.querySelector('.pagination').innerHTML, latestPagination); + + table.searchQuery = 'A'; + const refreshedA = table.loadData(); + assert.deepEqual(calls, ['A', 'B', 'A']); + pending[2].resolve({ results: [{ id: 4, file_path: 'A refreshed.mp4' }], total: 150 }); + await refreshedA; + assert.equal(tableWindow.document.querySelector('#results-tbody .file-path-cell').textContent, 'A refreshed.mp4'); + assert.match(tableWindow.document.querySelector('.pagination').textContent, /3/); + table.dispose(); +})(); + +(async () => { + const tableWindow = makeTableWindow(); + let timerCallback; + let nextTimerId = 0; + tableWindow.setTimeout = (callback, delay) => { + assert.equal(delay, 30000); + timerCallback = callback; + return ++nextTimerId; + }; + tableWindow.clearTimeout = () => {}; + let requests = 0; + const table = new tableWindow.__TableManager({ + getScanResults: async () => { + requests++; + if (requests === 1) return { results: [{ id: 5, file_path: 'accepted.mp4' }], total: 1 }; + if (requests === 2 || requests === 4) { + const error = new Error('initializing'); + error.status = 503; + error.data = { duplicate_status: { ready: false, updated_at: null, stale: true } }; + throw error; + } + return { results: [{ id: 6, file_path: 'ready.mp4', duplicate_group_size: 2, duplicate_mode: 'hash' }], + total: 2, duplicate_status: { ready: true, updated_at: '2026-10-10T12:00:00Z', stale: false } }; + } + }); + await table.init(); + const oldRow = tableWindow.document.querySelector('#results-tbody tr'); + table.filter = 'duplicates'; + await table.loadData(); + assert.equal(oldRow.isConnected, false); + assert.equal(tableWindow.document.querySelectorAll('.file-checkbox').length, 0); + assert.equal(tableWindow.document.querySelector('.table-container').style.display, 'none'); + assert.equal(tableWindow.document.querySelector('.pagination').style.display, 'none'); + assert.match(tableWindow.document.querySelector('#duplicate-index-status').textContent, /Preparing duplicate results/); + assert.equal(table.duplicateIndexNotReady, true); + tableWindow.document.querySelector('#select-all').click(); + assert.deepEqual(Array.from(table.selectedFiles), []); + timerCallback(); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(requests, 3); + assert.equal(tableWindow.document.querySelector('#results-tbody .file-path-cell').title, 'ready.mp4'); + assert.equal(tableWindow.document.querySelector('#duplicate-index-status').hidden, true); + + const acceptedDuplicateRow = tableWindow.document.querySelector('#results-tbody tr'); + await table.loadData(); + assert.equal(tableWindow.document.querySelector('#results-tbody tr'), acceptedDuplicateRow); + assert.equal(tableWindow.document.querySelector('.table-container').style.display, ''); + assert.match(tableWindow.document.querySelector('#duplicate-index-status').textContent, /Preparing duplicate results/); + + table.filter = 'duplicates'; + let rejectOldDuplicate; + table.api.getScanResults = params => { + if (params.duplicate_mode) return new Promise((resolve, reject) => { rejectOldDuplicate = reject; }); + return Promise.resolve({ results: [{ id: 7, file_path: 'normal.mp4' }], total: 1 }); + }; + const oldDuplicateRequest = table.loadData(); + table.filter = 'all'; + await table.loadData(); + const oldError = new Error('still initializing'); + oldError.data = { duplicate_status: { ready: false, updated_at: null, stale: true } }; + rejectOldDuplicate(oldError); + await oldDuplicateRequest; + assert.equal(tableWindow.document.querySelector('#results-tbody .file-path-cell').textContent, 'normal.mp4'); + assert.equal(tableWindow.document.querySelector('#duplicate-index-status').hidden, true); + assert.equal(table.duplicateIndexNotReady, false); + assert.equal(table.duplicateRetryTimer, null); + table.dispose(); +})(); + +(async () => { + const tableWindow = makeTableWindow(700); + const table = new tableWindow.__TableManager({ + getScanResults: async params => ({ + results: [{ id: params.search === 'new' ? 22 : 11, + file_path: `${params.search || 'old'}.mp4` }], + total: 1 + }) + }); + await table.init(); + assert.equal(tableWindow.document.querySelectorAll('#results-tbody .file-checkbox').length, 0); + table.searchQuery = 'new'; + await table.loadData(); + tableWindow.innerWidth = 900; + tableWindow.dispatchEvent(new tableWindow.Event('resize')); + assert.equal(tableWindow.document.querySelectorAll('.mobile-results .file-checkbox').length, 0); + assert.deepEqual( + Array.from(tableWindow.document.querySelectorAll('#results-tbody .file-checkbox'), checkbox => checkbox.value), + ['22'] + ); + tableWindow.document.querySelector('#select-all').click(); + assert.deepEqual(Array.from(table.selectedFiles), [22]); + table.dispose(); +})(); + +(async () => { + const appWindow = makeTableWindow(); + appWindow.fetch = async () => ({ + ok: false, status: 503, + headers: { get: name => name === 'Retry-After' ? '30' : null }, + json: async () => ({ error: 'Duplicate index is initializing', duplicate_status: { ready: false, updated_at: null, stale: true } }) + }); + const api = new appWindow.__APIClient(); + await assert.rejects(api.getStats(), error => error.status === 503 && error.data.duplicate_status.ready === false); + + let unblockStats; + const calls = []; + const instance = { + stats: { init: () => new Promise(resolve => { unblockStats = resolve; }) }, + table: { init: () => { calls.push('table'); return Promise.resolve(); } }, + loadScanPaths: () => { calls.push('paths'); }, + detectOngoingOperations: () => { calls.push('operations'); } + }; + await appWindow.__PixelProbeApp.prototype.init.call(instance); + assert.deepEqual(calls, ['table', 'paths', 'operations']); + unblockStats(); +})(); + for (const [statusFile, statusClass, statusText] of statusCases) { const file = { id: 100, file_path: payload, file_size: 0, ...statusFile }; const desktopStatus = table.renderRow(file).children[1].querySelector('.badge'); @@ -266,6 +469,34 @@ async function verifyViewer() { assert.match(statsWindow.document.querySelector('#integrity-details').textContent, /with no recorded recheck attempt 2/); assert.match(statsWindow.document.querySelector('#integrity-details').textContent, /Legacy integrity outcomes were not recorded/); assert.match(statsWindow.document.querySelector('#integrity-refresh-status').textContent, /Last refreshed/); + dashboard.renderStats({ total_files: 8, healthy_files: 3, corrupted_files: 1, warning_files: 0, + pending_files: 0, scanning_files: 0, duplicate_files: null, duplicate_groups: null, + duplicate_extra_files: null, duplicate_status: { ready: false, updated_at: null, stale: true } }); + assert.equal(statsWindow.document.querySelector('#duplicate-files').textContent, '2'); + assert.match(statsWindow.document.querySelector('#duplicate-details').textContent, /Refresh delayed/); + dashboard.renderStats({ total_files: 8, healthy_files: 3, corrupted_files: 1, warning_files: 0, + pending_files: 0, scanning_files: 0, duplicate_files: 4, duplicate_groups: 2, + duplicate_extra_files: 2, duplicate_status: { ready: true, updated_at: '2026-10-10T12:00:00Z', stale: true } }); + assert.equal(statsWindow.document.querySelector('#duplicate-files').textContent, '4'); + assert.match(statsWindow.document.querySelector('#duplicate-details').textContent, /Refresh delayed/); + + let finishStats; + let statsCalls = 0; + const coalescedDashboard = new statsWindow.__StatsDashboard({ + getStats: () => { + statsCalls++; + return new Promise(resolve => { finishStats = resolve; }); + } + }); + const statsRequestOne = coalescedDashboard.updateStats(); + const statsRequestTwo = coalescedDashboard.updateStats(); + assert.equal(statsCalls, 1); + finishStats({ total_files: 1, healthy_files: 1, corrupted_files: 0, warning_files: 0, + pending_files: 0, scanning_files: 0, duplicate_files: null, duplicate_groups: null, + duplicate_extra_files: null, duplicate_status: { ready: false, updated_at: null, stale: true } }); + await Promise.all([statsRequestOne, statsRequestTwo]); + assert.equal(statsWindow.document.querySelector('#duplicate-files').textContent, 'Preparing...'); + shouldFail = true; dashboard.startAutoRefresh(); await dashboard._statsPoll(); diff --git a/tests/test_duplicate_index_postgres.py b/tests/test_duplicate_index_postgres.py new file mode 100644 index 0000000..1a59190 --- /dev/null +++ b/tests/test_duplicate_index_postgres.py @@ -0,0 +1,493 @@ +import os +import threading +import time +from datetime import datetime, timedelta, timezone +from uuid import uuid4 + +import pytest +from flask import Flask +from sqlalchemy import create_engine, event, literal, text +from sqlalchemy.pool import NullPool + +from pixelprobe.models import (db as model_db, DuplicateDirtyKey, + DuplicateGroupSummary, DuplicateIndexState, + ScanResult) +from pixelprobe.services import duplicate_service +from pixelprobe.services.duplicate_service import ( + duplicate_index_status, duplicate_statistics, refresh_duplicate_index, +) + + +@pytest.fixture +def postgres_duplicates(): + uri = os.environ['PIXELPROBE_TEST_POSTGRES_URI'] + schema = f'dupidx_{uuid4().hex[:12]}' + admin_engine = create_engine(uri) + with admin_engine.begin() as connection: + connection.execute(text(f'CREATE SCHEMA {schema}')) + + app = Flask(__name__) + app.config.update( + SQLALCHEMY_DATABASE_URI=uri, + SQLALCHEMY_ENGINE_OPTIONS={ + 'connect_args': {'options': f'-csearch_path={schema}'}, + }, + SQLALCHEMY_TRACK_MODIFICATIONS=False, + ) + model_db.init_app(app) + try: + with app.app_context(): + model_db.create_all() + from pixelprobe.migrations.duplicates import run_duplicate_index_migrations + run_duplicate_index_migrations(model_db) + yield app + finally: + with app.app_context(): + model_db.session.remove() + model_db.engine.dispose() + with admin_engine.begin() as connection: + connection.execute(text(f'DROP SCHEMA {schema} CASCADE')) + admin_engine.dispose() + + +pytestmark = [ + pytest.mark.postgres, + pytest.mark.skipif( + not os.environ.get('PIXELPROBE_TEST_POSTGRES_URI'), + reason='PIXELPROBE_TEST_POSTGRES_URI not set', + ), +] + + +def _scan(path, file_hash='hash-a', size=10, **fields): + return ScanResult(file_path=path, file_hash=file_hash, + file_size=size, **fields) + + +def _dirty_keys(): + return set(model_db.session.query( + DuplicateDirtyKey.mode, DuplicateDirtyKey.key_text, + DuplicateDirtyKey.key_size, + ).all()) + + +def _summary_rows(): + return set(model_db.session.query( + DuplicateGroupSummary.mode, DuplicateGroupSummary.key_text, + DuplicateGroupSummary.key_size, DuplicateGroupSummary.group_size, + DuplicateGroupSummary.group_id, + ).all()) + + +def _refresh_until_clean(limit=5): + for _ in range(limit): + assert refresh_duplicate_index() + model_db.session.expire_all() + if not _dirty_keys(): + return + assert not _dirty_keys() + + +def _group_oracle(mode): + groups, _ = duplicate_service._duplicate_groups(mode) + return set(model_db.session.query( + groups.c.key_0, groups.c.key_1 if mode == 'hash' else literal(-1), + groups.c.group_size, groups.c.group_id, + ).all()) + + +def _assert_summary_matches_oracle(): + model_db.session.expire_all() + actual = _summary_rows() + expected = set() + for mode in ('hash', 'name'): + for key_text, key_size, group_size, group_id in _group_oracle(mode): + expected.add((mode, key_text, key_size, group_size, group_id)) + assert actual == expected + + +def test_postgres_backfill_bulk_mutations_and_case_sensitive_names(postgres_duplicates): + with postgres_duplicates.app_context(): + assert duplicate_index_status() == { + 'ready': False, 'updated_at': None, 'stale': False, + } + assert duplicate_statistics() == { + 'duplicate_files': None, 'duplicate_groups': None, + 'duplicate_extra_files': None, 'filename_duplicate_files': None, + 'filename_duplicate_groups': None, + 'filename_duplicate_extra_files': None, + } + model_db.session.add_all([ + _scan('/one/Photo.JPG'), _scan('/two/Photo.JPG'), + _scan('/three/photo.jpg'), + ]) + model_db.session.commit() + + assert refresh_duplicate_index() + model_db.session.expire_all() + assert duplicate_index_status()['ready'] is True + assert _summary_rows() == { + ('hash', 'hash-a', 10, 3, 1), + ('name', 'Photo.JPG', -1, 2, 1), + } + _refresh_until_clean() + + model_db.session.add(_scan('/four/Photo.JPG', file_hash='hash-b', size=20)) + model_db.session.commit() + _refresh_until_clean() + _assert_summary_matches_oracle() + assert ('name', 'Photo.JPG', -1, 3, 1) in _summary_rows() + assert ('hash', 'hash-a', 10, 3, 1) in _summary_rows() + + model_db.session.query(ScanResult).filter_by(file_path='/one/Photo.JPG').update( + {'file_hash': 'hash-b', 'file_size': 20}, synchronize_session=False) + model_db.session.commit() + _refresh_until_clean() + _assert_summary_matches_oracle() + assert ('hash', 'hash-a', 10, 2, 2) in _summary_rows() + assert ('hash', 'hash-b', 20, 2, 1) in _summary_rows() + + model_db.session.query(ScanResult).filter_by(file_path='/two/Photo.JPG').delete( + synchronize_session=False) + model_db.session.commit() + _refresh_until_clean() + _assert_summary_matches_oracle() + assert ('hash', 'hash-a', 10, 2, 3) not in _summary_rows() + assert ('name', 'Photo.JPG', -1, 2, 1) in _summary_rows() + + from pixelprobe.migrations.duplicates import run_duplicate_index_migrations + run_duplicate_index_migrations(model_db) + assert refresh_duplicate_index() + model_db.session.expire_all() + _assert_summary_matches_oracle() + + +def test_postgres_trigger_tracks_eligibility_and_ignores_irrelevant_updates(postgres_duplicates): + with postgres_duplicates.app_context(): + rows = [ + _scan('/a/shared.bin'), _scan('/b/shared.bin'), + _scan('/c/Same.bin', file_hash='hash-c', size=11), + _scan('/d/Same.bin', file_hash='hash-c', size=11), + ] + model_db.session.add_all(rows) + model_db.session.commit() + assert refresh_duplicate_index() + _refresh_until_clean() + model_db.session.expire_all() + + model_db.session.query(ScanResult).filter_by(file_path='/a/shared.bin').update( + {'scan_status': 'completed'}, synchronize_session=False) + model_db.session.query(ScanResult).filter_by(file_path='/b/shared.bin').update( + {'scan_tool': 'scanner-v2'}, synchronize_session=False) + model_db.session.query(ScanResult).filter_by(file_path='/c/Same.bin').update( + {'file_path': '/elsewhere/Same.bin'}, synchronize_session=False) + model_db.session.commit() + assert _dirty_keys() == set() + + transitions = [ + ('bitrot_suspected', True), ('scan_status', 'failed'), + ('scan_tool', 'error'), ('last_integrity_outcome', 'unreadable'), + ('file_exists', False), ('file_hash', None), ('file_size', None), + ] + baseline = { + 'bitrot_suspected': False, 'scan_status': 'pending', + 'scan_tool': None, 'last_integrity_outcome': None, + 'file_exists': True, 'file_hash': 'hash-a', 'file_size': 10, + } + for field, invalid_value in transitions: + model_db.session.query(ScanResult).filter_by(file_path='/a/shared.bin').update( + baseline, synchronize_session=False) + model_db.session.commit() + _refresh_until_clean() + _assert_summary_matches_oracle() + model_db.session.query(ScanResult).filter_by(file_path='/a/shared.bin').update( + {field: invalid_value}, synchronize_session=False) + model_db.session.commit() + assert ('hash', 'hash-a', 10) in _dirty_keys(), ( + f'eligibility change did not enqueue: {field}' + ) + _refresh_until_clean() + _assert_summary_matches_oracle() + + +def test_postgres_concurrent_insert_requeues_claimed_key(postgres_duplicates, monkeypatch): + app = postgres_duplicates + entered_refresh = threading.Event() + allow_refresh = threading.Event() + writer_started = threading.Event() + writer_done = threading.Event() + errors = [] + writer_pid = {} + + with app.app_context(): + model_db.session.add_all([ + _scan('/a/group.dat'), _scan('/b/group.dat'), + ]) + model_db.session.commit() + assert refresh_duplicate_index() + _refresh_until_clean() + model_db.session.add(_scan('/c/group.dat')) + model_db.session.commit() + model_db.session.expire_all() + assert ('hash', 'hash-a', 10) in _dirty_keys() + + original = duplicate_service._refresh_claimed_keys + + def blocked_refresh(conn, keys, deadline): + entered_refresh.set() + if not allow_refresh.wait(5): + raise AssertionError('test did not release the refresh worker') + return original(conn, keys, deadline) + + monkeypatch.setattr(duplicate_service, '_refresh_claimed_keys', blocked_refresh) + + def refresh_worker(): + try: + with app.app_context(): + calls = {'count': 0} + + def dispatch(): + calls['count'] += 1 + return calls['count'] <= 5 + + refresh_duplicate_index(batch_size=1, can_dispatch=dispatch) + except Exception as exc: + errors.append(exc) + + def insert_worker(): + try: + with app.app_context(): + connection = model_db.engine.connect() + pid = connection.execute(text('SELECT pg_backend_pid()')).scalar_one() + connection.commit() + writer_pid['value'] = pid + writer_started.set() + connection.execute(text(""" + INSERT INTO scan_results (file_path, file_hash, file_size, + file_exists, bitrot_suspected, marked_as_good, has_warnings) + VALUES ('/d/group.dat', 'hash-a', 10, TRUE, FALSE, FALSE, FALSE) + """)) + connection.commit() + connection.close() + writer_done.set() + errors.append(('writer_pid', pid)) + except Exception as exc: + errors.append(exc) + + refresh_thread = threading.Thread(target=refresh_worker) + refresh_thread.start() + assert entered_refresh.wait(5) + writer_thread = threading.Thread(target=insert_worker) + writer_thread.start() + assert writer_started.wait(5) + + blockers = [] + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + worker_errors = [error for error in errors if isinstance(error, Exception)] + if worker_errors: + raise worker_errors[0] + with app.app_context(): + connection = model_db.engine.connect() + blockers = connection.execute(text( + 'SELECT pg_blocking_pids(:pid)'), {'pid': writer_pid['value']}, + ).scalar_one() + connection.close() + if blockers: + break + threading.Event().wait(0.02) + if not blockers: + with app.app_context(): + activity = model_db.session.execute(text(""" + SELECT wait_event_type, wait_event, state, query + FROM pg_stat_activity WHERE pid = :pid + """), {'pid': writer_pid['value']}).one_or_none() + assert blockers, f'writer was not blocked; done={writer_done.is_set()}, activity={activity}' + + allow_refresh.set() + refresh_thread.join(10) + writer_thread.join(10) + assert not refresh_thread.is_alive() + assert not writer_thread.is_alive() + assert not [error for error in errors if isinstance(error, Exception)] + + with app.app_context(): + model_db.session.expire_all() + assert ('hash', 'hash-a', 10) in _dirty_keys() + _refresh_until_clean() + _assert_summary_matches_oracle() + assert ('hash', 'hash-a', 10, 4, 1) in _summary_rows() + + +def test_postgres_incremental_failure_rolls_back_summary_and_retries(postgres_duplicates, monkeypatch): + with postgres_duplicates.app_context(): + model_db.session.add_all([_scan('/a/fail.dat'), _scan('/b/fail.dat')]) + model_db.session.commit() + assert refresh_duplicate_index() + _refresh_until_clean() + baseline = _summary_rows() + + model_db.session.query(ScanResult).filter_by(file_path='/a/fail.dat').update( + {'file_size': 99}, synchronize_session=False) + model_db.session.commit() + model_db.session.expire_all() + pending = _dirty_keys() + assert pending + original = duplicate_service._refresh_claimed_keys + + def fail_after_writes(conn, keys, deadline): + original(conn, keys, deadline) + raise RuntimeError('forced transaction rollback') + + monkeypatch.setattr(duplicate_service, '_refresh_claimed_keys', fail_after_writes) + with pytest.raises(RuntimeError, match='forced transaction rollback'): + refresh_duplicate_index() + model_db.session.expire_all() + assert _summary_rows() == baseline + assert _dirty_keys() == pending + assert duplicate_index_status()['stale'] is True + + monkeypatch.setattr(duplicate_service, '_refresh_claimed_keys', original) + _refresh_until_clean() + _assert_summary_matches_oracle() + assert duplicate_index_status()['stale'] is False + + +def test_postgres_backfill_failure_is_retryable_and_migration_is_idempotent( + postgres_duplicates, monkeypatch): + with postgres_duplicates.app_context(): + model_db.session.add_all([_scan('/a/retry.dat'), _scan('/b/retry.dat')]) + model_db.session.commit() + from pixelprobe.migrations.duplicates import run_duplicate_index_migrations + run_duplicate_index_migrations(model_db) + model_db.session.expire_all() + assert duplicate_index_status()['ready'] is False + original = duplicate_service._initial_grouping + + def fail_after_backfill(conn): + original(conn) + raise RuntimeError('forced backfill failure') + + monkeypatch.setattr(duplicate_service, '_initial_grouping', fail_after_backfill) + with pytest.raises(RuntimeError, match='forced backfill failure'): + refresh_duplicate_index() + model_db.session.expire_all() + state = model_db.session.get(DuplicateIndexState, 1) + assert state.ready is False + assert _summary_rows() == set() + + monkeypatch.setattr(duplicate_service, '_initial_grouping', original) + assert refresh_duplicate_index() + model_db.session.expire_all() + assert duplicate_index_status()['ready'] is True + _refresh_until_clean() + _assert_summary_matches_oracle() + run_duplicate_index_migrations(model_db) + assert refresh_duplicate_index() + model_db.session.expire_all() + _assert_summary_matches_oracle() + + +def test_postgres_refresh_failures_release_advisory_lock(postgres_duplicates, monkeypatch): + lock_id = duplicate_service._DUPLICATE_REFRESH_LOCK_ID + + def assert_lock_free(): + verifier_engine = create_engine( + os.environ['PIXELPROBE_TEST_POSTGRES_URI'], poolclass=NullPool, + ) + try: + connection = verifier_engine.connect() + acquired = connection.execute(text( + 'SELECT pg_try_advisory_lock(:lock_id)'), {'lock_id': lock_id}, + ).scalar_one() + assert acquired is True + connection.execute(text( + 'SELECT pg_advisory_unlock(:lock_id)'), {'lock_id': lock_id}, + ) + connection.commit() + connection.close() + finally: + verifier_engine.dispose() + + with postgres_duplicates.app_context(): + original_ensure = duplicate_service._ensure_duplicate_indexes + monkeypatch.setattr(duplicate_service, '_ensure_duplicate_indexes', + lambda can_dispatch=None: (_ for _ in ()).throw( + RuntimeError('index prep failed'))) + with pytest.raises(RuntimeError, match='index prep failed'): + refresh_duplicate_index() + assert_lock_free() + + monkeypatch.setattr(duplicate_service, '_ensure_duplicate_indexes', original_ensure) + assert refresh_duplicate_index(can_dispatch=lambda: False) is False + assert_lock_free() + + model_db.session.add_all([_scan('/a/sql-fail.dat'), _scan('/b/sql-fail.dat')]) + model_db.session.commit() + assert refresh_duplicate_index() + _refresh_until_clean() + original_grouping = duplicate_service._initial_grouping + + def sql_failure(conn): + conn.execute(text('SELECT 1 / 0')) + + monkeypatch.setattr(duplicate_service, '_initial_grouping', sql_failure) + model_db.session.query(DuplicateIndexState).filter_by(id=1).update( + {'ready': False}, synchronize_session=False) + model_db.session.commit() + with pytest.raises(Exception): + refresh_duplicate_index() + assert_lock_free() + monkeypatch.setattr(duplicate_service, '_initial_grouping', original_grouping) + + +def test_postgres_index_preparation_resets_statement_timeout(postgres_duplicates): + with postgres_duplicates.app_context(): + observed_timeout = {} + + def fail_index_creation(connection, cursor, statement, parameters, context, executemany): + if statement.startswith('CREATE INDEX CONCURRENTLY'): + raise RuntimeError('forced concurrent index failure') + + def capture_reset(connection, cursor, statement, parameters, context, executemany): + if statement.strip().upper() == 'RESET STATEMENT_TIMEOUT': + cursor.execute('SHOW statement_timeout') + observed_timeout['value'] = cursor.fetchone()[0] + + event.listen(model_db.engine, 'before_cursor_execute', fail_index_creation) + event.listen(model_db.engine, 'after_cursor_execute', capture_reset) + try: + with pytest.raises(RuntimeError, match='forced concurrent index failure'): + duplicate_service._ensure_duplicate_indexes() + finally: + event.remove(model_db.engine, 'before_cursor_execute', fail_index_creation) + event.remove(model_db.engine, 'after_cursor_execute', capture_reset) + assert observed_timeout.get('value') == '0' + assert duplicate_service._ensure_duplicate_indexes() is True + + +def test_postgres_stale_status_uses_queue_age_or_refresh_failure(postgres_duplicates): + with postgres_duplicates.app_context(): + model_db.session.add_all([_scan('/a/stale.dat'), _scan('/b/stale.dat')]) + model_db.session.commit() + assert refresh_duplicate_index() + _refresh_until_clean() + model_db.session.add(DuplicateDirtyKey( + mode='hash', key_text='hash-a', key_size=10, + enqueued_at=datetime.now(timezone.utc) - timedelta(seconds=61), + )) + model_db.session.commit() + assert duplicate_index_status()['stale'] is True + model_db.session.query(DuplicateDirtyKey).delete() + model_db.session.query(DuplicateIndexState).filter_by(id=1).update( + {'refresh_failed': True}, synchronize_session=False) + model_db.session.commit() + assert duplicate_index_status()['stale'] is True + model_db.session.query(DuplicateIndexState).filter_by(id=1).update( + {'refresh_failed': False}, synchronize_session=False) + model_db.session.add(DuplicateDirtyKey( + mode='hash', key_text='hash-a', key_size=10, + enqueued_at=datetime.now(timezone.utc), + )) + model_db.session.commit() + assert duplicate_index_status()['stale'] is False diff --git a/tests/test_duplicates.py b/tests/test_duplicates.py index 1cce5fd..661b7ba 100644 --- a/tests/test_duplicates.py +++ b/tests/test_duplicates.py @@ -1,12 +1,18 @@ import os from uuid import uuid4 +from datetime import datetime, timezone import pytest from flask import Flask from sqlalchemy import create_engine, text -from pixelprobe.models import db as model_db, ScanResult -from pixelprobe.services.duplicate_service import duplicate_members, duplicate_statistics +from pixelprobe.models import (db as model_db, DuplicateGroupSummary, + DuplicateIndexState, ScanResult) +from pixelprobe.services.duplicate_service import (_duplicate_groups, + duplicate_index_status, + duplicate_members, + duplicate_statistics, + refresh_duplicate_index) from pixelprobe.services.stats_service import StatsService @@ -31,8 +37,26 @@ def seed_duplicates(session): return rows -def assert_grouping(session): +def prime_sqlite_cache(session): + for mode in ('hash', 'name'): + groups, _ = _duplicate_groups(mode) + for row in session.query(groups).all(): + session.add(DuplicateGroupSummary( + mode=mode, + key_text=row.key_0, + key_size=row.key_1 if mode == 'hash' else -1, + group_size=row.group_size, + group_id=row.group_id, + )) + session.add(DuplicateIndexState(id=1, ready=True, + updated_at=datetime.now(timezone.utc))) + session.commit() + + +def assert_grouping(session, prime=True): seed_duplicates(session) + if prime: + prime_sqlite_cache(session) stats = duplicate_statistics() assert stats == { 'duplicate_files': 3, 'duplicate_groups': 1, 'duplicate_extra_files': 2, @@ -50,8 +74,25 @@ def test_duplicate_eligibility_and_count_semantics(db): assert StatsService().get_file_statistics()['duplicate_files'] == 3 +def test_duplicate_api_waits_for_cache_before_serving_duplicate_results( + db, authenticated_client): + status = duplicate_index_status() + assert status == {'ready': False, 'updated_at': None, 'stale': False} + response = authenticated_client.get('/api/scan-results?duplicate_mode=hash') + assert response.status_code == 503 + assert response.headers['Retry-After'] == '30' + assert response.get_json() == { + 'error': 'Duplicate index is initializing', + 'duplicate_status': status, + } + stats = authenticated_client.get('/api/stats').get_json() + assert stats['duplicate_files'] is None + assert stats['duplicate_status']['ready'] is False + + def test_duplicate_api_global_groups_pagination_and_stats(db, authenticated_client, monkeypatch): seed_duplicates(db.session) + prime_sqlite_cache(db.session) monkeypatch.setattr('pixelprobe.api.scan_routes.get_configured_scan_paths', lambda: ['/media']) response = authenticated_client.get('/api/scan-results?duplicate_mode=hash&path=/media&per_page=1&sort_field=file_path&sort_order=asc') assert response.status_code == 200 @@ -89,7 +130,24 @@ def test_postgres_duplicate_basename_and_grouping(): try: with app.app_context(): model_db.create_all() - assert_grouping(model_db.session) + from pixelprobe.migrations.duplicates import run_duplicate_index_migrations + run_duplicate_index_migrations(model_db) + seed_duplicates(model_db.session) + assert refresh_duplicate_index() + assert duplicate_index_status()['ready'] is True + assert refresh_duplicate_index() + assert duplicate_statistics()['duplicate_files'] == 3 + stats = duplicate_statistics() + assert stats == { + 'duplicate_files': 3, 'duplicate_groups': 1, + 'duplicate_extra_files': 2, + 'filename_duplicate_files': 3, 'filename_duplicate_groups': 1, + 'filename_duplicate_extra_files': 2, + } + for mode in ('hash', 'name'): + members = duplicate_members(mode) + assert model_db.session.query(members).count() == 3 + assert {row.group_size for row in model_db.session.query(members)} == {3} model_db.session.remove() model_db.engine.dispose() finally: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 8a0fa3e..7c7e2eb 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -237,6 +237,11 @@ def test_sync_job_registered(self, scheduler): # the sync job must never match that prefix or it would delete itself assert not job.id.startswith('schedule_') + def test_duplicate_refresh_uses_its_own_single_thread_executor(self, scheduler): + job = scheduler.scheduler.get_job('duplicate_index_refresh') + assert job is not None + assert job.executor == 'duplicate_index' + def test_sync_reloads_when_definitions_change(self, scheduler, app, db): with app.app_context(): schedule = ScanSchedule(name='Sync Test', cron_expression='0 2 * * *', diff --git a/tests/unit/test_stats_service.py b/tests/unit/test_stats_service.py index a00e102..2dcda2e 100644 --- a/tests/unit/test_stats_service.py +++ b/tests/unit/test_stats_service.py @@ -26,8 +26,12 @@ def duplicate_stats(self): 'filename_duplicate_groups': 3, 'filename_duplicate_extra_files': 6, } - with patch('pixelprobe.services.stats_service.duplicate_statistics', return_value=counts) as mocked: - yield counts, mocked + status = {'ready': True, 'updated_at': None, 'stale': False} + with patch('pixelprobe.services.stats_service.duplicate_index_status', + return_value=status), \ + patch('pixelprobe.services.stats_service.duplicate_statistics', + return_value=counts) as mocked: + yield counts, status, mocked @patch('pixelprobe.services.stats_service.db') def test_get_file_statistics_success(self, mock_db, stats_service, duplicate_stats): @@ -49,9 +53,10 @@ def test_get_file_statistics_success(self, mock_db, stats_service, duplicate_sta assert stats['healthy_files'] == 85 assert stats['marked_as_good'] == 3 assert stats['warning_files'] == 2 - counts, mocked = duplicate_stats + counts, status, mocked = duplicate_stats assert {key: stats[key] for key in counts} == counts - mocked.assert_called_once_with() + assert stats['duplicate_status'] == status + mocked.assert_called_once_with(status) @patch('pixelprobe.services.stats_service.db') @patch('pixelprobe.services.stats_service.ScanResult') @@ -69,9 +74,10 @@ def test_get_file_statistics_fallback(self, mock_scan_result, mock_db, stats_ser assert stats['total_files'] == 100 assert stats['completed_files'] == 80 - counts, mocked = duplicate_stats + counts, status, mocked = duplicate_stats assert {key: stats[key] for key in counts} == counts - mocked.assert_called_once_with() + assert stats['duplicate_status'] == status + mocked.assert_called_once_with(status) @patch('pixelprobe.services.stats_service.db') def test_get_system_info(self, mock_db, stats_service): From a95a88ecc888e4f923e1522299e27c539985c3ee Mon Sep 17 00:00:00 2001 From: ttlequals0 Date: Sat, 10 Oct 2026 08:25:26 -0400 Subject: [PATCH 4/5] Return explicit JSON responses for scan results --- pixelprobe/api/scan_routes.py | 6 +++--- tests/test_duplicates.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/pixelprobe/api/scan_routes.py b/pixelprobe/api/scan_routes.py index e1a3b4a..bf0da03 100644 --- a/pixelprobe/api/scan_routes.py +++ b/pixelprobe/api/scan_routes.py @@ -1,4 +1,4 @@ -from flask import Blueprint, request, current_app +from flask import Blueprint, jsonify, request, current_app import os import threading import logging @@ -218,7 +218,7 @@ def get_scan_results(): } if duplicate_status is not None: empty_response['duplicate_status'] = duplicate_status - return empty_response + return jsonify(empty_response) # Apply search filter if search_query: @@ -338,7 +338,7 @@ def __init__(self, items, total): } if duplicate_status is not None: response['duplicate_status'] = duplicate_status - return response + return jsonify(response) @scan_bp.route('/scan-results/') @auth_required diff --git a/tests/test_duplicates.py b/tests/test_duplicates.py index 661b7ba..3ec9b34 100644 --- a/tests/test_duplicates.py +++ b/tests/test_duplicates.py @@ -115,6 +115,36 @@ def test_duplicate_api_global_groups_pagination_and_stats(db, authenticated_clie assert authenticated_client.get('/api/scan-results?duplicate_mode=visual').status_code == 400 +def test_scan_results_json_preserves_script_like_search_and_filename( + db, authenticated_client): + html_like = '' + result = ScanResult(file_path=f'/media/{html_like}.mp4', file_size=10) + db.session.add(result) + db.session.commit() + + response = authenticated_client.get( + '/api/scan-results', query_string={'search': html_like}) + + assert response.status_code == 200 + assert response.content_type == 'application/json' + assert response.get_json()['results'][0]['file_name'] == f'{html_like}.mp4' + + +def test_scan_results_invalid_path_returns_json_empty_response( + db, authenticated_client, monkeypatch): + monkeypatch.setattr( + 'pixelprobe.api.scan_routes.get_configured_scan_paths', lambda: ['/media']) + + response = authenticated_client.get( + '/api/scan-results', query_string={'path': '/outside/'}) + + assert response.status_code == 200 + assert response.content_type == 'application/json' + assert response.get_json() == { + 'results': [], 'total': 0, 'page': 1, 'per_page': 100, 'pages': 0, + } + + @pytest.mark.postgres @pytest.mark.skipif(not os.environ.get('PIXELPROBE_TEST_POSTGRES_URI'), reason='PIXELPROBE_TEST_POSTGRES_URI not set') def test_postgres_duplicate_basename_and_grouping(): From 38893e1b79605818c5832e3cc22bd2446630786c Mon Sep 17 00:00:00 2001 From: ttlequals0 Date: Sat, 10 Oct 2026 09:41:28 -0400 Subject: [PATCH 5/5] Collapse duplicate details by default for 2.10.2 --- CHANGELOG.MD | 6 +++++ pixelprobe/version.py | 2 +- static/css/desktop.css | 20 +++++++++++----- static/js/app.js | 49 +++++++++++++++++++++++++++++++++++++- templates/index.html | 12 ++++++++-- tests/frontend_dom_test.js | 33 +++++++++++++++++++++---- 6 files changed, 108 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 555f8ee..6cfe537 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.10.2] - 2026-10-10 + +### Changed + +- Duplicate statistics keep the file count visible and place supporting details behind a Details toggle. + ## [2.10.1] - 2026-10-10 ### Changed diff --git a/pixelprobe/version.py b/pixelprobe/version.py index ab14855..00a9b46 100644 --- a/pixelprobe/version.py +++ b/pixelprobe/version.py @@ -4,7 +4,7 @@ # Default version - this is the single source of truth -_DEFAULT_VERSION = '2.10.1' +_DEFAULT_VERSION = '2.10.2' # Allow override via environment variable for CI/CD, but default to the hardcoded version diff --git a/static/css/desktop.css b/static/css/desktop.css index 6f4a66f..fa24d89 100644 --- a/static/css/desktop.css +++ b/static/css/desktop.css @@ -417,7 +417,8 @@ body { color: var(--warning-color, #b7791f); } -.integrity-details-toggle { +.integrity-details-toggle, +.duplicate-details-toggle { display: inline-flex; align-items: center; align-self: flex-start; @@ -433,24 +434,31 @@ body { } .integrity-details-toggle:hover, -.integrity-details-toggle:focus-visible { +.integrity-details-toggle:focus-visible, +.duplicate-details-toggle:hover, +.duplicate-details-toggle:focus-visible { color: var(--primary-green); } -.integrity-details-toggle i { +.integrity-details-toggle i, +.duplicate-details-toggle i { transition: transform 0.2s ease; } -.integrity-details-toggle[aria-expanded="true"] .fa-chevron-down { +.integrity-details-toggle[aria-expanded="true"] .fa-chevron-down, +.duplicate-details-toggle[aria-expanded="true"] .fa-chevron-down { transform: rotate(180deg); } .integrity-details-toggle .integrity-refresh-warning, -.integrity-details-toggle.is-stale { +.integrity-details-toggle.is-stale, +.duplicate-details-toggle .duplicate-refresh-warning, +.duplicate-details-toggle.is-stale { color: var(--warning-color, #b7791f); } -.integrity-refresh-warning[hidden] { +.integrity-refresh-warning[hidden], +.duplicate-refresh-warning[hidden] { display: none !important; } diff --git a/static/js/app.js b/static/js/app.js index 2fdf0f9..f4dc95a 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -386,6 +386,7 @@ class StatsDashboard { this.statsRequest = null; this.statsPollInFlight = false; this.setupIntegrityDetailsToggle(); + this.setupDuplicateDetailsToggle(); } setupIntegrityDetailsToggle() { @@ -400,6 +401,18 @@ class StatsDashboard { }); } + setupDuplicateDetailsToggle() { + const toggle = document.querySelector('#duplicate-details-toggle'); + const panel = document.querySelector('#duplicate-detail-panel'); + if (!toggle || !panel || toggle.dataset.bound) return; + toggle.dataset.bound = 'true'; + toggle.addEventListener('click', () => { + const expanded = toggle.getAttribute('aria-expanded') === 'true'; + toggle.setAttribute('aria-expanded', String(!expanded)); + panel.hidden = expanded; + }); + } + async init() { try { await this.updateStats(); @@ -451,12 +464,20 @@ class StatsDashboard { this.lastDuplicateStats = { fileCount, groupCount, extraCount, updatedAt: status.updated_at || null }; this.updateStatCard('duplicate-files', fileCount); const details = document.querySelector('#duplicate-details'); + const toggle = document.querySelector('#duplicate-details-toggle'); + const warning = document.querySelector('.duplicate-refresh-warning'); if (details) { const lastUpdated = status.updated_at ? ` Last updated ${new Date(status.updated_at).toLocaleString()}.` : ''; const refresh = status.stale ? '. Refresh delayed.' : ''; details.textContent = `${groupCount} ${groupCount === 1 ? 'group' : 'groups'}; ${extraCount} extra ${extraCount === 1 ? 'file' : 'files'}${refresh}${lastUpdated}`; details.classList.toggle('is-stale', Boolean(status.stale)); } + if (toggle) { + toggle.classList.toggle('is-stale', Boolean(status.stale)); + toggle.title = status.stale ? 'Duplicate stats refresh is delayed. Open details for the latest counts.' : + 'Show duplicate file details'; + } + if (warning) warning.hidden = !status.stale; return; } @@ -467,6 +488,13 @@ class StatsDashboard { details.textContent = 'Refresh delayed; showing the last available duplicate counts.'; details.classList.add('is-stale'); } + const toggle = document.querySelector('#duplicate-details-toggle'); + const warning = document.querySelector('.duplicate-refresh-warning'); + if (toggle) { + toggle.classList.add('is-stale'); + toggle.title = 'Duplicate stats refresh is delayed. Open details for the latest counts.'; + } + if (warning) warning.hidden = false; } else { const count = document.querySelector('#duplicate-files'); if (count) count.textContent = 'Preparing...'; @@ -475,6 +503,13 @@ class StatsDashboard { details.textContent = 'Preparing duplicate counts.'; details.classList.remove('is-stale'); } + const toggle = document.querySelector('#duplicate-details-toggle'); + const warning = document.querySelector('.duplicate-refresh-warning'); + if (toggle) { + toggle.classList.remove('is-stale'); + toggle.title = 'Show duplicate file details'; + } + if (warning) warning.hidden = true; } } @@ -518,7 +553,7 @@ class StatsDashboard { const element = document.querySelector('#integrity-refresh-status'); if (!element) return; const toggle = document.querySelector('#integrity-details-toggle'); - const warning = document.querySelector('.integrity-refresh-warning'); + const warning = document.querySelector('#integrity-details-toggle .integrity-refresh-warning'); element.classList.toggle('is-stale', Boolean(error)); if (toggle) { toggle.classList.toggle('is-stale', Boolean(error)); @@ -541,17 +576,29 @@ class StatsDashboard { renderDuplicateRefreshFailure() { const details = document.querySelector('#duplicate-details'); + const toggle = document.querySelector('#duplicate-details-toggle'); + const warning = document.querySelector('.duplicate-refresh-warning'); if (!details) return; if (this.lastDuplicateStats) { const updated = this.lastDuplicateStats.updatedAt ? ` Last updated ${new Date(this.lastDuplicateStats.updatedAt).toLocaleString()}.` : ''; details.textContent = `Refresh delayed; showing the last available duplicate counts.${updated}`; details.classList.add('is-stale'); + if (toggle) { + toggle.classList.add('is-stale'); + toggle.title = 'Duplicate stats refresh failed. Open details for the last available counts.'; + } + if (warning) warning.hidden = false; } else { const count = document.querySelector('#duplicate-files'); if (count) count.textContent = 'Preparing...'; details.textContent = 'Preparing duplicate counts. Refresh will retry automatically.'; details.classList.remove('is-stale'); + if (toggle) { + toggle.classList.remove('is-stale'); + toggle.title = 'Show duplicate file details'; + } + if (warning) warning.hidden = true; } } diff --git a/templates/index.html b/templates/index.html index cb15b43..d8199a7 100644 --- a/templates/index.html +++ b/templates/index.html @@ -207,8 +207,16 @@

Scan Progress

Preparing...
Duplicate Files
-
Preparing duplicate counts.
-
All files in matching recorded SHA-256 groups.
+ +
-
diff --git a/tests/frontend_dom_test.js b/tests/frontend_dom_test.js index dd561a2..d9d6b2f 100644 --- a/tests/frontend_dom_test.js +++ b/tests/frontend_dom_test.js @@ -423,13 +423,14 @@ async function verifyViewer() { (async () => { await verifyViewer(); + const templateDom = new JSDOM(fs.readFileSync('templates/index.html', 'utf8')); + const duplicateCard = templateDom.window.document.querySelector('[aria-label="Duplicate files by recorded content hash"]'); + const integrityCard = templateDom.window.document.querySelector('[aria-label="Integrity coverage"]'); const statsDom = new JSDOM(`
-
-
- - `, +
+ ${duplicateCard.outerHTML}${integrityCard.outerHTML}`, { runScripts: 'outside-only' }); const statsWindow = statsDom.window; statsWindow.Chart = function () {}; @@ -459,6 +460,15 @@ async function verifyViewer() { }); await dashboard.updateStats(); assert.equal(statsWindow.document.querySelector('#integrity-detail-panel').hidden, true); + assert.equal(statsWindow.document.querySelector('#duplicate-detail-panel').hidden, true); + assert.equal(statsWindow.document.querySelector('#duplicate-files').textContent, '2'); + assert.equal(statsWindow.document.querySelector('#duplicate-detail-panel').contains( + statsWindow.document.querySelector('#duplicate-files')), false); + assert.equal(statsWindow.document.querySelector('#duplicate-details-toggle').getAttribute('aria-expanded'), 'false'); + assert.equal(statsWindow.document.querySelector('#duplicate-details-toggle').getAttribute('aria-controls'), 'duplicate-detail-panel'); + statsWindow.document.querySelector('#duplicate-details-toggle').click(); + assert.equal(statsWindow.document.querySelector('#duplicate-detail-panel').hidden, false); + assert.equal(statsWindow.document.querySelector('#duplicate-details-toggle').getAttribute('aria-expanded'), 'true'); statsWindow.document.querySelector('#integrity-details-toggle').click(); assert.equal(statsWindow.document.querySelector('#integrity-detail-panel').hidden, false); assert.equal(statsWindow.document.querySelector('#integrity-details-toggle').getAttribute('aria-expanded'), 'true'); @@ -474,11 +484,26 @@ async function verifyViewer() { duplicate_extra_files: null, duplicate_status: { ready: false, updated_at: null, stale: true } }); assert.equal(statsWindow.document.querySelector('#duplicate-files').textContent, '2'); assert.match(statsWindow.document.querySelector('#duplicate-details').textContent, /Refresh delayed/); + assert.equal(statsWindow.document.querySelector('.duplicate-refresh-warning').hidden, false); dashboard.renderStats({ total_files: 8, healthy_files: 3, corrupted_files: 1, warning_files: 0, pending_files: 0, scanning_files: 0, duplicate_files: 4, duplicate_groups: 2, duplicate_extra_files: 2, duplicate_status: { ready: true, updated_at: '2026-10-10T12:00:00Z', stale: true } }); assert.equal(statsWindow.document.querySelector('#duplicate-files').textContent, '4'); assert.match(statsWindow.document.querySelector('#duplicate-details').textContent, /Refresh delayed/); + assert.equal(statsWindow.document.querySelector('#duplicate-detail-panel').hidden, false); + assert.equal(statsWindow.document.querySelector('.duplicate-refresh-warning').hidden, false); + assert.match(statsWindow.document.querySelector('#duplicate-details-toggle').title, /delayed/); + dashboard.renderStats({ total_files: 8, healthy_files: 3, corrupted_files: 1, warning_files: 0, + pending_files: 0, scanning_files: 0, duplicate_files: 4, duplicate_groups: 2, + duplicate_extra_files: 2, duplicate_status: { ready: true, updated_at: null, stale: false } }); + assert.equal(statsWindow.document.querySelector('#duplicate-detail-panel').hidden, false); + assert.equal(statsWindow.document.querySelector('.duplicate-refresh-warning').hidden, true); + statsWindow.document.querySelector('#duplicate-details-toggle').click(); + assert.equal(statsWindow.document.querySelector('#duplicate-detail-panel').hidden, true); + assert.equal(statsWindow.document.querySelector('#duplicate-details-toggle').getAttribute('aria-expanded'), 'false'); + await dashboard.updateStats(); + assert.equal(statsWindow.document.querySelector('#duplicate-detail-panel').hidden, true); + assert.equal(statsWindow.document.querySelector('#duplicate-files').textContent, '2'); let finishStats; let statsCalls = 0;