diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 82179f5..e958392 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.10.5] - 2026-10-29 + +### Added + +- CSV and JSON exports mark each row's content duplicate group and filename duplicate group, with the group's file count, so a script can keep one file per group ([#88](https://github.com/ttlequals0/PixelProbe/issues/88)). Exports honor the Duplicates filter and its content or filename mode. They return HTTP 503 with `Retry-After: 30` until the duplicate index is ready, and the duplicate columns stay empty while it initializes. + +### Fixed + +- The per-file Actions menu did not open on mobile cards or desktop table rows. Since 2.9.3 the toggle called the menu handler on the table renderer instead of the application, which raised a TypeError on every tap. + +### Security + +- Updated Pillow to 13.0.0. It fixes 28 vulnerabilities reported against 12.3.0, including CVE-2026-112393, a heap buffer overflow when saving TIFF data to a memory buffer. +- Frontend build dependencies: undici 8.12.3, fast-uri 4.2.3, and serialize-javascript 7.1.4 via lockfile overrides. Build-time only; nothing ships to the browser. + ## [2.10.4] - 2026-10-10 ### Fixed diff --git a/docs/api.md b/docs/api.md index 21274f1..238d340 100644 --- a/docs/api.md +++ b/docs/api.md @@ -758,8 +758,9 @@ Export scan results in multiple formats (CSV, JSON, or PDF). **Query parameters (GET):** - `format` (string): Output format - `csv`, `json`, or `pdf` (default: csv) -- `filter` (string): Filter type - `all`, `corrupted`, `healthy`, `pending`, `error` (default: all). `warning` is **not** handled on GET; use the POST form for it. +- `filter` (string): Filter type - `all`, `corrupted`, `healthy`, `warning`, `pending`, `error`, `duplicates` (default: all) - `search` (string): Search term to filter by file path +- `duplicate_mode` (string): With `filter=duplicates`, `hash` (same recorded content, default) or `name` (same filename) **Request body (POST):** ```json @@ -771,9 +772,11 @@ Export scan results in multiple formats (CSV, JSON, or PDF). } ``` -In the POST form, `filter` accepts `all`, `corrupted`, `healthy`, or `warning`. If `file_ids` is provided, only those specific results are exported and `filter`/`search` are ignored. +The POST form accepts the same `filter`, `search`, and `duplicate_mode` values. If `file_ids` is provided, only those specific results are exported and `filter`/`search` are ignored. -**Response:** File download in requested format +CSV and JSON rows carry four duplicate columns. `Content Duplicate Group` and `Content Duplicate Files` (`content_duplicate_group` and `content_duplicate_group_size` in JSON) mark files whose recorded SHA-256 hash and size match another existing file. `Filename Duplicate Group` and `Filename Duplicate Files` (`filename_duplicate_group` and `filename_duplicate_group_size`) mark files that share a filename. The group value is the lowest scan result ID in the group, so a script can keep one row per group value and remove the rest. The columns are empty for files that are not duplicates, and for every row while the duplicate index is still initializing. + +**Response:** File download in requested format. `filter=duplicates` returns `503` with `Retry-After: 30` until the duplicate index is ready, and `400` for an unknown `duplicate_mode`. ### Maintenance endpoints diff --git a/openapi.yaml b/openapi.yaml index ffb4caa..48a95a2 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -2699,15 +2699,26 @@ paths: in: query schema: type: string - enum: [all, corrupted, healthy, pending, error] + enum: [all, corrupted, healthy, warning, pending, error, duplicates] default: all - name: search in: query schema: type: string + - name: duplicate_mode + in: query + description: With filter=duplicates, hash (same recorded content) or name (same filename) + schema: + type: string + enum: [all, hash, name] + default: all responses: '200': - description: Export file generated and returned + description: Export file generated and returned. CSV and JSON rows carry content and filename duplicate group columns. + '400': + description: Unknown duplicate_mode + '503': + description: Duplicate index is initializing; retry after the Retry-After interval post: summary: Export scan results (POST) @@ -2739,16 +2750,24 @@ paths: enum: [csv, json, pdf] filter: type: string - enum: [all, corrupted, healthy, pending, warning] + enum: [all, corrupted, healthy, warning, pending, error, duplicates] search: type: string + duplicate_mode: + type: string + enum: [all, hash, name] + default: all file_ids: type: array items: type: integer responses: '200': - description: Export file generated and returned + description: Export file generated and returned. CSV and JSON rows carry content and filename duplicate group columns. + '400': + description: Unknown duplicate_mode + '503': + description: Duplicate index is initializing; retry after the Retry-After interval # Maintenance Endpoints /test-cleanup: diff --git a/package-lock.json b/package-lock.json index 9f4be8a..3b4c882 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1363,9 +1363,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-4.2.1.tgz", - "integrity": "sha512-TmHQgewjHtMq1E5QKA0tOE0yeYGQs25KZC/ziJpubRtWI15W92e6vPFydWOeZBVROSHBYw/QhD9d5OefdD6LDg==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-4.2.3.tgz", + "integrity": "sha512-I1PXwP5eAhlNlcdpjt9CgB59op2ZvZfxw+BkqGFVIUOgNpCVTcTqp3HhTX7brxj5vsip+nSY4nTvff+eIA76hg==", "dev": true, "funding": [ { @@ -2734,9 +2734,9 @@ } }, "node_modules/serialize-javascript": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-7.1.2.tgz", - "integrity": "sha512-GL2BWwVa6JydKO6l/ljVgjAZF4QJ3S7dWDWi53s5GZT8PJzD2fNxi67HANQGKAqPrwEpL5ba7gUBGi0Ls/sEoQ==", + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-7.1.4.tgz", + "integrity": "sha512-FqlEuoTq/zRmA4Lg6aOqyH3lCHbUtCB0bmHCeFniX2GBdDqMJM3FwdvEhlE5Pim1dlt2FfBdVPmUNNiGhhKqpw==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -3052,9 +3052,9 @@ } }, "node_modules/undici": { - "version": "8.10.2", - "resolved": "https://registry.npmjs.org/undici/-/undici-8.10.2.tgz", - "integrity": "sha512-/y4/bH9YNU5hi9NIrpOuvGXFcxrj3CMrV+/AYpowAYTpHn8gX/XPFjNy766FPoYY0miQhdW977JFWKGNhBdwyQ==", + "version": "8.12.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.12.3.tgz", + "integrity": "sha512-+fdndUTwBP86cVjihTp21hZHo7d6oj1c/47XPA3lL8mM0XHs+C4EkcjmgRsFnFk2f/UONGDy9ItBItmXsLOQHw==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index bf44daf..9974117 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ }, "overrides": { "ajv": "^8.20.0", - "fast-uri": "^4.1.5", - "serialize-javascript": "^7.1.2" + "fast-uri": "^4.2.3", + "serialize-javascript": "^7.1.4", + "undici": "^8.12.3" } } diff --git a/pixelprobe/api/export_routes.py b/pixelprobe/api/export_routes.py index 6848d53..d146e2b 100644 --- a/pixelprobe/api/export_routes.py +++ b/pixelprobe/api/export_routes.py @@ -1,4 +1,5 @@ from flask import Blueprint, request, send_file, Response, make_response, stream_with_context +from sqlalchemy import null import os import csv import io @@ -10,6 +11,7 @@ from pixelprobe.models import db, ScanResult from pixelprobe.auth import auth_required +from pixelprobe.services.duplicate_service import duplicate_index_status, duplicate_members from pixelprobe.utils.security import PathTraversalError, open_authorized_media_file logger = logging.getLogger(__name__) @@ -94,6 +96,48 @@ def _apply_status_filter(query, filter_type): return query +class _InvalidDuplicateMode(ValueError): + pass + + +class _DuplicateIndexNotReady(Exception): + def __init__(self, status): + super().__init__('Duplicate index is initializing') + self.status = status + + +def _filtered_export_query(filter_type, search, duplicate_mode): + """Build the export query; returns (query, export_type).""" + if duplicate_mode not in ('all', 'hash', 'name'): + raise _InvalidDuplicateMode('duplicate_mode must be all, hash, or name') + query = ScanResult.query + if search: + query = query.filter(ScanResult.file_path.contains(search)) + if filter_type != 'duplicates': + return _apply_status_filter(query, filter_type), filter_type + mode = 'hash' if duplicate_mode == 'all' else duplicate_mode + status = duplicate_index_status() + if not status['ready']: + raise _DuplicateIndexNotReady(status) + members = duplicate_members(mode) + return query.join(members, ScanResult.id == members.c.file_id), f'duplicates_{mode}' + + +def _duplicate_export_columns(): + """Outer joins and labeled columns for duplicate annotations; blank until the index is ready.""" + labels = ('content_duplicate_group', 'content_duplicate_group_size', + 'filename_duplicate_group', 'filename_duplicate_group_size') + if not duplicate_index_status()['ready']: + return [], tuple(null().label(label) for label in labels) + hash_members = duplicate_members('hash') + name_members = duplicate_members('name') + joins = [(hash_members, ScanResult.id == hash_members.c.file_id), + (name_members, ScanResult.id == name_members.c.file_id)] + columns = (hash_members.c.group_id.label(labels[0]), hash_members.c.group_size.label(labels[1]), + name_members.c.group_id.label(labels[2]), name_members.c.group_size.label(labels[3])) + return joins, columns + + def _pdf_export_rows(query): """Load only the fields rendered by the bounded generic PDF export.""" return (query.with_entities( @@ -209,18 +253,10 @@ def export_scan_results(): # Export based on current filter and search filter_type = data.get('filter', 'all') search = data.get('search', '') - - query = ScanResult.query - - # Apply search filter - if search: - query = query.filter(ScanResult.file_path.contains(search)) - - query = _apply_status_filter(query, filter_type) - + duplicate_mode = data.get('duplicate_mode', 'all') + query, export_type = _filtered_export_query(filter_type, search, duplicate_mode) result_total = query.count() results = _pdf_export_rows(query) if format_type == 'pdf' else None - export_type = filter_type if filter_type != 'all' else 'all' logger.info("Exporting %s scan results to %s", result_total, format_type.upper()) else: # GET request - support format, filter, and search parameters @@ -232,31 +268,28 @@ def export_scan_results(): if format_type not in ['csv', 'json', 'pdf']: format_type = 'csv' - # Build query based on filter - query = ScanResult.query - - # Apply search filter if provided - if search: - query = query.filter(ScanResult.file_path.contains(search)) - - query = _apply_status_filter(query, filter_type) - + duplicate_mode = request.args.get('duplicate_mode', 'all') + query, export_type = _filtered_export_query(filter_type, search, duplicate_mode) result_total = query.count() results = _pdf_export_rows(query) if format_type == 'pdf' else None - export_type = filter_type if filter_type != 'all' else 'all' logger.info("Exporting %s scan results to %s", result_total, format_type.upper()) + duplicate_joins, duplicate_columns = _duplicate_export_columns() + def result_batches(): columns = (ScanResult.id, ScanResult.file_path, ScanResult.file_size, ScanResult.file_type, ScanResult.creation_date, ScanResult.is_corrupted, ScanResult.corruption_details, ScanResult.scan_date, ScanResult.scan_status, ScanResult.discovered_date, ScanResult.marked_as_good, ScanResult.has_warnings, - ScanResult.warning_details, ScanResult.error_message) + ScanResult.warning_details, ScanResult.error_message) + duplicate_columns last_id = 0 while True: session = db.session.session_factory() try: - batch = (query.with_session(session).filter( + batch_query = query.with_session(session) + for members, on_clause in duplicate_joins: + batch_query = batch_query.outerjoin(members, on_clause) + batch = (batch_query.filter( ScanResult.id > last_id, ScanResult.id <= export_max_id) .order_by(ScanResult.id).with_entities(*columns).limit(500).all()) finally: @@ -298,6 +331,10 @@ def generate_json(): 'has_warnings': getattr(result, 'has_warnings', False), 'warning_details': getattr(result, 'warning_details', None), 'error_message': getattr(result, 'error_message', None), + 'content_duplicate_group': result.content_duplicate_group, + 'content_duplicate_group_size': result.content_duplicate_group_size, + 'filename_duplicate_group': result.filename_duplicate_group, + 'filename_duplicate_group_size': result.filename_duplicate_group_size, 'details': { 'corruption': result.corruption_details, 'warning': getattr(result, 'warning_details', None), @@ -460,7 +497,11 @@ def generate_csv(): 'Scan Date', 'Scan Status', 'Discovered Date', - 'Marked as Good' + 'Marked as Good', + 'Content Duplicate Group', + 'Content Duplicate Files', + 'Filename Duplicate Group', + 'Filename Duplicate Files' ]) yield output.getvalue(); output.seek(0); output.truncate(0) for result in result_batches(): @@ -485,13 +526,21 @@ def generate_csv(): result.scan_date.isoformat() if result.scan_date else '', getattr(result, 'scan_status', 'completed'), # Default to completed for old records getattr(result, 'discovered_date', result.scan_date).isoformat() if getattr(result, 'discovered_date', result.scan_date) else '', - 'Yes' if result.marked_as_good else 'No' + 'Yes' if result.marked_as_good else 'No', + result.content_duplicate_group if result.content_duplicate_group is not None else '', + result.content_duplicate_group_size if result.content_duplicate_group_size is not None else '', + result.filename_duplicate_group if result.filename_duplicate_group is not None else '', + result.filename_duplicate_group_size if result.filename_duplicate_group_size is not None else '' ]) yield output.getvalue(); output.seek(0); output.truncate(0) filename = f"pixelprobe_{export_type}_{timestamp}.csv" release_request_session() return Response(stream_with_context(generate_csv()), mimetype='text/csv', headers={'Content-Disposition': f'attachment; filename={filename}'}) + except _InvalidDuplicateMode as e: + return {'error': str(e)}, 400 + except _DuplicateIndexNotReady as e: + return {'error': str(e), 'duplicate_status': e.status}, 503, {'Retry-After': '30'} except Exception as e: logger.error(f"Error exporting: {str(e)}", exc_info=True) return {'error': 'Export failed'}, 500 diff --git a/pixelprobe/version.py b/pixelprobe/version.py index 81517ae..3fd345e 100644 --- a/pixelprobe/version.py +++ b/pixelprobe/version.py @@ -4,7 +4,7 @@ # Default version - this is the single source of truth -_DEFAULT_VERSION = '2.10.4' +_DEFAULT_VERSION = '2.10.5' # Allow override via environment variable for CI/CD, but default to the hardcoded version diff --git a/requirements.txt b/requirements.txt index e11f6eb..1636253 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ Flask-CORS==6.0.0 Flask-Limiter==3.9.0 Flask-WTF==1.2.2 flask-restx==1.3.2 -Pillow==12.3.0 # 12.3.0 resolves 13 CVEs in 12.2.0 (CVE-2026-59197..59205, 55379, 55380, 55798, 54058..54060) +Pillow==13.0.0 # 13.0.0 resolves 28 CVEs in 12.3.0, including CVE-2026-112393 (TIFF heap overflow) pillow-heif==1.3.0 # HEIC/HEIF support for Pillow (requires libheif system library); 1.3.0 resolves CVE-2026-28231 python-magic==0.4.27 ffmpeg-python==0.2.0 diff --git a/static/js/app.js b/static/js/app.js index 7994b22..3bc26ba 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1977,7 +1977,7 @@ class TableManager { actions.appendChild(make('View', 'fa-eye', () => app.viewFile(id), 'btn-primary')); const dropdown = document.createElement('div'); dropdown.className = 'action-dropdown'; const menu = document.createElement('ul'); menu.className = 'dropdown-menu'; menu.style.display = 'none'; - const toggle = make('Actions', 'fa-tasks', (event) => this.toggleActionDropdown(event, menu.id)); + const toggle = make('Actions', 'fa-tasks', (event) => app.toggleActionDropdown(event, menu.id)); menu.id = `file-action-menu-${id}-${compact ? 'table' : 'mobile'}`; [['Rescan', 'fa-sync', () => app.rescanFile(id)], ['Cleanup', 'fa-search', () => app.orphanCheckFile(id)], ['Integrity Check', 'fa-shield-alt', () => app.changeCheckFile(id)]].forEach(([text, icon, handler]) => { const item = document.createElement('li'); const action = document.createElement('a'); action.href = '#'; action.className = 'dropdown-item'; action.innerHTML = ``; action.append(` ${text}`); action.addEventListener('click', (event) => { event.preventDefault(); handler(); }); item.appendChild(action); menu.appendChild(item); @@ -4436,6 +4436,7 @@ class PixelProbeApp { // Export all files in current filter/search requestBody.filter = this.table.filter; requestBody.search = this.table.searchQuery; + if (this.table.filter === 'duplicates') requestBody.duplicate_mode = this.table.duplicateMode; } const response = await fetch('/api/export', { @@ -4470,6 +4471,8 @@ class PixelProbeApp { window.URL.revokeObjectURL(url); this.showNotification(`${formatUpper} export complete`, 'success'); + } else if (response.status === 503) { + this.showNotification('Duplicate index is initializing. Try the export again shortly.', 'warning'); } else { throw new Error('Export failed'); } diff --git a/tests/frontend_dom_test.js b/tests/frontend_dom_test.js index 1479ee8..3784ede 100644 --- a/tests/frontend_dom_test.js +++ b/tests/frontend_dom_test.js @@ -46,6 +46,28 @@ for (const mode of ['hash', 'name']) { } assert.equal(row.querySelector('.duplicate-label'), null); +// Regression: the per-file Actions toggle must open its menu on mobile cards and table rows. +window.app.toggleActionDropdown = window.__PixelProbeApp.prototype.toggleActionDropdown; +window.app.closeAllDropdowns = window.__PixelProbeApp.prototype.closeAllDropdowns; +const mobileResults = window.document.createElement('div'); +mobileResults.className = 'mobile-results'; +window.document.body.appendChild(mobileResults); +const actionCard = table.renderMobileCard({ id: 9, file_path: '/media/card.mp4', scan_status: 'completed' }); +mobileResults.appendChild(actionCard); +const actionMenu = actionCard.querySelector('.action-dropdown .dropdown-menu'); +assert.equal(actionMenu.style.display, 'none'); +actionCard.querySelector('.action-dropdown button').click(); +assert.equal(actionMenu.style.display, 'block'); +let rescanned = null; +window.app.rescanFile = (id) => { rescanned = id; }; +actionMenu.querySelector('.dropdown-item').click(); +assert.equal(rescanned, 9); +const tableRow = table.renderRow({ id: 10, file_path: '/media/row.mp4', scan_status: 'completed' }); +window.document.querySelector('#results-tbody').appendChild(tableRow); +tableRow.querySelector('.action-dropdown button').click(); +assert.equal(tableRow.querySelector('.action-dropdown .dropdown-menu').style.display, 'block'); +assert.equal(actionMenu.style.display, 'none'); + const duplicateControls = window.document.createElement('div'); duplicateControls.innerHTML = ''; window.document.body.appendChild(duplicateControls); diff --git a/tests/test_duplicates.py b/tests/test_duplicates.py index 3ec9b34..4d63207 100644 --- a/tests/test_duplicates.py +++ b/tests/test_duplicates.py @@ -1,3 +1,6 @@ +import csv +import io +import json import os from uuid import uuid4 from datetime import datetime, timezone @@ -184,3 +187,53 @@ def test_postgres_duplicate_basename_and_grouping(): with engine.begin() as connection: connection.execute(text(f'DROP SCHEMA {schema} CASCADE')) engine.dispose() + + +def _csv_rows(response): + return list(csv.DictReader(io.StringIO(response.get_data(as_text=True)))) + + +def test_export_annotates_duplicate_groups_and_filters_duplicates(db, authenticated_client): + rows = seed_duplicates(db.session) + prime_sqlite_cache(db.session) + content_ids = {rows[0].id, rows[1].id, rows[2].id} + name_ids = {rows[0].id, rows[1].id, rows[11].id} + + exported = _csv_rows(authenticated_client.get('/api/export?format=csv')) + assert len(exported) == len(rows) + by_id = {int(row['ID']): row for row in exported} + assert {int(by_id[i]['Content Duplicate Group']) for i in content_ids} == {rows[0].id} + assert {by_id[i]['Content Duplicate Files'] for i in content_ids} == {'3'} + assert {int(by_id[i]['Filename Duplicate Group']) for i in name_ids} == {rows[0].id} + assert {by_id[i]['Filename Duplicate Files'] for i in name_ids} == {'3'} + assert by_id[rows[3].id]['Content Duplicate Group'] == '' + assert by_id[rows[12].id]['Filename Duplicate Group'] == '' + + as_json = json.loads(authenticated_client.get('/api/export?format=json').get_data(as_text=True)) + by_id = {row['id']: row for row in as_json} + assert by_id[rows[2].id]['content_duplicate_group'] == rows[0].id + assert by_id[rows[2].id]['content_duplicate_group_size'] == 3 + assert by_id[rows[2].id]['filename_duplicate_group'] is None + assert by_id[rows[11].id]['filename_duplicate_group_size'] == 3 + + filtered = _csv_rows(authenticated_client.get('/api/export?format=csv&filter=duplicates')) + assert {int(row['ID']) for row in filtered} == content_ids + posted = authenticated_client.post('/api/export', json={ + 'format': 'csv', 'filter': 'duplicates', 'duplicate_mode': 'name', 'search': 'no-hash'}) + assert posted.status_code == 200 + assert {int(row['ID']) for row in _csv_rows(posted)} == {rows[11].id} + pdf = authenticated_client.post('/api/export', json={'format': 'pdf', 'filter': 'duplicates'}) + assert pdf.status_code == 200 + assert pdf.mimetype == 'application/pdf' + assert authenticated_client.get('/api/export?filter=duplicates&duplicate_mode=visual').status_code == 400 + + +def test_export_duplicate_filter_waits_for_index_and_leaves_columns_blank(db, authenticated_client): + rows = seed_duplicates(db.session) + response = authenticated_client.get('/api/export?format=csv&filter=duplicates') + assert response.status_code == 503 + assert response.headers['Retry-After'] == '30' + exported = _csv_rows(authenticated_client.get('/api/export?format=csv')) + assert len(exported) == len(rows) + assert {row['Content Duplicate Group'] for row in exported} == {''} + assert {row['Filename Duplicate Files'] for row in exported} == {''}