diff --git a/.env.example b/.env.example index 58849ba..119e60e 100644 --- a/.env.example +++ b/.env.example @@ -1,76 +1,48 @@ -# PixelProbe Environment Configuration -# Copy this file to .env and customize for your deployment +# Copy this file to .env and customize the deployment values. -# REQUIRED: Secure secret key for Flask sessions -# Generate with: python -c "import secrets; print(secrets.token_hex(32))" -SECRET_KEY=your-secure-random-key-here-replace-this +# Required +SECRET_KEY=replace-with-a-random-secret +POSTGRES_PASSWORD=replace-with-a-strong-password -# Database configuration (PostgreSQL REQUIRED since v2.2.0) -# When using docker-compose.yml, the POSTGRES_* variables below are used instead. -# DATABASE_URL is only needed for standalone (non-Docker) deployments. -# DATABASE_URL=postgresql://pixelprobe:changeme@localhost:5432/pixelprobe - -# PostgreSQL credentials (used by docker-compose.yml) -POSTGRES_PASSWORD=changeme - -# Container identity. Use your host UID:GID when mounted media or instance -# directories are owned by a non-default account. -PUID=10001 -PGID=10001 - -# Cookie security defaults. Keep HTTPS enabled in production. Set false only -# for an explicitly plain-HTTP local or LAN deployment. -SESSION_COOKIE_SECURE=true -SESSION_COOKIE_HTTPONLY=true -SESSION_COOKIE_SAMESITE=Lax -REMEMBER_COOKIE_DURATION_DAYS=30 - -# Scanning configuration -# Comma-separated list of directories to monitor (inside container) -SCAN_PATHS=/media - -# Path to your media files on the host system (mounted read-only into container) +# Host directories and container identity MEDIA_PATH=/path/to/your/media - -# Performance tuning -# MAX_WORKERS: parallel file scanning threads per task (default: 10) -MAX_WORKERS=10 - -# Scheduler ownership (default: true). In multi-container deployments set -# false on the web container so only the Celery worker runs the scheduler. -# SCHEDULER_ENABLED=true - -# Gunicorn listen address; comma-separated list allowed (default: 0.0.0.0:5000) -# For IPv6/dual-stack (e.g. rootless podman) use: GUNICORN_BIND=[::]:5000 -# GUNICORN_BIND=0.0.0.0:5000 -# BATCH_SIZE: files per database batch commit (default: 100) -BATCH_SIZE=100 - -# Celery configuration -# CELERY_CONCURRENCY: number of concurrent Celery tasks (default: 4) -CELERY_CONCURRENCY=4 - -# Timezone configuration -TZ=America/New_York - -# Scheduling (optional) -# Periodic scan schedule (examples): -# PERIODIC_SCAN_SCHEDULE=cron:0 2 * * * # Daily at 2 AM -# PERIODIC_SCAN_SCHEDULE=interval:hours:6 # Every 6 hours -PERIODIC_SCAN_SCHEDULE= - -# Cleanup schedule (examples): -# CLEANUP_SCHEDULE=cron:0 3 * * 0 # Weekly on Sunday at 3 AM -# CLEANUP_SCHEDULE=interval:days:7 # Every 7 days -CLEANUP_SCHEDULE= - -# Exclusions -# Path exclusions (comma-separated) -EXCLUDED_PATHS=/media/temp,/media/cache -# Extension exclusions (comma-separated) -EXCLUDED_EXTENSIONS=.tmp,.temp,.cache - -# SSRF trusted hosts (optional) -# Hostnames and/or CIDR ranges that bypass private-IP blocking for healthcheck pings. -# Comma-separated. Example: myhost.local,192.168.5.0/24 -TRUSTED_INTERNAL_HOSTS= +SCAN_PATHS=/media +# PUID=10001 +# PGID=10001 + +# Keep HTTPS cookie protection enabled. Set false only for plain HTTP. +# SESSION_COOKIE_SECURE=false + +# Optional startup settings. Uncomment only when changing the Compose defaults. +# TZ=UTC +# CELERY_CONCURRENCY sets the worker process count. +# CELERY_CONCURRENCY=4 +# CELERY_LOG_LEVEL=INFO +# MAX_WORKERS caps selected-file rescan threads; the API defaults to 4. +# MAX_WORKERS=10 +# BATCH_SIZE sets the legacy discovery database lookup batch size. +# BATCH_SIZE=100 +# MAX_OUTPUT_SIZE=10000 +# OUTPUT_ROTATION_ENABLED=true +# FLASK_ENV=production + +# Startup seeds this value only when no saved row exists; saved values win. +# FREEZE_DETECTION_ENABLED=true + +# Legacy startup import only. Active exclusions are stored in PostgreSQL after +# the one-time import; these values do not override saved rules at runtime. +# EXCLUDED_PATHS=/media/temp,/media/cache +# EXCLUDED_EXTENSIONS=.tmp,.temp,.cache +# Legacy exclusions.json keys: paths, extensions, filename_patterns. +# Built-in filename patterns remain excluded after configured rules are cleared. + +# Optional SSRF allowlist for internal hosts/CIDRs. +# TRUSTED_INTERNAL_HOSTS=healthcheck.internal.local,192.168.5.0/24 + +# Deprecated compatibility override read by app.py if passed to the process. +# Current Compose uses POSTGRES_* and does not forward DATABASE_URL. +# DATABASE_URL=postgresql://pixelprobe:password@localhost:5432/pixelprobe + +# Optional startup schedules run alongside schedules stored in the database. +# PERIODIC_SCAN_SCHEDULE=cron:0 2 * * * +# CLEANUP_SCHEDULE=cron:0 3 * * 0 diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 6cfe537..82179f5 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.10.4] - 2026-10-10 + +### Fixed + +- Mobile dialogs and the sidebar fit the visible browser height and keep background scrolling locked until the last overlay closes. + +## [2.10.3] - 2026-10-10 + +### Changed + +- Compose shares app and worker startup settings, and `.env.example` separates required values from optional overrides. The configuration guide documents six shared read-only media mounts and the one-time import and database runtime behavior for exclusions. + +### Fixed + +- Saved exclusions now apply to scans, scheduled scans, and selected-file rescans. Legacy JSON and environment rules import once without restoring removed database entries, and API replacement and re-add operations handle existing rows. + ## [2.10.2] - 2026-10-10 ### Changed diff --git a/README.md b/README.md index 0c0072e..ec2823d 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ cd PixelProbe cp .env.example .env ``` -Before startup, edit `.env`: replace `SECRET_KEY` and `POSTGRES_PASSWORD`, set `MEDIA_PATH`, and choose `PUID` and `PGID`. Create the host runtime directory and make it writable by those configured values. The Compose default is `10001:10001`. +Before startup, edit `.env`: set `SECRET_KEY`, `POSTGRES_PASSWORD`, and `MEDIA_PATH`. `MEDIA_PATH` must name an existing host directory; Compose will fail if it is missing. The `PUID` and `PGID` defaults are `10001:10001`; change them only if the host directories need a different identity. Create the host `instance` directory and make it writable by the configured identity. ```bash mkdir -p instance diff --git a/docker-compose.yml b/docker-compose.yml index 5fc2bb0..9a348cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,13 +13,40 @@ # so this is the main scan-throughput knob (8-12 reasonable; chunk work # is mostly ffmpeg/imagemagick subprocess-bound) # -# - MAX_WORKERS: Parallel workers for selected-file rescans (default: 10) +# - MAX_WORKERS: Maximum thread count for selected-file rescans (default: 10). +# The API defaults to 4 threads; this value sets the maximum. # # Connection math (keep under PostgreSQL max_connections, default 100): # web: 4 gunicorn workers x (DB_POOL_SIZE 5 + DB_MAX_OVERFLOW 10) = 60 max # celery: CELERY_CONCURRENCY children x (5 + 10) worst-case, few in practice # plus ~MAX_WORKERS single checker connections during selected-file rescans +x-pixelprobe-environment: &pixelprobe-environment + SECRET_KEY: ${SECRET_KEY} + TZ: ${TZ:-UTC} + DB_POOL_SIZE: ${DB_POOL_SIZE:-5} + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + POSTGRES_DB: pixelprobe + POSTGRES_USER: pixelprobe + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set} + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + FLASK_ENV: ${FLASK_ENV:-production} + SCAN_PATHS: ${SCAN_PATHS:-/media} + EXCLUDED_PATHS: ${EXCLUDED_PATHS:-} + EXCLUDED_EXTENSIONS: ${EXCLUDED_EXTENSIONS:-.txt,.log,.md} + MAX_WORKERS: ${MAX_WORKERS:-10} + BATCH_SIZE: ${BATCH_SIZE:-100} + MAX_OUTPUT_SIZE: ${MAX_OUTPUT_SIZE:-10000} + OUTPUT_ROTATION_ENABLED: ${OUTPUT_ROTATION_ENABLED:-true} + # Legacy migration input; stored Tunables become authoritative after seeding. + FREEZE_DETECTION_ENABLED: ${FREEZE_DETECTION_ENABLED:-true} + PILLOW_BLOCKS_MAX: 256 + PERIODIC_SCAN_SCHEDULE: ${PERIODIC_SCAN_SCHEDULE:-} + CLEANUP_SCHEDULE: ${CLEANUP_SCHEDULE:-} + TRUSTED_INTERNAL_HOSTS: ${TRUSTED_INTERNAL_HOSTS:-} + services: # PostgreSQL database # BREAKING (v2.7.0): default moved from postgres:15 to postgres:18. An @@ -71,11 +98,11 @@ services: # PixelProbe application pixelprobe: - image: ttlequals0/pixelprobe:${PIXELPROBE_VERSION:-2.10.0} + image: ttlequals0/pixelprobe:${PIXELPROBE_VERSION:-2.10.3} container_name: pixelprobe-app environment: + <<: *pixelprobe-environment # Security - SECRET_KEY: ${SECRET_KEY} SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE:-true} SESSION_COOKIE_HTTPONLY: ${SESSION_COOKIE_HTTPONLY:-true} SESSION_COOKIE_SAMESITE: ${SESSION_COOKIE_SAMESITE:-Lax} @@ -84,43 +111,10 @@ services: GUNICORN_BIND: ${GUNICORN_BIND:-0.0.0.0:5000} GUNICORN_WORKERS: ${GUNICORN_WORKERS:-4} GUNICORN_TIMEOUT: ${GUNICORN_TIMEOUT:-300} - DB_POOL_SIZE: ${DB_POOL_SIZE:-5} - TZ: ${TZ:-UTC} # Scheduler runs in celery-worker; keep the web container out of the # scheduler lock entirely (set to "true" only in single-container setups) SCHEDULER_ENABLED: "false" - - # PostgreSQL database configuration - POSTGRES_HOST: postgres - POSTGRES_PORT: 5432 - POSTGRES_DB: pixelprobe - POSTGRES_USER: pixelprobe - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set} - - # Celery configuration - CELERY_BROKER_URL: redis://redis:6379/0 - CELERY_RESULT_BACKEND: redis://redis:6379/0 - - # Application settings - FLASK_ENV: ${FLASK_ENV:-production} - SCAN_PATHS: ${SCAN_PATHS:-/media} - EXCLUDED_PATHS: ${EXCLUDED_PATHS:-} - EXCLUDED_EXTENSIONS: ${EXCLUDED_EXTENSIONS:-.txt,.log,.md} - MAX_WORKERS: ${MAX_WORKERS:-10} - BATCH_SIZE: ${BATCH_SIZE:-100} - MAX_OUTPUT_SIZE: ${MAX_OUTPUT_SIZE:-10000} - OUTPUT_ROTATION_ENABLED: ${OUTPUT_ROTATION_ENABLED:-true} - FREEZE_DETECTION_ENABLED: ${FREEZE_DETECTION_ENABLED:-true} - PILLOW_BLOCKS_MAX: 256 - - - # Scheduled scans - PERIODIC_SCAN_SCHEDULE: ${PERIODIC_SCAN_SCHEDULE:-} - CLEANUP_SCHEDULE: ${CLEANUP_SCHEDULE:-} - - # SSRF trusted hosts (hostnames/CIDRs that bypass private-IP blocking) - TRUSTED_INTERNAL_HOSTS: ${TRUSTED_INTERNAL_HOSTS:-} volumes: # Media files to scan @@ -167,47 +161,14 @@ services: # P1 Celery Worker for distributed task processing celery-worker: - image: ttlequals0/pixelprobe:${PIXELPROBE_VERSION:-2.10.0} + image: ttlequals0/pixelprobe:${PIXELPROBE_VERSION:-2.10.3} container_name: pixelprobe-celery-worker command: python celery_worker.py environment: + <<: *pixelprobe-environment # Celery configuration - CELERY_BROKER_URL: redis://redis:6379/0 - CELERY_RESULT_BACKEND: redis://redis:6379/0 CELERY_LOG_LEVEL: ${CELERY_LOG_LEVEL:-INFO} CELERY_CONCURRENCY: ${CELERY_CONCURRENCY:-4} - DB_POOL_SIZE: ${DB_POOL_SIZE:-5} - TZ: ${TZ:-UTC} - - # Database connection (worker needs database access) - POSTGRES_HOST: postgres - POSTGRES_PORT: 5432 - POSTGRES_DB: pixelprobe - POSTGRES_USER: pixelprobe - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set} - - # Worker-specific settings - SECRET_KEY: ${SECRET_KEY} - FLASK_ENV: ${FLASK_ENV:-production} - MAX_WORKERS: ${MAX_WORKERS:-10} # Parallel file scanning workers per task - BATCH_SIZE: ${BATCH_SIZE:-100} - MAX_OUTPUT_SIZE: ${MAX_OUTPUT_SIZE:-10000} - OUTPUT_ROTATION_ENABLED: ${OUTPUT_ROTATION_ENABLED:-true} - FREEZE_DETECTION_ENABLED: ${FREEZE_DETECTION_ENABLED:-true} - PILLOW_BLOCKS_MAX: 256 - - # Scan configuration (required for scheduler) - SCAN_PATHS: ${SCAN_PATHS:-/media} - EXCLUDED_PATHS: ${EXCLUDED_PATHS:-} - EXCLUDED_EXTENSIONS: ${EXCLUDED_EXTENSIONS:-.txt,.log,.md} - - # Scheduled scans (scheduler runs in celery-worker) - PERIODIC_SCAN_SCHEDULE: ${PERIODIC_SCAN_SCHEDULE:-} - CLEANUP_SCHEDULE: ${CLEANUP_SCHEDULE:-} - - # SSRF trusted hosts (hostnames/CIDRs that bypass private-IP blocking) - TRUSTED_INTERNAL_HOSTS: ${TRUSTED_INTERNAL_HOSTS:-} - volumes: # Media files access (read-only for scanning) diff --git a/docs/configuration.md b/docs/configuration.md index 540e883..2c07b13 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,6 +1,6 @@ # PixelProbe configuration guide -Reference for PixelProbe environment variables and performance tuning. +Reference for deployment environment variables, saved scanner settings, and performance tuning. ## Table of contents @@ -17,7 +17,22 @@ Reference for PixelProbe environment variables and performance tuning. ## Environment variables -All configuration is done via environment variables, either in `.env` file or directly in `docker-compose.yml`. +Deployment settings come from the process environment. Scanner tunables and scan paths can also be saved in the database through the UI or API. Their source and update behavior differ, as summarized below. + +### Configuration sources and when changes take effect + +| Setting | Source and precedence | When a change takes effect | +|---|---|---| +| Compose substitutions such as `MEDIA_PATH` and `POSTGRES_PASSWORD` | Host shell, then the project `.env` file; `.env` values are used by Compose for interpolation and are not automatically copied into containers. Only variables listed under a service's `environment` or `env_file` reach that container. If both are set, `environment` wins. Removing an environment entry can expose a same-name value in `env_file`, so check for conflicts before dropping defaults. | Recreate the affected container after changing values. `docker compose restart` restarts the existing container with its old configuration. | +| Scanner tunables such as freeze detection and decode limits | Saved `AppConfig` rows take priority. At startup, legacy environment variables seed any missing rows. A saved row wins while present; if a row is reset while its legacy variable remains set, the next startup can seed it again. | Workers refresh cached values within 60 seconds. Selected-file scans snapshot settings for each submitted file; other checks use cached values as they reach each setting lookup. An edit is not guaranteed to change a file already in progress. | +| Scan paths | At startup, `SCAN_PATHS` is inserted into the active database path list if those rows do not already exist. Active database paths drive scan selection; the environment is a fallback when none are active. Environment paths also remain authorization roots in both services. | Database path edits affect new requests. Changing `SCAN_PATHS` requires recreating both services; the web service runs startup sync. | +| Scan capacity | `CELERY_CONCURRENCY` sets Celery process count at worker start. `MAX_WORKERS` caps selected-file scan threads; that API defaults to 4 threads. | Recreate the worker after changing concurrency. Recreate both app and worker after changing the selected-scan cap. | +| Exclusions | On first startup, the v2.10.3 migration imports `exclusions.json` and legacy `EXCLUDED_PATHS` / `EXCLUDED_EXTENSIONS` values into PostgreSQL. Existing rows, including inactive rows, take precedence for an exact type/value pair. The migration marker prevents later imports. Active database exclusions are used by scanners and the scheduler. | The next file checker and scheduled scan load current active rules. An in-progress checker, discovery task, or scheduled scan keeps its captured rules; chunk checkers load current rules. | +| Stored scan output | `MAX_OUTPUT_SIZE` and `OUTPUT_ROTATION_ENABLED` are read when the model module imports. | Recreate app and worker processes after changing either value. | +| Library and framework settings | `PILLOW_BLOCKS_MAX` is read by Pillow; the bundled Compose value is `256`. `FLASK_ENV` defaults to `production` in Compose and remains configurable. | Recreate affected containers after changing either value. | +| Legacy discovery batch | `BATCH_SIZE` controls lookup batch size in the legacy media-checker discovery path. It does not set parallel discovery inserts or scan chunk commits. | Recreate the affected process after changing its environment. | +| Schedules | Saved UI/API schedules are loaded from PostgreSQL and reloaded when their definitions change. `PERIODIC_SCAN_SCHEDULE` and `CLEANUP_SCHEDULE` add process-level default jobs. | Database schedule edits are synchronized by the scheduler. Environment schedule changes require recreating the worker. | +| Startup settings such as database credentials, broker URLs, `SECRET_KEY`, and `TRUSTED_INTERNAL_HOSTS` | Process environment. Trusted hosts are cached after they are first read. | Recreate the affected container. | ## Effective Compose deployment @@ -25,10 +40,10 @@ The root `docker-compose.yml` is the deployment source of truth. It starts Postg | Service | Scheduler | Media mount | Identity | Writable paths | |---|---|---|---|---| -| `pixelprobe` | Disabled by default | `${MEDIA_PATH}:/media:ro` | `${PUID:-10001}:${PGID:-10001}` | `./instance`, `/app/runtime`, `/tmp`, `/app/logs` | -| `celery-worker` | Enabled by default | `${MEDIA_PATH}:/media:ro` | `${PUID:-10001}:${PGID:-10001}` | `/app/runtime`, `/tmp` | +| `pixelprobe` | Disabled in bundled Compose | `${MEDIA_PATH}:/media:ro` | `${PUID:-10001}:${PGID:-10001}` | `./instance`, `/app/runtime`, `/tmp` | +| `celery-worker` | Enabled in bundled Compose by the application default | `${MEDIA_PATH}:/media:ro` | `${PUID:-10001}:${PGID:-10001}` | `/app/runtime`, `/tmp` | -The services share the same UID/GID and read-only media view. Do not change only one service identity or mount path. Create the host `instance` directory before startup and give it to the configured identity, for example `sudo chown 10001:10001 instance` when using defaults. +The services share the same UID/GID and read-only media view. Do not change only one service identity or mount path. Both host media and `instance` paths must exist before startup. Compose rejects a missing media path instead of creating it. Give `instance` to the configured identity, for example `sudo chown 10001:10001 instance` when using defaults. The web service publishes `${PORT:-5000}:5000`. PostgreSQL and Valkey are internal to the Compose network. The scheduler lease prevents duplicate ownership, but the normal deployment deliberately enables scheduler work only in `celery-worker`. @@ -37,7 +52,7 @@ The web service publishes `${PORT:-5000}:5000`. PostgreSQL and Valkey are intern | Variable | Description | Example | |----------|-------------|---------| | `SECRET_KEY` | Flask session secret key (64 chars) | Generate with: `python -c "import secrets; print(secrets.token_hex(32))"` | -| `POSTGRES_PASSWORD` | PostgreSQL database password | `your-secure-password` | +| `POSTGRES_PASSWORD` | PostgreSQL database password | Set a private value in `.env` | | `MEDIA_PATH` | Host path to media files (Docker only) | `/mnt/media` | ### Database variables @@ -53,11 +68,13 @@ The web service publishes `${PORT:-5000}:5000`. PostgreSQL and Valkey are intern | `DB_MAX_OVERFLOW` | `10` | Extra SQLAlchemy connections when the pool is exhausted | | `DATABASE_ECHO` | `false` | Enable SQL query logging (debug) | +`DATABASE_URL` remains accepted as a deprecated compatibility override and takes precedence when explicitly set. Do not set it alongside `POSTGRES_*` unless you intend it to select that database. The bundled Compose file does not pass it to the application. + ### Application variables | Variable | Default | Description | |----------|---------|-------------| -| `FLASK_ENV` | `development` | Flask environment (`production`, `development`, `testing`). The code default is `development`; the Docker image sets `production` | +| `FLASK_ENV` | `production` in the bundled Compose file | Flask environment (`production`, `development`, `testing`). The code default is `development`; the image and Compose deployment default to `production`. | | `SCAN_PATHS` | (empty) | Comma-separated paths to scan inside container. The code default is empty; the bundled compose file sets `/media` | | `TZ` | `UTC` | Timezone for timestamps (e.g., `America/New_York`) | | `PORT` | `5000` | Host port published by the compose file; gunicorn always binds 5000 inside the container | @@ -71,7 +88,7 @@ The web service publishes `${PORT:-5000}:5000`. PostgreSQL and Valkey are intern | Variable | Default | Description | Recommendations | |----------|---------|-------------|-----------------| -| `MAX_WORKERS` | `10` | Parallel file scanning workers per task | 10-24 for most systems | +| `MAX_WORKERS` | `10` | Upper bound on thread workers accepted by the selected-file scan API. The API request defaults to 4 workers and clamps larger values to this limit. | Raise only for selected rescans and with PostgreSQL connection headroom. | | `BATCH_SIZE` | `100` | Legacy media-checker discovery lookup batch. It does not set parallel discovery inserts or scan chunk commits. | Leave at `100` unless diagnosing that legacy path. | | `MAX_OUTPUT_SIZE` | `10000` | Maximum stored scan-output characters before model-level rotation truncates the stored text. | Keep the default unless operators accept less retained diagnostic output. | | `OUTPUT_ROTATION_ENABLED` | `true` | Enable model-level stored-output rotation. | Keep `true` for bounded stored output. | @@ -79,9 +96,9 @@ The web service publishes `${PORT:-5000}:5000`. PostgreSQL and Valkey are intern | `CHUNK_REVIVE_STALENESS_SECS` | `600` | How stale the scan liveness timestamp must be before the stuck-scan sweeper treats the chunk workers as gone and re-queues their chunks (recovers scans interrupted by container restarts) | Must exceed several heartbeat intervals | **Performance notes:** -- `MAX_WORKERS` controls parallelism within each scan task -- Each worker creates 1 database connection -- Worst-case connections = web app 4 gunicorn workers x (5 base + 10 overflow) = 60, plus CELERY_CONCURRENCY x (5 + 10) from the worker children, plus ~MAX_WORKERS checker connections +- `MAX_WORKERS` caps selected-file scan threads; it does not set Celery process count. +- `CELERY_CONCURRENCY` sets the number of Celery worker processes. +- Worst-case connections = web app 4 gunicorn workers x (5 base + 10 overflow) = 60, plus CELERY_CONCURRENCY x (5 + 10) from worker children, plus selected-scan connections. - Keep the total under PostgreSQL max_connections (default: 100). The extra large profile below exceeds 100 and needs `max_connections` raised to 150+ ### Celery variables @@ -95,7 +112,7 @@ The web service publishes `${PORT:-5000}:5000`. PostgreSQL and Valkey are intern **Celery notes:** - `CELERY_CONCURRENCY` controls how many scan tasks run simultaneously -- Independent from `MAX_WORKERS` (which controls parallelism within each task) +- Independent from `MAX_WORKERS` (which caps selected-file scan threads) - Recommended: 4-8 for most systems ### Redis variables @@ -113,8 +130,6 @@ The web service publishes `${PORT:-5000}:5000`. PostgreSQL and Valkey are intern | Variable | Default | Description | |----------|---------|-------------| -| `EXCLUDED_PATHS` | (empty) | Comma-separated paths to exclude from scanning | -| `EXCLUDED_EXTENSIONS` | `.txt,.log,.md` | Comma-separated file extensions to exclude | | `PERIODIC_SCAN_SCHEDULE` | (empty) | Automated scan schedule (cron or interval format) | | `CLEANUP_SCHEDULE` | (empty) | Automated cleanup schedule (cron or interval format) | @@ -190,7 +205,7 @@ Confirming that kept records were deleted is your decision. The UI offers it aft **Image processing:** -`PILLOW_BLOCKS_MAX` (set to `256` in the bundled compose file) is a Pillow-internal memory allocator tuning knob read by the Pillow library itself, not by PixelProbe code. +`PILLOW_BLOCKS_MAX` (set to `256` in the bundled Compose file) is read by Pillow, not PixelProbe. The installed Pillow default is `0`, so keep `256` in custom stacks unless you have measured a reason to change it. ### Monitoring variables (future) @@ -215,52 +230,32 @@ services: CELERY_CONCURRENCY: 2 ``` -### Multiple scan paths - -To scan multiple directories: - -**Method 1: multiple volume mounts** -```yaml -pixelprobe: - environment: - SCAN_PATHS: /movies,/tv-shows,/photos - volumes: - - /mnt/movies:/movies:ro - - /mnt/tv-shows:/tv-shows:ro - - /mnt/photos:/photos:ro - -celery-worker: - environment: - SCAN_PATHS: /movies,/tv-shows,/photos - volumes: - - /mnt/movies:/movies:ro - - /mnt/tv-shows:/tv-shows:ro - - /mnt/photos:/photos:ro -``` - -**Method 2: single parent volume** -```yaml -pixelprobe: - environment: - SCAN_PATHS: /media/movies,/media/tv,/media/photos - volumes: - - /mnt/all-media:/media:ro -``` +See [Docker setup: media directories](docker-setup.md#media-directories) for a Portainer-ready example with six read-only mounts shared by the web and worker services. ### User permissions (important) -Both `pixelprobe` and `celery-worker` MUST run as the same user to access media files: +Both `pixelprobe` and `celery-worker` must run as the same user to access media files: ```yaml pixelprobe: user: "${PUID:-10001}:${PGID:-10001}" volumes: - - ${MEDIA_PATH}:/media:ro + - type: bind + source: ${MEDIA_PATH:?Set MEDIA_PATH to an existing host directory} + target: /media + read_only: true + bind: + create_host_path: false celery-worker: user: "${PUID:-10001}:${PGID:-10001}" # MUST match pixelprobe volumes: - - ${MEDIA_PATH}:/media:ro + - type: bind + source: ${MEDIA_PATH:?Set MEDIA_PATH to an existing host directory} + target: /media + read_only: true + bind: + create_host_path: false ``` Find your UID/GID: @@ -312,36 +307,32 @@ ALTER SYSTEM SET max_parallel_workers_per_gather = 4; ## Performance tuning -The values in this section are operator starting points, not benchmark results. Measure CPU, memory, database connections, and media storage behavior on the library that will run the deployment before increasing concurrency. +The values in this section are operator starting points, not benchmark results. `CELERY_CONCURRENCY` controls directory-scan throughput. `MAX_WORKERS` only caps thread count for selected-file scans. Measure CPU, memory, database connections, and media storage before increasing concurrency. ### Recommended settings by system size -`BATCH_SIZE` is omitted from these profiles because it does not tune parallel discovery inserts or scan chunk commits. Leave its legacy lookup default at `100` unless investigating that specific code path. +`BATCH_SIZE` is omitted from these profiles because it only tunes a legacy discovery lookup. Leave `MAX_WORKERS` at its default unless selected-file scans need a higher cap. #### Small library (< 10,000 files) ```bash -MAX_WORKERS=4 CELERY_CONCURRENCY=2 REDIS_MAX_MEMORY=512mb ``` #### Medium library (10,000 - 100,000 files) ```bash -MAX_WORKERS=10 CELERY_CONCURRENCY=4 REDIS_MAX_MEMORY=1gb ``` #### Large library (100,000 - 1,000,000 files) ```bash -MAX_WORKERS=16 CELERY_CONCURRENCY=6 REDIS_MAX_MEMORY=2gb ``` #### Extra large library (1,000,000+ files) ```bash -MAX_WORKERS=24 CELERY_CONCURRENCY=8 REDIS_MAX_MEMORY=4gb ``` @@ -384,20 +375,19 @@ pixelprobe: ## Scanning configuration -### Exclusion configuration +### Exclusion behavior -Exclude specific paths or file types from scanning: +The Tools > Exclusions UI stores paths and extensions in PostgreSQL. The API also supports filename patterns. Scanners and the scheduler use active database rules. This makes imported `EXCLUDED_EXTENSIONS` apply to all scanner paths, including selected-file corruption rescans. Removing configured rules does not remove built-in filename patterns: `._*`, `.DS_Store`, `Thumbs.db`, `.gitkeep`, and `.placeholder` always remain excluded. -**Via environment variables:** -```bash -EXCLUDED_PATHS=/media/temp,/media/incomplete,/media/.cache -EXCLUDED_EXTENSIONS=.tmp,.partial,.!qB,.part,.crdownload -``` +At startup, the v2.10.3 migration imports legacy settings once. It reads `paths`, `extensions`, and `filename_patterns` from a root-level `exclusions.json`, plus `EXCLUDED_PATHS` and `EXCLUDED_EXTENSIONS` from the process environment. JSON and environment values are combined; environment extensions are lowercased, while JSON values keep their spelling. Existing database rows for an exact type and value win, including inactive rows. The `legacy_exclusions_imported_v1` marker prevents a later startup from importing those sources again, even after configured rules are removed. JSON may be absent; malformed JSON, non-list fields, empty/non-string entries, or entries longer than 500 characters stop startup without importing rows or setting the marker. -**Via web interface:** -1. Navigate to Tools > Exclusions -2. Add paths or extensions -3. Click Save +Extension matching uses lowercase file suffixes. Use lowercase `exclusions.json` extensions; environment imports and API writes normalize extensions to lowercase. + +Filename patterns use Python `fnmatch` matching; matching is case-sensitive in the Linux containers. + +After import, environment variables do not override database rules at runtime. Edit paths and extensions in Tools > Exclusions or manage all three types through the API. A `PUT /api/exclusions` request replaces the `paths` and `extensions` lists; an omitted key or an empty list clears that configured type. It changes `filename_patterns` only when that key is supplied; an empty list clears configured patterns. The single-rule API supports `filename_pattern`. The GET endpoint lists configured active rules; built-in filename patterns remain active regardless. + +Each file checker uses the active rules it loaded when it was created. An in-progress checker keeps that snapshot. Parallel discovery carries its captured policy into its discovery task, while chunk checkers load current rules. The scheduler reloads exclusion paths before each scheduled scan; a scan already in progress keeps the root list it started with. Excluded files are skipped before file reads, hashing, decoding, or cache writes. Directory-scoped cleanup and integrity checks apply paths, extensions, and filename patterns they can translate to SQL. That maintenance filter supports `*` and `?`; bracket character classes are not applied. Exact-file maintenance requests and unscoped maintenance do not filter candidates by exclusion rules. ### Schedule configuration @@ -645,36 +635,28 @@ Tokens support optional expiration dates. ## Configuration best practices -Start with the defaults and raise `MAX_WORKERS` and `CELERY_CONCURRENCY` gradually, watching `docker stats` for CPU and memory pressure. Back up the database and `.env` before upgrades, use strong passwords, and keep `SECRET_KEY` secret. +Start with the defaults and raise `CELERY_CONCURRENCY` gradually, watching `docker stats` for CPU and memory pressure. Raise `MAX_WORKERS` only when selected-file scans need more threads. Back up the database and `.env` before upgrades; keep database passwords and `SECRET_KEY` secret. ## Examples ### Home media server (20K files) ```bash -MAX_WORKERS=8 CELERY_CONCURRENCY=3 REDIS_MAX_MEMORY=1gb -POSTGRES_PASSWORD=strong-password-here SCAN_PATHS=/movies,/tv ``` ### Professional archive (500K files) ```bash -MAX_WORKERS=20 CELERY_CONCURRENCY=6 REDIS_MAX_MEMORY=4gb -POSTGRES_PASSWORD=very-strong-password SCAN_PATHS=/archive/video,/archive/images -OUTPUT_ROTATION_ENABLED=true -MAX_OUTPUT_SIZE=50000 ``` ### Multi-user production (2M files) ```bash -MAX_WORKERS=24 CELERY_CONCURRENCY=8 REDIS_MAX_MEMORY=8gb -POSTGRES_PASSWORD=enterprise-strength-password SCAN_PATHS=/storage/media1,/storage/media2,/storage/media3 ``` @@ -684,9 +666,9 @@ See [troubleshooting.md](troubleshooting.md) for solutions to common configurati ## Scanner settings -These are edited under **System > Tunables** in the web interface, or through `GET`, `PUT` and `DELETE` on `/api/settings`. They are stored in the database, so a change reaches a running scan without a restart and survives a container rebuild. +These are edited under **System > Tunables** in the web interface, or through `GET`, `PUT` and `DELETE` on `/api/settings`. They are stored in the database and survive a container rebuild. Workers refresh cached values within 60 seconds. Selected-file scans snapshot settings for each submitted file; other checks use cached values at each setting lookup, so a change is not guaranteed to affect work already in progress. -Each one used to be an environment variable. On first start after upgrading, any of those variables still set in your environment is copied into the database once, so nothing changes under you. After that the stored value wins and the variable is ignored. Removing a setting through the API restores its default. +These settings used to be environment variables. At startup, any legacy variable still set seeds its database row only if that row is missing. A saved row wins while it exists. Resetting a setting through the API deletes its row and restores the default until the next startup; if its legacy variable is still set, that value can be seeded again. Remove the legacy variable before resetting a setting if you want the default to persist across restarts. ### Detection diff --git a/docs/docker-setup.md b/docs/docker-setup.md index 5f6e526..cf28fd8 100644 --- a/docs/docker-setup.md +++ b/docs/docker-setup.md @@ -4,7 +4,7 @@ The Docker Compose setup for PixelProbe and what each container does. ## Container overview -PixelProbe uses 4 main containers: +The Compose stack has four containers: | Container | Purpose | Ports | Dependencies | |-----------|---------|-------|--------------| @@ -31,22 +31,13 @@ services: CELERY_CONCURRENCY: 2 ``` -## Environment variables (.env file) +## Environment variables -Create a `.env` file in the same directory as your `docker-compose.yml`: +The project `.env` file supplies values for substitutions in Compose. It does not automatically add every value to the container environment. The root Compose file forwards only the variables listed under each service's `environment` section. If a service also has `env_file`, entries under `environment` take precedence. Removing a default-valued `environment` entry can expose a value with the same name in `env_file`; check for conflicts before dropping defaults. See the [Compose services reference](https://docs.docker.com/reference/compose-file/services/). -```bash -# Required secrets -POSTGRES_PASSWORD=your-secure-database-password-here -SECRET_KEY=your-secret-key-for-sessions-here - -# Optional settings -TZ=America/New_York -CELERY_CONCURRENCY=4 -MAX_WORKERS=10 -BATCH_SIZE=100 -REDIS_MAX_MEMORY=2gb -``` +After changing a value used by Compose, run `docker compose up -d` to recreate containers whose configuration changed. `docker compose restart` restarts existing containers and does not apply new environment or Compose settings, as described in the [`docker compose restart` reference](https://docs.docker.com/reference/cli/docker/compose/restart/). + +The restart policy handles a stopped process. A failed healthcheck marks a running container unhealthy; it does not trigger the restart policy or stop an active scan by itself. Healthchecks are status signals for operators and dependent services. See Docker's [restart policy documentation](https://docs.docker.com/engine/containers/start-containers-automatically/). ## Container responsibilities @@ -119,32 +110,92 @@ Rule of thumb: set `shared_buffers` to about 25% of the container's memory limit ### Media directories -Mount your media directories as **read-only** to prevent accidental modifications: +For a Portainer stack based on the root Compose file, add the six host path variables in the stack's environment settings. Replace the existing `/media` bind under both application services with these mounts, then set the matching `SCAN_PATHS` values. Keep the web service's `./instance` volume and the rest of the root service configuration. ```yaml -volumes: - - /media/movies:/movies:ro - - /media/tv:/tv:ro - - /media/photos:/photos:ro - - /media/music:/music:ro +x-media-films: &media-films + type: bind + source: ${MEDIA_FILMS_PATH:?Set MEDIA_FILMS_PATH to an existing host directory} + target: /library/films + read_only: true + bind: + create_host_path: false +x-media-series: &media-series + type: bind + source: ${MEDIA_SERIES_PATH:?Set MEDIA_SERIES_PATH to an existing host directory} + target: /library/series + read_only: true + bind: + create_host_path: false +x-media-photos: &media-photos + type: bind + source: ${MEDIA_PHOTOS_PATH:?Set MEDIA_PHOTOS_PATH to an existing host directory} + target: /library/photos + read_only: true + bind: + create_host_path: false +x-media-music: &media-music + type: bind + source: ${MEDIA_MUSIC_PATH:?Set MEDIA_MUSIC_PATH to an existing host directory} + target: /library/music + read_only: true + bind: + create_host_path: false +x-media-home-video: &media-home-video + type: bind + source: ${MEDIA_HOME_VIDEO_PATH:?Set MEDIA_HOME_VIDEO_PATH to an existing host directory} + target: /library/home-video + read_only: true + bind: + create_host_path: false +x-media-misc: &media-misc + type: bind + source: ${MEDIA_MISC_PATH:?Set MEDIA_MISC_PATH to an existing host directory} + target: /library/misc + read_only: true + bind: + create_host_path: false + +services: + pixelprobe: + environment: + SCAN_PATHS: /library/films,/library/series,/library/photos,/library/music,/library/home-video,/library/misc + volumes: + - *media-films + - *media-series + - *media-photos + - *media-music + - *media-home-video + - *media-misc + - ./instance:/app/instance + + celery-worker: + environment: + SCAN_PATHS: /library/films,/library/series,/library/photos,/library/music,/library/home-video,/library/misc + volumes: + - *media-films + - *media-series + - *media-photos + - *media-music + - *media-home-video + - *media-misc ``` -**Important:** the root Compose file already runs both application services as -the same configured user. Keep that setting in any override: +The root Compose file runs both application services as the same configured user. Keep those identities and the web service's existing `instance` volume when adding media mounts: + +`SCAN_PATHS` is synced into the database at web startup, and both services use it for path authorization. Recreate both services after changing it. Existing active database paths remain in use, so disable old roots in the UI if they should no longer be scanned. + +Files such as `.mount-ok` used by host healthchecks are not read by the scanner and do not guard scan roots. For PixelProbe's mount identity check, set `require_mount` on a scan-path configuration; see [Required mount policy](operational-evidence.md#required-mount-policy). ```yaml services: pixelprobe: # ... other settings ... user: "${PUID:-10001}:${PGID:-10001}" - volumes: - - /media/movies:/movies:ro celery-worker: # ... other settings ... - user: "${PUID:-10001}:${PGID:-10001}" # MUST match pixelprobe user - volumes: - - /media/movies:/movies:ro + user: "${PUID:-10001}:${PGID:-10001}" ``` To find your user's UID and GID on the host: @@ -158,7 +209,7 @@ Or use environment variables: user: "${PUID:-10001}:${PGID:-10001}" ``` -If the web app and Celery worker run as different users, the worker may not read the mounted media directories. It then reports "No valid files provided" even when the files exist. +If the web app and Celery worker run as different users, the worker may not read the mounted media directories. The root Compose file also uses `create_host_path: false`, so a missing host directory fails at startup instead of being created as a directory. ### Database persistence @@ -196,16 +247,16 @@ pixelprobe:5000 <-> redis:6379 <-> celery-worker ```bash # Start all containers -docker-compose up -d +docker compose up -d # Check status -docker-compose ps +docker compose ps # View logs -docker-compose logs -f +docker compose logs -f # Stop all containers -docker-compose down +docker compose down ``` ## Monitoring @@ -228,12 +279,12 @@ docker exec pixelprobe-postgres psql -U pixelprobe -c "SELECT count(*) FROM pg_s ## Troubleshooting ### Workers not processing -1. Check Redis is running: `docker-compose ps redis` -2. Check worker logs: `docker-compose logs celery-worker` +1. Check Redis is running: `docker compose ps redis` +2. Check worker logs: `docker compose logs celery-worker` 3. Verify queue: `docker exec pixelprobe-redis valkey-cli LLEN pixelprobe` ### Database connection issues -1. Check PostgreSQL is healthy: `docker-compose ps postgres` +1. Check PostgreSQL is healthy: `docker compose ps postgres` 2. Test connection: `docker exec pixelprobe-postgres pg_isready` 3. Check password in `.env` file @@ -249,7 +300,7 @@ docker exec pixelprobe-postgres psql -U pixelprobe -c "SELECT count(*) FROM pg_s 2. Mount media as read-only (`:ro` flag) 3. Don't expose ports unless needed (remove `ports:` sections) 4. Use a firewall if exposing ports -5. Pull latest images periodically +5. Review published versions periodically and update image tags intentionally ## Backup strategy @@ -271,11 +322,11 @@ tar -czf pixelprobe_config_$(date +%Y%m%d).tar.gz docker-compose.yml .env ## Upgrade process 1. Backup database -2. Stop containers: `docker-compose down` +2. Stop containers: `docker compose down` 3. Update image version in `docker-compose.yml` -4. Pull new image: `docker-compose pull` -5. Start containers: `docker-compose up -d` -6. Check logs: `docker-compose logs -f` +4. Pull new image: `docker compose pull` +5. Start containers: `docker compose up -d` +6. Check logs: `docker compose logs -f` ## PostgreSQL 15 to 18 migration (required for v2.7.0+) @@ -291,7 +342,7 @@ DUMP=pixelprobe_pg15_$(date +%Y%m%d).sql.gz docker exec pixelprobe-postgres pg_dump -U pixelprobe pixelprobe | gzip > "$DUMP" # 2. Stop the stack -docker-compose down +docker compose down # 3. Keep the old volume as a fallback (rename instead of delete) docker volume create pixelprobe_postgres_data_pg15_backup @@ -302,7 +353,7 @@ docker volume rm pixelprobe_postgres_data # /var/lib/postgresql volume mount) - e.g. git pull or edit your copy # 5. Start ONLY postgres on the new compose file (creates a fresh v18 volume) -docker-compose up -d postgres +docker compose up -d postgres # wait for: docker exec pixelprobe-postgres pg_isready -U pixelprobe # 6. Restore the dump @@ -313,7 +364,7 @@ docker exec pixelprobe-postgres psql -U pixelprobe -d pixelprobe -c "ALTER DATAB docker exec pixelprobe-postgres psql -U pixelprobe -d pixelprobe -c "REINDEX DATABASE pixelprobe;" # 8. Start the rest of the stack and verify -docker-compose up -d +docker compose up -d # sanity check: row counts should match your pre-migration numbers docker exec pixelprobe-postgres psql -U pixelprobe -d pixelprobe -c "SELECT COUNT(*) FROM scan_results;" ``` diff --git a/docs/examples/integration-guide.md b/docs/examples/integration-guide.md index b89a56e..523ce6c 100644 --- a/docs/examples/integration-guide.md +++ b/docs/examples/integration-guide.md @@ -742,55 +742,10 @@ class MediaValidator { ## Docker Compose integration -```yaml -version: '3.8' - -services: - pixelprobe: - image: pixelprobe:latest - container_name: pixelprobe - ports: - - "5000:5000" - volumes: - - /media:/media:ro - - pixelprobe_data:/app/data - environment: - - SECRET_KEY=${SECRET_KEY} - - TZ=${TZ:-UTC} - restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:5000/healthz"] - interval: 30s - timeout: 10s - retries: 3 - - # Media server integration - jellyfin: - image: jellyfin/jellyfin - volumes: - - /media:/media - depends_on: - - pixelprobe - - # Monitoring - prometheus: - image: prom/prometheus - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - grafana: - image: grafana/grafana - ports: - - "3000:3000" - volumes: - - grafana_data:/var/lib/grafana - -volumes: - pixelprobe_data: - grafana_data: -``` +Use the root [`docker-compose.yml`](../../docker-compose.yml) for the supported +PixelProbe stack. It includes PostgreSQL, Valkey, the web app, and the Celery +worker with the required identity and read-only media settings. For Portainer +media mounts, see [Docker setup](../docker-setup.md#media-directories). ## Webhook integration diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 188c503..18b78e4 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -681,7 +681,7 @@ deploy: ## Environment variables -Scanner detection, performance and timeout values are no longer environment variables. They are stored in the database and edited under System > Tunables or through `/api/settings`, so a change reaches a running scan without a restart. See [Configuration](configuration.md#scanner-settings). +Scanner detection, performance, and timeout values are stored in the database and edited under System > Tunables or through `/api/settings`. Workers refresh cached values within 60 seconds. Selected-file scans snapshot settings for each submitted file; other checks use cached values at each setting lookup. An edit is not guaranteed to change a file already in progress. See [Configuration](configuration.md#scanner-settings). ### Required @@ -709,8 +709,6 @@ Only one variable is truly required - the app refuses to start without it: | `CELERY_RESULT_BACKEND` | `redis://localhost:6379/0` | Celery result backend | | `CELERY_CONCURRENCY` | `4` | Worker pool size (main scan-throughput knob) | | `SCAN_PATHS` | (empty) | Comma-separated directories to scan | -| `EXCLUDED_PATHS` | (empty) | Comma-separated excluded paths | -| `EXCLUDED_EXTENSIONS` | `.txt,.log,.md` | Comma-separated excluded extensions | | `MAX_WORKERS` | `10` | Thread pool size for selected-file rescans | | `BATCH_SIZE` | `100` | Legacy media-checker discovery lookup batch; not parallel discovery inserts or scan chunk commits | | `SCHEDULER_ENABLED` | `true` | Whether this process may compete for the scheduler lock | @@ -719,113 +717,9 @@ Only one variable is truly required - the app refuses to start without it: | `CHUNK_HEARTBEAT_INTERVAL_SECS` | `120` | Chunk liveness heartbeat interval | | `CHUNK_REVIVE_STALENESS_SECS` | `600` | Staleness threshold before chunk revival | -## Deployment example +## Deployment -Condensed from the repository's `docker-compose.yml`: - -```yaml -services: - postgres: - image: postgres:18-alpine - environment: - POSTGRES_DB: pixelprobe - POSTGRES_USER: pixelprobe - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set} - volumes: - # postgres:18+ images require the mount at /var/lib/postgresql (NOT .../data) - - postgres_data:/var/lib/postgresql - healthcheck: - test: ["CMD-SHELL", "pg_isready -U pixelprobe"] - interval: 10s - timeout: 5s - retries: 5 - networks: - - pixelprobe-network - - # Valkey (Redis-compatible). Service name stays "redis" so redis:// URLs work. - redis: - image: valkey/valkey:9-alpine - command: > - valkey-server - --maxmemory ${REDIS_MAX_MEMORY:-2gb} - --maxmemory-policy noeviction - healthcheck: - test: ["CMD", "valkey-cli", "ping"] - interval: 10s - timeout: 5s - retries: 5 - networks: - - pixelprobe-network - - pixelprobe: - image: ttlequals0/pixelprobe:latest - environment: - SECRET_KEY: ${SECRET_KEY} - # Scheduler runs in celery-worker; keep the web container out of the lock - SCHEDULER_ENABLED: "false" - POSTGRES_HOST: postgres - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set} - CELERY_BROKER_URL: redis://redis:6379/0 - CELERY_RESULT_BACKEND: redis://redis:6379/0 - SCAN_PATHS: ${SCAN_PATHS:-/media} - TRUSTED_INTERNAL_HOSTS: ${TRUSTED_INTERNAL_HOSTS:-} - volumes: - - ${MEDIA_PATH:-./media}:/media:ro - # Instance folder for configs (no database files with PostgreSQL) - - ./instance:/app/instance - ports: - - "${PORT:-5000}:5000" - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:5000/healthz"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 120s # migrations run before workers serve requests - networks: - - pixelprobe-network - - celery-worker: - image: ttlequals0/pixelprobe:latest - command: python celery_worker.py - environment: - SECRET_KEY: ${SECRET_KEY} - POSTGRES_HOST: postgres - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set} - CELERY_BROKER_URL: redis://redis:6379/0 - CELERY_RESULT_BACKEND: redis://redis:6379/0 - CELERY_CONCURRENCY: ${CELERY_CONCURRENCY:-4} - SCAN_PATHS: ${SCAN_PATHS:-/media} - TRUSTED_INTERNAL_HOSTS: ${TRUSTED_INTERNAL_HOSTS:-} - volumes: - - ${MEDIA_PATH:-./media}:/media:ro - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - restart: unless-stopped - networks: - - pixelprobe-network - -networks: - pixelprobe-network: - driver: bridge - -volumes: - postgres_data: -``` - -Notes: -- Postgres and Redis ports are NOT published to the host: the broker has no auth, and app/worker reach both on the compose network -- Valkey runs with `noeviction` so queued tasks are never silently dropped; size it with `REDIS_MAX_MEMORY` (default 2gb) -- The app and celery-worker MUST run as the same user so both can read mounted media files (`user: "${PUID:-10001}:${PGID:-10001}"` by default) -- The container healthcheck hits the unauthenticated `/healthz` liveness endpoint, with a 120s `start_period` because startup migrations run before workers serve requests +The root [`docker-compose.yml`](../docker-compose.yml) is the supported deployment configuration. It defines the PostgreSQL and Valkey services, web and worker settings, read-only media mounts, and healthchecks. See [Docker setup](docker-setup.md) for deployment and Portainer guidance. ## Technology stack diff --git a/docs/installation.md b/docs/installation.md index 6276a9d..82af94b 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -83,53 +83,35 @@ Copy the example environment file and edit it: cp .env.example .env ``` -Edit `.env` with your preferred text editor and set the required variables: +Edit `.env` with your preferred text editor. Generate a secret key with: ```bash # Generate a secure secret key python3 -c "import secrets; print(secrets.token_hex(32))" -# Edit .env file -nano .env # or vim, code, etc. +# Edit .env with your preferred text editor, then set SECRET_KEY, +# POSTGRES_PASSWORD, and MEDIA_PATH from the Docker host. +nano .env ``` -**Required settings in `.env`:** -```bash -# Security (use the generated secret key from above) -SECRET_KEY=your-generated-64-character-hex-key-here - -# Database password (choose a secure password) -POSTGRES_PASSWORD=your-secure-database-password - -# Media path on host system (absolute path) -MEDIA_PATH=/path/to/your/media/directory - -# Scan paths inside container (usually /media) -SCAN_PATHS=/media -``` - -**Optional settings:** -```bash -# Timezone (default: UTC) -TZ=America/New_York +Set `MEDIA_PATH` to an existing host directory. The Compose file fails startup +if the bind source is missing. `SCAN_PATHS` defaults to `/media`; active paths +saved in the app take priority. See [Configuration](configuration.md) for +environment settings and database-backed tunables. -# Performance tuning (see configuration.md for details) -MAX_WORKERS=10 -CELERY_CONCURRENCY=4 -BATCH_SIZE=100 - -# Port (default: 5000) -PORT=5000 -``` +Optional Compose settings include `TZ`, `PORT`, and `CELERY_CONCURRENCY`. +`MAX_WORKERS` caps threads for selected-file scans. It does not set the number +of Celery processes. Avoid adding the legacy `BATCH_SIZE` unless you are +diagnosing media discovery lookups. ### 3. Start the application ```bash -docker-compose up -d +docker compose up -d ``` This will: -1. Pull the latest PixelProbe image from Docker Hub +1. Start the PixelProbe image selected by `PIXELPROBE_VERSION` in Compose 2. Start PostgreSQL database 3. Start Redis message broker 4. Start PixelProbe web application @@ -147,7 +129,7 @@ The Compose file also drops Linux capabilities, enables `no-new-privileges`, and ### 4. Verify containers are running ```bash -docker-compose ps +docker compose ps ``` You should see all containers in "Up" state: @@ -163,11 +145,11 @@ pixelprobe-celery-worker Up ```bash # All containers -docker-compose logs -f +docker compose logs -f # Specific container -docker-compose logs -f pixelprobe-app -docker-compose logs -f pixelprobe-celery-worker +docker compose logs -f pixelprobe +docker compose logs -f celery-worker ``` ## Manual installation @@ -184,13 +166,17 @@ cd PixelProbe ### 2. Set up PostgreSQL database ```bash -# Create database user and database -sudo -u postgres psql << EOF -CREATE USER pixelprobe WITH PASSWORD 'your-secure-password'; +sudo -u postgres psql +``` + +At the `psql` prompt, create the user and database, then set the password: + +```sql +CREATE USER pixelprobe; +\password pixelprobe CREATE DATABASE pixelprobe OWNER pixelprobe; GRANT ALL PRIVILEGES ON DATABASE pixelprobe TO pixelprobe; \q -EOF ``` ### 3. Set up Redis @@ -228,18 +214,18 @@ cp .env.example .env nano .env # Edit configuration ``` -Set the following in `.env`: +Set the generated secret and chosen database password in `.env`. The remaining +values below are connection settings for a local PostgreSQL and Redis setup: + ```bash -SECRET_KEY=your-generated-secret-key FLASK_ENV=production POSTGRES_HOST=localhost POSTGRES_PORT=5432 POSTGRES_DB=pixelprobe POSTGRES_USER=pixelprobe -POSTGRES_PASSWORD=your-secure-password CELERY_BROKER_URL=redis://localhost:6379/0 CELERY_RESULT_BACKEND=redis://localhost:6379/0 -SCAN_PATHS=/path/to/your/media +SCAN_PATHS=/media ``` Notes: @@ -380,16 +366,16 @@ After installation: ```bash # Stop containers -docker-compose down +docker compose down -# Pull latest image -docker-compose pull +# Pull the image version selected in Compose +docker compose pull # Start containers -docker-compose up -d +docker compose up -d # Check logs -docker-compose logs -f +docker compose logs -f ``` ### Manual installation @@ -421,6 +407,6 @@ Common installation issues: ## Getting help 1. Check [troubleshooting.md](troubleshooting.md) for common issues -2. Review logs: `docker-compose logs` or check terminal output +2. Review logs: `docker compose logs` or check terminal output 3. Search existing issues: https://github.com/ttlequals0/PixelProbe/issues 4. Create new issue with logs and system info diff --git a/docs/performance-tuning.md b/docs/performance-tuning.md index 6fdcada..9eeb82e 100644 --- a/docs/performance-tuning.md +++ b/docs/performance-tuning.md @@ -6,15 +6,15 @@ Configuration values in this guide are operator starting points, not measured pe ### Scanning performance - `CELERY_CONCURRENCY=4` - Number of Celery worker processes (default: 4). This is the main scan-throughput knob: scans are split into chunks that fan out across these workers. -- `MAX_WORKERS=10` - ThreadPoolExecutor threads for parallel file validation within a scan task (default: 10). Distinct from Celery concurrency. +- `MAX_WORKERS=10` - Upper bound on thread workers accepted by the selected-file scan API. The API defaults to 4 workers and clamps larger requests to this limit. Distinct from Celery process count. - `BATCH_SIZE=100` - Legacy media-checker discovery lookup batch. Leave at 100; it does not control parallel discovery inserts or scan chunk commits. -- Freeze detection, its confirmation limits, and the scan timeouts are settings rather than environment variables. Edit them under System > Tunables or through `/api/settings`; a change reaches a running scan without a restart. See [Configuration](configuration.md#scanner-settings). +- Freeze detection, its confirmation limits, and scan timeouts are saved in the database. Edit them under System > Tunables or through `/api/settings`. Workers refresh cached values within 60 seconds. Selected-file scans snapshot settings for each submitted file; other checks use cached values at each setting lookup, so an edit is not guaranteed to change a file already in progress. See [Configuration](configuration.md#scanner-settings). - The data integrity check is close to free. The allocation gate reuses the block count from the `stat` the scanner already performs, and files that fail it are queried with `SEEK_HOLE`, which reads no file data. A file it marks incomplete skips decoding entirely, which on a large damaged file saves a full-length pass. - Freeze corroboration runs only on files that already produced a candidate. Each event costs one packet probe (a fraction of a second, no decode) and, when the packets are present, one decode of just that window. At most 12 events per file are probed; the rest fall through to the uncorroborated path. - `time_budget_minutes` - Per-schedule setting on file-changes schedules. Caps how long a rolling integrity check runs; the next run resumes where the last one stopped. ### Database performance -- `DATABASE_URL` - Deprecated since v2.2.0; use the `POSTGRES_*` variables instead +- `DATABASE_URL` - Deprecated compatibility override. If explicitly set, it takes precedence over `POSTGRES_*`; the bundled Compose file does not pass it. - `DB_POOL_SIZE=5` - SQLAlchemy connection pool size per process (default: 5) - `DB_MAX_OVERFLOW=10` - Extra connections beyond the pool (default: 10) - PostgreSQL connection pooling is automatically configured with: @@ -23,7 +23,7 @@ Configuration values in this guide are operator starting points, not measured pe - Pool timeout: 30 seconds - Connection timeout: 10 seconds -Keep pool math under PostgreSQL `max_connections` (default 100): 4 gunicorn workers x (5 + 10) = 60 max for the web app, plus Celery children and checker connections. +Keep pool math under PostgreSQL `max_connections` (default 100): 4 gunicorn workers x (5 + 10) = 60 max for the web app, plus `CELERY_CONCURRENCY` worker children and selected-scan connections. `CELERY_CONCURRENCY` sets Celery processes; `MAX_WORKERS` caps selected-file scan threads. ### Redis - `REDIS_MAX_MEMORY=2gb` - Memory limit for the Redis/Valkey task queue (default: 2gb, recommended: 1-4gb). @@ -88,7 +88,7 @@ Freeze detection fully decodes every video, and that decode is the dominant cost ### For memory-constrained environments - Reduce `CELERY_CONCURRENCY` to 2 -- Reduce `MAX_WORKERS` to 4 +- Reduce `MAX_WORKERS` to 4 when selected-file scans use many threads - Lower `REDIS_MAX_MEMORY` to 1gb ### For high-performance storage diff --git a/openapi.yaml b/openapi.yaml index c8e3190..ffb4caa 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -2481,7 +2481,8 @@ paths: get: summary: Get exclusion settings description: | - Get current exclusion settings from database. + Get active path, extension, and filename-pattern exclusions from PostgreSQL. + Built-in filename patterns are not included in this response and remain active. **Example:** ```bash @@ -2492,11 +2493,32 @@ paths: responses: '200': description: Exclusions retrieved successfully + content: + application/json: + schema: + type: object + required: [paths, extensions, filename_patterns] + properties: + paths: + type: array + items: + type: string + extensions: + type: array + items: + type: string + filename_patterns: + type: array + items: + type: string put: - summary: Update all exclusion settings + summary: Update exclusion settings description: | - Update all exclusion settings in database. + Replace the paths and extensions lists in the database. An omitted path or extension + key clears that type. Update filename patterns only when that key is supplied; an + empty filename_patterns list clears configured patterns. Built-in filename patterns + remain active. **Example:** ```bash @@ -2524,6 +2546,10 @@ paths: type: array items: type: string + filename_patterns: + type: array + items: + type: string responses: '200': description: Exclusions updated successfully @@ -2548,7 +2574,7 @@ paths: required: true schema: type: string - enum: [path, extension] + enum: [path, extension, filename_pattern] requestBody: required: true content: @@ -2582,7 +2608,7 @@ paths: required: true schema: type: string - enum: [path, extension] + enum: [path, extension, filename_pattern] requestBody: required: true content: diff --git a/pixelprobe/api/admin_routes.py b/pixelprobe/api/admin_routes.py index 5c3e01b..494bc40 100644 --- a/pixelprobe/api/admin_routes.py +++ b/pixelprobe/api/admin_routes.py @@ -596,7 +596,7 @@ def delete_schedule(schedule_id): @admin_bp.route('/exclusions', methods=['GET']) @admin_required def get_exclusions(): - """Get current exclusion settings from database""" + """Get active path, extension, and filename pattern exclusions.""" try: from pixelprobe.models import Exclusion @@ -607,42 +607,56 @@ def get_exclusions(): ).all() extension_exclusions = Exclusion.query.filter_by( - exclusion_type='extension', - is_active=True - ).all() + exclusion_type='extension', is_active=True).all() + filename_patterns = Exclusion.query.filter_by( + exclusion_type='filename_pattern', is_active=True).all() return { 'paths': [e.value for e in path_exclusions], - 'extensions': [e.value for e in extension_exclusions] + 'extensions': [e.value for e in extension_exclusions], + 'filename_patterns': [e.value for e in filename_patterns] } except Exception as e: - logger.error(f"Error reading exclusions: {e}") - return {'paths': [], 'extensions': []} + logger.error(f"Error reading exclusions: {e}", exc_info=True) + db.session.rollback() + return {'error': 'Unable to read exclusions'}, 500 @admin_bp.route('/exclusions', methods=['PUT']) @admin_required def update_exclusions(): - """Update all exclusion settings in database""" - data = request.get_json() + """Replace path and extension lists; update patterns only when supplied.""" + data = request.get_json(silent=True) + if not isinstance(data, dict): + return {'error': 'Invalid data format'}, 400 try: from pixelprobe.models import Exclusion # Validate data structure - if not isinstance(data.get('paths', []), list) or not isinstance(data.get('extensions', []), list): + fields = {'paths': 'path', 'extensions': 'extension', + 'filename_patterns': 'filename_pattern'} + if any(key in data and not isinstance(data[key], list) for key in fields): return {'error': 'Invalid data format'}, 400 - - # Clear existing exclusions - Exclusion.query.update({'is_active': False}) - - # Add new exclusions - for path in data.get('paths', []): - exclusion = Exclusion(exclusion_type='path', value=path, is_active=True) - db.session.add(exclusion) - - for extension in data.get('extensions', []): - exclusion = Exclusion(exclusion_type='extension', value=extension, is_active=True) - db.session.add(exclusion) + if any(not isinstance(value, str) or not value.strip() or len(value) > 500 + for key in fields if key in data for value in data[key]): + return {'error': 'Exclusion values must be non-empty strings of at most 500 characters'}, 400 + + for field, exclusion_type in fields.items(): + if field not in data and field == 'filename_patterns': + continue + values = list(dict.fromkeys(data.get(field, []))) + if exclusion_type == 'extension': + values = [value.lower() for value in values] + Exclusion.query.filter_by(exclusion_type=exclusion_type).update( + {'is_active': False}) + for value in values: + row = Exclusion.query.filter_by( + exclusion_type=exclusion_type, value=value).first() + if row: + row.is_active = True + else: + db.session.add(Exclusion( + exclusion_type=exclusion_type, value=value, is_active=True)) db.session.commit() return {'message': 'Exclusions updated'} @@ -654,37 +668,36 @@ def update_exclusions(): @admin_bp.route('/exclusions/', methods=['POST']) @admin_required def add_exclusion(exclusion_type): - """Add a single exclusion (path or extension) to database""" + """Add a path, extension, or filename pattern exclusion.""" # Validate exclusion type - if exclusion_type not in ['path', 'extension']: + if exclusion_type not in ['path', 'extension', 'filename_pattern']: return {'error': 'Invalid exclusion type'}, 400 - data = request.get_json() + data = request.get_json(silent=True) + if not isinstance(data, dict): + return {'error': 'Invalid data format'}, 400 value = data.get('item') or data.get('value') # Support both 'item' and 'value' - - if not value: + + if not isinstance(value, str) or not value.strip() or len(value) > 500: return {'error': 'Value is required'}, 400 + value = value.strip() + if exclusion_type == 'extension': + value = value.lower() try: from pixelprobe.models import Exclusion # Check if already exists existing = Exclusion.query.filter_by( - exclusion_type=exclusion_type, - value=value, - is_active=True - ).first() + exclusion_type=exclusion_type, value=value).first() - if existing: + if existing and existing.is_active: return {'error': f'{exclusion_type.capitalize()} already exists'}, 400 - - # Add new exclusion - exclusion = Exclusion( - exclusion_type=exclusion_type, - value=value, - is_active=True - ) - db.session.add(exclusion) + if existing: + existing.is_active = True + else: + db.session.add(Exclusion( + exclusion_type=exclusion_type, value=value, is_active=True)) db.session.commit() AuditLogger.log_action('add_exclusion', {'type': exclusion_type, 'value': value}) @@ -701,13 +714,14 @@ def add_exclusion(exclusion_type): def remove_exclusion(exclusion_type): """Remove a single exclusion (path or extension) from database""" # Validate exclusion type - if exclusion_type not in ['path', 'extension']: + if exclusion_type not in ['path', 'extension', 'filename_pattern']: return {'error': 'Invalid exclusion type'}, 400 - data = request.get_json() + data = request.get_json(silent=True) + if not isinstance(data, dict): + return {'error': 'Invalid data format'}, 400 value = data.get('item') or data.get('value') # Support both 'item' and 'value' - - if not value: + if not isinstance(value, str) or not value.strip() or len(value) > 500: return {'error': 'Value is required'}, 400 try: diff --git a/pixelprobe/media_checker.py b/pixelprobe/media_checker.py index 1b6487e..320e2eb 100644 --- a/pixelprobe/media_checker.py +++ b/pixelprobe/media_checker.py @@ -24,6 +24,7 @@ import shutil from concurrent.futures import ThreadPoolExecutor, as_completed import threading +import fnmatch from pixelprobe.utils.security import ( authorized_fd_path, open_authorized_media_file, rewind_authorized_fd_path, safe_subprocess_run, validate_file_path, ensure_cli_safe_path, @@ -376,56 +377,31 @@ def get_default_filename_patterns(): ] def load_exclusions(): - """Load exclusion patterns from exclusions.json file with default exclusions - - Returns: - tuple: (excluded_paths, excluded_extensions) for backward compatibility - """ - # Default exclusions that are always applied - default_excluded_paths = [] - default_excluded_extensions = [] - - try: - # Load user-defined exclusions from file - exclusions_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'exclusions.json') - user_paths = [] - user_extensions = [] - - if os.path.exists(exclusions_file): - with open(exclusions_file, 'r') as f: - data = json.load(f) - user_paths = data.get('paths', []) - user_extensions = data.get('extensions', []) - - # Combine default and user exclusions - excluded_paths = list(set(default_excluded_paths + user_paths)) - excluded_extensions = list(set(default_excluded_extensions + user_extensions)) - - return excluded_paths, excluded_extensions - except Exception as e: - logger.error(f"Error loading exclusions.json: {e}") - # Return defaults on error - return default_excluded_paths, default_excluded_extensions + """Load path and extension exclusions from the active policy source.""" + paths, extensions, _ = load_exclusions_with_patterns() + return paths, extensions def load_exclusions_with_patterns(): - """Load exclusion patterns including filename patterns. - - Reads exclusions.json once and returns all exclusion data. - - Returns: - tuple: (excluded_paths, excluded_extensions, excluded_patterns) - """ + """Load active database policy in the app, retaining standalone JSON use.""" default_patterns = get_default_filename_patterns() + from flask import has_app_context + if has_app_context(): + from pixelprobe.models import Exclusion + rows = Exclusion.query.filter_by(is_active=True).all() + paths = [row.value for row in rows if row.exclusion_type == 'path'] + extensions = [row.value for row in rows if row.exclusion_type == 'extension'] + patterns = [row.value for row in rows if row.exclusion_type == 'filename_pattern'] + return paths, extensions, list(dict.fromkeys(default_patterns + patterns)) try: exclusions_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'exclusions.json') if os.path.exists(exclusions_file): with open(exclusions_file, 'r') as f: data = json.load(f) - paths = list(set(data.get('paths', []))) - extensions = list(set(data.get('extensions', []))) + paths = list(dict.fromkeys(data.get('paths', []))) + extensions = list(dict.fromkeys(data.get('extensions', []))) user_patterns = data.get('filename_patterns', []) - excluded_patterns = list(set(default_patterns + user_patterns)) + excluded_patterns = list(dict.fromkeys(default_patterns + user_patterns)) return paths, extensions, excluded_patterns except Exception as e: logger.error(f"Error loading exclusions.json: {e}") @@ -841,7 +817,6 @@ def _is_supported_file(self, file_path): return False # Check if filename matches exclusion patterns - import fnmatch for pattern in self.excluded_patterns: if fnmatch.fnmatch(filename, pattern): logger.debug(f"Excluding {filename} - matches pattern {pattern}") @@ -1282,6 +1257,14 @@ def scan_file(self, file_path, force_rescan=False): with self.scan_lock: self.current_scan_file = file_path self.scan_start_time = scan_start_time + + extension = Path(file_path).suffix.lower() + filename = os.path.basename(file_path) + if (extension in self.excluded_extensions + or any(is_path_under(file_path, path) for path in self.excluded_paths) + or any(fnmatch.fnmatch(filename, pattern) + for pattern in self.excluded_patterns)): + return {'file_path': file_path, 'outcome': 'excluded', 'excluded': True} media_file = None checked_path = file_path diff --git a/pixelprobe/migrations/startup.py b/pixelprobe/migrations/startup.py index 1d527c9..8737737 100644 --- a/pixelprobe/migrations/startup.py +++ b/pixelprobe/migrations/startup.py @@ -7,6 +7,7 @@ import os import logging +import json from contextlib import contextmanager from contextvars import ContextVar @@ -27,6 +28,8 @@ # Migrations are idempotent: a timed-out one is retried on the next boot. MIGRATION_LOCK_TIMEOUT_MS = env_int('MIGRATION_LOCK_TIMEOUT_MS', 10000, floor=1000) MIGRATION_STATEMENT_TIMEOUT_MS = env_int('MIGRATION_STATEMENT_TIMEOUT_MS', 300000, floor=10000) +LEGACY_EXCLUSIONS_FILE = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'exclusions.json') +LEGACY_EXCLUSIONS_MARKER = 'legacy_exclusions_imported_v1' _migration_owner_connection = ContextVar('migration_owner_connection', default=None) @@ -374,6 +377,57 @@ def run_v2_6_53_migrations(db): raise +def run_v2_10_3_exclusion_migration(db): + """Import legacy exclusion sources once, preserving explicit DB rows.""" + try: + with migration_connection(db) as conn: + if conn.execute(text( + "SELECT 1 FROM app_configs WHERE key = :key"), + {'key': LEGACY_EXCLUSIONS_MARKER}).first(): + return + + legacy = {'path': [], 'extension': [], 'filename_pattern': []} + try: + with open(LEGACY_EXCLUSIONS_FILE, encoding='utf-8') as source: + data = json.load(source) + except FileNotFoundError: + data = {} + if not isinstance(data, dict): + raise ValueError('Legacy exclusions must be a JSON object') + for key, exclusion_type in ( + ('paths', 'path'), ('extensions', 'extension'), + ('filename_patterns', 'filename_pattern')): + values = data.get(key, []) + if not isinstance(values, list) or any( + not isinstance(value, str) or not value.strip() + or len(value) > 500 for value in values): + raise ValueError(f'Legacy exclusions field {key} must be a list of non-empty strings') + legacy[exclusion_type].extend(values) + legacy['path'].extend( + value.strip() for value in os.environ.get('EXCLUDED_PATHS', '').split(',') + if value.strip()) + legacy['extension'].extend( + value.strip().lower() for value in os.environ.get('EXCLUDED_EXTENSIONS', '').split(',') + if value.strip()) + + for exclusion_type, values in legacy.items(): + for value in dict.fromkeys(values): + conn.execute(text(""" + INSERT INTO exclusions (exclusion_type, value, created_at, is_active) + VALUES (:type, :value, CURRENT_TIMESTAMP, TRUE) + ON CONFLICT (exclusion_type, value) DO NOTHING + """), {'type': exclusion_type, 'value': value}) + conn.execute(text(""" + INSERT INTO app_configs (key, value, description) + VALUES (:key, 'true', 'Legacy exclusion import completed') + ON CONFLICT (key) DO NOTHING + """), {'key': LEGACY_EXCLUSIONS_MARKER}) + conn.commit() + except Exception as e: + logger.error(f"Legacy exclusion import failed: {e}") + raise + + def run_v2_6_60_migrations(db): """Add scan_schedules.time_budget_minutes for budgeted integrity runs. @@ -808,6 +862,9 @@ def _run_all_migrations(db, connection): logger.info("Creating duplicate index state and invalidation triggers...") run_duplicate_index_migrations(db) + logger.info("Importing legacy exclusion settings...") + run_v2_10_3_exclusion_migration(db) + logger.info("Creating performance indexes...") create_performance_indexes(db) logger.info("Performance indexes created successfully") diff --git a/pixelprobe/models.py b/pixelprobe/models.py index 559ce61..2710dc6 100644 --- a/pixelprobe/models.py +++ b/pixelprobe/models.py @@ -256,7 +256,7 @@ class Exclusion(db.Model): __tablename__ = 'exclusions' id = db.Column(db.Integer, primary_key=True) - exclusion_type = db.Column(db.String(20), nullable=False) # 'path' or 'extension' + exclusion_type = db.Column(db.String(20), nullable=False) value = db.Column(db.String(500), nullable=False) created_at = db.Column(db.DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) is_active = db.Column(db.Boolean, nullable=False, default=True) diff --git a/pixelprobe/scheduler.py b/pixelprobe/scheduler.py index b834549..358b0c2 100644 --- a/pixelprobe/scheduler.py +++ b/pixelprobe/scheduler.py @@ -91,7 +91,6 @@ def __init__(self, app=None): 'SCHEDULE_RETRY_MAX_COUNT', self.DEFAULT_RETRY_MAX_COUNT, min_value=0 ) - # Load exclusions from environment self._load_exclusions() @staticmethod @@ -407,14 +406,17 @@ def init_app(self, app): logger.error(f"Failed to queue missed schedule fires: {e}") def _load_exclusions(self): - """Load path and extension exclusions from environment variables""" - excluded_paths_env = os.environ.get('EXCLUDED_PATHS', '') - if excluded_paths_env: - self.excluded_paths = [p.strip() for p in excluded_paths_env.split(',') if p.strip()] - - excluded_extensions_env = os.environ.get('EXCLUDED_EXTENSIONS', '') - if excluded_extensions_env: - self.excluded_extensions = [e.strip().lower() for e in excluded_extensions_env.split(',') if e.strip()] + """Refresh scheduled-scan exclusions from the shared database policy.""" + from flask import has_app_context + if not has_app_context(): + self.excluded_paths = [] + self.excluded_extensions = [] + return + from pixelprobe.models import Exclusion + rows = Exclusion.query.filter_by(is_active=True).all() + self.excluded_paths = [row.value for row in rows if row.exclusion_type == 'path'] + self.excluded_extensions = [ + row.value.lower() for row in rows if row.exclusion_type == 'extension'] def _schedule_default_tasks(self): """Schedule default tasks based on environment variables""" @@ -574,6 +576,7 @@ def _run_periodic_scan(self): try: with self.app.app_context(): + self._load_exclusions() # Check if ANY scan is already running before proceeding scan_state = ScanState.get_or_create() if scan_state.is_active and scan_state.phase not in TERMINAL_SCAN_PHASES: @@ -628,6 +631,7 @@ def _run_scheduled_scan(self, schedule_id: int): try: with self.app.app_context(): + self._load_exclusions() # Check if ANY scan is already running before proceeding scan_state = ScanState.get_or_create() if scan_state.is_active and scan_state.phase not in TERMINAL_SCAN_PHASES: diff --git a/pixelprobe/services/scan_service.py b/pixelprobe/services/scan_service.py index 44da77b..864d586 100644 --- a/pixelprobe/services/scan_service.py +++ b/pixelprobe/services/scan_service.py @@ -893,6 +893,24 @@ def _snapshot_run_member(self, scan_id: str, file_path: str, observed=None): member.completed_at = datetime.now(timezone.utc) db.session.commit() return True + if isinstance(observed, dict) and observed.get('outcome') == 'excluded': + member.status = 'skipped' + member.outcome = 'excluded' + member.file_hash = None + member.file_size = None + member.last_modified = None + member.is_corrupted = None + member.has_warnings = None + member.corruption_details = None + member.warning_details = None + member.file_type = None + member.scan_tool = None + member.scan_output = None + member.marked_as_good = None + member.error_message = None + member.completed_at = datetime.now(timezone.utc) + db.session.commit() + return True row = (ScanResult.query.filter_by(file_path=file_path) .populate_existing().first()) if not row: @@ -1115,7 +1133,7 @@ def _sequential_scan_selected_chunks(self, checker: PixelProbe, chunks: List[Sca result = checker.scan_file(file_path, force_rescan=force_rescan) recorded = self._snapshot_run_member(scan_state.scan_id, file_path, result) outcome = result.get('outcome') if isinstance(result, dict) else None - if not recorded or outcome != 'completed': + if not recorded or outcome not in ('completed', 'excluded'): chunk_failed = True if not recorded: self._mark_selected_member_error( @@ -1291,7 +1309,7 @@ def scan_file(path=file_path, values=settings): recorded = self._snapshot_run_member( scan_state.scan_id, file_path, observed) outcome = observed.get('outcome') if isinstance(observed, dict) else None - if not recorded or outcome != 'completed': + if not recorded or outcome not in ('completed', 'excluded'): chunk_failed[directory] = True if not recorded: self._mark_selected_member_error( diff --git a/pixelprobe/version.py b/pixelprobe/version.py index 00a9b46..81517ae 100644 --- a/pixelprobe/version.py +++ b/pixelprobe/version.py @@ -4,7 +4,7 @@ # Default version - this is the single source of truth -_DEFAULT_VERSION = '2.10.2' +_DEFAULT_VERSION = '2.10.4' # Allow override via environment variable for CI/CD, but default to the hardcoded version diff --git a/static/css/mobile.css b/static/css/mobile.css index 964be13..3517f4a 100644 --- a/static/css/mobile.css +++ b/static/css/mobile.css @@ -5,7 +5,8 @@ #media-viewer-modal .modal-content { width: calc(100% - 1rem); max-width: none; - height: calc(100dvh - 1rem); + height: calc(100vh - 1rem); + height: calc(100dvh - env(safe-area-inset-top) - env(safe-area-inset-bottom) - 1rem); margin: 0.5rem; } @@ -63,6 +64,11 @@ top: 0 !important; width: var(--sidebar-width) !important; height: 100vh !important; + height: 100dvh !important; + box-sizing: border-box; + padding-top: env(safe-area-inset-top); + padding-bottom: env(safe-area-inset-bottom); + overscroll-behavior-y: contain; z-index: 1000 !important; transform: translateX(-100%) !important; transition: transform 0.3s ease !important; @@ -81,10 +87,9 @@ .sidebar-overlay { display: none; position: fixed; - top: 0; - left: 0; - right: 0; - bottom: 0; + inset: 0; + height: 100vh; + height: 100dvh; background-color: rgba(0, 0, 0, 0.5); z-index: 999; } @@ -383,6 +388,7 @@ width: 100%; margin: 0; height: 100vh; + height: calc(100dvh - env(safe-area-inset-top) - env(safe-area-inset-bottom)); border-radius: 0; display: flex; flex-direction: column; @@ -420,6 +426,7 @@ width: 100%; margin: 0 !important; height: 100vh; + height: calc(100dvh - env(safe-area-inset-top) - env(safe-area-inset-bottom)); border-radius: 0; display: flex; flex-direction: column; @@ -709,15 +716,36 @@ } /* Modal adjustments for mobile */ + .modal { + inset: 0; + width: auto; + height: 100vh; + height: 100dvh; + padding: env(safe-area-inset-top) env(safe-area-inset-right) + env(safe-area-inset-bottom) env(safe-area-inset-left); + overflow: hidden; + overscroll-behavior-y: contain; + } + .modal-content { - margin: 10px; + display: flex; + flex-direction: column; + margin: 10px auto; width: calc(100% - 20px); max-height: calc(100vh - 20px); + max-height: calc(100dvh - env(safe-area-inset-top) - env(safe-area-inset-bottom) - 20px); + } + + .modal-header { + flex: 0 0 auto; } .modal-body { - max-height: calc(100vh - 120px); + flex: 1 1 auto; + min-height: 0; + max-height: none; overflow-y: auto; + overscroll-behavior-y: contain; -webkit-overflow-scrolling: touch; } @@ -1177,7 +1205,8 @@ max-width: 100%; width: 100%; height: 100vh; - max-height: 100vh; + height: calc(100dvh - env(safe-area-inset-top) - env(safe-area-inset-bottom)); + max-height: calc(100dvh - env(safe-area-inset-top) - env(safe-area-inset-bottom)); border-radius: 0; margin: 0; display: flex; diff --git a/static/js/app.js b/static/js/app.js index f4dc95a..7994b22 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -110,6 +110,68 @@ class ThemeManager { } } +function initializeMobileScrollLock() { + const body = document.body; + const properties = ['overflow', 'overflow-x', 'overflow-y', 'position', 'top', 'left', 'right', 'width']; + let previousBodyStyle = null; + + const sync = () => { + const sidebarOpen = Boolean(document.querySelector('.sidebar.active')); + const modalOpen = Array.from(document.querySelectorAll('.modal')).some( + modal => getComputedStyle(modal).display !== 'none'); + const shouldLock = window.innerWidth <= 768 && (sidebarOpen || modalOpen); + + if (shouldLock && !previousBodyStyle) { + previousBodyStyle = { + scrollY: window.scrollY, + values: Object.fromEntries(properties.map(property => [ + property, + [body.style.getPropertyValue(property), body.style.getPropertyPriority(property)], + ])), + }; + body.style.position = 'fixed'; + body.style.top = `-${previousBodyStyle.scrollY}px`; + body.style.left = '0'; + body.style.right = '0'; + body.style.width = 'auto'; + body.style.overflow = 'hidden'; + } else if (!shouldLock && previousBodyStyle) { + const { scrollY, values } = previousBodyStyle; + previousBodyStyle = null; + properties.forEach(property => { + const [value, priority] = values[property]; + if (value) body.style.setProperty(property, value, priority); + else body.style.removeProperty(property); + }); + window.scrollTo(0, scrollY); + } + }; + + const attributeObserver = new MutationObserver(sync); + const observeOverlays = () => { + attributeObserver.disconnect(); + document.querySelectorAll('.modal, .sidebar').forEach(overlay => { + attributeObserver.observe(overlay, { + attributes: true, + attributeFilter: ['class', 'style'], + }); + }); + }; + const childObserver = new MutationObserver(records => { + const changed = records.some(record => + [...record.addedNodes, ...record.removedNodes].some(node => + node.nodeType === Node.ELEMENT_NODE && node.matches('.modal, .sidebar'))); + if (changed) { + observeOverlays(); + sync(); + } + }); + observeOverlays(); + childObserver.observe(body, { childList: true }); + window.addEventListener('resize', sync); + sync(); +} + // Sidebar Management class SidebarManager { constructor() { @@ -165,13 +227,11 @@ class SidebarManager { toggleMobile() { this.sidebar?.classList.toggle('active'); this.overlay?.classList.toggle('active'); - document.body.style.overflow = this.sidebar?.classList.contains('active') ? 'hidden' : ''; } closeMobile() { this.sidebar?.classList.remove('active'); this.overlay?.classList.remove('active'); - document.body.style.overflow = ''; } toggleDesktop() { @@ -5327,6 +5387,7 @@ class PixelProbeApp { async loadExclusions() { try { const response = await fetch('/api/exclusions'); + if (!response.ok) throw new Error('Failed to load exclusions'); const data = await response.json(); this.renderExclusionList('#excluded-paths-list', data.paths, 'path', 'No excluded paths'); @@ -5474,6 +5535,7 @@ function bindTemplateActions() { // Initialize when DOM is ready document.addEventListener('DOMContentLoaded', () => { + initializeMobileScrollLock(); window.app = new PixelProbeApp(); window.app.init(); bindTemplateActions(); diff --git a/tests/frontend_dom_test.js b/tests/frontend_dom_test.js index d9d6b2f..1479ee8 100644 --- a/tests/frontend_dom_test.js +++ b/tests/frontend_dom_test.js @@ -8,7 +8,7 @@ window.fetch = async () => ({ ok: true, json: async () => ({}) }); window.Chart = function () {}; let source = fs.readFileSync('static/js/app.js', 'utf8'); source = source.replace('document.addEventListener(\'DOMContentLoaded\'', '/* test suppresses bootstrap */ document.addEventListener(\'testDOMContentLoaded\''); -source += '\nwindow.__TableManager = TableManager; window.__PixelProbeApp = PixelProbeApp; window.__StatsDashboard = StatsDashboard; window.__ProgressManager = ProgressManager; window.__APIClient = APIClient;'; +source += '\nwindow.__TableManager = TableManager; window.__PixelProbeApp = PixelProbeApp; window.__StatsDashboard = StatsDashboard; window.__ProgressManager = ProgressManager; window.__APIClient = APIClient; window.__initializeMobileScrollLock = initializeMobileScrollLock;'; window.eval(source); const table = new window.__TableManager({ getScanResults: async () => ({}) }); @@ -537,10 +537,74 @@ async function verifyViewer() { throw error; }); +(async () => { + const lockDom = new JSDOM('', { runScripts: 'outside-only' }); + const lockWindow = lockDom.window; + const lockBody = lockWindow.document.body; + Object.defineProperty(lockWindow, 'innerWidth', { configurable: true, value: 390, writable: true }); + Object.defineProperty(lockWindow, 'scrollY', { configurable: true, value: 125 }); + let restoredScroll; + lockWindow.scrollTo = (_x, y) => { restoredScroll = y; }; + lockBody.style.setProperty('overflow', 'scroll', 'important'); + lockBody.style.setProperty('position', 'relative'); + lockWindow.eval(source); + lockWindow.__initializeMobileScrollLock(); + const sidebar = lockWindow.document.querySelector('.sidebar'); + sidebar.classList.add('active'); + await new Promise(resolve => setTimeout(resolve, 0)); + assert.equal(lockBody.style.position, 'fixed'); + + const firstModal = lockWindow.document.querySelector('#schedule-modal'); + firstModal.style.display = 'block'; + await new Promise(resolve => setTimeout(resolve, 0)); + sidebar.classList.remove('active'); + await new Promise(resolve => setTimeout(resolve, 0)); + assert.equal(lockBody.style.position, 'fixed'); + + const secondModal = lockWindow.document.createElement('div'); + secondModal.className = 'modal'; + secondModal.style.display = 'block'; + lockBody.appendChild(secondModal); + await new Promise(resolve => setTimeout(resolve, 0)); + firstModal.style.display = 'none'; + await new Promise(resolve => setTimeout(resolve, 0)); + assert.equal(lockBody.style.position, 'fixed'); + secondModal.remove(); + await new Promise(resolve => setTimeout(resolve, 0)); + assert.equal(lockBody.style.getPropertyValue('overflow'), 'scroll'); + assert.equal(lockBody.style.getPropertyPriority('overflow'), 'important'); + assert.equal(lockBody.style.position, 'relative'); + assert.equal(restoredScroll, 125); + + firstModal.style.display = 'block'; + await new Promise(resolve => setTimeout(resolve, 0)); + lockWindow.innerWidth = 900; + lockWindow.dispatchEvent(new lockWindow.Event('resize')); + assert.equal(lockBody.style.position, 'relative'); + assert.equal(restoredScroll, 125); + + const postcss = require('postcss'); + const css = postcss.parse(fs.readFileSync('static/css/mobile.css', 'utf8')); + const declarations = selector => { + const found = []; + css.walkRules(selector, rule => rule.walkDecls(decl => found.push([decl.prop, decl.value]))); + return found; + }; + assert(declarations('.sidebar').some(([prop, value]) => prop === 'height' && value === '100dvh')); + assert(declarations('.modal').some(([prop, value]) => prop === 'height' && value === '100dvh')); + assert(declarations('.modal-content').some(([prop, value]) => prop === 'max-height' && value.includes('100dvh'))); + assert(declarations('.modal-body').some(([prop, value]) => prop === 'min-height' && value === '0')); + assert(declarations('.modal-body').some(([prop, value]) => prop === 'overflow-y' && value === 'auto')); +})().catch((error) => { + process.exitCode = 1; + throw error; +}); + (async () => { const payload = 'Stored "value" '; const componentDom = new JSDOM(`
+
`, { runScripts: 'outside-only' }); const componentWindow = componentDom.window; @@ -567,12 +631,32 @@ async function verifyViewer() { schedule.querySelector('[title="Configure Healthcheck"]').click(); assert.equal(healthcheckName, payload); componentApp.renderExclusionList('#excluded-paths-list', [payload], 'path', 'None'); + componentApp.renderExclusionList('#excluded-extensions-list', ['.tmp'], 'extension', 'None'); const exclusion = componentWindow.document.querySelector('#excluded-paths-list'); assert.equal(exclusion.querySelector('img'), null); exclusion.querySelector('button').click(); assert.equal(exclusionValue, payload); + const pathListBeforeFailure = exclusion.innerHTML; + const extensionListBeforeFailure = componentWindow.document.querySelector( + '#excluded-extensions-list').innerHTML; + let parsedFailureResponse = false; + componentWindow.fetch = async () => ({ + ok: false, + status: 500, + json: async () => { + parsedFailureResponse = true; + return { paths: [], extensions: [] }; + }, + }); + await componentApp.loadExclusions(); + assert.equal(parsedFailureResponse, false); + assert.equal(exclusion.innerHTML, pathListBeforeFailure); + assert.equal(componentWindow.document.querySelector( + '#excluded-extensions-list').innerHTML, extensionListBeforeFailure); + assert.equal(componentWindow.document.querySelector( + '.notification-error').textContent, 'Failed to load exclusions'); componentApp.showNotification(payload, 'warning'); - assert.equal(componentWindow.document.querySelector('.notification').textContent, payload); + assert.equal(componentWindow.document.querySelector('.notification-warning').textContent, payload); const confirmation = componentApp.showConfirmModal(payload, payload); assert.equal(componentWindow.document.querySelector('#confirm-title').textContent, payload); assert.equal(componentWindow.document.querySelector('#confirm-message').textContent, payload); diff --git a/tests/integration/test_admin_endpoints.py b/tests/integration/test_admin_endpoints.py index 5b6e062..eaf910f 100644 --- a/tests/integration/test_admin_endpoints.py +++ b/tests/integration/test_admin_endpoints.py @@ -335,6 +335,67 @@ def test_invalid_exclusion_type(self, authenticated_client, db): assert response.status_code == 400 assert 'Invalid exclusion type' in response.get_json()['error'] + def test_put_preserves_patterns_and_reactivates_removed_value( + self, authenticated_client, db, app): + from pixelprobe.models import Exclusion + + with app.app_context(): + pattern = Exclusion(exclusion_type='filename_pattern', value='*.skip') + removed = Exclusion(exclusion_type='path', value='/library/removed', + is_active=False) + db.session.add_all([pattern, removed]) + db.session.commit() + + response = authenticated_client.put('/api/exclusions', json={ + 'paths': ['/library/keep'], 'extensions': ['.TMP']}) + assert response.status_code == 200 + response = authenticated_client.put('/api/exclusions', json={ + 'paths': ['/library/keep'], 'extensions': ['.TMP']}) + assert response.status_code == 200 + assert pattern.is_active is True + assert removed.is_active is False + + response = authenticated_client.post('/api/exclusions/path', + json={'item': '/library/removed'}) + assert response.status_code == 200 + assert removed.is_active is True + + from pixelprobe.media_checker import load_exclusions_with_patterns + paths, extensions, patterns = load_exclusions_with_patterns() + assert '/library/removed' in paths + assert '.tmp' in extensions + assert '*.skip' in patterns + + response = authenticated_client.delete('/api/exclusions/path', + json={'item': '/library/removed'}) + assert response.status_code == 200 + assert '/library/removed' not in load_exclusions_with_patterns()[0] + + response = authenticated_client.get('/api/exclusions') + assert response.get_json()['filename_patterns'] == ['*.skip'] + assert response.get_json()['extensions'] == ['.tmp'] + + response = authenticated_client.post('/api/exclusions/filename_pattern', + json={'item': '*.temp'}) + assert response.status_code == 200 + response = authenticated_client.delete('/api/exclusions/filename_pattern', + json={'item': '*.temp'}) + assert response.status_code == 200 + + response = authenticated_client.put('/api/exclusions', json={ + 'paths': [], 'extensions': [], 'filename_patterns': []}) + assert response.status_code == 200 + paths, extensions, patterns = load_exclusions_with_patterns() + assert paths == [] + assert extensions == [] + assert '*.skip' not in patterns + assert '._*' in patterns + + def test_delete_rejects_non_string_value(self, authenticated_client, db): + response = authenticated_client.delete('/api/exclusions/path', + json={'item': ['bad']}) + assert response.status_code == 400 + class TestIgnoredPatternsEndpoints: """Test ignored patterns endpoints""" diff --git a/tests/test_exclusions_postgres.py b/tests/test_exclusions_postgres.py new file mode 100644 index 0000000..6e08ce9 --- /dev/null +++ b/tests/test_exclusions_postgres.py @@ -0,0 +1,154 @@ +"""PostgreSQL coverage for the one-time exclusion migration and live policy.""" + +import json +import os +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from flask import Flask +from sqlalchemy import create_engine, text + +from pixelprobe.models import Exclusion, AppConfig, db + +POSTGRES_URI = os.environ.get('PIXELPROBE_TEST_POSTGRES_URI') + + +@pytest.fixture +def exclusion_database(tmp_path, monkeypatch): + from pixelprobe.migrations import startup + + schema = f"exclusions_{uuid4().hex[:12]}" + admin_engine = create_engine(POSTGRES_URI) + with admin_engine.begin() as conn: + conn.execute(text(f'CREATE SCHEMA {schema}')) + + app = Flask(__name__) + app.config.update( + SQLALCHEMY_DATABASE_URI=POSTGRES_URI, + SQLALCHEMY_TRACK_MODIFICATIONS=False, + SQLALCHEMY_ENGINE_OPTIONS={ + 'connect_args': {'options': f'-csearch_path={schema}'}}, + ) + db.init_app(app) + + try: + with app.app_context(): + db.create_all() + migration_db = SimpleNamespace(engine=db.engine, metadata=db.metadata) + json_path = tmp_path / 'legacy-exclusions.json' + monkeypatch.setattr(startup, 'LEGACY_EXCLUSIONS_FILE', str(json_path)) + monkeypatch.delenv('EXCLUDED_PATHS', raising=False) + monkeypatch.delenv('EXCLUDED_EXTENSIONS', raising=False) + try: + yield app, migration_db, json_path + finally: + db.session.remove() + db.engine.dispose() + finally: + with admin_engine.begin() as conn: + conn.execute(text(f'DROP SCHEMA {schema} CASCADE')) + admin_engine.dispose() + + +@pytest.mark.postgres +@pytest.mark.skipif(not POSTGRES_URI, reason='PIXELPROBE_TEST_POSTGRES_URI not set') +def test_legacy_exclusions_import_once_and_database_tombstones_win( + exclusion_database, monkeypatch): + from pixelprobe.migrations.startup import run_v2_10_3_exclusion_migration + from pixelprobe.media_checker import load_exclusions_with_patterns + + app, migration_db, json_path = exclusion_database + json_path.write_text(json.dumps({ + 'paths': ['/media/from-json', '/media/hidden'], + 'extensions': ['.tmp', '.MP4'], + 'filename_patterns': ['sample-*', 'Sample-*'], + }), encoding='utf-8') + monkeypatch.setenv('EXCLUDED_PATHS', '/media/from-env') + monkeypatch.setenv('EXCLUDED_EXTENSIONS', '.PART') + + with app.app_context(): + db.session.add(Exclusion( + exclusion_type='path', value='/media/hidden', is_active=False)) + db.session.commit() + + run_v2_10_3_exclusion_migration(migration_db) + run_v2_10_3_exclusion_migration(migration_db) + + active = {(row.exclusion_type, row.value) for row in Exclusion.query + if row.is_active} + assert active == { + ('path', '/media/from-json'), + ('path', '/media/from-env'), + ('extension', '.tmp'), + ('extension', '.MP4'), + ('extension', '.part'), + ('filename_pattern', 'sample-*'), + ('filename_pattern', 'Sample-*'), + } + assert Exclusion.query.filter_by( + exclusion_type='path', value='/media/hidden').one().is_active is False + assert db.session.query(AppConfig).filter_by( + key='legacy_exclusions_imported_v1').count() == 1 + + paths, extensions, patterns = load_exclusions_with_patterns() + assert set(paths) == {'/media/from-json', '/media/from-env'} + assert set(extensions) == {'.tmp', '.MP4', '.part'} + assert 'sample-*' in patterns + assert 'Sample-*' in patterns + assert {'.DS_Store', 'Thumbs.db', '._*', '.gitkeep', '.placeholder'} <= set(patterns) + + # A new app context observes the policy through a separate session. + db.session.remove() + with app.app_context(): + paths, extensions, patterns = load_exclusions_with_patterns() + assert '/media/from-env' in paths + assert '.part' in extensions + assert 'Sample-*' in patterns + + # A later malformed legacy file is ignored after the marker is stored. + json_path.write_text('{invalid', encoding='utf-8') + run_v2_10_3_exclusion_migration(migration_db) + + # Even a physical row deletion cannot cause the legacy source to reseed it. + Exclusion.query.filter_by(value='/media/from-json').delete() + db.session.commit() + run_v2_10_3_exclusion_migration(migration_db) + assert Exclusion.query.filter_by(value='/media/from-json').count() == 0 + + +@pytest.mark.postgres +@pytest.mark.skipif(not POSTGRES_URI, reason='PIXELPROBE_TEST_POSTGRES_URI not set') +def test_invalid_legacy_exclusions_fail_without_rows_or_marker( + exclusion_database): + from pixelprobe.migrations.startup import run_v2_10_3_exclusion_migration + + app, migration_db, json_path = exclusion_database + json_path.write_text(json.dumps({ + 'paths': ['/media/should-not-import'], + 'extensions': 'not-a-list', + }), encoding='utf-8') + + with app.app_context(): + with pytest.raises(ValueError, match='extensions'): + run_v2_10_3_exclusion_migration(migration_db) + assert Exclusion.query.count() == 0 + assert AppConfig.query.filter_by( + key='legacy_exclusions_imported_v1').count() == 0 + + +@pytest.mark.postgres +@pytest.mark.skipif(not POSTGRES_URI, reason='PIXELPROBE_TEST_POSTGRES_URI not set') +def test_unreadable_legacy_exclusions_fail_without_rows_or_marker( + exclusion_database): + from pixelprobe.migrations.startup import run_v2_10_3_exclusion_migration + + app, migration_db, json_path = exclusion_database + json_path.mkdir() + + with app.app_context(): + with pytest.raises(OSError): + run_v2_10_3_exclusion_migration(migration_db) + assert Exclusion.query.count() == 0 + assert AppConfig.query.filter_by( + key='legacy_exclusions_imported_v1').count() == 0 diff --git a/tests/test_media_checker.py b/tests/test_media_checker.py index edcde66..17ab6a3 100644 --- a/tests/test_media_checker.py +++ b/tests/test_media_checker.py @@ -54,6 +54,31 @@ def side_effect(cmd, *args, **kwargs): class TestMediaChecker: """Test the core PixelProbe media checking functionality""" + + @pytest.mark.parametrize(('file_path', 'policy'), [ + ('/library/secret.private', {'excluded_patterns': ['*.private']}), + ('/library/tmp/movie.mkv', {'excluded_paths': ['/library/tmp']}), + ('/library/movie.tmp', {'excluded_extensions': ['.tmp']}), + ]) + def test_scan_file_exclusion_returns_before_open_or_cache( + self, monkeypatch, file_path, policy): + checker = PixelProbe(allowed_paths=['/library'], **policy) + monkeypatch.setattr('pixelprobe.media_checker.open_authorized_media_file', + lambda *_args, **_kwargs: pytest.fail('excluded file was opened')) + monkeypatch.setattr(checker, 'get_file_info', + lambda *_args, **_kwargs: pytest.fail('excluded file was statted')) + monkeypatch.setattr(checker, 'calculate_file_hash', + lambda *_args, **_kwargs: pytest.fail('excluded file was hashed')) + monkeypatch.setattr(checker, '_save_to_cache', + lambda *_args, **_kwargs: pytest.fail('excluded result was cached')) + + result = checker.scan_file(file_path, force_rescan=True) + + assert result == { + 'file_path': file_path, + 'outcome': 'excluded', + 'excluded': True, + } def test_corrupted_mp4_detection(self, test_data_dir): """Test detection of corrupted MP4 files""" diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 7c7e2eb..74d9c70 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -7,7 +7,7 @@ import pixelprobe.progress_utils as pu import pixelprobe.scheduler_lock as sl from pixelprobe.scheduler import MediaScheduler -from pixelprobe.models import db, ScanSchedule +from pixelprobe.models import db, Exclusion, ScanSchedule class TestSchedulerLock: @@ -130,6 +130,24 @@ def test_update_schedules_method_exists(self, scheduler): assert hasattr(scheduler, 'update_schedules') assert callable(getattr(scheduler, 'update_schedules')) + def test_periodic_scan_refreshes_exclusions_from_database( + self, scheduler, app, db, monkeypatch): + with app.app_context(): + db.session.add(Exclusion(exclusion_type='path', value='/library/blocked')) + db.session.commit() + monkeypatch.setattr( + 'pixelprobe.utils.helpers.get_configured_scan_paths', + lambda: ['/library/blocked', '/library/allowed']) + dispatched = [] + monkeypatch.setattr( + scheduler, '_execute_scan_request', + lambda endpoint, payload, *_args, **_kwargs: dispatched.append(payload) or object()) + monkeypatch.setattr(scheduler, '_handle_scan_response', lambda *_args: None) + + scheduler._run_periodic_scan() + + assert dispatched[0]['directories'] == ['/library/allowed'] + def test_shutdown_stops_lease_before_scheduler_executors(self): scheduler = MediaScheduler() scheduler.app = MagicMock() diff --git a/tests/unit/test_rolling_integrity_queue.py b/tests/unit/test_rolling_integrity_queue.py index 9eb2ad0..81f37d4 100644 --- a/tests/unit/test_rolling_integrity_queue.py +++ b/tests/unit/test_rolling_integrity_queue.py @@ -8,8 +8,11 @@ from datetime import datetime, timedelta -from pixelprobe.models import ScanResult -from pixelprobe.services.maintenance_service import fetch_cleanup_batch, fetch_integrity_batch +from pixelprobe.models import Exclusion, ScanResult +from pixelprobe.media_checker import load_exclusions_with_patterns +from pixelprobe.services.maintenance_service import ( + _apply_maintenance_scope, fetch_cleanup_batch, fetch_integrity_batch, +) # Naive UTC datetimes, matching what the DateTime column stores. NOW = datetime(2026, 7, 1, 12, 0, 0) @@ -32,6 +35,31 @@ def seed(db, path, checked=None, size=1000): class TestQueueOrdering: + def test_maintenance_uses_database_policy_but_keeps_exact_file_scope(self, db): + wanted = seed(db, '/media/a/kept.mkv', checked=None) + seed(db, '/media/a/excluded/file.mkv', checked=None) + seed(db, '/media/a/ignored.tmp', checked=None) + seed(db, '/media/a/._resource.mkv', checked=None) + db.session.add_all([ + Exclusion(exclusion_type='path', value='/media/a/excluded'), + Exclusion(exclusion_type='extension', value='.tmp'), + Exclusion(exclusion_type='filename_pattern', value='._*'), + ]) + db.session.commit() + + paths, extensions, patterns = load_exclusions_with_patterns() + batch = fetch_integrity_batch( + None, WATERMARK, set(), 10, scan_roots=['/media/a'], + excluded_paths=paths, excluded_extensions=extensions, + excluded_patterns=patterns) + assert [entry['id'] for entry in batch] == [wanted] + + exact = _apply_maintenance_scope( + ScanResult.query, file_paths=['/media/a/excluded/file.mkv'], + scan_roots=['/media/a'], excluded_paths=paths, + excluded_extensions=extensions, excluded_patterns=patterns).all() + assert [row.file_path for row in exact] == ['/media/a/excluded/file.mkv'] + def test_never_checked_files_come_first(self, db): checked = seed(db, '/m/checked.mkv', checked=NOW - timedelta(days=2)) never = seed(db, '/m/never.mkv', checked=None) diff --git a/tests/unit/test_scan_service.py b/tests/unit/test_scan_service.py index 0c408dc..2c1a925 100644 --- a/tests/unit/test_scan_service.py +++ b/tests/unit/test_scan_service.py @@ -367,6 +367,34 @@ def test_missing_selected_observation_does_not_copy_global_result(self, scan_ser assert result.scan_status == 'completed' assert result.file_hash == 'before' + def test_excluded_selected_member_does_not_copy_stale_result(self, scan_service, app, db): + with app.app_context(): + run_id = 'selected-excluded-observation' + file_path = '/library/excluded.png' + state = ScanState.create_new_scan(scan_id=run_id) + state.start_scan(['selected_files'], force_rescan=True) + state.phase = 'scanning' + result = ScanResult(file_path=file_path, scan_status='completed', + file_hash='before', is_corrupted=True) + db.session.add(result) + db.session.flush() + member = ScanRunFile(scan_id=run_id, scan_result_id=result.id, + file_path=file_path, status='processing') + db.session.add(member) + db.session.commit() + + assert scan_service._snapshot_run_member( + run_id, file_path, {'outcome': 'excluded', 'excluded': True}) + + db.session.refresh(member) + db.session.refresh(result) + assert member.status == 'skipped' + assert member.outcome == 'excluded' + assert member.file_hash is None + assert member.is_corrupted is None + assert result.scan_status == 'completed' + assert result.file_hash == 'before' + def test_cancellation_reclaims_only_owned_scanning_result(self, scan_service, app, db): with app.app_context(): state = ScanState.create_new_scan(scan_id='selected-owned-cancel') diff --git a/tests/unit/test_tasks_parallel.py b/tests/unit/test_tasks_parallel.py index 1e9701d..b289ad1 100644 --- a/tests/unit/test_tasks_parallel.py +++ b/tests/unit/test_tasks_parallel.py @@ -7,7 +7,8 @@ import pytest from pixelprobe.models import ( - ScanConfiguration, ScanResult, ScanState, ScanChunk, ScanRunFile, ScanRunRoot, ScanTask, + Exclusion, ScanConfiguration, ScanResult, ScanState, ScanChunk, ScanRunFile, + ScanRunRoot, ScanTask, ) from pixelprobe.services.scan_reporting import add_files_batch_to_db @@ -75,6 +76,46 @@ def _authorized_media_root(db, tmp_path, *names): class TestDiscoveryMembership: + def test_parallel_policy_uses_database_and_keeps_discovery_snapshot(self, tp, app, db, + tmp_path): + with app.app_context(): + blocked = str(tmp_path / 'blocked') + db.session.add_all([ + ScanConfiguration(path=str(tmp_path), is_active=True), + Exclusion(exclusion_type='path', value=blocked), + Exclusion(exclusion_type='extension', value='.MP4'), + Exclusion(exclusion_type='filename_pattern', value='*.Private.mp4'), + ]) + state = _make_scan_state(db, 'policy-snapshot', phase='discovering') + root = ScanRunRoot(scan_id=state.scan_id, root_path=str(tmp_path), + resolved_path=str(tmp_path), status='pending') + db.session.add(root) + db.session.commit() + + checker = tp._build_chunk_checker(state.scan_id) + assert checker._is_supported_file(f'{blocked}/movie.mkv') is False + assert checker._is_supported_file('/other/movie.mp4') is True + assert checker._is_supported_file('/other/movie.Private.mp4') is False + assert checker._is_supported_file('/other/movie.private.mp4') is True + + paths, extensions, patterns = tp.load_exclusions_with_patterns() + tasks = tp._dispatch_discovery_tasks( + state.scan_id, [str(tmp_path)], paths, extensions, patterns, + commit=True, publish=False) + assert tasks[0].payload['excluded_paths'] == [blocked] + assert tasks[0].payload['excluded_extensions'] == ['.MP4'] + assert '*.Private.mp4' in tasks[0].payload['excluded_patterns'] + assert checker.excluded_paths == [blocked] + + newly_excluded = str(tmp_path / 'newly-blocked') + db.session.add(Exclusion(exclusion_type='path', value=newly_excluded)) + db.session.commit() + assert checker._is_supported_file(f'{newly_excluded}/movie.mkv') is True + refreshed_checker = tp._build_chunk_checker(state.scan_id) + assert refreshed_checker._is_supported_file( + f'{newly_excluded}/movie.mkv') is False + assert tasks[0].payload['excluded_paths'] == [blocked] + def test_nonforce_discovery_members_only_new_or_pending_inventory(self, app, db, tmp_path): with app.app_context(): paths = [str(tmp_path / name) for name in ('new.mkv', 'pending.mkv', 'done.mkv',