From 12bc27e6c608b11f9557c47998607a7250bc0cf0 Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sat, 29 Aug 2026 05:49:26 +0800 Subject: [PATCH 01/15] feat: prewarm nearby MediaInfo --- CHANGELOG.en.md | 13 + CHANGELOG.md | 9 + README.en.md | 62 ++- README.md | 47 ++- app.env.example | 126 ++++++ cmd/plex-gateway/main.go | 89 +++- cmd/plex-gateway/main_test.go | 44 +- docker-compose.example.yml | 30 +- docs/architecture.md | 28 +- docs/media-lifecycle-architecture.md | 76 ++-- docs/roadmap.md | 15 +- internal/config/config.go | 52 ++- internal/config/config_test.go | 50 ++- internal/gateway/decision.go | 1 + internal/gateway/gateway.go | 90 +++- internal/gateway/gateway_test.go | 72 +++- internal/mediainfo/service.go | 287 ++++++++++--- internal/mediainfo/submit_test.go | 261 ++++++++++++ internal/metrics/metrics.go | 194 ++++++--- internal/metrics/metrics_test.go | 69 +++- internal/partcache/partcache.go | 6 + internal/partcache/partcache_test.go | 11 + internal/playback/service.go | 57 ++- internal/playback/service_test.go | 5 +- internal/plexmeta/episode.go | 320 +++++++++++++++ internal/plexmeta/episode_test.go | 103 +++++ internal/prewarm/discovery.go | 447 ++++++++++++++++++++ internal/prewarm/discovery_test.go | 230 +++++++++++ internal/prewarm/service.go | 593 +++++++++++++++++++++++++++ internal/prewarm/service_test.go | 509 +++++++++++++++++++++++ 30 files changed, 3633 insertions(+), 263 deletions(-) create mode 100644 app.env.example create mode 100644 internal/mediainfo/submit_test.go create mode 100644 internal/plexmeta/episode.go create mode 100644 internal/plexmeta/episode_test.go create mode 100644 internal/prewarm/discovery.go create mode 100644 internal/prewarm/discovery_test.go create mode 100644 internal/prewarm/service.go create mode 100644 internal/prewarm/service_test.go diff --git a/CHANGELOG.en.md b/CHANGELOG.en.md index fd0123e..1589aa3 100644 --- a/CHANGELOG.en.md +++ b/CHANGELOG.en.md @@ -9,6 +9,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Added bounded remote ffprobe, an L1 LRU plus SQLite MediaInfo cache, and + single-item Plex metadata response enrichment. Cold-wait timeouts preserve the + original response while probing continues in the background. +- Current-Part prewarming now starts after a cloud redirect is ready. An isolated + Plex management token can additionally discover and rate-limit a configurable + nearby window, defaulting to two previous and three following items. Missing or + invalid management credentials disable only nearby discovery. +- Added per-client rapid-switch coordination, cross-User-Agent failure handoff, + and distinct prewarm metrics for fresh cache hits, joined work, new queue + admissions, and rejections. + ## [0.1.1] - 2026-08-28 ### Added diff --git a/CHANGELOG.md b/CHANGELOG.md index 6223497..44280eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ ## [未发布] +### 新增 + +- 增加 MediaInfo L1 LRU 与 SQLite 持久缓存、受限远程 ffprobe 和单项 Plex metadata + 响应增强；冷缓存等待超时后保持原响应并在后台完成探测。 +- 云端重定向就绪后立即预热当前 Part，并可使用独立 Plex 管理 Token 按默认前 2、后 3 + 的窗口限速预热邻近媒体。Token 缺失或失效只禁用邻近发现。 +- 增加按客户端隔离的快速切换协调、跨 User-Agent 失败接力，以及缓存命中、加入现有 + 任务、新入队和拒绝的预热指标。 + ## [0.1.1] - 2026-08-28 ### 新增 diff --git a/README.en.md b/README.en.md index ad4a5f9..e921a78 100644 --- a/README.en.md +++ b/README.en.md @@ -83,6 +83,10 @@ The gateway remains a transparent proxy when `MEDIAVAULT_URL` and `PATH_MAPPINGS` are both unset. Configure them together to enable cloud redirect handling. +Before using `docker-compose.example.yml`, copy `app.env.example` to `app.env` +and set the deployment addresses, path mappings, and optional token. The +Compose example reads it through `env_file: ./app.env`. + ```sh PLEX_URL=http://plex:32400 \ MEDIAVAULT_URL=http://mediavault:7811 \ @@ -98,6 +102,7 @@ Important environment variables: | Variable | Default | Purpose | | --- | --- | --- | | `PLEX_URL` | required | Internal Plex origin; credentials are rejected. | +| `PLEX_TOKEN` | disabled | Optional Plex management token used only for startup validation and nearby-item discovery. Current-item prewarming remains available without it. | | `MEDIAVAULT_URL` | disabled | Internal MediaVault HTTP(S) origin. | | `PATH_MAPPINGS` | disabled | JSON array of Plex-to-container STRM prefixes. | | `LISTEN_ADDR` | `:32400` | Gateway listener. | @@ -119,9 +124,19 @@ Important environment variables: | `MEDIAINFO_COLD_WAIT` | `5s` | Cold-cache wait ceiling for one metadata item; timeout returns the original Plex response while probing continues. | | `MEDIAINFO_RESPONSE_MAX_BYTES` | `8388608` | Maximum size of one Plex metadata response buffered for enrichment. | | `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `4` | Maximum single-item metadata responses buffered and waiting for MediaInfo concurrently. | - -Plex tokens are not stored in configuration. Ordinary Plex requests remain -transparent. For cloud playback, all client request headers are forwarded to +| `MEDIAINFO_PREWARM_BEFORE` | `2` | Number of nearby items before the current item to prewarm. | +| `MEDIAINFO_PREWARM_AFTER` | `3` | Number of nearby items after the current item to prewarm. | +| `MEDIAINFO_PREWARM_INTERVAL` | `5s` | Delay between nearby-item submissions; the current item does not wait. | + +Ordinary Plex requests remain transparent, while cloud playback sends client +headers to the trusted MediaVault origin under the contract below. The optional +management `PLEX_TOKEN` is injected through the environment and sent only to the +configured Plex origin for nearby-item discovery. It is not written to the +database, logs, or tasks and is never sent to MediaVault, the CDN, or ffprobe. +Compose deployments can place +it in an ignored `app.env`; see [app.env.example](app.env.example). + +For cloud playback, all client request headers are forwarded to the configured MediaVault origin so it can generate a direct URL for the same client context. The internal lookup always uses `GET`, allowing a client `HEAD` request to receive the same 302 even when MediaVault does not support `HEAD`. @@ -129,10 +144,43 @@ Deploy MediaVault as a trusted upstream and keep resolver requests and headers out of logs. Clients connect to the gateway as they would connect to Plex and authenticate -through Plex. The gateway does not need a Plex username, password, or static -token. MediaVault's API key for `/api/v1` integrations is also not required for +through Plex. The gateway does not need a Plex username or password, and +playback does not depend on the management token. MediaVault's API key for `/api/v1` integrations is also not required for the STRM `/redirect` playback contract. +### Nearby-item MediaInfo prewarming + +The gateway performs one bounded in-memory enqueue only after Plex has +authorized the cloud Part, MediaVault has returned the final direct URL, and the +gateway has written the 302. This signal means that the cloud redirect is ready; +it does not claim that the client followed the redirect or actually started +playback. Current-item prewarming does not require a management token. A valid +`PLEX_TOKEN` additionally enables nearby-item discovery. + +The current item is submitted immediately at interactive priority. The +background coordinator prefers explicit Plex playQueue order and permits movie, +cross-show, and multi-Media/Part entries. Every candidate is stored under its +own PartID and STRM fingerprint. Without a usable queue, Plex episode and season +response order is used, including specials and entries with missing indices. + +The default window is two items before and three after, with following items +submitted first. The combined configurable window is capped at 50. Nearby items +enter the low-priority queue every five seconds by default, while the MediaInfo +worker defaults to one concurrent probe to avoid MediaVault/CDN bursts. A rapid +A/B/C switch submits each new current item immediately and cancels only the old +window entries that were not submitted. Submitted work retains its own identity +and is deduplicated by singleflight. + +The gateway persists parsed MediaInfo, not short-lived CDN URLs or raw head/tail +bytes. MediaVault upload precaching applies only to files uploaded through +MediaVault. A future stable MediaInfo or precache API can become the preferred +provider, with CDN ffprobe as fallback. `MEDIAINFO_NEGATIVE_TTL` suppresses +repeated failures; the coordinator does not perform blind retries without the +final probe error class. Plex discovery, STRM reads, fingerprinting, and SQLite +access all run after the 302 path. A missing or invalid token disables only +nearby-item discovery, not current-item prewarming, transparent proxying, +metadata enrichment, or playback redirects. + ### Metadata concurrency protection Some clients fan out many `GET/HEAD /library/metadata/{ratingKey}` requests @@ -235,8 +283,8 @@ That topology is outside the exclusive-gateway deployment contract. ## Endpoints -- `GET /health` returns process health plus a credential-free MediaInfo - availability, cache, and queue summary. +- `GET /health` returns process health plus credential-free MediaInfo cache, + probe-queue, and nearby-item prewarm summaries. - `GET /metrics` returns fixed-shape JSON counters plus resolver and complete redirect-path latency totals, samples, last values, and maxima. When metadata protection is enabled, it also reports admission, timeout, active, and queued diff --git a/README.md b/README.md index ac33f42..7942f7a 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,10 @@ MediaVault URL 输入会被重写为配置的内部 MediaVault origin。同源 当 `MEDIAVAULT_URL` 和 `PATH_MAPPINGS` 都未设置时，Gateway 仍作为透明代理运行。 要启用云端重定向处理，请同时配置这两个变量。 +使用 `docker-compose.example.yml` 前，先将 `app.env.example` 复制为 `app.env`， +再按部署环境填写地址、路径映射和可选 Token。Compose 示例会通过 +`env_file: ./app.env` 读取该文件。 + ```sh PLEX_URL=http://plex:32400 \ MEDIAVAULT_URL=http://mediavault:7811 \ @@ -86,6 +90,7 @@ go run ./cmd/plex-gateway | 变量 | 默认值 | 用途 | | --- | --- | --- | | `PLEX_URL` | 必填 | Plex 内部 origin；配置中的凭据会被拒绝。 | +| `PLEX_TOKEN` | 禁用 | 可选 Plex 管理 Token，仅用于启动验证和邻近媒体发现。缺失时仍预热当前播放项。 | | `MEDIAVAULT_URL` | 禁用 | MediaVault 内部 HTTP(S) origin。 | | `PATH_MAPPINGS` | 禁用 | Plex 到容器内 STRM 路径的 JSON 映射数组。 | | `LISTEN_ADDR` | `:32400` | Gateway 监听地址。 | @@ -107,17 +112,51 @@ go run ./cmd/plex-gateway | `MEDIAINFO_COLD_WAIT` | `5s` | 单项 metadata 冷缓存等待上限；超时返回原 Plex 响应，探测继续。 | | `MEDIAINFO_RESPONSE_MAX_BYTES` | `8388608` | 可以缓冲并尝试增强的单个 Plex metadata 响应上限。 | | `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `4` | 同时缓冲和等待 MediaInfo 的单项 metadata 响应上限。 | - -Plex Token 不会保存到配置中。普通 Plex 请求保持透明转发。对于云端播放，客户端的 +| `MEDIAINFO_PREWARM_BEFORE` | `2` | 当前项之前的邻近媒体预热数量。 | +| `MEDIAINFO_PREWARM_AFTER` | `3` | 当前项之后的邻近媒体预热数量。 | +| `MEDIAINFO_PREWARM_INTERVAL` | `5s` | 邻近项提交到后台探测队列的间隔；当前项不等待。 | + +普通 Plex 请求保持透明转发，云端播放请求按下文契约将客户端 header 发送给可信 +MediaVault。可选管理 `PLEX_TOKEN` 通过环境变量注入，只发送到配置的 Plex origin， +用于邻近媒体发现；不会写入数据库、日志或任务，也不会发送给 MediaVault、CDN 或 +ffprobe。Compose 部署可将它放在不纳入 +Git 的 `app.env` 中。推荐让 Compose 只通过 `env_file: ./app.env` 加载运行配置，完整的 +中文变量说明和默认值参考 [app.env.example](app.env.example)。 + +对于云端播放，客户端的 所有请求 header 都会转发到配置的 MediaVault origin，使其能够为相同客户端上下文 生成直链。内部查找始终使用 `GET`，因此即使 MediaVault 不支持 `HEAD`，客户端的 `HEAD` 请求也能收到相同的 302。请将 MediaVault 部署为可信上游，并避免将解析请求 及其 header 写入日志。 客户端像连接 Plex 一样连接 Gateway，并通过 Plex 完成身份认证。Gateway 不需要 -单独配置 Plex 用户名、密码或静态 Token。STRM `/redirect` 播放协议也不需要 +单独配置 Plex 用户名或密码，播放也不依赖管理 Token。STRM `/redirect` 播放协议不需要 MediaVault 用于 `/api/v1` 集成的 API key。 +### 邻近媒体 MediaInfo 预热 + +Gateway 在云端 Part 已通过 Plex 授权、MediaVault 已返回最终直链且 Gateway 已写出 +302 后，只向有界内存协调器投递一次事件。该信号表示“云端重定向已就绪”，不宣称 +客户端已经跟随 302 或真实起播。当前项预热不需要管理 Token；配置有效 +`PLEX_TOKEN` 后才会额外发现邻近媒体。 + +当前项会立即以交互优先级提交。后台优先使用明确的 Plex playQueue 顺序，并允许其中的 +电影、跨剧条目和多 Media/Part；候选始终采用自身的 PartID 与 STRM fingerprint 落库， +不会写入当前项。没有可靠 playQueue 时，按 Plex 返回的剧集和季顺序查找邻近项，包括 +S00 和缺失索引的条目。 + +默认窗口为前 `2`、后 `3`，后续项先提交；两个方向合计最多配置 `50`。邻近项默认每隔 +`5s` 进入低优先级队列，MediaInfo worker 默认并发为 `1`，用于避免 MediaVault/CDN +请求突发。快速切换 A、B、C 时，新当前项立即进入交互队列，旧窗口中尚未提交的候选会被 +取消；已经提交的任务仍按自己的身份完成并由 singleflight 去重。整个发现、STRM 读取、 +指纹计算和 SQLite 查询均在 302 响应之后执行。 + +Gateway 只持久化解析后的 MediaInfo，不缓存短期 CDN URL 或原始文件头尾。MediaVault 的 +上传预缓存只覆盖经其上传的新文件；若未来提供稳定 MediaInfo/预缓存读取 API，可作为优先 +Provider，未命中时再回退 CDN ffprobe。探测失败由 `MEDIAINFO_NEGATIVE_TTL` 抑制重复请求； +协调器不执行无法感知最终错误类型的盲目重试。Token 缺失或失效只禁用邻近媒体发现， +不影响当前项预热、透明代理、metadata 增强或播放重定向。 + ### Metadata 并发保护 部分客户端会在打开大型媒体库时并发请求大量 @@ -202,7 +241,7 @@ Gateway 重新执行检查。 ## Endpoints -- `GET /health` 返回进程健康状态，以及不含媒体身份和凭据的 MediaInfo 可用性、缓存和队列摘要。 +- `GET /health` 返回进程健康状态，以及不含媒体身份和凭据的 MediaInfo 缓存、探测队列和邻近媒体预热摘要。 - `GET /metrics` 返回固定结构的 JSON 计数器，以及 resolver 和完整重定向链路的 延迟总计、样本数、最近值和最大值。启用 metadata 保护时还会返回准入、超时、 活动和排队计数，所有指标都不包含请求标签或凭据。 diff --git a/app.env.example b/app.env.example new file mode 100644 index 0000000..4c41281 --- /dev/null +++ b/app.env.example @@ -0,0 +1,126 @@ +# Plex 内部地址，必填。必须指向 Gateway 可直连的 Plex origin，不能填写 Gateway 自己。 +PLEX_URL=http://plex:32400 + +# Plex 管理 Token。仅用于启动校验和邻近媒体发现；留空仍会预热当前播放项。 +# 不会写入日志、数据库或转发给 MediaVault/CDN。 +PLEX_TOKEN= + +# MediaVault 内部地址。启用 STRM 302 时必须与 PATH_MAPPINGS 同时配置。 +MEDIAVAULT_URL=http://mediavault:7811 + +# Plex 媒体路径到 Gateway 容器内 STRM 路径的 JSON 映射。多个映射写在同一行。 +PATH_MAPPINGS=[{"plex_prefix":"/media/cloud","local_prefix":"/strm"},{"plex_prefix":"/media/archive","local_prefix":"/archive-strm"}] + +# Gateway 监听地址。容器内通常保持 :32400。 +LISTEN_ADDR=:32400 + +# Gateway SQLite 数据库路径。容器部署应放在持久化的 /app_data 中。 +DATABASE_PATH=/app_data/plex-gateway.db + +# 日志级别，可选 debug、info、warn、error。 +LOG_LEVEL=info + +# 是否记录脱敏后的 Plex 请求顺序。排障时临时开启，生产默认关闭。 +TRACE_ENABLED=false + +# 识别为云端控制文件的扩展名，多个值以逗号分隔。 +CLOUD_EXTENSIONS=.strm + +# Part 内存缓存有效期。 +PART_CACHE_TTL=24h + +# MediaVault 直链解析的端到端超时。 +RESOLVER_TIMEOUT=15s + +# 观察 Plex metadata 响应时允许缓冲的最大字节数。 +OBSERVE_MAX_BYTES=8388608 + +# 无响应体 Part 授权请求和 Plex 管理请求的超时。 +PART_PROBE_TIMEOUT=15s + +# 是否限制单项详细 metadata 请求进入 Plex 的并发量。 +METADATA_GUARD_ENABLED=false + +# 所有客户端共享的单项详细 metadata 并发上限。 +METADATA_GUARD_GLOBAL_CONCURRENCY=8 + +# 单个 Plex 客户端的单项详细 metadata 并发上限，不能大于全局上限。 +METADATA_GUARD_CLIENT_CONCURRENCY=4 + +# 是否为逗号分隔的批量 metadata 请求启用独立并发保护。 +METADATA_GUARD_BATCH_ENABLED=false + +# 批量 metadata 请求的全局并发上限。 +METADATA_GUARD_BATCH_CONCURRENCY=3 + +# metadata 并发槽位的最长等待时间，超时返回 429。 +METADATA_GUARD_QUEUE_TIMEOUT=10s + +# 是否启用 MediaInfo 缓存、远程探测和 Plex metadata 增强。 +MEDIAINFO_ENABLED=true + +# ffprobe 可执行文件路径。官方镜像中可直接使用 ffprobe。 +MEDIAINFO_FFPROBE_PATH=ffprobe + +# 单次 MediaInfo 远程探测的总超时。 +MEDIAINFO_PROBE_TIMEOUT=20s + +# ffprobe 的分析窗口参数，不是 HTTP 总下载量硬上限；应结合超时、并发和实际流量调优。 +MEDIAINFO_PROBE_SIZE=8388608 + +# ffprobe 分析时长上限。 +MEDIAINFO_ANALYZE_DURATION=5s + +# ffprobe JSON 输出允许读取的最大字节数。 +MEDIAINFO_OUTPUT_MAX_BYTES=2097152 + +# MediaInfo worker 并发数。CDN 风控敏感环境建议保持 1。 +MEDIAINFO_CONCURRENCY=1 + +# 交互式冷缓存任务的内存队列上限。 +MEDIAINFO_INTERACTIVE_QUEUE_SIZE=256 + +# 邻近项预热等后台任务的内存队列上限。 +MEDIAINFO_BACKGROUND_QUEUE_SIZE=256 + +# 已知 MediaInfo 结果的新鲜期。过期后仍可返回旧值并在后台刷新。 +MEDIAINFO_RECORD_TTL=720h + +# 长期未访问记录的保留期，不能短于新鲜期。 +MEDIAINFO_RECORD_RETENTION=4320h + +# L1 内存缓存最多保存的 MediaInfo 记录数。 +MEDIAINFO_L1_MAX_ENTRIES=10000 + +# 同一 Part、STRM fingerprint 和 User-Agent 探测失败后的抑制时间。 +MEDIAINFO_NEGATIVE_TTL=15m + +# 没有活动客户端上下文时，后台探测使用的 fallback User-Agent。 +MEDIAINFO_USER_AGENT=Infuse-Library/8.4.4 + +# 单项 metadata 冷缓存最多等待多久。超时后原响应立即返回，探测继续在后台执行。 +MEDIAINFO_COLD_WAIT=5s + +# 单个 Plex metadata 响应允许缓冲并尝试增强的最大字节数。 +MEDIAINFO_RESPONSE_MAX_BYTES=8388608 + +# 同时缓冲并等待 MediaInfo 的单项 metadata 响应数量。 +MEDIAINFO_ENRICHMENT_CONCURRENCY=4 + +# 当前播放项之前预热的邻近媒体数量。前后都设为 0 时仅立即预热当前项。 +MEDIAINFO_PREWARM_BEFORE=2 + +# 当前播放项之后预热的邻近媒体数量。前后合计不能超过 50。 +MEDIAINFO_PREWARM_AFTER=3 + +# 邻近项进入后台探测队列的间隔。当前播放项不等待该间隔并立即优先提交。 +MEDIAINFO_PREWARM_INTERVAL=5s + +# HTTP 请求头读取超时。 +READ_HEADER_TIMEOUT=15s + +# HTTP keep-alive 空闲超时。 +IDLE_TIMEOUT=90s + +# 进程停止时等待 HTTP、预热和 MediaInfo worker 收尾的最长时间。 +SHUTDOWN_TIMEOUT=15s diff --git a/cmd/plex-gateway/main.go b/cmd/plex-gateway/main.go index d57e6ad..f0dfff4 100644 --- a/cmd/plex-gateway/main.go +++ b/cmd/plex-gateway/main.go @@ -21,6 +21,8 @@ import ( "github.com/InfinityPacer/plex-gateway/internal/metrics" "github.com/InfinityPacer/plex-gateway/internal/partcache" "github.com/InfinityPacer/plex-gateway/internal/pathmap" + "github.com/InfinityPacer/plex-gateway/internal/playback" + "github.com/InfinityPacer/plex-gateway/internal/prewarm" "github.com/InfinityPacer/plex-gateway/internal/resolver" "github.com/InfinityPacer/plex-gateway/internal/trace" "github.com/InfinityPacer/plex-gateway/internal/version" @@ -73,10 +75,19 @@ func run() error { mediaInfoService, mediaInfoReason := initializeMediaInfo( context.Background(), databaseStore, databaseReason, cfg.MediaInfo, cfg.PlexURL, cfg.PartProbeTimeout, strmResolver, registry, logger, ) - defer closeRuntime(mediaInfoService, databaseStore, cfg.ShutdownTimeout, logger) + prewarmService, prewarmReason := initializePrewarm( + context.Background(), cfg.PlexToken, cfg.PlexURL, cfg.PartProbeTimeout, + playback.NewPartPreparer(mapper, strmResolver, cfg.CloudExtensions), + mediaInfoService, cfg.MediaInfo.PrewarmBefore, cfg.MediaInfo.PrewarmAfter, + cfg.MediaInfo.PrewarmInterval, registry, logger, + ) + defer closeRuntime(prewarmService, mediaInfoService, databaseStore, cfg.ShutdownTimeout, logger) mediaInfoStatus := func() mediainfo.Status { return mediaInfoService.Status() } + prewarmStatus := func() prewarm.Status { + return prewarmService.Status() + } handler := gateway.New(gateway.Options{ Upstream: cfg.PlexURL, Logger: logger, @@ -102,6 +113,8 @@ func run() error { MediaInfoColdWait: cfg.MediaInfo.ColdWait, MediaInfoResponseMaxBytes: cfg.MediaInfo.ResponseMaxBytes, MediaInfoEnrichmentConcurrency: cfg.MediaInfo.EnrichmentWaiters, + MediaInfoPrewarmer: prewarmService, + MediaInfoPrewarmStatus: prewarmStatus, }) server := &http.Server{ Addr: cfg.ListenAddr, @@ -126,6 +139,11 @@ func run() error { "mediainfo_enabled", cfg.MediaInfo.Enabled, "mediainfo_available", mediaInfoService != nil, "mediainfo_reason", mediaInfoReason, + "mediainfo_prewarm_available", prewarmService != nil, + "mediainfo_prewarm_reason", prewarmReason, + "mediainfo_prewarm_before", cfg.MediaInfo.PrewarmBefore, + "mediainfo_prewarm_after", cfg.MediaInfo.PrewarmAfter, + "mediainfo_prewarm_interval_ms", cfg.MediaInfo.PrewarmInterval.Milliseconds(), ) serveErrors <- server.ListenAndServe() }() @@ -149,6 +167,66 @@ func run() error { return nil } +func initializePrewarm( + ctx context.Context, + token string, + plexURL *url.URL, + timeout time.Duration, + partPreparer *playback.PartPreparer, + mediaInfoService *mediainfo.Service, + beforeCount int, + afterCount int, + submitInterval time.Duration, + registry *metrics.Metrics, + logger *slog.Logger, +) (*prewarm.Service, string) { + if mediaInfoService == nil || partPreparer == nil { + return nil, "mediainfo_unavailable" + } + var discovery *prewarm.Discovery + reason := "ready" + if beforeCount+afterCount == 0 { + reason = "current_only_window_disabled" + } else if strings.TrimSpace(token) == "" { + reason = "current_only_token_unconfigured" + } else { + client := &http.Client{Timeout: timeout} + candidate, err := prewarm.NewDiscovery(prewarm.DiscoveryOptions{ + BaseURL: plexURL, Token: token, Client: client, + UserAgent: "plex-gateway/" + version.String(), + }) + if err != nil { + reason = "current_only_configuration_invalid" + } else { + validateContext, cancel := context.WithTimeout(ctx, timeout) + err = candidate.Validate(validateContext) + cancel() + if err != nil { + reason = "current_only_plex_token_invalid" + } else { + discovery = candidate + } + } + if discovery == nil { + logger.Warn("mediainfo_neighbor_prewarm_unavailable", "reason", reason) + } + } + serviceOptions := prewarm.ServiceOptions{ + Playback: partPreparer, MediaInfo: mediaInfoService, + Logger: logger, Metrics: registry, DiscoveryTimeout: timeout, + BeforeCount: beforeCount, AfterCount: afterCount, SubmitInterval: submitInterval, + } + if discovery != nil { + serviceOptions.Discovery = discovery + } + service, err := prewarm.NewService(serviceOptions) + if err != nil { + logger.Warn("mediainfo_prewarm_unavailable", "reason", "coordinator") + return nil, "coordinator_unavailable" + } + return service, reason +} + func initializeMediaInfo( ctx context.Context, databaseStore *database.SQLite, @@ -229,7 +307,14 @@ func initializeDatabase(ctx context.Context, path string, required bool, logger return store, "ready" } -func closeRuntime(service *mediainfo.Service, store *database.SQLite, timeout time.Duration, logger *slog.Logger) { +func closeRuntime(prewarmService *prewarm.Service, service *mediainfo.Service, store *database.SQLite, timeout time.Duration, logger *slog.Logger) { + if prewarmService != nil { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + if err := prewarmService.Close(ctx); err != nil { + logger.Warn("mediainfo_prewarm_shutdown_incomplete", "error_kind", "timeout") + } + cancel() + } serviceClosed := true if service != nil { ctx, cancel := context.WithTimeout(context.Background(), timeout) diff --git a/cmd/plex-gateway/main_test.go b/cmd/plex-gateway/main_test.go index 2b5ba3a..3936ecf 100644 --- a/cmd/plex-gateway/main_test.go +++ b/cmd/plex-gateway/main_test.go @@ -16,6 +16,8 @@ import ( "github.com/InfinityPacer/plex-gateway/internal/config" "github.com/InfinityPacer/plex-gateway/internal/metrics" + "github.com/InfinityPacer/plex-gateway/internal/pathmap" + "github.com/InfinityPacer/plex-gateway/internal/playback" "github.com/InfinityPacer/plex-gateway/internal/resolver" ) @@ -36,15 +38,23 @@ func (control *countingControlResolver) ResolveTarget(context.Context, string, r func TestInitializeMediaInfoDoesNotProbeWithoutTask(t *testing.T) { unexpectedPlexPath := make(chan string, 1) plex := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/identity" { + switch r.URL.Path { + case "/identity": + _, _ = io.WriteString(w, ``) + case "/library/sections": + if r.Header.Get("X-Plex-Token") != "management-token" { + http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"MediaContainer":{"Metadata":[]}}`) + default: select { case unexpectedPlexPath <- r.URL.Path: default: } http.NotFound(w, r) - return } - _, _ = io.WriteString(w, ``) })) defer plex.Close() plexURL, err := url.Parse(plex.URL) @@ -94,5 +104,31 @@ func TestInitializeMediaInfoDoesNotProbeWithoutTask(t *testing.T) { if _, err := os.Stat(ffprobePath + ".called"); !errors.Is(err, os.ErrNotExist) { t.Fatalf("ffprobe process unexpectedly started: %v", err) } - closeRuntime(service, databaseStore, time.Second, logger) + mapper, err := pathmap.New([]pathmap.Mapping{{PlexPrefix: "/media/cloud", LocalPrefix: directory}}) + if err != nil { + t.Fatal(err) + } + partPreparer := playback.NewPartPreparer(mapper, control, []string{".strm"}) + invalidTokenService, reason := initializePrewarm( + t.Context(), "invalid-token", plexURL, time.Second, partPreparer, service, + 2, 3, time.Millisecond, metrics.New(), logger, + ) + if invalidTokenService == nil || reason != "current_only_plex_token_invalid" || + !invalidTokenService.Status().Available || invalidTokenService.Status().NeighborAvailable { + t.Fatalf("invalid token prewarm service=%v reason=%q status=%#v", invalidTokenService != nil, reason, invalidTokenService.Status()) + } + closeContext, closeCancel := context.WithTimeout(t.Context(), time.Second) + if err := invalidTokenService.Close(closeContext); err != nil { + closeCancel() + t.Fatal(err) + } + closeCancel() + prewarmService, reason := initializePrewarm( + t.Context(), "management-token", plexURL, time.Second, partPreparer, service, + 2, 3, time.Millisecond, metrics.New(), logger, + ) + if prewarmService == nil || reason != "ready" || !prewarmService.Status().Available { + t.Fatalf("prewarm service=%v reason=%q", prewarmService != nil, reason) + } + closeRuntime(prewarmService, service, databaseStore, time.Second, logger) } diff --git a/docker-compose.example.yml b/docker-compose.example.yml index 6df49dd..621888d 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -6,34 +6,8 @@ services: restart: unless-stopped ports: - "32400:32400" - environment: - PLEX_URL: "http://plex:32400" - MEDIAVAULT_URL: "http://mediavault:7811" - PATH_MAPPINGS: >- - [{"plex_prefix":"/media/cloud","local_prefix":"/strm"},{"plex_prefix":"/media/archive","local_prefix":"/archive-strm"}] - LISTEN_ADDR: ":32400" - DATABASE_PATH: "/app_data/plex-gateway.db" - LOG_LEVEL: "info" - TRACE_ENABLED: "false" - PART_CACHE_TTL: "24h" - RESOLVER_TIMEOUT: "15s" - OBSERVE_MAX_BYTES: "8388608" - PART_PROBE_TIMEOUT: "15s" - METADATA_GUARD_ENABLED: "false" - METADATA_GUARD_GLOBAL_CONCURRENCY: "8" - METADATA_GUARD_CLIENT_CONCURRENCY: "4" - METADATA_GUARD_BATCH_ENABLED: "false" - METADATA_GUARD_BATCH_CONCURRENCY: "3" - METADATA_GUARD_QUEUE_TIMEOUT: "10s" - MEDIAINFO_ENABLED: "true" - # Background probes use this only when no active client request supplies a User-Agent. - MEDIAINFO_USER_AGENT: "Infuse-Library/8.4.4" - MEDIAINFO_COLD_WAIT: "5s" - MEDIAINFO_RESPONSE_MAX_BYTES: "8388608" - MEDIAINFO_ENRICHMENT_CONCURRENCY: "4" - READ_HEADER_TIMEOUT: "15s" - IDLE_TIMEOUT: "90s" - SHUTDOWN_TIMEOUT: "15s" + env_file: + - ./app.env volumes: - ./strm:/strm:ro - ./archive-strm:/archive-strm:ro diff --git a/docs/architecture.md b/docs/architecture.md index 7c10033..d16c8be 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -166,10 +166,12 @@ proxied and retains Plex's normal Web compatibility. ## Analysis-plane boundary -MediaVault's documented ffprobe assistance is an internal recognition and -renaming feature, and MediaInfoKeeper is Emby-specific. Neither is treated as a -public MediaInfo API. The gateway does not inspect MediaVault internals or -duplicate those features in the playback plane. +MediaVault's documented ffprobe assistance and upload-time head/tail precache +are internal recognition and scan optimizations, and MediaInfoKeeper is +Emby-specific. None is treated as a public MediaInfo API. The precache covers +only files uploaded through MediaVault and cannot be consumed by the gateway +without a stable interface. The gateway does not inspect MediaVault internals, +mount its private cache, or duplicate those features in the playback plane. MediaInfo Phase D begins with a provider boundary, L1 and SQLite persistence, bounded remote probing, proactive prewarming, and metadata response enrichment. @@ -184,9 +186,9 @@ Intro and credits work starts with an isolated Plex marker API probe. See The Part cache is in memory and has a configurable TTL. Metadata responses refresh entries by stable `Part.id`; the full `Part.key` is retained only as observed metadata because its change stamp may vary. A restart begins with an -empty cache; normal metadata, decision, start, and play queue traffic repopulate -it. A direct cold-start Part request falls back to Plex until one of those -authenticated metadata paths observes the Part. +empty cache; normal metadata, decision, and play queue traffic repopulate it. A +direct cold-start Part request falls back to Plex until one of those authenticated +metadata paths observes the Part. Direct URLs are intentionally not persisted. They are short-lived credentials owned by MediaVault and the cloud provider. @@ -198,9 +200,9 @@ The gateway requires: - network access to Plex and MediaVault; - read-only mounts for each configured STRM tree; - no Plex database, MediaVault database, 115 cookie, or cloud-provider API - access. Playback requires no static Plex Token; MediaInfo discovery and - prewarming may optionally use `PLEX_TOKEN` injected from an ignored - deployment `app.env`. + access. Playback and current-item MediaInfo prewarming require no static Plex + Token; nearby-item discovery may optionally use `PLEX_TOKEN` injected from an + ignored deployment `app.env`. Before listening, the process verifies that a fresh invalid Plex Token is rejected by the internal Plex origin. Plex **Allowed Networks** must not include @@ -211,9 +213,9 @@ playback failures after startup remain fail-open to authenticated Plex traffic. The MediaVault API key documented for `/api/v1` integrations is not required by the STRM `/redirect` contract and is not a gateway configuration value. Active playback authorization comes only from the client request; the optional -management Token is used only for background discovery, prewarming, and -reconciliation. Users do not provide their Plex username or password to the -gateway. +management Token is used only to validate and discover nearby Plex items. +Current-item prewarming does not require it. Users do not provide their Plex +username or password to the gateway. Client-provided Plex authentication headers and query parameters are forwarded unchanged to Plex. Cloud playback headers are additionally forwarded to diff --git a/docs/media-lifecycle-architecture.md b/docs/media-lifecycle-architecture.md index 28477bf..679154a 100644 --- a/docs/media-lifecycle-architecture.md +++ b/docs/media-lifecycle-architecture.md @@ -411,12 +411,14 @@ MoviePilot 本地下载路径应在真实文件仍在本地时执行 ffprobe。 3. 用户可控的受限远程 ffprobe provider； 4. 标记缺失，保留 Gateway 保护，不反复无界探测。 -远程探测必须限制 Range、probe size、timeout、并发、重试和负缓存。分享挂载失效时应 -标记 source unavailable，不应持续重试拖垮 Plex 或 115。 +远程探测必须限制 ffprobe 的 `probesize`、`analyzeduration`、timeout、并发和负缓存， +并监测实际网络读取量。`probesize` 是分析窗口，不是 HTTP 总下载量硬上限；Gateway 不能 +仅凭该参数承诺固定 Range 字节数。分享挂载失效时应标记 source unavailable，不应持续 +重试拖垮 Plex 或 115。 ### 10.2 记录契约、存储与投影 -`MediaInfoRecord` 必须是版本化公开契约，至少包含： +统一的 `MediaInfoRecord` 目标契约必须版本化，至少包含： - `schema_version`、记录身份、provider revision、上游关联和内容 fingerprint； - container、duration、size； @@ -424,6 +426,9 @@ MoviePilot 本地下载路径应在真实文件仍在本地时执行 ffprobe。 - provider、`probed_at`、`ready|stale|negative` 状态； - 负缓存期限和生成该记录时的 backing fingerprint。 +当前 Gateway SQLite 只持久化成功记录，负缓存状态及期限保存在有界内存中；因此当前表 +结构不是上述跨生产者公开契约的完整实现。 + 发布方式优先使用原子 sidecar 或稳定只读 API。消费者不得直接读取 MoviePilot 插件表。 内容或 STRM fingerprint 变化时记录立即失效；provider schema 不兼容时消费者必须拒绝 使用而不是猜测字段。 @@ -461,7 +466,7 @@ Gateway fallback 是当前优先实现的兜底能力，不要求媒体先经过 MediaVault 尚未公开的 MediaInfo API： ```text -Plex Part / configured STRM root +Plex Part / cloud redirect │ ▼ Gateway discovery and prewarm queue @@ -481,7 +486,7 @@ singleflight 或短租约时加入；PostgreSQL 仅在 SQLite 单写者成为实 探测任务使用稳定身份和内容指纹，不使用短期 CDN URL 作为缓存键。至少关联 Plex server identity、ratingKey、PartID、STRM fingerprint、可用时的 provider/content fingerprint、 MediaInfo schema 版本和探测实现版本。探测失败不得覆盖已知良好记录；相同指纹使用 -singleflight、负缓存与退避。 +singleflight 和负缓存抑制重复失败。当前不执行无法识别最终错误类型的自动重试或退避。 Gateway fallback 默认将成功记录视为 `30d` fresh，并将最近使用记录保留 `180d`。 fresh 到期后可以立即返回仍在 retention 内的已知良好记录，同时在后台低优先级复验； @@ -493,17 +498,18 @@ Gateway 支持以下范围： | 范围 | 触发方式 | 约束 | | --- | --- | --- | | 单 Part | metadata miss/stale 或成功播放 | 自动、高优先级、去重 | -| 下一集 | 当前集确认播放后的低优先预取 | 当前实现范围；必须精确识别 Part，只向前一步 | +| 邻近媒体 | 当前项云端重定向就绪后的低优先预取 | 当前实现范围；默认前 2 后 3，可配置双向窗口与间隔 | | 单季 | 显式预热或计划任务 | 后续批量能力；分页、checkpoint | | 整剧 | 显式后台任务 | 后续批量能力；按季分页，不由一次播放自动扩散 | | STRM 目录 | 管理员扫描或 reconciliation | 后续批量能力；限制到配置 mapping root，可取消和恢复 | -当前实现不执行启动全库、单季、整剧或目录批量预热。精确当前项始终高于下一集预取； -下一集只能在当前集确认播放后进入低优先队列，快速切换产生的新当前项可以提升同一 -singleflight 或抢占尚未完成的纯推测任务。远程 probe 初始并发候选值为 `1`，经 CDN、 -CPU、Plex 和 MediaVault 压测后再决定是否提高。每次播放可以触发 freshness 判断，但 -不等于每次执行完整 ffprobe；未过期记录只更新访问时间，过期记录优先异步校验，可靠 -指纹变化才重新探测。 +当前实现不执行启动全库、单季、整剧或目录批量预热。当前项在 302 写出后立即以交互 +优先级提交；邻近项默认按后 3、前 2 的顺序，每隔 `5s` 进入低优先级队列。该信号只表示 +云端重定向就绪，不代表客户端已跟随 302 或真正起播。快速切换产生的新当前项优先于旧 +窗口中尚未提交的候选；已经提交的任务保留自身 PartID 与 STRM fingerprint，并由 +singleflight 去重。远程 probe 初始并发候选值为 `1`，经 CDN、CPU、Plex 和 MediaVault +压测后再决定是否提高。每次播放可以触发 freshness 判断，但不等于每次执行完整 ffprobe； +未过期记录只更新访问时间，过期记录优先异步校验，可靠指纹变化才重新探测。 一次真实 MediaVault/115 环境的匿名初始样本中，六个文件的 `/redirect` 耗时为 `0.018–0.477s`，受限 ffprobe 耗时为 `0.33–0.46s`，组合约为 `0.46–0.89s`。解析直链和 @@ -514,12 +520,18 @@ MediaVault redirect 与 ffprobe 必须在同一次任务中使用相同 User-Age 失败源。 首次冷缓存长期返回空 MediaInfo 不可作为正常体验，metadata 请求一直无响应同样不可 -接受。处理顺序为：Gateway 启动时从 SQLite 恢复 L1；精确冷 miss 时创建最高优先级 +接受。处理顺序为：Gateway 启动时按需从 SQLite 回填 L1；精确冷 miss 时创建最高优先级 任务，并允许当前单项 metadata response 在可配置的硬截止时间内等待。当前 PoC 候选 上限为 `5s`；窗口内完成就补充当前 response，超时或失败必须立即返回原始 Plex -response，任务继续在后台完成。当前集确认播放后，管理面只发现并低优先预热下一集， -不扩散为季、整剧或目录扫描。提高首次完整率应优先使用准确的一步预取，而不是无限增加 -请求等待或启动无界批量任务。 +response，任务继续在后台完成。当前项云端重定向已就绪后，管理面只发现并限速预热 +可配置邻近窗口，不扩散为季、整剧或目录扫描。提高首次完整率应优先使用有界窗口，而不是 +无限增加请求等待或启动无界批量任务。 + +Gateway 不缓存原始文件头尾。MediaVault 的上传预缓存利用上传时仍在本地的数据，适合 +减少其后续扫描对 115 的请求，但只覆盖经 MediaVault 上传的新文件；Gateway 若从 CDN +下载相同头尾再缓存，并不能降低首次探测风险。Gateway 只持久化解析后的 MediaInfo。 +未来若 MediaVault 提供稳定的 MediaInfo 或预缓存读取 API，可作为优先 Provider，未命中 +时再回退 CDN ffprobe。 ## 11. Plex MediaInfo 投影 @@ -597,25 +609,27 @@ Gateway 不应负责： - 在播放 decision、Part、universal start 或 302 请求中执行 ffprobe 或数据库访问； - 写 Plex 数据库。 -Gateway analysis worker 当前只执行精确单 Part 和一步下一集探测。单季、整剧和 STRM +Gateway analysis worker 当前只执行精确当前 Part 和可配置邻近窗口探测。单季、整剧和 STRM 目录探测保留为后续有界批量能力；这些任务不拥有下载、做种或清理决策，也不能由一次 播放无界扩散到整库。 ### 12.1 Plex 管理凭据与管理面 -Gateway 保存一个可选 `PLEX_TOKEN`，用于后台枚举、Part 发现、预热和 reconciliation。 +Gateway 保存一个可选 `PLEX_TOKEN`，当前用于启动校验、Part 发现和邻近媒体预热。 它与活动请求中的客户端 Plex Token 分离，不替代当前用户的播放授权，也不复制到任务 记录。Go 程序只读取环境变量；Docker Compose 部署使用不纳入 Git 的 `app.env`，通过 `env_file: ./app.env` 注入。`.env` 保留给 Compose 默认插值语义。 -Token 缺失或失效时，透明代理和客户端播放继续工作，只有后台发现与预热降级。启动时 -验证 Token；稳定 Plex server identity 从无需管理 Token 的 PMS `/identity` 获取。日志、 +Token 缺失或失效时，透明代理、客户端播放和当前项预热继续工作，只有邻近媒体发现降级。 +启动时验证 Token；稳定 Plex server identity 从无需管理 Token 的 PMS `/identity` 获取。日志、 metrics、任务、CLI 和管理响应都不得输出 Token 或 machine identifier。管理 Token 只允许 -发送到配置的 Plex origin；MediaVault、CDN 和 ffprobe 请求使用独立构造的干净 header， -禁止继承 `X-Plex-Token`、Cookie 或 Authorization。 +发送到配置的 Plex origin，不能注入客户端播放请求或 MediaInfo 任务。播放链路继续按既有 +契约向可信 MediaVault 透传客户端 header；CDN ffprobe 只复用探测任务选定的客户端或 +fallback User-Agent，不继承管理 Token。 -Gateway 当前没有后台页面。当前管理面只需要查询状态、验证 Token 和控制一步下一集 -预热；创建单季、整剧和目录任务、批任务 checkpoint 与恢复留到后续。以后若增加 HTTP +Gateway 当前没有后台页面。当前管理面只提供状态查询和启动时 Token 验证，当前项及邻近 +窗口预热由成功 302 自动触发；创建显式控制接口、单季、整剧和目录任务、批任务 checkpoint +与恢复留到后续。以后若增加 HTTP API 或页面，使用默认关闭的独立 admin listener 或 Unix socket 和独立认证，不能挂到 客户端可访问的 Plex listener。 @@ -688,7 +702,7 @@ provider、目标 backing、decision revision、人工确认或策略依据， | Plex 未扫描到 Part | 延迟重试定向刷新，不猜测 PartID，不写数据库 | | MediaInfo 冷探测超时 | 在硬截止时间返回原始 Plex metadata，后台继续任务，不影响播放和观看状态 | | MediaInfo 探测失败 | 保留已知良好记录并进入负缓存；没有旧记录时保持 Plex 原始字段 | -| Plex 管理 Token 失效 | 停止主动发现和预热，客户端请求继续使用自己的 Token | +| Plex 管理 Token 失效 | 停止邻近媒体发现；当前项预热和客户端请求继续工作 | | Plex DB helper 拒绝写入 | 保留权威 MediaInfo，播放按 Gateway 能力降级，不删除本地文件 | | 分享挂载失效 | 标记 borrowed source unavailable，保留观看状态，等待换源、转存或人工退休 | | 分享转存部分成功 | 逐项验证自有 backing，未成功项不得标记为 `verified` | @@ -698,8 +712,8 @@ provider、目标 backing、decision revision、人工确认或策略依据， ## 15. 安全与隐私 - MoviePilot 到 MediaVault 的管理调用只使用 Header API Key，不使用 query API Key； -- Gateway 可以通过部署专用 `app.env` 注入 `PLEX_TOKEN`，仅用于后台发现、预热和 - reconciliation； +- Gateway 可以通过部署专用 `app.env` 注入 `PLEX_TOKEN`，当前仅用于启动校验和邻近 + 媒体发现； - 客户端 Plex Token 只属于当前请求，不转换为后台管理凭据； - Gateway 不保存 115 Cookie、分享提取码或 MediaVault 管理 API Key； - 分享引用、云盘 file ID、下载 hash 和本地路径只在必要的私有状态中保存； @@ -742,10 +756,10 @@ provider、目标 backing、decision revision、人工确认或策略依据， ### Phase D：MediaInfo 兜底与投影 PoC -- 优先实现 Gateway L1、SQLite、受限 ffprobe、`PLEX_TOKEN` 主动预热和 response +- 优先实现 Gateway L1、SQLite、受限 ffprobe、`PLEX_TOKEN` 邻近发现和 response enrichment； -- 实现精确单 Part 和确认播放后的一步下一集任务，以及优先级、singleflight、推测任务 - 抢占、负缓存和已知良好保护； +- 实现精确当前 Part 和重定向就绪后的双向邻近窗口，以及优先级、限速提交、singleflight、 + 推测任务抢占、负缓存和已知良好保护； - 将单季、整剧、STRM 目录任务、批任务 checkpoint 和恢复留到后续批量预热能力； - 验证 `5s` 冷等待候选上限、首次 metadata 完整率和 metadata p50/p95/p99； - 验证 decision、Part、universal start 和 302 不执行分析 I/O，单项 metadata 的有界等待 @@ -782,7 +796,7 @@ provider、目标 backing、decision revision、人工确认或策略依据， ## 17. 待评估决策 1. `5s` 冷等待能否覆盖 HDR、Dolby Vision、大型 MKV 和高延迟 CDN 的 p95/p99； -2. L1 启动恢复和 Plex 管理 Token 主动预热能覆盖多少首次请求； +2. L1 启动恢复、当前项预热和 Plex 管理 Token 邻近发现能覆盖多少首次请求； 3. MediaInfo 公开契约使用原子 sidecar、只读 API，还是同时提供； 4. MediaVault 是否愿意提供版本化的分享任务和 MediaInfo API； 5. 未提供接口时，自有 115 provider 的允许范围与维护成本； diff --git a/docs/roadmap.md b/docs/roadmap.md index 6327b6b..e38615f 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -70,20 +70,23 @@ manifest segments, and every genuine transcode request remain Plex-owned. ## MediaInfo Phase D -MediaInfo is not an active dependency of redirect playback. The production Plex -snapshot currently exposes STRM Parts without container, codec, resolution, or -stream elements; the current playback plane does not synthesize those fields. +MediaInfo is not a dependency of redirect playback. Eligible single-item +metadata responses can now be enriched from the fail-open L1 and SQLite cache; +missing, slow, or failed probes preserve the original Plex response and do not +change the 302 path. MediaInfo remains isolated from redirect playback. Phase D prioritizes the Gateway fallback while preserving the broader lifecycle design: 1. use L1 and SQLite for single-instance fallback storage; 2. accept optional `PLEX_TOKEN` from an ignored deployment `app.env` for - discovery and prewarming; + nearby-item discovery; current-item prewarming remains token-independent; 3. evaluate bounded remote `ffprobe`, with an initial `5s` cold metadata wait ceiling based on the current sample; -4. support exact single-Part analysis and one-step next-episode prefetch after - confirmed playback without blocking decision, Part, universal start, or 302; +4. support exact current-Part analysis and a configurable nearby-item window + after a cloud redirect is ready, with immediate current-item submission and + rate-limited background neighbors, without blocking decision, Part, + universal start, or 302; defer season, show, and configured STRM-root batch tasks; 5. enrich authorized Plex metadata responses first, then evaluate an official Plex API, other supported PMS interfaces, and an isolated database helper diff --git a/internal/config/config.go b/internal/config/config.go index 62d16fd..f76cde0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -41,15 +41,21 @@ const ( defaultMediaInfoColdWait = 5 * time.Second defaultMediaInfoBodyLimit = 8 << 20 defaultMediaInfoWaiters = 4 + defaultPrewarmBefore = 2 + defaultPrewarmAfter = 3 + defaultPrewarmInterval = 5 * time.Second + maxPrewarmWindow = 50 ) // Config contains process-level settings for transparent Plex proxying and -// optional STRM redirect handling. Credentials remain request-scoped and are -// never loaded into this structure. +// optional STRM redirect handling. PlexToken is an isolated management +// credential for nearby-item discovery and is never mixed into client playback +// context. type Config struct { ListenAddr string DatabasePath string PlexURL *url.URL + PlexToken string MediaVaultURL *url.URL PathMappings []pathmap.Mapping CloudExtensions []string @@ -86,6 +92,11 @@ type MediaInfoConfig struct { ColdWait time.Duration ResponseMaxBytes int64 EnrichmentWaiters int + // PrewarmBefore and PrewarmAfter bound speculative neighbors around a + // cloud redirect. PrewarmInterval spaces their background submissions. + PrewarmBefore int + PrewarmAfter int + PrewarmInterval time.Duration } // MetadataGuardConfig bounds detailed metadata fan-out before requests enter @@ -123,6 +134,10 @@ func Load() (Config, error) { } plexURL.RawQuery = "" plexURL.Fragment = "" + plexToken := strings.TrimSpace(os.Getenv("PLEX_TOKEN")) + if strings.ContainsAny(plexToken, "\r\n") { + return Config{}, errors.New("PLEX_TOKEN must not contain line breaks") + } mediaVaultURL, err := optionalHTTPURL("MEDIAVAULT_URL") if err != nil { @@ -266,6 +281,21 @@ func Load() (Config, error) { if err != nil { return Config{}, err } + mediaInfoPrewarmBefore, err := envNonNegativeInt("MEDIAINFO_PREWARM_BEFORE", defaultPrewarmBefore) + if err != nil { + return Config{}, err + } + mediaInfoPrewarmAfter, err := envNonNegativeInt("MEDIAINFO_PREWARM_AFTER", defaultPrewarmAfter) + if err != nil { + return Config{}, err + } + if mediaInfoPrewarmBefore+mediaInfoPrewarmAfter > maxPrewarmWindow { + return Config{}, fmt.Errorf("MEDIAINFO_PREWARM_BEFORE and MEDIAINFO_PREWARM_AFTER must total at most %d", maxPrewarmWindow) + } + mediaInfoPrewarmInterval, err := envDuration("MEDIAINFO_PREWARM_INTERVAL", defaultPrewarmInterval) + if err != nil { + return Config{}, err + } readHeaderTimeout, err := envDuration("READ_HEADER_TIMEOUT", defaultReadHeaderTimeout) if err != nil { @@ -296,6 +326,7 @@ func Load() (Config, error) { ListenAddr: listenAddr, DatabasePath: databasePath, PlexURL: plexURL, + PlexToken: plexToken, MediaVaultURL: mediaVaultURL, PathMappings: pathMappings, CloudExtensions: cloudExtensions, @@ -320,6 +351,8 @@ func Load() (Config, error) { RecordTTL: mediaInfoTTL, RecordRetention: mediaInfoRetention, L1MaxEntries: mediaInfoL1Entries, NegativeTTL: mediaInfoNegativeTTL, UserAgent: mediaInfoUserAgent, ColdWait: mediaInfoColdWait, ResponseMaxBytes: mediaInfoBodyLimit, EnrichmentWaiters: mediaInfoWaiters, + PrewarmBefore: mediaInfoPrewarmBefore, PrewarmAfter: mediaInfoPrewarmAfter, + PrewarmInterval: mediaInfoPrewarmInterval, }, LogLevel: logLevel, TraceEnabled: traceEnabled, @@ -444,3 +477,18 @@ func envPositiveInt(name string, fallback int) (int, error) { } return value, nil } + +func envNonNegativeInt(name string, fallback int) (int, error) { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return fallback, nil + } + value, err := strconv.Atoi(raw) + if err != nil { + return 0, fmt.Errorf("parse %s: %w", name, err) + } + if value < 0 { + return 0, fmt.Errorf("%s must not be negative", name) + } + return value, nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3b617a3..c01d356 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -18,6 +18,7 @@ func TestLoadRejectsCredentialInPlexURL(t *testing.T) { func TestLoadDefaults(t *testing.T) { t.Setenv("PLEX_URL", "http://plex:32400") + t.Setenv("PLEX_TOKEN", "") t.Setenv("MEDIAVAULT_URL", "") t.Setenv("PATH_MAPPINGS", "") t.Setenv("LISTEN_ADDR", "") @@ -34,6 +35,9 @@ func TestLoadDefaults(t *testing.T) { if got.PlexURL.String() != "http://plex:32400" { t.Fatalf("PlexURL = %q", got.PlexURL) } + if got.PlexToken != "" { + t.Fatal("PlexToken unexpectedly configured") + } if got.TraceEnabled { t.Fatal("TraceEnabled = true") } @@ -43,7 +47,7 @@ func TestLoadDefaults(t *testing.T) { if got.DatabasePath != "./data/plex-gateway.db" { t.Fatalf("DatabasePath = %q", got.DatabasePath) } - if !got.MediaInfo.Enabled || got.MediaInfo.ProbeTimeout != 20*time.Second || got.MediaInfo.Concurrency != 1 || got.MediaInfo.UserAgent != "Infuse-Library/8.4.4" || got.MediaInfo.ColdWait != 5*time.Second || got.MediaInfo.ResponseMaxBytes != 8<<20 || got.MediaInfo.EnrichmentWaiters != 4 { + if !got.MediaInfo.Enabled || got.MediaInfo.ProbeTimeout != 20*time.Second || got.MediaInfo.Concurrency != 1 || got.MediaInfo.UserAgent != "Infuse-Library/8.4.4" || got.MediaInfo.ColdWait != 5*time.Second || got.MediaInfo.ResponseMaxBytes != 8<<20 || got.MediaInfo.EnrichmentWaiters != 4 || got.MediaInfo.PrewarmBefore != 2 || got.MediaInfo.PrewarmAfter != 3 || got.MediaInfo.PrewarmInterval != 5*time.Second { t.Fatalf("unexpected MediaInfo defaults: %#v", got.MediaInfo) } if got.MediaVaultURL != nil || len(got.PathMappings) != 0 { @@ -59,6 +63,7 @@ func TestLoadDefaults(t *testing.T) { func TestLoadCloudConfiguration(t *testing.T) { t.Setenv("PLEX_URL", "http://plex:32400") + t.Setenv("PLEX_TOKEN", "management-token") t.Setenv("MEDIAVAULT_URL", "http://mediavault:7811") t.Setenv("PATH_MAPPINGS", `[ {"plex_prefix":"/media/cloud","local_prefix":"/media/cloud"}, @@ -93,6 +98,9 @@ func TestLoadCloudConfiguration(t *testing.T) { t.Setenv("MEDIAINFO_COLD_WAIT", "4s") t.Setenv("MEDIAINFO_RESPONSE_MAX_BYTES", "2097152") t.Setenv("MEDIAINFO_ENRICHMENT_CONCURRENCY", "3") + t.Setenv("MEDIAINFO_PREWARM_BEFORE", "4") + t.Setenv("MEDIAINFO_PREWARM_AFTER", "6") + t.Setenv("MEDIAINFO_PREWARM_INTERVAL", "9s") got, err := Load() if err != nil { @@ -101,6 +109,9 @@ func TestLoadCloudConfiguration(t *testing.T) { if got.MediaVaultURL.String() != "http://mediavault:7811" || len(got.PathMappings) != 2 { t.Fatalf("cloud configuration = %#v", got) } + if got.PlexToken != "management-token" { + t.Fatalf("PlexToken = %q", got.PlexToken) + } if !reflect.DeepEqual(got.CloudExtensions, []string{".strm"}) { t.Fatalf("CloudExtensions = %v", got.CloudExtensions) } @@ -113,11 +124,21 @@ func TestLoadCloudConfiguration(t *testing.T) { if got.DatabasePath != "/app_data/test.db" { t.Fatalf("DatabasePath = %q", got.DatabasePath) } - if !got.MediaInfo.Enabled || got.MediaInfo.FFProbePath != "/usr/bin/ffprobe" || got.MediaInfo.ProbeTimeout != 12*time.Second || got.MediaInfo.ProbeSize != 4194304 || got.MediaInfo.AnalyzeDuration != 3*time.Second || got.MediaInfo.OutputMaxBytes != 1048576 || got.MediaInfo.Concurrency != 2 || got.MediaInfo.InteractiveQueueSize != 64 || got.MediaInfo.BackgroundQueueSize != 128 || got.MediaInfo.RecordTTL != 168*time.Hour || got.MediaInfo.RecordRetention != 720*time.Hour || got.MediaInfo.L1MaxEntries != 5000 || got.MediaInfo.NegativeTTL != 10*time.Minute || got.MediaInfo.UserAgent != "plex-gateway-test" || got.MediaInfo.ColdWait != 4*time.Second || got.MediaInfo.ResponseMaxBytes != 2097152 || got.MediaInfo.EnrichmentWaiters != 3 { + if !got.MediaInfo.Enabled || got.MediaInfo.FFProbePath != "/usr/bin/ffprobe" || got.MediaInfo.ProbeTimeout != 12*time.Second || got.MediaInfo.ProbeSize != 4194304 || got.MediaInfo.AnalyzeDuration != 3*time.Second || got.MediaInfo.OutputMaxBytes != 1048576 || got.MediaInfo.Concurrency != 2 || got.MediaInfo.InteractiveQueueSize != 64 || got.MediaInfo.BackgroundQueueSize != 128 || got.MediaInfo.RecordTTL != 168*time.Hour || got.MediaInfo.RecordRetention != 720*time.Hour || got.MediaInfo.L1MaxEntries != 5000 || got.MediaInfo.NegativeTTL != 10*time.Minute || got.MediaInfo.UserAgent != "plex-gateway-test" || got.MediaInfo.ColdWait != 4*time.Second || got.MediaInfo.ResponseMaxBytes != 2097152 || got.MediaInfo.EnrichmentWaiters != 3 || got.MediaInfo.PrewarmBefore != 4 || got.MediaInfo.PrewarmAfter != 6 || got.MediaInfo.PrewarmInterval != 9*time.Second { t.Fatalf("MediaInfo configuration = %#v", got.MediaInfo) } } +func TestLoadRejectsPlexTokenWithLineBreak(t *testing.T) { + t.Setenv("PLEX_URL", "http://plex:32400") + t.Setenv("PLEX_TOKEN", "secret\nvalue") + + _, err := Load() + if err == nil || !strings.Contains(err.Error(), "line breaks") { + t.Fatalf("Load() error = %v", err) + } +} + func TestLoadRejectsMediaInfoRetentionBelowFreshTTL(t *testing.T) { t.Setenv("PLEX_URL", "http://plex:32400") t.Setenv("MEDIAINFO_RECORD_TTL", "720h") @@ -150,3 +171,28 @@ func TestLoadRequiresCompleteCloudConfiguration(t *testing.T) { t.Fatalf("Load() error = %v", err) } } + +func TestLoadRejectsOversizedPrewarmWindow(t *testing.T) { + t.Setenv("PLEX_URL", "http://plex:32400") + t.Setenv("MEDIAINFO_PREWARM_BEFORE", "25") + t.Setenv("MEDIAINFO_PREWARM_AFTER", "26") + + _, err := Load() + if err == nil || !strings.Contains(err.Error(), "total at most 50") { + t.Fatalf("Load() error = %v", err) + } +} + +func TestLoadAllowsCurrentOnlyPrewarm(t *testing.T) { + t.Setenv("PLEX_URL", "http://plex:32400") + t.Setenv("MEDIAINFO_PREWARM_BEFORE", "0") + t.Setenv("MEDIAINFO_PREWARM_AFTER", "0") + + got, err := Load() + if err != nil { + t.Fatal(err) + } + if got.MediaInfo.PrewarmBefore != 0 || got.MediaInfo.PrewarmAfter != 0 { + t.Fatalf("prewarm window = %d/%d", got.MediaInfo.PrewarmBefore, got.MediaInfo.PrewarmAfter) + } +} diff --git a/internal/gateway/decision.go b/internal/gateway/decision.go index 3ca1e20..0d9ebca 100644 --- a/internal/gateway/decision.go +++ b/internal/gateway/decision.go @@ -92,6 +92,7 @@ func (h *decisionHandler) selectCloudPart(r *http.Request, attempt playback.Atte if preparation.State != playback.PreparationReady { return playback.PreparedPart{}, decisionSelectionPassthrough } + preparation.Part.RatingKey = strings.TrimPrefix(attempt.MetadataPath, "/library/metadata/") return preparation.Part, decisionSelectionCloud } diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 3490f85..9f86ef3 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -18,6 +18,7 @@ import ( "github.com/InfinityPacer/plex-gateway/internal/partcache" "github.com/InfinityPacer/plex-gateway/internal/pathmap" "github.com/InfinityPacer/plex-gateway/internal/playback" + "github.com/InfinityPacer/plex-gateway/internal/prewarm" "github.com/InfinityPacer/plex-gateway/internal/resolver" "github.com/InfinityPacer/plex-gateway/internal/trace" ) @@ -44,6 +45,12 @@ type Options struct { MediaInfoColdWait time.Duration MediaInfoResponseMaxBytes int64 MediaInfoEnrichmentConcurrency int + MediaInfoPrewarmer cloudRedirectPrewarmer + MediaInfoPrewarmStatus func() prewarm.Status +} + +type cloudRedirectPrewarmer interface { + TryEnqueue(prewarm.PlaybackContext) bool } // New builds the fail-open Plex proxy and optional Direct Play interceptor. @@ -75,9 +82,10 @@ func New(options Options) http.Handler { }, MaxBodyBytes: options.ObserveMaxBytes, OnParts: func(observation observe.Observation) { + ratingKey := observedRatingKey(observation.Request) for _, part := range observation.Parts { if cloudPlayback != nil { - cloudPlayback.Remember(playback.PreparedPart{Part: part}) + cloudPlayback.Remember(playback.PreparedPart{Part: part, RatingKey: ratingKey}) } } }, @@ -126,10 +134,11 @@ func New(options Options) http.Handler { CloudExtensions: options.CloudExtensions, }) partPlayback := &playbackHandler{ - service: cloudPlayback, - plex: plex, - logger: logger, - metrics: registry, + service: cloudPlayback, + plex: plex, + logger: logger, + metrics: registry, + prewarmer: options.MediaInfoPrewarmer, } decision := &decisionHandler{ plex: plex, @@ -153,9 +162,13 @@ func New(options Options) http.Handler { mux := http.NewServeMux() mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) { status := mediainfo.Status{} + prewarmStatus := prewarm.Status{} if options.MediaInfoStatus != nil { status = options.MediaInfoStatus() } + if options.MediaInfoPrewarmStatus != nil { + prewarmStatus = options.MediaInfoPrewarmStatus() + } w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(http.StatusOK) _ = json.NewEncoder(w).Encode(struct { @@ -163,13 +176,15 @@ func New(options Options) http.Handler { MediaInfo struct { Enabled bool `json:"enabled"` mediainfo.Status + Prewarm prewarm.Status `json:"prewarm"` } `json:"mediainfo"` }{ Status: "ok", MediaInfo: struct { Enabled bool `json:"enabled"` mediainfo.Status - }{Enabled: options.MediaInfoEnabled, Status: status}, + Prewarm prewarm.Status `json:"prewarm"` + }{Enabled: options.MediaInfoEnabled, Status: status, Prewarm: prewarmStatus}, }) }) mux.Handle("GET /metrics", registry.Handler()) @@ -203,10 +218,11 @@ func New(options Options) http.Handler { } type playbackHandler struct { - service *playback.Service - plex http.Handler - logger *slog.Logger - metrics *metrics.Metrics + service *playback.Service + plex http.Handler + logger *slog.Logger + metrics *metrics.Metrics + prewarmer cloudRedirectPrewarmer } func (h *playbackHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -279,7 +295,7 @@ func (h *playbackHandler) servePrepared( return } h.metrics.ObserveResolverLatency(result.ResolverLatency) - h.redirect(w, part.Part.ID, result.DirectURL, started, source) + h.redirect(w, r, part, result.DirectURL, started, source) } // requestCanceled separates downstream disconnects from failures in Plex or @@ -297,15 +313,24 @@ func requestFinished(r *http.Request) bool { return r != nil && r.Context().Err() != nil } -func (h *playbackHandler) redirect(w http.ResponseWriter, partID string, directURL resolver.DirectURL, started time.Time, source string) { +func (h *playbackHandler) redirect(w http.ResponseWriter, request *http.Request, part playback.PreparedPart, directURL resolver.DirectURL, started time.Time, source string) { latency := time.Since(started) h.metrics.IncRedirectSuccess() h.metrics.ObserveRedirectLatency(latency) w.Header().Set("Location", directURL.String()) w.Header().Set("Cache-Control", "no-store") w.WriteHeader(http.StatusFound) + if h.prewarmer != nil { + h.prewarmer.TryEnqueue(prewarm.PlaybackContext{ + RatingKey: part.RatingKey, PartID: part.Part.ID, Target: part.Target, + WindowKey: playbackWindowKey(request), + PlayQueueID: playbackRequestValue(request, "playQueueID"), + PlayQueueItemID: playbackRequestValue(request, "playQueueItemID"), + UserAgent: request.UserAgent(), + }) + } h.logger.Info("play", - "part", partID, + "part", part.Part.ID, "source", source, "resolver", "mediavault", "result", http.StatusFound, @@ -313,6 +338,45 @@ func (h *playbackHandler) redirect(w http.ResponseWriter, partID string, directU ) } +func observedRatingKey(request *http.Request) string { + if request == nil || request.URL == nil || !isMetadataItemPath(request.URL.Path) { + return "" + } + return strings.TrimPrefix(request.URL.Path, "/library/metadata/") +} + +func playbackRequestValue(request *http.Request, name string) string { + if request == nil || request.URL == nil { + return "" + } + value, ok := singleQueryValue(request.URL.Query(), name) + if !ok { + return "" + } + return value +} + +// playbackWindowKey chooses a stable non-credential client identity so rapid +// item changes replace only that client's speculative window. Ambiguous values +// are ignored and the coordinator assigns an isolated anonymous window. +func playbackWindowKey(request *http.Request) string { + if request == nil || request.URL == nil { + return "" + } + for _, name := range []string{ + "X-Plex-Client-Identifier", + "X-Plex-Session-Identifier", + "X-Plex-Session-Id", + "X-Plex-Playback-Session-Id", + } { + value, present, valid := requestIdentity(request, name) + if valid && present { + return name + "\x00" + value + } + } + return "" +} + func (h *playbackHandler) fallback(w http.ResponseWriter, r *http.Request, partID, reason string) { h.metrics.IncPlexFallback() h.logger.Info("play_fallback", "part", partID, "reason", reason) diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index 9c03a04..236bdd2 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -19,6 +19,7 @@ import ( "github.com/InfinityPacer/plex-gateway/internal/metrics" "github.com/InfinityPacer/plex-gateway/internal/partcache" "github.com/InfinityPacer/plex-gateway/internal/pathmap" + "github.com/InfinityPacer/plex-gateway/internal/prewarm" "github.com/InfinityPacer/plex-gateway/internal/resolver" "github.com/InfinityPacer/plex-gateway/internal/trace" ) @@ -143,12 +144,15 @@ func TestMetadataObservationEnablesCloudPartRedirect(t *testing.T) { })) defer plex.Close() - handler, registry, _ := newCloudHandler(t, plex.URL, mediaVault.URL, []pathmap.Mapping{{PlexPrefix: plexRoot, LocalPrefix: localRoot}}) + handler, registry, cache := newCloudHandler(t, plex.URL, mediaVault.URL, []pathmap.Mapping{{PlexPrefix: plexRoot, LocalPrefix: localRoot}}) metadata := httptest.NewRecorder() handler.ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/library/metadata/42", nil)) if metadata.Code != http.StatusOK { t.Fatalf("metadata status = %d", metadata.Code) } + if observed, ok := cache.Get("123"); !ok || observed.RatingKey != "42" { + t.Fatalf("observed Part = %#v, found = %v", observed, ok) + } playback := httptest.NewRecorder() playbackRequest := httptest.NewRequest(http.MethodGet, "/library/parts/123/7/file", nil) @@ -176,6 +180,63 @@ func TestMetadataObservationEnablesCloudPartRedirect(t *testing.T) { } } +func TestSuccessfulCloudRedirectEnqueuesCredentialFreePlaybackContext(t *testing.T) { + localRoot := t.TempDir() + if err := os.WriteFile(filepath.Join(localRoot, "Episode.strm"), []byte("http://public.invalid/redirect/pickcode\n"), 0o600); err != nil { + t.Fatal(err) + } + plex := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Location", "http://public.invalid/redirect/pickcode") + w.WriteHeader(http.StatusMovedPermanently) + })) + defer plex.Close() + mediaVault := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Location", "https://cdn.invalid/Episode.mkv") + w.WriteHeader(http.StatusFound) + })) + defer mediaVault.Close() + upstream, _ := url.Parse(plex.URL) + mapper, err := pathmap.New([]pathmap.Mapping{{PlexPrefix: "/media/cloud", LocalPrefix: localRoot}}) + if err != nil { + t.Fatal(err) + } + control, err := resolver.NewMediaVaultSTRMResolver(mediaVault.URL, nil, 0) + if err != nil { + t.Fatal(err) + } + cache := partcache.New(time.Hour) + cache.Put(partcache.PartInfo{ + PartID: "123", RatingKey: "42", PlexFilePath: "/media/cloud/Episode.strm", + PartKey: "/library/parts/123/7/file", + }) + events := make(chan prewarm.PlaybackContext, 1) + handler := New(Options{ + Upstream: upstream, PartCache: cache, PathMapper: mapper, Resolver: control, + CloudExtensions: []string{".strm"}, MediaInfoPrewarmer: recordingPrewarmer{events: events}, + }) + request := httptest.NewRequest(http.MethodGet, "/library/parts/123/7/file?playQueueID=5&playQueueItemID=99&X-Plex-Token=client-secret", nil) + request.Header.Set("User-Agent", "Client-Test/1.0") + request.Header.Set("X-Plex-Client-Identifier", "client-device") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusFound { + t.Fatalf("status = %d", response.Code) + } + select { + case event := <-events: + want := prewarm.PlaybackContext{ + RatingKey: "42", PartID: "123", Target: "http://public.invalid/redirect/pickcode", + WindowKey: "X-Plex-Client-Identifier\x00client-device", + PlayQueueID: "5", PlayQueueItemID: "99", UserAgent: "Client-Test/1.0", + } + if event != want { + t.Fatalf("event = %#v, want %#v", event, want) + } + case <-time.After(time.Second): + t.Fatal("prewarm event was not enqueued") + } +} + func TestAuthorizedPlexSTRMRedirectEnablesCloudPartRedirect(t *testing.T) { localRoot := t.TempDir() if err := os.WriteFile(filepath.Join(localRoot, "Movie.strm"), []byte("http://public.invalid/redirect/pickcode\n"), 0o600); err != nil { @@ -883,6 +944,15 @@ type cancelingResolver struct { cancel context.CancelFunc } +type recordingPrewarmer struct { + events chan<- prewarm.PlaybackContext +} + +func (prewarmer recordingPrewarmer) TryEnqueue(event prewarm.PlaybackContext) bool { + prewarmer.events <- event + return true +} + func (r cancelingResolver) ReadTarget(string) (string, error) { return "http://public.invalid/redirect/pickcode", nil } diff --git a/internal/mediainfo/service.go b/internal/mediainfo/service.go index 63c3ef7..9561daa 100644 --- a/internal/mediainfo/service.go +++ b/internal/mediainfo/service.go @@ -18,6 +18,7 @@ const ( defaultWorkerQueueSize = 256 defaultWorkerCount = 1 defaultNegativeLimit = 4096 + defaultRetryLimit = 256 defaultTouchQueueSize = 1024 defaultGCInterval = 24 * time.Hour defaultStoreTimeout = 5 * time.Second @@ -25,9 +26,10 @@ const ( ) var ( - ErrServiceUnavailable = errors.New("MediaInfo service is unavailable") - ErrQueueFull = errors.New("MediaInfo queue is full") - ErrNegativeCache = errors.New("MediaInfo probe is in backoff") + ErrServiceUnavailable = errors.New("MediaInfo service is unavailable") + ErrQueueFull = errors.New("MediaInfo queue is full") + ErrRetryRegistrationFull = errors.New("MediaInfo retry registration is full") + ErrNegativeCache = errors.New("MediaInfo probe is in backoff") ) // Priority separates interactive misses from administrator prewarming. A @@ -49,6 +51,26 @@ type Request struct { ClientUserAgent string } +// SubmitDisposition explains how a non-blocking Submit request was handled. +// A joined request may be retried later with its own User-Agent when the +// shared flight fails. +type SubmitDisposition string + +const ( + SubmitFreshCache SubmitDisposition = "fresh_cache" + SubmitJoinedExistingFlight SubmitDisposition = "joined_existing_flight" + SubmitNewlyQueued SubmitDisposition = "newly_queued" + SubmitRejected SubmitDisposition = "rejected" +) + +// SubmitResult is the detailed outcome of SubmitDetailed. Err is non-nil only +// for a rejected request; accepted cache hits, joins, and queue admissions +// have a nil Err. +type SubmitResult struct { + Disposition SubmitDisposition + Err error +} + func (request Request) validate() error { if err := request.Key.Validate(); err != nil { return err @@ -143,16 +165,20 @@ type Service struct { workerDone chan struct{} touches chan recordTouch - mu sync.Mutex - condition *sync.Cond - interactive []*job - background []*job - interactiveLimit int - backgroundLimit int - flights map[string]*flight - negative map[string]time.Time - closed bool - active atomic.Int64 + mu sync.Mutex + condition *sync.Cond + interactive []*job + background []*job + retryInteractive []*job + retryBackground []*job + interactiveLimit int + backgroundLimit int + flights map[string]*flight + negative map[string]time.Time + retryRegistrations int + retryLimit int + closed bool + active atomic.Int64 } type recordTouch struct { @@ -163,11 +189,17 @@ type recordTouch struct { } type flight struct { - done chan struct{} - job *job - userAgent string - record Record - err error + done chan struct{} + job *job + userAgent string + retryWaiters []retryRegistration + record Record + err error +} + +type retryRegistration struct { + request Request + userAgentKey string } type job struct { @@ -248,6 +280,7 @@ func NewService(options ServiceOptions) (*Service, error) { logger: logger, metrics: options.Metrics, probeTimeout: probeTimeout, recordTTL: recordTTL, recordRetention: recordRetention, negativeTTL: negativeTTL, negativeLimit: defaultNegativeLimit, + retryLimit: defaultRetryLimit, backgroundUserAgent: backgroundUserAgent, gcInterval: gcInterval, touchInterval: touchInterval, now: now, ctx: ctx, cancel: cancel, workerDone: make(chan struct{}), @@ -338,24 +371,38 @@ func (service *Service) Ensure(ctx context.Context, request Request) (Record, er } } -// Submit schedules work without creating a request-bound waiter. +// Submit schedules work without creating a request-bound waiter. It retains +// the original error-only contract; use SubmitDetailed when the admission +// disposition is needed. func (service *Service) Submit(request Request) error { + return service.SubmitDetailed(request).Err +} + +// SubmitDetailed schedules work and reports whether the request used a fresh +// cache record, joined an existing flight, was newly queued, or was rejected. +// A request that joins a different-User-Agent flight is registered once, with +// a bounded per-service budget, so a provider failure can hand the flight to +// that User-Agent without probing blindly with the same identity. +func (service *Service) SubmitDetailed(request Request) SubmitResult { if service == nil { - return ErrServiceUnavailable + return SubmitResult{Disposition: SubmitRejected, Err: ErrServiceUnavailable} } request, err := service.normalizeRequest(request) if err != nil { - return err + return SubmitResult{Disposition: SubmitRejected, Err: err} } _, found, fresh := service.lookup(service.ctx, request.Key) if found && fresh { - return nil + return SubmitResult{Disposition: SubmitFreshCache} } if found { request.Priority = PriorityBackground } - _, err = service.begin(request) - return err + _, disposition, err := service.beginDetailed(request, true) + if err != nil { + return SubmitResult{Disposition: SubmitRejected, Err: err} + } + return SubmitResult{Disposition: disposition} } func (service *Service) normalizeRequest(request Request) (Request, error) { @@ -428,13 +475,18 @@ func (service *Service) touchCached(record Record, now time.Time) Record { } func (service *Service) begin(request Request) (*flight, error) { + flight, _, err := service.beginDetailed(request, false) + return flight, err +} + +func (service *Service) beginDetailed(request Request, registerRetry bool) (*flight, SubmitDisposition, error) { if service == nil { - return nil, ErrServiceUnavailable + return nil, SubmitRejected, ErrServiceUnavailable } var err error request, err = service.normalizeRequest(request) if err != nil { - return nil, err + return nil, SubmitRejected, err } key := request.Key.cacheKey() negativeKey := service.negativeKey(request) @@ -442,12 +494,12 @@ func (service *Service) begin(request Request) (*flight, error) { service.mu.Lock() if service.closed { service.mu.Unlock() - return nil, ErrServiceUnavailable + return nil, SubmitRejected, ErrServiceUnavailable } if until, exists := service.negative[negativeKey]; exists { if now.Before(until) { service.mu.Unlock() - return nil, ErrNegativeCache + return nil, SubmitRejected, ErrNegativeCache } delete(service.negative, negativeKey) } @@ -455,21 +507,47 @@ func (service *Service) begin(request Request) (*flight, error) { if request.Priority == PriorityInteractive { service.promoteLocked(existing.job) } + if registerRetry && request.ClientUserAgent != existing.userAgent { + if err := service.registerRetryLocked(existing, request); err != nil { + service.mu.Unlock() + return nil, SubmitRejected, err + } + } service.mu.Unlock() - return existing, nil + return existing, SubmitJoinedExistingFlight, nil } flight := &flight{done: make(chan struct{}), userAgent: request.ClientUserAgent} queued := &job{request: request, flight: flight} flight.job = queued if !service.enqueueLocked(queued) { service.mu.Unlock() - return nil, ErrQueueFull + return nil, SubmitRejected, ErrQueueFull } service.flights[key] = flight service.condition.Signal() service.mu.Unlock() service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeQueued() }) - return flight, nil + return flight, SubmitNewlyQueued, nil +} + +// registerRetryLocked records one distinct waiting User-Agent for an active +// flight. The caller must hold service.mu. Registration is intentionally +// bounded because User-Agent is untrusted request input. +func (service *Service) registerRetryLocked(existing *flight, request Request) error { + userAgentKey := retryUserAgentKey(request.ClientUserAgent) + for _, registration := range existing.retryWaiters { + if registration.userAgentKey == userAgentKey { + return nil + } + } + if service.retryRegistrations >= service.retryLimit { + return ErrRetryRegistrationFull + } + existing.retryWaiters = append(existing.retryWaiters, retryRegistration{ + request: request, userAgentKey: userAgentKey, + }) + service.retryRegistrations++ + return nil } func (service *Service) enqueueLocked(queued *job) bool { @@ -488,22 +566,38 @@ func (service *Service) enqueueLocked(queued *job) bool { } func (service *Service) promoteLocked(queued *job) { - if queued == nil || queued.claimed || queued.request.Priority == PriorityInteractive || - len(service.interactive) >= service.interactiveLimit { + if queued == nil || queued.claimed || queued.request.Priority == PriorityInteractive { return } - for index, candidate := range service.background { - if candidate != queued { - continue - } - copy(service.background[index:], service.background[index+1:]) - service.background[len(service.background)-1] = nil - service.background = service.background[:len(service.background)-1] + if removeQueuedJob(&service.retryBackground, queued) { queued.request.Priority = PriorityInteractive - service.interactive = append(service.interactive, queued) + service.retryInteractive = append(service.retryInteractive, queued) service.condition.Signal() return } + if len(service.interactive) >= service.interactiveLimit { + return + } + if !removeQueuedJob(&service.background, queued) { + return + } + queued.request.Priority = PriorityInteractive + service.interactive = append(service.interactive, queued) + service.condition.Signal() +} + +func removeQueuedJob(queue *[]*job, target *job) bool { + for index, candidate := range *queue { + if candidate != target { + continue + } + copy((*queue)[index:], (*queue)[index+1:]) + last := len(*queue) - 1 + (*queue)[last] = nil + *queue = (*queue)[:last] + return true + } + return false } func (service *Service) worker() { @@ -561,17 +655,25 @@ func (service *Service) janitorWorker() { func (service *Service) nextJob() (*job, bool) { service.mu.Lock() defer service.mu.Unlock() - for !service.closed && len(service.interactive) == 0 && len(service.background) == 0 { + for !service.closed && service.queuedJobsLocked() == 0 { service.condition.Wait() } if service.closed { return nil, false } var queued *job - if len(service.interactive) > 0 { + if len(service.retryInteractive) > 0 { + queued = service.retryInteractive[0] + service.retryInteractive[0] = nil + service.retryInteractive = service.retryInteractive[1:] + } else if len(service.interactive) > 0 { queued = service.interactive[0] service.interactive[0] = nil service.interactive = service.interactive[1:] + } else if len(service.retryBackground) > 0 { + queued = service.retryBackground[0] + service.retryBackground[0] = nil + service.retryBackground = service.retryBackground[1:] } else { queued = service.background[0] service.background[0] = nil @@ -581,6 +683,19 @@ func (service *Service) nextJob() (*job, bool) { return queued, true } +func (service *Service) queuedJobsLocked() int { + return len(service.retryInteractive) + len(service.interactive) + + len(service.retryBackground) + len(service.background) +} + +func (service *Service) enqueueRetryLocked(queued *job) { + if queued.request.Priority == PriorityInteractive { + service.retryInteractive = append(service.retryInteractive, queued) + return + } + service.retryBackground = append(service.retryBackground, queued) +} + func (service *Service) run(queued *job) { started := service.now() service.active.Add(1) @@ -636,6 +751,7 @@ func (service *Service) run(queued *job) { func (service *Service) finish(queued *job, record Record, err error) { key := queued.request.Key.cacheKey() + var retry *flight service.mu.Lock() flight := service.flights[key] if flight != queued.flight { @@ -644,12 +760,19 @@ func (service *Service) finish(queued *job, record Record, err error) { } delete(service.flights, key) if err != nil && !errors.Is(err, context.Canceled) { - service.rememberNegativeLocked(service.negativeKey(queued.request), service.now()) + now := service.now() + service.rememberNegativeLocked(service.negativeKey(queued.request), now) + retry = service.scheduleRetryLocked(flight, queued.request.ClientUserAgent, now) + } else { + service.releaseRetryRegistrationsLocked(flight) } flight.record = record flight.err = err close(flight.done) service.mu.Unlock() + if retry != nil { + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeQueued() }) + } if err != nil { service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeFailure() }) service.logger.Info("mediainfo_probe_failed", "part", queued.request.Key.PartID, "error_kind", probeErrorKind(err)) @@ -658,11 +781,77 @@ func (service *Service) finish(queued *job, record Record, err error) { service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeSuccess() }) } +// scheduleRetryLocked promotes the first eligible Submit registration to a +// new flight and carries the remaining registrations forward. The caller must +// hold service.mu. A failed User-Agent is already in negative backoff, so it +// cannot be selected for an immediate blind retry. +func (service *Service) scheduleRetryLocked(failed *flight, failedUserAgent string, now time.Time) *flight { + var selected *retryRegistration + remaining := make([]retryRegistration, 0, len(failed.retryWaiters)) + for index := range failed.retryWaiters { + registration := failed.retryWaiters[index] + service.retryRegistrations-- + if registration.request.ClientUserAgent == failedUserAgent || + service.negativeActiveLocked(registration.request, now) { + continue + } + if selected == nil { + candidate := registration + selected = &candidate + continue + } + remaining = append(remaining, registration) + } + failed.retryWaiters = nil + if selected == nil { + return nil + } + + next := &flight{done: make(chan struct{}), userAgent: selected.request.ClientUserAgent} + next.retryWaiters = remaining + service.retryRegistrations += len(remaining) + next.job = &job{request: selected.request, flight: next} + service.enqueueRetryLocked(next.job) + service.flights[next.job.request.Key.cacheKey()] = next + service.condition.Signal() + return next +} + +func (service *Service) releaseRetryRegistrationsLocked(flight *flight) { + service.retryRegistrations -= len(flight.retryWaiters) + if service.retryRegistrations < 0 { + service.retryRegistrations = 0 + } + flight.retryWaiters = nil +} + +func (service *Service) negativeActiveLocked(request Request, now time.Time) bool { + key := service.negativeKey(request) + until, exists := service.negative[key] + if !exists { + return false + } + if now.Before(until) { + return true + } + delete(service.negative, key) + return false +} + func (service *Service) negativeKey(request Request) string { - digest := sha256.Sum256([]byte(request.ClientUserAgent)) + digest := userAgentDigest(request.ClientUserAgent) return request.Key.cacheKey() + "\x00" + string(digest[:]) } +func retryUserAgentKey(userAgent string) string { + digest := userAgentDigest(userAgent) + return string(digest[:]) +} + +func userAgentDigest(userAgent string) [sha256.Size]byte { + return sha256.Sum256([]byte(userAgent)) +} + func (service *Service) rememberNegativeLocked(key string, now time.Time) { for candidate, until := range service.negative { if !now.Before(until) { @@ -702,8 +891,9 @@ func (service *Service) Status() Status { available := !service.closed status := Status{ Available: available, CacheEntries: service.cache.Len(), ActiveProbes: service.active.Load(), - InteractiveQueued: len(service.interactive), BackgroundQueued: len(service.background), - NegativeEntries: len(service.negative), + InteractiveQueued: len(service.retryInteractive) + len(service.interactive), + BackgroundQueued: len(service.retryBackground) + len(service.background), + NegativeEntries: len(service.negative), } service.mu.Unlock() return status @@ -731,10 +921,13 @@ func (service *Service) Close(ctx context.Context) error { for key, flight := range service.flights { delete(service.flights, key) flight.err = ErrServiceUnavailable + service.releaseRetryRegistrationsLocked(flight) close(flight.done) } service.interactive = nil service.background = nil + service.retryInteractive = nil + service.retryBackground = nil service.condition.Broadcast() } service.mu.Unlock() diff --git a/internal/mediainfo/submit_test.go b/internal/mediainfo/submit_test.go new file mode 100644 index 0000000..1feb805 --- /dev/null +++ b/internal/mediainfo/submit_test.go @@ -0,0 +1,261 @@ +package mediainfo + +import ( + "context" + "errors" + "fmt" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestServiceSubmitDetailedReportsAdmissionDisposition(t *testing.T) { + now := time.Unix(1_800_000_000, 0).UTC() + prober := &blockingProber{started: make(chan string, 1), release: make(chan struct{})} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, now), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", Now: func() time.Time { return now }, + }) + + fresh := completeRecord(now) + fresh.Key.PlexServerID = "server" + fresh.Key.PartID = "fresh" + service.cache.Put(fresh, now) + freshResult := service.SubmitDetailed(testRequest("fresh", PriorityBackground)) + if freshResult.Disposition != SubmitFreshCache || freshResult.Err != nil { + t.Fatalf("fresh cache result = %#v", freshResult) + } + + queuedRequest := testRequest("queued", PriorityBackground) + queuedResult := service.SubmitDetailed(queuedRequest) + if queuedResult.Disposition != SubmitNewlyQueued || queuedResult.Err != nil { + t.Fatalf("new queue result = %#v", queuedResult) + } + select { + case <-prober.started: + case <-time.After(time.Second): + t.Fatal("queued probe did not start") + } + + joined := testRequest("queued", PriorityBackground) + joined.ClientUserAgent = "different-client" + joinedResult := service.SubmitDetailed(joined) + if joinedResult.Disposition != SubmitJoinedExistingFlight || joinedResult.Err != nil { + t.Fatalf("joined result = %#v", joinedResult) + } + + close(prober.release) + waitFor(t, func() bool { + _, ok := service.Get(queuedRequest.Key) + return ok + }) +} + +func TestServiceSubmitDetailedReportsRejection(t *testing.T) { + prober := &blockingProber{started: make(chan string, 1), release: make(chan struct{})} + service := newTestServiceWithQueues(t, prober, 1, 1, 1) + if result := service.SubmitDetailed(testRequest("active", PriorityBackground)); result.Disposition != SubmitNewlyQueued || result.Err != nil { + t.Fatalf("active result = %#v", result) + } + select { + case <-prober.started: + case <-time.After(time.Second): + t.Fatal("active probe did not start") + } + if result := service.SubmitDetailed(testRequest("queued", PriorityBackground)); result.Disposition != SubmitNewlyQueued || result.Err != nil { + t.Fatalf("queued result = %#v", result) + } + result := service.SubmitDetailed(testRequest("overflow", PriorityBackground)) + if result.Disposition != SubmitRejected || !errors.Is(result.Err, ErrQueueFull) { + t.Fatalf("overflow result = %#v", result) + } +} + +func TestServiceSubmitRequeuesJoinedUserAgentAfterFailure(t *testing.T) { + now := time.Unix(1_800_000_000, 0).UTC() + started := make(chan string, 2) + releaseFailure := make(chan struct{}) + var calls atomic.Int64 + prober := proberFunc(func(ctx context.Context, _ string, userAgent string) (Media, error) { + call := calls.Add(1) + started <- userAgent + if call == 1 { + select { + case <-ctx.Done(): + return Media{}, ctx.Err() + case <-releaseFailure: + return Media{}, errors.New("first User-Agent rejected") + } + } + return Media{ + Complete: true, Container: "mkv", DurationMS: 60_000, + Streams: []Stream{{Index: 0, Type: "video", Codec: "hevc", Width: 1920, Height: 1080}}, + }, nil + }) + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, now), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", NegativeTTL: time.Hour, Now: func() time.Time { return now }, + }) + first := testRequest("submit-cross-ua", PriorityInteractive) + first.ClientUserAgent = "client-agent-a" + second := first + second.ClientUserAgent = "client-agent-b" + if result := service.SubmitDetailed(first); result.Disposition != SubmitNewlyQueued || result.Err != nil { + t.Fatalf("first submit result = %#v", result) + } + if got := <-started; got != first.ClientUserAgent { + t.Fatalf("first probe User-Agent = %q", got) + } + if result := service.SubmitDetailed(second); result.Disposition != SubmitJoinedExistingFlight || result.Err != nil { + t.Fatalf("second submit result = %#v", result) + } + + close(releaseFailure) + if got := <-started; got != second.ClientUserAgent { + t.Fatalf("retry probe User-Agent = %q", got) + } + waitFor(t, func() bool { + _, ok := service.Get(first.Key) + return ok + }) + if got := calls.Load(); got != 2 { + t.Fatalf("probe calls = %d, want 2", got) + } +} + +func TestServicePromotesPendingUserAgentRetryForCurrentPlayback(t *testing.T) { + started := make(chan string, 8) + releases := map[string]chan struct{}{ + "first": make(chan struct{}), + "blocker": make(chan struct{}), + "retry": make(chan struct{}), + "other": make(chan struct{}), + } + prober := proberFunc(func(ctx context.Context, target, _ string) (Media, error) { + target = strings.TrimPrefix(target, "https://cdn.example.test/") + started <- target + select { + case <-ctx.Done(): + return Media{}, ctx.Err() + case <-releases[target]: + } + if target == "first" { + return Media{}, errors.New("first User-Agent rejected") + } + return Media{ + Complete: true, Container: "mkv", DurationMS: 60_000, + Streams: []Stream{{Index: 0, Type: "video", Codec: "hevc", Width: 1920, Height: 1080}}, + }, nil + }) + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", Concurrency: 1, InteractiveQueueSize: 8, BackgroundQueueSize: 8, + }) + first := testRequest("first", PriorityBackground) + first.ClientUserAgent = "client-a" + retry := first + retry.Target = "retry" + retry.ClientUserAgent = "client-b" + if result := service.SubmitDetailed(first); result.Disposition != SubmitNewlyQueued { + t.Fatalf("first result = %#v", result) + } + if got := <-started; got != "first" { + t.Fatalf("first target = %q", got) + } + if result := service.SubmitDetailed(retry); result.Disposition != SubmitJoinedExistingFlight { + t.Fatalf("retry registration = %#v", result) + } + if result := service.SubmitDetailed(testRequest("blocker", PriorityInteractive)); result.Disposition != SubmitNewlyQueued { + t.Fatalf("blocker result = %#v", result) + } + close(releases["first"]) + if got := <-started; got != "blocker" { + t.Fatalf("second target = %q", got) + } + if result := service.SubmitDetailed(testRequest("other", PriorityInteractive)); result.Disposition != SubmitNewlyQueued { + t.Fatalf("other result = %#v", result) + } + current := retry + current.Priority = PriorityInteractive + if result := service.SubmitDetailed(current); result.Disposition != SubmitJoinedExistingFlight { + t.Fatalf("current join result = %#v", result) + } + close(releases["blocker"]) + if got := <-started; got != "retry" { + t.Fatalf("promoted target = %q, want retry", got) + } + close(releases["retry"]) + if got := <-started; got != "other" { + t.Fatalf("following target = %q, want other", got) + } + close(releases["other"]) +} + +func TestServiceSubmitDoesNotRequeueSameUserAgent(t *testing.T) { + now := time.Unix(1_800_000_000, 0).UTC() + started := make(chan string, 1) + releaseFailure := make(chan struct{}) + var calls atomic.Int64 + prober := proberFunc(func(ctx context.Context, _ string, userAgent string) (Media, error) { + calls.Add(1) + started <- userAgent + select { + case <-ctx.Done(): + return Media{}, ctx.Err() + case <-releaseFailure: + return Media{}, errors.New("probe rejected") + } + }) + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, now), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", NegativeTTL: time.Hour, Now: func() time.Time { return now }, + }) + request := testRequest("submit-same-ua", PriorityBackground) + request.ClientUserAgent = "same-client" + if result := service.SubmitDetailed(request); result.Disposition != SubmitNewlyQueued || result.Err != nil { + t.Fatalf("first submit result = %#v", result) + } + <-started + if result := service.SubmitDetailed(request); result.Disposition != SubmitJoinedExistingFlight || result.Err != nil { + t.Fatalf("same User-Agent join result = %#v", result) + } + close(releaseFailure) + waitFor(t, func() bool { return calls.Load() == 1 && service.Status().ActiveProbes == 0 }) + select { + case got := <-started: + t.Fatalf("same User-Agent was requeued as %q", got) + default: + } +} + +func TestServiceSubmitBoundsRetryRegistrations(t *testing.T) { + prober := &blockingProber{started: make(chan string, 1), release: make(chan struct{})} + service := newTestService(t, prober, 1) + first := testRequest("submit-registration-limit", PriorityBackground) + first.ClientUserAgent = "owner" + if result := service.SubmitDetailed(first); result.Err != nil { + t.Fatalf("first submit result = %#v", result) + } + <-prober.started + for index := 0; index < defaultRetryLimit+32; index++ { + request := first + request.ClientUserAgent = fmt.Sprintf("client-%d", index) + result := service.SubmitDetailed(request) + if index < defaultRetryLimit { + if result.Disposition != SubmitJoinedExistingFlight || result.Err != nil { + t.Fatalf("registration %d result = %#v", index, result) + } + continue + } + if result.Disposition != SubmitRejected || !errors.Is(result.Err, ErrRetryRegistrationFull) { + t.Fatalf("overflow registration %d result = %#v", index, result) + } + } + service.mu.Lock() + registrations := service.retryRegistrations + service.mu.Unlock() + if registrations != defaultRetryLimit { + t.Fatalf("retry registrations = %d, want %d", registrations, defaultRetryLimit) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index ec44d39..89cf26a 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -15,35 +15,44 @@ import ( // surface bounded and prevents request credentials or media URLs from being // copied into an operational endpoint. type Metrics struct { - plexRequestsTotal atomic.Uint64 - cloudPartHits atomic.Uint64 - cloudPartMisses atomic.Uint64 - redirectSuccess atomic.Uint64 - redirectFailure atomic.Uint64 - plexFallbackTotal atomic.Uint64 - activeRequests atomic.Int64 - metadataAdmitted atomic.Uint64 - metadataTimeouts atomic.Uint64 - metadataActive atomic.Int64 - metadataQueued atomic.Int64 - metadataBatchAdmitted atomic.Uint64 - metadataBatchTimeouts atomic.Uint64 - metadataBatchActive atomic.Int64 - metadataBatchQueued atomic.Int64 - mediaInfoCacheHits atomic.Uint64 - mediaInfoCacheMisses atomic.Uint64 - mediaInfoProbeQueued atomic.Uint64 - mediaInfoProbeSuccess atomic.Uint64 - mediaInfoProbeFailure atomic.Uint64 - mediaInfoStoreFailure atomic.Uint64 - mediaInfoProbeActive atomic.Int64 - mediaInfoEnriched atomic.Uint64 - mediaInfoFailOpen atomic.Uint64 - mediaInfoWaitActive atomic.Int64 - mediaInfoWaitRejected atomic.Uint64 - resolverLatency latencyMetrics - redirectLatency latencyMetrics - mediaInfoProbeLatency latencyMetrics + plexRequestsTotal atomic.Uint64 + cloudPartHits atomic.Uint64 + cloudPartMisses atomic.Uint64 + redirectSuccess atomic.Uint64 + redirectFailure atomic.Uint64 + plexFallbackTotal atomic.Uint64 + activeRequests atomic.Int64 + metadataAdmitted atomic.Uint64 + metadataTimeouts atomic.Uint64 + metadataActive atomic.Int64 + metadataQueued atomic.Int64 + metadataBatchAdmitted atomic.Uint64 + metadataBatchTimeouts atomic.Uint64 + metadataBatchActive atomic.Int64 + metadataBatchQueued atomic.Int64 + mediaInfoCacheHits atomic.Uint64 + mediaInfoCacheMisses atomic.Uint64 + mediaInfoProbeQueued atomic.Uint64 + mediaInfoProbeSuccess atomic.Uint64 + mediaInfoProbeFailure atomic.Uint64 + mediaInfoStoreFailure atomic.Uint64 + mediaInfoProbeActive atomic.Int64 + mediaInfoEnriched atomic.Uint64 + mediaInfoFailOpen atomic.Uint64 + mediaInfoWaitActive atomic.Int64 + mediaInfoWaitRejected atomic.Uint64 + mediaInfoPrewarmTriggered atomic.Uint64 + mediaInfoPrewarmReplaced atomic.Uint64 + mediaInfoPrewarmDiscoverySuccess atomic.Uint64 + mediaInfoPrewarmDiscoveryFailure atomic.Uint64 + mediaInfoPrewarmFreshCache atomic.Uint64 + mediaInfoPrewarmJoinedFlight atomic.Uint64 + mediaInfoPrewarmQueued atomic.Uint64 + mediaInfoPrewarmRejected atomic.Uint64 + mediaInfoPrewarmSkipped atomic.Uint64 + resolverLatency latencyMetrics + redirectLatency latencyMetrics + mediaInfoProbeLatency latencyMetrics } type latencyMetrics struct { @@ -64,25 +73,34 @@ type Snapshot struct { PlexFallbackTotal uint64 `json:"plex_fallback_total"` ActiveRequests int64 `json:"active_requests"` - MetadataGuardAdmittedTotal uint64 `json:"metadata_guard_admitted_total"` - MetadataGuardTimeoutsTotal uint64 `json:"metadata_guard_timeouts_total"` - MetadataGuardActive int64 `json:"metadata_guard_active"` - MetadataGuardQueued int64 `json:"metadata_guard_queued"` - MetadataBatchGuardAdmittedTotal uint64 `json:"metadata_batch_guard_admitted_total"` - MetadataBatchGuardTimeoutsTotal uint64 `json:"metadata_batch_guard_timeouts_total"` - MetadataBatchGuardActive int64 `json:"metadata_batch_guard_active"` - MetadataBatchGuardQueued int64 `json:"metadata_batch_guard_queued"` - MediaInfoCacheHitsTotal uint64 `json:"mediainfo_cache_hits_total"` - MediaInfoCacheMissesTotal uint64 `json:"mediainfo_cache_misses_total"` - MediaInfoProbeQueuedTotal uint64 `json:"mediainfo_probe_queued_total"` - MediaInfoProbeSuccessTotal uint64 `json:"mediainfo_probe_success_total"` - MediaInfoProbeFailureTotal uint64 `json:"mediainfo_probe_failure_total"` - MediaInfoStoreFailureTotal uint64 `json:"mediainfo_store_failure_total"` - MediaInfoProbeActive int64 `json:"mediainfo_probe_active"` - MediaInfoEnrichedTotal uint64 `json:"mediainfo_enriched_total"` - MediaInfoFailOpenTotal uint64 `json:"mediainfo_fail_open_total"` - MediaInfoWaitActive int64 `json:"mediainfo_wait_active"` - MediaInfoWaitRejectedTotal uint64 `json:"mediainfo_wait_rejected_total"` + MetadataGuardAdmittedTotal uint64 `json:"metadata_guard_admitted_total"` + MetadataGuardTimeoutsTotal uint64 `json:"metadata_guard_timeouts_total"` + MetadataGuardActive int64 `json:"metadata_guard_active"` + MetadataGuardQueued int64 `json:"metadata_guard_queued"` + MetadataBatchGuardAdmittedTotal uint64 `json:"metadata_batch_guard_admitted_total"` + MetadataBatchGuardTimeoutsTotal uint64 `json:"metadata_batch_guard_timeouts_total"` + MetadataBatchGuardActive int64 `json:"metadata_batch_guard_active"` + MetadataBatchGuardQueued int64 `json:"metadata_batch_guard_queued"` + MediaInfoCacheHitsTotal uint64 `json:"mediainfo_cache_hits_total"` + MediaInfoCacheMissesTotal uint64 `json:"mediainfo_cache_misses_total"` + MediaInfoProbeQueuedTotal uint64 `json:"mediainfo_probe_queued_total"` + MediaInfoProbeSuccessTotal uint64 `json:"mediainfo_probe_success_total"` + MediaInfoProbeFailureTotal uint64 `json:"mediainfo_probe_failure_total"` + MediaInfoStoreFailureTotal uint64 `json:"mediainfo_store_failure_total"` + MediaInfoProbeActive int64 `json:"mediainfo_probe_active"` + MediaInfoEnrichedTotal uint64 `json:"mediainfo_enriched_total"` + MediaInfoFailOpenTotal uint64 `json:"mediainfo_fail_open_total"` + MediaInfoWaitActive int64 `json:"mediainfo_wait_active"` + MediaInfoWaitRejectedTotal uint64 `json:"mediainfo_wait_rejected_total"` + MediaInfoPrewarmTriggeredTotal uint64 `json:"mediainfo_prewarm_triggered_total"` + MediaInfoPrewarmReplacedTotal uint64 `json:"mediainfo_prewarm_replaced_total"` + MediaInfoPrewarmDiscoverySuccessTotal uint64 `json:"mediainfo_prewarm_discovery_success_total"` + MediaInfoPrewarmDiscoveryFailureTotal uint64 `json:"mediainfo_prewarm_discovery_failure_total"` + MediaInfoPrewarmFreshCacheTotal uint64 `json:"mediainfo_prewarm_fresh_cache_total"` + MediaInfoPrewarmJoinedFlightTotal uint64 `json:"mediainfo_prewarm_joined_flight_total"` + MediaInfoPrewarmQueuedTotal uint64 `json:"mediainfo_prewarm_queued_total"` + MediaInfoPrewarmRejectedTotal uint64 `json:"mediainfo_prewarm_rejected_total"` + MediaInfoPrewarmSkippedTotal uint64 `json:"mediainfo_prewarm_skipped_total"` ResolverLatencyMSTotal uint64 `json:"resolver_latency_ms_total"` ResolverLatencySamples uint64 `json:"resolver_latency_samples"` @@ -184,6 +202,39 @@ func (m *Metrics) DecMediaInfoWaitActive() { decrementGauge(&m.mediaInfoWaitActi // because the bounded enrichment pool was full. func (m *Metrics) IncMediaInfoWaitRejected() { m.mediaInfoWaitRejected.Add(1) } +// IncMediaInfoPrewarmTriggered records an accepted in-memory redirect event. +func (m *Metrics) IncMediaInfoPrewarmTriggered() { m.mediaInfoPrewarmTriggered.Add(1) } + +// IncMediaInfoPrewarmReplaced records a pending or active nearby-item window +// replaced by a newer playback in the same client session. +func (m *Metrics) IncMediaInfoPrewarmReplaced() { m.mediaInfoPrewarmReplaced.Add(1) } + +// IncMediaInfoPrewarmDiscoverySuccess records a usable nearby-item window. +func (m *Metrics) IncMediaInfoPrewarmDiscoverySuccess() { + m.mediaInfoPrewarmDiscoverySuccess.Add(1) +} + +// IncMediaInfoPrewarmDiscoveryFailure records a bounded Plex discovery error. +func (m *Metrics) IncMediaInfoPrewarmDiscoveryFailure() { + m.mediaInfoPrewarmDiscoveryFailure.Add(1) +} + +// IncMediaInfoPrewarmFreshCache records a prewarm satisfied by fresh MediaInfo. +func (m *Metrics) IncMediaInfoPrewarmFreshCache() { m.mediaInfoPrewarmFreshCache.Add(1) } + +// IncMediaInfoPrewarmJoinedFlight records a prewarm joined to active work. +func (m *Metrics) IncMediaInfoPrewarmJoinedFlight() { m.mediaInfoPrewarmJoinedFlight.Add(1) } + +// IncMediaInfoPrewarmQueued records a newly queued MediaInfo probe. +func (m *Metrics) IncMediaInfoPrewarmQueued() { m.mediaInfoPrewarmQueued.Add(1) } + +// IncMediaInfoPrewarmRejected records a MediaInfo admission rejection. +func (m *Metrics) IncMediaInfoPrewarmRejected() { m.mediaInfoPrewarmRejected.Add(1) } + +// IncMediaInfoPrewarmSkipped records a duplicate, unsafe, unavailable, or +// otherwise non-actionable speculative event. +func (m *Metrics) IncMediaInfoPrewarmSkipped() { m.mediaInfoPrewarmSkipped.Add(1) } + // IncMetadataGuardAdmitted records a detailed metadata request admitted to // Plex after both client and global limits were acquired. func (m *Metrics) IncMetadataGuardAdmitted() { @@ -325,25 +376,34 @@ func (m *Metrics) Snapshot() Snapshot { PlexFallbackTotal: m.plexFallbackTotal.Load(), ActiveRequests: m.activeRequests.Load(), - MetadataGuardAdmittedTotal: m.metadataAdmitted.Load(), - MetadataGuardTimeoutsTotal: m.metadataTimeouts.Load(), - MetadataGuardActive: m.metadataActive.Load(), - MetadataGuardQueued: m.metadataQueued.Load(), - MetadataBatchGuardAdmittedTotal: m.metadataBatchAdmitted.Load(), - MetadataBatchGuardTimeoutsTotal: m.metadataBatchTimeouts.Load(), - MetadataBatchGuardActive: m.metadataBatchActive.Load(), - MetadataBatchGuardQueued: m.metadataBatchQueued.Load(), - MediaInfoCacheHitsTotal: m.mediaInfoCacheHits.Load(), - MediaInfoCacheMissesTotal: m.mediaInfoCacheMisses.Load(), - MediaInfoProbeQueuedTotal: m.mediaInfoProbeQueued.Load(), - MediaInfoProbeSuccessTotal: m.mediaInfoProbeSuccess.Load(), - MediaInfoProbeFailureTotal: m.mediaInfoProbeFailure.Load(), - MediaInfoStoreFailureTotal: m.mediaInfoStoreFailure.Load(), - MediaInfoProbeActive: m.mediaInfoProbeActive.Load(), - MediaInfoEnrichedTotal: m.mediaInfoEnriched.Load(), - MediaInfoFailOpenTotal: m.mediaInfoFailOpen.Load(), - MediaInfoWaitActive: m.mediaInfoWaitActive.Load(), - MediaInfoWaitRejectedTotal: m.mediaInfoWaitRejected.Load(), + MetadataGuardAdmittedTotal: m.metadataAdmitted.Load(), + MetadataGuardTimeoutsTotal: m.metadataTimeouts.Load(), + MetadataGuardActive: m.metadataActive.Load(), + MetadataGuardQueued: m.metadataQueued.Load(), + MetadataBatchGuardAdmittedTotal: m.metadataBatchAdmitted.Load(), + MetadataBatchGuardTimeoutsTotal: m.metadataBatchTimeouts.Load(), + MetadataBatchGuardActive: m.metadataBatchActive.Load(), + MetadataBatchGuardQueued: m.metadataBatchQueued.Load(), + MediaInfoCacheHitsTotal: m.mediaInfoCacheHits.Load(), + MediaInfoCacheMissesTotal: m.mediaInfoCacheMisses.Load(), + MediaInfoProbeQueuedTotal: m.mediaInfoProbeQueued.Load(), + MediaInfoProbeSuccessTotal: m.mediaInfoProbeSuccess.Load(), + MediaInfoProbeFailureTotal: m.mediaInfoProbeFailure.Load(), + MediaInfoStoreFailureTotal: m.mediaInfoStoreFailure.Load(), + MediaInfoProbeActive: m.mediaInfoProbeActive.Load(), + MediaInfoEnrichedTotal: m.mediaInfoEnriched.Load(), + MediaInfoFailOpenTotal: m.mediaInfoFailOpen.Load(), + MediaInfoWaitActive: m.mediaInfoWaitActive.Load(), + MediaInfoWaitRejectedTotal: m.mediaInfoWaitRejected.Load(), + MediaInfoPrewarmTriggeredTotal: m.mediaInfoPrewarmTriggered.Load(), + MediaInfoPrewarmReplacedTotal: m.mediaInfoPrewarmReplaced.Load(), + MediaInfoPrewarmDiscoverySuccessTotal: m.mediaInfoPrewarmDiscoverySuccess.Load(), + MediaInfoPrewarmDiscoveryFailureTotal: m.mediaInfoPrewarmDiscoveryFailure.Load(), + MediaInfoPrewarmFreshCacheTotal: m.mediaInfoPrewarmFreshCache.Load(), + MediaInfoPrewarmJoinedFlightTotal: m.mediaInfoPrewarmJoinedFlight.Load(), + MediaInfoPrewarmQueuedTotal: m.mediaInfoPrewarmQueued.Load(), + MediaInfoPrewarmRejectedTotal: m.mediaInfoPrewarmRejected.Load(), + MediaInfoPrewarmSkippedTotal: m.mediaInfoPrewarmSkipped.Load(), ResolverLatencyMSTotal: resolverTotal, ResolverLatencySamples: resolverSamples, diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index b3365ba..7cd1444 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -38,6 +38,15 @@ func TestSnapshotAndJSONHandler(t *testing.T) { registry.IncMediaInfoFailOpen() registry.IncMediaInfoWaitActive() registry.IncMediaInfoWaitRejected() + registry.IncMediaInfoPrewarmTriggered() + registry.IncMediaInfoPrewarmReplaced() + registry.IncMediaInfoPrewarmDiscoverySuccess() + registry.IncMediaInfoPrewarmDiscoveryFailure() + registry.IncMediaInfoPrewarmFreshCache() + registry.IncMediaInfoPrewarmJoinedFlight() + registry.IncMediaInfoPrewarmQueued() + registry.IncMediaInfoPrewarmRejected() + registry.IncMediaInfoPrewarmSkipped() registry.ObserveMediaInfoProbeLatency(44 * time.Millisecond) registry.ObserveResolverLatency(12 * time.Millisecond) registry.ObserveResolverLatency(7 * time.Millisecond) @@ -67,25 +76,34 @@ func TestSnapshotAndJSONHandler(t *testing.T) { PlexFallbackTotal: 1, ActiveRequests: 1, - MetadataGuardAdmittedTotal: 1, - MetadataGuardTimeoutsTotal: 1, - MetadataGuardActive: 1, - MetadataGuardQueued: 1, - MetadataBatchGuardAdmittedTotal: 1, - MetadataBatchGuardTimeoutsTotal: 1, - MetadataBatchGuardActive: 1, - MetadataBatchGuardQueued: 1, - MediaInfoCacheHitsTotal: 1, - MediaInfoCacheMissesTotal: 1, - MediaInfoProbeQueuedTotal: 1, - MediaInfoProbeSuccessTotal: 1, - MediaInfoProbeFailureTotal: 1, - MediaInfoStoreFailureTotal: 1, - MediaInfoProbeActive: 1, - MediaInfoEnrichedTotal: 1, - MediaInfoFailOpenTotal: 1, - MediaInfoWaitActive: 1, - MediaInfoWaitRejectedTotal: 1, + MetadataGuardAdmittedTotal: 1, + MetadataGuardTimeoutsTotal: 1, + MetadataGuardActive: 1, + MetadataGuardQueued: 1, + MetadataBatchGuardAdmittedTotal: 1, + MetadataBatchGuardTimeoutsTotal: 1, + MetadataBatchGuardActive: 1, + MetadataBatchGuardQueued: 1, + MediaInfoCacheHitsTotal: 1, + MediaInfoCacheMissesTotal: 1, + MediaInfoProbeQueuedTotal: 1, + MediaInfoProbeSuccessTotal: 1, + MediaInfoProbeFailureTotal: 1, + MediaInfoStoreFailureTotal: 1, + MediaInfoProbeActive: 1, + MediaInfoEnrichedTotal: 1, + MediaInfoFailOpenTotal: 1, + MediaInfoWaitActive: 1, + MediaInfoWaitRejectedTotal: 1, + MediaInfoPrewarmTriggeredTotal: 1, + MediaInfoPrewarmReplacedTotal: 1, + MediaInfoPrewarmDiscoverySuccessTotal: 1, + MediaInfoPrewarmDiscoveryFailureTotal: 1, + MediaInfoPrewarmFreshCacheTotal: 1, + MediaInfoPrewarmJoinedFlightTotal: 1, + MediaInfoPrewarmQueuedTotal: 1, + MediaInfoPrewarmRejectedTotal: 1, + MediaInfoPrewarmSkippedTotal: 1, ResolverLatencyMSTotal: 19, ResolverLatencySamples: 2, @@ -217,6 +235,15 @@ func TestCountersAreSafeForConcurrentUpdates(t *testing.T) { registry.IncMediaInfoWaitActive() registry.DecMediaInfoWaitActive() registry.IncMediaInfoWaitRejected() + registry.IncMediaInfoPrewarmTriggered() + registry.IncMediaInfoPrewarmReplaced() + registry.IncMediaInfoPrewarmDiscoverySuccess() + registry.IncMediaInfoPrewarmDiscoveryFailure() + registry.IncMediaInfoPrewarmFreshCache() + registry.IncMediaInfoPrewarmJoinedFlight() + registry.IncMediaInfoPrewarmQueued() + registry.IncMediaInfoPrewarmRejected() + registry.IncMediaInfoPrewarmSkipped() registry.ObserveMediaInfoProbeLatency(3 * time.Millisecond) registry.ObserveResolverLatency(time.Millisecond) registry.ObserveRedirectLatency(2 * time.Millisecond) @@ -238,6 +265,10 @@ func TestCountersAreSafeForConcurrentUpdates(t *testing.T) { got.MediaInfoProbeQueuedTotal != want || got.MediaInfoProbeSuccessTotal != want || got.MediaInfoProbeFailureTotal != want || got.MediaInfoStoreFailureTotal != want || got.MediaInfoProbeActive != 0 || got.MediaInfoEnrichedTotal != want || got.MediaInfoFailOpenTotal != want || got.MediaInfoWaitActive != 0 || got.MediaInfoWaitRejectedTotal != want || + got.MediaInfoPrewarmTriggeredTotal != want || got.MediaInfoPrewarmReplacedTotal != want || + got.MediaInfoPrewarmDiscoverySuccessTotal != want || got.MediaInfoPrewarmDiscoveryFailureTotal != want || + got.MediaInfoPrewarmFreshCacheTotal != want || got.MediaInfoPrewarmJoinedFlightTotal != want || + got.MediaInfoPrewarmQueuedTotal != want || got.MediaInfoPrewarmRejectedTotal != want || got.MediaInfoPrewarmSkippedTotal != want || got.ResolverLatencySamples != want || got.ResolverLatencyMSTotal != want || got.RedirectLatencySamples != want || got.RedirectLatencyMSTotal != 2*want || got.MediaInfoProbeLatencySamples != want || got.MediaInfoProbeLatencyMSTotal != 3*want { diff --git a/internal/partcache/partcache.go b/internal/partcache/partcache.go index 7573968..8d1eea1 100644 --- a/internal/partcache/partcache.go +++ b/internal/partcache/partcache.go @@ -12,6 +12,7 @@ import ( // stamp may vary after a library refresh. type PartInfo struct { PartID string + RatingKey string PlexFilePath string PartKey string UpdatedAt time.Time @@ -60,6 +61,11 @@ func (c *Cache) Put(info PartInfo) { c.mu.Lock() defer c.mu.Unlock() c.sweepIfDue(now) + if info.RatingKey == "" { + if current, exists := c.entries[info.PartID]; exists && now.Before(current.expiresAt) { + info.RatingKey = current.info.RatingKey + } + } c.entries[info.PartID] = entry{ info: info, expiresAt: now.Add(c.ttl), diff --git a/internal/partcache/partcache_test.go b/internal/partcache/partcache_test.go index 6d2a6e8..b16e7ae 100644 --- a/internal/partcache/partcache_test.go +++ b/internal/partcache/partcache_test.go @@ -60,6 +60,17 @@ func TestPutSweepsExpiredUnrelatedEntries(t *testing.T) { } } +func TestObservationWithoutRatingKeyDoesNotEraseKnownAssociation(t *testing.T) { + cache := New(time.Minute) + cache.Put(PartInfo{PartID: "123", RatingKey: "42", PlexFilePath: "/cloud/episode.strm"}) + cache.Put(PartInfo{PartID: "123", PlexFilePath: "/cloud/episode.strm", PartKey: "/library/parts/123/file"}) + + got, ok := cache.Get("123") + if !ok || got.RatingKey != "42" || got.PartKey != "/library/parts/123/file" { + t.Fatalf("PartInfo = %#v, found = %v", got, ok) + } +} + func TestEmptyPartIDIsIgnored(t *testing.T) { cache := New(time.Minute) cache.Put(PartInfo{PlexFilePath: "/media/cloud/empty.strm"}) diff --git a/internal/playback/service.go b/internal/playback/service.go index 7358215..026253f 100644 --- a/internal/playback/service.go +++ b/internal/playback/service.go @@ -29,8 +29,9 @@ const ( // PreparedPart binds the Plex identity used for authorization to the exact // control target read from its mapped STRM file. type PreparedPart struct { - Part plexmeta.Part - Target string + Part plexmeta.Part + RatingKey string + Target string } // Preparation is a typed result so HTTP adapters can preserve transparent @@ -104,22 +105,35 @@ type Options struct { // Service prepares, authorizes, and resolves cloud Parts. It never writes an // HTTP response; protocol adapters remain responsible for redirect or fallback. type Service struct { - cache *partcache.Cache + cache *partcache.Cache + resolver resolver.ControlResolver + authorizePart AuthorizePartFunc + preparer *PartPreparer +} + +// PartPreparer owns the pure cloud-file classification, path mapping, and STRM +// target read shared by live playback and speculative analysis. +type PartPreparer struct { mapper *pathmap.Mapper resolver resolver.ControlResolver - authorizePart AuthorizePartFunc cloudExtensions map[string]struct{} } +// NewPartPreparer creates the side-effect-bounded preparation boundary. It +// never contacts Plex, MediaVault, or the media origin. +func NewPartPreparer(mapper *pathmap.Mapper, controlResolver resolver.ControlResolver, cloudExtensions []string) *PartPreparer { + return &PartPreparer{ + mapper: mapper, resolver: controlResolver, + cloudExtensions: normalizeExtensions(cloudExtensions), + } +} + // New creates a playback service. Missing cloud collaborators intentionally // produce PreparationUnavailable so a proxy-only deployment remains valid. func New(options Options) *Service { return &Service{ - cache: options.Cache, - mapper: options.Mapper, - resolver: options.Resolver, - authorizePart: options.AuthorizePart, - cloudExtensions: normalizeExtensions(options.CloudExtensions), + cache: options.Cache, resolver: options.Resolver, authorizePart: options.AuthorizePart, + preparer: NewPartPreparer(options.Mapper, options.Resolver, options.CloudExtensions), } } @@ -133,27 +147,38 @@ func (s *Service) PrepareCached(partID string) Preparation { if !found { return Preparation{State: PreparationMissing, Reason: "cache_miss"} } - return s.Prepare(plexmeta.Part{ID: info.PartID, Key: info.PartKey, File: info.PlexFilePath}) + preparation := s.Prepare(plexmeta.Part{ID: info.PartID, Key: info.PartKey, File: info.PlexFilePath}) + preparation.Part.RatingKey = info.RatingKey + return preparation } // Prepare maps a cloud Part to its local STRM file and reads one validated // control target without contacting Plex, MediaVault, or the media origin. func (s *Service) Prepare(partInfo plexmeta.Part) Preparation { - if !s.enabled() { + if s == nil || s.preparer == nil { + return Preparation{State: PreparationUnavailable, Reason: "cloud_disabled"} + } + return s.preparer.Prepare(partInfo) +} + +// Prepare applies cloud Part rules without playback authorization or direct +// URL resolution. +func (preparer *PartPreparer) Prepare(partInfo plexmeta.Part) Preparation { + if preparer == nil || preparer.mapper == nil || preparer.resolver == nil { return Preparation{State: PreparationUnavailable, Reason: "cloud_disabled"} } prepared := PreparedPart{Part: partInfo} if partInfo.ID == "" || partInfo.File == "" { return Preparation{State: PreparationFailed, Part: prepared, Reason: "invalid_part"} } - if _, ok := s.cloudExtensions[strings.ToLower(path.Ext(partInfo.File))]; !ok { + if _, ok := preparer.cloudExtensions[strings.ToLower(path.Ext(partInfo.File))]; !ok { return Preparation{State: PreparationLocal, Part: prepared} } - localPath, err := s.mapper.Resolve(partInfo.File) + localPath, err := preparer.mapper.Resolve(partInfo.File) if err != nil { return Preparation{State: PreparationFailed, Part: prepared, Reason: "path_mapping", Cloud: true} } - target, err := s.resolver.ReadTarget(localPath) + target, err := preparer.resolver.ReadTarget(localPath) if err != nil { return Preparation{State: PreparationFailed, Part: prepared, Reason: "strm_target", Cloud: true} } @@ -170,6 +195,7 @@ func (s *Service) Remember(part PreparedPart) { } s.cache.Put(partcache.PartInfo{ PartID: partInfo.ID, + RatingKey: part.RatingKey, PlexFilePath: partInfo.File, PartKey: partInfo.Key, UpdatedAt: time.Now().UTC(), @@ -203,7 +229,8 @@ func (s *Service) Play(input PlayInput) (PlayResult, error) { } func (s *Service) enabled() bool { - return s != nil && s.cache != nil && s.mapper != nil && s.resolver != nil && s.authorizePart != nil + return s != nil && s.cache != nil && s.preparer != nil && s.preparer.mapper != nil && + s.preparer.resolver != nil && s.resolver != nil && s.authorizePart != nil } // resolverRequest preserves every client header and promotes query-carried diff --git a/internal/playback/service_test.go b/internal/playback/service_test.go index 7fdf25c..c45b363 100644 --- a/internal/playback/service_test.go +++ b/internal/playback/service_test.go @@ -39,11 +39,12 @@ func TestServicePreparationOwnsCloudClassificationAndTargetRead(t *testing.T) { } cache.Put(partcache.PartInfo{ PartID: "cloud", + RatingKey: "42", PartKey: "/library/parts/cloud/1/file", PlexFilePath: "/media/cloud/Movie.strm", }) got := service.PrepareCached("cloud") - if got.State != PreparationReady || got.Part.Target != control.target { + if got.State != PreparationReady || got.Part.Target != control.target || got.Part.RatingKey != "42" { t.Fatalf("cloud preparation = %#v", got) } if control.readPath != filepath.Join(localRoot, "Movie.strm") { @@ -105,7 +106,7 @@ func TestServicePlayPreservesClientContextAndRefreshesCache(t *testing.T) { control.playback.Header.Get("Range") != "bytes=100-200" { t.Fatalf("resolver request = %#v", control.playback) } - if cached, ok := cache.Get(part.Part.ID); !ok || cached.PartKey != part.Part.Key { + if cached, ok := cache.Get(part.Part.ID); !ok || cached.PartKey != part.Part.Key || cached.RatingKey != part.RatingKey { t.Fatalf("refreshed cache = %#v, found = %v", cached, ok) } } diff --git a/internal/plexmeta/episode.go b/internal/plexmeta/episode.go new file mode 100644 index 0000000..6e1942a --- /dev/null +++ b/internal/plexmeta/episode.go @@ -0,0 +1,320 @@ +package plexmeta + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "strconv" + "strings" +) + +// Episode identifies the Plex hierarchy used for best-effort next-item +// discovery. Unknown season or episode indices use -1. Parts preserves Plex +// order; Part is the first usable Part, while UniquePart reports the stricter +// single-Media/single-Part shape used by metadata enrichment. +type Episode struct { + RatingKey string + ParentRatingKey string + GrandparentRatingKey string + SeasonIndex int + EpisodeIndex int + PlayQueueItemID string + Part Part + Parts []Part + UniquePart bool +} + +// Season identifies one Plex season. Unknown indices use -1 and returned order +// remains available to discovery as a fallback. +type Season struct { + RatingKey string + Index int +} + +// PlayQueueItem preserves queue order and type so discovery never skips over +// an intervening movie, clip, or item from another show. +type PlayQueueItem struct { + Type string + RatingKey string + GrandparentRatingKey string + PlayQueueItemID string +} + +// ParseEpisodes extracts episode hierarchy without rewriting the Plex body. +func ParseEpisodes(body []byte, contentType string) ([]Episode, error) { + trimmed := bytes.TrimSpace(body) + if len(trimmed) == 0 { + return nil, errors.New("Plex episode response is empty") + } + if strings.Contains(strings.ToLower(contentType), "json") || trimmed[0] == '{' || trimmed[0] == '[' { + return parseJSONEpisodes(trimmed) + } + return parseXMLEpisodes(trimmed) +} + +// ParseSeasons extracts season identities used only for bounded cross-season +// discovery. +func ParseSeasons(body []byte, contentType string) ([]Season, error) { + trimmed := bytes.TrimSpace(body) + if len(trimmed) == 0 { + return nil, errors.New("Plex season response is empty") + } + if strings.Contains(strings.ToLower(contentType), "json") || trimmed[0] == '{' || trimmed[0] == '[' { + return parseJSONSeasons(trimmed) + } + return parseXMLSeasons(trimmed) +} + +// ParsePlayQueue extracts the ordered identity fields needed to prove that an +// item is the immediate successor of the active queue entry. +func ParsePlayQueue(body []byte, contentType string) ([]PlayQueueItem, error) { + trimmed := bytes.TrimSpace(body) + if len(trimmed) == 0 { + return nil, errors.New("Plex play queue response is empty") + } + if strings.Contains(strings.ToLower(contentType), "json") || trimmed[0] == '{' || trimmed[0] == '[' { + root, err := parseJSONMediaContainer(trimmed) + if err != nil { + return nil, err + } + items := objectList(root["Metadata"]) + result := make([]PlayQueueItem, 0, len(items)) + for _, item := range items { + typeName, _ := stringValue(item["type"]) + ratingKey, _ := stringValue(item["ratingKey"]) + grandparentRatingKey, _ := stringValue(item["grandparentRatingKey"]) + playQueueItemID, _ := stringValue(item["playQueueItemID"]) + result = append(result, PlayQueueItem{ + Type: typeName, RatingKey: ratingKey, + GrandparentRatingKey: grandparentRatingKey, + PlayQueueItemID: playQueueItemID, + }) + } + return result, nil + } + type item struct { + XMLName xml.Name + Type string `xml:"type,attr"` + RatingKey string `xml:"ratingKey,attr"` + GrandparentRatingKey string `xml:"grandparentRatingKey,attr"` + PlayQueueItemID string `xml:"playQueueItemID,attr"` + } + type container struct { + Items []item `xml:",any"` + } + var value container + if err := xml.Unmarshal(trimmed, &value); err != nil { + return nil, fmt.Errorf("parse Plex XML play queue: %w", err) + } + result := make([]PlayQueueItem, 0, len(value.Items)) + for _, candidate := range value.Items { + if candidate.XMLName.Local == "MediaContainer" { + continue + } + result = append(result, PlayQueueItem{ + Type: candidate.Type, RatingKey: candidate.RatingKey, + GrandparentRatingKey: candidate.GrandparentRatingKey, + PlayQueueItemID: candidate.PlayQueueItemID, + }) + } + return result, nil +} + +func parseXMLEpisodes(body []byte) ([]Episode, error) { + type media struct { + Parts []Part `xml:"Part"` + } + type video struct { + Type string `xml:"type,attr"` + RatingKey string `xml:"ratingKey,attr"` + ParentRatingKey string `xml:"parentRatingKey,attr"` + GrandparentRatingKey string `xml:"grandparentRatingKey,attr"` + ParentIndex *int `xml:"parentIndex,attr"` + Index *int `xml:"index,attr"` + PlayQueueItemID string `xml:"playQueueItemID,attr"` + Media []media `xml:"Media"` + } + type container struct { + Videos []video `xml:"Video"` + } + var value container + if err := xml.Unmarshal(body, &value); err != nil { + return nil, fmt.Errorf("parse Plex XML episodes: %w", err) + } + episodes := make([]Episode, 0, len(value.Videos)) + for _, item := range value.Videos { + if item.Type != "episode" { + continue + } + if item.RatingKey == "" { + continue + } + seasonIndex := -1 + if item.ParentIndex != nil && *item.ParentIndex >= 0 { + seasonIndex = *item.ParentIndex + } + episodeIndex := -1 + if item.Index != nil && *item.Index >= 0 { + episodeIndex = *item.Index + } + episode := Episode{ + RatingKey: item.RatingKey, ParentRatingKey: item.ParentRatingKey, + GrandparentRatingKey: item.GrandparentRatingKey, + SeasonIndex: seasonIndex, EpisodeIndex: episodeIndex, + PlayQueueItemID: item.PlayQueueItemID, + } + for _, mediaItem := range item.Media { + episode.Parts = append(episode.Parts, mediaItem.Parts...) + } + for _, part := range episode.Parts { + if part.ID != "" && part.File != "" { + episode.Part = part + break + } + } + if len(item.Media) == 1 && len(item.Media[0].Parts) == 1 { + episode.UniquePart = item.Media[0].Parts[0].ID != "" && item.Media[0].Parts[0].File != "" + } + episodes = append(episodes, episode) + } + return episodes, nil +} + +func parseJSONEpisodes(body []byte) ([]Episode, error) { + root, err := parseJSONMediaContainer(body) + if err != nil { + return nil, err + } + items := objectList(root["Metadata"]) + episodes := make([]Episode, 0, len(items)) + for _, item := range items { + typeName, _ := stringValue(item["type"]) + ratingKey, _ := stringValue(item["ratingKey"]) + seasonIndex, seasonOK := nonNegativeJSONInt(item["parentIndex"]) + episodeIndex, episodeOK := nonNegativeJSONInt(item["index"]) + if typeName != "episode" { + continue + } + if ratingKey == "" { + continue + } + if !seasonOK { + seasonIndex = -1 + } + if !episodeOK { + episodeIndex = -1 + } + parentRatingKey, _ := stringValue(item["parentRatingKey"]) + grandparentRatingKey, _ := stringValue(item["grandparentRatingKey"]) + playQueueItemID, _ := stringValue(item["playQueueItemID"]) + episode := Episode{ + RatingKey: ratingKey, ParentRatingKey: parentRatingKey, + GrandparentRatingKey: grandparentRatingKey, + SeasonIndex: seasonIndex, EpisodeIndex: episodeIndex, + PlayQueueItemID: playQueueItemID, + } + media := objectList(item["Media"]) + for _, mediaItem := range media { + for _, rawPart := range objectList(mediaItem["Part"]) { + if part, ok := partFromMap(rawPart); ok { + episode.Parts = append(episode.Parts, part) + } + } + } + for _, part := range episode.Parts { + if part.ID != "" && part.File != "" { + episode.Part = part + break + } + } + if len(media) == 1 && len(objectList(media[0]["Part"])) == 1 { + episode.UniquePart = episode.Part.ID != "" && episode.Part.File != "" + } + episodes = append(episodes, episode) + } + return episodes, nil +} + +func parseXMLSeasons(body []byte) ([]Season, error) { + type directory struct { + Type string `xml:"type,attr"` + RatingKey string `xml:"ratingKey,attr"` + Index *int `xml:"index,attr"` + } + type container struct { + Directories []directory `xml:"Directory"` + } + var value container + if err := xml.Unmarshal(body, &value); err != nil { + return nil, fmt.Errorf("parse Plex XML seasons: %w", err) + } + seasons := make([]Season, 0, len(value.Directories)) + for _, item := range value.Directories { + if item.Type != "season" { + continue + } + if item.RatingKey == "" { + continue + } + index := -1 + if item.Index != nil && *item.Index >= 0 { + index = *item.Index + } + seasons = append(seasons, Season{RatingKey: item.RatingKey, Index: index}) + } + return seasons, nil +} + +func parseJSONSeasons(body []byte) ([]Season, error) { + root, err := parseJSONMediaContainer(body) + if err != nil { + return nil, err + } + items := objectList(root["Metadata"]) + seasons := make([]Season, 0, len(items)) + for _, item := range items { + typeName, _ := stringValue(item["type"]) + ratingKey, _ := stringValue(item["ratingKey"]) + index, indexOK := nonNegativeJSONInt(item["index"]) + if typeName != "season" { + continue + } + if ratingKey == "" { + continue + } + if !indexOK { + index = -1 + } + seasons = append(seasons, Season{RatingKey: ratingKey, Index: index}) + } + return seasons, nil +} + +func parseJSONMediaContainer(body []byte) (map[string]any, error) { + var value any + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.UseNumber() + if err := decoder.Decode(&value); err != nil { + return nil, fmt.Errorf("parse Plex JSON hierarchy: %w", err) + } + root, ok := value.(map[string]any) + if !ok { + return nil, errors.New("Plex JSON hierarchy has no object root") + } + container, ok := root["MediaContainer"].(map[string]any) + if !ok { + return nil, errors.New("Plex JSON hierarchy has no MediaContainer") + } + return container, nil +} + +func nonNegativeJSONInt(value any) (int, bool) { + text, ok := stringValue(value) + if !ok { + return 0, false + } + parsed, err := strconv.Atoi(text) + return parsed, err == nil && parsed >= 0 +} diff --git a/internal/plexmeta/episode_test.go b/internal/plexmeta/episode_test.go new file mode 100644 index 0000000..481c2c1 --- /dev/null +++ b/internal/plexmeta/episode_test.go @@ -0,0 +1,103 @@ +package plexmeta + +import "testing" + +func TestParseEpisodesXMLAndJSON(t *testing.T) { + fixtures := []struct { + name string + contentType string + body string + }{ + { + name: "XML", contentType: "application/xml", + body: ``, + }, + { + name: "JSON", contentType: "application/json", + body: `{"MediaContainer":{"Metadata":[{"type":"episode","ratingKey":42,"parentRatingKey":7,"grandparentRatingKey":3,"parentIndex":1,"index":2,"playQueueItemID":99,"Media":[{"Part":[{"id":9,"key":"/library/parts/9/file","file":"/cloud/e02.strm"}]}]}]}}`, + }, + } + for _, fixture := range fixtures { + t.Run(fixture.name, func(t *testing.T) { + episodes, err := ParseEpisodes([]byte(fixture.body), fixture.contentType) + if err != nil { + t.Fatal(err) + } + if len(episodes) != 1 { + t.Fatalf("episodes = %#v", episodes) + } + got := episodes[0] + if got.RatingKey != "42" || got.ParentRatingKey != "7" || got.GrandparentRatingKey != "3" || + got.SeasonIndex != 1 || got.EpisodeIndex != 2 || got.PlayQueueItemID != "99" || + !got.UniquePart || got.Part.ID != "9" || got.Part.File != "/cloud/e02.strm" { + t.Fatalf("episode = %#v", got) + } + }) + } +} + +func TestParseEpisodesRejectsAmbiguousPart(t *testing.T) { + episodes, err := ParseEpisodes([]byte(``), "application/xml") + if err != nil { + t.Fatal(err) + } + if len(episodes) != 1 || episodes[0].UniquePart { + t.Fatalf("episodes = %#v", episodes) + } +} + +func TestParseEpisodesPreservesIncompleteHierarchy(t *testing.T) { + for _, fixture := range []struct { + contentType string + body string + }{ + {"application/xml", ``}, + {"application/json", `{"MediaContainer":{"Metadata":[{"type":"episode","ratingKey":"42","index":2}]}}`}, + } { + episodes, err := ParseEpisodes([]byte(fixture.body), fixture.contentType) + if err != nil { + t.Fatal(err) + } + if len(episodes) != 1 || episodes[0].RatingKey != "42" || + episodes[0].SeasonIndex != -1 || episodes[0].EpisodeIndex != 2 { + t.Fatalf("ParseEpisodes(%s) = %#v", fixture.contentType, episodes) + } + } +} + +func TestParseSeasonsXMLAndJSON(t *testing.T) { + for _, fixture := range []struct { + contentType string + body string + }{ + {"application/xml", ``}, + {"application/json", `{"MediaContainer":{"Metadata":[{"type":"season","ratingKey":7,"index":1},{"type":"show","ratingKey":3,"index":9}]}}`}, + } { + seasons, err := ParseSeasons([]byte(fixture.body), fixture.contentType) + if err != nil { + t.Fatal(err) + } + if len(seasons) != 1 || seasons[0] != (Season{RatingKey: "7", Index: 1}) { + t.Fatalf("seasons = %#v", seasons) + } + } +} + +func TestParsePlayQueuePreservesInterveningTypes(t *testing.T) { + for _, fixture := range []struct { + contentType string + body string + }{ + {"application/xml", ``}, + {"application/json", `{"MediaContainer":{"Metadata":[{"type":"episode","ratingKey":42,"grandparentRatingKey":3,"playQueueItemID":99},{"type":"movie","ratingKey":50,"playQueueItemID":100}]}}`}, + } { + items, err := ParsePlayQueue([]byte(fixture.body), fixture.contentType) + if err != nil { + t.Fatal(err) + } + if len(items) != 2 || items[0].RatingKey != "42" || items[0].PlayQueueItemID != "99" || + items[1].Type != "movie" || items[1].RatingKey != "50" { + t.Fatalf("items = %#v", items) + } + } +} diff --git a/internal/prewarm/discovery.go b/internal/prewarm/discovery.go new file mode 100644 index 0000000..0aaa60f --- /dev/null +++ b/internal/prewarm/discovery.go @@ -0,0 +1,447 @@ +// Package prewarm discovers and schedules nearby speculative MediaInfo records +// after a cloud redirect is ready. +package prewarm + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/InfinityPacer/plex-gateway/internal/plexmeta" +) + +const defaultPlexResponseLimit int64 = 8 << 20 + +var ( + ErrNoCandidates = errors.New("no nearby candidates") + ErrUntrustedCurrent = errors.New("current media identity is ambiguous") + errPlayQueueUnavailable = errors.New("play queue is unavailable") +) + +// PlaybackContext carries the exact authorized Part and STRM target captured +// when a cloud redirect is ready. Target is transient analysis input: it is +// never logged, persisted, or combined with the Plex management credential. +type PlaybackContext struct { + RatingKey string + PartID string + Target string + // WindowKey identifies one client playback window. It should remain stable + // while the client moves between items and must differ across clients. + WindowKey string + PlayQueueID string + PlayQueueItemID string + UserAgent string +} + +// Candidate is one nearby speculative Plex item and its first usable Part. +type Candidate struct { + RatingKey string + Part plexmeta.Part +} + +// Discovery reads only the configured Plex origin with a management token. +// The token is held by the client and never copied into playback events. +type Discovery struct { + baseURL *url.URL + token string + client *http.Client + maxBytes int64 + userAgent string +} + +// DiscoveryOptions defines the isolated Plex management client. +type DiscoveryOptions struct { + BaseURL *url.URL + Token string + Client *http.Client + MaxBytes int64 + UserAgent string +} + +// NewDiscovery rejects credentials that cannot safely be represented in an +// HTTP header and pins every request to one configured Plex origin. +func NewDiscovery(options DiscoveryOptions) (*Discovery, error) { + if options.BaseURL == nil || (options.BaseURL.Scheme != "http" && options.BaseURL.Scheme != "https") || + options.BaseURL.Host == "" || options.BaseURL.User != nil { + return nil, errors.New("Plex management origin is invalid") + } + token := strings.TrimSpace(options.Token) + if token == "" || strings.ContainsAny(token, "\r\n") { + return nil, errors.New("Plex management token is invalid") + } + defaultTransport := http.DefaultTransport.(*http.Transport).Clone() + defaultTransport.Proxy = nil + transport := http.RoundTripper(defaultTransport) + timeout := 15 * time.Second + if options.Client != nil { + if options.Client.Transport != nil { + transport = options.Client.Transport + } + if options.Client.Timeout > 0 { + timeout = options.Client.Timeout + } + } + client := &http.Client{ + Transport: transport, + Timeout: timeout, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + } + maxBytes := options.MaxBytes + if maxBytes <= 0 { + maxBytes = defaultPlexResponseLimit + } + userAgent := strings.TrimSpace(options.UserAgent) + if strings.ContainsAny(userAgent, "\r\n") { + return nil, errors.New("Plex management User-Agent is invalid") + } + baseURL := *options.BaseURL + baseURL.RawQuery = "" + baseURL.Fragment = "" + baseURL.User = nil + return &Discovery{baseURL: &baseURL, token: token, client: client, maxBytes: maxBytes, userAgent: userAgent}, nil +} + +// Validate confirms that the management token can read an authenticated Plex +// library endpoint. Failure disables prewarming without affecting proxying. +func (discovery *Discovery) Validate(ctx context.Context) error { + _, _, err := discovery.get(ctx, "/library/sections", url.Values{ + "X-Plex-Container-Start": {"0"}, + "X-Plex-Container-Size": {"1"}, + }) + return err +} + +// Neighbors verifies that the current metadata contains the active Part, then +// prefers the explicit play-queue order and falls back to the episode hierarchy. +// Returned candidates are ordered with following items first because continuous +// playback is more likely than backward navigation. +func (discovery *Discovery) Neighbors(ctx context.Context, current PlaybackContext, before, after int) ([]Candidate, error) { + if !numericIdentity(current.RatingKey) || !numericIdentity(current.PartID) { + return nil, ErrUntrustedCurrent + } + if before < 0 || after < 0 || before+after == 0 { + return nil, ErrNoCandidates + } + body, contentType, err := discovery.get(ctx, "/library/metadata/"+current.RatingKey, nil) + if err != nil || !partsContainID(body, contentType, current.PartID) { + return nil, ErrUntrustedCurrent + } + currentEpisode, hasEpisode := exactEpisode(body, contentType, current.RatingKey) + if numericIdentity(current.PlayQueueID) && numericIdentity(current.PlayQueueItemID) { + candidates, queueErr := discovery.neighborsFromPlayQueue(ctx, current, before, after) + if queueErr == nil && len(candidates) > 0 { + return candidates, nil + } + } + if !hasEpisode || currentEpisode.ParentRatingKey == "" || currentEpisode.GrandparentRatingKey == "" { + return nil, ErrNoCandidates + } + return discovery.neighborsFromLibrary(ctx, currentEpisode, before, after) +} + +func (discovery *Discovery) neighborsFromPlayQueue(ctx context.Context, playback PlaybackContext, before, after int) ([]Candidate, error) { + body, contentType, err := discovery.get(ctx, "/playQueues/"+playback.PlayQueueID, nil) + if err != nil { + return nil, err + } + items, err := plexmeta.ParsePlayQueue(body, contentType) + if err != nil { + return nil, err + } + position := -1 + for index, item := range items { + if item.PlayQueueItemID != playback.PlayQueueItemID { + continue + } + if position >= 0 || item.RatingKey != playback.RatingKey { + return nil, errPlayQueueUnavailable + } + position = index + } + if position < 0 { + return nil, errPlayQueueUnavailable + } + ratingKeys := make([]string, 0, before+after) + for index := position + 1; index < len(items) && len(ratingKeys) < after; index++ { + if numericIdentity(items[index].RatingKey) { + ratingKeys = append(ratingKeys, items[index].RatingKey) + } + } + previous := 0 + for index := position - 1; index >= 0 && previous < before; index-- { + if numericIdentity(items[index].RatingKey) { + ratingKeys = append(ratingKeys, items[index].RatingKey) + previous++ + } + } + return discovery.confirmCandidates(ctx, ratingKeys) +} + +func (discovery *Discovery) neighborsFromLibrary(ctx context.Context, current plexmeta.Episode, before, after int) ([]Candidate, error) { + body, contentType, err := discovery.get(ctx, "/library/metadata/"+current.ParentRatingKey+"/children", nil) + if err != nil { + return nil, err + } + episodes, err := plexmeta.ParseEpisodes(body, contentType) + if err != nil { + return nil, err + } + previousRefs, nextRefs, positioned := splitEpisodeNeighbors(episodes, current) + if !positioned { + return nil, ErrNoCandidates + } + previousRefs = takeFirst(previousRefs, before) + nextRefs = takeFirst(nextRefs, after) + if len(previousRefs) < before || len(nextRefs) < after { + body, contentType, err = discovery.get(ctx, "/library/metadata/"+current.GrandparentRatingKey+"/children", nil) + if err == nil { + seasons, parseErr := plexmeta.ParseSeasons(body, contentType) + if parseErr == nil { + previousRefs, nextRefs = discovery.extendAcrossSeasons( + ctx, seasons, current, previousRefs, nextRefs, before, after, + ) + } + } + } + ratingKeys := make([]string, 0, len(previousRefs)+len(nextRefs)) + ratingKeys = append(ratingKeys, nextRefs...) + ratingKeys = append(ratingKeys, previousRefs...) + return discovery.confirmCandidates(ctx, ratingKeys) +} + +func (discovery *Discovery) extendAcrossSeasons( + ctx context.Context, + seasons []plexmeta.Season, + current plexmeta.Episode, + previousRefs, nextRefs []string, + before, after int, +) ([]string, []string) { + position := seasonPosition(seasons, current.ParentRatingKey, current.SeasonIndex) + if position < 0 { + return previousRefs, nextRefs + } + for index := position + 1; index < len(seasons) && len(nextRefs) < after; index++ { + if ctx.Err() != nil { + break + } + episodes, err := discovery.readSeasonEpisodes(ctx, seasons[index].RatingKey) + if err != nil { + continue + } + for _, episode := range episodes { + if usableEpisodeReference(episode) { + nextRefs = append(nextRefs, episode.RatingKey) + if len(nextRefs) == after { + break + } + } + } + } + for index := position - 1; index >= 0 && len(previousRefs) < before; index-- { + if ctx.Err() != nil { + break + } + episodes, err := discovery.readSeasonEpisodes(ctx, seasons[index].RatingKey) + if err != nil { + continue + } + for episodeIndex := len(episodes) - 1; episodeIndex >= 0; episodeIndex-- { + if usableEpisodeReference(episodes[episodeIndex]) { + previousRefs = append(previousRefs, episodes[episodeIndex].RatingKey) + if len(previousRefs) == before { + break + } + } + } + } + return previousRefs, nextRefs +} + +func (discovery *Discovery) readSeasonEpisodes(ctx context.Context, ratingKey string) ([]plexmeta.Episode, error) { + if !numericIdentity(ratingKey) { + return nil, ErrNoCandidates + } + body, contentType, err := discovery.get(ctx, "/library/metadata/"+ratingKey+"/children", nil) + if err != nil { + return nil, err + } + return plexmeta.ParseEpisodes(body, contentType) +} + +func (discovery *Discovery) confirmCandidates(ctx context.Context, ratingKeys []string) ([]Candidate, error) { + candidates := make([]Candidate, 0, len(ratingKeys)) + seen := make(map[string]struct{}, len(ratingKeys)) + for _, ratingKey := range ratingKeys { + if ctx.Err() != nil { + break + } + if _, exists := seen[ratingKey]; exists || !numericIdentity(ratingKey) { + continue + } + seen[ratingKey] = struct{}{} + candidate, err := discovery.confirmCandidate(ctx, ratingKey) + if err == nil { + candidates = append(candidates, candidate) + } + } + if len(candidates) == 0 { + return nil, ErrNoCandidates + } + return candidates, nil +} + +func (discovery *Discovery) confirmCandidate(ctx context.Context, ratingKey string) (Candidate, error) { + body, contentType, err := discovery.get(ctx, "/library/metadata/"+ratingKey, nil) + if err != nil { + return Candidate{}, ErrNoCandidates + } + parts, err := plexmeta.ParseParts(body, contentType) + if err != nil { + return Candidate{}, ErrNoCandidates + } + for _, part := range parts { + if part.ID != "" && part.File != "" { + return Candidate{RatingKey: ratingKey, Part: part}, nil + } + } + return Candidate{}, ErrNoCandidates +} + +func exactEpisode(body []byte, contentType, ratingKey string) (plexmeta.Episode, bool) { + episodes, err := plexmeta.ParseEpisodes(body, contentType) + if err != nil || len(episodes) != 1 || episodes[0].RatingKey != ratingKey { + return plexmeta.Episode{}, false + } + return episodes[0], true +} + +func (discovery *Discovery) get(ctx context.Context, endpointPath string, query url.Values) ([]byte, string, error) { + if discovery == nil || discovery.baseURL == nil || discovery.client == nil { + return nil, "", errors.New("Plex management client is unavailable") + } + target := *discovery.baseURL + target.Path = strings.TrimRight(target.Path, "/") + endpointPath + target.RawPath = "" + target.RawQuery = query.Encode() + request, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil) + if err != nil { + return nil, "", errors.New("create Plex management request") + } + request.Header.Set("X-Plex-Token", discovery.token) + request.Header.Set("Accept", "application/json") + request.Header.Set("Accept-Encoding", "identity") + if discovery.userAgent != "" { + request.Header.Set("User-Agent", discovery.userAgent) + } + request.Host = discovery.baseURL.Host + response, err := discovery.client.Do(request) + if err != nil { + return nil, "", errors.New("Plex management request failed") + } + defer response.Body.Close() + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + return nil, "", fmt.Errorf("Plex management request returned HTTP %d", response.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(response.Body, discovery.maxBytes+1)) + if err != nil || int64(len(body)) > discovery.maxBytes { + return nil, "", errors.New("Plex management response exceeds the limit") + } + return body, response.Header.Get("Content-Type"), nil +} + +func splitEpisodeNeighbors(episodes []plexmeta.Episode, current plexmeta.Episode) ([]string, []string, bool) { + currentPosition := -1 + for index, episode := range episodes { + if episode.RatingKey == current.RatingKey { + currentPosition = index + break + } + } + if currentPosition < 0 && current.EpisodeIndex >= 0 { + for index, episode := range episodes { + if episode.EpisodeIndex >= current.EpisodeIndex { + currentPosition = index + break + } + } + } + if currentPosition < 0 { + return nil, nil, false + } + previous := make([]string, 0, currentPosition) + for index := currentPosition - 1; index >= 0; index-- { + if usableEpisodeReference(episodes[index]) { + previous = append(previous, episodes[index].RatingKey) + } + } + next := make([]string, 0, len(episodes)-currentPosition-1) + start := currentPosition + 1 + if episodes[currentPosition].RatingKey != current.RatingKey { + start = currentPosition + } + for index := start; index < len(episodes); index++ { + if usableEpisodeReference(episodes[index]) { + next = append(next, episodes[index].RatingKey) + } + } + return previous, next, true +} + +func seasonPosition(seasons []plexmeta.Season, ratingKey string, index int) int { + for position, season := range seasons { + if season.RatingKey == ratingKey { + return position + } + } + if index >= 0 { + for position, season := range seasons { + if season.Index == index { + return position + } + } + } + return -1 +} + +func takeFirst(values []string, limit int) []string { + if limit <= 0 { + return nil + } + if len(values) <= limit { + return values + } + return values[:limit] +} + +func usableEpisodeReference(episode plexmeta.Episode) bool { + return numericIdentity(episode.RatingKey) +} + +func partsContainID(body []byte, contentType, partID string) bool { + parts, err := plexmeta.ParseParts(body, contentType) + if err != nil { + return false + } + for _, part := range parts { + if part.ID == partID { + return true + } + } + return false +} + +func numericIdentity(value string) bool { + if value == "" { + return false + } + parsed, err := strconv.ParseUint(value, 10, 64) + return err == nil && parsed > 0 +} diff --git a/internal/prewarm/discovery_test.go b/internal/prewarm/discovery_test.go new file mode 100644 index 0000000..ae66ea5 --- /dev/null +++ b/internal/prewarm/discovery_test.go @@ -0,0 +1,230 @@ +package prewarm + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strconv" + "testing" + "time" +) + +func TestDiscoveryUsesBidirectionalPlayQueueAndKeepsTokenAtPlex(t *testing.T) { + var paths []string + plex := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("X-Plex-Token"); got != "management-secret" { + t.Fatalf("token = %q", got) + } + if r.URL.Query().Get("X-Plex-Token") != "" { + t.Fatal("token leaked into query") + } + paths = append(paths, r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/library/metadata/42": + _, _ = io.WriteString(w, episodeJSON("42", "7", "3", 1, 2, "9", "/cloud/e02.strm")) + case "/playQueues/5": + _, _ = io.WriteString(w, `{"MediaContainer":{"Metadata":[`+ + `{"type":"movie","ratingKey":"40","playQueueItemID":"98"},`+ + `{"type":"episode","ratingKey":"42","playQueueItemID":"99"},`+ + `{"type":"movie","ratingKey":"70","playQueueItemID":"100"},`+ + `{"type":"episode","ratingKey":"44","playQueueItemID":"101"}]}}`) + case "/library/metadata/40": + _, _ = io.WriteString(w, itemJSON("8", "/cloud/previous.strm")) + case "/library/metadata/70": + _, _ = io.WriteString(w, itemJSON("11", "/cloud/movie.strm")) + case "/library/metadata/44": + _, _ = io.WriteString(w, itemJSON("12", "/cloud/e04.strm")) + default: + http.NotFound(w, r) + } + })) + defer plex.Close() + + discovery := newTestDiscovery(t, plex.URL) + candidates, err := discovery.Neighbors(t.Context(), PlaybackContext{ + RatingKey: "42", PartID: "9", PlayQueueID: "5", PlayQueueItemID: "99", + }, 2, 3) + if err != nil { + t.Fatal(err) + } + if got := candidateIDs(candidates); !reflect.DeepEqual(got, []string{"11", "12", "8"}) { + t.Fatalf("candidate parts = %v", got) + } + wantPaths := []string{ + "/library/metadata/42", "/playQueues/5", "/library/metadata/70", + "/library/metadata/44", "/library/metadata/40", + } + if !reflect.DeepEqual(paths, wantPaths) { + t.Fatalf("paths = %v, want %v", paths, wantPaths) + } +} + +func TestDiscoveryBuildsLibraryWindowAcrossSpecialAndRegularSeasons(t *testing.T) { + var paths []string + plex := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/library/metadata/42": + _, _ = io.WriteString(w, episodeJSON("42", "7", "3", 1, 2, "9", "/cloud/e02.strm")) + case "/library/metadata/7/children": + _, _ = io.WriteString(w, episodeListJSON( + episodeRefJSON("41", "7", "3", 1, 1), + episodeRefJSON("42", "7", "3", 1, 2), + episodeRefJSON("43", "7", "3", 1, 3), + )) + case "/library/metadata/3/children": + _, _ = io.WriteString(w, `{"MediaContainer":{"Metadata":[`+ + `{"type":"season","ratingKey":"6","index":0},`+ + `{"type":"season","ratingKey":"7","index":1},`+ + `{"type":"season","ratingKey":"8","index":2}]}}`) + case "/library/metadata/6/children": + _, _ = io.WriteString(w, episodeListJSON(episodeRefJSON("5", "6", "3", 0, 1))) + case "/library/metadata/8/children": + _, _ = io.WriteString(w, episodeListJSON( + episodeRefJSON("50", "8", "3", 2, 1), + episodeRefJSON("51", "8", "3", 2, 2), + )) + case "/library/metadata/5": + _, _ = io.WriteString(w, itemJSON("5", "/cloud/special.strm")) + case "/library/metadata/41": + _, _ = io.WriteString(w, itemJSON("8", "/cloud/e01.strm")) + case "/library/metadata/43": + _, _ = io.WriteString(w, itemJSON("10", "/cloud/e03.strm")) + case "/library/metadata/50": + _, _ = io.WriteString(w, itemJSON("12", "/cloud/s02e01.strm")) + case "/library/metadata/51": + _, _ = io.WriteString(w, itemJSON("13", "/cloud/s02e02.strm")) + default: + http.NotFound(w, r) + } + })) + defer plex.Close() + + discovery := newTestDiscovery(t, plex.URL) + candidates, err := discovery.Neighbors(t.Context(), PlaybackContext{RatingKey: "42", PartID: "9"}, 2, 3) + if err != nil { + t.Fatal(err) + } + if got := candidateIDs(candidates); !reflect.DeepEqual(got, []string{"10", "12", "13", "8", "5"}) { + t.Fatalf("candidate parts = %v", got) + } + if len(paths) != 10 { + t.Fatalf("paths = %v", paths) + } +} + +func TestDiscoveryAcceptsActivePartAmongMultipleAndChoosesFirstCandidatePart(t *testing.T) { + plex := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/library/metadata/42": + _, _ = io.WriteString(w, `{"MediaContainer":{"Metadata":[{"type":"episode","ratingKey":"42","parentRatingKey":"7","grandparentRatingKey":"3","parentIndex":1,"index":2,"Media":[{"Part":[{"id":"8","file":"/a.strm"},{"id":"9","file":"/b.strm"}]}]}]}}`) + case "/playQueues/5": + _, _ = io.WriteString(w, `{"MediaContainer":{"Metadata":[{"ratingKey":"42","playQueueItemID":"99"},{"ratingKey":"44","playQueueItemID":"100"}]}}`) + case "/library/metadata/44": + _, _ = io.WriteString(w, `{"MediaContainer":{"Metadata":[{"ratingKey":"44","Media":[{"Part":[{"id":"11","file":"/first.strm"},{"id":"12","file":"/second.strm"}]}]}]}}`) + default: + http.NotFound(w, r) + } + })) + defer plex.Close() + discovery := newTestDiscovery(t, plex.URL) + candidates, err := discovery.Neighbors(t.Context(), PlaybackContext{ + RatingKey: "42", PartID: "9", PlayQueueID: "5", PlayQueueItemID: "99", + }, 0, 1) + if err != nil { + t.Fatal(err) + } + if len(candidates) != 1 || candidates[0].Part.ID != "11" { + t.Fatalf("candidates = %#v", candidates) + } +} + +func TestDiscoveryRejectsCurrentMetadataWithoutActivePart(t *testing.T) { + plex := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, itemJSON("8", "/cloud/other.strm")) + })) + defer plex.Close() + discovery := newTestDiscovery(t, plex.URL) + _, err := discovery.Neighbors(t.Context(), PlaybackContext{RatingKey: "42", PartID: "9"}, 2, 3) + if err != ErrUntrustedCurrent { + t.Fatalf("error = %v", err) + } +} + +func TestDiscoveryDoesNotFollowRedirectWithManagementToken(t *testing.T) { + received := make(chan http.Header, 1) + target := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + received <- r.Header.Clone() + })) + defer target.Close() + plex := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Location", target.URL) + w.WriteHeader(http.StatusFound) + })) + defer plex.Close() + discovery := newTestDiscovery(t, plex.URL) + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + if err := discovery.Validate(ctx); err == nil { + t.Fatal("Validate() accepted redirect") + } + select { + case header := <-received: + t.Fatalf("redirect target received headers %#v", header) + default: + } +} + +func newTestDiscovery(t *testing.T, rawURL string) *Discovery { + t.Helper() + baseURL, err := url.Parse(rawURL) + if err != nil { + t.Fatal(err) + } + discovery, err := NewDiscovery(DiscoveryOptions{ + BaseURL: baseURL, Token: "management-secret", Client: &http.Client{Timeout: time.Second}, + }) + if err != nil { + t.Fatal(err) + } + return discovery +} + +func candidateIDs(candidates []Candidate) []string { + result := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + result = append(result, candidate.Part.ID) + } + return result +} + +func itemJSON(partID, file string) string { + return `{"MediaContainer":{"Metadata":[{"Media":[{"Part":[{"id":"` + partID + `","file":"` + file + `"}]}]}]}}` +} + +func episodeListJSON(items ...string) string { + result := `{"MediaContainer":{"Metadata":[` + for index, item := range items { + if index > 0 { + result += "," + } + result += item + } + return result + `]}}` +} + +func episodeRefJSON(ratingKey, parentRatingKey, grandparentRatingKey string, season, episode int) string { + return `{"type":"episode","ratingKey":"` + ratingKey + `","parentRatingKey":"` + parentRatingKey + `","grandparentRatingKey":"` + grandparentRatingKey + `","parentIndex":` + strconv.Itoa(season) + `,"index":` + strconv.Itoa(episode) + `}` +} + +func episodeJSON(ratingKey, parentRatingKey, grandparentRatingKey string, season, episode int, partID, file string) string { + return `{"MediaContainer":{"Metadata":[{"type":"episode","ratingKey":"` + ratingKey + `","parentRatingKey":"` + parentRatingKey + `","grandparentRatingKey":"` + grandparentRatingKey + `","parentIndex":` + + strconv.Itoa(season) + `,"index":` + strconv.Itoa(episode) + `,"Media":[{"Part":[{"id":"` + partID + `","key":"/library/parts/` + partID + `/file","file":"` + file + `"}]}]}]}}` +} diff --git a/internal/prewarm/service.go b/internal/prewarm/service.go new file mode 100644 index 0000000..5a8ab87 --- /dev/null +++ b/internal/prewarm/service.go @@ -0,0 +1,593 @@ +package prewarm + +import ( + "context" + "crypto/sha256" + "errors" + "log/slog" + "strconv" + "strings" + "sync" + "time" + + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" + "github.com/InfinityPacer/plex-gateway/internal/playback" + "github.com/InfinityPacer/plex-gateway/internal/plexmeta" +) + +const ( + defaultDiscoveryTimeout = 15 * time.Second + defaultTriggerCooldown = 5 * time.Minute + defaultSubmitInterval = 5 * time.Second + defaultRecentLimit = 4096 + defaultCurrentQueueSize = 64 + defaultWindowGroupLimit = 8 +) + +type neighborDiscovery interface { + Neighbors(context.Context, PlaybackContext, int, int) ([]Candidate, error) +} + +type partPreparer interface { + Prepare(plexmeta.Part) playback.Preparation +} + +type mediaInfoSubmitter interface { + SubmitDetailed(mediainfo.Request) mediainfo.SubmitResult +} + +// Metrics receives fixed-cardinality prewarm lifecycle events. +type Metrics interface { + IncMediaInfoPrewarmTriggered() + IncMediaInfoPrewarmReplaced() + IncMediaInfoPrewarmDiscoverySuccess() + IncMediaInfoPrewarmDiscoveryFailure() + IncMediaInfoPrewarmFreshCache() + IncMediaInfoPrewarmJoinedFlight() + IncMediaInfoPrewarmQueued() + IncMediaInfoPrewarmRejected() + IncMediaInfoPrewarmSkipped() +} + +// ServiceOptions binds the discovery, STRM preparation, and MediaInfo queues. +// TryEnqueue only changes bounded in-memory state; all collaborators run on a +// worker after the redirect response has been written. +type ServiceOptions struct { + Discovery neighborDiscovery + Playback partPreparer + MediaInfo mediaInfoSubmitter + Logger *slog.Logger + Metrics Metrics + DiscoveryTimeout time.Duration + TriggerCooldown time.Duration + BeforeCount int + AfterCount int + SubmitInterval time.Duration + CurrentQueueSize int + Now func() time.Time +} + +type currentJob struct { + playback PlaybackContext + key string +} + +// windowState is the latest-wins queue for one client/session window. The +// state is protected by Service.mu; its worker is counted in Service.workers. +type windowState struct { + key string + pending *PlaybackContext + pendingKey string + activeKey string + activeCancel context.CancelFunc + wake chan struct{} +} + +// Service schedules immediate current-item analysis independently from +// speculative nearby-item discovery. Current events are FIFO and durable only +// in memory; each nearby window is latest-wins within its own bounded group. +type Service struct { + discovery neighborDiscovery + playback partPreparer + mediaInfo mediaInfoSubmitter + logger *slog.Logger + metrics Metrics + discoveryTimeout time.Duration + cooldown time.Duration + beforeCount int + afterCount int + submitInterval time.Duration + now func() time.Time + + ctx context.Context + cancel context.CancelFunc + done chan struct{} + workers sync.WaitGroup + + currentQueue chan currentJob + + mu sync.Mutex + currentQueued int + currentActive int + currentRecent map[string]time.Time + windowRecent map[string]time.Time + windows map[string]*windowState + windowSeq uint64 + closed bool +} + +// NewService starts the bounded current-item worker. Nearby workers are +// created only for admitted session windows and never exceed the fixed group +// limit. +func NewService(options ServiceOptions) (*Service, error) { + if options.Playback == nil || options.MediaInfo == nil { + return nil, errors.New("prewarm dependencies are unavailable") + } + logger := options.Logger + if logger == nil { + logger = slog.Default() + } + discoveryTimeout := options.DiscoveryTimeout + if discoveryTimeout <= 0 { + discoveryTimeout = defaultDiscoveryTimeout + } + cooldown := options.TriggerCooldown + if cooldown <= 0 { + cooldown = defaultTriggerCooldown + } + beforeCount := options.BeforeCount + if beforeCount < 0 { + return nil, errors.New("prewarm before count is invalid") + } + afterCount := options.AfterCount + if afterCount < 0 { + return nil, errors.New("prewarm after count is invalid") + } + submitInterval := options.SubmitInterval + if submitInterval <= 0 { + submitInterval = defaultSubmitInterval + } + queueSize := options.CurrentQueueSize + if queueSize < 0 { + return nil, errors.New("prewarm current queue size is invalid") + } + if queueSize == 0 { + queueSize = defaultCurrentQueueSize + } + now := options.Now + if now == nil { + now = time.Now + } + ctx, cancel := context.WithCancel(context.Background()) + service := &Service{ + discovery: options.Discovery, playback: options.Playback, mediaInfo: options.MediaInfo, + logger: logger, metrics: options.Metrics, discoveryTimeout: discoveryTimeout, + cooldown: cooldown, beforeCount: beforeCount, afterCount: afterCount, + submitInterval: submitInterval, now: now, ctx: ctx, cancel: cancel, + done: make(chan struct{}), currentQueue: make(chan currentJob, queueSize), + currentRecent: make(map[string]time.Time), windowRecent: make(map[string]time.Time), + windows: make(map[string]*windowState), + } + service.workers.Add(1) + go service.currentWorker() + go func() { + service.workers.Wait() + close(service.done) + }() + return service, nil +} + +// TryEnqueue performs validation and bounded memory operations only. It never +// contacts Plex, reads a STRM file, invokes MediaInfo, or opens SQLite/CDN. +// The current item is admitted independently of RatingKey and of nearby +// discovery, so an incomplete Plex metadata response cannot lose playback. +func (service *Service) TryEnqueue(playbackContext PlaybackContext) bool { + if service == nil || !validPlaybackContext(playbackContext) { + return false + } + now := service.now() + currentKey := currentContextKey(playbackContext) + var startWindow *windowState + currentAccepted := false + windowAccepted := false + replaced := false + skipped := false + + service.mu.Lock() + if service.closed { + service.mu.Unlock() + return false + } + service.expireRecentLocked(service.currentRecent, now) + service.expireRecentLocked(service.windowRecent, now) + + if until, exists := service.currentRecent[currentKey]; exists && now.Before(until) { + skipped = true + } else { + job := currentJob{playback: playbackContext, key: currentKey} + select { + case service.currentQueue <- job: + service.currentQueued++ + service.rememberRecentLocked(service.currentRecent, currentKey, now) + currentAccepted = true + default: + // The queue is deliberately non-blocking so a redirect cannot wait on + // a slow MediaInfo backend. The current request remains fail-open. + skipped = true + } + } + + if service.discovery != nil && numericIdentity(playbackContext.RatingKey) && service.beforeCount+service.afterCount > 0 { + groupKey := service.windowGroupKeyLocked(playbackContext) + eventKey := groupKey + "\x00" + playbackContextKey(playbackContext) + if until, exists := service.windowRecent[eventKey]; exists && now.Before(until) { + skipped = true + } else if window := service.windows[groupKey]; window != nil { + if window.activeKey == eventKey || window.pendingKey == eventKey { + skipped = true + } else { + if window.pending != nil || window.activeCancel != nil { + replaced = true + } + copyOfContext := playbackContext + window.pending = ©OfContext + window.pendingKey = eventKey + service.rememberRecentLocked(service.windowRecent, eventKey, now) + if window.activeCancel != nil { + window.activeCancel() + } + select { + case window.wake <- struct{}{}: + default: + } + windowAccepted = true + } + } else if len(service.windows) >= defaultWindowGroupLimit { + skipped = true + } else { + copyOfContext := playbackContext + window := &windowState{ + key: groupKey, pending: ©OfContext, pendingKey: eventKey, + wake: make(chan struct{}, 1), + } + service.windows[groupKey] = window + service.rememberRecentLocked(service.windowRecent, eventKey, now) + service.workers.Add(1) + startWindow = window + windowAccepted = true + } + } + service.mu.Unlock() + + if startWindow != nil { + go service.windowWorker(startWindow) + } + if currentAccepted || windowAccepted { + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmTriggered() }) + } + if replaced { + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmReplaced() }) + } + if skipped { + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmSkipped() }) + } + return currentAccepted || windowAccepted +} + +func (service *Service) currentWorker() { + defer service.workers.Done() + for { + select { + case <-service.ctx.Done(): + return + case job := <-service.currentQueue: + service.mu.Lock() + if service.currentQueued > 0 { + service.currentQueued-- + } + if service.closed { + service.mu.Unlock() + return + } + service.currentActive++ + service.mu.Unlock() + + submitted := service.submit(service.ctx, Candidate{ + RatingKey: job.playback.RatingKey, + Part: plexmeta.Part{ID: job.playback.PartID}, + }, job.playback.Target, job.playback.UserAgent, mediainfo.PriorityInteractive, "current") + + service.mu.Lock() + if !submitted { + delete(service.currentRecent, job.key) + } + if service.currentActive > 0 { + service.currentActive-- + } + service.mu.Unlock() + } + } +} + +func (service *Service) windowWorker(window *windowState) { + defer service.workers.Done() + for { + service.mu.Lock() + if service.closed || service.windows[window.key] != window { + service.mu.Unlock() + return + } + if window.pending != nil { + current := *window.pending + eventKey := window.pendingKey + window.pending = nil + window.pendingKey = "" + jobContext, cancel := context.WithCancel(service.ctx) + window.activeKey = eventKey + window.activeCancel = cancel + service.mu.Unlock() + + service.runWindow(jobContext, current) + cancel() + + service.mu.Lock() + if window.activeKey == eventKey { + window.activeKey = "" + window.activeCancel = nil + } + closed := service.closed + service.mu.Unlock() + if closed { + return + } + continue + } + if service.windows[window.key] == window && window.pending == nil && window.activeCancel == nil { + delete(service.windows, window.key) + } + service.mu.Unlock() + return + } +} + +func (service *Service) runWindow(ctx context.Context, current PlaybackContext) { + if service.beforeCount+service.afterCount == 0 || ctx.Err() != nil { + return + } + discoveryContext, cancel := context.WithTimeout(ctx, service.discoveryTimeout) + candidates, err := service.discovery.Neighbors( + discoveryContext, current, service.beforeCount, service.afterCount, + ) + cancel() + if err != nil { + if ctx.Err() == nil { + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmDiscoveryFailure() }) + service.logger.Info("mediainfo_prewarm_skipped", "part", current.PartID, "reason", discoveryErrorKind(err)) + } + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmSkipped() }) + return + } + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmDiscoverySuccess() }) + for _, candidate := range candidates { + if !waitForInterval(ctx, service.submitInterval) { + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmSkipped() }) + return + } + if ctx.Err() != nil { + return + } + preparation := service.playback.Prepare(candidate.Part) + if ctx.Err() != nil { + return + } + if preparation.State != playback.PreparationReady { + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmSkipped() }) + continue + } + service.submit( + ctx, candidate, preparation.Part.Target, current.UserAgent, + mediainfo.PriorityBackground, "neighbor", + ) + } +} + +func (service *Service) submit( + ctx context.Context, + candidate Candidate, + target, userAgent string, + priority mediainfo.Priority, + kind string, +) bool { + if ctx.Err() != nil { + return false + } + fingerprint, err := mediainfo.FingerprintSTRMTarget(target) + if err != nil { + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmSkipped() }) + return false + } + result := service.mediaInfo.SubmitDetailed(mediainfo.Request{ + Key: mediainfo.Key{PartID: candidate.Part.ID, STRMFingerprint: fingerprint}, + RatingKey: candidate.RatingKey, Target: target, + Priority: priority, ClientUserAgent: userAgent, + }) + switch result.Disposition { + case mediainfo.SubmitFreshCache: + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmFreshCache() }) + case mediainfo.SubmitJoinedExistingFlight: + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmJoinedFlight() }) + case mediainfo.SubmitNewlyQueued: + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmQueued() }) + service.logger.Info("mediainfo_prewarm_queued", "part", candidate.Part.ID, "kind", kind) + case mediainfo.SubmitRejected: + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmRejected() }) + return false + default: + service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmRejected() }) + return false + } + return true +} + +func waitForInterval(ctx context.Context, interval time.Duration) bool { + timer := time.NewTimer(interval) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +// Status is a credential-free coordinator snapshot. +type Status struct { + Available bool `json:"available"` + NeighborAvailable bool `json:"neighbor_available"` + Active bool `json:"active"` + Queued int `json:"queued"` + Before int `json:"before"` + After int `json:"after"` + IntervalMS int64 `json:"interval_ms"` +} + +// Status reports bounded current and window queue state without media +// identity. A window remains counted only while its worker is alive. +func (service *Service) Status() Status { + if service == nil { + return Status{} + } + service.mu.Lock() + defer service.mu.Unlock() + queued := service.currentQueued + active := service.currentActive > 0 + for _, window := range service.windows { + if window.pending != nil { + queued++ + } + if window.activeCancel != nil { + active = true + } + } + return Status{ + Available: !service.closed, NeighborAvailable: !service.closed && service.discovery != nil, + Active: active, Queued: queued, + Before: service.beforeCount, After: service.afterCount, + IntervalMS: service.submitInterval.Milliseconds(), + } +} + +// Close cancels current and nearby workers and waits for every worker that was +// admitted before the close. The MediaInfo service owns already submitted +// jobs; this coordinator never closes that collaborator. +func (service *Service) Close(ctx context.Context) error { + if service == nil { + return nil + } + service.mu.Lock() + if !service.closed { + service.closed = true + for _, window := range service.windows { + window.pending = nil + window.pendingKey = "" + if window.activeCancel != nil { + window.activeCancel() + } + } + service.cancel() + } + service.mu.Unlock() + if ctx == nil { + ctx = context.Background() + } + select { + case <-service.done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (service *Service) expireRecentLocked(recent map[string]time.Time, now time.Time) { + for key, until := range recent { + if !now.Before(until) { + delete(recent, key) + } + } +} + +func (service *Service) rememberRecentLocked(recent map[string]time.Time, key string, now time.Time) { + service.expireRecentLocked(recent, now) + if len(recent) >= defaultRecentLimit { + var earliestKey string + var earliest time.Time + for candidate, until := range recent { + if earliestKey == "" || until.Before(earliest) { + earliestKey, earliest = candidate, until + } + } + delete(recent, earliestKey) + } + recent[key] = now.Add(service.cooldown) +} + +func (service *Service) metric(call func(Metrics)) { + if service != nil && service.metrics != nil { + call(service.metrics) + } +} + +// windowGroupKeyLocked chooses an explicit playback window first. A PlayQueue +// ID is a useful fallback for clients that expose it. Without either value we +// allocate a private anonymous group instead of allowing unrelated clients to +// cancel one another; callers that want latest-wins across episode changes +// should set WindowKey to a stable client playback-session identity. +func (service *Service) windowGroupKeyLocked(playbackContext PlaybackContext) string { + if windowKey := strings.TrimSpace(playbackContext.WindowKey); windowKey != "" { + digest := sha256.Sum256([]byte("window\x00" + windowKey)) + return string(digest[:]) + } + if playQueueID := strings.TrimSpace(playbackContext.PlayQueueID); playQueueID != "" { + digest := sha256.Sum256([]byte("queue\x00" + playQueueID)) + return string(digest[:]) + } + service.windowSeq++ + return "anonymous\x00" + strconv.FormatUint(service.windowSeq, 10) +} + +func validPlaybackContext(playbackContext PlaybackContext) bool { + return numericIdentity(playbackContext.PartID) && + (playbackContext.RatingKey == "" || numericIdentity(playbackContext.RatingKey)) && + strings.TrimSpace(playbackContext.Target) != "" && !strings.ContainsAny(playbackContext.Target, "\r\n") && + !strings.ContainsAny(playbackContext.UserAgent, "\r\n") && + !strings.ContainsAny(playbackContext.WindowKey, "\r\n") && + (playbackContext.PlayQueueID == "" || numericIdentity(playbackContext.PlayQueueID)) && + (playbackContext.PlayQueueItemID == "" || numericIdentity(playbackContext.PlayQueueItemID)) +} + +func currentContextKey(playbackContext PlaybackContext) string { + digest := sha256.Sum256([]byte(playbackContext.PartID + "\x00" + playbackContext.Target + "\x00" + playbackContext.UserAgent)) + return string(digest[:]) +} + +func playbackContextKey(playbackContext PlaybackContext) string { + targetDigest := sha256.Sum256([]byte(playbackContext.Target)) + userAgentDigest := sha256.Sum256([]byte(playbackContext.UserAgent)) + windowDigest := sha256.Sum256([]byte(playbackContext.WindowKey)) + return playbackContext.RatingKey + "\x00" + playbackContext.PartID + "\x00" + + playbackContext.PlayQueueID + "\x00" + playbackContext.PlayQueueItemID + "\x00" + + string(windowDigest[:]) + "\x00" + string(targetDigest[:]) + "\x00" + string(userAgentDigest[:]) +} + +func discoveryErrorKind(err error) string { + switch { + case errors.Is(err, ErrNoCandidates): + return "no_candidates" + case errors.Is(err, ErrUntrustedCurrent): + return "identity" + case errors.Is(err, context.DeadlineExceeded): + return "timeout" + default: + return "plex" + } +} diff --git a/internal/prewarm/service_test.go b/internal/prewarm/service_test.go new file mode 100644 index 0000000..9dfe954 --- /dev/null +++ b/internal/prewarm/service_test.go @@ -0,0 +1,509 @@ +package prewarm + +import ( + "context" + "errors" + "io" + "log/slog" + "sync" + "testing" + "time" + + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" + "github.com/InfinityPacer/plex-gateway/internal/playback" + "github.com/InfinityPacer/plex-gateway/internal/plexmeta" +) + +type discoveryFunc func(context.Context, PlaybackContext, int, int) ([]Candidate, error) + +func (function discoveryFunc) Neighbors(ctx context.Context, current PlaybackContext, before, after int) ([]Candidate, error) { + return function(ctx, current, before, after) +} + +type preparerFunc func(plexmeta.Part) playback.Preparation + +func (function preparerFunc) Prepare(part plexmeta.Part) playback.Preparation { return function(part) } + +type submitterFunc func(mediainfo.Request) error + +func (function submitterFunc) SubmitDetailed(request mediainfo.Request) mediainfo.SubmitResult { + if err := function(request); err != nil { + return mediainfo.SubmitResult{Disposition: mediainfo.SubmitRejected, Err: err} + } + return mediainfo.SubmitResult{Disposition: mediainfo.SubmitNewlyQueued} +} + +type detailedSubmitterFunc func(mediainfo.Request) mediainfo.SubmitResult + +func (function detailedSubmitterFunc) SubmitDetailed(request mediainfo.Request) mediainfo.SubmitResult { + return function(request) +} + +func TestServiceTryEnqueueDoesNoIOInline(t *testing.T) { + discoveryStarted := make(chan struct{}) + releaseDiscovery := make(chan struct{}) + submitted := make(chan mediainfo.Request, 1) + service := newTestService(t, + discoveryFunc(func(context.Context, PlaybackContext, int, int) ([]Candidate, error) { + close(discoveryStarted) + <-releaseDiscovery + return nil, ErrNoCandidates + }), + preparerFunc(func(plexmeta.Part) playback.Preparation { + t.Fatal("Prepare ran inline") + return playback.Preparation{} + }), + submitterFunc(func(request mediainfo.Request) error { + submitted <- request + return nil + }), + ) + started := time.Now() + if !service.TryEnqueue(testPlayback("42", "9", "current")) { + t.Fatal("TryEnqueue() = false") + } + if elapsed := time.Since(started); elapsed > 50*time.Millisecond { + t.Fatalf("TryEnqueue blocked for %s", elapsed) + } + select { + case request := <-submitted: + if request.Priority != mediainfo.PriorityInteractive || request.Key.PartID != "9" { + t.Fatalf("current request = %#v", request) + } + case <-time.After(time.Second): + t.Fatal("current item was not submitted") + } + select { + case <-discoveryStarted: + case <-time.After(time.Second): + t.Fatal("background discovery did not start") + } + close(releaseDiscovery) +} + +func TestServiceSubmitsCurrentImmediatelyAndSpacesNeighbors(t *testing.T) { + type observed struct { + request mediainfo.Request + at time.Time + } + submitted := make(chan observed, 3) + service := newConfiguredTestService(t, ServiceOptions{ + Discovery: discoveryFunc(func(_ context.Context, _ PlaybackContext, before, after int) ([]Candidate, error) { + if before != 2 || after != 3 { + t.Fatalf("window = %d/%d", before, after) + } + return []Candidate{ + {RatingKey: "43", Part: plexmeta.Part{ID: "10", File: "/cloud/e03.strm"}}, + {RatingKey: "41", Part: plexmeta.Part{ID: "8", File: "/cloud/e01.strm"}}, + }, nil + }), + Playback: preparerFunc(func(part plexmeta.Part) playback.Preparation { + return playback.Preparation{State: playback.PreparationReady, Part: playback.PreparedPart{ + Part: part, Target: "http://mediavault/redirect/" + part.ID, + }} + }), + MediaInfo: submitterFunc(func(request mediainfo.Request) error { + submitted <- observed{request: request, at: time.Now()} + return nil + }), + BeforeCount: 2, AfterCount: 3, SubmitInterval: 25 * time.Millisecond, + }) + started := time.Now() + service.TryEnqueue(testPlayback("42", "9", "current")) + first := <-submitted + second := <-submitted + third := <-submitted + if first.request.Key.PartID != "9" || first.request.Priority != mediainfo.PriorityInteractive || + second.request.Key.PartID != "10" || second.request.Priority != mediainfo.PriorityBackground || + third.request.Key.PartID != "8" || third.request.Priority != mediainfo.PriorityBackground { + t.Fatalf("submission order = %#v %#v %#v", first.request, second.request, third.request) + } + if first.at.Sub(started) > 20*time.Millisecond { + t.Fatalf("current submission delay = %s", first.at.Sub(started)) + } + if second.at.Sub(first.at) < 20*time.Millisecond || third.at.Sub(second.at) < 20*time.Millisecond { + t.Fatalf("neighbor spacing = %s, %s", second.at.Sub(first.at), third.at.Sub(second.at)) + } +} + +func TestServiceLatestPlaybackCancelsOlderNeighborDiscovery(t *testing.T) { + started := make(chan string, 3) + submitted := make(chan mediainfo.Request, 8) + service := newTestService(t, + discoveryFunc(func(ctx context.Context, current PlaybackContext, _, _ int) ([]Candidate, error) { + started <- current.RatingKey + if current.RatingKey != "44" { + <-ctx.Done() + return nil, ctx.Err() + } + return []Candidate{{RatingKey: "45", Part: plexmeta.Part{ID: "12", File: "/cloud/e05.strm"}}}, nil + }), + preparerFunc(func(part plexmeta.Part) playback.Preparation { + return playback.Preparation{State: playback.PreparationReady, Part: playback.PreparedPart{ + Part: part, Target: "http://mediavault/redirect/next", + }} + }), + submitterFunc(func(request mediainfo.Request) error { + submitted <- request + return nil + }), + ) + service.TryEnqueue(testPlayback("42", "9", "a")) + if got := <-started; got != "42" { + t.Fatalf("first discovery = %q", got) + } + service.TryEnqueue(testPlayback("43", "10", "b")) + service.TryEnqueue(testPlayback("44", "11", "c")) + + deadline := time.After(time.Second) + seenCurrent := map[string]bool{} + seenCurrentC := false + seenNeighbor := false + for !seenCurrentC || !seenNeighbor { + select { + case request := <-submitted: + if request.Priority == mediainfo.PriorityInteractive { + seenCurrent[request.Key.PartID] = true + seenCurrentC = request.Key.PartID == "11" || seenCurrentC + } + if request.Key.PartID == "12" && request.RatingKey == "45" && request.ClientUserAgent == "Client/c" { + seenNeighbor = true + } + case <-deadline: + t.Fatalf("currentC=%v neighbor=%v", seenCurrentC, seenNeighbor) + } + } + for _, partID := range []string{"9", "10", "11"} { + if !seenCurrent[partID] { + t.Fatalf("current item %s was not submitted", partID) + } + } +} + +func TestServiceKeepsNeighborWindowsIndependentAcrossClients(t *testing.T) { + started := make(chan string, 2) + release := make(chan struct{}) + service := newTestService(t, + discoveryFunc(func(ctx context.Context, current PlaybackContext, _, _ int) ([]Candidate, error) { + started <- current.WindowKey + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-release: + return nil, ErrNoCandidates + } + }), + preparerFunc(func(plexmeta.Part) playback.Preparation { return playback.Preparation{} }), + submitterFunc(func(mediainfo.Request) error { return nil }), + ) + first := testPlayback("42", "9", "first") + first.WindowKey = "client-a" + second := testPlayback("43", "10", "second") + second.WindowKey = "client-b" + service.TryEnqueue(first) + service.TryEnqueue(second) + seen := map[string]bool{} + for len(seen) < 2 { + select { + case key := <-started: + seen[key] = true + case <-time.After(time.Second): + t.Fatalf("started windows = %#v", seen) + } + } + close(release) +} + +func TestServiceRetiresCompletedNeighborWindow(t *testing.T) { + done := make(chan struct{}, 1) + service := newTestService(t, + discoveryFunc(func(context.Context, PlaybackContext, int, int) ([]Candidate, error) { + done <- struct{}{} + return nil, ErrNoCandidates + }), + preparerFunc(func(plexmeta.Part) playback.Preparation { return playback.Preparation{} }), + submitterFunc(func(mediainfo.Request) error { return nil }), + ) + service.TryEnqueue(testPlayback("42", "9", "retire")) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("discovery did not run") + } + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + service.mu.Lock() + windowCount := len(service.windows) + service.mu.Unlock() + if windowCount == 0 { + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("completed neighbor window was not retired") +} + +func TestServiceSubmitsCurrentWithoutRatingKey(t *testing.T) { + submitted := make(chan mediainfo.Request, 1) + service := newTestService(t, + discoveryFunc(func(context.Context, PlaybackContext, int, int) ([]Candidate, error) { + t.Fatal("neighbor discovery must not run without a rating key") + return nil, nil + }), + preparerFunc(func(plexmeta.Part) playback.Preparation { return playback.Preparation{} }), + submitterFunc(func(request mediainfo.Request) error { + submitted <- request + return nil + }), + ) + current := testPlayback("", "9", "no-rating-key") + if !service.TryEnqueue(current) { + t.Fatal("TryEnqueue() = false") + } + select { + case request := <-submitted: + if request.Key.PartID != "9" || request.RatingKey != "" { + t.Fatalf("request = %#v", request) + } + case <-time.After(time.Second): + t.Fatal("current item was not submitted") + } +} + +func TestServiceRecordsDetailedSubmitDisposition(t *testing.T) { + metrics := &recordingMetrics{} + results := []mediainfo.SubmitDisposition{ + mediainfo.SubmitFreshCache, + mediainfo.SubmitJoinedExistingFlight, + mediainfo.SubmitNewlyQueued, + mediainfo.SubmitRejected, + } + index := 0 + service := newConfiguredTestService(t, ServiceOptions{ + Discovery: discoveryFunc(func(context.Context, PlaybackContext, int, int) ([]Candidate, error) { + return nil, ErrNoCandidates + }), + Playback: preparerFunc(func(plexmeta.Part) playback.Preparation { return playback.Preparation{} }), + MediaInfo: detailedSubmitterFunc(func(mediainfo.Request) mediainfo.SubmitResult { + result := mediainfo.SubmitResult{Disposition: results[index]} + index++ + return result + }), + Metrics: metrics, BeforeCount: 0, AfterCount: 0, + }) + for _, partID := range []string{"1", "2", "3", "4"} { + service.TryEnqueue(testPlayback("42", partID, "result-"+partID)) + } + deadline := time.Now().Add(time.Second) + for metrics.totalSubmits() < len(results) && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if metrics.fresh != 1 || metrics.joined != 1 || metrics.queued != 1 || metrics.rejected != 1 { + t.Fatalf("metrics = %#v", metrics) + } +} + +func TestServiceRetriesCurrentAfterAdmissionRejection(t *testing.T) { + requests := make(chan mediainfo.Request, 2) + attempt := 0 + service := newConfiguredTestService(t, ServiceOptions{ + Playback: preparerFunc(func(plexmeta.Part) playback.Preparation { return playback.Preparation{} }), + MediaInfo: detailedSubmitterFunc(func(request mediainfo.Request) mediainfo.SubmitResult { + requests <- request + attempt++ + if attempt == 1 { + return mediainfo.SubmitResult{Disposition: mediainfo.SubmitRejected, Err: mediainfo.ErrQueueFull} + } + return mediainfo.SubmitResult{Disposition: mediainfo.SubmitNewlyQueued} + }), + BeforeCount: 0, AfterCount: 0, + }) + current := testPlayback("42", "9", "retry") + service.TryEnqueue(current) + select { + case <-requests: + case <-time.After(time.Second): + t.Fatal("first submission did not run") + } + deadline := time.Now().Add(time.Second) + for service.Status().Active && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if !service.TryEnqueue(current) { + t.Fatal("second TryEnqueue() = false") + } + select { + case <-requests: + case <-time.After(time.Second): + t.Fatal("second submission did not run") + } +} + +type recordingMetrics struct { + mu sync.Mutex + fresh, joined, queued, rejected, skipped int +} + +func (*recordingMetrics) IncMediaInfoPrewarmTriggered() {} +func (*recordingMetrics) IncMediaInfoPrewarmReplaced() {} +func (*recordingMetrics) IncMediaInfoPrewarmDiscoverySuccess() {} +func (*recordingMetrics) IncMediaInfoPrewarmDiscoveryFailure() {} +func (metrics *recordingMetrics) IncMediaInfoPrewarmFreshCache() { + metrics.mu.Lock() + defer metrics.mu.Unlock() + metrics.fresh++ +} +func (metrics *recordingMetrics) IncMediaInfoPrewarmJoinedFlight() { + metrics.mu.Lock() + defer metrics.mu.Unlock() + metrics.joined++ +} +func (metrics *recordingMetrics) IncMediaInfoPrewarmQueued() { + metrics.mu.Lock() + defer metrics.mu.Unlock() + metrics.queued++ +} +func (metrics *recordingMetrics) IncMediaInfoPrewarmRejected() { + metrics.mu.Lock() + defer metrics.mu.Unlock() + metrics.rejected++ +} +func (metrics *recordingMetrics) IncMediaInfoPrewarmSkipped() { + metrics.mu.Lock() + defer metrics.mu.Unlock() + metrics.skipped++ +} +func (metrics *recordingMetrics) totalSubmits() int { + metrics.mu.Lock() + defer metrics.mu.Unlock() + return metrics.fresh + metrics.joined + metrics.queued + metrics.rejected +} + +func TestServiceDeduplicatesRecentPlayback(t *testing.T) { + var mu sync.Mutex + calls := 0 + done := make(chan struct{}, 1) + service := newTestService(t, + discoveryFunc(func(context.Context, PlaybackContext, int, int) ([]Candidate, error) { + mu.Lock() + calls++ + mu.Unlock() + done <- struct{}{} + return nil, ErrNoCandidates + }), + preparerFunc(func(plexmeta.Part) playback.Preparation { return playback.Preparation{} }), + submitterFunc(func(mediainfo.Request) error { return nil }), + ) + current := testPlayback("42", "9", "current") + service.TryEnqueue(current) + <-done + for service.Status().Active { + time.Sleep(time.Millisecond) + } + service.TryEnqueue(current) + time.Sleep(20 * time.Millisecond) + mu.Lock() + defer mu.Unlock() + if calls != 1 { + t.Fatalf("discovery calls = %d", calls) + } +} + +func TestServiceDoesNotDeduplicateChangedTargetOrUserAgent(t *testing.T) { + called := make(chan PlaybackContext, 3) + service := newTestService(t, + discoveryFunc(func(_ context.Context, current PlaybackContext, _, _ int) ([]Candidate, error) { + called <- current + return nil, ErrNoCandidates + }), + preparerFunc(func(plexmeta.Part) playback.Preparation { return playback.Preparation{} }), + submitterFunc(func(mediainfo.Request) error { return nil }), + ) + base := testPlayback("42", "9", "base") + service.TryEnqueue(base) + <-called + for service.Status().Active { + time.Sleep(time.Millisecond) + } + changedTarget := base + changedTarget.Target += "?revision=2" + service.TryEnqueue(changedTarget) + <-called + for service.Status().Active { + time.Sleep(time.Millisecond) + } + changedAgent := changedTarget + changedAgent.UserAgent = "Client/other" + service.TryEnqueue(changedAgent) + <-called +} + +func TestServiceAllowsCurrentOnlyWindow(t *testing.T) { + submitted := make(chan mediainfo.Request, 1) + service := newConfiguredTestService(t, ServiceOptions{ + Discovery: discoveryFunc(func(context.Context, PlaybackContext, int, int) ([]Candidate, error) { + t.Fatal("discovery must not run for a zero window") + return nil, nil + }), + Playback: preparerFunc(func(plexmeta.Part) playback.Preparation { + t.Fatal("neighbor preparation must not run for a zero window") + return playback.Preparation{} + }), + MediaInfo: submitterFunc(func(request mediainfo.Request) error { + submitted <- request + return nil + }), + BeforeCount: 0, AfterCount: 0, SubmitInterval: time.Millisecond, + }) + service.TryEnqueue(testPlayback("42", "9", "current-only")) + select { + case request := <-submitted: + if request.Key.PartID != "9" || request.Priority != mediainfo.PriorityInteractive { + t.Fatalf("request = %#v", request) + } + case <-time.After(time.Second): + t.Fatal("current item was not submitted") + } +} + +func newTestService(t *testing.T, discovery neighborDiscovery, preparer partPreparer, submitter mediaInfoSubmitter) *Service { + t.Helper() + return newConfiguredTestService(t, ServiceOptions{ + Discovery: discovery, Playback: preparer, MediaInfo: submitter, + BeforeCount: 2, AfterCount: 3, SubmitInterval: time.Millisecond, + }) +} + +func newConfiguredTestService(t *testing.T, options ServiceOptions) *Service { + t.Helper() + if options.Logger == nil { + options.Logger = slog.New(slog.NewTextHandler(io.Discard, nil)) + } + if options.DiscoveryTimeout == 0 { + options.DiscoveryTimeout = time.Second + } + if options.TriggerCooldown == 0 { + options.TriggerCooldown = time.Minute + } + service, err := NewService(options) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := service.Close(ctx); err != nil && !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + }) + return service +} + +func testPlayback(ratingKey, partID, suffix string) PlaybackContext { + return PlaybackContext{ + RatingKey: ratingKey, PartID: partID, + Target: "http://mediavault/redirect/" + suffix, + WindowKey: "test-session", + UserAgent: "Client/" + suffix, + } +} From e46e35b94665764f6448a27331507d26362453a2 Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sat, 29 Aug 2026 06:44:39 +0800 Subject: [PATCH 02/15] feat: project STRM MediaInfo streams --- CHANGELOG.en.md | 7 + CHANGELOG.md | 4 + README.en.md | 20 + README.md | 14 + docs/architecture.md | 12 +- docs/media-lifecycle-architecture.md | 11 +- docs/roadmap.md | 5 +- internal/gateway/metadata_enrichment.go | 43 +- internal/gateway/metadata_enrichment_test.go | 129 +++- internal/mediainfo/ffprobe.go | 74 ++- internal/mediainfo/ffprobe_test.go | 19 + internal/mediainfo/model.go | 6 +- internal/mediainfo/model_test.go | 2 +- internal/mediainfo/provider.go | 2 +- internal/mediainfo/provider_test.go | 2 +- internal/mediainfo/service.go | 14 +- internal/mediainfo/service_test.go | 32 +- internal/mediainfo/sqlite_test.go | 4 +- internal/mediainfo/touch_lifecycle_test.go | 2 + internal/plexmeta/projection.go | 605 +++++++++++++++++-- internal/plexmeta/projection_test.go | 327 +++++++++- 21 files changed, 1244 insertions(+), 90 deletions(-) diff --git a/CHANGELOG.en.md b/CHANGELOG.en.md index 1589aa3..3817756 100644 --- a/CHANGELOG.en.md +++ b/CHANGELOG.en.md @@ -21,6 +21,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added per-client rapid-switch coordination, cross-User-Agent failure handoff, and distinct prewarm metrics for fresh cache hits, joined work, new queue admissions, and rejections. +- Parts without Plex Streams now receive descriptive video, audio, and subtitle + Streams without Plex IDs or playback-selection state, including HDR10, Dolby + Vision, bit depth, bitrate, channel, and language fields. +- Infuse-style background library synchronization consumes existing MediaInfo + cache records without turning per-item browsing into full-library cold probes. +- MediaInfo ffprobe records now use a new provider revision so records produced + by the previous interpretation are not reused. ## [0.1.1] - 2026-08-28 diff --git a/CHANGELOG.md b/CHANGELOG.md index 44280eb..09898ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ 的窗口限速预热邻近媒体。Token 缺失或失效只禁用邻近发现。 - 增加按客户端隔离的快速切换协调、跨 User-Agent 失败接力，以及缓存命中、加入现有 任务、新入队和拒绝的预热指标。 +- 为没有 Stream 的 STRM Part 创建不含 Plex ID 或播放选择状态的描述性视频、音频和 + 字幕 Stream，并补充 HDR10、Dolby Vision、bit depth、bitrate、声道和语言等字段。 +- Infuse 风格的后台媒体库同步只读取现有 MediaInfo 缓存，不再因逐项浏览触发全库冷探测。 +- MediaInfo ffprobe 记录升级到新的 Provider revision，旧解释版本的缓存不会被复用。 ## [0.1.1] - 2026-08-28 diff --git a/README.en.md b/README.en.md index e921a78..4899427 100644 --- a/README.en.md +++ b/README.en.md @@ -148,6 +148,26 @@ through Plex. The gateway does not need a Plex username or password, and playback does not depend on the management token. MediaVault's API key for `/api/v1` integrations is also not required for the STRM `/redirect` playback contract. +### MediaInfo response enrichment + +For an authenticated single-item STRM metadata request, the gateway can fill +missing Media, Part, and Stream technical fields from its L1 or SQLite record. +When a Plex Part has no Stream elements, ffprobe stream types and source indexes +are used to create descriptive video, audio, and subtitle Streams with fields +such as HDR10, Dolby Vision, bit depth, codec, bitrate, channel layout, and +language code. Generated Streams never contain Plex Stream IDs or playback +selection fields such as `selected`, `default`, or `decision`. When Plex already +has Streams, only missing fields on identity-matched Streams are filled. Missing +sibling Streams are not created and existing Plex values are not overwritten. + +Background library synchronization can enumerate an entire library one item at +a time. Requests with `skipRefresh` whose product ends in `-Library` may consume +an existing MediaInfo cache record but never admit a cold probe. Ordinary +single-item access, a successful cloud 302, and the nearby window retain their +bounded probe behavior, so browsing a library cannot expand into full-library +CDN ffprobe work. Any cache miss, timeout, unsupported structure, or projection +failure returns the original Plex response. + ### Nearby-item MediaInfo prewarming The gateway performs one bounded in-memory enqueue only after Plex has diff --git a/README.md b/README.md index 7942f7a..5a5e2b5 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,20 @@ Git 的 `app.env` 中。推荐让 Compose 只通过 `env_file: ./app.env` 加载 单独配置 Plex 用户名或密码，播放也不依赖管理 Token。STRM `/redirect` 播放协议不需要 MediaVault 用于 `/api/v1` 集成的 API key。 +### MediaInfo 响应增强 + +对于已认证的单项 STRM metadata 请求，Gateway 可以从 L1 或 SQLite 记录补充缺失的 +Media、Part 和 Stream 技术字段。若 Plex 的 Part 完全没有 Stream，Gateway 会使用 +ffprobe 的流类型和源索引创建描述性的视频、音频和字幕 Stream，并投影 HDR10、Dolby +Vision、bit depth、codec、bitrate、声道布局和语言码等字段。合成 Stream 不包含 Plex +Stream ID，也不生成 `selected`、`default` 或 `decision` 等播放选择字段。Plex 已经存在 +Stream 时只补充身份匹配 Stream 的缺失字段，不创建缺少的其他流，也不覆盖 Plex 值。 + +Infuse 等客户端的后台媒体库同步可能逐项请求整个库。带 `skipRefresh` 且产品名以 +`-Library` 结尾的后台同步请求只读取现有 MediaInfo 缓存，不创建冷探测。普通单项访问、 +成功的云端 302 和邻近窗口仍可按既定边界提交探测，因此浏览媒体库不会扩散成全库 CDN +ffprobe。任何缓存 miss、超时、结构不支持或投影失败都会返回原始 Plex response。 + ### 邻近媒体 MediaInfo 预热 Gateway 在云端 Part 已通过 Plex 授权、MediaVault 已返回最终直链且 Gateway 已写出 diff --git a/docs/architecture.md b/docs/architecture.md index d16c8be..74c4675 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,9 +21,13 @@ gateway; the isolated analysis worker may read a bounded amount for probing. - Successful XML or JSON Plex responses are observed to populate the in-memory `Part.id` cache without changing their bytes. When MediaInfo is available, an authenticated single-item metadata response may also fill missing whitelisted - technical fields from the exact STRM fingerprint. The transform is bounded, - never creates Stream identities, and preserves the original Plex response on - timeout, ambiguity, unsupported encoding, or any other failure. + technical fields from the exact STRM fingerprint. A Part with no Stream + elements may receive descriptive Streams identified only by ffprobe stream + type and source index. The gateway never invents Plex Stream IDs or playback + selection state, and an existing Stream set is never expanded. The transform + is bounded and preserves the original Plex response on timeout, ambiguity, + unsupported encoding, or any other failure. Background library crawlers + identified by `skipRefresh` and a `-Library` product consume cache only. - `GET` and `HEAD` requests below `/library/parts/{partID}/...` are eligible for interception only when the cached `Part.file` has a configured cloud extension and maps to a readable local STRM file. @@ -175,6 +179,8 @@ mount its private cache, or duplicate those features in the playback plane. MediaInfo Phase D begins with a provider boundary, L1 and SQLite persistence, bounded remote probing, proactive prewarming, and metadata response enrichment. +A provider revision separates records whenever normalized probe semantics +change, preventing an old interpretation from being projected after an upgrade. A stable MediaVault or MoviePilot provider can replace remote probing without changing playback. Plex persistence remains a feasibility question across an official API, another supported PMS interface, and an isolated database helper. diff --git a/docs/media-lifecycle-architecture.md b/docs/media-lifecycle-architecture.md index 679154a..cc957a0 100644 --- a/docs/media-lifecycle-architecture.md +++ b/docs/media-lifecycle-architecture.md @@ -564,9 +564,14 @@ Gateway response enrichment 先用于验证 Plex 字段映射和客户端消费 打开 Plex 数据库。如果不启用 helper，Gateway 仍需保留 metadata 请求准入、缓存与 熔断，并可在已确认的 STRM 项目上做客户端响应 enrichment。 -response enrichment 只修改当前已认证 Plex response 中已经存在的精确 Media 和 Part， -Plex 已有字段不覆盖。首版只处理成功的单项 `GET /library/metadata/{ratingKey}`；XML、 -JSON、gzip、body 大小或结构不支持，以及转换失败时，都必须原样返回 Plex response。 +response enrichment 只修改当前已认证 Plex response 中的精确 Media 和 Part，Plex 已有 +字段不覆盖。Part 完全没有 Stream 时，可以按 ffprobe 的流类型和源索引创建描述性 +Stream，但不生成 Plex Stream ID、`selected`、`default`、`decision` 等播放选择状态。 +Plex 已有 Stream 时只补充身份匹配项的缺失字段，不创建缺少的其他流。首版只处理成功的 +单项 `GET /library/metadata/{ratingKey}`；XML、JSON、gzip、body 大小或结构不支持，以及 +转换失败时，都必须原样返回 Plex response。带 `skipRefresh` 且产品名以 `-Library` +结尾的后台媒体库同步只消费现有缓存，不准入冷探测，避免一次媒体库同步扩散为全库 CDN +ffprobe。 修改成功后同步更新 Content-Length，并移除失效的 ETag、Content-MD5 或 Digest。HDR 和 Dolby Vision 映射必须使用同一片源的本地文件与 STRM Plex response 建立 fixture，不能 仅按 ffprobe 字段名猜测。 diff --git a/docs/roadmap.md b/docs/roadmap.md index e38615f..2d0542e 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -90,7 +90,10 @@ Gateway fallback while preserving the broader lifecycle design: defer season, show, and configured STRM-root batch tasks; 5. enrich authorized Plex metadata responses first, then evaluate an official Plex API, other supported PMS interfaces, and an isolated database helper - without selecting a persistence path in advance; + without selecting a persistence path in advance. Parts without Plex Streams + may receive descriptive ffprobe-indexed Streams, while existing Stream sets + are never expanded and Plex IDs or playback-selection state are never + synthesized. Background `-Library` synchronization remains cache-only; 6. add Redis or replace SQLite with PostgreSQL only after measured multi-instance or capacity requirements exist. diff --git a/internal/gateway/metadata_enrichment.go b/internal/gateway/metadata_enrichment.go index 77de5b9..d982a15 100644 --- a/internal/gateway/metadata_enrichment.go +++ b/internal/gateway/metadata_enrichment.go @@ -26,6 +26,7 @@ const ( ) type mediaInfoEnsurer interface { + Get(mediainfo.Key) (mediainfo.Record, bool) Ensure(context.Context, mediainfo.Request) (mediainfo.Record, error) } @@ -136,16 +137,26 @@ func (handler *metadataEnrichmentHandler) ServeHTTP(w http.ResponseWriter, reque handler.failOpen(capture, rawBody) return } - waitContext, cancel := context.WithTimeout(request.Context(), handler.coldWait) - record, err := handler.service.Ensure(waitContext, mediainfo.Request{ - Key: mediainfo.Key{PartID: target.Part.ID, STRMFingerprint: fingerprint}, - RatingKey: ratingKey, Target: strmTarget, Priority: mediainfo.PriorityInteractive, - ClientUserAgent: request.UserAgent(), - }) - cancel() - if err != nil { - handler.failOpen(capture, rawBody) - return + key := mediainfo.Key{PartID: target.Part.ID, STRMFingerprint: fingerprint} + var record mediainfo.Record + if mediaInfoCacheOnlyRequest(request) { + var found bool + record, found = handler.service.Get(key) + if !found { + handler.replay(capture, rawBody) + return + } + } else { + waitContext, cancel := context.WithTimeout(request.Context(), handler.coldWait) + record, err = handler.service.Ensure(waitContext, mediainfo.Request{ + Key: key, RatingKey: ratingKey, Target: strmTarget, + Priority: mediainfo.PriorityInteractive, ClientUserAgent: request.UserAgent(), + }) + cancel() + if err != nil { + handler.failOpen(capture, rawBody) + return + } } currentTarget, err := handler.resolver.ReadTarget(localPath) if err != nil { @@ -204,6 +215,18 @@ func mediaInfoMetadataRequest(request *http.Request) (string, bool) { return strings.TrimPrefix(request.URL.Path, "/library/metadata/"), true } +// mediaInfoCacheOnlyRequest identifies background library crawlers that may +// enumerate every item. They can consume an existing record, while active +// playback remains responsible for admitting new remote analysis work. +func mediaInfoCacheOnlyRequest(request *http.Request) bool { + if request == nil || request.URL == nil { + return false + } + product := strings.ToLower(strings.TrimSpace(request.Header.Get("X-Plex-Product"))) + _, skipsRefresh := request.URL.Query()["skipRefresh"] + return skipsRefresh && strings.HasSuffix(product, "-library") +} + func normalizedExtensionSet(extensions []string) map[string]struct{} { if len(extensions) == 0 { extensions = []string{".strm"} diff --git a/internal/gateway/metadata_enrichment_test.go b/internal/gateway/metadata_enrichment_test.go index 99910a5..3cf0cfa 100644 --- a/internal/gateway/metadata_enrichment_test.go +++ b/internal/gateway/metadata_enrichment_test.go @@ -26,11 +26,21 @@ import ( type fakeMediaInfoEnsurer struct { calls atomic.Int64 + gets atomic.Int64 mu sync.Mutex seen []mediainfo.Request + getFn func(mediainfo.Key) (mediainfo.Record, bool) fn func(context.Context, mediainfo.Request) (mediainfo.Record, error) } +func (ensurer *fakeMediaInfoEnsurer) Get(key mediainfo.Key) (mediainfo.Record, bool) { + ensurer.gets.Add(1) + if ensurer.getFn == nil { + return mediainfo.Record{}, false + } + return ensurer.getFn(key) +} + func (ensurer *fakeMediaInfoEnsurer) Ensure(ctx context.Context, request mediainfo.Request) (mediainfo.Record, error) { ensurer.calls.Add(1) ensurer.mu.Lock() @@ -118,6 +128,71 @@ func TestMetadataEnrichmentPreservesGzipJSON(t *testing.T) { } } +func TestMetadataEnrichmentKeepsBackgroundLibrarySyncCacheOnly(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + raw := fixture.xmlBody() + ensurer := &fakeMediaInfoEnsurer{fn: func(context.Context, mediainfo.Request) (mediainfo.Record, error) { + t.Fatal("background library sync must not admit a cold probe") + return mediainfo.Record{}, nil + }} + handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(raw, "application/xml")) + request := authenticatedRequest(http.MethodGet, "/library/metadata/42?skipRefresh=1") + request.Header.Set("X-Plex-Product", "Infuse-Library") + + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), raw) { + t.Fatalf("response = %d %q", response.Code, response.Body.Bytes()) + } + if ensurer.gets.Load() != 1 || ensurer.calls.Load() != 0 { + t.Fatalf("Get calls = %d, Ensure calls = %d", ensurer.gets.Load(), ensurer.calls.Load()) + } +} + +func TestMetadataEnrichmentUsesCachedRecordForBackgroundLibrarySync(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + ensurer := &fakeMediaInfoEnsurer{getFn: func(key mediainfo.Key) (mediainfo.Record, bool) { + return mediainfo.Record{Key: key, Media: completeProjectionMedia()}, true + }} + handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(fixture.xmlBody(), "application/xml")) + request := authenticatedRequest(http.MethodGet, "/library/metadata/42?skipRefresh=1") + request.Header.Set("X-Plex-Product", "Infuse-Library") + + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `container="mkv"`) { + t.Fatalf("response = %d %q", response.Code, response.Body.String()) + } + if ensurer.gets.Load() != 1 || ensurer.calls.Load() != 0 { + t.Fatalf("Get calls = %d, Ensure calls = %d", ensurer.gets.Load(), ensurer.calls.Load()) + } +} + +func TestMediaInfoCacheOnlyRequestRequiresLibraryProductAndSkipRefresh(t *testing.T) { + tests := []struct { + name string + path string + product string + want bool + }{ + {name: "background library sync", path: "/library/metadata/42?skipRefresh=1", product: "Infuse-Library", want: true}, + {name: "case insensitive product", path: "/library/metadata/42?skipRefresh=1", product: "INFUSE-LIBRARY", want: true}, + {name: "interactive library request", path: "/library/metadata/42", product: "Infuse-Library", want: false}, + {name: "ordinary client with skip refresh", path: "/library/metadata/42?skipRefresh=1", product: "Plex for iOS", want: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, test.path, nil) + request.Header.Set("X-Plex-Product", test.product) + if got := mediaInfoCacheOnlyRequest(request); got != test.want { + t.Fatalf("mediaInfoCacheOnlyRequest() = %v, want %v", got, test.want) + } + }) + } +} + func TestMetadataEnrichmentTimeoutReturnsOriginalResponse(t *testing.T) { fixture := newMetadataEnrichmentFixture(t) raw := fixture.xmlBody() @@ -254,6 +329,58 @@ func TestMetadataEnrichmentBypassesIneligibleAndUnbufferableResponses(t *testing } } +func TestMetadataEnrichmentAdmitsExistingStreamHDRAndSTRMControlSize(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + media := completeProjectionMedia() + media.Size = 987654321 + media.Bitrate = 29430119 + media.Streams[0] = mediainfo.Stream{ + Index: 0, Type: "video", Codec: "hevc", Profile: "Main 10", Bitrate: 27510000, + Width: 3840, Height: 2160, FrameRate: "24000/1001", PixelFormat: "yuv420p10le", BitDepth: 10, + ColorSpace: "bt2020nc", ColorRange: "tv", ColorPrimaries: "bt2020", ColorTransfer: "smpte2084", + HDRFormat: "dolby_vision", + DolbyVision: &mediainfo.DolbyVision{ + VersionMajor: 1, Profile: 7, Level: 6, RPUPresent: 1, + ELPresent: 1, BLPresent: 1, BLCompatID: 6, + }, + } + tests := []struct { + name string + contentType string + body []byte + want [][]byte + }{ + { + name: "XML", contentType: "application/xml", + body: []byte(``), + want: [][]byte{[]byte(`size="987654321"`), []byte(`displayTitle="4K DoVi/HDR10"`), []byte(`DOVIPresent="1"`)}, + }, + { + name: "JSON", contentType: "application/json", + body: []byte(`{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"container":"mkv","duration":60000,"bitrate":8000,"width":3840,"height":2160,"aspectRatio":"16:9","audioChannels":6,"audioCodec":"aac","videoCodec":"hevc","videoResolution":"4k","videoFrameRate":"23.976","videoProfile":"Main 10","audioProfile":"LC","Part":[{"id":9,"file":"/media/cloud/episode.strm","duration":60000,"size":301,"container":"mkv","videoProfile":"Main 10","audioProfile":"LC","Stream":[{"id":101,"streamType":1,"index":0,"codec":"hevc","profile":"Main 10","level":153,"bitrate":8000,"languageCode":"eng","title":"Video","width":3840,"height":2160,"frameRate":23.976,"refFrames":4,"pixelFormat":"yuv420p10le","bitDepth":10,"colorSpace":"bt2020nc","colorRange":"tv","colorPrimaries":"bt2020","colorTrc":"smpte2084","chromaLocation":"left","sampleAspectRatio":"1:1","displayAspectRatio":"16:9"}]}]}]}]}}`), + want: [][]byte{[]byte(`"size":987654321`), []byte(`"displayTitle":"4K DoVi/HDR10"`), []byte(`"DOVIPresent":true`)}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { + return mediainfo.Record{Key: request.Key, Media: media}, nil + }} + handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(test.body, test.contentType)) + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedMetadataRequest(http.MethodGet)) + if ensurer.calls.Load() != 1 { + t.Fatalf("Ensure calls = %d", ensurer.calls.Load()) + } + for _, expected := range test.want { + if !bytes.Contains(response.Body.Bytes(), expected) { + t.Fatalf("response missing %q: %s", expected, response.Body.Bytes()) + } + } + }) + } +} + func TestMetadataEnrichmentSkipsLocalAndCompleteCloudParts(t *testing.T) { fixture := newMetadataEnrichmentFixture(t) ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, _ mediainfo.Request) (mediainfo.Record, error) { @@ -269,7 +396,7 @@ func TestMetadataEnrichmentSkipsLocalAndCompleteCloudParts(t *testing.T) { }, { name: "complete cloud Part", - body: []byte(``), + body: []byte(``), }, } for _, test := range tests { diff --git a/internal/mediainfo/ffprobe.go b/internal/mediainfo/ffprobe.go index e4105ff..d760348 100644 --- a/internal/mediainfo/ffprobe.go +++ b/internal/mediainfo/ffprobe.go @@ -333,6 +333,9 @@ func normalizeStream(raw ffprobeStream) Stream { if raw.BitsPerRawSample.integer() > int64(bitDepth) { bitDepth = int(raw.BitsPerRawSample.integer()) } + if bitDepth == 0 { + bitDepth = pixelFormatBitDepth(raw.PixelFormat) + } stream := Stream{ Index: raw.Index, Type: strings.ToLower(raw.CodecType), @@ -378,6 +381,60 @@ func normalizeStream(raw ffprobeStream) Stream { return stream } +// pixelFormatBitDepth recovers the nominal component depth from common +// ffmpeg pixel-format spellings when ffprobe omits both bits_per_sample and +// bits_per_raw_sample. It intentionally returns zero for ambiguous formats. +func pixelFormatBitDepth(value string) int { + value = strings.ToLower(strings.TrimSpace(value)) + if value == "" { + return 0 + } + if strings.HasPrefix(value, "gray") { + if depth := pixelFormatDigits(value[len("gray"):]); depth > 0 { + return depth + } + } + for index := 0; index < len(value); index++ { + if value[index] != 'p' { + continue + } + if index+1 == len(value) { + if strings.HasPrefix(value, "yuv") || strings.HasPrefix(value, "gbr") || strings.HasPrefix(value, "rgb") { + return 8 + } + continue + } + if value[index+1] == '0' && index+3 < len(value) && isASCIIDigit(value[index+2]) && isASCIIDigit(value[index+3]) { + if depth := pixelFormatDigits(value[index+1 : index+4]); depth > 0 { + return depth + } + } + if depth := pixelFormatDigits(value[index+1:]); depth > 0 { + return depth + } + } + return 0 +} + +func pixelFormatDigits(value string) int { + end := 0 + for end < len(value) && isASCIIDigit(value[end]) { + end++ + } + if end == 0 { + return 0 + } + depth, err := strconv.Atoi(value[:end]) + if err != nil || depth < 8 || depth > 32 { + return 0 + } + return depth +} + +func isASCIIDigit(value byte) bool { + return value >= '0' && value <= '9' +} + func normalizeDisposition(raw ffprobeDisposition) Disposition { return Disposition{ Default: raw.Default != 0, Dub: raw.Dub != 0, Original: raw.Original != 0, @@ -436,7 +493,7 @@ func hdrFormat(stream Stream) string { } func applyMediaSummary(media *Media, stream Stream) { - if media == nil { + if media == nil || !primaryStream(stream) { return } switch stream.Type { @@ -465,6 +522,9 @@ func mediaComplete(media Media) bool { } mediaStreams := 0 for _, stream := range media.Streams { + if !primaryStream(stream) { + continue + } switch stream.Type { case "video": mediaStreams++ @@ -485,6 +545,18 @@ func mediaComplete(media Media) bool { return mediaStreams > 0 } +func primaryStream(stream Stream) bool { + if stream.Disposition.AttachedPicture { + return false + } + switch stream.Type { + case "video", "audio", "subtitle": + return true + default: + return false + } +} + func normalizeContainer(formatName string) string { formats := strings.Split(strings.ToLower(formatName), ",") for _, format := range formats { diff --git a/internal/mediainfo/ffprobe_test.go b/internal/mediainfo/ffprobe_test.go index cd91b5f..1e69262 100644 --- a/internal/mediainfo/ffprobe_test.go +++ b/internal/mediainfo/ffprobe_test.go @@ -101,6 +101,25 @@ func TestParseFFProbePreservesPlexRelevantFields(t *testing.T) { } } +func TestParseFFProbeInfersBitDepthFromPixelFormat(t *testing.T) { + body := []byte(`{ + "streams": [ + {"index":0,"codec_name":"hevc","codec_type":"video","width":3840,"height":2160,"pix_fmt":"yuv420p10le"}, + {"index":1,"codec_name":"hevc","codec_type":"video","width":1920,"height":1080,"pix_fmt":"p010le"}, + {"index":2,"codec_name":"h264","codec_type":"video","width":1280,"height":720,"pix_fmt":"yuv420p"} + ], + "format":{"format_name":"matroska","duration":"60"} +}`) + + media, err := parseFFProbe(body) + if err != nil { + t.Fatal(err) + } + if len(media.Streams) != 3 || media.Streams[0].BitDepth != 10 || media.Streams[1].BitDepth != 10 || media.Streams[2].BitDepth != 8 { + t.Fatalf("inferred bit depths = %#v", media.Streams) + } +} + func TestParseFFProbeRejectsIncompleteVideo(t *testing.T) { _, err := parseFFProbe([]byte(`{ "streams":[{"index":0,"codec_type":"video","codec_name":"hevc"}], diff --git a/internal/mediainfo/model.go b/internal/mediainfo/model.go index 5ebda19..5882873 100644 --- a/internal/mediainfo/model.go +++ b/internal/mediainfo/model.go @@ -19,8 +19,10 @@ const ( SchemaVersion = 1 // ProviderMediaVaultFFProbe identifies the redirect plus bounded ffprobe path. ProviderMediaVaultFFProbe = "mediavault-ffprobe" - // ProviderRevisionFFProbeJSONV1 identifies the normalized ffprobe contract. - ProviderRevisionFFProbeJSONV1 = "ffprobe-json-v1" + // ProviderRevisionFFProbeJSONV2 identifies the normalized ffprobe contract. + // It includes pixel-format bit-depth inference and therefore must not reuse + // records produced by the earlier interpretation. + ProviderRevisionFFProbeJSONV2 = "ffprobe-json-v2" maxSTRMFingerprintBytes = 64 << 10 ) diff --git a/internal/mediainfo/model_test.go b/internal/mediainfo/model_test.go index b4ba0bd..1b24209 100644 --- a/internal/mediainfo/model_test.go +++ b/internal/mediainfo/model_test.go @@ -174,7 +174,7 @@ func completeRecord(now time.Time) Record { }, RatingKey: "456", Provider: ProviderMediaVaultFFProbe, - ProviderRevision: ProviderRevisionFFProbeJSONV1, + ProviderRevision: ProviderRevisionFFProbeJSONV2, SchemaVersion: SchemaVersion, Media: Media{ Complete: true, diff --git a/internal/mediainfo/provider.go b/internal/mediainfo/provider.go index ea28d20..ebf75c4 100644 --- a/internal/mediainfo/provider.go +++ b/internal/mediainfo/provider.go @@ -61,7 +61,7 @@ func NewMediaVaultFFProbeProvider(controlResolver resolver.ControlResolver, prob // Descriptor returns the durable compatibility identity for this provider. func (*MediaVaultFFProbeProvider) Descriptor() ProviderDescriptor { - return ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV1} + return ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV2} } // Probe forwards only the transport headers required for deterministic diff --git a/internal/mediainfo/provider_test.go b/internal/mediainfo/provider_test.go index 9e804d4..488f546 100644 --- a/internal/mediainfo/provider_test.go +++ b/internal/mediainfo/provider_test.go @@ -57,7 +57,7 @@ func TestMediaVaultProviderUsesSameClientUserAgentForRedirectAndProbe(t *testing if probe.target != "https://cdn.example.test/movie.mkv" || probe.userAgent != "Infuse-Library/8.4.4" { t.Fatalf("probe target=%q User-Agent=%q", probe.target, probe.userAgent) } - if provider.Descriptor() != (ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV1}) || !result.Media.Complete { + if provider.Descriptor() != (ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV2}) || !result.Media.Complete { t.Fatalf("provider result = %#v", result) } } diff --git a/internal/mediainfo/service.go b/internal/mediainfo/service.go index 9561daa..258bd36 100644 --- a/internal/mediainfo/service.go +++ b/internal/mediainfo/service.go @@ -306,15 +306,21 @@ func NewService(options ServiceOptions) (*Service, error) { return service, nil } -// Get returns one exact fresh L1 record and renews its retention asynchronously. +// Get returns one exact retained record from L1 or the durable store without +// scheduling analysis or revalidation. Cache-only consumers can therefore use +// known-good stale data without turning enumeration into remote probe work. func (service *Service) Get(key Key) (Record, bool) { if service == nil { return Record{}, false } - now := service.now().UTC() - record, ok := service.cache.Get(key, now) + if strings.TrimSpace(key.PlexServerID) == "" { + key.PlexServerID = service.plexServerID + } + if key.PlexServerID != service.plexServerID { + return Record{}, false + } + record, ok, _ := service.lookup(service.ctx, key) if ok { - record = service.touchCached(record, now) service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoCacheHits() }) } else { service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoCacheMisses() }) diff --git a/internal/mediainfo/service_test.go b/internal/mediainfo/service_test.go index 4e77706..d0d4fec 100644 --- a/internal/mediainfo/service_test.go +++ b/internal/mediainfo/service_test.go @@ -56,7 +56,7 @@ type fakeProvider struct { } func (*fakeProvider) Descriptor() ProviderDescriptor { - return ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV1} + return ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV2} } func (fake *fakeProvider) Probe(ctx context.Context, request ProviderRequest) (ProviderResult, error) { @@ -323,11 +323,39 @@ func TestServiceReprobesIncompatibleProviderRevision(t *testing.T) { if err != nil { t.Fatal(err) } - if got.ProviderRevision != ProviderRevisionFFProbeJSONV1 || provider.calls.Load() != 1 { + if got.ProviderRevision != ProviderRevisionFFProbeJSONV2 || provider.calls.Load() != 1 { t.Fatalf("Ensure() revision=%q provider_calls=%d", got.ProviderRevision, provider.calls.Load()) } } +func TestServiceGetNormalizesPlexServerIdentity(t *testing.T) { + now := time.Date(2026, 8, 29, 8, 0, 0, 0, time.UTC) + record := completeRecord(now.Add(-31 * 24 * time.Hour)) + record.Key.PlexServerID = "server" + store := &fakeRecordStore{records: map[string]Record{record.Key.cacheKey(): record}} + provider := &fakeProvider{prober: &blockingProber{}} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, now), Store: store, + Provider: provider, + PlexServerID: "server", Now: func() time.Time { return now }, + }) + + key := record.Key + key.PlexServerID = "" + got, found := service.Get(key) + if !found || got.Key != record.Key || got.Fresh(now) { + t.Fatalf("Get() found=%v key=%#v, want %#v", found, got.Key, record.Key) + } + if provider.calls.Load() != 0 { + t.Fatalf("cache-only Get scheduled %d provider calls", provider.calls.Load()) + } + + key.PlexServerID = "other-server" + if _, found := service.Get(key); found { + t.Fatal("Get() returned a record for another Plex server") + } +} + func TestServiceReturnsStaleKnownGoodAndRevalidatesInBackground(t *testing.T) { now := time.Unix(1_800_000_000, 0).UTC() request := testRequest("stale", PriorityInteractive) diff --git a/internal/mediainfo/sqlite_test.go b/internal/mediainfo/sqlite_test.go index b977037..6693c8e 100644 --- a/internal/mediainfo/sqlite_test.go +++ b/internal/mediainfo/sqlite_test.go @@ -170,7 +170,7 @@ func TestSQLiteStoreLoadsMostRecentlyAccessedWithinLimit(t *testing.T) { } func testProviderDescriptor() ProviderDescriptor { - return ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV1} + return ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV2} } func TestSQLiteStoreDoesNotRestoreOtherProviderRevision(t *testing.T) { @@ -181,7 +181,7 @@ func TestSQLiteStoreDoesNotRestoreOtherProviderRevision(t *testing.T) { t.Fatal(err) } loaded, err := store.LoadCompatibleRetained(t.Context(), now, 100, ProviderDescriptor{ - Name: ProviderMediaVaultFFProbe, Revision: "ffprobe-json-v2", + Name: ProviderMediaVaultFFProbe, Revision: "ffprobe-json-v1", }) if err != nil { t.Fatal(err) diff --git a/internal/mediainfo/touch_lifecycle_test.go b/internal/mediainfo/touch_lifecycle_test.go index 7359039..b85ab0c 100644 --- a/internal/mediainfo/touch_lifecycle_test.go +++ b/internal/mediainfo/touch_lifecycle_test.go @@ -179,8 +179,10 @@ func TestServiceCloseCancelsPendingTouches(t *testing.T) { const touchInterval = time.Hour now := time.Unix(1_800_000_000, 0).UTC() first := completeRecord(now.Add(-2 * touchInterval)) + first.Key.PlexServerID = "server" first.Key.PartID = "close-first" second := completeRecord(now.Add(-2 * touchInterval)) + second.Key.PlexServerID = "server" second.Key.PartID = "close-second" store := &cancelAwareTouchStore{ started: make(chan struct{}, 2), diff --git a/internal/plexmeta/projection.go b/internal/plexmeta/projection.go index 55b4e8a..a55e566 100644 --- a/internal/plexmeta/projection.go +++ b/internal/plexmeta/projection.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "math" "strconv" "strings" @@ -15,8 +16,8 @@ import ( // Target identifies the single Plex item and reports whether the response is // missing a whitelisted technical field that a MediaInfo record can project. -// A missing Stream is not itself a reason to enrich: projection never creates -// Stream elements because their Plex identity is not known to the gateway. +// A Part with no Stream elements is eligible for descriptive Stream creation +// because ffprobe supplies the stable stream type and source index. type Target struct { Part Part // NeedsEnrichment is true when a whitelisted field is absent from the @@ -46,10 +47,10 @@ func SelectEnrichmentPart(body []byte, contentType, ratingKey string) (Part, err return target.Part, nil } -// EnrichMetadata fills only absent whitelisted Media, Part, and already -// present Stream attributes. SelectEnrichmentTarget supplies the exact Part -// to callers that need to build a probe request before this function runs. -// The input body is never modified in place. When no field is filled, the +// EnrichMetadata fills only absent whitelisted Media, Part, and matching +// Stream attributes. When a Part has no Stream elements, it creates +// descriptive video, audio, and subtitle elements from the probe record. The +// input body is never modified in place. When no field is filled, the // original bytes are returned unchanged. func EnrichMetadata(body []byte, contentType, ratingKey string, media mediainfo.Media) ([]byte, bool, error) { document, selection, err := parseProjection(body, contentType, ratingKey) @@ -129,9 +130,10 @@ func trimProjectionBody(body []byte) []byte { // whitelist contains only descriptive technical Plex fields; identifiers and // playback-selection fields are never generated. type projectionAttribute struct { - name string - value string - number bool + name string + value string + number bool + boolean bool } func textProjectionAttribute(name, value string) projectionAttribute { @@ -142,6 +144,29 @@ func integerProjectionAttribute(name string, value int64) projectionAttribute { return projectionAttribute{name: name, value: strconv.FormatInt(value, 10), number: true} } +func booleanProjectionAttribute(name string, value bool) projectionAttribute { + if value { + return projectionAttribute{name: name, value: "1", boolean: true} + } + return projectionAttribute{name: name, value: "0", boolean: true} +} + +const strmControlSizeThreshold int64 = 1 << 20 + +// bitrateKbps converts ffprobe's bits-per-second value to Plex's integer +// kilobits-per-second representation. A positive sub-kilobit value remains +// visible as one rather than being rounded down to zero. +func bitrateKbps(value int64) int64 { + if value <= 0 { + return 0 + } + value /= 1000 + if value == 0 { + return 1 + } + return value +} + func mediaProjectionAttributes(media mediainfo.Media) []projectionAttribute { attributes := make([]projectionAttribute, 0, 14) if media.Container != "" { @@ -151,7 +176,7 @@ func mediaProjectionAttributes(media mediainfo.Media) []projectionAttribute { attributes = append(attributes, integerProjectionAttribute("duration", media.DurationMS)) } if media.Bitrate > 0 { - attributes = append(attributes, integerProjectionAttribute("bitrate", media.Bitrate)) + attributes = append(attributes, integerProjectionAttribute("bitrate", bitrateKbps(media.Bitrate))) } if media.Width > 0 { attributes = append(attributes, integerProjectionAttribute("width", int64(media.Width))) @@ -174,8 +199,8 @@ func mediaProjectionAttributes(media mediainfo.Media) []projectionAttribute { if media.VideoResolution != "" { attributes = append(attributes, textProjectionAttribute("videoResolution", media.VideoResolution)) } - if media.FrameRate != "" { - attributes = append(attributes, textProjectionAttribute("videoFrameRate", media.FrameRate)) + if frameRate := plexMediaFrameRate(media.FrameRate); frameRate != "" { + attributes = append(attributes, textProjectionAttribute("videoFrameRate", frameRate)) } if media.VideoProfile != "" { attributes = append(attributes, textProjectionAttribute("videoProfile", media.VideoProfile)) @@ -207,9 +232,9 @@ func partProjectionAttributes(media mediainfo.Media) []projectionAttribute { } func streamProjectionAttributes(stream mediainfo.Stream) []projectionAttribute { - attributes := make([]projectionAttribute, 0, 20) + attributes := make([]projectionAttribute, 0, 35) if stream.Codec != "" { - attributes = append(attributes, textProjectionAttribute("codec", stream.Codec)) + attributes = append(attributes, textProjectionAttribute("codec", plexCodec(stream.Codec))) } if stream.Profile != "" { attributes = append(attributes, textProjectionAttribute("profile", stream.Profile)) @@ -218,7 +243,7 @@ func streamProjectionAttributes(stream mediainfo.Stream) []projectionAttribute { attributes = append(attributes, integerProjectionAttribute("level", int64(stream.Level))) } if stream.Bitrate > 0 { - attributes = append(attributes, integerProjectionAttribute("bitrate", stream.Bitrate)) + attributes = append(attributes, integerProjectionAttribute("bitrate", bitrateKbps(stream.Bitrate))) } if stream.Width > 0 { attributes = append(attributes, integerProjectionAttribute("width", int64(stream.Width))) @@ -226,9 +251,9 @@ func streamProjectionAttributes(stream mediainfo.Stream) []projectionAttribute { if stream.Height > 0 { attributes = append(attributes, integerProjectionAttribute("height", int64(stream.Height))) } - frameRate := firstProjectionString(stream.FrameRate, stream.AverageFrameRate) + frameRate := plexFrameRate(firstProjectionString(stream.FrameRate, stream.AverageFrameRate)) if frameRate != "" { - attributes = append(attributes, textProjectionAttribute("frameRate", frameRate)) + attributes = append(attributes, projectionAttribute{name: "frameRate", value: frameRate, number: true}) } if stream.ReferenceFrames > 0 { attributes = append(attributes, integerProjectionAttribute("refFrames", int64(stream.ReferenceFrames))) @@ -266,15 +291,203 @@ func streamProjectionAttributes(stream mediainfo.Stream) []projectionAttribute { if stream.Channels > 0 { attributes = append(attributes, integerProjectionAttribute("channels", int64(stream.Channels))) } - if stream.ChannelLayout != "" { - attributes = append(attributes, textProjectionAttribute("channelLayout", stream.ChannelLayout)) + if stream.ChannelLayout != "" && canonicalStreamType(stream.Type) == "2" { + attributes = append(attributes, textProjectionAttribute("audioChannelLayout", stream.ChannelLayout)) } - if stream.Language != "" { - attributes = append(attributes, textProjectionAttribute("language", stream.Language)) + if code, ok := plexLanguageCode(stream.Language); ok { + attributes = append(attributes, textProjectionAttribute("languageCode", code)) } if stream.Title != "" { attributes = append(attributes, textProjectionAttribute("title", stream.Title)) } + if canonicalStreamType(stream.Type) == "1" { + if displayTitle, extendedDisplayTitle := videoDisplayTitles(stream); displayTitle != "" { + attributes = append(attributes, textProjectionAttribute("displayTitle", displayTitle)) + if extendedDisplayTitle != "" { + attributes = append(attributes, textProjectionAttribute("extendedDisplayTitle", extendedDisplayTitle)) + } + } + } + if stream.DolbyVision != nil && canonicalStreamType(stream.Type) == "1" { + attributes = append(attributes, dolbyVisionProjectionAttributes(stream.DolbyVision)...) + } + attributes = append(attributes, dispositionProjectionAttributes(stream.Disposition)...) + return attributes +} + +// plexCodec keeps the normalized codec vocabulary expected by Plex clients. +// ffprobe may expose a container-specific subtitle codec name that Plex +// represents using its short format name. +func plexCodec(value string) string { + value = strings.ToLower(strings.TrimSpace(value)) + switch value { + case "hdmv_pgs_subtitle", "pgs": + return "pgs" + case "subrip": + return "srt" + default: + return value + } +} + +func plexLanguageCode(value string) (string, bool) { + value = strings.TrimSpace(value) + if len(value) != 2 && len(value) != 3 { + return "", false + } + for index := 0; index < len(value); index++ { + if (value[index] < 'a' || value[index] > 'z') && (value[index] < 'A' || value[index] > 'Z') { + return "", false + } + } + return strings.ToLower(value), true +} + +func videoDisplayTitles(stream mediainfo.Stream) (string, string) { + label := "" + switch strings.ToLower(strings.TrimSpace(stream.HDRFormat)) { + case "dolby_vision", "dolbyvision", "dovi", "dv": + label = "DoVi" + if stream.DolbyVision != nil && dolbyVisionHDR10Compatible(stream.DolbyVision.BLCompatID) { + label = "DoVi/HDR10" + } + case "hdr10": + label = "HDR10" + case "hlg": + label = "HLG" + } + resolution := plexVideoResolution(stream) + displayTitle := strings.TrimSpace(strings.Join([]string{resolution, label}, " ")) + if displayTitle == "" { + return "", "" + } + codec := plexDisplayCodec(stream.Codec) + codecAndProfile := strings.TrimSpace(strings.Join([]string{codec, strings.TrimSpace(stream.Profile)}, " ")) + if codecAndProfile == "" { + return displayTitle, "" + } + return displayTitle, displayTitle + " (" + codecAndProfile + ")" +} + +func dolbyVisionHDR10Compatible(compatibilityID int) bool { + // Plex labels the two observed HDR10-compatible signal identifiers as a + // combined DoVi/HDR10 presentation. Other identifiers may describe SDR or + // HLG compatibility and must not be promoted to HDR10. + return compatibilityID == 1 || compatibilityID == 6 +} + +// plexFrameRate converts ffprobe ratios to the decimal representation emitted +// by Plex Stream metadata while preserving already-decimal values. +func plexFrameRate(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + numerator, denominator, fraction := strings.Cut(value, "/") + if !fraction { + parsed, err := strconv.ParseFloat(value, 64) + if err != nil || parsed <= 0 { + return "" + } + return strings.TrimRight(strings.TrimRight(strconv.FormatFloat(parsed, 'f', 3, 64), "0"), ".") + } + n, err := strconv.ParseFloat(strings.TrimSpace(numerator), 64) + if err != nil || n <= 0 { + return "" + } + d, err := strconv.ParseFloat(strings.TrimSpace(denominator), 64) + if err != nil || d <= 0 { + return "" + } + return strings.TrimRight(strings.TrimRight(strconv.FormatFloat(n/d, 'f', 3, 64), "0"), ".") +} + +// plexMediaFrameRate converts ffprobe's measured frame rate to Plex's nominal +// media classification. Stream metadata retains the precise decimal value so +// clients can distinguish a 23.976 stream from the Media-level 24p label. +func plexMediaFrameRate(value string) string { + decimal := plexFrameRate(value) + if decimal == "" { + return "" + } + fps, err := strconv.ParseFloat(decimal, 64) + if err != nil || fps <= 0 || math.IsNaN(fps) || math.IsInf(fps, 0) { + return "" + } + nominal := math.Round(fps) + if nominal <= 0 || math.IsInf(nominal, 0) { + return "" + } + return strconv.FormatFloat(nominal, 'f', 0, 64) + "p" +} + +func plexVideoResolution(stream mediainfo.Stream) string { + switch { + case stream.Width >= 3800 || stream.Height >= 2100: + return "4K" + case stream.Height >= 1400: + return "2K" + case stream.Height >= 1000: + return "1080p" + case stream.Height >= 700: + return "720p" + case stream.Height > 0: + return strconv.Itoa(stream.Height) + "p" + } + return "" +} + +func plexDisplayCodec(value string) string { + switch plexCodec(value) { + case "hevc": + return "HEVC" + case "h264": + return "H.264" + case "mpeg4": + return "MPEG-4" + case "av1": + return "AV1" + case "vp9": + return "VP9" + default: + return strings.ToUpper(plexCodec(value)) + } +} + +func dolbyVisionProjectionAttributes(dolby *mediainfo.DolbyVision) []projectionAttribute { + if dolby == nil { + return nil + } + attributes := []projectionAttribute{ + integerProjectionAttribute("DOVIBLCompatID", int64(dolby.BLCompatID)), + booleanProjectionAttribute("DOVIBLPresent", dolby.BLPresent != 0), + booleanProjectionAttribute("DOVIELPresent", dolby.ELPresent != 0), + booleanProjectionAttribute("DOVIPresent", true), + booleanProjectionAttribute("DOVIRPUPresent", dolby.RPUPresent != 0), + } + if dolby.Level > 0 { + attributes = append(attributes, integerProjectionAttribute("DOVILevel", int64(dolby.Level))) + } + if dolby.Profile > 0 { + attributes = append(attributes, integerProjectionAttribute("DOVIProfile", int64(dolby.Profile))) + } + if dolby.VersionMajor > 0 || dolby.VersionMinor > 0 { + attributes = append(attributes, textProjectionAttribute("DOVIVersion", fmt.Sprintf("%d.%d", dolby.VersionMajor, dolby.VersionMinor))) + } + return attributes +} + +func dispositionProjectionAttributes(disposition mediainfo.Disposition) []projectionAttribute { + attributes := make([]projectionAttribute, 0, 3) + if disposition.Forced { + attributes = append(attributes, booleanProjectionAttribute("forced", true)) + } + if disposition.HearingImpaired { + attributes = append(attributes, booleanProjectionAttribute("hearingImpaired", true)) + } + if disposition.VisualImpaired { + attributes = append(attributes, booleanProjectionAttribute("visualImpaired", true)) + } return attributes } @@ -296,16 +509,21 @@ var mediaProjectionAttributeNames = []string{ var partProjectionAttributeNames = []string{"duration", "size", "container", "videoProfile", "audioProfile"} var commonStreamProjectionAttributeNames = []string{ - "codec", "profile", "level", "bitrate", "language", "title", + "codec", "profile", "level", "bitrate", "languageCode", "title", } var videoStreamProjectionAttributeNames = []string{ "width", "height", "frameRate", "refFrames", "pixelFormat", "bitDepth", "colorSpace", "colorRange", "colorPrimaries", "colorTrc", "chromaLocation", - "sampleAspectRatio", "displayAspectRatio", + "sampleAspectRatio", "displayAspectRatio", "displayTitle", "extendedDisplayTitle", +} + +var dolbyVisionProjectionAttributeNames = []string{ + "DOVIBLCompatID", "DOVIBLPresent", "DOVIELPresent", "DOVILevel", + "DOVIPresent", "DOVIProfile", "DOVIRPUPresent", "DOVIVersion", } -var audioStreamProjectionAttributeNames = []string{"samplingRate", "channels", "channelLayout"} +var audioStreamProjectionAttributeNames = []string{"samplingRate", "channels", "audioChannelLayout"} type streamKey struct { typeName string @@ -321,6 +539,9 @@ func buildStreamProjections(media mediainfo.Media) ([]streamProjection, error) { projections := make([]streamProjection, 0, len(media.Streams)) seen := make(map[streamKey]struct{}, len(media.Streams)) for _, stream := range media.Streams { + if !projectableStream(stream) { + continue + } key, ok := mediaStreamKey(stream.Type, stream.Index) if !ok { continue @@ -337,6 +558,18 @@ func buildStreamProjections(media mediainfo.Media) ([]streamProjection, error) { return projections, nil } +func projectableStream(stream mediainfo.Stream) bool { + if stream.Disposition.AttachedPicture { + return false + } + switch canonicalStreamType(stream.Type) { + case "1", "2", "3": + return true + default: + return false + } +} + func mediaStreamKey(typeName string, index int) (streamKey, bool) { if index < 0 { return streamKey{}, false @@ -529,11 +762,19 @@ func (selection *xmlProjectionSelection) enrich(media mediainfo.Media) (bool, er } changed := addMissingXMLAttributes(selection.media, mediaProjectionAttributes(media)) changed = addMissingXMLAttributes(selection.part, partProjectionAttributes(media)) || changed + changed = replaceSTRMPartSizeXML(selection.part, media.Size) || changed streamElements := directXMLElements(selection.part, "Stream") streamIndex, err := indexXMLStreams(streamElements) if err != nil { return false, err } + if len(streamElements) == 0 { + for _, projection := range streamProjections { + selection.part.children = append(selection.part.children, &xmlProjectionNode{element: newXMLStream(projection)}) + changed = true + } + return changed, nil + } for _, projection := range streamProjections { stream, ok := streamIndex[projection.key] if !ok { @@ -544,6 +785,44 @@ func (selection *xmlProjectionSelection) enrich(media mediainfo.Media) (bool, er return changed, nil } +func newXMLStream(projection streamProjection) *xmlProjectionElement { + attributes := make([]xml.Attr, 0, len(projection.attributes)+2) + attributes = append(attributes, + xml.Attr{Name: xml.Name{Local: "streamType"}, Value: projection.key.typeName}, + xml.Attr{Name: xml.Name{Local: "index"}, Value: strconv.Itoa(projection.key.index)}, + ) + for _, attribute := range projection.attributes { + attributes = append(attributes, xml.Attr{Name: xml.Name{Local: attribute.name}, Value: attribute.value}) + } + return &xmlProjectionElement{start: xml.StartElement{Name: xml.Name{Local: "Stream"}, Attr: attributes}} +} + +func replaceSTRMPartSizeXML(part *xmlProjectionElement, mediaSize int64) bool { + if mediaSize <= strmControlSizeThreshold { + return false + } + file, present, err := xmlAttribute(part.start.Attr, "file") + if err != nil || !present || !isSTRMPath(file) { + return false + } + for index := range part.start.Attr { + if part.start.Attr[index].Name.Local != "size" { + continue + } + current, err := strconv.ParseInt(strings.TrimSpace(part.start.Attr[index].Value), 10, 64) + if err != nil || current < 0 || current > strmControlSizeThreshold || mediaSize <= current { + return false + } + part.start.Attr[index].Value = strconv.FormatInt(mediaSize, 10) + return true + } + return false +} + +func isSTRMPath(value string) bool { + return strings.HasSuffix(strings.ToLower(strings.TrimSpace(value)), ".strm") +} + func partFromXML(element *xmlProjectionElement) (Part, error) { id, present, err := xmlAttribute(element.start.Attr, "id") if err != nil { @@ -602,10 +881,18 @@ func xmlTargetNeedsEnrichment(media, part *xmlProjectionElement) (bool, error) { needs = true } } + if correction, err := xmlSTRMPartSizeNeedsCorrection(part); err != nil { + return false, err + } else if correction { + needs = true + } streams := directXMLElements(part, "Stream") if _, err := indexXMLStreams(streams); err != nil { return false, err } + if len(streams) == 0 { + return true, nil + } for _, stream := range streams { key, present, err := xmlStreamIdentity(stream) if err != nil { @@ -615,7 +902,7 @@ func xmlTargetNeedsEnrichment(media, part *xmlProjectionElement) (bool, error) { continue } for _, name := range streamProjectionAttributeNames(key.typeName) { - _, present, err := xmlAttribute(stream.start.Attr, name) + present, err := xmlProjectionAttributePresent(stream, name) if err != nil { return false, err } @@ -623,10 +910,80 @@ func xmlTargetNeedsEnrichment(media, part *xmlProjectionElement) (bool, error) { needs = true } } + if key.typeName == "1" { + incomplete, err := xmlDolbyVisionFieldsIncomplete(stream) + if err != nil { + return false, err + } + needs = needs || incomplete + } } return needs, nil } +func xmlSTRMPartSizeNeedsCorrection(part *xmlProjectionElement) (bool, error) { + file, present, err := xmlAttribute(part.start.Attr, "file") + if err != nil || !present || !isSTRMPath(file) { + return false, err + } + size, present, err := xmlAttribute(part.start.Attr, "size") + if err != nil || !present { + return false, err + } + value, err := strconv.ParseInt(strings.TrimSpace(size), 10, 64) + return err == nil && value >= 0 && value <= strmControlSizeThreshold, nil +} + +func xmlDolbyVisionFieldsIncomplete(stream *xmlProjectionElement) (bool, error) { + displayTitle, present, err := xmlAttribute(stream.start.Attr, "displayTitle") + if err != nil { + return false, err + } + dolbyKnown := present && strings.Contains(strings.ToLower(displayTitle), "dovi") + for _, name := range dolbyVisionProjectionAttributeNames { + _, fieldPresent, err := xmlAttribute(stream.start.Attr, name) + if err != nil { + return false, err + } + dolbyKnown = dolbyKnown || fieldPresent + } + if !dolbyKnown { + return false, nil + } + for _, name := range dolbyVisionProjectionAttributeNames { + _, fieldPresent, err := xmlAttribute(stream.start.Attr, name) + if err != nil || !fieldPresent { + return !fieldPresent, err + } + } + return false, nil +} + +func xmlProjectionAttributePresent(stream *xmlProjectionElement, name string) (bool, error) { + _, present, err := xmlAttribute(stream.start.Attr, name) + if err != nil || present { + return present, err + } + for _, alias := range projectionAttributeAliases(name) { + _, present, err = xmlAttribute(stream.start.Attr, alias) + if err != nil || present { + return present, err + } + } + return false, nil +} + +func projectionAttributeAliases(name string) []string { + switch name { + case "languageCode": + return []string{"language"} + case "audioChannelLayout": + return []string{"channelLayout"} + default: + return nil + } +} + func indexXMLStreams(streams []*xmlProjectionElement) (map[streamKey]*xmlProjectionElement, error) { result := make(map[streamKey]*xmlProjectionElement, len(streams)) for _, stream := range streams { @@ -989,35 +1346,55 @@ func (selection *jsonProjectionSelection) enrich(media mediainfo.Media) (bool, e return false, err } changed = changed || partChanged - streamRaw, exists := (*selection.part)["Stream"] - if !exists { - return changed, nil - } - streams, err := decodeJSONProjectionList(streamRaw, "Stream") + partSizeChanged, err := replaceSTRMPartSizeJSON(*selection.part, media.Size) if err != nil { return false, err } - streamIndex, err := indexJSONStreams(streams.items) - if err != nil { - return false, err - } - for _, projection := range streamProjections { - stream, ok := streamIndex[projection.key] - if !ok { - continue + changed = changed || partSizeChanged + streamRaw, exists := (*selection.part)["Stream"] + if !exists { + if len(streamProjections) > 0 { + if err := setGeneratedJSONStreams(*selection.part, streamProjections); err != nil { + return false, err + } + changed = true } - streamChanged, err := addMissingJSONAttributes(*stream, projection.attributes) + } else { + streams, err := decodeJSONProjectionList(streamRaw, "Stream") if err != nil { return false, err } - changed = changed || streamChanged - } - if changed { - encoded, err := encodeJSONProjectionList(streams) - if err != nil { - return false, err + if len(streams.items) == 0 { + if len(streamProjections) > 0 { + if err := setGeneratedJSONStreams(*selection.part, streamProjections); err != nil { + return false, err + } + changed = true + } + } else { + streamIndex, err := indexJSONStreams(streams.items) + if err != nil { + return false, err + } + for _, projection := range streamProjections { + stream, ok := streamIndex[projection.key] + if !ok { + continue + } + streamChanged, err := addMissingJSONAttributes(*stream, projection.attributes) + if err != nil { + return false, err + } + changed = changed || streamChanged + } + if changed { + encoded, err := encodeJSONProjectionList(streams) + if err != nil { + return false, err + } + (*selection.part)["Stream"] = encoded + } } - (*selection.part)["Stream"] = encoded } if changed { if err := selection.commit(); err != nil { @@ -1027,6 +1404,74 @@ func (selection *jsonProjectionSelection) enrich(media mediainfo.Media) (bool, e return changed, nil } +func setGeneratedJSONStreams(part jsonObject, projections []streamProjection) error { + streams := jsonProjectionList{items: make([]*jsonObject, 0, len(projections)), array: true} + for _, projection := range projections { + stream := make(jsonObject) + if _, err := addMissingJSONAttributes(stream, generatedStreamAttributes(projection)); err != nil { + return err + } + streams.items = append(streams.items, &stream) + } + encoded, err := encodeJSONProjectionList(streams) + if err != nil { + return err + } + part["Stream"] = encoded + return nil +} + +func generatedStreamAttributes(projection streamProjection) []projectionAttribute { + attributes := make([]projectionAttribute, 0, len(projection.attributes)+2) + attributes = append(attributes, + integerProjectionAttribute("streamType", int64(streamTypeNumber(projection.key.typeName))), + integerProjectionAttribute("index", int64(projection.key.index)), + ) + return append(attributes, projection.attributes...) +} + +func streamTypeNumber(typeName string) int { + switch typeName { + case "1": + return 1 + case "2": + return 2 + case "3": + return 3 + default: + return 0 + } +} + +func replaceSTRMPartSizeJSON(part jsonObject, mediaSize int64) (bool, error) { + if mediaSize <= strmControlSizeThreshold { + return false, nil + } + file, err := requiredJSONString(part, "file") + if err != nil || !isSTRMPath(file) { + return false, nil + } + raw, present := part["size"] + if !present { + return false, nil + } + current, ok := jsonRawInt64(raw) + if !ok || current < 0 || current > strmControlSizeThreshold || mediaSize <= current { + return false, nil + } + part["size"] = json.RawMessage(strconv.FormatInt(mediaSize, 10)) + return true, nil +} + +func jsonRawInt64(raw json.RawMessage) (int64, bool) { + value, ok := jsonRawScalarString(raw) + if !ok { + return 0, false + } + parsed, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + return parsed, err == nil +} + func (selection *jsonProjectionSelection) commit() error { parts, err := encodeJSONProjectionList(selection.parts) if err != nil { @@ -1130,14 +1575,20 @@ func jsonTargetNeedsEnrichment(media, part *jsonObject) (bool, error) { needs = true } } + if jsonSTRMPartSizeNeedsCorrection(*part) { + needs = true + } streamRaw, exists := (*part)["Stream"] if !exists { - return needs, nil + return true, nil } streams, err := decodeJSONProjectionList(streamRaw, "Stream") if err != nil { return false, err } + if len(streams.items) == 0 { + return true, nil + } if _, err := indexJSONStreams(streams.items); err != nil { return false, err } @@ -1150,14 +1601,64 @@ func jsonTargetNeedsEnrichment(media, part *jsonObject) (bool, error) { continue } for _, name := range streamProjectionAttributeNames(key.typeName) { - if _, present := (*stream)[name]; !present { + if !jsonProjectionAttributePresent(*stream, name) { needs = true } } + if key.typeName == "1" && jsonDolbyVisionFieldsIncomplete(*stream) { + needs = true + } } return needs, nil } +func jsonSTRMPartSizeNeedsCorrection(part jsonObject) bool { + file, err := requiredJSONString(part, "file") + if err != nil || !isSTRMPath(file) { + return false + } + raw, present := part["size"] + if !present { + return false + } + value, ok := jsonRawInt64(raw) + return ok && value >= 0 && value <= strmControlSizeThreshold +} + +func jsonDolbyVisionFieldsIncomplete(stream jsonObject) bool { + dolbyKnown := false + if raw, present := stream["displayTitle"]; present { + if value, ok := jsonRawScalarString(raw); ok { + dolbyKnown = strings.Contains(strings.ToLower(value), "dovi") + } + } + for _, name := range dolbyVisionProjectionAttributeNames { + _, present := stream[name] + dolbyKnown = dolbyKnown || present + } + if !dolbyKnown { + return false + } + for _, name := range dolbyVisionProjectionAttributeNames { + if _, present := stream[name]; !present { + return true + } + } + return false +} + +func jsonProjectionAttributePresent(stream jsonObject, name string) bool { + if _, present := stream[name]; present { + return true + } + for _, alias := range projectionAttributeAliases(name) { + if _, present := stream[alias]; present { + return true + } + } + return false +} + func indexJSONStreams(streams []*jsonObject) (map[streamKey]*jsonObject, error) { result := make(map[streamKey]*jsonObject, len(streams)) for _, stream := range streams { @@ -1209,7 +1710,13 @@ func addMissingJSONAttributes(object jsonObject, attributes []projectionAttribut } var raw []byte var err error - if attribute.number { + if attribute.boolean { + if attribute.value == "1" { + raw = []byte("true") + } else { + raw = []byte("false") + } + } else if attribute.number { raw = []byte(attribute.value) } else { raw, err = json.Marshal(attribute.value) diff --git a/internal/plexmeta/projection_test.go b/internal/plexmeta/projection_test.go index d23a7d9..90ab7f4 100644 --- a/internal/plexmeta/projection_test.go +++ b/internal/plexmeta/projection_test.go @@ -132,14 +132,289 @@ func TestEnrichMetadataJSONFillsOnlyMissingSafeFields(t *testing.T) { } } -func TestEnrichMetadataDoesNotCreateStreams(t *testing.T) { - body := []byte(``) - result, changed, err := EnrichMetadata(body, "application/xml", "42", projectionTestMedia()) - if err != nil { - t.Fatal(err) +func TestEnrichMetadataCreatesStreamsWhenPartHasNone(t *testing.T) { + media := projectionStreamMedia() + tests := []struct { + name string + contentType string + body []byte + }{ + { + name: "XML", + contentType: "application/xml", + body: []byte(``), + }, + { + name: "JSON", + contentType: "application/json", + body: []byte(`{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"Part":[{"id":7,"file":"/cloud/movie.strm"}]}]}]}}`), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, changed, err := EnrichMetadata(test.body, test.contentType, "42", media) + if err != nil { + t.Fatal(err) + } + if !changed || !bytes.Contains(result, []byte("Stream")) { + t.Fatalf("result = %s, want generated Stream elements", result) + } + if test.contentType == "application/xml" { + attrs := xmlProjectionAttributeSets(t, result) + if attrs["Media"]["bitrate"] != "29430" || attrs["Part"]["size"] != "987654321" { + t.Fatalf("projected XML units = media=%#v part=%#v", attrs["Media"], attrs["Part"]) + } + if attrs["Stream#1"]["streamType"] != "1" || attrs["Stream#1"]["index"] != "0" || + attrs["Stream#2"]["streamType"] != "2" || attrs["Stream#2"]["index"] != "1" || + attrs["Stream#3"]["streamType"] != "3" || attrs["Stream#3"]["index"] != "2" { + t.Fatalf("generated XML stream identities = %#v", attrs) + } + if attrs["Stream#1"]["bitrate"] != "29430" || attrs["Stream#1"]["frameRate"] != "23.976" || + attrs["Stream#1"]["displayTitle"] != "4K HDR10" || + attrs["Stream#1"]["extendedDisplayTitle"] != "4K HDR10 (HEVC Main 10)" || + attrs["Stream#1"]["colorTrc"] != "smpte2084" { + t.Fatalf("generated HDR10 attributes = %#v", attrs["Stream#1"]) + } + if attrs["Stream#2"]["bitrate"] != "5000" || attrs["Stream#2"]["audioChannelLayout"] != "7.1" { + t.Fatalf("generated audio attributes = %#v", attrs["Stream#2"]) + } + for _, name := range []string{"id", "selected", "default", "decision"} { + for _, streamName := range []string{"Stream#1", "Stream#2", "Stream#3"} { + if _, present := attrs[streamName][name]; present { + t.Fatalf("generated XML Stream contains forbidden field %q: %#v", name, attrs[streamName]) + } + } + } + if attrs["Stream#3"]["codec"] != "pgs" || attrs["Stream#3"]["languageCode"] != "zho" || attrs["Stream#3"]["forced"] != "1" { + t.Fatalf("generated subtitle attributes = %#v", attrs["Stream#3"]) + } + return + } + var document map[string]any + if err := json.Unmarshal(result, &document); err != nil { + t.Fatal(err) + } + part := document["MediaContainer"].(map[string]any)["Metadata"].([]any)[0].(map[string]any)["Media"].([]any)[0].(map[string]any)["Part"].([]any)[0].(map[string]any) + mediaObject := document["MediaContainer"].(map[string]any)["Metadata"].([]any)[0].(map[string]any)["Media"].([]any)[0].(map[string]any) + if mediaObject["bitrate"] != float64(29430) || part["size"] != float64(987654321) { + t.Fatalf("projected JSON units = media=%#v part=%#v", mediaObject, part) + } + streams := part["Stream"].([]any) + if len(streams) != 3 { + t.Fatalf("generated JSON streams = %#v", streams) + } + for _, rawStream := range streams { + stream := rawStream.(map[string]any) + for _, name := range []string{"id", "selected", "default", "decision"} { + if _, present := stream[name]; present { + t.Fatalf("generated JSON Stream contains forbidden field %q: %#v", name, stream) + } + } + } + if streams[0].(map[string]any)["streamType"] != float64(1) || streams[0].(map[string]any)["index"] != float64(0) || + streams[1].(map[string]any)["streamType"] != float64(2) || streams[1].(map[string]any)["index"] != float64(1) || + streams[2].(map[string]any)["streamType"] != float64(3) || streams[2].(map[string]any)["index"] != float64(2) { + t.Fatalf("generated JSON stream identities = %#v", streams) + } + if streams[2].(map[string]any)["codec"] != "pgs" || streams[2].(map[string]any)["languageCode"] != "zho" || streams[2].(map[string]any)["forced"] != true { + t.Fatalf("generated JSON subtitle attributes = %#v", streams[2]) + } + if streams[0].(map[string]any)["bitrate"] != float64(29430) || streams[0].(map[string]any)["frameRate"] != 23.976 || + streams[0].(map[string]any)["displayTitle"] != "4K HDR10" || + streams[0].(map[string]any)["extendedDisplayTitle"] != "4K HDR10 (HEVC Main 10)" || + streams[0].(map[string]any)["colorTrc"] != "smpte2084" { + t.Fatalf("generated JSON HDR10 attributes = %#v", streams[0]) + } + if streams[1].(map[string]any)["bitrate"] != float64(5000) || streams[1].(map[string]any)["audioChannelLayout"] != "7.1" { + t.Fatalf("generated JSON audio attributes = %#v", streams[1]) + } + }) + } +} + +func TestEnrichMetadataUsesDolbyVisionPlexFields(t *testing.T) { + media := projectionStreamMedia() + media.Streams[0].HDRFormat = "dolby_vision" + media.Streams[0].DolbyVision = &mediainfo.DolbyVision{ + VersionMajor: 1, VersionMinor: 0, Profile: 7, Level: 6, + RPUPresent: 1, ELPresent: 1, BLPresent: 1, BLCompatID: 6, + } + for _, test := range []struct { + name string + contentType string + body []byte + }{ + { + name: "XML", + contentType: "application/xml", + body: []byte(``), + }, + { + name: "JSON", + contentType: "application/json", + body: []byte(`{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"Part":[{"id":7,"file":"/cloud/movie.strm"}]}]}]}}`), + }, + } { + t.Run(test.name, func(t *testing.T) { + result, changed, err := EnrichMetadata(test.body, test.contentType, "42", media) + if err != nil || !changed { + t.Fatalf("EnrichMetadata changed=%v err=%v", changed, err) + } + if test.contentType == "application/xml" { + attrs := xmlProjectionAttributeSets(t, result)["Stream#1"] + assertDolbyVisionAttributes(t, attrs) + return + } + var document map[string]any + if err := json.Unmarshal(result, &document); err != nil { + t.Fatal(err) + } + part := document["MediaContainer"].(map[string]any)["Metadata"].([]any)[0].(map[string]any)["Media"].([]any)[0].(map[string]any)["Part"].([]any)[0].(map[string]any) + attrs := part["Stream"].([]any)[0].(map[string]any) + if attrs["displayTitle"] != "4K DoVi/HDR10" || attrs["extendedDisplayTitle"] != "4K DoVi/HDR10 (HEVC Main 10)" || + attrs["DOVIBLCompatID"] != float64(6) || attrs["DOVIBLPresent"] != true || + attrs["DOVIELPresent"] != true || attrs["DOVILevel"] != float64(6) || + attrs["DOVIPresent"] != true || attrs["DOVIProfile"] != float64(7) || + attrs["DOVIRPUPresent"] != true || attrs["DOVIVersion"] != "1.0" { + t.Fatalf("Dolby Vision JSON attributes = %#v", attrs) + } + }) + } +} + +func TestEnrichMetadataProjectsPlexMediaFrameRateAndPreciseStreamRate(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + {name: "film rate", input: "24000/1001", want: "24p"}, + {name: "ntsc rate", input: "30000/1001", want: "30p"}, + {name: "pal rate", input: "25/1", want: "25p"}, + } + for _, format := range []struct { + name string + contentType string + body []byte + }{ + { + name: "XML", + contentType: "application/xml", + body: []byte(``), + }, + { + name: "JSON", + contentType: "application/json", + body: []byte(`{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"Part":[{"id":7,"file":"/cloud/movie.strm","Stream":[{"streamType":1,"index":0}]}]}]}]}}`), + }, + } { + for _, test := range tests { + t.Run(format.name+"/"+test.name, func(t *testing.T) { + media := projectionStreamMedia() + media.FrameRate = test.input + result, changed, err := EnrichMetadata(format.body, format.contentType, "42", media) + if err != nil || !changed { + t.Fatalf("EnrichMetadata changed=%v err=%v", changed, err) + } + + if format.contentType == "application/xml" { + attrs := xmlProjectionAttributeSets(t, result) + if attrs["Media"]["videoFrameRate"] != test.want || attrs["Stream#1"]["frameRate"] != "23.976" { + t.Fatalf("projected XML frame rates = media=%q stream=%q", attrs["Media"]["videoFrameRate"], attrs["Stream#1"]["frameRate"]) + } + return + } + + var document map[string]any + if err := json.Unmarshal(result, &document); err != nil { + t.Fatal(err) + } + mediaObject := document["MediaContainer"].(map[string]any)["Metadata"].([]any)[0].(map[string]any)["Media"].([]any)[0].(map[string]any) + part := mediaObject["Part"].([]any)[0].(map[string]any) + stream := part["Stream"].([]any)[0].(map[string]any) + if mediaObject["videoFrameRate"] != test.want || stream["frameRate"] != 23.976 { + t.Fatalf("projected JSON frame rates = media=%#v stream=%#v", mediaObject["videoFrameRate"], stream["frameRate"]) + } + }) + } + } +} + +func assertDolbyVisionAttributes(t *testing.T, attrs map[string]string) { + t.Helper() + if attrs["displayTitle"] != "4K DoVi/HDR10" || attrs["extendedDisplayTitle"] != "4K DoVi/HDR10 (HEVC Main 10)" || + attrs["DOVIBLCompatID"] != "6" || attrs["DOVIBLPresent"] != "1" || attrs["DOVIELPresent"] != "1" || + attrs["DOVILevel"] != "6" || attrs["DOVIPresent"] != "1" || attrs["DOVIProfile"] != "7" || + attrs["DOVIRPUPresent"] != "1" || attrs["DOVIVersion"] != "1.0" { + t.Fatalf("Dolby Vision XML attributes = %#v", attrs) + } +} + +func TestDolbyVisionDisplayTitleDoesNotAssumeHDR10FromBaseLayer(t *testing.T) { + stream := mediainfo.Stream{ + Type: "video", Codec: "hevc", Profile: "Main 10", Width: 3840, Height: 2160, + HDRFormat: "dolby_vision", + DolbyVision: &mediainfo.DolbyVision{ + VersionMajor: 1, Profile: 5, Level: 6, RPUPresent: 1, BLPresent: 1, + }, + } + displayTitle, extendedDisplayTitle := videoDisplayTitles(stream) + if displayTitle != "4K DoVi" || extendedDisplayTitle != "4K DoVi (HEVC Main 10)" { + t.Fatalf("Dolby Vision-only titles = %q, %q", displayTitle, extendedDisplayTitle) + } +} + +func TestEnrichMetadataReplacesOnlyObviousSTRMPartSize(t *testing.T) { + media := projectionTestMedia() + media.Size = 987654321 + for _, test := range []struct { + name string + body string + want string + }{ + { + name: "XML control size", + body: ``, + want: `size="987654321"`, + }, + { + name: "XML non-control size", + body: ``, + want: `size="2097152"`, + }, + } { + t.Run(test.name, func(t *testing.T) { + result, changed, err := EnrichMetadata([]byte(test.body), "application/xml", "42", media) + if err != nil || !changed || !bytes.Contains(result, []byte(test.want)) { + t.Fatalf("result=%s changed=%v err=%v", result, changed, err) + } + }) + } + + media.Size = strmControlSizeThreshold + body := []byte(``) + result, changed, err := EnrichMetadata(body, "application/xml", "42", media) + if err != nil || !changed || !bytes.Contains(result, []byte(`size="301"`)) { + t.Fatalf("threshold result=%s changed=%v err=%v", result, changed, err) + } +} + +func TestEnrichMetadataDoesNotCreateMissingStreamsWhenPlexHasOne(t *testing.T) { + media := projectionStreamMedia() + body := []byte(``) + result, changed, err := EnrichMetadata(body, "application/xml", "42", media) + if err != nil || !changed { + t.Fatalf("EnrichMetadata changed=%v err=%v", changed, err) } - if !changed || bytes.Contains(result, []byte("` + return `` } func completeProjectionJSON() string { - return `{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"container":"mkv","duration":123000,"bitrate":12000000,"width":3840,"height":2160,"aspectRatio":"16:9","audioChannels":6,"audioCodec":"aac","videoCodec":"hevc","videoResolution":"4k","videoFrameRate":"23.976","videoProfile":"Main 10","audioProfile":"LC","Part":[{"id":7,"key":"/library/parts/7/1/file","file":"/cloud/movie.strm","duration":123000,"size":987654321,"container":"mkv","videoProfile":"Main 10","audioProfile":"LC","Stream":[{"streamType":1,"index":0,"codec":"hevc","profile":"Main 10","level":153,"bitrate":12000000,"width":3840,"height":2160,"frameRate":"23.976","refFrames":4,"pixelFormat":"yuv420p10le","bitDepth":10,"colorSpace":"bt2020nc","colorRange":"tv","colorPrimaries":"bt2020","colorTrc":"smpte2084","chromaLocation":"left","sampleAspectRatio":"1:1","displayAspectRatio":"16:9","language":"eng","title":"Video"},{"streamType":2,"index":1,"codec":"aac","profile":"LC","level":1,"bitrate":640000,"language":"eng","title":"English","samplingRate":48000,"channels":6,"channelLayout":"5.1"}]}]}]}]}}` + return `{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"container":"mkv","duration":123000,"bitrate":12000,"width":3840,"height":2160,"aspectRatio":"16:9","audioChannels":6,"audioCodec":"aac","videoCodec":"hevc","videoResolution":"4k","videoFrameRate":"23.976","videoProfile":"Main 10","audioProfile":"LC","Part":[{"id":7,"key":"/library/parts/7/1/file","file":"/cloud/movie.strm","duration":123000,"size":987654321,"container":"mkv","videoProfile":"Main 10","audioProfile":"LC","Stream":[{"streamType":1,"index":0,"codec":"hevc","profile":"Main 10","level":153,"bitrate":12000,"width":3840,"height":2160,"frameRate":23.976,"refFrames":4,"pixelFormat":"yuv420p10le","bitDepth":10,"colorSpace":"bt2020nc","colorRange":"tv","colorPrimaries":"bt2020","colorTrc":"smpte2084","chromaLocation":"left","sampleAspectRatio":"1:1","displayAspectRatio":"16:9","languageCode":"eng","title":"Video","displayTitle":"4K HDR10","extendedDisplayTitle":"4K HDR10 (HEVC Main 10)"},{"streamType":2,"index":1,"codec":"aac","profile":"LC","level":1,"bitrate":640,"languageCode":"eng","title":"English","samplingRate":48000,"channels":6,"audioChannelLayout":"5.1"}]}]}]}]}}` } From b80a0baaad3a569b454a451e63c19ed631d44315 Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sat, 29 Aug 2026 06:52:15 +0800 Subject: [PATCH 03/15] fix: update Infuse fallback user agent --- CHANGELOG.en.md | 2 ++ CHANGELOG.md | 1 + README.en.md | 2 +- README.md | 2 +- app.env.example | 2 +- internal/config/config.go | 2 +- internal/config/config_test.go | 2 +- 7 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.en.md b/CHANGELOG.en.md index 3817756..d4b47a0 100644 --- a/CHANGELOG.en.md +++ b/CHANGELOG.en.md @@ -28,6 +28,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 cache records without turning per-item browsing into full-library cold probes. - MediaInfo ffprobe records now use a new provider revision so records produced by the previous interpretation are not reused. +- Updated the background-probe fallback User-Agent to the currently verified + Infuse Library version. ## [0.1.1] - 2026-08-28 diff --git a/CHANGELOG.md b/CHANGELOG.md index 09898ad..5df0f3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ 字幕 Stream，并补充 HDR10、Dolby Vision、bit depth、bitrate、声道和语言等字段。 - Infuse 风格的后台媒体库同步只读取现有 MediaInfo 缓存，不再因逐项浏览触发全库冷探测。 - MediaInfo ffprobe 记录升级到新的 Provider revision，旧解释版本的缓存不会被复用。 +- 后台探测 fallback User-Agent 更新为当前已验证的 Infuse Library 版本。 ## [0.1.1] - 2026-08-28 diff --git a/README.en.md b/README.en.md index 4899427..a961245 100644 --- a/README.en.md +++ b/README.en.md @@ -120,7 +120,7 @@ Important environment variables: | `METADATA_GUARD_QUEUE_TIMEOUT` | `10s` | Maximum admission wait before returning `429`. | | `MEDIAINFO_ENABLED` | `true` | Enable the MediaInfo cache, bounded probes, and single-item metadata enrichment; initialization failures do not affect transparent proxying. | | `DATABASE_PATH` | `./data/plex-gateway.db` | Gateway SQLite database path; the container image defaults to `/app_data/plex-gateway.db`. | -| `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.4.4` | Fallback User-Agent for background probes without an active client context. | +| `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.5.1` | Fallback User-Agent for background probes without an active client context. | | `MEDIAINFO_COLD_WAIT` | `5s` | Cold-cache wait ceiling for one metadata item; timeout returns the original Plex response while probing continues. | | `MEDIAINFO_RESPONSE_MAX_BYTES` | `8388608` | Maximum size of one Plex metadata response buffered for enrichment. | | `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `4` | Maximum single-item metadata responses buffered and waiting for MediaInfo concurrently. | diff --git a/README.md b/README.md index 5a5e2b5..412570a 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ go run ./cmd/plex-gateway | `METADATA_GUARD_QUEUE_TIMEOUT` | `10s` | 等待准入的最长时间，超时返回 `429`。 | | `MEDIAINFO_ENABLED` | `true` | 启用 MediaInfo 缓存、受限探测和单项 metadata 增强；初始化失败不影响透明代理。 | | `DATABASE_PATH` | `./data/plex-gateway.db` | Gateway SQLite 持久数据库路径；容器镜像默认使用 `/app_data/plex-gateway.db`。 | -| `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.4.4` | 没有活动客户端上下文时，后台探测使用的 fallback User-Agent。 | +| `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.5.1` | 没有活动客户端上下文时，后台探测使用的 fallback User-Agent。 | | `MEDIAINFO_COLD_WAIT` | `5s` | 单项 metadata 冷缓存等待上限；超时返回原 Plex 响应，探测继续。 | | `MEDIAINFO_RESPONSE_MAX_BYTES` | `8388608` | 可以缓冲并尝试增强的单个 Plex metadata 响应上限。 | | `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `4` | 同时缓冲和等待 MediaInfo 的单项 metadata 响应上限。 | diff --git a/app.env.example b/app.env.example index 4c41281..139a764 100644 --- a/app.env.example +++ b/app.env.example @@ -96,7 +96,7 @@ MEDIAINFO_L1_MAX_ENTRIES=10000 MEDIAINFO_NEGATIVE_TTL=15m # 没有活动客户端上下文时，后台探测使用的 fallback User-Agent。 -MEDIAINFO_USER_AGENT=Infuse-Library/8.4.4 +MEDIAINFO_USER_AGENT=Infuse-Library/8.5.1 # 单项 metadata 冷缓存最多等待多久。超时后原响应立即返回，探测继续在后台执行。 MEDIAINFO_COLD_WAIT=5s diff --git a/internal/config/config.go b/internal/config/config.go index f76cde0..36711f0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -37,7 +37,7 @@ const ( defaultMediaInfoRetention = 180 * 24 * time.Hour defaultMediaInfoL1Entries = 10_000 defaultMediaInfoNegative = 15 * time.Minute - defaultMediaInfoAgent = "Infuse-Library/8.4.4" + defaultMediaInfoAgent = "Infuse-Library/8.5.1" defaultMediaInfoColdWait = 5 * time.Second defaultMediaInfoBodyLimit = 8 << 20 defaultMediaInfoWaiters = 4 diff --git a/internal/config/config_test.go b/internal/config/config_test.go index c01d356..6132b2e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -47,7 +47,7 @@ func TestLoadDefaults(t *testing.T) { if got.DatabasePath != "./data/plex-gateway.db" { t.Fatalf("DatabasePath = %q", got.DatabasePath) } - if !got.MediaInfo.Enabled || got.MediaInfo.ProbeTimeout != 20*time.Second || got.MediaInfo.Concurrency != 1 || got.MediaInfo.UserAgent != "Infuse-Library/8.4.4" || got.MediaInfo.ColdWait != 5*time.Second || got.MediaInfo.ResponseMaxBytes != 8<<20 || got.MediaInfo.EnrichmentWaiters != 4 || got.MediaInfo.PrewarmBefore != 2 || got.MediaInfo.PrewarmAfter != 3 || got.MediaInfo.PrewarmInterval != 5*time.Second { + if !got.MediaInfo.Enabled || got.MediaInfo.ProbeTimeout != 20*time.Second || got.MediaInfo.Concurrency != 1 || got.MediaInfo.UserAgent != "Infuse-Library/8.5.1" || got.MediaInfo.ColdWait != 5*time.Second || got.MediaInfo.ResponseMaxBytes != 8<<20 || got.MediaInfo.EnrichmentWaiters != 4 || got.MediaInfo.PrewarmBefore != 2 || got.MediaInfo.PrewarmAfter != 3 || got.MediaInfo.PrewarmInterval != 5*time.Second { t.Fatalf("unexpected MediaInfo defaults: %#v", got.MediaInfo) } if got.MediaVaultURL != nil || len(got.PathMappings) != 0 { From bdcb09f443f6a2341a4562e6ff7f43f5abb7bef1 Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sat, 29 Aug 2026 07:20:27 +0800 Subject: [PATCH 04/15] feat: recover remote media size --- internal/mediainfo/ffprobe.go | 94 +++++++++++++- internal/mediainfo/ffprobe_test.go | 186 +++++++++++++++++++++++++++ internal/mediainfo/model.go | 8 +- internal/mediainfo/model_test.go | 2 +- internal/mediainfo/provider.go | 2 +- internal/mediainfo/provider_test.go | 2 +- internal/mediainfo/service_test.go | 4 +- internal/mediainfo/sqlite_test.go | 2 +- internal/plexmeta/projection_test.go | 31 +++++ 9 files changed, 320 insertions(+), 11 deletions(-) diff --git a/internal/mediainfo/ffprobe.go b/internal/mediainfo/ffprobe.go index d760348..80563e3 100644 --- a/internal/mediainfo/ffprobe.go +++ b/internal/mediainfo/ffprobe.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "net/http" "net/url" "os/exec" "strconv" @@ -18,6 +19,7 @@ const ( defaultProbeSize = 8 << 20 defaultAnalyzeDuration = 5 * time.Second defaultProbeOutputLimit = 2 << 20 + defaultSizeProbeTimeout = 2 * time.Second ) var ( @@ -49,6 +51,7 @@ type FFProber struct { probeSize int64 analyzeDuration time.Duration outputLimit int64 + sizeClient *http.Client } // NewFFProber resolves the ffprobe binary and validates all resource limits. @@ -83,6 +86,7 @@ func NewFFProber(options FFProbeOptions) (*FFProber, error) { probeSize: probeSize, analyzeDuration: analyzeDuration, outputLimit: outputLimit, + sizeClient: newSizeProbeClient(), }, nil } @@ -111,7 +115,8 @@ func (prober *FFProber) Probe(ctx context.Context, directURL, userAgent string) stdout := &limitedBuffer{limit: prober.outputLimit, cancel: cancelCommand} stderr := &limitedBuffer{limit: 64 << 10, cancel: cancelCommand} - command := exec.CommandContext(commandCtx, prober.binary, prober.arguments(parsed.String(), userAgent)...) + target := parsed.String() + command := exec.CommandContext(commandCtx, prober.binary, prober.arguments(target, userAgent)...) command.Stdout = stdout command.Stderr = stderr if err := command.Run(); err != nil { @@ -133,9 +138,96 @@ func (prober *FFProber) Probe(ctx context.Context, directURL, userAgent string) if err != nil { return Media{}, err } + if media.Size <= 0 { + if size, ok := prober.probeRemoteSize(probeCtx, target, userAgent); ok { + media.Size = size + } + } return media, nil } +func newSizeProbeClient() *http.Client { + return &http.Client{ + Timeout: defaultSizeProbeTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + +// probeRemoteSize asks only for the first byte so a format with missing +// ffprobe size can still expose its authoritative Content-Range total. The +// body is deliberately never read because the media path must stay outside +// the gateway. +func (prober *FFProber) probeRemoteSize(ctx context.Context, directURL, userAgent string) (int64, bool) { + if ctx == nil { + ctx = context.Background() + } + sizeCtx, cancel := context.WithTimeout(ctx, defaultSizeProbeTimeout) + defer cancel() + request, err := http.NewRequestWithContext(sizeCtx, http.MethodGet, directURL, nil) + if err != nil { + return 0, false + } + request.Header.Set("Range", "bytes=0-0") + request.Header.Set("User-Agent", userAgent) + + client := prober.sizeClient + if client == nil { + client = newSizeProbeClient() + } + response, err := client.Do(request) + if err != nil || response == nil || response.Body == nil { + return 0, false + } + defer response.Body.Close() + if response.StatusCode != http.StatusPartialContent { + return 0, false + } + return contentRangeTotal(response.Header.Get("Content-Range")) +} + +func contentRangeTotal(value string) (int64, bool) { + fields := strings.Fields(value) + if len(fields) != 2 || !strings.EqualFold(fields[0], "bytes") { + return 0, false + } + rangeAndTotal := strings.Split(fields[1], "/") + if len(rangeAndTotal) != 2 || rangeAndTotal[1] == "*" { + return 0, false + } + bounds := strings.Split(rangeAndTotal[0], "-") + if len(bounds) != 2 { + return 0, false + } + start, ok := nonNegativeInt64(bounds[0]) + if !ok { + return 0, false + } + end, ok := nonNegativeInt64(bounds[1]) + if !ok { + return 0, false + } + total, ok := nonNegativeInt64(rangeAndTotal[1]) + if !ok || total == 0 || start > end || end >= total { + return 0, false + } + return total, true +} + +func nonNegativeInt64(value string) (int64, bool) { + if value == "" { + return 0, false + } + for index := 0; index < len(value); index++ { + if !isASCIIDigit(value[index]) { + return 0, false + } + } + parsed, err := strconv.ParseInt(value, 10, 64) + return parsed, err == nil +} + func (prober *FFProber) arguments(directURL, userAgent string) []string { showEntries := strings.Join([]string{ "format=format_name,format_long_name,start_time,duration,size,bit_rate", diff --git a/internal/mediainfo/ffprobe_test.go b/internal/mediainfo/ffprobe_test.go index 1e69262..a0e6bf6 100644 --- a/internal/mediainfo/ffprobe_test.go +++ b/internal/mediainfo/ffprobe_test.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "sync/atomic" "testing" "time" ) @@ -101,6 +102,41 @@ func TestParseFFProbePreservesPlexRelevantFields(t *testing.T) { } } +func TestParseFFProbeDoesNotInferCodecFieldsFromDolbyVisionProfile(t *testing.T) { + body := []byte(`{ + "streams": [ + { + "index": 0, + "codec_name": "hevc", + "codec_type": "video", + "width": 3840, + "height": 2160, + "side_data_list": [ + {"side_data_type":"DOVI configuration record","dv_version_major":1,"dv_version_minor":0,"dv_profile":5,"dv_level":6,"rpu_present_flag":1,"el_present_flag":0,"bl_present_flag":1,"dv_bl_signal_compatibility_id":0} + ] + } + ], + "format":{"format_name":"matroska","duration":"60"} +}`) + + media, err := parseFFProbe(body) + if err != nil { + t.Fatal(err) + } + if len(media.Streams) != 1 { + t.Fatalf("streams = %#v", media.Streams) + } + video := media.Streams[0] + if video.DolbyVision == nil || video.DolbyVision.Profile != 5 || video.HDRFormat != "dolby_vision" { + t.Fatalf("Dolby Vision fields = %#v", video) + } + if video.Profile != "" || video.BitDepth != 0 || video.PixelFormat != "" || + video.ColorSpace != "" || video.ColorRange != "" || + video.ColorPrimaries != "" || video.ColorTransfer != "" { + t.Fatalf("codec or color fields were inferred from Dolby Vision profile = %#v", video) + } +} + func TestParseFFProbeInfersBitDepthFromPixelFormat(t *testing.T) { body := []byte(`{ "streams": [ @@ -181,6 +217,90 @@ func TestFFProberAgainstHTTPMedia(t *testing.T) { } } +func TestFFProberFillsMissingSizeFromContentRange(t *testing.T) { + transport := &rangeProbeTransport{status: http.StatusPartialContent, contentRange: "bytes 0-0/123456"} + prober := newOutputScriptProber(t, completeFFProbeJSONWithoutSize, &http.Client{Transport: transport}) + + media, err := prober.Probe(t.Context(), "https://cdn.example.test/movie.mkv", "Infuse-Library/8.4.4") + if err != nil { + t.Fatal(err) + } + if media.Size != 123456 { + t.Fatalf("media size = %d", media.Size) + } + if transport.calls.Load() != 1 || transport.request == nil { + t.Fatalf("range requests = %d request=%#v", transport.calls.Load(), transport.request) + } + if transport.request.Method != http.MethodGet || + transport.request.Header.Get("Range") != "bytes=0-0" || + transport.request.Header.Get("User-Agent") != "Infuse-Library/8.4.4" { + t.Fatalf("range request = %#v", transport.request) + } +} + +func TestFFProberSizeFallbackFailsOpenForInvalidResponses(t *testing.T) { + tests := []struct { + name string + status int + contentRange string + }{ + {name: "full response", status: http.StatusOK, contentRange: "bytes 0-0/123456"}, + {name: "missing total", status: http.StatusPartialContent, contentRange: "bytes 0-0/*"}, + {name: "invalid unit", status: http.StatusPartialContent, contentRange: "octets 0-0/123456"}, + {name: "invalid range", status: http.StatusPartialContent, contentRange: "bytes 0-0"}, + {name: "zero total", status: http.StatusPartialContent, contentRange: "bytes 0-0/0"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + transport := &rangeProbeTransport{status: test.status, contentRange: test.contentRange} + prober := newOutputScriptProber(t, completeFFProbeJSONWithoutSize, &http.Client{Transport: transport}) + media, err := prober.Probe(t.Context(), "https://cdn.example.test/movie.mkv", "Infuse-Library/8.4.4") + if err != nil { + t.Fatal(err) + } + if media.Size != 0 { + t.Fatalf("invalid size was accepted = %d", media.Size) + } + if transport.calls.Load() != 1 { + t.Fatalf("range requests = %d", transport.calls.Load()) + } + }) + } +} + +func TestFFProberSizeFallbackTimeoutFailsOpen(t *testing.T) { + transport := &deadlineErrorTransport{} + prober := newOutputScriptProber(t, completeFFProbeJSONWithoutSize, &http.Client{Transport: transport}) + media, err := prober.Probe(t.Context(), "https://cdn.example.test/movie.mkv", "Infuse-Library/8.4.4") + if err != nil { + t.Fatal(err) + } + if media.Size != 0 { + t.Fatalf("timeout changed size = %d", media.Size) + } + if transport.calls.Load() != 1 || transport.deadline.IsZero() { + t.Fatalf("timeout request calls=%d deadline=%s", transport.calls.Load(), transport.deadline) + } + if transport.deadline.After(transport.observedAt.Add(defaultSizeProbeTimeout + 100*time.Millisecond)) { + t.Fatalf("size probe deadline = %s, exceeds %s cap", transport.deadline.Sub(transport.observedAt), defaultSizeProbeTimeout) + } +} + +func TestFFProberDoesNotRangeProbeExistingSize(t *testing.T) { + transport := &rangeProbeTransport{status: http.StatusPartialContent, contentRange: "bytes 0-0/123456"} + prober := newOutputScriptProber(t, completeFFProbeJSONWithSize, &http.Client{Transport: transport}) + media, err := prober.Probe(t.Context(), "https://cdn.example.test/movie.mkv", "Infuse-Library/8.4.4") + if err != nil { + t.Fatal(err) + } + if media.Size != 987654 { + t.Fatalf("existing size changed = %d", media.Size) + } + if transport.calls.Load() != 0 { + t.Fatalf("unexpected range requests = %d", transport.calls.Load()) + } +} + func TestFFProberPreservesCancellation(t *testing.T) { prober := newScriptProber(t, "exec sleep 5", 5*time.Second, defaultProbeOutputLimit) ctx, cancel := context.WithCancel(t.Context()) @@ -222,3 +342,69 @@ func newScriptProber(t *testing.T, body string, timeout time.Duration, outputLim } return prober } + +const ( + completeFFProbeJSONWithoutSize = `{"streams":[{"index":0,"codec_name":"hevc","codec_type":"video","width":1920,"height":1080}],"format":{"format_name":"matroska","duration":"60"}}` + completeFFProbeJSONWithSize = `{"streams":[{"index":0,"codec_name":"hevc","codec_type":"video","width":1920,"height":1080}],"format":{"format_name":"matroska","duration":"60","size":"987654"}}` +) + +func newOutputScriptProber(t *testing.T, output string, client *http.Client) *FFProber { + t.Helper() + path := filepath.Join(t.TempDir(), "ffprobe-output-test") + script := "#!/bin/sh\nprintf '%s' '" + output + "'\n" + if err := os.WriteFile(path, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + prober, err := NewFFProber(FFProbeOptions{ + Binary: path, Timeout: 5 * time.Second, ProbeSize: defaultProbeSize, + AnalyzeDuration: defaultAnalyzeDuration, OutputLimit: defaultProbeOutputLimit, + }) + if err != nil { + t.Fatal(err) + } + prober.sizeClient = client + return prober +} + +type rangeProbeTransport struct { + calls atomic.Int64 + request *http.Request + status int + contentRange string +} + +func (transport *rangeProbeTransport) RoundTrip(request *http.Request) (*http.Response, error) { + transport.calls.Add(1) + transport.request = request.Clone(request.Context()) + header := make(http.Header) + if transport.contentRange != "" { + header.Set("Content-Range", transport.contentRange) + } + return &http.Response{ + StatusCode: transport.status, + Header: header, + Body: unreadableResponseBody{}, + Request: request, + }, nil +} + +type deadlineErrorTransport struct { + calls atomic.Int64 + deadline time.Time + observedAt time.Time +} + +func (transport *deadlineErrorTransport) RoundTrip(request *http.Request) (*http.Response, error) { + transport.calls.Add(1) + transport.observedAt = time.Now() + transport.deadline, _ = request.Context().Deadline() + return nil, context.DeadlineExceeded +} + +type unreadableResponseBody struct{} + +func (unreadableResponseBody) Read([]byte) (int, error) { + panic("media response body was read") +} + +func (unreadableResponseBody) Close() error { return nil } diff --git a/internal/mediainfo/model.go b/internal/mediainfo/model.go index 5882873..10ad575 100644 --- a/internal/mediainfo/model.go +++ b/internal/mediainfo/model.go @@ -19,10 +19,10 @@ const ( SchemaVersion = 1 // ProviderMediaVaultFFProbe identifies the redirect plus bounded ffprobe path. ProviderMediaVaultFFProbe = "mediavault-ffprobe" - // ProviderRevisionFFProbeJSONV2 identifies the normalized ffprobe contract. - // It includes pixel-format bit-depth inference and therefore must not reuse - // records produced by the earlier interpretation. - ProviderRevisionFFProbeJSONV2 = "ffprobe-json-v2" + // ProviderRevisionFFProbeJSONV3 identifies the normalized ffprobe contract. + // It includes the bounded Content-Range size fallback and therefore must not + // reuse records produced by the earlier interpretation. + ProviderRevisionFFProbeJSONV3 = "ffprobe-json-v3" maxSTRMFingerprintBytes = 64 << 10 ) diff --git a/internal/mediainfo/model_test.go b/internal/mediainfo/model_test.go index 1b24209..1ac2f50 100644 --- a/internal/mediainfo/model_test.go +++ b/internal/mediainfo/model_test.go @@ -174,7 +174,7 @@ func completeRecord(now time.Time) Record { }, RatingKey: "456", Provider: ProviderMediaVaultFFProbe, - ProviderRevision: ProviderRevisionFFProbeJSONV2, + ProviderRevision: ProviderRevisionFFProbeJSONV3, SchemaVersion: SchemaVersion, Media: Media{ Complete: true, diff --git a/internal/mediainfo/provider.go b/internal/mediainfo/provider.go index ebf75c4..c19498d 100644 --- a/internal/mediainfo/provider.go +++ b/internal/mediainfo/provider.go @@ -61,7 +61,7 @@ func NewMediaVaultFFProbeProvider(controlResolver resolver.ControlResolver, prob // Descriptor returns the durable compatibility identity for this provider. func (*MediaVaultFFProbeProvider) Descriptor() ProviderDescriptor { - return ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV2} + return ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV3} } // Probe forwards only the transport headers required for deterministic diff --git a/internal/mediainfo/provider_test.go b/internal/mediainfo/provider_test.go index 488f546..1462ec3 100644 --- a/internal/mediainfo/provider_test.go +++ b/internal/mediainfo/provider_test.go @@ -57,7 +57,7 @@ func TestMediaVaultProviderUsesSameClientUserAgentForRedirectAndProbe(t *testing if probe.target != "https://cdn.example.test/movie.mkv" || probe.userAgent != "Infuse-Library/8.4.4" { t.Fatalf("probe target=%q User-Agent=%q", probe.target, probe.userAgent) } - if provider.Descriptor() != (ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV2}) || !result.Media.Complete { + if provider.Descriptor() != (ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV3}) || !result.Media.Complete { t.Fatalf("provider result = %#v", result) } } diff --git a/internal/mediainfo/service_test.go b/internal/mediainfo/service_test.go index d0d4fec..89cc470 100644 --- a/internal/mediainfo/service_test.go +++ b/internal/mediainfo/service_test.go @@ -56,7 +56,7 @@ type fakeProvider struct { } func (*fakeProvider) Descriptor() ProviderDescriptor { - return ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV2} + return ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV3} } func (fake *fakeProvider) Probe(ctx context.Context, request ProviderRequest) (ProviderResult, error) { @@ -323,7 +323,7 @@ func TestServiceReprobesIncompatibleProviderRevision(t *testing.T) { if err != nil { t.Fatal(err) } - if got.ProviderRevision != ProviderRevisionFFProbeJSONV2 || provider.calls.Load() != 1 { + if got.ProviderRevision != ProviderRevisionFFProbeJSONV3 || provider.calls.Load() != 1 { t.Fatalf("Ensure() revision=%q provider_calls=%d", got.ProviderRevision, provider.calls.Load()) } } diff --git a/internal/mediainfo/sqlite_test.go b/internal/mediainfo/sqlite_test.go index 6693c8e..ed6a343 100644 --- a/internal/mediainfo/sqlite_test.go +++ b/internal/mediainfo/sqlite_test.go @@ -170,7 +170,7 @@ func TestSQLiteStoreLoadsMostRecentlyAccessedWithinLimit(t *testing.T) { } func testProviderDescriptor() ProviderDescriptor { - return ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV2} + return ProviderDescriptor{Name: ProviderMediaVaultFFProbe, Revision: ProviderRevisionFFProbeJSONV3} } func TestSQLiteStoreDoesNotRestoreOtherProviderRevision(t *testing.T) { diff --git a/internal/plexmeta/projection_test.go b/internal/plexmeta/projection_test.go index 90ab7f4..eca5847 100644 --- a/internal/plexmeta/projection_test.go +++ b/internal/plexmeta/projection_test.go @@ -364,6 +364,37 @@ func TestDolbyVisionDisplayTitleDoesNotAssumeHDR10FromBaseLayer(t *testing.T) { } } +func TestEnrichMetadataDoesNotProjectUnknownCodecFieldsFromDolbyVisionProfile(t *testing.T) { + media := mediainfo.Media{ + Complete: true, Container: "mkv", DurationMS: 60000, + VideoCodec: "hevc", VideoResolution: "4k", Width: 3840, Height: 2160, + Streams: []mediainfo.Stream{ + { + Index: 0, Type: "video", Codec: "hevc", Width: 3840, Height: 2160, + HDRFormat: "dolby_vision", + DolbyVision: &mediainfo.DolbyVision{ + VersionMajor: 1, Profile: 5, Level: 6, RPUPresent: 1, BLPresent: 1, + }, + }, + }, + } + body := []byte(``) + + result, changed, err := EnrichMetadata(body, "application/xml", "42", media) + if err != nil || !changed { + t.Fatalf("EnrichMetadata changed=%v err=%v", changed, err) + } + attrs := xmlProjectionAttributeSets(t, result)["Stream#1"] + if attrs["DOVIProfile"] != "5" || attrs["DOVIPresent"] != "1" || attrs["displayTitle"] != "4K DoVi" { + t.Fatalf("Dolby Vision projection = %#v", attrs) + } + for _, name := range []string{"profile", "bitDepth", "pixelFormat", "colorSpace", "colorRange", "colorPrimaries", "colorTrc"} { + if value, present := attrs[name]; present { + t.Fatalf("unknown field %q was projected as %q: %#v", name, value, attrs) + } + } +} + func TestEnrichMetadataReplacesOnlyObviousSTRMPartSize(t *testing.T) { media := projectionTestMedia() media.Size = 987654321 From ecd5d957a2451012ce9dc76a63668fe7fd18e20d Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sat, 29 Aug 2026 07:39:38 +0800 Subject: [PATCH 05/15] fix: preserve MediaInfo at probe deadline --- CHANGELOG.en.md | 3 ++ CHANGELOG.md | 2 + README.en.md | 10 ++++ README.md | 7 +++ docs/architecture.md | 5 ++ docs/media-lifecycle-architecture.md | 6 +++ internal/mediainfo/service.go | 5 +- internal/mediainfo/service_test.go | 72 ++++++++++++++++++++++++++++ 8 files changed, 109 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.en.md b/CHANGELOG.en.md index d4b47a0..57ce865 100644 --- a/CHANGELOG.en.md +++ b/CHANGELOG.en.md @@ -28,6 +28,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 cache records without turning per-item browsing into full-library cold probes. - MediaInfo ffprobe records now use a new provider revision so records produced by the previous interpretation are not reused. +- When ffprobe omits the total media size, a bounded one-byte Range request with + the same User-Agent recovers Part size from a valid `Content-Range`. Failures + preserve the otherwise valid MediaInfo and fail open. - Updated the background-probe fallback User-Agent to the currently verified Infuse Library version. diff --git a/CHANGELOG.md b/CHANGELOG.md index 5df0f3b..0a2399a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ 字幕 Stream，并补充 HDR10、Dolby Vision、bit depth、bitrate、声道和语言等字段。 - Infuse 风格的后台媒体库同步只读取现有 MediaInfo 缓存，不再因逐项浏览触发全库冷探测。 - MediaInfo ffprobe 记录升级到新的 Provider revision，旧解释版本的缓存不会被复用。 +- 当 ffprobe 缺少媒体总大小时，使用同一 User-Agent 发起有界的单字节 Range 请求， + 并从有效 `Content-Range` 补充 Part size，失败保持已有 MediaInfo 并正常回退。 - 后台探测 fallback User-Agent 更新为当前已验证的 Infuse Library 版本。 ## [0.1.1] - 2026-08-28 diff --git a/README.en.md b/README.en.md index a961245..7555922 100644 --- a/README.en.md +++ b/README.en.md @@ -160,6 +160,16 @@ selection fields such as `selected`, `default`, or `decision`. When Plex already has Streams, only missing fields on identity-matched Streams are filled. Missing sibling Streams are not created and existing Plex values are not overwritten. +When ffprobe cannot report the total media size, the gateway sends one +`Range: bytes=0-0` request to the same temporary direct URL with the same +User-Agent used for that MediaVault resolution and ffprobe task. The request is +capped at two seconds, accepts only a `206` response +with a valid `Content-Range` total, and never reads the media response body. +Timeouts, redirects, full `200` responses, and malformed headers are ignored +without discarding otherwise valid MediaInfo or delaying the 302 playback path. +The recovered size is persisted with the MediaInfo record, so fresh cache hits +do not repeat the CDN request. + Background library synchronization can enumerate an entire library one item at a time. Requests with `skipRefresh` whose product ends in `-Library` may consume an existing MediaInfo cache record but never admit a cold probe. Ordinary diff --git a/README.md b/README.md index 412570a..3eb89d6 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,13 @@ Vision、bit depth、codec、bitrate、声道布局和语言码等字段。合 Stream ID，也不生成 `selected`、`default` 或 `decision` 等播放选择字段。Plex 已经存在 Stream 时只补充身份匹配 Stream 的缺失字段，不创建缺少的其他流，也不覆盖 Plex 值。 +当 ffprobe 无法返回媒体总大小时，Gateway 会使用与本次 MediaVault 解析和 ffprobe 相同的 +User-Agent 对同一临时直链发送一次 `Range: bytes=0-0`。该请求最多等待 `2s`，只接受包含有效 +`Content-Range` 总大小的 `206` 响应，并且不读取媒体响应体。超时、重定向、完整 `200` +响应或字段格式异常都会忽略，不影响已经成功取得的其他 MediaInfo，也不会阻塞 302 +播放路径。成功解析后的大小随 MediaInfo 一起进入 L1 和 SQLite，fresh cache hit 不会 +重复请求 CDN。 + Infuse 等客户端的后台媒体库同步可能逐项请求整个库。带 `skipRefresh` 且产品名以 `-Library` 结尾的后台同步请求只读取现有 MediaInfo 缓存，不创建冷探测。普通单项访问、 成功的云端 302 和邻近窗口仍可按既定边界提交探测，因此浏览媒体库不会扩散成全库 CDN diff --git a/docs/architecture.md b/docs/architecture.md index 74c4675..8b1204b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -181,6 +181,11 @@ MediaInfo Phase D begins with a provider boundary, L1 and SQLite persistence, bounded remote probing, proactive prewarming, and metadata response enrichment. A provider revision separates records whenever normalized probe semantics change, preventing an old interpretation from being projected after an upgrade. +A successful ffprobe result that omits format size may perform one same-UA +`bytes=0-0` request against the same direct URL. Only a valid partial response +is accepted, the body is not consumed, and every error preserves the other +probe fields. This fallback remains inside the MediaInfo worker and never enters +the synchronous 302 path. A stable MediaVault or MoviePilot provider can replace remote probing without changing playback. Plex persistence remains a feasibility question across an official API, another supported PMS interface, and an isolated database helper. diff --git a/docs/media-lifecycle-architecture.md b/docs/media-lifecycle-architecture.md index cc957a0..a081cca 100644 --- a/docs/media-lifecycle-architecture.md +++ b/docs/media-lifecycle-architecture.md @@ -519,6 +519,12 @@ MediaVault redirect 与 ffprobe 必须在同一次任务中使用相同 User-Age 实现，仍需覆盖 HDR、Dolby Vision、多音轨、字幕、大型 MKV、尾部索引容器、高延迟和 失败源。 +ffprobe 成功但未返回 format size 时，Gateway 可以在同一 worker 中使用相同 User-Agent +对同一直链发起一次 `Range: bytes=0-0`。该请求使用独立 `2s` 上限，只接受有效的 +`206 Content-Range` 总大小，不读取响应体，也不跟随重定向。失败只保留 size 为空，不能 +丢弃其他已验证字段或延长 302 路径。成功结果随当前 Provider revision 持久化，缓存命中 +在 fresh 期间不重复访问 CDN，fresh 到期后的 retained 记录仍可触发后台复验。 + 首次冷缓存长期返回空 MediaInfo 不可作为正常体验，metadata 请求一直无响应同样不可 接受。处理顺序为：Gateway 启动时按需从 SQLite 回填 L1；精确冷 miss 时创建最高优先级 任务，并允许当前单项 metadata response 在可配置的硬截止时间内等待。当前 PoC 候选 diff --git a/internal/mediainfo/service.go b/internal/mediainfo/service.go index 258bd36..e23ab11 100644 --- a/internal/mediainfo/service.go +++ b/internal/mediainfo/service.go @@ -725,7 +725,10 @@ func (service *Service) run(queued *job) { service.finish(queued, Record{}, err) return } - if err := probeCtx.Err(); err != nil { + // A provider can return a complete result while an optional best-effort + // enrichment consumes the final instant of the probe deadline. Preserve that + // result unless the service lifecycle itself has already been canceled. + if err := service.ctx.Err(); err != nil { service.finish(queued, Record{}, err) return } diff --git a/internal/mediainfo/service_test.go b/internal/mediainfo/service_test.go index 89cc470..14b4cb1 100644 --- a/internal/mediainfo/service_test.go +++ b/internal/mediainfo/service_test.go @@ -77,6 +77,25 @@ type blockingProber struct { release chan struct{} } +type deadlineResultProber struct { + deadlineObserved chan struct{} + release chan struct{} +} + +func (prober deadlineResultProber) Probe(ctx context.Context, _, _ string) (Media, error) { + <-ctx.Done() + if prober.deadlineObserved != nil { + close(prober.deadlineObserved) + } + if prober.release != nil { + <-prober.release + } + return Media{ + Complete: true, Container: "mkv", DurationMS: 60_000, + Streams: []Stream{{Index: 0, Type: "video", Codec: "hevc", Width: 1920, Height: 1080}}, + }, nil +} + func (prober *blockingProber) Probe(ctx context.Context, target, _ string) (Media, error) { prober.calls.Add(1) if prober.started != nil { @@ -161,6 +180,59 @@ func TestServiceClientWaitCancellationDoesNotCancelBackgroundJob(t *testing.T) { }) } +func TestServicePreservesSuccessfulResultReturnedAtProbeDeadline(t *testing.T) { + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, + Provider: &fakeProvider{prober: deadlineResultProber{}}, + PlexServerID: "server", ProbeTimeout: 20 * time.Millisecond, + }) + + record, err := service.Ensure(t.Context(), testRequest("deadline-success", PriorityInteractive)) + if err != nil { + t.Fatal(err) + } + if !record.Media.Complete || len(record.Media.Streams) != 1 || record.Media.Streams[0].Codec != "hevc" { + t.Fatalf("record = %#v", record) + } +} + +func TestServiceRejectsSuccessfulResultReturnedAfterShutdownCancellation(t *testing.T) { + prober := deadlineResultProber{ + deadlineObserved: make(chan struct{}), + release: make(chan struct{}), + } + store := &fakeRecordStore{} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: store, + Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", ProbeTimeout: 20 * time.Millisecond, + }) + request := testRequest("shutdown-canceled", PriorityBackground) + if err := service.Submit(request); err != nil { + t.Fatal(err) + } + <-prober.deadlineObserved + + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + closed := make(chan error, 1) + go func() { closed <- service.Close(ctx) }() + <-service.ctx.Done() + close(prober.release) + if err := <-closed; err != nil { + t.Fatal(err) + } + if _, ok := service.cache.GetKnown(request.Key, time.Now()); ok { + t.Fatal("result returned after shutdown cancellation was written to L1") + } + store.mu.Lock() + _, persisted := store.records[request.Key.cacheKey()] + store.mu.Unlock() + if persisted { + t.Fatal("result returned after shutdown cancellation was persisted") + } +} + func TestServicePrioritizesRapidInteractiveSwitchesOverPrewarm(t *testing.T) { prober := &blockingProber{started: make(chan string, 4), release: make(chan struct{}, 4)} service := newTestService(t, prober, 1) From 3cbc34b752817eeb66fd2fdeb13fafb208d005e4 Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sat, 29 Aug 2026 08:07:41 +0800 Subject: [PATCH 06/15] docs: align MediaInfo implementation status --- docs/roadmap.md | 3 +-- internal/metrics/metrics.go | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 2d0542e..e33696a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -81,8 +81,7 @@ Gateway fallback while preserving the broader lifecycle design: 1. use L1 and SQLite for single-instance fallback storage; 2. accept optional `PLEX_TOKEN` from an ignored deployment `app.env` for nearby-item discovery; current-item prewarming remains token-independent; -3. evaluate bounded remote `ffprobe`, with an initial `5s` cold metadata wait - ceiling based on the current sample; +3. use bounded remote `ffprobe` with a default `5s` cold metadata wait ceiling; 4. support exact current-Part analysis and a configurable nearby-item window after a cloud redirect is ready, with immediate current-item submission and rate-limited background neighbors, without blocking decision, Part, diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 89cf26a..7f937a0 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -154,10 +154,11 @@ func (m *Metrics) IncPlexFallback() { m.plexFallbackTotal.Add(1) } -// IncMediaInfoCacheHits records a fresh L1 result. +// IncMediaInfoCacheHits records an exact key satisfied by L1 or SQLite, +// including a retained stale result returned while refresh runs separately. func (m *Metrics) IncMediaInfoCacheHits() { m.mediaInfoCacheHits.Add(1) } -// IncMediaInfoCacheMisses records an exact key not satisfied by L1. +// IncMediaInfoCacheMisses records an exact key not satisfied by L1 or SQLite. func (m *Metrics) IncMediaInfoCacheMisses() { m.mediaInfoCacheMisses.Add(1) } // IncMediaInfoProbeQueued records a newly scheduled singleflight job. From 3eb7c2560db8907391d8f9871eba58e3966bb4f5 Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sat, 29 Aug 2026 16:23:16 +0800 Subject: [PATCH 07/15] feat: enrich Plex playback with MediaInfo --- Dockerfile | 1 + README.en.md | 48 +++- README.md | 42 ++- app.env.example | 10 +- cmd/plex-gateway/main.go | 29 ++- cmd/plex-gateway/maintenance.go | 121 +++++++++ cmd/plex-gateway/maintenance_test.go | 121 +++++++++ docs/architecture.md | 6 + docs/roadmap.md | 20 +- internal/config/config.go | 8 +- internal/config/config_test.go | 8 +- internal/gateway/decision.go | 70 ++++- internal/gateway/decision_capture.go | 25 ++ internal/gateway/decision_capture_test.go | 40 +++ internal/gateway/decision_test.go | 165 ++++++++++++ internal/gateway/gateway.go | 19 +- internal/gateway/gateway_test.go | 46 ++++ internal/gateway/metadata_enrichment.go | 123 ++++++++- internal/gateway/metadata_enrichment_test.go | 104 +++++++- internal/mediainfo/cache.go | 8 + internal/mediainfo/ffprobe_test.go | 2 +- internal/mediainfo/service.go | 166 ++++++++++-- internal/mediainfo/service_test.go | 49 ++++ internal/mediainfo/sqlite.go | 65 +++++ internal/mediainfo/sqlite_test.go | 64 +++++ internal/mediainfo/submit_test.go | 10 + internal/plexmeta/decision_projection.go | 240 +++++++++++++++++ internal/plexmeta/decision_projection_test.go | 120 +++++++++ internal/plexmeta/projection.go | 244 ++++++++++++++---- internal/plexmeta/projection_test.go | 150 ++++++++++- scripts/reset-mediainfo-cache.sh | 9 + 31 files changed, 1997 insertions(+), 136 deletions(-) create mode 100644 cmd/plex-gateway/maintenance.go create mode 100644 cmd/plex-gateway/maintenance_test.go create mode 100644 internal/plexmeta/decision_projection.go create mode 100644 internal/plexmeta/decision_projection_test.go create mode 100755 scripts/reset-mediainfo-cache.sh diff --git a/Dockerfile b/Dockerfile index e6d1348..243b333 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ RUN apk add --no-cache ca-certificates ffmpeg \ && adduser -D -H -u 1000 -G gateway -s /sbin/nologin gateway \ && install -d -o gateway -g gateway /app_data COPY --from=build /plex-gateway /usr/local/bin/plex-gateway +COPY --chmod=755 scripts/reset-mediainfo-cache.sh /usr/local/bin/plex-gateway-reset-mediainfo-cache WORKDIR /app_data USER gateway EXPOSE 32400 diff --git a/README.en.md b/README.en.md index 7555922..9a84ea4 100644 --- a/README.en.md +++ b/README.en.md @@ -34,6 +34,12 @@ playback session and exact Media/Part, authorizes the Part again, and returns the same CDN redirect without starting Plex Transcoder or synthesizing Plex metadata. +The gateway applies the same MediaInfo projection rules to every client. It +does not branch on product name, device name, or User-Agent for format +compatibility. Part redirects never wait for MediaInfo. A Plex decision may +wait up to `MEDIAINFO_COLD_WAIT` on a cold cache; probe timeout or failure +returns Plex's original response unchanged. + Native clients that accept a raw media redirect on those routes can use this path. Plex Web cloud playback is not supported: its browser player fetches `start.mpd` as a DASH manifest, while the redirect target is the original media @@ -48,7 +54,7 @@ media in Plex Web remains ordinary proxied Plex traffic. | Local Plex media | Supported by transparent proxy | The gateway does not rewrite local Parts. | | Infuse Direct Play | Direct Play verified | Uses the Plex Part redirect path. | | Plex iOS ExperimentalPlayer | Direct Play verified | STRM uses the universal decision/start redirect path. | -| Plex for Apple TV | Direct Play verified | STRM uses the universal decision/start redirect path; local media remains transparent proxy traffic. | +| Plex for Apple TV | Direct Play verified | STRM uses the universal decision/start redirect path and the same MediaInfo projection rules as other clients. Actual HDR and Dolby Vision output still depends on the source and client decoder capabilities. | | Plex Web cloud playback | Unsupported | The browser expects a DASH manifest and enforces final-origin CORS. | This is an independent community project. It is not affiliated with, endorsed @@ -111,11 +117,11 @@ Important environment variables: | `OBSERVE_MAX_BYTES` | `8388608` | Maximum metadata body copied for observation. | | `PART_PROBE_TIMEOUT` | `15s` | Timeout for the bodyless Plex Part authorization probe. | | `CLOUD_EXTENSIONS` | `.strm` | Comma-separated cloud control-file extensions. | -| `TRACE_ENABLED` | `false` | Enable sanitized Plex request-order tracing. | -| `METADATA_GUARD_ENABLED` | `false` | Limit single-item detailed metadata requests before they enter Plex. | +| `TRACE_ENABLED` | `true` | Enable sanitized Plex request-order tracing. | +| `METADATA_GUARD_ENABLED` | `true` | Limit single-item detailed metadata requests before they enter Plex. | | `METADATA_GUARD_GLOBAL_CONCURRENCY` | `8` | Shared detailed metadata concurrency limit across all clients. | | `METADATA_GUARD_CLIENT_CONCURRENCY` | `4` | Detailed metadata concurrency limit for each Plex client identifier. | -| `METADATA_GUARD_BATCH_ENABLED` | `false` | Limit comma-separated batch metadata reads before they enter Plex. | +| `METADATA_GUARD_BATCH_ENABLED` | `true` | Limit comma-separated batch metadata reads before they enter Plex. | | `METADATA_GUARD_BATCH_CONCURRENCY` | `3` | Shared batch metadata concurrency limit across all clients. | | `METADATA_GUARD_QUEUE_TIMEOUT` | `10s` | Maximum admission wait before returning `429`. | | `MEDIAINFO_ENABLED` | `true` | Enable the MediaInfo cache, bounded probes, and single-item metadata enrichment; initialization failures do not affect transparent proxying. | @@ -123,7 +129,7 @@ Important environment variables: | `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.5.1` | Fallback User-Agent for background probes without an active client context. | | `MEDIAINFO_COLD_WAIT` | `5s` | Cold-cache wait ceiling for one metadata item; timeout returns the original Plex response while probing continues. | | `MEDIAINFO_RESPONSE_MAX_BYTES` | `8388608` | Maximum size of one Plex metadata response buffered for enrichment. | -| `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `4` | Maximum single-item metadata responses buffered and waiting for MediaInfo concurrently. | +| `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `8` | Maximum single-item metadata responses buffered and waiting for MediaInfo concurrently. | | `MEDIAINFO_PREWARM_BEFORE` | `2` | Number of nearby items before the current item to prewarm. | | `MEDIAINFO_PREWARM_AFTER` | `3` | Number of nearby items after the current item to prewarm. | | `MEDIAINFO_PREWARM_INTERVAL` | `5s` | Delay between nearby-item submissions; the current item does not wait. | @@ -154,7 +160,7 @@ For an authenticated single-item STRM metadata request, the gateway can fill missing Media, Part, and Stream technical fields from its L1 or SQLite record. When a Plex Part has no Stream elements, ffprobe stream types and source indexes are used to create descriptive video, audio, and subtitle Streams with fields -such as HDR10, Dolby Vision, bit depth, codec, bitrate, channel layout, and +such as HDR10, Dolby Vision, bit depth, codec, codecID, bitrate, channel layout, and language code. Generated Streams never contain Plex Stream IDs or playback selection fields such as `selected`, `default`, or `decision`. When Plex already has Streams, only missing fields on identity-matched Streams are filled. Missing @@ -322,6 +328,36 @@ That topology is outside the exclusive-gateway deployment contract. - Every other endpoint follows the Plex proxy or Direct Play interception rules described in [docs/architecture.md](docs/architecture.md). +### Resetting the MediaInfo hot and cold caches + +Run this command from the Compose project directory: + +```sh +./scripts/reset-mediainfo-cache.sh +``` + +The script hot-clears the cache through a container-loopback maintenance +endpoint, so the Gateway and Plex proxy remain online. It first backs up the +complete SQLite database under `app_data/backups/`, then clears L1, the current +MediaInfo queues, and the rebuildable `media_info_records`. The database, +migration history, and tables owned by other modules are preserved. + +Set the Compose service name when it is not `plex-gateway`: + +```sh +PLEX_GATEWAY_COMPOSE_SERVICE=gateway ./scripts/reset-mediainfo-cache.sh +``` + +The image also contains `/usr/local/bin/plex-gateway-reset-mediainfo-cache`, so +it can be invoked directly: + +```sh +docker exec plex-gateway plex-gateway-reset-mediainfo-cache +``` + +The maintenance endpoint accepts only container-local loopback requests. It is +not exposed through the public Gateway endpoint or reverse proxy. + ## Probe a Plex item `plex-probe` records Part identifiers and paths for one metadata item. Keep the diff --git a/README.md b/README.md index 3eb89d6..40ce877 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,10 @@ Part 标记为 Direct Play 后，Gateway 才会创建短期授权。如果客户 对应的授权，再次为 Part 请求授权，并返回同一个 CDN 重定向，而不会启动 Plex Transcoder 或合成 Plex 元数据。 +Gateway 对所有客户端使用同一套 MediaInfo 投影规则，不根据产品名、设备名或 +User-Agent 特判格式兼容性。Part 302 不等待 MediaInfo；Plex decision 在冷缓存时最多 +等待 `MEDIAINFO_COLD_WAIT`，探测超时或失败会原样返回 Plex 响应。 + 能够在这些路径上接受原始媒体重定向的原生客户端可以使用此链路。Plex Web 不支持 云端播放：其浏览器播放器会将 `start.mpd` 作为 DASH manifest 获取，而重定向目标 是原始媒体文件，且可能不允许跨源浏览器请求。为 Gateway 的重定向添加 CORS header @@ -39,7 +43,7 @@ Transcoder 或合成 Plex 元数据。 | Plex 本地媒体 | 透明代理支持 | Gateway 不会重写本地 Part。 | | Infuse Direct Play | 已验证 Direct Play | 使用 Plex Part 重定向路径。 | | Plex iOS ExperimentalPlayer | 已验证 Direct Play | STRM 使用通用 decision/start 重定向链路。 | -| Plex for Apple TV | 已验证 Direct Play | STRM 使用通用 decision/start 重定向链路，本地媒体保持透明代理。 | +| Plex for Apple TV | 已验证 Direct Play | STRM 使用通用 decision/start 重定向链路，并与其他客户端使用相同 MediaInfo 投影规则。实际 HDR/Dolby Vision 输出仍取决于片源与客户端解码能力。 | | Plex Web 云端播放 | 不支持 | 浏览器需要 DASH manifest，并受最终源站 CORS 限制。 | 本项目是独立的社区项目，与 Plex、MediaVault、Infuse 或其各自所有者没有关联， @@ -99,11 +103,11 @@ go run ./cmd/plex-gateway | `OBSERVE_MAX_BYTES` | `8388608` | 观察解析时复制的最大元数据 body 大小。 | | `PART_PROBE_TIMEOUT` | `15s` | 无 body 的 Plex Part 授权 probe 超时时间。 | | `CLOUD_EXTENSIONS` | `.strm` | 以逗号分隔的云端控制文件扩展名。 | -| `TRACE_ENABLED` | `false` | 启用经过脱敏的 Plex 请求顺序追踪。 | -| `METADATA_GUARD_ENABLED` | `false` | 限制单项详细 metadata 请求进入 Plex 的并发量。 | +| `TRACE_ENABLED` | `true` | 启用经过脱敏的 Plex 请求顺序追踪。 | +| `METADATA_GUARD_ENABLED` | `true` | 限制单项详细 metadata 请求进入 Plex 的并发量。 | | `METADATA_GUARD_GLOBAL_CONCURRENCY` | `8` | 所有客户端共享的详细 metadata 并发上限。 | | `METADATA_GUARD_CLIENT_CONCURRENCY` | `4` | 每个 Plex 客户端标识的详细 metadata 并发上限。 | -| `METADATA_GUARD_BATCH_ENABLED` | `false` | 限制逗号分隔的批量 metadata 读取进入 Plex 的并发量。 | +| `METADATA_GUARD_BATCH_ENABLED` | `true` | 限制逗号分隔的批量 metadata 读取进入 Plex 的并发量。 | | `METADATA_GUARD_BATCH_CONCURRENCY` | `3` | 所有客户端共享的批量 metadata 并发上限。 | | `METADATA_GUARD_QUEUE_TIMEOUT` | `10s` | 等待准入的最长时间，超时返回 `429`。 | | `MEDIAINFO_ENABLED` | `true` | 启用 MediaInfo 缓存、受限探测和单项 metadata 增强；初始化失败不影响透明代理。 | @@ -111,7 +115,7 @@ go run ./cmd/plex-gateway | `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.5.1` | 没有活动客户端上下文时，后台探测使用的 fallback User-Agent。 | | `MEDIAINFO_COLD_WAIT` | `5s` | 单项 metadata 冷缓存等待上限；超时返回原 Plex 响应，探测继续。 | | `MEDIAINFO_RESPONSE_MAX_BYTES` | `8388608` | 可以缓冲并尝试增强的单个 Plex metadata 响应上限。 | -| `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `4` | 同时缓冲和等待 MediaInfo 的单项 metadata 响应上限。 | +| `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `8` | 同时缓冲和等待 MediaInfo 的单项 metadata 响应上限。 | | `MEDIAINFO_PREWARM_BEFORE` | `2` | 当前项之前的邻近媒体预热数量。 | | `MEDIAINFO_PREWARM_AFTER` | `3` | 当前项之后的邻近媒体预热数量。 | | `MEDIAINFO_PREWARM_INTERVAL` | `5s` | 邻近项提交到后台探测队列的间隔；当前项不等待。 | @@ -138,7 +142,7 @@ MediaVault 用于 `/api/v1` 集成的 API key。 对于已认证的单项 STRM metadata 请求，Gateway 可以从 L1 或 SQLite 记录补充缺失的 Media、Part 和 Stream 技术字段。若 Plex 的 Part 完全没有 Stream，Gateway 会使用 ffprobe 的流类型和源索引创建描述性的视频、音频和字幕 Stream，并投影 HDR10、Dolby -Vision、bit depth、codec、bitrate、声道布局和语言码等字段。合成 Stream 不包含 Plex +Vision、bit depth、codec、codecID、bitrate、声道布局和语言码等字段。合成 Stream 不包含 Plex Stream ID，也不生成 `selected`、`default` 或 `decision` 等播放选择字段。Plex 已经存在 Stream 时只补充身份匹配 Stream 的缺失字段，不创建缺少的其他流，也不覆盖 Plex 值。 @@ -269,6 +273,32 @@ Gateway 重新执行检查。 - 其他所有 endpoint 都遵循 [docs/architecture.md](docs/architecture.md) 中说明 的 Plex 代理或 Direct Play 拦截规则。 +### 清空 MediaInfo 冷热缓存 + +在 Compose 项目目录执行： + +```sh +./scripts/reset-mediainfo-cache.sh +``` + +脚本通过容器内 loopback 维护接口热清理缓存，Gateway 和 Plex 代理不会停止。清理前 +会把完整 SQLite 数据库备份到 `app_data/backups/`，然后同步清空 L1、当前 MediaInfo +队列和可重建的 `media_info_records`。数据库、迁移记录和其他模块表不会被删除。 + +Compose 服务名不是 `plex-gateway` 时，可以通过环境变量指定： + +```sh +PLEX_GATEWAY_COMPOSE_SERVICE=gateway ./scripts/reset-mediainfo-cache.sh +``` + +镜像内同时提供 `/usr/local/bin/plex-gateway-reset-mediainfo-cache`，因此也可以直接执行： + +```sh +docker exec plex-gateway plex-gateway-reset-mediainfo-cache +``` + +维护接口只接受容器本机 loopback 请求，不通过公开 Gateway endpoint 或反向代理开放。 + ## 探测 Plex 媒体项 `plex-probe` 会记录一个元数据项的 Part 标识符和路径。请将 Token 放在环境变量中， diff --git a/app.env.example b/app.env.example index 139a764..24a7636 100644 --- a/app.env.example +++ b/app.env.example @@ -20,8 +20,8 @@ DATABASE_PATH=/app_data/plex-gateway.db # 日志级别，可选 debug、info、warn、error。 LOG_LEVEL=info -# 是否记录脱敏后的 Plex 请求顺序。排障时临时开启，生产默认关闭。 -TRACE_ENABLED=false +# 是否记录脱敏后的 Plex 请求顺序。默认开启，便于识别不同客户端的实际请求链路。 +TRACE_ENABLED=true # 识别为云端控制文件的扩展名，多个值以逗号分隔。 CLOUD_EXTENSIONS=.strm @@ -39,7 +39,7 @@ OBSERVE_MAX_BYTES=8388608 PART_PROBE_TIMEOUT=15s # 是否限制单项详细 metadata 请求进入 Plex 的并发量。 -METADATA_GUARD_ENABLED=false +METADATA_GUARD_ENABLED=true # 所有客户端共享的单项详细 metadata 并发上限。 METADATA_GUARD_GLOBAL_CONCURRENCY=8 @@ -48,7 +48,7 @@ METADATA_GUARD_GLOBAL_CONCURRENCY=8 METADATA_GUARD_CLIENT_CONCURRENCY=4 # 是否为逗号分隔的批量 metadata 请求启用独立并发保护。 -METADATA_GUARD_BATCH_ENABLED=false +METADATA_GUARD_BATCH_ENABLED=true # 批量 metadata 请求的全局并发上限。 METADATA_GUARD_BATCH_CONCURRENCY=3 @@ -105,7 +105,7 @@ MEDIAINFO_COLD_WAIT=5s MEDIAINFO_RESPONSE_MAX_BYTES=8388608 # 同时缓冲并等待 MediaInfo 的单项 metadata 响应数量。 -MEDIAINFO_ENRICHMENT_CONCURRENCY=4 +MEDIAINFO_ENRICHMENT_CONCURRENCY=8 # 当前播放项之前预热的邻近媒体数量。前后都设为 0 时仅立即预热当前项。 MEDIAINFO_PREWARM_BEFORE=2 diff --git a/cmd/plex-gateway/main.go b/cmd/plex-gateway/main.go index f0dfff4..a9e4a67 100644 --- a/cmd/plex-gateway/main.go +++ b/cmd/plex-gateway/main.go @@ -29,16 +29,24 @@ import ( ) func main() { - if len(os.Args) > 1 && os.Args[1] == "version" { - fmt.Println(version.String()) - return - } - if len(os.Args) > 1 && os.Args[1] == "healthcheck" { - if err := healthcheck(); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) + if len(os.Args) > 1 { + switch os.Args[1] { + case "version": + fmt.Println(version.String()) + return + case "healthcheck": + if err := healthcheck(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return + case "cache-reset": + if err := requestMediaInfoCacheReset(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return } - return } if err := run(); err != nil { fmt.Fprintln(os.Stderr, err) @@ -88,7 +96,7 @@ func run() error { prewarmStatus := func() prewarm.Status { return prewarmService.Status() } - handler := gateway.New(gateway.Options{ + gatewayHandler := gateway.New(gateway.Options{ Upstream: cfg.PlexURL, Logger: logger, Tracer: trace.New(cfg.TraceEnabled, logger), @@ -116,6 +124,7 @@ func run() error { MediaInfoPrewarmer: prewarmService, MediaInfoPrewarmStatus: prewarmStatus, }) + handler := newMaintenanceHandler(gatewayHandler, mediaInfoService, cfg.DatabasePath, logger) server := &http.Server{ Addr: cfg.ListenAddr, Handler: handler, diff --git a/cmd/plex-gateway/maintenance.go b/cmd/plex-gateway/maintenance.go new file mode 100644 index 0000000..67c9dc2 --- /dev/null +++ b/cmd/plex-gateway/maintenance.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" +) + +const mediaInfoCacheResetPath = "/_plex-gateway/maintenance/mediainfo-cache/reset" + +type cacheResetResponse struct { + Status string `json:"status"` + DeletedRecords int64 `json:"deleted_records"` + Backup string `json:"backup"` +} + +// newMaintenanceHandler exposes container-local maintenance without widening +// the public Gateway API. Remote callers receive no proxy fallback for the +// reserved path. +func newMaintenanceHandler( + next http.Handler, + service *mediainfo.Service, + databasePath string, + logger *slog.Logger, +) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("POST "+mediaInfoCacheResetPath, func(w http.ResponseWriter, request *http.Request) { + if !loopbackRemote(request.RemoteAddr) { + http.NotFound(w, request) + return + } + if service == nil { + http.Error(w, "MediaInfo service unavailable", http.StatusServiceUnavailable) + return + } + ctx, cancel := context.WithTimeout(request.Context(), 30*time.Second) + defer cancel() + result, err := service.ResetCache(ctx, filepath.Join(filepath.Dir(databasePath), "backups")) + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, mediainfo.ErrCacheResetInProgress) { + status = http.StatusConflict + } + logger.Warn("mediainfo_cache_reset_failed", "error_kind", "maintenance") + http.Error(w, "MediaInfo cache reset failed", status) + return + } + logger.Info("mediainfo_cache_reset", "deleted_records", result.DeletedRecords) + w.Header().Set("Content-Type", "application/json; charset=utf-8") + _ = json.NewEncoder(w).Encode(cacheResetResponse{ + Status: "ok", DeletedRecords: result.DeletedRecords, Backup: filepath.Base(result.BackupPath), + }) + }) + mux.Handle("/", next) + return mux +} + +func loopbackRemote(remoteAddress string) bool { + host, _, err := net.SplitHostPort(strings.TrimSpace(remoteAddress)) + if err != nil { + return false + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +func requestMediaInfoCacheReset() error { + endpoint, err := localGatewayEndpoint(mediaInfoCacheResetPath) + if err != nil { + return err + } + request, err := http.NewRequest(http.MethodPost, endpoint, nil) + if err != nil { + return fmt.Errorf("create MediaInfo cache reset request: %w", err) + } + client := &http.Client{Timeout: 35 * time.Second} + response, err := client.Do(request) + if err != nil { + return fmt.Errorf("request MediaInfo cache reset: %w", err) + } + defer response.Body.Close() + body, err := io.ReadAll(io.LimitReader(response.Body, 64<<10)) + if err != nil { + return fmt.Errorf("read MediaInfo cache reset response: %w", err) + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("MediaInfo cache reset returned HTTP %d", response.StatusCode) + } + var result cacheResetResponse + if err := json.Unmarshal(body, &result); err != nil || result.Status != "ok" { + return errors.New("MediaInfo cache reset returned an invalid response") + } + fmt.Printf("mediainfo_cache_reset deleted=%d backup=%s\n", result.DeletedRecords, result.Backup) + return nil +} + +func localGatewayEndpoint(path string) (string, error) { + listenAddr := strings.TrimSpace(os.Getenv("LISTEN_ADDR")) + if listenAddr == "" { + listenAddr = ":32400" + } + host, port, err := net.SplitHostPort(listenAddr) + if err != nil { + return "", fmt.Errorf("parse LISTEN_ADDR: %w", err) + } + if host == "" || host == "0.0.0.0" || host == "::" { + host = "127.0.0.1" + } + return "http://" + net.JoinHostPort(host, port) + path, nil +} diff --git a/cmd/plex-gateway/maintenance_test.go b/cmd/plex-gateway/maintenance_test.go new file mode 100644 index 0000000..2c627bb --- /dev/null +++ b/cmd/plex-gateway/maintenance_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" +) + +type maintenanceStore struct { + resets atomic.Int64 +} + +func (*maintenanceStore) Put(context.Context, mediainfo.Record) error { return nil } + +func (*maintenanceStore) Get(context.Context, mediainfo.Key) (mediainfo.Record, bool, error) { + return mediainfo.Record{}, false, nil +} + +func (*maintenanceStore) Touch(context.Context, mediainfo.Key, time.Time, time.Time) error { + return nil +} + +func (store *maintenanceStore) BackupAndDeleteAll( + _ context.Context, + backupDir string, + _ time.Time, +) (mediainfo.ResetResult, error) { + store.resets.Add(1) + return mediainfo.ResetResult{DeletedRecords: 3, BackupPath: filepath.Join(backupDir, "backup.db")}, nil +} + +type maintenanceProvider struct{} + +func (maintenanceProvider) Descriptor() mediainfo.ProviderDescriptor { + return mediainfo.ProviderDescriptor{ + Name: mediainfo.ProviderMediaVaultFFProbe, Revision: mediainfo.ProviderRevisionFFProbeJSONV3, + } +} + +func (maintenanceProvider) Probe(context.Context, mediainfo.ProviderRequest) (mediainfo.ProviderResult, error) { + return mediainfo.ProviderResult{}, nil +} + +func TestMaintenanceHandlerHotResetsFromLoopbackOnly(t *testing.T) { + store := &maintenanceStore{} + service, err := mediainfo.NewService(mediainfo.ServiceOptions{ + Cache: mediainfo.NewCache(nil, time.Now()), Store: store, Provider: maintenanceProvider{}, + PlexServerID: "server", BackgroundUserAgent: "Infuse/1.0", + }) + if err != nil { + t.Fatal(err) + } + defer service.Close(t.Context()) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + handler := newMaintenanceHandler(http.NotFoundHandler(), service, "/app_data/plex-gateway.db", logger) + + remoteRequest := httptest.NewRequest(http.MethodPost, mediaInfoCacheResetPath, nil) + remoteRequest.RemoteAddr = "203.0.113.10:1234" + remoteResponse := httptest.NewRecorder() + handler.ServeHTTP(remoteResponse, remoteRequest) + if remoteResponse.Code != http.StatusNotFound || store.resets.Load() != 0 { + t.Fatalf("remote response=%d resets=%d", remoteResponse.Code, store.resets.Load()) + } + + localRequest := httptest.NewRequest(http.MethodPost, mediaInfoCacheResetPath, nil) + localRequest.RemoteAddr = "127.0.0.1:1234" + localResponse := httptest.NewRecorder() + handler.ServeHTTP(localResponse, localRequest) + if localResponse.Code != http.StatusOK || store.resets.Load() != 1 { + t.Fatalf("local response=%d resets=%d body=%s", localResponse.Code, store.resets.Load(), localResponse.Body.String()) + } + var result cacheResetResponse + if err := json.Unmarshal(localResponse.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + if result.Status != "ok" || result.DeletedRecords != 3 || result.Backup != "backup.db" { + t.Fatalf("response = %#v", result) + } +} + +func TestRequestMediaInfoCacheResetUsesLocalListener(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost || request.URL.Path != mediaInfoCacheResetPath { + t.Fatalf("request = %s %s", request.Method, request.URL.Path) + } + _ = json.NewEncoder(w).Encode(cacheResetResponse{Status: "ok", DeletedRecords: 2, Backup: "backup.db"}) + })) + defer server.Close() + parsed, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + t.Setenv("LISTEN_ADDR", parsed.Host) + if err := requestMediaInfoCacheReset(); err != nil { + t.Fatal(err) + } +} + +func TestLoopbackRemoteRejectsForwardedOrMalformedAddresses(t *testing.T) { + for _, address := range []string{"127.0.0.1:1", "[::1]:1"} { + if !loopbackRemote(address) { + t.Fatalf("loopback address rejected: %s", address) + } + } + for _, address := range []string{"203.0.113.1:1", "127.0.0.1", strings.Repeat("x", 100)} { + if loopbackRemote(address) { + t.Fatalf("non-loopback address accepted: %s", address) + } + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 8b1204b..0eae3d0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -154,6 +154,12 @@ Universal start routes are redirected only after the same STRM selection and authorization checks; manifest segments and genuine transcode traffic remain Plex-owned. +All clients use the same MediaInfo projection rules. The gateway does not infer +format compatibility from a product name, device name, or User-Agent. Part and +universal-start redirects never wait for MediaInfo. A decision may perform one +bounded cache lookup or interactive probe; timeout or failure preserves Plex's +original response. + ## Client compatibility boundary The redirect plane requires a client that can consume the final raw media URL. diff --git a/docs/roadmap.md b/docs/roadmap.md index e33696a..b24f67a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -41,13 +41,14 @@ Runtime evidence from Plex Web and the official mobile client established a separate compatibility leaf: their original universal decision requested neither Direct Play nor Direct Stream and Plex rejected an STRM Part before the client reached `/library/parts`. With the same complete client profile, changing -only those two flags produced Plex's normal Direct Play decision and Part key. +those playback flags plus `hasMDE` produced Plex's normal Direct Play decision +and Part key. The compatibility adapter therefore: - performs an authenticated, bounded metadata read using the active client; - selects `Metadata -> Media[mediaIndex] -> Part[partIndex]` without flattening; -- rewrites only `directPlay` and `directStream` for a mapped STRM Part; +- rewrites `directPlay`, `directStream`, and `hasMDE` for a mapped STRM Part; - keeps Plex responsible for the decision response and playback session; - fails open for local media, ambiguous indices, authorization failures, and every unrecognized request. @@ -71,12 +72,15 @@ manifest segments, and every genuine transcode request remain Plex-owned. ## MediaInfo Phase D MediaInfo is not a dependency of redirect playback. Eligible single-item -metadata responses can now be enriched from the fail-open L1 and SQLite cache; -missing, slow, or failed probes preserve the original Plex response and do not -change the 302 path. - -MediaInfo remains isolated from redirect playback. Phase D prioritizes the -Gateway fallback while preserving the broader lifecycle design: +metadata and Direct Play decision responses can be enriched from the fail-open +L1 and SQLite cache. A cold decision may wait up to `MEDIAINFO_COLD_WAIT` for an +interactive probe; missing, slow, or failed probes preserve the original Plex +response. Part and universal-start redirects never wait for MediaInfo. All +clients use the same projection rules without product, device, or User-Agent +format-compatibility branches. + +MediaInfo probing and persistence remain isolated from redirect playback. Phase +D prioritizes the Gateway fallback while preserving the broader lifecycle design: 1. use L1 and SQLite for single-instance fallback storage; 2. accept optional `PLEX_TOKEN` from an ignored deployment `app.env` for diff --git a/internal/config/config.go b/internal/config/config.go index 36711f0..c3dbef1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -40,7 +40,7 @@ const ( defaultMediaInfoAgent = "Infuse-Library/8.5.1" defaultMediaInfoColdWait = 5 * time.Second defaultMediaInfoBodyLimit = 8 << 20 - defaultMediaInfoWaiters = 4 + defaultMediaInfoWaiters = defaultMetadataGlobal defaultPrewarmBefore = 2 defaultPrewarmAfter = 3 defaultPrewarmInterval = 5 * time.Second @@ -172,11 +172,11 @@ func Load() (Config, error) { return Config{}, err } - traceEnabled, err := envBool("TRACE_ENABLED", false) + traceEnabled, err := envBool("TRACE_ENABLED", true) if err != nil { return Config{}, err } - metadataGuardEnabled, err := envBool("METADATA_GUARD_ENABLED", false) + metadataGuardEnabled, err := envBool("METADATA_GUARD_ENABLED", true) if err != nil { return Config{}, err } @@ -191,7 +191,7 @@ func Load() (Config, error) { if metadataPerClient > metadataGlobal { return Config{}, errors.New("METADATA_GUARD_CLIENT_CONCURRENCY must not exceed METADATA_GUARD_GLOBAL_CONCURRENCY") } - metadataBatchEnabled, err := envBool("METADATA_GUARD_BATCH_ENABLED", false) + metadataBatchEnabled, err := envBool("METADATA_GUARD_BATCH_ENABLED", true) if err != nil { return Config{}, err } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6132b2e..9908c7f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -38,16 +38,16 @@ func TestLoadDefaults(t *testing.T) { if got.PlexToken != "" { t.Fatal("PlexToken unexpectedly configured") } - if got.TraceEnabled { - t.Fatal("TraceEnabled = true") + if !got.TraceEnabled { + t.Fatal("TraceEnabled = false") } - if got.MetadataGuard.Enabled || got.MetadataGuard.GlobalConcurrency != 8 || got.MetadataGuard.PerClientConcurrency != 4 || got.MetadataGuard.BatchEnabled || got.MetadataGuard.BatchConcurrency != 3 || got.MetadataGuard.QueueTimeout != 10*time.Second { + if !got.MetadataGuard.Enabled || got.MetadataGuard.GlobalConcurrency != 8 || got.MetadataGuard.PerClientConcurrency != 4 || !got.MetadataGuard.BatchEnabled || got.MetadataGuard.BatchConcurrency != 3 || got.MetadataGuard.QueueTimeout != 10*time.Second { t.Fatalf("unexpected metadata guard defaults: %#v", got.MetadataGuard) } if got.DatabasePath != "./data/plex-gateway.db" { t.Fatalf("DatabasePath = %q", got.DatabasePath) } - if !got.MediaInfo.Enabled || got.MediaInfo.ProbeTimeout != 20*time.Second || got.MediaInfo.Concurrency != 1 || got.MediaInfo.UserAgent != "Infuse-Library/8.5.1" || got.MediaInfo.ColdWait != 5*time.Second || got.MediaInfo.ResponseMaxBytes != 8<<20 || got.MediaInfo.EnrichmentWaiters != 4 || got.MediaInfo.PrewarmBefore != 2 || got.MediaInfo.PrewarmAfter != 3 || got.MediaInfo.PrewarmInterval != 5*time.Second { + if !got.MediaInfo.Enabled || got.MediaInfo.ProbeTimeout != 20*time.Second || got.MediaInfo.Concurrency != 1 || got.MediaInfo.UserAgent != "Infuse-Library/8.5.1" || got.MediaInfo.ColdWait != 5*time.Second || got.MediaInfo.ResponseMaxBytes != 8<<20 || got.MediaInfo.EnrichmentWaiters != 8 || got.MediaInfo.PrewarmBefore != 2 || got.MediaInfo.PrewarmAfter != 3 || got.MediaInfo.PrewarmInterval != 5*time.Second { t.Fatalf("unexpected MediaInfo defaults: %#v", got.MediaInfo) } if got.MediaVaultURL != nil || len(got.PathMappings) != 0 { diff --git a/internal/gateway/decision.go b/internal/gateway/decision.go index 0d9ebca..a8b8bc1 100644 --- a/internal/gateway/decision.go +++ b/internal/gateway/decision.go @@ -1,6 +1,7 @@ package gateway import ( + "context" "errors" "io" "log/slog" @@ -11,6 +12,8 @@ import ( "strings" "time" + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" + "github.com/InfinityPacer/plex-gateway/internal/metrics" "github.com/InfinityPacer/plex-gateway/internal/playback" "github.com/InfinityPacer/plex-gateway/internal/plexmeta" ) @@ -25,11 +28,19 @@ const ( // leaving Plex responsible for the decision response and session semantics. // Any uncertainty preserves the original request unchanged. type decisionHandler struct { - plex http.Handler - probe *decisionMetadataProbe - service *playback.Service - grants *playback.GrantStore - logger *slog.Logger + plex http.Handler + probe *decisionMetadataProbe + service *playback.Service + mediaInfo decisionMediaInfoService + coldWait time.Duration + grants *playback.GrantStore + logger *slog.Logger + metrics *metrics.Metrics +} + +type decisionMediaInfoService interface { + GetMemory(mediainfo.Key) (mediainfo.Record, bool) + Ensure(context.Context, mediainfo.Request) (mediainfo.Record, error) } func (h *decisionHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -46,7 +57,6 @@ func (h *decisionHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.plex.ServeHTTP(w, r) return } - h.service.Remember(selection) request := r.Clone(r.Context()) requestURL := *r.URL @@ -59,14 +69,58 @@ func (h *decisionHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { capture := newDecisionResponseCapture(w, defaultDecisionMetadataMaxBytes) h.plex.ServeHTTP(capture, request) granted := false - if capture.successful() && plexmeta.IsDirectPlayDecision(capture.body(), capture.Header().Get("Content-Type"), selection.Part) { - granted = h.grants.Put(attempt, selection) + if capture.successful() { + body := capture.body() + if plexmeta.IsDirectPlayDecision(body, capture.Header().Get("Content-Type"), selection.Part) { + if record, found := h.mediaInfoForDecision(r, selection); found { + enriched, changed, err := plexmeta.EnrichDecision(body, capture.Header().Get("Content-Type"), selection.Part, record.Media) + if err != nil { + h.logger.Warn("decision_mediainfo_fail_open", "part", selection.Part.ID, "error_kind", errorKind(err)) + } else if changed { + if err := capture.replaceDecodedBody(enriched); err != nil { + h.logger.Warn("decision_mediainfo_fail_open", "part", selection.Part.ID, "error_kind", errorKind(err)) + } else if h.metrics != nil { + h.metrics.IncMediaInfoEnriched() + } + } + } + granted = h.grants.Put(attempt, selection) + } } if err := capture.commit(); err != nil && granted { h.grants.Delete(attempt) } } +// mediaInfoForDecision reuses an exact cached STRM identity and admits at most +// one bounded interactive probe on a cold playback. Failure leaves Plex's +// original decision response usable and the shared worker may still warm a +// later request after the client's wait expires. +func (h *decisionHandler) mediaInfoForDecision(request *http.Request, part playback.PreparedPart) (mediainfo.Record, bool) { + if h == nil || h.mediaInfo == nil || request == nil { + return mediainfo.Record{}, false + } + fingerprint, err := mediainfo.FingerprintSTRMTarget(part.Target) + if err != nil { + return mediainfo.Record{}, false + } + key := mediainfo.Key{PartID: part.Part.ID, STRMFingerprint: fingerprint} + if record, found := h.mediaInfo.GetMemory(key); found { + return record, true + } + wait := h.coldWait + if wait <= 0 { + wait = defaultMediaInfoColdWait + } + ctx, cancel := context.WithTimeout(request.Context(), wait) + defer cancel() + record, err := h.mediaInfo.Ensure(ctx, mediainfo.Request{ + Key: key, RatingKey: part.RatingKey, Target: part.Target, + Priority: mediainfo.PriorityInteractive, ClientUserAgent: request.UserAgent(), + }) + return record, err == nil && record.Media.Complete +} + func (h *decisionHandler) selectCloudPart(r *http.Request, attempt playback.Attempt, query url.Values) (playback.PreparedPart, decisionSelectionState) { if h.probe == nil || h.service == nil { return playback.PreparedPart{}, decisionSelectionUnauthenticated diff --git a/internal/gateway/decision_capture.go b/internal/gateway/decision_capture.go index d1c27d6..4d101bd 100644 --- a/internal/gateway/decision_capture.go +++ b/internal/gateway/decision_capture.go @@ -3,8 +3,10 @@ package gateway import ( "bytes" "compress/gzip" + "errors" "io" "net/http" + "strconv" "strings" ) @@ -134,4 +136,27 @@ func (w *decisionResponseCapture) body() []byte { return decoded } +// replaceDecodedBody swaps one complete bounded decision response before it is +// exposed to the client. Wire encoding is preserved while stale entity +// validators are removed because the response representation has changed. +func (w *decisionResponseCapture) replaceDecodedBody(body []byte) error { + if !w.successful() || w.committed || w.passthrough { + return errors.New("decision response is not replaceable") + } + wireBody, err := encodeMetadataBody(body, w.Header().Get("Content-Encoding")) + if err != nil { + return err + } + if int64(len(wireBody)) > w.limit { + return errors.New("enriched decision response exceeds the limit") + } + w.captured.Reset() + _, _ = w.captured.Write(wireBody) + for _, name := range []string{"ETag", "Last-Modified", "Content-MD5", "Digest"} { + w.Header().Del(name) + } + w.Header().Set("Content-Length", strconv.Itoa(len(wireBody))) + return nil +} + var _ interface{ Unwrap() http.ResponseWriter } = (*decisionResponseCapture)(nil) diff --git a/internal/gateway/decision_capture_test.go b/internal/gateway/decision_capture_test.go index c089d55..fb18c07 100644 --- a/internal/gateway/decision_capture_test.go +++ b/internal/gateway/decision_capture_test.go @@ -3,12 +3,52 @@ package gateway import ( "bytes" "compress/gzip" + "io" "net/http" "net/http/httptest" "net/url" + "strconv" "testing" ) +func TestDecisionCaptureReplacesGzipBodyAndEntityHeaders(t *testing.T) { + destination := httptest.NewRecorder() + capture := newDecisionResponseCapture(destination, 1<<20) + capture.Header().Set("Content-Encoding", "gzip") + capture.Header().Set("Content-Length", "1") + capture.Header().Set("ETag", `"stale"`) + capture.WriteHeader(http.StatusOK) + + var original bytes.Buffer + writer := gzip.NewWriter(&original) + _, _ = writer.Write([]byte("original")) + if err := writer.Close(); err != nil { + t.Fatal(err) + } + _, _ = capture.Write(original.Bytes()) + + if err := capture.replaceDecodedBody([]byte("enriched")); err != nil { + t.Fatal(err) + } + if err := capture.commit(); err != nil { + t.Fatal(err) + } + reader, err := gzip.NewReader(bytes.NewReader(destination.Body.Bytes())) + if err != nil { + t.Fatal(err) + } + decoded, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + if err := reader.Close(); err != nil { + t.Fatal(err) + } + if string(decoded) != "enriched" || destination.Header().Get("ETag") != "" || destination.Header().Get("Content-Length") != strconv.Itoa(destination.Body.Len()) { + t.Fatalf("headers=%#v body=%q", destination.Header(), decoded) + } +} + func TestNormalizePlaybackAttemptUsesStableSessionIdentity(t *testing.T) { decision := grantTestRequest("") decision.Header.Set("X-Plex-Playback-Session-Id", "header-session") diff --git a/internal/gateway/decision_test.go b/internal/gateway/decision_test.go index eadee9d..6f3cb57 100644 --- a/internal/gateway/decision_test.go +++ b/internal/gateway/decision_test.go @@ -1,6 +1,9 @@ package gateway import ( + "context" + "io" + "log/slog" "net/http" "net/http/httptest" "net/url" @@ -8,10 +11,172 @@ import ( "path/filepath" "strings" "testing" + "time" + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" + "github.com/InfinityPacer/plex-gateway/internal/metrics" + "github.com/InfinityPacer/plex-gateway/internal/partcache" "github.com/InfinityPacer/plex-gateway/internal/pathmap" + "github.com/InfinityPacer/plex-gateway/internal/playback" + "github.com/InfinityPacer/plex-gateway/internal/resolver" ) +type decisionMediaInfoStub struct { + record mediainfo.Record + memoryHit bool + waitForCancel bool + ensureCalls int + request mediainfo.Request +} + +func (stub *decisionMediaInfoStub) GetMemory(mediainfo.Key) (mediainfo.Record, bool) { + return stub.record, stub.memoryHit +} + +func (stub *decisionMediaInfoStub) Ensure(ctx context.Context, request mediainfo.Request) (mediainfo.Record, error) { + stub.ensureCalls++ + stub.request = request + if stub.waitForCancel { + <-ctx.Done() + return mediainfo.Record{}, ctx.Err() + } + return stub.record, nil +} + +func TestCloudDecisionProjectsMediaInfoWithoutClientSpecificPolicy(t *testing.T) { + clients := []struct { + name string + product string + platform string + userAgent string + }{ + {name: "Infuse", product: "Infuse-Direct", platform: "iOS", userAgent: "Infuse-Direct/8.5.3"}, + {name: "Plex iOS", product: "Plex for iOS", platform: "iOS", userAgent: "PlexMobile/8.0"}, + {name: "Plex Apple TV", product: "Plex for Apple TV", platform: "tvOS", userAgent: "PlexTV/8.0"}, + } + var projectedBody string + for _, client := range clients { + t.Run(client.name, func(t *testing.T) { + media := completeProjectionMedia() + media.Size = 987654321 + mediaInfo := &decisionMediaInfoStub{record: mediainfo.Record{Media: media}} + handler := newDecisionProjectionHandler(t, mediaInfo, 100*time.Millisecond) + request := decisionProjectionRequest() + request.Header.Set("User-Agent", client.userAgent) + request.Header.Set("X-Plex-Product", client.product) + request.Header.Set("X-Plex-Platform", client.platform) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `decision="directplay"`) || + !strings.Contains(response.Body.String(), `size="987654321"`) || + !strings.Contains(response.Body.String(), `container="mkv"`) || + !strings.Contains(response.Body.String(), ` time.Second { + t.Fatalf("decision fail-open took %s", elapsed) + } + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `size="301"`) || strings.Contains(response.Body.String(), `container="mkv"`) { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } + if mediaInfo.ensureCalls != 1 { + t.Fatalf("Ensure calls = %d", mediaInfo.ensureCalls) + } +} + +func newDecisionProjectionHandler(t *testing.T, mediaInfo decisionMediaInfoService, coldWait time.Duration) http.Handler { + t.Helper() + localRoot := t.TempDir() + if err := os.WriteFile(filepath.Join(localRoot, "episode.strm"), []byte("http://mediavault.invalid/redirect/pick/episode.mkv\n"), 0o600); err != nil { + t.Fatal(err) + } + mapper, err := pathmap.New([]pathmap.Mapping{{PlexPrefix: "/media/cloud", LocalPrefix: localRoot}}) + if err != nil { + t.Fatal(err) + } + controlResolver, err := resolver.NewMediaVaultSTRMResolver("http://mediavault.invalid", nil, 0) + if err != nil { + t.Fatal(err) + } + metadataServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/library/metadata/42" { + http.NotFound(w, request) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"MediaContainer":{"Metadata":[{"Media":[{"Part":[{"id":9,"key":"/library/parts/9/1/file","file":"/media/cloud/episode.strm"}]}]}]}}`)) + })) + t.Cleanup(metadataServer.Close) + upstream, err := url.Parse(metadataServer.URL) + if err != nil { + t.Fatal(err) + } + cloudPlayback := playback.New(playback.Options{ + Cache: partcache.New(time.Hour), Mapper: mapper, Resolver: controlResolver, + AuthorizePart: func(*http.Request, string, string) (bool, error) { return true, nil }, + CloudExtensions: []string{".strm"}, + }) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + return &decisionHandler{ + plex: http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/video/:/transcode/universal/decision" || request.URL.Query().Get("directPlay") != "1" { + http.NotFound(w, request) + return + } + w.Header().Set("Content-Type", "application/xml") + w.Header().Set("Content-Length", "1") + w.Header().Set("ETag", `"stale"`) + _, _ = w.Write([]byte(``)) + }), + probe: &decisionMetadataProbe{ + upstream: upstream, + client: &http.Client{Timeout: time.Second}, + maxBytes: 1 << 20, + }, + service: cloudPlayback, mediaInfo: mediaInfo, coldWait: coldWait, + grants: playback.NewGrantStore(time.Minute, 16), logger: logger, metrics: metrics.New(), + } +} + +func decisionProjectionRequest() *http.Request { + query := url.Values{ + "path": {"/library/metadata/42"}, + "mediaIndex": {"0"}, + "partIndex": {"0"}, + "directPlay": {"0"}, + "directStream": {"0"}, + "X-Plex-Playback-Session-Id": {"decision-projection-session"}, + } + request := httptest.NewRequest(http.MethodGet, "/video/:/transcode/universal/decision?"+query.Encode(), nil) + request.Header.Set("X-Plex-Token", "client-token") + return request +} + func TestCloudDecisionForcesDirectPlayAndContinuesThroughPartRedirect(t *testing.T) { localRoot := t.TempDir() if err := os.WriteFile(filepath.Join(localRoot, "D.strm"), []byte("http://public.invalid/redirect/pickcode/D.mkv\n"), 0o600); err != nil { diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 9f86ef3..7f83e09 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -141,10 +141,13 @@ func New(options Options) http.Handler { prewarmer: options.MediaInfoPrewarmer, } decision := &decisionHandler{ - plex: plex, - service: cloudPlayback, - logger: logger, - grants: playback.NewGrantStore(defaultDecisionGrantTTL, defaultDecisionGrantLimit), + plex: plex, + service: cloudPlayback, + mediaInfo: options.MediaInfoService, + coldWait: options.MediaInfoColdWait, + logger: logger, + metrics: registry, + grants: playback.NewGrantStore(defaultDecisionGrantTTL, defaultDecisionGrantLimit), probe: &decisionMetadataProbe{ upstream: options.Upstream, client: &http.Client{ @@ -200,13 +203,15 @@ func New(options Options) http.Handler { } mux.Handle("GET /library/parts/{partID}/{rest...}", partPlayback) mux.Handle("HEAD /library/parts/{partID}/{rest...}", partPlayback) - metadata := newMetadataGuard(options.MetadataGuard, plex, registry, logger) - metadata = newMetadataEnrichmentHandler(metadataEnrichmentOptions{ + metadata := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ Service: options.MediaInfoService, Mapper: options.PathMapper, Resolver: options.Resolver, CloudExtensions: options.CloudExtensions, ColdWait: options.MediaInfoColdWait, ResponseLimit: options.MediaInfoResponseMaxBytes, Concurrency: options.MediaInfoEnrichmentConcurrency, Metrics: registry, - }, metadata) + }, plex) + // Admission must happen before response buffering so a queued request does + // not consume an enrichment slot or bypass projection under client bursts. + metadata = newMetadataGuard(options.MetadataGuard, metadata, registry, logger) mux.Handle("/", metadata) withActiveRequests := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index 236bdd2..e3d6391 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -281,6 +281,52 @@ func TestAuthorizedPlexSTRMRedirectEnablesCloudPartRedirect(t *testing.T) { } } +func TestCloudPartRedirectDoesNotApplyClientSpecificPolicy(t *testing.T) { + localRoot := t.TempDir() + if err := os.WriteFile(filepath.Join(localRoot, "Movie.strm"), []byte("http://public.invalid/redirect/pickcode\n"), 0o600); err != nil { + t.Fatal(err) + } + + mediaVault := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Location", "https://cdn.invalid/Movie.mkv?signature=private") + w.WriteHeader(http.StatusFound) + })) + defer mediaVault.Close() + plex := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Location", "http://public.invalid/redirect/pickcode") + w.WriteHeader(http.StatusMovedPermanently) + })) + defer plex.Close() + + handler, _, cache := newCloudHandler(t, plex.URL, mediaVault.URL, []pathmap.Mapping{{ + PlexPrefix: "/media/cloud", LocalPrefix: localRoot, + }}) + cache.Put(partcache.PartInfo{PartID: "123", PlexFilePath: "/media/cloud/Movie.strm"}) + clients := []struct { + product string + platform string + userAgent string + }{ + {product: "Infuse-Library", platform: "iOS", userAgent: "Infuse-Library/8.5.1"}, + {product: "Plex for iOS", platform: "iOS", userAgent: "PlexMobile/8.0"}, + {product: "Plex for Apple TV", platform: "tvOS", userAgent: "PlexTV/8.45"}, + } + for _, client := range clients { + request := httptest.NewRequest(http.MethodGet, "/library/parts/123/7/file", nil) + request.Header.Set("X-Plex-Token", "valid-token") + request.Header.Set("X-Plex-Product", client.product) + request.Header.Set("X-Plex-Platform", client.platform) + request.Header.Set("User-Agent", client.userAgent) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusFound || response.Header().Get("Location") != "https://cdn.invalid/Movie.mkv?signature=private" { + t.Fatalf("client %q status=%d Location=%q", client.product, response.Code, response.Header().Get("Location")) + } + } +} + func TestCloudPartHeadUsesMediaVaultGet(t *testing.T) { localRoot := t.TempDir() const controlTarget = "http://public.invalid/redirect/pickcode/Movie.mkv" diff --git a/internal/gateway/metadata_enrichment.go b/internal/gateway/metadata_enrichment.go index d982a15..4443245 100644 --- a/internal/gateway/metadata_enrichment.go +++ b/internal/gateway/metadata_enrichment.go @@ -21,12 +21,13 @@ import ( const ( defaultMediaInfoColdWait = 5 * time.Second + defaultMediaInfoCacheLookupWait = 100 * time.Millisecond defaultMediaInfoResponseLimit = 8 << 20 - defaultMediaInfoEnrichmentSlots = 4 + defaultMediaInfoEnrichmentSlots = 8 ) type mediaInfoEnsurer interface { - Get(mediainfo.Key) (mediainfo.Record, bool) + GetContext(context.Context, mediainfo.Key) (mediainfo.Record, bool) Ensure(context.Context, mediainfo.Request) (mediainfo.Record, error) } @@ -86,6 +87,10 @@ func newMetadataEnrichmentHandler(options metadataEnrichmentOptions, next http.H func (handler *metadataEnrichmentHandler) ServeHTTP(w http.ResponseWriter, request *http.Request) { ratingKey, eligible := mediaInfoMetadataRequest(request) if !eligible { + if mediaInfoCollectionRequest(request) { + handler.serveCachedCollection(w, request) + return + } handler.next.ServeHTTP(w, request) return } @@ -141,7 +146,9 @@ func (handler *metadataEnrichmentHandler) ServeHTTP(w http.ResponseWriter, reque var record mediainfo.Record if mediaInfoCacheOnlyRequest(request) { var found bool - record, found = handler.service.Get(key) + lookupContext, cancel := context.WithTimeout(request.Context(), defaultMediaInfoCacheLookupWait) + record, found = handler.service.GetContext(lookupContext, key) + cancel() if !found { handler.replay(capture, rawBody) return @@ -173,11 +180,99 @@ func (handler *metadataEnrichmentHandler) ServeHTTP(w http.ResponseWriter, reque handler.failOpen(capture, rawBody) return } - wireBody, err := encodeMetadataBody(enriched, encoding) + if err := handler.writeEnrichedResponse(w, capture, enriched, encoding); err != nil { + handler.failOpen(capture, rawBody) + return + } + handler.metrics.IncMediaInfoEnriched() +} + +// serveCachedCollection enriches already-probed items without admitting remote +// analysis. Browsing a season therefore cannot turn one response into a burst +// of MediaVault or CDN requests. +func (handler *metadataEnrichmentHandler) serveCachedCollection(w http.ResponseWriter, request *http.Request) { + capture := newBoundedResponseCapture(w, handler.responseLimit) + handler.next.ServeHTTP(capture, request) + if capture.passthrough { + return + } + rawBody := capture.body.Bytes() + decoded, encoding, err := decodeMetadataBody(rawBody, capture.header.Get("Content-Encoding"), handler.responseLimit) if err != nil { handler.failOpen(capture, rawBody) return } + targets, err := plexmeta.SelectEnrichmentTargets(decoded, capture.header.Get("Content-Type")) + if err != nil { + handler.failOpen(capture, rawBody) + return + } + records := make(map[string]mediainfo.Media) + lookupContext, cancel := context.WithTimeout(request.Context(), defaultMediaInfoCacheLookupWait) + defer cancel() + for _, target := range targets { + if !target.NeedsEnrichment || !handler.cloudPart(target.Part) { + continue + } + media, found := handler.cachedMedia(lookupContext, target.Part) + if found { + records[target.RatingKey] = media + } + } + if len(records) == 0 { + handler.replay(capture, rawBody) + return + } + enriched, changed, err := plexmeta.EnrichMetadataTargets(decoded, capture.header.Get("Content-Type"), records) + if err != nil || changed == 0 { + handler.failOpen(capture, rawBody) + return + } + if err := handler.writeEnrichedResponse(w, capture, enriched, encoding); err != nil { + handler.failOpen(capture, rawBody) + return + } + handler.metrics.IncMediaInfoEnriched() +} + +func (handler *metadataEnrichmentHandler) cachedMedia(ctx context.Context, part plexmeta.Part) (mediainfo.Media, bool) { + localPath, err := handler.mapper.Resolve(part.File) + if err != nil { + return mediainfo.Media{}, false + } + strmTarget, err := handler.resolver.ReadTarget(localPath) + if err != nil { + return mediainfo.Media{}, false + } + fingerprint, err := mediainfo.FingerprintSTRMTarget(strmTarget) + if err != nil { + return mediainfo.Media{}, false + } + record, found := handler.service.GetContext(ctx, mediainfo.Key{PartID: part.ID, STRMFingerprint: fingerprint}) + if !found { + return mediainfo.Media{}, false + } + currentTarget, err := handler.resolver.ReadTarget(localPath) + if err != nil { + return mediainfo.Media{}, false + } + currentFingerprint, err := mediainfo.FingerprintSTRMTarget(currentTarget) + if err != nil || currentFingerprint != fingerprint { + return mediainfo.Media{}, false + } + return record.Media, true +} + +func (handler *metadataEnrichmentHandler) writeEnrichedResponse( + w http.ResponseWriter, + capture *boundedResponseCapture, + enriched []byte, + encoding string, +) error { + wireBody, err := encodeMetadataBody(enriched, encoding) + if err != nil { + return err + } headers := capture.header.Clone() for _, name := range []string{"ETag", "Last-Modified", "Content-MD5", "Digest"} { headers.Del(name) @@ -187,7 +282,7 @@ func (handler *metadataEnrichmentHandler) ServeHTTP(w http.ResponseWriter, reque } headers.Set("Content-Length", strconv.Itoa(len(wireBody))) writeCapturedResponse(w, headers, capture.statusCode(), wireBody) - handler.metrics.IncMediaInfoEnriched() + return nil } func (handler *metadataEnrichmentHandler) cloudPart(part plexmeta.Part) bool { @@ -215,6 +310,24 @@ func mediaInfoMetadataRequest(request *http.Request) (string, bool) { return strings.TrimPrefix(request.URL.Path, "/library/metadata/"), true } +func mediaInfoCollectionRequest(request *http.Request) bool { + if request == nil || request.URL == nil || request.Method != http.MethodGet || request.Header.Get("Range") != "" { + return false + } + _, tokenPresent, tokenValid := requestIdentity(request, "X-Plex-Token") + if !tokenPresent || !tokenValid { + return false + } + const prefix = "/library/metadata/" + const suffix = "/children" + value := strings.TrimPrefix(request.URL.Path, prefix) + if value == request.URL.Path || !strings.HasSuffix(value, suffix) { + return false + } + ratingKey := strings.TrimSuffix(value, suffix) + return ratingKey != "" && !strings.Contains(ratingKey, "/") && isMetadataItemPath(prefix+ratingKey) +} + // mediaInfoCacheOnlyRequest identifies background library crawlers that may // enumerate every item. They can consume an existing record, while active // playback remains responsible for admitting new remote analysis work. diff --git a/internal/gateway/metadata_enrichment_test.go b/internal/gateway/metadata_enrichment_test.go index 3cf0cfa..aa78a24 100644 --- a/internal/gateway/metadata_enrichment_test.go +++ b/internal/gateway/metadata_enrichment_test.go @@ -25,22 +25,52 @@ import ( ) type fakeMediaInfoEnsurer struct { - calls atomic.Int64 - gets atomic.Int64 - mu sync.Mutex - seen []mediainfo.Request - getFn func(mediainfo.Key) (mediainfo.Record, bool) - fn func(context.Context, mediainfo.Request) (mediainfo.Record, error) + calls atomic.Int64 + gets atomic.Int64 + mu sync.Mutex + seen []mediainfo.Request + getFn func(mediainfo.Key) (mediainfo.Record, bool) + getContextFn func(context.Context, mediainfo.Key) (mediainfo.Record, bool) + fn func(context.Context, mediainfo.Request) (mediainfo.Record, error) } -func (ensurer *fakeMediaInfoEnsurer) Get(key mediainfo.Key) (mediainfo.Record, bool) { +func (ensurer *fakeMediaInfoEnsurer) GetContext(ctx context.Context, key mediainfo.Key) (mediainfo.Record, bool) { ensurer.gets.Add(1) + if ensurer.getContextFn != nil { + return ensurer.getContextFn(ctx, key) + } if ensurer.getFn == nil { return mediainfo.Record{}, false } return ensurer.getFn(key) } +func TestMetadataEnrichmentCollectionFailsOpenWhenCacheStoreStalls(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + raw := []byte(``) + ensurer := &fakeMediaInfoEnsurer{ + getContextFn: func(ctx context.Context, _ mediainfo.Key) (mediainfo.Record, bool) { + <-ctx.Done() + return mediainfo.Record{}, false + }, + fn: func(context.Context, mediainfo.Request) (mediainfo.Record, error) { + t.Fatal("collection metadata must not admit a cold probe") + return mediainfo.Record{}, nil + }, + } + handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(raw, "application/xml")) + request := authenticatedRequest(http.MethodGet, "/library/metadata/41/children") + started := time.Now() + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("cache-only collection blocked for %s", elapsed) + } + if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), raw) { + t.Fatalf("response = %d %q", response.Code, response.Body.Bytes()) + } +} + func (ensurer *fakeMediaInfoEnsurer) Ensure(ctx context.Context, request mediainfo.Request) (mediainfo.Record, error) { ensurer.calls.Add(1) ensurer.mu.Lock() @@ -170,6 +200,57 @@ func TestMetadataEnrichmentUsesCachedRecordForBackgroundLibrarySync(t *testing.T } } +func TestMetadataEnrichmentUsesOnlyCachedRecordsForCollection(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + raw := []byte(``) + ensurer := &fakeMediaInfoEnsurer{ + getFn: func(key mediainfo.Key) (mediainfo.Record, bool) { + if key.PartID != "9" { + t.Fatalf("unexpected cached Part = %#v", key) + } + return mediainfo.Record{Key: key, Media: completeProjectionMedia()}, true + }, + fn: func(context.Context, mediainfo.Request) (mediainfo.Record, error) { + t.Fatal("collection metadata must not admit a cold probe") + return mediainfo.Record{}, nil + }, + } + handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(raw, "application/xml")) + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedRequest(http.MethodGet, "/library/metadata/7/children")) + + if response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte(`ratingKey="42">`) + ensurer := &fakeMediaInfoEnsurer{getFn: func(mediainfo.Key) (mediainfo.Record, bool) { + return mediainfo.Record{}, false + }, fn: func(context.Context, mediainfo.Request) (mediainfo.Record, error) { + t.Fatal("collection cache miss must not admit a cold probe") + return mediainfo.Record{}, nil + }} + handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(raw, "application/xml")) + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedRequest(http.MethodGet, "/library/metadata/7/children")) + + if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), raw) { + t.Fatalf("response = %d %q", response.Code, response.Body.Bytes()) + } + if ensurer.gets.Load() != 1 || ensurer.calls.Load() != 0 { + t.Fatalf("Get calls = %d, Ensure calls = %d", ensurer.gets.Load(), ensurer.calls.Load()) + } +} + func TestMediaInfoCacheOnlyRequestRequiresLibraryProductAndSkipRefresh(t *testing.T) { tests := []struct { name string @@ -298,6 +379,9 @@ func TestMetadataEnrichmentBypassesIneligibleAndUnbufferableResponses(t *testing {name: "HEAD", request: authenticatedMetadataRequest(http.MethodHead), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK}, {name: "batch", request: authenticatedRequest(http.MethodGet, "/library/metadata/42,43"), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, {name: "children", request: authenticatedRequest(http.MethodGet, "/library/metadata/42/children"), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, + {name: "HEAD children", request: authenticatedRequest(http.MethodHead, "/library/metadata/42/children"), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK}, + {name: "range children", request: rangeChildrenRequest(), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, + {name: "missing token children", request: httptest.NewRequest(http.MethodGet, "/library/metadata/42/children", nil), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, {name: "Part playback", request: authenticatedRequest(http.MethodGet, "/library/parts/9/1/file.strm"), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, {name: "decision", request: authenticatedRequest(http.MethodGet, "/video/:/transcode/universal/decision"), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, {name: "universal start", request: authenticatedRequest(http.MethodGet, "/video/:/transcode/universal/start.mpd"), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, @@ -483,6 +567,12 @@ func rangeMetadataRequest() *http.Request { return request } +func rangeChildrenRequest() *http.Request { + request := authenticatedRequest(http.MethodGet, "/library/metadata/42/children") + request.Header.Set("Range", "bytes=0-100") + return request +} + func metadataBodyHandler(body []byte, contentType string) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { w.Header().Set("Content-Type", contentType) diff --git a/internal/mediainfo/cache.go b/internal/mediainfo/cache.go index b694441..4781890 100644 --- a/internal/mediainfo/cache.go +++ b/internal/mediainfo/cache.go @@ -191,6 +191,14 @@ func (cache *Cache) Len() int { return cache.records.Len() } +// Purge removes every in-memory record. Durable records are managed by the +// store so maintenance can reset both cache tiers under one service boundary. +func (cache *Cache) Purge() { + if cache != nil { + cache.records.Purge() + } +} + func cloneRecord(record Record) Record { cloned := record cloned.Media.Streams = append([]Stream(nil), record.Media.Streams...) diff --git a/internal/mediainfo/ffprobe_test.go b/internal/mediainfo/ffprobe_test.go index a0e6bf6..e399731 100644 --- a/internal/mediainfo/ffprobe_test.go +++ b/internal/mediainfo/ffprobe_test.go @@ -356,7 +356,7 @@ func newOutputScriptProber(t *testing.T, output string, client *http.Client) *FF t.Fatal(err) } prober, err := NewFFProber(FFProbeOptions{ - Binary: path, Timeout: 5 * time.Second, ProbeSize: defaultProbeSize, + Binary: path, Timeout: defaultProbeTimeout, ProbeSize: defaultProbeSize, AnalyzeDuration: defaultAnalyzeDuration, OutputLimit: defaultProbeOutputLimit, }) if err != nil { diff --git a/internal/mediainfo/service.go b/internal/mediainfo/service.go index e23ab11..359cd4f 100644 --- a/internal/mediainfo/service.go +++ b/internal/mediainfo/service.go @@ -12,17 +12,18 @@ import ( ) const ( - defaultRecordTTL = 30 * 24 * time.Hour - defaultRecordRetention = 180 * 24 * time.Hour - defaultNegativeTTL = 15 * time.Minute - defaultWorkerQueueSize = 256 - defaultWorkerCount = 1 - defaultNegativeLimit = 4096 - defaultRetryLimit = 256 - defaultTouchQueueSize = 1024 - defaultGCInterval = 24 * time.Hour - defaultStoreTimeout = 5 * time.Second - defaultTouchInterval = time.Hour + defaultRecordTTL = 30 * 24 * time.Hour + defaultRecordRetention = 180 * 24 * time.Hour + defaultNegativeTTL = 15 * time.Minute + defaultWorkerQueueSize = 256 + defaultWorkerCount = 1 + defaultNegativeLimit = 4096 + defaultRetryLimit = 256 + defaultTouchQueueSize = 1024 + defaultGCInterval = 24 * time.Hour + defaultStoreTimeout = 5 * time.Second + defaultTouchInterval = time.Hour + maxClientUserAgentBytes = 4 << 10 ) var ( @@ -30,6 +31,7 @@ var ( ErrQueueFull = errors.New("MediaInfo queue is full") ErrRetryRegistrationFull = errors.New("MediaInfo retry registration is full") ErrNegativeCache = errors.New("MediaInfo probe is in backoff") + ErrCacheResetInProgress = errors.New("MediaInfo cache reset is in progress") ) // Priority separates interactive misses from administrator prewarming. A @@ -81,6 +83,9 @@ func (request Request) validate() error { if strings.ContainsAny(request.ClientUserAgent, "\r\n") { return errors.New("client User-Agent is invalid") } + if len(request.ClientUserAgent) > maxClientUserAgentBytes { + return errors.New("client User-Agent is too large") + } return nil } @@ -91,6 +96,16 @@ type RecordStore interface { Touch(context.Context, Key, time.Time, time.Time) error } +// ResetResult reports the durable effect of one hot cache reset. +type ResetResult struct { + DeletedRecords int64 + BackupPath string +} + +type recordResetStore interface { + BackupAndDeleteAll(context.Context, string, time.Time) (ResetResult, error) +} + // RecordJanitor removes records after their retention window. It is separate // from request-time storage so alternate stores need not expose SQLite details. type RecordJanitor interface { @@ -143,6 +158,7 @@ type ServiceOptions struct { type Service struct { cache *Cache store RecordStore + resetStore recordResetStore janitor RecordJanitor provider Provider providerDescriptor ProviderDescriptor @@ -179,6 +195,9 @@ type Service struct { retryLimit int closed bool active atomic.Int64 + generation atomic.Uint64 + resetting atomic.Bool + resetMu sync.RWMutex } type recordTouch struct { @@ -203,9 +222,10 @@ type retryRegistration struct { } type job struct { - request Request - flight *flight - claimed bool + request Request + flight *flight + generation uint64 + claimed bool } // NewService starts the configured worker count after validating every @@ -249,6 +269,9 @@ func NewService(options ServiceOptions) (*Service, error) { if backgroundUserAgent == "" || strings.ContainsAny(backgroundUserAgent, "\r\n") { return nil, errors.New("MediaInfo background User-Agent is invalid") } + if len(backgroundUserAgent) > maxClientUserAgentBytes { + return nil, errors.New("MediaInfo background User-Agent is too large") + } gcInterval := options.GCInterval if gcInterval <= 0 { gcInterval = defaultGCInterval @@ -288,6 +311,7 @@ func NewService(options ServiceOptions) (*Service, error) { interactiveLimit: interactiveSize, backgroundLimit: backgroundSize, flights: make(map[string]*flight), negative: make(map[string]time.Time), } + service.resetStore, _ = options.Store.(recordResetStore) service.condition = sync.NewCond(&service.mu) for index := 0; index < concurrency; index++ { service.workers.Add(1) @@ -313,13 +337,25 @@ func (service *Service) Get(key Key) (Record, bool) { if service == nil { return Record{}, false } + return service.GetContext(service.ctx, key) +} + +// GetContext performs a cache-only lookup bounded by the caller's response +// deadline and never schedules remote analysis. +func (service *Service) GetContext(ctx context.Context, key Key) (Record, bool) { + if service == nil { + return Record{}, false + } + if ctx == nil { + ctx = context.Background() + } if strings.TrimSpace(key.PlexServerID) == "" { key.PlexServerID = service.plexServerID } if key.PlexServerID != service.plexServerID { return Record{}, false } - record, ok, _ := service.lookup(service.ctx, key) + record, ok, _ := service.lookup(ctx, key) if ok { service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoCacheHits() }) } else { @@ -328,6 +364,32 @@ func (service *Service) Get(key Key) (Record, bool) { return record, ok } +// GetMemory returns one exact retained L1 record without durable storage I/O or +// remote work. Playback-policy callers use it to keep decision and Part paths +// independent from SQLite latency. +func (service *Service) GetMemory(key Key) (Record, bool) { + if service == nil || service.resetting.Load() { + return Record{}, false + } + if strings.TrimSpace(key.PlexServerID) == "" { + key.PlexServerID = service.plexServerID + } + if key.PlexServerID != service.plexServerID { + return Record{}, false + } + service.resetMu.RLock() + defer service.resetMu.RUnlock() + if service.resetting.Load() { + return Record{}, false + } + now := service.now().UTC() + record, found := service.cache.GetKnown(key, now) + if !found || !service.compatible(record) { + return Record{}, false + } + return service.touchCached(record, now), true +} + // Ensure returns a cached result or waits for a shared job until ctx ends. The // worker uses the service context, so a client disconnect or five-second wait // expiry does not cancel work needed by another client or a later request. @@ -429,6 +491,14 @@ func (service *Service) normalizeRequest(request Request) (Request, error) { } func (service *Service) lookup(ctx context.Context, key Key) (Record, bool, bool) { + if service.resetting.Load() { + return Record{}, false, false + } + service.resetMu.RLock() + defer service.resetMu.RUnlock() + if service.resetting.Load() { + return Record{}, false, false + } now := service.now().UTC() if record, ok := service.cache.GetKnown(key, now); ok && service.compatible(record) { record = service.touchCached(record, now) @@ -489,6 +559,9 @@ func (service *Service) beginDetailed(request Request, registerRetry bool) (*fli if service == nil { return nil, SubmitRejected, ErrServiceUnavailable } + if service.resetting.Load() { + return nil, SubmitRejected, ErrCacheResetInProgress + } var err error request, err = service.normalizeRequest(request) if err != nil { @@ -523,7 +596,7 @@ func (service *Service) beginDetailed(request Request, registerRetry bool) (*fli return existing, SubmitJoinedExistingFlight, nil } flight := &flight{done: make(chan struct{}), userAgent: request.ClientUserAgent} - queued := &job{request: request, flight: flight} + queued := &job{request: request, flight: flight, generation: service.generation.Load()} flight.job = queued if !service.enqueueLocked(queued) { service.mu.Unlock() @@ -748,6 +821,12 @@ func (service *Service) run(queued *job) { service.finish(queued, Record{}, ErrProbeIncomplete) return } + service.resetMu.RLock() + if queued.generation != service.generation.Load() || service.resetting.Load() { + service.resetMu.RUnlock() + service.finish(queued, Record{}, ErrCacheResetInProgress) + return + } storeCtx, cancelStore := context.WithTimeout(service.ctx, defaultStoreTimeout) if err := service.store.Put(storeCtx, record); err != nil { service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoStoreFailure() }) @@ -755,6 +834,7 @@ func (service *Service) run(queued *job) { } cancelStore() service.cache.Put(record, now) + service.resetMu.RUnlock() service.finish(queued, record, nil) } @@ -819,7 +899,7 @@ func (service *Service) scheduleRetryLocked(failed *flight, failedUserAgent stri next := &flight{done: make(chan struct{}), userAgent: selected.request.ClientUserAgent} next.retryWaiters = remaining service.retryRegistrations += len(remaining) - next.job = &job{request: selected.request, flight: next} + next.job = &job{request: selected.request, flight: next, generation: failed.job.generation} service.enqueueRetryLocked(next.job) service.flights[next.job.request.Key.cacheKey()] = next service.condition.Signal() @@ -908,6 +988,58 @@ func (service *Service) Status() Status { return status } +// ResetCache hot-clears rebuildable MediaInfo state while the proxy remains +// available. Requests arriving after the reset may populate new records. +func (service *Service) ResetCache(ctx context.Context, backupDir string) (ResetResult, error) { + if service == nil || service.resetStore == nil { + return ResetResult{}, ErrServiceUnavailable + } + if ctx == nil { + ctx = context.Background() + } + if !service.resetting.CompareAndSwap(false, true) { + return ResetResult{}, ErrCacheResetInProgress + } + defer service.resetting.Store(false) + service.generation.Add(1) + + service.mu.Lock() + if service.closed { + service.mu.Unlock() + return ResetResult{}, ErrServiceUnavailable + } + for key, flight := range service.flights { + delete(service.flights, key) + flight.err = ErrCacheResetInProgress + service.releaseRetryRegistrationsLocked(flight) + close(flight.done) + } + service.interactive = nil + service.background = nil + service.retryInteractive = nil + service.retryBackground = nil + service.negative = make(map[string]time.Time) + for { + select { + case touch := <-service.touches: + touch.reservation.release() + default: + service.mu.Unlock() + goto queuesCleared + } + } + +queuesCleared: + service.resetMu.Lock() + defer service.resetMu.Unlock() + result, err := service.resetStore.BackupAndDeleteAll(ctx, backupDir, service.now().UTC()) + if err != nil { + return ResetResult{}, err + } + service.cache.Purge() + return result, nil +} + // Close cancels active processes and waits for workers without closing the // caller-owned persistent store. func (service *Service) Close(ctx context.Context) error { diff --git a/internal/mediainfo/service_test.go b/internal/mediainfo/service_test.go index 14b4cb1..a0c637e 100644 --- a/internal/mediainfo/service_test.go +++ b/internal/mediainfo/service_test.go @@ -14,6 +14,7 @@ type fakeRecordStore struct { mu sync.Mutex records map[string]Record touches atomic.Int64 + gets atomic.Int64 } func (store *fakeRecordStore) Put(_ context.Context, record Record) error { @@ -27,12 +28,32 @@ func (store *fakeRecordStore) Put(_ context.Context, record Record) error { } func (store *fakeRecordStore) Get(_ context.Context, key Key) (Record, bool, error) { + store.gets.Add(1) store.mu.Lock() defer store.mu.Unlock() record, ok := store.records[key.cacheKey()] return cloneRecord(record), ok, nil } +func TestServiceGetMemoryNeverLoadsDurableStorage(t *testing.T) { + now := time.Date(2026, 8, 29, 8, 0, 0, 0, time.UTC) + record := completeRecord(now) + store := &fakeRecordStore{records: map[string]Record{record.Key.cacheKey(): record}} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, now), Store: store, + Provider: &fakeProvider{prober: &blockingProber{}}, PlexServerID: record.Key.PlexServerID, + Now: func() time.Time { return now }, + }) + if _, found := service.GetMemory(record.Key); found || store.gets.Load() != 0 { + t.Fatalf("cold GetMemory found=%v store_gets=%d", found, store.gets.Load()) + } + service.cache.Put(record, now) + got, found := service.GetMemory(record.Key) + if !found || got.Key != record.Key || store.gets.Load() != 0 { + t.Fatalf("warm GetMemory found=%v key=%#v store_gets=%d", found, got.Key, store.gets.Load()) + } +} + func (store *fakeRecordStore) Touch(_ context.Context, key Key, accessedAt, retainUntil time.Time) error { store.touches.Add(1) store.mu.Lock() @@ -49,6 +70,14 @@ func (store *fakeRecordStore) Touch(_ context.Context, key Key, accessedAt, reta return nil } +func (store *fakeRecordStore) BackupAndDeleteAll(_ context.Context, backupDir string, _ time.Time) (ResetResult, error) { + store.mu.Lock() + defer store.mu.Unlock() + deleted := int64(len(store.records)) + store.records = make(map[string]Record) + return ResetResult{DeletedRecords: deleted, BackupPath: backupDir + "/backup.db"}, nil +} + type fakeProvider struct { calls atomic.Int64 prober Prober @@ -163,6 +192,26 @@ func TestServiceOwnsPlexServerIdentity(t *testing.T) { } } +func TestServiceResetCacheClearsL1AndDurableRecords(t *testing.T) { + now := time.Now().UTC() + record := completeRecord(now) + store := &fakeRecordStore{records: map[string]Record{record.Key.cacheKey(): record}} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache([]Record{record}, now), Store: store, + Provider: &fakeProvider{prober: &blockingProber{}}, PlexServerID: record.Key.PlexServerID, + }) + result, err := service.ResetCache(t.Context(), t.TempDir()) + if err != nil { + t.Fatal(err) + } + if result.DeletedRecords != 1 || service.Status().CacheEntries != 0 { + t.Fatalf("reset result=%#v status=%#v", result, service.Status()) + } + if _, found := service.Get(record.Key); found { + t.Fatal("reset record remained readable") + } +} + func TestServiceClientWaitCancellationDoesNotCancelBackgroundJob(t *testing.T) { prober := &blockingProber{release: make(chan struct{})} service := newTestService(t, prober, 1) diff --git a/internal/mediainfo/sqlite.go b/internal/mediainfo/sqlite.go index a32d605..106d420 100644 --- a/internal/mediainfo/sqlite.go +++ b/internal/mediainfo/sqlite.go @@ -6,6 +6,8 @@ import ( "encoding/json" "errors" "fmt" + "os" + "path/filepath" "strings" "time" @@ -214,6 +216,69 @@ func (store *SQLiteStore) DeleteUnretained(ctx context.Context, now time.Time) ( return deleted, nil } +// BackupAndDeleteAll creates a transactionally consistent full-database +// backup, then removes only rebuildable MediaInfo rows while retaining the +// shared schema registry and tables owned by other modules. +func (store *SQLiteStore) BackupAndDeleteAll( + ctx context.Context, + backupDir string, + now time.Time, +) (ResetResult, error) { + if store == nil || store.db == nil { + return ResetResult{}, errors.New("MediaInfo store is unavailable") + } + backupDir = filepath.Clean(backupDir) + if err := os.MkdirAll(backupDir, 0o700); err != nil { + return ResetResult{}, fmt.Errorf("create Gateway backup directory: %w", err) + } + if err := os.Chmod(backupDir, 0o700); err != nil { + return ResetResult{}, fmt.Errorf("protect Gateway backup directory: %w", err) + } + if now.IsZero() { + now = time.Now().UTC() + } else { + now = now.UTC() + } + backupPath := filepath.Join( + backupDir, + "plex-gateway-before-mediainfo-reset-"+now.Format("20060102T150405.000000000Z")+".db", + ) + + connection, err := store.db.Conn(ctx) + if err != nil { + return ResetResult{}, fmt.Errorf("reserve Gateway database connection: %w", err) + } + defer connection.Close() + var quotedPath string + if err := connection.QueryRowContext(ctx, "SELECT quote(?)", backupPath).Scan("edPath); err != nil { + return ResetResult{}, fmt.Errorf("quote Gateway backup path: %w", err) + } + if _, err := connection.ExecContext(ctx, "VACUUM INTO "+quotedPath); err != nil { + return ResetResult{}, fmt.Errorf("backup Gateway database: %w", err) + } + if err := os.Chmod(backupPath, 0o600); err != nil { + return ResetResult{}, fmt.Errorf("protect Gateway database backup: %w", err) + } + + transaction, err := connection.BeginTx(ctx, nil) + if err != nil { + return ResetResult{}, fmt.Errorf("begin MediaInfo cache reset: %w", err) + } + defer transaction.Rollback() + result, err := transaction.ExecContext(ctx, "DELETE FROM media_info_records") + if err != nil { + return ResetResult{}, fmt.Errorf("delete all MediaInfo records: %w", err) + } + deleted, err := result.RowsAffected() + if err != nil { + return ResetResult{}, fmt.Errorf("read deleted MediaInfo count: %w", err) + } + if err := transaction.Commit(); err != nil { + return ResetResult{}, fmt.Errorf("commit MediaInfo cache reset: %w", err) + } + return ResetResult{DeletedRecords: deleted, BackupPath: backupPath}, nil +} + // Touch extends retention without rewriting the MediaInfo JSON payload. func (store *SQLiteStore) Touch(ctx context.Context, key Key, accessedAt, retainUntil time.Time) error { if store == nil || store.db == nil { diff --git a/internal/mediainfo/sqlite_test.go b/internal/mediainfo/sqlite_test.go index ed6a343..593179e 100644 --- a/internal/mediainfo/sqlite_test.go +++ b/internal/mediainfo/sqlite_test.go @@ -2,6 +2,7 @@ package mediainfo import ( "context" + "os" "path/filepath" "strings" "testing" @@ -212,3 +213,66 @@ func TestSQLiteStoreTouchExtendsRetentionMonotonically(t *testing.T) { t.Fatalf("Get() = %#v, %t, %v", got, ok, err) } } + +func TestSQLiteStoreBackupAndDeleteAllPreservesSharedDatabase(t *testing.T) { + databasePath := filepath.Join(t.TempDir(), "plex-gateway.db") + store, gatewayDB := openTestSQLiteStore(t, databasePath) + now := time.Unix(1_800_000_000, 123_456_789).UTC() + record := completeRecord(now) + if err := store.Put(t.Context(), record); err != nil { + t.Fatal(err) + } + if _, err := gatewayDB.SQLDB().ExecContext(t.Context(), ` + CREATE TABLE other_module_records (id TEXT PRIMARY KEY); + INSERT INTO other_module_records (id) VALUES ('preserve'); + `); err != nil { + t.Fatal(err) + } + result, err := store.BackupAndDeleteAll(t.Context(), filepath.Join(filepath.Dir(databasePath), "backups"), now) + if err != nil { + t.Fatal(err) + } + if result.DeletedRecords != 1 { + t.Fatalf("deleted records = %d", result.DeletedRecords) + } + backupInfo, err := os.Stat(result.BackupPath) + if err != nil { + t.Fatal(err) + } + if backupInfo.Mode().Perm() != 0o600 { + t.Fatalf("backup mode = %o", backupInfo.Mode().Perm()) + } + assertSQLiteCounts(t, gatewayDB, 0, 1, 1) + if err := gatewayDB.Close(); err != nil { + t.Fatal(err) + } + + backupDB, err := database.OpenSQLite(t.Context(), result.BackupPath) + if err != nil { + t.Fatal(err) + } + defer backupDB.Close() + assertSQLiteCounts(t, backupDB, 1, 1, 1) +} + +func assertSQLiteCounts(t *testing.T, gatewayDB *database.SQLite, mediaInfo, otherModule, migrations int) { + t.Helper() + queries := []struct { + name string + sql string + want int + }{ + {name: "MediaInfo", sql: "SELECT COUNT(*) FROM media_info_records", want: mediaInfo}, + {name: "other module", sql: "SELECT COUNT(*) FROM other_module_records", want: otherModule}, + {name: "migration", sql: "SELECT COUNT(*) FROM gateway_schema_migrations WHERE module = 'mediainfo'", want: migrations}, + } + for _, query := range queries { + var count int + if err := gatewayDB.SQLDB().QueryRowContext(t.Context(), query.sql).Scan(&count); err != nil { + t.Fatalf("read %s count: %v", query.name, err) + } + if count != query.want { + t.Fatalf("%s count = %d, want %d", query.name, count, query.want) + } + } +} diff --git a/internal/mediainfo/submit_test.go b/internal/mediainfo/submit_test.go index 1feb805..c17fe05 100644 --- a/internal/mediainfo/submit_test.go +++ b/internal/mediainfo/submit_test.go @@ -72,6 +72,16 @@ func TestServiceSubmitDetailedReportsRejection(t *testing.T) { } } +func TestServiceRejectsOversizedClientUserAgent(t *testing.T) { + service := newTestService(t, &blockingProber{}, 1) + request := testRequest("oversized-user-agent", PriorityInteractive) + request.ClientUserAgent = strings.Repeat("x", maxClientUserAgentBytes+1) + result := service.SubmitDetailed(request) + if result.Disposition != SubmitRejected || result.Err == nil { + t.Fatalf("oversized User-Agent result = %#v", result) + } +} + func TestServiceSubmitRequeuesJoinedUserAgentAfterFailure(t *testing.T) { now := time.Unix(1_800_000_000, 0).UTC() started := make(chan string, 2) diff --git a/internal/plexmeta/decision_projection.go b/internal/plexmeta/decision_projection.go new file mode 100644 index 0000000..7c510a7 --- /dev/null +++ b/internal/plexmeta/decision_projection.go @@ -0,0 +1,240 @@ +package plexmeta + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" +) + +// EnrichDecision fills descriptive Media, Part, and Stream fields for the one +// Direct Play Part already selected by Plex. It preserves every Plex-owned +// decision, identifier, session, and unknown field. A missing, ambiguous, or +// non-Direct-Play Part is rejected without modifying the input bytes. +func EnrichDecision(body []byte, contentType string, expected Part, media mediainfo.Media) ([]byte, bool, error) { + if expected.ID == "" && expected.Key == "" { + return nil, false, errors.New("expected Plex Part identity is empty") + } + document, err := parseProjectionDocument(body, contentType) + if err != nil { + return nil, false, err + } + + var selection projectionSelection + var sizeChanged bool + switch typed := document.(type) { + case *xmlProjectionDocument: + xmlSelection, err := typed.selectDecisionPart(expected) + if err != nil { + return nil, false, err + } + selection = xmlSelection + sizeChanged, err = replaceDecisionPartSizeXML(xmlSelection.part, media.Size) + if err != nil { + return nil, false, err + } + case *jsonProjectionDocument: + jsonSelection, err := typed.selectDecisionPart(expected) + if err != nil { + return nil, false, err + } + selection = jsonSelection + sizeChanged, err = replaceDecisionPartSizeJSON(*jsonSelection.part, media.Size) + if err != nil { + return nil, false, err + } + default: + return nil, false, errors.New("unsupported Plex decision document") + } + + changed, err := selection.enrich(media) + if err != nil { + return nil, false, err + } + if sizeChanged && !changed { + if jsonSelection, ok := selection.(*jsonProjectionSelection); ok { + if err := jsonSelection.commit(); err != nil { + return nil, false, err + } + } + } + if !changed && !sizeChanged { + return append([]byte(nil), body...), false, nil + } + result, err := document.encode() + if err != nil { + return nil, false, err + } + return result, true, nil +} + +func (document *xmlProjectionDocument) selectDecisionPart(expected Part) (*xmlProjectionSelection, error) { + roots := xmlProjectionElements(document.nodes) + if len(roots) != 1 || roots[0].start.Name.Local != "MediaContainer" { + return nil, errors.New("Plex XML decision has no unique MediaContainer root") + } + var selected *xmlProjectionSelection + for _, video := range directXMLElements(roots[0], "Video") { + for _, media := range directXMLElements(video, "Media") { + for _, part := range directXMLElements(media, "Part") { + decision, _, err := xmlAttribute(part.start.Attr, "decision") + if err != nil { + return nil, err + } + id, _, err := xmlAttribute(part.start.Attr, "id") + if err != nil { + return nil, err + } + key, _, err := xmlAttribute(part.start.Attr, "key") + if err != nil { + return nil, err + } + if !strings.EqualFold(decision, "directplay") || !decisionPartMatches(id, key, expected) { + continue + } + if selected != nil { + return nil, errors.New("Plex XML decision has an ambiguous Direct Play Part") + } + selected = &xmlProjectionSelection{ + value: Target{Part: expected, NeedsEnrichment: true}, + media: media, + part: part, + } + } + } + } + if selected == nil { + return nil, errors.New("Plex XML decision has no matching Direct Play Part") + } + return selected, nil +} + +func (document *jsonProjectionDocument) selectDecisionPart(expected Part) (*jsonProjectionSelection, error) { + container, metadata, err := document.directMetadata() + if err != nil { + return nil, err + } + var selected *jsonProjectionSelection + for _, item := range metadata.items { + mediaRaw, ok := (*item)["Media"] + if !ok { + continue + } + mediaItems, err := decodeJSONProjectionList(mediaRaw, "Media") + if err != nil { + return nil, err + } + for _, mediaItem := range mediaItems.items { + partRaw, ok := (*mediaItem)["Part"] + if !ok { + continue + } + parts, err := decodeJSONProjectionList(partRaw, "Part") + if err != nil { + return nil, err + } + for _, part := range parts.items { + decision, err := optionalJSONDecisionString(*part, "decision") + if err != nil { + return nil, err + } + id, err := optionalJSONDecisionScalar(*part, "id") + if err != nil { + return nil, err + } + key, err := optionalJSONDecisionString(*part, "key") + if err != nil { + return nil, err + } + if !strings.EqualFold(decision, "directplay") || !decisionPartMatches(id, key, expected) { + continue + } + if selected != nil { + return nil, errors.New("Plex JSON decision has an ambiguous Direct Play Part") + } + selected = &jsonProjectionSelection{ + value: Target{Part: expected, NeedsEnrichment: true}, + document: document, + container: container, + metadata: metadata, + metadataItem: item, + media: mediaItems, + parts: parts, + mediaItem: mediaItem, + part: part, + } + } + } + } + if selected == nil { + return nil, errors.New("Plex JSON decision has no matching Direct Play Part") + } + return selected, nil +} + +func optionalJSONDecisionString(object jsonObject, name string) (string, error) { + raw, ok := object[name] + if !ok { + return "", nil + } + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return "", fmt.Errorf("Plex JSON decision %s is not a string", name) + } + return value, nil +} + +func optionalJSONDecisionScalar(object jsonObject, name string) (string, error) { + raw, ok := object[name] + if !ok { + return "", nil + } + value, ok := jsonRawScalarString(raw) + if !ok { + return "", fmt.Errorf("Plex JSON decision %s is not a scalar", name) + } + return value, nil +} + +func replaceDecisionPartSizeXML(part *xmlProjectionElement, mediaSize int64) (bool, error) { + if mediaSize <= strmControlSizeThreshold { + return false, nil + } + for index := range part.start.Attr { + if part.start.Attr[index].Name.Local != "size" { + continue + } + current, err := strconv.ParseInt(strings.TrimSpace(part.start.Attr[index].Value), 10, 64) + if err != nil { + return false, errors.New("Plex XML decision Part has an invalid size") + } + if current < 0 || current > strmControlSizeThreshold || mediaSize <= current { + return false, nil + } + part.start.Attr[index].Value = strconv.FormatInt(mediaSize, 10) + return true, nil + } + return false, nil +} + +func replaceDecisionPartSizeJSON(part jsonObject, mediaSize int64) (bool, error) { + if mediaSize <= strmControlSizeThreshold { + return false, nil + } + raw, present := part["size"] + if !present { + return false, nil + } + current, ok := jsonRawInt64(raw) + if !ok { + return false, errors.New("Plex JSON decision Part has an invalid size") + } + if current < 0 || current > strmControlSizeThreshold || mediaSize <= current { + return false, nil + } + part["size"] = json.RawMessage(strconv.FormatInt(mediaSize, 10)) + return true, nil +} diff --git a/internal/plexmeta/decision_projection_test.go b/internal/plexmeta/decision_projection_test.go new file mode 100644 index 0000000..52e5f42 --- /dev/null +++ b/internal/plexmeta/decision_projection_test.go @@ -0,0 +1,120 @@ +package plexmeta + +import ( + "bytes" + "encoding/json" + "testing" +) + +func TestEnrichDecisionProjectsMatchingDirectPlayPart(t *testing.T) { + media := projectionTestMedia() + tests := []struct { + name string + contentType string + body []byte + assert func(*testing.T, []byte) + }{ + { + name: "XML", + contentType: "application/xml", + body: []byte(``), + assert: func(t *testing.T, result []byte) { + attributes := xmlProjectionAttributeSets(t, result) + if attributes["Media"]["decision"] != "directplay" || attributes["Media"]["container"] != "mkv" || attributes["Media"]["bitrate"] != "12000" { + t.Fatalf("Media attributes = %#v", attributes["Media"]) + } + if attributes["Part"]["decision"] != "directplay" || attributes["Part"]["size"] != "987654321" || attributes["Part"]["container"] != "mkv" { + t.Fatalf("Part attributes = %#v", attributes["Part"]) + } + if attributes["Stream#1"]["codec"] != "hevc" || attributes["Stream#1"]["bitDepth"] != "10" || attributes["Stream#2"]["codec"] != "aac" { + t.Fatalf("Stream attributes = %#v", attributes) + } + for _, fragment := range [][]byte{[]byte(`decisionExtra="preserve"`), []byte(`videoExtra="preserve"`), []byte(`mediaExtra="preserve"`), []byte(`partExtra="preserve"`)} { + if !bytes.Contains(result, fragment) { + t.Fatalf("decision field %q was not preserved: %s", fragment, result) + } + } + }, + }, + { + name: "JSON", + contentType: "application/json", + body: []byte(`{"MediaContainer":{"decisionExtra":"preserve","Metadata":[{"videoExtra":"preserve","Media":[{"id":117278,"decision":"directplay","mediaExtra":"preserve","Part":[{"id":118649,"key":"/library/parts/118649/1/file","size":301,"decision":"directplay","partExtra":"preserve"}]}]}]}}`), + assert: func(t *testing.T, result []byte) { + var document map[string]any + if err := json.Unmarshal(result, &document); err != nil { + t.Fatal(err) + } + container := document["MediaContainer"].(map[string]any) + item := container["Metadata"].([]any)[0].(map[string]any) + mediaObject := item["Media"].([]any)[0].(map[string]any) + part := mediaObject["Part"].([]any)[0].(map[string]any) + streams := part["Stream"].([]any) + if mediaObject["decision"] != "directplay" || mediaObject["container"] != "mkv" || mediaObject["bitrate"] != float64(12000) { + t.Fatalf("Media fields = %#v", mediaObject) + } + if part["decision"] != "directplay" || part["size"] != float64(987654321) || part["container"] != "mkv" { + t.Fatalf("Part fields = %#v", part) + } + if len(streams) != 2 || streams[0].(map[string]any)["codec"] != "hevc" || streams[1].(map[string]any)["codec"] != "aac" { + t.Fatalf("Stream fields = %#v", streams) + } + if container["decisionExtra"] != "preserve" || item["videoExtra"] != "preserve" || mediaObject["mediaExtra"] != "preserve" || part["partExtra"] != "preserve" { + t.Fatalf("unknown decision fields were not preserved: %#v", document) + } + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + original := append([]byte(nil), test.body...) + result, changed, err := EnrichDecision(test.body, test.contentType, Part{ + ID: "118649", Key: "/library/parts/118649/1/file", File: "/cloud/movie.strm", + }, media) + if err != nil || !changed { + t.Fatalf("EnrichDecision changed=%v err=%v", changed, err) + } + if !bytes.Equal(test.body, original) { + t.Fatal("EnrichDecision modified its input body") + } + test.assert(t, result) + }) + } +} + +func TestEnrichDecisionRejectsNonMatchingOrAmbiguousPart(t *testing.T) { + tests := []struct { + name string + contentType string + body []byte + }{ + { + name: "XML transcode", + contentType: "application/xml", + body: []byte(``), + }, + { + name: "XML ambiguous", + contentType: "application/xml", + body: []byte(``), + }, + { + name: "JSON wrong Part", + contentType: "application/json", + body: []byte(`{"MediaContainer":{"Metadata":[{"Media":[{"Part":[{"id":8,"key":"/library/parts/8/1/file","decision":"directplay"}]}]}]}}`), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + original := append([]byte(nil), test.body...) + result, changed, err := EnrichDecision(test.body, test.contentType, Part{ID: "7", Key: "/library/parts/7/1/file"}, projectionTestMedia()) + if err == nil || result != nil || changed { + t.Fatalf("result=%q changed=%v err=%v", result, changed, err) + } + if !bytes.Equal(test.body, original) { + t.Fatal("failed decision projection modified its input") + } + }) + } +} diff --git a/internal/plexmeta/projection.go b/internal/plexmeta/projection.go index a55e566..7aa6029 100644 --- a/internal/plexmeta/projection.go +++ b/internal/plexmeta/projection.go @@ -19,7 +19,8 @@ import ( // A Part with no Stream elements is eligible for descriptive Stream creation // because ffprobe supplies the stable stream type and source index. type Target struct { - Part Part + RatingKey string + Part Part // NeedsEnrichment is true when a whitelisted field is absent from the // selected Media, Part, or matchable existing Stream elements. NeedsEnrichment bool @@ -71,8 +72,63 @@ func EnrichMetadata(body []byte, contentType, ratingKey string, media mediainfo. return result, true, nil } +// SelectEnrichmentTargets returns independently projectable items from a Plex +// collection response. Items with multiple Media versions or Parts are left +// untouched because a cache record cannot be mapped to them without guessing. +func SelectEnrichmentTargets(body []byte, contentType string) ([]Target, error) { + document, err := parseProjectionDocument(body, contentType) + if err != nil { + return nil, err + } + selections, err := document.selectTargets() + if err != nil { + return nil, err + } + targets := make([]Target, 0, len(selections)) + for _, selection := range selections { + targets = append(targets, selection.target()) + } + return targets, nil +} + +// EnrichMetadataTargets projects cached MediaInfo into matching items of one +// collection response. Missing records leave their items unchanged. +func EnrichMetadataTargets(body []byte, contentType string, records map[string]mediainfo.Media) ([]byte, int, error) { + document, err := parseProjectionDocument(body, contentType) + if err != nil { + return nil, 0, err + } + selections, err := document.selectTargets() + if err != nil { + return nil, 0, err + } + changedItems := 0 + for _, selection := range selections { + media, ok := records[selection.target().RatingKey] + if !ok { + continue + } + changed, err := selection.enrich(media) + if err != nil { + return nil, 0, err + } + if changed { + changedItems++ + } + } + if changedItems == 0 { + return append([]byte(nil), body...), 0, nil + } + result, err := document.encode() + if err != nil { + return nil, 0, err + } + return result, changedItems, nil +} + type projectionDocument interface { selectTarget(string) (projectionSelection, error) + selectTargets() ([]projectionSelection, error) encode() ([]byte, error) } @@ -86,30 +142,34 @@ func parseProjection(body []byte, contentType, ratingKey string) (projectionDocu if err != nil { return nil, nil, err } + document, err := parseProjectionDocument(body, contentType) + if err != nil { + return nil, nil, err + } + selection, err := document.selectTarget(ratingKey) + if err != nil { + return nil, nil, err + } + return document, selection, nil +} + +func parseProjectionDocument(body []byte, contentType string) (projectionDocument, error) { trimmed := trimProjectionBody(body) if len(trimmed) == 0 { - return nil, nil, errors.New("Plex metadata response is empty") + return nil, errors.New("Plex metadata response is empty") } if strings.Contains(strings.ToLower(contentType), "json") || trimmed[0] == '{' || trimmed[0] == '[' { document, err := parseJSONProjectionDocument(body) if err != nil { - return nil, nil, err - } - selection, err := document.selectTarget(ratingKey) - if err != nil { - return nil, nil, err + return nil, err } - return document, selection, nil + return document, nil } document, err := parseXMLProjectionDocument(body) if err != nil { - return nil, nil, err - } - selection, err := document.selectTarget(ratingKey) - if err != nil { - return nil, nil, err + return nil, err } - return document, selection, nil + return document, nil } func normalizeProjectionRatingKey(value string) (string, error) { @@ -236,6 +296,9 @@ func streamProjectionAttributes(stream mediainfo.Stream) []projectionAttribute { if stream.Codec != "" { attributes = append(attributes, textProjectionAttribute("codec", plexCodec(stream.Codec))) } + if codecID := plexCodecID(stream.CodecTag); codecID != "" { + attributes = append(attributes, textProjectionAttribute("codecID", codecID)) + } if stream.Profile != "" { attributes = append(attributes, textProjectionAttribute("profile", stream.Profile)) } @@ -315,6 +378,19 @@ func streamProjectionAttributes(stream mediainfo.Stream) []projectionAttribute { return attributes } +func plexCodecID(value string) string { + value = strings.TrimSpace(value) + if value == "" || strings.Contains(value, "[0]") || strings.EqualFold(value, "0x0000") { + return "" + } + for index := 0; index < len(value); index++ { + if value[index] < 0x21 || value[index] > 0x7e { + return "" + } + } + return value +} + // plexCodec keeps the normalized codec vocabulary expected by Plex clients. // ffprobe may expose a container-specific subtitle codec name that Plex // represents using its short format name. @@ -714,14 +790,46 @@ func (document *xmlProjectionDocument) selectTarget(ratingKey string) (projectio if len(videos) != 1 { return nil, errors.New("Plex XML metadata must contain one Video item") } - item := videos[0] - itemRatingKey, present, err := xmlAttribute(item.start.Attr, "ratingKey") + selection, err := xmlProjectionSelectionFromItem(videos[0]) if err != nil { return nil, err } - if !present || itemRatingKey == "" || itemRatingKey != ratingKey { + if selection.value.RatingKey != ratingKey { return nil, errors.New("Plex XML metadata ratingKey does not match the requested item") } + return selection, nil +} + +func (document *xmlProjectionDocument) selectTargets() ([]projectionSelection, error) { + roots := xmlProjectionElements(document.nodes) + if len(roots) != 1 || roots[0].start.Name.Local != "MediaContainer" { + return nil, errors.New("Plex XML metadata has no unique MediaContainer root") + } + videos := directXMLElements(roots[0], "Video") + selections := make([]projectionSelection, 0, len(videos)) + seen := make(map[string]struct{}, len(videos)) + for _, item := range videos { + selection, err := xmlProjectionSelectionFromItem(item) + if err != nil { + continue + } + if _, duplicate := seen[selection.value.RatingKey]; duplicate { + return nil, errors.New("Plex XML metadata has duplicate ratingKey items") + } + seen[selection.value.RatingKey] = struct{}{} + selections = append(selections, selection) + } + return selections, nil +} + +func xmlProjectionSelectionFromItem(item *xmlProjectionElement) (*xmlProjectionSelection, error) { + itemRatingKey, present, err := xmlAttribute(item.start.Attr, "ratingKey") + if err != nil { + return nil, err + } + if !present || itemRatingKey == "" { + return nil, errors.New("Plex XML metadata item has no ratingKey") + } mediaItems := directXMLElements(item, "Media") if len(mediaItems) != 1 { return nil, errors.New("Plex XML metadata must contain one Media") @@ -739,7 +847,7 @@ func (document *xmlProjectionDocument) selectTarget(ratingKey string) (projectio return nil, err } return &xmlProjectionSelection{ - value: Target{Part: part, NeedsEnrichment: needs}, + value: Target{RatingKey: itemRatingKey, Part: part, NeedsEnrichment: needs}, media: mediaItems[0], part: parts[0], }, nil @@ -1246,33 +1354,74 @@ func encodeJSONProjectionList(list jsonProjectionList) (json.RawMessage, error) } func (document *jsonProjectionDocument) selectTarget(ratingKey string) (projectionSelection, error) { + container, metadata, err := document.directMetadata() + if err != nil { + return nil, err + } + if len(metadata.items) != 1 { + return nil, errors.New("Plex JSON metadata must contain one Metadata item") + } + selection, err := jsonProjectionSelectionFromItem(document, container, metadata, metadata.items[0]) + if err != nil { + return nil, err + } + if selection.value.RatingKey != ratingKey { + return nil, errors.New("Plex JSON metadata ratingKey does not match the requested item") + } + return selection, nil +} + +func (document *jsonProjectionDocument) selectTargets() ([]projectionSelection, error) { + container, metadata, err := document.directMetadata() + if err != nil { + return nil, err + } + selections := make([]projectionSelection, 0, len(metadata.items)) + seen := make(map[string]struct{}, len(metadata.items)) + for _, item := range metadata.items { + selection, err := jsonProjectionSelectionFromItem(document, container, metadata, item) + if err != nil { + continue + } + if _, duplicate := seen[selection.value.RatingKey]; duplicate { + return nil, errors.New("Plex JSON metadata has duplicate ratingKey items") + } + seen[selection.value.RatingKey] = struct{}{} + selections = append(selections, selection) + } + return selections, nil +} + +func (document *jsonProjectionDocument) directMetadata() (jsonObject, jsonProjectionList, error) { containerRaw, ok := document.root["MediaContainer"] if !ok { - return nil, errors.New("Plex JSON metadata has no MediaContainer") + return nil, jsonProjectionList{}, errors.New("Plex JSON metadata has no MediaContainer") } container, err := decodeJSONProjectionObject(bytes.TrimSpace(containerRaw)) if err != nil { - return nil, fmt.Errorf("parse Plex JSON MediaContainer: %w", err) + return nil, jsonProjectionList{}, fmt.Errorf("parse Plex JSON MediaContainer: %w", err) } metadataRaw, ok := container["Metadata"] if !ok { - return nil, errors.New("Plex JSON metadata has no Metadata item") + return nil, jsonProjectionList{}, errors.New("Plex JSON metadata has no Metadata item") } metadata, err := decodeJSONProjectionList(metadataRaw, "Metadata") if err != nil { - return nil, err + return nil, jsonProjectionList{}, err } - if len(metadata.items) != 1 { - return nil, errors.New("Plex JSON metadata must contain one Metadata item") - } - item := metadata.items[0] + return container, metadata, nil +} + +func jsonProjectionSelectionFromItem( + document *jsonProjectionDocument, + container jsonObject, + metadata jsonProjectionList, + item *jsonObject, +) (*jsonProjectionSelection, error) { itemRatingKey, err := requiredJSONScalarString(*item, "ratingKey") if err != nil { return nil, err } - if itemRatingKey != ratingKey { - return nil, errors.New("Plex JSON metadata ratingKey does not match the requested item") - } mediaRaw, ok := (*item)["Media"] if !ok { return nil, errors.New("Plex JSON metadata has no Media") @@ -1306,26 +1455,28 @@ func (document *jsonProjectionDocument) selectTarget(ratingKey string) (projecti return nil, err } return &jsonProjectionSelection{ - value: Target{Part: part, NeedsEnrichment: needs}, - document: document, - container: container, - metadata: metadata, - media: mediaItems, - parts: parts, - mediaItem: mediaItem, - part: partItem, + value: Target{RatingKey: itemRatingKey, Part: part, NeedsEnrichment: needs}, + document: document, + container: container, + metadata: metadata, + metadataItem: item, + media: mediaItems, + parts: parts, + mediaItem: mediaItem, + part: partItem, }, nil } type jsonProjectionSelection struct { - value Target - document *jsonProjectionDocument - container jsonObject - metadata jsonProjectionList - media jsonProjectionList - parts jsonProjectionList - mediaItem *jsonObject - part *jsonObject + value Target + document *jsonProjectionDocument + container jsonObject + metadata jsonProjectionList + metadataItem *jsonObject + media jsonProjectionList + parts jsonProjectionList + mediaItem *jsonObject + part *jsonObject } func (selection *jsonProjectionSelection) target() Target { @@ -1482,8 +1633,7 @@ func (selection *jsonProjectionSelection) commit() error { if err != nil { return err } - // The metadata list has exactly one item by selection contract. - (*selection.metadata.items[0])["Media"] = media + (*selection.metadataItem)["Media"] = media metadata, err := encodeJSONProjectionList(selection.metadata) if err != nil { return err diff --git a/internal/plexmeta/projection_test.go b/internal/plexmeta/projection_test.go index eca5847..4a409d3 100644 --- a/internal/plexmeta/projection_test.go +++ b/internal/plexmeta/projection_test.go @@ -41,6 +41,136 @@ func TestSelectEnrichmentTargetStrictlySelectsJSONItem(t *testing.T) { } } +func TestEnrichMetadataTargetsProjectsOnlyMappedCollectionItems(t *testing.T) { + media := projectionStreamMedia() + tests := []struct { + name string + contentType string + body []byte + wantCloud []byte + wantLocal []byte + }{ + { + name: "XML", + contentType: "application/xml", + body: []byte(``), + wantCloud: []byte(`ratingKey="42">`), + }, + { + name: "JSON", + contentType: "application/json", + body: []byte(`{"MediaContainer":{"Metadata":[` + + `{"ratingKey":"42","Media":[{"Part":[{"id":7,"file":"/cloud/a.strm"}]}]},` + + `{"ratingKey":"43","Media":[{"Part":[{"id":8,"file":"/cloud/b.strm"}]},{"Part":[{"id":9,"file":"/cloud/c.strm"}]}]},` + + `{"ratingKey":"44","Media":[{"Part":[{"id":10,"file":"/cloud/d.strm"},{"id":11,"file":"/cloud/e.strm"}]}]},` + + `{"ratingKey":"45","Media":[{}]}]}}`), + }, + } { + t.Run(test.name, func(t *testing.T) { + original := append([]byte(nil), test.body...) + targets, err := SelectEnrichmentTargets(test.body, test.contentType) + if err != nil { + t.Fatal(err) + } + if len(targets) != 1 || targets[0].RatingKey != "42" || targets[0].Part.ID != "7" { + t.Fatalf("targets = %#v", targets) + } + enriched, changed, err := EnrichMetadataTargets(test.body, test.contentType, map[string]mediainfo.Media{"42": media}) + if err != nil { + t.Fatal(err) + } + if changed != 1 || bytes.Count(enriched, []byte("HDR10")) != 2 { + t.Fatalf("changed=%d body=%s", changed, enriched) + } + if bytes.Contains(enriched, []byte(`ratingKey="43"`)) && bytes.Contains(enriched, []byte(`ratingKey="43">`), + }, + { + name: "JSON", + contentType: "application/json", + body: []byte(`{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"Part":[{"id":7,"file":"/cloud/a.strm"}]}]},{"ratingKey":"42","Media":[{"Part":[{"id":8,"file":"/cloud/b.strm"}]}]}]}}`), + }, + } { + t.Run(test.name, func(t *testing.T) { + original := append([]byte(nil), test.body...) + if _, err := SelectEnrichmentTargets(test.body, test.contentType); err == nil { + t.Fatal("duplicate ratingKey collection unexpectedly succeeded") + } + if result, changed, err := EnrichMetadataTargets(test.body, test.contentType, map[string]mediainfo.Media{"42": projectionStreamMedia()}); err == nil || result != nil || changed != 0 { + t.Fatalf("result=%q changed=%d err=%v", result, changed, err) + } + if !bytes.Equal(test.body, original) { + t.Fatal("EnrichMetadataTargets modified its input body after an error") + } + }) + } +} + func TestEnrichMetadataXMLFillsOnlyMissingSafeFields(t *testing.T) { body := []byte(``) original := append([]byte(nil), body...) @@ -235,6 +365,7 @@ func TestEnrichMetadataCreatesStreamsWhenPartHasNone(t *testing.T) { func TestEnrichMetadataUsesDolbyVisionPlexFields(t *testing.T) { media := projectionStreamMedia() media.Streams[0].HDRFormat = "dolby_vision" + media.Streams[0].CodecTag = "dvh1" media.Streams[0].DolbyVision = &mediainfo.DolbyVision{ VersionMajor: 1, VersionMinor: 0, Profile: 7, Level: 6, RPUPresent: 1, ELPresent: 1, BLPresent: 1, BLCompatID: 6, @@ -263,6 +394,9 @@ func TestEnrichMetadataUsesDolbyVisionPlexFields(t *testing.T) { if test.contentType == "application/xml" { attrs := xmlProjectionAttributeSets(t, result)["Stream#1"] assertDolbyVisionAttributes(t, attrs) + if attrs["codecID"] != "dvh1" { + t.Fatalf("Dolby Vision codecID = %q", attrs["codecID"]) + } return } var document map[string]any @@ -271,7 +405,7 @@ func TestEnrichMetadataUsesDolbyVisionPlexFields(t *testing.T) { } part := document["MediaContainer"].(map[string]any)["Metadata"].([]any)[0].(map[string]any)["Media"].([]any)[0].(map[string]any)["Part"].([]any)[0].(map[string]any) attrs := part["Stream"].([]any)[0].(map[string]any) - if attrs["displayTitle"] != "4K DoVi/HDR10" || attrs["extendedDisplayTitle"] != "4K DoVi/HDR10 (HEVC Main 10)" || + if attrs["codecID"] != "dvh1" || attrs["displayTitle"] != "4K DoVi/HDR10" || attrs["extendedDisplayTitle"] != "4K DoVi/HDR10 (HEVC Main 10)" || attrs["DOVIBLCompatID"] != float64(6) || attrs["DOVIBLPresent"] != true || attrs["DOVIELPresent"] != true || attrs["DOVILevel"] != float64(6) || attrs["DOVIPresent"] != true || attrs["DOVIProfile"] != float64(7) || @@ -282,6 +416,20 @@ func TestEnrichMetadataUsesDolbyVisionPlexFields(t *testing.T) { } } +func TestPlexCodecIDRejectsFFProbePlaceholderTags(t *testing.T) { + for value, want := range map[string]string{ + "dvh1": "dvh1", + " hvc1 ": "hvc1", + "[0][0][0][0]": "", + "0x0000": "", + "bad\nvalue": "", + } { + if got := plexCodecID(value); got != want { + t.Fatalf("plexCodecID(%q) = %q, want %q", value, got, want) + } + } +} + func TestEnrichMetadataProjectsPlexMediaFrameRateAndPreciseStreamRate(t *testing.T) { tests := []struct { name string diff --git a/scripts/reset-mediainfo-cache.sh b/scripts/reset-mediainfo-cache.sh new file mode 100755 index 0000000..e483a4f --- /dev/null +++ b/scripts/reset-mediainfo-cache.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +if [ -x /usr/local/bin/plex-gateway ]; then + exec /usr/local/bin/plex-gateway cache-reset "$@" +fi + +service=${PLEX_GATEWAY_COMPOSE_SERVICE:-plex-gateway} +exec docker compose exec -T "$service" /usr/local/bin/plex-gateway-reset-mediainfo-cache "$@" From 1e69468971880dd71bf77b11bc4f669dc369429d Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sat, 29 Aug 2026 23:07:20 +0800 Subject: [PATCH 08/15] feat: harden MediaInfo playback controls --- CHANGELOG.en.md | 40 ++- CHANGELOG.md | 27 +- README.en.md | 81 +++-- README.md | 59 +++- VERSION | 2 +- app.env.example | 17 +- cmd/plex-gateway/main.go | 2 + docs/architecture.md | 64 +++- docs/performance-matrix.md | 79 +++++ docs/roadmap.md | 41 ++- internal/config/config.go | 8 +- internal/config/config_test.go | 16 + internal/gateway/decision.go | 19 ++ internal/gateway/decision_capture.go | 39 ++- internal/gateway/decision_capture_test.go | 33 +- internal/gateway/decision_test.go | 250 +++++++++++++++- internal/gateway/gateway.go | 2 + internal/gateway/gateway_test.go | 11 +- internal/gateway/metadata_enrichment.go | 54 +++- internal/gateway/metadata_enrichment_test.go | 300 +++++++++++++++++++ internal/gateway/metadata_guard.go | 2 +- internal/gateway/playback_veto.go | 69 +++++ internal/gateway/playback_veto_test.go | 53 ++++ internal/gateway/universal_start_test.go | 8 +- internal/mediainfo/benchmark_test.go | 37 +++ internal/plexmeta/decision_reject.go | 34 +++ internal/plexmeta/decision_reject_test.go | 33 ++ 27 files changed, 1267 insertions(+), 113 deletions(-) create mode 100644 docs/performance-matrix.md create mode 100644 internal/gateway/playback_veto.go create mode 100644 internal/gateway/playback_veto_test.go create mode 100644 internal/plexmeta/decision_reject.go create mode 100644 internal/plexmeta/decision_reject_test.go diff --git a/CHANGELOG.en.md b/CHANGELOG.en.md index 57ce865..722f8a0 100644 --- a/CHANGELOG.en.md +++ b/CHANGELOG.en.md @@ -9,15 +9,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.1.2] - 2026-08-29 + ### Added - Added bounded remote ffprobe, an L1 LRU plus SQLite MediaInfo cache, and single-item Plex metadata response enrichment. Cold-wait timeouts preserve the original response while probing continues in the background. -- Current-Part prewarming now starts after a cloud redirect is ready. An isolated - Plex management token can additionally discover and rate-limit a configurable - nearby window, defaulting to two previous and three following items. Missing or - invalid management credentials disable only nearby discovery. +- Current-Part prewarming is submitted at interactive priority after a cloud + redirect is ready. An isolated Plex management token can additionally discover + and rate-limit a configurable nearby window, defaulting to two previous and + three following items. The current job is only prioritized in the queue and + cannot preempt a probe already running. Missing or invalid management + credentials disable only nearby discovery. - Added per-client rapid-switch coordination, cross-User-Agent failure handoff, and distinct prewarm metrics for fresh cache hits, joined work, new queue admissions, and rejections. @@ -33,6 +37,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 preserve the otherwise valid MediaInfo and fail open. - Updated the background-probe fallback User-Agent to the currently verified Infuse Library version. +- Added a disabled-by-default experimental Apple TV Plex Dolby Vision Profile 5 + playback veto. It only reuses fresh MediaInfo already obtained by the + decision path, stores no state, does not intercept Parts, and starts no + additional probe. +- Added a public performance matrix. Local media and 302 routes perform no + MediaInfo analysis I/O, and one decision waits for at most one cold probe. + +### Changed + +- This release persists fallback MediaInfo only in the gateway's SQLite store. + Production Plex DB writes remain a next-stage evaluation after coverage, + compatibility, backup, and rollback requirements are measured; the final API + or database-helper route is not selected here. + +### Fixed + +- Limited one continuous metadata browsing burst to one synchronous cold probe. + After the admitted synchronous wait releases its slot, a fixed five-second + window begins. Other cold misses immediately preserve the Plex response, and + rejected requests do not renew the window. An admitted probe may finish in the + background after the request wait ends and persist the result to L1 and SQLite, + preventing starvation under continuous traffic. +- Set the Metadata Guard defaults to global 8, per-client 4, and batch 3. After + Plex MediaInfo writes provide high coverage and the resulting load is retested, + global 16, per-client 4, and batch 3 remains a candidate for evaluation. ## [0.1.1] - 2026-08-28 @@ -66,6 +95,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 gateway redirects playback. - Logs and metrics exclude Plex tokens and complete signed media URLs. -[Unreleased]: https://github.com/InfinityPacer/plex-gateway/compare/v0.1.1...HEAD +[Unreleased]: https://github.com/InfinityPacer/plex-gateway/compare/v0.1.2...HEAD +[0.1.2]: https://github.com/InfinityPacer/plex-gateway/compare/v0.1.1...v0.1.2 [0.1.1]: https://github.com/InfinityPacer/plex-gateway/compare/v0.1.0...v0.1.1 [0.1.0]: https://github.com/InfinityPacer/plex-gateway/releases/tag/v0.1.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a2399a..3e35067 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,15 @@ ## [未发布] +## [0.1.2] - 2026-08-29 + ### 新增 - 增加 MediaInfo L1 LRU 与 SQLite 持久缓存、受限远程 ffprobe 和单项 Plex metadata 响应增强；冷缓存等待超时后保持原响应并在后台完成探测。 -- 云端重定向就绪后立即预热当前 Part，并可使用独立 Plex 管理 Token 按默认前 2、后 3 - 的窗口限速预热邻近媒体。Token 缺失或失效只禁用邻近发现。 +- 云端重定向就绪后将当前 Part 以交互优先级提交，并可使用独立 Plex 管理 Token 按默认 + 前 2、后 3 的窗口限速预热邻近媒体。当前任务只会优先排队，不能抢占正在运行的探测。 + Token 缺失或失效只禁用邻近发现。 - 增加按客户端隔离的快速切换协调、跨 User-Agent 失败接力，以及缓存命中、加入现有 任务、新入队和拒绝的预热指标。 - 为没有 Stream 的 STRM Part 创建不含 Plex ID 或播放选择状态的描述性视频、音频和 @@ -24,6 +27,23 @@ - 当 ffprobe 缺少媒体总大小时，使用同一 User-Agent 发起有界的单字节 Range 请求， 并从有效 `Content-Range` 补充 Part size，失败保持已有 MediaInfo 并正常回退。 - 后台探测 fallback User-Agent 更新为当前已验证的 Infuse Library 版本。 +- 增加默认关闭的实验性 Apple TV Plex Dolby Vision Profile 5 播放 veto。它只复用 + decision 已取得的新鲜 MediaInfo，不保存状态、不拦截 Part，也不发起额外探测。 +- 增加公开性能矩阵。本地媒体与 302 路径不执行 MediaInfo 分析 I/O，同一 decision + 最多等待一次冷探测。 + +### 变更 + +- 本版本只将兜底 MediaInfo 持久化到 Gateway SQLite。Plex DB 生产写入保留到下一阶段， + 待覆盖率、兼容性、备份和回滚方案完成评估后再决定采用 API 或数据库辅助方案。 + +### 修复 + +- 限制一次连续 metadata 浏览突发最多同步等待一个冷探测。释放本次同步准入后建立固定 + 5 秒窗口，窗口内的其他冷 miss 立即透传 Plex，拒绝请求不会续期窗口。已准入的探测 + 在请求等待结束后仍可在后台完成，并成功写入 L1 和 SQLite，避免持续流量造成饥饿。 +- 将单项 metadata Guard 的默认值固定为全局 8、单客户端 4、批量 3。Plex MediaInfo + 写库获得较高覆盖率并完成重测后，再评估全局 16、单客户端 4、批量 3 的候选值。 ## [0.1.1] - 2026-08-28 @@ -54,6 +74,7 @@ - Gateway 返回云端播放重定向前，由 Plex 使用当前客户端凭据授权对应 Part。 - 日志和指标不包含 Plex Token 或完整的签名媒体 URL。 -[未发布]: https://github.com/InfinityPacer/plex-gateway/compare/v0.1.1...HEAD +[未发布]: https://github.com/InfinityPacer/plex-gateway/compare/v0.1.2...HEAD +[0.1.2]: https://github.com/InfinityPacer/plex-gateway/compare/v0.1.1...v0.1.2 [0.1.1]: https://github.com/InfinityPacer/plex-gateway/compare/v0.1.0...v0.1.1 [0.1.0]: https://github.com/InfinityPacer/plex-gateway/releases/tag/v0.1.0 diff --git a/README.en.md b/README.en.md index 9a84ea4..e843c4b 100644 --- a/README.en.md +++ b/README.en.md @@ -34,11 +34,15 @@ playback session and exact Media/Part, authorizes the Part again, and returns the same CDN redirect without starting Plex Transcoder or synthesizing Plex metadata. -The gateway applies the same MediaInfo projection rules to every client. It -does not branch on product name, device name, or User-Agent for format -compatibility. Part redirects never wait for MediaInfo. A Plex decision may -wait up to `MEDIAINFO_COLD_WAIT` on a cold cache; probe timeout or failure -returns Plex's original response unchanged. +The gateway applies the same MediaInfo projection rules to every client. A +disabled-by-default experimental playback veto checks the Plex for Apple TV, +tvOS, Dolby Vision Profile 5 combination only after Plex returns Direct Play +for the same STRM Part and MediaInfo enrichment already has a fresh record. A +match returns a Plex-compatible unsupported decision; every other case keeps +the existing playback path. The veto stores no verdict, does not intercept the +Part path, and starts no additional probe. Apple TV hardware supports Profile +5, while compatibility still varies with Plex version, container, and source, +so the veto remains disabled by default. Native clients that accept a raw media redirect on those routes can use this path. Plex Web cloud playback is not supported: its browser player fetches @@ -54,7 +58,7 @@ media in Plex Web remains ordinary proxied Plex traffic. | Local Plex media | Supported by transparent proxy | The gateway does not rewrite local Parts. | | Infuse Direct Play | Direct Play verified | Uses the Plex Part redirect path. | | Plex iOS ExperimentalPlayer | Direct Play verified | STRM uses the universal decision/start redirect path. | -| Plex for Apple TV | Direct Play verified | STRM uses the universal decision/start redirect path and the same MediaInfo projection rules as other clients. Actual HDR and Dolby Vision output still depends on the source and client decoder capabilities. | +| Plex for Apple TV | Direct Play verified | HDR STRM uses the universal decision/start redirect path. Apple TV 4K supports Dolby Vision Profile 5 Direct Play, while Plex version, container, and source combinations may still affect compatibility; the gateway does not reject it by default. | | Plex Web cloud playback | Unsupported | The browser expects a DASH manifest and enforces final-origin CORS. | This is an independent community project. It is not affiliated with, endorsed @@ -128,6 +132,7 @@ Important environment variables: | `DATABASE_PATH` | `./data/plex-gateway.db` | Gateway SQLite database path; the container image defaults to `/app_data/plex-gateway.db`. | | `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.5.1` | Fallback User-Agent for background probes without an active client context. | | `MEDIAINFO_COLD_WAIT` | `5s` | Cold-cache wait ceiling for one metadata item; timeout returns the original Plex response while probing continues. | +| `PLAYBACK_VETO_ENABLED` | `false` | Enable the experimental Apple TV Plex Dolby Vision Profile 5 veto; it only reuses fresh MediaInfo already obtained by the decision path. | | `MEDIAINFO_RESPONSE_MAX_BYTES` | `8388608` | Maximum size of one Plex metadata response buffered for enrichment. | | `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `8` | Maximum single-item metadata responses buffered and waiting for MediaInfo concurrently. | | `MEDIAINFO_PREWARM_BEFORE` | `2` | Number of nearby items before the current item to prewarm. | @@ -154,6 +159,23 @@ through Plex. The gateway does not need a Plex username or password, and playback does not depend on the management token. MediaVault's API key for `/api/v1` integrations is also not required for the STRM `/redirect` playback contract. +### Playback veto + +With `PLAYBACK_VETO_ENABLED=false`, no veto function is created and the decision +path performs only one nil check. When enabled, the gateway returns an +unsupported decision only for complete MediaInfo matching Plex for Apple TV, +tvOS, Dolby Vision Profile 5, and BL compatibility ID 0. It reuses the fresh +record already obtained by MediaInfo enrichment, performs no additional cache, +SQLite, MediaVault, or CDN work, stores no state, and never enters Part or +universal start handling. Missing, stale, conflicting, or unmatched evidence +always fails open. + +This is an experimental switch for deployments with a known source +compatibility issue, not a claim that Apple TV generally lacks Dolby Vision +Profile 5 support. See +[docs/architecture.md](docs/architecture.md) and +[docs/performance-matrix.md](docs/performance-matrix.md). + ### MediaInfo response enrichment For an authenticated single-item STRM metadata request, the gateway can fill @@ -165,6 +187,21 @@ language code. Generated Streams never contain Plex Stream IDs or playback selection fields such as `selected`, `default`, or `decision`. When Plex already has Streams, only missing fields on identity-matched Streams are filled. Missing sibling Streams are not created and existing Plex values are not overwritten. +This release writes fallback probe results only to the gateway's SQLite store and +does not write to the Plex DB. Production Plex DB writes are a next-stage +evaluation after coverage, compatibility, backup, and rollback validation. + +The L1 hot path returns without synchronously reading SQLite for the request. +Access renewal may touch SQLite asynchronously, so persistence I/O stays out of +the request critical path. Hot-cache projection remains concurrent. One +continuous browsing burst may admit only one synchronous remote probe. After +the admitted synchronous wait releases its slot, a fixed five-second window +begins. Other cold misses immediately preserve the Plex response, and rejected +requests do not renew the window. An admitted probe may finish in the background +after the request wait ends and persist the result to L1 and SQLite. Another +browsing probe is admitted only after the window ends. Playback decisions retain +their independent cold-probe budget, so this limit only suppresses metadata fan-out +from home and season screens. When ffprobe cannot report the total media size, the gateway sends one `Range: bytes=0-0` request to the same temporary direct URL with the same @@ -179,10 +216,12 @@ do not repeat the CDN request. Background library synchronization can enumerate an entire library one item at a time. Requests with `skipRefresh` whose product ends in `-Library` may consume an existing MediaInfo cache record but never admit a cold probe. Ordinary -single-item access, a successful cloud 302, and the nearby window retain their -bounded probe behavior, so browsing a library cannot expand into full-library -CDN ffprobe work. Any cache miss, timeout, unsupported structure, or projection -failure returns the original Plex response. +single-item access follows the same fixed cold-probe window. Cold misses inside +the window are not admitted and do not renew it; successful cloud 302 and +nearby-window work retain their existing boundaries. Browsing a library +therefore cannot expand into full-library CDN ffprobe work. Any cache miss, +timeout, unsupported structure, or projection failure returns the original Plex +response. ### Nearby-item MediaInfo prewarming @@ -193,19 +232,22 @@ it does not claim that the client followed the redirect or actually started playback. Current-item prewarming does not require a management token. A valid `PLEX_TOKEN` additionally enables nearby-item discovery. -The current item is submitted immediately at interactive priority. The -background coordinator prefers explicit Plex playQueue order and permits movie, -cross-show, and multi-Media/Part entries. Every candidate is stored under its -own PartID and STRM fingerprint. Without a usable queue, Plex episode and season -response order is used, including specials and entries with missing indices. +The current item enters the interactive-priority queue. The background +coordinator prefers explicit Plex playQueue order and permits movie, cross-show, +and multi-Media/Part entries. Every candidate is stored under its own PartID and +STRM fingerprint. The current item can only be prioritized in the queue and +cannot preempt a probe already running. Without a usable queue, Plex episode and +season response order is used, including specials and entries with missing +indices. The default window is two items before and three after, with following items submitted first. The combined configurable window is capped at 50. Nearby items enter the low-priority queue every five seconds by default, while the MediaInfo worker defaults to one concurrent probe to avoid MediaVault/CDN bursts. A rapid A/B/C switch submits each new current item immediately and cancels only the old -window entries that were not submitted. Submitted work retains its own identity -and is deduplicated by singleflight. +window entries that were not submitted. Submitted or running work retains its +own identity and is deduplicated by singleflight. A running background probe is +not preempted by a current item. The gateway persists parsed MediaInfo, not short-lived CDN URLs or raw head/tail bytes. MediaVault upload precaching applies only to files uploaded through @@ -238,6 +280,11 @@ Batch reads do not consume the global or per-client slots reserved for interactive single-item metadata requests, and metadata mutations do not enter the batch pool. +The current Guard defaults are global 8 for single-item metadata, per-client 4, +and batch 3. After Plex MediaInfo writes provide high coverage and the resulting +load is retested, global 16, per-client 4, and batch 3 may be evaluated. This +release adds no Guard mode or configuration item. + ## Advertise the gateway through Plex In Plex Web, open the server's **Settings > Network > Custom server access diff --git a/README.md b/README.md index 40ce877..07a31fa 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,12 @@ Part 标记为 Direct Play 后，Gateway 才会创建短期授权。如果客户 对应的授权，再次为 Part 请求授权，并返回同一个 CDN 重定向，而不会启动 Plex Transcoder 或合成 Plex 元数据。 -Gateway 对所有客户端使用同一套 MediaInfo 投影规则，不根据产品名、设备名或 -User-Agent 特判格式兼容性。Part 302 不等待 MediaInfo；Plex decision 在冷缓存时最多 -等待 `MEDIAINFO_COLD_WAIT`，探测超时或失败会原样返回 Plex 响应。 +Gateway 对所有客户端使用同一套 MediaInfo 投影规则。默认关闭的实验性播放 veto 只在 +Plex 已返回同一 STRM Part 的 Direct Play decision、且本次 MediaInfo 增强已经取得新鲜 +记录时检查 Apple TV Plex 与 Dolby Vision Profile 5 组合。命中时返回 Plex 兼容的 +不支持决策，其他情况保持原播放链路。该判断不保存 verdict、不拦截 Part，也不会触发 +额外探测。Apple TV 硬件支持 Profile 5，实际兼容性还受 Plex 版本、容器和片源影响， +因此 veto 默认关闭。 能够在这些路径上接受原始媒体重定向的原生客户端可以使用此链路。Plex Web 不支持 云端播放：其浏览器播放器会将 `start.mpd` 作为 DASH manifest 获取，而重定向目标 @@ -43,7 +46,7 @@ User-Agent 特判格式兼容性。Part 302 不等待 MediaInfo；Plex decision | Plex 本地媒体 | 透明代理支持 | Gateway 不会重写本地 Part。 | | Infuse Direct Play | 已验证 Direct Play | 使用 Plex Part 重定向路径。 | | Plex iOS ExperimentalPlayer | 已验证 Direct Play | STRM 使用通用 decision/start 重定向链路。 | -| Plex for Apple TV | 已验证 Direct Play | STRM 使用通用 decision/start 重定向链路，并与其他客户端使用相同 MediaInfo 投影规则。实际 HDR/Dolby Vision 输出仍取决于片源与客户端解码能力。 | +| Plex for Apple TV | 已验证 Direct Play | HDR STRM 使用通用 decision/start 重定向链路。Apple TV 4K 支持 DV Profile 5 Direct Play，但 Plex 版本、容器和片源组合仍可能影响兼容性；Gateway 默认不拒绝。 | | Plex Web 云端播放 | 不支持 | 浏览器需要 DASH manifest，并受最终源站 CORS 限制。 | 本项目是独立的社区项目，与 Plex、MediaVault、Infuse 或其各自所有者没有关联， @@ -114,6 +117,7 @@ go run ./cmd/plex-gateway | `DATABASE_PATH` | `./data/plex-gateway.db` | Gateway SQLite 持久数据库路径；容器镜像默认使用 `/app_data/plex-gateway.db`。 | | `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.5.1` | 没有活动客户端上下文时，后台探测使用的 fallback User-Agent。 | | `MEDIAINFO_COLD_WAIT` | `5s` | 单项 metadata 冷缓存等待上限；超时返回原 Plex 响应，探测继续。 | +| `PLAYBACK_VETO_ENABLED` | `false` | 启用实验性 Apple TV Plex Dolby Vision Profile 5 veto；只复用 decision 已取得的新鲜 MediaInfo，不发起额外探测。 | | `MEDIAINFO_RESPONSE_MAX_BYTES` | `8388608` | 可以缓冲并尝试增强的单个 Plex metadata 响应上限。 | | `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `8` | 同时缓冲和等待 MediaInfo 的单项 metadata 响应上限。 | | `MEDIAINFO_PREWARM_BEFORE` | `2` | 当前项之前的邻近媒体预热数量。 | @@ -137,6 +141,19 @@ Git 的 `app.env` 中。推荐让 Compose 只通过 `env_file: ./app.env` 加载 单独配置 Plex 用户名或密码，播放也不依赖管理 Token。STRM `/redirect` 播放协议不需要 MediaVault 用于 `/api/v1` 集成的 API key。 +### 播放 veto + +`PLAYBACK_VETO_ENABLED=false` 时不创建 veto 函数，decision 主线路只经过一次 nil 判断。 +显式开启后，Gateway 只对 Apple TV Plex、tvOS、Dolby Vision Profile 5 且 BL +compatibility ID 为 0 的完整 MediaInfo 返回不支持 decision。它只读取 MediaInfo 增强 +已经取得的新鲜记录，不新增缓存读取、SQLite、MediaVault 或 CDN 请求，不创建状态， +也不进入 Part 和 universal start 路径。证据缺失、过期、冲突或不匹配一律 fail-open。 + +这是用于已知片源兼容问题的实验性开关，不代表 Apple TV 普遍不支持 Dolby Vision +Profile 5。配置和性能验收见 +[docs/architecture.md](docs/architecture.md) 与 +[docs/performance-matrix.md](docs/performance-matrix.md)。 + ### MediaInfo 响应增强 对于已认证的单项 STRM metadata 请求，Gateway 可以从 L1 或 SQLite 记录补充缺失的 @@ -145,6 +162,15 @@ ffprobe 的流类型和源索引创建描述性的视频、音频和字幕 Strea Vision、bit depth、codec、codecID、bitrate、声道布局和语言码等字段。合成 Stream 不包含 Plex Stream ID，也不生成 `selected`、`default` 或 `decision` 等播放选择字段。Plex 已经存在 Stream 时只补充身份匹配 Stream 的缺失字段，不创建缺少的其他流，也不覆盖 Plex 值。 +当前版本只将兜底探测结果写入 Gateway SQLite，不写入 Plex DB。Plex DB 生产写入属于下一 +阶段评估内容，需先完成覆盖率、兼容性、备份和回滚验证。 + +L1 热路径直接返回，不会为了本次请求同步读取 SQLite。访问续期可能异步 touch SQLite， +不会把持久化 I/O 放进请求关键路径。热缓存投影可以并发执行，一次连续的浏览请求突发只 +允许一个请求同步等待远程探测。释放本次同步准入后建立固定 5 秒窗口，窗口内其他冷 miss +立即返回原始 Plex 响应，拒绝请求不会续期窗口。已准入的探测在请求等待结束后仍可在后台 +完成，并成功写入 L1 和 SQLite。窗口结束后才允许下一个浏览冷探测。播放 decision 保留 +独立的冷探测预算，因此该限制只抑制首屏和季页面的 metadata 请求风暴。 当 ffprobe 无法返回媒体总大小时，Gateway 会使用与本次 MediaVault 解析和 ffprobe 相同的 User-Agent 对同一临时直链发送一次 `Range: bytes=0-0`。该请求最多等待 `2s`，只接受包含有效 @@ -154,9 +180,11 @@ User-Agent 对同一临时直链发送一次 `Range: bytes=0-0`。该请求最 重复请求 CDN。 Infuse 等客户端的后台媒体库同步可能逐项请求整个库。带 `skipRefresh` 且产品名以 -`-Library` 结尾的后台同步请求只读取现有 MediaInfo 缓存，不创建冷探测。普通单项访问、 -成功的云端 302 和邻近窗口仍可按既定边界提交探测，因此浏览媒体库不会扩散成全库 CDN -ffprobe。任何缓存 miss、超时、结构不支持或投影失败都会返回原始 Plex response。 +`-Library` 结尾的后台同步请求只读取现有 MediaInfo 缓存，不创建冷探测。普通单项访问 +遵循同一个固定冷探测窗口，窗口内的冷 miss 不准入且不会续期；成功的云端 302 和邻近窗口 +继续按既定边界提交。 +因此浏览媒体库不会扩散成全库 CDN ffprobe。任何缓存 miss、超时、结构不支持或投影失败 +都会返回原始 Plex response。 ### 邻近媒体 MediaInfo 预热 @@ -165,16 +193,17 @@ Gateway 在云端 Part 已通过 Plex 授权、MediaVault 已返回最终直链 客户端已经跟随 302 或真实起播。当前项预热不需要管理 Token；配置有效 `PLEX_TOKEN` 后才会额外发现邻近媒体。 -当前项会立即以交互优先级提交。后台优先使用明确的 Plex playQueue 顺序，并允许其中的 +当前项会进入交互优先级队列。后台优先使用明确的 Plex playQueue 顺序，并允许其中的 电影、跨剧条目和多 Media/Part；候选始终采用自身的 PartID 与 STRM fingerprint 落库， -不会写入当前项。没有可靠 playQueue 时，按 Plex 返回的剧集和季顺序查找邻近项，包括 -S00 和缺失索引的条目。 +不会写入当前项。当前项只能优先排队，不能抢占正在运行的探测。没有可靠 playQueue 时， +按 Plex 返回的剧集和季顺序查找邻近项，包括 S00 和缺失索引的条目。 默认窗口为前 `2`、后 `3`，后续项先提交；两个方向合计最多配置 `50`。邻近项默认每隔 `5s` 进入低优先级队列，MediaInfo worker 默认并发为 `1`，用于避免 MediaVault/CDN 请求突发。快速切换 A、B、C 时，新当前项立即进入交互队列，旧窗口中尚未提交的候选会被 -取消；已经提交的任务仍按自己的身份完成并由 singleflight 去重。整个发现、STRM 读取、 -指纹计算和 SQLite 查询均在 302 响应之后执行。 +取消；已经提交或正在运行的任务仍按自己的身份完成并由 singleflight 去重。正在运行的 +后台探测不会被当前项抢占。整个发现、STRM 读取、指纹计算和 SQLite 查询均在 302 响应 +之后执行。 Gateway 只持久化解析后的 MediaInfo，不缓存短期 CDN URL 或原始文件头尾。MediaVault 的 上传预缓存只覆盖经其上传的新文件；若未来提供稳定 MediaInfo/预缓存读取 API，可作为优先 @@ -197,6 +226,10 @@ Gateway 会在这些单项详细 metadata 请求进入 Plex 前应用全局和 `GET/HEAD /library/metadata/1,2,...`。批量读取不会占用交互式单项 metadata 的全局或 单客户端槽位，metadata PUT 等修改请求也不会进入批量池。 +当前默认 Guard 为单项全局 8、单客户端 4、批量 3。Plex MediaInfo 写库完成较高覆盖率 +并经过重新压测后，才评估单项全局 16、单客户端 4、批量 3 的候选值。当前版本不新增 +Guard 模式或配置项。 + ## 通过 Plex 发布 Gateway 地址 在 Plex Web 中打开服务器的 **Settings > Network > Custom server access URLs**， @@ -269,7 +302,7 @@ Gateway 重新执行检查。 - `GET /health` 返回进程健康状态，以及不含媒体身份和凭据的 MediaInfo 缓存、探测队列和邻近媒体预热摘要。 - `GET /metrics` 返回固定结构的 JSON 计数器，以及 resolver 和完整重定向链路的 延迟总计、样本数、最近值和最大值。启用 metadata 保护时还会返回准入、超时、 - 活动和排队计数，所有指标都不包含请求标签或凭据。 + 活动和排队计数。所有指标都不包含请求标签或凭据。 - 其他所有 endpoint 都遵循 [docs/architecture.md](docs/architecture.md) 中说明 的 Plex 代理或 Direct Play 拦截规则。 diff --git a/VERSION b/VERSION index 17e51c3..d917d3e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.1 +0.1.2 diff --git a/app.env.example b/app.env.example index 24a7636..87ad25b 100644 --- a/app.env.example +++ b/app.env.example @@ -41,16 +41,17 @@ PART_PROBE_TIMEOUT=15s # 是否限制单项详细 metadata 请求进入 Plex 的并发量。 METADATA_GUARD_ENABLED=true -# 所有客户端共享的单项详细 metadata 并发上限。 +# 所有客户端共享的单项详细 metadata 并发上限。当前默认值为 8。 +# Plex MediaInfo 写库并完成重新压测后，才评估是否使用 16。 METADATA_GUARD_GLOBAL_CONCURRENCY=8 -# 单个 Plex 客户端的单项详细 metadata 并发上限，不能大于全局上限。 +# 单个 Plex 客户端的单项详细 metadata 并发上限，当前默认值为 4。 METADATA_GUARD_CLIENT_CONCURRENCY=4 # 是否为逗号分隔的批量 metadata 请求启用独立并发保护。 METADATA_GUARD_BATCH_ENABLED=true -# 批量 metadata 请求的全局并发上限。 +# 批量 metadata 请求的全局并发上限，当前默认值为 3。 METADATA_GUARD_BATCH_CONCURRENCY=3 # metadata 并发槽位的最长等待时间，超时返回 429。 @@ -98,22 +99,26 @@ MEDIAINFO_NEGATIVE_TTL=15m # 没有活动客户端上下文时，后台探测使用的 fallback User-Agent。 MEDIAINFO_USER_AGENT=Infuse-Library/8.5.1 -# 单项 metadata 冷缓存最多等待多久。超时后原响应立即返回，探测继续在后台执行。 +# 单项 metadata 冷缓存最多等待多久。超时后原响应立即返回，已准入的探测继续在后台执行。 MEDIAINFO_COLD_WAIT=5s +# 是否启用实验性 Apple TV Plex Dolby Vision Profile 5 播放 veto。默认关闭； +# 仅复用 decision 已取得的新鲜 MediaInfo，不发起额外探测，也不影响 Part 302。 +PLAYBACK_VETO_ENABLED=false + # 单个 Plex metadata 响应允许缓冲并尝试增强的最大字节数。 MEDIAINFO_RESPONSE_MAX_BYTES=8388608 # 同时缓冲并等待 MediaInfo 的单项 metadata 响应数量。 MEDIAINFO_ENRICHMENT_CONCURRENCY=8 -# 当前播放项之前预热的邻近媒体数量。前后都设为 0 时仅立即预热当前项。 +# 当前播放项之前预热的邻近媒体数量。前后都设为 0 时仅提交当前项。 MEDIAINFO_PREWARM_BEFORE=2 # 当前播放项之后预热的邻近媒体数量。前后合计不能超过 50。 MEDIAINFO_PREWARM_AFTER=3 -# 邻近项进入后台探测队列的间隔。当前播放项不等待该间隔并立即优先提交。 +# 邻近项进入后台探测队列的间隔。当前播放项只优先排队，不能抢占运行中的探测。 MEDIAINFO_PREWARM_INTERVAL=5s # HTTP 请求头读取超时。 diff --git a/cmd/plex-gateway/main.go b/cmd/plex-gateway/main.go index a9e4a67..c4d6f37 100644 --- a/cmd/plex-gateway/main.go +++ b/cmd/plex-gateway/main.go @@ -123,6 +123,7 @@ func run() error { MediaInfoEnrichmentConcurrency: cfg.MediaInfo.EnrichmentWaiters, MediaInfoPrewarmer: prewarmService, MediaInfoPrewarmStatus: prewarmStatus, + PlaybackVeto: cfg.PlaybackVeto, }) handler := newMaintenanceHandler(gatewayHandler, mediaInfoService, cfg.DatabasePath, logger) server := &http.Server{ @@ -153,6 +154,7 @@ func run() error { "mediainfo_prewarm_before", cfg.MediaInfo.PrewarmBefore, "mediainfo_prewarm_after", cfg.MediaInfo.PrewarmAfter, "mediainfo_prewarm_interval_ms", cfg.MediaInfo.PrewarmInterval.Milliseconds(), + "playback_veto_enabled", cfg.PlaybackVeto, ) serveErrors <- server.ListenAndServe() }() diff --git a/docs/architecture.md b/docs/architecture.md index 0eae3d0..dbf5615 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -18,6 +18,11 @@ gateway; the isolated analysis worker may read a bounded amount for probing. consume interactive metadata slots. Neither pool caches or rewrites Plex responses. Library listings, metadata mutations, playback paths, timeline, and watch-state traffic do not enter these pools. +- The current Guard defaults are global 8 for single-item metadata, per-client + 4, and batch 3. A global 16, per-client 4, batch 3 candidate is reserved for + evaluation after Plex MediaInfo writes provide high coverage and the resulting + load is retested. No additional Guard mode or configuration is part of this + release. - Successful XML or JSON Plex responses are observed to populate the in-memory `Part.id` cache without changing their bytes. When MediaInfo is available, an authenticated single-item metadata response may also fill missing whitelisted @@ -27,7 +32,13 @@ gateway; the isolated analysis worker may read a bounded amount for probing. selection state, and an existing Stream set is never expanded. The transform is bounded and preserves the original Plex response on timeout, ambiguity, unsupported encoding, or any other failure. Background library crawlers - identified by `skipRefresh` and a `-Library` product consume cache only. + identified by `skipRefresh` and a `-Library` product consume cache only. Hot + records remain concurrently projectable; one continuous browsing burst may + wait on only one cold probe. After the admitted synchronous wait releases its + slot, a fixed five-second window begins. Other cold misses preserve the Plex + response, and rejected requests do not renew the window. An admitted probe + may finish in the background after the request wait ends and persist to L1 + and SQLite, preventing guard-delayed request waves from starving later work. - `GET` and `HEAD` requests below `/library/parts/{partID}/...` are eligible for interception only when the cached `Part.file` has a configured cloud extension and maps to a readable local STRM file. @@ -41,6 +52,11 @@ gateway; the isolated analysis worker may read a bounded amount for probing. Plex's response remain intact. The complete XML or JSON decision response is observed without rewriting it; a start grant exists only when the same Part is explicitly marked `directplay`. +- After Plex accepts the exact STRM Part for Direct Play, an optional playback + veto may inspect the fresh MediaInfo record already obtained for response + enrichment. The built-in veto covers only Plex for Apple TV on tvOS with + Dolby Vision Profile 5 and BL compatibility ID 0. A match returns a + Plex-compatible unsupported envelope and creates no Direct Play grant. - Eligible universal `start`, `start.mpd`, and `start.m3u8` requests repeat the Part authorization using the exact selection captured by a short-lived Direct Play decision grant for the same client session and Media/Part, then resolve @@ -75,7 +91,7 @@ control-file and metadata readers. | State | Authority and writer | Invalidation | | --- | --- | --- | | Plex library, descriptive metadata, and watch state | Plex only; the gateway forwards timeline/scrobble traffic unchanged. | Plex lifecycle. | -| Technical MediaInfo | A validated producer record, or Gateway SQLite for fallback-probed media; Plex fields and response enrichment are projections. | Source or STRM fingerprint change, schema change, or freshness policy. | +| Technical MediaInfo | A validated producer record, or Gateway SQLite for fallback-probed media; Plex fields and response enrichment are projections. Plex DB production writes are not enabled in this release. | Source or STRM fingerprint change, schema change, or freshness policy. | | Part cache | Derived by metadata observation or an authenticated playback selection, keyed by `Part.id`. | Configured TTL or process restart. | | Direct Play grant | Published only after a complete Plex decision explicitly accepts the exact Part; a new related decision revokes the previous grant first. | Five-minute TTL, bounded eviction, or explicit revocation. | | MediaVault direct URL | Produced for the active authorized request and copied only to `Location`. | Never persisted or reused by the gateway. | @@ -154,11 +170,22 @@ Universal start routes are redirected only after the same STRM selection and authorization checks; manifest segments and genuine transcode traffic remain Plex-owned. -All clients use the same MediaInfo projection rules. The gateway does not infer -format compatibility from a product name, device name, or User-Agent. Part and -universal-start redirects never wait for MediaInfo. A decision may perform one -bounded cache lookup or interactive probe; timeout or failure preserves Plex's -original response. +All clients use the same MediaInfo projection rules. Part and universal-start +redirects never wait for MediaInfo. An L1 hit does not synchronously read SQLite +for the request; access renewal may touch SQLite asynchronously. An L1 miss may +use the persistent record or one bounded interactive probe, and timeout or +failure preserves the existing Direct Play path. Metadata browsing has a +separate single synchronous cold-probe slot. After the admitted synchronous wait +releases its slot, a fixed five-second window begins. Misses rejected inside that +window do not renew it, and an admitted probe can finish in the background and +write L1 and SQLite after the request waiter leaves. + +`PLAYBACK_VETO_ENABLED=false` leaves a nil hook in the decision handler. The +experimental hook is disabled by default. When enabled, it only consumes the +fresh MediaInfo record already obtained by that request and checks the built-in +Apple TV Plex Dolby Vision Profile 5 combination. It has no cache, persistence, +Part interception, remote I/O, or positive allow semantics. Missing or +conflicting facts preserve Plex's Direct Play response. ## Client compatibility boundary @@ -174,6 +201,15 @@ or modifying the Plex Web client would violate or expand the playback-plane boundary; none is enabled by this project. Local media remains transparently proxied and retains Plex's normal Web compatibility. +## Playback veto contract + +The veto is a best-effort compatibility override, not an authorization or +capability engine. It runs only after Plex has accepted the exact Part for +Direct Play and MediaInfo enrichment already has a complete, fresh record. +Returning a veto replaces that decision response and prevents creation of its +short-lived start grant. It does not block a direct Part request and therefore +must not be treated as a security boundary. + ## Analysis-plane boundary MediaVault's documented ffprobe assistance and upload-time head/tail precache @@ -193,11 +229,19 @@ is accepted, the body is not consumed, and every error preserves the other probe fields. This fallback remains inside the MediaInfo worker and never enters the synchronous 302 path. A stable MediaVault or MoviePilot provider can replace remote probing without -changing playback. Plex persistence remains a feasibility question across an -official API, another supported PMS interface, and an isolated database helper. -Intro and credits work starts with an isolated Plex marker API probe. See +changing playback. Plex DB production writing is a next-stage evaluation only, +covering an official API, another supported PMS interface, and an isolated +database helper with explicit coverage, backup, and rollback tests. This release +does not select or enable a Plex DB write path. Intro and credits work starts +with an isolated Plex marker API probe. See [media-lifecycle-architecture.md](media-lifecycle-architecture.md). +Performance acceptance and reproducible microbenchmarks are defined in +[performance-matrix.md](performance-matrix.md). Local proxying and the 302 path +remain higher priority than analysis coverage; a feature that moves MediaInfo, +SQLite, or CDN I/O into Part or universal-start handling is an architecture +regression. + ## Cache lifecycle The Part cache is in memory and has a configurable TTL. Metadata responses diff --git a/docs/performance-matrix.md b/docs/performance-matrix.md new file mode 100644 index 0000000..038482b --- /dev/null +++ b/docs/performance-matrix.md @@ -0,0 +1,79 @@ +# 性能矩阵 + +性能是 `plex-gateway` 的首要验收条件。功能覆盖不能以增加本地媒体延迟、把分析 I/O +放入 302 路径，或扩大 Plex、MediaVault 和 CDN 请求风暴为代价。 + +本文区分三类数据：本机微基准、脱敏真实链路样本和 NAS 候选版本端到端验收。微基准 +只能衡量代码增量，不能替代客户端首帧、Plex 响应或 CDN 网络结果。 + +## 不可破坏的边界 + +- 本地 Part 在识别为非云媒体后立即交给 Plex，不读取 MediaInfo、不访问 SQLite、不执行 + veto，也不请求 MediaVault 或 CDN。 +- 普通 STRM Part 和 universal start 不等待 MediaInfo，也不执行 veto。 +- L1 命中直接返回，不因本次请求同步读取 SQLite。访问续期可能异步 touch SQLite， + 因此持久化 I/O 不得阻塞热路径。 +- 一个 universal decision 最多等待一次 `MEDIAINFO_COLD_WAIT`。veto 只复用响应增强已经 + 取得的新鲜记录，不增加缓存读取或远程探测。 +- 同一 MediaInfo key 的并发请求必须共享 singleflight。并发数不能突破配置的 worker + 上限；当前项只能优先排队，不能抢占正在运行的后台 probe。 +- 302 响应只包含控制流。媒体字节始终由客户端直接从 CDN 读取。 + +## 当前基准 + +基准日期为 2026-08-29。代码运行于 `darwin/arm64`、Apple M3 Max、Go 1.26.5。 +每项运行三到五轮；下表给出观测范围。工作站数据只用于发现代码回退，换架构或 Go +版本后必须重新采集。 + +| 操作 | 结果 | 分配 | 说明 | +| --- | ---: | ---: | --- | +| MediaInfo L1 cache `Get` | 469.0-476.0 ns/op | 960 B, 7 allocs | 纯缓存结构基准。 | +| MediaInfo service `GetMemory` | 1.356-1.423 us/op | 2880 B, 21 allocs | 包含记录防御性复制、兼容性校验和访问续期判断。 | +| STRM target fingerprint | 382.0-387.4 ns/op | 448 B, 6 allocs | MediaVault redirect URL 规范化和 SHA-256。 | +| SQLite 单记录 `Get` | 14.60-15.37 us/op | 4016 B, 74 allocs | 独立本机 SQLite 读取；生产启动会先恢复到 L1。 | +| 本机 HTTP 合成媒体 ffprobe | 22.48-33.36 ms/op | 约 119-125 KiB, 232-234 allocs | 只验证 worker 代码，不代表 NAS/CDN。 | + +真实 MediaVault/CDN 抽样为 5/5 成功：MediaVault redirect p50 141 ms、范围 +109-533 ms；受限 CDN ffprobe p50 601 ms、范围 403-674 ms；合计 p50 761 ms、范围 +543-955 ms。每次读取约 1.22 MiB 并发生 2 次 seek。样本量较小，只证明当前探测通常能在 +默认 5 秒冷等待内完成，不能作为 CDN SLA。 + +当前 Metadata Guard 默认值为单项全局 8、单客户端 4、批量 3。单项全局 16、单客户端 4、 +批量 3 只有在 Plex MediaInfo 写库达到较高覆盖率并完成重新压测后才作为候选，不属于当前 +版本的性能承诺。 + +## 端到端验收矩阵 + +| 场景 | 请求期允许的 I/O | 必须采集 | 验收条件 | +| --- | --- | --- | --- | +| 本地媒体透明代理 | Plex | 直连 Plex 与经 Gateway 的 p50/p95/p99、错误率、上游请求数 | 0 功能错误；Gateway 增量 p95 <= 20 ms、p99 <= 50 ms，且相对增加 <= 10%。 | +| STRM Part 302 | Plex Part 授权、MediaVault redirect | 302 p50/p95/p99、Plex/MV 次数、响应字节 | 100% 健康样本返回 302；p95 < 2 s、p99 < 5 s；Gateway 不传输媒体字节。 | +| universal start | Plex Part 授权、MediaVault redirect | grant 命中、上游次数、302 延迟 | 不读取 metadata、STRM 或 MediaInfo；无 grant 时透明回退。 | +| Decision，Infuse | Plex metadata 和 decision | decision 延迟、veto 结果 | veto 始终放弃判断；不增加缓存、MediaVault 或 CDN 请求。 | +| Decision，L1 热缓存 | Plex metadata 和 decision，L1 | p50/p95/p99、SQLite/MV/CDN 次数 | 0 SQLite、0 MV、0 CDN；p95 <= 1 s、p99 <= 2 s。 | +| SQLite 启动恢复 | 启动期 SQLite，播放期 L1 | 恢复耗时、首个 decision 延迟 | 首个请求 0 ffprobe、0 请求期 SQLite。 | +| Decision，真实冷探测 | Plex、MediaVault、CDN、SQLite 写入 | resolver/probe/总延迟、读取字节、seek、fail-open | 成功样本 p95 < 2 s、p99 <= 5 s；超过冷等待返回可用的 fail-open 响应。 | +| Apple TV DV5，veto 开启 | Plex metadata、decision 和既有 MediaInfo 流程 | reject 延迟、grant、额外 I/O | 新鲜完整记录命中时 reject 且不创建 grant；相对 veto 关闭增加 0 次缓存、MV、CDN 请求。 | +| 同一 Part 并发 2/4/8/16 | 一个共享 probe | 唯一 probe 数、延迟、队列、active | 每个 key 最多 1 次 probe，无重复 CDN 读取。 | +| 不同 Part 并发 2/4/8 | 受 worker 上限约束 | active、队列、CPU、内存、错误率 | 不突破 `MEDIAINFO_CONCURRENCY`，不拖慢本地或 302 路径。 | +| A/B/C 快速切换 | 当前项交互 probe、后台邻近预热 | 每项延迟、身份对应、替换/队列指标 | 新当前项优先排队；不能抢占正在运行的 probe；无跨项 MediaInfo、grant 或 redirect。 | + +表中的验收条件是候选版本的门槛，不代表本机微基准已经替代端到端验收。当前已有证据 +包括本机代码基准和 5/5 成功的 MediaVault/CDN 探测抽样；本地透明代理、302 热冷路径、 +决策热冷路径的 p95/p99、并发矩阵和快速切换矩阵，仍需使用候选版本在目标 NAS 上串行采集。 + +客户端首次画面必须单独测量。Gateway 302 时间、客户端跟随 Location、CDN 首字节和 +播放器首帧是不同阶段，不能用 302 延迟代替起播性能。 + +## 复现命令 + +```sh +go test -run '^$' \ + -bench 'Benchmark(MediaInfoL1Get|MediaInfoServiceGetMemory|FingerprintSTRMTarget|MediaInfoSQLiteGet|MediaInfoHTTPFFProbe)$' \ + -benchmem -count=5 ./internal/mediainfo +``` + +真实 NAS 验收必须使用候选镜像、固定样本和串行 A/B，记录版本、配置、样本数、分位数、 +错误率和聚合 I/O。公开结果不得包含真实域名、地址、媒体标题、PartID、ratingKey、Token、 +Cookie、签名 URL、完整 User-Agent 或原始 trace。候选代码、Go 版本、Plex/MediaVault +版本、配置或网络路径变化都会使旧结果失效。 diff --git a/docs/roadmap.md b/docs/roadmap.md index b24f67a..61c9632 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -76,8 +76,16 @@ metadata and Direct Play decision responses can be enriched from the fail-open L1 and SQLite cache. A cold decision may wait up to `MEDIAINFO_COLD_WAIT` for an interactive probe; missing, slow, or failed probes preserve the original Plex response. Part and universal-start redirects never wait for MediaInfo. All -clients use the same projection rules without product, device, or User-Agent -format-compatibility branches. +clients use the same projection rules. An L1 hit does not synchronously read +SQLite for the request, although access renewal may touch SQLite asynchronously. +One metadata browsing burst permits one synchronous cold probe. Each admitted +the admitted synchronous wait releases its slot, a fixed five-second window +begins. Cold misses rejected inside that window fail open without renewing it, +while an admitted probe may finish in the background and write L1 and SQLite +after the request waiter leaves. +A default-disabled experimental veto may reuse an already available fresh record +for the Apple TV Plex DV Profile 5 combination, but it performs no additional +probe and owns no Part state. MediaInfo probing and persistence remain isolated from redirect playback. Phase D prioritizes the Gateway fallback while preserving the broader lifecycle design: @@ -87,19 +95,32 @@ D prioritizes the Gateway fallback while preserving the broader lifecycle design nearby-item discovery; current-item prewarming remains token-independent; 3. use bounded remote `ffprobe` with a default `5s` cold metadata wait ceiling; 4. support exact current-Part analysis and a configurable nearby-item window - after a cloud redirect is ready, with immediate current-item submission and + after a cloud redirect is ready, with current-item priority in the queue and rate-limited background neighbors, without blocking decision, Part, - universal start, or 302; + universal start, or 302. Queue priority cannot preempt a probe already + running; defer season, show, and configured STRM-root batch tasks; -5. enrich authorized Plex metadata responses first, then evaluate an official - Plex API, other supported PMS interfaces, and an isolated database helper - without selecting a persistence path in advance. Parts without Plex Streams - may receive descriptive ffprobe-indexed Streams, while existing Stream sets - are never expanded and Plex IDs or playback-selection state are never - synthesized. Background `-Library` synchronization remains cache-only; +5. enrich authorized Plex metadata responses first. This release writes fallback + records only to Gateway SQLite. Production Plex DB writing is a next-stage + evaluation across an official Plex API, another supported PMS interface, and + an isolated database helper, with coverage, backup, and rollback tests before + any path is selected. Parts without Plex Streams may receive descriptive + ffprobe-indexed Streams, while existing Stream sets are never expanded and + Plex IDs or playback-selection state are never synthesized. Background + `-Library` synchronization remains cache-only, and native-client metadata + fan-out cannot schedule a full-library cold scan; 6. add Redis or replace SQLite with PostgreSQL only after measured multi-instance or capacity requirements exist. +The performance matrix in [performance-matrix.md](performance-matrix.md) is a +delivery gate for this work. Local media must remain transparent, Part and +universal-start routes must remain free of analysis I/O, an L1 hit must not +block on SQLite, and one decision may consume at most one configured cold-wait +budget. The current Guard defaults are global 8, per-client 4, and batch 3. +After production Plex MediaInfo writing provides high coverage and the load is +retested, global 16, per-client 4, and batch 3 may be evaluated. No new Guard +mode or configuration is planned for this release. + The complete MoviePilot, MediaVault, 115 share, ownership, seeding, cleanup, MediaInfo, and projection design is documented in [media-lifecycle-architecture.md](media-lifecycle-architecture.md). Gateway diff --git a/internal/config/config.go b/internal/config/config.go index c3dbef1..c9b74fc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -40,7 +40,7 @@ const ( defaultMediaInfoAgent = "Infuse-Library/8.5.1" defaultMediaInfoColdWait = 5 * time.Second defaultMediaInfoBodyLimit = 8 << 20 - defaultMediaInfoWaiters = defaultMetadataGlobal + defaultMediaInfoWaiters = 8 defaultPrewarmBefore = 2 defaultPrewarmAfter = 3 defaultPrewarmInterval = 5 * time.Second @@ -65,6 +65,7 @@ type Config struct { PartProbeTimeout time.Duration MetadataGuard MetadataGuardConfig MediaInfo MediaInfoConfig + PlaybackVeto bool LogLevel string TraceEnabled bool ReadHeaderTimeout time.Duration @@ -296,6 +297,10 @@ func Load() (Config, error) { if err != nil { return Config{}, err } + playbackVeto, err := envBool("PLAYBACK_VETO_ENABLED", false) + if err != nil { + return Config{}, err + } readHeaderTimeout, err := envDuration("READ_HEADER_TIMEOUT", defaultReadHeaderTimeout) if err != nil { @@ -354,6 +359,7 @@ func Load() (Config, error) { PrewarmBefore: mediaInfoPrewarmBefore, PrewarmAfter: mediaInfoPrewarmAfter, PrewarmInterval: mediaInfoPrewarmInterval, }, + PlaybackVeto: playbackVeto, LogLevel: logLevel, TraceEnabled: traceEnabled, ReadHeaderTimeout: readHeaderTimeout, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9908c7f..4b80f12 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -50,6 +50,9 @@ func TestLoadDefaults(t *testing.T) { if !got.MediaInfo.Enabled || got.MediaInfo.ProbeTimeout != 20*time.Second || got.MediaInfo.Concurrency != 1 || got.MediaInfo.UserAgent != "Infuse-Library/8.5.1" || got.MediaInfo.ColdWait != 5*time.Second || got.MediaInfo.ResponseMaxBytes != 8<<20 || got.MediaInfo.EnrichmentWaiters != 8 || got.MediaInfo.PrewarmBefore != 2 || got.MediaInfo.PrewarmAfter != 3 || got.MediaInfo.PrewarmInterval != 5*time.Second { t.Fatalf("unexpected MediaInfo defaults: %#v", got.MediaInfo) } + if got.PlaybackVeto { + t.Fatal("PlaybackVeto = true") + } if got.MediaVaultURL != nil || len(got.PathMappings) != 0 { t.Fatalf("cloud redirect unexpectedly enabled: %#v", got) } @@ -196,3 +199,16 @@ func TestLoadAllowsCurrentOnlyPrewarm(t *testing.T) { t.Fatalf("prewarm window = %d/%d", got.MediaInfo.PrewarmBefore, got.MediaInfo.PrewarmAfter) } } + +func TestLoadPlaybackVetoCanBeEnabled(t *testing.T) { + t.Setenv("PLEX_URL", "http://plex:32400") + t.Setenv("PLAYBACK_VETO_ENABLED", "true") + + got, err := Load() + if err != nil { + t.Fatal(err) + } + if !got.PlaybackVeto { + t.Fatal("PlaybackVeto = false") + } +} diff --git a/internal/gateway/decision.go b/internal/gateway/decision.go index a8b8bc1..dd99157 100644 --- a/internal/gateway/decision.go +++ b/internal/gateway/decision.go @@ -34,6 +34,7 @@ type decisionHandler struct { mediaInfo decisionMediaInfoService coldWait time.Duration grants *playback.GrantStore + veto playbackVeto logger *slog.Logger metrics *metrics.Metrics } @@ -77,6 +78,13 @@ func (h *decisionHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if err != nil { h.logger.Warn("decision_mediainfo_fail_open", "part", selection.Part.ID, "error_kind", errorKind(err)) } else if changed { + if h.veto != nil && record.Fresh(time.Now().UTC()) { + if reason, reject := h.veto(r, record.Media); reject { + h.logger.Info("playback_veto", "part", selection.Part.ID, "reason", reason) + writeIncompatibleDecision(w, r) + return + } + } if err := capture.replaceDecodedBody(enriched); err != nil { h.logger.Warn("decision_mediainfo_fail_open", "part", selection.Part.ID, "error_kind", errorKind(err)) } else if h.metrics != nil { @@ -92,6 +100,17 @@ func (h *decisionHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } +func writeIncompatibleDecision(w http.ResponseWriter, r *http.Request) { + contentType, body := plexmeta.IncompatibleDecision(r.Header.Get("Accept")) + resetDecisionBodyHeaders(w.Header(), len(body)) + w.Header().Set("Content-Type", contentType) + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusOK) + if r.Method != http.MethodHead { + _, _ = w.Write(body) + } +} + // mediaInfoForDecision reuses an exact cached STRM identity and admits at most // one bounded interactive probe on a cold playback. Failure leaves Plex's // original decision response usable and the shared worker may still warm a diff --git a/internal/gateway/decision_capture.go b/internal/gateway/decision_capture.go index 4d101bd..c9995d6 100644 --- a/internal/gateway/decision_capture.go +++ b/internal/gateway/decision_capture.go @@ -137,26 +137,41 @@ func (w *decisionResponseCapture) body() []byte { } // replaceDecodedBody swaps one complete bounded decision response before it is -// exposed to the client. Wire encoding is preserved while stale entity -// validators are removed because the response representation has changed. +// exposed to the client. The replacement is emitted without upstream content +// coding because the response representation has changed. func (w *decisionResponseCapture) replaceDecodedBody(body []byte) error { if !w.successful() || w.committed || w.passthrough { return errors.New("decision response is not replaceable") } - wireBody, err := encodeMetadataBody(body, w.Header().Get("Content-Encoding")) - if err != nil { - return err - } - if int64(len(wireBody)) > w.limit { + if int64(len(body)) > w.limit { return errors.New("enriched decision response exceeds the limit") } w.captured.Reset() - _, _ = w.captured.Write(wireBody) - for _, name := range []string{"ETag", "Last-Modified", "Content-MD5", "Digest"} { - w.Header().Del(name) - } - w.Header().Set("Content-Length", strconv.Itoa(len(wireBody))) + _, _ = w.captured.Write(body) + resetDecisionBodyHeaders(w.Header(), len(body)) return nil } +// resetDecisionBodyHeaders removes representation metadata that describes the +// upstream body before installing a replacement. Validators and transfer +// framing are no longer valid once the decision document has changed. +func resetDecisionBodyHeaders(header http.Header, bodyLength int) { + for _, name := range []string{ + "Accept-Ranges", + "Content-Encoding", + "Content-Length", + "Content-MD5", + "Content-Range", + "Digest", + "ETag", + "Last-Modified", + "Trailer", + "Transfer-Encoding", + "Vary", + } { + header.Del(name) + } + header.Set("Content-Length", strconv.Itoa(bodyLength)) +} + var _ interface{ Unwrap() http.ResponseWriter } = (*decisionResponseCapture)(nil) diff --git a/internal/gateway/decision_capture_test.go b/internal/gateway/decision_capture_test.go index fb18c07..a0c04fe 100644 --- a/internal/gateway/decision_capture_test.go +++ b/internal/gateway/decision_capture_test.go @@ -3,7 +3,6 @@ package gateway import ( "bytes" "compress/gzip" - "io" "net/http" "net/http/httptest" "net/url" @@ -11,12 +10,18 @@ import ( "testing" ) -func TestDecisionCaptureReplacesGzipBodyAndEntityHeaders(t *testing.T) { +func TestDecisionCaptureReplacesGzipBodyAndRepresentationHeaders(t *testing.T) { destination := httptest.NewRecorder() capture := newDecisionResponseCapture(destination, 1<<20) capture.Header().Set("Content-Encoding", "gzip") capture.Header().Set("Content-Length", "1") - capture.Header().Set("ETag", `"stale"`) + for _, name := range []string{ + "Accept-Ranges", "Content-MD5", "Content-Range", "Digest", "ETag", + "Last-Modified", "Trailer", "Transfer-Encoding", "Vary", + } { + capture.Header().Set(name, "stale") + } + capture.Header().Set("Content-Type", "application/xml") capture.WriteHeader(http.StatusOK) var original bytes.Buffer @@ -33,19 +38,19 @@ func TestDecisionCaptureReplacesGzipBodyAndEntityHeaders(t *testing.T) { if err := capture.commit(); err != nil { t.Fatal(err) } - reader, err := gzip.NewReader(bytes.NewReader(destination.Body.Bytes())) - if err != nil { - t.Fatal(err) + if destination.Body.String() != "enriched" || destination.Header().Get("Content-Encoding") != "" || destination.Header().Get("Content-Length") != strconv.Itoa(len("enriched")) { + t.Fatalf("headers=%#v body=%q", destination.Header(), destination.Body.String()) } - decoded, err := io.ReadAll(reader) - if err != nil { - t.Fatal(err) - } - if err := reader.Close(); err != nil { - t.Fatal(err) + if destination.Header().Get("Content-Type") != "application/xml" { + t.Fatalf("Content-Type = %q", destination.Header().Get("Content-Type")) } - if string(decoded) != "enriched" || destination.Header().Get("ETag") != "" || destination.Header().Get("Content-Length") != strconv.Itoa(destination.Body.Len()) { - t.Fatalf("headers=%#v body=%q", destination.Header(), decoded) + for _, name := range []string{ + "Accept-Ranges", "Content-MD5", "Content-Range", "Digest", "ETag", + "Last-Modified", "Trailer", "Transfer-Encoding", "Vary", + } { + if got := destination.Header().Get(name); got != "" { + t.Errorf("%s = %q, want empty", name, got) + } } } diff --git a/internal/gateway/decision_test.go b/internal/gateway/decision_test.go index 6f3cb57..5eb5c23 100644 --- a/internal/gateway/decision_test.go +++ b/internal/gateway/decision_test.go @@ -1,6 +1,8 @@ package gateway import ( + "bytes" + "compress/gzip" "context" "io" "log/slog" @@ -9,6 +11,7 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -18,6 +21,7 @@ import ( "github.com/InfinityPacer/plex-gateway/internal/partcache" "github.com/InfinityPacer/plex-gateway/internal/pathmap" "github.com/InfinityPacer/plex-gateway/internal/playback" + "github.com/InfinityPacer/plex-gateway/internal/plexmeta" "github.com/InfinityPacer/plex-gateway/internal/resolver" ) @@ -26,10 +30,12 @@ type decisionMediaInfoStub struct { memoryHit bool waitForCancel bool ensureCalls int + memoryCalls int request mediainfo.Request } func (stub *decisionMediaInfoStub) GetMemory(mediainfo.Key) (mediainfo.Record, bool) { + stub.memoryCalls++ return stub.record, stub.memoryHit } @@ -60,7 +66,7 @@ func TestCloudDecisionProjectsMediaInfoWithoutClientSpecificPolicy(t *testing.T) media := completeProjectionMedia() media.Size = 987654321 mediaInfo := &decisionMediaInfoStub{record: mediainfo.Record{Media: media}} - handler := newDecisionProjectionHandler(t, mediaInfo, 100*time.Millisecond) + handler := newDecisionProjectionHandler(t, mediaInfo, 100*time.Millisecond, nil) request := decisionProjectionRequest() request.Header.Set("User-Agent", client.userAgent) request.Header.Set("X-Plex-Product", client.product) @@ -92,7 +98,7 @@ func TestCloudDecisionProjectsMediaInfoWithoutClientSpecificPolicy(t *testing.T) func TestCloudDecisionMediaInfoTimeoutFailsOpen(t *testing.T) { mediaInfo := &decisionMediaInfoStub{waitForCancel: true} - handler := newDecisionProjectionHandler(t, mediaInfo, 10*time.Millisecond) + handler := newDecisionProjectionHandler(t, mediaInfo, 10*time.Millisecond, nil) response := httptest.NewRecorder() started := time.Now() @@ -109,7 +115,243 @@ func TestCloudDecisionMediaInfoTimeoutFailsOpen(t *testing.T) { } } -func newDecisionProjectionHandler(t *testing.T, mediaInfo decisionMediaInfoService, coldWait time.Duration) http.Handler { +func TestWriteIncompatibleDecisionClearsRepresentationHeadersAndHonorsHEAD(t *testing.T) { + for _, test := range []struct { + name string + method string + accept string + }{ + {name: "XML GET", method: http.MethodGet, accept: "application/xml"}, + {name: "JSON GET", method: http.MethodGet, accept: "application/json"}, + {name: "XML HEAD", method: http.MethodHead, accept: "application/xml"}, + {name: "JSON HEAD", method: http.MethodHead, accept: "application/json"}, + } { + t.Run(test.name, func(t *testing.T) { + response := httptest.NewRecorder() + for _, name := range []string{ + "Accept-Ranges", "Content-Encoding", "Content-MD5", "Content-Range", "Digest", + "ETag", "Last-Modified", "Trailer", "Transfer-Encoding", "Vary", + } { + response.Header().Set(name, "stale") + } + request := httptest.NewRequest(test.method, "/video/:/transcode/universal/decision", nil) + request.Header.Set("Accept", test.accept) + + writeIncompatibleDecision(response, request) + + contentType, body := plexmeta.IncompatibleDecision(test.accept) + if response.Code != http.StatusOK || response.Header().Get("Content-Type") != contentType || + response.Header().Get("Content-Length") != strconv.Itoa(len(body)) || response.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("status=%d headers=%#v", response.Code, response.Header()) + } + wantBody := string(body) + if test.method == http.MethodHead { + wantBody = "" + } + if response.Body.String() != wantBody { + t.Fatalf("body=%q want=%q", response.Body.String(), wantBody) + } + for _, name := range []string{ + "Accept-Ranges", "Content-Encoding", "Content-MD5", "Content-Range", "Digest", + "ETag", "Last-Modified", "Trailer", "Transfer-Encoding", "Vary", + } { + if got := response.Header().Get(name); got != "" { + t.Errorf("%s = %q, want empty", name, got) + } + } + }) + } +} + +func TestCloudDecisionJSONProjectionClearsRepresentationHeaders(t *testing.T) { + media := completeProjectionMedia() + mediaInfo := &decisionMediaInfoStub{record: freshDecisionRecord(media), memoryHit: true} + handler := newDecisionProjectionHandler(t, mediaInfo, 100*time.Millisecond, nil).(*decisionHandler) + handler.plex = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + for _, name := range []string{ + "Accept-Ranges", "Content-MD5", "Content-Range", "Digest", "ETag", "Last-Modified", + "Trailer", "Transfer-Encoding", "Vary", + } { + w.Header().Set(name, "stale") + } + w.Header().Set("Content-Length", "1") + _, _ = w.Write([]byte(`{"MediaContainer":{"Metadata":[{"Media":[{"decision":"directplay","Part":[{"id":9,"key":"/library/parts/9/1/file","size":301,"decision":"directplay"}]}]}]}}`)) + }) + response := httptest.NewRecorder() + handler.ServeHTTP(response, decisionProjectionRequest()) + + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"container":"mkv"`) { + t.Fatalf("status=%d headers=%#v body=%s", response.Code, response.Header(), response.Body.String()) + } + if response.Header().Get("Content-Length") != strconv.Itoa(response.Body.Len()) || response.Header().Get("Content-Type") != "application/json" { + t.Fatalf("body headers=%#v body_len=%d", response.Header(), response.Body.Len()) + } + for _, name := range []string{ + "Accept-Ranges", "Content-Encoding", "Content-MD5", "Content-Range", "Digest", "ETag", + "Last-Modified", "Trailer", "Transfer-Encoding", "Vary", + } { + if got := response.Header().Get(name); got != "" { + t.Errorf("%s = %q, want empty", name, got) + } + } +} + +func TestCloudDecisionVetoReplacesGzipResponseWithIdentityBody(t *testing.T) { + media := completeProjectionMedia() + media.Streams[0].HDRFormat = "dolby_vision" + media.Streams[0].DolbyVision = &mediainfo.DolbyVision{Profile: 5, BLCompatID: 0} + mediaInfo := &decisionMediaInfoStub{record: freshDecisionRecord(media), memoryHit: true} + handler := newDecisionProjectionHandler(t, mediaInfo, 100*time.Millisecond, newPlaybackVeto(true)).(*decisionHandler) + handler.plex = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/xml") + w.Header().Set("Content-Encoding", "gzip") + for _, name := range []string{ + "Accept-Ranges", "Content-MD5", "Content-Range", "Digest", "ETag", "Last-Modified", + "Trailer", "Transfer-Encoding", "Vary", + } { + w.Header().Set(name, "stale") + } + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + _, _ = writer.Write([]byte(``)) + if err := writer.Close(); err != nil { + t.Fatal(err) + } + w.Header().Set("Content-Length", strconv.Itoa(compressed.Len())) + _, _ = w.Write(compressed.Bytes()) + }) + request := decisionProjectionRequest() + request.Header.Set("Accept", "application/xml") + request.Header.Set("X-Plex-Product", "Plex for Apple TV") + request.Header.Set("X-Plex-Platform", "tvOS") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `transcodeDecisionCode="4005"`) || response.Header().Get("Content-Encoding") != "" { + t.Fatalf("status=%d headers=%#v body=%s", response.Code, response.Header(), response.Body.String()) + } + if response.Header().Get("Content-Length") != strconv.Itoa(response.Body.Len()) { + t.Fatalf("Content-Length=%q body_len=%d", response.Header().Get("Content-Length"), response.Body.Len()) + } + for _, name := range []string{ + "Accept-Ranges", "Content-MD5", "Content-Range", "Digest", "ETag", "Last-Modified", + "Trailer", "Transfer-Encoding", "Vary", + } { + if got := response.Header().Get(name); got != "" { + t.Errorf("%s = %q, want empty", name, got) + } + } +} + +func TestCloudDecisionOptionalVetoUsesExistingFreshMediaInfo(t *testing.T) { + media := completeProjectionMedia() + media.Streams[0].HDRFormat = "dolby_vision" + media.Streams[0].DolbyVision = &mediainfo.DolbyVision{Profile: 5, BLCompatID: 0} + mediaInfo := &decisionMediaInfoStub{record: freshDecisionRecord(media), memoryHit: true} + handler := newDecisionProjectionHandler(t, mediaInfo, 100*time.Millisecond, newPlaybackVeto(true)) + request := decisionProjectionRequest() + request.Header.Set("X-Plex-Product", "Plex for Apple TV") + request.Header.Set("X-Plex-Platform", "tvOS") + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `transcodeDecisionCode="4005"`) || strings.Contains(response.Body.String(), `decision="directplay"`) { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } + if mediaInfo.memoryCalls != 1 || mediaInfo.ensureCalls != 0 { + t.Fatalf("GetMemory calls=%d Ensure calls=%d", mediaInfo.memoryCalls, mediaInfo.ensureCalls) + } +} + +func TestCloudDecisionDisabledVetoPreservesDV5WithSameMediaInfoIO(t *testing.T) { + media := completeProjectionMedia() + media.Streams[0].HDRFormat = "dolby_vision" + media.Streams[0].DolbyVision = &mediainfo.DolbyVision{Profile: 5, BLCompatID: 0} + mediaInfo := &decisionMediaInfoStub{record: freshDecisionRecord(media), memoryHit: true} + handler := newDecisionProjectionHandler(t, mediaInfo, 100*time.Millisecond, nil) + request := decisionProjectionRequest() + request.Header.Set("X-Plex-Product", "Plex for Apple TV") + request.Header.Set("X-Plex-Platform", "tvOS") + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `decision="directplay"`) { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } + if mediaInfo.memoryCalls != 1 || mediaInfo.ensureCalls != 0 { + t.Fatalf("GetMemory calls=%d Ensure calls=%d", mediaInfo.memoryCalls, mediaInfo.ensureCalls) + } +} + +func TestCloudDecisionOptionalVetoAbstainsOnStaleMediaInfo(t *testing.T) { + media := completeProjectionMedia() + media.Streams[0].HDRFormat = "dolby_vision" + media.Streams[0].DolbyVision = &mediainfo.DolbyVision{Profile: 5, BLCompatID: 0} + now := time.Now().UTC() + record := freshDecisionRecord(media) + record.ProbedAt = now.Add(-2 * time.Hour) + record.ExpiresAt = now.Add(-time.Hour) + mediaInfo := &decisionMediaInfoStub{record: record, memoryHit: true} + handler := newDecisionProjectionHandler(t, mediaInfo, 100*time.Millisecond, newPlaybackVeto(true)) + request := decisionProjectionRequest() + request.Header.Set("X-Plex-Product", "Plex for Apple TV") + request.Header.Set("X-Plex-Platform", "tvOS") + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `decision="directplay"`) { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } +} + +func TestCloudDecisionVetoDoesNotOverrideCompletePlexDecision(t *testing.T) { + media := completeProjectionMedia() + media.Streams[0].HDRFormat = "dolby_vision" + media.Streams[0].DolbyVision = &mediainfo.DolbyVision{Profile: 5, BLCompatID: 0} + mediaInfo := &decisionMediaInfoStub{record: freshDecisionRecord(media), memoryHit: true} + handler := newDecisionProjectionHandler(t, mediaInfo, 100*time.Millisecond, newPlaybackVeto(true)).(*decisionHandler) + base := []byte(``) + complete, changed, err := plexmeta.EnrichDecision(base, "application/xml", plexmeta.Part{ID: "9", Key: "/library/parts/9/1/file"}, media) + if err != nil || !changed { + t.Fatalf("prepare complete decision changed=%v err=%v", changed, err) + } + handler.plex = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write(complete) + }) + request := decisionProjectionRequest() + request.Header.Set("X-Plex-Product", "Plex for Apple TV") + request.Header.Set("X-Plex-Platform", "tvOS") + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `decision="directplay"`) || strings.Contains(response.Body.String(), `transcodeDecisionCode="4005"`) { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } +} + +func freshDecisionRecord(media mediainfo.Media) mediainfo.Record { + now := time.Now().UTC() + return mediainfo.Record{ + Key: mediainfo.Key{PlexServerID: "plex", PartID: "9", STRMFingerprint: strings.Repeat("a", 64)}, + Provider: mediainfo.ProviderMediaVaultFFProbe, ProviderRevision: mediainfo.ProviderRevisionFFProbeJSONV3, + SchemaVersion: mediainfo.SchemaVersion, Media: media, + ProbedAt: now.Add(-time.Minute), ExpiresAt: now.Add(time.Hour), + LastAccessedAt: now, RetainUntil: now.Add(24 * time.Hour), + } +} + +func newDecisionProjectionHandler( + t *testing.T, + mediaInfo decisionMediaInfoService, + coldWait time.Duration, + veto playbackVeto, +) http.Handler { t.Helper() localRoot := t.TempDir() if err := os.WriteFile(filepath.Join(localRoot, "episode.strm"), []byte("http://mediavault.invalid/redirect/pick/episode.mkv\n"), 0o600); err != nil { @@ -158,7 +400,7 @@ func newDecisionProjectionHandler(t *testing.T, mediaInfo decisionMediaInfoServi client: &http.Client{Timeout: time.Second}, maxBytes: 1 << 20, }, - service: cloudPlayback, mediaInfo: mediaInfo, coldWait: coldWait, + service: cloudPlayback, mediaInfo: mediaInfo, coldWait: coldWait, veto: veto, grants: playback.NewGrantStore(time.Minute, 16), logger: logger, metrics: metrics.New(), } } diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 7f83e09..f7899d4 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -47,6 +47,7 @@ type Options struct { MediaInfoEnrichmentConcurrency int MediaInfoPrewarmer cloudRedirectPrewarmer MediaInfoPrewarmStatus func() prewarm.Status + PlaybackVeto bool } type cloudRedirectPrewarmer interface { @@ -148,6 +149,7 @@ func New(options Options) http.Handler { logger: logger, metrics: registry, grants: playback.NewGrantStore(defaultDecisionGrantTTL, defaultDecisionGrantLimit), + veto: newPlaybackVeto(options.PlaybackVeto), probe: &decisionMetadataProbe{ upstream: options.Upstream, client: &http.Client{ diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index e3d6391..71aac57 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -281,7 +281,7 @@ func TestAuthorizedPlexSTRMRedirectEnablesCloudPartRedirect(t *testing.T) { } } -func TestCloudPartRedirectDoesNotApplyClientSpecificPolicy(t *testing.T) { +func TestCloudPartRedirectDoesNotApplyPlaybackVeto(t *testing.T) { localRoot := t.TempDir() if err := os.WriteFile(filepath.Join(localRoot, "Movie.strm"), []byte("http://public.invalid/redirect/pickcode\n"), 0o600); err != nil { t.Fatal(err) @@ -298,9 +298,9 @@ func TestCloudPartRedirectDoesNotApplyClientSpecificPolicy(t *testing.T) { })) defer plex.Close() - handler, _, cache := newCloudHandler(t, plex.URL, mediaVault.URL, []pathmap.Mapping{{ + handler, _, cache := newCloudHandlerWithVeto(t, plex.URL, mediaVault.URL, []pathmap.Mapping{{ PlexPrefix: "/media/cloud", LocalPrefix: localRoot, - }}) + }}, true) cache.Put(partcache.PartInfo{PartID: "123", PlexFilePath: "/media/cloud/Movie.strm"}) clients := []struct { product string @@ -958,6 +958,10 @@ func TestMetricsEndpointIsLocalAndStable(t *testing.T) { } func newCloudHandler(t *testing.T, plexURL, mediaVaultURL string, mappings []pathmap.Mapping) (http.Handler, *metrics.Metrics, *partcache.Cache) { + return newCloudHandlerWithVeto(t, plexURL, mediaVaultURL, mappings, false) +} + +func newCloudHandlerWithVeto(t *testing.T, plexURL, mediaVaultURL string, mappings []pathmap.Mapping, playbackVeto bool) (http.Handler, *metrics.Metrics, *partcache.Cache) { t.Helper() upstream, err := url.Parse(plexURL) if err != nil { @@ -983,6 +987,7 @@ func newCloudHandler(t *testing.T, plexURL, mediaVaultURL string, mappings []pat Resolver: strmResolver, Metrics: registry, CloudExtensions: []string{".strm"}, + PlaybackVeto: playbackVeto, }), registry, cache } diff --git a/internal/gateway/metadata_enrichment.go b/internal/gateway/metadata_enrichment.go index 4443245..270af12 100644 --- a/internal/gateway/metadata_enrichment.go +++ b/internal/gateway/metadata_enrichment.go @@ -10,6 +10,7 @@ import ( "path" "strconv" "strings" + "sync" "time" "github.com/InfinityPacer/plex-gateway/internal/mediainfo" @@ -22,6 +23,7 @@ import ( const ( defaultMediaInfoColdWait = 5 * time.Second defaultMediaInfoCacheLookupWait = 100 * time.Millisecond + defaultMetadataColdProbeQuiet = 5 * time.Second defaultMediaInfoResponseLimit = 8 << 20 defaultMediaInfoEnrichmentSlots = 8 ) @@ -55,6 +57,12 @@ type metadataEnrichmentHandler struct { responseLimit int64 waiters chan struct{} metrics *metrics.Metrics + + coldProbeMu sync.Mutex + coldProbeActive bool + coldProbeBlockedUntil time.Time + coldProbeQuiet time.Duration + now func() time.Time } func newMetadataEnrichmentHandler(options metadataEnrichmentOptions, next http.Handler) http.Handler { @@ -80,7 +88,8 @@ func newMetadataEnrichmentHandler(options metadataEnrichmentOptions, next http.H return &metadataEnrichmentHandler{ next: next, service: options.Service, mapper: options.Mapper, resolver: options.Resolver, cloudExtensions: normalizedExtensionSet(options.CloudExtensions), coldWait: coldWait, - responseLimit: responseLimit, waiters: make(chan struct{}, concurrency), metrics: registry, + responseLimit: responseLimit, waiters: make(chan struct{}, concurrency), + coldProbeQuiet: defaultMetadataColdProbeQuiet, now: time.Now, metrics: registry, } } @@ -143,17 +152,19 @@ func (handler *metadataEnrichmentHandler) ServeHTTP(w http.ResponseWriter, reque return } key := mediainfo.Key{PartID: target.Part.ID, STRMFingerprint: fingerprint} - var record mediainfo.Record - if mediaInfoCacheOnlyRequest(request) { - var found bool - lookupContext, cancel := context.WithTimeout(request.Context(), defaultMediaInfoCacheLookupWait) - record, found = handler.service.GetContext(lookupContext, key) - cancel() - if !found { + lookupContext, cancel := context.WithTimeout(request.Context(), defaultMediaInfoCacheLookupWait) + record, found := handler.service.GetContext(lookupContext, key) + cancel() + if !found { + cacheOnly := mediaInfoCacheOnlyRequest(request) + if cacheOnly || !handler.acquireColdProbe() { + if !cacheOnly { + handler.metrics.IncMediaInfoWaitRejected() + } handler.replay(capture, rawBody) return } - } else { + defer handler.releaseColdProbe() waitContext, cancel := context.WithTimeout(request.Context(), handler.coldWait) record, err = handler.service.Ensure(waitContext, mediainfo.Request{ Key: key, RatingKey: ratingKey, Target: strmTarget, @@ -187,6 +198,31 @@ func (handler *metadataEnrichmentHandler) ServeHTTP(w http.ResponseWriter, reque handler.metrics.IncMediaInfoEnriched() } +// A completed probe opens a fixed quiet interval. Rejected requests never move +// its end, so continuous metadata traffic cannot starve the next probe. +func (handler *metadataEnrichmentHandler) acquireColdProbe() bool { + handler.coldProbeMu.Lock() + defer handler.coldProbeMu.Unlock() + + now := handler.now() + if handler.coldProbeActive || now.Before(handler.coldProbeBlockedUntil) { + return false + } + handler.coldProbeActive = true + return true +} + +func (handler *metadataEnrichmentHandler) releaseColdProbe() { + handler.coldProbeMu.Lock() + defer handler.coldProbeMu.Unlock() + if !handler.coldProbeActive { + return + } + now := handler.now() + handler.coldProbeActive = false + handler.coldProbeBlockedUntil = now.Add(handler.coldProbeQuiet) +} + // serveCachedCollection enriches already-probed items without admitting remote // analysis. Browsing a season therefore cannot turn one response into a burst // of MediaVault or CDN requests. diff --git a/internal/gateway/metadata_enrichment_test.go b/internal/gateway/metadata_enrichment_test.go index aa78a24..ccfcfb1 100644 --- a/internal/gateway/metadata_enrichment_test.go +++ b/internal/gateway/metadata_enrichment_test.go @@ -362,6 +362,306 @@ func TestMetadataEnrichmentPoolSaturationSkipsWaiting(t *testing.T) { } } +func TestMetadataEnrichmentColdProbeSaturationFailsOpen(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + raw := fixture.xmlBody() + started := make(chan struct{}) + release := make(chan struct{}) + ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { + close(started) + <-release + return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil + }} + registry := metrics.New() + handler := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ + Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, + CloudExtensions: []string{".strm"}, ColdWait: time.Second, + ResponseLimit: 1 << 20, Concurrency: 2, Metrics: registry, + }, metadataBodyHandler(raw, "application/xml")) + + firstDone := make(chan *httptest.ResponseRecorder, 1) + go func() { + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedMetadataRequest(http.MethodGet)) + firstDone <- response + }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("first cold probe did not start") + } + + second := httptest.NewRecorder() + handler.ServeHTTP(second, authenticatedMetadataRequest(http.MethodGet)) + if !bytes.Equal(second.Body.Bytes(), raw) || ensurer.calls.Load() != 1 { + t.Fatalf("second response=%s Ensure calls=%d", second.Body.Bytes(), ensurer.calls.Load()) + } + close(release) + select { + case first := <-firstDone: + if bytes.Equal(first.Body.Bytes(), raw) { + t.Fatal("first response was not enriched") + } + case <-time.After(time.Second): + t.Fatal("first cold probe did not finish") + } + if snapshot := registry.Snapshot(); snapshot.MediaInfoWaitRejectedTotal != 1 || snapshot.MediaInfoWaitActive != 0 { + t.Fatalf("metrics = %#v", snapshot) + } +} + +func TestMetadataEnrichmentColdProbeQuietWindowSuppressesSequentialMiss(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + raw := fixture.xmlBody() + ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { + return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil + }} + handler := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ + Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, + CloudExtensions: []string{".strm"}, ColdWait: time.Second, + ResponseLimit: 1 << 20, Concurrency: 2, Metrics: metrics.New(), + }, metadataBodyHandler(raw, "application/xml")) + + first := httptest.NewRecorder() + handler.ServeHTTP(first, authenticatedMetadataRequest(http.MethodGet)) + if bytes.Equal(first.Body.Bytes(), raw) { + t.Fatal("first cold miss was not enriched") + } + + second := httptest.NewRecorder() + handler.ServeHTTP(second, authenticatedMetadataRequest(http.MethodGet)) + if !bytes.Equal(second.Body.Bytes(), raw) || ensurer.calls.Load() != 1 { + t.Fatalf("second response=%s Ensure calls=%d", second.Body.Bytes(), ensurer.calls.Load()) + } +} + +func TestMetadataEnrichmentColdProbeQuietWindowDoesNotSlideUnderContinuousTraffic(t *testing.T) { + start := time.Date(2026, time.August, 29, 12, 0, 0, 0, time.UTC) + now := start + handler := &metadataEnrichmentHandler{ + coldProbeQuiet: 5 * time.Second, + now: func() time.Time { return now }, + } + if !handler.acquireColdProbe() { + t.Fatal("first cold probe was rejected") + } + handler.releaseColdProbe() + if handler.acquireColdProbe() { + t.Fatal("probe was admitted immediately after release") + } + for second := 1; second < 5; second++ { + now = start.Add(time.Duration(second) * time.Second) + if handler.acquireColdProbe() { + t.Fatalf("probe was admitted during the quiet window at %ds", second) + } + } + now = start.Add(5 * time.Second) + if !handler.acquireColdProbe() { + t.Fatal("probe was not admitted after the fixed quiet interval") + } + handler.releaseColdProbe() +} + +func TestMetadataEnrichmentColdProbeQuietWindowIsSharedAcrossClientsAndRecovers(t *testing.T) { + start := time.Date(2026, time.August, 29, 12, 0, 0, 0, time.UTC) + now := start + handler := &metadataEnrichmentHandler{ + coldProbeQuiet: 5 * time.Second, + now: func() time.Time { return now }, + } + + clients := []string{"apple-tv", "infuse", "plex-ios", "plex-web", "senplayer", "tablet", "desktop", "tv"} + type admissionResult struct { + client string + admitted bool + } + results := make(chan admissionResult, len(clients)) + var group sync.WaitGroup + group.Add(len(clients)) + for _, client := range clients { + go func(client string) { + defer group.Done() + results <- admissionResult{client: client, admitted: handler.acquireColdProbe()} + }(client) + } + group.Wait() + close(results) + + var admitted int + var admittedClient string + for result := range results { + if result.admitted { + admitted++ + admittedClient = result.client + } + } + if admitted != 1 || admittedClient == "" { + t.Fatalf("admitted probes across clients = %d (client %q), want 1", admitted, admittedClient) + } + handler.releaseColdProbe() + + now = start.Add(4 * time.Second) + if handler.acquireColdProbe() { + t.Fatal("probe was admitted before the shared quiet window expired") + } + now = start.Add(5 * time.Second) + if !handler.acquireColdProbe() { + t.Fatal("probe was not admitted when the shared quiet window expired") + } + handler.releaseColdProbe() +} + +func TestMetadataEnrichmentColdProbeQuietWindowStartsAfterActiveProbeCompletes(t *testing.T) { + start := time.Date(2026, time.August, 29, 12, 0, 0, 0, time.UTC) + now := start + handler := &metadataEnrichmentHandler{ + coldProbeQuiet: 5 * time.Second, + now: func() time.Time { return now }, + } + if !handler.acquireColdProbe() { + t.Fatal("probe was rejected before an active probe") + } + + now = start.Add(30 * time.Second) + handler.releaseColdProbe() + now = start.Add(34 * time.Second) + if handler.acquireColdProbe() { + t.Fatal("probe was admitted before the completion-based quiet window expired") + } + now = start.Add(35 * time.Second) + if !handler.acquireColdProbe() { + t.Fatal("probe was not admitted after the completion-based quiet window expired") + } + handler.releaseColdProbe() +} + +func TestMetadataEnrichmentCachedProjectionBypassesColdProbeSlot(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + raw := fixture.xmlBody() + started := make(chan struct{}) + release := make(chan struct{}) + var lookups atomic.Int64 + ensurer := &fakeMediaInfoEnsurer{ + getContextFn: func(_ context.Context, key mediainfo.Key) (mediainfo.Record, bool) { + if lookups.Add(1) == 1 { + return mediainfo.Record{}, false + } + return mediainfo.Record{Key: key, Media: completeProjectionMedia()}, true + }, + fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { + close(started) + <-release + return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil + }, + } + handler := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ + Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, + CloudExtensions: []string{".strm"}, ColdWait: time.Second, + ResponseLimit: 1 << 20, Concurrency: 2, Metrics: metrics.New(), + }, metadataBodyHandler(raw, "application/xml")) + + firstDone := make(chan struct{}) + go func() { + defer close(firstDone) + handler.ServeHTTP(httptest.NewRecorder(), authenticatedMetadataRequest(http.MethodGet)) + }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("first cold probe did not start") + } + + second := httptest.NewRecorder() + handler.ServeHTTP(second, authenticatedMetadataRequest(http.MethodGet)) + if !strings.Contains(second.Body.String(), `container="mkv"`) { + t.Fatalf("cached response was not enriched: %s", second.Body.String()) + } + if ensurer.calls.Load() != 1 { + t.Fatalf("Ensure calls = %d", ensurer.calls.Load()) + } + close(release) + select { + case <-firstDone: + case <-time.After(time.Second): + t.Fatal("first cold probe did not finish") + } +} + +func TestMetadataEnrichmentBurstDoesNotQueueBehindColdProbe(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + raw := fixture.xmlBody() + started := make(chan struct{}) + release := make(chan struct{}) + ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { + close(started) + <-release + return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil + }} + registry := metrics.New() + enrichment := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ + Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, + CloudExtensions: []string{".strm"}, ColdWait: time.Second, + ResponseLimit: 1 << 20, Concurrency: 8, Metrics: registry, + }, metadataBodyHandler(raw, "application/xml")) + handler := newMetadataGuard(MetadataGuardOptions{ + Enabled: true, GlobalConcurrency: 16, PerClientConcurrency: 4, + QueueTimeout: 500 * time.Millisecond, + }, enrichment, registry, nil) + + firstDone := make(chan struct{}) + go func() { + defer close(firstDone) + handler.ServeHTTP(httptest.NewRecorder(), authenticatedMetadataRequest(http.MethodGet)) + }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("first cold probe did not start") + } + + const burst = 64 + responses := make(chan *httptest.ResponseRecorder, burst) + var group sync.WaitGroup + group.Add(burst) + for range burst { + go func() { + defer group.Done() + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedMetadataRequest(http.MethodGet)) + responses <- response + }() + } + burstDone := make(chan struct{}) + go func() { + group.Wait() + close(burstDone) + }() + select { + case <-burstDone: + case <-time.After(time.Second): + t.Fatal("cold metadata burst remained queued") + } + close(responses) + for response := range responses { + if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), raw) { + t.Fatalf("burst response = %d %q", response.Code, response.Body.Bytes()) + } + } + if ensurer.calls.Load() != 1 { + t.Fatalf("Ensure calls = %d", ensurer.calls.Load()) + } + if snapshot := registry.Snapshot(); snapshot.MetadataGuardTimeoutsTotal != 0 || snapshot.MediaInfoWaitRejectedTotal != burst { + t.Fatalf("metrics = %#v", snapshot) + } + + close(release) + select { + case <-firstDone: + case <-time.After(time.Second): + t.Fatal("first cold probe did not finish") + } +} + func TestMetadataEnrichmentBypassesIneligibleAndUnbufferableResponses(t *testing.T) { fixture := newMetadataEnrichmentFixture(t) raw := fixture.xmlBody() diff --git a/internal/gateway/metadata_guard.go b/internal/gateway/metadata_guard.go index bffe546..0caeac3 100644 --- a/internal/gateway/metadata_guard.go +++ b/internal/gateway/metadata_guard.go @@ -13,7 +13,7 @@ import ( ) const ( - defaultMetadataGlobalConcurrency = 8 + defaultMetadataGlobalConcurrency = 16 defaultMetadataPerClientConcurrency = 4 defaultMetadataBatchConcurrency = 3 defaultMetadataQueueTimeout = 10 * time.Second diff --git a/internal/gateway/playback_veto.go b/internal/gateway/playback_veto.go new file mode 100644 index 0000000..eead558 --- /dev/null +++ b/internal/gateway/playback_veto.go @@ -0,0 +1,69 @@ +package gateway + +import ( + "net/http" + "strings" + + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" +) + +type playbackVeto func(*http.Request, mediainfo.Media) (reason string, reject bool) + +func newPlaybackVeto(enabled bool) playbackVeto { + if !enabled { + return nil + } + return vetoAppleTVDolbyVisionProfile5 +} + +// vetoAppleTVDolbyVisionProfile5 is an optional compatibility override for +// deployments where this exact client and media combination is known to fail. +// Unknown, incomplete, or conflicting facts always preserve Plex's decision. +func vetoAppleTVDolbyVisionProfile5(request *http.Request, media mediainfo.Media) (string, bool) { + if !media.Complete || request == nil || request.URL == nil { + return "", false + } + product, productOK := plexRequestFact(request, "X-Plex-Product") + platform, platformOK := plexRequestFact(request, "X-Plex-Platform") + if !productOK || !platformOK || !strings.EqualFold(product, "Plex for Apple TV") || !strings.EqualFold(platform, "tvOS") { + return "", false + } + var video *mediainfo.Stream + for index := range media.Streams { + if !strings.EqualFold(strings.TrimSpace(media.Streams[index].Type), "video") { + continue + } + if video != nil { + return "", false + } + video = &media.Streams[index] + } + if video == nil || !strings.EqualFold(strings.TrimSpace(video.HDRFormat), "dolby_vision") || video.DolbyVision == nil { + return "", false + } + if video.DolbyVision.Profile != 5 || video.DolbyVision.BLCompatID != 0 { + return "", false + } + return "dolby_vision_profile_5", true +} + +func plexRequestFact(request *http.Request, name string) (string, bool) { + queryValues, queryPresent := request.URL.Query()[name] + headerValues := request.Header.Values(name) + if queryPresent && (len(queryValues) != 1 || strings.TrimSpace(queryValues[0]) == "") { + return "", false + } + if len(headerValues) > 1 || len(headerValues) == 1 && strings.TrimSpace(headerValues[0]) == "" { + return "", false + } + if queryPresent && len(headerValues) == 1 && queryValues[0] != headerValues[0] { + return "", false + } + if queryPresent { + return strings.TrimSpace(queryValues[0]), true + } + if len(headerValues) == 1 { + return strings.TrimSpace(headerValues[0]), true + } + return "", false +} diff --git a/internal/gateway/playback_veto_test.go b/internal/gateway/playback_veto_test.go new file mode 100644 index 0000000..3f2f9cd --- /dev/null +++ b/internal/gateway/playback_veto_test.go @@ -0,0 +1,53 @@ +package gateway + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" +) + +func TestPlaybackVetoIsAbsentWhenDisabled(t *testing.T) { + if veto := newPlaybackVeto(false); veto != nil { + t.Fatal("disabled playback veto is not nil") + } +} + +func TestPlaybackVetoRejectsOnlyExactAppleTVDolbyVisionProfile5(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, "/decision", nil) + request.Header.Set("X-Plex-Product", "Plex for Apple TV") + request.Header.Set("X-Plex-Platform", "tvOS") + media := mediainfo.Media{ + Complete: true, + Streams: []mediainfo.Stream{{ + Type: "video", HDRFormat: "dolby_vision", + DolbyVision: &mediainfo.DolbyVision{Profile: 5, BLCompatID: 0}, + }}, + } + veto := newPlaybackVeto(true) + if reason, reject := veto(request, media); !reject || reason != "dolby_vision_profile_5" { + t.Fatalf("veto() = %q, %v", reason, reject) + } + + request.Header.Set("X-Plex-Product", "Infuse") + if reason, reject := veto(request, media); reject || reason != "" { + t.Fatalf("Infuse veto() = %q, %v", reason, reject) + } +} + +func TestPlaybackVetoAbstainsOnConflictingClientFacts(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, "/decision?X-Plex-Platform=tvOS", nil) + request.Header.Set("X-Plex-Product", "Plex for Apple TV") + request.Header.Set("X-Plex-Platform", "iOS") + media := mediainfo.Media{ + Complete: true, + Streams: []mediainfo.Stream{{ + Type: "video", HDRFormat: "dolby_vision", + DolbyVision: &mediainfo.DolbyVision{Profile: 5, BLCompatID: 0}, + }}, + } + if reason, reject := newPlaybackVeto(true)(request, media); reject || reason != "" { + t.Fatalf("veto() = %q, %v", reason, reject) + } +} diff --git a/internal/gateway/universal_start_test.go b/internal/gateway/universal_start_test.go index e928cc0..d5572e6 100644 --- a/internal/gateway/universal_start_test.go +++ b/internal/gateway/universal_start_test.go @@ -100,10 +100,10 @@ func TestGrantedCloudTranscodeStartRedirectsToDirectURL(t *testing.T) { })) defer plex.Close() - handler, registry, cache := newCloudHandler(t, plex.URL, mediaVault.URL, []pathmap.Mapping{{ + handler, registry, cache := newCloudHandlerWithVeto(t, plex.URL, mediaVault.URL, []pathmap.Mapping{{ PlexPrefix: "/media/cloud", LocalPrefix: localRoot, - }}) + }}, true) performCloudDecision(t, handler, query, http.StatusOK) response := httptest.NewRecorder() @@ -622,10 +622,10 @@ func TestLocalTranscodeStartRemainsUnchanged(t *testing.T) { })) defer plex.Close() - handler, _, _ := newCloudHandler(t, plex.URL, "http://mediavault.invalid:7811", []pathmap.Mapping{{ + handler, _, _ := newCloudHandlerWithVeto(t, plex.URL, "http://mediavault.invalid:7811", []pathmap.Mapping{{ PlexPrefix: "/media/cloud", LocalPrefix: t.TempDir(), - }}) + }}, true) response := httptest.NewRecorder() handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/video/:/transcode/universal/start.mpd?"+originalQuery, nil)) if response.Code != http.StatusAccepted || startQuery != originalQuery { diff --git a/internal/mediainfo/benchmark_test.go b/internal/mediainfo/benchmark_test.go index bcbebef..c4924fd 100644 --- a/internal/mediainfo/benchmark_test.go +++ b/internal/mediainfo/benchmark_test.go @@ -24,6 +24,43 @@ func BenchmarkMediaInfoL1Get(b *testing.B) { } } +func BenchmarkMediaInfoServiceGetMemory(b *testing.B) { + now := time.Now().UTC() + record := completeRecord(now) + service, err := NewService(ServiceOptions{ + Cache: NewCache([]Record{record}, now), Store: &fakeRecordStore{}, + Provider: &fakeProvider{}, PlexServerID: record.Key.PlexServerID, + BackgroundUserAgent: "Infuse-Library/8.5.1", Now: func() time.Time { return now }, + }) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := service.Close(ctx); err != nil { + b.Error(err) + } + }) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if _, ok := service.GetMemory(record.Key); !ok { + b.Fatal("service L1 miss") + } + } +} + +func BenchmarkFingerprintSTRMTarget(b *testing.B) { + const target = "http://mediavault:7811/redirect/pickcode/movie.mkv" + b.ReportAllocs() + for range b.N { + if _, err := FingerprintSTRMTarget(target); err != nil { + b.Fatal(err) + } + } +} + func BenchmarkMediaInfoL1PutAtCapacity(b *testing.B) { now := time.Now() records := make([]Record, defaultCacheEntries) diff --git a/internal/plexmeta/decision_reject.go b/internal/plexmeta/decision_reject.go new file mode 100644 index 0000000..dd26201 --- /dev/null +++ b/internal/plexmeta/decision_reject.go @@ -0,0 +1,34 @@ +package plexmeta + +import ( + "encoding/json" + "strings" +) + +const ( + incompatibleDirectPlayDecisionCode = 1000 + incompatibleTranscodeDecisionCode = 4005 + incompatibleGeneralDecisionCode = 2000 +) + +// IncompatibleDecision builds the minimal Plex decision envelope observed for +// a source that cannot be Direct Played or transcoded. It intentionally omits +// Video, Media, Part, and session identities so a reject cannot expose or start +// another playback path. +func IncompatibleDecision(accept string) (contentType string, body []byte) { + if strings.Contains(strings.ToLower(accept), "json") { + body, _ = json.Marshal(map[string]any{ + "MediaContainer": map[string]any{ + "size": 0, + "directPlayDecisionCode": incompatibleDirectPlayDecisionCode, + "transcodeDecisionCode": incompatibleTranscodeDecisionCode, + "generalDecisionCode": incompatibleGeneralDecisionCode, + }, + }) + return "application/json; charset=utf-8", body + } + return "application/xml; charset=utf-8", []byte( + `` + + ``, + ) +} diff --git a/internal/plexmeta/decision_reject_test.go b/internal/plexmeta/decision_reject_test.go new file mode 100644 index 0000000..118dc91 --- /dev/null +++ b/internal/plexmeta/decision_reject_test.go @@ -0,0 +1,33 @@ +package plexmeta + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestIncompatibleDecisionBuildsXMLAndJSONEnvelopes(t *testing.T) { + contentType, body := IncompatibleDecision("application/xml") + if contentType != "application/xml; charset=utf-8" || !strings.Contains(string(body), `transcodeDecisionCode="4005"`) || strings.Contains(string(body), " Date: Sun, 30 Aug 2026 00:16:57 +0800 Subject: [PATCH 09/15] feat: prioritize MediaInfo probe scheduling --- CHANGELOG.en.md | 18 +- CHANGELOG.md | 10 +- README.en.md | 36 +- README.md | 31 +- app.env.example | 20 +- cmd/plex-gateway/main.go | 13 +- cmd/plex-gateway/main_test.go | 35 +- docs/architecture.md | 63 +- docs/media-lifecycle-architecture.md | 12 +- docs/performance-matrix.md | 13 +- docs/roadmap.md | 22 +- internal/config/config.go | 79 +- internal/config/config_test.go | 26 +- internal/gateway/decision.go | 2 +- internal/gateway/metadata_enrichment.go | 46 +- internal/gateway/metadata_enrichment_test.go | 41 +- internal/mediainfo/benchmark_test.go | 62 ++ internal/mediainfo/scheduler_contract_test.go | 750 ++++++++++++++++ internal/mediainfo/service.go | 847 ++++++++++++++---- internal/mediainfo/service_test.go | 3 + internal/metrics/metrics.go | 175 +++- internal/metrics/metrics_test.go | 61 +- internal/prewarm/service.go | 46 +- internal/prewarm/service_test.go | 45 +- 24 files changed, 2038 insertions(+), 418 deletions(-) create mode 100644 internal/mediainfo/scheduler_contract_test.go diff --git a/CHANGELOG.en.md b/CHANGELOG.en.md index 722f8a0..cf6425a 100644 --- a/CHANGELOG.en.md +++ b/CHANGELOG.en.md @@ -16,15 +16,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added bounded remote ffprobe, an L1 LRU plus SQLite MediaInfo cache, and single-item Plex metadata response enrichment. Cold-wait timeouts preserve the original response while probing continues in the background. -- Current-Part prewarming is submitted at interactive priority after a cloud - redirect is ready. An isolated Plex management token can additionally discover - and rate-limit a configurable nearby window, defaulting to two previous and - three following items. The current job is only prioritized in the queue and +- Current-Part prewarming is submitted to P0 after a cloud redirect is ready. An + isolated Plex management token can additionally discover a configurable P1 + nearby window, defaulting to two previous and three following items. The + current job is only prioritized in the queue and cannot preempt a probe already running. Missing or invalid management credentials disable only nearby discovery. - Added per-client rapid-switch coordination, cross-User-Agent failure handoff, and distinct prewarm metrics for fresh cache hits, joined work, new queue admissions, and rejections. +- Unified actual playback, nearby prewarming, and fixed-window metadata misses + under one P0/P1/P2 MediaInfo scheduler. P1/P2 use a five-minute pending TTL + and a global five-second remote-start interval while P0 keeps reserved capacity. - Parts without Plex Streams now receive descriptive video, audio, and subtitle Streams without Plex IDs or playback-selection state, including HDR10, Dolby Vision, bit depth, bitrate, channel, and language fields. @@ -55,10 +58,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Limited one continuous metadata browsing burst to one synchronous cold probe. After the admitted synchronous wait releases its slot, a fixed five-second - window begins. Other cold misses immediately preserve the Plex response, and - rejected requests do not renew the window. An admitted probe may finish in the - background after the request wait ends and persist the result to L1 and SQLite, - preventing starvation under continuous traffic. + window begins. Other cold misses preserve the Plex response immediately and + may offer one deduplicated P2 task without renewing the window. Workers check + SQLite first, so only genuine misses enter the remote-start limiter. - Set the Metadata Guard defaults to global 8, per-client 4, and batch 3. After Plex MediaInfo writes provide high coverage and the resulting load is retested, global 16, per-client 4, and batch 3 remains a candidate for evaluation. diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e35067..f353869 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,11 +15,13 @@ - 增加 MediaInfo L1 LRU 与 SQLite 持久缓存、受限远程 ffprobe 和单项 Plex metadata 响应增强；冷缓存等待超时后保持原响应并在后台完成探测。 -- 云端重定向就绪后将当前 Part 以交互优先级提交，并可使用独立 Plex 管理 Token 按默认 - 前 2、后 3 的窗口限速预热邻近媒体。当前任务只会优先排队，不能抢占正在运行的探测。 +- 云端重定向就绪后将当前 Part 投递 P0，并可使用独立 Plex 管理 Token 按默认前 2、后 3 + 的窗口发现并投递 P1 邻近媒体。当前任务只会优先排队，不能抢占正在运行的探测。 Token 缺失或失效只禁用邻近发现。 - 增加按客户端隔离的快速切换协调、跨 User-Agent 失败接力，以及缓存命中、加入现有 任务、新入队和拒绝的预热指标。 +- 使用统一的 P0/P1/P2 MediaInfo 调度器处理实际播放、邻近预热和 fixed-window metadata + miss。P1/P2 采用 5 分钟 pending TTL 和全局 5 秒远程启动间隔，P0 使用独立容量。 - 为没有 Stream 的 STRM Part 创建不含 Plex ID 或播放选择状态的描述性视频、音频和 字幕 Stream，并补充 HDR10、Dolby Vision、bit depth、bitrate、声道和语言等字段。 - Infuse 风格的后台媒体库同步只读取现有 MediaInfo 缓存，不再因逐项浏览触发全库冷探测。 @@ -40,8 +42,8 @@ ### 修复 - 限制一次连续 metadata 浏览突发最多同步等待一个冷探测。释放本次同步准入后建立固定 - 5 秒窗口，窗口内的其他冷 miss 立即透传 Plex，拒绝请求不会续期窗口。已准入的探测 - 在请求等待结束后仍可在后台完成，并成功写入 L1 和 SQLite，避免持续流量造成饥饿。 + 5 秒窗口，窗口内的其他冷 miss 立即透传 Plex，并可非阻塞投递唯一 P2 任务；这些请求 + 不会续期窗口。后台 worker 先检查 SQLite，只有真实 miss 才进入远程启动限速。 - 将单项 metadata Guard 的默认值固定为全局 8、单客户端 4、批量 3。Plex MediaInfo 写库获得较高覆盖率并完成重测后，再评估全局 16、单客户端 4、批量 3 的候选值。 diff --git a/README.en.md b/README.en.md index e843c4b..5adb166 100644 --- a/README.en.md +++ b/README.en.md @@ -130,6 +130,11 @@ Important environment variables: | `METADATA_GUARD_QUEUE_TIMEOUT` | `10s` | Maximum admission wait before returning `429`. | | `MEDIAINFO_ENABLED` | `true` | Enable the MediaInfo cache, bounded probes, and single-item metadata enrichment; initialization failures do not affect transparent proxying. | | `DATABASE_PATH` | `./data/plex-gateway.db` | Gateway SQLite database path; the container image defaults to `/app_data/plex-gateway.db`. | +| `MEDIAINFO_PLAYBACK_QUEUE_SIZE` | `16` | Independent P0 queue capacity for actual playback work. | +| `MEDIAINFO_NEIGHBOR_QUEUE_SIZE` | `50` | P1 queue capacity for nearby-item prewarming. | +| `MEDIAINFO_METADATA_QUEUE_SIZE` | `50` | P2 queue capacity for fixed-window metadata misses. | +| `MEDIAINFO_PENDING_TTL` | `5m` | Lifetime of P1/P2 work that has not started. | +| `MEDIAINFO_BACKGROUND_INTERVAL` | `5s` | Global minimum interval between P1/P2 remote probes after L1 and SQLite miss. | | `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.5.1` | Fallback User-Agent for background probes without an active client context. | | `MEDIAINFO_COLD_WAIT` | `5s` | Cold-cache wait ceiling for one metadata item; timeout returns the original Plex response while probing continues. | | `PLAYBACK_VETO_ENABLED` | `false` | Enable the experimental Apple TV Plex Dolby Vision Profile 5 veto; it only reuses fresh MediaInfo already obtained by the decision path. | @@ -137,7 +142,6 @@ Important environment variables: | `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `8` | Maximum single-item metadata responses buffered and waiting for MediaInfo concurrently. | | `MEDIAINFO_PREWARM_BEFORE` | `2` | Number of nearby items before the current item to prewarm. | | `MEDIAINFO_PREWARM_AFTER` | `3` | Number of nearby items after the current item to prewarm. | -| `MEDIAINFO_PREWARM_INTERVAL` | `5s` | Delay between nearby-item submissions; the current item does not wait. | Ordinary Plex requests remain transparent, while cloud playback sends client headers to the trusted MediaVault origin under the contract below. The optional @@ -217,11 +221,11 @@ Background library synchronization can enumerate an entire library one item at a time. Requests with `skipRefresh` whose product ends in `-Library` may consume an existing MediaInfo cache record but never admit a cold probe. Ordinary single-item access follows the same fixed cold-probe window. Cold misses inside -the window are not admitted and do not renew it; successful cloud 302 and -nearby-window work retain their existing boundaries. Browsing a library -therefore cannot expand into full-library CDN ffprobe work. Any cache miss, -timeout, unsupported structure, or projection failure returns the original Plex -response. +the window return the original response immediately and only offer bounded P2 +work without renewing it; successful cloud 302 and nearby-window work retain +their existing boundaries. Browsing a library therefore cannot expand without +bound into full-library CDN ffprobe work. Any cache miss, timeout, unsupported +structure, or projection failure returns the original Plex response. ### Nearby-item MediaInfo prewarming @@ -232,7 +236,7 @@ it does not claim that the client followed the redirect or actually started playback. Current-item prewarming does not require a management token. A valid `PLEX_TOKEN` additionally enables nearby-item discovery. -The current item enters the interactive-priority queue. The background +The current item enters the P0 actual-playback queue. The background coordinator prefers explicit Plex playQueue order and permits movie, cross-show, and multi-Media/Part entries. Every candidate is stored under its own PartID and STRM fingerprint. The current item can only be prioritized in the queue and @@ -241,13 +245,14 @@ season response order is used, including specials and entries with missing indices. The default window is two items before and three after, with following items -submitted first. The combined configurable window is capped at 50. Nearby items -enter the low-priority queue every five seconds by default, while the MediaInfo -worker defaults to one concurrent probe to avoid MediaVault/CDN bursts. A rapid -A/B/C switch submits each new current item immediately and cancels only the old -window entries that were not submitted. Submitted or running work retains its -own identity and is deduplicated by singleflight. A running background probe is -not preempted by a current item. +submitted first. The combined configurable window is capped at 50. Prepared +neighbors are offered to P1 immediately without blocking. Only genuine L1 and +SQLite misses are subject to the global five-second minimum remote-start +interval, while the MediaInfo worker defaults to one concurrent probe to avoid +MediaVault/CDN bursts. A rapid A/B/C switch submits each new current item to P0 +immediately and cancels only old-window entries that were not offered. Offered +or running work retains its own identity and is deduplicated by singleflight. A +running background probe is not preempted by a current item. The gateway persists parsed MediaInfo, not short-lived CDN URLs or raw head/tail bytes. MediaVault upload precaching applies only to files uploaded through @@ -371,7 +376,8 @@ That topology is outside the exclusive-gateway deployment contract. - `GET /metrics` returns fixed-shape JSON counters plus resolver and complete redirect-path latency totals, samples, last values, and maxima. When metadata protection is enabled, it also reports admission, timeout, active, and queued - counts. No metric contains request labels or credentials. + counts. MediaInfo scheduler outcomes are additionally split into fixed + P0/P1/P2 sections. No metric contains request labels or credentials. - Every other endpoint follows the Plex proxy or Direct Play interception rules described in [docs/architecture.md](docs/architecture.md). diff --git a/README.md b/README.md index 07a31fa..771c6d4 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,11 @@ go run ./cmd/plex-gateway | `METADATA_GUARD_QUEUE_TIMEOUT` | `10s` | 等待准入的最长时间，超时返回 `429`。 | | `MEDIAINFO_ENABLED` | `true` | 启用 MediaInfo 缓存、受限探测和单项 metadata 增强；初始化失败不影响透明代理。 | | `DATABASE_PATH` | `./data/plex-gateway.db` | Gateway SQLite 持久数据库路径；容器镜像默认使用 `/app_data/plex-gateway.db`。 | +| `MEDIAINFO_PLAYBACK_QUEUE_SIZE` | `16` | P0 实际播放任务的独立队列容量。 | +| `MEDIAINFO_NEIGHBOR_QUEUE_SIZE` | `50` | P1 邻近项预热的队列容量。 | +| `MEDIAINFO_METADATA_QUEUE_SIZE` | `50` | P2 fixed-window metadata miss 的队列容量。 | +| `MEDIAINFO_PENDING_TTL` | `5m` | P1/P2 未开始任务的保留时间。 | +| `MEDIAINFO_BACKGROUND_INTERVAL` | `5s` | L1/SQLite miss 后 P1/P2 远程探测的全局最小启动间隔。 | | `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.5.1` | 没有活动客户端上下文时，后台探测使用的 fallback User-Agent。 | | `MEDIAINFO_COLD_WAIT` | `5s` | 单项 metadata 冷缓存等待上限；超时返回原 Plex 响应，探测继续。 | | `PLAYBACK_VETO_ENABLED` | `false` | 启用实验性 Apple TV Plex Dolby Vision Profile 5 veto；只复用 decision 已取得的新鲜 MediaInfo，不发起额外探测。 | @@ -122,7 +127,6 @@ go run ./cmd/plex-gateway | `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `8` | 同时缓冲和等待 MediaInfo 的单项 metadata 响应上限。 | | `MEDIAINFO_PREWARM_BEFORE` | `2` | 当前项之前的邻近媒体预热数量。 | | `MEDIAINFO_PREWARM_AFTER` | `3` | 当前项之后的邻近媒体预热数量。 | -| `MEDIAINFO_PREWARM_INTERVAL` | `5s` | 邻近项提交到后台探测队列的间隔；当前项不等待。 | 普通 Plex 请求保持透明转发，云端播放请求按下文契约将客户端 header 发送给可信 MediaVault。可选管理 `PLEX_TOKEN` 通过环境变量注入，只发送到配置的 Plex origin， @@ -181,10 +185,10 @@ User-Agent 对同一临时直链发送一次 `Range: bytes=0-0`。该请求最 Infuse 等客户端的后台媒体库同步可能逐项请求整个库。带 `skipRefresh` 且产品名以 `-Library` 结尾的后台同步请求只读取现有 MediaInfo 缓存，不创建冷探测。普通单项访问 -遵循同一个固定冷探测窗口，窗口内的冷 miss 不准入且不会续期；成功的云端 302 和邻近窗口 -继续按既定边界提交。 -因此浏览媒体库不会扩散成全库 CDN ffprobe。任何缓存 miss、超时、结构不支持或投影失败 -都会返回原始 Plex response。 +遵循同一个固定冷探测窗口，窗口内的冷 miss 立即返回原始响应，仅非阻塞投递有界 P2 +任务且不会续期；成功的云端 302 和邻近窗口继续按既定边界提交。因此浏览媒体库不会无界 +扩散成全库 CDN ffprobe。任何缓存 miss、超时、结构不支持或投影失败都会返回原始 Plex +response。 ### 邻近媒体 MediaInfo 预热 @@ -193,17 +197,17 @@ Gateway 在云端 Part 已通过 Plex 授权、MediaVault 已返回最终直链 客户端已经跟随 302 或真实起播。当前项预热不需要管理 Token；配置有效 `PLEX_TOKEN` 后才会额外发现邻近媒体。 -当前项会进入交互优先级队列。后台优先使用明确的 Plex playQueue 顺序，并允许其中的 +当前项会进入 P0 实际播放队列。后台优先使用明确的 Plex playQueue 顺序，并允许其中的 电影、跨剧条目和多 Media/Part；候选始终采用自身的 PartID 与 STRM fingerprint 落库， 不会写入当前项。当前项只能优先排队，不能抢占正在运行的探测。没有可靠 playQueue 时， 按 Plex 返回的剧集和季顺序查找邻近项，包括 S00 和缺失索引的条目。 -默认窗口为前 `2`、后 `3`，后续项先提交；两个方向合计最多配置 `50`。邻近项默认每隔 -`5s` 进入低优先级队列，MediaInfo worker 默认并发为 `1`，用于避免 MediaVault/CDN -请求突发。快速切换 A、B、C 时，新当前项立即进入交互队列，旧窗口中尚未提交的候选会被 -取消；已经提交或正在运行的任务仍按自己的身份完成并由 singleflight 去重。正在运行的 -后台探测不会被当前项抢占。整个发现、STRM 读取、指纹计算和 SQLite 查询均在 302 响应 -之后执行。 +默认窗口为前 `2`、后 `3`，后续项先提交；两个方向合计最多配置 `50`。邻近项发现和准备 +完成后立即非阻塞投递 P1；只有 L1 和 SQLite 都未命中时，远程探测才受全局最小 `5s` +启动间隔约束。MediaInfo worker 默认并发为 `1`，用于避免 MediaVault/CDN 请求突发。 +快速切换 A、B、C 时，新当前项立即进入 P0，旧窗口中尚未投递的候选会被取消；已经投递 +或正在运行的任务仍按自己的身份完成并由 singleflight 去重。正在运行的后台探测不会被 +当前项抢占。整个发现、STRM 读取、指纹计算和 SQLite 查询均在 302 响应之后执行。 Gateway 只持久化解析后的 MediaInfo，不缓存短期 CDN URL 或原始文件头尾。MediaVault 的 上传预缓存只覆盖经其上传的新文件；若未来提供稳定 MediaInfo/预缓存读取 API，可作为优先 @@ -302,7 +306,8 @@ Gateway 重新执行检查。 - `GET /health` 返回进程健康状态，以及不含媒体身份和凭据的 MediaInfo 缓存、探测队列和邻近媒体预热摘要。 - `GET /metrics` 返回固定结构的 JSON 计数器，以及 resolver 和完整重定向链路的 延迟总计、样本数、最近值和最大值。启用 metadata 保护时还会返回准入、超时、 - 活动和排队计数。所有指标都不包含请求标签或凭据。 + 活动和排队计数；MediaInfo 调度器另按固定 P0/P1/P2 结构返回准入和丢弃结果。所有指标 + 都不包含请求标签或凭据。 - 其他所有 endpoint 都遵循 [docs/architecture.md](docs/architecture.md) 中说明 的 Plex 代理或 Direct Play 拦截规则。 diff --git a/app.env.example b/app.env.example index 87ad25b..46e7031 100644 --- a/app.env.example +++ b/app.env.example @@ -78,11 +78,20 @@ MEDIAINFO_OUTPUT_MAX_BYTES=2097152 # MediaInfo worker 并发数。CDN 风控敏感环境建议保持 1。 MEDIAINFO_CONCURRENCY=1 -# 交互式冷缓存任务的内存队列上限。 -MEDIAINFO_INTERACTIVE_QUEUE_SIZE=256 +# P0 实际播放任务的独立内存队列上限，不与后台任务共享容量。 +MEDIAINFO_PLAYBACK_QUEUE_SIZE=16 -# 邻近项预热等后台任务的内存队列上限。 -MEDIAINFO_BACKGROUND_QUEUE_SIZE=256 +# P1 当前播放项前后邻近预热任务的内存队列上限。 +MEDIAINFO_NEIGHBOR_QUEUE_SIZE=50 + +# P2 fixed-window 内前台 metadata miss 的内存队列上限。 +MEDIAINFO_METADATA_QUEUE_SIZE=50 + +# P1/P2 未开始任务的保留时间。过期任务不会访问 MediaVault 或 CDN。 +MEDIAINFO_PENDING_TTL=5m + +# P1/P2 在确认 L1 和 SQLite 都未命中后，实际启动远程探测的最小间隔。 +MEDIAINFO_BACKGROUND_INTERVAL=5s # 已知 MediaInfo 结果的新鲜期。过期后仍可返回旧值并在后台刷新。 MEDIAINFO_RECORD_TTL=720h @@ -118,9 +127,6 @@ MEDIAINFO_PREWARM_BEFORE=2 # 当前播放项之后预热的邻近媒体数量。前后合计不能超过 50。 MEDIAINFO_PREWARM_AFTER=3 -# 邻近项进入后台探测队列的间隔。当前播放项只优先排队，不能抢占运行中的探测。 -MEDIAINFO_PREWARM_INTERVAL=5s - # HTTP 请求头读取超时。 READ_HEADER_TIMEOUT=15s diff --git a/cmd/plex-gateway/main.go b/cmd/plex-gateway/main.go index c4d6f37..c9f10e3 100644 --- a/cmd/plex-gateway/main.go +++ b/cmd/plex-gateway/main.go @@ -87,7 +87,7 @@ func run() error { context.Background(), cfg.PlexToken, cfg.PlexURL, cfg.PartProbeTimeout, playback.NewPartPreparer(mapper, strmResolver, cfg.CloudExtensions), mediaInfoService, cfg.MediaInfo.PrewarmBefore, cfg.MediaInfo.PrewarmAfter, - cfg.MediaInfo.PrewarmInterval, registry, logger, + registry, logger, ) defer closeRuntime(prewarmService, mediaInfoService, databaseStore, cfg.ShutdownTimeout, logger) mediaInfoStatus := func() mediainfo.Status { @@ -153,7 +153,7 @@ func run() error { "mediainfo_prewarm_reason", prewarmReason, "mediainfo_prewarm_before", cfg.MediaInfo.PrewarmBefore, "mediainfo_prewarm_after", cfg.MediaInfo.PrewarmAfter, - "mediainfo_prewarm_interval_ms", cfg.MediaInfo.PrewarmInterval.Milliseconds(), + "mediainfo_background_interval_ms", cfg.MediaInfo.BackgroundInterval.Milliseconds(), "playback_veto_enabled", cfg.PlaybackVeto, ) serveErrors <- server.ListenAndServe() @@ -187,7 +187,6 @@ func initializePrewarm( mediaInfoService *mediainfo.Service, beforeCount int, afterCount int, - submitInterval time.Duration, registry *metrics.Metrics, logger *slog.Logger, ) (*prewarm.Service, string) { @@ -225,7 +224,7 @@ func initializePrewarm( serviceOptions := prewarm.ServiceOptions{ Playback: partPreparer, MediaInfo: mediaInfoService, Logger: logger, Metrics: registry, DiscoveryTimeout: timeout, - BeforeCount: beforeCount, AfterCount: afterCount, SubmitInterval: submitInterval, + BeforeCount: beforeCount, AfterCount: afterCount, } if discovery != nil { serviceOptions.Discovery = discovery @@ -294,8 +293,10 @@ func initializeMediaInfo( service, err := mediainfo.NewService(mediainfo.ServiceOptions{ Cache: cache, Store: store, Janitor: store, Provider: provider, PlexServerID: plexServerID, Logger: logger, Metrics: registry, Concurrency: cfg.Concurrency, - InteractiveQueueSize: cfg.InteractiveQueueSize, BackgroundQueueSize: cfg.BackgroundQueueSize, - ProbeTimeout: cfg.ProbeTimeout, RecordTTL: cfg.RecordTTL, + PlaybackQueueSize: cfg.PlaybackQueueSize, NeighborQueueSize: cfg.NeighborQueueSize, + MetadataQueueSize: cfg.MetadataQueueSize, PendingTTL: cfg.PendingTTL, + BackgroundInterval: cfg.BackgroundInterval, + ProbeTimeout: cfg.ProbeTimeout, RecordTTL: cfg.RecordTTL, RecordRetention: cfg.RecordRetention, NegativeTTL: cfg.NegativeTTL, BackgroundUserAgent: cfg.UserAgent, }) diff --git a/cmd/plex-gateway/main_test.go b/cmd/plex-gateway/main_test.go index 3936ecf..f9113e4 100644 --- a/cmd/plex-gateway/main_test.go +++ b/cmd/plex-gateway/main_test.go @@ -71,20 +71,23 @@ func TestInitializeMediaInfoDoesNotProbeWithoutTask(t *testing.T) { logger := slog.New(slog.NewTextHandler(io.Discard, nil)) databaseStore, databaseReason := initializeDatabase(t.Context(), filepath.Join(directory, "plex-gateway.db"), true, logger) service, reason := initializeMediaInfo(t.Context(), databaseStore, databaseReason, config.MediaInfoConfig{ - Enabled: true, - FFProbePath: ffprobePath, - ProbeTimeout: time.Second, - ProbeSize: 1 << 20, - AnalyzeDuration: time.Second, - OutputMaxBytes: 1 << 20, - Concurrency: 1, - InteractiveQueueSize: 2, - BackgroundQueueSize: 2, - RecordTTL: 24 * time.Hour, - RecordRetention: 48 * time.Hour, - L1MaxEntries: 10, - NegativeTTL: time.Minute, - UserAgent: "plex-gateway-test", + Enabled: true, + FFProbePath: ffprobePath, + ProbeTimeout: time.Second, + ProbeSize: 1 << 20, + AnalyzeDuration: time.Second, + OutputMaxBytes: 1 << 20, + Concurrency: 1, + PlaybackQueueSize: 2, + NeighborQueueSize: 2, + MetadataQueueSize: 2, + PendingTTL: time.Minute, + BackgroundInterval: time.Millisecond, + RecordTTL: 24 * time.Hour, + RecordRetention: 48 * time.Hour, + L1MaxEntries: 10, + NegativeTTL: time.Minute, + UserAgent: "plex-gateway-test", }, plexURL, time.Second, control, metrics.New(), logger) if service == nil || databaseStore == nil || reason != "ready" { t.Fatalf("initializeMediaInfo() service=%v store=%v reason=%q", service != nil, databaseStore != nil, reason) @@ -111,7 +114,7 @@ func TestInitializeMediaInfoDoesNotProbeWithoutTask(t *testing.T) { partPreparer := playback.NewPartPreparer(mapper, control, []string{".strm"}) invalidTokenService, reason := initializePrewarm( t.Context(), "invalid-token", plexURL, time.Second, partPreparer, service, - 2, 3, time.Millisecond, metrics.New(), logger, + 2, 3, metrics.New(), logger, ) if invalidTokenService == nil || reason != "current_only_plex_token_invalid" || !invalidTokenService.Status().Available || invalidTokenService.Status().NeighborAvailable { @@ -125,7 +128,7 @@ func TestInitializeMediaInfoDoesNotProbeWithoutTask(t *testing.T) { closeCancel() prewarmService, reason := initializePrewarm( t.Context(), "management-token", plexURL, time.Second, partPreparer, service, - 2, 3, time.Millisecond, metrics.New(), logger, + 2, 3, metrics.New(), logger, ) if prewarmService == nil || reason != "ready" || !prewarmService.Status().Available { t.Fatalf("prewarm service=%v reason=%q", prewarmService != nil, reason) diff --git a/docs/architecture.md b/docs/architecture.md index dbf5615..3461937 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -35,10 +35,10 @@ gateway; the isolated analysis worker may read a bounded amount for probing. identified by `skipRefresh` and a `-Library` product consume cache only. Hot records remain concurrently projectable; one continuous browsing burst may wait on only one cold probe. After the admitted synchronous wait releases its - slot, a fixed five-second window begins. Other cold misses preserve the Plex - response, and rejected requests do not renew the window. An admitted probe - may finish in the background after the request wait ends and persist to L1 - and SQLite, preventing guard-delayed request waves from starving later work. + slot, a fixed five-second window begins. Other eligible cold misses preserve + the Plex response and may offer bounded P2 work without renewing the window. + An admitted probe may finish in the background after the request wait ends + and persist to L1 and SQLite. - `GET` and `HEAD` requests below `/library/parts/{partID}/...` are eligible for interception only when the cached `Part.file` has a configured cloud extension and maps to a readable local STRM file. @@ -173,12 +173,55 @@ Plex-owned. All clients use the same MediaInfo projection rules. Part and universal-start redirects never wait for MediaInfo. An L1 hit does not synchronously read SQLite for the request; access renewal may touch SQLite asynchronously. An L1 miss may -use the persistent record or one bounded interactive probe, and timeout or -failure preserves the existing Direct Play path. Metadata browsing has a -separate single synchronous cold-probe slot. After the admitted synchronous wait -releases its slot, a fixed five-second window begins. Misses rejected inside that -window do not renew it, and an admitted probe can finish in the background and -write L1 and SQLite after the request waiter leaves. +use the persistent record or one bounded playback probe, and timeout or failure +preserves the existing Direct Play path. Metadata browsing has a separate single +synchronous cold-probe slot. After the admitted synchronous wait releases its +slot, a fixed five-second window begins. Misses inside that window do not renew +it: eligible foreground misses may be offered to the bounded background +scheduler, while the original Plex response is returned immediately. An +admitted probe can finish in the background and write L1 and SQLite after the +request waiter leaves. + +## MediaInfo scheduling contract + +The in-memory scheduler owns all remote-probe admission. It has three fixed +priority classes: + +1. P0 represents the item that reached an actual playback decision or successful + cloud redirect. Its queue reserves 16 entries and is never displaced by + background work. +2. P1 represents the configured two previous and three following neighbors of an + actual playback item. Its queue admits at most 50 entries. +3. P2 represents an eligible foreground metadata miss that could not use the + synchronous cold-probe slot. Its queue admits at most 50 entries. + +P1 and P2 pending entries expire after five minutes and share one global +five-second remote start interval after both L1 and SQLite miss. Fresh SQLite +records complete without waiting for that interval. P0 bypasses the interval and +runs before all pending background work, but it does not cancel a probe that has +already started. Expired, rejected, or superseded work never reaches MediaVault +or the CDN and does not create negative-cache state. + +The exact task identity is Plex server ID, Part ID, and STRM fingerprint. A +queued or running identity owns one flight across all clients and priorities. +When a higher-priority request observes the same queued identity, the existing +job is promoted in place rather than duplicated. Promotion to P0 refreshes the +unclaimed job with the active playback User-Agent and request identity and +removes its background expiry. A P0 arrival joins an already running flight; if +that attempt fails with a different User-Agent, at most one P0 retry may be +retained. P2 never registers User-Agent fallback probes. + +The HTTP metadata path may only offer a P2 task through bounded in-memory work. +SQLite lookup, MediaVault resolution, ffprobe, persistence, and retry execution +belong to workers. Background synchronization identified by `-Library` plus +`skipRefresh` remains cache-only and cannot populate P2. Queue state is not +persisted and is discarded on cache reset, shutdown, or process restart. + +`/metrics` keeps aggregate scheduler counters and a fixed, label-free P0/P1/P2 +breakdown for offers, admissions, joins, promotions, capacity drops, expiry, and +fingerprint supersession. `/health` reports the current queue depth for each +class. Neither surface contains Part identities, URLs, credentials, or full +User-Agent values. `PLAYBACK_VETO_ENABLED=false` leaves a nil hook in the decision handler. The experimental hook is disabled by default. When enabled, it only consumes the diff --git a/docs/media-lifecycle-architecture.md b/docs/media-lifecycle-architecture.md index a081cca..d20f72a 100644 --- a/docs/media-lifecycle-architecture.md +++ b/docs/media-lifecycle-architecture.md @@ -498,16 +498,16 @@ Gateway 支持以下范围： | 范围 | 触发方式 | 约束 | | --- | --- | --- | | 单 Part | metadata miss/stale 或成功播放 | 自动、高优先级、去重 | -| 邻近媒体 | 当前项云端重定向就绪后的低优先预取 | 当前实现范围；默认前 2 后 3，可配置双向窗口与间隔 | +| 邻近媒体 | 当前项云端重定向就绪后的 P1 预取 | 当前实现范围；默认前 2 后 3，可配置双向窗口，远程启动统一限速 | | 单季 | 显式预热或计划任务 | 后续批量能力；分页、checkpoint | | 整剧 | 显式后台任务 | 后续批量能力；按季分页，不由一次播放自动扩散 | | STRM 目录 | 管理员扫描或 reconciliation | 后续批量能力；限制到配置 mapping root，可取消和恢复 | -当前实现不执行启动全库、单季、整剧或目录批量预热。当前项在 302 写出后立即以交互 -优先级提交；邻近项默认按后 3、前 2 的顺序，每隔 `5s` 进入低优先级队列。该信号只表示 -云端重定向就绪，不代表客户端已跟随 302 或真正起播。快速切换产生的新当前项优先于旧 -窗口中尚未提交的候选；已经提交的任务保留自身 PartID 与 STRM fingerprint，并由 -singleflight 去重。远程 probe 初始并发候选值为 `1`，经 CDN、CPU、Plex 和 MediaVault +当前实现不执行启动全库、单季、整剧或目录批量预热。当前项在 302 写出后立即投递 P0； +邻近项默认按后 3、前 2 的顺序准备并非阻塞投递 P1。只有 L1 和 SQLite 都未命中的 P1/P2 +才共享 `5s` 远程启动间隔。该信号只表示云端重定向就绪，不代表客户端已跟随 302 或真正 +起播。快速切换产生的新当前项优先于旧窗口中尚未投递的候选；已经投递的任务保留自身 +PartID 与 STRM fingerprint，并由 singleflight 去重。远程 probe 初始并发候选值为 `1`，经 CDN、CPU、Plex 和 MediaVault 压测后再决定是否提高。每次播放可以触发 freshness 判断，但不等于每次执行完整 ffprobe； 未过期记录只更新访问时间，过期记录优先异步校验，可靠指纹变化才重新探测。 diff --git a/docs/performance-matrix.md b/docs/performance-matrix.md index 038482b..7c2ff68 100644 --- a/docs/performance-matrix.md +++ b/docs/performance-matrix.md @@ -16,12 +16,14 @@ - 一个 universal decision 最多等待一次 `MEDIAINFO_COLD_WAIT`。veto 只复用响应增强已经 取得的新鲜记录，不增加缓存读取或远程探测。 - 同一 MediaInfo key 的并发请求必须共享 singleflight。并发数不能突破配置的 worker - 上限；当前项只能优先排队，不能抢占正在运行的后台 probe。 + 上限；P0 实际播放优先于 P1/P2 排队任务，但不能抢占正在运行的后台 probe。 +- P1/P2 在 worker 中先检查 SQLite。只有 L1 和 SQLite 都 miss 的任务才受全局远程启动 + 间隔限制；SQLite 热命中不能因 CDN 风控节流额外等待 5 秒。 - 302 响应只包含控制流。媒体字节始终由客户端直接从 CDN 读取。 ## 当前基准 -基准日期为 2026-08-29。代码运行于 `darwin/arm64`、Apple M3 Max、Go 1.26.5。 +基准采集于 2026-08-29 至 2026-08-30。代码运行于 `darwin/arm64`、Apple M3 Max、Go 1.26.5。 每项运行三到五轮；下表给出观测范围。工作站数据只用于发现代码回退，换架构或 Go 版本后必须重新采集。 @@ -29,6 +31,8 @@ | --- | ---: | ---: | --- | | MediaInfo L1 cache `Get` | 469.0-476.0 ns/op | 960 B, 7 allocs | 纯缓存结构基准。 | | MediaInfo service `GetMemory` | 1.356-1.423 us/op | 2880 B, 21 allocs | 包含记录防御性复制、兼容性校验和访问续期判断。 | +| MediaInfo service fresh `Offer` | 1.410-1.453 us/op | 2912 B, 22 allocs | P0/P1/P2 非阻塞投递前的 L1 热命中路径。 | +| MediaInfo service exact-key join `Offer` | 521.0-550.1 ns/op | 512 B, 7 allocs | 已有 flight 的内存去重，不读取 SQLite。 | | STRM target fingerprint | 382.0-387.4 ns/op | 448 B, 6 allocs | MediaVault redirect URL 规范化和 SHA-256。 | | SQLite 单记录 `Get` | 14.60-15.37 us/op | 4016 B, 74 allocs | 独立本机 SQLite 读取；生产启动会先恢复到 L1。 | | 本机 HTTP 合成媒体 ffprobe | 22.48-33.36 ms/op | 约 119-125 KiB, 232-234 allocs | 只验证 worker 代码，不代表 NAS/CDN。 | @@ -56,7 +60,8 @@ | Apple TV DV5，veto 开启 | Plex metadata、decision 和既有 MediaInfo 流程 | reject 延迟、grant、额外 I/O | 新鲜完整记录命中时 reject 且不创建 grant；相对 veto 关闭增加 0 次缓存、MV、CDN 请求。 | | 同一 Part 并发 2/4/8/16 | 一个共享 probe | 唯一 probe 数、延迟、队列、active | 每个 key 最多 1 次 probe，无重复 CDN 读取。 | | 不同 Part 并发 2/4/8 | 受 worker 上限约束 | active、队列、CPU、内存、错误率 | 不突破 `MEDIAINFO_CONCURRENCY`，不拖慢本地或 302 路径。 | -| A/B/C 快速切换 | 当前项交互 probe、后台邻近预热 | 每项延迟、身份对应、替换/队列指标 | 新当前项优先排队；不能抢占正在运行的 probe；无跨项 MediaInfo、grant 或 redirect。 | +| A/B/C 快速切换 | 当前项 P0 probe、P1 邻近预热 | 每项延迟、身份对应、替换/队列指标 | 新当前项优先排队；不能抢占正在运行的 probe；无跨项 MediaInfo、grant 或 redirect。 | +| 长剧集 metadata 突发 | Plex、L1；P2 后台可查 SQLite/探测 | 前台响应、P2 队列/过期/远程启动次数 | 前台不等待 P2；唯一 pending P2 <= 50；远程启动间隔 >= 5 s；过期任务 0 MV/CDN。 | 表中的验收条件是候选版本的门槛，不代表本机微基准已经替代端到端验收。当前已有证据 包括本机代码基准和 5/5 成功的 MediaVault/CDN 探测抽样；本地透明代理、302 热冷路径、 @@ -69,7 +74,7 @@ ```sh go test -run '^$' \ - -bench 'Benchmark(MediaInfoL1Get|MediaInfoServiceGetMemory|FingerprintSTRMTarget|MediaInfoSQLiteGet|MediaInfoHTTPFFProbe)$' \ + -bench 'Benchmark(MediaInfoL1Get|MediaInfoServiceGetMemory|MediaInfoServiceOfferFresh|MediaInfoServiceOfferJoin|FingerprintSTRMTarget|MediaInfoSQLiteGet|MediaInfoHTTPFFProbe)$' \ -benchmem -count=5 ./internal/mediainfo ``` diff --git a/docs/roadmap.md b/docs/roadmap.md index 61c9632..22ceff3 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -78,11 +78,11 @@ interactive probe; missing, slow, or failed probes preserve the original Plex response. Part and universal-start redirects never wait for MediaInfo. All clients use the same projection rules. An L1 hit does not synchronously read SQLite for the request, although access renewal may touch SQLite asynchronously. -One metadata browsing burst permits one synchronous cold probe. Each admitted -the admitted synchronous wait releases its slot, a fixed five-second window -begins. Cold misses rejected inside that window fail open without renewing it, -while an admitted probe may finish in the background and write L1 and SQLite -after the request waiter leaves. +One metadata browsing burst permits one synchronous cold probe. After the +admitted synchronous wait releases its slot, a fixed five-second window begins. +Cold misses inside that window fail open immediately and may offer bounded P2 +work without renewing it. Background work writes L1 and SQLite only after a +successful probe. A default-disabled experimental veto may reuse an already available fresh record for the Apple TV Plex DV Profile 5 combination, but it performs no additional probe and owns no Part state. @@ -95,10 +95,10 @@ D prioritizes the Gateway fallback while preserving the broader lifecycle design nearby-item discovery; current-item prewarming remains token-independent; 3. use bounded remote `ffprobe` with a default `5s` cold metadata wait ceiling; 4. support exact current-Part analysis and a configurable nearby-item window - after a cloud redirect is ready, with current-item priority in the queue and - rate-limited background neighbors, without blocking decision, Part, - universal start, or 302. Queue priority cannot preempt a probe already - running; + after a cloud redirect is ready. P0 actual playback, P1 nearby items, and P2 + foreground metadata misses share one exact-key scheduler. P1/P2 remote starts + are rate-limited without blocking Part, universal start, or 302. Queue + priority cannot preempt a probe already running; defer season, show, and configured STRM-root batch tasks; 5. enrich authorized Plex metadata responses first. This release writes fallback records only to Gateway SQLite. Production Plex DB writing is a next-stage @@ -107,8 +107,8 @@ D prioritizes the Gateway fallback while preserving the broader lifecycle design any path is selected. Parts without Plex Streams may receive descriptive ffprobe-indexed Streams, while existing Stream sets are never expanded and Plex IDs or playback-selection state are never synthesized. Background - `-Library` synchronization remains cache-only, and native-client metadata - fan-out cannot schedule a full-library cold scan; + `-Library` synchronization remains cache-only. Foreground fan-out can only + offer bounded, expiring P2 work and never waits for that background queue; 6. add Redis or replace SQLite with PostgreSQL only after measured multi-instance or capacity requirements exist. diff --git a/internal/config/config.go b/internal/config/config.go index c9b74fc..4fb6d02 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -32,7 +32,11 @@ const ( defaultMediaInfoAnalyze = 5 * time.Second defaultMediaInfoOutput = 2 << 20 defaultMediaInfoWorkers = 1 - defaultMediaInfoQueue = 256 + defaultPlaybackQueue = 16 + defaultNeighborQueue = 50 + defaultMetadataQueue = 50 + defaultPendingTTL = 5 * time.Minute + defaultBackgroundInterval = 5 * time.Second defaultMediaInfoTTL = 30 * 24 * time.Hour defaultMediaInfoRetention = 180 * 24 * time.Hour defaultMediaInfoL1Entries = 10_000 @@ -43,7 +47,6 @@ const ( defaultMediaInfoWaiters = 8 defaultPrewarmBefore = 2 defaultPrewarmAfter = 3 - defaultPrewarmInterval = 5 * time.Second maxPrewarmWindow = 50 ) @@ -76,28 +79,30 @@ type Config struct { // MediaInfoConfig bounds the optional analysis plane. It is enabled by default // but capability initialization remains fail-open for transparent Plex proxying. type MediaInfoConfig struct { - Enabled bool - FFProbePath string - ProbeTimeout time.Duration - ProbeSize int64 - AnalyzeDuration time.Duration - OutputMaxBytes int64 - Concurrency int - InteractiveQueueSize int - BackgroundQueueSize int - RecordTTL time.Duration - RecordRetention time.Duration - L1MaxEntries int - NegativeTTL time.Duration - UserAgent string - ColdWait time.Duration - ResponseMaxBytes int64 - EnrichmentWaiters int + Enabled bool + FFProbePath string + ProbeTimeout time.Duration + ProbeSize int64 + AnalyzeDuration time.Duration + OutputMaxBytes int64 + Concurrency int + PlaybackQueueSize int + NeighborQueueSize int + MetadataQueueSize int + PendingTTL time.Duration + BackgroundInterval time.Duration + RecordTTL time.Duration + RecordRetention time.Duration + L1MaxEntries int + NegativeTTL time.Duration + UserAgent string + ColdWait time.Duration + ResponseMaxBytes int64 + EnrichmentWaiters int // PrewarmBefore and PrewarmAfter bound speculative neighbors around a - // cloud redirect. PrewarmInterval spaces their background submissions. - PrewarmBefore int - PrewarmAfter int - PrewarmInterval time.Duration + // cloud redirect. Remote starts remain governed by BackgroundInterval. + PrewarmBefore int + PrewarmAfter int } // MetadataGuardConfig bounds detailed metadata fan-out before requests enter @@ -236,11 +241,23 @@ func Load() (Config, error) { if err != nil { return Config{}, err } - mediaInfoInteractiveQueue, err := envPositiveInt("MEDIAINFO_INTERACTIVE_QUEUE_SIZE", defaultMediaInfoQueue) + mediaInfoPlaybackQueue, err := envPositiveInt("MEDIAINFO_PLAYBACK_QUEUE_SIZE", defaultPlaybackQueue) + if err != nil { + return Config{}, err + } + mediaInfoNeighborQueue, err := envPositiveInt("MEDIAINFO_NEIGHBOR_QUEUE_SIZE", defaultNeighborQueue) + if err != nil { + return Config{}, err + } + mediaInfoMetadataQueue, err := envPositiveInt("MEDIAINFO_METADATA_QUEUE_SIZE", defaultMetadataQueue) if err != nil { return Config{}, err } - mediaInfoBackgroundQueue, err := envPositiveInt("MEDIAINFO_BACKGROUND_QUEUE_SIZE", defaultMediaInfoQueue) + mediaInfoPendingTTL, err := envDuration("MEDIAINFO_PENDING_TTL", defaultPendingTTL) + if err != nil { + return Config{}, err + } + mediaInfoBackgroundInterval, err := envDuration("MEDIAINFO_BACKGROUND_INTERVAL", defaultBackgroundInterval) if err != nil { return Config{}, err } @@ -293,10 +310,6 @@ func Load() (Config, error) { if mediaInfoPrewarmBefore+mediaInfoPrewarmAfter > maxPrewarmWindow { return Config{}, fmt.Errorf("MEDIAINFO_PREWARM_BEFORE and MEDIAINFO_PREWARM_AFTER must total at most %d", maxPrewarmWindow) } - mediaInfoPrewarmInterval, err := envDuration("MEDIAINFO_PREWARM_INTERVAL", defaultPrewarmInterval) - if err != nil { - return Config{}, err - } playbackVeto, err := envBool("PLAYBACK_VETO_ENABLED", false) if err != nil { return Config{}, err @@ -351,13 +364,13 @@ func Load() (Config, error) { Enabled: mediaInfoEnabled, FFProbePath: mediaInfoFFProbe, ProbeTimeout: mediaInfoProbeTimeout, ProbeSize: mediaInfoProbeSize, AnalyzeDuration: mediaInfoAnalyze, OutputMaxBytes: mediaInfoOutput, - Concurrency: mediaInfoWorkers, InteractiveQueueSize: mediaInfoInteractiveQueue, - BackgroundQueueSize: mediaInfoBackgroundQueue, - RecordTTL: mediaInfoTTL, RecordRetention: mediaInfoRetention, L1MaxEntries: mediaInfoL1Entries, + Concurrency: mediaInfoWorkers, PlaybackQueueSize: mediaInfoPlaybackQueue, + NeighborQueueSize: mediaInfoNeighborQueue, MetadataQueueSize: mediaInfoMetadataQueue, + PendingTTL: mediaInfoPendingTTL, BackgroundInterval: mediaInfoBackgroundInterval, + RecordTTL: mediaInfoTTL, RecordRetention: mediaInfoRetention, L1MaxEntries: mediaInfoL1Entries, NegativeTTL: mediaInfoNegativeTTL, UserAgent: mediaInfoUserAgent, ColdWait: mediaInfoColdWait, ResponseMaxBytes: mediaInfoBodyLimit, EnrichmentWaiters: mediaInfoWaiters, PrewarmBefore: mediaInfoPrewarmBefore, PrewarmAfter: mediaInfoPrewarmAfter, - PrewarmInterval: mediaInfoPrewarmInterval, }, PlaybackVeto: playbackVeto, LogLevel: logLevel, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4b80f12..e200525 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -47,7 +47,12 @@ func TestLoadDefaults(t *testing.T) { if got.DatabasePath != "./data/plex-gateway.db" { t.Fatalf("DatabasePath = %q", got.DatabasePath) } - if !got.MediaInfo.Enabled || got.MediaInfo.ProbeTimeout != 20*time.Second || got.MediaInfo.Concurrency != 1 || got.MediaInfo.UserAgent != "Infuse-Library/8.5.1" || got.MediaInfo.ColdWait != 5*time.Second || got.MediaInfo.ResponseMaxBytes != 8<<20 || got.MediaInfo.EnrichmentWaiters != 8 || got.MediaInfo.PrewarmBefore != 2 || got.MediaInfo.PrewarmAfter != 3 || got.MediaInfo.PrewarmInterval != 5*time.Second { + if !got.MediaInfo.Enabled || got.MediaInfo.ProbeTimeout != 20*time.Second || got.MediaInfo.Concurrency != 1 || + got.MediaInfo.PlaybackQueueSize != 16 || got.MediaInfo.NeighborQueueSize != 50 || + got.MediaInfo.MetadataQueueSize != 50 || got.MediaInfo.PendingTTL != 5*time.Minute || + got.MediaInfo.BackgroundInterval != 5*time.Second || got.MediaInfo.UserAgent != "Infuse-Library/8.5.1" || + got.MediaInfo.ColdWait != 5*time.Second || got.MediaInfo.ResponseMaxBytes != 8<<20 || + got.MediaInfo.EnrichmentWaiters != 8 || got.MediaInfo.PrewarmBefore != 2 || got.MediaInfo.PrewarmAfter != 3 { t.Fatalf("unexpected MediaInfo defaults: %#v", got.MediaInfo) } if got.PlaybackVeto { @@ -91,8 +96,11 @@ func TestLoadCloudConfiguration(t *testing.T) { t.Setenv("MEDIAINFO_ANALYZE_DURATION", "3s") t.Setenv("MEDIAINFO_OUTPUT_MAX_BYTES", "1048576") t.Setenv("MEDIAINFO_CONCURRENCY", "2") - t.Setenv("MEDIAINFO_INTERACTIVE_QUEUE_SIZE", "64") - t.Setenv("MEDIAINFO_BACKGROUND_QUEUE_SIZE", "128") + t.Setenv("MEDIAINFO_PLAYBACK_QUEUE_SIZE", "12") + t.Setenv("MEDIAINFO_NEIGHBOR_QUEUE_SIZE", "64") + t.Setenv("MEDIAINFO_METADATA_QUEUE_SIZE", "128") + t.Setenv("MEDIAINFO_PENDING_TTL", "7m") + t.Setenv("MEDIAINFO_BACKGROUND_INTERVAL", "9s") t.Setenv("MEDIAINFO_RECORD_TTL", "168h") t.Setenv("MEDIAINFO_RECORD_RETENTION", "720h") t.Setenv("MEDIAINFO_L1_MAX_ENTRIES", "5000") @@ -103,7 +111,6 @@ func TestLoadCloudConfiguration(t *testing.T) { t.Setenv("MEDIAINFO_ENRICHMENT_CONCURRENCY", "3") t.Setenv("MEDIAINFO_PREWARM_BEFORE", "4") t.Setenv("MEDIAINFO_PREWARM_AFTER", "6") - t.Setenv("MEDIAINFO_PREWARM_INTERVAL", "9s") got, err := Load() if err != nil { @@ -127,7 +134,16 @@ func TestLoadCloudConfiguration(t *testing.T) { if got.DatabasePath != "/app_data/test.db" { t.Fatalf("DatabasePath = %q", got.DatabasePath) } - if !got.MediaInfo.Enabled || got.MediaInfo.FFProbePath != "/usr/bin/ffprobe" || got.MediaInfo.ProbeTimeout != 12*time.Second || got.MediaInfo.ProbeSize != 4194304 || got.MediaInfo.AnalyzeDuration != 3*time.Second || got.MediaInfo.OutputMaxBytes != 1048576 || got.MediaInfo.Concurrency != 2 || got.MediaInfo.InteractiveQueueSize != 64 || got.MediaInfo.BackgroundQueueSize != 128 || got.MediaInfo.RecordTTL != 168*time.Hour || got.MediaInfo.RecordRetention != 720*time.Hour || got.MediaInfo.L1MaxEntries != 5000 || got.MediaInfo.NegativeTTL != 10*time.Minute || got.MediaInfo.UserAgent != "plex-gateway-test" || got.MediaInfo.ColdWait != 4*time.Second || got.MediaInfo.ResponseMaxBytes != 2097152 || got.MediaInfo.EnrichmentWaiters != 3 || got.MediaInfo.PrewarmBefore != 4 || got.MediaInfo.PrewarmAfter != 6 || got.MediaInfo.PrewarmInterval != 9*time.Second { + if !got.MediaInfo.Enabled || got.MediaInfo.FFProbePath != "/usr/bin/ffprobe" || got.MediaInfo.ProbeTimeout != 12*time.Second || + got.MediaInfo.ProbeSize != 4194304 || got.MediaInfo.AnalyzeDuration != 3*time.Second || + got.MediaInfo.OutputMaxBytes != 1048576 || got.MediaInfo.Concurrency != 2 || + got.MediaInfo.PlaybackQueueSize != 12 || got.MediaInfo.NeighborQueueSize != 64 || + got.MediaInfo.MetadataQueueSize != 128 || got.MediaInfo.PendingTTL != 7*time.Minute || + got.MediaInfo.BackgroundInterval != 9*time.Second || got.MediaInfo.RecordTTL != 168*time.Hour || + got.MediaInfo.RecordRetention != 720*time.Hour || got.MediaInfo.L1MaxEntries != 5000 || + got.MediaInfo.NegativeTTL != 10*time.Minute || got.MediaInfo.UserAgent != "plex-gateway-test" || + got.MediaInfo.ColdWait != 4*time.Second || got.MediaInfo.ResponseMaxBytes != 2097152 || + got.MediaInfo.EnrichmentWaiters != 3 || got.MediaInfo.PrewarmBefore != 4 || got.MediaInfo.PrewarmAfter != 6 { t.Fatalf("MediaInfo configuration = %#v", got.MediaInfo) } } diff --git a/internal/gateway/decision.go b/internal/gateway/decision.go index dd99157..b53c6d2 100644 --- a/internal/gateway/decision.go +++ b/internal/gateway/decision.go @@ -135,7 +135,7 @@ func (h *decisionHandler) mediaInfoForDecision(request *http.Request, part playb defer cancel() record, err := h.mediaInfo.Ensure(ctx, mediainfo.Request{ Key: key, RatingKey: part.RatingKey, Target: part.Target, - Priority: mediainfo.PriorityInteractive, ClientUserAgent: request.UserAgent(), + Priority: mediainfo.PriorityPlayback, ClientUserAgent: request.UserAgent(), }) return record, err == nil && record.Media.Complete } diff --git a/internal/gateway/metadata_enrichment.go b/internal/gateway/metadata_enrichment.go index 270af12..627d25d 100644 --- a/internal/gateway/metadata_enrichment.go +++ b/internal/gateway/metadata_enrichment.go @@ -29,8 +29,10 @@ const ( ) type mediaInfoEnsurer interface { + GetMemory(mediainfo.Key) (mediainfo.Record, bool) GetContext(context.Context, mediainfo.Key) (mediainfo.Record, bool) Ensure(context.Context, mediainfo.Request) (mediainfo.Record, error) + Offer(mediainfo.Request) mediainfo.SubmitResult } // metadataEnrichmentOptions keeps analysis collaborators outside the playback @@ -152,28 +154,36 @@ func (handler *metadataEnrichmentHandler) ServeHTTP(w http.ResponseWriter, reque return } key := mediainfo.Key{PartID: target.Part.ID, STRMFingerprint: fingerprint} - lookupContext, cancel := context.WithTimeout(request.Context(), defaultMediaInfoCacheLookupWait) - record, found := handler.service.GetContext(lookupContext, key) - cancel() + record, found := handler.service.GetMemory(key) if !found { cacheOnly := mediaInfoCacheOnlyRequest(request) - if cacheOnly || !handler.acquireColdProbe() { - if !cacheOnly { + if cacheOnly { + lookupContext, cancel := context.WithTimeout(request.Context(), defaultMediaInfoCacheLookupWait) + record, found = handler.service.GetContext(lookupContext, key) + cancel() + if !found { + handler.replay(capture, rawBody) + return + } + } else { + probeRequest := mediainfo.Request{ + Key: key, RatingKey: ratingKey, Target: strmTarget, + Priority: mediainfo.PriorityMetadata, ClientUserAgent: request.UserAgent(), + } + if !handler.acquireColdProbe() { handler.metrics.IncMediaInfoWaitRejected() + _ = handler.service.Offer(probeRequest) + handler.replay(capture, rawBody) + return + } + defer handler.releaseColdProbe() + waitContext, cancel := context.WithTimeout(request.Context(), handler.coldWait) + record, err = handler.service.Ensure(waitContext, probeRequest) + cancel() + if err != nil { + handler.failOpen(capture, rawBody) + return } - handler.replay(capture, rawBody) - return - } - defer handler.releaseColdProbe() - waitContext, cancel := context.WithTimeout(request.Context(), handler.coldWait) - record, err = handler.service.Ensure(waitContext, mediainfo.Request{ - Key: key, RatingKey: ratingKey, Target: strmTarget, - Priority: mediainfo.PriorityInteractive, ClientUserAgent: request.UserAgent(), - }) - cancel() - if err != nil { - handler.failOpen(capture, rawBody) - return } } currentTarget, err := handler.resolver.ReadTarget(localPath) diff --git a/internal/gateway/metadata_enrichment_test.go b/internal/gateway/metadata_enrichment_test.go index ccfcfb1..efba7fb 100644 --- a/internal/gateway/metadata_enrichment_test.go +++ b/internal/gateway/metadata_enrichment_test.go @@ -27,13 +27,24 @@ import ( type fakeMediaInfoEnsurer struct { calls atomic.Int64 gets atomic.Int64 + memoryGets atomic.Int64 mu sync.Mutex seen []mediainfo.Request + offered []mediainfo.Request + memoryFn func(mediainfo.Key) (mediainfo.Record, bool) getFn func(mediainfo.Key) (mediainfo.Record, bool) getContextFn func(context.Context, mediainfo.Key) (mediainfo.Record, bool) fn func(context.Context, mediainfo.Request) (mediainfo.Record, error) } +func (ensurer *fakeMediaInfoEnsurer) GetMemory(key mediainfo.Key) (mediainfo.Record, bool) { + ensurer.memoryGets.Add(1) + if ensurer.memoryFn == nil { + return mediainfo.Record{}, false + } + return ensurer.memoryFn(key) +} + func (ensurer *fakeMediaInfoEnsurer) GetContext(ctx context.Context, key mediainfo.Key) (mediainfo.Record, bool) { ensurer.gets.Add(1) if ensurer.getContextFn != nil { @@ -79,12 +90,25 @@ func (ensurer *fakeMediaInfoEnsurer) Ensure(ctx context.Context, request mediain return ensurer.fn(ctx, request) } +func (ensurer *fakeMediaInfoEnsurer) Offer(request mediainfo.Request) mediainfo.SubmitResult { + ensurer.mu.Lock() + ensurer.offered = append(ensurer.offered, request) + ensurer.mu.Unlock() + return mediainfo.SubmitResult{Disposition: mediainfo.SubmitNewlyQueued} +} + func (ensurer *fakeMediaInfoEnsurer) requests() []mediainfo.Request { ensurer.mu.Lock() defer ensurer.mu.Unlock() return append([]mediainfo.Request(nil), ensurer.seen...) } +func (ensurer *fakeMediaInfoEnsurer) offers() []mediainfo.Request { + ensurer.mu.Lock() + defer ensurer.mu.Unlock() + return append([]mediainfo.Request(nil), ensurer.offered...) +} + func TestMetadataEnrichmentAddsMissingXMLFields(t *testing.T) { fixture := newMetadataEnrichmentFixture(t) raw := []byte(``) @@ -396,6 +420,9 @@ func TestMetadataEnrichmentColdProbeSaturationFailsOpen(t *testing.T) { if !bytes.Equal(second.Body.Bytes(), raw) || ensurer.calls.Load() != 1 { t.Fatalf("second response=%s Ensure calls=%d", second.Body.Bytes(), ensurer.calls.Load()) } + if offered := ensurer.offers(); len(offered) != 1 || offered[0].Priority != mediainfo.PriorityMetadata { + t.Fatalf("background offers = %#v", offered) + } close(release) select { case first := <-firstDone: @@ -433,6 +460,9 @@ func TestMetadataEnrichmentColdProbeQuietWindowSuppressesSequentialMiss(t *testi if !bytes.Equal(second.Body.Bytes(), raw) || ensurer.calls.Load() != 1 { t.Fatalf("second response=%s Ensure calls=%d", second.Body.Bytes(), ensurer.calls.Load()) } + if offered := ensurer.offers(); len(offered) != 1 || offered[0].Priority != mediainfo.PriorityMetadata { + t.Fatalf("quiet-window offers = %#v", offered) + } } func TestMetadataEnrichmentColdProbeQuietWindowDoesNotSlideUnderContinuousTraffic(t *testing.T) { @@ -542,7 +572,7 @@ func TestMetadataEnrichmentCachedProjectionBypassesColdProbeSlot(t *testing.T) { release := make(chan struct{}) var lookups atomic.Int64 ensurer := &fakeMediaInfoEnsurer{ - getContextFn: func(_ context.Context, key mediainfo.Key) (mediainfo.Record, bool) { + memoryFn: func(key mediainfo.Key) (mediainfo.Record, bool) { if lookups.Add(1) == 1 { return mediainfo.Record{}, false } @@ -579,6 +609,9 @@ func TestMetadataEnrichmentCachedProjectionBypassesColdProbeSlot(t *testing.T) { if ensurer.calls.Load() != 1 { t.Fatalf("Ensure calls = %d", ensurer.calls.Load()) } + if offered := ensurer.offers(); len(offered) != 0 { + t.Fatalf("background offers after cache hit = %d", len(offered)) + } close(release) select { case <-firstDone: @@ -650,6 +683,12 @@ func TestMetadataEnrichmentBurstDoesNotQueueBehindColdProbe(t *testing.T) { if ensurer.calls.Load() != 1 { t.Fatalf("Ensure calls = %d", ensurer.calls.Load()) } + if offered := ensurer.offers(); len(offered) != burst { + t.Fatalf("background offers = %d, want %d", len(offered), burst) + } + if got := ensurer.gets.Load(); got != 0 { + t.Fatalf("quiet-window burst performed %d durable-capable lookups", got) + } if snapshot := registry.Snapshot(); snapshot.MetadataGuardTimeoutsTotal != 0 || snapshot.MediaInfoWaitRejectedTotal != burst { t.Fatalf("metrics = %#v", snapshot) } diff --git a/internal/mediainfo/benchmark_test.go b/internal/mediainfo/benchmark_test.go index c4924fd..c11298b 100644 --- a/internal/mediainfo/benchmark_test.go +++ b/internal/mediainfo/benchmark_test.go @@ -51,6 +51,68 @@ func BenchmarkMediaInfoServiceGetMemory(b *testing.B) { } } +func BenchmarkMediaInfoServiceOfferFresh(b *testing.B) { + now := time.Now().UTC() + record := completeRecord(now) + request := Request{ + Key: record.Key, RatingKey: record.RatingKey, Target: "https://control.example.test/fresh", + Priority: PriorityPlayback, ClientUserAgent: "Infuse-Library/8.5.1", + } + service, err := NewService(ServiceOptions{ + Cache: NewCache([]Record{record}, now), Store: &fakeRecordStore{}, + Provider: &fakeProvider{}, PlexServerID: record.Key.PlexServerID, + BackgroundUserAgent: "Infuse-Library/8.5.1", Now: func() time.Time { return now }, + }) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := service.Close(ctx); err != nil { + b.Error(err) + } + }) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if result := service.Offer(request); result.Disposition != SubmitFreshCache || result.Err != nil { + b.Fatalf("Offer() = %#v", result) + } + } +} + +func BenchmarkMediaInfoServiceOfferJoin(b *testing.B) { + prober := &blockingProber{started: make(chan string, 1), release: make(chan struct{})} + service, err := NewService(ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, + Provider: &fakeProvider{prober: prober}, PlexServerID: "server", + BackgroundUserAgent: "Infuse-Library/8.5.1", BackgroundInterval: time.Millisecond, + }) + if err != nil { + b.Fatal(err) + } + request := testRequest("benchmark-join", PriorityPlayback) + if result := service.Offer(request); result.Err != nil { + b.Fatal(result.Err) + } + <-prober.started + b.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := service.Close(ctx); err != nil { + b.Error(err) + } + }) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if result := service.Offer(request); result.Disposition != SubmitJoinedExistingFlight || result.Err != nil { + b.Fatalf("Offer() = %#v", result) + } + } +} + func BenchmarkFingerprintSTRMTarget(b *testing.B) { const target = "http://mediavault:7811/redirect/pickcode/movie.mkv" b.ReportAllocs() diff --git a/internal/mediainfo/scheduler_contract_test.go b/internal/mediainfo/scheduler_contract_test.go new file mode 100644 index 0000000..6056bec --- /dev/null +++ b/internal/mediainfo/scheduler_contract_test.go @@ -0,0 +1,750 @@ +package mediainfo + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + gatewaymetrics "github.com/InfinityPacer/plex-gateway/internal/metrics" +) + +type atomicClock struct { + nanos atomic.Int64 +} + +type blockingGetStore struct { + *fakeRecordStore + started chan struct{} + release chan struct{} + once sync.Once +} + +func (store *blockingGetStore) Get(ctx context.Context, key Key) (Record, bool, error) { + store.once.Do(func() { close(store.started) }) + select { + case <-ctx.Done(): + return Record{}, false, ctx.Err() + case <-store.release: + return store.fakeRecordStore.Get(ctx, key) + } +} + +func newAtomicClock(now time.Time) *atomicClock { + clock := &atomicClock{} + clock.nanos.Store(now.UnixNano()) + return clock +} + +func (clock *atomicClock) Now() time.Time { + return time.Unix(0, clock.nanos.Load()).UTC() +} + +func (clock *atomicClock) Advance(duration time.Duration) { + clock.nanos.Add(duration.Nanoseconds()) +} + +func TestServiceOfferIsMemoryOnly(t *testing.T) { + store := &fakeRecordStore{} + prober := &blockingProber{started: make(chan string, 2), release: make(chan struct{}, 2)} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: store, + Provider: &fakeProvider{prober: prober}, PlexServerID: "server", + }) + + if result := service.Offer(testRequest("offer-blocker", PriorityPlayback)); result.Err != nil { + t.Fatal(result.Err) + } + <-prober.started + store.gets.Store(0) + result := service.Offer(testRequest("offer-memory-only", PriorityMetadata)) + if result.Disposition != SubmitNewlyQueued || result.Err != nil { + t.Fatalf("Offer() result = %#v", result) + } + if got := store.gets.Load(); got != 0 { + t.Fatalf("Offer() performed %d durable reads", got) + } + prober.release <- struct{}{} +} + +func TestServicePromotesP2ToP0InPlaceAndUsesPlaybackRequest(t *testing.T) { + prober := &blockingProber{started: make(chan string, 3), release: make(chan struct{}, 3)} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", Concurrency: 1, PlaybackQueueSize: 2, + NeighborQueueSize: 2, MetadataQueueSize: 2, BackgroundInterval: time.Millisecond, + }) + active := testRequest("promotion-blocker", PriorityPlayback) + if result := service.Offer(active); result.Err != nil { + t.Fatal(result.Err) + } + if got := <-prober.started; got != "https://cdn.example.test/promotion-blocker" { + t.Fatalf("active target = %q", got) + } + + metadata := testRequest("promotion-p2", PriorityMetadata) + metadata.ClientUserAgent = "metadata-agent" + if result := service.Offer(metadata); result.Disposition != SubmitNewlyQueued || result.Err != nil { + t.Fatalf("P2 offer = %#v", result) + } + playback := metadata + playback.Priority = PriorityPlayback + playback.RatingKey = "playback-rating" + playback.Target = "playback-target" + playback.ClientUserAgent = "playback-agent" + if result := service.Offer(playback); result.Disposition != SubmitJoinedExistingFlight || result.Err != nil { + t.Fatalf("P0 promotion = %#v", result) + } + status := service.Status() + if status.PlaybackQueued != 1 || status.MetadataQueued != 0 { + t.Fatalf("status after promotion = %#v", status) + } + + prober.release <- struct{}{} + if got := <-prober.started; got != "https://cdn.example.test/playback-target" { + t.Fatalf("promoted target = %q", got) + } + prober.release <- struct{}{} +} + +func TestServicePromotesP1ToP0WithoutAddingAFlight(t *testing.T) { + prober := &blockingProber{started: make(chan string, 3), release: make(chan struct{}, 3)} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", Concurrency: 1, PlaybackQueueSize: 1, + NeighborQueueSize: 2, MetadataQueueSize: 2, BackgroundInterval: time.Millisecond, + }) + if result := service.Offer(testRequest("p1-blocker", PriorityPlayback)); result.Err != nil { + t.Fatal(result.Err) + } + <-prober.started + neighbor := testRequest("promotion-p1", PriorityNeighbor) + if result := service.Offer(neighbor); result.Err != nil { + t.Fatal(result.Err) + } + if result := service.Offer(Request{ + Key: neighbor.Key, RatingKey: "new-rating", Target: "new-target", + Priority: PriorityPlayback, ClientUserAgent: "playback-agent", + }); result.Disposition != SubmitJoinedExistingFlight || result.Err != nil { + t.Fatalf("P1 promotion = %#v", result) + } + if got := service.Status().PlaybackQueued; got != 1 { + t.Fatalf("playback queue = %d, want 1", got) + } + prober.release <- struct{}{} + if got := <-prober.started; got != "https://cdn.example.test/new-target" { + t.Fatalf("promoted target = %q", got) + } + prober.release <- struct{}{} +} + +func TestServiceP0PromotionDuringPreflightDoesNotRetrySameUserAgent(t *testing.T) { + store := &blockingGetStore{ + fakeRecordStore: &fakeRecordStore{}, started: make(chan struct{}), release: make(chan struct{}), + } + started := make(chan string, 2) + var calls atomic.Int64 + probe := proberFunc(func(_ context.Context, _ string, userAgent string) (Media, error) { + calls.Add(1) + started <- userAgent + return Media{}, errors.New("probe rejected") + }) + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: store, Provider: &fakeProvider{prober: probe}, + PlexServerID: "server", Concurrency: 1, BackgroundInterval: time.Millisecond, + }) + request := testRequest("preflight-promotion", PriorityMetadata) + request.ClientUserAgent = "metadata-agent" + if result := service.Offer(request); result.Err != nil { + t.Fatal(result.Err) + } + select { + case <-store.started: + case <-time.After(time.Second): + t.Fatal("background preflight did not start") + } + request.Priority = PriorityPlayback + request.ClientUserAgent = "playback-agent" + if result := service.Offer(request); result.Disposition != SubmitJoinedExistingFlight || result.Err != nil { + t.Fatalf("P0 promotion = %#v", result) + } + close(store.release) + select { + case got := <-started: + if got != "playback-agent" { + t.Fatalf("probe User-Agent = %q", got) + } + case <-time.After(time.Second): + t.Fatal("promoted probe did not start") + } + waitFor(t, func() bool { return service.Status().ActiveProbes == 0 }) + select { + case got := <-started: + t.Fatalf("unexpected same-User-Agent retry with %q", got) + case <-time.After(50 * time.Millisecond): + } + if got := calls.Load(); got != 1 { + t.Fatalf("probe calls = %d, want 1", got) + } +} + +func TestServicePendingTTLExpiresBeforeProviderAndDoesNotNegativeCache(t *testing.T) { + clock := newAtomicClock(time.Unix(1_800_000_000, 0)) + prober := &blockingProber{started: make(chan string, 2), release: make(chan struct{}, 2)} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, clock.Now()), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", Concurrency: 1, PendingTTL: 5 * time.Minute, + BackgroundInterval: time.Millisecond, Now: clock.Now, + }) + if result := service.Offer(testRequest("ttl-blocker", PriorityPlayback)); result.Err != nil { + t.Fatal(result.Err) + } + <-prober.started + pending := testRequest("ttl-expired", PriorityMetadata) + flight, err := service.begin(pending) + if err != nil { + t.Fatal(err) + } + clock.Advance(6 * time.Minute) + prober.release <- struct{}{} + select { + case <-flight.done: + case <-time.After(time.Second): + t.Fatal("expired flight did not finish") + } + if !errors.Is(flight.err, ErrPendingExpired) { + t.Fatalf("expired flight error = %v", flight.err) + } + if got := service.provider.(*fakeProvider).calls.Load(); got != 1 { + t.Fatalf("provider calls = %d, want blocker only", got) + } + service.mu.Lock() + negative := len(service.negative) + service.mu.Unlock() + if negative != 0 { + t.Fatalf("negative entries after expiration = %d", negative) + } +} + +func TestServiceP0WakesWorkerWaitingForBackgroundInterval(t *testing.T) { + prober := &blockingProber{started: make(chan string, 4), release: make(chan struct{}, 4)} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", Concurrency: 1, BackgroundInterval: 200 * time.Millisecond, + }) + if result := service.Offer(testRequest("interval-first", PriorityNeighbor)); result.Err != nil { + t.Fatal(result.Err) + } + if got := <-prober.started; got != "https://cdn.example.test/interval-first" { + t.Fatalf("first target = %q", got) + } + if result := service.Offer(testRequest("interval-waiting", PriorityNeighbor)); result.Err != nil { + t.Fatal(result.Err) + } + prober.release <- struct{}{} + + select { + case got := <-prober.started: + t.Fatalf("background started early with %q", got) + case <-time.After(30 * time.Millisecond): + } + urgent := testRequest("interval-playback", PriorityPlayback) + if result := service.Offer(urgent); result.Err != nil { + t.Fatal(result.Err) + } + select { + case got := <-prober.started: + if got != "https://cdn.example.test/interval-playback" { + t.Fatalf("urgent target = %q", got) + } + case <-time.After(100 * time.Millisecond): + t.Fatal("P0 did not wake the worker") + } + prober.release <- struct{}{} + select { + case got := <-prober.started: + if got != "https://cdn.example.test/interval-waiting" { + t.Fatalf("waiting target = %q", got) + } + case <-time.After(time.Second): + t.Fatal("background job did not resume after interval") + } + prober.release <- struct{}{} +} + +func TestServiceP0AdmissionIsIndependentFromBackgroundCapacity(t *testing.T) { + prober := &blockingProber{started: make(chan string, 4), release: make(chan struct{}, 4)} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", Concurrency: 1, PlaybackQueueSize: 1, + NeighborQueueSize: 1, MetadataQueueSize: 1, BackgroundInterval: time.Hour, + }) + if result := service.Offer(testRequest("capacity-blocker", PriorityPlayback)); result.Err != nil { + t.Fatal(result.Err) + } + <-prober.started + if result := service.Offer(testRequest("capacity-neighbor", PriorityNeighbor)); result.Err != nil { + t.Fatal(result.Err) + } + if result := service.Offer(testRequest("capacity-metadata", PriorityMetadata)); result.Err != nil { + t.Fatal(result.Err) + } + if result := service.Offer(testRequest("capacity-playback", PriorityPlayback)); result.Err != nil { + t.Fatalf("P0 was rejected while background queues were full: %#v", result) + } + if got := service.Status().PlaybackQueued; got != 1 { + t.Fatalf("playback queue = %d, want 1", got) + } + prober.release <- struct{}{} +} + +func TestServiceActivePlaybackJoinKeepsLatestUserAgentForOneFallback(t *testing.T) { + started := make(chan string, 2) + releaseFailure := make(chan struct{}) + var calls atomic.Int64 + probe := proberFunc(func(ctx context.Context, _, userAgent string) (Media, error) { + call := calls.Add(1) + started <- userAgent + if call == 1 { + select { + case <-ctx.Done(): + return Media{}, ctx.Err() + case <-releaseFailure: + } + return Media{}, errors.New("playback User-Agent rejected") + } + return Media{ + Complete: true, Container: "mkv", DurationMS: 60_000, + Streams: []Stream{{Index: 0, Type: "video", Codec: "hevc", Width: 1920, Height: 1080}}, + }, nil + }) + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: probe}, + PlexServerID: "server", BackgroundInterval: time.Millisecond, + }) + first := testRequest("active-playback-fallback", PriorityPlayback) + first.ClientUserAgent = "playback-a" + if result := service.Offer(first); result.Err != nil { + t.Fatal(result.Err) + } + if got := <-started; got != "playback-a" { + t.Fatalf("first User-Agent = %q", got) + } + latest := first + latest.ClientUserAgent = "playback-b" + latest.Target = "latest-target" + if result := service.Offer(latest); result.Disposition != SubmitJoinedExistingFlight || result.Err != nil { + t.Fatalf("active join = %#v", result) + } + close(releaseFailure) + if got := <-started; got != "playback-b" { + t.Fatalf("fallback User-Agent = %q", got) + } + waitFor(t, func() bool { return calls.Load() == 2 && service.Status().ActiveProbes == 0 }) +} + +func TestServiceMetadataJoinDoesNotRegisterUserAgentFallback(t *testing.T) { + started := make(chan string, 1) + releaseFailure := make(chan struct{}) + var calls atomic.Int64 + probe := proberFunc(func(ctx context.Context, _, userAgent string) (Media, error) { + calls.Add(1) + started <- userAgent + select { + case <-ctx.Done(): + return Media{}, ctx.Err() + case <-releaseFailure: + return Media{}, errors.New("metadata probe rejected") + } + }) + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: probe}, + PlexServerID: "server", BackgroundInterval: time.Millisecond, + }) + first := testRequest("metadata-no-fallback", PriorityMetadata) + first.ClientUserAgent = "metadata-a" + if result := service.Offer(first); result.Err != nil { + t.Fatal(result.Err) + } + if got := <-started; got != "metadata-a" { + t.Fatalf("first User-Agent = %q", got) + } + joined := first + joined.ClientUserAgent = "metadata-b" + if result := service.Offer(joined); result.Disposition != SubmitJoinedExistingFlight || result.Err != nil { + t.Fatalf("metadata join = %#v", result) + } + close(releaseFailure) + waitFor(t, func() bool { return service.Status().ActiveProbes == 0 }) + if got := calls.Load(); got != 1 { + t.Fatalf("metadata probe calls = %d, want 1", got) + } +} + +func TestServiceWorkerSkipsFreshDurableRecord(t *testing.T) { + prober := &blockingProber{started: make(chan string, 2), release: make(chan struct{}, 2)} + store := &fakeRecordStore{} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: store, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", BackgroundInterval: time.Millisecond, + }) + blocker := testRequest("l2-blocker", PriorityPlayback) + if result := service.Offer(blocker); result.Err != nil { + t.Fatal(result.Err) + } + <-prober.started + pending := testRequest("l2-hit", PriorityMetadata) + now := time.Now().UTC() + record := completeRecord(now) + record.Key = pending.Key + record.RatingKey = pending.RatingKey + record.ExpiresAt = now.Add(time.Hour) + record.RetainUntil = now.Add(24 * time.Hour) + record.LastAccessedAt = now + store.mu.Lock() + store.records = map[string]Record{pending.Key.cacheKey(): record} + store.mu.Unlock() + flight, err := service.begin(pending) + if err != nil { + t.Fatal(err) + } + prober.release <- struct{}{} + select { + case <-flight.done: + case <-time.After(time.Second): + t.Fatal("L2 hit flight did not finish") + } + if flight.err != nil || prober.calls.Load() != 1 { + t.Fatalf("L2 hit err=%v provider_calls=%d", flight.err, prober.calls.Load()) + } +} + +func TestServiceFreshOfferRenewsAccessWithoutDurableRead(t *testing.T) { + createdAt := time.Unix(1_800_000_000, 0).UTC() + clock := newAtomicClock(createdAt.Add(2 * time.Hour)) + request := testRequest("offer-touch", PriorityPlayback) + record := completeRecord(createdAt) + record.Key = request.Key + record.RatingKey = request.RatingKey + store := &fakeRecordStore{records: map[string]Record{record.Key.cacheKey(): record}} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache([]Record{record}, createdAt), Store: store, + Provider: &fakeProvider{prober: &blockingProber{}}, PlexServerID: "server", + Now: clock.Now, TouchInterval: time.Hour, + }) + + result := service.Offer(request) + if result.Disposition != SubmitFreshCache || result.Err != nil { + t.Fatalf("Offer() = %#v", result) + } + if got := store.gets.Load(); got != 0 { + t.Fatalf("durable reads = %d", got) + } + waitFor(t, func() bool { return store.touches.Load() == 1 }) + got, found := service.cache.GetKnown(request.Key, clock.Now()) + if !found || !got.LastAccessedAt.Equal(clock.Now()) { + t.Fatalf("renewed record found=%v last_access=%s", found, got.LastAccessedAt) + } +} + +func TestServiceBackgroundColdMissReadsDurableStoreOnce(t *testing.T) { + prober := &blockingProber{started: make(chan string, 1), release: make(chan struct{}, 1)} + store := &fakeRecordStore{} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: store, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", BackgroundInterval: time.Millisecond, + }) + if result := service.Offer(testRequest("one-l2-read", PriorityNeighbor)); result.Err != nil { + t.Fatal(result.Err) + } + select { + case <-prober.started: + case <-time.After(time.Second): + t.Fatal("background provider did not start") + } + if got := store.gets.Load(); got != 1 { + t.Fatalf("durable reads = %d, want 1", got) + } + prober.release <- struct{}{} +} + +func TestServiceFreshDurableHitBypassesBackgroundInterval(t *testing.T) { + now := time.Now().UTC() + request := testRequest("l2-no-rate-limit", PriorityMetadata) + record := completeRecord(now) + record.Key = request.Key + record.RatingKey = request.RatingKey + store := &fakeRecordStore{records: map[string]Record{record.Key.cacheKey(): record}} + provider := &fakeProvider{prober: &blockingProber{}} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, now), Store: store, Provider: provider, + PlexServerID: "server", BackgroundInterval: time.Hour, + }) + flight, _, err := service.beginDetailed(request, false) + if err != nil { + t.Fatal(err) + } + select { + case <-flight.done: + case <-time.After(100 * time.Millisecond): + t.Fatal("fresh L2 hit waited for the remote background interval") + } + if flight.err != nil || provider.calls.Load() != 0 || store.gets.Load() != 1 { + t.Fatalf("flight_err=%v provider_calls=%d store_gets=%d", flight.err, provider.calls.Load(), store.gets.Load()) + } +} + +func TestServiceP0BurstWakesConfiguredWorkers(t *testing.T) { + prober := &blockingProber{started: make(chan string, 3), release: make(chan struct{}, 3)} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", Concurrency: 3, PlaybackQueueSize: 3, + }) + for _, partID := range []string{"burst-a", "burst-b", "burst-c"} { + if result := service.Offer(testRequest(partID, PriorityPlayback)); result.Err != nil { + t.Fatal(result.Err) + } + } + for range 3 { + select { + case <-prober.started: + case <-time.After(200 * time.Millisecond): + t.Fatal("P0 burst did not use all configured workers") + } + } + for range 3 { + prober.release <- struct{}{} + } +} + +func TestServiceP0PromotionFailsOpenWhenP0QueueIsFull(t *testing.T) { + prober := &blockingProber{started: make(chan string, 1), release: make(chan struct{}, 1)} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", Concurrency: 1, PlaybackQueueSize: 1, + NeighborQueueSize: 1, MetadataQueueSize: 1, BackgroundInterval: time.Hour, + }) + if result := service.Offer(testRequest("promotion-full-active", PriorityPlayback)); result.Err != nil { + t.Fatal(result.Err) + } + <-prober.started + if result := service.Offer(testRequest("promotion-full-p0", PriorityPlayback)); result.Err != nil { + t.Fatal(result.Err) + } + metadata := testRequest("promotion-full-target", PriorityMetadata) + if result := service.Offer(metadata); result.Err != nil { + t.Fatal(result.Err) + } + metadata.Priority = PriorityPlayback + result := service.Offer(metadata) + if result.Disposition != SubmitRejected || !errors.Is(result.Err, ErrQueueFull) { + t.Fatalf("promotion result = %#v", result) + } + if status := service.Status(); status.PlaybackQueued != 1 || status.MetadataQueued != 1 { + t.Fatalf("status after rejected promotion = %#v", status) + } + prober.release <- struct{}{} +} + +func TestServicePlaybackRetryChainAllowsOnlyOneFallback(t *testing.T) { + started := make(chan string, 3) + releases := make(chan struct{}, 2) + var calls atomic.Int64 + probe := proberFunc(func(ctx context.Context, _, userAgent string) (Media, error) { + calls.Add(1) + started <- userAgent + select { + case <-ctx.Done(): + return Media{}, ctx.Err() + case <-releases: + return Media{}, errors.New("User-Agent rejected") + } + }) + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, + Provider: &fakeProvider{prober: probe}, PlexServerID: "server", + }) + request := testRequest("retry-chain", PriorityPlayback) + request.ClientUserAgent = "playback-a" + if result := service.Offer(request); result.Err != nil { + t.Fatal(result.Err) + } + <-started + request.ClientUserAgent = "playback-b" + if result := service.Offer(request); result.Err != nil { + t.Fatal(result.Err) + } + releases <- struct{}{} + if got := <-started; got != "playback-b" { + t.Fatalf("fallback User-Agent = %q", got) + } + request.ClientUserAgent = "playback-c" + if result := service.Offer(request); result.Err != nil { + t.Fatal(result.Err) + } + releases <- struct{}{} + waitFor(t, func() bool { return service.Status().ActiveProbes == 0 }) + select { + case got := <-started: + t.Fatalf("unexpected third retry with User-Agent %q", got) + case <-time.After(50 * time.Millisecond): + } + if got := calls.Load(); got != 2 { + t.Fatalf("probe calls = %d, want 2", got) + } +} + +func TestServiceCountsRetryDroppedByFullPriorityQueue(t *testing.T) { + started := make(chan string, 2) + releaseFirst := make(chan struct{}) + var calls atomic.Int64 + probe := proberFunc(func(ctx context.Context, target, _ string) (Media, error) { + if calls.Add(1) == 1 { + started <- target + select { + case <-ctx.Done(): + return Media{}, ctx.Err() + case <-releaseFirst: + return Media{}, errors.New("first User-Agent rejected") + } + } + started <- target + return Media{ + Complete: true, Container: "mkv", DurationMS: 60_000, + Streams: []Stream{{Index: 0, Type: "video", Codec: "hevc", Width: 1920, Height: 1080}}, + }, nil + }) + registry := gatewaymetrics.New() + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, + Provider: &fakeProvider{prober: probe}, PlexServerID: "server", + Concurrency: 1, PlaybackQueueSize: 1, Metrics: registry, + }) + request := testRequest("retry-full", PriorityPlayback) + request.ClientUserAgent = "playback-a" + if result := service.Offer(request); result.Err != nil { + t.Fatal(result.Err) + } + <-started + request.ClientUserAgent = "playback-b" + if result := service.Offer(request); result.Err != nil { + t.Fatal(result.Err) + } + if result := service.Offer(testRequest("retry-full-blocker", PriorityPlayback)); result.Err != nil { + t.Fatal(result.Err) + } + close(releaseFirst) + waitFor(t, func() bool { + return registry.Snapshot().MediaInfoProbeDroppedFullTotal == 1 + }) + if got := registry.Snapshot().MediaInfoProbeDroppedFullTotal; got != 1 { + t.Fatalf("dropped full = %d, want 1", got) + } +} + +func TestServiceSupersedesQueuedBackgroundWithNewFingerprint(t *testing.T) { + for _, priority := range []Priority{PriorityNeighbor, PriorityMetadata} { + t.Run(priorityName(priority), func(t *testing.T) { + prober := &blockingProber{started: make(chan string, 3), release: make(chan struct{}, 3)} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", Concurrency: 1, NeighborQueueSize: 4, + MetadataQueueSize: 4, BackgroundInterval: time.Millisecond, + }) + if result := service.Offer(testRequest("fingerprint-blocker", PriorityPlayback)); result.Err != nil { + t.Fatal(result.Err) + } + <-prober.started + old := testRequest("fingerprint-same-part", priority) + old.Key.STRMFingerprint = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + old.Target = "old-target" + oldFlight, err := service.begin(old) + if err != nil { + t.Fatal(err) + } + newer := old + newer.Key.STRMFingerprint = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + newer.Target = "new-target" + newFlight, err := service.begin(newer) + if err != nil { + t.Fatal(err) + } + select { + case <-oldFlight.done: + if !errors.Is(oldFlight.err, ErrSuperseded) { + t.Fatalf("old fingerprint error = %v", oldFlight.err) + } + case <-time.After(time.Second): + t.Fatal("old fingerprint was not superseded") + } + prober.release <- struct{}{} + if got := <-prober.started; got != "https://cdn.example.test/new-target" { + t.Fatalf("new fingerprint target = %q", got) + } + prober.release <- struct{}{} + select { + case <-newFlight.done: + case <-time.After(time.Second): + t.Fatal("new fingerprint flight did not finish") + } + }) + } +} + +func priorityName(priority Priority) string { + if priority == PriorityNeighbor { + return "neighbor" + } + return "metadata" +} + +func TestServiceTrySubmitAliasesOffer(t *testing.T) { + service := newTestService(t, &blockingProber{}, 1) + request := testRequest("try-submit", PriorityMetadata) + offered := service.Offer(request) + tried := service.TrySubmit(request) + if offered.Disposition != SubmitNewlyQueued || offered.Err != nil { + t.Fatalf("Offer() = %#v", offered) + } + if tried.Disposition != SubmitJoinedExistingFlight || tried.Err != nil { + t.Fatalf("TrySubmit() = %#v", tried) + } +} + +func TestServiceCloseCompletesAllPriorityQueues(t *testing.T) { + prober := &blockingProber{started: make(chan string, 1), release: make(chan struct{})} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, time.Now()), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", Concurrency: 1, PlaybackQueueSize: 2, + NeighborQueueSize: 2, MetadataQueueSize: 2, + }) + if result := service.Offer(testRequest("close-active", PriorityPlayback)); result.Err != nil { + t.Fatal(result.Err) + } + <-prober.started + flights := make([]*flight, 0, 3) + for _, priority := range []Priority{PriorityPlayback, PriorityNeighbor, PriorityMetadata} { + flight, err := service.begin(testRequest("close-"+string(rune('a'+len(flights))), priority)) + if err != nil { + t.Fatal(err) + } + flights = append(flights, flight) + } + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + if err := service.Close(ctx); err != nil { + t.Fatal(err) + } + for _, flight := range flights { + select { + case <-flight.done: + if !errors.Is(flight.err, ErrServiceUnavailable) { + t.Fatalf("closed flight error = %v", flight.err) + } + default: + t.Fatal("queued flight was not completed during close") + } + } +} diff --git a/internal/mediainfo/service.go b/internal/mediainfo/service.go index 359cd4f..ef4527d 100644 --- a/internal/mediainfo/service.go +++ b/internal/mediainfo/service.go @@ -12,18 +12,22 @@ import ( ) const ( - defaultRecordTTL = 30 * 24 * time.Hour - defaultRecordRetention = 180 * 24 * time.Hour - defaultNegativeTTL = 15 * time.Minute - defaultWorkerQueueSize = 256 - defaultWorkerCount = 1 - defaultNegativeLimit = 4096 - defaultRetryLimit = 256 - defaultTouchQueueSize = 1024 - defaultGCInterval = 24 * time.Hour - defaultStoreTimeout = 5 * time.Second - defaultTouchInterval = time.Hour - maxClientUserAgentBytes = 4 << 10 + defaultRecordTTL = 30 * 24 * time.Hour + defaultRecordRetention = 180 * 24 * time.Hour + defaultNegativeTTL = 15 * time.Minute + defaultPlaybackQueueSize = 16 + defaultNeighborQueueSize = 50 + defaultMetadataQueueSize = 50 + defaultPendingTTL = 5 * time.Minute + defaultBackgroundInterval = 5 * time.Second + defaultWorkerCount = 1 + defaultNegativeLimit = 4096 + defaultRetryLimit = 256 + defaultTouchQueueSize = 1024 + defaultGCInterval = 24 * time.Hour + defaultStoreTimeout = 5 * time.Second + defaultTouchInterval = time.Hour + maxClientUserAgentBytes = 4 << 10 ) var ( @@ -32,15 +36,25 @@ var ( ErrRetryRegistrationFull = errors.New("MediaInfo retry registration is full") ErrNegativeCache = errors.New("MediaInfo probe is in backoff") ErrCacheResetInProgress = errors.New("MediaInfo cache reset is in progress") + ErrPendingExpired = errors.New("MediaInfo pending request expired") + ErrSuperseded = errors.New("MediaInfo request was superseded") ) -// Priority separates interactive misses from administrator prewarming. A -// playback-related metadata request must not wait behind season or root scans. +// Priority orders MediaInfo work by user intent. P0 playback work can bypass +// the shared background start interval; P1 and P2 are bounded background work. type Priority uint8 const ( - PriorityBackground Priority = iota - PriorityInteractive + PriorityPlayback Priority = iota + PriorityNeighbor + PriorityMetadata +) + +const ( + // Deprecated: use PriorityPlayback. + PriorityInteractive = PriorityPlayback + // Deprecated: use PriorityNeighbor. + PriorityBackground = PriorityNeighbor ) // Request describes one exact STRM control target. Target is used only for the @@ -77,6 +91,9 @@ func (request Request) validate() error { if err := request.Key.Validate(); err != nil { return err } + if request.Priority > PriorityMetadata { + return errors.New("MediaInfo request priority is invalid") + } if request.Target == "" { return errors.New("STRM target is required") } @@ -121,7 +138,13 @@ type Prober interface { type ServiceMetrics interface { IncMediaInfoCacheHits() IncMediaInfoCacheMisses() - IncMediaInfoProbeQueued() + IncMediaInfoProbeOffered(uint8) + IncMediaInfoProbeQueued(uint8) + IncMediaInfoProbeJoined(uint8) + IncMediaInfoProbePromoted(uint8) + IncMediaInfoProbeDroppedFull(uint8) + IncMediaInfoProbeDroppedExpired(uint8) + IncMediaInfoProbeSuperseded(uint8) IncMediaInfoProbeSuccess() IncMediaInfoProbeFailure() IncMediaInfoStoreFailure() @@ -133,24 +156,31 @@ type ServiceMetrics interface { // ServiceOptions owns the background analysis lifecycle. Resolver requests use // only UserAgent and identity encoding; client credentials are never inherited. type ServiceOptions struct { - Cache *Cache - Store RecordStore - Janitor RecordJanitor - Provider Provider - PlexServerID string - Logger *slog.Logger - Metrics ServiceMetrics - Concurrency int + Cache *Cache + Store RecordStore + Janitor RecordJanitor + Provider Provider + PlexServerID string + Logger *slog.Logger + Metrics ServiceMetrics + Concurrency int + // Deprecated: use PlaybackQueueSize. InteractiveQueueSize int - BackgroundQueueSize int - ProbeTimeout time.Duration - RecordTTL time.Duration - RecordRetention time.Duration - NegativeTTL time.Duration - BackgroundUserAgent string - GCInterval time.Duration - TouchInterval time.Duration - Now func() time.Time + // Deprecated: use NeighborQueueSize. + BackgroundQueueSize int + ProbeTimeout time.Duration + RecordTTL time.Duration + RecordRetention time.Duration + NegativeTTL time.Duration + PendingTTL time.Duration + BackgroundInterval time.Duration + PlaybackQueueSize int + NeighborQueueSize int + MetadataQueueSize int + BackgroundUserAgent string + GCInterval time.Duration + TouchInterval time.Duration + Now func() time.Time } // Service schedules bounded MediaInfo work independently from client request @@ -169,6 +199,8 @@ type Service struct { recordTTL time.Duration recordRetention time.Duration negativeTTL time.Duration + pendingTTL time.Duration + backgroundInterval time.Duration negativeLimit int backgroundUserAgent string gcInterval time.Duration @@ -180,19 +212,24 @@ type Service struct { workers sync.WaitGroup workerDone chan struct{} touches chan recordTouch + wake chan struct{} mu sync.Mutex - condition *sync.Cond - interactive []*job - background []*job - retryInteractive []*job - retryBackground []*job - interactiveLimit int - backgroundLimit int + playback []*job + neighbor []*job + metadata []*job + retryPlayback []*job + retryNeighbor []*job + retryMetadata []*job + playbackLimit int + neighborLimit int + metadataLimit int + nextBackgroundAt time.Time flights map[string]*flight negative map[string]time.Time retryRegistrations int retryLimit int + retrySequence uint64 closed bool active atomic.Int64 generation atomic.Uint64 @@ -210,8 +247,11 @@ type recordTouch struct { type flight struct { done chan struct{} job *job + request Request + priority Priority userAgent string retryWaiters []retryRegistration + retryDepth uint8 record Record err error } @@ -219,13 +259,16 @@ type flight struct { type retryRegistration struct { request Request userAgentKey string + sequence uint64 } type job struct { - request Request - flight *flight - generation uint64 - claimed bool + request Request + flight *flight + generation uint64 + expiresAt time.Time + cacheChecked bool + claimed bool } // NewService starts the configured worker count after validating every @@ -238,13 +281,31 @@ func NewService(options ServiceOptions) (*Service, error) { if concurrency <= 0 { concurrency = defaultWorkerCount } - interactiveSize := options.InteractiveQueueSize - if interactiveSize <= 0 { - interactiveSize = defaultWorkerQueueSize + playbackSize := options.PlaybackQueueSize + if playbackSize <= 0 { + playbackSize = options.InteractiveQueueSize + } + if playbackSize <= 0 { + playbackSize = defaultPlaybackQueueSize + } + neighborSize := options.NeighborQueueSize + if neighborSize <= 0 { + neighborSize = options.BackgroundQueueSize + } + if neighborSize <= 0 { + neighborSize = defaultNeighborQueueSize } - backgroundSize := options.BackgroundQueueSize - if backgroundSize <= 0 { - backgroundSize = defaultWorkerQueueSize + metadataSize := options.MetadataQueueSize + if metadataSize <= 0 { + metadataSize = defaultMetadataQueueSize + } + pendingTTL := options.PendingTTL + if pendingTTL <= 0 { + pendingTTL = defaultPendingTTL + } + backgroundInterval := options.BackgroundInterval + if backgroundInterval <= 0 { + backgroundInterval = defaultBackgroundInterval } probeTimeout := options.ProbeTimeout if probeTimeout <= 0 { @@ -302,17 +363,17 @@ func NewService(options ServiceOptions) (*Service, error) { provider: options.Provider, providerDescriptor: providerDescriptor, plexServerID: plexServerID, logger: logger, metrics: options.Metrics, probeTimeout: probeTimeout, recordTTL: recordTTL, recordRetention: recordRetention, - negativeTTL: negativeTTL, negativeLimit: defaultNegativeLimit, + negativeTTL: negativeTTL, pendingTTL: pendingTTL, + backgroundInterval: backgroundInterval, negativeLimit: defaultNegativeLimit, retryLimit: defaultRetryLimit, backgroundUserAgent: backgroundUserAgent, gcInterval: gcInterval, touchInterval: touchInterval, now: now, ctx: ctx, cancel: cancel, workerDone: make(chan struct{}), - touches: make(chan recordTouch, defaultTouchQueueSize), - interactiveLimit: interactiveSize, backgroundLimit: backgroundSize, + touches: make(chan recordTouch, defaultTouchQueueSize), wake: make(chan struct{}, 1), + playbackLimit: playbackSize, neighborLimit: neighborSize, metadataLimit: metadataSize, flights: make(map[string]*flight), negative: make(map[string]time.Time), } service.resetStore, _ = options.Store.(recordResetStore) - service.condition = sync.NewCond(&service.mu) for index := 0; index < concurrency; index++ { service.workers.Add(1) go service.worker() @@ -410,12 +471,13 @@ func (service *Service) Ensure(ctx context.Context, request Request) (Record, er service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoCacheHits() }) if !fresh { revalidation := request - revalidation.Priority = PriorityBackground + revalidation.Priority = PriorityNeighbor _, _ = service.begin(revalidation) } return record, nil } service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoCacheMisses() }) + retries := 0 for { if err := ctx.Err(); err != nil { return Record{}, err @@ -428,10 +490,11 @@ func (service *Service) Ensure(ctx context.Context, request Request) (Record, er case <-ctx.Done(): return Record{}, ctx.Err() case <-flight.done: - if flight.err != nil && flight.userAgent != request.ClientUserAgent { + if flight.err != nil && flight.userAgent != request.ClientUserAgent && retries == 0 { // Successful technical metadata is UA-independent and remains // shared. A failed provider attempt is UA-sensitive, so a waiter - // with another client identity may retry within its own deadline. + // with another client identity may make one bounded retry. + retries++ continue } return cloneRecord(flight.record), flight.err @@ -446,6 +509,33 @@ func (service *Service) Submit(request Request) error { return service.SubmitDetailed(request).Err } +// Offer attempts a best-effort admission using only in-memory state. It is +// safe for response paths because it never reads the durable store or waits +// for a worker; a worker performs the authoritative L2 check before probing. +func (service *Service) Offer(request Request) SubmitResult { + if service == nil { + return SubmitResult{Disposition: SubmitRejected, Err: ErrServiceUnavailable} + } + request, err := service.normalizeRequest(request) + if err != nil { + return SubmitResult{Disposition: SubmitRejected, Err: err} + } + if _, ok := service.memoryFresh(request.Key); ok { + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoCacheHits() }) + return SubmitResult{Disposition: SubmitFreshCache} + } + _, disposition, err := service.beginDetailed(request, true) + if err != nil { + return SubmitResult{Disposition: SubmitRejected, Err: err} + } + return SubmitResult{Disposition: disposition} +} + +// TrySubmit is the concise alias used by non-blocking callers. +func (service *Service) TrySubmit(request Request) SubmitResult { + return service.Offer(request) +} + // SubmitDetailed schedules work and reports whether the request used a fresh // cache record, joined an existing flight, was newly queued, or was rejected. // A request that joins a different-User-Agent flight is registered once, with @@ -463,9 +553,6 @@ func (service *Service) SubmitDetailed(request Request) SubmitResult { if found && fresh { return SubmitResult{Disposition: SubmitFreshCache} } - if found { - request.Priority = PriorityBackground - } _, disposition, err := service.beginDetailed(request, true) if err != nil { return SubmitResult{Disposition: SubmitRejected, Err: err} @@ -473,6 +560,24 @@ func (service *Service) SubmitDetailed(request Request) SubmitResult { return SubmitResult{Disposition: disposition} } +func (service *Service) memoryFresh(key Key) (Record, bool) { + if service == nil || service.resetting.Load() { + return Record{}, false + } + service.resetMu.RLock() + if service.resetting.Load() { + service.resetMu.RUnlock() + return Record{}, false + } + now := service.now().UTC() + record, ok := service.cache.Get(key, now) + service.resetMu.RUnlock() + if !ok || !service.compatible(record) { + return Record{}, false + } + return service.touchCached(record, now), true +} + func (service *Service) normalizeRequest(request Request) (Request, error) { if strings.TrimSpace(request.Key.PlexServerID) == "" { request.Key.PlexServerID = service.plexServerID @@ -567,6 +672,7 @@ func (service *Service) beginDetailed(request Request, registerRetry bool) (*fli if err != nil { return nil, SubmitRejected, err } + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeOffered(uint8(request.Priority)) }) key := request.Key.cacheKey() negativeKey := service.negativeKey(request) now := service.now() @@ -583,36 +689,93 @@ func (service *Service) beginDetailed(request Request, registerRetry bool) (*fli delete(service.negative, negativeKey) } if existing, exists := service.flights[key]; exists { - if request.Priority == PriorityInteractive { - service.promoteLocked(existing.job) + accepted, promoted := service.updateExistingLocked(existing, request) + if !accepted { + service.mu.Unlock() + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeDroppedFull(uint8(request.Priority)) }) + return nil, SubmitRejected, ErrQueueFull } - if registerRetry && request.ClientUserAgent != existing.userAgent { + if registerRetry && existing.job.claimed && request.ClientUserAgent != existing.userAgent { if err := service.registerRetryLocked(existing, request); err != nil { service.mu.Unlock() return nil, SubmitRejected, err } } service.mu.Unlock() + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeJoined(uint8(request.Priority)) }) + if promoted { + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbePromoted(uint8(request.Priority)) }) + } return existing, SubmitJoinedExistingFlight, nil } - flight := &flight{done: make(chan struct{}), userAgent: request.ClientUserAgent} - queued := &job{request: request, flight: flight, generation: service.generation.Load()} + service.supersedeQueuedBackgroundLocked(request) + flight := &flight{ + done: make(chan struct{}), request: request, priority: request.Priority, + userAgent: request.ClientUserAgent, + } + queued := &job{ + request: request, flight: flight, generation: service.generation.Load(), + expiresAt: service.pendingExpiry(request.Priority, now), + } flight.job = queued if !service.enqueueLocked(queued) { service.mu.Unlock() + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeDroppedFull(uint8(request.Priority)) }) return nil, SubmitRejected, ErrQueueFull } service.flights[key] = flight - service.condition.Signal() + service.signalWakeLocked() service.mu.Unlock() - service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeQueued() }) + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeQueued(uint8(request.Priority)) }) return flight, SubmitNewlyQueued, nil } +func (service *Service) pendingExpiry(priority Priority, now time.Time) time.Time { + if priority == PriorityPlayback { + return time.Time{} + } + return now.Add(service.pendingTTL) +} + +func (service *Service) updateExistingLocked(existing *flight, request Request) (bool, bool) { + if existing == nil || existing.job == nil { + return false, false + } + if request.Priority < existing.priority { + if existing.job.claimed { + existing.priority = request.Priority + existing.request = request + service.signalWakeLocked() + } else if service.promoteLocked(existing.job, request) { + existing.priority = request.Priority + existing.request = request + } else { + return false, false + } + return true, true + } + if request.Priority != existing.priority { + return true, false + } + // The latest same-priority playback/neighbor request carries the current + // target and identity metadata. A queued job may also replace its transport + // User-Agent because no provider call has started yet. + existing.request = request + if !existing.job.claimed { + existing.job.request = request + existing.userAgent = request.ClientUserAgent + } + return true, false +} + // registerRetryLocked records one distinct waiting User-Agent for an active // flight. The caller must hold service.mu. Registration is intentionally // bounded because User-Agent is untrusted request input. func (service *Service) registerRetryLocked(existing *flight, request Request) error { + if existing == nil || existing.job == nil || existing.retryDepth >= 1 || + request.Priority > PriorityNeighbor || request.Priority > existing.priority { + return nil + } userAgentKey := retryUserAgentKey(request.ClientUserAgent) for _, registration := range existing.retryWaiters { if registration.userAgentKey == userAgentKey { @@ -623,46 +786,55 @@ func (service *Service) registerRetryLocked(existing *flight, request Request) e return ErrRetryRegistrationFull } existing.retryWaiters = append(existing.retryWaiters, retryRegistration{ - request: request, userAgentKey: userAgentKey, + request: request, userAgentKey: userAgentKey, sequence: service.retrySequence, }) + service.retrySequence++ service.retryRegistrations++ return nil } func (service *Service) enqueueLocked(queued *job) bool { - if queued.request.Priority == PriorityInteractive { - if len(service.interactive) >= service.interactiveLimit { + if queued == nil { + return false + } + switch queued.request.Priority { + case PriorityPlayback: + if service.playbackCountLocked() >= service.playbackLimit { return false } - service.interactive = append(service.interactive, queued) - return true - } - if len(service.background) >= service.backgroundLimit { + service.playback = append(service.playback, queued) + case PriorityNeighbor: + if service.neighborCountLocked() >= service.neighborLimit { + return false + } + service.neighbor = append(service.neighbor, queued) + case PriorityMetadata: + if service.metadataCountLocked() >= service.metadataLimit { + return false + } + service.metadata = append(service.metadata, queued) + default: return false } - service.background = append(service.background, queued) return true } -func (service *Service) promoteLocked(queued *job) { - if queued == nil || queued.claimed || queued.request.Priority == PriorityInteractive { - return - } - if removeQueuedJob(&service.retryBackground, queued) { - queued.request.Priority = PriorityInteractive - service.retryInteractive = append(service.retryInteractive, queued) - service.condition.Signal() - return +func (service *Service) promoteLocked(queued *job, request Request) bool { + if queued == nil || queued.claimed || request.Priority >= queued.request.Priority { + return false } - if len(service.interactive) >= service.interactiveLimit { - return + if !service.priorityCapacityAvailableLocked(request.Priority) { + return false } - if !removeQueuedJob(&service.background, queued) { - return + if !service.removeFromQueueLocked(queued) { + return false } - queued.request.Priority = PriorityInteractive - service.interactive = append(service.interactive, queued) - service.condition.Signal() + queued.request = request + queued.expiresAt = service.pendingExpiry(request.Priority, service.now()) + queued.flight.userAgent = request.ClientUserAgent + service.prependToQueueLocked(queued) + service.signalWakeLocked() + return true } func removeQueuedJob(queue *[]*job, target *job) bool { @@ -679,6 +851,105 @@ func removeQueuedJob(queue *[]*job, target *job) bool { return false } +func (service *Service) removeFromQueueLocked(target *job) bool { + for _, queue := range service.queuedQueuesLocked() { + if removeQueuedJob(queue, target) { + return true + } + } + return false +} + +func (service *Service) prependToQueueLocked(queued *job) { + switch queued.request.Priority { + case PriorityPlayback: + service.playback = append([]*job{queued}, service.playback...) + case PriorityNeighbor: + service.neighbor = append([]*job{queued}, service.neighbor...) + case PriorityMetadata: + service.metadata = append([]*job{queued}, service.metadata...) + } +} + +func (service *Service) queuedQueuesLocked() []*[]*job { + return []*[]*job{ + &service.playback, &service.neighbor, &service.metadata, + &service.retryPlayback, &service.retryNeighbor, &service.retryMetadata, + } +} + +func (service *Service) priorityCapacityAvailableLocked(priority Priority) bool { + switch priority { + case PriorityPlayback: + return service.playbackCountLocked() < service.playbackLimit + case PriorityNeighbor: + return service.neighborCountLocked() < service.neighborLimit + case PriorityMetadata: + return service.metadataCountLocked() < service.metadataLimit + default: + return false + } +} + +func (service *Service) playbackCountLocked() int { + return len(service.playback) + len(service.retryPlayback) +} + +func (service *Service) neighborCountLocked() int { + return len(service.neighbor) + len(service.retryNeighbor) +} + +func (service *Service) metadataCountLocked() int { + return len(service.metadata) + len(service.retryMetadata) +} + +func (service *Service) supersedeQueuedBackgroundLocked(request Request) { + for _, queue := range []*[]*job{ + &service.neighbor, &service.metadata, &service.retryNeighbor, &service.retryMetadata, + } { + kept := (*queue)[:0] + for _, queued := range *queue { + if queued.request.Key.PlexServerID == request.Key.PlexServerID && + queued.request.Key.PartID == request.Key.PartID && + queued.request.Key.STRMFingerprint != request.Key.STRMFingerprint { + service.cancelQueuedLocked(queued, ErrSuperseded) + continue + } + kept = append(kept, queued) + } + *queue = kept + } +} + +func (service *Service) cancelQueuedLocked(queued *job, err error) { + if queued == nil || queued.claimed || queued.flight == nil { + return + } + key := queued.request.Key.cacheKey() + if current, ok := service.flights[key]; !ok || current != queued.flight { + return + } + delete(service.flights, key) + service.releaseRetryRegistrationsLocked(queued.flight) + queued.flight.err = err + close(queued.flight.done) + switch { + case errors.Is(err, ErrQueueFull): + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeDroppedFull(uint8(queued.request.Priority)) }) + case errors.Is(err, ErrPendingExpired): + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeDroppedExpired(uint8(queued.request.Priority)) }) + case errors.Is(err, ErrSuperseded): + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeSuperseded(uint8(queued.request.Priority)) }) + } +} + +func (service *Service) signalWakeLocked() { + select { + case service.wake <- struct{}{}: + default: + } +} + func (service *Service) worker() { defer service.workers.Done() for { @@ -732,47 +1003,230 @@ func (service *Service) janitorWorker() { } func (service *Service) nextJob() (*job, bool) { - service.mu.Lock() - defer service.mu.Unlock() - for !service.closed && service.queuedJobsLocked() == 0 { - service.condition.Wait() - } - if service.closed { - return nil, false - } - var queued *job - if len(service.retryInteractive) > 0 { - queued = service.retryInteractive[0] - service.retryInteractive[0] = nil - service.retryInteractive = service.retryInteractive[1:] - } else if len(service.interactive) > 0 { - queued = service.interactive[0] - service.interactive[0] = nil - service.interactive = service.interactive[1:] - } else if len(service.retryBackground) > 0 { - queued = service.retryBackground[0] - service.retryBackground[0] = nil - service.retryBackground = service.retryBackground[1:] - } else { - queued = service.background[0] - service.background[0] = nil - service.background = service.background[1:] + for { + service.mu.Lock() + if service.closed { + service.mu.Unlock() + return nil, false + } + service.expireQueuedLocked(service.now()) + if queued := popQueue(&service.playback, &service.retryPlayback); queued != nil { + queued.claimed = true + if service.playbackCountLocked() > 0 { + service.signalWakeLocked() + } + service.mu.Unlock() + return queued, true + } + backgroundQueued := service.backgroundJobsLocked() + now := service.now() + if backgroundQueued > 0 && (service.nextBackgroundAt.IsZero() || !now.Before(service.nextBackgroundAt)) { + queued := service.popPreparedBackgroundLocked() + if queued != nil { + queued.claimed = true + service.nextBackgroundAt = now.Add(service.backgroundInterval) + service.mu.Unlock() + return queued, true + } + } + if queued := service.popUnpreparedBackgroundLocked(); queued != nil { + queued.claimed = true + if service.backgroundJobsLocked() > 0 { + service.signalWakeLocked() + } + service.mu.Unlock() + service.preflight(queued) + continue + } + + wait := time.Duration(-1) + if backgroundQueued > 0 && !service.nextBackgroundAt.IsZero() { + wait = service.nextBackgroundAt.Sub(now) + if wait < 0 { + wait = 0 + } + } + service.mu.Unlock() + + if wait < 0 { + select { + case <-service.wake: + continue + case <-service.ctx.Done(): + return nil, false + } + } + timer := time.NewTimer(wait) + select { + case <-service.wake: + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + case <-timer.C: + case <-service.ctx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + return nil, false + } } - queued.claimed = true - return queued, true } func (service *Service) queuedJobsLocked() int { - return len(service.retryInteractive) + len(service.interactive) + - len(service.retryBackground) + len(service.background) + return len(service.retryPlayback) + len(service.playback) + + len(service.retryNeighbor) + len(service.neighbor) + + len(service.retryMetadata) + len(service.metadata) +} + +func (service *Service) backgroundJobsLocked() int { + return len(service.retryNeighbor) + len(service.neighbor) + + len(service.retryMetadata) + len(service.metadata) +} + +func popQueue(primary, retry *[]*job) *job { + if len(*primary) > 0 { + queued := (*primary)[0] + (*primary)[0] = nil + *primary = (*primary)[1:] + return queued + } + if len(*retry) > 0 { + queued := (*retry)[0] + (*retry)[0] = nil + *retry = (*retry)[1:] + return queued + } + return nil } -func (service *Service) enqueueRetryLocked(queued *job) { - if queued.request.Priority == PriorityInteractive { - service.retryInteractive = append(service.retryInteractive, queued) +func popUnprepared(queue *[]*job) *job { + return popBackgroundByPreparation(queue, false) +} + +func popPrepared(queue *[]*job) *job { + return popBackgroundByPreparation(queue, true) +} + +func popBackgroundByPreparation(queue *[]*job, prepared bool) *job { + for index, queued := range *queue { + if queued == nil || queued.cacheChecked != prepared { + continue + } + copy((*queue)[index:], (*queue)[index+1:]) + last := len(*queue) - 1 + (*queue)[last] = nil + *queue = (*queue)[:last] + return queued + } + return nil +} + +func (service *Service) popPreparedBackgroundLocked() *job { + for _, queue := range []*[]*job{ + &service.neighbor, &service.retryNeighbor, &service.metadata, &service.retryMetadata, + } { + if queued := popPrepared(queue); queued != nil { + return queued + } + } + return nil +} + +func (service *Service) popUnpreparedBackgroundLocked() *job { + for _, queue := range []*[]*job{ + &service.neighbor, &service.retryNeighbor, &service.metadata, &service.retryMetadata, + } { + if queued := popUnprepared(queue); queued != nil { + return queued + } + } + return nil +} + +func (service *Service) enqueueRetryLocked(queued *job) bool { + if queued == nil { + return false + } + switch queued.request.Priority { + case PriorityPlayback: + if service.playbackCountLocked() >= service.playbackLimit { + return false + } + service.retryPlayback = append(service.retryPlayback, queued) + case PriorityNeighbor: + if service.neighborCountLocked() >= service.neighborLimit { + return false + } + service.retryNeighbor = append(service.retryNeighbor, queued) + case PriorityMetadata: + if service.metadataCountLocked() >= service.metadataLimit { + return false + } + service.retryMetadata = append(service.retryMetadata, queued) + default: + return false + } + return true +} + +// preflight completes fresh L2 hits before the remote-start limiter. A durable +// miss is returned to its priority queue without extending its pending TTL. +func (service *Service) preflight(queued *job) { + if !queued.expiresAt.IsZero() && !service.now().Before(queued.expiresAt) { + service.finish(queued, Record{}, ErrPendingExpired) + return + } + if record, found, fresh := service.lookup(service.ctx, queued.request.Key); found && fresh { + service.finishCached(queued, service.recordForFlight(queued, record)) + return + } + + service.mu.Lock() + key := queued.request.Key.cacheKey() + flight := service.flights[key] + if service.closed || flight != queued.flight { + service.mu.Unlock() + return + } + queued.claimed = false + queued.cacheChecked = true + queued.request = flight.request + queued.request.Priority = flight.priority + flight.userAgent = queued.request.ClientUserAgent + if flight.priority == PriorityPlayback { + queued.expiresAt = time.Time{} + } + if !service.priorityCapacityAvailableLocked(queued.request.Priority) { + service.cancelQueuedLocked(queued, ErrQueueFull) + service.mu.Unlock() return } - service.retryBackground = append(service.retryBackground, queued) + service.prependToQueueLocked(queued) + service.signalWakeLocked() + service.mu.Unlock() +} + +func (service *Service) expireQueuedLocked(now time.Time) { + for _, queue := range []*[]*job{ + &service.playback, &service.neighbor, &service.metadata, + &service.retryPlayback, &service.retryNeighbor, &service.retryMetadata, + } { + kept := (*queue)[:0] + for _, queued := range *queue { + if queued != nil && !queued.expiresAt.IsZero() && !now.Before(queued.expiresAt) { + service.cancelQueuedLocked(queued, ErrPendingExpired) + continue + } + kept = append(kept, queued) + } + *queue = kept + } } func (service *Service) run(queued *job) { @@ -784,6 +1238,25 @@ func (service *Service) run(queued *job) { service.metric(func(metrics ServiceMetrics) { metrics.DecMediaInfoProbeActive() }) service.metric(func(metrics ServiceMetrics) { metrics.ObserveMediaInfoProbeLatency(service.now().Sub(started)) }) }() + if !queued.expiresAt.IsZero() && !service.now().Before(queued.expiresAt) { + service.finish(queued, Record{}, ErrPendingExpired) + return + } + // P0 reaches the worker without preflight and checks both cache tiers. A + // preflighted background miss only rechecks L1 so remote work does not repeat + // the same SQLite read. + var record Record + var found, fresh bool + if queued.cacheChecked { + record, found = service.memoryFresh(queued.request.Key) + fresh = found + } else { + record, found, fresh = service.lookup(service.ctx, queued.request.Key) + } + if found && fresh { + service.finishCached(queued, service.recordForFlight(queued, record)) + return + } probeCtx, cancel := context.WithTimeout(service.ctx, service.probeTimeout) defer cancel() @@ -810,8 +1283,9 @@ func (service *Service) run(queued *job) { return } now := service.now().UTC() - record := Record{ - Key: queued.request.Key, RatingKey: queued.request.RatingKey, + request := service.requestForFlight(queued) + record = Record{ + Key: queued.request.Key, RatingKey: request.RatingKey, Provider: service.providerDescriptor.Name, ProviderRevision: service.providerDescriptor.Revision, ContentFingerprint: result.ContentFingerprint, SchemaVersion: SchemaVersion, Media: result.Media, ProbedAt: now, ExpiresAt: now.Add(service.recordTTL), @@ -838,6 +1312,41 @@ func (service *Service) run(queued *job) { service.finish(queued, record, nil) } +func (service *Service) requestForFlight(queued *job) Request { + service.mu.Lock() + defer service.mu.Unlock() + if queued != nil && queued.flight != nil && queued.flight.request.Key.cacheKey() == queued.request.Key.cacheKey() { + return queued.flight.request + } + if queued == nil { + return Request{} + } + return queued.request +} + +func (service *Service) recordForFlight(queued *job, record Record) Record { + request := service.requestForFlight(queued) + if request.RatingKey != "" { + record.RatingKey = request.RatingKey + } + return record +} + +func (service *Service) finishCached(queued *job, record Record) { + key := queued.request.Key.cacheKey() + service.mu.Lock() + flight := service.flights[key] + if flight != queued.flight { + service.mu.Unlock() + return + } + delete(service.flights, key) + service.releaseRetryRegistrationsLocked(flight) + flight.record = record + close(flight.done) + service.mu.Unlock() +} + func (service *Service) finish(queued *job, record Record, err error) { key := queued.request.Key.cacheKey() var retry *flight @@ -848,10 +1357,10 @@ func (service *Service) finish(queued *job, record Record, err error) { return } delete(service.flights, key) - if err != nil && !errors.Is(err, context.Canceled) { + if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, ErrPendingExpired) && !errors.Is(err, ErrSuperseded) { now := service.now() service.rememberNegativeLocked(service.negativeKey(queued.request), now) - retry = service.scheduleRetryLocked(flight, queued.request.ClientUserAgent, now) + retry = service.scheduleRetryLocked(flight, flight.userAgent, now) } else { service.releaseRetryRegistrationsLocked(flight) } @@ -860,7 +1369,11 @@ func (service *Service) finish(queued *job, record Record, err error) { close(flight.done) service.mu.Unlock() if retry != nil { - service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeQueued() }) + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeQueued(uint8(retry.priority)) }) + } + if errors.Is(err, ErrPendingExpired) { + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeDroppedExpired(uint8(queued.request.Priority)) }) + return } if err != nil { service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeFailure() }) @@ -870,39 +1383,49 @@ func (service *Service) finish(queued *job, record Record, err error) { service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeSuccess() }) } -// scheduleRetryLocked promotes the first eligible Submit registration to a -// new flight and carries the remaining registrations forward. The caller must -// hold service.mu. A failed User-Agent is already in negative backoff, so it -// cannot be selected for an immediate blind retry. +// scheduleRetryLocked selects at most one highest-priority, most-recent +// User-Agent registration. The caller must hold service.mu. A failed +// User-Agent is already in negative backoff and cannot be retried immediately. func (service *Service) scheduleRetryLocked(failed *flight, failedUserAgent string, now time.Time) *flight { + if failed.retryDepth >= 1 { + service.releaseRetryRegistrationsLocked(failed) + return nil + } var selected *retryRegistration - remaining := make([]retryRegistration, 0, len(failed.retryWaiters)) for index := range failed.retryWaiters { registration := failed.retryWaiters[index] service.retryRegistrations-- if registration.request.ClientUserAgent == failedUserAgent || - service.negativeActiveLocked(registration.request, now) { + registration.request.Priority > PriorityNeighbor || service.negativeActiveLocked(registration.request, now) { continue } - if selected == nil { + if selected == nil || registration.request.Priority < selected.request.Priority || + (registration.request.Priority == selected.request.Priority && registration.sequence > selected.sequence) { candidate := registration selected = &candidate - continue } - remaining = append(remaining, registration) } failed.retryWaiters = nil if selected == nil { return nil } - next := &flight{done: make(chan struct{}), userAgent: selected.request.ClientUserAgent} - next.retryWaiters = remaining - service.retryRegistrations += len(remaining) - next.job = &job{request: selected.request, flight: next, generation: failed.job.generation} - service.enqueueRetryLocked(next.job) + next := &flight{ + done: make(chan struct{}), request: selected.request, + priority: selected.request.Priority, userAgent: selected.request.ClientUserAgent, + retryDepth: failed.retryDepth + 1, + } + next.job = &job{ + request: selected.request, flight: next, generation: failed.job.generation, + expiresAt: service.pendingExpiry(selected.request.Priority, now), cacheChecked: true, + } + next.job.flight = next + if !service.enqueueRetryLocked(next.job) { + service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeDroppedFull(uint8(next.priority)) }) + return nil + } service.flights[next.job.request.Key.cacheKey()] = next - service.condition.Signal() + service.signalWakeLocked() return next } @@ -966,6 +1489,9 @@ type Status struct { Available bool `json:"available"` CacheEntries int `json:"cache_entries"` ActiveProbes int64 `json:"active_probes"` + PlaybackQueued int `json:"playback_queued"` + NeighborQueued int `json:"neighbor_queued"` + MetadataQueued int `json:"metadata_queued"` InteractiveQueued int `json:"interactive_queued"` BackgroundQueued int `json:"background_queued"` NegativeEntries int `json:"negative_entries"` @@ -980,8 +1506,11 @@ func (service *Service) Status() Status { available := !service.closed status := Status{ Available: available, CacheEntries: service.cache.Len(), ActiveProbes: service.active.Load(), - InteractiveQueued: len(service.retryInteractive) + len(service.interactive), - BackgroundQueued: len(service.retryBackground) + len(service.background), + PlaybackQueued: service.playbackCountLocked(), + NeighborQueued: service.neighborCountLocked(), + MetadataQueued: service.metadataCountLocked(), + InteractiveQueued: service.playbackCountLocked(), + BackgroundQueued: service.neighborCountLocked() + service.metadataCountLocked(), NegativeEntries: len(service.negative), } service.mu.Unlock() @@ -1014,16 +1543,20 @@ func (service *Service) ResetCache(ctx context.Context, backupDir string) (Reset service.releaseRetryRegistrationsLocked(flight) close(flight.done) } - service.interactive = nil - service.background = nil - service.retryInteractive = nil - service.retryBackground = nil + service.playback = nil + service.neighbor = nil + service.metadata = nil + service.retryPlayback = nil + service.retryNeighbor = nil + service.retryMetadata = nil service.negative = make(map[string]time.Time) + service.nextBackgroundAt = time.Time{} for { select { case touch := <-service.touches: touch.reservation.release() default: + service.signalWakeLocked() service.mu.Unlock() goto queuesCleared } @@ -1065,11 +1598,13 @@ func (service *Service) Close(ctx context.Context) error { service.releaseRetryRegistrationsLocked(flight) close(flight.done) } - service.interactive = nil - service.background = nil - service.retryInteractive = nil - service.retryBackground = nil - service.condition.Broadcast() + service.playback = nil + service.neighbor = nil + service.metadata = nil + service.retryPlayback = nil + service.retryNeighbor = nil + service.retryMetadata = nil + service.signalWakeLocked() } service.mu.Unlock() if ctx == nil { diff --git a/internal/mediainfo/service_test.go b/internal/mediainfo/service_test.go index a0c637e..b2aaf4d 100644 --- a/internal/mediainfo/service_test.go +++ b/internal/mediainfo/service_test.go @@ -598,6 +598,9 @@ func newServiceForTest(t *testing.T, options ServiceOptions) *Service { if options.NegativeTTL == 0 { options.NegativeTTL = time.Minute } + if options.BackgroundInterval == 0 { + options.BackgroundInterval = time.Millisecond + } if options.BackgroundUserAgent == "" { options.BackgroundUserAgent = "Infuse-Library/8.4.4" } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 7f937a0..2b75f20 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -32,7 +32,13 @@ type Metrics struct { metadataBatchQueued atomic.Int64 mediaInfoCacheHits atomic.Uint64 mediaInfoCacheMisses atomic.Uint64 + mediaInfoProbeOffered atomic.Uint64 mediaInfoProbeQueued atomic.Uint64 + mediaInfoProbeJoined atomic.Uint64 + mediaInfoProbePromoted atomic.Uint64 + mediaInfoProbeDroppedFull atomic.Uint64 + mediaInfoProbeDroppedExpired atomic.Uint64 + mediaInfoProbeSuperseded atomic.Uint64 mediaInfoProbeSuccess atomic.Uint64 mediaInfoProbeFailure atomic.Uint64 mediaInfoStoreFailure atomic.Uint64 @@ -53,6 +59,19 @@ type Metrics struct { resolverLatency latencyMetrics redirectLatency latencyMetrics mediaInfoProbeLatency latencyMetrics + mediaInfoScheduler schedulerMetrics +} + +const mediaInfoPriorityCount = 3 + +type schedulerMetrics struct { + offered [mediaInfoPriorityCount]atomic.Uint64 + queued [mediaInfoPriorityCount]atomic.Uint64 + joined [mediaInfoPriorityCount]atomic.Uint64 + promoted [mediaInfoPriorityCount]atomic.Uint64 + droppedFull [mediaInfoPriorityCount]atomic.Uint64 + droppedExpired [mediaInfoPriorityCount]atomic.Uint64 + superseded [mediaInfoPriorityCount]atomic.Uint64 } type latencyMetrics struct { @@ -73,34 +92,41 @@ type Snapshot struct { PlexFallbackTotal uint64 `json:"plex_fallback_total"` ActiveRequests int64 `json:"active_requests"` - MetadataGuardAdmittedTotal uint64 `json:"metadata_guard_admitted_total"` - MetadataGuardTimeoutsTotal uint64 `json:"metadata_guard_timeouts_total"` - MetadataGuardActive int64 `json:"metadata_guard_active"` - MetadataGuardQueued int64 `json:"metadata_guard_queued"` - MetadataBatchGuardAdmittedTotal uint64 `json:"metadata_batch_guard_admitted_total"` - MetadataBatchGuardTimeoutsTotal uint64 `json:"metadata_batch_guard_timeouts_total"` - MetadataBatchGuardActive int64 `json:"metadata_batch_guard_active"` - MetadataBatchGuardQueued int64 `json:"metadata_batch_guard_queued"` - MediaInfoCacheHitsTotal uint64 `json:"mediainfo_cache_hits_total"` - MediaInfoCacheMissesTotal uint64 `json:"mediainfo_cache_misses_total"` - MediaInfoProbeQueuedTotal uint64 `json:"mediainfo_probe_queued_total"` - MediaInfoProbeSuccessTotal uint64 `json:"mediainfo_probe_success_total"` - MediaInfoProbeFailureTotal uint64 `json:"mediainfo_probe_failure_total"` - MediaInfoStoreFailureTotal uint64 `json:"mediainfo_store_failure_total"` - MediaInfoProbeActive int64 `json:"mediainfo_probe_active"` - MediaInfoEnrichedTotal uint64 `json:"mediainfo_enriched_total"` - MediaInfoFailOpenTotal uint64 `json:"mediainfo_fail_open_total"` - MediaInfoWaitActive int64 `json:"mediainfo_wait_active"` - MediaInfoWaitRejectedTotal uint64 `json:"mediainfo_wait_rejected_total"` - MediaInfoPrewarmTriggeredTotal uint64 `json:"mediainfo_prewarm_triggered_total"` - MediaInfoPrewarmReplacedTotal uint64 `json:"mediainfo_prewarm_replaced_total"` - MediaInfoPrewarmDiscoverySuccessTotal uint64 `json:"mediainfo_prewarm_discovery_success_total"` - MediaInfoPrewarmDiscoveryFailureTotal uint64 `json:"mediainfo_prewarm_discovery_failure_total"` - MediaInfoPrewarmFreshCacheTotal uint64 `json:"mediainfo_prewarm_fresh_cache_total"` - MediaInfoPrewarmJoinedFlightTotal uint64 `json:"mediainfo_prewarm_joined_flight_total"` - MediaInfoPrewarmQueuedTotal uint64 `json:"mediainfo_prewarm_queued_total"` - MediaInfoPrewarmRejectedTotal uint64 `json:"mediainfo_prewarm_rejected_total"` - MediaInfoPrewarmSkippedTotal uint64 `json:"mediainfo_prewarm_skipped_total"` + MetadataGuardAdmittedTotal uint64 `json:"metadata_guard_admitted_total"` + MetadataGuardTimeoutsTotal uint64 `json:"metadata_guard_timeouts_total"` + MetadataGuardActive int64 `json:"metadata_guard_active"` + MetadataGuardQueued int64 `json:"metadata_guard_queued"` + MetadataBatchGuardAdmittedTotal uint64 `json:"metadata_batch_guard_admitted_total"` + MetadataBatchGuardTimeoutsTotal uint64 `json:"metadata_batch_guard_timeouts_total"` + MetadataBatchGuardActive int64 `json:"metadata_batch_guard_active"` + MetadataBatchGuardQueued int64 `json:"metadata_batch_guard_queued"` + MediaInfoCacheHitsTotal uint64 `json:"mediainfo_cache_hits_total"` + MediaInfoCacheMissesTotal uint64 `json:"mediainfo_cache_misses_total"` + MediaInfoProbeOfferedTotal uint64 `json:"mediainfo_probe_offered_total"` + MediaInfoProbeQueuedTotal uint64 `json:"mediainfo_probe_queued_total"` + MediaInfoProbeJoinedTotal uint64 `json:"mediainfo_probe_joined_total"` + MediaInfoProbePromotedTotal uint64 `json:"mediainfo_probe_promoted_total"` + MediaInfoProbeDroppedFullTotal uint64 `json:"mediainfo_probe_dropped_full_total"` + MediaInfoProbeDroppedExpiredTotal uint64 `json:"mediainfo_probe_dropped_expired_total"` + MediaInfoProbeSupersededTotal uint64 `json:"mediainfo_probe_superseded_total"` + MediaInfoProbeSuccessTotal uint64 `json:"mediainfo_probe_success_total"` + MediaInfoProbeFailureTotal uint64 `json:"mediainfo_probe_failure_total"` + MediaInfoStoreFailureTotal uint64 `json:"mediainfo_store_failure_total"` + MediaInfoProbeActive int64 `json:"mediainfo_probe_active"` + MediaInfoScheduler SchedulerSnapshot `json:"mediainfo_scheduler"` + MediaInfoEnrichedTotal uint64 `json:"mediainfo_enriched_total"` + MediaInfoFailOpenTotal uint64 `json:"mediainfo_fail_open_total"` + MediaInfoWaitActive int64 `json:"mediainfo_wait_active"` + MediaInfoWaitRejectedTotal uint64 `json:"mediainfo_wait_rejected_total"` + MediaInfoPrewarmTriggeredTotal uint64 `json:"mediainfo_prewarm_triggered_total"` + MediaInfoPrewarmReplacedTotal uint64 `json:"mediainfo_prewarm_replaced_total"` + MediaInfoPrewarmDiscoverySuccessTotal uint64 `json:"mediainfo_prewarm_discovery_success_total"` + MediaInfoPrewarmDiscoveryFailureTotal uint64 `json:"mediainfo_prewarm_discovery_failure_total"` + MediaInfoPrewarmFreshCacheTotal uint64 `json:"mediainfo_prewarm_fresh_cache_total"` + MediaInfoPrewarmJoinedFlightTotal uint64 `json:"mediainfo_prewarm_joined_flight_total"` + MediaInfoPrewarmQueuedTotal uint64 `json:"mediainfo_prewarm_queued_total"` + MediaInfoPrewarmRejectedTotal uint64 `json:"mediainfo_prewarm_rejected_total"` + MediaInfoPrewarmSkippedTotal uint64 `json:"mediainfo_prewarm_skipped_total"` ResolverLatencyMSTotal uint64 `json:"resolver_latency_ms_total"` ResolverLatencySamples uint64 `json:"resolver_latency_samples"` @@ -118,6 +144,25 @@ type Snapshot struct { MediaInfoProbeLatencyMSMax uint64 `json:"mediainfo_probe_latency_ms_max"` } +// SchedulerPrioritySnapshot contains one fixed MediaInfo priority class. The +// shape is intentionally label-free so request identities cannot expand it. +type SchedulerPrioritySnapshot struct { + OfferedTotal uint64 `json:"offered_total"` + QueuedTotal uint64 `json:"queued_total"` + JoinedTotal uint64 `json:"joined_total"` + PromotedTotal uint64 `json:"promoted_total"` + DroppedFullTotal uint64 `json:"dropped_full_total"` + DroppedExpiredTotal uint64 `json:"dropped_expired_total"` + SupersededTotal uint64 `json:"superseded_total"` +} + +// SchedulerSnapshot exposes only the fixed P0, P1, and P2 classes. +type SchedulerSnapshot struct { + P0 SchedulerPrioritySnapshot `json:"p0"` + P1 SchedulerPrioritySnapshot `json:"p1"` + P2 SchedulerPrioritySnapshot `json:"p2"` +} + // New creates an empty process-local metrics registry. func New() *Metrics { return &Metrics{} @@ -161,8 +206,47 @@ func (m *Metrics) IncMediaInfoCacheHits() { m.mediaInfoCacheHits.Add(1) } // IncMediaInfoCacheMisses records an exact key not satisfied by L1 or SQLite. func (m *Metrics) IncMediaInfoCacheMisses() { m.mediaInfoCacheMisses.Add(1) } +// IncMediaInfoProbeOffered records one valid admission attempt at the scheduler. +func (m *Metrics) IncMediaInfoProbeOffered(priority uint8) { + m.mediaInfoProbeOffered.Add(1) + incrementPriority(&m.mediaInfoScheduler.offered, priority) +} + // IncMediaInfoProbeQueued records a newly scheduled singleflight job. -func (m *Metrics) IncMediaInfoProbeQueued() { m.mediaInfoProbeQueued.Add(1) } +func (m *Metrics) IncMediaInfoProbeQueued(priority uint8) { + m.mediaInfoProbeQueued.Add(1) + incrementPriority(&m.mediaInfoScheduler.queued, priority) +} + +// IncMediaInfoProbeJoined records a signal merged into an exact-key flight. +func (m *Metrics) IncMediaInfoProbeJoined(priority uint8) { + m.mediaInfoProbeJoined.Add(1) + incrementPriority(&m.mediaInfoScheduler.joined, priority) +} + +// IncMediaInfoProbePromoted records an exact-key job raised to a higher class. +func (m *Metrics) IncMediaInfoProbePromoted(priority uint8) { + m.mediaInfoProbePromoted.Add(1) + incrementPriority(&m.mediaInfoScheduler.promoted, priority) +} + +// IncMediaInfoProbeDroppedFull records work rejected by its bounded queue. +func (m *Metrics) IncMediaInfoProbeDroppedFull(priority uint8) { + m.mediaInfoProbeDroppedFull.Add(1) + incrementPriority(&m.mediaInfoScheduler.droppedFull, priority) +} + +// IncMediaInfoProbeDroppedExpired records background work discarded before I/O. +func (m *Metrics) IncMediaInfoProbeDroppedExpired(priority uint8) { + m.mediaInfoProbeDroppedExpired.Add(1) + incrementPriority(&m.mediaInfoScheduler.droppedExpired, priority) +} + +// IncMediaInfoProbeSuperseded records stale fingerprint work replaced in queue. +func (m *Metrics) IncMediaInfoProbeSuperseded(priority uint8) { + m.mediaInfoProbeSuperseded.Add(1) + incrementPriority(&m.mediaInfoScheduler.superseded, priority) +} // IncMediaInfoProbeSuccess records a complete normalized probe result. func (m *Metrics) IncMediaInfoProbeSuccess() { m.mediaInfoProbeSuccess.Add(1) } @@ -387,11 +471,18 @@ func (m *Metrics) Snapshot() Snapshot { MetadataBatchGuardQueued: m.metadataBatchQueued.Load(), MediaInfoCacheHitsTotal: m.mediaInfoCacheHits.Load(), MediaInfoCacheMissesTotal: m.mediaInfoCacheMisses.Load(), + MediaInfoProbeOfferedTotal: m.mediaInfoProbeOffered.Load(), MediaInfoProbeQueuedTotal: m.mediaInfoProbeQueued.Load(), + MediaInfoProbeJoinedTotal: m.mediaInfoProbeJoined.Load(), + MediaInfoProbePromotedTotal: m.mediaInfoProbePromoted.Load(), + MediaInfoProbeDroppedFullTotal: m.mediaInfoProbeDroppedFull.Load(), + MediaInfoProbeDroppedExpiredTotal: m.mediaInfoProbeDroppedExpired.Load(), + MediaInfoProbeSupersededTotal: m.mediaInfoProbeSuperseded.Load(), MediaInfoProbeSuccessTotal: m.mediaInfoProbeSuccess.Load(), MediaInfoProbeFailureTotal: m.mediaInfoProbeFailure.Load(), MediaInfoStoreFailureTotal: m.mediaInfoStoreFailure.Load(), MediaInfoProbeActive: m.mediaInfoProbeActive.Load(), + MediaInfoScheduler: m.mediaInfoScheduler.snapshot(), MediaInfoEnrichedTotal: m.mediaInfoEnriched.Load(), MediaInfoFailOpenTotal: m.mediaInfoFailOpen.Load(), MediaInfoWaitActive: m.mediaInfoWaitActive.Load(), @@ -423,6 +514,32 @@ func (m *Metrics) Snapshot() Snapshot { } } +func incrementPriority(counters *[mediaInfoPriorityCount]atomic.Uint64, priority uint8) { + if priority < mediaInfoPriorityCount { + counters[priority].Add(1) + } +} + +func (metrics *schedulerMetrics) snapshot() SchedulerSnapshot { + return SchedulerSnapshot{ + P0: metrics.prioritySnapshot(0), + P1: metrics.prioritySnapshot(1), + P2: metrics.prioritySnapshot(2), + } +} + +func (metrics *schedulerMetrics) prioritySnapshot(priority int) SchedulerPrioritySnapshot { + return SchedulerPrioritySnapshot{ + OfferedTotal: metrics.offered[priority].Load(), + QueuedTotal: metrics.queued[priority].Load(), + JoinedTotal: metrics.joined[priority].Load(), + PromotedTotal: metrics.promoted[priority].Load(), + DroppedFullTotal: metrics.droppedFull[priority].Load(), + DroppedExpiredTotal: metrics.droppedExpired[priority].Load(), + SupersededTotal: metrics.superseded[priority].Load(), + } +} + func decrementGauge(gauge *atomic.Int64) { for { current := gauge.Load() diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 7cd1444..85662d1 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -29,7 +29,13 @@ func TestSnapshotAndJSONHandler(t *testing.T) { registry.IncMetadataBatchGuardQueued() registry.IncMediaInfoCacheHits() registry.IncMediaInfoCacheMisses() - registry.IncMediaInfoProbeQueued() + registry.IncMediaInfoProbeOffered(0) + registry.IncMediaInfoProbeQueued(0) + registry.IncMediaInfoProbeJoined(0) + registry.IncMediaInfoProbePromoted(0) + registry.IncMediaInfoProbeDroppedFull(0) + registry.IncMediaInfoProbeDroppedExpired(0) + registry.IncMediaInfoProbeSuperseded(0) registry.IncMediaInfoProbeSuccess() registry.IncMediaInfoProbeFailure() registry.IncMediaInfoStoreFailure() @@ -76,21 +82,31 @@ func TestSnapshotAndJSONHandler(t *testing.T) { PlexFallbackTotal: 1, ActiveRequests: 1, - MetadataGuardAdmittedTotal: 1, - MetadataGuardTimeoutsTotal: 1, - MetadataGuardActive: 1, - MetadataGuardQueued: 1, - MetadataBatchGuardAdmittedTotal: 1, - MetadataBatchGuardTimeoutsTotal: 1, - MetadataBatchGuardActive: 1, - MetadataBatchGuardQueued: 1, - MediaInfoCacheHitsTotal: 1, - MediaInfoCacheMissesTotal: 1, - MediaInfoProbeQueuedTotal: 1, - MediaInfoProbeSuccessTotal: 1, - MediaInfoProbeFailureTotal: 1, - MediaInfoStoreFailureTotal: 1, - MediaInfoProbeActive: 1, + MetadataGuardAdmittedTotal: 1, + MetadataGuardTimeoutsTotal: 1, + MetadataGuardActive: 1, + MetadataGuardQueued: 1, + MetadataBatchGuardAdmittedTotal: 1, + MetadataBatchGuardTimeoutsTotal: 1, + MetadataBatchGuardActive: 1, + MetadataBatchGuardQueued: 1, + MediaInfoCacheHitsTotal: 1, + MediaInfoCacheMissesTotal: 1, + MediaInfoProbeOfferedTotal: 1, + MediaInfoProbeQueuedTotal: 1, + MediaInfoProbeJoinedTotal: 1, + MediaInfoProbePromotedTotal: 1, + MediaInfoProbeDroppedFullTotal: 1, + MediaInfoProbeDroppedExpiredTotal: 1, + MediaInfoProbeSupersededTotal: 1, + MediaInfoProbeSuccessTotal: 1, + MediaInfoProbeFailureTotal: 1, + MediaInfoStoreFailureTotal: 1, + MediaInfoProbeActive: 1, + MediaInfoScheduler: SchedulerSnapshot{P0: SchedulerPrioritySnapshot{ + OfferedTotal: 1, QueuedTotal: 1, JoinedTotal: 1, PromotedTotal: 1, + DroppedFullTotal: 1, DroppedExpiredTotal: 1, SupersededTotal: 1, + }}, MediaInfoEnrichedTotal: 1, MediaInfoFailOpenTotal: 1, MediaInfoWaitActive: 1, @@ -224,7 +240,13 @@ func TestCountersAreSafeForConcurrentUpdates(t *testing.T) { registry.DecMetadataBatchGuardQueued() registry.IncMediaInfoCacheHits() registry.IncMediaInfoCacheMisses() - registry.IncMediaInfoProbeQueued() + registry.IncMediaInfoProbeOffered(0) + registry.IncMediaInfoProbeQueued(1) + registry.IncMediaInfoProbeJoined(2) + registry.IncMediaInfoProbePromoted(0) + registry.IncMediaInfoProbeDroppedFull(1) + registry.IncMediaInfoProbeDroppedExpired(2) + registry.IncMediaInfoProbeSuperseded(0) registry.IncMediaInfoProbeSuccess() registry.IncMediaInfoProbeFailure() registry.IncMediaInfoStoreFailure() @@ -262,7 +284,10 @@ func TestCountersAreSafeForConcurrentUpdates(t *testing.T) { got.MetadataBatchGuardAdmittedTotal != want || got.MetadataBatchGuardTimeoutsTotal != want || got.MetadataBatchGuardActive != 0 || got.MetadataBatchGuardQueued != 0 || got.MediaInfoCacheHitsTotal != want || got.MediaInfoCacheMissesTotal != want || - got.MediaInfoProbeQueuedTotal != want || got.MediaInfoProbeSuccessTotal != want || + got.MediaInfoProbeOfferedTotal != want || got.MediaInfoProbeQueuedTotal != want || got.MediaInfoProbeJoinedTotal != want || + got.MediaInfoProbePromotedTotal != want || got.MediaInfoProbeDroppedFullTotal != want || + got.MediaInfoProbeDroppedExpiredTotal != want || got.MediaInfoProbeSupersededTotal != want || + got.MediaInfoProbeSuccessTotal != want || got.MediaInfoProbeFailureTotal != want || got.MediaInfoStoreFailureTotal != want || got.MediaInfoProbeActive != 0 || got.MediaInfoEnrichedTotal != want || got.MediaInfoFailOpenTotal != want || got.MediaInfoWaitActive != 0 || got.MediaInfoWaitRejectedTotal != want || got.MediaInfoPrewarmTriggeredTotal != want || got.MediaInfoPrewarmReplacedTotal != want || diff --git a/internal/prewarm/service.go b/internal/prewarm/service.go index 5a8ab87..6d4507b 100644 --- a/internal/prewarm/service.go +++ b/internal/prewarm/service.go @@ -18,7 +18,6 @@ import ( const ( defaultDiscoveryTimeout = 15 * time.Second defaultTriggerCooldown = 5 * time.Minute - defaultSubmitInterval = 5 * time.Second defaultRecentLimit = 4096 defaultCurrentQueueSize = 64 defaultWindowGroupLimit = 8 @@ -33,7 +32,7 @@ type partPreparer interface { } type mediaInfoSubmitter interface { - SubmitDetailed(mediainfo.Request) mediainfo.SubmitResult + Offer(mediainfo.Request) mediainfo.SubmitResult } // Metrics receives fixed-cardinality prewarm lifecycle events. @@ -62,7 +61,6 @@ type ServiceOptions struct { TriggerCooldown time.Duration BeforeCount int AfterCount int - SubmitInterval time.Duration CurrentQueueSize int Now func() time.Time } @@ -96,7 +94,6 @@ type Service struct { cooldown time.Duration beforeCount int afterCount int - submitInterval time.Duration now func() time.Time ctx context.Context @@ -143,10 +140,6 @@ func NewService(options ServiceOptions) (*Service, error) { if afterCount < 0 { return nil, errors.New("prewarm after count is invalid") } - submitInterval := options.SubmitInterval - if submitInterval <= 0 { - submitInterval = defaultSubmitInterval - } queueSize := options.CurrentQueueSize if queueSize < 0 { return nil, errors.New("prewarm current queue size is invalid") @@ -163,7 +156,7 @@ func NewService(options ServiceOptions) (*Service, error) { discovery: options.Discovery, playback: options.Playback, mediaInfo: options.MediaInfo, logger: logger, metrics: options.Metrics, discoveryTimeout: discoveryTimeout, cooldown: cooldown, beforeCount: beforeCount, afterCount: afterCount, - submitInterval: submitInterval, now: now, ctx: ctx, cancel: cancel, + now: now, ctx: ctx, cancel: cancel, done: make(chan struct{}), currentQueue: make(chan currentJob, queueSize), currentRecent: make(map[string]time.Time), windowRecent: make(map[string]time.Time), windows: make(map[string]*windowState), @@ -295,7 +288,7 @@ func (service *Service) currentWorker() { submitted := service.submit(service.ctx, Candidate{ RatingKey: job.playback.RatingKey, Part: plexmeta.Part{ID: job.playback.PartID}, - }, job.playback.Target, job.playback.UserAgent, mediainfo.PriorityInteractive, "current") + }, job.playback.Target, job.playback.UserAgent, mediainfo.PriorityPlayback, "current") service.mu.Lock() if !submitted { @@ -369,10 +362,6 @@ func (service *Service) runWindow(ctx context.Context, current PlaybackContext) } service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmDiscoverySuccess() }) for _, candidate := range candidates { - if !waitForInterval(ctx, service.submitInterval) { - service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmSkipped() }) - return - } if ctx.Err() != nil { return } @@ -386,7 +375,7 @@ func (service *Service) runWindow(ctx context.Context, current PlaybackContext) } service.submit( ctx, candidate, preparation.Part.Target, current.UserAgent, - mediainfo.PriorityBackground, "neighbor", + mediainfo.PriorityNeighbor, "neighbor", ) } } @@ -406,7 +395,7 @@ func (service *Service) submit( service.metric(func(metrics Metrics) { metrics.IncMediaInfoPrewarmSkipped() }) return false } - result := service.mediaInfo.SubmitDetailed(mediainfo.Request{ + result := service.mediaInfo.Offer(mediainfo.Request{ Key: mediainfo.Key{PartID: candidate.Part.ID, STRMFingerprint: fingerprint}, RatingKey: candidate.RatingKey, Target: target, Priority: priority, ClientUserAgent: userAgent, @@ -429,26 +418,14 @@ func (service *Service) submit( return true } -func waitForInterval(ctx context.Context, interval time.Duration) bool { - timer := time.NewTimer(interval) - defer timer.Stop() - select { - case <-ctx.Done(): - return false - case <-timer.C: - return true - } -} - // Status is a credential-free coordinator snapshot. type Status struct { - Available bool `json:"available"` - NeighborAvailable bool `json:"neighbor_available"` - Active bool `json:"active"` - Queued int `json:"queued"` - Before int `json:"before"` - After int `json:"after"` - IntervalMS int64 `json:"interval_ms"` + Available bool `json:"available"` + NeighborAvailable bool `json:"neighbor_available"` + Active bool `json:"active"` + Queued int `json:"queued"` + Before int `json:"before"` + After int `json:"after"` } // Status reports bounded current and window queue state without media @@ -473,7 +450,6 @@ func (service *Service) Status() Status { Available: !service.closed, NeighborAvailable: !service.closed && service.discovery != nil, Active: active, Queued: queued, Before: service.beforeCount, After: service.afterCount, - IntervalMS: service.submitInterval.Milliseconds(), } } diff --git a/internal/prewarm/service_test.go b/internal/prewarm/service_test.go index 9dfe954..474fa70 100644 --- a/internal/prewarm/service_test.go +++ b/internal/prewarm/service_test.go @@ -26,7 +26,7 @@ func (function preparerFunc) Prepare(part plexmeta.Part) playback.Preparation { type submitterFunc func(mediainfo.Request) error -func (function submitterFunc) SubmitDetailed(request mediainfo.Request) mediainfo.SubmitResult { +func (function submitterFunc) Offer(request mediainfo.Request) mediainfo.SubmitResult { if err := function(request); err != nil { return mediainfo.SubmitResult{Disposition: mediainfo.SubmitRejected, Err: err} } @@ -35,7 +35,7 @@ func (function submitterFunc) SubmitDetailed(request mediainfo.Request) mediainf type detailedSubmitterFunc func(mediainfo.Request) mediainfo.SubmitResult -func (function detailedSubmitterFunc) SubmitDetailed(request mediainfo.Request) mediainfo.SubmitResult { +func (function detailedSubmitterFunc) Offer(request mediainfo.Request) mediainfo.SubmitResult { return function(request) } @@ -67,7 +67,7 @@ func TestServiceTryEnqueueDoesNoIOInline(t *testing.T) { } select { case request := <-submitted: - if request.Priority != mediainfo.PriorityInteractive || request.Key.PartID != "9" { + if request.Priority != mediainfo.PriorityPlayback || request.Key.PartID != "9" { t.Fatalf("current request = %#v", request) } case <-time.After(time.Second): @@ -81,7 +81,7 @@ func TestServiceTryEnqueueDoesNoIOInline(t *testing.T) { close(releaseDiscovery) } -func TestServiceSubmitsCurrentImmediatelyAndSpacesNeighbors(t *testing.T) { +func TestServiceOffersCurrentAndNeighborsWithoutLocalRateLimit(t *testing.T) { type observed struct { request mediainfo.Request at time.Time @@ -106,23 +106,24 @@ func TestServiceSubmitsCurrentImmediatelyAndSpacesNeighbors(t *testing.T) { submitted <- observed{request: request, at: time.Now()} return nil }), - BeforeCount: 2, AfterCount: 3, SubmitInterval: 25 * time.Millisecond, + BeforeCount: 2, AfterCount: 3, }) started := time.Now() service.TryEnqueue(testPlayback("42", "9", "current")) - first := <-submitted - second := <-submitted - third := <-submitted - if first.request.Key.PartID != "9" || first.request.Priority != mediainfo.PriorityInteractive || - second.request.Key.PartID != "10" || second.request.Priority != mediainfo.PriorityBackground || - third.request.Key.PartID != "8" || third.request.Priority != mediainfo.PriorityBackground { - t.Fatalf("submission order = %#v %#v %#v", first.request, second.request, third.request) - } - if first.at.Sub(started) > 20*time.Millisecond { - t.Fatalf("current submission delay = %s", first.at.Sub(started)) - } - if second.at.Sub(first.at) < 20*time.Millisecond || third.at.Sub(second.at) < 20*time.Millisecond { - t.Fatalf("neighbor spacing = %s, %s", second.at.Sub(first.at), third.at.Sub(second.at)) + observedByPart := make(map[string]observed, 3) + for range 3 { + item := <-submitted + observedByPart[item.request.Key.PartID] = item + } + if observedByPart["9"].request.Priority != mediainfo.PriorityPlayback || + observedByPart["10"].request.Priority != mediainfo.PriorityNeighbor || + observedByPart["8"].request.Priority != mediainfo.PriorityNeighbor { + t.Fatalf("submissions = %#v", observedByPart) + } + for partID, item := range observedByPart { + if delay := item.at.Sub(started); delay > 100*time.Millisecond { + t.Fatalf("part %s offer delay = %s", partID, delay) + } } } @@ -162,7 +163,7 @@ func TestServiceLatestPlaybackCancelsOlderNeighborDiscovery(t *testing.T) { for !seenCurrentC || !seenNeighbor { select { case request := <-submitted: - if request.Priority == mediainfo.PriorityInteractive { + if request.Priority == mediainfo.PriorityPlayback { seenCurrent[request.Key.PartID] = true seenCurrentC = request.Key.PartID == "11" || seenCurrentC } @@ -453,12 +454,12 @@ func TestServiceAllowsCurrentOnlyWindow(t *testing.T) { submitted <- request return nil }), - BeforeCount: 0, AfterCount: 0, SubmitInterval: time.Millisecond, + BeforeCount: 0, AfterCount: 0, }) service.TryEnqueue(testPlayback("42", "9", "current-only")) select { case request := <-submitted: - if request.Key.PartID != "9" || request.Priority != mediainfo.PriorityInteractive { + if request.Key.PartID != "9" || request.Priority != mediainfo.PriorityPlayback { t.Fatalf("request = %#v", request) } case <-time.After(time.Second): @@ -470,7 +471,7 @@ func newTestService(t *testing.T, discovery neighborDiscovery, preparer partPrep t.Helper() return newConfiguredTestService(t, ServiceOptions{ Discovery: discovery, Playback: preparer, MediaInfo: submitter, - BeforeCount: 2, AfterCount: 3, SubmitInterval: time.Millisecond, + BeforeCount: 2, AfterCount: 3, }) } From a39cacd68aea1c35242395dbc825899a03e37891 Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sun, 30 Aug 2026 01:08:03 +0800 Subject: [PATCH 10/15] fix: keep metadata cold misses off the request path --- CHANGELOG.en.md | 16 +- CHANGELOG.md | 12 +- README.en.md | 26 +- README.md | 19 +- app.env.example | 6 +- docs/architecture.md | 30 +- docs/media-lifecycle-architecture.md | 20 +- docs/performance-matrix.md | 9 + docs/roadmap.md | 11 +- internal/gateway/decision.go | 9 +- internal/gateway/gateway.go | 4 +- internal/gateway/metadata_enrichment.go | 63 +-- .../metadata_enrichment_integration_test.go | 6 +- internal/gateway/metadata_enrichment_test.go | 396 +++--------------- 14 files changed, 158 insertions(+), 469 deletions(-) diff --git a/CHANGELOG.en.md b/CHANGELOG.en.md index cf6425a..30a482e 100644 --- a/CHANGELOG.en.md +++ b/CHANGELOG.en.md @@ -14,8 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added bounded remote ffprobe, an L1 LRU plus SQLite MediaInfo cache, and - single-item Plex metadata response enrichment. Cold-wait timeouts preserve the - original response while probing continues in the background. + single-item Plex metadata response enrichment. Browsing misses preserve the + original response immediately while only playback decisions use a cold wait. - Current-Part prewarming is submitted to P0 after a cloud redirect is ready. An isolated Plex management token can additionally discover a configurable P1 nearby window, defaulting to two previous and three following items. The @@ -25,7 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added per-client rapid-switch coordination, cross-User-Agent failure handoff, and distinct prewarm metrics for fresh cache hits, joined work, new queue admissions, and rejections. -- Unified actual playback, nearby prewarming, and fixed-window metadata misses +- Unified actual playback, nearby prewarming, and foreground metadata misses under one P0/P1/P2 MediaInfo scheduler. P1/P2 use a five-minute pending TTL and a global five-second remote-start interval while P0 keeps reserved capacity. - Parts without Plex Streams now receive descriptive video, audio, and subtitle @@ -56,11 +56,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Limited one continuous metadata browsing burst to one synchronous cold probe. - After the admitted synchronous wait releases its slot, a fixed five-second - window begins. Other cold misses preserve the Plex response immediately and - may offer one deduplicated P2 task without renewing the window. Workers check - SQLite first, so only genuine misses enter the remote-start limiter. +- Single-item metadata browsing misses now preserve the Plex response + immediately and only perform a non-blocking, exact-key P2 memory offer. The + request path no longer reads SQLite synchronously, waits for MediaVault/CDN, + or holds a five-second cold-probe window. Workers check SQLite first, so only + genuine misses enter the remote-start limiter. - Set the Metadata Guard defaults to global 8, per-client 4, and batch 3. After Plex MediaInfo writes provide high coverage and the resulting load is retested, global 16, per-client 4, and batch 3 remains a candidate for evaluation. diff --git a/CHANGELOG.md b/CHANGELOG.md index f353869..4c3f646 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,14 +14,14 @@ ### 新增 - 增加 MediaInfo L1 LRU 与 SQLite 持久缓存、受限远程 ffprobe 和单项 Plex metadata - 响应增强；冷缓存等待超时后保持原响应并在后台完成探测。 + 响应增强。浏览冷 miss 立即保持原响应并在后台探测，实际播放 decision 使用独立冷等待。 - 云端重定向就绪后将当前 Part 投递 P0，并可使用独立 Plex 管理 Token 按默认前 2、后 3 的窗口发现并投递 P1 邻近媒体。当前任务只会优先排队，不能抢占正在运行的探测。 Token 缺失或失效只禁用邻近发现。 - 增加按客户端隔离的快速切换协调、跨 User-Agent 失败接力，以及缓存命中、加入现有 任务、新入队和拒绝的预热指标。 -- 使用统一的 P0/P1/P2 MediaInfo 调度器处理实际播放、邻近预热和 fixed-window metadata - miss。P1/P2 采用 5 分钟 pending TTL 和全局 5 秒远程启动间隔，P0 使用独立容量。 +- 使用统一的 P0/P1/P2 MediaInfo 调度器处理实际播放、邻近预热和前台 metadata miss。 + P1/P2 采用 5 分钟 pending TTL 和全局 5 秒远程启动间隔，P0 使用独立容量。 - 为没有 Stream 的 STRM Part 创建不含 Plex ID 或播放选择状态的描述性视频、音频和 字幕 Stream，并补充 HDR10、Dolby Vision、bit depth、bitrate、声道和语言等字段。 - Infuse 风格的后台媒体库同步只读取现有 MediaInfo 缓存，不再因逐项浏览触发全库冷探测。 @@ -41,9 +41,9 @@ ### 修复 -- 限制一次连续 metadata 浏览突发最多同步等待一个冷探测。释放本次同步准入后建立固定 - 5 秒窗口，窗口内的其他冷 miss 立即透传 Plex，并可非阻塞投递唯一 P2 任务；这些请求 - 不会续期窗口。后台 worker 先检查 SQLite，只有真实 miss 才进入远程启动限速。 +- 单项 metadata 浏览冷 miss 立即透传 Plex，并只进行非阻塞、精确去重的 P2 内存投递； + 请求路径不再同步查询 SQLite、等待 MediaVault/CDN 或持有 5 秒冷探测窗口。后台 worker + 先检查 SQLite，只有真实 miss 才进入远程启动限速。 - 将单项 metadata Guard 的默认值固定为全局 8、单客户端 4、批量 3。Plex MediaInfo 写库获得较高覆盖率并完成重测后，再评估全局 16、单客户端 4、批量 3 的候选值。 diff --git a/README.en.md b/README.en.md index 5adb166..1242aa3 100644 --- a/README.en.md +++ b/README.en.md @@ -132,14 +132,14 @@ Important environment variables: | `DATABASE_PATH` | `./data/plex-gateway.db` | Gateway SQLite database path; the container image defaults to `/app_data/plex-gateway.db`. | | `MEDIAINFO_PLAYBACK_QUEUE_SIZE` | `16` | Independent P0 queue capacity for actual playback work. | | `MEDIAINFO_NEIGHBOR_QUEUE_SIZE` | `50` | P1 queue capacity for nearby-item prewarming. | -| `MEDIAINFO_METADATA_QUEUE_SIZE` | `50` | P2 queue capacity for fixed-window metadata misses. | +| `MEDIAINFO_METADATA_QUEUE_SIZE` | `50` | P2 queue capacity for foreground metadata misses. | | `MEDIAINFO_PENDING_TTL` | `5m` | Lifetime of P1/P2 work that has not started. | | `MEDIAINFO_BACKGROUND_INTERVAL` | `5s` | Global minimum interval between P1/P2 remote probes after L1 and SQLite miss. | | `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.5.1` | Fallback User-Agent for background probes without an active client context. | -| `MEDIAINFO_COLD_WAIT` | `5s` | Cold-cache wait ceiling for one metadata item; timeout returns the original Plex response while probing continues. | +| `MEDIAINFO_COLD_WAIT` | `5s` | Cold-cache wait ceiling for an actual playback decision; metadata browsing misses never use this wait. | | `PLAYBACK_VETO_ENABLED` | `false` | Enable the experimental Apple TV Plex Dolby Vision Profile 5 veto; it only reuses fresh MediaInfo already obtained by the decision path. | | `MEDIAINFO_RESPONSE_MAX_BYTES` | `8388608` | Maximum size of one Plex metadata response buffered for enrichment. | -| `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `8` | Maximum single-item metadata responses buffered and waiting for MediaInfo concurrently. | +| `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `8` | Maximum single-item metadata responses buffered for concurrent L1 projection. | | `MEDIAINFO_PREWARM_BEFORE` | `2` | Number of nearby items before the current item to prewarm. | | `MEDIAINFO_PREWARM_AFTER` | `3` | Number of nearby items after the current item to prewarm. | @@ -197,15 +197,12 @@ evaluation after coverage, compatibility, backup, and rollback validation. The L1 hot path returns without synchronously reading SQLite for the request. Access renewal may touch SQLite asynchronously, so persistence I/O stays out of -the request critical path. Hot-cache projection remains concurrent. One -continuous browsing burst may admit only one synchronous remote probe. After -the admitted synchronous wait releases its slot, a fixed five-second window -begins. Other cold misses immediately preserve the Plex response, and rejected -requests do not renew the window. An admitted probe may finish in the background -after the request wait ends and persist the result to L1 and SQLite. Another -browsing probe is admitted only after the window ends. Playback decisions retain -their independent cold-probe budget, so this limit only suppresses metadata fan-out -from home and season screens. +the request critical path. Hot-cache projection remains concurrent. A metadata +browsing miss immediately preserves the Plex response and only offers bounded P2 +work in memory; it never waits for SQLite, MediaVault, or the CDN. Successful +background work later persists the result to L1 and SQLite. Actual playback +decisions retain an independent P0 cold-probe budget so playback negotiation can +receive technical fields without blocking home or season screens. When ffprobe cannot report the total media size, the gateway sends one `Range: bytes=0-0` request to the same temporary direct URL with the same @@ -220,9 +217,8 @@ do not repeat the CDN request. Background library synchronization can enumerate an entire library one item at a time. Requests with `skipRefresh` whose product ends in `-Library` may consume an existing MediaInfo cache record but never admit a cold probe. Ordinary -single-item access follows the same fixed cold-probe window. Cold misses inside -the window return the original response immediately and only offer bounded P2 -work without renewing it; successful cloud 302 and nearby-window work retain +single-item cold misses also return the original response immediately and only +offer bounded P2 work; successful cloud 302 and nearby-window work retain their existing boundaries. Browsing a library therefore cannot expand without bound into full-library CDN ffprobe work. Any cache miss, timeout, unsupported structure, or projection failure returns the original Plex response. diff --git a/README.md b/README.md index 771c6d4..e4cc39f 100644 --- a/README.md +++ b/README.md @@ -117,14 +117,14 @@ go run ./cmd/plex-gateway | `DATABASE_PATH` | `./data/plex-gateway.db` | Gateway SQLite 持久数据库路径；容器镜像默认使用 `/app_data/plex-gateway.db`。 | | `MEDIAINFO_PLAYBACK_QUEUE_SIZE` | `16` | P0 实际播放任务的独立队列容量。 | | `MEDIAINFO_NEIGHBOR_QUEUE_SIZE` | `50` | P1 邻近项预热的队列容量。 | -| `MEDIAINFO_METADATA_QUEUE_SIZE` | `50` | P2 fixed-window metadata miss 的队列容量。 | +| `MEDIAINFO_METADATA_QUEUE_SIZE` | `50` | 前台 metadata 冷 miss 的 P2 队列容量。 | | `MEDIAINFO_PENDING_TTL` | `5m` | P1/P2 未开始任务的保留时间。 | | `MEDIAINFO_BACKGROUND_INTERVAL` | `5s` | L1/SQLite miss 后 P1/P2 远程探测的全局最小启动间隔。 | | `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.5.1` | 没有活动客户端上下文时，后台探测使用的 fallback User-Agent。 | -| `MEDIAINFO_COLD_WAIT` | `5s` | 单项 metadata 冷缓存等待上限；超时返回原 Plex 响应，探测继续。 | +| `MEDIAINFO_COLD_WAIT` | `5s` | 实际播放 decision 的冷缓存等待上限；metadata 浏览冷 miss 不使用该等待。 | | `PLAYBACK_VETO_ENABLED` | `false` | 启用实验性 Apple TV Plex Dolby Vision Profile 5 veto；只复用 decision 已取得的新鲜 MediaInfo，不发起额外探测。 | | `MEDIAINFO_RESPONSE_MAX_BYTES` | `8388608` | 可以缓冲并尝试增强的单个 Plex metadata 响应上限。 | -| `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `8` | 同时缓冲和等待 MediaInfo 的单项 metadata 响应上限。 | +| `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `8` | 同时缓冲并尝试 L1 投影的单项 metadata 响应上限。 | | `MEDIAINFO_PREWARM_BEFORE` | `2` | 当前项之前的邻近媒体预热数量。 | | `MEDIAINFO_PREWARM_AFTER` | `3` | 当前项之后的邻近媒体预热数量。 | @@ -170,11 +170,10 @@ Stream 时只补充身份匹配 Stream 的缺失字段，不创建缺少的其 阶段评估内容，需先完成覆盖率、兼容性、备份和回滚验证。 L1 热路径直接返回，不会为了本次请求同步读取 SQLite。访问续期可能异步 touch SQLite， -不会把持久化 I/O 放进请求关键路径。热缓存投影可以并发执行，一次连续的浏览请求突发只 -允许一个请求同步等待远程探测。释放本次同步准入后建立固定 5 秒窗口，窗口内其他冷 miss -立即返回原始 Plex 响应，拒绝请求不会续期窗口。已准入的探测在请求等待结束后仍可在后台 -完成，并成功写入 L1 和 SQLite。窗口结束后才允许下一个浏览冷探测。播放 decision 保留 -独立的冷探测预算，因此该限制只抑制首屏和季页面的 metadata 请求风暴。 +不会把持久化 I/O 放进请求关键路径。热缓存投影可以并发执行；metadata 浏览冷 miss 立即 +返回原始 Plex 响应，只通过内存投递有界 P2，不等待 SQLite、MediaVault 或 CDN。后台任务 +成功后才写入 L1 和 SQLite。实际播放 decision 保留独立的 P0 冷探测预算，因此技术字段可 +在播放协商时补齐，而首屏和季页面不会被远程分析阻塞。 当 ffprobe 无法返回媒体总大小时，Gateway 会使用与本次 MediaVault 解析和 ffprobe 相同的 User-Agent 对同一临时直链发送一次 `Range: bytes=0-0`。该请求最多等待 `2s`，只接受包含有效 @@ -185,8 +184,8 @@ User-Agent 对同一临时直链发送一次 `Range: bytes=0-0`。该请求最 Infuse 等客户端的后台媒体库同步可能逐项请求整个库。带 `skipRefresh` 且产品名以 `-Library` 结尾的后台同步请求只读取现有 MediaInfo 缓存，不创建冷探测。普通单项访问 -遵循同一个固定冷探测窗口，窗口内的冷 miss 立即返回原始响应，仅非阻塞投递有界 P2 -任务且不会续期；成功的云端 302 和邻近窗口继续按既定边界提交。因此浏览媒体库不会无界 +的冷 miss 也立即返回原始响应，仅非阻塞投递有界 P2；成功的云端 302 和邻近窗口继续按 +既定边界提交。因此浏览媒体库不会无界 扩散成全库 CDN ffprobe。任何缓存 miss、超时、结构不支持或投影失败都会返回原始 Plex response。 diff --git a/app.env.example b/app.env.example index 46e7031..de113fe 100644 --- a/app.env.example +++ b/app.env.example @@ -84,7 +84,7 @@ MEDIAINFO_PLAYBACK_QUEUE_SIZE=16 # P1 当前播放项前后邻近预热任务的内存队列上限。 MEDIAINFO_NEIGHBOR_QUEUE_SIZE=50 -# P2 fixed-window 内前台 metadata miss 的内存队列上限。 +# 前台 metadata 冷 miss 的 P2 内存队列上限。 MEDIAINFO_METADATA_QUEUE_SIZE=50 # P1/P2 未开始任务的保留时间。过期任务不会访问 MediaVault 或 CDN。 @@ -108,7 +108,7 @@ MEDIAINFO_NEGATIVE_TTL=15m # 没有活动客户端上下文时，后台探测使用的 fallback User-Agent。 MEDIAINFO_USER_AGENT=Infuse-Library/8.5.1 -# 单项 metadata 冷缓存最多等待多久。超时后原响应立即返回，已准入的探测继续在后台执行。 +# 实际播放 decision 冷缓存最多等待多久。metadata 浏览冷 miss 不等待，直接投递 P2 并返回。 MEDIAINFO_COLD_WAIT=5s # 是否启用实验性 Apple TV Plex Dolby Vision Profile 5 播放 veto。默认关闭； @@ -118,7 +118,7 @@ PLAYBACK_VETO_ENABLED=false # 单个 Plex metadata 响应允许缓冲并尝试增强的最大字节数。 MEDIAINFO_RESPONSE_MAX_BYTES=8388608 -# 同时缓冲并等待 MediaInfo 的单项 metadata 响应数量。 +# 同时缓冲并尝试 L1 投影的单项 metadata 响应数量。 MEDIAINFO_ENRICHMENT_CONCURRENCY=8 # 当前播放项之前预热的邻近媒体数量。前后都设为 0 时仅提交当前项。 diff --git a/docs/architecture.md b/docs/architecture.md index 3461937..4c2546c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -33,12 +33,10 @@ gateway; the isolated analysis worker may read a bounded amount for probing. is bounded and preserves the original Plex response on timeout, ambiguity, unsupported encoding, or any other failure. Background library crawlers identified by `skipRefresh` and a `-Library` product consume cache only. Hot - records remain concurrently projectable; one continuous browsing burst may - wait on only one cold probe. After the admitted synchronous wait releases its - slot, a fixed five-second window begins. Other eligible cold misses preserve - the Plex response and may offer bounded P2 work without renewing the window. - An admitted probe may finish in the background after the request wait ends - and persist to L1 and SQLite. + records remain concurrently projectable. An eligible browsing miss preserves + the Plex response and may only offer bounded P2 work in memory; the request + does not wait for SQLite, MediaVault, or CDN work. A successful background + probe persists to L1 and SQLite for later responses. - `GET` and `HEAD` requests below `/library/parts/{partID}/...` are eligible for interception only when the cached `Part.file` has a configured cloud extension and maps to a readable local STRM file. @@ -172,15 +170,13 @@ Plex-owned. All clients use the same MediaInfo projection rules. Part and universal-start redirects never wait for MediaInfo. An L1 hit does not synchronously read SQLite -for the request; access renewal may touch SQLite asynchronously. An L1 miss may -use the persistent record or one bounded playback probe, and timeout or failure -preserves the existing Direct Play path. Metadata browsing has a separate single -synchronous cold-probe slot. After the admitted synchronous wait releases its -slot, a fixed five-second window begins. Misses inside that window do not renew -it: eligible foreground misses may be offered to the bounded background -scheduler, while the original Plex response is returned immediately. An -admitted probe can finish in the background and write L1 and SQLite after the -request waiter leaves. +for the request; access renewal may touch SQLite asynchronously. A playback +decision L1 miss may use the persistent record or one bounded P0 probe, and +timeout or failure preserves the existing Direct Play path. Metadata browsing +never waits for that work: an eligible foreground miss may only be offered to +the bounded P2 scheduler while the original Plex response is returned +immediately. Successful worker results populate L1 and SQLite for later +responses. ## MediaInfo scheduling contract @@ -192,8 +188,8 @@ priority classes: background work. 2. P1 represents the configured two previous and three following neighbors of an actual playback item. Its queue admits at most 50 entries. -3. P2 represents an eligible foreground metadata miss that could not use the - synchronous cold-probe slot. Its queue admits at most 50 entries. +3. P2 represents an eligible foreground metadata miss. Its queue admits at most + 50 entries, while the HTTP request returns the original Plex response. P1 and P2 pending entries expire after five minutes and share one global five-second remote start interval after both L1 and SQLite miss. Fresh SQLite diff --git a/docs/media-lifecycle-architecture.md b/docs/media-lifecycle-architecture.md index d20f72a..35ce3f4 100644 --- a/docs/media-lifecycle-architecture.md +++ b/docs/media-lifecycle-architecture.md @@ -526,12 +526,12 @@ ffprobe 成功但未返回 format size 时，Gateway 可以在同一 worker 中 在 fresh 期间不重复访问 CDN，fresh 到期后的 retained 记录仍可触发后台复验。 首次冷缓存长期返回空 MediaInfo 不可作为正常体验，metadata 请求一直无响应同样不可 -接受。处理顺序为：Gateway 启动时按需从 SQLite 回填 L1；精确冷 miss 时创建最高优先级 -任务，并允许当前单项 metadata response 在可配置的硬截止时间内等待。当前 PoC 候选 -上限为 `5s`；窗口内完成就补充当前 response，超时或失败必须立即返回原始 Plex -response，任务继续在后台完成。当前项云端重定向已就绪后，管理面只发现并限速预热 -可配置邻近窗口，不扩散为季、整剧或目录扫描。提高首次完整率应优先使用有界窗口，而不是 -无限增加请求等待或启动无界批量任务。 +接受。处理顺序为：Gateway 启动时按需从 SQLite 回填 L1；单项 metadata 精确冷 miss +立即返回原始 Plex response，只在内存中投递有界 P2，由 worker 异步查询 SQLite 并在 +必要时探测 MediaVault/CDN。实际播放 decision 的 P0 冷探测可以使用独立、可配置的硬 +截止时间，当前候选上限为 `5s`。当前项云端重定向已就绪后，管理面只发现并限速预热 +可配置邻近窗口，不扩散为季、整剧或目录扫描。提高首次完整率应优先使用有界后台窗口， +而不是增加浏览请求等待或启动无界批量任务。 Gateway 不缓存原始文件头尾。MediaVault 的上传预缓存利用上传时仍在本地的数据，适合 减少其后续扫描对 115 的请求，但只覆盖经 MediaVault 上传的新文件；Gateway 若从 CDN @@ -711,7 +711,7 @@ provider、目标 backing、decision revision、人工确认或策略依据， | MediaVault 缺少需要的 API | 标记 capability unavailable/manual，选择可控 provider 或人工检查点 | | STRM 半写入 | 生产者必须临时文件写完后原子 rename；Gateway 读取失败则 fallback Plex | | Plex 未扫描到 Part | 延迟重试定向刷新，不猜测 PartID，不写数据库 | -| MediaInfo 冷探测超时 | 在硬截止时间返回原始 Plex metadata，后台继续任务，不影响播放和观看状态 | +| MediaInfo 播放冷探测超时 | 在 decision 硬截止时间返回 Plex 原始结果，后台任务可继续，不影响观看状态；metadata 浏览不等待冷探测 | | MediaInfo 探测失败 | 保留已知良好记录并进入负缓存；没有旧记录时保持 Plex 原始字段 | | Plex 管理 Token 失效 | 停止邻近媒体发现；当前项预热和客户端请求继续工作 | | Plex DB helper 拒绝写入 | 保留权威 MediaInfo，播放按 Gateway 能力降级，不删除本地文件 | @@ -772,9 +772,9 @@ provider、目标 backing、decision revision、人工确认或策略依据， - 实现精确当前 Part 和重定向就绪后的双向邻近窗口，以及优先级、限速提交、singleflight、 推测任务抢占、负缓存和已知良好保护； - 将单季、整剧、STRM 目录任务、批任务 checkpoint 和恢复留到后续批量预热能力； -- 验证 `5s` 冷等待候选上限、首次 metadata 完整率和 metadata p50/p95/p99； -- 验证 decision、Part、universal start 和 302 不执行分析 I/O，单项 metadata 的有界等待 - 单独计入和验收； +- 验证 decision 的 `5s` 冷等待候选上限、首次 metadata 完整率和 metadata p50/p95/p99； +- 验证 Part、universal start 和 302 不执行分析 I/O，单项 metadata 冷 miss 立即透传且 + 只进行非阻塞 P2 投递； - 比较 Plex 官方 API、其他受支持 PMS 接口和独立 Plex Helper，不提前固定写入方案； - Helper 候选只在 Plex 停止或经过验证的维护窗口执行； - 测试库验证一致性备份与 restore、schema allowlist、CAS、写入和 API 回读； diff --git a/docs/performance-matrix.md b/docs/performance-matrix.md index 7c2ff68..cdd2b06 100644 --- a/docs/performance-matrix.md +++ b/docs/performance-matrix.md @@ -13,6 +13,8 @@ - 普通 STRM Part 和 universal start 不等待 MediaInfo，也不执行 veto。 - L1 命中直接返回，不因本次请求同步读取 SQLite。访问续期可能异步 touch SQLite， 因此持久化 I/O 不得阻塞热路径。 +- 单项 metadata 冷 miss 只允许内存级 P2 投递并立即返回 Plex 原响应，不同步读取 SQLite、 + 请求 MediaVault 或等待 CDN probe。 - 一个 universal decision 最多等待一次 `MEDIAINFO_COLD_WAIT`。veto 只复用响应增强已经 取得的新鲜记录，不增加缓存读取或远程探测。 - 同一 MediaInfo key 的并发请求必须共享 singleflight。并发数不能突破配置的 worker @@ -46,6 +48,13 @@ 批量 3 只有在 Plex MediaInfo 写库达到较高覆盖率并完成重新压测后才作为候选，不属于当前 版本的性能承诺。 +2026-08-30 的 NAS 阻塞 A/B 将旧候选判定为不可接受：同一冷 metadata 请求直连 Plex +约 `0.52s`，经旧候选约 `5.15s`，Gateway 热缓存约 `0.15s`。同期实际 CDN ffprobe 平均 +约 `0.54s`、最大约 `0.84s`，且没有 probe 失败。约五秒的增量来自 metadata 请求同步等待 +后台调度窗口并持有 Guard 准入，不是 ffprobe 执行时间。该结果确立了“浏览 miss 立即透传， +仅实际播放 decision 可使用冷等待预算”的当前门槛；修正版仍需在同一 NAS 和客户端路径 +重新采集端到端分位数。 + ## 端到端验收矩阵 | 场景 | 请求期允许的 I/O | 必须采集 | 验收条件 | diff --git a/docs/roadmap.md b/docs/roadmap.md index 22ceff3..c6cda08 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -78,11 +78,9 @@ interactive probe; missing, slow, or failed probes preserve the original Plex response. Part and universal-start redirects never wait for MediaInfo. All clients use the same projection rules. An L1 hit does not synchronously read SQLite for the request, although access renewal may touch SQLite asynchronously. -One metadata browsing burst permits one synchronous cold probe. After the -admitted synchronous wait releases its slot, a fixed five-second window begins. -Cold misses inside that window fail open immediately and may offer bounded P2 -work without renewing it. Background work writes L1 and SQLite only after a -successful probe. +A metadata browsing miss returns the original Plex response immediately and may +only offer bounded P2 work in memory. It never waits for SQLite, MediaVault, or +the CDN. Background work writes L1 and SQLite only after a successful probe. A default-disabled experimental veto may reuse an already available fresh record for the Apple TV Plex DV Profile 5 combination, but it performs no additional probe and owns no Part state. @@ -93,7 +91,8 @@ D prioritizes the Gateway fallback while preserving the broader lifecycle design 1. use L1 and SQLite for single-instance fallback storage; 2. accept optional `PLEX_TOKEN` from an ignored deployment `app.env` for nearby-item discovery; current-item prewarming remains token-independent; -3. use bounded remote `ffprobe` with a default `5s` cold metadata wait ceiling; +3. use bounded remote `ffprobe` with a default `5s` cold playback-decision wait + ceiling; metadata browsing does not consume that budget; 4. support exact current-Part analysis and a configurable nearby-item window after a cloud redirect is ready. P0 actual playback, P1 nearby items, and P2 foreground metadata misses share one exact-key scheduler. P1/P2 remote starts diff --git a/internal/gateway/decision.go b/internal/gateway/decision.go index b53c6d2..9ac0b93 100644 --- a/internal/gateway/decision.go +++ b/internal/gateway/decision.go @@ -19,9 +19,10 @@ import ( ) const ( - defaultDecisionMetadataMaxBytes = 4 << 20 - defaultDecisionGrantTTL = 5 * time.Minute - defaultDecisionGrantLimit = 4096 + defaultDecisionMetadataMaxBytes = 4 << 20 + defaultDecisionGrantTTL = 5 * time.Minute + defaultDecisionGrantLimit = 4096 + defaultDecisionMediaInfoColdWait = 5 * time.Second ) // decisionHandler opts an eligible STRM Part into Plex Direct Play while @@ -129,7 +130,7 @@ func (h *decisionHandler) mediaInfoForDecision(request *http.Request, part playb } wait := h.coldWait if wait <= 0 { - wait = defaultMediaInfoColdWait + wait = defaultDecisionMediaInfoColdWait } ctx, cancel := context.WithTimeout(request.Context(), wait) defer cancel() diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index f7899d4..9f1dfbf 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -207,8 +207,8 @@ func New(options Options) http.Handler { mux.Handle("HEAD /library/parts/{partID}/{rest...}", partPlayback) metadata := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ Service: options.MediaInfoService, Mapper: options.PathMapper, Resolver: options.Resolver, - CloudExtensions: options.CloudExtensions, ColdWait: options.MediaInfoColdWait, - ResponseLimit: options.MediaInfoResponseMaxBytes, Concurrency: options.MediaInfoEnrichmentConcurrency, + CloudExtensions: options.CloudExtensions, + ResponseLimit: options.MediaInfoResponseMaxBytes, Concurrency: options.MediaInfoEnrichmentConcurrency, Metrics: registry, }, plex) // Admission must happen before response buffering so a queued request does diff --git a/internal/gateway/metadata_enrichment.go b/internal/gateway/metadata_enrichment.go index 627d25d..3187d4f 100644 --- a/internal/gateway/metadata_enrichment.go +++ b/internal/gateway/metadata_enrichment.go @@ -10,7 +10,6 @@ import ( "path" "strconv" "strings" - "sync" "time" "github.com/InfinityPacer/plex-gateway/internal/mediainfo" @@ -21,9 +20,7 @@ import ( ) const ( - defaultMediaInfoColdWait = 5 * time.Second defaultMediaInfoCacheLookupWait = 100 * time.Millisecond - defaultMetadataColdProbeQuiet = 5 * time.Second defaultMediaInfoResponseLimit = 8 << 20 defaultMediaInfoEnrichmentSlots = 8 ) @@ -31,7 +28,6 @@ const ( type mediaInfoEnsurer interface { GetMemory(mediainfo.Key) (mediainfo.Record, bool) GetContext(context.Context, mediainfo.Key) (mediainfo.Record, bool) - Ensure(context.Context, mediainfo.Request) (mediainfo.Record, error) Offer(mediainfo.Request) mediainfo.SubmitResult } @@ -43,7 +39,6 @@ type metadataEnrichmentOptions struct { Mapper *pathmap.Mapper Resolver resolver.ControlResolver CloudExtensions []string - ColdWait time.Duration ResponseLimit int64 Concurrency int Metrics *metrics.Metrics @@ -55,26 +50,15 @@ type metadataEnrichmentHandler struct { mapper *pathmap.Mapper resolver resolver.ControlResolver cloudExtensions map[string]struct{} - coldWait time.Duration responseLimit int64 waiters chan struct{} metrics *metrics.Metrics - - coldProbeMu sync.Mutex - coldProbeActive bool - coldProbeBlockedUntil time.Time - coldProbeQuiet time.Duration - now func() time.Time } func newMetadataEnrichmentHandler(options metadataEnrichmentOptions, next http.Handler) http.Handler { if next == nil || options.Service == nil || options.Mapper == nil || options.Resolver == nil { return next } - coldWait := options.ColdWait - if coldWait <= 0 { - coldWait = defaultMediaInfoColdWait - } responseLimit := options.ResponseLimit if responseLimit <= 0 { responseLimit = defaultMediaInfoResponseLimit @@ -89,9 +73,8 @@ func newMetadataEnrichmentHandler(options metadataEnrichmentOptions, next http.H } return &metadataEnrichmentHandler{ next: next, service: options.Service, mapper: options.Mapper, resolver: options.Resolver, - cloudExtensions: normalizedExtensionSet(options.CloudExtensions), coldWait: coldWait, - responseLimit: responseLimit, waiters: make(chan struct{}, concurrency), - coldProbeQuiet: defaultMetadataColdProbeQuiet, now: time.Now, metrics: registry, + cloudExtensions: normalizedExtensionSet(options.CloudExtensions), + responseLimit: responseLimit, waiters: make(chan struct{}, concurrency), metrics: registry, } } @@ -170,20 +153,9 @@ func (handler *metadataEnrichmentHandler) ServeHTTP(w http.ResponseWriter, reque Key: key, RatingKey: ratingKey, Target: strmTarget, Priority: mediainfo.PriorityMetadata, ClientUserAgent: request.UserAgent(), } - if !handler.acquireColdProbe() { - handler.metrics.IncMediaInfoWaitRejected() - _ = handler.service.Offer(probeRequest) - handler.replay(capture, rawBody) - return - } - defer handler.releaseColdProbe() - waitContext, cancel := context.WithTimeout(request.Context(), handler.coldWait) - record, err = handler.service.Ensure(waitContext, probeRequest) - cancel() - if err != nil { - handler.failOpen(capture, rawBody) - return - } + _ = handler.service.Offer(probeRequest) + handler.replay(capture, rawBody) + return } } currentTarget, err := handler.resolver.ReadTarget(localPath) @@ -208,31 +180,6 @@ func (handler *metadataEnrichmentHandler) ServeHTTP(w http.ResponseWriter, reque handler.metrics.IncMediaInfoEnriched() } -// A completed probe opens a fixed quiet interval. Rejected requests never move -// its end, so continuous metadata traffic cannot starve the next probe. -func (handler *metadataEnrichmentHandler) acquireColdProbe() bool { - handler.coldProbeMu.Lock() - defer handler.coldProbeMu.Unlock() - - now := handler.now() - if handler.coldProbeActive || now.Before(handler.coldProbeBlockedUntil) { - return false - } - handler.coldProbeActive = true - return true -} - -func (handler *metadataEnrichmentHandler) releaseColdProbe() { - handler.coldProbeMu.Lock() - defer handler.coldProbeMu.Unlock() - if !handler.coldProbeActive { - return - } - now := handler.now() - handler.coldProbeActive = false - handler.coldProbeBlockedUntil = now.Add(handler.coldProbeQuiet) -} - // serveCachedCollection enriches already-probed items without admitting remote // analysis. Browsing a season therefore cannot turn one response into a burst // of MediaVault or CDN requests. diff --git a/internal/gateway/metadata_enrichment_integration_test.go b/internal/gateway/metadata_enrichment_integration_test.go index edf5e26..035b120 100644 --- a/internal/gateway/metadata_enrichment_integration_test.go +++ b/internal/gateway/metadata_enrichment_integration_test.go @@ -46,7 +46,7 @@ func (provider *delayedIntegrationProvider) Probe(ctx context.Context, _ mediain return mediainfo.ProviderResult{Media: completeProjectionMedia()}, nil } -func TestGatewayColdMetadataTimeoutContinuesProbeAndWarmsNextResponse(t *testing.T) { +func TestGatewayColdMetadataReturnsBeforeProbeAndWarmsNextResponse(t *testing.T) { raw := []byte(``) gzippedRaw := gzipBytes(t, raw) plex := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { @@ -136,7 +136,7 @@ func TestGatewayColdMetadataTimeoutContinuesProbeAndWarmsNextResponse(t *testing } select { case <-provider.started: - default: + case <-time.After(time.Second): t.Fatal("cold request did not start the background probe") } close(provider.release) @@ -163,7 +163,7 @@ func TestGatewayColdMetadataTimeoutContinuesProbeAndWarmsNextResponse(t *testing t.Fatalf("provider calls = %d, want 1", provider.calls.Load()) } snapshot := registry.Snapshot() - if snapshot.MediaInfoFailOpenTotal != 1 || snapshot.MediaInfoEnrichedTotal != 1 { + if snapshot.MediaInfoFailOpenTotal != 0 || snapshot.MediaInfoEnrichedTotal != 1 { t.Fatalf("metrics = %#v", snapshot) } diff --git a/internal/gateway/metadata_enrichment_test.go b/internal/gateway/metadata_enrichment_test.go index efba7fb..2f06329 100644 --- a/internal/gateway/metadata_enrichment_test.go +++ b/internal/gateway/metadata_enrichment_test.go @@ -5,7 +5,6 @@ import ( "compress/gzip" "context" "encoding/json" - "errors" "io" "net/http" "net/http/httptest" @@ -25,16 +24,13 @@ import ( ) type fakeMediaInfoEnsurer struct { - calls atomic.Int64 gets atomic.Int64 memoryGets atomic.Int64 mu sync.Mutex - seen []mediainfo.Request offered []mediainfo.Request memoryFn func(mediainfo.Key) (mediainfo.Record, bool) getFn func(mediainfo.Key) (mediainfo.Record, bool) getContextFn func(context.Context, mediainfo.Key) (mediainfo.Record, bool) - fn func(context.Context, mediainfo.Request) (mediainfo.Record, error) } func (ensurer *fakeMediaInfoEnsurer) GetMemory(key mediainfo.Key) (mediainfo.Record, bool) { @@ -64,10 +60,6 @@ func TestMetadataEnrichmentCollectionFailsOpenWhenCacheStoreStalls(t *testing.T) <-ctx.Done() return mediainfo.Record{}, false }, - fn: func(context.Context, mediainfo.Request) (mediainfo.Record, error) { - t.Fatal("collection metadata must not admit a cold probe") - return mediainfo.Record{}, nil - }, } handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(raw, "application/xml")) request := authenticatedRequest(http.MethodGet, "/library/metadata/41/children") @@ -82,14 +74,6 @@ func TestMetadataEnrichmentCollectionFailsOpenWhenCacheStoreStalls(t *testing.T) } } -func (ensurer *fakeMediaInfoEnsurer) Ensure(ctx context.Context, request mediainfo.Request) (mediainfo.Record, error) { - ensurer.calls.Add(1) - ensurer.mu.Lock() - ensurer.seen = append(ensurer.seen, request) - ensurer.mu.Unlock() - return ensurer.fn(ctx, request) -} - func (ensurer *fakeMediaInfoEnsurer) Offer(request mediainfo.Request) mediainfo.SubmitResult { ensurer.mu.Lock() ensurer.offered = append(ensurer.offered, request) @@ -97,12 +81,6 @@ func (ensurer *fakeMediaInfoEnsurer) Offer(request mediainfo.Request) mediainfo. return mediainfo.SubmitResult{Disposition: mediainfo.SubmitNewlyQueued} } -func (ensurer *fakeMediaInfoEnsurer) requests() []mediainfo.Request { - ensurer.mu.Lock() - defer ensurer.mu.Unlock() - return append([]mediainfo.Request(nil), ensurer.seen...) -} - func (ensurer *fakeMediaInfoEnsurer) offers() []mediainfo.Request { ensurer.mu.Lock() defer ensurer.mu.Unlock() @@ -113,8 +91,8 @@ func TestMetadataEnrichmentAddsMissingXMLFields(t *testing.T) { fixture := newMetadataEnrichmentFixture(t) raw := []byte(``) registry := metrics.New() - ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { - return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil + ensurer := &fakeMediaInfoEnsurer{memoryFn: func(key mediainfo.Key) (mediainfo.Record, bool) { + return mediainfo.Record{Key: key, Media: completeProjectionMedia()}, true }} handler := fixture.handler(ensurer, registry, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/xml") @@ -142,9 +120,8 @@ func TestMetadataEnrichmentAddsMissingXMLFields(t *testing.T) { if response.Header().Get("Cache-Control") != "private, no-cache" || response.Header().Get("Content-Length") != strconv.Itoa(response.Body.Len()) { t.Fatalf("mutated headers = %#v", response.Header()) } - requests := ensurer.requests() - if len(requests) != 1 || requests[0].Key.PartID != "9" || len(requests[0].Key.STRMFingerprint) != 64 || requests[0].Target != fixture.target || requests[0].ClientUserAgent != "Infuse-Test/1.0" { - t.Fatalf("MediaInfo request = %#v", requests) + if ensurer.memoryGets.Load() != 1 || len(ensurer.offers()) != 0 { + t.Fatalf("memory gets=%d offers=%d", ensurer.memoryGets.Load(), len(ensurer.offers())) } snapshot := registry.Snapshot() if snapshot.MediaInfoEnrichedTotal != 1 || snapshot.MediaInfoWaitActive != 0 || snapshot.MediaInfoFailOpenTotal != 0 { @@ -156,8 +133,8 @@ func TestMetadataEnrichmentPreservesGzipJSON(t *testing.T) { fixture := newMetadataEnrichmentFixture(t) rawJSON := []byte(`{"MediaContainer":{"Metadata":[{"ratingKey":"42","type":"episode","Media":[{"Part":[{"id":"9","key":"/library/parts/9/1/file.strm","file":"/media/cloud/episode.strm","Stream":[{"id":"101","streamType":1,"index":0}]}]}]}]}}`) wire := gzipBytes(t, rawJSON) - ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { - return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil + ensurer := &fakeMediaInfoEnsurer{memoryFn: func(key mediainfo.Key) (mediainfo.Record, bool) { + return mediainfo.Record{Key: key, Media: completeProjectionMedia()}, true }} handler := fixture.handler(ensurer, metrics.New(), http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -185,10 +162,7 @@ func TestMetadataEnrichmentPreservesGzipJSON(t *testing.T) { func TestMetadataEnrichmentKeepsBackgroundLibrarySyncCacheOnly(t *testing.T) { fixture := newMetadataEnrichmentFixture(t) raw := fixture.xmlBody() - ensurer := &fakeMediaInfoEnsurer{fn: func(context.Context, mediainfo.Request) (mediainfo.Record, error) { - t.Fatal("background library sync must not admit a cold probe") - return mediainfo.Record{}, nil - }} + ensurer := &fakeMediaInfoEnsurer{} handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(raw, "application/xml")) request := authenticatedRequest(http.MethodGet, "/library/metadata/42?skipRefresh=1") request.Header.Set("X-Plex-Product", "Infuse-Library") @@ -199,8 +173,8 @@ func TestMetadataEnrichmentKeepsBackgroundLibrarySyncCacheOnly(t *testing.T) { if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), raw) { t.Fatalf("response = %d %q", response.Code, response.Body.Bytes()) } - if ensurer.gets.Load() != 1 || ensurer.calls.Load() != 0 { - t.Fatalf("Get calls = %d, Ensure calls = %d", ensurer.gets.Load(), ensurer.calls.Load()) + if ensurer.gets.Load() != 1 || len(ensurer.offers()) != 0 { + t.Fatalf("Get calls = %d", ensurer.gets.Load()) } } @@ -219,8 +193,8 @@ func TestMetadataEnrichmentUsesCachedRecordForBackgroundLibrarySync(t *testing.T if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `container="mkv"`) { t.Fatalf("response = %d %q", response.Code, response.Body.String()) } - if ensurer.gets.Load() != 1 || ensurer.calls.Load() != 0 { - t.Fatalf("Get calls = %d, Ensure calls = %d", ensurer.gets.Load(), ensurer.calls.Load()) + if ensurer.gets.Load() != 1 { + t.Fatalf("Get calls = %d", ensurer.gets.Load()) } } @@ -234,10 +208,6 @@ func TestMetadataEnrichmentUsesOnlyCachedRecordsForCollection(t *testing.T) { } return mediainfo.Record{Key: key, Media: completeProjectionMedia()}, true }, - fn: func(context.Context, mediainfo.Request) (mediainfo.Record, error) { - t.Fatal("collection metadata must not admit a cold probe") - return mediainfo.Record{}, nil - }, } handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(raw, "application/xml")) response := httptest.NewRecorder() @@ -249,8 +219,8 @@ func TestMetadataEnrichmentUsesOnlyCachedRecordsForCollection(t *testing.T) { if !bytes.Contains(response.Body.Bytes(), []byte(`ratingKey="43">`) ensurer := &fakeMediaInfoEnsurer{getFn: func(mediainfo.Key) (mediainfo.Record, bool) { return mediainfo.Record{}, false - }, fn: func(context.Context, mediainfo.Request) (mediainfo.Record, error) { - t.Fatal("collection cache miss must not admit a cold probe") - return mediainfo.Record{}, nil }} handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(raw, "application/xml")) response := httptest.NewRecorder() @@ -270,8 +237,8 @@ func TestMetadataEnrichmentLeavesCollectionUnchangedOnCacheMiss(t *testing.T) { if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), raw) { t.Fatalf("response = %d %q", response.Code, response.Body.Bytes()) } - if ensurer.gets.Load() != 1 || ensurer.calls.Load() != 0 { - t.Fatalf("Get calls = %d, Ensure calls = %d", ensurer.gets.Load(), ensurer.calls.Load()) + if ensurer.gets.Load() != 1 { + t.Fatalf("Get calls = %d", ensurer.gets.Load()) } } @@ -298,38 +265,39 @@ func TestMediaInfoCacheOnlyRequestRequiresLibraryProductAndSkipRefresh(t *testin } } -func TestMetadataEnrichmentTimeoutReturnsOriginalResponse(t *testing.T) { +func TestMetadataEnrichmentColdMissReturnsOriginalAndOffersP2(t *testing.T) { fixture := newMetadataEnrichmentFixture(t) raw := fixture.xmlBody() - ensurer := &fakeMediaInfoEnsurer{fn: func(ctx context.Context, _ mediainfo.Request) (mediainfo.Record, error) { - <-ctx.Done() - return mediainfo.Record{}, ctx.Err() - }} + ensurer := &fakeMediaInfoEnsurer{} handler := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, - CloudExtensions: []string{".strm"}, ColdWait: 10 * time.Millisecond, - ResponseLimit: 1 << 20, Concurrency: 1, Metrics: metrics.New(), + CloudExtensions: []string{".strm"}, + ResponseLimit: 1 << 20, Concurrency: 1, Metrics: metrics.New(), }, metadataBodyHandler(raw, "application/xml")) response := httptest.NewRecorder() started := time.Now() handler.ServeHTTP(response, authenticatedMetadataRequest(http.MethodGet)) if elapsed := time.Since(started); elapsed > 500*time.Millisecond { - t.Fatalf("cold timeout took %s", elapsed) + t.Fatalf("cold miss took %s", elapsed) } if !bytes.Equal(response.Body.Bytes(), raw) || response.Header().Get("ETag") != `"plex-etag"` { t.Fatalf("fail-open response changed: headers=%#v body=%s", response.Header(), response.Body.Bytes()) } + offered := ensurer.offers() + if len(offered) != 1 || offered[0].Priority != mediainfo.PriorityMetadata || offered[0].Key.PartID != "9" { + t.Fatalf("P2 offers = %#v", offered) + } } func TestMetadataEnrichmentRejectsFingerprintChange(t *testing.T) { fixture := newMetadataEnrichmentFixture(t) raw := fixture.xmlBody() - ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { + ensurer := &fakeMediaInfoEnsurer{memoryFn: func(key mediainfo.Key) (mediainfo.Record, bool) { if err := os.WriteFile(fixture.strmPath, []byte("http://mediavault:7811/redirect/changed/file.mkv\n"), 0o600); err != nil { t.Fatal(err) } - return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil + return mediainfo.Record{Key: key, Media: completeProjectionMedia()}, true }} handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(raw, "application/xml")) response := httptest.NewRecorder() @@ -344,17 +312,22 @@ func TestMetadataEnrichmentPoolSaturationSkipsWaiting(t *testing.T) { raw := fixture.xmlBody() started := make(chan struct{}) release := make(chan struct{}) - ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { - close(started) - <-release - return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil + ensurer := &fakeMediaInfoEnsurer{memoryFn: func(key mediainfo.Key) (mediainfo.Record, bool) { + return mediainfo.Record{Key: key, Media: completeProjectionMedia()}, true }} registry := metrics.New() + var upstreamCalls atomic.Int64 handler := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, - CloudExtensions: []string{".strm"}, ColdWait: time.Second, - ResponseLimit: 1 << 20, Concurrency: 1, Metrics: registry, - }, metadataBodyHandler(raw, "application/xml")) + CloudExtensions: []string{".strm"}, + ResponseLimit: 1 << 20, Concurrency: 1, Metrics: registry, + }, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if upstreamCalls.Add(1) == 1 { + close(started) + <-release + } + metadataBodyHandler(raw, "application/xml").ServeHTTP(w, request) + })) firstDone := make(chan *httptest.ResponseRecorder, 1) go func() { @@ -369,8 +342,8 @@ func TestMetadataEnrichmentPoolSaturationSkipsWaiting(t *testing.T) { } second := httptest.NewRecorder() handler.ServeHTTP(second, authenticatedMetadataRequest(http.MethodGet)) - if !bytes.Equal(second.Body.Bytes(), raw) || ensurer.calls.Load() != 1 { - t.Fatalf("saturated response=%s calls=%d", second.Body.Bytes(), ensurer.calls.Load()) + if !bytes.Equal(second.Body.Bytes(), raw) { + t.Fatalf("saturated response=%s", second.Body.Bytes()) } close(release) select { @@ -386,190 +359,9 @@ func TestMetadataEnrichmentPoolSaturationSkipsWaiting(t *testing.T) { } } -func TestMetadataEnrichmentColdProbeSaturationFailsOpen(t *testing.T) { - fixture := newMetadataEnrichmentFixture(t) - raw := fixture.xmlBody() - started := make(chan struct{}) - release := make(chan struct{}) - ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { - close(started) - <-release - return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil - }} - registry := metrics.New() - handler := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ - Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, - CloudExtensions: []string{".strm"}, ColdWait: time.Second, - ResponseLimit: 1 << 20, Concurrency: 2, Metrics: registry, - }, metadataBodyHandler(raw, "application/xml")) - - firstDone := make(chan *httptest.ResponseRecorder, 1) - go func() { - response := httptest.NewRecorder() - handler.ServeHTTP(response, authenticatedMetadataRequest(http.MethodGet)) - firstDone <- response - }() - select { - case <-started: - case <-time.After(time.Second): - t.Fatal("first cold probe did not start") - } - - second := httptest.NewRecorder() - handler.ServeHTTP(second, authenticatedMetadataRequest(http.MethodGet)) - if !bytes.Equal(second.Body.Bytes(), raw) || ensurer.calls.Load() != 1 { - t.Fatalf("second response=%s Ensure calls=%d", second.Body.Bytes(), ensurer.calls.Load()) - } - if offered := ensurer.offers(); len(offered) != 1 || offered[0].Priority != mediainfo.PriorityMetadata { - t.Fatalf("background offers = %#v", offered) - } - close(release) - select { - case first := <-firstDone: - if bytes.Equal(first.Body.Bytes(), raw) { - t.Fatal("first response was not enriched") - } - case <-time.After(time.Second): - t.Fatal("first cold probe did not finish") - } - if snapshot := registry.Snapshot(); snapshot.MediaInfoWaitRejectedTotal != 1 || snapshot.MediaInfoWaitActive != 0 { - t.Fatalf("metrics = %#v", snapshot) - } -} - -func TestMetadataEnrichmentColdProbeQuietWindowSuppressesSequentialMiss(t *testing.T) { - fixture := newMetadataEnrichmentFixture(t) - raw := fixture.xmlBody() - ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { - return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil - }} - handler := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ - Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, - CloudExtensions: []string{".strm"}, ColdWait: time.Second, - ResponseLimit: 1 << 20, Concurrency: 2, Metrics: metrics.New(), - }, metadataBodyHandler(raw, "application/xml")) - - first := httptest.NewRecorder() - handler.ServeHTTP(first, authenticatedMetadataRequest(http.MethodGet)) - if bytes.Equal(first.Body.Bytes(), raw) { - t.Fatal("first cold miss was not enriched") - } - - second := httptest.NewRecorder() - handler.ServeHTTP(second, authenticatedMetadataRequest(http.MethodGet)) - if !bytes.Equal(second.Body.Bytes(), raw) || ensurer.calls.Load() != 1 { - t.Fatalf("second response=%s Ensure calls=%d", second.Body.Bytes(), ensurer.calls.Load()) - } - if offered := ensurer.offers(); len(offered) != 1 || offered[0].Priority != mediainfo.PriorityMetadata { - t.Fatalf("quiet-window offers = %#v", offered) - } -} - -func TestMetadataEnrichmentColdProbeQuietWindowDoesNotSlideUnderContinuousTraffic(t *testing.T) { - start := time.Date(2026, time.August, 29, 12, 0, 0, 0, time.UTC) - now := start - handler := &metadataEnrichmentHandler{ - coldProbeQuiet: 5 * time.Second, - now: func() time.Time { return now }, - } - if !handler.acquireColdProbe() { - t.Fatal("first cold probe was rejected") - } - handler.releaseColdProbe() - if handler.acquireColdProbe() { - t.Fatal("probe was admitted immediately after release") - } - for second := 1; second < 5; second++ { - now = start.Add(time.Duration(second) * time.Second) - if handler.acquireColdProbe() { - t.Fatalf("probe was admitted during the quiet window at %ds", second) - } - } - now = start.Add(5 * time.Second) - if !handler.acquireColdProbe() { - t.Fatal("probe was not admitted after the fixed quiet interval") - } - handler.releaseColdProbe() -} - -func TestMetadataEnrichmentColdProbeQuietWindowIsSharedAcrossClientsAndRecovers(t *testing.T) { - start := time.Date(2026, time.August, 29, 12, 0, 0, 0, time.UTC) - now := start - handler := &metadataEnrichmentHandler{ - coldProbeQuiet: 5 * time.Second, - now: func() time.Time { return now }, - } - - clients := []string{"apple-tv", "infuse", "plex-ios", "plex-web", "senplayer", "tablet", "desktop", "tv"} - type admissionResult struct { - client string - admitted bool - } - results := make(chan admissionResult, len(clients)) - var group sync.WaitGroup - group.Add(len(clients)) - for _, client := range clients { - go func(client string) { - defer group.Done() - results <- admissionResult{client: client, admitted: handler.acquireColdProbe()} - }(client) - } - group.Wait() - close(results) - - var admitted int - var admittedClient string - for result := range results { - if result.admitted { - admitted++ - admittedClient = result.client - } - } - if admitted != 1 || admittedClient == "" { - t.Fatalf("admitted probes across clients = %d (client %q), want 1", admitted, admittedClient) - } - handler.releaseColdProbe() - - now = start.Add(4 * time.Second) - if handler.acquireColdProbe() { - t.Fatal("probe was admitted before the shared quiet window expired") - } - now = start.Add(5 * time.Second) - if !handler.acquireColdProbe() { - t.Fatal("probe was not admitted when the shared quiet window expired") - } - handler.releaseColdProbe() -} - -func TestMetadataEnrichmentColdProbeQuietWindowStartsAfterActiveProbeCompletes(t *testing.T) { - start := time.Date(2026, time.August, 29, 12, 0, 0, 0, time.UTC) - now := start - handler := &metadataEnrichmentHandler{ - coldProbeQuiet: 5 * time.Second, - now: func() time.Time { return now }, - } - if !handler.acquireColdProbe() { - t.Fatal("probe was rejected before an active probe") - } - - now = start.Add(30 * time.Second) - handler.releaseColdProbe() - now = start.Add(34 * time.Second) - if handler.acquireColdProbe() { - t.Fatal("probe was admitted before the completion-based quiet window expired") - } - now = start.Add(35 * time.Second) - if !handler.acquireColdProbe() { - t.Fatal("probe was not admitted after the completion-based quiet window expired") - } - handler.releaseColdProbe() -} - -func TestMetadataEnrichmentCachedProjectionBypassesColdProbeSlot(t *testing.T) { +func TestMetadataEnrichmentUsesL1RecordAfterColdOffer(t *testing.T) { fixture := newMetadataEnrichmentFixture(t) raw := fixture.xmlBody() - started := make(chan struct{}) - release := make(chan struct{}) var lookups atomic.Int64 ensurer := &fakeMediaInfoEnsurer{ memoryFn: func(key mediainfo.Key) (mediainfo.Record, bool) { @@ -578,27 +370,17 @@ func TestMetadataEnrichmentCachedProjectionBypassesColdProbeSlot(t *testing.T) { } return mediainfo.Record{Key: key, Media: completeProjectionMedia()}, true }, - fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { - close(started) - <-release - return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil - }, } handler := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, - CloudExtensions: []string{".strm"}, ColdWait: time.Second, - ResponseLimit: 1 << 20, Concurrency: 2, Metrics: metrics.New(), + CloudExtensions: []string{".strm"}, + ResponseLimit: 1 << 20, Concurrency: 2, Metrics: metrics.New(), }, metadataBodyHandler(raw, "application/xml")) - firstDone := make(chan struct{}) - go func() { - defer close(firstDone) - handler.ServeHTTP(httptest.NewRecorder(), authenticatedMetadataRequest(http.MethodGet)) - }() - select { - case <-started: - case <-time.After(time.Second): - t.Fatal("first cold probe did not start") + first := httptest.NewRecorder() + handler.ServeHTTP(first, authenticatedMetadataRequest(http.MethodGet)) + if !bytes.Equal(first.Body.Bytes(), raw) { + t.Fatalf("cold response = %s", first.Body.Bytes()) } second := httptest.NewRecorder() @@ -606,52 +388,26 @@ func TestMetadataEnrichmentCachedProjectionBypassesColdProbeSlot(t *testing.T) { if !strings.Contains(second.Body.String(), `container="mkv"`) { t.Fatalf("cached response was not enriched: %s", second.Body.String()) } - if ensurer.calls.Load() != 1 { - t.Fatalf("Ensure calls = %d", ensurer.calls.Load()) - } - if offered := ensurer.offers(); len(offered) != 0 { - t.Fatalf("background offers after cache hit = %d", len(offered)) - } - close(release) - select { - case <-firstDone: - case <-time.After(time.Second): - t.Fatal("first cold probe did not finish") + if offered := ensurer.offers(); len(offered) != 1 || offered[0].Priority != mediainfo.PriorityMetadata { + t.Fatalf("offers = %#v", offered) } } -func TestMetadataEnrichmentBurstDoesNotQueueBehindColdProbe(t *testing.T) { +func TestMetadataEnrichmentColdBurstReturnsWithoutSynchronousProbe(t *testing.T) { fixture := newMetadataEnrichmentFixture(t) raw := fixture.xmlBody() - started := make(chan struct{}) - release := make(chan struct{}) - ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { - close(started) - <-release - return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil - }} + ensurer := &fakeMediaInfoEnsurer{} registry := metrics.New() enrichment := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, - CloudExtensions: []string{".strm"}, ColdWait: time.Second, - ResponseLimit: 1 << 20, Concurrency: 8, Metrics: registry, + CloudExtensions: []string{".strm"}, + ResponseLimit: 1 << 20, Concurrency: 64, Metrics: registry, }, metadataBodyHandler(raw, "application/xml")) handler := newMetadataGuard(MetadataGuardOptions{ Enabled: true, GlobalConcurrency: 16, PerClientConcurrency: 4, QueueTimeout: 500 * time.Millisecond, }, enrichment, registry, nil) - firstDone := make(chan struct{}) - go func() { - defer close(firstDone) - handler.ServeHTTP(httptest.NewRecorder(), authenticatedMetadataRequest(http.MethodGet)) - }() - select { - case <-started: - case <-time.After(time.Second): - t.Fatal("first cold probe did not start") - } - const burst = 64 responses := make(chan *httptest.ResponseRecorder, burst) var group sync.WaitGroup @@ -680,34 +436,22 @@ func TestMetadataEnrichmentBurstDoesNotQueueBehindColdProbe(t *testing.T) { t.Fatalf("burst response = %d %q", response.Code, response.Body.Bytes()) } } - if ensurer.calls.Load() != 1 { - t.Fatalf("Ensure calls = %d", ensurer.calls.Load()) - } if offered := ensurer.offers(); len(offered) != burst { t.Fatalf("background offers = %d, want %d", len(offered), burst) } if got := ensurer.gets.Load(); got != 0 { - t.Fatalf("quiet-window burst performed %d durable-capable lookups", got) + t.Fatalf("cold burst performed %d durable-capable lookups", got) } - if snapshot := registry.Snapshot(); snapshot.MetadataGuardTimeoutsTotal != 0 || snapshot.MediaInfoWaitRejectedTotal != burst { + if snapshot := registry.Snapshot(); snapshot.MetadataGuardTimeoutsTotal != 0 || snapshot.MediaInfoWaitRejectedTotal != 0 { t.Fatalf("metrics = %#v", snapshot) } - - close(release) - select { - case <-firstDone: - case <-time.After(time.Second): - t.Fatal("first cold probe did not finish") - } } func TestMetadataEnrichmentBypassesIneligibleAndUnbufferableResponses(t *testing.T) { fixture := newMetadataEnrichmentFixture(t) raw := fixture.xmlBody() oversized := bytes.Repeat([]byte("x"), 1025) - ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, _ mediainfo.Request) (mediainfo.Record, error) { - return mediainfo.Record{}, errors.New("unexpected Ensure") - }} + ensurer := &fakeMediaInfoEnsurer{} tests := []struct { name string request *http.Request @@ -737,8 +481,8 @@ func TestMetadataEnrichmentBypassesIneligibleAndUnbufferableResponses(t *testing t.Run(test.name, func(t *testing.T) { handler := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, - CloudExtensions: []string{".strm"}, ColdWait: time.Second, - ResponseLimit: 1024, Concurrency: 1, Metrics: metrics.New(), + CloudExtensions: []string{".strm"}, + ResponseLimit: 1024, Concurrency: 1, Metrics: metrics.New(), }, test.next) response := httptest.NewRecorder() handler.ServeHTTP(response, test.request) @@ -747,8 +491,8 @@ func TestMetadataEnrichmentBypassesIneligibleAndUnbufferableResponses(t *testing } }) } - if ensurer.calls.Load() != 0 { - t.Fatalf("ineligible responses called Ensure %d times", ensurer.calls.Load()) + if offered := ensurer.offers(); len(offered) != 0 { + t.Fatalf("ineligible responses offered %d probes", len(offered)) } } @@ -786,14 +530,14 @@ func TestMetadataEnrichmentAdmitsExistingStreamHDRAndSTRMControlSize(t *testing. } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { - return mediainfo.Record{Key: request.Key, Media: media}, nil + ensurer := &fakeMediaInfoEnsurer{memoryFn: func(key mediainfo.Key) (mediainfo.Record, bool) { + return mediainfo.Record{Key: key, Media: media}, true }} handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(test.body, test.contentType)) response := httptest.NewRecorder() handler.ServeHTTP(response, authenticatedMetadataRequest(http.MethodGet)) - if ensurer.calls.Load() != 1 { - t.Fatalf("Ensure calls = %d", ensurer.calls.Load()) + if ensurer.memoryGets.Load() != 1 || len(ensurer.offers()) != 0 { + t.Fatalf("memory gets=%d offers=%d", ensurer.memoryGets.Load(), len(ensurer.offers())) } for _, expected := range test.want { if !bytes.Contains(response.Body.Bytes(), expected) { @@ -806,9 +550,7 @@ func TestMetadataEnrichmentAdmitsExistingStreamHDRAndSTRMControlSize(t *testing. func TestMetadataEnrichmentSkipsLocalAndCompleteCloudParts(t *testing.T) { fixture := newMetadataEnrichmentFixture(t) - ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, _ mediainfo.Request) (mediainfo.Record, error) { - return mediainfo.Record{}, errors.New("unexpected Ensure") - }} + ensurer := &fakeMediaInfoEnsurer{} tests := []struct { name string body []byte @@ -832,8 +574,8 @@ func TestMetadataEnrichmentSkipsLocalAndCompleteCloudParts(t *testing.T) { } }) } - if ensurer.calls.Load() != 0 { - t.Fatalf("skipped Parts called Ensure %d times", ensurer.calls.Load()) + if ensurer.memoryGets.Load() != 0 || len(ensurer.offers()) != 0 { + t.Fatalf("skipped Parts memory gets=%d offers=%d", ensurer.memoryGets.Load(), len(ensurer.offers())) } } @@ -866,8 +608,8 @@ func newMetadataEnrichmentFixture(t *testing.T) metadataEnrichmentFixture { func (fixture metadataEnrichmentFixture) handler(ensurer mediaInfoEnsurer, registry *metrics.Metrics, next http.Handler) http.Handler { return newMetadataEnrichmentHandler(metadataEnrichmentOptions{ Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, - CloudExtensions: []string{".strm"}, ColdWait: time.Second, - ResponseLimit: 1 << 20, Concurrency: 2, Metrics: registry, + CloudExtensions: []string{".strm"}, + ResponseLimit: 1 << 20, Concurrency: 2, Metrics: registry, }, next) } From eb50e7603817e3de134f42daa32b7c9ddb856ca0 Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sun, 30 Aug 2026 01:55:48 +0800 Subject: [PATCH 11/15] fix: prevent Plex on-demand metadata analysis --- README.md | 15 ++- app.env.example | 4 + cmd/plex-gateway/main.go | 22 ++-- docs/architecture.md | 7 ++ docs/performance-matrix.md | 9 ++ internal/config/config.go | 70 ++++++----- internal/config/config_test.go | 8 ++ internal/gateway/gateway.go | 2 + internal/gateway/gateway_test.go | 37 ++++++ internal/gateway/metadata_analysis_filter.go | 72 +++++++++++ .../gateway/metadata_analysis_filter_test.go | 113 ++++++++++++++++++ internal/metrics/metrics.go | 37 +++--- internal/metrics/metrics_test.go | 18 +-- 13 files changed, 347 insertions(+), 67 deletions(-) create mode 100644 internal/gateway/metadata_analysis_filter.go create mode 100644 internal/gateway/metadata_analysis_filter_test.go diff --git a/README.md b/README.md index e4cc39f..70d61fb 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ go run ./cmd/plex-gateway | `PART_PROBE_TIMEOUT` | `15s` | 无 body 的 Plex Part 授权 probe 超时时间。 | | `CLOUD_EXTENSIONS` | `.strm` | 以逗号分隔的云端控制文件扩展名。 | | `TRACE_ENABLED` | `true` | 启用经过脱敏的 Plex 请求顺序追踪。 | +| `METADATA_ANALYSIS_FILTER_ENABLED` | `true` | 从详细 metadata 读取中移除会触发 Plex 文件分析的 `checkFiles` 和 `asyncAugmentMetadata`。 | | `METADATA_GUARD_ENABLED` | `true` | 限制单项详细 metadata 请求进入 Plex 的并发量。 | | `METADATA_GUARD_GLOBAL_CONCURRENCY` | `8` | 所有客户端共享的详细 metadata 并发上限。 | | `METADATA_GUARD_CLIENT_CONCURRENCY` | `4` | 每个 Plex 客户端标识的详细 metadata 并发上限。 | @@ -221,6 +222,12 @@ Provider，未命中时再回退 CDN ffprobe。探测失败由 `MEDIAINFO_NEGATI Gateway 会在这些单项详细 metadata 请求进入 Plex 前应用全局和单客户端并发上限。 客户端标识只用于进程内限流，并以摘要形式保存，不会写入日志或 metrics。 +`METADATA_ANALYSIS_FILTER_ENABLED` 默认从单项和逗号分隔的详细 metadata 读取中移除 +`checkFiles` 与 `asyncAugmentMetadata`。这两个客户端控制参数会要求 Plex 检查媒体文件； +对没有真实视频文件可分析的 STRM 项目，突发请求可能反复启动 Plex Media Scanner。 +过滤器保留其他 query 的原始编码，不修改 Header、响应或 Plex 数据库。Plex 的库扫描 +仍负责本地媒体文件的新鲜度，该开关也可以独立关闭用于兼容性回归。 + 该保护不处理媒体库列表、时间线、观看状态、播放决策、`/library/parts` 或其他 Plex 路径。请求最多排队 `METADATA_GUARD_QUEUE_TIMEOUT`，超时后返回 `429`，不会绕过保护 继续请求 Plex。此功能不缓存 metadata，也不改变 Plex 响应。 @@ -229,9 +236,8 @@ Gateway 会在这些单项详细 metadata 请求进入 Plex 前应用全局和 `GET/HEAD /library/metadata/1,2,...`。批量读取不会占用交互式单项 metadata 的全局或 单客户端槽位，metadata PUT 等修改请求也不会进入批量池。 -当前默认 Guard 为单项全局 8、单客户端 4、批量 3。Plex MediaInfo 写库完成较高覆盖率 -并经过重新压测后，才评估单项全局 16、单客户端 4、批量 3 的候选值。当前版本不新增 -Guard 模式或配置项。 +当前默认 Guard 为单项全局 8、单客户端 4、批量 3。只有在分析参数过滤启用、Scanner +没有被请求触发并完成重新压测后，才评估提高或关闭单项 Guard。 ## 通过 Plex 发布 Gateway 地址 @@ -304,7 +310,8 @@ Gateway 重新执行检查。 - `GET /health` 返回进程健康状态，以及不含媒体身份和凭据的 MediaInfo 缓存、探测队列和邻近媒体预热摘要。 - `GET /metrics` 返回固定结构的 JSON 计数器，以及 resolver 和完整重定向链路的 - 延迟总计、样本数、最近值和最大值。启用 metadata 保护时还会返回准入、超时、 + 延迟总计、样本数、最近值和最大值。`metadata_analysis_params_removed_total` 记录已过滤 + 的详细 metadata 请求；启用 metadata 保护时还会返回准入、超时、 活动和排队计数；MediaInfo 调度器另按固定 P0/P1/P2 结构返回准入和丢弃结果。所有指标 都不包含请求标签或凭据。 - 其他所有 endpoint 都遵循 [docs/architecture.md](docs/architecture.md) 中说明 diff --git a/app.env.example b/app.env.example index de113fe..cb321ad 100644 --- a/app.env.example +++ b/app.env.example @@ -38,6 +38,10 @@ OBSERVE_MAX_BYTES=8388608 # 无响应体 Part 授权请求和 Plex 管理请求的超时。 PART_PROBE_TIMEOUT=15s +# 是否移除详细 metadata 请求中的 checkFiles 和 asyncAugmentMetadata。 +# 默认开启，避免无可分析媒体文件的 STRM 项目反复启动 Plex Media Scanner。 +METADATA_ANALYSIS_FILTER_ENABLED=true + # 是否限制单项详细 metadata 请求进入 Plex 的并发量。 METADATA_GUARD_ENABLED=true diff --git a/cmd/plex-gateway/main.go b/cmd/plex-gateway/main.go index c9f10e3..35a062d 100644 --- a/cmd/plex-gateway/main.go +++ b/cmd/plex-gateway/main.go @@ -97,16 +97,17 @@ func run() error { return prewarmService.Status() } gatewayHandler := gateway.New(gateway.Options{ - Upstream: cfg.PlexURL, - Logger: logger, - Tracer: trace.New(cfg.TraceEnabled, logger), - PartCache: cache, - PathMapper: mapper, - Resolver: strmResolver, - Metrics: registry, - CloudExtensions: cfg.CloudExtensions, - ObserveMaxBytes: cfg.ObserveMaxBytes, - PartProbeTimeout: cfg.PartProbeTimeout, + Upstream: cfg.PlexURL, + Logger: logger, + Tracer: trace.New(cfg.TraceEnabled, logger), + PartCache: cache, + PathMapper: mapper, + Resolver: strmResolver, + Metrics: registry, + CloudExtensions: cfg.CloudExtensions, + ObserveMaxBytes: cfg.ObserveMaxBytes, + PartProbeTimeout: cfg.PartProbeTimeout, + MetadataAnalysisFilter: cfg.MetadataAnalysisFilter, MetadataGuard: gateway.MetadataGuardOptions{ Enabled: cfg.MetadataGuard.Enabled, GlobalConcurrency: cfg.MetadataGuard.GlobalConcurrency, @@ -144,6 +145,7 @@ func run() error { "cloud_redirect", cfg.MediaVaultURL != nil, "path_mappings", len(cfg.PathMappings), "trace", cfg.TraceEnabled, + "metadata_analysis_filter", cfg.MetadataAnalysisFilter, "metadata_guard", cfg.MetadataGuard.Enabled, "metadata_batch_guard", cfg.MetadataGuard.BatchEnabled, "mediainfo_enabled", cfg.MediaInfo.Enabled, diff --git a/docs/architecture.md b/docs/architecture.md index 4c2546c..9de2190 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -213,6 +213,13 @@ belong to workers. Background synchronization identified by `-Library` plus `skipRefresh` remains cache-only and cannot populate P2. Queue state is not persisted and is discarded on cache reset, shutdown, or process restart. +Detailed numeric metadata reads pass through an independent analysis-parameter +filter before admission control. When enabled, it removes only `checkFiles` and +`asyncAugmentMetadata`, preserving all retained raw query segments and leaving +headers and responses untouched. This prevents browse bursts over STRM entries +from asking Plex to start file scanners. The filter and concurrency guard have +separate switches so capacity tests never need to restore on-demand analysis. + `/metrics` keeps aggregate scheduler counters and a fixed, label-free P0/P1/P2 breakdown for offers, admissions, joins, promotions, capacity drops, expiry, and fingerprint supersession. `/health` reports the current queue depth for each diff --git a/docs/performance-matrix.md b/docs/performance-matrix.md index cdd2b06..11bbf05 100644 --- a/docs/performance-matrix.md +++ b/docs/performance-matrix.md @@ -15,6 +15,8 @@ 因此持久化 I/O 不得阻塞热路径。 - 单项 metadata 冷 miss 只允许内存级 P2 投递并立即返回 Plex 原响应，不同步读取 SQLite、 请求 MediaVault 或等待 CDN probe。 +- 详细 metadata 请求默认移除 `checkFiles` 和 `asyncAugmentMetadata`，避免 STRM 浏览突发 + 启动 Plex Scanner；过滤器不解析 body、不缓存响应，也不规范化其余 query。 - 一个 universal decision 最多等待一次 `MEDIAINFO_COLD_WAIT`。veto 只复用响应增强已经 取得的新鲜记录，不增加缓存读取或远程探测。 - 同一 MediaInfo key 的并发请求必须共享 singleflight。并发数不能突破配置的 worker @@ -38,6 +40,9 @@ | STRM target fingerprint | 382.0-387.4 ns/op | 448 B, 6 allocs | MediaVault redirect URL 规范化和 SHA-256。 | | SQLite 单记录 `Get` | 14.60-15.37 us/op | 4016 B, 74 allocs | 独立本机 SQLite 读取；生产启动会先恢复到 L1。 | | 本机 HTTP 合成媒体 ffprobe | 22.48-33.36 ms/op | 约 119-125 KiB, 232-234 allocs | 只验证 worker 代码，不代表 NAS/CDN。 | +| Metadata 分析过滤器，非 metadata | 6.247-6.429 ns/op | 0 B, 0 allocs | 普通代理路径仅执行方法和前缀判断。 | +| Metadata 分析过滤器，无目标参数 | 75.36-77.49 ns/op | 32 B, 2 allocs | 保留原请求和完整 raw query。 | +| Metadata 分析过滤器，移除目标参数 | 377.3-386.3 ns/op | 768 B, 7 allocs | 克隆请求 URL，其他 query 字节不变。 | 真实 MediaVault/CDN 抽样为 5/5 成功：MediaVault redirect p50 141 ms、范围 109-533 ms；受限 CDN ffprobe p50 601 ms、范围 403-674 ms；合计 p50 761 ms、范围 @@ -71,6 +76,7 @@ | 不同 Part 并发 2/4/8 | 受 worker 上限约束 | active、队列、CPU、内存、错误率 | 不突破 `MEDIAINFO_CONCURRENCY`，不拖慢本地或 302 路径。 | | A/B/C 快速切换 | 当前项 P0 probe、P1 邻近预热 | 每项延迟、身份对应、替换/队列指标 | 新当前项优先排队；不能抢占正在运行的 probe；无跨项 MediaInfo、grant 或 redirect。 | | 长剧集 metadata 突发 | Plex、L1；P2 后台可查 SQLite/探测 | 前台响应、P2 队列/过期/远程启动次数 | 前台不等待 P2；唯一 pending P2 <= 50；远程启动间隔 >= 5 s；过期任务 0 MV/CDN。 | +| 长剧集分析参数过滤 A/B | Plex、L1 | 过滤计数、Scanner 进程、metadata p50/p95/p99、错误率 | 开启过滤后 Scanner 为 0；先保留 Guard 验证，再关闭 Guard 测试 Plex 原生吞吐。 | 表中的验收条件是候选版本的门槛，不代表本机微基准已经替代端到端验收。当前已有证据 包括本机代码基准和 5/5 成功的 MediaVault/CDN 探测抽样；本地透明代理、302 热冷路径、 @@ -85,6 +91,9 @@ go test -run '^$' \ -bench 'Benchmark(MediaInfoL1Get|MediaInfoServiceGetMemory|MediaInfoServiceOfferFresh|MediaInfoServiceOfferJoin|FingerprintSTRMTarget|MediaInfoSQLiteGet|MediaInfoHTTPFFProbe)$' \ -benchmem -count=5 ./internal/mediainfo + +go test -run '^$' -bench '^BenchmarkMetadataAnalysisFilter$' \ + -benchmem -count=5 ./internal/gateway ``` 真实 NAS 验收必须使用候选镜像、固定样本和串行 A/B，记录版本、配置、样本数、分位数、 diff --git a/internal/config/config.go b/internal/config/config.go index 4fb6d02..ab7ae67 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -55,25 +55,26 @@ const ( // credential for nearby-item discovery and is never mixed into client playback // context. type Config struct { - ListenAddr string - DatabasePath string - PlexURL *url.URL - PlexToken string - MediaVaultURL *url.URL - PathMappings []pathmap.Mapping - CloudExtensions []string - PartTTL time.Duration - ResolverTimeout time.Duration - ObserveMaxBytes int64 - PartProbeTimeout time.Duration - MetadataGuard MetadataGuardConfig - MediaInfo MediaInfoConfig - PlaybackVeto bool - LogLevel string - TraceEnabled bool - ReadHeaderTimeout time.Duration - IdleTimeout time.Duration - ShutdownTimeout time.Duration + ListenAddr string + DatabasePath string + PlexURL *url.URL + PlexToken string + MediaVaultURL *url.URL + PathMappings []pathmap.Mapping + CloudExtensions []string + PartTTL time.Duration + ResolverTimeout time.Duration + ObserveMaxBytes int64 + PartProbeTimeout time.Duration + MetadataAnalysisFilter bool + MetadataGuard MetadataGuardConfig + MediaInfo MediaInfoConfig + PlaybackVeto bool + LogLevel string + TraceEnabled bool + ReadHeaderTimeout time.Duration + IdleTimeout time.Duration + ShutdownTimeout time.Duration } // MediaInfoConfig bounds the optional analysis plane. It is enabled by default @@ -106,8 +107,8 @@ type MediaInfoConfig struct { } // MetadataGuardConfig bounds detailed metadata fan-out before requests enter -// Plex. The protection is opt-in because client behavior and Plex capacity vary -// across deployments. +// Plex. It remains independently configurable from metadata analysis filtering +// so operators can validate Plex capacity without restoring file inspection. type MetadataGuardConfig struct { Enabled bool GlobalConcurrency int @@ -186,6 +187,10 @@ func Load() (Config, error) { if err != nil { return Config{}, err } + metadataAnalysisFilter, err := envBool("METADATA_ANALYSIS_FILTER_ENABLED", true) + if err != nil { + return Config{}, err + } metadataGlobal, err := envPositiveInt("METADATA_GUARD_GLOBAL_CONCURRENCY", defaultMetadataGlobal) if err != nil { return Config{}, err @@ -341,17 +346,18 @@ func Load() (Config, error) { } return Config{ - ListenAddr: listenAddr, - DatabasePath: databasePath, - PlexURL: plexURL, - PlexToken: plexToken, - MediaVaultURL: mediaVaultURL, - PathMappings: pathMappings, - CloudExtensions: cloudExtensions, - PartTTL: partTTL, - ResolverTimeout: resolverTimeout, - ObserveMaxBytes: observeMaxBytes, - PartProbeTimeout: partProbeTimeout, + ListenAddr: listenAddr, + DatabasePath: databasePath, + PlexURL: plexURL, + PlexToken: plexToken, + MediaVaultURL: mediaVaultURL, + PathMappings: pathMappings, + CloudExtensions: cloudExtensions, + PartTTL: partTTL, + ResolverTimeout: resolverTimeout, + ObserveMaxBytes: observeMaxBytes, + PartProbeTimeout: partProbeTimeout, + MetadataAnalysisFilter: metadataAnalysisFilter, MetadataGuard: MetadataGuardConfig{ Enabled: metadataGuardEnabled, GlobalConcurrency: metadataGlobal, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index e200525..fc2fbbe 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -24,6 +24,7 @@ func TestLoadDefaults(t *testing.T) { t.Setenv("LISTEN_ADDR", "") t.Setenv("LOG_LEVEL", "") t.Setenv("TRACE_ENABLED", "") + t.Setenv("METADATA_ANALYSIS_FILTER_ENABLED", "") got, err := Load() if err != nil { @@ -44,6 +45,9 @@ func TestLoadDefaults(t *testing.T) { if !got.MetadataGuard.Enabled || got.MetadataGuard.GlobalConcurrency != 8 || got.MetadataGuard.PerClientConcurrency != 4 || !got.MetadataGuard.BatchEnabled || got.MetadataGuard.BatchConcurrency != 3 || got.MetadataGuard.QueueTimeout != 10*time.Second { t.Fatalf("unexpected metadata guard defaults: %#v", got.MetadataGuard) } + if !got.MetadataAnalysisFilter { + t.Fatal("MetadataAnalysisFilter = false") + } if got.DatabasePath != "./data/plex-gateway.db" { t.Fatalf("DatabasePath = %q", got.DatabasePath) } @@ -83,6 +87,7 @@ func TestLoadCloudConfiguration(t *testing.T) { t.Setenv("OBSERVE_MAX_BYTES", "4096") t.Setenv("PART_PROBE_TIMEOUT", "3s") t.Setenv("METADATA_GUARD_ENABLED", "true") + t.Setenv("METADATA_ANALYSIS_FILTER_ENABLED", "false") t.Setenv("METADATA_GUARD_GLOBAL_CONCURRENCY", "6") t.Setenv("METADATA_GUARD_CLIENT_CONCURRENCY", "3") t.Setenv("METADATA_GUARD_BATCH_ENABLED", "true") @@ -131,6 +136,9 @@ func TestLoadCloudConfiguration(t *testing.T) { if !got.MetadataGuard.Enabled || got.MetadataGuard.GlobalConcurrency != 6 || got.MetadataGuard.PerClientConcurrency != 3 || !got.MetadataGuard.BatchEnabled || got.MetadataGuard.BatchConcurrency != 2 || got.MetadataGuard.QueueTimeout != 5*time.Second { t.Fatalf("metadata guard = %#v", got.MetadataGuard) } + if got.MetadataAnalysisFilter { + t.Fatal("MetadataAnalysisFilter = true") + } if got.DatabasePath != "/app_data/test.db" { t.Fatalf("DatabasePath = %q", got.DatabasePath) } diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 9f1dfbf..7bec5b8 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -38,6 +38,7 @@ type Options struct { CloudExtensions []string ObserveMaxBytes int64 PartProbeTimeout time.Duration + MetadataAnalysisFilter bool MetadataGuard MetadataGuardOptions MediaInfoEnabled bool MediaInfoStatus func() mediainfo.Status @@ -214,6 +215,7 @@ func New(options Options) http.Handler { // Admission must happen before response buffering so a queued request does // not consume an enrichment slot or bypass projection under client bursts. metadata = newMetadataGuard(options.MetadataGuard, metadata, registry, logger) + metadata = newMetadataAnalysisFilter(options.MetadataAnalysisFilter, metadata, registry) mux.Handle("/", metadata) withActiveRequests := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index 71aac57..87c9811 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -67,6 +67,43 @@ func TestTransparentProxyPreservesRequestAndResponse(t *testing.T) { } } +func TestMetadataAnalysisFilterIsAppliedBeforePlexProxy(t *testing.T) { + registry := metrics.New() + upstream := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if got := request.URL.RawQuery; got != "X-Plex-Token=a%2Bb&includeMarkers=1" { + t.Fatalf("upstream query = %q", got) + } + if got := request.Header.Get("X-Plex-Client-Identifier"); got != "client-id" { + t.Fatalf("client identifier = %q", got) + } + writer.WriteHeader(http.StatusNoContent) + })) + defer upstream.Close() + upstreamURL, err := url.Parse(upstream.URL) + if err != nil { + t.Fatal(err) + } + handler := New(Options{ + Upstream: upstreamURL, + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + Tracer: trace.New(false, nil), + Metrics: registry, + MetadataAnalysisFilter: true, + }) + request := httptest.NewRequest(http.MethodGet, "/library/metadata/42?checkFiles=1&X-Plex-Token=a%2Bb&asyncAugmentMetadata=1&includeMarkers=1", nil) + request.Header.Set("X-Plex-Client-Identifier", "client-id") + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusNoContent { + t.Fatalf("status = %d, want %d", response.Code, http.StatusNoContent) + } + if got := registry.Snapshot().MetadataAnalysisParamsRemovedTotal; got != 1 { + t.Fatalf("removed total = %d, want 1", got) + } +} + func TestHealthDoesNotReachPlex(t *testing.T) { reached := false upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { diff --git a/internal/gateway/metadata_analysis_filter.go b/internal/gateway/metadata_analysis_filter.go new file mode 100644 index 0000000..155b11b --- /dev/null +++ b/internal/gateway/metadata_analysis_filter.go @@ -0,0 +1,72 @@ +package gateway + +import ( + "net/http" + "net/url" + "strings" + + "github.com/InfinityPacer/plex-gateway/internal/metrics" +) + +var metadataAnalysisParameters = map[string]struct{}{ + "asyncAugmentMetadata": {}, + "checkFiles": {}, +} + +// newMetadataAnalysisFilter prevents detailed metadata reads from asking Plex +// to inspect media files on demand. Library scans remain responsible for local +// file freshness, while STRM entries can use the Gateway MediaInfo projection. +func newMetadataAnalysisFilter(enabled bool, next http.Handler, registry *metrics.Metrics) http.Handler { + if !enabled { + return next + } + if registry == nil { + registry = metrics.New() + } + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if classifyMetadataRequest(request) == metadataRequestNone || request.URL.RawQuery == "" { + next.ServeHTTP(writer, request) + return + } + + rawQuery, changed := removeRawQueryParameters(request.URL.RawQuery, metadataAnalysisParameters) + if !changed { + next.ServeHTTP(writer, request) + return + } + + filtered := request.Clone(request.Context()) + requestURL := *request.URL + requestURL.RawQuery = rawQuery + filtered.URL = &requestURL + registry.IncMetadataAnalysisParamsRemoved() + next.ServeHTTP(writer, filtered) + }) +} + +// removeRawQueryParameters preserves every retained query segment byte for +// byte. Plex clients use several equivalent escaping forms, and filtering two +// control parameters must not normalize credentials or client context. +func removeRawQueryParameters(rawQuery string, blocked map[string]struct{}) (string, bool) { + segments := strings.Split(rawQuery, "&") + kept := make([]string, 0, len(segments)) + changed := false + for _, segment := range segments { + key := segment + if separator := strings.IndexByte(key, '='); separator >= 0 { + key = key[:separator] + } + decoded, err := url.QueryUnescape(key) + if err == nil { + if _, remove := blocked[decoded]; remove { + changed = true + continue + } + } + kept = append(kept, segment) + } + if !changed { + return rawQuery, false + } + return strings.Join(kept, "&"), true +} diff --git a/internal/gateway/metadata_analysis_filter_test.go b/internal/gateway/metadata_analysis_filter_test.go new file mode 100644 index 0000000..c2fb8ac --- /dev/null +++ b/internal/gateway/metadata_analysis_filter_test.go @@ -0,0 +1,113 @@ +package gateway + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/InfinityPacer/plex-gateway/internal/metrics" +) + +func TestMetadataAnalysisFilterRemovesOnlyAnalysisParameters(t *testing.T) { + registry := metrics.New() + var received string + handler := newMetadataAnalysisFilter(true, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + received = request.URL.RawQuery + writer.WriteHeader(http.StatusNoContent) + }), registry) + originalQuery := "z=last&checkFiles=1&X-Plex-Token=a%2Bb&asyncAugmentMetadata=1&a=first+value" + request := httptest.NewRequest(http.MethodGet, "/library/metadata/42?"+originalQuery, nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusNoContent { + t.Fatalf("status = %d, want %d", response.Code, http.StatusNoContent) + } + if received != "z=last&X-Plex-Token=a%2Bb&a=first+value" { + t.Fatalf("filtered query = %q", received) + } + if request.URL.RawQuery != originalQuery { + t.Fatalf("original request query mutated to %q", request.URL.RawQuery) + } + if got := registry.Snapshot().MetadataAnalysisParamsRemovedTotal; got != 1 { + t.Fatalf("removed total = %d, want 1", got) + } +} + +func TestMetadataAnalysisFilterAppliesToSupportedReads(t *testing.T) { + tests := []struct { + name string + method string + path string + want string + }{ + {name: "head", method: http.MethodHead, path: "/library/metadata/42?checkFiles=1&keep=yes", want: "keep=yes"}, + {name: "batch", method: http.MethodGet, path: "/library/metadata/42,43?asyncAugmentMetadata&keep=yes", want: "keep=yes"}, + {name: "children", method: http.MethodGet, path: "/library/metadata/42/children?checkFiles=1", want: "checkFiles=1"}, + {name: "write", method: http.MethodPut, path: "/library/metadata/42?checkFiles=1", want: "checkFiles=1"}, + {name: "non_numeric", method: http.MethodGet, path: "/library/metadata/abc?checkFiles=1", want: "checkFiles=1"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var received string + handler := newMetadataAnalysisFilter(true, http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) { + received = request.URL.RawQuery + }), metrics.New()) + + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(test.method, test.path, nil)) + + if received != test.want { + t.Fatalf("query = %q, want %q", received, test.want) + } + }) + } +} + +func TestMetadataAnalysisFilterPreservesMalformedAndDisabledQueries(t *testing.T) { + tests := []struct { + name string + enabled bool + query string + }{ + {name: "disabled", enabled: false, query: "checkFiles=1&keep=yes"}, + {name: "malformed_key", enabled: true, query: "checkFiles%ZZ=1&keep=yes"}, + {name: "unrelated", enabled: true, query: "checkFilesExtra=1&keep=yes"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var received string + handler := newMetadataAnalysisFilter(test.enabled, http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) { + received = request.URL.RawQuery + }), metrics.New()) + + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/library/metadata/42?"+test.query, nil)) + + if received != test.query { + t.Fatalf("query = %q, want %q", received, test.query) + } + }) + } +} + +func BenchmarkMetadataAnalysisFilter(b *testing.B) { + tests := []struct { + name string + path string + }{ + {name: "non_metadata", path: "/library/sections/49/all?type=4"}, + {name: "metadata_clean", path: "/library/metadata/42?includeMarkers=1"}, + {name: "metadata_filtered", path: "/library/metadata/42?checkFiles=1&asyncAugmentMetadata=1&includeMarkers=1"}, + } + for _, test := range tests { + b.Run(test.name, func(b *testing.B) { + handler := newMetadataAnalysisFilter(true, http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}), metrics.New()) + request := httptest.NewRequest(http.MethodGet, test.path, nil) + response := httptest.NewRecorder() + b.ReportAllocs() + for b.Loop() { + handler.ServeHTTP(response, request) + } + }) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 2b75f20..9c4aee2 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -22,6 +22,7 @@ type Metrics struct { redirectFailure atomic.Uint64 plexFallbackTotal atomic.Uint64 activeRequests atomic.Int64 + metadataAnalysisParamsRemoved atomic.Uint64 metadataAdmitted atomic.Uint64 metadataTimeouts atomic.Uint64 metadataActive atomic.Int64 @@ -84,13 +85,14 @@ type latencyMetrics struct { // Snapshot is the stable JSON representation of the process counters. // Counter names match the operational metric names used by the gateway. type Snapshot struct { - PlexRequestsTotal uint64 `json:"plex_requests_total"` - CloudPartHits uint64 `json:"cloud_part_hits"` - CloudPartMisses uint64 `json:"cloud_part_misses"` - RedirectSuccess uint64 `json:"redirect_success"` - RedirectFailure uint64 `json:"redirect_failure"` - PlexFallbackTotal uint64 `json:"plex_fallback_total"` - ActiveRequests int64 `json:"active_requests"` + PlexRequestsTotal uint64 `json:"plex_requests_total"` + CloudPartHits uint64 `json:"cloud_part_hits"` + CloudPartMisses uint64 `json:"cloud_part_misses"` + RedirectSuccess uint64 `json:"redirect_success"` + RedirectFailure uint64 `json:"redirect_failure"` + PlexFallbackTotal uint64 `json:"plex_fallback_total"` + ActiveRequests int64 `json:"active_requests"` + MetadataAnalysisParamsRemovedTotal uint64 `json:"metadata_analysis_params_removed_total"` MetadataGuardAdmittedTotal uint64 `json:"metadata_guard_admitted_total"` MetadataGuardTimeoutsTotal uint64 `json:"metadata_guard_timeouts_total"` @@ -199,6 +201,12 @@ func (m *Metrics) IncPlexFallback() { m.plexFallbackTotal.Add(1) } +// IncMetadataAnalysisParamsRemoved records one metadata request whose +// on-demand file-analysis controls were removed before proxying to Plex. +func (m *Metrics) IncMetadataAnalysisParamsRemoved() { + m.metadataAnalysisParamsRemoved.Add(1) +} + // IncMediaInfoCacheHits records an exact key satisfied by L1 or SQLite, // including a retained stale result returned while refresh runs separately. func (m *Metrics) IncMediaInfoCacheHits() { m.mediaInfoCacheHits.Add(1) } @@ -453,13 +461,14 @@ func (m *Metrics) Snapshot() Snapshot { redirectTotal, redirectSamples, redirectLast, redirectMax := m.redirectLatency.snapshot() mediaInfoTotal, mediaInfoSamples, mediaInfoLast, mediaInfoMax := m.mediaInfoProbeLatency.snapshot() return Snapshot{ - PlexRequestsTotal: m.plexRequestsTotal.Load(), - CloudPartHits: m.cloudPartHits.Load(), - CloudPartMisses: m.cloudPartMisses.Load(), - RedirectSuccess: m.redirectSuccess.Load(), - RedirectFailure: m.redirectFailure.Load(), - PlexFallbackTotal: m.plexFallbackTotal.Load(), - ActiveRequests: m.activeRequests.Load(), + PlexRequestsTotal: m.plexRequestsTotal.Load(), + CloudPartHits: m.cloudPartHits.Load(), + CloudPartMisses: m.cloudPartMisses.Load(), + RedirectSuccess: m.redirectSuccess.Load(), + RedirectFailure: m.redirectFailure.Load(), + PlexFallbackTotal: m.plexFallbackTotal.Load(), + ActiveRequests: m.activeRequests.Load(), + MetadataAnalysisParamsRemovedTotal: m.metadataAnalysisParamsRemoved.Load(), MetadataGuardAdmittedTotal: m.metadataAdmitted.Load(), MetadataGuardTimeoutsTotal: m.metadataTimeouts.Load(), diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 85662d1..4d481fd 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -19,6 +19,7 @@ func TestSnapshotAndJSONHandler(t *testing.T) { registry.IncRedirectSuccess() registry.IncRedirectFailure() registry.IncPlexFallback() + registry.IncMetadataAnalysisParamsRemoved() registry.IncMetadataGuardAdmitted() registry.IncMetadataGuardTimeouts() registry.IncMetadataGuardActive() @@ -74,13 +75,14 @@ func TestSnapshotAndJSONHandler(t *testing.T) { t.Fatalf("decode JSON: %v", err) } want := Snapshot{ - PlexRequestsTotal: 1, - CloudPartHits: 1, - CloudPartMisses: 1, - RedirectSuccess: 1, - RedirectFailure: 1, - PlexFallbackTotal: 1, - ActiveRequests: 1, + PlexRequestsTotal: 1, + CloudPartHits: 1, + CloudPartMisses: 1, + RedirectSuccess: 1, + RedirectFailure: 1, + PlexFallbackTotal: 1, + ActiveRequests: 1, + MetadataAnalysisParamsRemovedTotal: 1, MetadataGuardAdmittedTotal: 1, MetadataGuardTimeoutsTotal: 1, @@ -226,6 +228,7 @@ func TestCountersAreSafeForConcurrentUpdates(t *testing.T) { registry.IncRedirectSuccess() registry.IncRedirectFailure() registry.IncPlexFallback() + registry.IncMetadataAnalysisParamsRemoved() registry.IncMetadataGuardAdmitted() registry.IncMetadataGuardTimeouts() registry.IncMetadataGuardActive() @@ -279,6 +282,7 @@ func TestCountersAreSafeForConcurrentUpdates(t *testing.T) { got := registry.Snapshot() if got.PlexRequestsTotal != want || got.CloudPartHits != want || got.CloudPartMisses != want || got.RedirectSuccess != want || got.RedirectFailure != want || got.PlexFallbackTotal != want || got.ActiveRequests != 0 || + got.MetadataAnalysisParamsRemovedTotal != want || got.MetadataGuardAdmittedTotal != want || got.MetadataGuardTimeoutsTotal != want || got.MetadataGuardActive != 0 || got.MetadataGuardQueued != 0 || got.MetadataBatchGuardAdmittedTotal != want || got.MetadataBatchGuardTimeoutsTotal != want || From b9fe8103544beccf17f2c01d1c5be7dca8b48a45 Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sun, 30 Aug 2026 03:11:20 +0800 Subject: [PATCH 12/15] feat: coalesce Plex metadata requests --- CHANGELOG.en.md | 20 + CHANGELOG.md | 15 + README.md | 38 +- app.env.example | 16 +- cmd/plex-gateway/main.go | 9 + docs/architecture.md | 58 +- docs/media-lifecycle-architecture.md | 4 +- docs/performance-matrix.md | 28 +- docs/roadmap.md | 7 +- internal/config/config.go | 112 +- internal/config/config_test.go | 34 + internal/gateway/gateway.go | 32 +- internal/gateway/gateway_test.go | 2 +- internal/gateway/metadata_analysis_filter.go | 7 +- .../gateway/metadata_analysis_filter_test.go | 16 +- internal/gateway/metadata_coalesce.go | 684 +++++++++++ internal/gateway/metadata_coalesce_test.go | 1037 +++++++++++++++++ internal/metrics/metrics.go | 38 + internal/metrics/metrics_test.go | 18 + internal/observe/observe.go | 9 +- internal/observe/observe_test.go | 32 + internal/plexmeta/batch_split.go | 601 ++++++++++ internal/plexmeta/batch_split_test.go | 261 +++++ 23 files changed, 2976 insertions(+), 102 deletions(-) create mode 100644 internal/gateway/metadata_coalesce.go create mode 100644 internal/gateway/metadata_coalesce_test.go create mode 100644 internal/plexmeta/batch_split.go create mode 100644 internal/plexmeta/batch_split_test.go diff --git a/CHANGELOG.en.md b/CHANGELOG.en.md index 30a482e..01fa828 100644 --- a/CHANGELOG.en.md +++ b/CHANGELOG.en.md @@ -9,6 +9,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Added general Plex metadata micro-batching. Single-item GET requests with + identical authentication, query, and client context can share a native Plex + batch of 32 items by default and are split back into XML, JSON, or gzip + responses by rating key. + +### Changed + +- The metadata analysis filter now removes only `asyncAugmentMetadata` and + preserves `checkFiles`, retaining Plex `Part.accessible` and `Part.exists` + semantics. + +### Fixed + +- Failed batches issue at most one request per unique rating key through the + existing single-item Guard, fan out that result to duplicate callers, and + temporarily suspend batching for that request group. Canceled callers do not + fall back, and canceling all callers terminates the upstream batch. + ## [0.1.2] - 2026-08-29 ### Added diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c3f646..d343038 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,21 @@ ## [未发布] +### 新增 + +- 增加通用 Plex metadata 微批。认证、查询和客户端上下文完全一致的单项 GET 默认可合并为 + 最多 32 项的 Plex 原生 batch，并按 ratingKey 拆回 XML、JSON 或 gzip 单项响应。 + +### 变更 + +- metadata 分析过滤器只移除 `asyncAugmentMetadata`，继续透传 `checkFiles`，保留 Plex + `Part.accessible` 与 `Part.exists` 语义。 + +### 修复 + +- batch 异常时每个唯一 ratingKey 只通过既有单项 Guard 回退一次，重复调用者共享结果， + 并短暂熔断同组微批；取消的请求不会回退，全部调用者取消后终止上游 batch。 + ## [0.1.2] - 2026-08-29 ### 新增 diff --git a/README.md b/README.md index 70d61fb..cbe2f09 100644 --- a/README.md +++ b/README.md @@ -107,13 +107,17 @@ go run ./cmd/plex-gateway | `PART_PROBE_TIMEOUT` | `15s` | 无 body 的 Plex Part 授权 probe 超时时间。 | | `CLOUD_EXTENSIONS` | `.strm` | 以逗号分隔的云端控制文件扩展名。 | | `TRACE_ENABLED` | `true` | 启用经过脱敏的 Plex 请求顺序追踪。 | -| `METADATA_ANALYSIS_FILTER_ENABLED` | `true` | 从详细 metadata 读取中移除会触发 Plex 文件分析的 `checkFiles` 和 `asyncAugmentMetadata`。 | +| `METADATA_ANALYSIS_FILTER_ENABLED` | `true` | 从详细 metadata 读取中移除会触发 Plex 后台分析的 `asyncAugmentMetadata`；保留 `checkFiles`。 | | `METADATA_GUARD_ENABLED` | `true` | 限制单项详细 metadata 请求进入 Plex 的并发量。 | | `METADATA_GUARD_GLOBAL_CONCURRENCY` | `8` | 所有客户端共享的详细 metadata 并发上限。 | | `METADATA_GUARD_CLIENT_CONCURRENCY` | `4` | 每个 Plex 客户端标识的详细 metadata 并发上限。 | | `METADATA_GUARD_BATCH_ENABLED` | `true` | 限制逗号分隔的批量 metadata 读取进入 Plex 的并发量。 | | `METADATA_GUARD_BATCH_CONCURRENCY` | `3` | 所有客户端共享的批量 metadata 并发上限。 | | `METADATA_GUARD_QUEUE_TIMEOUT` | `10s` | 等待准入的最长时间，超时返回 `429`。 | +| `METADATA_COALESCE_ENABLED` | `true` | 将认证和表示上下文完全一致的单项 metadata GET 合并为 Plex 原生 batch。 | +| `METADATA_COALESCE_WINDOW` | `3ms` | 同组请求的微批聚合窗口，最大 `100ms`。 | +| `METADATA_COALESCE_MAX_ITEMS` | `32` | 单个合成 batch 的唯一 ratingKey 上限，允许 `2-64`。 | +| `METADATA_COALESCE_TIMEOUT` | `5s` | 合成 batch 请求 Plex 并完成拆分的总超时。 | | `MEDIAINFO_ENABLED` | `true` | 启用 MediaInfo 缓存、受限探测和单项 metadata 增强；初始化失败不影响透明代理。 | | `DATABASE_PATH` | `./data/plex-gateway.db` | Gateway SQLite 持久数据库路径；容器镜像默认使用 `/app_data/plex-gateway.db`。 | | `MEDIAINFO_PLAYBACK_QUEUE_SIZE` | `16` | P0 实际播放任务的独立队列容量。 | @@ -215,29 +219,39 @@ Provider，未命中时再回退 CDN ffprobe。探测失败由 `MEDIAINFO_NEGATI 协调器不执行无法感知最终错误类型的盲目重试。Token 缺失或失效只禁用邻近媒体发现， 不影响当前项预热、透明代理、metadata 增强或播放重定向。 -### Metadata 并发保护 +### Metadata 微批与并发保护 部分客户端会在打开大型媒体库时并发请求大量 `GET/HEAD /library/metadata/{ratingKey}`。启用 `METADATA_GUARD_ENABLED` 后， Gateway 会在这些单项详细 metadata 请求进入 Plex 前应用全局和单客户端并发上限。 客户端标识只用于进程内限流，并以摘要形式保存，不会写入日志或 metrics。 -`METADATA_ANALYSIS_FILTER_ENABLED` 默认从单项和逗号分隔的详细 metadata 读取中移除 -`checkFiles` 与 `asyncAugmentMetadata`。这两个客户端控制参数会要求 Plex 检查媒体文件； -对没有真实视频文件可分析的 STRM 项目，突发请求可能反复启动 Plex Media Scanner。 -过滤器保留其他 query 的原始编码，不修改 Header、响应或 Plex 数据库。Plex 的库扫描 -仍负责本地媒体文件的新鲜度，该开关也可以独立关闭用于兼容性回归。 +`METADATA_COALESCE_ENABLED` 默认将短窗口内认证、原始 query、完整客户端 Header、内容 +协商和远端身份完全一致的单项 `GET` 合成为 Plex 原生逗号 batch，默认最多包含 `32` 个唯一 +ratingKey，可配置上限为 `64`。Gateway 按 ratingKey 拆分 XML 或 JSON 响应，并为每个原请求重新计算 +`Content-Length`；gzip 会解压、拆分后重新编码。相同 ratingKey 的重复请求共享同一 batch +结果。Token、Cookie、Authorization、客户端 profile、query 或身份不同的请求绝不合批。 + +微批不缓存 metadata，也不处理 `HEAD`、Range、条件请求、请求体、播放路径、图片、时间线 +或写请求。Plex batch 超时、返回非 200、结构异常、缺项、编码不支持或超过 body 上限时， +每个唯一 ratingKey 最多经过单项 Guard 回退一次，重复请求共享该结果；不同条目的回退仍受 +Guard 并发与排队上限保护。同组请求会短暂停止再次合批，所有调用者取消后，上游 batch 也会取消。 + +`METADATA_ANALYSIS_FILTER_ENABLED` 默认从单项和逗号分隔的详细 metadata 读取中只移除 +`asyncAugmentMetadata`，避免浏览突发反复调度 Plex 后台分析。`checkFiles` 保持透传，因为 +它会影响 Plex 返回的 `Part.accessible` 与 `Part.exists`。过滤器保留其他 query 的原始编码， +不修改 Header、响应或 Plex 数据库，也可以独立关闭用于兼容性回归。 该保护不处理媒体库列表、时间线、观看状态、播放决策、`/library/parts` 或其他 Plex 路径。请求最多排队 `METADATA_GUARD_QUEUE_TIMEOUT`，超时后返回 `429`，不会绕过保护 -继续请求 Plex。此功能不缓存 metadata，也不改变 Plex 响应。 +继续请求 Plex。Guard 不缓存 metadata；微批仅在可证明等价的成功响应上进行协议拆分。 `METADATA_GUARD_BATCH_ENABLED` 使用独立的全局并发池保护 `GET/HEAD /library/metadata/1,2,...`。批量读取不会占用交互式单项 metadata 的全局或 单客户端槽位，metadata PUT 等修改请求也不会进入批量池。 -当前默认 Guard 为单项全局 8、单客户端 4、批量 3。只有在分析参数过滤启用、Scanner -没有被请求触发并完成重新压测后，才评估提高或关闭单项 Guard。 +当前默认 Guard 为单项全局 8、单客户端 4、批量 3。合成 batch 使用批量池，异常回退使用 +单项全局与单客户端池。提高或关闭 Guard 必须以候选版本的真实客户端压测为依据。 ## 通过 Plex 发布 Gateway 地址 @@ -312,8 +326,8 @@ Gateway 重新执行检查。 - `GET /metrics` 返回固定结构的 JSON 计数器，以及 resolver 和完整重定向链路的 延迟总计、样本数、最近值和最大值。`metadata_analysis_params_removed_total` 记录已过滤 的详细 metadata 请求；启用 metadata 保护时还会返回准入、超时、 - 活动和排队计数；MediaInfo 调度器另按固定 P0/P1/P2 结构返回准入和丢弃结果。所有指标 - 都不包含请求标签或凭据。 + 活动和排队计数；微批另记录 offered、batch、唯一 item、fallback 和 active；MediaInfo + 调度器按固定 P0/P1/P2 结构返回准入和丢弃结果。所有指标都不包含请求标签或凭据。 - 其他所有 endpoint 都遵循 [docs/architecture.md](docs/architecture.md) 中说明 的 Plex 代理或 Direct Play 拦截规则。 diff --git a/app.env.example b/app.env.example index cb321ad..083900c 100644 --- a/app.env.example +++ b/app.env.example @@ -38,8 +38,8 @@ OBSERVE_MAX_BYTES=8388608 # 无响应体 Part 授权请求和 Plex 管理请求的超时。 PART_PROBE_TIMEOUT=15s -# 是否移除详细 metadata 请求中的 checkFiles 和 asyncAugmentMetadata。 -# 默认开启，避免无可分析媒体文件的 STRM 项目反复启动 Plex Media Scanner。 +# 是否移除详细 metadata 请求中的 asyncAugmentMetadata。 +# 默认开启，避免浏览请求调度 Plex 后台分析；checkFiles 仍透传以保留 accessible/exists。 METADATA_ANALYSIS_FILTER_ENABLED=true # 是否限制单项详细 metadata 请求进入 Plex 的并发量。 @@ -61,6 +61,18 @@ METADATA_GUARD_BATCH_CONCURRENCY=3 # metadata 并发槽位的最长等待时间，超时返回 429。 METADATA_GUARD_QUEUE_TIMEOUT=10s +# 是否将认证、查询和客户端上下文完全一致的单项 metadata GET 合并为 Plex 原生批量读取。 +METADATA_COALESCE_ENABLED=true + +# 等待同组请求到达的微批窗口。允许范围不超过 100ms，默认值优先降低页面扇出。 +METADATA_COALESCE_WINDOW=3ms + +# 单个原生 Plex batch 最多包含的唯一 ratingKey，允许范围为 2-64。 +METADATA_COALESCE_MAX_ITEMS=32 + +# 单个合成 batch 请求 Plex 并完成拆分的总超时。 +METADATA_COALESCE_TIMEOUT=5s + # 是否启用 MediaInfo 缓存、远程探测和 Plex metadata 增强。 MEDIAINFO_ENABLED=true diff --git a/cmd/plex-gateway/main.go b/cmd/plex-gateway/main.go index 35a062d..ae5f077 100644 --- a/cmd/plex-gateway/main.go +++ b/cmd/plex-gateway/main.go @@ -116,6 +116,12 @@ func run() error { BatchConcurrency: cfg.MetadataGuard.BatchConcurrency, QueueTimeout: cfg.MetadataGuard.QueueTimeout, }, + MetadataCoalesce: gateway.MetadataCoalesceOptions{ + Enabled: cfg.MetadataCoalesce.Enabled, + Window: cfg.MetadataCoalesce.Window, + MaxItems: cfg.MetadataCoalesce.MaxItems, + Timeout: cfg.MetadataCoalesce.Timeout, + }, MediaInfoEnabled: cfg.MediaInfo.Enabled, MediaInfoStatus: mediaInfoStatus, MediaInfoService: mediaInfoService, @@ -148,6 +154,9 @@ func run() error { "metadata_analysis_filter", cfg.MetadataAnalysisFilter, "metadata_guard", cfg.MetadataGuard.Enabled, "metadata_batch_guard", cfg.MetadataGuard.BatchEnabled, + "metadata_coalesce", cfg.MetadataCoalesce.Enabled, + "metadata_coalesce_window_ms", cfg.MetadataCoalesce.Window.Milliseconds(), + "metadata_coalesce_max_items", cfg.MetadataCoalesce.MaxItems, "mediainfo_enabled", cfg.MediaInfo.Enabled, "mediainfo_available", mediaInfoService != nil, "mediainfo_reason", mediaInfoReason, diff --git a/docs/architecture.md b/docs/architecture.md index 9de2190..d08382d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -11,18 +11,29 @@ gateway; the isolated analysis worker may read a bounded amount for probing. ## Request ownership - All requests are transparently proxied to Plex by default. -- Optional admission guards limit single-item `GET` and `HEAD` - `/library/metadata/{ratingKey}` requests and comma-separated batch metadata - reads before they enter Plex. Single-item requests use global and per-client - limits; batch reads use a separate global pool so maintenance traffic cannot - consume interactive metadata slots. Neither pool caches or rewrites Plex - responses. Library listings, metadata mutations, playback paths, timeline, - and watch-state traffic do not enter these pools. -- The current Guard defaults are global 8 for single-item metadata, per-client - 4, and batch 3. A global 16, per-client 4, batch 3 candidate is reserved for - evaluation after Plex MediaInfo writes provide high coverage and the resulting - load is retested. No additional Guard mode or configuration is part of this - release. +- Equivalent authenticated single-item metadata `GET` requests can be combined + within a three-millisecond window into Plex's native comma-separated endpoint. + A group requires identical raw query, complete request headers, representation + context, host, protocol, and remote identity. The default batch contains at + most 32 unique rating keys, with 64 available for controlled A/B, and is split + by rating key into individual XML, JSON, or gzip responses. This is a bounded, + non-caching protocol transform. `HEAD`, + Range, conditional requests, bodies, ambiguous credentials, and every + non-metadata path bypass it. +- Optional admission guards limit direct or fallback single-item `GET` and + `HEAD` `/library/metadata/{ratingKey}` requests and comma-separated batch + metadata reads before they enter Plex. Single-item requests use global and + per-client limits; explicit and synthesized batch reads use a separate global + pool. A batch failure issues at most one fallback per unique rating key through + the single-item pool and fans the result out to duplicate callers. Different + keys remain bounded by the Guard, and batching is temporarily suspended for + the same request context. The current Guard defaults are global 8, per-client + 4, and batch 3. +- The metadata chain is analysis filter, per-item MediaInfo enrichment, + coalescer, Guard, and Plex. This ordering lets each original request preserve + its response projection while one synthesized batch consumes one batch Guard + slot. Library listings, metadata mutations, playback paths, timeline, and + watch-state traffic do not enter the coalescer or Guard pools. - Successful XML or JSON Plex responses are observed to populate the in-memory `Part.id` cache without changing their bytes. When MediaInfo is available, an authenticated single-item metadata response may also fill missing whitelisted @@ -73,7 +84,7 @@ gateway; the isolated analysis worker may read a bounded amount for probing. | `internal/gateway` | Plex routing, transparent proxying, protocol probes, response capture, 302/fallback output, and request metrics. | | `internal/mediainfo` | Bounded analysis scheduling, L1/persistent cache policy, provider contracts, and technical-media records. | | `internal/playback` | Shared STRM preparation, normalized playback attempts, Plex authorization/MediaVault resolution sequencing, and decision-grant state. | -| `internal/plexmeta` | XML and JSON Part/decision readers plus whitelisted metadata projection. | +| `internal/plexmeta` | XML and JSON Part/decision readers, native-batch splitting, and whitelisted metadata projection. | | `internal/partcache` | Derived, expiring `Part.id` index; never an authority for access. | | `internal/pathmap` | Lexical Plex-to-container path mapping. | | `internal/resolver` | STRM target validation and the MediaVault `/redirect` protocol. | @@ -161,6 +172,16 @@ applicable limits within its queue timeout receives `429` instead of bypassing protection and increasing Plex load. This protects Plex control-plane availability and does not change cloud redirect fallback behavior. +Metadata coalescing remains fail-open through the Guard rather than directly to +Plex. A non-200 batch, missing or duplicate item, unsupported encoding, +oversized body, timeout, or malformed envelope is never interpreted as a +single-item result. Each unique rating key performs at most one fallback with +its original filtered request, and duplicate callers share the captured result; +canceled callers are discarded. When every caller leaves, the upstream batch is +canceled. Synthesized responses remove batch validators, +recompute `Content-Length`, and use private no-cache semantics because their +wire representation differs from the batch response. + Cloud STRM support is Direct Play only. The decision adapter opts an eligible Part into Plex's own Direct Play decision path; it does not synthesize a decision, create a transcode session, or send a cloud URL to Plex Transcoder. @@ -214,11 +235,12 @@ belong to workers. Background synchronization identified by `-Library` plus persisted and is discarded on cache reset, shutdown, or process restart. Detailed numeric metadata reads pass through an independent analysis-parameter -filter before admission control. When enabled, it removes only `checkFiles` and -`asyncAugmentMetadata`, preserving all retained raw query segments and leaving -headers and responses untouched. This prevents browse bursts over STRM entries -from asking Plex to start file scanners. The filter and concurrency guard have -separate switches so capacity tests never need to restore on-demand analysis. +filter before enrichment, coalescing, and admission control. When enabled, it +removes only `asyncAugmentMetadata`, preserving all retained raw query segments, +headers, and `checkFiles`. Plex therefore retains ownership of synchronous +`Part.accessible` and `Part.exists` fields while browse bursts cannot schedule +asynchronous augmentation. The filter, coalescer, and Guard have separate +switches for compatibility and capacity tests. `/metrics` keeps aggregate scheduler counters and a fixed, label-free P0/P1/P2 breakdown for offers, admissions, joins, promotions, capacity drops, expiry, and diff --git a/docs/media-lifecycle-architecture.md b/docs/media-lifecycle-architecture.md index 35ce3f4..08e62e7 100644 --- a/docs/media-lifecycle-architecture.md +++ b/docs/media-lifecycle-architecture.md @@ -601,8 +601,8 @@ Gateway 应负责： - 透明转发 Plex API 和观看状态； - 只对已确认 STRM Part 进行 Direct Play/302 适配； - 对所有未知和已知项目的详细 metadata 请求执行通用有界准入，保护冷缓存和首次浏览； -- 对相同认证范围、ratingKey、query profile、内容协商和 Gateway 版本做 singleflight - 与短缓存，禁止跨 Plex 用户或 token 权限域复用； +- 对认证范围、完整 query、客户端 Header、内容协商和远端身份一致的单项读取做短窗口 + 微批，不缓存 metadata，禁止跨 Plex 用户、Token、Cookie 或 Authorization 权限域复用； - 过载时只对非关键详细 metadata 返回合规的缓存响应或明确 `429/503`，不能 fail-open 绕过保护；时间线、观看状态和本地播放请求不进入该限流池； - 云项目定向策略只能使用独立、稳定、只读的分类索引，不能依赖 MoviePilot 私有表， diff --git a/docs/performance-matrix.md b/docs/performance-matrix.md index 11bbf05..09c6d50 100644 --- a/docs/performance-matrix.md +++ b/docs/performance-matrix.md @@ -15,8 +15,10 @@ 因此持久化 I/O 不得阻塞热路径。 - 单项 metadata 冷 miss 只允许内存级 P2 投递并立即返回 Plex 原响应，不同步读取 SQLite、 请求 MediaVault 或等待 CDN probe。 -- 详细 metadata 请求默认移除 `checkFiles` 和 `asyncAugmentMetadata`，避免 STRM 浏览突发 - 启动 Plex Scanner；过滤器不解析 body、不缓存响应，也不规范化其余 query。 +- 详细 metadata 请求默认只移除 `asyncAugmentMetadata`，避免浏览突发调度 Plex 后台分析； + `checkFiles` 继续透传以保留 `Part.accessible` 与 `Part.exists`。 +- 通用 metadata 微批只合并认证、query、Header、表示和远端身份完全一致的单项 GET。 + 它不缓存响应；HEAD、Range、条件请求和任何不确定身份直接透传。 - 一个 universal decision 最多等待一次 `MEDIAINFO_COLD_WAIT`。veto 只复用响应增强已经 取得的新鲜记录，不增加缓存读取或远程探测。 - 同一 MediaInfo key 的并发请求必须共享 singleflight。并发数不能突破配置的 worker @@ -43,15 +45,25 @@ | Metadata 分析过滤器，非 metadata | 6.247-6.429 ns/op | 0 B, 0 allocs | 普通代理路径仅执行方法和前缀判断。 | | Metadata 分析过滤器，无目标参数 | 75.36-77.49 ns/op | 32 B, 2 allocs | 保留原请求和完整 raw query。 | | Metadata 分析过滤器，移除目标参数 | 377.3-386.3 ns/op | 768 B, 7 allocs | 克隆请求 URL，其他 query 字节不变。 | +| 32 项 metadata 直接内存 handler | 76.87-78.02 us/op | 约 219.5 KiB, 929 allocs | 仅作为微批代码增量对照，不包含 Plex 网络或数据库。 | +| 32 项 metadata 微批、拆分与 fan-out | 422.9-448.4 us/op | 约 577.1 KiB, 4856-4857 allocs | 相对直接 handler 增加约 0.35-0.37 ms，包含 goroutine、JSON 拆分及 32 个响应。 | 真实 MediaVault/CDN 抽样为 5/5 成功：MediaVault redirect p50 141 ms、范围 109-533 ms；受限 CDN ffprobe p50 601 ms、范围 403-674 ms；合计 p50 761 ms、范围 543-955 ms。每次读取约 1.22 MiB 并发生 2 次 seek。样本量较小，只证明当前探测通常能在 默认 5 秒冷等待内完成，不能作为 CDN SLA。 -当前 Metadata Guard 默认值为单项全局 8、单客户端 4、批量 3。单项全局 16、单客户端 4、 -批量 3 只有在 Plex MediaInfo 写库达到较高覆盖率并完成重新压测后才作为候选，不属于当前 -版本的性能承诺。 +真实 Plex 控制面 A/B 中，直连 Plex 与 Gateway 透明代理单项 metadata p50 均约 +`46-47ms`。Plex 原生 batch 的 16、32、64、100 项响应约为 `50ms`、`94ms`、`182ms`、 +`285ms`。Apple TV 长剧集样本曾在约一秒内发出 1010 个单项请求，其中 581 个唯一条目， +峰值约 180 请求/秒；固定并发 Guard 会将这种突发转换为多轮排队。上述数据支持默认使用 +32 项、3ms 微批：32 项约为 340 item/s，64 项约为 352 item/s，吞吐只增加约 3.5%， +但单批延迟和失败回退范围接近翻倍。64 保留为可配置 A/B 候选，不能替代候选部署后的 +客户端首屏验证。 + +当前 Metadata Guard 默认值为单项全局 8、单客户端 4、批量 3。合成 batch 占用批量池， +异常回退占用单项池。是否调整这些值只依据候选版本的 Plex 延迟、错误率和真实客户端 +请求矩阵，不与未来 Plex MediaInfo 写入方案绑定。 2026-08-30 的 NAS 阻塞 A/B 将旧候选判定为不可接受：同一冷 metadata 请求直连 Plex 约 `0.52s`，经旧候选约 `5.15s`，Gateway 热缓存约 `0.15s`。同期实际 CDN ffprobe 平均 @@ -76,7 +88,8 @@ | 不同 Part 并发 2/4/8 | 受 worker 上限约束 | active、队列、CPU、内存、错误率 | 不突破 `MEDIAINFO_CONCURRENCY`，不拖慢本地或 302 路径。 | | A/B/C 快速切换 | 当前项 P0 probe、P1 邻近预热 | 每项延迟、身份对应、替换/队列指标 | 新当前项优先排队；不能抢占正在运行的 probe；无跨项 MediaInfo、grant 或 redirect。 | | 长剧集 metadata 突发 | Plex、L1；P2 后台可查 SQLite/探测 | 前台响应、P2 队列/过期/远程启动次数 | 前台不等待 P2；唯一 pending P2 <= 50；远程启动间隔 >= 5 s；过期任务 0 MV/CDN。 | -| 长剧集分析参数过滤 A/B | Plex、L1 | 过滤计数、Scanner 进程、metadata p50/p95/p99、错误率 | 开启过滤后 Scanner 为 0；先保留 Guard 验证，再关闭 Guard 测试 Plex 原生吞吐。 | +| 长剧集 metadata 微批 | Plex、L1 | 原始单项数、合成 batch 数、唯一 item、fallback、p50/p95/p99 | 同组 batch <= 32；身份不跨组；正常响应 0 fallback；相对透明代理显著降低上游请求并改善页面完成时间。 | +| 长剧集分析参数过滤 A/B | Plex、L1 | 过滤计数、Scanner 进程、`accessible/exists`、metadata p50/p95/p99 | `asyncAugmentMetadata` 不到达 Plex；`checkFiles` 和对应字段保留；Scanner 为 0。 | 表中的验收条件是候选版本的门槛，不代表本机微基准已经替代端到端验收。当前已有证据 包括本机代码基准和 5/5 成功的 MediaVault/CDN 探测抽样；本地透明代理、302 热冷路径、 @@ -94,6 +107,9 @@ go test -run '^$' \ go test -run '^$' -bench '^BenchmarkMetadataAnalysisFilter$' \ -benchmem -count=5 ./internal/gateway + +go test -run '^$' -bench '^BenchmarkMetadataCoalescerBurst32$' \ + -benchmem -count=5 ./internal/gateway ``` 真实 NAS 验收必须使用候选镜像、固定样本和串行 A/B，记录版本、配置、样本数、分位数、 diff --git a/docs/roadmap.md b/docs/roadmap.md index c6cda08..f9d1c57 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -116,9 +116,10 @@ delivery gate for this work. Local media must remain transparent, Part and universal-start routes must remain free of analysis I/O, an L1 hit must not block on SQLite, and one decision may consume at most one configured cold-wait budget. The current Guard defaults are global 8, per-client 4, and batch 3. -After production Plex MediaInfo writing provides high coverage and the load is -retested, global 16, per-client 4, and batch 3 may be evaluated. No new Guard -mode or configuration is planned for this release. +Equivalent single-item metadata GET requests may use the bounded native Plex +micro-batch adapter before Guard admission; invalid batches return through the +single-item limits. Guard changes remain evidence-driven rather than tied to a +future Plex MediaInfo write strategy. The complete MoviePilot, MediaVault, 115 share, ownership, seeding, cleanup, MediaInfo, and projection design is documented in diff --git a/internal/config/config.go b/internal/config/config.go index ab7ae67..6f5823f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -14,40 +14,45 @@ import ( ) const ( - defaultListenAddr = ":32400" - defaultReadHeaderTimeout = 15 * time.Second - defaultIdleTimeout = 90 * time.Second - defaultShutdownTimeout = 15 * time.Second - defaultPartTTL = 24 * time.Hour - defaultResolverTimeout = 15 * time.Second - defaultObserveMaxBytes = 8 << 20 - defaultPartProbeTimeout = 15 * time.Second - defaultMetadataGlobal = 8 - defaultMetadataPerClient = 4 - defaultMetadataBatch = 3 - defaultMetadataQueueWait = 10 * time.Second - defaultDatabasePath = "./data/plex-gateway.db" - defaultMediaInfoProbe = 20 * time.Second - defaultMediaInfoSize = 8 << 20 - defaultMediaInfoAnalyze = 5 * time.Second - defaultMediaInfoOutput = 2 << 20 - defaultMediaInfoWorkers = 1 - defaultPlaybackQueue = 16 - defaultNeighborQueue = 50 - defaultMetadataQueue = 50 - defaultPendingTTL = 5 * time.Minute - defaultBackgroundInterval = 5 * time.Second - defaultMediaInfoTTL = 30 * 24 * time.Hour - defaultMediaInfoRetention = 180 * 24 * time.Hour - defaultMediaInfoL1Entries = 10_000 - defaultMediaInfoNegative = 15 * time.Minute - defaultMediaInfoAgent = "Infuse-Library/8.5.1" - defaultMediaInfoColdWait = 5 * time.Second - defaultMediaInfoBodyLimit = 8 << 20 - defaultMediaInfoWaiters = 8 - defaultPrewarmBefore = 2 - defaultPrewarmAfter = 3 - maxPrewarmWindow = 50 + defaultListenAddr = ":32400" + defaultReadHeaderTimeout = 15 * time.Second + defaultIdleTimeout = 90 * time.Second + defaultShutdownTimeout = 15 * time.Second + defaultPartTTL = 24 * time.Hour + defaultResolverTimeout = 15 * time.Second + defaultObserveMaxBytes = 8 << 20 + defaultPartProbeTimeout = 15 * time.Second + defaultMetadataGlobal = 8 + defaultMetadataPerClient = 4 + defaultMetadataBatch = 3 + defaultMetadataQueueWait = 10 * time.Second + defaultMetadataCoalesceWindow = 3 * time.Millisecond + defaultMetadataCoalesceMaxItems = 32 + defaultMetadataCoalesceTimeout = 5 * time.Second + maxMetadataCoalesceWindow = 100 * time.Millisecond + maxMetadataCoalesceItems = 64 + defaultDatabasePath = "./data/plex-gateway.db" + defaultMediaInfoProbe = 20 * time.Second + defaultMediaInfoSize = 8 << 20 + defaultMediaInfoAnalyze = 5 * time.Second + defaultMediaInfoOutput = 2 << 20 + defaultMediaInfoWorkers = 1 + defaultPlaybackQueue = 16 + defaultNeighborQueue = 50 + defaultMetadataQueue = 50 + defaultPendingTTL = 5 * time.Minute + defaultBackgroundInterval = 5 * time.Second + defaultMediaInfoTTL = 30 * 24 * time.Hour + defaultMediaInfoRetention = 180 * 24 * time.Hour + defaultMediaInfoL1Entries = 10_000 + defaultMediaInfoNegative = 15 * time.Minute + defaultMediaInfoAgent = "Infuse-Library/8.5.1" + defaultMediaInfoColdWait = 5 * time.Second + defaultMediaInfoBodyLimit = 8 << 20 + defaultMediaInfoWaiters = 8 + defaultPrewarmBefore = 2 + defaultPrewarmAfter = 3 + maxPrewarmWindow = 50 ) // Config contains process-level settings for transparent Plex proxying and @@ -68,6 +73,7 @@ type Config struct { PartProbeTimeout time.Duration MetadataAnalysisFilter bool MetadataGuard MetadataGuardConfig + MetadataCoalesce MetadataCoalesceConfig MediaInfo MediaInfoConfig PlaybackVeto bool LogLevel string @@ -118,6 +124,15 @@ type MetadataGuardConfig struct { QueueTimeout time.Duration } +// MetadataCoalesceConfig bounds transparent aggregation of equivalent +// single-item reads into Plex's native comma-separated metadata endpoint. +type MetadataCoalesceConfig struct { + Enabled bool + Window time.Duration + MaxItems int + Timeout time.Duration +} + // Load reads environment configuration and rejects values that would make the // proxy ambiguous or accidentally embed credentials in durable configuration. func Load() (Config, error) { @@ -214,6 +229,31 @@ func Load() (Config, error) { if err != nil { return Config{}, err } + metadataCoalesceEnabled, err := envBool("METADATA_COALESCE_ENABLED", true) + if err != nil { + return Config{}, err + } + metadataCoalesceWindow, err := envDuration("METADATA_COALESCE_WINDOW", defaultMetadataCoalesceWindow) + if err != nil { + return Config{}, err + } + metadataCoalesceMaxItems, err := envPositiveInt("METADATA_COALESCE_MAX_ITEMS", defaultMetadataCoalesceMaxItems) + if err != nil { + return Config{}, err + } + metadataCoalesceTimeout, err := envDuration("METADATA_COALESCE_TIMEOUT", defaultMetadataCoalesceTimeout) + if err != nil { + return Config{}, err + } + if metadataCoalesceWindow > maxMetadataCoalesceWindow { + return Config{}, fmt.Errorf("METADATA_COALESCE_WINDOW must not exceed %s", maxMetadataCoalesceWindow) + } + if metadataCoalesceMaxItems < 2 { + return Config{}, errors.New("METADATA_COALESCE_MAX_ITEMS must be at least 2") + } + if metadataCoalesceMaxItems > maxMetadataCoalesceItems { + return Config{}, fmt.Errorf("METADATA_COALESCE_MAX_ITEMS must not exceed %d", maxMetadataCoalesceItems) + } mediaInfoEnabled, err := envBool("MEDIAINFO_ENABLED", true) if err != nil { return Config{}, err @@ -366,6 +406,10 @@ func Load() (Config, error) { BatchConcurrency: metadataBatch, QueueTimeout: metadataQueueWait, }, + MetadataCoalesce: MetadataCoalesceConfig{ + Enabled: metadataCoalesceEnabled, Window: metadataCoalesceWindow, + MaxItems: metadataCoalesceMaxItems, Timeout: metadataCoalesceTimeout, + }, MediaInfo: MediaInfoConfig{ Enabled: mediaInfoEnabled, FFProbePath: mediaInfoFFProbe, ProbeTimeout: mediaInfoProbeTimeout, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index fc2fbbe..f5201a8 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -25,6 +25,7 @@ func TestLoadDefaults(t *testing.T) { t.Setenv("LOG_LEVEL", "") t.Setenv("TRACE_ENABLED", "") t.Setenv("METADATA_ANALYSIS_FILTER_ENABLED", "") + t.Setenv("METADATA_COALESCE_ENABLED", "") got, err := Load() if err != nil { @@ -48,6 +49,9 @@ func TestLoadDefaults(t *testing.T) { if !got.MetadataAnalysisFilter { t.Fatal("MetadataAnalysisFilter = false") } + if !got.MetadataCoalesce.Enabled || got.MetadataCoalesce.Window != 3*time.Millisecond || got.MetadataCoalesce.MaxItems != 32 || got.MetadataCoalesce.Timeout != 5*time.Second { + t.Fatalf("unexpected metadata coalesce defaults: %#v", got.MetadataCoalesce) + } if got.DatabasePath != "./data/plex-gateway.db" { t.Fatalf("DatabasePath = %q", got.DatabasePath) } @@ -93,6 +97,10 @@ func TestLoadCloudConfiguration(t *testing.T) { t.Setenv("METADATA_GUARD_BATCH_ENABLED", "true") t.Setenv("METADATA_GUARD_BATCH_CONCURRENCY", "2") t.Setenv("METADATA_GUARD_QUEUE_TIMEOUT", "5s") + t.Setenv("METADATA_COALESCE_ENABLED", "false") + t.Setenv("METADATA_COALESCE_WINDOW", "7ms") + t.Setenv("METADATA_COALESCE_MAX_ITEMS", "24") + t.Setenv("METADATA_COALESCE_TIMEOUT", "4s") t.Setenv("MEDIAINFO_ENABLED", "true") t.Setenv("DATABASE_PATH", "/app_data/test.db") t.Setenv("MEDIAINFO_FFPROBE_PATH", "/usr/bin/ffprobe") @@ -139,6 +147,9 @@ func TestLoadCloudConfiguration(t *testing.T) { if got.MetadataAnalysisFilter { t.Fatal("MetadataAnalysisFilter = true") } + if got.MetadataCoalesce.Enabled || got.MetadataCoalesce.Window != 7*time.Millisecond || got.MetadataCoalesce.MaxItems != 24 || got.MetadataCoalesce.Timeout != 4*time.Second { + t.Fatalf("metadata coalesce = %#v", got.MetadataCoalesce) + } if got.DatabasePath != "/app_data/test.db" { t.Fatalf("DatabasePath = %q", got.DatabasePath) } @@ -188,6 +199,29 @@ func TestLoadRejectsMetadataClientLimitAboveGlobalLimit(t *testing.T) { } } +func TestLoadRejectsUnsafeMetadataCoalesceBounds(t *testing.T) { + tests := []struct { + name string + env string + value string + }{ + {name: "window", env: "METADATA_COALESCE_WINDOW", value: "101ms"}, + {name: "too_few_items", env: "METADATA_COALESCE_MAX_ITEMS", value: "1"}, + {name: "items", env: "METADATA_COALESCE_MAX_ITEMS", value: "65"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Setenv("PLEX_URL", "http://plex:32400") + t.Setenv(test.env, test.value) + + _, err := Load() + if err == nil || !strings.Contains(err.Error(), "must") { + t.Fatalf("Load() error = %v", err) + } + }) + } +} + func TestLoadRequiresCompleteCloudConfiguration(t *testing.T) { t.Setenv("PLEX_URL", "http://plex:32400") t.Setenv("MEDIAVAULT_URL", "http://mediavault:7811") diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 7bec5b8..b3a0e90 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -18,6 +18,7 @@ import ( "github.com/InfinityPacer/plex-gateway/internal/partcache" "github.com/InfinityPacer/plex-gateway/internal/pathmap" "github.com/InfinityPacer/plex-gateway/internal/playback" + "github.com/InfinityPacer/plex-gateway/internal/plexmeta" "github.com/InfinityPacer/plex-gateway/internal/prewarm" "github.com/InfinityPacer/plex-gateway/internal/resolver" "github.com/InfinityPacer/plex-gateway/internal/trace" @@ -40,6 +41,7 @@ type Options struct { PartProbeTimeout time.Duration MetadataAnalysisFilter bool MetadataGuard MetadataGuardOptions + MetadataCoalesce MetadataCoalesceOptions MediaInfoEnabled bool MediaInfoStatus func() mediainfo.Status MediaInfoService *mediainfo.Service @@ -73,6 +75,13 @@ func New(options Options) http.Handler { } var cloudPlayback *playback.Service + rememberParts := func(ratingKey string, parts []plexmeta.Part) { + for _, part := range parts { + if cloudPlayback != nil { + cloudPlayback.Remember(playback.PreparedPart{Part: part, RatingKey: ratingKey}) + } + } + } transport := http.RoundTripper(newTransport()) if options.PartCache != nil { transport = observe.NewRoundTripper(transport, observe.Config{ @@ -83,13 +92,9 @@ func New(options Options) http.Handler { "/playQueues", }, MaxBodyBytes: options.ObserveMaxBytes, + SkipRequest: isMetadataCoalesceUpstreamRequest, OnParts: func(observation observe.Observation) { - ratingKey := observedRatingKey(observation.Request) - for _, part := range observation.Parts { - if cloudPlayback != nil { - cloudPlayback.Remember(playback.PreparedPart{Part: part, RatingKey: ratingKey}) - } - } + rememberParts(observedRatingKey(observation.Request), observation.Parts) }, }) } @@ -206,15 +211,22 @@ func New(options Options) http.Handler { } mux.Handle("GET /library/parts/{partID}/{rest...}", partPlayback) mux.Handle("HEAD /library/parts/{partID}/{rest...}", partPlayback) + metadataCoalesce := options.MetadataCoalesce + metadataCoalesce.OnMetadata = func(request *http.Request, body []byte, contentType string) { + parts, err := plexmeta.ParseParts(body, contentType) + if err != nil { + return + } + rememberParts(observedRatingKey(request), parts) + } + metadataSource := newMetadataGuard(options.MetadataGuard, plex, registry, logger) + metadataSource = newMetadataCoalescer(metadataCoalesce, metadataSource, registry, logger) metadata := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ Service: options.MediaInfoService, Mapper: options.PathMapper, Resolver: options.Resolver, CloudExtensions: options.CloudExtensions, ResponseLimit: options.MediaInfoResponseMaxBytes, Concurrency: options.MediaInfoEnrichmentConcurrency, Metrics: registry, - }, plex) - // Admission must happen before response buffering so a queued request does - // not consume an enrichment slot or bypass projection under client bursts. - metadata = newMetadataGuard(options.MetadataGuard, metadata, registry, logger) + }, metadataSource) metadata = newMetadataAnalysisFilter(options.MetadataAnalysisFilter, metadata, registry) mux.Handle("/", metadata) diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index 87c9811..650ffdf 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -70,7 +70,7 @@ func TestTransparentProxyPreservesRequestAndResponse(t *testing.T) { func TestMetadataAnalysisFilterIsAppliedBeforePlexProxy(t *testing.T) { registry := metrics.New() upstream := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - if got := request.URL.RawQuery; got != "X-Plex-Token=a%2Bb&includeMarkers=1" { + if got := request.URL.RawQuery; got != "checkFiles=1&X-Plex-Token=a%2Bb&includeMarkers=1" { t.Fatalf("upstream query = %q", got) } if got := request.Header.Get("X-Plex-Client-Identifier"); got != "client-id" { diff --git a/internal/gateway/metadata_analysis_filter.go b/internal/gateway/metadata_analysis_filter.go index 155b11b..0ad192e 100644 --- a/internal/gateway/metadata_analysis_filter.go +++ b/internal/gateway/metadata_analysis_filter.go @@ -10,12 +10,11 @@ import ( var metadataAnalysisParameters = map[string]struct{}{ "asyncAugmentMetadata": {}, - "checkFiles": {}, } -// newMetadataAnalysisFilter prevents detailed metadata reads from asking Plex -// to inspect media files on demand. Library scans remain responsible for local -// file freshness, while STRM entries can use the Gateway MediaInfo projection. +// newMetadataAnalysisFilter prevents detailed metadata reads from scheduling +// asynchronous Plex analysis. Synchronous file accessibility fields remain +// Plex-owned and are therefore preserved. func newMetadataAnalysisFilter(enabled bool, next http.Handler, registry *metrics.Metrics) http.Handler { if !enabled { return next diff --git a/internal/gateway/metadata_analysis_filter_test.go b/internal/gateway/metadata_analysis_filter_test.go index c2fb8ac..5291fce 100644 --- a/internal/gateway/metadata_analysis_filter_test.go +++ b/internal/gateway/metadata_analysis_filter_test.go @@ -24,7 +24,7 @@ func TestMetadataAnalysisFilterRemovesOnlyAnalysisParameters(t *testing.T) { if response.Code != http.StatusNoContent { t.Fatalf("status = %d, want %d", response.Code, http.StatusNoContent) } - if received != "z=last&X-Plex-Token=a%2Bb&a=first+value" { + if received != "z=last&checkFiles=1&X-Plex-Token=a%2Bb&a=first+value" { t.Fatalf("filtered query = %q", received) } if request.URL.RawQuery != originalQuery { @@ -42,11 +42,11 @@ func TestMetadataAnalysisFilterAppliesToSupportedReads(t *testing.T) { path string want string }{ - {name: "head", method: http.MethodHead, path: "/library/metadata/42?checkFiles=1&keep=yes", want: "keep=yes"}, + {name: "head", method: http.MethodHead, path: "/library/metadata/42?asyncAugmentMetadata=1&checkFiles=1&keep=yes", want: "checkFiles=1&keep=yes"}, {name: "batch", method: http.MethodGet, path: "/library/metadata/42,43?asyncAugmentMetadata&keep=yes", want: "keep=yes"}, - {name: "children", method: http.MethodGet, path: "/library/metadata/42/children?checkFiles=1", want: "checkFiles=1"}, - {name: "write", method: http.MethodPut, path: "/library/metadata/42?checkFiles=1", want: "checkFiles=1"}, - {name: "non_numeric", method: http.MethodGet, path: "/library/metadata/abc?checkFiles=1", want: "checkFiles=1"}, + {name: "children", method: http.MethodGet, path: "/library/metadata/42/children?asyncAugmentMetadata=1", want: "asyncAugmentMetadata=1"}, + {name: "write", method: http.MethodPut, path: "/library/metadata/42?asyncAugmentMetadata=1", want: "asyncAugmentMetadata=1"}, + {name: "non_numeric", method: http.MethodGet, path: "/library/metadata/abc?asyncAugmentMetadata=1", want: "asyncAugmentMetadata=1"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -70,9 +70,9 @@ func TestMetadataAnalysisFilterPreservesMalformedAndDisabledQueries(t *testing.T enabled bool query string }{ - {name: "disabled", enabled: false, query: "checkFiles=1&keep=yes"}, - {name: "malformed_key", enabled: true, query: "checkFiles%ZZ=1&keep=yes"}, - {name: "unrelated", enabled: true, query: "checkFilesExtra=1&keep=yes"}, + {name: "disabled", enabled: false, query: "asyncAugmentMetadata=1&keep=yes"}, + {name: "malformed_key", enabled: true, query: "asyncAugmentMetadata%ZZ=1&keep=yes"}, + {name: "unrelated", enabled: true, query: "asyncAugmentMetadataExtra=1&keep=yes"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/internal/gateway/metadata_coalesce.go b/internal/gateway/metadata_coalesce.go new file mode 100644 index 0000000..2c7e1a9 --- /dev/null +++ b/internal/gateway/metadata_coalesce.go @@ -0,0 +1,684 @@ +package gateway + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "hash" + "log/slog" + "net" + "net/http" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/InfinityPacer/plex-gateway/internal/metrics" + "github.com/InfinityPacer/plex-gateway/internal/plexmeta" +) + +const ( + defaultMetadataCoalesceWindow = 3 * time.Millisecond + defaultMetadataCoalesceMaxItems = 32 + defaultMetadataCoalesceTimeout = 5 * time.Second + defaultMetadataCoalesceResponseLimit = 8 << 20 + defaultMetadataCoalesceMaxGroups = 256 + defaultMetadataCoalesceMaxWaiters = 128 + defaultMetadataCoalesceCooldown = time.Second + maximumMetadataCoalesceWindow = 100 * time.Millisecond + maximumMetadataCoalesceItems = 64 +) + +// MetadataCoalesceOptions controls a bounded aggregation window for equivalent +// single-item metadata reads. Every unsafe or ambiguous response falls back to +// the original requests through the same upstream admission guard. +type MetadataCoalesceOptions struct { + Enabled bool + Window time.Duration + MaxItems int + Timeout time.Duration + OnMetadata func(*http.Request, []byte, string) +} + +type metadataCoalescer struct { + next http.Handler + window time.Duration + maxItems int + timeout time.Duration + bodyLimit int64 + maxGroups int + maxWaiters int + onMetadata func(*http.Request, []byte, string) + metrics *metrics.Metrics + logger *slog.Logger + + mu sync.Mutex + groups map[[sha256.Size]byte]*metadataCoalesceGroup + failed map[[sha256.Size]byte]time.Time +} + +type metadataCoalesceGroup struct { + key [sha256.Size]byte + representative *http.Request + items map[string][]*metadataCoalesceCall + order []string + timer *time.Timer + waiters int + + lifecycleMu sync.Mutex + cancel context.CancelFunc +} + +type metadataCoalesceCall struct { + writer http.ResponseWriter + request *http.Request + result chan metadataCoalesceResult + + mu sync.Mutex + active bool + group *metadataCoalesceGroup +} + +type metadataCoalesceResult struct { + direct bool + followers []*metadataCoalesceCall + headers http.Header + status int + body []byte +} + +type metadataCoalesceContextKey struct{} + +func newMetadataCoalescer(options MetadataCoalesceOptions, next http.Handler, registry *metrics.Metrics, logger *slog.Logger) http.Handler { + if next == nil || !options.Enabled { + return next + } + window := options.Window + if window <= 0 { + window = defaultMetadataCoalesceWindow + } else if window > maximumMetadataCoalesceWindow { + window = maximumMetadataCoalesceWindow + } + maxItems := options.MaxItems + if maxItems == 1 { + return next + } + if maxItems <= 0 { + maxItems = defaultMetadataCoalesceMaxItems + } else if maxItems > maximumMetadataCoalesceItems { + maxItems = maximumMetadataCoalesceItems + } + timeout := options.Timeout + if timeout <= 0 { + timeout = defaultMetadataCoalesceTimeout + } + if registry == nil { + registry = metrics.New() + } + if logger == nil { + logger = slog.Default() + } + return &metadataCoalescer{ + next: next, window: window, maxItems: maxItems, timeout: timeout, + bodyLimit: defaultMetadataCoalesceResponseLimit, onMetadata: options.OnMetadata, + maxGroups: defaultMetadataCoalesceMaxGroups, maxWaiters: defaultMetadataCoalesceMaxWaiters, + metrics: registry, logger: logger, groups: make(map[[sha256.Size]byte]*metadataCoalesceGroup), + failed: make(map[[sha256.Size]byte]time.Time), + } +} + +func (coalescer *metadataCoalescer) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + ratingKey, eligible := metadataCoalesceRatingKey(request) + if !eligible { + coalescer.next.ServeHTTP(writer, request) + return + } + + call := &metadataCoalesceCall{ + writer: writer, request: request, result: make(chan metadataCoalesceResult, 1), active: true, + } + coalescer.metrics.IncMetadataCoalesceOffered() + if !coalescer.enqueue(metadataCoalesceSignature(request), ratingKey, call) { + coalescer.next.ServeHTTP(writer, request) + return + } + + var result metadataCoalesceResult + select { + case result = <-call.result: + case <-request.Context().Done(): + if call.cancel() { + return + } + result = <-call.result + } + coalescer.serveResult(call, result) +} + +func (coalescer *metadataCoalescer) enqueue(key [sha256.Size]byte, ratingKey string, call *metadataCoalesceCall) bool { + coalescer.mu.Lock() + if deadline, blocked := coalescer.failed[key]; blocked { + if time.Now().Before(deadline) { + coalescer.mu.Unlock() + return false + } + delete(coalescer.failed, key) + } + group := coalescer.groups[key] + if group == nil { + if len(coalescer.groups) >= coalescer.maxGroups { + coalescer.mu.Unlock() + return false + } + group = &metadataCoalesceGroup{ + key: key, + representative: call.request, + items: make(map[string][]*metadataCoalesceCall), + } + coalescer.groups[key] = group + group.timer = time.AfterFunc(coalescer.window, func() { + coalescer.flush(key, group) + }) + } + if group.waiters >= coalescer.maxWaiters { + coalescer.mu.Unlock() + return false + } + call.group = group + group.waiters++ + if _, exists := group.items[ratingKey]; !exists { + group.order = append(group.order, ratingKey) + } + group.items[ratingKey] = append(group.items[ratingKey], call) + full := len(group.order) >= coalescer.maxItems + if full { + delete(coalescer.groups, key) + group.timer.Stop() + } + coalescer.mu.Unlock() + + if full { + go coalescer.execute(group) + } + return true +} + +func (coalescer *metadataCoalescer) flush(key [sha256.Size]byte, group *metadataCoalesceGroup) { + coalescer.mu.Lock() + if coalescer.groups[key] != group { + coalescer.mu.Unlock() + return + } + delete(coalescer.groups, key) + coalescer.mu.Unlock() + coalescer.execute(group) +} + +func (coalescer *metadataCoalescer) execute(group *metadataCoalesceGroup) { + order, items, representative, liveCalls := liveMetadataCoalesceItems(group) + if liveCalls == 0 { + return + } + if liveCalls == 1 { + for _, ratingKey := range order { + items[ratingKey][0].dispatch(metadataCoalesceResult{direct: true}) + return + } + } + + baseContext := context.WithoutCancel(representative.Context()) + ctx, cancel := context.WithTimeout(baseContext, coalescer.timeout) + ctx = context.WithValue(ctx, metadataCoalesceContextKey{}, true) + group.installCancel(cancel) + defer group.clearCancel() + defer cancel() + if !group.anyActive() { + return + } + + coalescer.metrics.IncMetadataCoalesceBatch(len(order)) + coalescer.metrics.IncMetadataCoalesceActive() + responses, headers, status, err := coalescer.requestBatch(ctx, representative, order) + coalescer.metrics.DecMetadataCoalesceActive() + if err != nil { + if !group.anyActive() { + return + } + coalescer.metrics.IncMetadataCoalesceFallback() + coalescer.markFailed(group.key) + coalescer.logger.Warn("metadata_coalesce_fallback", "reason", metadataCoalesceErrorKind(err)) + coalescer.fallback(order, items) + return + } + + wireBodies := make(map[string][]byte, len(order)) + responseHeaders := make(map[string]http.Header, len(order)) + for _, ratingKey := range order { + decoded := responses[ratingKey] + wireBody, encodeErr := encodeMetadataBody(decoded, headers.Get("Content-Encoding")) + if encodeErr != nil { + coalescer.metrics.IncMetadataCoalesceFallback() + coalescer.markFailed(group.key) + coalescer.logger.Warn("metadata_coalesce_fallback", "reason", "encode") + coalescer.fallback(order, items) + return + } + wireBodies[ratingKey] = wireBody + responseHeaders[ratingKey] = metadataCoalesceResponseHeaders(headers, len(wireBody)) + } + for _, ratingKey := range order { + decoded := responses[ratingKey] + if coalescer.onMetadata != nil { + coalescer.observe(items[ratingKey][0].request, decoded, responseHeaders[ratingKey].Get("Content-Type")) + } + for _, call := range items[ratingKey] { + call.dispatch(metadataCoalesceResult{ + headers: responseHeaders[ratingKey], status: status, body: wireBodies[ratingKey], + }) + } + } +} + +func liveMetadataCoalesceItems(group *metadataCoalesceGroup) ([]string, map[string][]*metadataCoalesceCall, *http.Request, int) { + order := make([]string, 0, len(group.order)) + items := make(map[string][]*metadataCoalesceCall, len(group.items)) + var representative *http.Request + liveCalls := 0 + for _, ratingKey := range group.order { + for _, call := range group.items[ratingKey] { + if !call.isActive() { + continue + } + if representative == nil { + representative = call.request + } + items[ratingKey] = append(items[ratingKey], call) + liveCalls++ + } + if len(items[ratingKey]) != 0 { + order = append(order, ratingKey) + } + } + return order, items, representative, liveCalls +} + +func (coalescer *metadataCoalescer) requestBatch( + ctx context.Context, + request *http.Request, + ratingKeys []string, +) (responses map[string][]byte, headers http.Header, status int, err error) { + defer func() { + recovered := recover() + if recovered == nil { + return + } + if recovered == http.ErrAbortHandler { + responses, headers, status = nil, nil, 0 + err = errors.New("metadata coalesce upstream body aborted") + return + } + panic(recovered) + }() + if request == nil || len(ratingKeys) == 0 { + return nil, nil, 0, errors.New("metadata coalesce request is empty") + } + batchRequest := request.Clone(ctx) + batchRequest.URL.Path = "/library/metadata/" + strings.Join(ratingKeys, ",") + batchRequest.URL.RawPath = "" + batchRequest.RequestURI = batchRequest.URL.RequestURI() + + capture := newMetadataCoalesceCapture(coalescer.bodyLimit) + coalescer.next.ServeHTTP(capture, batchRequest) + if err := ctx.Err(); err != nil { + return nil, nil, 0, err + } + if capture.overflow { + return nil, nil, 0, errors.New("metadata coalesce response exceeds limit") + } + if capture.statusCode() != http.StatusOK || capture.header.Get("Trailer") != "" || capture.header.Get("Content-Range") != "" { + return nil, nil, 0, fmt.Errorf("metadata coalesce upstream status %d", capture.statusCode()) + } + decoded, encoding, err := decodeMetadataBody(capture.body.Bytes(), capture.header.Get("Content-Encoding"), coalescer.bodyLimit) + if err != nil { + return nil, nil, 0, fmt.Errorf("decode metadata coalesce response: %w", err) + } + responses, err = plexmeta.SplitMetadataBatch(decoded, capture.header.Get("Content-Type"), ratingKeys) + if err != nil { + return nil, nil, 0, fmt.Errorf("split metadata coalesce response: %w", err) + } + headers = capture.header.Clone() + headers.Set("Content-Encoding", encoding) + if encoding == "" { + headers.Del("Content-Encoding") + } + return responses, headers, capture.statusCode(), nil +} + +func (coalescer *metadataCoalescer) fallback(order []string, items map[string][]*metadataCoalesceCall) { + for _, ratingKey := range order { + calls := items[ratingKey] + for index, leader := range calls { + followers := make([]*metadataCoalesceCall, 0, len(calls)-1) + followers = append(followers, calls[:index]...) + followers = append(followers, calls[index+1:]...) + if leader.dispatch(metadataCoalesceResult{direct: true, followers: followers}) { + break + } + } + } +} + +func (coalescer *metadataCoalescer) markFailed(key [sha256.Size]byte) { + now := time.Now() + coalescer.mu.Lock() + if len(coalescer.failed) >= coalescer.maxGroups { + for candidate, deadline := range coalescer.failed { + if !now.Before(deadline) { + delete(coalescer.failed, candidate) + } + } + } + if len(coalescer.failed) < coalescer.maxGroups { + coalescer.failed[key] = now.Add(defaultMetadataCoalesceCooldown) + } + coalescer.mu.Unlock() +} + +func (coalescer *metadataCoalescer) observe(request *http.Request, body []byte, contentType string) { + defer func() { _ = recover() }() + coalescer.onMetadata(request, body, contentType) +} + +func metadataCoalesceRatingKey(request *http.Request) (string, bool) { + if request == nil || request.URL == nil || request.Method != http.MethodGet || request.ContentLength > 0 || + request.URL.RawPath != "" || request.Body != nil && request.Body != http.NoBody || len(request.TransferEncoding) != 0 { + return "", false + } + if classifyMetadataRequest(request) != metadataRequestSingle { + return "", false + } + for _, header := range []string{"Range", "If-Match", "If-None-Match", "If-Modified-Since", "If-Unmodified-Since", "If-Range"} { + if request.Header.Get(header) != "" { + return "", false + } + } + _, tokenPresent, tokenValid := requestIdentity(request, "X-Plex-Token") + if !tokenPresent || !tokenValid { + return "", false + } + return strings.TrimPrefix(request.URL.Path, "/library/metadata/"), true +} + +func metadataCoalesceSignature(request *http.Request) [sha256.Size]byte { + digest := sha256.New() + writeMetadataCoalesceValue(digest, request.Method) + writeMetadataCoalesceValue(digest, request.Proto) + writeMetadataCoalesceValue(digest, request.Host) + writeMetadataCoalesceValue(digest, request.URL.RawQuery) + writeMetadataCoalesceValue(digest, metadataCoalesceRemoteIP(request.RemoteAddr)) + headerNames := make([]string, 0, len(request.Header)) + for name := range request.Header { + headerNames = append(headerNames, http.CanonicalHeaderKey(name)) + } + sort.Strings(headerNames) + for _, name := range headerNames { + writeMetadataCoalesceValue(digest, name) + for _, value := range request.Header.Values(name) { + writeMetadataCoalesceValue(digest, value) + } + } + var result [sha256.Size]byte + copy(result[:], digest.Sum(nil)) + return result +} + +func writeMetadataCoalesceValue(destination hash.Hash, value string) { + _, _ = destination.Write([]byte(strconv.Itoa(len(value)))) + _, _ = destination.Write([]byte{':'}) + _, _ = destination.Write([]byte(value)) +} + +func metadataCoalesceRemoteIP(remoteAddr string) string { + host, _, err := net.SplitHostPort(remoteAddr) + if err == nil { + return host + } + return remoteAddr +} + +func metadataCoalesceResponseHeaders(source http.Header, contentLength int) http.Header { + headers := source.Clone() + for _, name := range []string{"Content-Length", "ETag", "Last-Modified", "Content-MD5", "Digest", "Content-Digest", "Repr-Digest", "Content-Range", "Trailer"} { + headers.Del(name) + } + if !hasCacheDirective(headers.Get("Cache-Control"), "no-store") { + headers.Set("Cache-Control", "private, no-cache") + } + headers.Set("Content-Length", strconv.Itoa(contentLength)) + return headers +} + +func metadataCoalesceErrorKind(err error) string { + if err == nil { + return "none" + } + if errors.Is(err, context.DeadlineExceeded) { + return "timeout" + } + message := err.Error() + switch { + case strings.Contains(message, "status"): + return "status" + case strings.Contains(message, "exceeds"): + return "limit" + case strings.Contains(message, "decode"): + return "decode" + case strings.Contains(message, "split"): + return "split" + default: + return "upstream" + } +} + +func isMetadataCoalesceUpstreamRequest(request *http.Request) bool { + return request != nil && request.Context().Value(metadataCoalesceContextKey{}) == true +} + +func (call *metadataCoalesceCall) isActive() bool { + call.mu.Lock() + defer call.mu.Unlock() + return call.active +} + +func (call *metadataCoalesceCall) cancel() bool { + call.mu.Lock() + if !call.active { + call.mu.Unlock() + return false + } + call.active = false + group := call.group + call.mu.Unlock() + if group != nil { + group.cancelIfIdle() + } + return true +} + +func (call *metadataCoalesceCall) dispatch(result metadataCoalesceResult) bool { + call.mu.Lock() + defer call.mu.Unlock() + if !call.active { + return false + } + call.active = false + call.result <- result + return true +} + +func (coalescer *metadataCoalescer) serveResult(call *metadataCoalesceCall, result metadataCoalesceResult) { + if !result.direct { + writeCapturedResponse(call.writer, result.headers, result.status, result.body) + return + } + if len(result.followers) == 0 { + coalescer.next.ServeHTTP(call.writer, call.request) + return + } + + capture := newMetadataCoalesceTee(call.writer, coalescer.bodyLimit) + panicked := true + defer func() { + if panicked || !capture.reusable() { + coalescer.dispatchFallbackError(result.followers) + return + } + response := metadataCoalesceResult{ + headers: capture.Header().Clone(), status: capture.statusCode(), body: capture.body.Bytes(), + } + for _, follower := range result.followers { + follower.dispatch(response) + } + }() + coalescer.next.ServeHTTP(capture, call.request) + panicked = false +} + +func (coalescer *metadataCoalescer) dispatchFallbackError(calls []*metadataCoalesceCall) { + body := []byte(http.StatusText(http.StatusBadGateway) + "\n") + result := metadataCoalesceResult{ + headers: http.Header{ + "Content-Type": []string{"text/plain; charset=utf-8"}, + "Content-Length": []string{strconv.Itoa(len(body))}, + }, + status: http.StatusBadGateway, + body: body, + } + for _, call := range calls { + call.dispatch(result) + } +} + +func (group *metadataCoalesceGroup) anyActive() bool { + for _, calls := range group.items { + for _, call := range calls { + if call.isActive() { + return true + } + } + } + return false +} + +func (group *metadataCoalesceGroup) installCancel(cancel context.CancelFunc) { + group.lifecycleMu.Lock() + group.cancel = cancel + if !group.anyActive() { + cancel() + } + group.lifecycleMu.Unlock() +} + +func (group *metadataCoalesceGroup) clearCancel() { + group.lifecycleMu.Lock() + group.cancel = nil + group.lifecycleMu.Unlock() +} + +func (group *metadataCoalesceGroup) cancelIfIdle() { + group.lifecycleMu.Lock() + if group.cancel != nil && !group.anyActive() { + group.cancel() + } + group.lifecycleMu.Unlock() +} + +type metadataCoalesceCapture struct { + header http.Header + body bytes.Buffer + status int + limit int64 + overflow bool +} + +type metadataCoalesceTee struct { + destination http.ResponseWriter + body bytes.Buffer + status int + limit int64 + overflow bool + failed bool +} + +func newMetadataCoalesceTee(destination http.ResponseWriter, limit int64) *metadataCoalesceTee { + return &metadataCoalesceTee{destination: destination, limit: limit} +} + +func (capture *metadataCoalesceTee) Header() http.Header { return capture.destination.Header() } + +func (capture *metadataCoalesceTee) WriteHeader(status int) { + if capture.status == 0 { + capture.status = status + } + capture.destination.WriteHeader(status) +} + +func (capture *metadataCoalesceTee) Write(value []byte) (int, error) { + if capture.status == 0 { + capture.WriteHeader(http.StatusOK) + } + count, err := capture.destination.Write(value) + if err != nil || count != len(value) { + capture.failed = true + } + if !capture.overflow && int64(capture.body.Len())+int64(count) <= capture.limit { + _, _ = capture.body.Write(value[:count]) + } else { + capture.overflow = true + } + return count, err +} + +func (capture *metadataCoalesceTee) statusCode() int { + if capture.status == 0 { + return http.StatusOK + } + return capture.status +} + +func (capture *metadataCoalesceTee) reusable() bool { + return !capture.failed && !capture.overflow && capture.Header().Get("Trailer") == "" +} + +func newMetadataCoalesceCapture(limit int64) *metadataCoalesceCapture { + return &metadataCoalesceCapture{header: make(http.Header), limit: limit} +} + +func (capture *metadataCoalesceCapture) Header() http.Header { return capture.header } + +func (capture *metadataCoalesceCapture) WriteHeader(status int) { + if capture.status == 0 { + capture.status = status + } +} + +func (capture *metadataCoalesceCapture) Write(value []byte) (int, error) { + if capture.status == 0 { + capture.status = http.StatusOK + } + if capture.overflow || int64(capture.body.Len())+int64(len(value)) > capture.limit { + capture.overflow = true + return len(value), nil + } + return capture.body.Write(value) +} + +func (capture *metadataCoalesceCapture) statusCode() int { + if capture.status == 0 { + return http.StatusOK + } + return capture.status +} diff --git a/internal/gateway/metadata_coalesce_test.go b/internal/gateway/metadata_coalesce_test.go new file mode 100644 index 0000000..23ff780 --- /dev/null +++ b/internal/gateway/metadata_coalesce_test.go @@ -0,0 +1,1037 @@ +package gateway + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "encoding/xml" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/InfinityPacer/plex-gateway/internal/metrics" + "github.com/InfinityPacer/plex-gateway/internal/partcache" + "github.com/InfinityPacer/plex-gateway/internal/trace" +) + +type metadataCoalesceObservedRequest struct { + Method string + Path string + RawQuery string + RemoteAddr string + Header http.Header + Batch bool +} + +type metadataCoalesceTestUpstream struct { + mu sync.Mutex + requests []metadataCoalesceObservedRequest + serve func(http.ResponseWriter, *http.Request) +} + +func (upstream *metadataCoalesceTestUpstream) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + upstream.mu.Lock() + upstream.requests = append(upstream.requests, metadataCoalesceObservedRequest{ + Method: request.Method, Path: request.URL.Path, RawQuery: request.URL.RawQuery, + RemoteAddr: request.RemoteAddr, Header: request.Header.Clone(), + Batch: isMetadataCoalesceUpstreamRequest(request), + }) + upstream.mu.Unlock() + upstream.serve(writer, request) +} + +func (upstream *metadataCoalesceTestUpstream) snapshot() []metadataCoalesceObservedRequest { + upstream.mu.Lock() + defer upstream.mu.Unlock() + result := make([]metadataCoalesceObservedRequest, len(upstream.requests)) + copy(result, upstream.requests) + return result +} + +func newMetadataCoalesceTestHandler( + upstream *metadataCoalesceTestUpstream, + window, timeout time.Duration, +) (http.Handler, *metrics.Metrics) { + registry := metrics.New() + return newMetadataCoalescer(MetadataCoalesceOptions{ + Enabled: true, Window: window, MaxItems: 8, Timeout: timeout, + }, upstream, registry, slog.New(slog.NewTextHandler(io.Discard, nil))), registry +} + +func newMetadataCoalesceTestRequest(method, ratingKey string) *http.Request { + request := httptest.NewRequest(method, "http://gateway.test/library/metadata/"+ratingKey, nil) + request.Header.Set("X-Plex-Token", "token-a") + request.Header.Set("User-Agent", "metadata-client/1") + request.RemoteAddr = "198.51.100.10:4000" + return request +} + +func runMetadataCoalesceCalls(t *testing.T, handler http.Handler, requests ...*http.Request) []*httptest.ResponseRecorder { + t.Helper() + start := make(chan struct{}) + type result struct { + index int + response *httptest.ResponseRecorder + } + results := make(chan result, len(requests)) + for index, request := range requests { + go func(index int, request *http.Request) { + <-start + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + results <- result{index: index, response: response} + }(index, request) + } + close(start) + responses := make([]*httptest.ResponseRecorder, len(requests)) + deadline := time.After(2 * time.Second) + for range requests { + select { + case result := <-results: + responses[result.index] = result.response + case <-deadline: + t.Fatal("metadata coalescer request did not complete") + } + } + return responses +} + +func runMetadataCoalesceOrderedCalls(t *testing.T, coalescer *metadataCoalescer, handler http.Handler, requests ...*http.Request) []*httptest.ResponseRecorder { + t.Helper() + type result struct { + index int + response *httptest.ResponseRecorder + } + results := make(chan result, len(requests)) + for index, request := range requests { + go func(index int, request *http.Request) { + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + results <- result{index: index, response: response} + }(index, request) + waitForMetadataCoalesceWaiters(t, coalescer, index+1) + } + responses := make([]*httptest.ResponseRecorder, len(requests)) + deadline := time.After(2 * time.Second) + for range requests { + select { + case result := <-results: + responses[result.index] = result.response + case <-deadline: + t.Fatal("ordered metadata coalescer request did not complete") + } + } + return responses +} + +func metadataCoalesceTestKeys(request *http.Request) []string { + identifier := strings.TrimPrefix(request.URL.Path, "/library/metadata/") + return strings.Split(identifier, ",") +} + +func metadataCoalesceJSONBody(keys []string) []byte { + items := make([]string, 0, len(keys)) + for _, key := range keys { + items = append(items, `{"ratingKey":"`+key+`","title":"title-`+key+`","unknownItemField":"keep"}`) + } + return []byte(`{"MediaContainer":{"size":` + strconv.Itoa(len(keys)) + `,"unknownContainerField":{"keep":true},"Metadata":[` + strings.Join(items, ",") + `]}}`) +} + +func metadataCoalesceXMLBody(keys []string) []byte { + items := make([]string, 0, len(keys)) + for _, key := range keys { + items = append(items, `