From 90dabfa672fdf5baf65de0e07a3d96fdbe29ab85 Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Sat, 29 Aug 2026 04:12:56 +0800 Subject: [PATCH] feat: enrich Plex metadata with MediaInfo --- Dockerfile | 2 +- README.en.md | 7 +- README.md | 7 +- cmd/plex-gateway/main.go | 63 +- cmd/plex-gateway/main_test.go | 10 +- docker-compose.example.yml | 4 + docs/architecture.md | 15 +- internal/config/config.go | 32 +- internal/config/config_test.go | 15 +- internal/database/sqlite.go | 159 +++ internal/database/sqlite_test.go | 61 + internal/gateway/gateway.go | 42 +- internal/gateway/metadata_enrichment.go | 354 +++++ .../metadata_enrichment_integration_test.go | 196 +++ internal/gateway/metadata_enrichment_test.go | 418 ++++++ internal/mediainfo/benchmark_test.go | 7 +- internal/mediainfo/model.go | 19 +- internal/mediainfo/model_test.go | 19 + internal/mediainfo/negative_cache_test.go | 317 +++++ internal/mediainfo/service.go | 57 +- internal/mediainfo/sqlite.go | 105 +- internal/mediainfo/sqlite_test.go | 102 +- internal/metrics/metrics.go | 31 + internal/metrics/metrics_test.go | 16 + internal/plexmeta/projection.go | 1236 +++++++++++++++++ internal/plexmeta/projection_test.go | 349 +++++ 26 files changed, 3415 insertions(+), 228 deletions(-) create mode 100644 internal/database/sqlite.go create mode 100644 internal/database/sqlite_test.go create mode 100644 internal/gateway/metadata_enrichment.go create mode 100644 internal/gateway/metadata_enrichment_integration_test.go create mode 100644 internal/gateway/metadata_enrichment_test.go create mode 100644 internal/mediainfo/negative_cache_test.go create mode 100644 internal/plexmeta/projection.go create mode 100644 internal/plexmeta/projection_test.go diff --git a/Dockerfile b/Dockerfile index f18b548..e6d1348 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,6 @@ WORKDIR /app_data USER gateway EXPOSE 32400 ENV LISTEN_ADDR=:32400 -ENV MEDIAINFO_DB_PATH=/app_data/mediainfo.db +ENV DATABASE_PATH=/app_data/plex-gateway.db HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 CMD ["/usr/local/bin/plex-gateway", "healthcheck"] ENTRYPOINT ["/usr/local/bin/plex-gateway"] diff --git a/README.en.md b/README.en.md index a730a05..ad4a5f9 100644 --- a/README.en.md +++ b/README.en.md @@ -113,9 +113,12 @@ Important environment variables: | `METADATA_GUARD_BATCH_ENABLED` | `false` | Limit comma-separated batch metadata reads before they enter Plex. | | `METADATA_GUARD_BATCH_CONCURRENCY` | `3` | Shared batch metadata concurrency limit across all clients. | | `METADATA_GUARD_QUEUE_TIMEOUT` | `10s` | Maximum admission wait before returning `429`. | -| `MEDIAINFO_ENABLED` | `true` | Enable the optional MediaInfo cache and bounded probe subsystem; initialization failures do not affect transparent proxying. | -| `MEDIAINFO_DB_PATH` | `./data/mediainfo.db` | MediaInfo SQLite cache path; the container image defaults to `/app_data/mediainfo.db`. | +| `MEDIAINFO_ENABLED` | `true` | Enable the MediaInfo cache, bounded probes, and single-item metadata enrichment; initialization failures do not affect transparent proxying. | +| `DATABASE_PATH` | `./data/plex-gateway.db` | Gateway SQLite database path; the container image defaults to `/app_data/plex-gateway.db`. | | `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.4.4` | Fallback User-Agent for background probes without an active client context. | +| `MEDIAINFO_COLD_WAIT` | `5s` | Cold-cache wait ceiling for one metadata item; timeout returns the original Plex response while probing continues. | +| `MEDIAINFO_RESPONSE_MAX_BYTES` | `8388608` | Maximum size of one Plex metadata response buffered for enrichment. | +| `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `4` | Maximum single-item metadata responses buffered and waiting for MediaInfo concurrently. | Plex tokens are not stored in configuration. Ordinary Plex requests remain transparent. For cloud playback, all client request headers are forwarded to diff --git a/README.md b/README.md index 5d84e25..ac33f42 100644 --- a/README.md +++ b/README.md @@ -101,9 +101,12 @@ go run ./cmd/plex-gateway | `METADATA_GUARD_BATCH_ENABLED` | `false` | 限制逗号分隔的批量 metadata 读取进入 Plex 的并发量。 | | `METADATA_GUARD_BATCH_CONCURRENCY` | `3` | 所有客户端共享的批量 metadata 并发上限。 | | `METADATA_GUARD_QUEUE_TIMEOUT` | `10s` | 等待准入的最长时间，超时返回 `429`。 | -| `MEDIAINFO_ENABLED` | `true` | 启用可选 MediaInfo 缓存与受限探测子系统；初始化失败不影响透明代理。 | -| `MEDIAINFO_DB_PATH` | `./data/mediainfo.db` | MediaInfo SQLite 持久缓存路径；容器镜像默认使用 `/app_data/mediainfo.db`。 | +| `MEDIAINFO_ENABLED` | `true` | 启用 MediaInfo 缓存、受限探测和单项 metadata 增强；初始化失败不影响透明代理。 | +| `DATABASE_PATH` | `./data/plex-gateway.db` | Gateway SQLite 持久数据库路径；容器镜像默认使用 `/app_data/plex-gateway.db`。 | | `MEDIAINFO_USER_AGENT` | `Infuse-Library/8.4.4` | 没有活动客户端上下文时，后台探测使用的 fallback User-Agent。 | +| `MEDIAINFO_COLD_WAIT` | `5s` | 单项 metadata 冷缓存等待上限；超时返回原 Plex 响应，探测继续。 | +| `MEDIAINFO_RESPONSE_MAX_BYTES` | `8388608` | 可以缓冲并尝试增强的单个 Plex metadata 响应上限。 | +| `MEDIAINFO_ENRICHMENT_CONCURRENCY` | `4` | 同时缓冲和等待 MediaInfo 的单项 metadata 响应上限。 | Plex Token 不会保存到配置中。普通 Plex 请求保持透明转发。对于云端播放，客户端的 所有请求 header 都会转发到配置的 MediaVault origin，使其能够为相同客户端上下文 diff --git a/cmd/plex-gateway/main.go b/cmd/plex-gateway/main.go index e1b180b..d57e6ad 100644 --- a/cmd/plex-gateway/main.go +++ b/cmd/plex-gateway/main.go @@ -15,6 +15,7 @@ import ( "time" "github.com/InfinityPacer/plex-gateway/internal/config" + "github.com/InfinityPacer/plex-gateway/internal/database" "github.com/InfinityPacer/plex-gateway/internal/gateway" "github.com/InfinityPacer/plex-gateway/internal/mediainfo" "github.com/InfinityPacer/plex-gateway/internal/metrics" @@ -66,10 +67,13 @@ func run() error { return fmt.Errorf("create MediaVault resolver: %w", err) } } - mediaInfoService, mediaInfoStore, mediaInfoReason := initializeMediaInfo( - context.Background(), cfg.MediaInfo, cfg.PlexURL, cfg.PartProbeTimeout, strmResolver, registry, logger, + databaseStore, databaseReason := initializeDatabase( + context.Background(), cfg.DatabasePath, cfg.MediaInfo.Enabled && strmResolver != nil, logger, ) - defer closeMediaInfo(mediaInfoService, mediaInfoStore, cfg.ShutdownTimeout, logger) + mediaInfoService, mediaInfoReason := initializeMediaInfo( + context.Background(), databaseStore, databaseReason, cfg.MediaInfo, cfg.PlexURL, cfg.PartProbeTimeout, strmResolver, registry, logger, + ) + defer closeRuntime(mediaInfoService, databaseStore, cfg.ShutdownTimeout, logger) mediaInfoStatus := func() mediainfo.Status { return mediaInfoService.Status() } @@ -92,8 +96,12 @@ func run() error { BatchConcurrency: cfg.MetadataGuard.BatchConcurrency, QueueTimeout: cfg.MetadataGuard.QueueTimeout, }, - MediaInfoEnabled: cfg.MediaInfo.Enabled, - MediaInfoStatus: mediaInfoStatus, + MediaInfoEnabled: cfg.MediaInfo.Enabled, + MediaInfoStatus: mediaInfoStatus, + MediaInfoService: mediaInfoService, + MediaInfoColdWait: cfg.MediaInfo.ColdWait, + MediaInfoResponseMaxBytes: cfg.MediaInfo.ResponseMaxBytes, + MediaInfoEnrichmentConcurrency: cfg.MediaInfo.EnrichmentWaiters, }) server := &http.Server{ Addr: cfg.ListenAddr, @@ -143,23 +151,28 @@ func run() error { func initializeMediaInfo( ctx context.Context, + databaseStore *database.SQLite, + databaseReason string, cfg config.MediaInfoConfig, plexURL *url.URL, identityTimeout time.Duration, strmResolver resolver.ControlResolver, registry *metrics.Metrics, logger *slog.Logger, -) (*mediainfo.Service, *mediainfo.SQLiteStore, string) { +) (*mediainfo.Service, string) { if !cfg.Enabled { - return nil, nil, "disabled" + return nil, "disabled" } if strmResolver == nil { - return nil, nil, "cloud_unavailable" + return nil, "cloud_unavailable" + } + if databaseStore == nil { + return nil, databaseReason } plexServerID, err := gateway.ReadPlexServerIdentity(ctx, plexURL, identityTimeout) if err != nil { logger.Warn("mediainfo_unavailable", "reason", "plex_identity") - return nil, nil, "plex_identity_unavailable" + return nil, "plex_identity_unavailable" } prober, err := mediainfo.NewFFProber(mediainfo.FFProbeOptions{ Binary: cfg.FFProbePath, Timeout: cfg.ProbeTimeout, ProbeSize: cfg.ProbeSize, @@ -167,17 +180,17 @@ func initializeMediaInfo( }) if err != nil { logger.Warn("mediainfo_unavailable", "reason", "ffprobe") - return nil, nil, "ffprobe_unavailable" + return nil, "ffprobe_unavailable" } provider, err := mediainfo.NewMediaVaultFFProbeProvider(strmResolver, prober) if err != nil { logger.Warn("mediainfo_unavailable", "reason", "provider") - return nil, nil, "provider_unavailable" + return nil, "provider_unavailable" } - store, err := mediainfo.OpenSQLite(ctx, cfg.DatabasePath) + store, err := mediainfo.NewSQLiteStore(ctx, databaseStore) if err != nil { - logger.Warn("mediainfo_unavailable", "reason", "storage") - return nil, nil, "storage_unavailable" + logger.Warn("mediainfo_unavailable", "reason", "migration") + return nil, "migration_failed" } now := time.Now().UTC() if _, err := store.DeleteUnretained(ctx, now); err != nil { @@ -186,8 +199,7 @@ func initializeMediaInfo( records, err := store.LoadCompatibleRetained(ctx, now, cfg.L1MaxEntries, provider.Descriptor()) if err != nil { logger.Warn("mediainfo_unavailable", "reason", "restore") - _ = store.Close() - return nil, nil, "restore_failed" + return nil, "restore_failed" } cache := mediainfo.NewCacheWithLimit(records, now, cfg.L1MaxEntries) service, err := mediainfo.NewService(mediainfo.ServiceOptions{ @@ -200,13 +212,24 @@ func initializeMediaInfo( }) if err != nil { logger.Warn("mediainfo_unavailable", "reason", "worker") - _ = store.Close() - return nil, nil, "worker_unavailable" + return nil, "worker_unavailable" + } + return service, "ready" +} + +func initializeDatabase(ctx context.Context, path string, required bool, logger *slog.Logger) (*database.SQLite, string) { + if !required { + return nil, "unused" + } + store, err := database.OpenSQLite(ctx, path) + if err != nil { + logger.Warn("database_unavailable", "reason", "storage") + return nil, "storage_unavailable" } - return service, store, "ready" + return store, "ready" } -func closeMediaInfo(service *mediainfo.Service, store *mediainfo.SQLiteStore, timeout time.Duration, logger *slog.Logger) { +func closeRuntime(service *mediainfo.Service, store *database.SQLite, timeout time.Duration, logger *slog.Logger) { serviceClosed := true if service != nil { ctx, cancel := context.WithTimeout(context.Background(), timeout) diff --git a/cmd/plex-gateway/main_test.go b/cmd/plex-gateway/main_test.go index dabc455..2b5ba3a 100644 --- a/cmd/plex-gateway/main_test.go +++ b/cmd/plex-gateway/main_test.go @@ -59,9 +59,9 @@ func TestInitializeMediaInfoDoesNotProbeWithoutTask(t *testing.T) { } control := &countingControlResolver{} logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - service, store, reason := initializeMediaInfo(t.Context(), config.MediaInfoConfig{ + databaseStore, databaseReason := initializeDatabase(t.Context(), filepath.Join(directory, "plex-gateway.db"), true, logger) + service, reason := initializeMediaInfo(t.Context(), databaseStore, databaseReason, config.MediaInfoConfig{ Enabled: true, - DatabasePath: filepath.Join(directory, "mediainfo.db"), FFProbePath: ffprobePath, ProbeTimeout: time.Second, ProbeSize: 1 << 20, @@ -76,8 +76,8 @@ func TestInitializeMediaInfoDoesNotProbeWithoutTask(t *testing.T) { NegativeTTL: time.Minute, UserAgent: "plex-gateway-test", }, plexURL, time.Second, control, metrics.New(), logger) - if service == nil || store == nil || reason != "ready" { - t.Fatalf("initializeMediaInfo() service=%v store=%v reason=%q", service != nil, store != nil, reason) + if service == nil || databaseStore == nil || reason != "ready" { + t.Fatalf("initializeMediaInfo() service=%v store=%v reason=%q", service != nil, databaseStore != nil, reason) } if status := service.Status(); !status.Available || status.ActiveProbes != 0 || status.InteractiveQueued != 0 || status.BackgroundQueued != 0 { @@ -94,5 +94,5 @@ func TestInitializeMediaInfoDoesNotProbeWithoutTask(t *testing.T) { if _, err := os.Stat(ffprobePath + ".called"); !errors.Is(err, os.ErrNotExist) { t.Fatalf("ffprobe process unexpectedly started: %v", err) } - closeMediaInfo(service, store, time.Second, logger) + closeRuntime(service, databaseStore, time.Second, logger) } diff --git a/docker-compose.example.yml b/docker-compose.example.yml index ea90bbd..6df49dd 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -12,6 +12,7 @@ services: PATH_MAPPINGS: >- [{"plex_prefix":"/media/cloud","local_prefix":"/strm"},{"plex_prefix":"/media/archive","local_prefix":"/archive-strm"}] LISTEN_ADDR: ":32400" + DATABASE_PATH: "/app_data/plex-gateway.db" LOG_LEVEL: "info" TRACE_ENABLED: "false" PART_CACHE_TTL: "24h" @@ -27,6 +28,9 @@ services: MEDIAINFO_ENABLED: "true" # Background probes use this only when no active client request supplies a User-Agent. MEDIAINFO_USER_AGENT: "Infuse-Library/8.4.4" + MEDIAINFO_COLD_WAIT: "5s" + MEDIAINFO_RESPONSE_MAX_BYTES: "8388608" + MEDIAINFO_ENRICHMENT_CONCURRENCY: "4" READ_HEADER_TIMEOUT: "15s" IDLE_TIMEOUT: "90s" SHUTDOWN_TIMEOUT: "15s" diff --git a/docs/architecture.md b/docs/architecture.md index 2c763d4..7c10033 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -18,11 +18,12 @@ gateway; the isolated analysis worker may read a bounded amount for probing. consume interactive metadata slots. Neither pool caches or rewrites Plex responses. Library listings, metadata mutations, playback paths, timeline, and watch-state traffic do not enter these pools. -- Successful XML or JSON Plex responses are currently observed without changing - their bytes. Any `Media.Part` values found in those responses populate an - in-memory cache keyed by `Part.id`. Phase D may enrich a successful single-item - metadata response when explicitly enabled; unsupported or failed transforms - preserve the original Plex response. +- Successful XML or JSON Plex responses are observed to populate the in-memory + `Part.id` cache without changing their bytes. When MediaInfo is available, an + authenticated single-item metadata response may also fill missing whitelisted + technical fields from the exact STRM fingerprint. The transform is bounded, + never creates Stream identities, and preserves the original Plex response on + timeout, ambiguity, unsupported encoding, or any other failure. - `GET` and `HEAD` requests below `/library/parts/{partID}/...` are eligible for interception only when the cached `Part.file` has a configured cloud extension and maps to a readable local STRM file. @@ -50,9 +51,11 @@ gateway; the isolated analysis worker may read a bounded amount for probing. | Package | Responsibility | | --- | --- | | `cmd/plex-gateway` | Configuration, dependency assembly, and process lifecycle. | +| `internal/database` | Gateway SQLite connection lifecycle and independent per-module migrations. | | `internal/gateway` | Plex routing, transparent proxying, protocol probes, response capture, 302/fallback output, and request metrics. | +| `internal/mediainfo` | Bounded analysis scheduling, L1/persistent cache policy, provider contracts, and technical-media records. | | `internal/playback` | Shared STRM preparation, normalized playback attempts, Plex authorization/MediaVault resolution sequencing, and decision-grant state. | -| `internal/plexmeta` | Side-effect-free XML and JSON Part/decision readers. | +| `internal/plexmeta` | XML and JSON Part/decision readers plus whitelisted metadata projection. | | `internal/partcache` | Derived, expiring `Part.id` index; never an authority for access. | | `internal/pathmap` | Lexical Plex-to-container path mapping. | | `internal/resolver` | STRM target validation and the MediaVault `/redirect` protocol. | diff --git a/internal/config/config.go b/internal/config/config.go index 45a166c..62d16fd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -26,7 +26,7 @@ const ( defaultMetadataPerClient = 4 defaultMetadataBatch = 3 defaultMetadataQueueWait = 10 * time.Second - defaultMediaInfoDatabase = "./data/mediainfo.db" + defaultDatabasePath = "./data/plex-gateway.db" defaultMediaInfoProbe = 20 * time.Second defaultMediaInfoSize = 8 << 20 defaultMediaInfoAnalyze = 5 * time.Second @@ -38,6 +38,9 @@ const ( defaultMediaInfoL1Entries = 10_000 defaultMediaInfoNegative = 15 * time.Minute defaultMediaInfoAgent = "Infuse-Library/8.4.4" + defaultMediaInfoColdWait = 5 * time.Second + defaultMediaInfoBodyLimit = 8 << 20 + defaultMediaInfoWaiters = 4 ) // Config contains process-level settings for transparent Plex proxying and @@ -45,6 +48,7 @@ const ( // never loaded into this structure. type Config struct { ListenAddr string + DatabasePath string PlexURL *url.URL MediaVaultURL *url.URL PathMappings []pathmap.Mapping @@ -66,7 +70,6 @@ type Config struct { // but capability initialization remains fail-open for transparent Plex proxying. type MediaInfoConfig struct { Enabled bool - DatabasePath string FFProbePath string ProbeTimeout time.Duration ProbeSize int64 @@ -80,6 +83,9 @@ type MediaInfoConfig struct { L1MaxEntries int NegativeTTL time.Duration UserAgent string + ColdWait time.Duration + ResponseMaxBytes int64 + EnrichmentWaiters int } // MetadataGuardConfig bounds detailed metadata fan-out before requests enter @@ -186,9 +192,9 @@ func Load() (Config, error) { if err != nil { return Config{}, err } - mediaInfoDatabase := strings.TrimSpace(os.Getenv("MEDIAINFO_DB_PATH")) - if mediaInfoDatabase == "" { - mediaInfoDatabase = defaultMediaInfoDatabase + databasePath := strings.TrimSpace(os.Getenv("DATABASE_PATH")) + if databasePath == "" { + databasePath = defaultDatabasePath } mediaInfoFFProbe := strings.TrimSpace(os.Getenv("MEDIAINFO_FFPROBE_PATH")) if mediaInfoFFProbe == "" { @@ -248,6 +254,18 @@ func Load() (Config, error) { if strings.ContainsAny(mediaInfoUserAgent, "\r\n") { return Config{}, errors.New("MEDIAINFO_USER_AGENT must not contain line breaks") } + mediaInfoColdWait, err := envDuration("MEDIAINFO_COLD_WAIT", defaultMediaInfoColdWait) + if err != nil { + return Config{}, err + } + mediaInfoBodyLimit, err := envPositiveInt64("MEDIAINFO_RESPONSE_MAX_BYTES", defaultMediaInfoBodyLimit) + if err != nil { + return Config{}, err + } + mediaInfoWaiters, err := envPositiveInt("MEDIAINFO_ENRICHMENT_CONCURRENCY", defaultMediaInfoWaiters) + if err != nil { + return Config{}, err + } readHeaderTimeout, err := envDuration("READ_HEADER_TIMEOUT", defaultReadHeaderTimeout) if err != nil { @@ -276,6 +294,7 @@ func Load() (Config, error) { return Config{ ListenAddr: listenAddr, + DatabasePath: databasePath, PlexURL: plexURL, MediaVaultURL: mediaVaultURL, PathMappings: pathMappings, @@ -293,13 +312,14 @@ func Load() (Config, error) { QueueTimeout: metadataQueueWait, }, MediaInfo: MediaInfoConfig{ - Enabled: mediaInfoEnabled, DatabasePath: mediaInfoDatabase, FFProbePath: mediaInfoFFProbe, + Enabled: mediaInfoEnabled, FFProbePath: mediaInfoFFProbe, ProbeTimeout: mediaInfoProbeTimeout, ProbeSize: mediaInfoProbeSize, AnalyzeDuration: mediaInfoAnalyze, OutputMaxBytes: mediaInfoOutput, Concurrency: mediaInfoWorkers, InteractiveQueueSize: mediaInfoInteractiveQueue, BackgroundQueueSize: mediaInfoBackgroundQueue, RecordTTL: mediaInfoTTL, RecordRetention: mediaInfoRetention, L1MaxEntries: mediaInfoL1Entries, NegativeTTL: mediaInfoNegativeTTL, UserAgent: mediaInfoUserAgent, + ColdWait: mediaInfoColdWait, ResponseMaxBytes: mediaInfoBodyLimit, EnrichmentWaiters: mediaInfoWaiters, }, LogLevel: logLevel, TraceEnabled: traceEnabled, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5a0931c..3b617a3 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -40,7 +40,10 @@ func TestLoadDefaults(t *testing.T) { if got.MetadataGuard.Enabled || got.MetadataGuard.GlobalConcurrency != 8 || got.MetadataGuard.PerClientConcurrency != 4 || got.MetadataGuard.BatchEnabled || got.MetadataGuard.BatchConcurrency != 3 || got.MetadataGuard.QueueTimeout != 10*time.Second { t.Fatalf("unexpected metadata guard defaults: %#v", got.MetadataGuard) } - if !got.MediaInfo.Enabled || got.MediaInfo.ProbeTimeout != 20*time.Second || got.MediaInfo.Concurrency != 1 || got.MediaInfo.DatabasePath != "./data/mediainfo.db" || got.MediaInfo.UserAgent != "Infuse-Library/8.4.4" { + if got.DatabasePath != "./data/plex-gateway.db" { + t.Fatalf("DatabasePath = %q", got.DatabasePath) + } + if !got.MediaInfo.Enabled || got.MediaInfo.ProbeTimeout != 20*time.Second || got.MediaInfo.Concurrency != 1 || got.MediaInfo.UserAgent != "Infuse-Library/8.4.4" || got.MediaInfo.ColdWait != 5*time.Second || got.MediaInfo.ResponseMaxBytes != 8<<20 || got.MediaInfo.EnrichmentWaiters != 4 { t.Fatalf("unexpected MediaInfo defaults: %#v", got.MediaInfo) } if got.MediaVaultURL != nil || len(got.PathMappings) != 0 { @@ -73,7 +76,7 @@ func TestLoadCloudConfiguration(t *testing.T) { t.Setenv("METADATA_GUARD_BATCH_CONCURRENCY", "2") t.Setenv("METADATA_GUARD_QUEUE_TIMEOUT", "5s") t.Setenv("MEDIAINFO_ENABLED", "true") - t.Setenv("MEDIAINFO_DB_PATH", "/app_data/test.db") + t.Setenv("DATABASE_PATH", "/app_data/test.db") t.Setenv("MEDIAINFO_FFPROBE_PATH", "/usr/bin/ffprobe") t.Setenv("MEDIAINFO_PROBE_TIMEOUT", "12s") t.Setenv("MEDIAINFO_PROBE_SIZE", "4194304") @@ -87,6 +90,9 @@ func TestLoadCloudConfiguration(t *testing.T) { t.Setenv("MEDIAINFO_L1_MAX_ENTRIES", "5000") t.Setenv("MEDIAINFO_NEGATIVE_TTL", "10m") t.Setenv("MEDIAINFO_USER_AGENT", "plex-gateway-test") + t.Setenv("MEDIAINFO_COLD_WAIT", "4s") + t.Setenv("MEDIAINFO_RESPONSE_MAX_BYTES", "2097152") + t.Setenv("MEDIAINFO_ENRICHMENT_CONCURRENCY", "3") got, err := Load() if err != nil { @@ -104,7 +110,10 @@ func TestLoadCloudConfiguration(t *testing.T) { if !got.MetadataGuard.Enabled || got.MetadataGuard.GlobalConcurrency != 6 || got.MetadataGuard.PerClientConcurrency != 3 || !got.MetadataGuard.BatchEnabled || got.MetadataGuard.BatchConcurrency != 2 || got.MetadataGuard.QueueTimeout != 5*time.Second { t.Fatalf("metadata guard = %#v", got.MetadataGuard) } - if !got.MediaInfo.Enabled || got.MediaInfo.DatabasePath != "/app_data/test.db" || got.MediaInfo.FFProbePath != "/usr/bin/ffprobe" || got.MediaInfo.ProbeTimeout != 12*time.Second || got.MediaInfo.ProbeSize != 4194304 || got.MediaInfo.AnalyzeDuration != 3*time.Second || got.MediaInfo.OutputMaxBytes != 1048576 || got.MediaInfo.Concurrency != 2 || got.MediaInfo.InteractiveQueueSize != 64 || got.MediaInfo.BackgroundQueueSize != 128 || got.MediaInfo.RecordTTL != 168*time.Hour || got.MediaInfo.RecordRetention != 720*time.Hour || got.MediaInfo.L1MaxEntries != 5000 || got.MediaInfo.NegativeTTL != 10*time.Minute || got.MediaInfo.UserAgent != "plex-gateway-test" { + if got.DatabasePath != "/app_data/test.db" { + t.Fatalf("DatabasePath = %q", got.DatabasePath) + } + if !got.MediaInfo.Enabled || got.MediaInfo.FFProbePath != "/usr/bin/ffprobe" || got.MediaInfo.ProbeTimeout != 12*time.Second || got.MediaInfo.ProbeSize != 4194304 || got.MediaInfo.AnalyzeDuration != 3*time.Second || got.MediaInfo.OutputMaxBytes != 1048576 || got.MediaInfo.Concurrency != 2 || got.MediaInfo.InteractiveQueueSize != 64 || got.MediaInfo.BackgroundQueueSize != 128 || got.MediaInfo.RecordTTL != 168*time.Hour || got.MediaInfo.RecordRetention != 720*time.Hour || got.MediaInfo.L1MaxEntries != 5000 || got.MediaInfo.NegativeTTL != 10*time.Minute || got.MediaInfo.UserAgent != "plex-gateway-test" || got.MediaInfo.ColdWait != 4*time.Second || got.MediaInfo.ResponseMaxBytes != 2097152 || got.MediaInfo.EnrichmentWaiters != 3 { t.Fatalf("MediaInfo configuration = %#v", got.MediaInfo) } } diff --git a/internal/database/sqlite.go b/internal/database/sqlite.go new file mode 100644 index 0000000..74a8f6d --- /dev/null +++ b/internal/database/sqlite.go @@ -0,0 +1,159 @@ +// Package database owns the Gateway SQLite connection and cross-domain +// migration registry. Feature packages own their tables and submit migrations +// through this package. +package database + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +const migrationRegistrySchema = ` +CREATE TABLE IF NOT EXISTS gateway_schema_migrations ( + module TEXT NOT NULL, + version INTEGER NOT NULL, + applied_at_ms INTEGER NOT NULL, + PRIMARY KEY (module, version) +); +` + +// Migration is one ordered schema transition owned by a feature module. +// Versions must start at one and remain contiguous within that module. +type Migration struct { + Version int + SQL string +} + +// SQLite owns the process-wide Gateway database connection. +type SQLite struct { + db *sql.DB +} + +// OpenSQLite opens the Gateway database and establishes connection-wide +// durability and contention settings. The path must be a filesystem path, not +// a caller-controlled SQLite URI. +func OpenSQLite(ctx context.Context, path string) (*SQLite, error) { + path = strings.TrimSpace(path) + if path == "" || strings.IndexByte(path, 0) >= 0 || strings.HasPrefix(path, "file:") { + return nil, errors.New("Gateway database path is invalid") + } + cleanPath := filepath.Clean(path) + if err := os.MkdirAll(filepath.Dir(cleanPath), 0o750); err != nil { + return nil, fmt.Errorf("create Gateway database directory: %w", err) + } + file, err := os.OpenFile(cleanPath, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("create Gateway database: %w", err) + } + if err := file.Close(); err != nil { + return nil, fmt.Errorf("close Gateway database file: %w", err) + } + + db, err := sql.Open("sqlite", cleanPath) + if err != nil { + return nil, fmt.Errorf("open Gateway database: %w", err) + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + store := &SQLite{db: db} + if err := store.initialize(ctx); err != nil { + _ = db.Close() + return nil, err + } + return store, nil +} + +func (store *SQLite) initialize(ctx context.Context) error { + for _, statement := range []string{ + "PRAGMA journal_mode=WAL", + "PRAGMA synchronous=NORMAL", + "PRAGMA foreign_keys=ON", + "PRAGMA busy_timeout=5000", + } { + if _, err := store.db.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("configure Gateway database: %w", err) + } + } + if _, err := store.db.ExecContext(ctx, migrationRegistrySchema); err != nil { + return fmt.Errorf("initialize Gateway migration registry: %w", err) + } + return nil +} + +// ApplyMigrations advances one feature module without changing the schema +// version of any other module. A newer on-disk version is rejected so an older +// binary cannot silently operate on an unknown schema. +func (store *SQLite) ApplyMigrations(ctx context.Context, module string, migrations []Migration) error { + if store == nil || store.db == nil { + return errors.New("Gateway database is unavailable") + } + module = strings.TrimSpace(module) + if module == "" { + return errors.New("Gateway migration module is required") + } + for index, migration := range migrations { + if migration.Version != index+1 || strings.TrimSpace(migration.SQL) == "" { + return fmt.Errorf("Gateway migration %s version sequence is invalid", module) + } + } + + transaction, err := store.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin Gateway database migration: %w", err) + } + defer transaction.Rollback() + + var applied, current int + if err := transaction.QueryRowContext(ctx, ` + SELECT COUNT(*), COALESCE(MAX(version), 0) + FROM gateway_schema_migrations + WHERE module = ? + `, module).Scan(&applied, ¤t); err != nil { + return fmt.Errorf("read %s schema version: %w", module, err) + } + if applied != current { + return fmt.Errorf("%s schema migration history is not contiguous", module) + } + if current > len(migrations) { + return fmt.Errorf("%s schema version %d is newer than supported version %d", module, current, len(migrations)) + } + for _, migration := range migrations[current:] { + if _, err := transaction.ExecContext(ctx, migration.SQL); err != nil { + return fmt.Errorf("apply %s schema v%d: %w", module, migration.Version, err) + } + if _, err := transaction.ExecContext(ctx, ` + INSERT INTO gateway_schema_migrations (module, version, applied_at_ms) + VALUES (?, ?, ?) + `, module, migration.Version, time.Now().UTC().UnixMilli()); err != nil { + return fmt.Errorf("record %s schema v%d: %w", module, migration.Version, err) + } + } + if err := transaction.Commit(); err != nil { + return fmt.Errorf("commit Gateway database migration: %w", err) + } + return nil +} + +// SQLDB returns the shared handle for feature stores. Callers must not close it. +func (store *SQLite) SQLDB() *sql.DB { + if store == nil { + return nil + } + return store.db +} + +// Close flushes and closes the process-wide Gateway database. +func (store *SQLite) Close() error { + if store == nil || store.db == nil { + return nil + } + return store.db.Close() +} diff --git a/internal/database/sqlite_test.go b/internal/database/sqlite_test.go new file mode 100644 index 0000000..438d1ce --- /dev/null +++ b/internal/database/sqlite_test.go @@ -0,0 +1,61 @@ +package database + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestSQLiteTracksModuleMigrationsIndependently(t *testing.T) { + store, err := OpenSQLite(t.Context(), filepath.Join(t.TempDir(), "plex-gateway.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + if err := store.ApplyMigrations(t.Context(), "media", []Migration{{Version: 1, SQL: "CREATE TABLE media_records (id TEXT PRIMARY KEY)"}}); err != nil { + t.Fatal(err) + } + if err := store.ApplyMigrations(t.Context(), "jobs", []Migration{{Version: 1, SQL: "CREATE TABLE jobs (id TEXT PRIMARY KEY)"}}); err != nil { + t.Fatal(err) + } + rows, err := store.db.QueryContext(t.Context(), "SELECT module, version FROM gateway_schema_migrations ORDER BY module") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var got []string + for rows.Next() { + var module string + var version int + if err := rows.Scan(&module, &version); err != nil { + t.Fatal(err) + } + got = append(got, module) + } + if strings.Join(got, ",") != "jobs,media" { + t.Fatalf("migration modules = %v", got) + } +} + +func TestSQLiteRejectsNewerModuleWithoutAffectingOthers(t *testing.T) { + store, err := OpenSQLite(t.Context(), filepath.Join(t.TempDir(), "plex-gateway.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err := store.db.ExecContext(t.Context(), ` + INSERT INTO gateway_schema_migrations (module, version, applied_at_ms) + VALUES ('mediainfo', 1, 0), ('mediainfo', 2, 0), ('jobs', 1, 0) + `); err != nil { + t.Fatal(err) + } + + err = store.ApplyMigrations(t.Context(), "mediainfo", []Migration{{Version: 1, SQL: "SELECT 1"}}) + if err == nil || !strings.Contains(err.Error(), "newer than supported") { + t.Fatalf("ApplyMigrations() error = %v", err) + } + if err := store.ApplyMigrations(t.Context(), "jobs", []Migration{{Version: 1, SQL: "SELECT 1"}}); err != nil { + t.Fatalf("unrelated module migration failed: %v", err) + } +} diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 12cdc7a..3490f85 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -27,23 +27,28 @@ const defaultPartProbeTimeout = 15 * time.Second // Options describes the optional cloud playback plane around the transparent // Plex proxy. Nil cloud components keep the handler in proxy-only mode. type Options struct { - Upstream *url.URL - Logger *slog.Logger - Tracer *trace.Tracer - PartCache *partcache.Cache - PathMapper *pathmap.Mapper - Resolver resolver.ControlResolver - Metrics *metrics.Metrics - CloudExtensions []string - ObserveMaxBytes int64 - PartProbeTimeout time.Duration - MetadataGuard MetadataGuardOptions - MediaInfoEnabled bool - MediaInfoStatus func() mediainfo.Status + Upstream *url.URL + Logger *slog.Logger + Tracer *trace.Tracer + PartCache *partcache.Cache + PathMapper *pathmap.Mapper + Resolver resolver.ControlResolver + Metrics *metrics.Metrics + CloudExtensions []string + ObserveMaxBytes int64 + PartProbeTimeout time.Duration + MetadataGuard MetadataGuardOptions + MediaInfoEnabled bool + MediaInfoStatus func() mediainfo.Status + MediaInfoService *mediainfo.Service + MediaInfoColdWait time.Duration + MediaInfoResponseMaxBytes int64 + MediaInfoEnrichmentConcurrency int } // New builds the fail-open Plex proxy and optional Direct Play interceptor. -// Metadata observation never mutates the Plex response body. +// Observation only populates caches; optional MediaInfo response enrichment is +// bounded and returns the original Plex response whenever it cannot act safely. func New(options Options) http.Handler { logger := options.Logger if logger == nil { @@ -180,7 +185,14 @@ func New(options Options) http.Handler { } mux.Handle("GET /library/parts/{partID}/{rest...}", partPlayback) mux.Handle("HEAD /library/parts/{partID}/{rest...}", partPlayback) - mux.Handle("/", newMetadataGuard(options.MetadataGuard, plex, registry, logger)) + metadata := newMetadataGuard(options.MetadataGuard, plex, registry, logger) + metadata = newMetadataEnrichmentHandler(metadataEnrichmentOptions{ + Service: options.MediaInfoService, Mapper: options.PathMapper, Resolver: options.Resolver, + CloudExtensions: options.CloudExtensions, ColdWait: options.MediaInfoColdWait, + ResponseLimit: options.MediaInfoResponseMaxBytes, Concurrency: options.MediaInfoEnrichmentConcurrency, + Metrics: registry, + }, metadata) + mux.Handle("/", metadata) withActiveRequests := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { release := registry.BeginRequest() diff --git a/internal/gateway/metadata_enrichment.go b/internal/gateway/metadata_enrichment.go new file mode 100644 index 0000000..77de5b9 --- /dev/null +++ b/internal/gateway/metadata_enrichment.go @@ -0,0 +1,354 @@ +package gateway + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "net/http" + "path" + "strconv" + "strings" + "time" + + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" + "github.com/InfinityPacer/plex-gateway/internal/metrics" + "github.com/InfinityPacer/plex-gateway/internal/pathmap" + "github.com/InfinityPacer/plex-gateway/internal/plexmeta" + "github.com/InfinityPacer/plex-gateway/internal/resolver" +) + +const ( + defaultMediaInfoColdWait = 5 * time.Second + defaultMediaInfoResponseLimit = 8 << 20 + defaultMediaInfoEnrichmentSlots = 4 +) + +type mediaInfoEnsurer interface { + Ensure(context.Context, mediainfo.Request) (mediainfo.Record, error) +} + +// metadataEnrichmentOptions keeps analysis collaborators outside the playback +// handlers. A missing collaborator disables enrichment without changing Plex +// proxy behavior. +type metadataEnrichmentOptions struct { + Service mediaInfoEnsurer + Mapper *pathmap.Mapper + Resolver resolver.ControlResolver + CloudExtensions []string + ColdWait time.Duration + ResponseLimit int64 + Concurrency int + Metrics *metrics.Metrics +} + +type metadataEnrichmentHandler struct { + next http.Handler + service mediaInfoEnsurer + mapper *pathmap.Mapper + resolver resolver.ControlResolver + cloudExtensions map[string]struct{} + coldWait time.Duration + responseLimit int64 + waiters chan struct{} + metrics *metrics.Metrics +} + +func newMetadataEnrichmentHandler(options metadataEnrichmentOptions, next http.Handler) http.Handler { + if next == nil || options.Service == nil || options.Mapper == nil || options.Resolver == nil { + return next + } + coldWait := options.ColdWait + if coldWait <= 0 { + coldWait = defaultMediaInfoColdWait + } + responseLimit := options.ResponseLimit + if responseLimit <= 0 { + responseLimit = defaultMediaInfoResponseLimit + } + concurrency := options.Concurrency + if concurrency <= 0 { + concurrency = defaultMediaInfoEnrichmentSlots + } + registry := options.Metrics + if registry == nil { + registry = metrics.New() + } + return &metadataEnrichmentHandler{ + next: next, service: options.Service, mapper: options.Mapper, resolver: options.Resolver, + cloudExtensions: normalizedExtensionSet(options.CloudExtensions), coldWait: coldWait, + responseLimit: responseLimit, waiters: make(chan struct{}, concurrency), metrics: registry, + } +} + +func (handler *metadataEnrichmentHandler) ServeHTTP(w http.ResponseWriter, request *http.Request) { + ratingKey, eligible := mediaInfoMetadataRequest(request) + if !eligible { + handler.next.ServeHTTP(w, request) + return + } + select { + case handler.waiters <- struct{}{}: + handler.metrics.IncMediaInfoWaitActive() + defer func() { + <-handler.waiters + handler.metrics.DecMediaInfoWaitActive() + }() + default: + handler.metrics.IncMediaInfoWaitRejected() + handler.next.ServeHTTP(w, request) + return + } + + capture := newBoundedResponseCapture(w, handler.responseLimit) + handler.next.ServeHTTP(capture, request) + if capture.passthrough { + return + } + rawBody := capture.body.Bytes() + decoded, encoding, err := decodeMetadataBody(rawBody, capture.header.Get("Content-Encoding"), handler.responseLimit) + if err != nil { + handler.failOpen(capture, rawBody) + return + } + target, err := plexmeta.SelectEnrichmentTarget(decoded, capture.header.Get("Content-Type"), ratingKey) + if err != nil { + handler.failOpen(capture, rawBody) + return + } + if !target.NeedsEnrichment || !handler.cloudPart(target.Part) { + handler.replay(capture, rawBody) + return + } + localPath, err := handler.mapper.Resolve(target.Part.File) + if err != nil { + handler.failOpen(capture, rawBody) + return + } + strmTarget, err := handler.resolver.ReadTarget(localPath) + if err != nil { + handler.failOpen(capture, rawBody) + return + } + fingerprint, err := mediainfo.FingerprintSTRMTarget(strmTarget) + if err != nil { + handler.failOpen(capture, rawBody) + return + } + waitContext, cancel := context.WithTimeout(request.Context(), handler.coldWait) + record, err := handler.service.Ensure(waitContext, mediainfo.Request{ + Key: mediainfo.Key{PartID: target.Part.ID, STRMFingerprint: fingerprint}, + RatingKey: ratingKey, Target: strmTarget, Priority: mediainfo.PriorityInteractive, + ClientUserAgent: request.UserAgent(), + }) + cancel() + if err != nil { + handler.failOpen(capture, rawBody) + return + } + currentTarget, err := handler.resolver.ReadTarget(localPath) + if err != nil { + handler.failOpen(capture, rawBody) + return + } + currentFingerprint, err := mediainfo.FingerprintSTRMTarget(currentTarget) + if err != nil || currentFingerprint != fingerprint { + handler.failOpen(capture, rawBody) + return + } + enriched, changed, err := plexmeta.EnrichMetadata(decoded, capture.header.Get("Content-Type"), ratingKey, record.Media) + if err != nil || !changed { + handler.failOpen(capture, rawBody) + return + } + wireBody, err := encodeMetadataBody(enriched, encoding) + if err != nil { + handler.failOpen(capture, rawBody) + return + } + headers := capture.header.Clone() + for _, name := range []string{"ETag", "Last-Modified", "Content-MD5", "Digest"} { + headers.Del(name) + } + if !hasCacheDirective(headers.Get("Cache-Control"), "no-store") { + headers.Set("Cache-Control", "private, no-cache") + } + headers.Set("Content-Length", strconv.Itoa(len(wireBody))) + writeCapturedResponse(w, headers, capture.statusCode(), wireBody) + handler.metrics.IncMediaInfoEnriched() +} + +func (handler *metadataEnrichmentHandler) cloudPart(part plexmeta.Part) bool { + _, ok := handler.cloudExtensions[strings.ToLower(path.Ext(part.File))] + return ok +} + +func (handler *metadataEnrichmentHandler) failOpen(capture *boundedResponseCapture, rawBody []byte) { + handler.replay(capture, rawBody) + handler.metrics.IncMediaInfoFailOpen() +} + +func (handler *metadataEnrichmentHandler) replay(capture *boundedResponseCapture, rawBody []byte) { + writeCapturedResponse(capture.destination, capture.header, capture.statusCode(), rawBody) +} + +func mediaInfoMetadataRequest(request *http.Request) (string, bool) { + if request == nil || request.URL == nil || request.Method != http.MethodGet || request.Header.Get("Range") != "" { + return "", false + } + _, tokenPresent, tokenValid := requestIdentity(request, "X-Plex-Token") + if !tokenPresent || !tokenValid || !isMetadataItemPath(request.URL.Path) { + return "", false + } + return strings.TrimPrefix(request.URL.Path, "/library/metadata/"), true +} + +func normalizedExtensionSet(extensions []string) map[string]struct{} { + if len(extensions) == 0 { + extensions = []string{".strm"} + } + result := make(map[string]struct{}, len(extensions)) + for _, extension := range extensions { + extension = strings.ToLower(strings.TrimSpace(extension)) + if extension != "" { + result[extension] = struct{}{} + } + } + return result +} + +func hasCacheDirective(value, directive string) bool { + for _, component := range strings.Split(value, ",") { + name, _, _ := strings.Cut(strings.TrimSpace(component), "=") + if strings.EqualFold(name, directive) { + return true + } + } + return false +} + +func decodeMetadataBody(raw []byte, rawEncoding string, limit int64) ([]byte, string, error) { + encoding := strings.ToLower(strings.TrimSpace(rawEncoding)) + switch encoding { + case "", "identity": + return raw, encoding, nil + case "gzip": + reader, err := gzip.NewReader(bytes.NewReader(raw)) + if err != nil { + return nil, "", err + } + defer reader.Close() + decoded, err := io.ReadAll(io.LimitReader(reader, limit+1)) + if err != nil || int64(len(decoded)) > limit { + return nil, "", errors.New("metadata gzip body exceeds limit") + } + return decoded, encoding, nil + default: + return nil, "", errors.New("metadata content encoding is unsupported") + } +} + +func encodeMetadataBody(decoded []byte, encoding string) ([]byte, error) { + if encoding != "gzip" { + return decoded, nil + } + var output bytes.Buffer + writer := gzip.NewWriter(&output) + if _, err := writer.Write(decoded); err != nil { + return nil, err + } + if err := writer.Close(); err != nil { + return nil, err + } + return output.Bytes(), nil +} + +type boundedResponseCapture struct { + destination http.ResponseWriter + header http.Header + body bytes.Buffer + status int + limit int64 + passthrough bool +} + +func newBoundedResponseCapture(destination http.ResponseWriter, limit int64) *boundedResponseCapture { + return &boundedResponseCapture{destination: destination, header: make(http.Header), limit: limit} +} + +func (capture *boundedResponseCapture) Header() http.Header { return capture.header } + +func (capture *boundedResponseCapture) WriteHeader(status int) { + if capture.status != 0 { + return + } + capture.status = status + if status != http.StatusOK || capture.header.Get("Trailer") != "" || capture.header.Get("Content-Range") != "" { + capture.startPassthrough() + } +} + +func (capture *boundedResponseCapture) Write(value []byte) (int, error) { + if capture.status == 0 { + capture.WriteHeader(http.StatusOK) + } + if capture.passthrough { + return capture.destination.Write(value) + } + if int64(capture.body.Len())+int64(len(value)) > capture.limit { + if err := capture.startPassthrough(); err != nil { + return 0, err + } + return capture.destination.Write(value) + } + return capture.body.Write(value) +} + +// Flush is intentionally deferred while a bounded metadata body is captured. +// A passthrough response retains the downstream flusher behavior. +func (capture *boundedResponseCapture) Flush() { + if !capture.passthrough { + return + } + if flusher, ok := capture.destination.(http.Flusher); ok { + flusher.Flush() + } +} + +func (capture *boundedResponseCapture) statusCode() int { + if capture.status == 0 { + return http.StatusOK + } + return capture.status +} + +func (capture *boundedResponseCapture) startPassthrough() error { + if capture.passthrough { + return nil + } + capture.passthrough = true + writeHeader(capture.destination.Header(), capture.header) + capture.destination.WriteHeader(capture.statusCode()) + if capture.body.Len() == 0 { + return nil + } + _, err := capture.destination.Write(capture.body.Bytes()) + capture.body.Reset() + return err +} + +func writeCapturedResponse(destination http.ResponseWriter, header http.Header, status int, body []byte) { + writeHeader(destination.Header(), header) + destination.WriteHeader(status) + _, _ = destination.Write(body) +} + +func writeHeader(destination, source http.Header) { + for name := range destination { + destination.Del(name) + } + for name, values := range source { + destination[name] = append([]string(nil), values...) + } +} diff --git a/internal/gateway/metadata_enrichment_integration_test.go b/internal/gateway/metadata_enrichment_integration_test.go new file mode 100644 index 0000000..edf5e26 --- /dev/null +++ b/internal/gateway/metadata_enrichment_integration_test.go @@ -0,0 +1,196 @@ +package gateway + +import ( + "bytes" + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/InfinityPacer/plex-gateway/internal/database" + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" + "github.com/InfinityPacer/plex-gateway/internal/metrics" + "github.com/InfinityPacer/plex-gateway/internal/partcache" + "github.com/InfinityPacer/plex-gateway/internal/pathmap" + "github.com/InfinityPacer/plex-gateway/internal/resolver" +) + +type delayedIntegrationProvider struct { + calls atomic.Int64 + once sync.Once + started chan struct{} + release chan struct{} +} + +func (*delayedIntegrationProvider) Descriptor() mediainfo.ProviderDescriptor { + return mediainfo.ProviderDescriptor{Name: "integration", Revision: "v1"} +} + +func (provider *delayedIntegrationProvider) Probe(ctx context.Context, _ mediainfo.ProviderRequest) (mediainfo.ProviderResult, error) { + provider.calls.Add(1) + provider.once.Do(func() { close(provider.started) }) + select { + case <-ctx.Done(): + return mediainfo.ProviderResult{}, ctx.Err() + case <-provider.release: + } + return mediainfo.ProviderResult{Media: completeProjectionMedia()}, nil +} + +func TestGatewayColdMetadataTimeoutContinuesProbeAndWarmsNextResponse(t *testing.T) { + raw := []byte(``) + gzippedRaw := gzipBytes(t, raw) + plex := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/library/metadata/42" { + http.NotFound(w, request) + return + } + w.Header().Set("Content-Type", "application/xml") + w.Header().Set("ETag", `"plex-etag"`) + body := raw + if strings.Contains(request.Header.Get("Accept-Encoding"), "gzip") { + w.Header().Set("Content-Encoding", "gzip") + body = gzippedRaw + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body) + })) + defer plex.Close() + + directory := t.TempDir() + strmTarget := "http://mediavault:7811/redirect/pick/episode.mkv" + if err := os.WriteFile(filepath.Join(directory, "episode.strm"), []byte(strmTarget+"\n"), 0o600); err != nil { + t.Fatal(err) + } + mapper, err := pathmap.New([]pathmap.Mapping{{PlexPrefix: "/media/cloud", LocalPrefix: directory}}) + if err != nil { + t.Fatal(err) + } + controlResolver, err := resolver.NewMediaVaultSTRMResolver("http://mediavault:7811", nil, 0) + if err != nil { + t.Fatal(err) + } + gatewayDB, err := database.OpenSQLite(t.Context(), filepath.Join(directory, "plex-gateway.db")) + if err != nil { + t.Fatal(err) + } + store, err := mediainfo.NewSQLiteStore(t.Context(), gatewayDB) + if err != nil { + _ = gatewayDB.Close() + t.Fatal(err) + } + provider := &delayedIntegrationProvider{started: make(chan struct{}), release: make(chan struct{})} + service, err := mediainfo.NewService(mediainfo.ServiceOptions{ + Cache: mediainfo.NewCache(nil, time.Now()), Store: store, Provider: provider, + PlexServerID: "integration-server", Concurrency: 1, + InteractiveQueueSize: 4, BackgroundQueueSize: 4, + ProbeTimeout: time.Second, RecordTTL: time.Hour, RecordRetention: 24 * time.Hour, + BackgroundUserAgent: "integration-background", + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + }) + if err != nil { + _ = gatewayDB.Close() + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := service.Close(ctx); err != nil { + t.Error(err) + } + if err := gatewayDB.Close(); err != nil { + t.Error(err) + } + }) + + plexURL, err := url.Parse(plex.URL) + if err != nil { + t.Fatal(err) + } + registry := metrics.New() + handler := New(Options{ + Upstream: plexURL, PartCache: partcache.New(time.Hour), PathMapper: mapper, + Resolver: controlResolver, Metrics: registry, CloudExtensions: []string{".strm"}, + MediaInfoEnabled: true, MediaInfoStatus: service.Status, MediaInfoService: service, + MediaInfoColdWait: 20 * time.Millisecond, MediaInfoResponseMaxBytes: 1 << 20, + MediaInfoEnrichmentConcurrency: 1, + }) + + first := httptest.NewRecorder() + startedAt := time.Now() + handler.ServeHTTP(first, authenticatedMetadataRequest(http.MethodGet)) + if elapsed := time.Since(startedAt); elapsed > 500*time.Millisecond { + t.Fatalf("cold fail-open took %s", elapsed) + } + if first.Code != http.StatusOK || !bytes.Equal(first.Body.Bytes(), raw) || first.Header().Get("ETag") != `"plex-etag"` { + t.Fatalf("cold response status=%d headers=%#v body=%s", first.Code, first.Header(), first.Body.Bytes()) + } + select { + case <-provider.started: + default: + t.Fatal("cold request did not start the background probe") + } + close(provider.release) + + fingerprint, err := mediainfo.FingerprintSTRMTarget(strmTarget) + if err != nil { + t.Fatal(err) + } + key := mediainfo.Key{PlexServerID: "integration-server", PartID: "9", STRMFingerprint: fingerprint} + waitForPersistedMediaInfo(t, store, key) + + second := httptest.NewRecorder() + warmRequest := authenticatedMetadataRequest(http.MethodGet) + warmRequest.Header.Set("Accept-Encoding", "gzip") + handler.ServeHTTP(second, warmRequest) + if second.Code != http.StatusOK || bytes.Equal(second.Body.Bytes(), raw) { + t.Fatalf("warm response status=%d body=%s", second.Code, second.Body.Bytes()) + } + decodedWarm := gunzipBytes(t, second.Body.Bytes()) + if second.Header().Get("Content-Encoding") != "gzip" || second.Header().Get("ETag") != "" || !bytes.Contains(decodedWarm, []byte(`container="mkv"`)) { + t.Fatalf("warm response headers=%#v body=%s", second.Header(), second.Body.Bytes()) + } + if provider.calls.Load() != 1 { + t.Fatalf("provider calls = %d, want 1", provider.calls.Load()) + } + snapshot := registry.Snapshot() + if snapshot.MediaInfoFailOpenTotal != 1 || snapshot.MediaInfoEnrichedTotal != 1 { + t.Fatalf("metrics = %#v", snapshot) + } + + for _, requestPath := range []string{ + "/library/parts/999/1/file.strm", + "/video/:/transcode/universal/decision", + "/video/:/transcode/universal/start.mpd", + "/:/timeline", + } { + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedRequest(http.MethodGet, requestPath)) + } + if provider.calls.Load() != 1 { + t.Fatalf("non-metadata routes triggered MediaInfo provider: calls=%d", provider.calls.Load()) + } +} + +func waitForPersistedMediaInfo(t *testing.T, store *mediainfo.SQLiteStore, key mediainfo.Key) { + t.Helper() + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if record, found, err := store.Get(t.Context(), key); err != nil { + t.Fatal(err) + } else if found && record.Media.Complete { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("MediaInfo worker did not persist the completed probe") +} diff --git a/internal/gateway/metadata_enrichment_test.go b/internal/gateway/metadata_enrichment_test.go new file mode 100644 index 0000000..99910a5 --- /dev/null +++ b/internal/gateway/metadata_enrichment_test.go @@ -0,0 +1,418 @@ +package gateway + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" + "github.com/InfinityPacer/plex-gateway/internal/metrics" + "github.com/InfinityPacer/plex-gateway/internal/pathmap" + "github.com/InfinityPacer/plex-gateway/internal/resolver" +) + +type fakeMediaInfoEnsurer struct { + calls atomic.Int64 + mu sync.Mutex + seen []mediainfo.Request + fn func(context.Context, mediainfo.Request) (mediainfo.Record, error) +} + +func (ensurer *fakeMediaInfoEnsurer) Ensure(ctx context.Context, request mediainfo.Request) (mediainfo.Record, error) { + ensurer.calls.Add(1) + ensurer.mu.Lock() + ensurer.seen = append(ensurer.seen, request) + ensurer.mu.Unlock() + return ensurer.fn(ctx, request) +} + +func (ensurer *fakeMediaInfoEnsurer) requests() []mediainfo.Request { + ensurer.mu.Lock() + defer ensurer.mu.Unlock() + return append([]mediainfo.Request(nil), ensurer.seen...) +} + +func TestMetadataEnrichmentAddsMissingXMLFields(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + raw := []byte(``) + registry := metrics.New() + ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { + return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil + }} + handler := fixture.handler(ensurer, registry, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/xml") + w.Header().Set("Content-Length", strconv.Itoa(len(raw))) + w.Header().Set("ETag", `"plex-etag"`) + w.Header().Set("Last-Modified", "Wed, 01 Jan 2025 00:00:00 GMT") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(raw) + })) + + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedMetadataRequest(http.MethodGet)) + if response.Code != http.StatusOK { + t.Fatalf("status = %d", response.Code) + } + if bytes.Equal(response.Body.Bytes(), raw) { + t.Fatal("metadata body was not enriched") + } + if !strings.Contains(response.Body.String(), `container="mkv"`) || !strings.Contains(response.Body.String(), `codec="hevc"`) { + t.Fatalf("enriched XML missing expected fields: %s", response.Body.String()) + } + if response.Header().Get("ETag") != "" || response.Header().Get("Last-Modified") != "" { + t.Fatalf("stale validators retained: %#v", response.Header()) + } + if response.Header().Get("Cache-Control") != "private, no-cache" || response.Header().Get("Content-Length") != strconv.Itoa(response.Body.Len()) { + t.Fatalf("mutated headers = %#v", response.Header()) + } + requests := ensurer.requests() + if len(requests) != 1 || requests[0].Key.PartID != "9" || len(requests[0].Key.STRMFingerprint) != 64 || requests[0].Target != fixture.target || requests[0].ClientUserAgent != "Infuse-Test/1.0" { + t.Fatalf("MediaInfo request = %#v", requests) + } + snapshot := registry.Snapshot() + if snapshot.MediaInfoEnrichedTotal != 1 || snapshot.MediaInfoWaitActive != 0 || snapshot.MediaInfoFailOpenTotal != 0 { + t.Fatalf("metrics = %#v", snapshot) + } +} + +func TestMetadataEnrichmentPreservesGzipJSON(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + rawJSON := []byte(`{"MediaContainer":{"Metadata":[{"ratingKey":"42","type":"episode","Media":[{"Part":[{"id":"9","key":"/library/parts/9/1/file.strm","file":"/media/cloud/episode.strm","Stream":[{"id":"101","streamType":1,"index":0}]}]}]}]}}`) + wire := gzipBytes(t, rawJSON) + ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { + return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil + }} + handler := fixture.handler(ensurer, metrics.New(), http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") + w.Header().Set("Content-Length", strconv.Itoa(len(wire))) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(wire) + })) + + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedMetadataRequest(http.MethodGet)) + if response.Header().Get("Content-Encoding") != "gzip" || response.Header().Get("Content-Length") != strconv.Itoa(response.Body.Len()) { + t.Fatalf("gzip headers = %#v", response.Header()) + } + decoded := gunzipBytes(t, response.Body.Bytes()) + var value map[string]any + if err := json.Unmarshal(decoded, &value); err != nil { + t.Fatalf("decode enriched JSON: %v", err) + } + if !bytes.Contains(decoded, []byte(`"container":"mkv"`)) || !bytes.Contains(decoded, []byte(`"codec":"hevc"`)) { + t.Fatalf("enriched JSON missing expected fields: %s", decoded) + } +} + +func TestMetadataEnrichmentTimeoutReturnsOriginalResponse(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + raw := fixture.xmlBody() + ensurer := &fakeMediaInfoEnsurer{fn: func(ctx context.Context, _ mediainfo.Request) (mediainfo.Record, error) { + <-ctx.Done() + return mediainfo.Record{}, ctx.Err() + }} + handler := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ + Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, + CloudExtensions: []string{".strm"}, ColdWait: 10 * time.Millisecond, + ResponseLimit: 1 << 20, Concurrency: 1, Metrics: metrics.New(), + }, metadataBodyHandler(raw, "application/xml")) + + response := httptest.NewRecorder() + started := time.Now() + handler.ServeHTTP(response, authenticatedMetadataRequest(http.MethodGet)) + if elapsed := time.Since(started); elapsed > 500*time.Millisecond { + t.Fatalf("cold timeout took %s", elapsed) + } + if !bytes.Equal(response.Body.Bytes(), raw) || response.Header().Get("ETag") != `"plex-etag"` { + t.Fatalf("fail-open response changed: headers=%#v body=%s", response.Header(), response.Body.Bytes()) + } +} + +func TestMetadataEnrichmentRejectsFingerprintChange(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + raw := fixture.xmlBody() + ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { + if err := os.WriteFile(fixture.strmPath, []byte("http://mediavault:7811/redirect/changed/file.mkv\n"), 0o600); err != nil { + t.Fatal(err) + } + return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil + }} + handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(raw, "application/xml")) + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedMetadataRequest(http.MethodGet)) + if !bytes.Equal(response.Body.Bytes(), raw) { + t.Fatalf("response changed after STRM identity changed: %s", response.Body.Bytes()) + } +} + +func TestMetadataEnrichmentPoolSaturationSkipsWaiting(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + raw := fixture.xmlBody() + started := make(chan struct{}) + release := make(chan struct{}) + ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, request mediainfo.Request) (mediainfo.Record, error) { + close(started) + <-release + return mediainfo.Record{Key: request.Key, Media: completeProjectionMedia()}, nil + }} + registry := metrics.New() + handler := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ + Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, + CloudExtensions: []string{".strm"}, ColdWait: time.Second, + ResponseLimit: 1 << 20, Concurrency: 1, Metrics: registry, + }, metadataBodyHandler(raw, "application/xml")) + + firstDone := make(chan *httptest.ResponseRecorder, 1) + go func() { + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedMetadataRequest(http.MethodGet)) + firstDone <- response + }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("first enrichment did not start") + } + second := httptest.NewRecorder() + handler.ServeHTTP(second, authenticatedMetadataRequest(http.MethodGet)) + if !bytes.Equal(second.Body.Bytes(), raw) || ensurer.calls.Load() != 1 { + t.Fatalf("saturated response=%s calls=%d", second.Body.Bytes(), ensurer.calls.Load()) + } + close(release) + select { + case first := <-firstDone: + if bytes.Equal(first.Body.Bytes(), raw) { + t.Fatal("first response was not enriched") + } + case <-time.After(time.Second): + t.Fatal("first enrichment did not finish") + } + if registry.Snapshot().MediaInfoWaitRejectedTotal != 1 { + t.Fatalf("metrics = %#v", registry.Snapshot()) + } +} + +func TestMetadataEnrichmentBypassesIneligibleAndUnbufferableResponses(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + raw := fixture.xmlBody() + oversized := bytes.Repeat([]byte("x"), 1025) + ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, _ mediainfo.Request) (mediainfo.Record, error) { + return mediainfo.Record{}, errors.New("unexpected Ensure") + }} + tests := []struct { + name string + request *http.Request + next http.Handler + wantStatus int + wantBody []byte + }{ + {name: "HEAD", request: authenticatedMetadataRequest(http.MethodHead), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK}, + {name: "batch", request: authenticatedRequest(http.MethodGet, "/library/metadata/42,43"), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, + {name: "children", request: authenticatedRequest(http.MethodGet, "/library/metadata/42/children"), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, + {name: "Part playback", request: authenticatedRequest(http.MethodGet, "/library/parts/9/1/file.strm"), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, + {name: "decision", request: authenticatedRequest(http.MethodGet, "/video/:/transcode/universal/decision"), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, + {name: "universal start", request: authenticatedRequest(http.MethodGet, "/video/:/transcode/universal/start.mpd"), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, + {name: "timeline", request: authenticatedRequest(http.MethodGet, "/:/timeline"), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, + {name: "missing token", request: httptest.NewRequest(http.MethodGet, "/library/metadata/42", nil), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, + {name: "conflicting token", request: conflictingTokenRequest(), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, + {name: "range", request: rangeMetadataRequest(), next: metadataBodyHandler(raw, "application/xml"), wantStatus: http.StatusOK, wantBody: raw}, + {name: "unsupported encoding", request: authenticatedMetadataRequest(http.MethodGet), next: encodedMetadataHandler(raw, "br"), wantStatus: http.StatusOK, wantBody: raw}, + {name: "oversized", request: authenticatedMetadataRequest(http.MethodGet), next: metadataBodyHandler(oversized, "application/xml"), wantStatus: http.StatusOK, wantBody: oversized}, + {name: "not modified", request: authenticatedMetadataRequest(http.MethodGet), next: metadataStatusHandler(http.StatusNotModified, nil), wantStatus: http.StatusNotModified}, + {name: "partial content", request: authenticatedMetadataRequest(http.MethodGet), next: metadataStatusHandler(http.StatusPartialContent, []byte("partial")), wantStatus: http.StatusPartialContent, wantBody: []byte("partial")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + handler := newMetadataEnrichmentHandler(metadataEnrichmentOptions{ + Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, + CloudExtensions: []string{".strm"}, ColdWait: time.Second, + ResponseLimit: 1024, Concurrency: 1, Metrics: metrics.New(), + }, test.next) + response := httptest.NewRecorder() + handler.ServeHTTP(response, test.request) + if response.Code != test.wantStatus || !bytes.Equal(response.Body.Bytes(), test.wantBody) { + t.Fatalf("response status=%d body=%q, want status=%d body=%q", response.Code, response.Body.Bytes(), test.wantStatus, test.wantBody) + } + }) + } + if ensurer.calls.Load() != 0 { + t.Fatalf("ineligible responses called Ensure %d times", ensurer.calls.Load()) + } +} + +func TestMetadataEnrichmentSkipsLocalAndCompleteCloudParts(t *testing.T) { + fixture := newMetadataEnrichmentFixture(t) + ensurer := &fakeMediaInfoEnsurer{fn: func(_ context.Context, _ mediainfo.Request) (mediainfo.Record, error) { + return mediainfo.Record{}, errors.New("unexpected Ensure") + }} + tests := []struct { + name string + body []byte + }{ + { + name: "local Part", + body: []byte(``), + }, + { + name: "complete cloud Part", + body: []byte(``), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + handler := fixture.handler(ensurer, metrics.New(), metadataBodyHandler(test.body, "application/xml")) + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedMetadataRequest(http.MethodGet)) + if !bytes.Equal(response.Body.Bytes(), test.body) || response.Header().Get("ETag") != `"plex-etag"` { + t.Fatalf("response changed: headers=%#v body=%s", response.Header(), response.Body.Bytes()) + } + }) + } + if ensurer.calls.Load() != 0 { + t.Fatalf("skipped Parts called Ensure %d times", ensurer.calls.Load()) + } +} + +type metadataEnrichmentFixture struct { + mapper *pathmap.Mapper + resolver resolver.ControlResolver + strmPath string + target string +} + +func newMetadataEnrichmentFixture(t *testing.T) metadataEnrichmentFixture { + t.Helper() + directory := t.TempDir() + strmPath := filepath.Join(directory, "episode.strm") + target := "http://mediavault:7811/redirect/pick/episode.mkv" + if err := os.WriteFile(strmPath, []byte(target+"\n"), 0o600); err != nil { + t.Fatal(err) + } + mapper, err := pathmap.New([]pathmap.Mapping{{PlexPrefix: "/media/cloud", LocalPrefix: directory}}) + if err != nil { + t.Fatal(err) + } + strmResolver, err := resolver.NewMediaVaultSTRMResolver("http://mediavault:7811", nil, 0) + if err != nil { + t.Fatal(err) + } + return metadataEnrichmentFixture{mapper: mapper, resolver: strmResolver, strmPath: strmPath, target: target} +} + +func (fixture metadataEnrichmentFixture) handler(ensurer mediaInfoEnsurer, registry *metrics.Metrics, next http.Handler) http.Handler { + return newMetadataEnrichmentHandler(metadataEnrichmentOptions{ + Service: ensurer, Mapper: fixture.mapper, Resolver: fixture.resolver, + CloudExtensions: []string{".strm"}, ColdWait: time.Second, + ResponseLimit: 1 << 20, Concurrency: 2, Metrics: registry, + }, next) +} + +func (fixture metadataEnrichmentFixture) xmlBody() []byte { + return []byte(``) +} + +func completeProjectionMedia() mediainfo.Media { + return mediainfo.Media{ + Complete: true, Container: "mkv", DurationMS: 60_000, Size: 1_000_000, + Bitrate: 8_000, VideoCodec: "hevc", Width: 3840, Height: 2160, + Streams: []mediainfo.Stream{{Index: 0, Type: "video", Codec: "hevc", Width: 3840, Height: 2160, HDRFormat: "dolby_vision"}}, + } +} + +func authenticatedMetadataRequest(method string) *http.Request { + return authenticatedRequest(method, "/library/metadata/42") +} + +func authenticatedRequest(method, path string) *http.Request { + request := httptest.NewRequest(method, path, nil) + request.Header.Set("X-Plex-Token", "client-token") + request.Header.Set("User-Agent", "Infuse-Test/1.0") + return request +} + +func conflictingTokenRequest() *http.Request { + request := authenticatedRequest(http.MethodGet, "/library/metadata/42?X-Plex-Token=query-token") + request.Header.Set("X-Plex-Token", "header-token") + return request +} + +func rangeMetadataRequest() *http.Request { + request := authenticatedMetadataRequest(http.MethodGet) + request.Header.Set("Range", "bytes=0-100") + return request +} + +func metadataBodyHandler(body []byte, contentType string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.Header().Set("ETag", `"plex-etag"`) + w.WriteHeader(http.StatusOK) + if request.Method != http.MethodHead { + _, _ = w.Write(body) + } + }) +} + +func encodedMetadataHandler(body []byte, encoding string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/xml") + w.Header().Set("Content-Encoding", encoding) + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body) + }) +} + +func metadataStatusHandler(status int, body []byte) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/xml") + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + if status == http.StatusPartialContent { + w.Header().Set("Content-Range", "bytes 0-6/100") + } + w.WriteHeader(status) + _, _ = w.Write(body) + }) +} + +func gzipBytes(t *testing.T, body []byte) []byte { + t.Helper() + var output bytes.Buffer + writer := gzip.NewWriter(&output) + if _, err := writer.Write(body); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return output.Bytes() +} + +func gunzipBytes(t *testing.T, body []byte) []byte { + t.Helper() + reader, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + decoded, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + return decoded +} diff --git a/internal/mediainfo/benchmark_test.go b/internal/mediainfo/benchmark_test.go index a5e3f57..bcbebef 100644 --- a/internal/mediainfo/benchmark_test.go +++ b/internal/mediainfo/benchmark_test.go @@ -46,12 +46,9 @@ func BenchmarkMediaInfoL1PutAtCapacity(b *testing.B) { func BenchmarkMediaInfoSQLiteGet(b *testing.B) { now := time.Now() record := completeRecord(now) - store, err := OpenSQLite(context.Background(), filepath.Join(b.TempDir(), "mediainfo.db")) - if err != nil { - b.Fatal(err) - } + store, gatewayDB := openTestSQLiteStore(b, filepath.Join(b.TempDir(), "plex-gateway.db")) b.Cleanup(func() { - if err := store.Close(); err != nil { + if err := gatewayDB.Close(); err != nil { b.Error(err) } }) diff --git a/internal/mediainfo/model.go b/internal/mediainfo/model.go index 5164732..5ebda19 100644 --- a/internal/mediainfo/model.go +++ b/internal/mediainfo/model.go @@ -233,12 +233,7 @@ func FingerprintSTRM(path string) (string, error) { if target == "" { continue } - normalized, err := normalizeSTRMTarget(target) - if err != nil { - return "", err - } - digest := sha256.Sum256([]byte(normalized)) - return hex.EncodeToString(digest[:]), nil + return FingerprintSTRMTarget(target) } if err := scanner.Err(); err != nil { return "", errors.New("STRM control target exceeds fingerprint limit") @@ -246,6 +241,18 @@ func FingerprintSTRM(path string) (string, error) { return "", errors.New("STRM control file is empty") } +// FingerprintSTRMTarget hashes a target already read and validated from a STRM +// file. Callers that also need the target can avoid a second filesystem read, +// keeping the analysis identity bound to the exact control value being probed. +func FingerprintSTRMTarget(target string) (string, error) { + normalized, err := normalizeSTRMTarget(strings.TrimSpace(target)) + if err != nil { + return "", err + } + digest := sha256.Sum256([]byte(normalized)) + return hex.EncodeToString(digest[:]), nil +} + func normalizeSTRMTarget(target string) (string, error) { if strings.HasPrefix(target, "/") { return "path:" + target, nil diff --git a/internal/mediainfo/model_test.go b/internal/mediainfo/model_test.go index d6739e6..b4ba0bd 100644 --- a/internal/mediainfo/model_test.go +++ b/internal/mediainfo/model_test.go @@ -58,6 +58,25 @@ func TestFingerprintSTRMNormalizesFormattingAndMediaVaultOrigin(t *testing.T) { } } +func TestFingerprintSTRMTargetMatchesFileFingerprint(t *testing.T) { + target := "http://mediavault:7811/redirect/example/file.mkv" + path := filepath.Join(t.TempDir(), "sample.strm") + if err := os.WriteFile(path, []byte("\n"+target+"\n"), 0o600); err != nil { + t.Fatal(err) + } + fromFile, err := FingerprintSTRM(path) + if err != nil { + t.Fatal(err) + } + fromTarget, err := FingerprintSTRMTarget(target) + if err != nil { + t.Fatal(err) + } + if fromTarget != fromFile { + t.Fatalf("target fingerprint = %q, file fingerprint = %q", fromTarget, fromFile) + } +} + func TestCacheReturnsDetachedFreshRecord(t *testing.T) { now := time.Unix(1_800_000_000, 0) record := completeRecord(now) diff --git a/internal/mediainfo/negative_cache_test.go b/internal/mediainfo/negative_cache_test.go new file mode 100644 index 0000000..bf13146 --- /dev/null +++ b/internal/mediainfo/negative_cache_test.go @@ -0,0 +1,317 @@ +package mediainfo + +import ( + "bytes" + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +type scriptedMediaProber struct { + calls atomic.Int64 + errors []error + userAgents chan string +} + +type proberFunc func(context.Context, string, string) (Media, error) + +func (probe proberFunc) Probe(ctx context.Context, target, userAgent string) (Media, error) { + return probe(ctx, target, userAgent) +} + +type synchronizedBuffer struct { + mu sync.Mutex + buffer bytes.Buffer +} + +type observedDoneContext struct { + context.Context + once sync.Once + observed chan struct{} +} + +func (ctx *observedDoneContext) Done() <-chan struct{} { + ctx.once.Do(func() { close(ctx.observed) }) + return ctx.Context.Done() +} + +func (buffer *synchronizedBuffer) Write(value []byte) (int, error) { + buffer.mu.Lock() + defer buffer.mu.Unlock() + return buffer.buffer.Write(value) +} + +func (buffer *synchronizedBuffer) String() string { + buffer.mu.Lock() + defer buffer.mu.Unlock() + return buffer.buffer.String() +} + +func (prober *scriptedMediaProber) Probe(_ context.Context, _ string, userAgent string) (Media, error) { + index := int(prober.calls.Add(1)) - 1 + if prober.userAgents != nil { + prober.userAgents <- userAgent + } + if index < len(prober.errors) && prober.errors[index] != nil { + return Media{}, fmt.Errorf("probe failed for %s: %w", userAgent, prober.errors[index]) + } + return Media{ + Complete: true, Container: "mkv", DurationMS: 60_000, + Streams: []Stream{{Index: 0, Type: "video", Codec: "hevc", Width: 1920, Height: 1080}}, + }, nil +} + +func TestServiceNegativeCacheSeparatesUserAgents(t *testing.T) { + now := time.Unix(1_800_000_000, 0).UTC() + probeErr := errors.New("probe unavailable") + prober := &scriptedMediaProber{errors: []error{probeErr, probeErr}} + provider := &fakeProvider{prober: prober} + logBuffer := new(synchronizedBuffer) + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, now), Store: &fakeRecordStore{}, Provider: provider, + PlexServerID: "server", NegativeTTL: time.Hour, + Logger: slog.New(slog.NewTextHandler(logBuffer, nil)), Now: func() time.Time { return now }, + }) + requestA := testRequest("negative-ua", PriorityInteractive) + requestA.ClientUserAgent = "client-agent-a" + requestB := requestA + requestB.ClientUserAgent = "client-agent-b" + + if _, err := service.Ensure(t.Context(), requestA); !errors.Is(err, probeErr) { + t.Fatalf("first User-Agent error = %v", err) + } + if _, err := service.Ensure(t.Context(), requestA); !errors.Is(err, ErrNegativeCache) { + t.Fatalf("same User-Agent retry error = %v", err) + } + if _, err := service.Ensure(t.Context(), requestB); !errors.Is(err, probeErr) { + t.Fatalf("different User-Agent error = %v", err) + } + if _, err := service.Ensure(t.Context(), requestB); !errors.Is(err, ErrNegativeCache) { + t.Fatalf("different User-Agent retry error = %v", err) + } + if got := provider.calls.Load(); got != 2 { + t.Fatalf("provider calls = %d, want 2", got) + } + + service.mu.Lock() + if len(service.negative) != 2 { + service.mu.Unlock() + t.Fatalf("negative cache entries = %d, want 2", len(service.negative)) + } + for key := range service.negative { + if strings.Contains(key, requestA.ClientUserAgent) || strings.Contains(key, requestB.ClientUserAgent) { + service.mu.Unlock() + t.Fatalf("negative cache key contains raw User-Agent: %q", key) + } + } + service.mu.Unlock() + logs := logBuffer.String() + if strings.Contains(logs, requestA.ClientUserAgent) || strings.Contains(logs, requestB.ClientUserAgent) { + t.Fatalf("failure logs contain raw User-Agent: %s", logs) + } +} + +func TestServiceNegativeCacheUsesBackgroundUserAgentForEmptyRequest(t *testing.T) { + now := time.Unix(1_800_000_000, 0).UTC() + backgroundUserAgent := "background-agent" + probeErr := errors.New("probe unavailable") + prober := &scriptedMediaProber{ + errors: []error{probeErr}, + userAgents: make(chan string, 1), + } + provider := &fakeProvider{prober: prober} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, now), Store: &fakeRecordStore{}, Provider: provider, + PlexServerID: "server", BackgroundUserAgent: backgroundUserAgent, + NegativeTTL: time.Hour, Now: func() time.Time { return now }, + }) + request := testRequest("negative-background-ua", PriorityInteractive) + request.ClientUserAgent = "" + if _, err := service.Ensure(t.Context(), request); !errors.Is(err, probeErr) { + t.Fatalf("empty User-Agent error = %v", err) + } + select { + case got := <-prober.userAgents: + if got != backgroundUserAgent { + t.Fatalf("effective User-Agent = %q, want %q", got, backgroundUserAgent) + } + case <-time.After(time.Second): + t.Fatal("probe User-Agent was not captured") + } + + explicit := request + explicit.ClientUserAgent = backgroundUserAgent + if _, err := service.Ensure(t.Context(), explicit); !errors.Is(err, ErrNegativeCache) { + t.Fatalf("explicit background User-Agent retry error = %v", err) + } + if got := provider.calls.Load(); got != 1 { + t.Fatalf("provider calls = %d, want 1", got) + } + service.mu.Lock() + for key := range service.negative { + if strings.Contains(key, backgroundUserAgent) { + service.mu.Unlock() + t.Fatalf("negative cache key contains raw background User-Agent: %q", key) + } + } + service.mu.Unlock() +} + +func TestServiceDoesNotNegativeCacheCanceledProbe(t *testing.T) { + now := time.Unix(1_800_000_000, 0).UTC() + prober := &scriptedMediaProber{errors: []error{fmt.Errorf("probe canceled: %w", context.Canceled), nil}} + provider := &fakeProvider{prober: prober} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, now), Store: &fakeRecordStore{}, Provider: provider, + PlexServerID: "server", NegativeTTL: time.Hour, Now: func() time.Time { return now }, + }) + request := testRequest("canceled-probe", PriorityInteractive) + if _, err := service.Ensure(t.Context(), request); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled probe error = %v", err) + } + if _, err := service.Ensure(t.Context(), request); err != nil { + t.Fatalf("retry after cancellation error = %v", err) + } + if got := provider.calls.Load(); got != 2 { + t.Fatalf("provider calls = %d, want 2", got) + } + service.mu.Lock() + negativeEntries := len(service.negative) + service.mu.Unlock() + if negativeEntries != 0 { + t.Fatalf("negative cache entries after cancellation = %d", negativeEntries) + } +} + +func TestServiceSingleflightsSuccessfulProbesAcrossUserAgents(t *testing.T) { + now := time.Unix(1_800_000_000, 0).UTC() + prober := &blockingProber{started: make(chan string, 1), release: make(chan struct{})} + provider := &fakeProvider{prober: prober} + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, now), Store: &fakeRecordStore{}, Provider: provider, + PlexServerID: "server", Now: func() time.Time { return now }, + }) + requestA := testRequest("singleflight-ua", PriorityInteractive) + requestA.ClientUserAgent = "client-agent-a" + requestB := requestA + requestB.ClientUserAgent = "client-agent-b" + + flight, err := service.begin(requestA) + if err != nil { + t.Fatal(err) + } + select { + case <-prober.started: + case <-time.After(time.Second): + t.Fatal("probe did not start") + } + sameFlight, err := service.begin(requestB) + if err != nil { + t.Fatal(err) + } + if sameFlight != flight { + t.Fatal("different User-Agent did not join the existing Key flight") + } + close(prober.release) + select { + case <-flight.done: + if flight.err != nil { + t.Fatalf("singleflight error = %v", flight.err) + } + case <-time.After(time.Second): + t.Fatal("singleflight did not finish") + } + if got := provider.calls.Load(); got != 1 { + t.Fatalf("provider calls = %d, want 1", got) + } + if _, err := service.Ensure(t.Context(), requestB); err != nil { + t.Fatalf("successful record lookup with different User-Agent error = %v", err) + } + if got := provider.calls.Load(); got != 1 { + t.Fatalf("provider calls after cached lookup = %d, want 1", got) + } +} + +func TestServiceRetriesFailedSharedFlightWithWaitingUserAgent(t *testing.T) { + now := time.Unix(1_800_000_000, 0).UTC() + releaseFirst := make(chan struct{}) + userAgents := make(chan string, 2) + var calls atomic.Int64 + prober := proberFunc(func(ctx context.Context, _ string, userAgent string) (Media, error) { + call := calls.Add(1) + userAgents <- userAgent + if call == 1 { + select { + case <-ctx.Done(): + return Media{}, ctx.Err() + case <-releaseFirst: + return Media{}, errors.New("first User-Agent rejected") + } + } + return Media{ + Complete: true, Container: "mkv", DurationMS: 60_000, + Streams: []Stream{{Index: 0, Type: "video", Codec: "hevc", Width: 1920, Height: 1080}}, + }, nil + }) + service := newServiceForTest(t, ServiceOptions{ + Cache: NewCache(nil, now), Store: &fakeRecordStore{}, Provider: &fakeProvider{prober: prober}, + PlexServerID: "server", NegativeTTL: time.Hour, Now: func() time.Time { return now }, + }) + requestA := testRequest("concurrent-failed-ua", PriorityInteractive) + requestA.ClientUserAgent = "client-agent-a" + requestB := requestA + requestB.ClientUserAgent = "client-agent-b" + + resultA := make(chan error, 1) + go func() { + _, err := service.Ensure(t.Context(), requestA) + resultA <- err + }() + select { + case got := <-userAgents: + if got != requestA.ClientUserAgent { + t.Fatalf("first probe User-Agent = %q", got) + } + case <-time.After(time.Second): + t.Fatal("first probe did not start") + } + + waitObserved := make(chan struct{}) + waitContext, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + resultB := make(chan error, 1) + go func() { + _, err := service.Ensure(&observedDoneContext{Context: waitContext, observed: waitObserved}, requestB) + resultB <- err + }() + select { + case <-waitObserved: + case <-time.After(time.Second): + t.Fatal("second User-Agent did not join the shared flight") + } + close(releaseFirst) + + if err := <-resultA; err == nil || !strings.Contains(err.Error(), "first User-Agent rejected") { + t.Fatalf("first User-Agent error = %v", err) + } + select { + case got := <-userAgents: + if got != requestB.ClientUserAgent { + t.Fatalf("retry probe User-Agent = %q", got) + } + case <-time.After(time.Second): + t.Fatal("waiting User-Agent retry did not start") + } + if err := <-resultB; err != nil { + t.Fatalf("waiting User-Agent retry error = %v", err) + } + if got := calls.Load(); got != 2 { + t.Fatalf("probe calls = %d, want 2", got) + } +} diff --git a/internal/mediainfo/service.go b/internal/mediainfo/service.go index 7bd56c3..63c3ef7 100644 --- a/internal/mediainfo/service.go +++ b/internal/mediainfo/service.go @@ -2,6 +2,7 @@ package mediainfo import ( "context" + "crypto/sha256" "errors" "log/slog" "strings" @@ -162,10 +163,11 @@ type recordTouch struct { } type flight struct { - done chan struct{} - job *job - record Record - err error + done chan struct{} + job *job + userAgent string + record Record + err error } type job struct { @@ -313,15 +315,26 @@ func (service *Service) Ensure(ctx context.Context, request Request) (Record, er return record, nil } service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoCacheMisses() }) - flight, err := service.begin(request) - if err != nil { - return Record{}, err - } - select { - case <-ctx.Done(): - return Record{}, ctx.Err() - case <-flight.done: - return cloneRecord(flight.record), flight.err + for { + if err := ctx.Err(); err != nil { + return Record{}, err + } + flight, err := service.begin(request) + if err != nil { + return Record{}, err + } + select { + case <-ctx.Done(): + return Record{}, ctx.Err() + case <-flight.done: + if flight.err != nil && flight.userAgent != request.ClientUserAgent { + // Successful technical metadata is UA-independent and remains + // shared. A failed provider attempt is UA-sensitive, so a waiter + // with another client identity may retry within its own deadline. + continue + } + return cloneRecord(flight.record), flight.err + } } } @@ -352,6 +365,10 @@ func (service *Service) normalizeRequest(request Request) (Request, error) { if request.Key.PlexServerID != service.plexServerID { return Request{}, errors.New("MediaInfo request belongs to another Plex server") } + request.ClientUserAgent = strings.TrimSpace(request.ClientUserAgent) + if request.ClientUserAgent == "" { + request.ClientUserAgent = service.backgroundUserAgent + } if err := request.validate(); err != nil { return Request{}, err } @@ -420,18 +437,19 @@ func (service *Service) begin(request Request) (*flight, error) { return nil, err } key := request.Key.cacheKey() + negativeKey := service.negativeKey(request) now := service.now() service.mu.Lock() if service.closed { service.mu.Unlock() return nil, ErrServiceUnavailable } - if until, exists := service.negative[key]; exists { + if until, exists := service.negative[negativeKey]; exists { if now.Before(until) { service.mu.Unlock() return nil, ErrNegativeCache } - delete(service.negative, key) + delete(service.negative, negativeKey) } if existing, exists := service.flights[key]; exists { if request.Priority == PriorityInteractive { @@ -440,7 +458,7 @@ func (service *Service) begin(request Request) (*flight, error) { service.mu.Unlock() return existing, nil } - flight := &flight{done: make(chan struct{})} + flight := &flight{done: make(chan struct{}), userAgent: request.ClientUserAgent} queued := &job{request: request, flight: flight} flight.job = queued if !service.enqueueLocked(queued) { @@ -626,7 +644,7 @@ func (service *Service) finish(queued *job, record Record, err error) { } delete(service.flights, key) if err != nil && !errors.Is(err, context.Canceled) { - service.rememberNegativeLocked(key, service.now()) + service.rememberNegativeLocked(service.negativeKey(queued.request), service.now()) } flight.record = record flight.err = err @@ -640,6 +658,11 @@ func (service *Service) finish(queued *job, record Record, err error) { service.metric(func(metrics ServiceMetrics) { metrics.IncMediaInfoProbeSuccess() }) } +func (service *Service) negativeKey(request Request) string { + digest := sha256.Sum256([]byte(request.ClientUserAgent)) + return request.Key.cacheKey() + "\x00" + string(digest[:]) +} + func (service *Service) rememberNegativeLocked(key string, now time.Time) { for candidate, until := range service.negative { if !now.Before(until) { diff --git a/internal/mediainfo/sqlite.go b/internal/mediainfo/sqlite.go index 7d94a7a..a32d605 100644 --- a/internal/mediainfo/sqlite.go +++ b/internal/mediainfo/sqlite.go @@ -6,18 +6,16 @@ import ( "encoding/json" "errors" "fmt" - "os" - "path/filepath" "strings" "time" - _ "modernc.org/sqlite" + "github.com/InfinityPacer/plex-gateway/internal/database" ) const ( - sqliteDatabaseVersion = 1 - sqliteSchemaV1 = ` -CREATE TABLE IF NOT EXISTS media_info_records ( + mediaInfoMigrationModule = "mediainfo" + mediaInfoSchemaV1 = ` + CREATE TABLE IF NOT EXISTS media_info_records ( plex_server_id TEXT NOT NULL, part_id TEXT NOT NULL, strm_fingerprint TEXT NOT NULL, @@ -39,87 +37,24 @@ CREATE INDEX IF NOT EXISTS media_info_records_retain_until ` ) -// SQLiteStore persists successful MediaInfo results. It uses one database -// connection because SQLite is the single-instance writer in this design. +// SQLiteStore persists successful MediaInfo results in the shared Gateway +// database without owning the connection lifecycle. type SQLiteStore struct { db *sql.DB } -// OpenSQLite creates or opens a private SQLite database, enables WAL mode, and -// applies versioned migrations. The path is a filesystem path, not a -// caller-controlled SQLite URI. -func OpenSQLite(ctx context.Context, path string) (*SQLiteStore, error) { - path = strings.TrimSpace(path) - if path == "" || strings.IndexByte(path, 0) >= 0 || strings.HasPrefix(path, "file:") { - return nil, errors.New("MediaInfo database path is invalid") - } - cleanPath := filepath.Clean(path) - directory := filepath.Dir(cleanPath) - if err := os.MkdirAll(directory, 0o750); err != nil { - return nil, fmt.Errorf("create MediaInfo database directory: %w", err) - } - file, err := os.OpenFile(cleanPath, os.O_CREATE|os.O_RDWR, 0o600) - if err != nil { - return nil, fmt.Errorf("create MediaInfo database: %w", err) - } - if err := file.Close(); err != nil { - return nil, fmt.Errorf("close MediaInfo database file: %w", err) - } - - db, err := sql.Open("sqlite", cleanPath) - if err != nil { - return nil, fmt.Errorf("open MediaInfo database: %w", err) - } - db.SetMaxOpenConns(1) - db.SetMaxIdleConns(1) - store := &SQLiteStore{db: db} - if err := store.initialize(ctx); err != nil { - _ = db.Close() +// NewSQLiteStore attaches the MediaInfo domain to an open Gateway database and +// applies only the migrations owned by this domain. +func NewSQLiteStore(ctx context.Context, gatewayDB *database.SQLite) (*SQLiteStore, error) { + if gatewayDB == nil || gatewayDB.SQLDB() == nil { + return nil, errors.New("Gateway database is unavailable") + } + if err := gatewayDB.ApplyMigrations(ctx, mediaInfoMigrationModule, []database.Migration{ + {Version: 1, SQL: mediaInfoSchemaV1}, + }); err != nil { return nil, err } - return store, nil -} - -func (store *SQLiteStore) initialize(ctx context.Context) error { - for _, statement := range []string{ - "PRAGMA journal_mode=WAL", - "PRAGMA synchronous=NORMAL", - "PRAGMA foreign_keys=ON", - "PRAGMA busy_timeout=5000", - } { - if _, err := store.db.ExecContext(ctx, statement); err != nil { - return fmt.Errorf("configure MediaInfo database: %w", err) - } - } - return store.migrate(ctx) -} - -func (store *SQLiteStore) migrate(ctx context.Context) error { - transaction, err := store.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("begin MediaInfo database migration: %w", err) - } - defer transaction.Rollback() - - var version int - if err := transaction.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil { - return fmt.Errorf("read MediaInfo database version: %w", err) - } - if version > sqliteDatabaseVersion { - return fmt.Errorf("MediaInfo database version %d is newer than supported version %d", version, sqliteDatabaseVersion) - } - if version < 1 { - if _, err := transaction.ExecContext(ctx, sqliteSchemaV1); err != nil { - return fmt.Errorf("apply MediaInfo database schema v1: %w", err) - } - if _, err := transaction.ExecContext(ctx, "PRAGMA user_version = 1"); err != nil { - return fmt.Errorf("record MediaInfo database schema v1: %w", err) - } - } - if err := transaction.Commit(); err != nil { - return fmt.Errorf("commit MediaInfo database migration: %w", err) - } - return nil + return &SQLiteStore{db: gatewayDB.SQLDB()}, nil } // Put inserts or refreshes one complete record. A late older probe cannot @@ -306,14 +241,6 @@ WHERE plex_server_id = ? AND part_id = ? AND strm_fingerprint = ? return nil } -// Close flushes and closes the SQLite database. -func (store *SQLiteStore) Close() error { - if store == nil || store.db == nil { - return nil - } - return store.db.Close() -} - type rowScanner interface { Scan(dest ...any) error } diff --git a/internal/mediainfo/sqlite_test.go b/internal/mediainfo/sqlite_test.go index 22d92fa..b977037 100644 --- a/internal/mediainfo/sqlite_test.go +++ b/internal/mediainfo/sqlite_test.go @@ -2,37 +2,44 @@ package mediainfo import ( "context" - "database/sql" "path/filepath" "strings" "testing" "time" - _ "modernc.org/sqlite" + "github.com/InfinityPacer/plex-gateway/internal/database" ) +func openTestSQLiteStore(tb testing.TB, path string) (*SQLiteStore, *database.SQLite) { + tb.Helper() + gatewayDB, err := database.OpenSQLite(context.Background(), path) + if err != nil { + tb.Fatal(err) + } + store, err := NewSQLiteStore(context.Background(), gatewayDB) + if err != nil { + _ = gatewayDB.Close() + tb.Fatal(err) + } + return store, gatewayDB +} + func TestSQLiteStorePersistsRetainedRecordAcrossReopen(t *testing.T) { ctx := context.Background() - path := filepath.Join(t.TempDir(), "mediainfo.db") + path := filepath.Join(t.TempDir(), "plex-gateway.db") now := time.Unix(1_800_000_000, 0).UTC() record := completeRecord(now) - store, err := OpenSQLite(ctx, path) - if err != nil { - t.Fatal(err) - } + store, gatewayDB := openTestSQLiteStore(t, path) if err := store.Put(ctx, record); err != nil { t.Fatal(err) } - if err := store.Close(); err != nil { + if err := gatewayDB.Close(); err != nil { t.Fatal(err) } - store, err = OpenSQLite(ctx, path) - if err != nil { - t.Fatal(err) - } - defer store.Close() + store, gatewayDB = openTestSQLiteStore(t, path) + defer gatewayDB.Close() loaded, err := store.LoadCompatibleRetained(ctx, now, 100, testProviderDescriptor()) if err != nil { t.Fatal(err) @@ -43,45 +50,50 @@ func TestSQLiteStorePersistsRetainedRecordAcrossReopen(t *testing.T) { } func TestSQLiteStoreRecordsSchemaVersion(t *testing.T) { - store, err := OpenSQLite(t.Context(), filepath.Join(t.TempDir(), "mediainfo.db")) - if err != nil { - t.Fatal(err) - } - defer store.Close() + store, gatewayDB := openTestSQLiteStore(t, filepath.Join(t.TempDir(), "plex-gateway.db")) + defer gatewayDB.Close() var version int - if err := store.db.QueryRowContext(t.Context(), "PRAGMA user_version").Scan(&version); err != nil { + if err := store.db.QueryRowContext(t.Context(), ` + SELECT MAX(version) FROM gateway_schema_migrations WHERE module = ? + `, mediaInfoMigrationModule).Scan(&version); err != nil { t.Fatal(err) } - if version != sqliteDatabaseVersion { + if version != 1 { t.Fatalf("schema version = %d", version) } } func TestSQLiteStoreRejectsNewerSchema(t *testing.T) { - path := filepath.Join(t.TempDir(), "mediainfo.db") - database, err := sql.Open("sqlite", path) + path := filepath.Join(t.TempDir(), "plex-gateway.db") + gatewayDB, err := database.OpenSQLite(t.Context(), path) if err != nil { t.Fatal(err) } - if _, err := database.ExecContext(t.Context(), "PRAGMA user_version = 999"); err != nil { + if _, err := gatewayDB.SQLDB().ExecContext(t.Context(), ` + INSERT INTO gateway_schema_migrations (module, version, applied_at_ms) + VALUES (?, 1, ?), (?, 2, ?) + `, mediaInfoMigrationModule, time.Now().UTC().UnixMilli(), + mediaInfoMigrationModule, time.Now().UTC().UnixMilli()); err != nil { t.Fatal(err) } - if err := database.Close(); err != nil { + if err := gatewayDB.Close(); err != nil { t.Fatal(err) } - _, err = OpenSQLite(t.Context(), path) + gatewayDB, err = database.OpenSQLite(t.Context(), path) + if err != nil { + t.Fatal(err) + } + defer gatewayDB.Close() + _, err = NewSQLiteStore(t.Context(), gatewayDB) if err == nil || !strings.Contains(err.Error(), "newer than supported") { - t.Fatalf("OpenSQLite() error = %v", err) + t.Fatalf("NewSQLiteStore() error = %v", err) } } func TestSQLiteStoreRejectsLateOlderProbe(t *testing.T) { ctx := context.Background() - store, err := OpenSQLite(ctx, filepath.Join(t.TempDir(), "mediainfo.db")) - if err != nil { - t.Fatal(err) - } - defer store.Close() + store, gatewayDB := openTestSQLiteStore(t, filepath.Join(t.TempDir(), "plex-gateway.db")) + defer gatewayDB.Close() now := time.Unix(1_800_000_000, 0).UTC() newer := completeRecord(now) @@ -105,11 +117,8 @@ func TestSQLiteStoreRejectsLateOlderProbe(t *testing.T) { func TestSQLiteStoreDeletesUnretainedWithoutAffectingStaleOrFresh(t *testing.T) { ctx := context.Background() - store, err := OpenSQLite(ctx, filepath.Join(t.TempDir(), "mediainfo.db")) - if err != nil { - t.Fatal(err) - } - defer store.Close() + store, gatewayDB := openTestSQLiteStore(t, filepath.Join(t.TempDir(), "plex-gateway.db")) + defer gatewayDB.Close() now := time.Unix(1_800_000_000, 0).UTC() unretained := completeRecord(now.Add(-200 * 24 * time.Hour)) @@ -141,11 +150,8 @@ func TestSQLiteStoreDeletesUnretainedWithoutAffectingStaleOrFresh(t *testing.T) } func TestSQLiteStoreLoadsMostRecentlyAccessedWithinLimit(t *testing.T) { - store, err := OpenSQLite(t.Context(), filepath.Join(t.TempDir(), "mediainfo.db")) - if err != nil { - t.Fatal(err) - } - defer store.Close() + store, gatewayDB := openTestSQLiteStore(t, filepath.Join(t.TempDir(), "plex-gateway.db")) + defer gatewayDB.Close() now := time.Unix(1_800_000_000, 0).UTC() for index, age := range []time.Duration{3 * time.Hour, 2 * time.Hour, time.Hour} { record := completeRecord(now.Add(-age)) @@ -168,11 +174,8 @@ func testProviderDescriptor() ProviderDescriptor { } func TestSQLiteStoreDoesNotRestoreOtherProviderRevision(t *testing.T) { - store, err := OpenSQLite(t.Context(), filepath.Join(t.TempDir(), "mediainfo.db")) - if err != nil { - t.Fatal(err) - } - defer store.Close() + store, gatewayDB := openTestSQLiteStore(t, filepath.Join(t.TempDir(), "plex-gateway.db")) + defer gatewayDB.Close() now := time.Unix(1_800_000_000, 0).UTC() if err := store.Put(t.Context(), completeRecord(now)); err != nil { t.Fatal(err) @@ -189,11 +192,8 @@ func TestSQLiteStoreDoesNotRestoreOtherProviderRevision(t *testing.T) { } func TestSQLiteStoreTouchExtendsRetentionMonotonically(t *testing.T) { - store, err := OpenSQLite(t.Context(), filepath.Join(t.TempDir(), "mediainfo.db")) - if err != nil { - t.Fatal(err) - } - defer store.Close() + store, gatewayDB := openTestSQLiteStore(t, filepath.Join(t.TempDir(), "plex-gateway.db")) + defer gatewayDB.Close() now := time.Unix(1_800_000_000, 0).UTC() record := completeRecord(now) if err := store.Put(t.Context(), record); err != nil { diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 7ad54df..ec44d39 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -37,6 +37,10 @@ type Metrics struct { mediaInfoProbeFailure atomic.Uint64 mediaInfoStoreFailure atomic.Uint64 mediaInfoProbeActive atomic.Int64 + mediaInfoEnriched atomic.Uint64 + mediaInfoFailOpen atomic.Uint64 + mediaInfoWaitActive atomic.Int64 + mediaInfoWaitRejected atomic.Uint64 resolverLatency latencyMetrics redirectLatency latencyMetrics mediaInfoProbeLatency latencyMetrics @@ -75,6 +79,10 @@ type Snapshot struct { MediaInfoProbeFailureTotal uint64 `json:"mediainfo_probe_failure_total"` MediaInfoStoreFailureTotal uint64 `json:"mediainfo_store_failure_total"` MediaInfoProbeActive int64 `json:"mediainfo_probe_active"` + MediaInfoEnrichedTotal uint64 `json:"mediainfo_enriched_total"` + MediaInfoFailOpenTotal uint64 `json:"mediainfo_fail_open_total"` + MediaInfoWaitActive int64 `json:"mediainfo_wait_active"` + MediaInfoWaitRejectedTotal uint64 `json:"mediainfo_wait_rejected_total"` ResolverLatencyMSTotal uint64 `json:"resolver_latency_ms_total"` ResolverLatencySamples uint64 `json:"resolver_latency_samples"` @@ -157,6 +165,25 @@ func (m *Metrics) ObserveMediaInfoProbeLatency(duration time.Duration) { m.mediaInfoProbeLatency.observe(duration) } +// IncMediaInfoEnriched records a Plex metadata response that gained technical +// fields from an identity-matched MediaInfo record. +func (m *Metrics) IncMediaInfoEnriched() { m.mediaInfoEnriched.Add(1) } + +// IncMediaInfoFailOpen records an eligible response returned unchanged after +// enrichment could not complete safely. +func (m *Metrics) IncMediaInfoFailOpen() { m.mediaInfoFailOpen.Add(1) } + +// IncMediaInfoWaitActive marks one buffered metadata response waiting for an +// identity-matched record. +func (m *Metrics) IncMediaInfoWaitActive() { m.mediaInfoWaitActive.Add(1) } + +// DecMediaInfoWaitActive releases one metadata enrichment waiter. +func (m *Metrics) DecMediaInfoWaitActive() { decrementGauge(&m.mediaInfoWaitActive) } + +// IncMediaInfoWaitRejected records an eligible response that skipped waiting +// because the bounded enrichment pool was full. +func (m *Metrics) IncMediaInfoWaitRejected() { m.mediaInfoWaitRejected.Add(1) } + // IncMetadataGuardAdmitted records a detailed metadata request admitted to // Plex after both client and global limits were acquired. func (m *Metrics) IncMetadataGuardAdmitted() { @@ -313,6 +340,10 @@ func (m *Metrics) Snapshot() Snapshot { MediaInfoProbeFailureTotal: m.mediaInfoProbeFailure.Load(), MediaInfoStoreFailureTotal: m.mediaInfoStoreFailure.Load(), MediaInfoProbeActive: m.mediaInfoProbeActive.Load(), + MediaInfoEnrichedTotal: m.mediaInfoEnriched.Load(), + MediaInfoFailOpenTotal: m.mediaInfoFailOpen.Load(), + MediaInfoWaitActive: m.mediaInfoWaitActive.Load(), + MediaInfoWaitRejectedTotal: m.mediaInfoWaitRejected.Load(), ResolverLatencyMSTotal: resolverTotal, ResolverLatencySamples: resolverSamples, diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 1fd5b4f..b3365ba 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -34,6 +34,10 @@ func TestSnapshotAndJSONHandler(t *testing.T) { registry.IncMediaInfoProbeFailure() registry.IncMediaInfoStoreFailure() registry.IncMediaInfoProbeActive() + registry.IncMediaInfoEnriched() + registry.IncMediaInfoFailOpen() + registry.IncMediaInfoWaitActive() + registry.IncMediaInfoWaitRejected() registry.ObserveMediaInfoProbeLatency(44 * time.Millisecond) registry.ObserveResolverLatency(12 * time.Millisecond) registry.ObserveResolverLatency(7 * time.Millisecond) @@ -78,6 +82,10 @@ func TestSnapshotAndJSONHandler(t *testing.T) { MediaInfoProbeFailureTotal: 1, MediaInfoStoreFailureTotal: 1, MediaInfoProbeActive: 1, + MediaInfoEnrichedTotal: 1, + MediaInfoFailOpenTotal: 1, + MediaInfoWaitActive: 1, + MediaInfoWaitRejectedTotal: 1, ResolverLatencyMSTotal: 19, ResolverLatencySamples: 2, @@ -103,6 +111,7 @@ func TestSnapshotAndJSONHandler(t *testing.T) { registry.DecMetadataBatchGuardActive() registry.DecMetadataBatchGuardQueued() registry.DecMediaInfoProbeActive() + registry.DecMediaInfoWaitActive() if got := registry.Snapshot().ActiveRequests; got != 0 { t.Fatalf("active requests after release = %d, want 0", got) } @@ -155,6 +164,7 @@ func TestActiveRequestReleaseIsIdempotentAndClamped(t *testing.T) { registry.DecMetadataBatchGuardActive() registry.DecMetadataBatchGuardQueued() registry.DecMediaInfoProbeActive() + registry.DecMediaInfoWaitActive() if got := registry.Snapshot().ActiveRequests; got != 0 { t.Fatalf("clamped active requests = %d, want 0", got) } @@ -202,6 +212,11 @@ func TestCountersAreSafeForConcurrentUpdates(t *testing.T) { registry.IncMediaInfoStoreFailure() registry.IncMediaInfoProbeActive() registry.DecMediaInfoProbeActive() + registry.IncMediaInfoEnriched() + registry.IncMediaInfoFailOpen() + registry.IncMediaInfoWaitActive() + registry.DecMediaInfoWaitActive() + registry.IncMediaInfoWaitRejected() registry.ObserveMediaInfoProbeLatency(3 * time.Millisecond) registry.ObserveResolverLatency(time.Millisecond) registry.ObserveRedirectLatency(2 * time.Millisecond) @@ -222,6 +237,7 @@ func TestCountersAreSafeForConcurrentUpdates(t *testing.T) { got.MediaInfoCacheHitsTotal != want || got.MediaInfoCacheMissesTotal != want || got.MediaInfoProbeQueuedTotal != want || got.MediaInfoProbeSuccessTotal != want || got.MediaInfoProbeFailureTotal != want || got.MediaInfoStoreFailureTotal != want || got.MediaInfoProbeActive != 0 || + got.MediaInfoEnrichedTotal != want || got.MediaInfoFailOpenTotal != want || got.MediaInfoWaitActive != 0 || got.MediaInfoWaitRejectedTotal != want || got.ResolverLatencySamples != want || got.ResolverLatencyMSTotal != want || got.RedirectLatencySamples != want || got.RedirectLatencyMSTotal != 2*want || got.MediaInfoProbeLatencySamples != want || got.MediaInfoProbeLatencyMSTotal != 3*want { diff --git a/internal/plexmeta/projection.go b/internal/plexmeta/projection.go new file mode 100644 index 0000000..55b4e8a --- /dev/null +++ b/internal/plexmeta/projection.go @@ -0,0 +1,1236 @@ +package plexmeta + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "strconv" + "strings" + + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" +) + +// Target identifies the single Plex item and reports whether the response is +// missing a whitelisted technical field that a MediaInfo record can project. +// A missing Stream is not itself a reason to enrich: projection never creates +// Stream elements because their Plex identity is not known to the gateway. +type Target struct { + Part Part + // NeedsEnrichment is true when a whitelisted field is absent from the + // selected Media, Part, or matchable existing Stream elements. + NeedsEnrichment bool +} + +// SelectEnrichmentTarget strictly selects one ratingKey item and its only +// Media/Part pair. It performs no response mutation and is suitable for +// deciding whether a probe is needed before loading MediaInfo. +func SelectEnrichmentTarget(body []byte, contentType, ratingKey string) (Target, error) { + _, selection, err := parseProjection(body, contentType, ratingKey) + if err != nil { + return Target{}, err + } + return selection.target(), nil +} + +// SelectEnrichmentPart is the strict Part-only form of +// SelectEnrichmentTarget. The response must contain exactly one requested +// ratingKey item, one Media, and one Part. +func SelectEnrichmentPart(body []byte, contentType, ratingKey string) (Part, error) { + target, err := SelectEnrichmentTarget(body, contentType, ratingKey) + if err != nil { + return Part{}, err + } + return target.Part, nil +} + +// EnrichMetadata fills only absent whitelisted Media, Part, and already +// present Stream attributes. SelectEnrichmentTarget supplies the exact Part +// to callers that need to build a probe request before this function runs. +// The input body is never modified in place. When no field is filled, the +// original bytes are returned unchanged. +func EnrichMetadata(body []byte, contentType, ratingKey string, media mediainfo.Media) ([]byte, bool, error) { + document, selection, err := parseProjection(body, contentType, ratingKey) + if err != nil { + return nil, false, err + } + changed, err := selection.enrich(media) + if err != nil { + return nil, false, err + } + if !changed { + return append([]byte(nil), body...), false, nil + } + result, err := document.encode() + if err != nil { + return nil, false, err + } + return result, true, nil +} + +type projectionDocument interface { + selectTarget(string) (projectionSelection, error) + encode() ([]byte, error) +} + +type projectionSelection interface { + target() Target + enrich(mediainfo.Media) (bool, error) +} + +func parseProjection(body []byte, contentType, ratingKey string) (projectionDocument, projectionSelection, error) { + ratingKey, err := normalizeProjectionRatingKey(ratingKey) + if err != nil { + return nil, nil, err + } + trimmed := trimProjectionBody(body) + if len(trimmed) == 0 { + return nil, nil, errors.New("Plex metadata response is empty") + } + if strings.Contains(strings.ToLower(contentType), "json") || trimmed[0] == '{' || trimmed[0] == '[' { + document, err := parseJSONProjectionDocument(body) + if err != nil { + return nil, nil, err + } + selection, err := document.selectTarget(ratingKey) + if err != nil { + return nil, nil, err + } + return document, selection, nil + } + document, err := parseXMLProjectionDocument(body) + if err != nil { + return nil, nil, err + } + selection, err := document.selectTarget(ratingKey) + if err != nil { + return nil, nil, err + } + return document, selection, nil +} + +func normalizeProjectionRatingKey(value string) (string, error) { + value = strings.TrimSpace(value) + if value == "" || strings.ContainsAny(value, "/?#\x00\r\n") { + return "", errors.New("rating key must be a single path segment") + } + return value, nil +} + +func trimProjectionBody(body []byte) []byte { + trimmed := bytes.TrimSpace(body) + trimmed = bytes.TrimPrefix(trimmed, []byte{0xef, 0xbb, 0xbf}) + return bytes.TrimSpace(trimmed) +} + +// projectionAttribute is deliberately narrower than mediainfo.Media. The +// whitelist contains only descriptive technical Plex fields; identifiers and +// playback-selection fields are never generated. +type projectionAttribute struct { + name string + value string + number bool +} + +func textProjectionAttribute(name, value string) projectionAttribute { + return projectionAttribute{name: name, value: value} +} + +func integerProjectionAttribute(name string, value int64) projectionAttribute { + return projectionAttribute{name: name, value: strconv.FormatInt(value, 10), number: true} +} + +func mediaProjectionAttributes(media mediainfo.Media) []projectionAttribute { + attributes := make([]projectionAttribute, 0, 14) + if media.Container != "" { + attributes = append(attributes, textProjectionAttribute("container", media.Container)) + } + if media.DurationMS > 0 { + attributes = append(attributes, integerProjectionAttribute("duration", media.DurationMS)) + } + if media.Bitrate > 0 { + attributes = append(attributes, integerProjectionAttribute("bitrate", media.Bitrate)) + } + if media.Width > 0 { + attributes = append(attributes, integerProjectionAttribute("width", int64(media.Width))) + } + if media.Height > 0 { + attributes = append(attributes, integerProjectionAttribute("height", int64(media.Height))) + } + if media.AspectRatio != "" { + attributes = append(attributes, textProjectionAttribute("aspectRatio", media.AspectRatio)) + } + if media.AudioChannels > 0 { + attributes = append(attributes, integerProjectionAttribute("audioChannels", int64(media.AudioChannels))) + } + if media.AudioCodec != "" { + attributes = append(attributes, textProjectionAttribute("audioCodec", media.AudioCodec)) + } + if media.VideoCodec != "" { + attributes = append(attributes, textProjectionAttribute("videoCodec", media.VideoCodec)) + } + if media.VideoResolution != "" { + attributes = append(attributes, textProjectionAttribute("videoResolution", media.VideoResolution)) + } + if media.FrameRate != "" { + attributes = append(attributes, textProjectionAttribute("videoFrameRate", media.FrameRate)) + } + if media.VideoProfile != "" { + attributes = append(attributes, textProjectionAttribute("videoProfile", media.VideoProfile)) + } + if media.AudioProfile != "" { + attributes = append(attributes, textProjectionAttribute("audioProfile", media.AudioProfile)) + } + return attributes +} + +func partProjectionAttributes(media mediainfo.Media) []projectionAttribute { + attributes := make([]projectionAttribute, 0, 5) + if media.DurationMS > 0 { + attributes = append(attributes, integerProjectionAttribute("duration", media.DurationMS)) + } + if media.Size > 0 { + attributes = append(attributes, integerProjectionAttribute("size", media.Size)) + } + if media.Container != "" { + attributes = append(attributes, textProjectionAttribute("container", media.Container)) + } + if media.VideoProfile != "" { + attributes = append(attributes, textProjectionAttribute("videoProfile", media.VideoProfile)) + } + if media.AudioProfile != "" { + attributes = append(attributes, textProjectionAttribute("audioProfile", media.AudioProfile)) + } + return attributes +} + +func streamProjectionAttributes(stream mediainfo.Stream) []projectionAttribute { + attributes := make([]projectionAttribute, 0, 20) + if stream.Codec != "" { + attributes = append(attributes, textProjectionAttribute("codec", stream.Codec)) + } + if stream.Profile != "" { + attributes = append(attributes, textProjectionAttribute("profile", stream.Profile)) + } + if stream.Level > 0 { + attributes = append(attributes, integerProjectionAttribute("level", int64(stream.Level))) + } + if stream.Bitrate > 0 { + attributes = append(attributes, integerProjectionAttribute("bitrate", stream.Bitrate)) + } + if stream.Width > 0 { + attributes = append(attributes, integerProjectionAttribute("width", int64(stream.Width))) + } + if stream.Height > 0 { + attributes = append(attributes, integerProjectionAttribute("height", int64(stream.Height))) + } + frameRate := firstProjectionString(stream.FrameRate, stream.AverageFrameRate) + if frameRate != "" { + attributes = append(attributes, textProjectionAttribute("frameRate", frameRate)) + } + if stream.ReferenceFrames > 0 { + attributes = append(attributes, integerProjectionAttribute("refFrames", int64(stream.ReferenceFrames))) + } + if stream.PixelFormat != "" { + attributes = append(attributes, textProjectionAttribute("pixelFormat", stream.PixelFormat)) + } + if stream.BitDepth > 0 { + attributes = append(attributes, integerProjectionAttribute("bitDepth", int64(stream.BitDepth))) + } + if stream.ColorSpace != "" { + attributes = append(attributes, textProjectionAttribute("colorSpace", stream.ColorSpace)) + } + if stream.ColorRange != "" { + attributes = append(attributes, textProjectionAttribute("colorRange", stream.ColorRange)) + } + if stream.ColorPrimaries != "" { + attributes = append(attributes, textProjectionAttribute("colorPrimaries", stream.ColorPrimaries)) + } + if stream.ColorTransfer != "" { + attributes = append(attributes, textProjectionAttribute("colorTrc", stream.ColorTransfer)) + } + if stream.ChromaLocation != "" { + attributes = append(attributes, textProjectionAttribute("chromaLocation", stream.ChromaLocation)) + } + if stream.SampleAspectRatio != "" { + attributes = append(attributes, textProjectionAttribute("sampleAspectRatio", stream.SampleAspectRatio)) + } + if stream.DisplayAspectRatio != "" { + attributes = append(attributes, textProjectionAttribute("displayAspectRatio", stream.DisplayAspectRatio)) + } + if stream.SampleRate > 0 { + attributes = append(attributes, integerProjectionAttribute("samplingRate", int64(stream.SampleRate))) + } + if stream.Channels > 0 { + attributes = append(attributes, integerProjectionAttribute("channels", int64(stream.Channels))) + } + if stream.ChannelLayout != "" { + attributes = append(attributes, textProjectionAttribute("channelLayout", stream.ChannelLayout)) + } + if stream.Language != "" { + attributes = append(attributes, textProjectionAttribute("language", stream.Language)) + } + if stream.Title != "" { + attributes = append(attributes, textProjectionAttribute("title", stream.Title)) + } + return attributes +} + +func firstProjectionString(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} + +var mediaProjectionAttributeNames = []string{ + "container", "duration", "bitrate", "width", "height", "aspectRatio", + "audioChannels", "audioCodec", "videoCodec", "videoResolution", "videoFrameRate", + "videoProfile", "audioProfile", +} + +var partProjectionAttributeNames = []string{"duration", "size", "container", "videoProfile", "audioProfile"} + +var commonStreamProjectionAttributeNames = []string{ + "codec", "profile", "level", "bitrate", "language", "title", +} + +var videoStreamProjectionAttributeNames = []string{ + "width", "height", "frameRate", "refFrames", "pixelFormat", "bitDepth", + "colorSpace", "colorRange", "colorPrimaries", "colorTrc", "chromaLocation", + "sampleAspectRatio", "displayAspectRatio", +} + +var audioStreamProjectionAttributeNames = []string{"samplingRate", "channels", "channelLayout"} + +type streamKey struct { + typeName string + index int +} + +type streamProjection struct { + key streamKey + attributes []projectionAttribute +} + +func buildStreamProjections(media mediainfo.Media) ([]streamProjection, error) { + projections := make([]streamProjection, 0, len(media.Streams)) + seen := make(map[streamKey]struct{}, len(media.Streams)) + for _, stream := range media.Streams { + key, ok := mediaStreamKey(stream.Type, stream.Index) + if !ok { + continue + } + if _, exists := seen[key]; exists { + return nil, fmt.Errorf("MediaInfo contains duplicate stream identity %s/%d", key.typeName, key.index) + } + seen[key] = struct{}{} + projections = append(projections, streamProjection{ + key: key, + attributes: streamProjectionAttributes(stream), + }) + } + return projections, nil +} + +func mediaStreamKey(typeName string, index int) (streamKey, bool) { + if index < 0 { + return streamKey{}, false + } + typeName = canonicalStreamType(typeName) + if typeName == "" { + return streamKey{}, false + } + return streamKey{typeName: typeName, index: index}, true +} + +func canonicalStreamType(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "1", "video": + return "1" + case "2", "audio": + return "2" + case "3", "subtitle", "subtitles": + return "3" + default: + return strings.ToLower(strings.TrimSpace(value)) + } +} + +func streamProjectionAttributeNames(typeName string) []string { + names := append([]string(nil), commonStreamProjectionAttributeNames...) + switch typeName { + case "1": + names = append(names, videoStreamProjectionAttributeNames...) + case "2": + names = append(names, audioStreamProjectionAttributeNames...) + } + return names +} + +// XML projection ----------------------------------------------------------- + +type xmlProjectionDocument struct { + nodes []*xmlProjectionNode +} + +type xmlProjectionNode struct { + element *xmlProjectionElement + token xml.Token +} + +type xmlProjectionElement struct { + start xml.StartElement + children []*xmlProjectionNode +} + +func parseXMLProjectionDocument(body []byte) (*xmlProjectionDocument, error) { + decoder := xml.NewDecoder(bytes.NewReader(bytes.TrimPrefix(body, []byte{0xef, 0xbb, 0xbf}))) + document := &xmlProjectionDocument{} + stack := make([]*xmlProjectionElement, 0, 8) + for { + token, err := decoder.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("parse Plex XML metadata: %w", err) + } + switch typed := token.(type) { + case xml.StartElement: + element := &xmlProjectionElement{start: copyXMLStartElement(typed)} + node := &xmlProjectionNode{element: element} + appendXMLProjectionNode(document, stack, node) + stack = append(stack, element) + case xml.EndElement: + if len(stack) == 0 { + return nil, errors.New("Plex XML metadata has an unexpected closing element") + } + stack = stack[:len(stack)-1] + case xml.CharData: + data := append(xml.CharData(nil), typed...) + if len(stack) == 0 && len(bytes.TrimSpace(data)) != 0 { + return nil, errors.New("Plex XML metadata has non-whitespace text outside the root") + } + appendXMLProjectionNode(document, stack, &xmlProjectionNode{token: data}) + case xml.Comment: + appendXMLProjectionNode(document, stack, &xmlProjectionNode{token: append(xml.Comment(nil), typed...)}) + case xml.Directive: + appendXMLProjectionNode(document, stack, &xmlProjectionNode{token: append(xml.Directive(nil), typed...)}) + case xml.ProcInst: + appendXMLProjectionNode(document, stack, &xmlProjectionNode{token: xml.ProcInst{ + Target: typed.Target, + Inst: append([]byte(nil), typed.Inst...), + }}) + } + } + if len(stack) != 0 { + return nil, errors.New("Plex XML metadata has an unterminated element") + } + if len(xmlProjectionElements(document.nodes)) != 1 { + return nil, errors.New("Plex XML metadata must have one root element") + } + return document, nil +} + +func copyXMLStartElement(start xml.StartElement) xml.StartElement { + attributes := make([]xml.Attr, len(start.Attr)) + copy(attributes, start.Attr) + return xml.StartElement{Name: start.Name, Attr: attributes} +} + +func appendXMLProjectionNode(document *xmlProjectionDocument, stack []*xmlProjectionElement, node *xmlProjectionNode) { + if len(stack) == 0 { + document.nodes = append(document.nodes, node) + return + } + parent := stack[len(stack)-1] + parent.children = append(parent.children, node) +} + +func xmlProjectionElements(nodes []*xmlProjectionNode) []*xmlProjectionElement { + result := make([]*xmlProjectionElement, 0, len(nodes)) + for _, node := range nodes { + if node.element != nil { + result = append(result, node.element) + } + } + return result +} + +func directXMLElements(parent *xmlProjectionElement, localName string) []*xmlProjectionElement { + result := make([]*xmlProjectionElement, 0) + for _, node := range parent.children { + if node.element != nil && node.element.start.Name.Local == localName { + result = append(result, node.element) + } + } + return result +} + +func (document *xmlProjectionDocument) selectTarget(ratingKey string) (projectionSelection, error) { + roots := xmlProjectionElements(document.nodes) + if len(roots) != 1 || roots[0].start.Name.Local != "MediaContainer" { + return nil, errors.New("Plex XML metadata has no unique MediaContainer root") + } + videos := directXMLElements(roots[0], "Video") + if len(videos) != 1 { + return nil, errors.New("Plex XML metadata must contain one Video item") + } + item := videos[0] + itemRatingKey, present, err := xmlAttribute(item.start.Attr, "ratingKey") + if err != nil { + return nil, err + } + if !present || itemRatingKey == "" || itemRatingKey != ratingKey { + return nil, errors.New("Plex XML metadata ratingKey does not match the requested item") + } + mediaItems := directXMLElements(item, "Media") + if len(mediaItems) != 1 { + return nil, errors.New("Plex XML metadata must contain one Media") + } + parts := directXMLElements(mediaItems[0], "Part") + if len(parts) != 1 { + return nil, errors.New("Plex XML metadata must contain one Part") + } + part, err := partFromXML(parts[0]) + if err != nil { + return nil, err + } + needs, err := xmlTargetNeedsEnrichment(mediaItems[0], parts[0]) + if err != nil { + return nil, err + } + return &xmlProjectionSelection{ + value: Target{Part: part, NeedsEnrichment: needs}, + media: mediaItems[0], + part: parts[0], + }, nil +} + +type xmlProjectionSelection struct { + value Target + media *xmlProjectionElement + part *xmlProjectionElement +} + +func (selection *xmlProjectionSelection) target() Target { + return selection.value +} + +func (selection *xmlProjectionSelection) enrich(media mediainfo.Media) (bool, error) { + streamProjections, err := buildStreamProjections(media) + if err != nil { + return false, err + } + changed := addMissingXMLAttributes(selection.media, mediaProjectionAttributes(media)) + changed = addMissingXMLAttributes(selection.part, partProjectionAttributes(media)) || changed + streamElements := directXMLElements(selection.part, "Stream") + streamIndex, err := indexXMLStreams(streamElements) + if err != nil { + return false, err + } + for _, projection := range streamProjections { + stream, ok := streamIndex[projection.key] + if !ok { + continue + } + changed = addMissingXMLAttributes(stream, projection.attributes) || changed + } + return changed, nil +} + +func partFromXML(element *xmlProjectionElement) (Part, error) { + id, present, err := xmlAttribute(element.start.Attr, "id") + if err != nil { + return Part{}, err + } + if !present || id == "" { + return Part{}, errors.New("selected Plex Part has no id") + } + key, _, err := xmlAttribute(element.start.Attr, "key") + if err != nil { + return Part{}, err + } + file, present, err := xmlAttribute(element.start.Attr, "file") + if err != nil { + return Part{}, err + } + if !present || file == "" { + return Part{}, errors.New("selected Plex Part has no file") + } + return Part{ID: id, Key: key, File: file}, nil +} + +func xmlAttribute(attributes []xml.Attr, localName string) (string, bool, error) { + value := "" + present := false + for _, attribute := range attributes { + if attribute.Name.Local != localName { + continue + } + if present { + return "", false, fmt.Errorf("Plex XML metadata has duplicate %s attributes", localName) + } + value = attribute.Value + present = true + } + return value, present, nil +} + +func xmlTargetNeedsEnrichment(media, part *xmlProjectionElement) (bool, error) { + needs := false + for _, name := range mediaProjectionAttributeNames { + _, present, err := xmlAttribute(media.start.Attr, name) + if err != nil { + return false, err + } + if !present { + needs = true + } + } + for _, name := range partProjectionAttributeNames { + _, present, err := xmlAttribute(part.start.Attr, name) + if err != nil { + return false, err + } + if !present { + needs = true + } + } + streams := directXMLElements(part, "Stream") + if _, err := indexXMLStreams(streams); err != nil { + return false, err + } + for _, stream := range streams { + key, present, err := xmlStreamIdentity(stream) + if err != nil { + return false, err + } + if !present { + continue + } + for _, name := range streamProjectionAttributeNames(key.typeName) { + _, present, err := xmlAttribute(stream.start.Attr, name) + if err != nil { + return false, err + } + if !present { + needs = true + } + } + } + return needs, nil +} + +func indexXMLStreams(streams []*xmlProjectionElement) (map[streamKey]*xmlProjectionElement, error) { + result := make(map[streamKey]*xmlProjectionElement, len(streams)) + for _, stream := range streams { + key, present, err := xmlStreamIdentity(stream) + if err != nil { + return nil, err + } + if !present { + continue + } + if _, exists := result[key]; exists { + return nil, fmt.Errorf("Plex metadata has duplicate Stream identity %s/%d", key.typeName, key.index) + } + result[key] = stream + } + return result, nil +} + +func xmlStreamIdentity(stream *xmlProjectionElement) (streamKey, bool, error) { + typeName, typePresent, err := xmlAttribute(stream.start.Attr, "streamType") + if err != nil { + return streamKey{}, false, err + } + indexValue, indexPresent, err := xmlAttribute(stream.start.Attr, "index") + if err != nil { + return streamKey{}, false, err + } + if !typePresent || !indexPresent { + return streamKey{}, false, nil + } + if strings.TrimSpace(typeName) == "" { + return streamKey{}, false, errors.New("Plex metadata Stream has an empty streamType") + } + index, err := strconv.Atoi(strings.TrimSpace(indexValue)) + if err != nil || index < 0 { + return streamKey{}, false, errors.New("Plex metadata Stream has an invalid index") + } + key, ok := mediaStreamKey(typeName, index) + if !ok { + return streamKey{}, false, errors.New("Plex metadata Stream has an invalid identity") + } + return key, true, nil +} + +func addMissingXMLAttributes(element *xmlProjectionElement, attributes []projectionAttribute) bool { + changed := false + for _, attribute := range attributes { + _, present, err := xmlAttribute(element.start.Attr, attribute.name) + if err != nil { + // The target validation inspects all whitelisted attributes before + // mutation. This branch is unreachable for a valid selection. + continue + } + if present { + continue + } + element.start.Attr = append(element.start.Attr, xml.Attr{ + Name: xml.Name{Local: attribute.name}, + Value: attribute.value, + }) + changed = true + } + return changed +} + +func (document *xmlProjectionDocument) encode() ([]byte, error) { + var buffer bytes.Buffer + encoder := xml.NewEncoder(&buffer) + for _, node := range document.nodes { + if node.element != nil { + if err := encodeXMLProjectionElement(encoder, node.element); err != nil { + return nil, fmt.Errorf("encode Plex XML metadata: %w", err) + } + continue + } + if err := encoder.EncodeToken(node.token); err != nil { + return nil, fmt.Errorf("encode Plex XML metadata: %w", err) + } + } + if err := encoder.Flush(); err != nil { + return nil, fmt.Errorf("flush Plex XML metadata: %w", err) + } + return buffer.Bytes(), nil +} + +func encodeXMLProjectionElement(encoder *xml.Encoder, element *xmlProjectionElement) error { + if err := encoder.EncodeToken(element.start); err != nil { + return err + } + for _, node := range element.children { + if node.element != nil { + if err := encodeXMLProjectionElement(encoder, node.element); err != nil { + return err + } + continue + } + if err := encoder.EncodeToken(node.token); err != nil { + return err + } + } + return encoder.EncodeToken(element.start.End()) +} + +// JSON projection ---------------------------------------------------------- + +type jsonProjectionDocument struct { + root jsonObject + prefix []byte + suffix []byte +} + +type jsonObject map[string]json.RawMessage + +func parseJSONProjectionDocument(body []byte) (*jsonProjectionDocument, error) { + core, prefix, suffix, err := splitJSONProjectionBody(body) + if err != nil { + return nil, err + } + root, err := decodeJSONProjectionObject(core) + if err != nil { + return nil, fmt.Errorf("parse Plex JSON metadata: %w", err) + } + return &jsonProjectionDocument{root: root, prefix: prefix, suffix: suffix}, nil +} + +func splitJSONProjectionBody(body []byte) ([]byte, []byte, []byte, error) { + leading := len(body) - len(bytes.TrimLeft(body, " \t\r\n")) + start := leading + if bytes.HasPrefix(body[start:], []byte{0xef, 0xbb, 0xbf}) { + start += 3 + } + withoutBOM := body[start:] + left := len(withoutBOM) - len(bytes.TrimLeft(withoutBOM, " \t\r\n")) + coreWithSuffix := withoutBOM[left:] + right := len(coreWithSuffix) - len(bytes.TrimRight(coreWithSuffix, " \t\r\n")) + if len(coreWithSuffix)-right == 0 { + return nil, nil, nil, errors.New("Plex JSON metadata response is empty") + } + prefix := append([]byte(nil), body[:start+left]...) + suffix := append([]byte(nil), coreWithSuffix[len(coreWithSuffix)-right:]...) + return coreWithSuffix[:len(coreWithSuffix)-right], prefix, suffix, nil +} + +func decodeJSONProjectionObject(raw []byte) (jsonObject, error) { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + token, err := decoder.Token() + if err != nil { + return nil, err + } + if delimiter, ok := token.(json.Delim); !ok || delimiter != '{' { + return nil, errors.New("Plex JSON metadata value is not an object") + } + result := make(jsonObject) + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return nil, err + } + key, ok := keyToken.(string) + if !ok { + return nil, errors.New("Plex JSON metadata object has an invalid key") + } + if _, exists := result[key]; exists { + return nil, fmt.Errorf("Plex JSON metadata has duplicate %s fields", key) + } + var value json.RawMessage + if err := decoder.Decode(&value); err != nil { + return nil, err + } + result[key] = append(json.RawMessage(nil), value...) + } + if token, err := decoder.Token(); err != nil { + return nil, err + } else if delimiter, ok := token.(json.Delim); !ok || delimiter != '}' { + return nil, errors.New("Plex JSON metadata object is not closed") + } + if token, err := decoder.Token(); !errors.Is(err, io.EOF) || token != nil { + return nil, errors.New("Plex JSON metadata has trailing values") + } + return result, nil +} + +func decodeJSONProjectionArray(raw []byte) ([]json.RawMessage, error) { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + token, err := decoder.Token() + if err != nil { + return nil, err + } + if delimiter, ok := token.(json.Delim); !ok || delimiter != '[' { + return nil, errors.New("Plex JSON metadata value is not an array") + } + result := make([]json.RawMessage, 0) + for decoder.More() { + var value json.RawMessage + if err := decoder.Decode(&value); err != nil { + return nil, err + } + result = append(result, append(json.RawMessage(nil), value...)) + } + if token, err := decoder.Token(); err != nil { + return nil, err + } else if delimiter, ok := token.(json.Delim); !ok || delimiter != ']' { + return nil, errors.New("Plex JSON metadata array is not closed") + } + if token, err := decoder.Token(); !errors.Is(err, io.EOF) || token != nil { + return nil, errors.New("Plex JSON metadata has trailing values") + } + return result, nil +} + +type jsonProjectionList struct { + items []*jsonObject + array bool +} + +func decodeJSONProjectionList(raw json.RawMessage, name string) (jsonProjectionList, error) { + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 { + return jsonProjectionList{}, fmt.Errorf("Plex JSON metadata %s is empty", name) + } + if trimmed[0] == '{' { + object, err := decodeJSONProjectionObject(trimmed) + if err != nil { + return jsonProjectionList{}, fmt.Errorf("parse Plex JSON %s: %w", name, err) + } + return jsonProjectionList{items: []*jsonObject{&object}}, nil + } + if trimmed[0] != '[' { + return jsonProjectionList{}, fmt.Errorf("Plex JSON metadata %s is neither an object nor an array", name) + } + values, err := decodeJSONProjectionArray(trimmed) + if err != nil { + return jsonProjectionList{}, fmt.Errorf("parse Plex JSON %s: %w", name, err) + } + result := jsonProjectionList{items: make([]*jsonObject, 0, len(values)), array: true} + for _, value := range values { + object, err := decodeJSONProjectionObject(value) + if err != nil { + return jsonProjectionList{}, fmt.Errorf("Plex JSON metadata %s contains a non-object item: %w", name, err) + } + result.items = append(result.items, &object) + } + return result, nil +} + +func encodeJSONProjectionList(list jsonProjectionList) (json.RawMessage, error) { + if list.array { + values := make([]jsonObject, 0, len(list.items)) + for _, item := range list.items { + values = append(values, *item) + } + return json.Marshal(values) + } + if len(list.items) != 1 { + return nil, errors.New("Plex JSON metadata object list is not unique") + } + return json.Marshal(*list.items[0]) +} + +func (document *jsonProjectionDocument) selectTarget(ratingKey string) (projectionSelection, error) { + containerRaw, ok := document.root["MediaContainer"] + if !ok { + return nil, errors.New("Plex JSON metadata has no MediaContainer") + } + container, err := decodeJSONProjectionObject(bytes.TrimSpace(containerRaw)) + if err != nil { + return nil, fmt.Errorf("parse Plex JSON MediaContainer: %w", err) + } + metadataRaw, ok := container["Metadata"] + if !ok { + return nil, errors.New("Plex JSON metadata has no Metadata item") + } + metadata, err := decodeJSONProjectionList(metadataRaw, "Metadata") + if err != nil { + return nil, err + } + if len(metadata.items) != 1 { + return nil, errors.New("Plex JSON metadata must contain one Metadata item") + } + item := metadata.items[0] + itemRatingKey, err := requiredJSONScalarString(*item, "ratingKey") + if err != nil { + return nil, err + } + if itemRatingKey != ratingKey { + return nil, errors.New("Plex JSON metadata ratingKey does not match the requested item") + } + mediaRaw, ok := (*item)["Media"] + if !ok { + return nil, errors.New("Plex JSON metadata has no Media") + } + mediaItems, err := decodeJSONProjectionList(mediaRaw, "Media") + if err != nil { + return nil, err + } + if len(mediaItems.items) != 1 { + return nil, errors.New("Plex JSON metadata must contain one Media") + } + mediaItem := mediaItems.items[0] + partRaw, ok := (*mediaItem)["Part"] + if !ok { + return nil, errors.New("Plex JSON metadata has no Part") + } + parts, err := decodeJSONProjectionList(partRaw, "Part") + if err != nil { + return nil, err + } + if len(parts.items) != 1 { + return nil, errors.New("Plex JSON metadata must contain one Part") + } + partItem := parts.items[0] + part, err := partFromJSON(partItem) + if err != nil { + return nil, err + } + needs, err := jsonTargetNeedsEnrichment(mediaItem, partItem) + if err != nil { + return nil, err + } + return &jsonProjectionSelection{ + value: Target{Part: part, NeedsEnrichment: needs}, + document: document, + container: container, + metadata: metadata, + media: mediaItems, + parts: parts, + mediaItem: mediaItem, + part: partItem, + }, nil +} + +type jsonProjectionSelection struct { + value Target + document *jsonProjectionDocument + container jsonObject + metadata jsonProjectionList + media jsonProjectionList + parts jsonProjectionList + mediaItem *jsonObject + part *jsonObject +} + +func (selection *jsonProjectionSelection) target() Target { + return selection.value +} + +func (selection *jsonProjectionSelection) enrich(media mediainfo.Media) (bool, error) { + streamProjections, err := buildStreamProjections(media) + if err != nil { + return false, err + } + changed, err := addMissingJSONAttributes(*selection.mediaItem, mediaProjectionAttributes(media)) + if err != nil { + return false, err + } + partChanged, err := addMissingJSONAttributes(*selection.part, partProjectionAttributes(media)) + if err != nil { + return false, err + } + changed = changed || partChanged + streamRaw, exists := (*selection.part)["Stream"] + if !exists { + return changed, nil + } + streams, err := decodeJSONProjectionList(streamRaw, "Stream") + if err != nil { + return false, err + } + streamIndex, err := indexJSONStreams(streams.items) + if err != nil { + return false, err + } + for _, projection := range streamProjections { + stream, ok := streamIndex[projection.key] + if !ok { + continue + } + streamChanged, err := addMissingJSONAttributes(*stream, projection.attributes) + if err != nil { + return false, err + } + changed = changed || streamChanged + } + if changed { + encoded, err := encodeJSONProjectionList(streams) + if err != nil { + return false, err + } + (*selection.part)["Stream"] = encoded + } + if changed { + if err := selection.commit(); err != nil { + return false, err + } + } + return changed, nil +} + +func (selection *jsonProjectionSelection) commit() error { + parts, err := encodeJSONProjectionList(selection.parts) + if err != nil { + return err + } + (*selection.mediaItem)["Part"] = parts + media, err := encodeJSONProjectionList(selection.media) + if err != nil { + return err + } + // The metadata list has exactly one item by selection contract. + (*selection.metadata.items[0])["Media"] = media + metadata, err := encodeJSONProjectionList(selection.metadata) + if err != nil { + return err + } + selection.container["Metadata"] = metadata + container, err := json.Marshal(selection.container) + if err != nil { + return err + } + selection.document.root["MediaContainer"] = container + return nil +} + +func partFromJSON(object *jsonObject) (Part, error) { + id, err := requiredJSONScalarString(*object, "id") + if err != nil { + return Part{}, err + } + file, err := requiredJSONString(*object, "file") + if err != nil { + return Part{}, err + } + key, err := optionalJSONString(*object, "key") + if err != nil { + return Part{}, err + } + return Part{ID: id, Key: key, File: file}, nil +} + +func requiredJSONScalarString(object jsonObject, name string) (string, error) { + raw, ok := object[name] + if !ok { + return "", fmt.Errorf("Plex JSON metadata has no %s", name) + } + value, ok := jsonRawScalarString(raw) + if !ok || value == "" { + return "", fmt.Errorf("Plex JSON metadata %s is not a non-empty scalar", name) + } + return value, nil +} + +func requiredJSONString(object jsonObject, name string) (string, error) { + raw, ok := object[name] + if !ok { + return "", fmt.Errorf("Plex JSON metadata has no %s", name) + } + var value string + if err := json.Unmarshal(raw, &value); err != nil || value == "" { + return "", fmt.Errorf("Plex JSON metadata %s is not a non-empty string", name) + } + return value, nil +} + +func optionalJSONString(object jsonObject, name string) (string, error) { + raw, ok := object[name] + if !ok { + return "", nil + } + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return "", fmt.Errorf("Plex JSON metadata %s is not a string", name) + } + return value, nil +} + +func jsonRawScalarString(raw json.RawMessage) (string, bool) { + var value any + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + if err := decoder.Decode(&value); err != nil { + return "", false + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return "", false + } + return stringValue(value) +} + +func jsonTargetNeedsEnrichment(media, part *jsonObject) (bool, error) { + needs := false + for _, name := range mediaProjectionAttributeNames { + if _, present := (*media)[name]; !present { + needs = true + } + } + for _, name := range partProjectionAttributeNames { + if _, present := (*part)[name]; !present { + needs = true + } + } + streamRaw, exists := (*part)["Stream"] + if !exists { + return needs, nil + } + streams, err := decodeJSONProjectionList(streamRaw, "Stream") + if err != nil { + return false, err + } + if _, err := indexJSONStreams(streams.items); err != nil { + return false, err + } + for _, stream := range streams.items { + key, present, err := jsonStreamIdentity(*stream) + if err != nil { + return false, err + } + if !present { + continue + } + for _, name := range streamProjectionAttributeNames(key.typeName) { + if _, present := (*stream)[name]; !present { + needs = true + } + } + } + return needs, nil +} + +func indexJSONStreams(streams []*jsonObject) (map[streamKey]*jsonObject, error) { + result := make(map[streamKey]*jsonObject, len(streams)) + for _, stream := range streams { + key, present, err := jsonStreamIdentity(*stream) + if err != nil { + return nil, err + } + if !present { + continue + } + if _, exists := result[key]; exists { + return nil, fmt.Errorf("Plex metadata has duplicate Stream identity %s/%d", key.typeName, key.index) + } + result[key] = stream + } + return result, nil +} + +func jsonStreamIdentity(object jsonObject) (streamKey, bool, error) { + typeRaw, typePresent := object["streamType"] + indexRaw, indexPresent := object["index"] + if !typePresent || !indexPresent { + return streamKey{}, false, nil + } + typeName, ok := jsonRawScalarString(typeRaw) + if !ok || strings.TrimSpace(typeName) == "" { + return streamKey{}, false, errors.New("Plex JSON metadata Stream has an invalid streamType") + } + indexValue, ok := jsonRawScalarString(indexRaw) + if !ok { + return streamKey{}, false, errors.New("Plex JSON metadata Stream has an invalid index") + } + index, err := strconv.Atoi(strings.TrimSpace(indexValue)) + if err != nil || index < 0 { + return streamKey{}, false, errors.New("Plex JSON metadata Stream has an invalid index") + } + key, ok := mediaStreamKey(typeName, index) + if !ok { + return streamKey{}, false, errors.New("Plex JSON metadata Stream has an invalid identity") + } + return key, true, nil +} + +func addMissingJSONAttributes(object jsonObject, attributes []projectionAttribute) (bool, error) { + changed := false + for _, attribute := range attributes { + if _, present := object[attribute.name]; present { + continue + } + var raw []byte + var err error + if attribute.number { + raw = []byte(attribute.value) + } else { + raw, err = json.Marshal(attribute.value) + if err != nil { + return false, err + } + } + object[attribute.name] = append(json.RawMessage(nil), raw...) + changed = true + } + return changed, nil +} + +func (document *jsonProjectionDocument) encode() ([]byte, error) { + encoded, err := json.Marshal(document.root) + if err != nil { + return nil, fmt.Errorf("encode Plex JSON metadata: %w", err) + } + result := make([]byte, 0, len(document.prefix)+len(encoded)+len(document.suffix)) + result = append(result, document.prefix...) + result = append(result, encoded...) + result = append(result, document.suffix...) + return result, nil +} diff --git a/internal/plexmeta/projection_test.go b/internal/plexmeta/projection_test.go new file mode 100644 index 0000000..d23a7d9 --- /dev/null +++ b/internal/plexmeta/projection_test.go @@ -0,0 +1,349 @@ +package plexmeta + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "io" + "reflect" + "strconv" + "testing" + + "github.com/InfinityPacer/plex-gateway/internal/mediainfo" +) + +func TestSelectEnrichmentTargetStrictlySelectsXMLItem(t *testing.T) { + body := []byte(``) + want := Part{ID: "7", Key: "/library/parts/7/1/file", File: "/cloud/movie.strm"} + + target, err := SelectEnrichmentTarget(body, "application/xml", "42") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(target.Part, want) || !target.NeedsEnrichment { + t.Fatalf("target = %#v, want part %#v and enrichment", target, want) + } + part, err := SelectEnrichmentPart(body, "application/xml", "42") + if err != nil || !reflect.DeepEqual(part, want) { + t.Fatalf("part = %#v, err = %v", part, err) + } +} + +func TestSelectEnrichmentTargetStrictlySelectsJSONItem(t *testing.T) { + body := []byte(`{"MediaContainer":{"rootExtra":"preserve","Metadata":[{"ratingKey":42,"itemExtra":"preserve","Media":[{"mediaExtra":"preserve","Part":[{"id":7,"key":"/library/parts/7/1/file","file":"/cloud/movie.strm","partExtra":"preserve"}]}]}]}}`) + target, err := SelectEnrichmentTarget(body, "application/json", "42") + if err != nil { + t.Fatal(err) + } + want := Part{ID: "7", Key: "/library/parts/7/1/file", File: "/cloud/movie.strm"} + if !reflect.DeepEqual(target.Part, want) || !target.NeedsEnrichment { + t.Fatalf("target = %#v, want part %#v and enrichment", target, want) + } +} + +func TestEnrichMetadataXMLFillsOnlyMissingSafeFields(t *testing.T) { + body := []byte(``) + original := append([]byte(nil), body...) + + result, changed, err := EnrichMetadata(body, "application/xml", "42", projectionTestMedia()) + if err != nil { + t.Fatal(err) + } + if !changed { + t.Fatal("EnrichMetadata reported no change") + } + if !bytes.Equal(body, original) { + t.Fatal("EnrichMetadata modified its input body") + } + + attrs := xmlProjectionAttributeSets(t, result) + media := attrs["Media"] + part := attrs["Part"] + videoStream := attrs["Stream#1"] + audioStream := attrs["Stream#2"] + if media["container"] != "plex-container" || media["videoCodec"] != "plex-codec" || media["videoResolution"] != "4k" || media["duration"] != "123000" { + t.Fatalf("Media attributes = %#v", media) + } + if part["id"] != "7" || part["file"] != "/cloud/movie.strm" || part["duration"] != "999" || part["size"] != "987654321" || part["container"] != "mkv" { + t.Fatalf("Part attributes = %#v", part) + } + if videoStream["id"] != "plex-stream-id" || videoStream["codec"] != "plex-stream-codec" || videoStream["profile"] != "Main 10" || videoStream["width"] != "3840" || videoStream["bitDepth"] != "10" { + t.Fatalf("video Stream attributes = %#v", videoStream) + } + if audioStream["codec"] != "aac" || audioStream["channels"] != "6" || audioStream["samplingRate"] != "48000" { + t.Fatalf("audio Stream attributes = %#v", audioStream) + } + for _, name := range []string{"selected", "default", "decision"} { + if videoStream[name] != "1" && name != "decision" { + t.Fatalf("forbidden Stream attribute %q changed: %#v", name, videoStream) + } + } + if videoStream["decision"] != "directplay" || videoStream["streamExtra"] != "preserve" || audioStream["streamExtra"] != "audio" { + t.Fatalf("existing or unknown Stream attributes were not preserved: video=%#v audio=%#v", videoStream, audioStream) + } + if !bytes.Contains(result, []byte(`rootExtra="preserve"`)) || !bytes.Contains(result, []byte(`UnknownNode unknownAttr="keep"`)) || !bytes.Contains(result, []byte(`UnknownChild`)) { + t.Fatalf("unknown XML attributes/nodes were not preserved: %s", result) + } +} + +func TestEnrichMetadataJSONFillsOnlyMissingSafeFields(t *testing.T) { + body := []byte(`{"MediaContainer":{"rootExtra":"preserve","Metadata":[{"ratingKey":"42","itemExtra":"preserve","Media":[{"mediaExtra":"preserve","container":"plex-container","videoCodec":"plex-codec","Part":[{"id":7,"key":"/library/parts/7/1/file","file":"/cloud/movie.strm","partExtra":"preserve","duration":999,"Stream":[{"id":"plex-stream-id","streamType":1,"index":0,"codec":"plex-stream-codec","selected":1,"default":1,"decision":"directplay","streamExtra":"preserve"},{"streamType":2,"index":1,"streamExtra":"audio"}]}]}]}]}}`) + original := append([]byte(nil), body...) + + result, changed, err := EnrichMetadata(body, "application/json", "42", projectionTestMedia()) + if err != nil { + t.Fatal(err) + } + if !changed { + t.Fatal("EnrichMetadata reported no change") + } + if !bytes.Equal(body, original) { + t.Fatal("EnrichMetadata modified its input body") + } + + var document map[string]any + if err := json.Unmarshal(result, &document); err != nil { + t.Fatal(err) + } + container := document["MediaContainer"].(map[string]any) + item := container["Metadata"].([]any)[0].(map[string]any) + media := item["Media"].([]any)[0].(map[string]any) + part := media["Part"].([]any)[0].(map[string]any) + streams := part["Stream"].([]any) + videoStream := streams[0].(map[string]any) + audioStream := streams[1].(map[string]any) + if media["container"] != "plex-container" || media["videoCodec"] != "plex-codec" || media["videoResolution"] != "4k" || media["duration"] != float64(123000) { + t.Fatalf("Media fields = %#v", media) + } + if part["id"] != float64(7) || part["file"] != "/cloud/movie.strm" || part["duration"] != float64(999) || part["size"] != float64(987654321) || part["container"] != "mkv" { + t.Fatalf("Part fields = %#v", part) + } + if videoStream["id"] != "plex-stream-id" || videoStream["codec"] != "plex-stream-codec" || videoStream["profile"] != "Main 10" || videoStream["width"] != float64(3840) || videoStream["bitDepth"] != float64(10) { + t.Fatalf("video Stream fields = %#v", videoStream) + } + if audioStream["codec"] != "aac" || audioStream["channels"] != float64(6) || audioStream["samplingRate"] != float64(48000) { + t.Fatalf("audio Stream fields = %#v", audioStream) + } + if videoStream["selected"] != float64(1) || videoStream["default"] != float64(1) || videoStream["decision"] != "directplay" || videoStream["streamExtra"] != "preserve" || audioStream["streamExtra"] != "audio" { + t.Fatalf("existing or forbidden Stream fields changed: video=%#v audio=%#v", videoStream, audioStream) + } + if container["rootExtra"] != "preserve" || item["itemExtra"] != "preserve" || media["mediaExtra"] != "preserve" || part["partExtra"] != "preserve" { + t.Fatalf("unknown JSON fields were not preserved: %#v", document) + } +} + +func TestEnrichMetadataDoesNotCreateStreams(t *testing.T) { + body := []byte(``) + result, changed, err := EnrichMetadata(body, "application/xml", "42", projectionTestMedia()) + if err != nil { + t.Fatal(err) + } + if !changed || bytes.Contains(result, []byte("`}, + {name: "XML rating mismatch", contentType: "application/xml", body: ``}, + {name: "XML multiple media", contentType: "application/xml", body: ``}, + {name: "XML multiple parts", contentType: "application/xml", body: ``}, + {name: "JSON multiple items", contentType: "application/json", body: `{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"Part":[{"id":7,"file":"/a.strm"}]}]},{"ratingKey":"43","Media":[{"Part":[{"id":8,"file":"/b.strm"}]}]}]}}`}, + {name: "JSON multiple media", contentType: "application/json", body: `{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"Part":[{"id":7,"file":"/a.strm"}]},{"Part":[{"id":8,"file":"/b.strm"}]}]}]}}`}, + {name: "JSON multiple parts", contentType: "application/json", body: `{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"Part":[{"id":7,"file":"/a.strm"},{"id":8,"file":"/b.strm"}]}]}]}}`}, + {name: "JSON duplicate path field", contentType: "application/json", body: `{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"Part":[{"id":7,"id":8,"file":"/a.strm"}]}]}]}}`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + body := []byte(test.body) + original := append([]byte(nil), body...) + if _, err := SelectEnrichmentPart(body, test.contentType, "42"); err == nil { + t.Fatal("SelectEnrichmentPart unexpectedly succeeded") + } + if !bytes.Equal(body, original) { + t.Fatal("SelectEnrichmentPart modified its input body") + } + if result, changed, err := EnrichMetadata(body, test.contentType, "42", projectionTestMedia()); err == nil || result != nil || changed { + t.Fatalf("EnrichMetadata result=%q changed=%v err=%v", result, changed, err) + } + if !bytes.Equal(body, original) { + t.Fatal("EnrichMetadata modified its input body after an error") + } + }) + } +} + +func TestEnrichMetadataRejectsDuplicateStreamIdentity(t *testing.T) { + tests := []struct { + name string + contentType string + body string + }{ + {name: "XML", contentType: "application/xml", body: ``}, + {name: "JSON", contentType: "application/json", body: `{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"Part":[{"id":7,"file":"/a.strm","Stream":[{"streamType":1,"index":0},{"streamType":"video","index":0}]}]}]}]}}`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + body := []byte(test.body) + original := append([]byte(nil), body...) + if result, changed, err := EnrichMetadata(body, test.contentType, "42", projectionTestMedia()); err == nil || result != nil || changed { + t.Fatalf("EnrichMetadata result=%q changed=%v err=%v", result, changed, err) + } + if !bytes.Equal(body, original) { + t.Fatal("EnrichMetadata modified its input body after an error") + } + }) + } +} + +func TestEnrichMetadataRejectsMalformedStreamIdentity(t *testing.T) { + tests := []struct { + name string + contentType string + body string + }{ + {name: "XML", contentType: "application/xml", body: ``}, + {name: "JSON", contentType: "application/json", body: `{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"Part":[{"id":7,"file":"/a.strm","Stream":[{"streamType":1,"index":"not-an-index"}]}]}]}]}}`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + body := []byte(test.body) + original := append([]byte(nil), body...) + if _, err := SelectEnrichmentTarget(body, test.contentType, "42"); err == nil { + t.Fatal("SelectEnrichmentTarget unexpectedly accepted malformed Stream identity") + } + if !bytes.Equal(body, original) { + t.Fatal("SelectEnrichmentTarget modified its input body") + } + }) + } +} + +func TestEnrichMetadataSkipsUnmatchedStreams(t *testing.T) { + body := []byte(`{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"Part":[{"id":7,"file":"/a.strm","Stream":[{"streamType":1,"index":99,"codec":"keep"}]}]}]}]}}`) + result, changed, err := EnrichMetadata(body, "application/json", "42", projectionTestMedia()) + if err != nil { + t.Fatal(err) + } + if !changed { + t.Fatal("Media/Part fields should still have been projected") + } + var document map[string]any + if err := json.Unmarshal(result, &document); err != nil { + t.Fatal(err) + } + stream := document["MediaContainer"].(map[string]any)["Metadata"].([]any)[0].(map[string]any)["Media"].([]any)[0].(map[string]any)["Part"].([]any)[0].(map[string]any)["Stream"].([]any)[0].(map[string]any) + if len(stream) != 3 || stream["codec"] != "keep" || stream["streamType"] != float64(1) || stream["index"] != float64(99) { + t.Fatalf("unmatched Stream was changed or created: %#v", stream) + } +} + +func projectionTestMedia() mediainfo.Media { + return mediainfo.Media{ + Complete: true, + Container: "mkv", + DurationMS: 123000, + Size: 987654321, + Bitrate: 12000000, + VideoCodec: "hevc", + VideoProfile: "Main 10", + VideoResolution: "4k", + Width: 3840, + Height: 2160, + AspectRatio: "16:9", + FrameRate: "23.976", + AudioCodec: "aac", + AudioProfile: "LC", + AudioChannels: 6, + Streams: []mediainfo.Stream{ + { + Index: 0, Type: "video", Codec: "hevc", Profile: "Main 10", Level: 153, + Bitrate: 12000000, Width: 3840, Height: 2160, FrameRate: "23.976", + ReferenceFrames: 4, PixelFormat: "yuv420p10le", BitDepth: 10, + ColorSpace: "bt2020nc", ColorRange: "tv", ColorPrimaries: "bt2020", + ColorTransfer: "smpte2084", ChromaLocation: "left", + SampleAspectRatio: "1:1", DisplayAspectRatio: "16:9", Language: "eng", Title: "Video", + }, + { + Index: 1, Type: "audio", Codec: "aac", Profile: "LC", Level: 1, + Bitrate: 640000, SampleRate: 48000, Channels: 6, ChannelLayout: "5.1", + Language: "eng", Title: "English", + }, + }, + } +} + +func xmlProjectionAttributeSets(t *testing.T, body []byte) map[string]map[string]string { + t.Helper() + decoder := xml.NewDecoder(bytes.NewReader(body)) + result := make(map[string]map[string]string) + streamCount := 0 + for { + token, err := decoder.Token() + if err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + start, ok := token.(xml.StartElement) + if !ok { + continue + } + name := start.Name.Local + if name == "Stream" { + streamCount++ + name += "#" + strconv.Itoa(streamCount) + } + attributes := make(map[string]string, len(start.Attr)) + for _, attribute := range start.Attr { + attributes[attribute.Name.Local] = attribute.Value + } + if _, exists := result[name]; !exists { + result[name] = attributes + } + } + return result +} + +func completeProjectionXML() string { + return `` +} + +func completeProjectionJSON() string { + return `{"MediaContainer":{"Metadata":[{"ratingKey":"42","Media":[{"container":"mkv","duration":123000,"bitrate":12000000,"width":3840,"height":2160,"aspectRatio":"16:9","audioChannels":6,"audioCodec":"aac","videoCodec":"hevc","videoResolution":"4k","videoFrameRate":"23.976","videoProfile":"Main 10","audioProfile":"LC","Part":[{"id":7,"key":"/library/parts/7/1/file","file":"/cloud/movie.strm","duration":123000,"size":987654321,"container":"mkv","videoProfile":"Main 10","audioProfile":"LC","Stream":[{"streamType":1,"index":0,"codec":"hevc","profile":"Main 10","level":153,"bitrate":12000000,"width":3840,"height":2160,"frameRate":"23.976","refFrames":4,"pixelFormat":"yuv420p10le","bitDepth":10,"colorSpace":"bt2020nc","colorRange":"tv","colorPrimaries":"bt2020","colorTrc":"smpte2084","chromaLocation":"left","sampleAspectRatio":"1:1","displayAspectRatio":"16:9","language":"eng","title":"Video"},{"streamType":2,"index":1,"codec":"aac","profile":"LC","level":1,"bitrate":640000,"language":"eng","title":"English","samplingRate":48000,"channels":6,"channelLayout":"5.1"}]}]}]}]}}` +}