From ad0a61a223ea7d64a9e93208747f264e8ef36a5d Mon Sep 17 00:00:00 2001
From: David Peluola <79669430+Adeolu05@users.noreply.github.com>
Date: Mon, 28 Sep 2026 18:51:58 +0100
Subject: [PATCH] feat: share rate limits and bot prefs via optional Redis;
hash IPs in logs
- REDIS_URL (optional): extract/convert/events quotas are counted once
across all Gunicorn workers and the Telegram bot instead of per process;
Telegram format/timestamp preferences survive restarts (90-day TTL).
Unset or unreachable Redis falls back to in-memory with a throttled warning.
- Docker Compose runs a small Redis; render.yaml takes REDIS_URL and
generates LOG_HASH_SECRET.
- hash_identifier is now HMAC-keyed (plain SHA-256 of an IPv4 or Telegram ID
is reversible by enumeration); rate-limit block logs no longer contain the
raw client IP.
- Failure logs record the public video ID + host instead of a partially
stripped URL (which lost the ID for watch?v= links and kept share tokens
in youtu.be paths).
- Privacy page / bot copy: document IP-based limits, hashed log IDs, and
Telegram preferences; caption cache lasts days (72h default), not hours.
- CI: Redis service container for real-server integration tests.
Co-Authored-By: Claude Opus 5.5
---
.env.example | 8 +
.github/workflows/tests.yml | 11 ++
README.md | 4 +-
app/bot/telegram_bot.py | 21 +--
app/core/config.py | 10 ++
app/core/dependencies.py | 3 +-
app/core/redis_client.py | 53 +++++++
app/services/metadata_service.py | 26 +--
app/services/rate_limit_service.py | 36 ++++-
app/services/telemetry_service.py | 11 +-
app/services/user_prefs_service.py | 67 ++++++++
docker-compose.yml | 20 +++
render.yaml | 5 +
requirements.txt | 1 +
tests/test_shared_state_and_privacy.py | 209 +++++++++++++++++++++++++
web/app/privacy/page.tsx | 19 ++-
16 files changed, 469 insertions(+), 35 deletions(-)
create mode 100644 app/core/redis_client.py
create mode 100644 app/services/user_prefs_service.py
create mode 100644 tests/test_shared_state_and_privacy.py
diff --git a/.env.example b/.env.example
index 8e36cfa..b1a7734 100644
--- a/.env.example
+++ b/.env.example
@@ -36,6 +36,14 @@ MAX_TRANSCRIPT_SEGMENTS=20000
IN_PROCESS_CLEANUP_ENABLED=true
CLEANUP_INTERVAL_SECONDS=900
+# --- Shared state (recommended in production)
+# Redis shares rate-limit counts across Gunicorn workers + the Telegram bot and keeps
+# bot preferences across restarts. Unset/unreachable = per-process in-memory fallback.
+# REDIS_URL=redis://localhost:6379/0
+# TELEGRAM_PREFS_TTL_DAYS=90
+# Key for hashing IPs / Telegram IDs in logs (any long random string). Unset = random per process.
+# LOG_HASH_SECRET=
+
# --- Internal metrics dashboard (Basic Auth at /internal/metrics)
METRICS_USERNAME=admin
# Change this before any public deployment.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 57f27ef..e6cf2b9 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -8,6 +8,16 @@ on:
jobs:
unit:
runs-on: ubuntu-latest
+ services:
+ redis:
+ image: redis:7-alpine
+ ports:
+ - 6379:6379
+ options: >-
+ --health-cmd "redis-cli ping"
+ --health-interval 5s
+ --health-timeout 3s
+ --health-retries 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
@@ -19,6 +29,7 @@ jobs:
run: python -m unittest discover -s tests -v
env:
PYTHONPATH: .
+ TEST_REDIS_URL: redis://localhost:6379/15
web:
runs-on: ubuntu-latest
diff --git a/README.md b/README.md
index ed8303f..a266328 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@ Core Value: **Fast → Reliable → Structured → Downloadable.**
- **Clean Formats**: Download as **TXT, PDF, and DOCX**, or **SRT / VTT** subtitles with cue timings. Files are named after the video title.
- **Timestamps**: Toggle perfectly formatted timestamps on or off.
- **Anonymous & Secure**: No accounts required. Generated files auto-delete after ~1 hour (TTL). Optional short-term caption cache (`TRANSCRIPT_CACHE_TTL_HOURS`) is pruned by the same cleanup loop.
-- **Rate Limited & Guardrailed**: Separate extract vs convert quotas, max video duration (with caption-based fallback), segment/size caps, and timeout protections.
+- **Rate Limited & Guardrailed**: Separate extract vs convert quotas (shared across workers and the bot via optional Redis), max video duration (with caption-based fallback), segment/size caps, and timeout protections.
- **Telemetry & Analytics**: Built-in lightweight JSON-structured logging and analytics tracking.
## 🏗 System Architecture
@@ -54,7 +54,7 @@ A premium, modern SaaS web app interface designed with:
### 2. Telegram Bot (`app/bot/telegram_bot.py`)
A seriously structured, minimal utility bot:
- **Instant Processing**: Send a URL, get a file. No conversational fluff.
-- **In-Memory Preferences**: Remembers your preferred format (e.g., PDF) and timestamp setting while the bot process runs (reset on restart/deploy).
+- **Remembered Preferences**: Remembers your preferred format (e.g., PDF) and timestamp setting — in Redis when `REDIS_URL` is set (kept ~90 days), otherwise in memory until the bot restarts.
- **Single Message UX**: Status updates (Extracting... $\rightarrow$ Formatting... $\rightarrow$ Ready) happen within a single edited message to keep chat history clean.
- **Branded Onboarding**: Clean `/start` experience with inline keyboard routing.
diff --git a/app/bot/telegram_bot.py b/app/bot/telegram_bot.py
index 0bfe3b4..2cec538 100644
--- a/app/bot/telegram_bot.py
+++ b/app/bot/telegram_bot.py
@@ -22,6 +22,7 @@
from app.services.formatter_service import SUBTITLE_FORMATS, TranscriptFormatter
from app.services.file_service import FileGenerator, download_filename
from app.services.rate_limit_service import check_rate_limit
+from app.services import user_prefs_service
from app.services.metadata_service import MetadataService
from app.services.extract_guardrails import enforce_transcript_guardrails
from app.core.errors import AppError, ErrorCode, TranscriptFetchError
@@ -48,11 +49,6 @@
re.IGNORECASE,
)
-# ---------------------------------------------------------------------------
-# Per-user preferences (Phase 1 — in-memory)
-# ---------------------------------------------------------------------------
-_user_prefs: Dict[str, Dict] = {}
-
# ---------------------------------------------------------------------------
# Avatar path (bundled inside app/ for Docker availability)
# ---------------------------------------------------------------------------
@@ -60,18 +56,12 @@
def _get_prefs(user_id: str) -> dict:
- """Return preferences for *user_id*, creating defaults if absent."""
- if user_id not in _user_prefs:
- _user_prefs[user_id] = {
- "last_file_format": "txt",
- "last_include_timestamps": False,
- }
- return _user_prefs[user_id]
+ """Preferences for *user_id* (Redis-backed when REDIS_URL is set)."""
+ return user_prefs_service.get_prefs(user_id)
def _set_pref(user_id: str, **kwargs) -> None:
- prefs = _get_prefs(user_id)
- prefs.update(kwargs)
+ user_prefs_service.set_prefs(user_id, **kwargs)
def _emit_event(event_name: str, user_id: str, props: dict | None = None) -> None:
@@ -160,7 +150,8 @@ def _escape_md2(text: str) -> str:
"*Privacy*\n\n"
"\\• No account required\\.\n"
"\\• Generated files auto\\-delete after about 1 hour\\.\n"
- "\\• Short\\-term caption cache \\(hours\\) may reduce repeat fetches\\.\n"
+ "\\• Short\\-term caption cache \\(up to a few days\\) may reduce repeat fetches\\.\n"
+ "\\• Your format and timestamp preference is remembered for about 90 days\\.\n"
"\\• We do not keep permanent user accounts or history\\.\n"
"\\• See the website privacy page for full details\\."
)
diff --git a/app/core/config.py b/app/core/config.py
index b33047c..47b73b0 100644
--- a/app/core/config.py
+++ b/app/core/config.py
@@ -32,6 +32,16 @@ class Settings(BaseSettings):
rate_limit_download_enabled: bool = False
rate_limit_download_requests: int = 200
+ # Optional shared store (e.g. redis://...). When set, rate limits are counted
+ # across all Gunicorn workers + the Telegram bot, and bot preferences survive
+ # restarts. Empty or unreachable = per-process in-memory fallback.
+ redis_url: str = ""
+ telegram_prefs_ttl_days: int = 90
+
+ # Key for HMAC-hashing IPs / Telegram IDs in logs and telemetry. Unset = a
+ # random per-process key (hashes then only correlate within one process).
+ log_hash_secret: str = ""
+
# Proxy security: how many trusted proxies sit in front of the app.
# 0 = direct exposure (ignore X-Forwarded-For entirely, use socket IP).
# 1 = one reverse proxy (Nginx, Cloudflare, etc.).
diff --git a/app/core/dependencies.py b/app/core/dependencies.py
index cb17614..581d8be 100644
--- a/app/core/dependencies.py
+++ b/app/core/dependencies.py
@@ -4,6 +4,7 @@
from app.core.config import settings
from app.core.errors import AppError, ErrorCode
from app.services.metadata_service import MetadataService
+from app.services.telemetry_service import hash_identifier
from app.utils.logging_config import logger
@@ -56,7 +57,7 @@ def _extract_client_ip(request: Request) -> str:
def _raise_if_limited(client_ip: str, bucket: str) -> None:
if not check_rate_limit(client_ip, bucket=bucket):
- MetadataService.log_rate_limit_block(identifier=f"{bucket}:{client_ip}")
+ MetadataService.log_rate_limit_block(identifier=f"{bucket}:{hash_identifier(client_ip)}")
raise AppError(
ErrorCode.RATE_LIMIT_EXCEEDED,
"Rate limit exceeded. Try again later.",
diff --git a/app/core/redis_client.py b/app/core/redis_client.py
new file mode 100644
index 0000000..c6d4524
--- /dev/null
+++ b/app/core/redis_client.py
@@ -0,0 +1,53 @@
+"""
+Optional shared Redis connection (REDIS_URL).
+
+Callers must treat Redis as best-effort: get_redis() returns None when it is
+not configured, and any redis.RedisError should fall back to in-memory state so
+an outage degrades limits to per-process instead of taking the API down.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from typing import Optional
+
+import redis
+
+from app.core.config import settings
+from app.utils.logging_config import logger
+
+# Keep request latency bounded if Redis is slow or gone
+_SOCKET_TIMEOUT_SECONDS = 0.5
+_WARN_INTERVAL_SECONDS = 60
+
+_client: Optional[redis.Redis] = None
+_client_url = ""
+_lock = threading.Lock()
+_last_warning = 0.0
+
+
+def get_redis() -> Optional[redis.Redis]:
+ global _client, _client_url
+ url = (settings.redis_url or "").strip()
+ if not url:
+ return None
+ with _lock:
+ if _client is None or _client_url != url:
+ _client = redis.Redis.from_url(
+ url,
+ socket_timeout=_SOCKET_TIMEOUT_SECONDS,
+ socket_connect_timeout=_SOCKET_TIMEOUT_SECONDS,
+ decode_responses=True,
+ )
+ _client_url = url
+ return _client
+
+
+def warn_fallback(context: str, error: Exception) -> None:
+ """Log a Redis failure at most once a minute (a dead Redis would flood logs)."""
+ global _last_warning
+ now = time.monotonic()
+ if now - _last_warning >= _WARN_INTERVAL_SECONDS:
+ _last_warning = now
+ logger.warning("Redis unavailable (%s), using in-memory fallback: %s", context, error)
diff --git a/app/services/metadata_service.py b/app/services/metadata_service.py
index 12aea6d..c67d657 100644
--- a/app/services/metadata_service.py
+++ b/app/services/metadata_service.py
@@ -3,6 +3,7 @@
from urllib.parse import urlparse
from app.utils.logging_config import logger
+from app.utils.validators import validate_vimeo_url, validate_youtube_url
class MetadataService:
@@ -45,15 +46,20 @@ def log_success(
logger.info(json.dumps(log_data))
@staticmethod
- def _url_for_log(url: str, max_len: int = 160) -> str:
- """Host + path only (no query/fragment) to avoid logging tokens or full PII."""
- if not url:
- return ""
- p = urlparse(url)
- base = f"{p.scheme}://{p.netloc}{p.path}".strip()
- if len(base) > max_len:
- return base[: max_len - 1] + "…"
- return base
+ def _video_ref(url: str, platform: str) -> Dict[str, Optional[str]]:
+ """
+ Public video ID (as success logs already record) plus host — never the
+ raw URL, whose query/path can carry share tokens or tracking params.
+ """
+ validate = {"youtube": validate_youtube_url, "vimeo": validate_vimeo_url}.get(platform)
+ video_id = None
+ if validate and url:
+ try:
+ video_id = validate(url)
+ except ValueError:
+ pass
+ host = (urlparse(url).hostname or "")[:100] if url else ""
+ return {"video_id": video_id, "host": host}
@staticmethod
def log_failure(
@@ -67,7 +73,7 @@ def log_failure(
"""Logs a failed extraction workflow as structured JSON."""
log_data = {
"event": "transcript_failure",
- "url": MetadataService._url_for_log(url),
+ **MetadataService._video_ref(url, platform),
"provider": platform,
"error_code": error_code,
"error_message": error_message,
diff --git a/app/services/rate_limit_service.py b/app/services/rate_limit_service.py
index 982c65a..13abe42 100644
--- a/app/services/rate_limit_service.py
+++ b/app/services/rate_limit_service.py
@@ -1,15 +1,19 @@
import time
-from typing import Dict, Tuple
+from typing import Dict, Optional, Tuple
+
+import redis
from app.core.config import settings
+from app.core.redis_client import get_redis, warn_fallback
class RateLimitService:
"""
- In-memory sliding window counter keyed by (bucket, identifier).
+ Fixed-window counter keyed by (bucket, identifier).
- Multi-worker note: each Gunicorn worker has its own store, so effective
- limits can be up to ~N× the configured value under uniform load.
+ With REDIS_URL set, counts are shared by every Gunicorn worker and the
+ Telegram bot. Otherwise (or if Redis errors) each process keeps its own
+ in-memory store, so effective limits can be up to ~N× the configured value.
"""
def __init__(self) -> None:
@@ -37,6 +41,10 @@ def is_allowed(self, identifier: str, bucket: str = "extract") -> bool:
return True
window_seconds = max(1, int(settings.rate_limit_window_seconds))
+ shared = self._redis_is_allowed(identifier, bucket, limit, window_seconds)
+ if shared is not None:
+ return shared
+
now = time.time()
self._cleanup(now)
@@ -57,6 +65,26 @@ def is_allowed(self, identifier: str, bucket: str = "extract") -> bool:
return False
+ @staticmethod
+ def _redis_is_allowed(
+ identifier: str, bucket: str, limit: int, window_seconds: int
+ ) -> Optional[bool]:
+ """Shared count via Redis; None means use the in-memory fallback."""
+ client = get_redis()
+ if client is None:
+ return None
+ key = f"tf:rl:{bucket}:{identifier}"
+ try:
+ # SET NX starts the window with its TTL; INCR counts this request
+ pipe = client.pipeline(transaction=True)
+ pipe.set(key, 0, ex=window_seconds, nx=True)
+ pipe.incr(key)
+ _, count = pipe.execute()
+ except redis.RedisError as e:
+ warn_fallback("rate limit", e)
+ return None
+ return int(count) <= limit
+
def _cleanup(self, now: float) -> None:
expired = [k for k, v in self._store.items() if now > v[1]]
for k in expired:
diff --git a/app/services/telemetry_service.py b/app/services/telemetry_service.py
index cd6cf3f..5a01406 100644
--- a/app/services/telemetry_service.py
+++ b/app/services/telemetry_service.py
@@ -7,11 +7,18 @@
import json
import hashlib
+import hmac
+import secrets
from datetime import datetime, timezone
from typing import Any, Dict, Optional
+from app.core.config import settings
from app.utils.logging_config import logger
+# Plain SHA-256 of an IPv4 address or Telegram ID is reversible by enumeration,
+# so identifiers are keyed. Without LOG_HASH_SECRET the key lives per process.
+_HASH_KEY = (settings.log_hash_secret or secrets.token_hex(32)).encode()
+
# ---------------------------------------------------------------------------
# Event allowlist
# ---------------------------------------------------------------------------
@@ -73,8 +80,8 @@ def bucket_duration(seconds: Optional[int]) -> Optional[str]:
def hash_identifier(raw: str) -> str:
- """One-way SHA-256 hash of an identifier (e.g. Telegram user ID)."""
- return hashlib.sha256(raw.encode()).hexdigest()[:16]
+ """Keyed one-way hash of an identifier (IP, Telegram user ID) for logs."""
+ return hmac.new(_HASH_KEY, raw.encode(), hashlib.sha256).hexdigest()[:16]
# ---------------------------------------------------------------------------
diff --git a/app/services/user_prefs_service.py b/app/services/user_prefs_service.py
new file mode 100644
index 0000000..4eba9d0
--- /dev/null
+++ b/app/services/user_prefs_service.py
@@ -0,0 +1,67 @@
+"""
+Telegram per-user preferences (preferred file format, timestamps on/off).
+
+Stored in Redis when REDIS_URL is set, so they survive restarts and deploys,
+and expire after TELEGRAM_PREFS_TTL_DAYS of inactivity. Otherwise, or if Redis
+errors, they live in process memory as before.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict
+
+import redis
+
+from app.core.config import settings
+from app.core.redis_client import get_redis, warn_fallback
+
+FILE_FORMATS = ("txt", "pdf", "docx", "srt", "vtt")
+DEFAULTS: Dict[str, Any] = {"last_file_format": "txt", "last_include_timestamps": False}
+
+_memory: Dict[str, Dict[str, Any]] = {}
+
+
+def _key(user_id: str) -> str:
+ return f"tf:tgprefs:{user_id}"
+
+
+def _decode(raw: Dict[str, str]) -> Dict[str, Any]:
+ """Redis hash (all strings) → typed prefs, ignoring unknown/invalid values."""
+ prefs = dict(DEFAULTS)
+ if raw.get("last_file_format") in FILE_FORMATS:
+ prefs["last_file_format"] = raw["last_file_format"]
+ if raw.get("last_include_timestamps") in ("0", "1"):
+ prefs["last_include_timestamps"] = raw["last_include_timestamps"] == "1"
+ return prefs
+
+
+def get_prefs(user_id: str) -> Dict[str, Any]:
+ client = get_redis()
+ if client is not None:
+ try:
+ return _decode(client.hgetall(_key(user_id)))
+ except redis.RedisError as e:
+ warn_fallback("telegram prefs", e)
+ return dict(_memory.get(user_id, DEFAULTS))
+
+
+def set_prefs(user_id: str, **updates: Any) -> None:
+ unknown = set(updates) - set(DEFAULTS)
+ if unknown:
+ raise ValueError(f"Unknown preference(s): {sorted(unknown)}")
+
+ _memory[user_id] = {**_memory.get(user_id, DEFAULTS), **updates}
+
+ client = get_redis()
+ if client is None:
+ return
+ encoded = {
+ k: ("1" if v else "0") if isinstance(v, bool) else str(v) for k, v in updates.items()
+ }
+ try:
+ pipe = client.pipeline(transaction=True)
+ pipe.hset(_key(user_id), mapping=encoded)
+ pipe.expire(_key(user_id), max(1, settings.telegram_prefs_ttl_days) * 86400)
+ pipe.execute()
+ except redis.RedisError as e:
+ warn_fallback("telegram prefs", e)
diff --git a/docker-compose.yml b/docker-compose.yml
index 5636611..7613ad1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -41,8 +41,12 @@ services:
FILE_STORAGE_DIR: /app/data/tmp
TRANSCRIPT_CACHE_DIR: /app/data/transcript_cache
IN_PROCESS_CLEANUP_ENABLED: "true"
+ REDIS_URL: redis://redis:6379/0
volumes:
- transcript_data:/app/data
+ depends_on:
+ redis:
+ condition: service_healthy
restart: unless-stopped
logging:
driver: json-file
@@ -76,6 +80,7 @@ services:
TMPDIR: /tmp
FILE_STORAGE_DIR: /app/data/tmp
TRANSCRIPT_CACHE_DIR: /app/data/transcript_cache
+ REDIS_URL: redis://redis:6379/0
volumes:
- transcript_data:/app/data
restart: unless-stopped
@@ -88,5 +93,20 @@ services:
api:
condition: service_healthy
+ # Shared rate-limit counters (all API workers + bot) and Telegram preferences
+ redis:
+ image: redis:7-alpine
+ container_name: transcriptflow-redis
+ command: ["redis-server", "--appendonly", "yes", "--maxmemory", "64mb", "--maxmemory-policy", "volatile-ttl"]
+ volumes:
+ - redis_data:/data
+ restart: unless-stopped
+ healthcheck:
+ test: ["CMD", "redis-cli", "ping"]
+ interval: 10s
+ timeout: 3s
+ retries: 5
+
volumes:
transcript_data:
+ redis_data:
diff --git a/render.yaml b/render.yaml
index b51ec1b..2b84667 100644
--- a/render.yaml
+++ b/render.yaml
@@ -32,6 +32,11 @@ services:
value: "false"
- key: TRUSTED_PROXY_COUNT
value: "1"
+ # Point at a Render Key Value (Redis) instance so limits are shared across workers
+ - key: REDIS_URL
+ sync: false
+ - key: LOG_HASH_SECRET
+ generateValue: true
- key: MAX_VIDEO_DURATION_SECONDS
value: "10800"
- key: TRANSCRIPT_TIMEOUT_SECONDS
diff --git a/requirements.txt b/requirements.txt
index 31ef4ac..f4d527a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,3 +13,4 @@ python-dotenv==1.1.0
webvtt-py==0.5.1
requests==2.32.3
httpx==0.28.1
+redis==5.2.1
diff --git a/tests/test_shared_state_and_privacy.py b/tests/test_shared_state_and_privacy.py
new file mode 100644
index 0000000..9117b19
--- /dev/null
+++ b/tests/test_shared_state_and_privacy.py
@@ -0,0 +1,209 @@
+"""Redis-shared rate limits / bot prefs (with in-memory fallback) and log privacy."""
+
+import hashlib
+import json
+import os
+import unittest
+from unittest.mock import patch
+
+import redis
+
+from app.services import rate_limit_service as rls
+from app.services import telemetry_service
+from app.services import user_prefs_service as prefs
+from app.services.metadata_service import MetadataService
+
+
+class _FakePipeline:
+ def __init__(self, store):
+ self._store = store
+ self._ops = []
+
+ def __getattr__(self, name):
+ def queue(*args, **kwargs):
+ self._ops.append((name, args, kwargs))
+ return self
+ return queue
+
+ def execute(self):
+ return [getattr(self._store, name)(*a, **kw) for name, a, kw in self._ops]
+
+
+class FakeRedis:
+ """Just the commands the app uses; TTLs recorded, not enforced."""
+
+ def __init__(self):
+ self.data = {}
+ self.ttl = {}
+
+ def pipeline(self, transaction=True):
+ return _FakePipeline(self)
+
+ def set(self, key, value, ex=None, nx=False):
+ if nx and key in self.data:
+ return None
+ self.data[key] = str(value)
+ self.ttl[key] = ex
+ return True
+
+ def incr(self, key):
+ self.data[key] = str(int(self.data.get(key, 0)) + 1)
+ return int(self.data[key])
+
+ def hgetall(self, key):
+ return dict(self.data.get(key, {}))
+
+ def hset(self, key, mapping):
+ self.data.setdefault(key, {}).update(mapping)
+ return len(mapping)
+
+ def expire(self, key, seconds):
+ self.ttl[key] = seconds
+ return True
+
+
+class _BrokenRedis:
+ def pipeline(self, transaction=True):
+ raise redis.ConnectionError("down")
+
+ def hgetall(self, key):
+ raise redis.ConnectionError("down")
+
+
+def _limits(s, extract=2):
+ s.rate_limit_requests = extract
+ s.rate_limit_convert_requests = 40
+ s.rate_limit_events_requests = 100
+ s.rate_limit_download_requests = 200
+ s.rate_limit_window_seconds = 3600
+
+
+class TestSharedRateLimit(unittest.TestCase):
+ def test_count_is_shared_between_limiter_instances(self):
+ fake = FakeRedis()
+ worker_a, worker_b = rls.RateLimitService(), rls.RateLimitService()
+ with patch.object(rls, "get_redis", return_value=fake), patch.object(rls, "settings") as s:
+ _limits(s)
+ self.assertTrue(worker_a.is_allowed("1.1.1.1"))
+ self.assertTrue(worker_b.is_allowed("1.1.1.1"))
+ # Third request is blocked no matter which worker receives it
+ self.assertFalse(worker_a.is_allowed("1.1.1.1"))
+ self.assertTrue(worker_b.is_allowed("1.1.1.1", bucket="events"))
+ self.assertEqual(fake.ttl["tf:rl:extract:1.1.1.1"], 3600)
+
+ def test_falls_back_to_memory_when_redis_down(self):
+ svc = rls.RateLimitService()
+ with patch.object(rls, "get_redis", return_value=_BrokenRedis()), patch.object(
+ rls, "settings"
+ ) as s, patch.object(rls, "warn_fallback") as warn:
+ _limits(s)
+ self.assertTrue(svc.is_allowed("ip"))
+ self.assertTrue(svc.is_allowed("ip"))
+ self.assertFalse(svc.is_allowed("ip"))
+ warn.assert_called()
+
+
+@unittest.skipUnless(os.environ.get("TEST_REDIS_URL"), "TEST_REDIS_URL not set (CI provides one)")
+class TestRealRedis(unittest.TestCase):
+ """Same behaviour against a real server (MULTI/SET NX EX/INCR, hash + TTL)."""
+
+ def setUp(self):
+ self.client = redis.Redis.from_url(os.environ["TEST_REDIS_URL"], decode_responses=True)
+ self.client.flushdb()
+ self.addCleanup(self.client.flushdb)
+
+ def test_rate_limit_shared_with_ttl(self):
+ with patch.object(rls, "get_redis", return_value=self.client), patch.object(
+ rls, "settings"
+ ) as s:
+ _limits(s)
+ results = [rls.RateLimitService().is_allowed("ip") for _ in range(3)]
+ self.assertEqual(results, [True, True, False])
+ self.assertTrue(0 < self.client.ttl("tf:rl:extract:ip") <= 3600)
+
+ def test_prefs_roundtrip(self):
+ prefs._memory.clear()
+ with patch.object(prefs, "get_redis", return_value=self.client):
+ prefs.set_prefs("42", last_file_format="vtt", last_include_timestamps=True)
+ prefs._memory.clear()
+ self.assertEqual(prefs.get_prefs("42")["last_file_format"], "vtt")
+ self.assertGreater(self.client.ttl("tf:tgprefs:42"), 86400)
+
+
+class TestTelegramPrefs(unittest.TestCase):
+ def setUp(self):
+ prefs._memory.clear()
+
+ def test_defaults(self):
+ with patch.object(prefs, "get_redis", return_value=None):
+ self.assertEqual(prefs.get_prefs("42"), prefs.DEFAULTS)
+
+ def test_roundtrip_through_redis_survives_process_memory_loss(self):
+ fake = FakeRedis()
+ with patch.object(prefs, "get_redis", return_value=fake):
+ prefs.set_prefs("42", last_file_format="srt", last_include_timestamps=True)
+ prefs._memory.clear() # simulate a restart
+ self.assertEqual(
+ prefs.get_prefs("42"),
+ {"last_file_format": "srt", "last_include_timestamps": True},
+ )
+ self.assertEqual(fake.ttl["tf:tgprefs:42"], prefs.settings.telegram_prefs_ttl_days * 86400)
+
+ def test_invalid_stored_values_ignored(self):
+ fake = FakeRedis()
+ fake.data["tf:tgprefs:7"] = {"last_file_format": "exe", "last_include_timestamps": "yes"}
+ with patch.object(prefs, "get_redis", return_value=fake):
+ self.assertEqual(prefs.get_prefs("7"), prefs.DEFAULTS)
+
+ def test_memory_fallback_when_redis_down(self):
+ with patch.object(prefs, "get_redis", return_value=_BrokenRedis()), patch.object(
+ prefs, "warn_fallback"
+ ):
+ prefs.set_prefs("9", last_file_format="pdf")
+ self.assertEqual(prefs.get_prefs("9")["last_file_format"], "pdf")
+
+ def test_unknown_pref_rejected(self):
+ with self.assertRaises(ValueError):
+ prefs.set_prefs("1", theme="dark")
+
+
+class TestLogPrivacy(unittest.TestCase):
+ def test_hash_is_keyed_not_plain_sha256(self):
+ ip = "203.0.113.7"
+ hashed = telemetry_service.hash_identifier(ip)
+ self.assertEqual(hashed, telemetry_service.hash_identifier(ip))
+ self.assertNotEqual(hashed, hashlib.sha256(ip.encode()).hexdigest()[:16])
+
+ def _logged_json(self, mock_logger_method):
+ return json.loads(mock_logger_method.call_args.args[0])
+
+ @patch("app.services.metadata_service.logger")
+ def test_failure_log_has_video_id_not_url(self, mock_logger):
+ MetadataService.log_failure(
+ url="https://youtu.be/dQw4w9WgXcQ?si=SHARETOKEN",
+ platform="youtube",
+ error_code="TRANSCRIPT_NOT_AVAILABLE",
+ error_message="x",
+ processing_time_ms=1,
+ )
+ data = self._logged_json(mock_logger.error)
+ self.assertEqual(data["video_id"], "dQw4w9WgXcQ")
+ self.assertEqual(data["host"], "youtu.be")
+ self.assertNotIn("url", data)
+ self.assertNotIn("SHARETOKEN", json.dumps(data))
+
+ @patch("app.services.metadata_service.logger")
+ def test_rate_limit_block_log_hashes_ip(self, mock_logger):
+ from fastapi.testclient import TestClient
+ from app.main import app
+
+ with patch("app.core.dependencies.check_rate_limit", return_value=False):
+ r = TestClient(app).post("/api/v1/events", json={})
+ self.assertEqual(r.status_code, 429)
+ logged = mock_logger.warning.call_args.args[0]
+ self.assertNotIn("testclient", logged) # TestClient's socket "IP"
+ self.assertIn(telemetry_service.hash_identifier("testclient"), logged)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/web/app/privacy/page.tsx b/web/app/privacy/page.tsx
index 2fff989..ca227ac 100644
--- a/web/app/privacy/page.tsx
+++ b/web/app/privacy/page.tsx
@@ -69,7 +69,7 @@ export default function PrivacyPage() {
To improve reliability and reduce load on third-party caption sources, we may cache
- caption/transcript data for a video for a limited period (on the order of hours, not
+ caption/transcript data for a video for a limited period (up to a few days, not
permanent archives). This cache is operational, not a personal library or account
history. It is separate from the short-lived download files described above.
@@ -85,6 +85,23 @@ export default function PrivacyPage() {
to avoid storing unnecessary detail. Internal dashboards may show aggregated usage
metrics, not a permanent copy of your full transcript text, for operations.
+
+ To enforce fair-use limits we count requests per IP address (or per Telegram user) for
+ the length of the limit window, typically 24 hours. Logs record IP addresses and
+ Telegram user IDs only as keyed one-way hashes, and record the public video ID rather
+ than the full link you submitted.
+
+
+
+
+
+ Telegram preferences
+
+
+ The Telegram bot remembers your preferred file format and timestamp setting, keyed to
+ your Telegram user ID, so you do not have to choose them each time. This is forgotten
+ after about 90 days without use.
+