diff --git a/.env.example b/.env.example index 8e36cfa..b1a7734 100644 --- a/.env.example +++ b/.env.example @@ -36,6 +36,14 @@ MAX_TRANSCRIPT_SEGMENTS=20000 IN_PROCESS_CLEANUP_ENABLED=true CLEANUP_INTERVAL_SECONDS=900 +# --- Shared state (recommended in production) +# Redis shares rate-limit counts across Gunicorn workers + the Telegram bot and keeps +# bot preferences across restarts. Unset/unreachable = per-process in-memory fallback. +# REDIS_URL=redis://localhost:6379/0 +# TELEGRAM_PREFS_TTL_DAYS=90 +# Key for hashing IPs / Telegram IDs in logs (any long random string). Unset = random per process. +# LOG_HASH_SECRET= + # --- Internal metrics dashboard (Basic Auth at /internal/metrics) METRICS_USERNAME=admin # Change this before any public deployment. diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 57f27ef..e6cf2b9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,6 +8,16 @@ on: jobs: unit: runs-on: ubuntu-latest + services: + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -19,6 +29,7 @@ jobs: run: python -m unittest discover -s tests -v env: PYTHONPATH: . + TEST_REDIS_URL: redis://localhost:6379/15 web: runs-on: ubuntu-latest diff --git a/README.md b/README.md index ed8303f..a266328 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Core Value: **Fast → Reliable → Structured → Downloadable.** - **Clean Formats**: Download as **TXT, PDF, and DOCX**, or **SRT / VTT** subtitles with cue timings. Files are named after the video title. - **Timestamps**: Toggle perfectly formatted timestamps on or off. - **Anonymous & Secure**: No accounts required. Generated files auto-delete after ~1 hour (TTL). Optional short-term caption cache (`TRANSCRIPT_CACHE_TTL_HOURS`) is pruned by the same cleanup loop. -- **Rate Limited & Guardrailed**: Separate extract vs convert quotas, max video duration (with caption-based fallback), segment/size caps, and timeout protections. +- **Rate Limited & Guardrailed**: Separate extract vs convert quotas (shared across workers and the bot via optional Redis), max video duration (with caption-based fallback), segment/size caps, and timeout protections. - **Telemetry & Analytics**: Built-in lightweight JSON-structured logging and analytics tracking. ## 🏗 System Architecture @@ -54,7 +54,7 @@ A premium, modern SaaS web app interface designed with: ### 2. Telegram Bot (`app/bot/telegram_bot.py`) A seriously structured, minimal utility bot: - **Instant Processing**: Send a URL, get a file. No conversational fluff. -- **In-Memory Preferences**: Remembers your preferred format (e.g., PDF) and timestamp setting while the bot process runs (reset on restart/deploy). +- **Remembered Preferences**: Remembers your preferred format (e.g., PDF) and timestamp setting — in Redis when `REDIS_URL` is set (kept ~90 days), otherwise in memory until the bot restarts. - **Single Message UX**: Status updates (Extracting... $\rightarrow$ Formatting... $\rightarrow$ Ready) happen within a single edited message to keep chat history clean. - **Branded Onboarding**: Clean `/start` experience with inline keyboard routing. diff --git a/app/bot/telegram_bot.py b/app/bot/telegram_bot.py index 0bfe3b4..2cec538 100644 --- a/app/bot/telegram_bot.py +++ b/app/bot/telegram_bot.py @@ -22,6 +22,7 @@ from app.services.formatter_service import SUBTITLE_FORMATS, TranscriptFormatter from app.services.file_service import FileGenerator, download_filename from app.services.rate_limit_service import check_rate_limit +from app.services import user_prefs_service from app.services.metadata_service import MetadataService from app.services.extract_guardrails import enforce_transcript_guardrails from app.core.errors import AppError, ErrorCode, TranscriptFetchError @@ -48,11 +49,6 @@ re.IGNORECASE, ) -# --------------------------------------------------------------------------- -# Per-user preferences (Phase 1 — in-memory) -# --------------------------------------------------------------------------- -_user_prefs: Dict[str, Dict] = {} - # --------------------------------------------------------------------------- # Avatar path (bundled inside app/ for Docker availability) # --------------------------------------------------------------------------- @@ -60,18 +56,12 @@ def _get_prefs(user_id: str) -> dict: - """Return preferences for *user_id*, creating defaults if absent.""" - if user_id not in _user_prefs: - _user_prefs[user_id] = { - "last_file_format": "txt", - "last_include_timestamps": False, - } - return _user_prefs[user_id] + """Preferences for *user_id* (Redis-backed when REDIS_URL is set).""" + return user_prefs_service.get_prefs(user_id) def _set_pref(user_id: str, **kwargs) -> None: - prefs = _get_prefs(user_id) - prefs.update(kwargs) + user_prefs_service.set_prefs(user_id, **kwargs) def _emit_event(event_name: str, user_id: str, props: dict | None = None) -> None: @@ -160,7 +150,8 @@ def _escape_md2(text: str) -> str: "*Privacy*\n\n" "\\• No account required\\.\n" "\\• Generated files auto\\-delete after about 1 hour\\.\n" - "\\• Short\\-term caption cache \\(hours\\) may reduce repeat fetches\\.\n" + "\\• Short\\-term caption cache \\(up to a few days\\) may reduce repeat fetches\\.\n" + "\\• Your format and timestamp preference is remembered for about 90 days\\.\n" "\\• We do not keep permanent user accounts or history\\.\n" "\\• See the website privacy page for full details\\." ) diff --git a/app/core/config.py b/app/core/config.py index b33047c..47b73b0 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -32,6 +32,16 @@ class Settings(BaseSettings): rate_limit_download_enabled: bool = False rate_limit_download_requests: int = 200 + # Optional shared store (e.g. redis://...). When set, rate limits are counted + # across all Gunicorn workers + the Telegram bot, and bot preferences survive + # restarts. Empty or unreachable = per-process in-memory fallback. + redis_url: str = "" + telegram_prefs_ttl_days: int = 90 + + # Key for HMAC-hashing IPs / Telegram IDs in logs and telemetry. Unset = a + # random per-process key (hashes then only correlate within one process). + log_hash_secret: str = "" + # Proxy security: how many trusted proxies sit in front of the app. # 0 = direct exposure (ignore X-Forwarded-For entirely, use socket IP). # 1 = one reverse proxy (Nginx, Cloudflare, etc.). diff --git a/app/core/dependencies.py b/app/core/dependencies.py index cb17614..581d8be 100644 --- a/app/core/dependencies.py +++ b/app/core/dependencies.py @@ -4,6 +4,7 @@ from app.core.config import settings from app.core.errors import AppError, ErrorCode from app.services.metadata_service import MetadataService +from app.services.telemetry_service import hash_identifier from app.utils.logging_config import logger @@ -56,7 +57,7 @@ def _extract_client_ip(request: Request) -> str: def _raise_if_limited(client_ip: str, bucket: str) -> None: if not check_rate_limit(client_ip, bucket=bucket): - MetadataService.log_rate_limit_block(identifier=f"{bucket}:{client_ip}") + MetadataService.log_rate_limit_block(identifier=f"{bucket}:{hash_identifier(client_ip)}") raise AppError( ErrorCode.RATE_LIMIT_EXCEEDED, "Rate limit exceeded. Try again later.", diff --git a/app/core/redis_client.py b/app/core/redis_client.py new file mode 100644 index 0000000..c6d4524 --- /dev/null +++ b/app/core/redis_client.py @@ -0,0 +1,53 @@ +""" +Optional shared Redis connection (REDIS_URL). + +Callers must treat Redis as best-effort: get_redis() returns None when it is +not configured, and any redis.RedisError should fall back to in-memory state so +an outage degrades limits to per-process instead of taking the API down. +""" + +from __future__ import annotations + +import threading +import time +from typing import Optional + +import redis + +from app.core.config import settings +from app.utils.logging_config import logger + +# Keep request latency bounded if Redis is slow or gone +_SOCKET_TIMEOUT_SECONDS = 0.5 +_WARN_INTERVAL_SECONDS = 60 + +_client: Optional[redis.Redis] = None +_client_url = "" +_lock = threading.Lock() +_last_warning = 0.0 + + +def get_redis() -> Optional[redis.Redis]: + global _client, _client_url + url = (settings.redis_url or "").strip() + if not url: + return None + with _lock: + if _client is None or _client_url != url: + _client = redis.Redis.from_url( + url, + socket_timeout=_SOCKET_TIMEOUT_SECONDS, + socket_connect_timeout=_SOCKET_TIMEOUT_SECONDS, + decode_responses=True, + ) + _client_url = url + return _client + + +def warn_fallback(context: str, error: Exception) -> None: + """Log a Redis failure at most once a minute (a dead Redis would flood logs).""" + global _last_warning + now = time.monotonic() + if now - _last_warning >= _WARN_INTERVAL_SECONDS: + _last_warning = now + logger.warning("Redis unavailable (%s), using in-memory fallback: %s", context, error) diff --git a/app/services/metadata_service.py b/app/services/metadata_service.py index 12aea6d..c67d657 100644 --- a/app/services/metadata_service.py +++ b/app/services/metadata_service.py @@ -3,6 +3,7 @@ from urllib.parse import urlparse from app.utils.logging_config import logger +from app.utils.validators import validate_vimeo_url, validate_youtube_url class MetadataService: @@ -45,15 +46,20 @@ def log_success( logger.info(json.dumps(log_data)) @staticmethod - def _url_for_log(url: str, max_len: int = 160) -> str: - """Host + path only (no query/fragment) to avoid logging tokens or full PII.""" - if not url: - return "" - p = urlparse(url) - base = f"{p.scheme}://{p.netloc}{p.path}".strip() - if len(base) > max_len: - return base[: max_len - 1] + "…" - return base + def _video_ref(url: str, platform: str) -> Dict[str, Optional[str]]: + """ + Public video ID (as success logs already record) plus host — never the + raw URL, whose query/path can carry share tokens or tracking params. + """ + validate = {"youtube": validate_youtube_url, "vimeo": validate_vimeo_url}.get(platform) + video_id = None + if validate and url: + try: + video_id = validate(url) + except ValueError: + pass + host = (urlparse(url).hostname or "")[:100] if url else "" + return {"video_id": video_id, "host": host} @staticmethod def log_failure( @@ -67,7 +73,7 @@ def log_failure( """Logs a failed extraction workflow as structured JSON.""" log_data = { "event": "transcript_failure", - "url": MetadataService._url_for_log(url), + **MetadataService._video_ref(url, platform), "provider": platform, "error_code": error_code, "error_message": error_message, diff --git a/app/services/rate_limit_service.py b/app/services/rate_limit_service.py index 982c65a..13abe42 100644 --- a/app/services/rate_limit_service.py +++ b/app/services/rate_limit_service.py @@ -1,15 +1,19 @@ import time -from typing import Dict, Tuple +from typing import Dict, Optional, Tuple + +import redis from app.core.config import settings +from app.core.redis_client import get_redis, warn_fallback class RateLimitService: """ - In-memory sliding window counter keyed by (bucket, identifier). + Fixed-window counter keyed by (bucket, identifier). - Multi-worker note: each Gunicorn worker has its own store, so effective - limits can be up to ~N× the configured value under uniform load. + With REDIS_URL set, counts are shared by every Gunicorn worker and the + Telegram bot. Otherwise (or if Redis errors) each process keeps its own + in-memory store, so effective limits can be up to ~N× the configured value. """ def __init__(self) -> None: @@ -37,6 +41,10 @@ def is_allowed(self, identifier: str, bucket: str = "extract") -> bool: return True window_seconds = max(1, int(settings.rate_limit_window_seconds)) + shared = self._redis_is_allowed(identifier, bucket, limit, window_seconds) + if shared is not None: + return shared + now = time.time() self._cleanup(now) @@ -57,6 +65,26 @@ def is_allowed(self, identifier: str, bucket: str = "extract") -> bool: return False + @staticmethod + def _redis_is_allowed( + identifier: str, bucket: str, limit: int, window_seconds: int + ) -> Optional[bool]: + """Shared count via Redis; None means use the in-memory fallback.""" + client = get_redis() + if client is None: + return None + key = f"tf:rl:{bucket}:{identifier}" + try: + # SET NX starts the window with its TTL; INCR counts this request + pipe = client.pipeline(transaction=True) + pipe.set(key, 0, ex=window_seconds, nx=True) + pipe.incr(key) + _, count = pipe.execute() + except redis.RedisError as e: + warn_fallback("rate limit", e) + return None + return int(count) <= limit + def _cleanup(self, now: float) -> None: expired = [k for k, v in self._store.items() if now > v[1]] for k in expired: diff --git a/app/services/telemetry_service.py b/app/services/telemetry_service.py index cd6cf3f..5a01406 100644 --- a/app/services/telemetry_service.py +++ b/app/services/telemetry_service.py @@ -7,11 +7,18 @@ import json import hashlib +import hmac +import secrets from datetime import datetime, timezone from typing import Any, Dict, Optional +from app.core.config import settings from app.utils.logging_config import logger +# Plain SHA-256 of an IPv4 address or Telegram ID is reversible by enumeration, +# so identifiers are keyed. Without LOG_HASH_SECRET the key lives per process. +_HASH_KEY = (settings.log_hash_secret or secrets.token_hex(32)).encode() + # --------------------------------------------------------------------------- # Event allowlist # --------------------------------------------------------------------------- @@ -73,8 +80,8 @@ def bucket_duration(seconds: Optional[int]) -> Optional[str]: def hash_identifier(raw: str) -> str: - """One-way SHA-256 hash of an identifier (e.g. Telegram user ID).""" - return hashlib.sha256(raw.encode()).hexdigest()[:16] + """Keyed one-way hash of an identifier (IP, Telegram user ID) for logs.""" + return hmac.new(_HASH_KEY, raw.encode(), hashlib.sha256).hexdigest()[:16] # --------------------------------------------------------------------------- diff --git a/app/services/user_prefs_service.py b/app/services/user_prefs_service.py new file mode 100644 index 0000000..4eba9d0 --- /dev/null +++ b/app/services/user_prefs_service.py @@ -0,0 +1,67 @@ +""" +Telegram per-user preferences (preferred file format, timestamps on/off). + +Stored in Redis when REDIS_URL is set, so they survive restarts and deploys, +and expire after TELEGRAM_PREFS_TTL_DAYS of inactivity. Otherwise, or if Redis +errors, they live in process memory as before. +""" + +from __future__ import annotations + +from typing import Any, Dict + +import redis + +from app.core.config import settings +from app.core.redis_client import get_redis, warn_fallback + +FILE_FORMATS = ("txt", "pdf", "docx", "srt", "vtt") +DEFAULTS: Dict[str, Any] = {"last_file_format": "txt", "last_include_timestamps": False} + +_memory: Dict[str, Dict[str, Any]] = {} + + +def _key(user_id: str) -> str: + return f"tf:tgprefs:{user_id}" + + +def _decode(raw: Dict[str, str]) -> Dict[str, Any]: + """Redis hash (all strings) → typed prefs, ignoring unknown/invalid values.""" + prefs = dict(DEFAULTS) + if raw.get("last_file_format") in FILE_FORMATS: + prefs["last_file_format"] = raw["last_file_format"] + if raw.get("last_include_timestamps") in ("0", "1"): + prefs["last_include_timestamps"] = raw["last_include_timestamps"] == "1" + return prefs + + +def get_prefs(user_id: str) -> Dict[str, Any]: + client = get_redis() + if client is not None: + try: + return _decode(client.hgetall(_key(user_id))) + except redis.RedisError as e: + warn_fallback("telegram prefs", e) + return dict(_memory.get(user_id, DEFAULTS)) + + +def set_prefs(user_id: str, **updates: Any) -> None: + unknown = set(updates) - set(DEFAULTS) + if unknown: + raise ValueError(f"Unknown preference(s): {sorted(unknown)}") + + _memory[user_id] = {**_memory.get(user_id, DEFAULTS), **updates} + + client = get_redis() + if client is None: + return + encoded = { + k: ("1" if v else "0") if isinstance(v, bool) else str(v) for k, v in updates.items() + } + try: + pipe = client.pipeline(transaction=True) + pipe.hset(_key(user_id), mapping=encoded) + pipe.expire(_key(user_id), max(1, settings.telegram_prefs_ttl_days) * 86400) + pipe.execute() + except redis.RedisError as e: + warn_fallback("telegram prefs", e) diff --git a/docker-compose.yml b/docker-compose.yml index 5636611..7613ad1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,8 +41,12 @@ services: FILE_STORAGE_DIR: /app/data/tmp TRANSCRIPT_CACHE_DIR: /app/data/transcript_cache IN_PROCESS_CLEANUP_ENABLED: "true" + REDIS_URL: redis://redis:6379/0 volumes: - transcript_data:/app/data + depends_on: + redis: + condition: service_healthy restart: unless-stopped logging: driver: json-file @@ -76,6 +80,7 @@ services: TMPDIR: /tmp FILE_STORAGE_DIR: /app/data/tmp TRANSCRIPT_CACHE_DIR: /app/data/transcript_cache + REDIS_URL: redis://redis:6379/0 volumes: - transcript_data:/app/data restart: unless-stopped @@ -88,5 +93,20 @@ services: api: condition: service_healthy + # Shared rate-limit counters (all API workers + bot) and Telegram preferences + redis: + image: redis:7-alpine + container_name: transcriptflow-redis + command: ["redis-server", "--appendonly", "yes", "--maxmemory", "64mb", "--maxmemory-policy", "volatile-ttl"] + volumes: + - redis_data:/data + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 5 + volumes: transcript_data: + redis_data: diff --git a/render.yaml b/render.yaml index b51ec1b..2b84667 100644 --- a/render.yaml +++ b/render.yaml @@ -32,6 +32,11 @@ services: value: "false" - key: TRUSTED_PROXY_COUNT value: "1" + # Point at a Render Key Value (Redis) instance so limits are shared across workers + - key: REDIS_URL + sync: false + - key: LOG_HASH_SECRET + generateValue: true - key: MAX_VIDEO_DURATION_SECONDS value: "10800" - key: TRANSCRIPT_TIMEOUT_SECONDS diff --git a/requirements.txt b/requirements.txt index 31ef4ac..f4d527a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ python-dotenv==1.1.0 webvtt-py==0.5.1 requests==2.32.3 httpx==0.28.1 +redis==5.2.1 diff --git a/tests/test_shared_state_and_privacy.py b/tests/test_shared_state_and_privacy.py new file mode 100644 index 0000000..9117b19 --- /dev/null +++ b/tests/test_shared_state_and_privacy.py @@ -0,0 +1,209 @@ +"""Redis-shared rate limits / bot prefs (with in-memory fallback) and log privacy.""" + +import hashlib +import json +import os +import unittest +from unittest.mock import patch + +import redis + +from app.services import rate_limit_service as rls +from app.services import telemetry_service +from app.services import user_prefs_service as prefs +from app.services.metadata_service import MetadataService + + +class _FakePipeline: + def __init__(self, store): + self._store = store + self._ops = [] + + def __getattr__(self, name): + def queue(*args, **kwargs): + self._ops.append((name, args, kwargs)) + return self + return queue + + def execute(self): + return [getattr(self._store, name)(*a, **kw) for name, a, kw in self._ops] + + +class FakeRedis: + """Just the commands the app uses; TTLs recorded, not enforced.""" + + def __init__(self): + self.data = {} + self.ttl = {} + + def pipeline(self, transaction=True): + return _FakePipeline(self) + + def set(self, key, value, ex=None, nx=False): + if nx and key in self.data: + return None + self.data[key] = str(value) + self.ttl[key] = ex + return True + + def incr(self, key): + self.data[key] = str(int(self.data.get(key, 0)) + 1) + return int(self.data[key]) + + def hgetall(self, key): + return dict(self.data.get(key, {})) + + def hset(self, key, mapping): + self.data.setdefault(key, {}).update(mapping) + return len(mapping) + + def expire(self, key, seconds): + self.ttl[key] = seconds + return True + + +class _BrokenRedis: + def pipeline(self, transaction=True): + raise redis.ConnectionError("down") + + def hgetall(self, key): + raise redis.ConnectionError("down") + + +def _limits(s, extract=2): + s.rate_limit_requests = extract + s.rate_limit_convert_requests = 40 + s.rate_limit_events_requests = 100 + s.rate_limit_download_requests = 200 + s.rate_limit_window_seconds = 3600 + + +class TestSharedRateLimit(unittest.TestCase): + def test_count_is_shared_between_limiter_instances(self): + fake = FakeRedis() + worker_a, worker_b = rls.RateLimitService(), rls.RateLimitService() + with patch.object(rls, "get_redis", return_value=fake), patch.object(rls, "settings") as s: + _limits(s) + self.assertTrue(worker_a.is_allowed("1.1.1.1")) + self.assertTrue(worker_b.is_allowed("1.1.1.1")) + # Third request is blocked no matter which worker receives it + self.assertFalse(worker_a.is_allowed("1.1.1.1")) + self.assertTrue(worker_b.is_allowed("1.1.1.1", bucket="events")) + self.assertEqual(fake.ttl["tf:rl:extract:1.1.1.1"], 3600) + + def test_falls_back_to_memory_when_redis_down(self): + svc = rls.RateLimitService() + with patch.object(rls, "get_redis", return_value=_BrokenRedis()), patch.object( + rls, "settings" + ) as s, patch.object(rls, "warn_fallback") as warn: + _limits(s) + self.assertTrue(svc.is_allowed("ip")) + self.assertTrue(svc.is_allowed("ip")) + self.assertFalse(svc.is_allowed("ip")) + warn.assert_called() + + +@unittest.skipUnless(os.environ.get("TEST_REDIS_URL"), "TEST_REDIS_URL not set (CI provides one)") +class TestRealRedis(unittest.TestCase): + """Same behaviour against a real server (MULTI/SET NX EX/INCR, hash + TTL).""" + + def setUp(self): + self.client = redis.Redis.from_url(os.environ["TEST_REDIS_URL"], decode_responses=True) + self.client.flushdb() + self.addCleanup(self.client.flushdb) + + def test_rate_limit_shared_with_ttl(self): + with patch.object(rls, "get_redis", return_value=self.client), patch.object( + rls, "settings" + ) as s: + _limits(s) + results = [rls.RateLimitService().is_allowed("ip") for _ in range(3)] + self.assertEqual(results, [True, True, False]) + self.assertTrue(0 < self.client.ttl("tf:rl:extract:ip") <= 3600) + + def test_prefs_roundtrip(self): + prefs._memory.clear() + with patch.object(prefs, "get_redis", return_value=self.client): + prefs.set_prefs("42", last_file_format="vtt", last_include_timestamps=True) + prefs._memory.clear() + self.assertEqual(prefs.get_prefs("42")["last_file_format"], "vtt") + self.assertGreater(self.client.ttl("tf:tgprefs:42"), 86400) + + +class TestTelegramPrefs(unittest.TestCase): + def setUp(self): + prefs._memory.clear() + + def test_defaults(self): + with patch.object(prefs, "get_redis", return_value=None): + self.assertEqual(prefs.get_prefs("42"), prefs.DEFAULTS) + + def test_roundtrip_through_redis_survives_process_memory_loss(self): + fake = FakeRedis() + with patch.object(prefs, "get_redis", return_value=fake): + prefs.set_prefs("42", last_file_format="srt", last_include_timestamps=True) + prefs._memory.clear() # simulate a restart + self.assertEqual( + prefs.get_prefs("42"), + {"last_file_format": "srt", "last_include_timestamps": True}, + ) + self.assertEqual(fake.ttl["tf:tgprefs:42"], prefs.settings.telegram_prefs_ttl_days * 86400) + + def test_invalid_stored_values_ignored(self): + fake = FakeRedis() + fake.data["tf:tgprefs:7"] = {"last_file_format": "exe", "last_include_timestamps": "yes"} + with patch.object(prefs, "get_redis", return_value=fake): + self.assertEqual(prefs.get_prefs("7"), prefs.DEFAULTS) + + def test_memory_fallback_when_redis_down(self): + with patch.object(prefs, "get_redis", return_value=_BrokenRedis()), patch.object( + prefs, "warn_fallback" + ): + prefs.set_prefs("9", last_file_format="pdf") + self.assertEqual(prefs.get_prefs("9")["last_file_format"], "pdf") + + def test_unknown_pref_rejected(self): + with self.assertRaises(ValueError): + prefs.set_prefs("1", theme="dark") + + +class TestLogPrivacy(unittest.TestCase): + def test_hash_is_keyed_not_plain_sha256(self): + ip = "203.0.113.7" + hashed = telemetry_service.hash_identifier(ip) + self.assertEqual(hashed, telemetry_service.hash_identifier(ip)) + self.assertNotEqual(hashed, hashlib.sha256(ip.encode()).hexdigest()[:16]) + + def _logged_json(self, mock_logger_method): + return json.loads(mock_logger_method.call_args.args[0]) + + @patch("app.services.metadata_service.logger") + def test_failure_log_has_video_id_not_url(self, mock_logger): + MetadataService.log_failure( + url="https://youtu.be/dQw4w9WgXcQ?si=SHARETOKEN", + platform="youtube", + error_code="TRANSCRIPT_NOT_AVAILABLE", + error_message="x", + processing_time_ms=1, + ) + data = self._logged_json(mock_logger.error) + self.assertEqual(data["video_id"], "dQw4w9WgXcQ") + self.assertEqual(data["host"], "youtu.be") + self.assertNotIn("url", data) + self.assertNotIn("SHARETOKEN", json.dumps(data)) + + @patch("app.services.metadata_service.logger") + def test_rate_limit_block_log_hashes_ip(self, mock_logger): + from fastapi.testclient import TestClient + from app.main import app + + with patch("app.core.dependencies.check_rate_limit", return_value=False): + r = TestClient(app).post("/api/v1/events", json={}) + self.assertEqual(r.status_code, 429) + logged = mock_logger.warning.call_args.args[0] + self.assertNotIn("testclient", logged) # TestClient's socket "IP" + self.assertIn(telemetry_service.hash_identifier("testclient"), logged) + + +if __name__ == "__main__": + unittest.main() diff --git a/web/app/privacy/page.tsx b/web/app/privacy/page.tsx index 2fff989..ca227ac 100644 --- a/web/app/privacy/page.tsx +++ b/web/app/privacy/page.tsx @@ -69,7 +69,7 @@ export default function PrivacyPage() {

To improve reliability and reduce load on third-party caption sources, we may cache - caption/transcript data for a video for a limited period (on the order of hours, not + caption/transcript data for a video for a limited period (up to a few days, not permanent archives). This cache is operational, not a personal library or account history. It is separate from the short-lived download files described above.

@@ -85,6 +85,23 @@ export default function PrivacyPage() { to avoid storing unnecessary detail. Internal dashboards may show aggregated usage metrics, not a permanent copy of your full transcript text, for operations.

+

+ To enforce fair-use limits we count requests per IP address (or per Telegram user) for + the length of the limit window, typically 24 hours. Logs record IP addresses and + Telegram user IDs only as keyed one-way hashes, and record the public video ID rather + than the full link you submitted. +

+ + +
+

+ Telegram preferences +

+

+ The Telegram bot remembers your preferred file format and timestamp setting, keyed to + your Telegram user ID, so you do not have to choose them each time. This is forgotten + after about 90 days without use. +