Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,14 @@ MAX_TRANSCRIPT_SEGMENTS=20000
IN_PROCESS_CLEANUP_ENABLED=true
CLEANUP_INTERVAL_SECONDS=900

# --- Shared state (recommended in production)
# Redis shares rate-limit counts across Gunicorn workers + the Telegram bot and keeps
# bot preferences across restarts. Unset/unreachable = per-process in-memory fallback.
# REDIS_URL=redis://localhost:6379/0
# TELEGRAM_PREFS_TTL_DAYS=90
# Key for hashing IPs / Telegram IDs in logs (any long random string). Unset = random per process.
# LOG_HASH_SECRET=

# --- Internal metrics dashboard (Basic Auth at /internal/metrics)
METRICS_USERNAME=admin
# Change this before any public deployment.
Expand Down
11 changes: 11 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,16 @@ on:
jobs:
unit:
runs-on: ubuntu-latest
services:
redis:
image: redis:7-alpine
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 5s
--health-timeout 3s
--health-retries 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
Expand All @@ -19,6 +29,7 @@ jobs:
run: python -m unittest discover -s tests -v
env:
PYTHONPATH: .
TEST_REDIS_URL: redis://localhost:6379/15

web:
runs-on: ubuntu-latest
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ Core Value: **Fast → Reliable → Structured → Downloadable.**
- **Clean Formats**: Download as **TXT, PDF, and DOCX**, or **SRT / VTT** subtitles with cue timings. Files are named after the video title.
- **Timestamps**: Toggle perfectly formatted timestamps on or off.
- **Anonymous & Secure**: No accounts required. Generated files auto-delete after ~1 hour (TTL). Optional short-term caption cache (`TRANSCRIPT_CACHE_TTL_HOURS`) is pruned by the same cleanup loop.
- **Rate Limited & Guardrailed**: Separate extract vs convert quotas, max video duration (with caption-based fallback), segment/size caps, and timeout protections.
- **Rate Limited & Guardrailed**: Separate extract vs convert quotas (shared across workers and the bot via optional Redis), max video duration (with caption-based fallback), segment/size caps, and timeout protections.
- **Telemetry & Analytics**: Built-in lightweight JSON-structured logging and analytics tracking.

## 🏗 System Architecture
Expand Down Expand Up @@ -54,7 +54,7 @@ A premium, modern SaaS web app interface designed with:
### 2. Telegram Bot (`app/bot/telegram_bot.py`)
A seriously structured, minimal utility bot:
- **Instant Processing**: Send a URL, get a file. No conversational fluff.
- **In-Memory Preferences**: Remembers your preferred format (e.g., PDF) and timestamp setting while the bot process runs (reset on restart/deploy).
- **Remembered Preferences**: Remembers your preferred format (e.g., PDF) and timestamp setting — in Redis when `REDIS_URL` is set (kept ~90 days), otherwise in memory until the bot restarts.
- **Single Message UX**: Status updates (Extracting... $\rightarrow$ Formatting... $\rightarrow$ Ready) happen within a single edited message to keep chat history clean.
- **Branded Onboarding**: Clean `/start` experience with inline keyboard routing.

Expand Down
21 changes: 6 additions & 15 deletions app/bot/telegram_bot.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from app.services.formatter_service import SUBTITLE_FORMATS, TranscriptFormatter
from app.services.file_service import FileGenerator, download_filename
from app.services.rate_limit_service import check_rate_limit
from app.services import user_prefs_service
from app.services.metadata_service import MetadataService
from app.services.extract_guardrails import enforce_transcript_guardrails
from app.core.errors import AppError, ErrorCode, TranscriptFetchError
Expand All @@ -48,30 +49,19 @@
re.IGNORECASE,
)

# ---------------------------------------------------------------------------
# Per-user preferences (Phase 1 — in-memory)
# ---------------------------------------------------------------------------
_user_prefs: Dict[str, Dict] = {}

# ---------------------------------------------------------------------------
# Avatar path (bundled inside app/ for Docker availability)
# ---------------------------------------------------------------------------
_AVATAR_PATH = pathlib.Path(__file__).parent / "assets" / "telegram-avatar-1024.png"


def _get_prefs(user_id: str) -> dict:
"""Return preferences for *user_id*, creating defaults if absent."""
if user_id not in _user_prefs:
_user_prefs[user_id] = {
"last_file_format": "txt",
"last_include_timestamps": False,
}
return _user_prefs[user_id]
"""Preferences for *user_id* (Redis-backed when REDIS_URL is set)."""
return user_prefs_service.get_prefs(user_id)


def _set_pref(user_id: str, **kwargs) -> None:
prefs = _get_prefs(user_id)
prefs.update(kwargs)
user_prefs_service.set_prefs(user_id, **kwargs)


def _emit_event(event_name: str, user_id: str, props: dict | None = None) -> None:
Expand Down Expand Up @@ -160,7 +150,8 @@ def _escape_md2(text: str) -> str:
"*Privacy*\n\n"
"\\• No account required\\.\n"
"\\• Generated files auto\\-delete after about 1 hour\\.\n"
"\\• Short\\-term caption cache \\(hours\\) may reduce repeat fetches\\.\n"
"\\• Short\\-term caption cache \\(up to a few days\\) may reduce repeat fetches\\.\n"
"\\• Your format and timestamp preference is remembered for about 90 days\\.\n"
"\\• We do not keep permanent user accounts or history\\.\n"
"\\• See the website privacy page for full details\\."
)
Expand Down
10 changes: 10 additions & 0 deletions app/core/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,16 @@ class Settings(BaseSettings):
rate_limit_download_enabled: bool = False
rate_limit_download_requests: int = 200

# Optional shared store (e.g. redis://...). When set, rate limits are counted
# across all Gunicorn workers + the Telegram bot, and bot preferences survive
# restarts. Empty or unreachable = per-process in-memory fallback.
redis_url: str = ""
telegram_prefs_ttl_days: int = 90

# Key for HMAC-hashing IPs / Telegram IDs in logs and telemetry. Unset = a
# random per-process key (hashes then only correlate within one process).
log_hash_secret: str = ""

# Proxy security: how many trusted proxies sit in front of the app.
# 0 = direct exposure (ignore X-Forwarded-For entirely, use socket IP).
# 1 = one reverse proxy (Nginx, Cloudflare, etc.).
Expand Down
3 changes: 2 additions & 1 deletion app/core/dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from app.core.config import settings
from app.core.errors import AppError, ErrorCode
from app.services.metadata_service import MetadataService
from app.services.telemetry_service import hash_identifier
from app.utils.logging_config import logger


Expand Down Expand Up @@ -56,7 +57,7 @@ def _extract_client_ip(request: Request) -> str:

def _raise_if_limited(client_ip: str, bucket: str) -> None:
if not check_rate_limit(client_ip, bucket=bucket):
MetadataService.log_rate_limit_block(identifier=f"{bucket}:{client_ip}")
MetadataService.log_rate_limit_block(identifier=f"{bucket}:{hash_identifier(client_ip)}")
raise AppError(
ErrorCode.RATE_LIMIT_EXCEEDED,
"Rate limit exceeded. Try again later.",
Expand Down
53 changes: 53 additions & 0 deletions app/core/redis_client.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
"""
Optional shared Redis connection (REDIS_URL).

Callers must treat Redis as best-effort: get_redis() returns None when it is
not configured, and any redis.RedisError should fall back to in-memory state so
an outage degrades limits to per-process instead of taking the API down.
"""

from __future__ import annotations

import threading
import time
from typing import Optional

import redis

from app.core.config import settings
from app.utils.logging_config import logger

# Keep request latency bounded if Redis is slow or gone
_SOCKET_TIMEOUT_SECONDS = 0.5
_WARN_INTERVAL_SECONDS = 60

_client: Optional[redis.Redis] = None
_client_url = ""
_lock = threading.Lock()
_last_warning = 0.0


def get_redis() -> Optional[redis.Redis]:
global _client, _client_url
url = (settings.redis_url or "").strip()
if not url:
return None
with _lock:
if _client is None or _client_url != url:
_client = redis.Redis.from_url(
url,
socket_timeout=_SOCKET_TIMEOUT_SECONDS,
socket_connect_timeout=_SOCKET_TIMEOUT_SECONDS,
decode_responses=True,
)
_client_url = url
return _client


def warn_fallback(context: str, error: Exception) -> None:
"""Log a Redis failure at most once a minute (a dead Redis would flood logs)."""
global _last_warning
now = time.monotonic()
if now - _last_warning >= _WARN_INTERVAL_SECONDS:
_last_warning = now
logger.warning("Redis unavailable (%s), using in-memory fallback: %s", context, error)
26 changes: 16 additions & 10 deletions app/services/metadata_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from urllib.parse import urlparse

from app.utils.logging_config import logger
from app.utils.validators import validate_vimeo_url, validate_youtube_url


class MetadataService:
Expand Down Expand Up @@ -45,15 +46,20 @@ def log_success(
logger.info(json.dumps(log_data))

@staticmethod
def _url_for_log(url: str, max_len: int = 160) -> str:
"""Host + path only (no query/fragment) to avoid logging tokens or full PII."""
if not url:
return ""
p = urlparse(url)
base = f"{p.scheme}://{p.netloc}{p.path}".strip()
if len(base) > max_len:
return base[: max_len - 1] + "…"
return base
def _video_ref(url: str, platform: str) -> Dict[str, Optional[str]]:
"""
Public video ID (as success logs already record) plus host — never the
raw URL, whose query/path can carry share tokens or tracking params.
"""
validate = {"youtube": validate_youtube_url, "vimeo": validate_vimeo_url}.get(platform)
video_id = None
if validate and url:
try:
video_id = validate(url)
except ValueError:
pass
host = (urlparse(url).hostname or "")[:100] if url else ""
return {"video_id": video_id, "host": host}

@staticmethod
def log_failure(
Expand All @@ -67,7 +73,7 @@ def log_failure(
"""Logs a failed extraction workflow as structured JSON."""
log_data = {
"event": "transcript_failure",
"url": MetadataService._url_for_log(url),
**MetadataService._video_ref(url, platform),
"provider": platform,
"error_code": error_code,
"error_message": error_message,
Expand Down
36 changes: 32 additions & 4 deletions app/services/rate_limit_service.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,19 @@
import time
from typing import Dict, Tuple
from typing import Dict, Optional, Tuple

import redis

from app.core.config import settings
from app.core.redis_client import get_redis, warn_fallback


class RateLimitService:
"""
In-memory sliding window counter keyed by (bucket, identifier).
Fixed-window counter keyed by (bucket, identifier).

Multi-worker note: each Gunicorn worker has its own store, so effective
limits can be up to ~N× the configured value under uniform load.
With REDIS_URL set, counts are shared by every Gunicorn worker and the
Telegram bot. Otherwise (or if Redis errors) each process keeps its own
in-memory store, so effective limits can be up to ~N× the configured value.
"""

def __init__(self) -> None:
Expand Down Expand Up @@ -37,6 +41,10 @@ def is_allowed(self, identifier: str, bucket: str = "extract") -> bool:
return True

window_seconds = max(1, int(settings.rate_limit_window_seconds))
shared = self._redis_is_allowed(identifier, bucket, limit, window_seconds)
if shared is not None:
return shared

now = time.time()
self._cleanup(now)

Expand All @@ -57,6 +65,26 @@ def is_allowed(self, identifier: str, bucket: str = "extract") -> bool:

return False

@staticmethod
def _redis_is_allowed(
identifier: str, bucket: str, limit: int, window_seconds: int
) -> Optional[bool]:
"""Shared count via Redis; None means use the in-memory fallback."""
client = get_redis()
if client is None:
return None
key = f"tf:rl:{bucket}:{identifier}"
try:
# SET NX starts the window with its TTL; INCR counts this request
pipe = client.pipeline(transaction=True)
pipe.set(key, 0, ex=window_seconds, nx=True)
pipe.incr(key)
_, count = pipe.execute()
except redis.RedisError as e:
warn_fallback("rate limit", e)
return None
return int(count) <= limit

def _cleanup(self, now: float) -> None:
expired = [k for k, v in self._store.items() if now > v[1]]
for k in expired:
Expand Down
11 changes: 9 additions & 2 deletions app/services/telemetry_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,18 @@

import json
import hashlib
import hmac
import secrets
from datetime import datetime, timezone
from typing import Any, Dict, Optional

from app.core.config import settings
from app.utils.logging_config import logger

# Plain SHA-256 of an IPv4 address or Telegram ID is reversible by enumeration,
# so identifiers are keyed. Without LOG_HASH_SECRET the key lives per process.
_HASH_KEY = (settings.log_hash_secret or secrets.token_hex(32)).encode()

# ---------------------------------------------------------------------------
# Event allowlist
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -73,8 +80,8 @@ def bucket_duration(seconds: Optional[int]) -> Optional[str]:


def hash_identifier(raw: str) -> str:
"""One-way SHA-256 hash of an identifier (e.g. Telegram user ID)."""
return hashlib.sha256(raw.encode()).hexdigest()[:16]
"""Keyed one-way hash of an identifier (IP, Telegram user ID) for logs."""
return hmac.new(_HASH_KEY, raw.encode(), hashlib.sha256).hexdigest()[:16]


# ---------------------------------------------------------------------------
Expand Down
Loading
Loading