Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,26 @@ jobs:
run: python -m unittest discover -s tests -v
env:
PYTHONPATH: .

web:
runs-on: ubuntu-latest
defaults:
run:
working-directory: web
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
cache-dependency-path: web/package-lock.json
- name: Install dependencies
run: npm ci
- name: Lint
run: npm run lint
- name: Type-check
run: npx tsc --noEmit
- name: Build
run: npm run build
env:
NEXT_PUBLIC_API_BASE_URL: https://api.example.invalid
9 changes: 5 additions & 4 deletions app/api/v1_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,11 @@ async def extract_transcript(request: ExtractRequest):

layout_format = "timestamp" if request.include_timestamps else "clean"

# The worker thread outlives wait_for; the deadline makes it stop calling upstream too
deadline = time.monotonic() + settings.transcript_timeout_seconds
try:
transcript_data = await asyncio.wait_for(
asyncio.to_thread(get_transcript_from_url, request.url),
asyncio.to_thread(get_transcript_from_url, request.url, deadline),
timeout=settings.transcript_timeout_seconds,
)
except asyncio.TimeoutError:
Expand Down Expand Up @@ -124,9 +126,9 @@ async def extract_transcript(request: ExtractRequest):
err_msg = str(e)
processing_time_ms = int((time.time() - start_time) * 1000)
if isinstance(e, TranscriptFetchError):
code = e.code
code, status = e.code, e.status_code
else:
code = ErrorCode.INTERNAL_ERROR
code, status = ErrorCode.INTERNAL_ERROR, 500
logger.exception("Unexpected transcript fetch failure")
MetadataService.log_failure(
url=request.url,
Expand All @@ -144,7 +146,6 @@ async def extract_transcript(request: ExtractRequest):
processing_time_ms=processing_time_ms,
error_code=code.value,
)
status = 400 if code != ErrorCode.INTERNAL_ERROR else 500
client_msg = (
"An unexpected error occurred."
if code == ErrorCode.INTERNAL_ERROR
Expand Down
3 changes: 2 additions & 1 deletion app/bot/telegram_bot.py
Original file line number Diff line number Diff line change
Expand Up @@ -312,8 +312,9 @@ async def _process_url(

try:
# Extract (with timeout) -----------------------------------------
deadline = time.monotonic() + settings.transcript_timeout_seconds
transcript_data = await asyncio.wait_for(
asyncio.to_thread(get_transcript_from_url, url),
asyncio.to_thread(get_transcript_from_url, url, deadline),
timeout=settings.transcript_timeout_seconds,
)

Expand Down
11 changes: 11 additions & 0 deletions app/core/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ class ErrorCode(str, Enum):
VIDEO_TOO_LONG = "VIDEO_TOO_LONG"
RATE_LIMIT_EXCEEDED = "RATE_LIMIT_EXCEEDED"
UPSTREAM_TIMEOUT = "UPSTREAM_TIMEOUT"
# YouTube refused this server's IP (proxy missing/exhausted) — our problem, not the user's
UPSTREAM_BLOCKED = "UPSTREAM_BLOCKED"
FILE_NOT_FOUND = "FILE_NOT_FOUND"
FILE_EXPIRED = "FILE_EXPIRED"
CONVERT_FAILED = "CONVERT_FAILED"
Expand All @@ -35,11 +37,20 @@ class TranscriptFetchError(Exception):
failures by type instead of matching on message text.
"""

_STATUS = {
ErrorCode.UPSTREAM_BLOCKED: 503,
ErrorCode.UPSTREAM_TIMEOUT: 504,
}

def __init__(self, code: ErrorCode, message: str):
self.code = code
self.message = message
super().__init__(message)

@property
def status_code(self) -> int:
return self._STATUS.get(self.code, 400)


def success_response(data: Dict[str, Any]) -> Dict[str, Any]:
"""Wraps any payload in the standard success envelope."""
Expand Down
117 changes: 86 additions & 31 deletions app/services/transcript_service.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
from __future__ import annotations

import random
import re
import time
import requests
from concurrent.futures import Future, ThreadPoolExecutor
from typing import Any, Dict, List, Optional, Tuple

from youtube_transcript_api import (
Expand Down Expand Up @@ -35,6 +35,38 @@
"Please try again later."
)

YOUTUBE_TIMEOUT_MESSAGE = "Transcript provider did not respond in time."

_OEMBED_URL = "https://www.youtube.com/oembed"
# Title only names the download; never hold captions back for long waiting on it.
_METADATA_GRACE_SECONDS = 1.5
_metadata_pool = ThreadPoolExecutor(max_workers=4, thread_name_prefix="yt-metadata")


class _DeadlineExceeded(Exception):
"""The caller's time budget ran out before the next upstream request."""


class _DeadlineSession(requests.Session):
"""
Caps each request at the time left before *deadline* (time.monotonic()).
asyncio.wait_for cannot stop the worker thread, so this is what makes a
timed-out extract stop calling YouTube (and spending proxy bandwidth).
"""

def __init__(self, deadline: float):
super().__init__()
self._deadline = deadline

def request(self, *args, **kwargs):
remaining = self._deadline - time.monotonic()
if remaining <= 0:
raise _DeadlineExceeded()
timeout = kwargs.get("timeout")
kwargs["timeout"] = min(timeout, remaining) if isinstance(timeout, (int, float)) else remaining
return super().request(*args, **kwargs)


# Fallback search order after the client’s target language (never English-only).
_BASE_LANG_PRIORITY: Tuple[str, ...] = (
"en",
Expand Down Expand Up @@ -98,11 +130,12 @@ def _youtube_ip_block_attempts() -> int:
return min(n, 8)


def _youtube_api() -> YouTubeTranscriptApi:
def _youtube_api(deadline: Optional[float] = None) -> YouTubeTranscriptApi:
"""
Optional residential / generic proxies — see youtube-transcript-api README
https://github.com/jdepoix/youtube-transcript-api#working-around-ip-bans
"""
http_client = _DeadlineSession(deadline) if deadline is not None else None
proxy_config = None
try:
from youtube_transcript_api.proxies import GenericProxyConfig, WebshareProxyConfig
Expand Down Expand Up @@ -130,13 +163,14 @@ def _youtube_api() -> YouTubeTranscriptApi:
https_url=settings.youtube_https_proxy_url or None,
)

return YouTubeTranscriptApi(proxy_config=proxy_config) if proxy_config else YouTubeTranscriptApi()
return YouTubeTranscriptApi(proxy_config=proxy_config, http_client=http_client)


def get_transcript_from_url(url: str) -> Dict:
def get_transcript_from_url(url: str, deadline: Optional[float] = None) -> Dict:
"""
Fetch transcript. YouTube captions are always resolved to English (native or auto-translate).
Raises TranscriptFetchError for expected failures; anything else is a bug.
*deadline* (time.monotonic()) bounds upstream work so it stops once the caller gives up.
"""
try:
platform = detect_platform(url)
Expand All @@ -154,35 +188,36 @@ def get_transcript_from_url(url: str) -> Dict:
return out

if platform == "youtube":
result = get_youtube_transcript(video_id, url=url)
result = get_youtube_transcript(video_id, url=url, deadline=deadline)
else:
result = get_vimeo_transcript(video_id, url=url)
result = get_vimeo_transcript(video_id, url=url, deadline=deadline)
write_transcript_cache(platform, video_id, result)
return result


def _get_youtube_metadata(video_id: str) -> Dict:
"""Fetches video metadata directly from YouTube page."""
url = f"https://www.youtube.com/watch?v={video_id}"
headers = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
),
"Accept-Language": "en-US,en;q=0.9",
}
"""
Title via oEmbed: a small JSON endpoint that is rarely blocked, unlike the
watch page. Duration isn't exposed there; extract_guardrails estimates it
from the last caption cue when this is 0.
"""
fallback = {"title": f"YouTube Video {video_id}", "duration": 0}
try:
response = requests.get(url, headers=headers, timeout=10)
response = requests.get(
_OEMBED_URL,
params={"url": f"https://www.youtube.com/watch?v={video_id}", "format": "json"},
timeout=5,
)
response.raise_for_status()
html = response.text
title = str(response.json().get("title") or "").strip()
return {"title": title or fallback["title"], "duration": 0}
except (requests.RequestException, ValueError):
return fallback

title_match = re.search(r'<meta name="title" content="([^"]+)">', html)
title = title_match.group(1) if title_match else f"YouTube Video {video_id}"

duration_match = re.search(r'"approxDurationMs":"(\d+)"', html)
duration = int(int(duration_match.group(1)) / 1000) if duration_match else 0

return {"title": title, "duration": duration}
def _resolve_metadata(future: Future, video_id: str) -> Dict:
try:
return future.result(timeout=_METADATA_GRACE_SECONDS)
except Exception:
return {"title": f"YouTube Video {video_id}", "duration": 0}

Expand Down Expand Up @@ -254,9 +289,12 @@ def _youtube_transcript_fresh_core(
video_id: str,
languages: Optional[List[str]],
url: str,
metadata_future: Optional[Future] = None,
deadline: Optional[float] = None,
) -> Dict:
metadata = _get_youtube_metadata(video_id)
api = _youtube_api()
if metadata_future is None:
metadata_future = _metadata_pool.submit(_get_youtube_metadata, video_id)
api = _youtube_api(deadline)
tlist = api.list(video_id)

target = "en"
Expand All @@ -279,6 +317,7 @@ def _youtube_transcript_fresh_core(
{"text": item.text, "start": item.start, "duration": item.duration}
for item in fetched
]
metadata = _resolve_metadata(metadata_future, video_id)

return {
"provider": "youtube",
Expand All @@ -295,7 +334,9 @@ def _youtube_fetch_error(e: CouldNotRetrieveTranscript) -> TranscriptFetchError:
"""Map youtube-transcript-api failures to a user-safe TranscriptFetchError."""
na = ErrorCode.TRANSCRIPT_NOT_AVAILABLE
if isinstance(e, (IpBlocked, RequestBlocked)):
return TranscriptFetchError(na, YOUTUBE_TRANSCRIPT_UNAVAILABLE_EN_ONLY)
return TranscriptFetchError(
ErrorCode.UPSTREAM_BLOCKED, YOUTUBE_TRANSCRIPT_UNAVAILABLE_EN_ONLY
)
if isinstance(e, TranscriptsDisabled):
return TranscriptFetchError(na, "Transcripts are disabled for this video.")
if isinstance(e, VideoUnavailable):
Expand All @@ -312,24 +353,38 @@ def get_youtube_transcript(
video_id: str,
languages: Optional[List[str]] = None,
url: str = "",
deadline: Optional[float] = None,
) -> Dict:
"""
list() + find_transcript; never uses fetch(video_id) default ['en'] shortcut.
English-only: prefer English captions, else translate to English when available.
With residential proxy configured, retries a few times on IpBlocked / RequestBlocked.
With residential proxy configured, retries a few times on IpBlocked / RequestBlocked,
but never sleeps past *deadline*.
"""
# Title fetch runs alongside the caption requests (and survives retries)
metadata_future = _metadata_pool.submit(_get_youtube_metadata, video_id)
attempts = 1 if languages is not None else _youtube_ip_block_attempts()
for attempt in range(attempts):
try:
return _youtube_transcript_fresh_core(video_id, languages, url)
return _youtube_transcript_fresh_core(
video_id, languages, url, metadata_future=metadata_future, deadline=deadline
)
except (IpBlocked, RequestBlocked) as e:
if attempt < attempts - 1:
delay = settings.youtube_transcript_retry_backoff_seconds * (attempt + 1)
time.sleep(delay + random.uniform(0, 0.25))
delay = settings.youtube_transcript_retry_backoff_seconds * (attempt + 1)
delay += random.uniform(0, 0.25)
out_of_time = deadline is not None and time.monotonic() + delay >= deadline
if attempt < attempts - 1 and not out_of_time:
time.sleep(delay)
continue
raise _youtube_fetch_error(e) from e
except CouldNotRetrieveTranscript as e:
raise _youtube_fetch_error(e) from e
except (_DeadlineExceeded, requests.Timeout) as e:
raise TranscriptFetchError(ErrorCode.UPSTREAM_TIMEOUT, YOUTUBE_TIMEOUT_MESSAGE) from e
except requests.RequestException as e:
raise TranscriptFetchError(
ErrorCode.TRANSCRIPT_NOT_AVAILABLE, "Could not reach YouTube. Try again later."
) from e
raise AssertionError("unreachable")


Expand Down
25 changes: 22 additions & 3 deletions app/services/vimeo_service.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,30 @@
import requests
import re
import json
import time
import webvtt
from io import StringIO
from typing import List, Dict, Optional

from app.core.errors import ErrorCode, TranscriptFetchError

_NO_CAPTIONS = "No captions are available for this Vimeo video."
_REQUEST_TIMEOUT = 10

def get_vimeo_transcript(video_id: str, url: str = '') -> Dict:

def _timeout(deadline: Optional[float]) -> float:
"""Per-request timeout, never past the caller's deadline (time.monotonic())."""
if deadline is None:
return _REQUEST_TIMEOUT
remaining = deadline - time.monotonic()
if remaining <= 0:
raise TranscriptFetchError(
ErrorCode.UPSTREAM_TIMEOUT, "Transcript provider did not respond in time."
)
return min(_REQUEST_TIMEOUT, remaining)


def get_vimeo_transcript(video_id: str, url: str = '', deadline: Optional[float] = None) -> Dict:
"""
Fetches the transcript for a given Vimeo video ID.
Extracts text tracks from Vimeo player config and parses VTT format.
Expand All @@ -27,7 +42,7 @@ def get_vimeo_transcript(video_id: str, url: str = '') -> Dict:
'Accept-Language': 'en-US,en;q=0.9',
}

response = requests.get(player_url, headers=headers, timeout=10)
response = requests.get(player_url, headers=headers, timeout=_timeout(deadline))
response.raise_for_status()

html_content = response.text
Expand Down Expand Up @@ -75,7 +90,7 @@ def get_vimeo_transcript(video_id: str, url: str = '') -> Dict:
raise TranscriptFetchError(ErrorCode.TRANSCRIPT_NOT_AVAILABLE, _NO_CAPTIONS)

# Download the VTT file
vtt_response = requests.get(vtt_url, timeout=10)
vtt_response = requests.get(vtt_url, timeout=_timeout(deadline))
vtt_response.raise_for_status()

# Parse VTT content
Expand Down Expand Up @@ -106,6 +121,10 @@ def get_vimeo_transcript(video_id: str, url: str = '') -> Dict:
raise TranscriptFetchError(
ErrorCode.TRANSCRIPT_NOT_AVAILABLE, "Vimeo did not return this video. Try again later."
) from e
except requests.Timeout as e:
raise TranscriptFetchError(
ErrorCode.UPSTREAM_TIMEOUT, "Transcript provider did not respond in time."
) from e
except requests.RequestException as e:
raise TranscriptFetchError(
ErrorCode.TRANSCRIPT_NOT_AVAILABLE, "Could not reach Vimeo. Try again later."
Expand Down
Loading
Loading