diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e3d4b54..57f27ef 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,3 +19,26 @@ jobs: run: python -m unittest discover -s tests -v env: PYTHONPATH: . + + web: + runs-on: ubuntu-latest + defaults: + run: + working-directory: web + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: web/package-lock.json + - name: Install dependencies + run: npm ci + - name: Lint + run: npm run lint + - name: Type-check + run: npx tsc --noEmit + - name: Build + run: npm run build + env: + NEXT_PUBLIC_API_BASE_URL: https://api.example.invalid diff --git a/app/api/v1_routes.py b/app/api/v1_routes.py index 9083f04..f274313 100644 --- a/app/api/v1_routes.py +++ b/app/api/v1_routes.py @@ -90,9 +90,11 @@ async def extract_transcript(request: ExtractRequest): layout_format = "timestamp" if request.include_timestamps else "clean" + # The worker thread outlives wait_for; the deadline makes it stop calling upstream too + deadline = time.monotonic() + settings.transcript_timeout_seconds try: transcript_data = await asyncio.wait_for( - asyncio.to_thread(get_transcript_from_url, request.url), + asyncio.to_thread(get_transcript_from_url, request.url, deadline), timeout=settings.transcript_timeout_seconds, ) except asyncio.TimeoutError: @@ -124,9 +126,9 @@ async def extract_transcript(request: ExtractRequest): err_msg = str(e) processing_time_ms = int((time.time() - start_time) * 1000) if isinstance(e, TranscriptFetchError): - code = e.code + code, status = e.code, e.status_code else: - code = ErrorCode.INTERNAL_ERROR + code, status = ErrorCode.INTERNAL_ERROR, 500 logger.exception("Unexpected transcript fetch failure") MetadataService.log_failure( url=request.url, @@ -144,7 +146,6 @@ async def extract_transcript(request: ExtractRequest): processing_time_ms=processing_time_ms, error_code=code.value, ) - status = 400 if code != ErrorCode.INTERNAL_ERROR else 500 client_msg = ( "An unexpected error occurred." if code == ErrorCode.INTERNAL_ERROR diff --git a/app/bot/telegram_bot.py b/app/bot/telegram_bot.py index 6bc1a9f..0bfe3b4 100644 --- a/app/bot/telegram_bot.py +++ b/app/bot/telegram_bot.py @@ -312,8 +312,9 @@ async def _process_url( try: # Extract (with timeout) ----------------------------------------- + deadline = time.monotonic() + settings.transcript_timeout_seconds transcript_data = await asyncio.wait_for( - asyncio.to_thread(get_transcript_from_url, url), + asyncio.to_thread(get_transcript_from_url, url, deadline), timeout=settings.transcript_timeout_seconds, ) diff --git a/app/core/errors.py b/app/core/errors.py index 1178c97..f5ad5ab 100644 --- a/app/core/errors.py +++ b/app/core/errors.py @@ -10,6 +10,8 @@ class ErrorCode(str, Enum): VIDEO_TOO_LONG = "VIDEO_TOO_LONG" RATE_LIMIT_EXCEEDED = "RATE_LIMIT_EXCEEDED" UPSTREAM_TIMEOUT = "UPSTREAM_TIMEOUT" + # YouTube refused this server's IP (proxy missing/exhausted) — our problem, not the user's + UPSTREAM_BLOCKED = "UPSTREAM_BLOCKED" FILE_NOT_FOUND = "FILE_NOT_FOUND" FILE_EXPIRED = "FILE_EXPIRED" CONVERT_FAILED = "CONVERT_FAILED" @@ -35,11 +37,20 @@ class TranscriptFetchError(Exception): failures by type instead of matching on message text. """ + _STATUS = { + ErrorCode.UPSTREAM_BLOCKED: 503, + ErrorCode.UPSTREAM_TIMEOUT: 504, + } + def __init__(self, code: ErrorCode, message: str): self.code = code self.message = message super().__init__(message) + @property + def status_code(self) -> int: + return self._STATUS.get(self.code, 400) + def success_response(data: Dict[str, Any]) -> Dict[str, Any]: """Wraps any payload in the standard success envelope.""" diff --git a/app/services/transcript_service.py b/app/services/transcript_service.py index 922aa05..be9d452 100644 --- a/app/services/transcript_service.py +++ b/app/services/transcript_service.py @@ -1,9 +1,9 @@ from __future__ import annotations import random -import re import time import requests +from concurrent.futures import Future, ThreadPoolExecutor from typing import Any, Dict, List, Optional, Tuple from youtube_transcript_api import ( @@ -35,6 +35,38 @@ "Please try again later." ) +YOUTUBE_TIMEOUT_MESSAGE = "Transcript provider did not respond in time." + +_OEMBED_URL = "https://www.youtube.com/oembed" +# Title only names the download; never hold captions back for long waiting on it. +_METADATA_GRACE_SECONDS = 1.5 +_metadata_pool = ThreadPoolExecutor(max_workers=4, thread_name_prefix="yt-metadata") + + +class _DeadlineExceeded(Exception): + """The caller's time budget ran out before the next upstream request.""" + + +class _DeadlineSession(requests.Session): + """ + Caps each request at the time left before *deadline* (time.monotonic()). + asyncio.wait_for cannot stop the worker thread, so this is what makes a + timed-out extract stop calling YouTube (and spending proxy bandwidth). + """ + + def __init__(self, deadline: float): + super().__init__() + self._deadline = deadline + + def request(self, *args, **kwargs): + remaining = self._deadline - time.monotonic() + if remaining <= 0: + raise _DeadlineExceeded() + timeout = kwargs.get("timeout") + kwargs["timeout"] = min(timeout, remaining) if isinstance(timeout, (int, float)) else remaining + return super().request(*args, **kwargs) + + # Fallback search order after the client’s target language (never English-only). _BASE_LANG_PRIORITY: Tuple[str, ...] = ( "en", @@ -98,11 +130,12 @@ def _youtube_ip_block_attempts() -> int: return min(n, 8) -def _youtube_api() -> YouTubeTranscriptApi: +def _youtube_api(deadline: Optional[float] = None) -> YouTubeTranscriptApi: """ Optional residential / generic proxies — see youtube-transcript-api README https://github.com/jdepoix/youtube-transcript-api#working-around-ip-bans """ + http_client = _DeadlineSession(deadline) if deadline is not None else None proxy_config = None try: from youtube_transcript_api.proxies import GenericProxyConfig, WebshareProxyConfig @@ -130,13 +163,14 @@ def _youtube_api() -> YouTubeTranscriptApi: https_url=settings.youtube_https_proxy_url or None, ) - return YouTubeTranscriptApi(proxy_config=proxy_config) if proxy_config else YouTubeTranscriptApi() + return YouTubeTranscriptApi(proxy_config=proxy_config, http_client=http_client) -def get_transcript_from_url(url: str) -> Dict: +def get_transcript_from_url(url: str, deadline: Optional[float] = None) -> Dict: """ Fetch transcript. YouTube captions are always resolved to English (native or auto-translate). Raises TranscriptFetchError for expected failures; anything else is a bug. + *deadline* (time.monotonic()) bounds upstream work so it stops once the caller gives up. """ try: platform = detect_platform(url) @@ -154,35 +188,36 @@ def get_transcript_from_url(url: str) -> Dict: return out if platform == "youtube": - result = get_youtube_transcript(video_id, url=url) + result = get_youtube_transcript(video_id, url=url, deadline=deadline) else: - result = get_vimeo_transcript(video_id, url=url) + result = get_vimeo_transcript(video_id, url=url, deadline=deadline) write_transcript_cache(platform, video_id, result) return result def _get_youtube_metadata(video_id: str) -> Dict: - """Fetches video metadata directly from YouTube page.""" - url = f"https://www.youtube.com/watch?v={video_id}" - headers = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" - ), - "Accept-Language": "en-US,en;q=0.9", - } + """ + Title via oEmbed: a small JSON endpoint that is rarely blocked, unlike the + watch page. Duration isn't exposed there; extract_guardrails estimates it + from the last caption cue when this is 0. + """ + fallback = {"title": f"YouTube Video {video_id}", "duration": 0} try: - response = requests.get(url, headers=headers, timeout=10) + response = requests.get( + _OEMBED_URL, + params={"url": f"https://www.youtube.com/watch?v={video_id}", "format": "json"}, + timeout=5, + ) response.raise_for_status() - html = response.text + title = str(response.json().get("title") or "").strip() + return {"title": title or fallback["title"], "duration": 0} + except (requests.RequestException, ValueError): + return fallback - title_match = re.search(r'', html) - title = title_match.group(1) if title_match else f"YouTube Video {video_id}" - duration_match = re.search(r'"approxDurationMs":"(\d+)"', html) - duration = int(int(duration_match.group(1)) / 1000) if duration_match else 0 - - return {"title": title, "duration": duration} +def _resolve_metadata(future: Future, video_id: str) -> Dict: + try: + return future.result(timeout=_METADATA_GRACE_SECONDS) except Exception: return {"title": f"YouTube Video {video_id}", "duration": 0} @@ -254,9 +289,12 @@ def _youtube_transcript_fresh_core( video_id: str, languages: Optional[List[str]], url: str, + metadata_future: Optional[Future] = None, + deadline: Optional[float] = None, ) -> Dict: - metadata = _get_youtube_metadata(video_id) - api = _youtube_api() + if metadata_future is None: + metadata_future = _metadata_pool.submit(_get_youtube_metadata, video_id) + api = _youtube_api(deadline) tlist = api.list(video_id) target = "en" @@ -279,6 +317,7 @@ def _youtube_transcript_fresh_core( {"text": item.text, "start": item.start, "duration": item.duration} for item in fetched ] + metadata = _resolve_metadata(metadata_future, video_id) return { "provider": "youtube", @@ -295,7 +334,9 @@ def _youtube_fetch_error(e: CouldNotRetrieveTranscript) -> TranscriptFetchError: """Map youtube-transcript-api failures to a user-safe TranscriptFetchError.""" na = ErrorCode.TRANSCRIPT_NOT_AVAILABLE if isinstance(e, (IpBlocked, RequestBlocked)): - return TranscriptFetchError(na, YOUTUBE_TRANSCRIPT_UNAVAILABLE_EN_ONLY) + return TranscriptFetchError( + ErrorCode.UPSTREAM_BLOCKED, YOUTUBE_TRANSCRIPT_UNAVAILABLE_EN_ONLY + ) if isinstance(e, TranscriptsDisabled): return TranscriptFetchError(na, "Transcripts are disabled for this video.") if isinstance(e, VideoUnavailable): @@ -312,24 +353,38 @@ def get_youtube_transcript( video_id: str, languages: Optional[List[str]] = None, url: str = "", + deadline: Optional[float] = None, ) -> Dict: """ list() + find_transcript; never uses fetch(video_id) default ['en'] shortcut. English-only: prefer English captions, else translate to English when available. - With residential proxy configured, retries a few times on IpBlocked / RequestBlocked. + With residential proxy configured, retries a few times on IpBlocked / RequestBlocked, + but never sleeps past *deadline*. """ + # Title fetch runs alongside the caption requests (and survives retries) + metadata_future = _metadata_pool.submit(_get_youtube_metadata, video_id) attempts = 1 if languages is not None else _youtube_ip_block_attempts() for attempt in range(attempts): try: - return _youtube_transcript_fresh_core(video_id, languages, url) + return _youtube_transcript_fresh_core( + video_id, languages, url, metadata_future=metadata_future, deadline=deadline + ) except (IpBlocked, RequestBlocked) as e: - if attempt < attempts - 1: - delay = settings.youtube_transcript_retry_backoff_seconds * (attempt + 1) - time.sleep(delay + random.uniform(0, 0.25)) + delay = settings.youtube_transcript_retry_backoff_seconds * (attempt + 1) + delay += random.uniform(0, 0.25) + out_of_time = deadline is not None and time.monotonic() + delay >= deadline + if attempt < attempts - 1 and not out_of_time: + time.sleep(delay) continue raise _youtube_fetch_error(e) from e except CouldNotRetrieveTranscript as e: raise _youtube_fetch_error(e) from e + except (_DeadlineExceeded, requests.Timeout) as e: + raise TranscriptFetchError(ErrorCode.UPSTREAM_TIMEOUT, YOUTUBE_TIMEOUT_MESSAGE) from e + except requests.RequestException as e: + raise TranscriptFetchError( + ErrorCode.TRANSCRIPT_NOT_AVAILABLE, "Could not reach YouTube. Try again later." + ) from e raise AssertionError("unreachable") diff --git a/app/services/vimeo_service.py b/app/services/vimeo_service.py index f71487b..662daac 100644 --- a/app/services/vimeo_service.py +++ b/app/services/vimeo_service.py @@ -1,6 +1,7 @@ import requests import re import json +import time import webvtt from io import StringIO from typing import List, Dict, Optional @@ -8,8 +9,22 @@ from app.core.errors import ErrorCode, TranscriptFetchError _NO_CAPTIONS = "No captions are available for this Vimeo video." +_REQUEST_TIMEOUT = 10 -def get_vimeo_transcript(video_id: str, url: str = '') -> Dict: + +def _timeout(deadline: Optional[float]) -> float: + """Per-request timeout, never past the caller's deadline (time.monotonic()).""" + if deadline is None: + return _REQUEST_TIMEOUT + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TranscriptFetchError( + ErrorCode.UPSTREAM_TIMEOUT, "Transcript provider did not respond in time." + ) + return min(_REQUEST_TIMEOUT, remaining) + + +def get_vimeo_transcript(video_id: str, url: str = '', deadline: Optional[float] = None) -> Dict: """ Fetches the transcript for a given Vimeo video ID. Extracts text tracks from Vimeo player config and parses VTT format. @@ -27,7 +42,7 @@ def get_vimeo_transcript(video_id: str, url: str = '') -> Dict: 'Accept-Language': 'en-US,en;q=0.9', } - response = requests.get(player_url, headers=headers, timeout=10) + response = requests.get(player_url, headers=headers, timeout=_timeout(deadline)) response.raise_for_status() html_content = response.text @@ -75,7 +90,7 @@ def get_vimeo_transcript(video_id: str, url: str = '') -> Dict: raise TranscriptFetchError(ErrorCode.TRANSCRIPT_NOT_AVAILABLE, _NO_CAPTIONS) # Download the VTT file - vtt_response = requests.get(vtt_url, timeout=10) + vtt_response = requests.get(vtt_url, timeout=_timeout(deadline)) vtt_response.raise_for_status() # Parse VTT content @@ -106,6 +121,10 @@ def get_vimeo_transcript(video_id: str, url: str = '') -> Dict: raise TranscriptFetchError( ErrorCode.TRANSCRIPT_NOT_AVAILABLE, "Vimeo did not return this video. Try again later." ) from e + except requests.Timeout as e: + raise TranscriptFetchError( + ErrorCode.UPSTREAM_TIMEOUT, "Transcript provider did not respond in time." + ) from e except requests.RequestException as e: raise TranscriptFetchError( ErrorCode.TRANSCRIPT_NOT_AVAILABLE, "Could not reach Vimeo. Try again later." diff --git a/tests/test_extract_reliability.py b/tests/test_extract_reliability.py new file mode 100644 index 0000000..5a153fb --- /dev/null +++ b/tests/test_extract_reliability.py @@ -0,0 +1,155 @@ +"""Tests for extract time budget: concurrent metadata, deadlines, IP-block reporting.""" + +import threading +import time +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import requests +from fastapi.testclient import TestClient +from youtube_transcript_api import IpBlocked + +from app.core.errors import ErrorCode, TranscriptFetchError +from app.main import app +from app.services import transcript_service as ts +from app.services.vimeo_service import get_vimeo_transcript + +client = TestClient(app) + + +class _Fetched(list): + language_code = "en" + + +def _mock_api(on_list=None): + """YouTubeTranscriptApi stand-in whose list() returns one English track.""" + tr = MagicMock(language_code="en", is_translatable=False) + tr.fetch.return_value = _Fetched([SimpleNamespace(text="hi", start=0.0, duration=1.0)]) + tlist = MagicMock() + tlist.find_transcript.return_value = tr + + def _list(_video_id): + if on_list: + on_list() + return tlist + + api = MagicMock() + api.list.side_effect = _list + return api + + +class TestConcurrentMetadata(unittest.TestCase): + def test_title_fetch_overlaps_caption_fetch(self): + captions_started = threading.Event() + + def metadata(_vid): + # Only succeeds if it runs while captions are being fetched + if not captions_started.wait(timeout=1): + return {"title": "SEQUENTIAL", "duration": 0} + return {"title": "Concurrent", "duration": 0} + + with patch.object(ts, "_get_youtube_metadata", side_effect=metadata), patch.object( + ts, "YouTubeTranscriptApi", return_value=_mock_api(captions_started.set) + ): + result = ts.get_youtube_transcript("vid") + self.assertEqual(result["title"], "Concurrent") + + def test_slow_title_does_not_hold_captions(self): + release = threading.Event() + self.addCleanup(release.set) + + def metadata(_vid): + release.wait(timeout=5) + return {"title": "Late", "duration": 0} + + with patch.object(ts, "_get_youtube_metadata", side_effect=metadata), patch.object( + ts, "YouTubeTranscriptApi", return_value=_mock_api() + ), patch.object(ts, "_METADATA_GRACE_SECONDS", 0.05): + started = time.monotonic() + result = ts.get_youtube_transcript("vid") + self.assertLess(time.monotonic() - started, 1) + self.assertEqual(result["title"], "YouTube Video vid") + self.assertEqual(result["segments"][0]["text"], "hi") + + +class TestOembedMetadata(unittest.TestCase): + @patch("app.services.transcript_service.requests.get") + def test_title_from_oembed(self, mock_get): + mock_get.return_value.json.return_value = {"title": "Tom & Jerry"} + self.assertEqual( + ts._get_youtube_metadata("vid"), {"title": "Tom & Jerry", "duration": 0} + ) + + @patch("app.services.transcript_service.requests.get") + def test_fallback_on_failure(self, mock_get): + mock_get.side_effect = requests.ConnectionError() + self.assertEqual(ts._get_youtube_metadata("vid")["title"], "YouTube Video vid") + + +class TestDeadline(unittest.TestCase): + def test_session_refuses_requests_after_deadline(self): + session = ts._DeadlineSession(time.monotonic() - 1) + with self.assertRaises(ts._DeadlineExceeded): + session.get("https://example.invalid") + + def test_session_caps_request_timeout(self): + session = ts._DeadlineSession(time.monotonic() + 2) + with patch("requests.Session.request") as mock_request: + session.get("https://example.invalid", timeout=30) + self.assertLessEqual(mock_request.call_args.kwargs["timeout"], 2) + + @patch("app.services.transcript_service.time.sleep") + @patch("app.services.transcript_service._youtube_ip_block_attempts", return_value=5) + @patch("app.services.transcript_service._youtube_transcript_fresh_core") + def test_no_retry_sleep_past_deadline(self, mock_core, _attempts, mock_sleep): + mock_core.side_effect = IpBlocked("vid") + with self.assertRaises(TranscriptFetchError) as ctx: + ts.get_youtube_transcript("vid", deadline=time.monotonic() + 0.1) + mock_sleep.assert_not_called() + self.assertEqual(mock_core.call_count, 1) + self.assertEqual(ctx.exception.code, ErrorCode.UPSTREAM_BLOCKED) + + @patch("app.services.transcript_service._youtube_transcript_fresh_core") + def test_deadline_exceeded_is_upstream_timeout(self, mock_core): + mock_core.side_effect = ts._DeadlineExceeded() + with self.assertRaises(TranscriptFetchError) as ctx: + ts.get_youtube_transcript("vid", deadline=time.monotonic()) + self.assertEqual(ctx.exception.code, ErrorCode.UPSTREAM_TIMEOUT) + self.assertEqual(ctx.exception.status_code, 504) + + @patch("app.services.vimeo_service.requests.get") + def test_vimeo_stops_after_deadline(self, mock_get): + with self.assertRaises(TranscriptFetchError) as ctx: + get_vimeo_transcript("123", deadline=time.monotonic() - 1) + self.assertEqual(ctx.exception.code, ErrorCode.UPSTREAM_TIMEOUT) + mock_get.assert_not_called() + + +class TestBlockedResponse(unittest.TestCase): + @patch("app.api.v1_routes.get_transcript_from_url") + def test_ip_block_returns_503_with_own_code(self, mock_fetch): + mock_fetch.side_effect = ts._youtube_fetch_error(IpBlocked("vid")) + r = client.post( + "/api/v1/extract", + json={"url": "https://www.youtube.com/watch?v=testvideo12", "include_timestamps": False}, + ) + self.assertEqual(r.status_code, 503) + self.assertEqual(r.json()["error"]["code"], "UPSTREAM_BLOCKED") + self.assertIn("blocked this server", r.json()["error"]["message"]) + + @patch("app.api.v1_routes.get_transcript_from_url") + def test_api_passes_deadline_within_budget(self, mock_fetch): + mock_fetch.side_effect = TranscriptFetchError(ErrorCode.TRANSCRIPT_NOT_AVAILABLE, "x") + before = time.monotonic() + client.post( + "/api/v1/extract", + json={"url": "https://www.youtube.com/watch?v=testvideo12", "include_timestamps": False}, + ) + deadline = mock_fetch.call_args.args[1] + self.assertGreater(deadline, before) + self.assertLessEqual(deadline, time.monotonic() + ts.settings.transcript_timeout_seconds) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_get_transcript_from_url.py b/tests/test_get_transcript_from_url.py index 650eb99..7f86720 100644 --- a/tests/test_get_transcript_from_url.py +++ b/tests/test_get_transcript_from_url.py @@ -43,8 +43,8 @@ def test_vimeo_delegates(self, mock_vm, _mock_read, _mock_write): "segments": [], } url = "https://vimeo.com/999" - result = get_transcript_from_url(url) - mock_vm.assert_called_once_with("999", url=url) + result = get_transcript_from_url(url, deadline=123.0) + mock_vm.assert_called_once_with("999", url=url, deadline=123.0) self.assertEqual(result["provider"], "vimeo") @patch("app.services.transcript_service.get_youtube_transcript") diff --git a/web/app/components/home/BentoGrid.tsx b/web/app/components/home/BentoGrid.tsx deleted file mode 100644 index 9cfc828..0000000 --- a/web/app/components/home/BentoGrid.tsx +++ /dev/null @@ -1,84 +0,0 @@ -import React from 'react'; -import { Zap, Lock, Globe, Database, Bot, Share2 } from 'lucide-react'; - -export const BentoGrid: React.FC = () => { - return ( -
Built for scale. Designed for speed.
-- Don't leave your messenger. Send a link to @TranscriptFlowBot and receive your file instantly. The core engine runs 24/7 on our distributed infrastructure. -
-1-hour video processed in under 10 seconds.
-No permanent storage of processed data.
-- We generate cleaner data than raw captions. Perfect for feeding into GPT-4, Claude, or Llama for summarization and repurposing. -
-TXT, PDF, DOCX, SRT, VTT. Your choice.
-YouTube & Vimeo supported. Auto-detects languages.
-- v2.8.1 / stable / zero-latency -
-- {result.summary} -
-- The ultimate infrastructure for video-to-text conversion. - Process YouTube, Vimeo, and Audio instantly via our Web Dashboard or Telegram Bot. -
- -