From 6cccace887d8cbb864390f2e76a2671d66ae7877 Mon Sep 17 00:00:00 2001 From: David Peluola <79669430+Adeolu05@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:44:13 +0100 Subject: [PATCH] fix: keep extracts inside their time budget; flag YouTube IP blocks - Fetch the YouTube title via oEmbed, concurrently with captions, instead of a sequential watch-page scrape (up to 10s) before the caption request. Duration now comes from the guardrails' last-cue estimate. - Carry a monotonic deadline from the API/bot into the fetch: every YouTube request is capped at the time left, IP-block retries never sleep past it, and Vimeo requests are bounded too. Timed-out extracts stop calling upstream instead of running on in the worker thread. - New UPSTREAM_BLOCKED error code (503) for IpBlocked/RequestBlocked so the metrics dashboard separates proxy failures from videos without captions; YouTube timeouts map to UPSTREAM_TIMEOUT (504), network errors to a clear message instead of a 500. - CI: lint, type-check, and build the web app. - Web: remove seven unused components from an earlier homepage and fix the remaining lint errors (useHydrated replaces setState-in-effect guards). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests.yml | 23 +++ app/api/v1_routes.py | 9 +- app/bot/telegram_bot.py | 3 +- app/core/errors.py | 11 ++ app/services/transcript_service.py | 117 ++++++++++---- app/services/vimeo_service.py | 25 ++- tests/test_extract_reliability.py | 155 ++++++++++++++++++ tests/test_get_transcript_from_url.py | 4 +- web/app/components/home/BentoGrid.tsx | 84 ---------- web/app/components/home/BotDemo.tsx | 2 +- web/app/components/home/Generator.tsx | 217 -------------------------- web/app/components/home/Hero.tsx | 66 -------- web/app/components/layout/Footer.tsx | 4 - web/app/components/layout/Navbar.tsx | 87 ----------- web/app/components/ui/Button.tsx | 43 ----- web/app/types/index.ts | 22 --- web/components/Navbar.tsx | 5 +- web/components/ThemeToggle.tsx | 5 +- web/lib/useHydrated.ts | 12 ++ 19 files changed, 323 insertions(+), 571 deletions(-) create mode 100644 tests/test_extract_reliability.py delete mode 100644 web/app/components/home/BentoGrid.tsx delete mode 100644 web/app/components/home/Generator.tsx delete mode 100644 web/app/components/home/Hero.tsx delete mode 100644 web/app/components/layout/Footer.tsx delete mode 100644 web/app/components/layout/Navbar.tsx delete mode 100644 web/app/components/ui/Button.tsx delete mode 100644 web/app/types/index.ts create mode 100644 web/lib/useHydrated.ts diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e3d4b54..57f27ef 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,3 +19,26 @@ jobs: run: python -m unittest discover -s tests -v env: PYTHONPATH: . + + web: + runs-on: ubuntu-latest + defaults: + run: + working-directory: web + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: web/package-lock.json + - name: Install dependencies + run: npm ci + - name: Lint + run: npm run lint + - name: Type-check + run: npx tsc --noEmit + - name: Build + run: npm run build + env: + NEXT_PUBLIC_API_BASE_URL: https://api.example.invalid diff --git a/app/api/v1_routes.py b/app/api/v1_routes.py index 9083f04..f274313 100644 --- a/app/api/v1_routes.py +++ b/app/api/v1_routes.py @@ -90,9 +90,11 @@ async def extract_transcript(request: ExtractRequest): layout_format = "timestamp" if request.include_timestamps else "clean" + # The worker thread outlives wait_for; the deadline makes it stop calling upstream too + deadline = time.monotonic() + settings.transcript_timeout_seconds try: transcript_data = await asyncio.wait_for( - asyncio.to_thread(get_transcript_from_url, request.url), + asyncio.to_thread(get_transcript_from_url, request.url, deadline), timeout=settings.transcript_timeout_seconds, ) except asyncio.TimeoutError: @@ -124,9 +126,9 @@ async def extract_transcript(request: ExtractRequest): err_msg = str(e) processing_time_ms = int((time.time() - start_time) * 1000) if isinstance(e, TranscriptFetchError): - code = e.code + code, status = e.code, e.status_code else: - code = ErrorCode.INTERNAL_ERROR + code, status = ErrorCode.INTERNAL_ERROR, 500 logger.exception("Unexpected transcript fetch failure") MetadataService.log_failure( url=request.url, @@ -144,7 +146,6 @@ async def extract_transcript(request: ExtractRequest): processing_time_ms=processing_time_ms, error_code=code.value, ) - status = 400 if code != ErrorCode.INTERNAL_ERROR else 500 client_msg = ( "An unexpected error occurred." if code == ErrorCode.INTERNAL_ERROR diff --git a/app/bot/telegram_bot.py b/app/bot/telegram_bot.py index 6bc1a9f..0bfe3b4 100644 --- a/app/bot/telegram_bot.py +++ b/app/bot/telegram_bot.py @@ -312,8 +312,9 @@ async def _process_url( try: # Extract (with timeout) ----------------------------------------- + deadline = time.monotonic() + settings.transcript_timeout_seconds transcript_data = await asyncio.wait_for( - asyncio.to_thread(get_transcript_from_url, url), + asyncio.to_thread(get_transcript_from_url, url, deadline), timeout=settings.transcript_timeout_seconds, ) diff --git a/app/core/errors.py b/app/core/errors.py index 1178c97..f5ad5ab 100644 --- a/app/core/errors.py +++ b/app/core/errors.py @@ -10,6 +10,8 @@ class ErrorCode(str, Enum): VIDEO_TOO_LONG = "VIDEO_TOO_LONG" RATE_LIMIT_EXCEEDED = "RATE_LIMIT_EXCEEDED" UPSTREAM_TIMEOUT = "UPSTREAM_TIMEOUT" + # YouTube refused this server's IP (proxy missing/exhausted) — our problem, not the user's + UPSTREAM_BLOCKED = "UPSTREAM_BLOCKED" FILE_NOT_FOUND = "FILE_NOT_FOUND" FILE_EXPIRED = "FILE_EXPIRED" CONVERT_FAILED = "CONVERT_FAILED" @@ -35,11 +37,20 @@ class TranscriptFetchError(Exception): failures by type instead of matching on message text. """ + _STATUS = { + ErrorCode.UPSTREAM_BLOCKED: 503, + ErrorCode.UPSTREAM_TIMEOUT: 504, + } + def __init__(self, code: ErrorCode, message: str): self.code = code self.message = message super().__init__(message) + @property + def status_code(self) -> int: + return self._STATUS.get(self.code, 400) + def success_response(data: Dict[str, Any]) -> Dict[str, Any]: """Wraps any payload in the standard success envelope.""" diff --git a/app/services/transcript_service.py b/app/services/transcript_service.py index 922aa05..be9d452 100644 --- a/app/services/transcript_service.py +++ b/app/services/transcript_service.py @@ -1,9 +1,9 @@ from __future__ import annotations import random -import re import time import requests +from concurrent.futures import Future, ThreadPoolExecutor from typing import Any, Dict, List, Optional, Tuple from youtube_transcript_api import ( @@ -35,6 +35,38 @@ "Please try again later." ) +YOUTUBE_TIMEOUT_MESSAGE = "Transcript provider did not respond in time." + +_OEMBED_URL = "https://www.youtube.com/oembed" +# Title only names the download; never hold captions back for long waiting on it. +_METADATA_GRACE_SECONDS = 1.5 +_metadata_pool = ThreadPoolExecutor(max_workers=4, thread_name_prefix="yt-metadata") + + +class _DeadlineExceeded(Exception): + """The caller's time budget ran out before the next upstream request.""" + + +class _DeadlineSession(requests.Session): + """ + Caps each request at the time left before *deadline* (time.monotonic()). + asyncio.wait_for cannot stop the worker thread, so this is what makes a + timed-out extract stop calling YouTube (and spending proxy bandwidth). + """ + + def __init__(self, deadline: float): + super().__init__() + self._deadline = deadline + + def request(self, *args, **kwargs): + remaining = self._deadline - time.monotonic() + if remaining <= 0: + raise _DeadlineExceeded() + timeout = kwargs.get("timeout") + kwargs["timeout"] = min(timeout, remaining) if isinstance(timeout, (int, float)) else remaining + return super().request(*args, **kwargs) + + # Fallback search order after the client’s target language (never English-only). _BASE_LANG_PRIORITY: Tuple[str, ...] = ( "en", @@ -98,11 +130,12 @@ def _youtube_ip_block_attempts() -> int: return min(n, 8) -def _youtube_api() -> YouTubeTranscriptApi: +def _youtube_api(deadline: Optional[float] = None) -> YouTubeTranscriptApi: """ Optional residential / generic proxies — see youtube-transcript-api README https://github.com/jdepoix/youtube-transcript-api#working-around-ip-bans """ + http_client = _DeadlineSession(deadline) if deadline is not None else None proxy_config = None try: from youtube_transcript_api.proxies import GenericProxyConfig, WebshareProxyConfig @@ -130,13 +163,14 @@ def _youtube_api() -> YouTubeTranscriptApi: https_url=settings.youtube_https_proxy_url or None, ) - return YouTubeTranscriptApi(proxy_config=proxy_config) if proxy_config else YouTubeTranscriptApi() + return YouTubeTranscriptApi(proxy_config=proxy_config, http_client=http_client) -def get_transcript_from_url(url: str) -> Dict: +def get_transcript_from_url(url: str, deadline: Optional[float] = None) -> Dict: """ Fetch transcript. YouTube captions are always resolved to English (native or auto-translate). Raises TranscriptFetchError for expected failures; anything else is a bug. + *deadline* (time.monotonic()) bounds upstream work so it stops once the caller gives up. """ try: platform = detect_platform(url) @@ -154,35 +188,36 @@ def get_transcript_from_url(url: str) -> Dict: return out if platform == "youtube": - result = get_youtube_transcript(video_id, url=url) + result = get_youtube_transcript(video_id, url=url, deadline=deadline) else: - result = get_vimeo_transcript(video_id, url=url) + result = get_vimeo_transcript(video_id, url=url, deadline=deadline) write_transcript_cache(platform, video_id, result) return result def _get_youtube_metadata(video_id: str) -> Dict: - """Fetches video metadata directly from YouTube page.""" - url = f"https://www.youtube.com/watch?v={video_id}" - headers = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" - ), - "Accept-Language": "en-US,en;q=0.9", - } + """ + Title via oEmbed: a small JSON endpoint that is rarely blocked, unlike the + watch page. Duration isn't exposed there; extract_guardrails estimates it + from the last caption cue when this is 0. + """ + fallback = {"title": f"YouTube Video {video_id}", "duration": 0} try: - response = requests.get(url, headers=headers, timeout=10) + response = requests.get( + _OEMBED_URL, + params={"url": f"https://www.youtube.com/watch?v={video_id}", "format": "json"}, + timeout=5, + ) response.raise_for_status() - html = response.text + title = str(response.json().get("title") or "").strip() + return {"title": title or fallback["title"], "duration": 0} + except (requests.RequestException, ValueError): + return fallback - title_match = re.search(r'', html) - title = title_match.group(1) if title_match else f"YouTube Video {video_id}" - duration_match = re.search(r'"approxDurationMs":"(\d+)"', html) - duration = int(int(duration_match.group(1)) / 1000) if duration_match else 0 - - return {"title": title, "duration": duration} +def _resolve_metadata(future: Future, video_id: str) -> Dict: + try: + return future.result(timeout=_METADATA_GRACE_SECONDS) except Exception: return {"title": f"YouTube Video {video_id}", "duration": 0} @@ -254,9 +289,12 @@ def _youtube_transcript_fresh_core( video_id: str, languages: Optional[List[str]], url: str, + metadata_future: Optional[Future] = None, + deadline: Optional[float] = None, ) -> Dict: - metadata = _get_youtube_metadata(video_id) - api = _youtube_api() + if metadata_future is None: + metadata_future = _metadata_pool.submit(_get_youtube_metadata, video_id) + api = _youtube_api(deadline) tlist = api.list(video_id) target = "en" @@ -279,6 +317,7 @@ def _youtube_transcript_fresh_core( {"text": item.text, "start": item.start, "duration": item.duration} for item in fetched ] + metadata = _resolve_metadata(metadata_future, video_id) return { "provider": "youtube", @@ -295,7 +334,9 @@ def _youtube_fetch_error(e: CouldNotRetrieveTranscript) -> TranscriptFetchError: """Map youtube-transcript-api failures to a user-safe TranscriptFetchError.""" na = ErrorCode.TRANSCRIPT_NOT_AVAILABLE if isinstance(e, (IpBlocked, RequestBlocked)): - return TranscriptFetchError(na, YOUTUBE_TRANSCRIPT_UNAVAILABLE_EN_ONLY) + return TranscriptFetchError( + ErrorCode.UPSTREAM_BLOCKED, YOUTUBE_TRANSCRIPT_UNAVAILABLE_EN_ONLY + ) if isinstance(e, TranscriptsDisabled): return TranscriptFetchError(na, "Transcripts are disabled for this video.") if isinstance(e, VideoUnavailable): @@ -312,24 +353,38 @@ def get_youtube_transcript( video_id: str, languages: Optional[List[str]] = None, url: str = "", + deadline: Optional[float] = None, ) -> Dict: """ list() + find_transcript; never uses fetch(video_id) default ['en'] shortcut. English-only: prefer English captions, else translate to English when available. - With residential proxy configured, retries a few times on IpBlocked / RequestBlocked. + With residential proxy configured, retries a few times on IpBlocked / RequestBlocked, + but never sleeps past *deadline*. """ + # Title fetch runs alongside the caption requests (and survives retries) + metadata_future = _metadata_pool.submit(_get_youtube_metadata, video_id) attempts = 1 if languages is not None else _youtube_ip_block_attempts() for attempt in range(attempts): try: - return _youtube_transcript_fresh_core(video_id, languages, url) + return _youtube_transcript_fresh_core( + video_id, languages, url, metadata_future=metadata_future, deadline=deadline + ) except (IpBlocked, RequestBlocked) as e: - if attempt < attempts - 1: - delay = settings.youtube_transcript_retry_backoff_seconds * (attempt + 1) - time.sleep(delay + random.uniform(0, 0.25)) + delay = settings.youtube_transcript_retry_backoff_seconds * (attempt + 1) + delay += random.uniform(0, 0.25) + out_of_time = deadline is not None and time.monotonic() + delay >= deadline + if attempt < attempts - 1 and not out_of_time: + time.sleep(delay) continue raise _youtube_fetch_error(e) from e except CouldNotRetrieveTranscript as e: raise _youtube_fetch_error(e) from e + except (_DeadlineExceeded, requests.Timeout) as e: + raise TranscriptFetchError(ErrorCode.UPSTREAM_TIMEOUT, YOUTUBE_TIMEOUT_MESSAGE) from e + except requests.RequestException as e: + raise TranscriptFetchError( + ErrorCode.TRANSCRIPT_NOT_AVAILABLE, "Could not reach YouTube. Try again later." + ) from e raise AssertionError("unreachable") diff --git a/app/services/vimeo_service.py b/app/services/vimeo_service.py index f71487b..662daac 100644 --- a/app/services/vimeo_service.py +++ b/app/services/vimeo_service.py @@ -1,6 +1,7 @@ import requests import re import json +import time import webvtt from io import StringIO from typing import List, Dict, Optional @@ -8,8 +9,22 @@ from app.core.errors import ErrorCode, TranscriptFetchError _NO_CAPTIONS = "No captions are available for this Vimeo video." +_REQUEST_TIMEOUT = 10 -def get_vimeo_transcript(video_id: str, url: str = '') -> Dict: + +def _timeout(deadline: Optional[float]) -> float: + """Per-request timeout, never past the caller's deadline (time.monotonic()).""" + if deadline is None: + return _REQUEST_TIMEOUT + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TranscriptFetchError( + ErrorCode.UPSTREAM_TIMEOUT, "Transcript provider did not respond in time." + ) + return min(_REQUEST_TIMEOUT, remaining) + + +def get_vimeo_transcript(video_id: str, url: str = '', deadline: Optional[float] = None) -> Dict: """ Fetches the transcript for a given Vimeo video ID. Extracts text tracks from Vimeo player config and parses VTT format. @@ -27,7 +42,7 @@ def get_vimeo_transcript(video_id: str, url: str = '') -> Dict: 'Accept-Language': 'en-US,en;q=0.9', } - response = requests.get(player_url, headers=headers, timeout=10) + response = requests.get(player_url, headers=headers, timeout=_timeout(deadline)) response.raise_for_status() html_content = response.text @@ -75,7 +90,7 @@ def get_vimeo_transcript(video_id: str, url: str = '') -> Dict: raise TranscriptFetchError(ErrorCode.TRANSCRIPT_NOT_AVAILABLE, _NO_CAPTIONS) # Download the VTT file - vtt_response = requests.get(vtt_url, timeout=10) + vtt_response = requests.get(vtt_url, timeout=_timeout(deadline)) vtt_response.raise_for_status() # Parse VTT content @@ -106,6 +121,10 @@ def get_vimeo_transcript(video_id: str, url: str = '') -> Dict: raise TranscriptFetchError( ErrorCode.TRANSCRIPT_NOT_AVAILABLE, "Vimeo did not return this video. Try again later." ) from e + except requests.Timeout as e: + raise TranscriptFetchError( + ErrorCode.UPSTREAM_TIMEOUT, "Transcript provider did not respond in time." + ) from e except requests.RequestException as e: raise TranscriptFetchError( ErrorCode.TRANSCRIPT_NOT_AVAILABLE, "Could not reach Vimeo. Try again later." diff --git a/tests/test_extract_reliability.py b/tests/test_extract_reliability.py new file mode 100644 index 0000000..5a153fb --- /dev/null +++ b/tests/test_extract_reliability.py @@ -0,0 +1,155 @@ +"""Tests for extract time budget: concurrent metadata, deadlines, IP-block reporting.""" + +import threading +import time +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import requests +from fastapi.testclient import TestClient +from youtube_transcript_api import IpBlocked + +from app.core.errors import ErrorCode, TranscriptFetchError +from app.main import app +from app.services import transcript_service as ts +from app.services.vimeo_service import get_vimeo_transcript + +client = TestClient(app) + + +class _Fetched(list): + language_code = "en" + + +def _mock_api(on_list=None): + """YouTubeTranscriptApi stand-in whose list() returns one English track.""" + tr = MagicMock(language_code="en", is_translatable=False) + tr.fetch.return_value = _Fetched([SimpleNamespace(text="hi", start=0.0, duration=1.0)]) + tlist = MagicMock() + tlist.find_transcript.return_value = tr + + def _list(_video_id): + if on_list: + on_list() + return tlist + + api = MagicMock() + api.list.side_effect = _list + return api + + +class TestConcurrentMetadata(unittest.TestCase): + def test_title_fetch_overlaps_caption_fetch(self): + captions_started = threading.Event() + + def metadata(_vid): + # Only succeeds if it runs while captions are being fetched + if not captions_started.wait(timeout=1): + return {"title": "SEQUENTIAL", "duration": 0} + return {"title": "Concurrent", "duration": 0} + + with patch.object(ts, "_get_youtube_metadata", side_effect=metadata), patch.object( + ts, "YouTubeTranscriptApi", return_value=_mock_api(captions_started.set) + ): + result = ts.get_youtube_transcript("vid") + self.assertEqual(result["title"], "Concurrent") + + def test_slow_title_does_not_hold_captions(self): + release = threading.Event() + self.addCleanup(release.set) + + def metadata(_vid): + release.wait(timeout=5) + return {"title": "Late", "duration": 0} + + with patch.object(ts, "_get_youtube_metadata", side_effect=metadata), patch.object( + ts, "YouTubeTranscriptApi", return_value=_mock_api() + ), patch.object(ts, "_METADATA_GRACE_SECONDS", 0.05): + started = time.monotonic() + result = ts.get_youtube_transcript("vid") + self.assertLess(time.monotonic() - started, 1) + self.assertEqual(result["title"], "YouTube Video vid") + self.assertEqual(result["segments"][0]["text"], "hi") + + +class TestOembedMetadata(unittest.TestCase): + @patch("app.services.transcript_service.requests.get") + def test_title_from_oembed(self, mock_get): + mock_get.return_value.json.return_value = {"title": "Tom & Jerry"} + self.assertEqual( + ts._get_youtube_metadata("vid"), {"title": "Tom & Jerry", "duration": 0} + ) + + @patch("app.services.transcript_service.requests.get") + def test_fallback_on_failure(self, mock_get): + mock_get.side_effect = requests.ConnectionError() + self.assertEqual(ts._get_youtube_metadata("vid")["title"], "YouTube Video vid") + + +class TestDeadline(unittest.TestCase): + def test_session_refuses_requests_after_deadline(self): + session = ts._DeadlineSession(time.monotonic() - 1) + with self.assertRaises(ts._DeadlineExceeded): + session.get("https://example.invalid") + + def test_session_caps_request_timeout(self): + session = ts._DeadlineSession(time.monotonic() + 2) + with patch("requests.Session.request") as mock_request: + session.get("https://example.invalid", timeout=30) + self.assertLessEqual(mock_request.call_args.kwargs["timeout"], 2) + + @patch("app.services.transcript_service.time.sleep") + @patch("app.services.transcript_service._youtube_ip_block_attempts", return_value=5) + @patch("app.services.transcript_service._youtube_transcript_fresh_core") + def test_no_retry_sleep_past_deadline(self, mock_core, _attempts, mock_sleep): + mock_core.side_effect = IpBlocked("vid") + with self.assertRaises(TranscriptFetchError) as ctx: + ts.get_youtube_transcript("vid", deadline=time.monotonic() + 0.1) + mock_sleep.assert_not_called() + self.assertEqual(mock_core.call_count, 1) + self.assertEqual(ctx.exception.code, ErrorCode.UPSTREAM_BLOCKED) + + @patch("app.services.transcript_service._youtube_transcript_fresh_core") + def test_deadline_exceeded_is_upstream_timeout(self, mock_core): + mock_core.side_effect = ts._DeadlineExceeded() + with self.assertRaises(TranscriptFetchError) as ctx: + ts.get_youtube_transcript("vid", deadline=time.monotonic()) + self.assertEqual(ctx.exception.code, ErrorCode.UPSTREAM_TIMEOUT) + self.assertEqual(ctx.exception.status_code, 504) + + @patch("app.services.vimeo_service.requests.get") + def test_vimeo_stops_after_deadline(self, mock_get): + with self.assertRaises(TranscriptFetchError) as ctx: + get_vimeo_transcript("123", deadline=time.monotonic() - 1) + self.assertEqual(ctx.exception.code, ErrorCode.UPSTREAM_TIMEOUT) + mock_get.assert_not_called() + + +class TestBlockedResponse(unittest.TestCase): + @patch("app.api.v1_routes.get_transcript_from_url") + def test_ip_block_returns_503_with_own_code(self, mock_fetch): + mock_fetch.side_effect = ts._youtube_fetch_error(IpBlocked("vid")) + r = client.post( + "/api/v1/extract", + json={"url": "https://www.youtube.com/watch?v=testvideo12", "include_timestamps": False}, + ) + self.assertEqual(r.status_code, 503) + self.assertEqual(r.json()["error"]["code"], "UPSTREAM_BLOCKED") + self.assertIn("blocked this server", r.json()["error"]["message"]) + + @patch("app.api.v1_routes.get_transcript_from_url") + def test_api_passes_deadline_within_budget(self, mock_fetch): + mock_fetch.side_effect = TranscriptFetchError(ErrorCode.TRANSCRIPT_NOT_AVAILABLE, "x") + before = time.monotonic() + client.post( + "/api/v1/extract", + json={"url": "https://www.youtube.com/watch?v=testvideo12", "include_timestamps": False}, + ) + deadline = mock_fetch.call_args.args[1] + self.assertGreater(deadline, before) + self.assertLessEqual(deadline, time.monotonic() + ts.settings.transcript_timeout_seconds) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_get_transcript_from_url.py b/tests/test_get_transcript_from_url.py index 650eb99..7f86720 100644 --- a/tests/test_get_transcript_from_url.py +++ b/tests/test_get_transcript_from_url.py @@ -43,8 +43,8 @@ def test_vimeo_delegates(self, mock_vm, _mock_read, _mock_write): "segments": [], } url = "https://vimeo.com/999" - result = get_transcript_from_url(url) - mock_vm.assert_called_once_with("999", url=url) + result = get_transcript_from_url(url, deadline=123.0) + mock_vm.assert_called_once_with("999", url=url, deadline=123.0) self.assertEqual(result["provider"], "vimeo") @patch("app.services.transcript_service.get_youtube_transcript") diff --git a/web/app/components/home/BentoGrid.tsx b/web/app/components/home/BentoGrid.tsx deleted file mode 100644 index 9cfc828..0000000 --- a/web/app/components/home/BentoGrid.tsx +++ /dev/null @@ -1,84 +0,0 @@ -import React from 'react'; -import { Zap, Lock, Globe, Database, Bot, Share2 } from 'lucide-react'; - -export const BentoGrid: React.FC = () => { - return ( -
-
-

System Architecture

-

Built for scale. Designed for speed.

-
- -
- - {/* Large Card: Telegram Bot */} -
-
- -
-
-
- -
-

Telegram Native

-

- Don't leave your messenger. Send a link to @TranscriptFlowBot and receive your file instantly. The core engine runs 24/7 on our distributed infrastructure. -

-
-
- - {/* Card: Speed */} -
- -
-

Lightning Fast

-

1-hour video processed in under 10 seconds.

-
-
- - {/* Card: Security */} -
- -
-

Secure Core

-

No permanent storage of processed data.

-
-
- - {/* Medium Card: AI Ready */} -
-
-
- -
-
-

AI-Ready Output

-

- We generate cleaner data than raw captions. Perfect for feeding into GPT-4, Claude, or Llama for summarization and repurposing. -

-
-
-
- - {/* Card: Multi-Format */} -
- -
-

Multi-Format

-

TXT, PDF, DOCX, SRT, VTT. Your choice.

-
-
- - {/* Card: Global */} -
- -
-

Multi-Platform

-

YouTube & Vimeo supported. Auto-detects languages.

-
-
- -
-
- ); -}; diff --git a/web/app/components/home/BotDemo.tsx b/web/app/components/home/BotDemo.tsx index 26393ea..00d6fc9 100644 --- a/web/app/components/home/BotDemo.tsx +++ b/web/app/components/home/BotDemo.tsx @@ -23,7 +23,7 @@ export const BotDemo: React.FC = () => { { delay: 7500, msg: { id: 5, text: "Transcript ready:", sender: 'bot' as const, file: true } }, ]; - let timeouts: ReturnType[] = []; + const timeouts: ReturnType[] = []; const runSequence = () => { setMessages([]); diff --git a/web/app/components/home/Generator.tsx b/web/app/components/home/Generator.tsx deleted file mode 100644 index 61f8d80..0000000 --- a/web/app/components/home/Generator.tsx +++ /dev/null @@ -1,217 +0,0 @@ -'use client'; - -import React, { useState, useRef } from 'react'; -import { CheckCircle2, AlertCircle, Copy, Download, Youtube, Video, ChevronRight } from 'lucide-react'; -import { ProcessingStatus, TranscriptResult, FileFormat } from '../../types'; -import { PUBLIC_API_BASE } from '../../../lib/publicApiBase'; - -export const Generator: React.FC = () => { - const [url, setUrl] = useState(''); - const [status, setStatus] = useState(ProcessingStatus.IDLE); - const [result, setResult] = useState(null); - const [format, setFormat] = useState('txt'); - const [error, setError] = useState(null); - const [downloadUrl, setDownloadUrl] = useState(''); - - const resultRef = useRef(null); - - const handleGenerate = async () => { - if (!url) return; - setError(null); - setResult(null); - setDownloadUrl(''); - setStatus(ProcessingStatus.ANALYZING); - - const apiBase = PUBLIC_API_BASE; - - try { - setStatus(ProcessingStatus.EXTRACTING); - - const extractRes = await fetch(`${apiBase}/api/v1/extract`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - url, - include_timestamps: false, - }), - }); - - const data = await extractRes.json(); - - if (!extractRes.ok || !data.success) { - const msg = data?.error?.message ?? data?.detail ?? 'extraction failed'; - throw new Error(typeof msg === 'string' ? msg : 'extraction failed'); - } - - setStatus(ProcessingStatus.FORMATTING); - - let downloadPath = data.file_download_url as string; - - if (format !== 'txt') { - const convertRes = await fetch(`${apiBase}/api/v1/convert`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - file_id: data.file_id, - format, - }), - }); - const conv = await convertRes.json(); - if (!convertRes.ok || !conv.success) { - const msg = conv?.error?.message ?? conv?.detail ?? 'conversion failed'; - throw new Error(typeof msg === 'string' ? msg : 'conversion failed'); - } - downloadPath = conv.file_download_url; - } - - setDownloadUrl(`${apiBase}${downloadPath}`); - - setResult({ - videoId: extractVideoId(url), - title: data.title || 'video transcript', - segments: [], - summary: `processed ${data.word_count} words. archival artifact generated.`, - }); - - setStatus(ProcessingStatus.COMPLETE); - setTimeout(() => resultRef.current?.scrollIntoView({ behavior: 'smooth' }), 100); - } catch (err: unknown) { - setError(err instanceof Error ? err.message : 'unexpected engine error.'); - setStatus(ProcessingStatus.ERROR); - } - }; - - const extractVideoId = (url: string): string => { - const match = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/|vimeo\.com\/)([^&\s]+)/); - return match ? match[1] : 'unknown'; - }; - - const handleCopy = async () => { - if (result) { - const text = result.segments.map(s => s.text).join('\n'); - await navigator.clipboard.writeText(text); - } - }; - - const handleDownload = () => { - if (downloadUrl) { - const a = document.createElement('a'); - a.href = downloadUrl; - a.download = `transcript.${format}`; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - } - }; - - const isProcessing = [ProcessingStatus.ANALYZING, ProcessingStatus.EXTRACTING, ProcessingStatus.FORMATTING].includes(status); - - return ( -
- {/* Colorful Background Glow */} -
- -
-
-

- extraction engine -

-

- v2.8.1 / stable / zero-latency -

-
-
- processing.online -
-
- -
-
-
- setUrl(e.target.value)} - placeholder="https://content-url.com" - className="w-full bg-transparent text-white px-0 py-4 focus:outline-none transition-all font-normal placeholder-neutral-800 text-lg tracking-tight" - disabled={isProcessing} - /> -
-
- -
-
- -
- -
-
- - - - {error && ( -
- - {error} -
- )} -
- - {/* Results View */} - {result && downloadUrl && ( -
-
-
-
-

{result.title}

-
- - id: {result.videoId} - -
-
- - -
- -
-

- {result.summary} -

-
-
-
- )} -
- ); -}; - diff --git a/web/app/components/home/Hero.tsx b/web/app/components/home/Hero.tsx deleted file mode 100644 index b4d6fdd..0000000 --- a/web/app/components/home/Hero.tsx +++ /dev/null @@ -1,66 +0,0 @@ -'use client'; - -import React from 'react'; -import { ArrowRight, PlayCircle } from 'lucide-react'; -import { Button } from '../ui/Button'; - -interface HeroProps { - onStart: () => void; -} - -export const Hero: React.FC = ({ onStart }) => { - return ( -
- - {/* Background Decor */} -
-
-
-
- -
-
- - V1.0 ENGINE ONLINE -
- -

- EXTRACT.
- - TRANSCRIBE. -
- CONQUER. -

- -

- The ultimate infrastructure for video-to-text conversion. - Process YouTube, Vimeo, and Audio instantly via our Web Dashboard or Telegram Bot. -

- -
- - -
- - {/* Stats Grid - Mini Glass Cards */} -
- {[ - { label: 'Speed', value: '< 10s' }, - { label: 'Accuracy', value: '99.8%' }, - { label: 'Users', value: '2.5k+' }, - { label: 'Uptime', value: '99.9%' } - ].map((stat, i) => ( -
-
{stat.value}
-
{stat.label}
-
- ))} -
-
-
- ); -}; diff --git a/web/app/components/layout/Footer.tsx b/web/app/components/layout/Footer.tsx deleted file mode 100644 index 3f68a78..0000000 --- a/web/app/components/layout/Footer.tsx +++ /dev/null @@ -1,4 +0,0 @@ -/** - * @deprecated Prefer importing SiteFooter from @/components/SiteFooter. - */ -export { SiteFooter as Footer } from "../../../components/SiteFooter"; diff --git a/web/app/components/layout/Navbar.tsx b/web/app/components/layout/Navbar.tsx deleted file mode 100644 index 7a733bc..0000000 --- a/web/app/components/layout/Navbar.tsx +++ /dev/null @@ -1,87 +0,0 @@ -'use client'; - -import React from 'react'; -import { Bot, Menu, X, Terminal } from 'lucide-react'; - -export const Navbar: React.FC = () => { - const [isOpen, setIsOpen] = React.useState(false); - - return ( - - ); -}; diff --git a/web/app/components/ui/Button.tsx b/web/app/components/ui/Button.tsx deleted file mode 100644 index 6400693..0000000 --- a/web/app/components/ui/Button.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import React from 'react'; -import { Loader2 } from 'lucide-react'; - -interface ButtonProps extends React.ButtonHTMLAttributes { - variant?: 'primary' | 'secondary' | 'glass'; - isLoading?: boolean; - glow?: boolean; -} - -export const Button: React.FC = ({ - children, - variant = 'primary', - isLoading = false, - glow = false, - className = '', - ...props -}) => { - const baseStyles = "relative inline-flex items-center justify-center px-8 py-3 font-orbitron font-medium tracking-wider text-sm transition-all duration-300 rounded-sm overflow-hidden group"; - - const variants = { - primary: "bg-singularity text-black hover:bg-singularity-hover disabled:opacity-50 disabled:cursor-not-allowed", - secondary: "bg-transparent border border-singularity text-singularity hover:bg-singularity/10", - glass: "bg-white/5 border border-white/10 text-white hover:bg-white/10 backdrop-blur-md", - }; - - const glowEffect = glow ? "shadow-[0_0_20px_rgba(0,212,255,0.4)] hover:shadow-[0_0_30px_rgba(0,212,255,0.6)]" : ""; - - return ( - - ); -}; diff --git a/web/app/types/index.ts b/web/app/types/index.ts deleted file mode 100644 index 85400f1..0000000 --- a/web/app/types/index.ts +++ /dev/null @@ -1,22 +0,0 @@ -export interface TranscriptSegment { - start: number; - text: string; -} - -export interface TranscriptResult { - videoId: string; - title: string; - segments: TranscriptSegment[]; - summary?: string; -} - -export enum ProcessingStatus { - IDLE = 'IDLE', - ANALYZING = 'ANALYZING', - EXTRACTING = 'EXTRACTING', - FORMATTING = 'FORMATTING', - COMPLETE = 'COMPLETE', - ERROR = 'ERROR', -} - -export type FileFormat = 'txt' | 'docx' | 'pdf'; diff --git a/web/components/Navbar.tsx b/web/components/Navbar.tsx index 4f3c2de..d9bab9e 100644 --- a/web/components/Navbar.tsx +++ b/web/components/Navbar.tsx @@ -3,6 +3,7 @@ import { useEffect, useState, useCallback } from 'react'; import Link from 'next/link'; import { useTheme } from 'next-themes'; +import { useHydrated } from '../lib/useHydrated'; interface NavbarProps { links: { label: string; href?: string; onClick?: () => void }[]; @@ -13,9 +14,7 @@ export function Navbar({ links, showThemeToggle = false }: NavbarProps) { const [scrolled, setScrolled] = useState(false); const [menuOpen, setMenuOpen] = useState(false); const { theme, setTheme } = useTheme(); - const [mounted, setMounted] = useState(false); - - useEffect(() => setMounted(true), []); + const mounted = useHydrated(); useEffect(() => { const onScroll = () => setScrolled(window.scrollY > 8); diff --git a/web/components/ThemeToggle.tsx b/web/components/ThemeToggle.tsx index 3f478c0..a5b7e4a 100644 --- a/web/components/ThemeToggle.tsx +++ b/web/components/ThemeToggle.tsx @@ -1,13 +1,12 @@ 'use client'; import { useTheme } from 'next-themes'; -import { useEffect, useState } from 'react'; +import { useHydrated } from '../lib/useHydrated'; export function ThemeToggle() { const { theme, setTheme } = useTheme(); - const [mounted, setMounted] = useState(false); + const mounted = useHydrated(); - useEffect(() => setMounted(true), []); if (!mounted) return ; const isDark = theme === 'dark'; diff --git a/web/lib/useHydrated.ts b/web/lib/useHydrated.ts new file mode 100644 index 0000000..484fee0 --- /dev/null +++ b/web/lib/useHydrated.ts @@ -0,0 +1,12 @@ +import { useSyncExternalStore } from 'react'; + +const subscribe = () => () => {}; + +/** + * False during SSR and the hydration render, true afterwards. Use it to gate UI + * that depends on client-only state (e.g. the resolved next-themes theme) + * without a setState-in-effect "mounted" flag. + */ +export function useHydrated(): boolean { + return useSyncExternalStore(subscribe, () => true, () => false); +}