diff --git a/.env.example b/.env.example index b1a7734..f298f19 100644 --- a/.env.example +++ b/.env.example @@ -68,6 +68,12 @@ TELEGRAM_BOT_TOKEN=YOUR_TOKEN_HERE # WEBSHARE_PROXY_PASSWORD= # WEBSHARE_PROXY_LOCATIONS= +# --- PDF fonts +# Unicode TTF for PDFs beyond Western European text (Polish, Vietnamese, Cyrillic, Greek). +# Docker installs DejaVu Sans + Nanum Gothic (Korean); set this for local dev if needed. +# Arabic, Hebrew, Indic and Thai scripts get DOCX/TXT only (no PDF). +# PDF_FONT_PATH= + # --- Transcript disk cache TRANSCRIPT_CACHE_ENABLED=true TRANSCRIPT_CACHE_TTL_HOURS=72 diff --git a/Dockerfile b/Dockerfile index a3513cb..4fa6206 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,9 +10,13 @@ RUN groupadd --system --gid 1000 app \ WORKDIR /app -# System deps for reportlab / cffi +# System deps for reportlab / cffi. PDF fonts: DejaVu Sans embeds Latin-extended, +# Greek and Cyrillic (Helvetica only covers Western European); Nanum Gothic embeds +# Korean (ReportLab's built-in Korean CID fonts render blank in Chrome's viewer) RUN apt-get update && apt-get install -y --no-install-recommends \ libffi-dev \ + fonts-dejavu-core \ + fonts-nanum \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt . diff --git a/README.md b/README.md index a266328..8a66985 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Core Value: **Fast → Reliable → Structured → Downloadable.** - **Multi-Platform Support**: Extract transcripts from **YouTube** and **Vimeo** instantly. - **Clean Formats**: Download as **TXT, PDF, and DOCX**, or **SRT / VTT** subtitles with cue timings. Files are named after the video title. +- **23 Languages**: Pick the transcript language on the web or with `/language` in Telegram. YouTube auto-translates when a video has no captions in that language; Vimeo uses the uploader's tracks. - **Timestamps**: Toggle perfectly formatted timestamps on or off. - **Anonymous & Secure**: No accounts required. Generated files auto-delete after ~1 hour (TTL). Optional short-term caption cache (`TRANSCRIPT_CACHE_TTL_HOURS`) is pruned by the same cleanup loop. - **Rate Limited & Guardrailed**: Separate extract vs convert quotas (shared across workers and the bot via optional Redis), max video duration (with caption-based fallback), segment/size caps, and timeout protections. diff --git a/app/api/v1_routes.py b/app/api/v1_routes.py index f274313..6240e14 100644 --- a/app/api/v1_routes.py +++ b/app/api/v1_routes.py @@ -16,6 +16,13 @@ verify_rate_limit_extract, ) from app.core.errors import AppError, ErrorCode, TranscriptFetchError, success_response +from app.core.languages import ( + DEFAULT_LANGUAGE, + LANGUAGE_CODES, + language_name, + languages_match, + public_language_list, +) from app.services.analytics_service import AnalyticsService from app.services.extract_guardrails import enforce_transcript_guardrails from app.services.file_service import ( @@ -27,6 +34,7 @@ write_sidecar, ) from app.services.formatter_service import SUBTITLE_FORMATS, TranscriptFormatter +from app.services.pdf_fonts import UnsupportedPdfScript, pdf_supported from app.services.metadata_service import MetadataService from app.services.telemetry_service import MAX_PAYLOAD_BYTES, TelemetryService from app.services.transcript_service import get_transcript_from_url @@ -68,6 +76,8 @@ def _validate_file_id(file_id: str) -> str: class ExtractRequest(BaseModel): url: str = Field(..., min_length=20, max_length=2048) include_timestamps: bool = False + # Target caption language (see app.core.languages); YouTube auto-translates + language: str = DEFAULT_LANGUAGE @router.get("/health") @@ -88,13 +98,18 @@ async def extract_transcript(request: ExtractRequest): 400, ) + if request.language not in LANGUAGE_CODES: + raise AppError(ErrorCode.UNSUPPORTED_LANGUAGE, "That language isn't supported.", 400) + layout_format = "timestamp" if request.include_timestamps else "clean" # The worker thread outlives wait_for; the deadline makes it stop calling upstream too deadline = time.monotonic() + settings.transcript_timeout_seconds try: transcript_data = await asyncio.wait_for( - asyncio.to_thread(get_transcript_from_url, request.url, deadline), + asyncio.to_thread( + get_transcript_from_url, request.url, deadline, request.language + ), timeout=settings.transcript_timeout_seconds, ) except asyncio.TimeoutError: @@ -205,7 +220,12 @@ async def extract_transcript(request: ExtractRequest): text_sidecar.write_text(formatted_text, encoding="utf-8") # Segments feed SRT/VTT convert; title names every download of this transcript await asyncio.to_thread(write_sidecar, filename, "segments", transcript_data["segments"]) - write_sidecar(filename, "meta", {"title": transcript_data.get("title")}) + out_language = transcript_data.get("language", DEFAULT_LANGUAGE) + # Language picks the CJK PDF font (zh-Hant vs zh-Hans) at convert time + write_sidecar(filename, "meta", {"title": transcript_data.get("title"), "language": out_language}) + can_pdf = await asyncio.to_thread(pdf_supported, formatted_text) + source_language = transcript_data.get("source_language") or out_language + translated = bool(transcript_data.get("translated")) MetadataService.log_success( video_id=transcript_data["video_id"], @@ -235,7 +255,16 @@ async def extract_transcript(request: ExtractRequest): "provider": transcript_data.get("provider", platform), "video_id": transcript_data["video_id"], "title": transcript_data["title"], - "language": transcript_data.get("language", "en"), + "language": out_language, + "language_name": language_name(out_language), + "requested_language": request.language, + "source_language": source_language, + "source_language_name": language_name(source_language), + "translated": translated, + # Neither a native track nor a translation: captions are in another language + "language_fallback": not translated + and not languages_match(out_language, request.language), + "pdf_supported": can_pdf, "duration_seconds": duration, "word_count": metrics["word_count"], "reading_time_seconds": metrics["reading_time_seconds"], @@ -330,8 +359,13 @@ async def convert_file(request: ConvertRequest): try: try: file_path = await asyncio.to_thread( - FileGenerator.generate_file, content, request.format + FileGenerator.generate_file, + content, + request.format, + (read_sidecar(request.file_id, "meta") or {}).get("language", ""), ) + except UnsupportedPdfScript as e: + raise AppError(ErrorCode.CONVERT_FAILED, str(e), 400) except Exception as e: logger.error("Convert generation failed: %s", e) raise AppError( @@ -359,6 +393,8 @@ async def get_public_config(): return success_response( { "file_ttl_hours": settings.file_ttl_hours, + "default_language": DEFAULT_LANGUAGE, + "languages": public_language_list(), } ) diff --git a/app/bot/telegram_bot.py b/app/bot/telegram_bot.py index 2cec538..ff5659d 100644 --- a/app/bot/telegram_bot.py +++ b/app/bot/telegram_bot.py @@ -27,6 +27,8 @@ from app.services.extract_guardrails import enforce_transcript_guardrails from app.core.errors import AppError, ErrorCode, TranscriptFetchError from app.core.config import settings +from app.core.languages import SUPPORTED_LANGUAGES, LANGUAGE_CODES, language_name, languages_match +from app.services.pdf_fonts import UnsupportedPdfScript from app.utils.logging_config import logger from app.services.analytics_service import AnalyticsService from app.services.telemetry_service import TelemetryService, hash_identifier, bucket_duration @@ -109,7 +111,7 @@ def _escape_md2(text: str) -> str: WELCOME_TEXT = ( "*Transcript Flow*\n\n" "Turn YouTube or Vimeo videos into clean transcripts\\.\n\n" - "\\• YouTube: English captions only at this time\n" + "\\• 23 languages: YouTube auto\\-translates captions \\(/language\\)\n" "\\• TXT \\(default\\)\n" "\\• PDF / DOCX\n" "\\• SRT / VTT subtitles\n" @@ -127,9 +129,10 @@ def _escape_md2(text: str) -> str: "2\\. Preview the transcript\\.\n" "3\\. Download as TXT, PDF, DOCX, SRT, or VTT\\.\n\n" "Power users:\n" - "`/extract `\n\n" + "`/extract `\n" + "`/language` to choose the transcript language\n\n" "If extraction fails:\n" - "\\• YouTube transcripts are English\\-only for now\\.\n" + "\\• Pick another language with /language\\.\n" "\\• Captions may be missing or unavailable for some videos\\.\n" "\\• Transcripts may be disabled for the video\\." ) @@ -151,7 +154,7 @@ def _escape_md2(text: str) -> str: "\\• No account required\\.\n" "\\• Generated files auto\\-delete after about 1 hour\\.\n" "\\• Short\\-term caption cache \\(up to a few days\\) may reduce repeat fetches\\.\n" - "\\• Your format and timestamp preference is remembered for about 90 days\\.\n" + "\\• Your format, timestamp and language preference is remembered for about 90 days\\.\n" "\\• We do not keep permanent user accounts or history\\.\n" "\\• See the website privacy page for full details\\." ) @@ -206,10 +209,24 @@ def _ready_keyboard(include_timestamps: bool) -> InlineKeyboardMarkup: ) -def _ready_text(title: str, duration_s: int, word_count: int, reading_time_s: int) -> str: +def _language_label(transcript_data: dict, requested: str) -> str: + """e.g. "Español (auto-translated)" or "English (Español not available)".""" + actual = transcript_data.get("language") or requested + label = language_name(actual) + if transcript_data.get("translated"): + return f"{label} (auto-translated)" + if not languages_match(actual, requested): + return f"{label} ({language_name(requested)} not available)" + return label + + +def _ready_text( + title: str, duration_s: int, word_count: int, reading_time_s: int, language: str +) -> str: return ( f"Transcript ready.\n\n" f"Title: {title}\n" + f"Language: {language}\n" f"Duration: {_duration_label(duration_s)}\n" f"Words: {word_count}\n" f"Read time: {_reading_time_label(reading_time_s)}" @@ -227,18 +244,43 @@ def _start_keyboard() -> InlineKeyboardMarkup: InlineKeyboardButton("Privacy", callback_data="privacy"), InlineKeyboardButton("Help", callback_data="help"), ], + [ + InlineKeyboardButton("Language", callback_data="language"), + ], ]) -async def _build_file(transcript_data: dict, formatted_text: str, ext: str) -> str: - """Generate *ext* off the event loop (PDF builds can take seconds).""" +def _language_keyboard(current: str) -> InlineKeyboardMarkup: + buttons = [ + InlineKeyboardButton( + f"{'✓ ' if code == current else ''}{native}", callback_data=f"lang|{code}" + ) + for code, native, _ in SUPPORTED_LANGUAGES + ] + return InlineKeyboardMarkup([buttons[i:i + 3] for i in range(0, len(buttons), 3)]) + + +async def _build_file( + transcript_data: dict, formatted_text: str, ext: str +) -> tuple[str, str]: + """ + Generate *ext* off the event loop (PDF builds can take seconds). + Returns (path, ext actually produced): PDF falls back to DOCX for scripts + the PDF renderer can't typeset (Arabic, Hindi...). + """ if ext in SUBTITLE_FORMATS: content = await asyncio.to_thread( TranscriptFormatter.format_subtitles, transcript_data["segments"], ext ) else: content = formatted_text - return await asyncio.to_thread(FileGenerator.generate_file, content, ext) + language = transcript_data.get("language", "") + try: + path = await asyncio.to_thread(FileGenerator.generate_file, content, ext, language) + return path, ext + except UnsupportedPdfScript: + path = await asyncio.to_thread(FileGenerator.generate_file, content, "docx") + return path, "docx" def _extract_url_from_text(text: str) -> str | None: @@ -286,6 +328,7 @@ async def _process_url( prefs = _get_prefs(user_id) file_ext = prefs["last_file_format"] include_ts = prefs["last_include_timestamps"] + language = prefs["language"] fmt_type = _format_type_for_prefs(include_ts) started = time.time() @@ -297,6 +340,7 @@ async def _process_url( f"Extracting transcript\u2026\n" f"Provider: {platform}\n" f"Format: {file_ext}\n" + f"Language: {language_name(language)}\n" f"Timestamps: {_ts_label(include_ts)}" ), ) @@ -305,7 +349,7 @@ async def _process_url( # Extract (with timeout) ----------------------------------------- deadline = time.monotonic() + settings.transcript_timeout_seconds transcript_data = await asyncio.wait_for( - asyncio.to_thread(get_transcript_from_url, url, deadline), + asyncio.to_thread(get_transcript_from_url, url, deadline, language), timeout=settings.transcript_timeout_seconds, ) @@ -325,7 +369,7 @@ async def _process_url( await status_msg.edit_text("Transcript Flow\n\nPreparing file\u2026") video_id = transcript_data.get("video_id", "unknown") - file_path = await _build_file(transcript_data, formatted_text, file_ext) + file_path, file_ext = await _build_file(transcript_data, formatted_text, file_ext) # State D — Ready ------------------------------------------------ ready = _ready_text( @@ -333,6 +377,7 @@ async def _process_url( duration, metrics["word_count"], metrics["reading_time_seconds"], + _language_label(transcript_data, language), ) await status_msg.edit_text(ready, reply_markup=_ready_keyboard(include_ts)) @@ -351,6 +396,8 @@ async def _process_url( processing_time_ms = int((time.time() - started) * 1000) _emit_event("tg_extract_succeeded", user_id, { "provider": platform, + "language": transcript_data.get("language"), + "translated": bool(transcript_data.get("translated")), "duration_seconds_bucket": bucket_duration(duration), "file_format": file_ext, }) @@ -463,6 +510,38 @@ async def privacy_cmd(update: Update, context: ContextTypes.DEFAULT_TYPE) -> Non await update.message.reply_text(PRIVACY_TEXT, parse_mode="MarkdownV2") +async def _send_language_picker(context: ContextTypes.DEFAULT_TYPE, chat_id: int, user_id: str) -> None: + current = _get_prefs(user_id)["language"] + await context.bot.send_message( + chat_id=chat_id, + text=( + f"Transcript language: {language_name(current)}\n\n" + "Pick a language. If a video has no captions in it, YouTube auto-translates. " + "Vimeo can only use the captions the uploader added." + ), + reply_markup=_language_keyboard(current), + ) + + +async def language_cmd(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + user_id = str(update.effective_user.id) + _emit_event("tg_cmd_language", user_id) + await _send_language_picker(context, update.effective_chat.id, user_id) + + +async def _handle_language_choice(query, code: str) -> None: + if code not in LANGUAGE_CODES: + await query.answer() + return + user_id = str(query.from_user.id) + _set_pref(user_id, language=code) + _emit_event("tg_language_set", user_id, {"language": code}) + await query.answer(f"Language: {language_name(code)}") + await query.edit_message_text( + f"Transcript language: {language_name(code)}\n\nSend a link \u2014 it applies from your next transcript.", + ) + + async def status_cmd(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: _emit_event("tg_cmd_status", str(update.effective_user.id)) @@ -527,6 +606,13 @@ async def button_callback(update: Update, context: ContextTypes.DEFAULT_TYPE) -> parse_mode="MarkdownV2", ) + elif data == "language": + await query.answer() + await _send_language_picker(context, query.message.chat_id, str(query.from_user.id)) + + elif data.startswith("lang|"): + await _handle_language_choice(query, data.split("|", 1)[1]) + elif data == "privacy": await query.answer() await context.bot.send_message( @@ -597,7 +683,13 @@ async def _handle_download(query, context: ContextTypes.DEFAULT_TYPE, ext: str) if ext == context.user_data.get("file_ext") and cached_path and os.path.exists(cached_path): file_path = cached_path else: - file_path = await _build_file(transcript_data, formatted_text, ext) + file_path, built_ext = await _build_file(transcript_data, formatted_text, ext) + if built_ext != ext: + await context.bot.send_message( + chat_id=query.message.chat_id, + text="PDF isn't available for this language yet, so here's a DOCX instead.", + ) + ext = built_ext # Send file ------------------------------------------------------- with open(file_path, "rb") as f: @@ -710,7 +802,7 @@ async def _handle_timestamp_toggle(query, context: ContextTypes.DEFAULT_TYPE) -> # Regenerate file in current preferred format ------------------------- file_ext = prefs["last_file_format"] - file_path = await _build_file(transcript_data, formatted_text, file_ext) + file_path, file_ext = await _build_file(transcript_data, formatted_text, file_ext) # Update cache -------------------------------------------------------- context.user_data["formatted_text"] = formatted_text @@ -724,6 +816,7 @@ async def _handle_timestamp_toggle(query, context: ContextTypes.DEFAULT_TYPE) -> transcript_data.get("duration_seconds", 0), metrics["word_count"], metrics["reading_time_seconds"], + _language_label(transcript_data, prefs["language"]), ) try: await context.bot.edit_message_text( @@ -788,6 +881,7 @@ def run_bot() -> None: application.add_handler(CommandHandler("formats", formats_cmd)) application.add_handler(CommandHandler("privacy", privacy_cmd)) application.add_handler(CommandHandler("status", status_cmd)) + application.add_handler(CommandHandler("language", language_cmd)) # Plain-text message handler (catches raw links) application.add_handler(MessageHandler(filters.TEXT & (~filters.COMMAND), handle_plain_text)) diff --git a/app/core/config.py b/app/core/config.py index 47b73b0..15eb049 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -62,6 +62,10 @@ class Settings(BaseSettings): in_process_cleanup_enabled: bool = True cleanup_interval_seconds: int = 900 # 15 minutes + # TTF used for PDFs whose text is beyond Helvetica's Western European set + # (Polish, Vietnamese, Cyrillic, Greek...). Empty = search common system paths. + pdf_font_path: str = "" + # Telegram bot preview length preview_chars: int = 1500 diff --git a/app/core/errors.py b/app/core/errors.py index f5ad5ab..4232e4f 100644 --- a/app/core/errors.py +++ b/app/core/errors.py @@ -6,6 +6,7 @@ class ErrorCode(str, Enum): INVALID_URL = "INVALID_URL" + UNSUPPORTED_LANGUAGE = "UNSUPPORTED_LANGUAGE" TRANSCRIPT_NOT_AVAILABLE = "TRANSCRIPT_NOT_AVAILABLE" VIDEO_TOO_LONG = "VIDEO_TOO_LONG" RATE_LIMIT_EXCEEDED = "RATE_LIMIT_EXCEEDED" diff --git a/app/core/languages.py b/app/core/languages.py new file mode 100644 index 0000000..d27231d --- /dev/null +++ b/app/core/languages.py @@ -0,0 +1,84 @@ +""" +Transcript output languages offered in the web picker and Telegram /language. + +YouTube auto-translates captions into any of these when the video lacks a +native track; Vimeo can only pick among the tracks the uploader provided. +Codes follow YouTube's caption codes (note zh-Hans / zh-Hant). +""" + +from __future__ import annotations + +from typing import Dict, List + +DEFAULT_LANGUAGE = "en" + +# (code, native name, English name). Order is the order shown to users. +SUPPORTED_LANGUAGES: List[tuple[str, str, str]] = [ + ("en", "English", "English"), + ("es", "Español", "Spanish"), + ("fr", "Français", "French"), + ("de", "Deutsch", "German"), + ("pt", "Português", "Portuguese"), + ("it", "Italiano", "Italian"), + ("nl", "Nederlands", "Dutch"), + ("pl", "Polski", "Polish"), + ("tr", "Türkçe", "Turkish"), + ("ro", "Română", "Romanian"), + ("sv", "Svenska", "Swedish"), + ("cs", "Čeština", "Czech"), + ("id", "Bahasa Indonesia", "Indonesian"), + ("vi", "Tiếng Việt", "Vietnamese"), + ("ru", "Русский", "Russian"), + ("uk", "Українська", "Ukrainian"), + ("el", "Ελληνικά", "Greek"), + ("ja", "日本語", "Japanese"), + ("ko", "한국어", "Korean"), + ("zh-Hans", "简体中文", "Chinese (Simplified)"), + ("zh-Hant", "繁體中文", "Chinese (Traditional)"), + ("hi", "हिन्दी", "Hindi"), + ("ar", "العربية", "Arabic"), +] + +LANGUAGE_CODES = frozenset(code for code, _, _ in SUPPORTED_LANGUAGES) +_NATIVE_NAMES: Dict[str, str] = {code: native for code, native, _ in SUPPORTED_LANGUAGES} + + +_ZH_TRADITIONAL = frozenset({"zh-hant", "zh-tw", "zh-hk", "zh-mo"}) + + +def lang_base(code: str) -> str: + return (code or "").strip().split("-")[0].lower() + + +def is_traditional_chinese(code: str) -> bool: + return (code or "").strip().lower() in _ZH_TRADITIONAL + + +def languages_match(code: str, target: str) -> bool: + """ + Same language for our purposes: regional variants are interchangeable + (en-GB ~ en), but Chinese scripts are not (zh-Hans != zh-Hant). + """ + c, t = (code or "").strip().lower(), (target or "").strip().lower() + if not c or not t: + return False + if c == t: + return True + if lang_base(t) == "zh": + return lang_base(c) == "zh" and is_traditional_chinese(c) == is_traditional_chinese(t) + return lang_base(c) == lang_base(t) + + +def language_name(code: str) -> str: + """Native name for *code*, matching regional variants (en-GB -> English).""" + if code in _NATIVE_NAMES: + return _NATIVE_NAMES[code] + match = next((c for c, _, _ in SUPPORTED_LANGUAGES if languages_match(code, c)), None) + return _NATIVE_NAMES[match] if match else code + + +def public_language_list() -> List[Dict[str, str]]: + return [ + {"code": code, "name": native, "english_name": english} + for code, native, english in SUPPORTED_LANGUAGES + ] diff --git a/app/services/file_service.py b/app/services/file_service.py index 88b370b..137e610 100644 --- a/app/services/file_service.py +++ b/app/services/file_service.py @@ -11,6 +11,7 @@ from docx import Document from app.core.config import settings +from app.services.pdf_fonts import pdf_font_for from app.utils.logging_config import logger @@ -102,12 +103,18 @@ def generate_docx(content: str) -> str: return str(filepath) @staticmethod - def generate_pdf(content: str) -> str: + def generate_pdf(content: str, language: str = "") -> str: + """Raises pdf_fonts.UnsupportedPdfScript for scripts ReportLab can't typeset.""" from reportlab.lib.pagesizes import letter from reportlab.lib.styles import getSampleStyleSheet from reportlab.platypus import Paragraph, SimpleDocTemplate, Spacer from reportlab.lib.units import inch + # Pick the font before creating the file so an unsupported script leaves nothing behind + font = pdf_font_for(content, language) + if font.drop: + content = "".join(ch for ch in content if ch not in font.drop) + filepath = FileGenerator._get_temp_filepath("pdf") doc = SimpleDocTemplate( @@ -121,7 +128,9 @@ def generate_pdf(content: str) -> str: story = [] styles = getSampleStyleSheet() - style = styles["BodyText"] + style = styles["BodyText"].clone("Transcript", fontName=font.name) + if font.cjk_wrap: + style.wordWrap = "CJK" for para_text in content.split("\n\n"): if para_text.strip(): @@ -134,12 +143,12 @@ def generate_pdf(content: str) -> str: return str(filepath) @staticmethod - def generate_file(content: str, file_format: str) -> str: + def generate_file(content: str, file_format: str, language: str = "") -> str: """Returns the absolute path to the generated file on disk.""" if file_format == "docx": return FileGenerator.generate_docx(content) if file_format == "pdf": - return FileGenerator.generate_pdf(content) + return FileGenerator.generate_pdf(content, language) if file_format in ("srt", "vtt"): return FileGenerator.generate_txt(content, file_format) return FileGenerator.generate_txt(content) diff --git a/app/services/pdf_fonts.py b/app/services/pdf_fonts.py new file mode 100644 index 0000000..0ca8163 --- /dev/null +++ b/app/services/pdf_fonts.py @@ -0,0 +1,169 @@ +""" +Pick a PDF font that can actually draw the transcript's characters. + +Decided from the text itself, not the requested language: YouTube/Vimeo may +fall back to the original captions, so a "Spanish" request can return Russian. + +- Western European text -> Helvetica (built in, not embedded; unchanged output) +- Chinese / Japanese -> ReportLab's built-in CID fonts (no files needed) +- Korean -> an embedded Korean TTF (Nanum Gothic in Docker); the built-in Korean + CID fonts render blank in PDFium (Chrome's viewer) +- Other alphabetic scripts (Polish, Vietnamese, Cyrillic, Greek...) -> an embedded + Unicode TTF (DejaVu Sans in Docker), verified glyph by glyph +- Scripts that need shaping or right-to-left layout (Arabic, Hebrew, Indic, Thai...) + -> UnsupportedPdfScript: ReportLab would draw them as disconnected or reversed glyphs +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterable, Optional, Set, Tuple + +from app.core.config import settings +from app.core.languages import is_traditional_chinese + +UNSUPPORTED_PDF_MESSAGE = ( + "PDF isn't available for this language yet. Download DOCX or TXT instead." +) + +_COMPLEX_SCRIPTS: Tuple[Tuple[int, int], ...] = ( + (0x0590, 0x08FF), # Hebrew, Arabic, Syriac, Thaana, NKo, Samaritan... + (0x0900, 0x0DFF), # Devanagari through Sinhala (Indic) + (0x0E00, 0x0FFF), # Thai, Lao, Tibetan + (0x1000, 0x109F), # Myanmar + (0x1780, 0x17FF), # Khmer + (0xFB1D, 0xFDFF), # Hebrew / Arabic presentation forms + (0xFE70, 0xFEFF), # Arabic presentation forms B +) +_HANGUL = ((0x1100, 0x11FF), (0x3130, 0x318F), (0xAC00, 0xD7AF)) +_KANA = ((0x3040, 0x30FF), (0x31F0, 0x31FF)) +_HAN = ((0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF)) + +_TTF_CANDIDATES = ( + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", # Debian/Ubuntu (Docker image) + "/usr/share/fonts/dejavu/DejaVuSans.ttf", # Fedora/Alpine + "/Library/Fonts/Arial Unicode.ttf", # macOS dev machines + "C:/Windows/Fonts/arial.ttf", # Windows dev machines +) +_KOREAN_TTF_CANDIDATES = ( + "/usr/share/fonts/truetype/nanum/NanumGothic.ttf", # Debian fonts-nanum (Docker image) + "C:/Windows/Fonts/malgun.ttf", # Windows dev machines + "/System/Library/Fonts/Supplemental/AppleGothic.ttf", # macOS dev machines +) +_UNICODE_FONT_NAME = "TranscriptUnicode" +_KOREAN_FONT_NAME = "TranscriptKorean" + + +class UnsupportedPdfScript(Exception): + """Text uses a script this PDF pipeline cannot typeset correctly.""" + + def __init__(self) -> None: + super().__init__(UNSUPPORTED_PDF_MESSAGE) + + +@dataclass(frozen=True) +class PdfFont: + name: str + # CJK text has no spaces between words, so lines must break per character + cjk_wrap: bool = False + # Non-letter symbols the font can't draw (e.g. emoji, "♪"); removed rather than + # letting one caption symbol block the whole PDF + drop: frozenset = frozenset() + + +def _in_ranges(cp: int, ranges: Iterable[Tuple[int, int]]) -> bool: + return any(lo <= cp <= hi for lo, hi in ranges) + + +def _has(chars: Set[str], ranges: Iterable[Tuple[int, int]]) -> bool: + ranges = tuple(ranges) + return any(_in_ranges(ord(ch), ranges) for ch in chars) + + +def _is_western(ch: str) -> bool: + try: + ch.encode("cp1252") + return True + except UnicodeEncodeError: + return False + + +_register_lock = threading.Lock() +_registered: Set[str] = set() +# font name -> code points it can draw (None = no font file found) +_ttf_glyphs: Dict[str, Optional[Set[int]]] = {} + + +def _register_cid(name: str) -> str: + from reportlab.pdfbase import pdfmetrics + from reportlab.pdfbase.cidfonts import UnicodeCIDFont + + with _register_lock: + if name not in _registered: + pdfmetrics.registerFont(UnicodeCIDFont(name)) + _registered.add(name) + return name + + +def _register_ttf(name: str, candidates: Iterable[str]) -> Optional[Set[int]]: + """Register the first existing TTF under *name* once; returns its code points.""" + from reportlab.pdfbase import pdfmetrics + from reportlab.pdfbase.ttfonts import TTFont + + with _register_lock: + if name not in _ttf_glyphs: + path = next((Path(c) for c in candidates if c and Path(c).is_file()), None) + glyphs = None + if path is not None: + font = TTFont(name, str(path)) + pdfmetrics.registerFont(font) + glyphs = set(font.face.charToGlyph) + _ttf_glyphs[name] = glyphs + return _ttf_glyphs[name] + + +def _embedded_font(name: str, candidates: Iterable[str], chars: Set[str], **kw) -> PdfFont: + """Embedded TTF if it covers every letter; symbols it lacks are dropped.""" + glyphs = _register_ttf(name, candidates) + if glyphs is None or not all(ord(ch) in glyphs for ch in chars if ch.isalpha()): + raise UnsupportedPdfScript() + return PdfFont(name, drop=frozenset(ch for ch in chars if ord(ch) not in glyphs), **kw) + + +def pdf_font_for(text: str, language: str = "") -> PdfFont: + """Font able to render *text*; raises UnsupportedPdfScript otherwise.""" + chars = {ch for ch in set(text) if not ch.isspace()} + + if _has(chars, _COMPLEX_SCRIPTS): + raise UnsupportedPdfScript() + if _has(chars, _HANGUL): + return _embedded_font(_KOREAN_FONT_NAME, _KOREAN_TTF_CANDIDATES, chars, cjk_wrap=True) + if _has(chars, _KANA): + return PdfFont(_register_cid("HeiseiKakuGo-W5"), cjk_wrap=True) + if _has(chars, _HAN): + name = "MSung-Light" if is_traditional_chinese(language) else "STSong-Light" + return PdfFont(_register_cid(name), cjk_wrap=True) + + beyond_western = {ch for ch in chars if not _is_western(ch)} + if not beyond_western: + return PdfFont("Helvetica") + + # Only letters decide; symbols the chosen font lacks are dropped + try: + return _embedded_font( + _UNICODE_FONT_NAME, (settings.pdf_font_path, *_TTF_CANDIDATES), chars + ) + except UnsupportedPdfScript: + if any(ch.isalpha() for ch in beyond_western): + raise + return PdfFont("Helvetica", drop=frozenset(beyond_western)) + + +def pdf_supported(text: str) -> bool: + try: + pdf_font_for(text) + return True + except UnsupportedPdfScript: + return False diff --git a/app/services/telemetry_service.py b/app/services/telemetry_service.py index 5a01406..45e597d 100644 --- a/app/services/telemetry_service.py +++ b/app/services/telemetry_service.py @@ -33,6 +33,7 @@ "download_succeeded", "download_failed", "new_transcript_clicked", + "language_changed", # Telegram "tg_link_received", "tg_extract_succeeded", @@ -48,6 +49,8 @@ "tg_cmd_privacy", "tg_cmd_status", "tg_sample_clicked", + "tg_cmd_language", + "tg_language_set", } # Props keys we accept (everything else is stripped) @@ -57,6 +60,8 @@ "include_timestamps", "duration_seconds_bucket", "error_code", + "language", + "translated", } # Maximum payload size (bytes) — reject anything larger diff --git a/app/services/transcript_cache_service.py b/app/services/transcript_cache_service.py index fbd8572..4c36a72 100644 --- a/app/services/transcript_cache_service.py +++ b/app/services/transcript_cache_service.py @@ -26,14 +26,17 @@ def _safe_video_stem(video_id: str) -> str: return s or "invalid" -def cache_path(provider: str, video_id: str) -> Path: - return cache_dir() / f"{provider}_{_safe_video_stem(video_id)}.json" +def cache_path(provider: str, video_id: str, language: str = "en") -> Path: + # Language is part of the key: the same video yields different captions per target + return cache_dir() / f"{provider}_{_safe_video_stem(video_id)}_{_safe_video_stem(language)}.json" -def read_transcript_cache(provider: str, video_id: str) -> Optional[Dict[str, Any]]: +def read_transcript_cache( + provider: str, video_id: str, language: str = "en" +) -> Optional[Dict[str, Any]]: if not settings.transcript_cache_enabled: return None - path = cache_path(provider, video_id) + path = cache_path(provider, video_id, language) if not path.is_file(): return None ttl_sec = max(0, int(settings.transcript_cache_ttl_hours * 3600)) @@ -49,10 +52,12 @@ def read_transcript_cache(provider: str, video_id: str) -> Optional[Dict[str, An return None -def write_transcript_cache(provider: str, video_id: str, data: Dict[str, Any]) -> None: +def write_transcript_cache( + provider: str, video_id: str, data: Dict[str, Any], language: str = "en" +) -> None: if not settings.transcript_cache_enabled: return - path = cache_path(provider, video_id) + path = cache_path(provider, video_id, language) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(".json.tmp") try: diff --git a/app/services/transcript_service.py b/app/services/transcript_service.py index be9d452..b2e9fdc 100644 --- a/app/services/transcript_service.py +++ b/app/services/transcript_service.py @@ -28,9 +28,15 @@ ) from app.core.config import settings from app.core.errors import ErrorCode, TranscriptFetchError +from app.core.languages import ( + DEFAULT_LANGUAGE, + is_traditional_chinese, + lang_base, + languages_match, +) # User-facing when YouTube IpBlocked / RequestBlocked (cloud IPs are often banned). -YOUTUBE_TRANSCRIPT_UNAVAILABLE_EN_ONLY = ( +YOUTUBE_BLOCKED_MESSAGE = ( "We could not load captions because YouTube blocked this server. " "Please try again later." ) @@ -166,9 +172,13 @@ def _youtube_api(deadline: Optional[float] = None) -> YouTubeTranscriptApi: return YouTubeTranscriptApi(proxy_config=proxy_config, http_client=http_client) -def get_transcript_from_url(url: str, deadline: Optional[float] = None) -> Dict: +def get_transcript_from_url( + url: str, deadline: Optional[float] = None, language: str = DEFAULT_LANGUAGE +) -> Dict: """ - Fetch transcript. YouTube captions are always resolved to English (native or auto-translate). + Fetch captions in *language*: YouTube uses a native track or auto-translates; + Vimeo picks the matching uploaded track. Either may fall back to the original + captions; the result's "language" says what was actually returned. Raises TranscriptFetchError for expected failures; anything else is a bug. *deadline* (time.monotonic()) bounds upstream work so it stops once the caller gives up. """ @@ -180,7 +190,7 @@ def get_transcript_from_url(url: str, deadline: Optional[float] = None) -> Dict: except ValueError as e: raise TranscriptFetchError(ErrorCode.INVALID_URL, str(e)) from e - cached = read_transcript_cache(platform, video_id) + cached = read_transcript_cache(platform, video_id, language) if cached is not None: out = dict(cached) if url: @@ -188,10 +198,14 @@ def get_transcript_from_url(url: str, deadline: Optional[float] = None) -> Dict: return out if platform == "youtube": - result = get_youtube_transcript(video_id, url=url, deadline=deadline) + result = get_youtube_transcript( + video_id, url=url, deadline=deadline, target_language=language + ) else: - result = get_vimeo_transcript(video_id, url=url, deadline=deadline) - write_transcript_cache(platform, video_id, result) + result = get_vimeo_transcript( + video_id, url=url, deadline=deadline, target_language=language + ) + write_transcript_cache(platform, video_id, result, language) return result @@ -229,8 +243,13 @@ def _language_priority_tuple() -> Tuple[str, ...]: return tuple(merged) -def _norm_lang_base(code: str) -> str: - return (code or "").strip().split("-")[0].lower() +def _target_aliases(target: str) -> Tuple[str, ...]: + """Caption codes YouTube may use for a native track in *target*.""" + if is_traditional_chinese(target): + return ("zh-Hant", "zh-TW", "zh-HK") + if lang_base(target) == "zh": + return ("zh-Hans", "zh-CN", "zh") + return (target, lang_base(target)) def _select_transcript_for_priority( @@ -245,44 +264,36 @@ def _select_transcript_for_priority( raise NoTranscriptFound(tlist.video_id, list(cleaned), tlist) from None -def _translation_api_code_for_target(tr: Transcript, target_base: str) -> Optional[str]: - """Exact language_code to pass to translate(), matching YouTube’s list.""" - if not tr.is_translatable or not target_base: +def _translation_api_code_for_target(tr: Transcript, target: str) -> Optional[str]: + """Exact language_code to pass to translate(), matching YouTube's list.""" + if not tr.is_translatable or not target: return None - for tl in tr.translation_languages: - if _norm_lang_base(tl.language_code) == target_base: - return tl.language_code - return None + codes = [tl.language_code for tl in tr.translation_languages] + exact = next((c for c in codes if c.lower() == target.lower()), None) + return exact or next((c for c in codes if languages_match(c, target)), None) -def _fetch_for_target_language(tr: Transcript, target_language: str) -> Tuple[Any, str]: +def _fetch_for_target_language(tr: Transcript, target_language: str) -> Tuple[Any, str, bool]: """ - Return (fetched_transcript, language_code). Prefer native track in *target_language*, - else YouTube translate(), else original captions. + Return (fetched_transcript, language_code, translated). Prefer native track in + *target_language*, else YouTube translate(), else original captions. """ - tgt = _norm_lang_base(target_language) - src = _norm_lang_base(tr.language_code) - if tgt and src == tgt: + if languages_match(tr.language_code, target_language): fetched = tr.fetch() - return fetched, fetched.language_code + return fetched, fetched.language_code, False - api_code = _translation_api_code_for_target(tr, tgt) if tgt else None + api_code = _translation_api_code_for_target(tr, target_language) if api_code: try: fetched = tr.translate(api_code).fetch() - return fetched, fetched.language_code - except (TranslationLanguageNotAvailable, NotTranslatable, Exception): - pass - - if tgt: - try: - fetched = tr.translate(target_language).fetch() - return fetched, fetched.language_code + return fetched, fetched.language_code, True except (TranslationLanguageNotAvailable, NotTranslatable): + # Only "can't translate" falls back. Blocks/timeouts must surface: a silent + # fallback would be mislabelled "not available" and cached for days. pass fetched = tr.fetch() - return fetched, fetched.language_code + return fetched, fetched.language_code, False def _youtube_transcript_fresh_core( @@ -291,27 +302,23 @@ def _youtube_transcript_fresh_core( url: str, metadata_future: Optional[Future] = None, deadline: Optional[float] = None, + target_language: str = DEFAULT_LANGUAGE, ) -> Dict: if metadata_future is None: metadata_future = _metadata_pool.submit(_get_youtube_metadata, video_id) api = _youtube_api(deadline) tlist = api.list(video_id) - target = "en" - + translated = False if languages is not None: transcript = tlist.find_transcript(tuple(languages)) fetched = transcript.fetch() out_lang = fetched.language_code else: - priority = tuple( - dict.fromkeys( - [target, _norm_lang_base(target), *list(_language_priority_tuple())] - ) - ) - priority = tuple(p for p in priority if p) + # Native track in the target first; otherwise the best source to translate from + priority = (*_target_aliases(target_language), *_language_priority_tuple()) transcript = _select_transcript_for_priority(tlist, priority) - fetched, out_lang = _fetch_for_target_language(transcript, target) + fetched, out_lang, translated = _fetch_for_target_language(transcript, target_language) segments = [ {"text": item.text, "start": item.start, "duration": item.duration} @@ -325,6 +332,8 @@ def _youtube_transcript_fresh_core( "video_id": video_id, "title": metadata["title"], "language": out_lang, + "source_language": transcript.language_code, + "translated": translated, "duration_seconds": metadata["duration"], "segments": segments, } @@ -335,7 +344,7 @@ def _youtube_fetch_error(e: CouldNotRetrieveTranscript) -> TranscriptFetchError: na = ErrorCode.TRANSCRIPT_NOT_AVAILABLE if isinstance(e, (IpBlocked, RequestBlocked)): return TranscriptFetchError( - ErrorCode.UPSTREAM_BLOCKED, YOUTUBE_TRANSCRIPT_UNAVAILABLE_EN_ONLY + ErrorCode.UPSTREAM_BLOCKED, YOUTUBE_BLOCKED_MESSAGE ) if isinstance(e, TranscriptsDisabled): return TranscriptFetchError(na, "Transcripts are disabled for this video.") @@ -354,10 +363,11 @@ def get_youtube_transcript( languages: Optional[List[str]] = None, url: str = "", deadline: Optional[float] = None, + target_language: str = DEFAULT_LANGUAGE, ) -> Dict: """ list() + find_transcript; never uses fetch(video_id) default ['en'] shortcut. - English-only: prefer English captions, else translate to English when available. + Prefers a native track in *target_language*, else YouTube auto-translation. With residential proxy configured, retries a few times on IpBlocked / RequestBlocked, but never sleeps past *deadline*. """ @@ -367,7 +377,12 @@ def get_youtube_transcript( for attempt in range(attempts): try: return _youtube_transcript_fresh_core( - video_id, languages, url, metadata_future=metadata_future, deadline=deadline + video_id, + languages, + url, + metadata_future=metadata_future, + deadline=deadline, + target_language=target_language, ) except (IpBlocked, RequestBlocked) as e: delay = settings.youtube_transcript_retry_backoff_seconds * (attempt + 1) diff --git a/app/services/user_prefs_service.py b/app/services/user_prefs_service.py index 4eba9d0..e4d68f8 100644 --- a/app/services/user_prefs_service.py +++ b/app/services/user_prefs_service.py @@ -13,10 +13,15 @@ import redis from app.core.config import settings +from app.core.languages import DEFAULT_LANGUAGE, LANGUAGE_CODES from app.core.redis_client import get_redis, warn_fallback FILE_FORMATS = ("txt", "pdf", "docx", "srt", "vtt") -DEFAULTS: Dict[str, Any] = {"last_file_format": "txt", "last_include_timestamps": False} +DEFAULTS: Dict[str, Any] = { + "last_file_format": "txt", + "last_include_timestamps": False, + "language": DEFAULT_LANGUAGE, +} _memory: Dict[str, Dict[str, Any]] = {} @@ -30,6 +35,8 @@ def _decode(raw: Dict[str, str]) -> Dict[str, Any]: prefs = dict(DEFAULTS) if raw.get("last_file_format") in FILE_FORMATS: prefs["last_file_format"] = raw["last_file_format"] + if raw.get("language") in LANGUAGE_CODES: + prefs["language"] = raw["language"] if raw.get("last_include_timestamps") in ("0", "1"): prefs["last_include_timestamps"] = raw["last_include_timestamps"] == "1" return prefs diff --git a/app/services/vimeo_service.py b/app/services/vimeo_service.py index 662daac..da6f6d3 100644 --- a/app/services/vimeo_service.py +++ b/app/services/vimeo_service.py @@ -7,6 +7,7 @@ from typing import List, Dict, Optional from app.core.errors import ErrorCode, TranscriptFetchError +from app.core.languages import DEFAULT_LANGUAGE, languages_match _NO_CAPTIONS = "No captions are available for this Vimeo video." _REQUEST_TIMEOUT = 10 @@ -24,7 +25,12 @@ def _timeout(deadline: Optional[float]) -> float: return min(_REQUEST_TIMEOUT, remaining) -def get_vimeo_transcript(video_id: str, url: str = '', deadline: Optional[float] = None) -> Dict: +def get_vimeo_transcript( + video_id: str, + url: str = '', + deadline: Optional[float] = None, + target_language: str = DEFAULT_LANGUAGE, +) -> Dict: """ Fetches the transcript for a given Vimeo video ID. Extracts text tracks from Vimeo player config and parses VTT format. @@ -69,14 +75,12 @@ def get_vimeo_transcript(video_id: str, url: str = '', deadline: Optional[float] if not text_tracks or len(text_tracks) == 0: raise TranscriptFetchError(ErrorCode.TRANSCRIPT_NOT_AVAILABLE, _NO_CAPTIONS) - # Use the first available text track (preferably English) - selected_track = None - for track in text_tracks: - if track.get('lang') == 'en': - selected_track = track - break - - # If no English, use the first one + # Vimeo can't translate: prefer the requested language, then English, then any track + selected_track = next( + (t for t in text_tracks if languages_match(t.get('lang', ''), target_language)), + None, + ) or next((t for t in text_tracks if languages_match(t.get('lang', ''), 'en')), None) + if not selected_track and len(text_tracks) > 0: selected_track = text_tracks[0] @@ -103,6 +107,8 @@ def get_vimeo_transcript(video_id: str, url: str = '', deadline: Optional[float] "video_id": video_id, "title": title, "language": language, + "source_language": language, + "translated": False, "duration_seconds": duration, "segments": segments } diff --git a/tests/test_extract_reliability.py b/tests/test_extract_reliability.py index 5a153fb..bd9093c 100644 --- a/tests/test_extract_reliability.py +++ b/tests/test_extract_reliability.py @@ -12,6 +12,7 @@ from app.core.errors import ErrorCode, TranscriptFetchError from app.main import app +from app.services.rate_limit_service import rate_limiter from app.services import transcript_service as ts from app.services.vimeo_service import get_vimeo_transcript @@ -127,6 +128,10 @@ def test_vimeo_stops_after_deadline(self, mock_get): class TestBlockedResponse(unittest.TestCase): + + def setUp(self): + # All TestClient requests share one "IP"; other suites' extracts must not exhaust its quota + rate_limiter._store.clear() @patch("app.api.v1_routes.get_transcript_from_url") def test_ip_block_returns_503_with_own_code(self, mock_fetch): mock_fetch.side_effect = ts._youtube_fetch_error(IpBlocked("vid")) diff --git a/tests/test_get_transcript_from_url.py b/tests/test_get_transcript_from_url.py index 7f86720..04dcc5d 100644 --- a/tests/test_get_transcript_from_url.py +++ b/tests/test_get_transcript_from_url.py @@ -43,8 +43,8 @@ def test_vimeo_delegates(self, mock_vm, _mock_read, _mock_write): "segments": [], } url = "https://vimeo.com/999" - result = get_transcript_from_url(url, deadline=123.0) - mock_vm.assert_called_once_with("999", url=url, deadline=123.0) + result = get_transcript_from_url(url, deadline=123.0, language="de") + mock_vm.assert_called_once_with("999", url=url, deadline=123.0, target_language="de") self.assertEqual(result["provider"], "vimeo") @patch("app.services.transcript_service.get_youtube_transcript") diff --git a/tests/test_languages.py b/tests/test_languages.py new file mode 100644 index 0000000..88abaa1 --- /dev/null +++ b/tests/test_languages.py @@ -0,0 +1,215 @@ +"""Language picker: matching rules, target-language fetch, cache keys, API, PDF fonts.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from fastapi.testclient import TestClient + +from app.core.languages import LANGUAGE_CODES, language_name, languages_match +from app.main import app +from app.services.rate_limit_service import rate_limiter +from app.services import pdf_fonts +from app.services import transcript_cache_service as tcs +from app.services import transcript_service as ts +from app.services.file_service import FileGenerator + +client = TestClient(app) + + +class TestLanguageMatching(unittest.TestCase): + def test_regional_variants_match(self): + self.assertTrue(languages_match("en-GB", "en")) + self.assertTrue(languages_match("pt-BR", "pt")) + self.assertFalse(languages_match("es", "en")) + + def test_chinese_scripts_do_not_match(self): + self.assertTrue(languages_match("zh-TW", "zh-Hant")) + self.assertTrue(languages_match("zh-CN", "zh-Hans")) + self.assertFalse(languages_match("zh-Hans", "zh-Hant")) + + def test_language_name_handles_variants(self): + self.assertEqual(language_name("en-GB"), "English") + self.assertEqual(language_name("zh-TW"), "繁體中文") + self.assertEqual(language_name("fil"), "fil") + + +class _Fetched(list): + def __init__(self, code, items): + super().__init__(items) + self.language_code = code + + +def _track(code, translations=()): + tr = MagicMock(language_code=code, is_translatable=bool(translations)) + tr.translation_languages = [SimpleNamespace(language_code=c) for c in translations] + tr.fetch.return_value = _Fetched(code, [SimpleNamespace(text=f"{code} text", start=0.0, duration=1.0)]) + tr.translate.side_effect = lambda c: MagicMock( + fetch=MagicMock(return_value=_Fetched(c, [SimpleNamespace(text=f"{c} text", start=0.0, duration=1.0)])) + ) + return tr + + +def _fetch_with_tracks(tracks, target): + """Run the YouTube core against a fake caption list.""" + by_code = {t.language_code: t for t in tracks} + tlist = MagicMock(video_id="vid") + + def find(codes): + for c in codes: + if c in by_code: + return by_code[c] + raise ts.NoTranscriptFound("vid", list(codes), tlist) + + tlist.find_transcript.side_effect = find + tlist.__iter__.side_effect = lambda: iter(tracks) + api = MagicMock() + api.list.return_value = tlist + with patch.object(ts, "YouTubeTranscriptApi", return_value=api), patch.object( + ts, "_get_youtube_metadata", return_value={"title": "T", "duration": 0} + ): + return ts.get_youtube_transcript("vid", target_language=target) + + +class TestTargetLanguageFetch(unittest.TestCase): + def test_native_track_preferred(self): + out = _fetch_with_tracks([_track("en", ["es"]), _track("es")], "es") + self.assertEqual(out["language"], "es") + self.assertFalse(out["translated"]) + + def test_translates_when_no_native_track(self): + out = _fetch_with_tracks([_track("en", ["es", "ja"])], "ja") + self.assertEqual(out["language"], "ja") + self.assertEqual(out["source_language"], "en") + self.assertTrue(out["translated"]) + + def test_traditional_chinese_not_served_simplified(self): + out = _fetch_with_tracks([_track("zh-Hans", ["zh-Hans", "zh-Hant"])], "zh-Hant") + self.assertEqual(out["language"], "zh-Hant") + self.assertTrue(out["translated"]) + + def test_blocked_translation_is_an_error_not_a_fallback(self): + source = _track("en", ["ja"]) + source.translate.side_effect = None + source.translate.return_value.fetch.side_effect = ts.IpBlocked("vid") + with patch.object(ts, "write_transcript_cache") as write_cache: + with self.assertRaises(ts.TranscriptFetchError) as ctx: + _fetch_with_tracks([source], "ja") + self.assertEqual(ctx.exception.code.value, "UPSTREAM_BLOCKED") + write_cache.assert_not_called() + + def test_falls_back_to_original_when_untranslatable(self): + out = _fetch_with_tracks([_track("de")], "fr") + self.assertEqual(out["language"], "de") + self.assertFalse(out["translated"]) + + +class TestCacheKey(unittest.TestCase): + def test_language_in_cache_path(self): + self.assertNotEqual( + tcs.cache_path("youtube", "vid", "en"), tcs.cache_path("youtube", "vid", "es") + ) + + +class TestApi(unittest.TestCase): + + def setUp(self): + # All TestClient requests share one "IP"; other suites' extracts must not exhaust its quota + rate_limiter._store.clear() + def _extract(self, mock_fetch, data, language="es"): + mock_fetch.return_value = data + return client.post( + "/api/v1/extract", + json={ + "url": "https://www.youtube.com/watch?v=testvideo12", + "include_timestamps": False, + "language": language, + }, + ) + + def test_config_lists_languages(self): + body = client.get("/api/v1/config").json() + self.assertEqual(body["default_language"], "en") + self.assertEqual({l["code"] for l in body["languages"]}, set(LANGUAGE_CODES)) + + def test_unknown_language_rejected(self): + r = client.post( + "/api/v1/extract", + json={"url": "https://www.youtube.com/watch?v=testvideo12", "language": "xx"}, + ) + self.assertEqual(r.status_code, 400) + self.assertEqual(r.json()["error"]["code"], "UNSUPPORTED_LANGUAGE") + + @patch("app.api.v1_routes.get_transcript_from_url") + def test_translated_response(self, mock_fetch): + r = self._extract(mock_fetch, { + "provider": "youtube", "video_id": "testvideo12", "title": "T", + "language": "es", "source_language": "en", "translated": True, + "duration_seconds": 5, "segments": [{"text": "Hola", "start": 0, "duration": 1}], + }) + body = r.json() + self.assertEqual(mock_fetch.call_args.args[2], "es") + self.assertEqual(body["language_name"], "Español") + self.assertEqual(body["source_language_name"], "English") + self.assertTrue(body["translated"]) + self.assertFalse(body["language_fallback"]) + self.assertTrue(body["pdf_supported"]) + + @patch("app.api.v1_routes.get_transcript_from_url") + def test_arabic_fallback_disables_pdf(self, mock_fetch): + r = self._extract(mock_fetch, { + "provider": "youtube", "video_id": "testvideo12", "title": "T", + "language": "ar", "source_language": "ar", "translated": False, + "duration_seconds": 5, "segments": [{"text": "مرحبا بالعالم", "start": 0, "duration": 1}], + }) + body = r.json() + self.assertTrue(body["language_fallback"]) + self.assertFalse(body["pdf_supported"]) + + conv = client.post("/api/v1/convert", json={"file_id": body["file_id"], "format": "pdf"}) + self.assertEqual(conv.status_code, 400) + self.assertIn("PDF isn't available", conv.json()["error"]["message"]) + docx = client.post("/api/v1/convert", json={"file_id": body["file_id"], "format": "docx"}) + self.assertEqual(docx.status_code, 200) + + +class TestPdfFonts(unittest.TestCase): + def test_western_stays_helvetica(self): + self.assertEqual(pdf_fonts.pdf_font_for("Café déjà vu").name, "Helvetica") + + def test_cjk_uses_cid_font_with_cjk_wrap(self): + font = pdf_fonts.pdf_font_for("你好世界", "zh-Hant") + self.assertEqual(font.name, "MSung-Light") + self.assertTrue(font.cjk_wrap) + self.assertEqual(pdf_fonts.pdf_font_for("こんにちは").name, "HeiseiKakuGo-W5") + + def test_complex_scripts_refused(self): + for text in ("مرحبا", "नमस्ते", "שלום", "สวัสดี"): + with self.assertRaises(pdf_fonts.UnsupportedPdfScript): + pdf_fonts.pdf_font_for(text) + + def test_symbols_never_block_a_western_pdf(self): + with patch.object(pdf_fonts, "_register_ttf", return_value=None): + font = pdf_fonts.pdf_font_for("[♪ music ♪] hello 🎉") + self.assertEqual(font.name, "Helvetica") + self.assertEqual(font.drop, frozenset({"♪", "🎉"})) + + def test_letters_without_font_refused(self): + with patch.object(pdf_fonts, "_register_ttf", return_value=None): + with self.assertRaises(pdf_fonts.UnsupportedPdfScript): + pdf_fonts.pdf_font_for("Привет") + + def test_letters_missing_from_font_refused(self): + with patch.object(pdf_fonts, "_register_ttf", return_value={ord("П")}): + with self.assertRaises(pdf_fonts.UnsupportedPdfScript): + pdf_fonts.pdf_font_for("Привет") + + @unittest.skipUnless(pdf_fonts._register_ttf("TranscriptUnicode", pdf_fonts._TTF_CANDIDATES), "no Unicode TTF on this machine") + def test_cyrillic_pdf_builds_with_embedded_font(self): + path = FileGenerator.generate_pdf("Привет, как дела") + with open(path, "rb") as f: + self.assertIn(b"/FontFile2", f.read()) # embedded TrueType, not Helvetica + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_shared_state_and_privacy.py b/tests/test_shared_state_and_privacy.py index 9117b19..0f1da13 100644 --- a/tests/test_shared_state_and_privacy.py +++ b/tests/test_shared_state_and_privacy.py @@ -141,17 +141,23 @@ def test_defaults(self): def test_roundtrip_through_redis_survives_process_memory_loss(self): fake = FakeRedis() with patch.object(prefs, "get_redis", return_value=fake): - prefs.set_prefs("42", last_file_format="srt", last_include_timestamps=True) + prefs.set_prefs( + "42", last_file_format="srt", last_include_timestamps=True, language="ja" + ) prefs._memory.clear() # simulate a restart self.assertEqual( prefs.get_prefs("42"), - {"last_file_format": "srt", "last_include_timestamps": True}, + {"last_file_format": "srt", "last_include_timestamps": True, "language": "ja"}, ) self.assertEqual(fake.ttl["tf:tgprefs:42"], prefs.settings.telegram_prefs_ttl_days * 86400) def test_invalid_stored_values_ignored(self): fake = FakeRedis() - fake.data["tf:tgprefs:7"] = {"last_file_format": "exe", "last_include_timestamps": "yes"} + fake.data["tf:tgprefs:7"] = { + "last_file_format": "exe", + "last_include_timestamps": "yes", + "language": "klingon", + } with patch.object(prefs, "get_redis", return_value=fake): self.assertEqual(prefs.get_prefs("7"), prefs.DEFAULTS) diff --git a/tests/test_v1_api.py b/tests/test_v1_api.py index 0d426ec..0fc407d 100644 --- a/tests/test_v1_api.py +++ b/tests/test_v1_api.py @@ -7,6 +7,7 @@ from app.core.errors import ErrorCode, TranscriptFetchError from app.main import app +from app.services.rate_limit_service import rate_limiter client = TestClient(app) @@ -25,6 +26,10 @@ class TestV1Extract(unittest.TestCase): + + def setUp(self): + # All TestClient requests share one "IP"; other suites' extracts must not exhaust its quota + rate_limiter._store.clear() @patch("app.api.v1_routes.get_transcript_from_url") def test_extract_success_envelope(self, mock_fetch): mock_fetch.return_value = {**_TRANSCRIPT} @@ -93,6 +98,10 @@ def test_extract_internal_error_hides_exception_detail(self, mock_fetch): class TestV1Convert(unittest.TestCase): + + def setUp(self): + # All TestClient requests share one "IP"; other suites' extracts must not exhaust its quota + rate_limiter._store.clear() @patch("app.api.v1_routes.get_transcript_from_url") def test_convert_txt_returns_download_url(self, mock_fetch): mock_fetch.return_value = {**_TRANSCRIPT} diff --git a/web/app/components/home/Capabilities.tsx b/web/app/components/home/Capabilities.tsx index de741c9..66fa52d 100644 --- a/web/app/components/home/Capabilities.tsx +++ b/web/app/components/home/Capabilities.tsx @@ -3,6 +3,7 @@ import React from 'react'; const items = [ { label: 'Sources', description: 'YouTube and Vimeo links' }, { label: 'Output', description: 'TXT (default), PDF, DOCX, SRT, VTT' }, + { label: 'Languages', description: '23 languages, auto-translated by YouTube' }, { label: 'Formatting', description: 'Clean text or timestamps' }, { label: 'Preview', description: 'Short preview before download' }, { label: 'Privacy', description: 'No account required' }, diff --git a/web/components/TranscribeClient.tsx b/web/components/TranscribeClient.tsx index f2e23b9..0672599 100644 --- a/web/components/TranscribeClient.tsx +++ b/web/components/TranscribeClient.tsx @@ -12,6 +12,12 @@ interface SuccessData { video_id: string; title: string; language: string; + language_name: string; + requested_language: string; + source_language_name: string; + translated: boolean; + language_fallback: boolean; + pdf_supported: boolean; duration_seconds: number; word_count: number; reading_time_seconds: number; @@ -28,6 +34,33 @@ interface ErrorData { type DownloadFormat = 'txt' | 'pdf' | 'docx' | 'srt' | 'vtt'; +interface Language { + code: string; + name: string; + english_name: string; +} + +// Used until /config answers (or if it fails) +const FALLBACK_LANGUAGES: Language[] = [{ code: 'en', name: 'English', english_name: 'English' }]; +const LANGUAGE_KEY = 'tf_language'; + +// Per-viewer convenience only; storage can be unavailable (private mode, blocked site data) +function readStoredLanguage(): string | null { + try { + return localStorage.getItem(LANGUAGE_KEY); + } catch { + return null; + } +} + +function storeLanguage(code: string) { + try { + localStorage.setItem(LANGUAGE_KEY, code); + } catch { + /* ignore */ + } +} + // TXT is the primary action; the rest go through /convert. const SECONDARY_FORMATS: { format: DownloadFormat; label: string }[] = [ { format: 'pdf', label: 'PDF' }, @@ -62,6 +95,8 @@ export function TranscribeClient() { const [downloading, setDownloading] = useState(null); const [convertError, setConvertError] = useState(null); const [fileTtlHours, setFileTtlHours] = useState(1); + const [languages, setLanguages] = useState(FALLBACK_LANGUAGES); + const [language, setLanguage] = useState('en'); const resultRef = useRef(null); const API_BASE = PUBLIC_API_BASE; @@ -72,22 +107,37 @@ export function TranscribeClient() { } }, [state]); - // Fetch public config (file TTL) on mount + // Fetch public config (file TTL, languages) on mount useEffect(() => { trackEvent('app_opened'); fetch(`${API_BASE}/api/v1/config`) .then(r => r.json()) - .then(d => { if (d.success && d.file_ttl_hours) setFileTtlHours(d.file_ttl_hours); }) + .then(d => { + if (!d.success) return; + if (d.file_ttl_hours) setFileTtlHours(d.file_ttl_hours); + if (Array.isArray(d.languages) && d.languages.length) { + const list = d.languages as Language[]; + setLanguages(list); + const stored = readStoredLanguage(); + if (stored && list.some(l => l.code === stored)) setLanguage(stored); + } + }) .catch(() => { }); }, [API_BASE]); + const handleLanguageChange = (code: string) => { + setLanguage(code); + storeLanguage(code); + trackEvent('language_changed', { language: code }); + }; + const handleGenerate = async () => { if (!url.trim()) return; setState('processing'); setStep(0); setSuccess(null); setError(null); - trackEvent('extract_clicked', { include_timestamps: timestamps }); + trackEvent('extract_clicked', { include_timestamps: timestamps, language }); const stepTimer = setInterval(() => { setStep(prev => (prev < 2 ? prev + 1 : prev)); @@ -100,6 +150,7 @@ export function TranscribeClient() { body: JSON.stringify({ url: url.trim(), include_timestamps: timestamps, + language, }), }); @@ -112,6 +163,8 @@ export function TranscribeClient() { provider: data.provider, duration_seconds_bucket: data.duration_seconds <= 600 ? '0-600' : data.duration_seconds <= 1800 ? '600-1800' : data.duration_seconds <= 3600 ? '1800-3600' : '3600+', include_timestamps: timestamps, + language: data.language, + translated: Boolean(data.translated), }); setTimeout(() => { setSuccess(data as SuccessData); @@ -163,7 +216,7 @@ export function TranscribeClient() { trackEvent('download_succeeded', { file_format: format }); } else { const fmt = format.toUpperCase(); - setConvertError(`Could not generate ${fmt}. Try TXT or retry.`); + setConvertError(data.error?.message ?? `Could not generate ${fmt}. Try TXT or retry.`); trackEvent('download_failed', { file_format: format, error_code: data.error?.code ?? 'CONVERT_FAILED' }); } } catch { @@ -239,9 +292,29 @@ export function TranscribeClient() { - - YouTube transcripts are English-only for now. - + {/* Language */} + + + Transcript language + + handleLanguageChange(e.target.value)} + disabled={isProcessing} + className="bg-transparent py-1 pr-2 text-sm font-medium focus:outline-none cursor-pointer disabled:opacity-50" + style={{ color: 'var(--text)', borderBottom: '1px solid var(--border)' }} + > + {languages.map(l => ( + + {l.name === l.english_name ? l.name : `${l.name} · ${l.english_name}`} + + ))} + + + YouTube auto-translates when a video has no captions in this language. + + {/* Generate / Extracting button */} ))} + + {/* What language actually came back */} + {(success.translated || success.language_fallback) && ( + + {success.translated + ? `Auto-translated by YouTube from ${success.source_language_name}. Machine translation can contain mistakes.` + : `${languages.find(l => l.code === success.requested_language)?.name ?? success.requested_language} captions weren't available for this video, so this transcript is in ${success.language_name}.`} + + )} {/* Preview block */} @@ -345,6 +428,7 @@ export function TranscribeClient() { Preview (truncated) handleDownload(format)} - disabled={downloading !== null} + disabled={downloading !== null || (format === 'pdf' && !success.pdf_supported)} + title={format === 'pdf' && !success.pdf_supported ? "PDF isn't available for this language yet. Use DOCX or TXT." : undefined} className="inline-flex items-center gap-3 px-8 py-4 font-bold text-[10px] uppercase tracking-widest transition-all rounded-full active:scale-[0.97] cursor-pointer disabled:opacity-40" style={{ color: 'var(--text)', border: '1px solid var(--border)', background: 'transparent' }} > @@ -397,6 +482,12 @@ export function TranscribeClient() { + {!success.pdf_supported && ( + + PDF isn't available for this language yet. DOCX, TXT, SRT and VTT work normally. + + )} + {/* Convert error message */} {convertError && (
- YouTube transcripts are English-only for now. -
+ YouTube auto-translates when a video has no captions in this language. +
+ {success.translated + ? `Auto-translated by YouTube from ${success.source_language_name}. Machine translation can contain mistakes.` + : `${languages.find(l => l.code === success.requested_language)?.name ?? success.requested_language} captions weren't available for this video, so this transcript is in ${success.language_name}.`} +
Preview (truncated)
+ PDF isn't available for this language yet. DOCX, TXT, SRT and VTT work normally. +