Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,12 @@ TELEGRAM_BOT_TOKEN=YOUR_TOKEN_HERE
# WEBSHARE_PROXY_PASSWORD=
# WEBSHARE_PROXY_LOCATIONS=

# --- PDF fonts
# Unicode TTF for PDFs beyond Western European text (Polish, Vietnamese, Cyrillic, Greek).
# Docker installs DejaVu Sans + Nanum Gothic (Korean); set this for local dev if needed.
# Arabic, Hebrew, Indic and Thai scripts get DOCX/TXT only (no PDF).
# PDF_FONT_PATH=

# --- Transcript disk cache
TRANSCRIPT_CACHE_ENABLED=true
TRANSCRIPT_CACHE_TTL_HOURS=72
Expand Down
6 changes: 5 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,13 @@ RUN groupadd --system --gid 1000 app \

WORKDIR /app

# System deps for reportlab / cffi
# System deps for reportlab / cffi. PDF fonts: DejaVu Sans embeds Latin-extended,
# Greek and Cyrillic (Helvetica only covers Western European); Nanum Gothic embeds
# Korean (ReportLab's built-in Korean CID fonts render blank in Chrome's viewer)
RUN apt-get update && apt-get install -y --no-install-recommends \
libffi-dev \
fonts-dejavu-core \
fonts-nanum \
&& rm -rf /var/lib/apt/lists/*

COPY requirements.txt .
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ Core Value: **Fast → Reliable → Structured → Downloadable.**

- **Multi-Platform Support**: Extract transcripts from **YouTube** and **Vimeo** instantly.
- **Clean Formats**: Download as **TXT, PDF, and DOCX**, or **SRT / VTT** subtitles with cue timings. Files are named after the video title.
- **23 Languages**: Pick the transcript language on the web or with `/language` in Telegram. YouTube auto-translates when a video has no captions in that language; Vimeo uses the uploader's tracks.
- **Timestamps**: Toggle perfectly formatted timestamps on or off.
- **Anonymous & Secure**: No accounts required. Generated files auto-delete after ~1 hour (TTL). Optional short-term caption cache (`TRANSCRIPT_CACHE_TTL_HOURS`) is pruned by the same cleanup loop.
- **Rate Limited & Guardrailed**: Separate extract vs convert quotas (shared across workers and the bot via optional Redis), max video duration (with caption-based fallback), segment/size caps, and timeout protections.
Expand Down
44 changes: 40 additions & 4 deletions app/api/v1_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,13 @@
verify_rate_limit_extract,
)
from app.core.errors import AppError, ErrorCode, TranscriptFetchError, success_response
from app.core.languages import (
DEFAULT_LANGUAGE,
LANGUAGE_CODES,
language_name,
languages_match,
public_language_list,
)
from app.services.analytics_service import AnalyticsService
from app.services.extract_guardrails import enforce_transcript_guardrails
from app.services.file_service import (
Expand All @@ -27,6 +34,7 @@
write_sidecar,
)
from app.services.formatter_service import SUBTITLE_FORMATS, TranscriptFormatter
from app.services.pdf_fonts import UnsupportedPdfScript, pdf_supported
from app.services.metadata_service import MetadataService
from app.services.telemetry_service import MAX_PAYLOAD_BYTES, TelemetryService
from app.services.transcript_service import get_transcript_from_url
Expand Down Expand Up @@ -68,6 +76,8 @@ def _validate_file_id(file_id: str) -> str:
class ExtractRequest(BaseModel):
url: str = Field(..., min_length=20, max_length=2048)
include_timestamps: bool = False
# Target caption language (see app.core.languages); YouTube auto-translates
language: str = DEFAULT_LANGUAGE


@router.get("/health")
Expand All @@ -88,13 +98,18 @@ async def extract_transcript(request: ExtractRequest):
400,
)

if request.language not in LANGUAGE_CODES:
raise AppError(ErrorCode.UNSUPPORTED_LANGUAGE, "That language isn't supported.", 400)

layout_format = "timestamp" if request.include_timestamps else "clean"

# The worker thread outlives wait_for; the deadline makes it stop calling upstream too
deadline = time.monotonic() + settings.transcript_timeout_seconds
try:
transcript_data = await asyncio.wait_for(
asyncio.to_thread(get_transcript_from_url, request.url, deadline),
asyncio.to_thread(
get_transcript_from_url, request.url, deadline, request.language
),
timeout=settings.transcript_timeout_seconds,
)
except asyncio.TimeoutError:
Expand Down Expand Up @@ -205,7 +220,12 @@ async def extract_transcript(request: ExtractRequest):
text_sidecar.write_text(formatted_text, encoding="utf-8")
# Segments feed SRT/VTT convert; title names every download of this transcript
await asyncio.to_thread(write_sidecar, filename, "segments", transcript_data["segments"])
write_sidecar(filename, "meta", {"title": transcript_data.get("title")})
out_language = transcript_data.get("language", DEFAULT_LANGUAGE)
# Language picks the CJK PDF font (zh-Hant vs zh-Hans) at convert time
write_sidecar(filename, "meta", {"title": transcript_data.get("title"), "language": out_language})
can_pdf = await asyncio.to_thread(pdf_supported, formatted_text)
source_language = transcript_data.get("source_language") or out_language
translated = bool(transcript_data.get("translated"))

MetadataService.log_success(
video_id=transcript_data["video_id"],
Expand Down Expand Up @@ -235,7 +255,16 @@ async def extract_transcript(request: ExtractRequest):
"provider": transcript_data.get("provider", platform),
"video_id": transcript_data["video_id"],
"title": transcript_data["title"],
"language": transcript_data.get("language", "en"),
"language": out_language,
"language_name": language_name(out_language),
"requested_language": request.language,
"source_language": source_language,
"source_language_name": language_name(source_language),
"translated": translated,
# Neither a native track nor a translation: captions are in another language
"language_fallback": not translated
and not languages_match(out_language, request.language),
"pdf_supported": can_pdf,
"duration_seconds": duration,
"word_count": metrics["word_count"],
"reading_time_seconds": metrics["reading_time_seconds"],
Expand Down Expand Up @@ -330,8 +359,13 @@ async def convert_file(request: ConvertRequest):
try:
try:
file_path = await asyncio.to_thread(
FileGenerator.generate_file, content, request.format
FileGenerator.generate_file,
content,
request.format,
(read_sidecar(request.file_id, "meta") or {}).get("language", ""),
)
except UnsupportedPdfScript as e:
raise AppError(ErrorCode.CONVERT_FAILED, str(e), 400)
except Exception as e:
logger.error("Convert generation failed: %s", e)
raise AppError(
Expand Down Expand Up @@ -359,6 +393,8 @@ async def get_public_config():
return success_response(
{
"file_ttl_hours": settings.file_ttl_hours,
"default_language": DEFAULT_LANGUAGE,
"languages": public_language_list(),
}
)

Expand Down
118 changes: 106 additions & 12 deletions app/bot/telegram_bot.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
from app.services.extract_guardrails import enforce_transcript_guardrails
from app.core.errors import AppError, ErrorCode, TranscriptFetchError
from app.core.config import settings
from app.core.languages import SUPPORTED_LANGUAGES, LANGUAGE_CODES, language_name, languages_match
from app.services.pdf_fonts import UnsupportedPdfScript
from app.utils.logging_config import logger
from app.services.analytics_service import AnalyticsService
from app.services.telemetry_service import TelemetryService, hash_identifier, bucket_duration
Expand Down Expand Up @@ -109,7 +111,7 @@ def _escape_md2(text: str) -> str:
WELCOME_TEXT = (
"*Transcript Flow*\n\n"
"Turn YouTube or Vimeo videos into clean transcripts\\.\n\n"
"\\• YouTube: English captions only at this time\n"
"\\• 23 languages: YouTube auto\\-translates captions \\(/language\\)\n"
"\\• TXT \\(default\\)\n"
"\\• PDF / DOCX\n"
"\\• SRT / VTT subtitles\n"
Expand All @@ -127,9 +129,10 @@ def _escape_md2(text: str) -> str:
"2\\. Preview the transcript\\.\n"
"3\\. Download as TXT, PDF, DOCX, SRT, or VTT\\.\n\n"
"Power users:\n"
"`/extract <link>`\n\n"
"`/extract <link>`\n"
"`/language` to choose the transcript language\n\n"
"If extraction fails:\n"
"\\• YouTube transcripts are English\\-only for now\\.\n"
"\\• Pick another language with /language\\.\n"
"\\• Captions may be missing or unavailable for some videos\\.\n"
"\\• Transcripts may be disabled for the video\\."
)
Expand All @@ -151,7 +154,7 @@ def _escape_md2(text: str) -> str:
"\\• No account required\\.\n"
"\\• Generated files auto\\-delete after about 1 hour\\.\n"
"\\• Short\\-term caption cache \\(up to a few days\\) may reduce repeat fetches\\.\n"
"\\• Your format and timestamp preference is remembered for about 90 days\\.\n"
"\\• Your format, timestamp and language preference is remembered for about 90 days\\.\n"
"\\• We do not keep permanent user accounts or history\\.\n"
"\\• See the website privacy page for full details\\."
)
Expand Down Expand Up @@ -206,10 +209,24 @@ def _ready_keyboard(include_timestamps: bool) -> InlineKeyboardMarkup:
)


def _ready_text(title: str, duration_s: int, word_count: int, reading_time_s: int) -> str:
def _language_label(transcript_data: dict, requested: str) -> str:
"""e.g. "Español (auto-translated)" or "English (Español not available)"."""
actual = transcript_data.get("language") or requested
label = language_name(actual)
if transcript_data.get("translated"):
return f"{label} (auto-translated)"
if not languages_match(actual, requested):
return f"{label} ({language_name(requested)} not available)"
return label


def _ready_text(
title: str, duration_s: int, word_count: int, reading_time_s: int, language: str
) -> str:
return (
f"Transcript ready.\n\n"
f"Title: {title}\n"
f"Language: {language}\n"
f"Duration: {_duration_label(duration_s)}\n"
f"Words: {word_count}\n"
f"Read time: {_reading_time_label(reading_time_s)}"
Expand All @@ -227,18 +244,43 @@ def _start_keyboard() -> InlineKeyboardMarkup:
InlineKeyboardButton("Privacy", callback_data="privacy"),
InlineKeyboardButton("Help", callback_data="help"),
],
[
InlineKeyboardButton("Language", callback_data="language"),
],
])


async def _build_file(transcript_data: dict, formatted_text: str, ext: str) -> str:
"""Generate *ext* off the event loop (PDF builds can take seconds)."""
def _language_keyboard(current: str) -> InlineKeyboardMarkup:
buttons = [
InlineKeyboardButton(
f"{'✓ ' if code == current else ''}{native}", callback_data=f"lang|{code}"
)
for code, native, _ in SUPPORTED_LANGUAGES
]
return InlineKeyboardMarkup([buttons[i:i + 3] for i in range(0, len(buttons), 3)])


async def _build_file(
transcript_data: dict, formatted_text: str, ext: str
) -> tuple[str, str]:
"""
Generate *ext* off the event loop (PDF builds can take seconds).
Returns (path, ext actually produced): PDF falls back to DOCX for scripts
the PDF renderer can't typeset (Arabic, Hindi...).
"""
if ext in SUBTITLE_FORMATS:
content = await asyncio.to_thread(
TranscriptFormatter.format_subtitles, transcript_data["segments"], ext
)
else:
content = formatted_text
return await asyncio.to_thread(FileGenerator.generate_file, content, ext)
language = transcript_data.get("language", "")
try:
path = await asyncio.to_thread(FileGenerator.generate_file, content, ext, language)
return path, ext
except UnsupportedPdfScript:
path = await asyncio.to_thread(FileGenerator.generate_file, content, "docx")
return path, "docx"


def _extract_url_from_text(text: str) -> str | None:
Expand Down Expand Up @@ -286,6 +328,7 @@ async def _process_url(
prefs = _get_prefs(user_id)
file_ext = prefs["last_file_format"]
include_ts = prefs["last_include_timestamps"]
language = prefs["language"]
fmt_type = _format_type_for_prefs(include_ts)
started = time.time()

Expand All @@ -297,6 +340,7 @@ async def _process_url(
f"Extracting transcript\u2026\n"
f"Provider: {platform}\n"
f"Format: {file_ext}\n"
f"Language: {language_name(language)}\n"
f"Timestamps: {_ts_label(include_ts)}"
),
)
Expand All @@ -305,7 +349,7 @@ async def _process_url(
# Extract (with timeout) -----------------------------------------
deadline = time.monotonic() + settings.transcript_timeout_seconds
transcript_data = await asyncio.wait_for(
asyncio.to_thread(get_transcript_from_url, url, deadline),
asyncio.to_thread(get_transcript_from_url, url, deadline, language),
timeout=settings.transcript_timeout_seconds,
)

Expand All @@ -325,14 +369,15 @@ async def _process_url(
await status_msg.edit_text("Transcript Flow\n\nPreparing file\u2026")

video_id = transcript_data.get("video_id", "unknown")
file_path = await _build_file(transcript_data, formatted_text, file_ext)
file_path, file_ext = await _build_file(transcript_data, formatted_text, file_ext)

# State D — Ready ------------------------------------------------
ready = _ready_text(
transcript_data.get("title", "Untitled"),
duration,
metrics["word_count"],
metrics["reading_time_seconds"],
_language_label(transcript_data, language),
)
await status_msg.edit_text(ready, reply_markup=_ready_keyboard(include_ts))

Expand All @@ -351,6 +396,8 @@ async def _process_url(
processing_time_ms = int((time.time() - started) * 1000)
_emit_event("tg_extract_succeeded", user_id, {
"provider": platform,
"language": transcript_data.get("language"),
"translated": bool(transcript_data.get("translated")),
"duration_seconds_bucket": bucket_duration(duration),
"file_format": file_ext,
})
Expand Down Expand Up @@ -463,6 +510,38 @@ async def privacy_cmd(update: Update, context: ContextTypes.DEFAULT_TYPE) -> Non
await update.message.reply_text(PRIVACY_TEXT, parse_mode="MarkdownV2")


async def _send_language_picker(context: ContextTypes.DEFAULT_TYPE, chat_id: int, user_id: str) -> None:
current = _get_prefs(user_id)["language"]
await context.bot.send_message(
chat_id=chat_id,
text=(
f"Transcript language: {language_name(current)}\n\n"
"Pick a language. If a video has no captions in it, YouTube auto-translates. "
"Vimeo can only use the captions the uploader added."
),
reply_markup=_language_keyboard(current),
)


async def language_cmd(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
user_id = str(update.effective_user.id)
_emit_event("tg_cmd_language", user_id)
await _send_language_picker(context, update.effective_chat.id, user_id)


async def _handle_language_choice(query, code: str) -> None:
if code not in LANGUAGE_CODES:
await query.answer()
return
user_id = str(query.from_user.id)
_set_pref(user_id, language=code)
_emit_event("tg_language_set", user_id, {"language": code})
await query.answer(f"Language: {language_name(code)}")
await query.edit_message_text(
f"Transcript language: {language_name(code)}\n\nSend a link \u2014 it applies from your next transcript.",
)


async def status_cmd(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
_emit_event("tg_cmd_status", str(update.effective_user.id))

Expand Down Expand Up @@ -527,6 +606,13 @@ async def button_callback(update: Update, context: ContextTypes.DEFAULT_TYPE) ->
parse_mode="MarkdownV2",
)

elif data == "language":
await query.answer()
await _send_language_picker(context, query.message.chat_id, str(query.from_user.id))

elif data.startswith("lang|"):
await _handle_language_choice(query, data.split("|", 1)[1])

elif data == "privacy":
await query.answer()
await context.bot.send_message(
Expand Down Expand Up @@ -597,7 +683,13 @@ async def _handle_download(query, context: ContextTypes.DEFAULT_TYPE, ext: str)
if ext == context.user_data.get("file_ext") and cached_path and os.path.exists(cached_path):
file_path = cached_path
else:
file_path = await _build_file(transcript_data, formatted_text, ext)
file_path, built_ext = await _build_file(transcript_data, formatted_text, ext)
if built_ext != ext:
await context.bot.send_message(
chat_id=query.message.chat_id,
text="PDF isn't available for this language yet, so here's a DOCX instead.",
)
ext = built_ext

# Send file -------------------------------------------------------
with open(file_path, "rb") as f:
Expand Down Expand Up @@ -710,7 +802,7 @@ async def _handle_timestamp_toggle(query, context: ContextTypes.DEFAULT_TYPE) ->

# Regenerate file in current preferred format -------------------------
file_ext = prefs["last_file_format"]
file_path = await _build_file(transcript_data, formatted_text, file_ext)
file_path, file_ext = await _build_file(transcript_data, formatted_text, file_ext)

# Update cache --------------------------------------------------------
context.user_data["formatted_text"] = formatted_text
Expand All @@ -724,6 +816,7 @@ async def _handle_timestamp_toggle(query, context: ContextTypes.DEFAULT_TYPE) ->
transcript_data.get("duration_seconds", 0),
metrics["word_count"],
metrics["reading_time_seconds"],
_language_label(transcript_data, prefs["language"]),
)
try:
await context.bot.edit_message_text(
Expand Down Expand Up @@ -788,6 +881,7 @@ def run_bot() -> None:
application.add_handler(CommandHandler("formats", formats_cmd))
application.add_handler(CommandHandler("privacy", privacy_cmd))
application.add_handler(CommandHandler("status", status_cmd))
application.add_handler(CommandHandler("language", language_cmd))

# Plain-text message handler (catches raw links)
application.add_handler(MessageHandler(filters.TEXT & (~filters.COMMAND), handle_plain_text))
Expand Down
Loading
Loading