diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c186d8..c80db78 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,6 +57,9 @@ jobs: - name: Launcher smoke test shell: bash run: bash tests/launcher-smoke.sh + - name: Segmented download test + shell: bash + run: bash tests/segment-download.sh - name: Package if: startsWith(github.ref, 'refs/tags/v') shell: bash @@ -442,3 +445,5 @@ jobs: with: files: artifacts/**/* generate_release_notes: true + # Tags with a hyphen (v0.5.0-rc1, ...) are prereleases, so they don't show as "Latest". + prerelease: ${{ contains(github.ref_name, '-') }} diff --git a/README.md b/README.md index f29fb72..de240ad 100644 --- a/README.md +++ b/README.md @@ -120,9 +120,10 @@ Common knobs (`./start-zonos2.sh --help` / `start-zonos2.bat -Help` lists everyt | Skip download prompt / browser | `-y`, `--no-browser` | `-Yes`, `-NoBrowser` | | Extra `zonos2-server` flags | `-- --batch 16` | appended verbatim | -Model dir, download URL, and more can be overridden via `ZONOS2_*` environment variables (see -`--help`). Prefer to launch the server by hand, or want CUDA? See -[Manual & advanced setup](#manual--advanced-setup). +Large model files download over several parallel connections (~2× faster than a single stream on +Hugging Face); tune with `ZONOS2_DL_CONNECTIONS` (default 8). Model dir, download URL, and more can +be overridden via `ZONOS2_*` environment variables (see `--help`). Prefer to launch the server by +hand, or want CUDA? See [Manual & advanced setup](#manual--advanced-setup). ### 3. Generate Speech diff --git a/scripts/start-zonos2.ps1 b/scripts/start-zonos2.ps1 index 6e99d44..5e29977 100644 --- a/scripts/start-zonos2.ps1 +++ b/scripts/start-zonos2.ps1 @@ -62,10 +62,66 @@ $Sizes = @{ } function Get-SizeGB([string]$name) { if ($Sizes.ContainsKey($name)) { $Sizes[$name] } else { '?' } } +# Segmented parallel download: one HEAD resolves the exact size + final CDN URL (HF's resolve/ +# endpoint 302s to a Xet CDN that honors Range), then N concurrent range requests are stitched +# back together — the trick hf_transfer uses to beat single-stream HF (~2x+). Falls back to a +# single stream on any hiccup. Tunable via ZONOS2_DL_CONNECTIONS (default 8, max 16). +function Download-Segmented([string]$name, [string]$url, [string]$dst) { + $conns = 8 + if ($env:ZONOS2_DL_CONNECTIONS) { + $parsed = 0 + if ([int]::TryParse($env:ZONOS2_DL_CONNECTIONS, [ref]$parsed) -and $parsed -gt 0) { $conns = $parsed } + } + if ($conns -lt 1) { $conns = 1 } + if ($conns -gt 16) { $conns = 16 } + if ($conns -le 1) { return $false } + # resolve final URL + exact size from a single redirect-following HEAD (-match is case-insensitive) + $head = & curl.exe -sIL $url + if ($LASTEXITCODE -ne 0) { return $false } + $size = [int64]0; $eff = $url + foreach ($line in $head) { + $l = $line.TrimEnd("`r") + if ($l -match '^content-length:\s*(\d+)') { $v = [int64]$Matches[1]; if ($v -gt $size) { $size = $v } } + elseif ($l -match '^location:\s*(\S+)') { $eff = $Matches[1] } + } + if ($size -lt 33554432) { return $false } # < 32 MB: not worth segmenting + $seg = [int64]([math]::Floor($size / $conns)) + Get-ChildItem "$dst.part*" -ErrorAction SilentlyContinue | Remove-Item -Force -ErrorAction SilentlyContinue + Write-Host "downloading $name ($(Get-SizeGB $name) GB, $conns connections) ..." + $procs = @() + for ($k = 0; $k -lt $conns; $k++) { + $s = [int64]$k * $seg + if ($k -eq ($conns - 1)) { $e = $size - 1 } else { $e = [int64]($k + 1) * $seg - 1 } + $procs += Start-Process -FilePath curl.exe -PassThru -NoNewWindow -ArgumentList ` + @('-sfL', '--retry', '3', '--range', "$s-$e", '-o', "$dst.part$k", $eff) + } + $ok = $true + foreach ($p in $procs) { $p.WaitForExit(); if ($p.ExitCode -ne 0) { $ok = $false } } + if (-not $ok) { + Get-ChildItem "$dst.part*" -ErrorAction SilentlyContinue | Remove-Item -Force -ErrorAction SilentlyContinue + return $false + } + $out = [System.IO.File]::Create("$dst.part") # concatenate segments in order, freeing each + try { + for ($k = 0; $k -lt $conns; $k++) { + $part = "$dst.part$k" + $in = [System.IO.File]::OpenRead($part) + try { $in.CopyTo($out) } finally { $in.Close() } + Remove-Item -Force $part -ErrorAction SilentlyContinue + } + } finally { $out.Close() } + if ((Get-Item "$dst.part").Length -ne $size) { + Remove-Item -Force "$dst.part" -ErrorAction SilentlyContinue; return $false + } + Move-Item -Force "$dst.part" $dst + return $true +} + # curl -C - --fail exits 33 when the .part is already complete (HTTP 416): restart clean once. function Download-One([string]$name) { $url = "$BaseUrl/$name" $dst = Join-Path $ModelDir $name + if (Download-Segmented $name $url $dst) { return $true } # fast path; falls through on failure Write-Host "downloading $name ($(Get-SizeGB $name) GB) ..." & curl.exe -L --fail --retry 3 -C - --progress-bar -o "$dst.part" $url if ($LASTEXITCODE -eq 33) { diff --git a/scripts/start-zonos2.sh b/scripts/start-zonos2.sh index de6f7fa..eff0b20 100755 --- a/scripts/start-zonos2.sh +++ b/scripts/start-zonos2.sh @@ -33,6 +33,7 @@ usage: $0 [options] [-- ] --no-browser don't open the web UI env overrides: ZONOS2_QUANT ZONOS2_MODEL_DIR ZONOS2_BASE_URL ZONOS2_HOST ZONOS2_PORT ZONOS2_SERVER_BIN ZONOS2_ASSUME_YES ZONOS2_NO_BROWSER ZONOS2_FFMPEG_URL + ZONOS2_DL_CONNECTIONS (parallel download connections, default 8, max 16) ffmpeg (voice cloning only) is fetched into $ZONOS2_MODEL_DIR/bin if not already there or on PATH; basic TTS needs no ffmpeg. EOF @@ -55,8 +56,6 @@ while [ $# -gt 0 ]; do esac done -[ -x "$ZONOS2_SERVER_BIN" ] || die "zonos2-server not found at $ZONOS2_SERVER_BIN (run this script from the extracted release archive, or set ZONOS2_SERVER_BIN)" - model_size_gb() { case "$1" in zonos2-f16.gguf) echo 15.3 ;; @@ -70,11 +69,55 @@ model_size_gb() { esac } +# Segmented parallel download: one HEAD resolves the exact size + final CDN URL (HF's resolve/ +# endpoint 302s to a Xet CDN that honors Range), then N concurrent range requests are stitched +# back together — the trick hf_transfer uses to beat single-stream HF (~2x+ here). Falls back to +# a single stream (below) on any hiccup. Tunable via ZONOS2_DL_CONNECTIONS (default 8, max 16). +download_segmented() { + seg_name="$1"; seg_url="$2"; seg_dst="$3" + conns="${ZONOS2_DL_CONNECTIONS:-8}" + case "$conns" in ''|*[!0-9]*) conns=8 ;; esac + [ "$conns" -lt 1 ] && conns=1 + [ "$conns" -gt 16 ] && conns=16 + [ "$conns" -le 1 ] && return 1 + # resolve final URL + exact size from a single redirect-following HEAD + seg_head=$(curl -sIL "$seg_url" | tr -d '\r') || return 1 + seg_size=$(printf '%s\n' "$seg_head" | awk 'tolower($1)=="content-length:"{v=$2} END{print v}') + seg_eff=$(printf '%s\n' "$seg_head" | awk 'tolower($1)=="location:"{v=$2} END{print v}') + [ -n "$seg_eff" ] || seg_eff="$seg_url" + case "$seg_size" in ''|*[!0-9]*) return 1 ;; esac + [ "$seg_size" -lt 33554432 ] && return 1 # < 32 MB: not worth segmenting + seg_len=$((seg_size / conns)) + rm -f "$seg_dst".part[0-9]* + echo "downloading $seg_name ($(model_size_gb "$seg_name") GB, $conns connections) ..." + k=0; pids="" + while [ "$k" -lt "$conns" ]; do + s=$((k * seg_len)) + if [ "$k" -eq $((conns - 1)) ]; then e=$((seg_size - 1)); else e=$(((k + 1) * seg_len - 1)); fi + curl -sfL --retry 3 --range "$s-$e" -o "$seg_dst.part$k" "$seg_eff" & + pids="$pids $!" + k=$((k + 1)) + done + seg_ok=1 + for p in $pids; do wait "$p" || seg_ok=0; done + if [ "$seg_ok" != 1 ]; then rm -f "$seg_dst".part[0-9]*; return 1; fi + : > "$seg_dst.part" + k=0 + while [ "$k" -lt "$conns" ]; do + cat "$seg_dst.part$k" >> "$seg_dst.part" || { rm -f "$seg_dst".part*; return 1; } + rm -f "$seg_dst.part$k" + k=$((k + 1)) + done + if [ "$(wc -c < "$seg_dst.part")" != "$seg_size" ]; then rm -f "$seg_dst.part"; return 1; fi + mv "$seg_dst.part" "$seg_dst" +} + # curl -C - --fail exits 33 when the .part is already complete (HTTP 416): restart clean once. download_one() { name="$1" url="$ZONOS2_BASE_URL/$name" dst="$ZONOS2_MODEL_DIR/$name" + download_segmented "$name" "$url" "$dst" && return 0 # fast path; falls through on any failure echo "downloading $name ($(model_size_gb "$name") GB) ..." rc=0 curl -L --fail --retry 3 -C - --progress-bar -o "$dst.part" "$url" || rc=$? @@ -122,6 +165,12 @@ ffmpeg_download() { [ -x "$FFMPEG_BIN" ] } +# Sourced by tests/segment-download.sh to exercise download_segmented in isolation; a normal +# run leaves ZONOS2_LIB_ONLY unset and proceeds into the main flow below. +[ "${ZONOS2_LIB_ONLY:-}" = 1 ] && return 0 + +[ -x "$ZONOS2_SERVER_BIN" ] || die "zonos2-server not found at $ZONOS2_SERVER_BIN (run this script from the extracted release archive, or set ZONOS2_SERVER_BIN)" + BACKBONE="zonos2-$ZONOS2_QUANT.gguf" MISSING=() for f in "$BACKBONE" dac.gguf spk-encoder.gguf; do diff --git a/src/app.cpp b/src/app.cpp index b3a28d1..389b386 100644 --- a/src/app.cpp +++ b/src/app.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -297,34 +298,110 @@ static int sys_exit(int r) { #endif } -// Content-Length of the (possibly redirected) URL, for an accurate progress bar; 0 on failure. -static long long remote_size(const std::string & url) { +// Number of parallel connections for a segmented download (ZONOS2_DL_CONNECTIONS, default 8). +static int dl_connections() { + int n = 8; + if (const char * e = getenv("ZONOS2_DL_CONNECTIONS"); e && *e) { int v = std::atoi(e); if (v > 0) n = v; } + return n < 1 ? 1 : (n > 16 ? 16 : n); +} + +// One HEAD (following redirects) yields both the exact Content-Length and the final CDN URL: +// HF's resolve/ endpoint 302-redirects to a Xet CDN that honors Range, so we resolve once and +// reuse that URL across all segments. Sets eff to the last Location seen (or url if none) and +// size to the largest Content-Length (the CDN 200, not the redirect stub). false on failure. +static bool probe_remote(const std::string & url, std::string & eff, long long & size) { std::error_code ec; fs::create_directories(models_dir(), ec); const fs::path tmp = models_dir() / ".head"; const std::string cmd = "curl -sIL " + shq(url) + " -o " + shq(tmp.string()); - long long best = 0; - if (sys_exit(std::system(cmd.c_str())) == 0) { - std::ifstream f(tmp); - static const std::string key = "content-length:"; - for (std::string line; std::getline(f, line); ) { - if (!line.empty() && line.back() == '\r') line.pop_back(); - if (line.size() <= key.size()) continue; - std::string head = line.substr(0, key.size()); - for (char & c : head) c = static_cast(std::tolower((unsigned char) c)); - if (head == key) { // take the largest across redirect hops (the CDN 200) - long long v = std::atoll(line.c_str() + key.size()); - if (v > best) best = v; + eff = url; size = 0; + if (sys_exit(std::system(cmd.c_str())) != 0) { fs::remove(tmp, ec); return false; } + std::ifstream f(tmp); + auto header = [](std::string line, const char * key, std::string & val) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + const size_t klen = std::strlen(key); + if (line.size() <= klen) return false; + std::string head = line.substr(0, klen); + for (char & c : head) c = static_cast(std::tolower((unsigned char) c)); + if (head != key) return false; + size_t b = klen; while (b < line.size() && (line[b] == ' ' || line[b] == '\t')) ++b; + val = line.substr(b); + return true; + }; + for (std::string line, v; std::getline(f, line); ) { + if (header(line, "content-length:", v)) { long long n = std::atoll(v.c_str()); if (n > size) size = n; } + else if (header(line, "location:", v) && !v.empty()) eff = v; // last Location = final CDN URL + } + fs::remove(tmp, ec); + return size > 0; +} + +// Segmented download: split [0,total) into `conns` byte ranges fetched concurrently from the +// resolved CDN URL, concatenate, and verify the size. Returns false (caller falls back to a +// single stream) on any segment error, a short concat, or a size mismatch. No new dependency — +// just parallel curl range requests, the same trick hf_transfer uses to beat single-stream HF. +static bool download_segmented(const std::string & name, const std::string & eff, + long long total, int conns) { + const fs::path dst = models_dir() / name; + auto part_k = [&](int k) { return fs::path(dst.string() + ".part" + std::to_string(k)); }; + std::error_code ec; + for (int k = 0; k < conns; ++k) fs::remove(part_k(k), ec); // clear any stale segments + + const long long seg = total / conns; + std::vector rc(conns, -1); + std::vector ths; + ths.reserve(conns); + for (int k = 0; k < conns; ++k) { + const long long s = static_cast(k) * seg; + const long long e = (k == conns - 1) ? total - 1 : static_cast(k + 1) * seg - 1; + ths.emplace_back([&rc, k, s, e, eff, p = part_k(k)] { + const std::string cmd = "curl -sfL --retry 3 --range " + std::to_string(s) + "-" + + std::to_string(e) + " -o " + shq(p.string()) + " " + shq(eff); + rc[k] = sys_exit(std::system(cmd.c_str())); + }); + } + for (auto & t : ths) t.join(); + + bool ok = true; + for (int r : rc) if (r != 0) ok = false; + const fs::path whole = fs::path(dst.string() + ".part"); + if (ok) { + fs::remove(whole, ec); + std::ofstream out(whole, std::ios::binary); + for (int k = 0; k < conns && ok; ++k) { // append, freeing each segment as we go + std::ifstream in(part_k(k), std::ios::binary); + if (!in) { ok = false; break; } + out << in.rdbuf(); + in.close(); + fs::remove(part_k(k), ec); + } + out.close(); + if (ok) { + const auto sz = fs::file_size(whole, ec); + if (!ec && static_cast(sz) == total) { + fs::rename(whole, dst, ec); + if (!ec) { + fprintf(stderr, "zonos2-app: %s fetched with %d parallel connections\n", name.c_str(), conns); + return true; + } } } } - fs::remove(tmp, ec); - return best; + for (int k = 0; k < conns; ++k) fs::remove(part_k(k), ec); // failure: leave no partial segments + fs::remove(whole, ec); + return false; } -// Fetch one file to / via curl, resuming a prior .part; rc 33 (HTTP 416 on an -// already-complete .part) means restart clean once. Returns true on success. -static bool download_one(const std::string & name) { +// Fetch one file to /: parallel segmented download when the size is known and +// worth it, else a single resumable stream (rc 33 = HTTP 416 on an already-complete .part → +// restart clean once). `eff` is the resolved URL and `exact` the Content-Length from probe_remote +// (exact == 0 means unknown → single stream only). Returns true on success. +static bool download_one(const std::string & name, const std::string & eff, long long exact) { + const int conns = dl_connections(); + if (exact >= 32LL * 1024 * 1024 && conns > 1 && !eff.empty() + && download_segmented(name, eff, exact, conns)) + return true; + const fs::path dst = models_dir() / name; const fs::path part = fs::path(dst.string() + ".part"); const std::string url = base_url() + "/" + name; @@ -383,11 +460,12 @@ static void run_download(std::string quant, bool want_spk, bool gpu) { if (name == "spk-encoder.gguf") spk_path = final_path.string(); continue; } - long long total = remote_size(base_url() + "/" + name); - if (total <= 0) total = model_size_hint(name); + std::string eff; long long exact = 0; + const bool probed = probe_remote(base_url() + "/" + name, eff, exact); + const long long total = probed ? exact : model_size_hint(name); // exact drives segmentation { std::lock_guard lk(g_dl.mu); g_dl.total = total; } - if (!download_one(name)) { + if (!download_one(name, eff, probed ? exact : 0)) { if (fatal) { ok = false; err = "failed to download " + name + " from " + base_url(); break; } fprintf(stderr, "zonos2-app: warning: could not download %s — voice cloning disabled\n", name.c_str()); continue; // spk-encoder is optional @@ -592,11 +670,18 @@ coco::stray start(saucer::application * app) { webview->expose("download_status", []() -> std::string { std::lock_guard lk(g_dl.mu); long long done = 0; - if (!g_dl.file.empty()) { // live byte count: the .part while downloading, else the final file + if (!g_dl.file.empty()) { // live byte count: sum every .part* (single .part or N segments) std::error_code ec; - auto sz = fs::file_size(models_dir() / (g_dl.file + ".part"), ec); - if (ec) { ec.clear(); sz = fs::file_size(models_dir() / g_dl.file, ec); } - if (!ec) done = static_cast(sz); + const std::string prefix = g_dl.file + ".part"; + for (fs::directory_iterator it(models_dir(), ec), end; !ec && it != end; it.increment(ec)) { + std::error_code fe; + if (it->path().filename().string().rfind(prefix, 0) == 0) + if (auto sz = it->file_size(fe); !fe) done += static_cast(sz); + } + if (done == 0) { // fully downloaded: parts gone, final file in place + auto sz = fs::file_size(models_dir() / g_dl.file, ec); + if (!ec) done = static_cast(sz); + } } json j = {{"active", g_dl.active}, {"finished", g_dl.finished}, {"ok", g_dl.ok}, {"file", g_dl.file}, {"idx", g_dl.idx}, {"count", g_dl.count}, diff --git a/tests/app-download-smoke.sh b/tests/app-download-smoke.sh index 23bfc17..fb5c551 100755 --- a/tests/app-download-smoke.sh +++ b/tests/app-download-smoke.sh @@ -36,12 +36,14 @@ fail() { echo "FAIL: $1"; [ -f "$WORK/app.log" ] && { echo "--- app log ---"; ca PORT="${ZONOS2_TEST_APP_HTTP_PORT:-18989}" echo "== app: model download self-test ==" -# Distinctive random payloads with different sizes so the HEAD/Content-Length path is exercised. +# Distinctive random payloads. The backbone is >32 MB so the segmented (parallel-range) path +# is exercised; dac/spk stay under the threshold and take the single-stream path. tests/range_server.py +# answers Range requests with 206 (the stdlib http.server ignores Range), emulating HF's CDN. mkdir -p "$WORK/serve" -head -c 3000000 /dev/urandom > "$WORK/serve/zonos2-q6_k.gguf" -head -c 500000 /dev/urandom > "$WORK/serve/dac.gguf" -head -c 40000 /dev/urandom > "$WORK/serve/spk-encoder.gguf" -python3 -m http.server "$PORT" --bind 127.0.0.1 --directory "$WORK/serve" >/dev/null 2>&1 & +head -c 40000000 /dev/urandom > "$WORK/serve/zonos2-q6_k.gguf" +head -c 500000 /dev/urandom > "$WORK/serve/dac.gguf" +head -c 40000 /dev/urandom > "$WORK/serve/spk-encoder.gguf" +python3 "$ROOT/tests/range_server.py" "$PORT" "$WORK/serve" >/dev/null 2>&1 & HTTP_PID=$! up=0 for _ in $(seq 1 50); do @@ -59,6 +61,9 @@ ZONOS2_BASE_URL="http://127.0.0.1:$PORT" ZONOS2_APP_SELFTEST_DOWNLOAD=q6_k \ "$APP" >"$WORK/app.log" 2>&1 || fail "self-test exited nonzero" grep -q "selftest-download: ok=1" "$WORK/app.log" || fail "self-test did not report ok=1" pass "self-test downloaded and reported success" +grep -q "zonos2-q6_k.gguf fetched with .* parallel connections" "$WORK/app.log" \ + || fail "backbone did not use the segmented (parallel) download path" +pass "backbone fetched via segmented parallel download" for f in zonos2-q6_k.gguf dac.gguf spk-encoder.gguf; do cmp -s "$WORK/serve/$f" "$MODELS/$f" || fail "$f missing or not byte-identical" diff --git a/tests/range_server.py b/tests/range_server.py new file mode 100644 index 0000000..45176ce --- /dev/null +++ b/tests/range_server.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# Minimal static file server WITH HTTP Range support, for the download tests. The stdlib +# http.server ignores `Range:` and always returns the whole file (200), so it can't exercise +# the segmented (parallel byte-range) downloader — a real CDN like Hugging Face answers each +# range with 206 + Content-Range, which is what this emulates. HEAD still reports Content-Length +# (used to resolve size), and non-range GETs fall back to the stdlib handler. +# +# Usage: range_server.py +import os +import re +import sys +from functools import partial +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer + + +class RangeHandler(SimpleHTTPRequestHandler): + def do_GET(self): + rng = self.headers.get("Range") + m = re.match(r"bytes=(\d+)-(\d*)$", rng) if rng else None + path = self.translate_path(self.path) + if not m or not os.path.isfile(path): + return super().do_GET() # no/!parseable range -> full file + size = os.path.getsize(path) + start = int(m.group(1)) + end = int(m.group(2)) if m.group(2) else size - 1 + end = min(end, size - 1) + if start > end or start >= size: + self.send_error(416) + return + length = end - start + 1 + self.send_response(206) + self.send_header("Content-Type", self.guess_type(path)) + self.send_header("Content-Range", f"bytes {start}-{end}/{size}") + self.send_header("Content-Length", str(length)) + self.send_header("Accept-Ranges", "bytes") + self.end_headers() + with open(path, "rb") as f: + f.seek(start) + remaining = length + while remaining > 0: + chunk = f.read(min(65536, remaining)) + if not chunk: + break + self.wfile.write(chunk) + remaining -= len(chunk) + + +def main(): + port = int(sys.argv[1]) + directory = sys.argv[2] + handler = partial(RangeHandler, directory=directory) + ThreadingHTTPServer(("127.0.0.1", port), handler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/tests/segment-download.sh b/tests/segment-download.sh new file mode 100755 index 0000000..4da8a11 --- /dev/null +++ b/tests/segment-download.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Exercise the launcher's segmented (parallel-range) download in isolation: source +# start-zonos2.sh with ZONOS2_LIB_ONLY=1 (loads the functions, skips the main flow), serve a +# >32 MB file from a local threaded http.server, and assert download_segmented reassembles it +# byte-identically. Also checks the small-file threshold (returns 1 → caller falls back). +# +# Usage: bash tests/segment-download.sh +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" +command -v curl >/dev/null 2>&1 || { echo "SKIP: curl not available"; exit 0; } +command -v python3 >/dev/null 2>&1 || { echo "SKIP: python3 not available"; exit 0; } +command -v cmp >/dev/null 2>&1 || { echo "SKIP: cmp not available"; exit 0; } + +# Load the launcher's functions without running it. +ZONOS2_LIB_ONLY=1 . scripts/start-zonos2.sh +# model_size_gb (called for the progress line) only knows the real names; stub it for fixtures. +model_size_gb() { echo 0.04; } + +WORK="$(mktemp -d)" +HTTP_PID="" +cleanup() { + local rc=$? + [ -n "$HTTP_PID" ] && { kill "$HTTP_PID" 2>/dev/null || true; wait "$HTTP_PID" 2>/dev/null || true; } + rm -rf "$WORK" + exit "$rc" +} +trap cleanup EXIT +pass() { echo " ok: $1"; } +fail() { echo "FAIL: $1"; exit 1; } + +PORT="${ZONOS2_TEST_SEG_PORT:-18991}" +mkdir -p "$WORK/serve" +head -c 40000000 /dev/urandom > "$WORK/serve/big.bin" # > 32 MB -> segmented +head -c 100000 /dev/urandom > "$WORK/serve/small.bin" # < 32 MB -> threshold returns 1 +# range_server.py answers Range with 206 (stdlib http.server ignores Range), emulating HF's CDN. +python3 "$ROOT/tests/range_server.py" "$PORT" "$WORK/serve" >/dev/null 2>&1 & +HTTP_PID=$! +for _ in $(seq 1 50); do curl -fsI "http://127.0.0.1:$PORT/big.bin" >/dev/null 2>&1 && break; sleep 0.2; done + +echo "== segmented reassembly (8 connections) ==" +ZONOS2_DL_CONNECTIONS=8 \ + download_segmented "big.bin" "http://127.0.0.1:$PORT/big.bin" "$WORK/big.out" >"$WORK/seg.log" 2>&1 \ + || fail "download_segmented returned failure on a >32MB file" +cmp -s "$WORK/serve/big.bin" "$WORK/big.out" || fail "reassembled file is not byte-identical" +pass "40 MB file reassembled byte-identical" +grep -q "8 connections" "$WORK/seg.log" || fail "did not announce the parallel connections" +pass "announced parallel connections" +ls "$WORK"/big.out.part* >/dev/null 2>&1 && fail "leftover .partN segments" || pass "no leftover segments" + +echo "== small file skips segmentation ==" +if ZONOS2_DL_CONNECTIONS=8 \ + download_segmented "small.bin" "http://127.0.0.1:$PORT/small.bin" "$WORK/small.out" >/dev/null 2>&1; then + fail "small file should have returned non-zero (below the 32MB threshold)" +fi +[ -e "$WORK/small.out" ] && fail "small file should not have been written by download_segmented" +pass "sub-threshold file declined (caller falls back to single stream)" + +echo "== single connection disables segmentation ==" +if ZONOS2_DL_CONNECTIONS=1 \ + download_segmented "big.bin" "http://127.0.0.1:$PORT/big.bin" "$WORK/one.out" >/dev/null 2>&1; then + fail "ZONOS2_DL_CONNECTIONS=1 should decline segmentation" +fi +pass "single-connection setting declines segmentation" + +echo "All segmented download tests passed."