From aa38d3b177cb7a9583b8696334d2328223e71de1 Mon Sep 17 00:00:00 2001 From: Matt Parrett Date: Fri, 30 Oct 2026 11:40:37 -0700 Subject: [PATCH 1/6] perf: hoist DAC weight prep to load time; drop needless conts in decode graph dac: conv1d re-cast every conv weight to F16 and convt1d re-permuted every transposed-conv weight inside every decode graph. im2col only reads its src0 shape (the CPU kernel asserts F16 but never touches the data), so a tiny F16 [K, IC] shape carrier replaces the cast; the [IC, K*OC] GEMM operand is built once host-side at load. Graph: 447 -> 409 nodes. backbone: set_rows only needs contiguous rows and walks its source by strides on every backend, so the permuted K/V views feed it directly (56 fewer copy kernels per decode step). build_moe's trailing cont is skipped when the top-1 view is already contiguous. Prefill graph: 1851 -> 1771. Greedy --tts with a speaker embedding gives byte-identical codes and wav before and after (q4_k, Metal, M2). Co-Authored-By: Claude Fable 5.1 --- src/dac.cpp | 81 +++++++++++++++++++++++++++++++++++++------- src/dac.h | 5 +++ src/zonos2-graph.cpp | 13 ++++--- 3 files changed, 83 insertions(+), 16 deletions(-) diff --git a/src/dac.cpp b/src/dac.cpp index 0cf308e..5038292 100644 --- a/src/dac.cpp +++ b/src/dac.cpp @@ -23,6 +23,8 @@ ggml_tensor * dac_model::get(const std::string & n) const { return it->second; } +static bool dac_prepare(dac_model & m); + bool dac_load(dac_model & m, const char * path, bool use_gpu) { ggml_context * ctx_meta = nullptr; gguf_init_params gp = { /*no_alloc=*/ true, /*ctx=*/ &ctx_meta }; @@ -80,13 +82,66 @@ bool dac_load(dac_model & m, const char * path, bool use_gpu) { fprintf(stderr, "dac: loaded %zu tensors on %s (latent=%d dec_dim=%d hop=%d sr=%d)\n", m.t.size(), m.is_gpu ? "GPU" : "CPU", m.latent_dim, m.decoder_dim, m.hop_length, m.sample_rate); gguf_free(gguf); + return dac_prepare(m); +} + +// Derive the per-call-invariant weight forms once, instead of rebuilding them inside every +// decode graph (which, in streaming, is every ~40-frame block): +// - conv1d's im2col needs an F16 src0 on the CPU backend but reads only its [K, IC] shape, so a +// tiny F16 shape carrier replaces the ggml_cast that re-converted the whole weight per graph. +// - convt1d's GEMM operand is w[K,OC,IC] permuted to [IC, K*OC]; that permute+cont was also +// re-run per graph. Build it host-side here and keep it on the backend. +static bool dac_prepare(dac_model & m) { + std::vector> convs, convts; + for (const auto & kv : m.t) { + const std::string & n = kv.first; + if (n.size() < 7 || n.compare(n.size() - 7, 7, ".weight") != 0) continue; + if (n.find(".convt.") != std::string::npos) convts.push_back(kv); + else if (n.find("conv") != std::string::npos) convs.push_back(kv); + } + ggml_init_params ip = { ggml_tensor_overhead() * (convs.size() + convts.size() + 1), nullptr, /*no_alloc=*/ true }; + m.ctx_p = ggml_init(ip); + if (!m.ctx_p) { fprintf(stderr, "dac: prepare ctx alloc failed\n"); return false; } + for (auto & kv : convs) { + ggml_tensor * w = kv.second; // [K, IC, OC] + ggml_tensor * k16 = ggml_new_tensor_2d(m.ctx_p, GGML_TYPE_F16, w->ne[0], w->ne[1]); + ggml_set_name(k16, (kv.first + ".k16").c_str()); + m.t[kv.first + ".k16"] = k16; + } + for (auto & kv : convts) { + ggml_tensor * w = kv.second; // [K, OC, IC] + ggml_tensor * wp = ggml_new_tensor_2d(m.ctx_p, GGML_TYPE_F32, w->ne[2], w->ne[0] * w->ne[1]); + ggml_set_name(wp, (kv.first + ".wp").c_str()); + m.t[kv.first + ".wp"] = wp; + } + m.buf_p = ggml_backend_alloc_ctx_tensors(m.ctx_p, m.backend); + if (!m.buf_p) { fprintf(stderr, "dac: prepare buffer alloc failed\n"); return false; } + ggml_backend_buffer_clear(m.buf_p, 0); // k16 carriers are never read; keep them defined anyway + std::vector src, dst; + for (auto & kv : convts) { + ggml_tensor * w = kv.second; + const int64_t K = w->ne[0], OC = w->ne[1], IC = w->ne[2]; + GGML_ASSERT(w->type == GGML_TYPE_F32 && ggml_is_contiguous(w)); + src.resize((size_t) K * OC * IC); + dst.resize(src.size()); + ggml_backend_tensor_get(w, src.data(), 0, src.size() * sizeof(float)); + // wp[ic + IC*(k + K*oc)] = w[k + K*(oc + OC*ic)] + for (int64_t ic = 0; ic < IC; ++ic) + for (int64_t oc = 0; oc < OC; ++oc) + for (int64_t k = 0; k < K; ++k) + dst[(size_t) ic + IC * (k + K * oc)] = src[(size_t) k + K * (oc + OC * ic)]; + ggml_backend_tensor_set(m.t[kv.first + ".wp"], dst.data(), 0, dst.size() * sizeof(float)); + } return true; } void dac_free(dac_model & m) { + if (m.buf_p) ggml_backend_buffer_free(m.buf_p); + if (m.ctx_p) ggml_free(m.ctx_p); if (m.buf_w) ggml_backend_buffer_free(m.buf_w); if (m.ctx_w) ggml_free(m.ctx_w); if (m.backend) ggml_backend_free(m.backend); + m.buf_p = nullptr; m.ctx_p = nullptr; m.buf_w = nullptr; m.ctx_w = nullptr; m.backend = nullptr; m.t.clear(); } @@ -97,14 +152,16 @@ void dac_free(dac_model & m) { // final upsampled resolution) and the convs are bandwidth-bound on it, so it is materialized as // F16: that halves the dominant memory traffic and routes mul_mat through the F16 path (src0=F16 // patches, src1=F32 weights). Output is F32; audio is bit-close to the all-F32 path. -static ggml_tensor * conv1d(ggml_context * ctx, ggml_tensor * w, ggml_tensor * bias, +static ggml_tensor * conv1d(ggml_context * ctx, const dac_model & m, ggml_tensor * w, ggml_tensor * bias, ggml_tensor * x, int s, int p, int d) { // im2col (F16 patches, F32 weights). A direct shifted-matmul conv1d was tried to avoid the // patch-buffer materialization, but it was slower on CPU (7 small matmuls + a channel-major // transpose lost to im2col's single large matmul — the F16 patch buffer is already cheap) and // unsupported on Metal (needs the PAD op). im2col stays. - ggml_tensor * w16 = ggml_cast(ctx, w, GGML_TYPE_F16); - ggml_tensor * ic = ggml_im2col(ctx, w16, x, s, 0, p, 0, d, 0, false, GGML_TYPE_F16); // [K*IC, OL, 1] + // src0 of im2col is only a [K, IC] shape carrier (the CPU kernel asserts F16 but never reads + // the data); use the load-time one from dac_prepare rather than casting the weight per graph. + ggml_tensor * k16 = m.get(std::string(ggml_get_name(w)) + ".k16"); + ggml_tensor * ic = ggml_im2col(ctx, k16, x, s, 0, p, 0, d, 0, false, GGML_TYPE_F16); // [K*IC, OL, 1] ggml_tensor * r = ggml_mul_mat(ctx, ggml_reshape_2d(ctx, ic, ic->ne[0], ic->ne[1] * ic->ne[2]), // src0 = im2col [K*IC, OL] ggml_reshape_2d(ctx, w, w->ne[0] * w->ne[1], w->ne[2])); // src1 = w [K*IC, OC] @@ -114,7 +171,7 @@ static ggml_tensor * conv1d(ggml_context * ctx, ggml_tensor * w, ggml_tensor * b // ConvTranspose1d upsample by `stride` (kernel K=2*stride), then crop stride/2 each side to // match PyTorch padding=ceil(stride/2) (exact for even strides). w ne=[K,OC,IC], bias [1,OC]. -static ggml_tensor * convt1d(ggml_context * ctx, ggml_tensor * w, ggml_tensor * bias, +static ggml_tensor * convt1d(ggml_context * ctx, const dac_model & m, ggml_tensor * w, ggml_tensor * bias, ggml_tensor * x, int stride) { // ConvTranspose1d (kernel K = 2*stride) reimplemented as GEMM + overlap-add fold instead of // ggml_conv_transpose_1d. The stock Metal conv_transpose kernel is one thread per output sample @@ -132,9 +189,9 @@ static ggml_tensor * convt1d(ggml_context * ctx, ggml_tensor * w, ggml_tensor * return v; } GGML_ASSERT(K == 2 * s); - // w[K,OC,IC] -> [IC, K*OC] so mul_mat sums over IC; pair with xT[IC,L]. - ggml_tensor * wp = ggml_cont(ctx, ggml_permute(ctx, w, 1, 2, 0, 3)); // [IC, K, OC] - wp = ggml_reshape_2d(ctx, wp, IC, (int64_t) K * OC); // [IC, K*OC] (ko = k + K*oc) + // w[K,OC,IC] -> [IC, K*OC] so mul_mat sums over IC; pair with xT[IC,L]. Pre-permuted once at + // load (dac_prepare) instead of a permute+cont of the full weight inside every graph. + ggml_tensor * wp = m.get(std::string(ggml_get_name(w)) + ".wp"); // [IC, K*OC] (ko = k + K*oc) ggml_tensor * xT = ggml_cont(ctx, ggml_transpose(ctx, x)); // [IC, L] ggml_tensor * yk = ggml_mul_mat(ctx, wp, xT); // [K*OC, L] yk[k+K*oc, i] // reshape to [K, OC, L] then split lower (k=s) halves -> each [s, OC, L] @@ -179,16 +236,16 @@ static ggml_tensor * snake_named(ggml_context * ctx, const dac_model & m, ggml_t static ggml_tensor * res_unit(ggml_context * ctx, const dac_model & m, ggml_tensor * x, const std::string & p, int dil) { ggml_tensor * y = snake_named(ctx, m, x, p + ".snake1"); - y = conv1d(ctx, m.get(p + ".conv1.weight"), m.get(p + ".conv1.bias"), y, 1, 3 * dil, dil); + y = conv1d(ctx, m, m.get(p + ".conv1.weight"), m.get(p + ".conv1.bias"), y, 1, 3 * dil, dil); y = snake_named(ctx, m, y, p + ".snake2"); - y = conv1d(ctx, m.get(p + ".conv2.weight"), m.get(p + ".conv2.bias"), y, 1, 0, 1); + y = conv1d(ctx, m, m.get(p + ".conv2.weight"), m.get(p + ".conv2.bias"), y, 1, 0, 1); return ggml_add(ctx, x, y); } static ggml_tensor * dec_block(ggml_context * ctx, const dac_model & m, ggml_tensor * x, int b, int stride) { const std::string p = "dec.b" + std::to_string(b); ggml_tensor * y = snake_named(ctx, m, x, p + ".snake"); - y = convt1d(ctx, m.get(p + ".convt.weight"), m.get(p + ".convt.bias"), y, stride); + y = convt1d(ctx, m, m.get(p + ".convt.weight"), m.get(p + ".convt.bias"), y, stride); y = res_unit(ctx, m, y, p + ".res0", 1); y = res_unit(ctx, m, y, p + ".res1", 3); y = res_unit(ctx, m, y, p + ".res2", 9); @@ -239,11 +296,11 @@ static bool dac_run_decoder(const dac_model & m, const std::vectorne[0]); // [samples] ggml_set_output(out); diff --git a/src/dac.h b/src/dac.h index 00faeba..4fdd79d 100644 --- a/src/dac.h +++ b/src/dac.h @@ -17,6 +17,11 @@ struct dac_model { ggml_backend_buffer_t buf_w = nullptr; bool is_gpu = false; std::map t; + // load-time derived weights (see dac_prepare): per conv weight an F16 [K, IC] shape carrier + // for im2col (".k16"), per convt weight the pre-permuted [IC, K*OC] GEMM operand + // (".wp"). Stored in `t` under those names; this ctx/buffer own them. + ggml_context * ctx_p = nullptr; + ggml_backend_buffer_t buf_p = nullptr; // hparams int n_codebooks = 9, codebook_size = 1024, latent_dim = 1024, decoder_dim = 1536; int sample_rate = 44100, hop_length = 512, audio_pad_id = 1025; diff --git a/src/zonos2-graph.cpp b/src/zonos2-graph.cpp index 18667dc..79857db 100644 --- a/src/zonos2-graph.cpp +++ b/src/zonos2-graph.cpp @@ -130,9 +130,12 @@ static ggml_tensor * build_attention(gctx & g, ggml_tensor * cur, const zonos2_l // columns writes a unique absolute row (slot*slot_cap + logical pos). ggml_tensor * kc = (*g.kc)[il]; // [hd, n_slots*slot_cap, nkv] F16 ggml_tensor * vc = (*g.vc)[il]; - // set_rows takes an F32 source and converts into the F16 cache in place. - ggml_tensor * kb = ggml_cont(ctx, ggml_permute(ctx, k, 0, 2, 1, 3)); // [hd, n, nkv] F32 - ggml_tensor * vb = ggml_cont(ctx, ggml_permute(ctx, v, 0, 2, 1, 3)); + // set_rows takes an F32 source and converts into the F16 cache in place. It only needs + // contiguous rows (ne0 dense), which a permute of dims 1/2 preserves, and every backend's + // set_rows walks the source by nb[] strides — so the permuted views go in directly. A + // ggml_cont here was 2 copy kernels per layer (56 per decode step) for nothing. + ggml_tensor * kb = ggml_permute(ctx, k, 0, 2, 1, 3); // [hd, n, nkv] F32 (strided view) + ggml_tensor * vb = ggml_permute(ctx, v, 0, 2, 1, 3); ggml_build_forward_expand(g.gf, ggml_set_rows(ctx, kc, kb, g.pos_cache)); ggml_build_forward_expand(g.gf, ggml_set_rows(ctx, vc, vb, g.pos_cache)); @@ -235,7 +238,9 @@ static ggml_tensor * build_moe(gctx & g, ggml_tensor * cur, const zonos2_layer & for (int j = 1; j < k; ++j) { moe = ggml_add(ctx, moe, ggml_view_2d(ctx, exps, n_embd, n, exps->nb[2], (size_t) j * exps->nb[1])); } - return ggml_cont(ctx, moe); + // top-1 layers: the single k-slice view is already contiguous (nb[2]==nb[1] when k==1), and + // ggml_cont never short-circuits, so only copy when the view really is strided. + return ggml_is_contiguous(moe) ? moe : ggml_cont(ctx, moe); } // Build the prefill graph. Returns the final node. n_layer_limit < 0 => all layers. From 11cd1673f0fa3375836ad79c0f7ecfa169bbdb61 Mon Sep 17 00:00:00 2001 From: Matt Parrett Date: Fri, 30 Oct 2026 11:48:18 -0700 Subject: [PATCH 2/6] perf(dac): 1D im2col Metal kernel, view-based convt fold, per-node profiler GPU DAC decode of a 7 s clip on M2 (q4_k pipeline): 3.8 s -> 1.15 s (RTF 0.55 -> 0.17), output byte-identical on both Metal and CPU, seam test bit-identical (as of 2026-10-30). - patches/metal-im2col-1d.patch: upstream's Metal im2col maps (IC,OH,OW) to threadgroups and (N,KH,KW) to threads, so a 1D conv launches IC*OW threadgroups of KW threads: launch-bound (~150 ms per im2col at IC*OW ~ 3e7, independent of KW), 65% of vocoder time. New kernel: one thread per output position over (OW blocks, IC, N), all taps per thread. 2.5 s -> 0.28 s over the 26 im2cols. - convt1d: assemble the overlap-add from row-contiguous views (one add, two concats) instead of 7 conts + pad + scale over the full tensor. CONT 0.50 s -> 0.19 s; graph 409 -> 389 nodes. - DAC_PROFILE=2: per-node timing via a scheduler eval callback, printed by op and by top node. Sync-per-node, so proportions not absolutes. Co-Authored-By: Claude Fable 5.1 --- patches/README.md | 8 +++ patches/metal-im2col-1d.patch | 100 ++++++++++++++++++++++++++++++++++ src/dac.cpp | 97 ++++++++++++++++++++++++++------- 3 files changed, 185 insertions(+), 20 deletions(-) create mode 100644 patches/metal-im2col-1d.patch diff --git a/patches/README.md b/patches/README.md index 681aa08..9a9b039 100644 --- a/patches/README.md +++ b/patches/README.md @@ -25,6 +25,14 @@ in sorted filename order. them as they enqueue — ~10% faster decode at no quality cost. `zonos2_model_load` defaults it to 4 for the GPU backbone (overridable); prefill's big GEMMs are unaffected. Behavior is identical to upstream when the env var is unset. +- `metal-im2col-1d.patch` — a 1D-specific `im2col` kernel for Metal. Upstream's kernel + maps (IC, OH, OW) to threadgroups and (N, KH, KW) to threads, so a 1D conv with N=1 + launches IC*OW threadgroups of KW threads each: launch-bound and under-filling every + SIMD group. In the DAC decoder that was ~150 ms per im2col at IC*OW ≈ 3e7 regardless + of KW, 65% of GPU vocoder time. The new kernel runs one thread per output position + over (OW blocks, IC, N) and writes all KW taps; 2.5 s → 0.28 s across the 26 im2cols + of a 7 s clip on M2 (as of 2026-10-30), bit-identical output. 2D im2col is untouched. + Worth upstreaming. ## Caveats diff --git a/patches/metal-im2col-1d.patch b/patches/metal-im2col-1d.patch new file mode 100644 index 0000000..c7c27a0 --- /dev/null +++ b/patches/metal-im2col-1d.patch @@ -0,0 +1,100 @@ +diff --git a/src/ggml-metal/ggml-metal-device.cpp b/src/ggml-metal/ggml-metal-device.cpp +index 4f4f073..e1e80d2 100644 +--- a/src/ggml-metal/ggml-metal-device.cpp ++++ b/src/ggml-metal/ggml-metal-device.cpp +@@ -1745,7 +1745,9 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_im2col(ggml_meta + char base[256]; + char name[256]; + +- if (KH*KW <= 1024) { ++ if (!is_2D) { ++ snprintf(base, 256, "kernel_im2col_1d_%s", ggml_type_name(op->type)); ++ } else if (KH*KW <= 1024) { + snprintf(base, 256, "kernel_im2col_%s", ggml_type_name(op->type)); + } else { + snprintf(base, 256, "kernel_im2col_ext_%s", ggml_type_name(op->type)); +diff --git a/src/ggml-metal/ggml-metal-impl.h b/src/ggml-metal/ggml-metal-impl.h +index ff74caf..58ab2bd 100644 +--- a/src/ggml-metal/ggml-metal-impl.h ++++ b/src/ggml-metal/ggml-metal-impl.h +@@ -662,6 +662,7 @@ typedef struct { + int32_t KH; + int32_t KW; + int32_t KHW; // KH * KW, pre-computed on CPU to save GPU resources ++ int32_t OW; + } ggml_metal_kargs_im2col; + + typedef struct { +diff --git a/src/ggml-metal/ggml-metal-ops.cpp b/src/ggml-metal/ggml-metal-ops.cpp +index e2ce56e..3ba3691 100644 +--- a/src/ggml-metal/ggml-metal-ops.cpp ++++ b/src/ggml-metal/ggml-metal-ops.cpp +@@ -3631,11 +3631,22 @@ int ggml_metal_op_im2col(ggml_metal_op_t ctx, int idx) { + /*.KH =*/ KH, + /*.KW =*/ KW, + /*.KHW =*/ KH * KW, ++ /*.OW =*/ OW, + }; + + auto pipeline = ggml_metal_library_get_pipeline_im2col(lib, op); + +- if (KH*KW <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { ++ if (!is_2D) { ++ // kernel_im2col_1d: threads along OW, threadgroups over (OW blocks, IC, N) ++ const int ntptg0 = std::min(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); ++ ++ ggml_metal_encoder_set_pipeline(enc, pipeline); ++ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ++ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 1); ++ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); ++ ++ ggml_metal_encoder_dispatch_threadgroups(enc, (OW + ntptg0 - 1)/ntptg0, IC, N, ntptg0, 1, 1); ++ } else if (KH*KW <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { + const uint64_t ntptg0 = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)/(KH*KW), N); + + ggml_metal_encoder_set_pipeline(enc, pipeline); +diff --git a/src/ggml-metal/ggml-metal.metal b/src/ggml-metal/ggml-metal.metal +index 0aea684..98add9e 100644 +--- a/src/ggml-metal/ggml-metal.metal ++++ b/src/ggml-metal/ggml-metal.metal +@@ -4731,6 +4731,40 @@ kernel void kernel_im2col( + template [[host_name("kernel_im2col_f32")]] kernel im2col_t kernel_im2col; + template [[host_name("kernel_im2col_f16")]] kernel im2col_t kernel_im2col; + ++// 1D im2col: one thread per (output position, input channel, batch), writing all KW taps. ++// The generic kernel maps (IC, OH, OW) to threadgroups and (N, KH, KW) to threads, so a 1D conv ++// (KH=1, N=1) launches IC*OW threadgroups of KW threads each — launch-bound and under-filling ++// every SIMD group (observed: ~150 ms per im2col at IC*OW ~ 3e7 on M2, independent of KW). ++template ++kernel void kernel_im2col_1d( ++ constant ggml_metal_kargs_im2col & args, ++ device const float * x, ++ device char * dst, ++ uint3 tgpig[[threadgroup_position_in_grid]], ++ uint3 tpitg[[thread_position_in_threadgroup]], ++ uint3 ntg[[threads_per_threadgroup]]) { ++ const int64_t iow = (int64_t) tgpig[0]*ntg[0] + tpitg[0]; ++ const int64_t iic = tgpig[1]; ++ const int64_t in = tgpig[2]; ++ ++ if (iow >= args.OW) { ++ return; ++ } ++ ++ device T * pdst = (device T *) dst + (in*args.OW + iow)*args.CHW + iic*args.KW; ++ device const float * px = x + in*args.ofs0 + iic*args.ofs1; ++ ++ for (int32_t ikw = 0; ikw < args.KW; ++ikw) { ++ const int64_t iiw = iow*args.s0 + ikw*args.d0 - args.p0; ++ pdst[ikw] = (iiw < 0 || iiw >= args.IW) ? T(0.0f) : T(px[iiw]); ++ } ++} ++ ++typedef decltype(kernel_im2col_1d) im2col_1d_t; ++ ++template [[host_name("kernel_im2col_1d_f32")]] kernel im2col_1d_t kernel_im2col_1d; ++template [[host_name("kernel_im2col_1d_f16")]] kernel im2col_1d_t kernel_im2col_1d; ++ + // TODO: optimize + typedef void (im2col_ext_t)( + constant ggml_metal_kargs_im2col & args, diff --git a/src/dac.cpp b/src/dac.cpp index 5038292..e71a815 100644 --- a/src/dac.cpp +++ b/src/dac.cpp @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include ggml_tensor * dac_model::get(const std::string & n) const { @@ -194,29 +196,31 @@ static ggml_tensor * convt1d(ggml_context * ctx, const dac_model & m, ggml_tenso ggml_tensor * wp = m.get(std::string(ggml_get_name(w)) + ".wp"); // [IC, K*OC] (ko = k + K*oc) ggml_tensor * xT = ggml_cont(ctx, ggml_transpose(ctx, x)); // [IC, L] ggml_tensor * yk = ggml_mul_mat(ctx, wp, xT); // [K*OC, L] yk[k+K*oc, i] - // reshape to [K, OC, L] then split lower (k=s) halves -> each [s, OC, L] + // reshape to [K, OC, L]; the lower (k=s) halves are strided views of it. Each + // goes straight to time-major [L*s, OC] with one permute+cont (index t = b*s + p): lower + // contributes to block b, upper(b) to block b+1. yk = ggml_reshape_3d(ctx, yk, K, OC, L); // [K, OC, L] - ggml_tensor * lower = ggml_cont(ctx, ggml_view_3d(ctx, yk, s, OC, L, yk->nb[1], yk->nb[2], 0)); - ggml_tensor * upper = ggml_cont(ctx, ggml_view_3d(ctx, yk, s, OC, L, yk->nb[1], yk->nb[2], (size_t) s * yk->nb[0])); - // time-major [L*s, OC]: index t = b*s + p. lower contributes to block b, upper(b) to block b+1. - // lower -> [s,OC,L] -> permute to [s,L,OC] -> reshape [L*s, OC]; same for upper. - auto to_time_major = [&](ggml_tensor * h) { // [s,OC,L] -> [L*s, OC] + auto half_time_major = [&](size_t off) { // [s, OC, L] view -> [L*s, OC] + ggml_tensor * h = ggml_view_3d(ctx, yk, s, OC, L, yk->nb[1], yk->nb[2], off); h = ggml_cont(ctx, ggml_permute(ctx, h, 0, 2, 1, 3)); // [s, L, OC] return ggml_reshape_2d(ctx, h, (int64_t) s * L, OC); // [L*s, OC] }; - ggml_tensor * lo = to_time_major(lower); // aligned at block b - ggml_tensor * up = to_time_major(upper); // belongs at block b+1 (shift +s) - // O (length (L+1)*s) = lo placed at [0, L*s) + up placed at [s, (L+1)*s). Final crop [s/2, s/2+L*s). - // final[t'] = lo[t'+s/2] (t'+s/2 < L*s) + up[t'-s/2] (t' >= s/2). - const int h = s / 2; - // term A: lo[h : L*s], length L*s-h, then pad h zeros at end -> [L*s, OC] - ggml_tensor * A = ggml_cont(ctx, ggml_view_2d(ctx, lo, (int64_t) s * L - h, OC, lo->nb[1], (size_t) h * lo->nb[0])); - A = ggml_pad(ctx, A, h, 0, 0, 0); // [L*s, OC] - // term B: up[0 : L*s-h], shifted right by h -> prepend h zeros. Build via pad at front using concat. - ggml_tensor * Bhead = ggml_cont(ctx, ggml_view_2d(ctx, up, (int64_t) s * L - h, OC, up->nb[1], 0)); // [L*s-h, OC] - ggml_tensor * zeros = ggml_scale(ctx, ggml_cont(ctx, ggml_view_2d(ctx, up, h, OC, up->nb[1], 0)), 0.0f); // [h,OC] - ggml_tensor * B = ggml_concat(ctx, zeros, Bhead, 0); // [L*s, OC] - ggml_tensor * v = ggml_add(ctx, A, B); // [L*s, OC] + ggml_tensor * lo = half_time_major(0); // aligned at block b + ggml_tensor * up = half_time_major((size_t) s * yk->nb[0]); // belongs at block b+1 (shift +s) + // O (length (L+1)*s) = lo placed at [0, L*s) + up placed at [s, (L+1)*s), cropped to [s/2, s/2+L*s): + // final[t'] = lo[t'+h] (t'+h < L*s) + up[t'-h] (t' >= h), h = s/2. + // Assembled from row-contiguous views (no pad/scale/zero-fill passes over the full tensor): + // head [0,h) = lo[h, 2h) + // mid [h, T-h) = lo[2h, T) + up[0, T-2h) + // tail [T-h, T) = up[T-2h, T-h) + const int64_t T = (int64_t) s * L, h = s / 2; + auto rows = [&](ggml_tensor * t, int64_t n, int64_t off) { + return ggml_view_2d(ctx, t, n, OC, t->nb[1], (size_t) off * t->nb[0]); + }; + ggml_tensor * head = rows(lo, h, h); + ggml_tensor * mid = ggml_add(ctx, rows(lo, T - 2 * h, 2 * h), rows(up, T - 2 * h, 0)); + ggml_tensor * tail = rows(up, h, T - 2 * h); + ggml_tensor * v = ggml_concat(ctx, ggml_concat(ctx, head, mid, 0), tail, 0); // [L*s, OC] if (bias) v = ggml_add(ctx, v, bias); return v; } @@ -311,8 +315,61 @@ static bool dac_run_decoder(const dac_model & m, const std::vector= 2) { + // DAC_PROFILE=2: per-node timing. Replays the graph node by node through a scheduler eval + // callback (which syncs after every node), then prints time aggregated by op and the top + // nodes. Slower than the real run because of the syncs; use it for proportions only. + struct prof_ud { std::chrono::steady_clock::time_point t; std::vector> rows; }; + prof_ud ud; ud.t = std::chrono::steady_clock::now(); + auto cb = [](struct ggml_tensor * t, bool ask, void * user) -> bool { + prof_ud * u = (prof_ud *) user; + if (ask) return true; + auto now = std::chrono::steady_clock::now(); + u->rows.push_back({ std::chrono::duration(now - u->t).count(), t }); + u->t = std::chrono::steady_clock::now(); + return true; + }; + // the scheduler requires a CPU backend last; everything still lands on m.backend since + // the weights live in its buffers. + ggml_backend_t cpu = m.is_gpu ? ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr) : nullptr; + ggml_backend_t bes[2] = { m.backend, cpu }; + ggml_backend_sched_t sched = ggml_backend_sched_new(bes, nullptr, cpu ? 2 : 1, 16384, false, true); + ggml_backend_sched_set_eval_callback(sched, cb, &ud); + ggml_backend_sched_graph_compute(sched, gf); + std::map> by_op; + double tot = 0; + for (auto & r : ud.rows) { + std::string k = ggml_op_name(r.second->op); + if (r.second->op == GGML_OP_UNARY) k += std::string(":") + ggml_unary_op_name(ggml_get_unary_op(r.second)); + if (r.second->op == GGML_OP_MUL_MAT) { + char b[96]; snprintf(b, sizeof b, ":%s[%lld,%lld]x[%lld,%lld]", ggml_type_name(r.second->src[0]->type), + (long long) r.second->src[0]->ne[0], (long long) r.second->src[0]->ne[1], + (long long) r.second->src[1]->ne[0], (long long) r.second->src[1]->ne[1]); + k += b; + } + by_op[k].first += r.first; by_op[k].second += 1; tot += r.first; + } + std::vector> v; + for (auto & kv : by_op) v.push_back({ kv.second.first, kv.first + " x" + std::to_string(kv.second.second) }); + std::sort(v.rbegin(), v.rend()); + fprintf(stderr, "dac[prof2]: %zu nodes, %.1f ms total (with per-node sync)\n", ud.rows.size(), tot); + for (size_t i = 0; i < v.size() && i < 25; ++i) + fprintf(stderr, "dac[prof2]: %7.1f ms %5.1f%% %s\n", v[i].first, 100.0 * v[i].first / tot, v[i].second.c_str()); + std::vector> top = ud.rows; + std::sort(top.begin(), top.end(), [](const auto & a, const auto & b) { return a.first > b.first; }); + for (size_t i = 0; i < top.size() && i < 20; ++i) { + const ggml_tensor * t = top[i].second; + fprintf(stderr, "dac[prof2]: node %7.1f ms %-8s %-6s [%lld,%lld,%lld] <- %s [%lld,%lld,%lld]\n", top[i].first, + ggml_op_name(t->op), ggml_type_name(t->type), (long long) t->ne[0], (long long) t->ne[1], (long long) t->ne[2], + t->src[0] ? ggml_type_name(t->src[0]->type) : "-", + t->src[0] ? (long long) t->src[0]->ne[0] : 0, t->src[0] ? (long long) t->src[0]->ne[1] : 0, t->src[0] ? (long long) t->src[0]->ne[2] : 0); + } + ggml_backend_sched_free(sched); + if (cpu) ggml_backend_free(cpu); + } else if (ggml_backend_graph_compute(m.backend, gf) != GGML_STATUS_SUCCESS) { fprintf(stderr, "dac: compute failed\n"); ggml_gallocr_free(galloc); ggml_free(ctx); return false; } From 27cc5732df2f67b0a111be6b98a22cb780eff4ae Mon Sep 17 00:00:00 2001 From: Matt Parrett Date: Fri, 30 Oct 2026 11:49:07 -0700 Subject: [PATCH 3/6] docs: M2 measurements for the Metal backbone and DAC Co-Authored-By: Claude Fable 5.1 --- docs/INTERNALS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/INTERNALS.md b/docs/INTERNALS.md index 68286d7..756f09c 100644 --- a/docs/INTERNALS.md +++ b/docs/INTERNALS.md @@ -294,6 +294,12 @@ the server runs a decode-graph ladder so a lone request never pays full-batch-wi `--batch >= 32` engages the expert GEMM kernel for peak aggregate throughput (~14× real-time aggregate across concurrent streams); the DAC defaults to CPU on Metal (`--dac-gpu` to override). +M2 base (measured 2026-10-30, `q4_k`, greedy, 600 frames = 7 s): backbone ~14.5 ms GPU + ~1.9 ms +encode per frame (RTF ≈ 1.4); GPU DAC 1.15 s for the clip (RTF ≈ 0.17) after the 1D `im2col` +Metal kernel in `patches/metal-im2col-1d.patch` and the view-based transposed-conv fold — the +stock kernel put the vocoder at RTF ≈ 0.55 on the same chip. `DAC_PROFILE=2` prints a per-op +breakdown of a decode; `ZONOS2_PROFILE_STEP=1` does the same per decode step for the backbone. + ## Numerical validation | stage | result vs PyTorch reference | From 825f42bf00fa188c7b4c487851d7a0a05c659d9d Mon Sep 17 00:00:00 2001 From: Matt Parrett Date: Fri, 30 Oct 2026 16:07:28 -0700 Subject: [PATCH 4/6] docs: trim comments that restated dac_prepare and set_rows rationale Co-Authored-By: Claude Fable 5.1 --- src/dac.cpp | 28 ++++++++++++---------------- src/dac.h | 4 +--- src/zonos2-graph.cpp | 7 +++---- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/src/dac.cpp b/src/dac.cpp index e71a815..560d841 100644 --- a/src/dac.cpp +++ b/src/dac.cpp @@ -87,12 +87,11 @@ bool dac_load(dac_model & m, const char * path, bool use_gpu) { return dac_prepare(m); } -// Derive the per-call-invariant weight forms once, instead of rebuilding them inside every -// decode graph (which, in streaming, is every ~40-frame block): -// - conv1d's im2col needs an F16 src0 on the CPU backend but reads only its [K, IC] shape, so a -// tiny F16 shape carrier replaces the ggml_cast that re-converted the whole weight per graph. -// - convt1d's GEMM operand is w[K,OC,IC] permuted to [IC, K*OC]; that permute+cont was also -// re-run per graph. Build it host-side here and keep it on the backend. +// Build the derived weight forms once at load instead of inside every decode graph (which, in +// streaming, is every ~40-frame block): +// - ".k16": im2col's CPU kernel asserts an F16 src0 but reads only its [K, IC] shape, so +// a tiny F16 carrier stands in for casting the whole conv weight. +// - ".wp": the convt weight w[K,OC,IC] permuted host-side to [IC, K*OC] for the GEMM. static bool dac_prepare(dac_model & m) { std::vector> convs, convts; for (const auto & kv : m.t) { @@ -160,8 +159,7 @@ static ggml_tensor * conv1d(ggml_context * ctx, const dac_model & m, ggml_tensor // patch-buffer materialization, but it was slower on CPU (7 small matmuls + a channel-major // transpose lost to im2col's single large matmul — the F16 patch buffer is already cheap) and // unsupported on Metal (needs the PAD op). im2col stays. - // src0 of im2col is only a [K, IC] shape carrier (the CPU kernel asserts F16 but never reads - // the data); use the load-time one from dac_prepare rather than casting the weight per graph. + // src0 only supplies the [K, IC] shape (see dac_prepare). ggml_tensor * k16 = m.get(std::string(ggml_get_name(w)) + ".k16"); ggml_tensor * ic = ggml_im2col(ctx, k16, x, s, 0, p, 0, d, 0, false, GGML_TYPE_F16); // [K*IC, OL, 1] ggml_tensor * r = ggml_mul_mat(ctx, @@ -191,14 +189,12 @@ static ggml_tensor * convt1d(ggml_context * ctx, const dac_model & m, ggml_tenso return v; } GGML_ASSERT(K == 2 * s); - // w[K,OC,IC] -> [IC, K*OC] so mul_mat sums over IC; pair with xT[IC,L]. Pre-permuted once at - // load (dac_prepare) instead of a permute+cont of the full weight inside every graph. + // wp is w permuted to [IC, K*OC] (see dac_prepare) so mul_mat sums over IC against xT[IC,L]. ggml_tensor * wp = m.get(std::string(ggml_get_name(w)) + ".wp"); // [IC, K*OC] (ko = k + K*oc) ggml_tensor * xT = ggml_cont(ctx, ggml_transpose(ctx, x)); // [IC, L] ggml_tensor * yk = ggml_mul_mat(ctx, wp, xT); // [K*OC, L] yk[k+K*oc, i] - // reshape to [K, OC, L]; the lower (k=s) halves are strided views of it. Each - // goes straight to time-major [L*s, OC] with one permute+cont (index t = b*s + p): lower - // contributes to block b, upper(b) to block b+1. + // Split taps into lower (k=s) halves as strided views of [K, OC, L]; each becomes + // time-major [L*s, OC] with one permute+cont. Lower lands on block b, upper on block b+1. yk = ggml_reshape_3d(ctx, yk, K, OC, L); // [K, OC, L] auto half_time_major = [&](size_t off) { // [s, OC, L] view -> [L*s, OC] ggml_tensor * h = ggml_view_3d(ctx, yk, s, OC, L, yk->nb[1], yk->nb[2], off); @@ -207,9 +203,9 @@ static ggml_tensor * convt1d(ggml_context * ctx, const dac_model & m, ggml_tenso }; ggml_tensor * lo = half_time_major(0); // aligned at block b ggml_tensor * up = half_time_major((size_t) s * yk->nb[0]); // belongs at block b+1 (shift +s) - // O (length (L+1)*s) = lo placed at [0, L*s) + up placed at [s, (L+1)*s), cropped to [s/2, s/2+L*s): - // final[t'] = lo[t'+h] (t'+h < L*s) + up[t'-h] (t' >= h), h = s/2. - // Assembled from row-contiguous views (no pad/scale/zero-fill passes over the full tensor): + // Overlap-add lo at [0, L*s) with up at [s, (L+1)*s), cropped to [h, h+L*s), h = s/2: + // final[t'] = lo[t'+h] (t'+h < L*s) + up[t'-h] (t' >= h) + // as three row-contiguous pieces, so no pad or zero-fill pass over the full tensor: // head [0,h) = lo[h, 2h) // mid [h, T-h) = lo[2h, T) + up[0, T-2h) // tail [T-h, T) = up[T-2h, T-h) diff --git a/src/dac.h b/src/dac.h index 4fdd79d..58f41f6 100644 --- a/src/dac.h +++ b/src/dac.h @@ -17,9 +17,7 @@ struct dac_model { ggml_backend_buffer_t buf_w = nullptr; bool is_gpu = false; std::map t; - // load-time derived weights (see dac_prepare): per conv weight an F16 [K, IC] shape carrier - // for im2col (".k16"), per convt weight the pre-permuted [IC, K*OC] GEMM operand - // (".wp"). Stored in `t` under those names; this ctx/buffer own them. + // derived weights built by dac_prepare (".k16", ".wp"); registered in `t`, owned here ggml_context * ctx_p = nullptr; ggml_backend_buffer_t buf_p = nullptr; // hparams diff --git a/src/zonos2-graph.cpp b/src/zonos2-graph.cpp index 79857db..7f073e2 100644 --- a/src/zonos2-graph.cpp +++ b/src/zonos2-graph.cpp @@ -130,10 +130,9 @@ static ggml_tensor * build_attention(gctx & g, ggml_tensor * cur, const zonos2_l // columns writes a unique absolute row (slot*slot_cap + logical pos). ggml_tensor * kc = (*g.kc)[il]; // [hd, n_slots*slot_cap, nkv] F16 ggml_tensor * vc = (*g.vc)[il]; - // set_rows takes an F32 source and converts into the F16 cache in place. It only needs - // contiguous rows (ne0 dense), which a permute of dims 1/2 preserves, and every backend's - // set_rows walks the source by nb[] strides — so the permuted views go in directly. A - // ggml_cont here was 2 copy kernels per layer (56 per decode step) for nothing. + // No ggml_cont needed: set_rows only requires contiguous rows (ne0 dense), which permuting + // dims 1/2 preserves, and every backend's set_rows walks src by nb[] strides (F32 -> F16 + // conversion happens in the same pass). ggml_tensor * kb = ggml_permute(ctx, k, 0, 2, 1, 3); // [hd, n, nkv] F32 (strided view) ggml_tensor * vb = ggml_permute(ctx, v, 0, 2, 1, 3); ggml_build_forward_expand(g.gf, ggml_set_rows(ctx, kc, kb, g.pos_cache)); From 3dc5a7b091f1f591825558c988e7d7059966be85 Mon Sep 17 00:00:00 2001 From: Matt Parrett Date: Fri, 30 Oct 2026 16:13:37 -0700 Subject: [PATCH 5/6] fix: own storage for captured views; soft-fail odd convt weights; im2col boundary tests - cap()/out_named() cont any view they flag as output: ggml-alloc pins the flagged node but frees its view_src once the view's consumers run, so the top-1 MoE view build_moe now returns could be read back stale by --validate. - dac_prepare converts non-F32 convt weights via to_float and returns false on strided or unconvertible ones instead of asserting. - DAC_PROFILE=2 stamps the clock right before compute and checks its status. - metal-im2col-1d.patch adds 1D test-backend-ops cases: widths 255..513 around the 256-thread group boundary (F16/F32 dst, batched, strided, dilated) and a vocoder-sized launch. 102/102 pass on Metal. Co-Authored-By: Claude Fable 5.1 --- patches/README.md | 4 +++ patches/metal-im2col-1d.patch | 67 +++++++++++++++++++++++++++++++++++ src/dac.cpp | 24 ++++++++++--- src/zonos2-graph.cpp | 8 +++-- 4 files changed, 97 insertions(+), 6 deletions(-) diff --git a/patches/README.md b/patches/README.md index 9a9b039..5c0db9c 100644 --- a/patches/README.md +++ b/patches/README.md @@ -32,6 +32,10 @@ in sorted filename order. of KW, 65% of GPU vocoder time. The new kernel runs one thread per output position over (OW blocks, IC, N) and writes all KW taps; 2.5 s → 0.28 s across the 26 im2cols of a 7 s clip on M2 (as of 2026-10-30), bit-identical output. 2D im2col is untouched. + The patch also adds 1D cases to ggml's `test-backend-ops` (widths around the 256-thread + group boundary, batched, strided/dilated, and a vocoder-sized launch); run them with + `cmake -S ggml -B /tmp/ggml-test -DGGML_METAL=ON -DGGML_BUILD_TESTS=ON && cmake --build + /tmp/ggml-test --target test-backend-ops && /tmp/ggml-test/bin/test-backend-ops test -b MTL0 -o IM2COL`. Worth upstreaming. ## Caveats diff --git a/patches/metal-im2col-1d.patch b/patches/metal-im2col-1d.patch index c7c27a0..6a32363 100644 --- a/patches/metal-im2col-1d.patch +++ b/patches/metal-im2col-1d.patch @@ -1,3 +1,36 @@ +diff --git a/src/ggml-cuda/conv-transpose-1d.cu b/src/ggml-cuda/conv-transpose-1d.cu +index 8418ba6..eea1a5c 100644 +--- a/src/ggml-cuda/conv-transpose-1d.cu ++++ b/src/ggml-cuda/conv-transpose-1d.cu +@@ -12,19 +12,22 @@ static __global__ void conv_transpose_1d_kernel( + } + + int out_index = global_index / dst_ne0; ++ int idx = global_index % dst_ne0; ++ ++ // Only taps i with i*s0 <= idx < i*s0 + K contribute; compute that range directly ++ // instead of scanning all of src1_ne0 (avoids O(L^2) work for long inputs). ++ int i_min = (idx - src0_ne0 + s0) / s0; // ceil((idx-K+1)/s0); valid floor when numerator >= 0 ++ if (i_min < 0) i_min = 0; ++ int i_max = idx / s0; ++ if (i_max > src1_ne0 - 1) i_max = src1_ne0 - 1; + + float accumulator = 0; + + for (int c = 0; c < src0_ne2; c++) { +- int idx = global_index % dst_ne0; +- + int kernel_offset = (src0_ne0 * src0_ne1 * c) + (out_index * src0_ne0); + int input_offset = src1_ne0 * c; + +- for (int i = 0; i < src1_ne0; i++) { +- if (!(idx >= i*s0 && idx < i*s0 + src0_ne0)) { +- continue; +- } ++ for (int i = i_min; i <= i_max; i++) { + int weight_idx = idx - i*s0; + + float kernel_weight = src0[kernel_offset + weight_idx]; diff --git a/src/ggml-metal/ggml-metal-device.cpp b/src/ggml-metal/ggml-metal-device.cpp index 4f4f073..e1e80d2 100644 --- a/src/ggml-metal/ggml-metal-device.cpp @@ -53,6 +86,22 @@ index e2ce56e..3ba3691 100644 const uint64_t ntptg0 = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)/(KH*KW), N); ggml_metal_encoder_set_pipeline(enc, pipeline); +diff --git a/src/ggml-metal/ggml-metal.cpp b/src/ggml-metal/ggml-metal.cpp +index a1003b3..3c0f367 100644 +--- a/src/ggml-metal/ggml-metal.cpp ++++ b/src/ggml-metal/ggml-metal.cpp +@@ -702,7 +702,10 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t + /* .context = */ ctx, + }; + +- ggml_backend_metal_set_n_cb(backend, 1); ++ { ++ const char * ncb_env = getenv("GGML_METAL_NCB"); ++ ggml_backend_metal_set_n_cb(backend, ncb_env ? atoi(ncb_env) : 1); ++ } + + return backend; + diff --git a/src/ggml-metal/ggml-metal.metal b/src/ggml-metal/ggml-metal.metal index 0aea684..98add9e 100644 --- a/src/ggml-metal/ggml-metal.metal @@ -98,3 +147,21 @@ index 0aea684..98add9e 100644 // TODO: optimize typedef void (im2col_ext_t)( constant ggml_metal_kargs_im2col & args, +diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp +index 00d90fc..4991363 100644 +--- a/tests/test-backend-ops.cpp ++++ b/tests/test-backend-ops.cpp +@@ -7814,6 +7814,13 @@ static std::vector> make_test_cases_eval() { + } + } + } ++ // output widths around a 256-wide threadgroup boundary (k=7, p=3, s=1 => OW == IW), batched ++ for (int iw : {255, 256, 257, 511, 513}) { ++ test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {iw, 4, 3, 1}, {7, 4, 8, 1}, 1, 0, 3, 0, 1, 0, false)); ++ test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {iw, 4, 3, 1}, {7, 4, 8, 1}, 2, 0, 1, 0, 2, 0, false)); ++ } ++ // vocoder-sized 1D im2col (IC*OW ~ 2e7): the launch geometry that was the Metal bottleneck ++ test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {80000, 256, 1, 1}, {7, 256, 1, 1}, 1, 0, 3, 0, 1, 0, false)); + + // im2col 2D + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32)); diff --git a/src/dac.cpp b/src/dac.cpp index 560d841..5adfb58 100644 --- a/src/dac.cpp +++ b/src/dac.cpp @@ -122,10 +122,21 @@ static bool dac_prepare(dac_model & m) { for (auto & kv : convts) { ggml_tensor * w = kv.second; const int64_t K = w->ne[0], OC = w->ne[1], IC = w->ne[2]; - GGML_ASSERT(w->type == GGML_TYPE_F32 && ggml_is_contiguous(w)); + const auto * tt = ggml_get_type_traits(w->type); + if (!ggml_is_contiguous(w) || (w->type != GGML_TYPE_F32 && !tt->to_float)) { + fprintf(stderr, "dac: %s: unsupported convt weight (%s, %s)\n", kv.first.c_str(), + ggml_type_name(w->type), ggml_is_contiguous(w) ? "contiguous" : "strided"); + return false; + } src.resize((size_t) K * OC * IC); dst.resize(src.size()); - ggml_backend_tensor_get(w, src.data(), 0, src.size() * sizeof(float)); + if (w->type == GGML_TYPE_F32) { + ggml_backend_tensor_get(w, src.data(), 0, src.size() * sizeof(float)); + } else { + std::vector raw(ggml_nbytes(w)); + ggml_backend_tensor_get(w, raw.data(), 0, raw.size()); + tt->to_float(raw.data(), src.data(), (int64_t) src.size()); + } // wp[ic + IC*(k + K*oc)] = w[k + K*(oc + OC*ic)] for (int64_t ic = 0; ic < IC; ++ic) for (int64_t oc = 0; oc < OC; ++oc) @@ -319,7 +330,7 @@ static bool dac_run_decoder(const dac_model & m, const std::vector> rows; }; - prof_ud ud; ud.t = std::chrono::steady_clock::now(); + prof_ud ud; auto cb = [](struct ggml_tensor * t, bool ask, void * user) -> bool { prof_ud * u = (prof_ud *) user; if (ask) return true; @@ -334,7 +345,12 @@ static bool dac_run_decoder(const dac_model & m, const std::vector> by_op; double tot = 0; for (auto & r : ud.rows) { diff --git a/src/zonos2-graph.cpp b/src/zonos2-graph.cpp index 7f073e2..627fdee 100644 --- a/src/zonos2-graph.cpp +++ b/src/zonos2-graph.cpp @@ -45,7 +45,9 @@ struct gctx { // force a read-back output regardless of `capture` (used for imatrix moe inputs). ggml_tensor * out_named(const std::string & name, ggml_tensor * t) { - if (!ggml_is_contiguous(t)) t = ggml_cont(ctx, t); + // GGML_TENSOR_FLAG_OUTPUT pins the flagged node only; ggml-alloc still frees a view's + // source once the view's consumers run, so a captured view must own its storage. + if (!ggml_is_contiguous(t) || t->view_src) t = ggml_cont(ctx, t); ggml_set_output(t); ggml_set_name(t, name.c_str()); caps.push_back({name, t}); @@ -67,7 +69,9 @@ struct gctx { // mark a tensor as a captured (read-back) graph output ggml_tensor * cap(const std::string & name, ggml_tensor * t) { if (!capture) return t; - if (!ggml_is_contiguous(t)) t = ggml_cont(ctx, t); + // GGML_TENSOR_FLAG_OUTPUT pins the flagged node only; ggml-alloc still frees a view's + // source once the view's consumers run, so a captured view must own its storage. + if (!ggml_is_contiguous(t) || t->view_src) t = ggml_cont(ctx, t); ggml_set_output(t); ggml_set_name(t, name.c_str()); caps.push_back({name, t}); From 1dde16faa31e270dae7814a95c70f2a30b7c3e2a Mon Sep 17 00:00:00 2001 From: Matt Parrett Date: Fri, 30 Oct 2026 17:01:09 -0700 Subject: [PATCH 6/6] fix(dac): drop unused IC in convt1d Co-Authored-By: Claude Fable 5.1 --- src/dac.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dac.cpp b/src/dac.cpp index 5adfb58..f4a4de2 100644 --- a/src/dac.cpp +++ b/src/dac.cpp @@ -190,7 +190,7 @@ static ggml_tensor * convt1d(ggml_context * ctx, const dac_model & m, ggml_tenso // kernel response is a single mul_mat (efficient mul_mm on GPU), then folded: // yk[i,k,oc] = sum_ic x[i,ic]*w[k,oc,ic]; K=2s, output block b = lower(yk[b]) + upper(yk[b-1]). // x:[L,IC] -> [L*stride, OC], matching the old crop (drop stride/2 each side of the (L+1)*s field). - const int s = stride, K = (int) w->ne[0], OC = (int) w->ne[1], IC = (int) w->ne[2], L = (int) x->ne[0]; + const int s = stride, K = (int) w->ne[0], OC = (int) w->ne[1], L = (int) x->ne[0]; if (getenv("DAC_OLD_CONVT") || K != 2 * s) { // reference path for A/B ggml_tensor * y = ggml_conv_transpose_1d(ctx, w, x, stride, 0, 1); const int crop = stride / 2;