From 47061ba60f99c024a99cb962dca1a780e2a90fa3 Mon Sep 17 00:00:00 2001 From: grorge Date: Mon, 26 Oct 2026 19:39:14 +0800 Subject: [PATCH 1/2] fix: stop shelling out for array hashing and VLM decoding fnv1aHash() piped through `python3.10 ../hash_script.py`, a script that is not in the repository, so debugArray() always threw. Compute the 64-bit FNV-1a hash of the array's row-major bytes directly instead. The VLM example built a shell command by concatenating ModelPath. Spawn decode.py with posix_spawnp and an argv array so the path is passed verbatim and no shell is involved. Co-Authored-By: Claude Opus 5.5 --- example/vlm.cpp | 22 +++++++++++++++++++--- mlx/base.cpp | 32 +++++++++++--------------------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/example/vlm.cpp b/example/vlm.cpp index 41d231b..4a27836 100644 --- a/example/vlm.cpp +++ b/example/vlm.cpp @@ -6,8 +6,14 @@ #include #include #include +#include #include +#include #include +#include + +extern char **environ; + int main() { // std::string ModelPath = "../../Llama-3.2-11B-Vision-Instruct-4bit/"; // auto Model = mllama::Model::fromPretrained(ModelPath, {64, 4}); @@ -32,9 +38,19 @@ int main() { mx::array TokenArr = mx::array(TokenList.data(), {static_cast(TokenList.size())}); mx::save("output.npy", TokenArr); - std::system((std::string("python3.10 ../example/decode.py ") + ModelPath + - " output.npy") - .c_str()); + // Run the detokenizer without a shell so ModelPath is passed verbatim. + std::string Python = "python3.10"; + std::string Script = "../example/decode.py"; + std::string Output = "output.npy"; + std::vector Argv = {Python.data(), Script.data(), ModelPath.data(), + Output.data(), nullptr}; + pid_t Pid; + if (posix_spawnp(&Pid, Argv[0], nullptr, nullptr, Argv.data(), environ) == + 0) { + waitpid(Pid, nullptr, 0); + } else { + spdlog::error("Failed to run {}", Script); + } const auto End{std::chrono::steady_clock::now()}; const std::chrono::duration ElapsedSeconds{End - Start}; spdlog::info("Elapsed time: {} s. TPS: {}.", ElapsedSeconds.count(), diff --git a/mlx/base.cpp b/mlx/base.cpp index e8630cd..e9950b2 100644 --- a/mlx/base.cpp +++ b/mlx/base.cpp @@ -1,8 +1,10 @@ #include "base.h" #include "../model/utils.h" #include "spdlog/spdlog.h" +#include #include #include +#include #include namespace mlx::core::nn { @@ -76,29 +78,17 @@ Module::getWeigts(const std::string &Prefix) { } // namespace mlx::core::nn +// 64-bit FNV-1a over the array's row-major bytes. uint64_t fnv1aHash(const mx::array &X) { - std::string FileName = "./temp_array.npy"; - mx::save(FileName.c_str(), X); - std::string Command = "python3.10 ../hash_script.py " + FileName; - - FILE *Pipe = popen(Command.c_str(), "r"); - if (!Pipe) { - throw std::runtime_error("Failed to open pipe for Python script."); - } - - char Buffer[128]; - std::string Result; - while (fgets(Buffer, sizeof(Buffer), Pipe) != nullptr) { - Result += Buffer; - } - pclose(Pipe); - - try { - uint64_t HashVal = std::stoull(Result); - return HashVal; - } catch (const std::exception &e) { - throw std::runtime_error("Failed to parse hash value from Python output."); + mx::array Contiguous = mx::contiguous(X); + Contiguous.eval(); + const auto *Bytes = Contiguous.data(); + uint64_t Hash = 14695981039346656037ULL; + for (size_t I = 0; I < Contiguous.nbytes(); ++I) { + Hash ^= Bytes[I]; + Hash *= 1099511628211ULL; } + return Hash; } void debugArray(const mx::array &X, const std::string &Name) { From f1fffafb59d8dd28b70da91d7668b6e1572e28bd Mon Sep 17 00:00:00 2001 From: grorge Date: Mon, 26 Oct 2026 19:51:21 +0800 Subject: [PATCH 2/2] refactor: take read-only parameters by const reference Pass mx::array, std::string and std::shared_ptr parameters that are only read by const reference instead of by value, as reported by clang-tidy performance-unnecessary-value-param. Declarations and definitions are updated together. Virtual functions (Embedding::forward, Linear::forward and Prompt::prepare) are left unchanged so their overrides keep matching. Co-Authored-By: Claude Opus 5.5 --- mlx/activations.cpp | 6 +++--- mlx/activations.h | 6 +++--- mlx/base.cpp | 4 ++-- mlx/base.h | 9 +++++---- mlx/convolution.cpp | 4 ++-- mlx/convolution.h | 4 ++-- mlx/embedding.cpp | 2 +- mlx/embedding.h | 2 +- mlx/normalization.cpp | 4 ++-- mlx/normalization.h | 4 ++-- mlx/positional_encoding.cpp | 2 +- mlx/positional_encoding.h | 2 +- mlx/quantized.cpp | 10 +++++----- mlx/quantized.h | 6 +++--- model/converter.cpp | 4 ++-- model/converter.h | 5 +++-- model/llm/transformer.cpp | 14 +++++++------- model/llm/transformer.h | 12 ++++++------ model/mllama/language.cpp | 8 ++++---- model/mllama/language.h | 8 ++++---- model/vlm_base.cpp | 2 +- model/vlm_base.h | 2 +- model/whisper/decoding.cpp | 11 ++++++----- model/whisper/decoding.h | 9 +++++---- model/whisper_transcribe.cpp | 7 ++++--- model/whisper_transcribe.h | 6 +++--- 26 files changed, 79 insertions(+), 74 deletions(-) diff --git a/mlx/activations.cpp b/mlx/activations.cpp index 485ce13..469e00a 100644 --- a/mlx/activations.cpp +++ b/mlx/activations.cpp @@ -2,7 +2,7 @@ #include "math.h" #include namespace mlx::core { -mx::array gelu(mx::array X) { +mx::array gelu(const mx::array &X) { // auto Result = X * (1 + mx::erf(X / std::sqrt(2))) / 2; auto Result = X * (mx::array({1}, X.dtype()) + @@ -10,8 +10,8 @@ mx::array gelu(mx::array X) { mx::array({2}, X.dtype()); return Result; } -mx::array silu(mx::array X) { return X * mx::sigmoid(X); } -mx::array geluApprox(mx::array X) { +mx::array silu(const mx::array &X) { return X * mx::sigmoid(X); } +mx::array geluApprox(const mx::array &X) { return mx::array({0.5}, X.dtype()) * X * (mx::array({1}, X.dtype()) + mx::tanh(mx::array({std::sqrt(2.0 / M_PI)}, X.dtype()) * diff --git a/mlx/activations.h b/mlx/activations.h index 939b6d3..4bf2a2f 100644 --- a/mlx/activations.h +++ b/mlx/activations.h @@ -3,7 +3,7 @@ #include #include namespace mlx::core { -mx::array gelu(mx::array X); -mx::array silu(mx::array X); -mx::array geluApprox(mx::array X); +mx::array gelu(const mx::array &X); +mx::array silu(const mx::array &X); +mx::array geluApprox(const mx::array &X); } // namespace mlx::core \ No newline at end of file diff --git a/mlx/base.cpp b/mlx/base.cpp index e9950b2..b470d0c 100644 --- a/mlx/base.cpp +++ b/mlx/base.cpp @@ -9,7 +9,7 @@ namespace mlx::core::nn { -mx::array &Module::registerParameter(std::string Name, mx::array &&W) { +mx::array &Module::registerParameter(const std::string &Name, mx::array &&W) { Parameters.insert({Name, W}); return Parameters.at(Name); } @@ -40,7 +40,7 @@ std::shared_ptr Module::toQuantized( } return shared_from_this(); } -void Module::apply(std::string Key, mx::array Value) { +void Module::apply(std::string Key, const mx::array &Value) { std::vector SplitKey = splitString(Key, '.'); if (SplitKey.size() == 1) { if (Parameters.find(Key) == Parameters.end()) { diff --git a/mlx/base.h b/mlx/base.h index 302c55d..a89eef7 100644 --- a/mlx/base.h +++ b/mlx/base.h @@ -16,7 +16,7 @@ class Module : public std::enable_shared_from_this { std::string Name; std::unordered_map Parameters{}; std::unordered_map> Submodules{}; - mx::array ®isterParameter(std::string Name, mx::array &&W); + mx::array ®isterParameter(const std::string &Name, mx::array &&W); std::unordered_map getWeigts(const std::string &Prefix = "model"); virtual std::shared_ptr toQuantized( @@ -24,9 +24,10 @@ class Module : public std::enable_shared_from_this { const std::unordered_map &Parameters = {}); virtual bool hasQuantize() { return false; } void update(std::unordered_map Parameters); - void apply(std::string Key, mx::array Parameters); + void apply(std::string Key, const mx::array &Parameters); template - void registerModule(std::string ModuleName, std::shared_ptr M) { + void registerModule(const std::string &ModuleName, + const std::shared_ptr &M) { using DecayedT = std::decay_t; if (!std::is_base_of::value) { spdlog::error("Invalid subModule."); @@ -42,7 +43,7 @@ class Module : public std::enable_shared_from_this { } } template - void registerLayer(std::string ModuleName, + void registerLayer(const std::string &ModuleName, std::vector> &Layers) { if (!std::is_base_of::value) { spdlog::error("Invalid subModule."); diff --git a/mlx/convolution.cpp b/mlx/convolution.cpp index 4c055c3..752e7e5 100644 --- a/mlx/convolution.cpp +++ b/mlx/convolution.cpp @@ -7,7 +7,7 @@ namespace mlx::core::nn { -mx::array Conv1d::forward(mx::array Input) { +mx::array Conv1d::forward(const mx::array &Input) { auto Y = mx::conv1d(Input, Parameters.at("weight"), Stride, Padding, Dilation, Groups); if (Parameters.find("bias") != Parameters.end()) { @@ -16,7 +16,7 @@ mx::array Conv1d::forward(mx::array Input) { return Y; } -mx::array Conv2d::forward(mx::array Input) { +mx::array Conv2d::forward(const mx::array &Input) { auto Y = mx::conv2d(Input, Parameters.at("weight"), Stride, Padding, Dilation, Groups); if (Parameters.find("bias") != Parameters.end()) { diff --git a/mlx/convolution.h b/mlx/convolution.h index 7c42c0c..1c0ea2d 100644 --- a/mlx/convolution.h +++ b/mlx/convolution.h @@ -21,7 +21,7 @@ class Conv1d : public nn::Module { registerParameter("bias", mx::zeros({OutChannels})); } } - mx::array forward(mx::array Input); + mx::array forward(const mx::array &Input); }; class Conv2d : public nn::Module { @@ -50,7 +50,7 @@ class Conv2d : public nn::Module { registerParameter("bias", mx::zeros({OutChannels})); } } - mx::array forward(mx::array Input); + mx::array forward(const mx::array &Input); }; } // namespace mlx::core::nn \ No newline at end of file diff --git a/mlx/embedding.cpp b/mlx/embedding.cpp index edd121f..400b71f 100644 --- a/mlx/embedding.cpp +++ b/mlx/embedding.cpp @@ -9,7 +9,7 @@ namespace mlx::core::nn { mx::array Embedding::forward(mx::array Input) { return take(Parameters.at("weight"), Input, 0); } -mx::array Embedding::asLinear(mx::array Input) { +mx::array Embedding::asLinear(const mx::array &Input) { return matmul(Input, transpose(Parameters.at("weight"))); } std::shared_ptr diff --git a/mlx/embedding.h b/mlx/embedding.h index ee79599..fc4a1ef 100644 --- a/mlx/embedding.h +++ b/mlx/embedding.h @@ -13,7 +13,7 @@ class Embedding : public Module { mx::random::normal({NumEmbeddings, Dims}, 0.0, Scale)); } virtual mx::array forward(mx::array Input); - mx::array asLinear(mx::array Input); + mx::array asLinear(const mx::array &Input); std::shared_ptr toQuantized(int GroupSize = 64, int Bits = 4, const std::string &Prefix = "", const std::unordered_map &Parameters = {}) diff --git a/mlx/normalization.cpp b/mlx/normalization.cpp index d5e6387..54fda36 100644 --- a/mlx/normalization.cpp +++ b/mlx/normalization.cpp @@ -2,10 +2,10 @@ #include #include namespace mlx::core::nn { -mx::array RMSNorm::forward(mx::array Input) { +mx::array RMSNorm::forward(const mx::array &Input) { return mx::fast::rms_norm(Input, Parameters.at("weight"), Eps); } -mx::array LayerNorm::forward(mx::array Input) { +mx::array LayerNorm::forward(const mx::array &Input) { std::optional Weight = {}; std::optional Bias = {}; if (Parameters.find("weight") != Parameters.end()) { diff --git a/mlx/normalization.h b/mlx/normalization.h index 15a3ebd..23e086d 100644 --- a/mlx/normalization.h +++ b/mlx/normalization.h @@ -10,7 +10,7 @@ class RMSNorm : public nn::Module { RMSNorm(int Dims, float Eps = 1e-5) : Eps(Eps) { registerParameter("weight", mx::ones({Dims})); } - mx::array forward(mx::array Input); + mx::array forward(const mx::array &Input); }; class LayerNorm : public nn::Module { @@ -27,7 +27,7 @@ class LayerNorm : public nn::Module { } } } - mx::array forward(mx::array Input); + mx::array forward(const mx::array &Input); }; } // namespace mlx::core::nn \ No newline at end of file diff --git a/mlx/positional_encoding.cpp b/mlx/positional_encoding.cpp index fba24b1..538d412 100644 --- a/mlx/positional_encoding.cpp +++ b/mlx/positional_encoding.cpp @@ -1,7 +1,7 @@ #include "positional_encoding.h" namespace mlx::core::nn { -mx::array RoPE::forward(mx::array Input, int Offset) { +mx::array RoPE::forward(const mx::array &Input, int Offset) { return mx::fast::rope(Input, Dims, Tranditional, Base, Scale, Offset); } diff --git a/mlx/positional_encoding.h b/mlx/positional_encoding.h index 5d99790..fd22567 100644 --- a/mlx/positional_encoding.h +++ b/mlx/positional_encoding.h @@ -14,7 +14,7 @@ class RoPE : public Module { RoPE(int Dims, bool Traditional = false, float Base = 10000, float Scale = 1.0) : Dims(Dims), Tranditional(Traditional), Base(Base), Scale(Scale) {} - mx::array forward(mx::array Input, int Offset = 0); + mx::array forward(const mx::array &Input, int Offset = 0); }; } // namespace mlx::core::nn \ No newline at end of file diff --git a/mlx/quantized.cpp b/mlx/quantized.cpp index 8db4a13..d994764 100644 --- a/mlx/quantized.cpp +++ b/mlx/quantized.cpp @@ -22,9 +22,9 @@ mx::array QuantizedLinear::forward(mx::array Input) { } return Out; } -std::shared_ptr -QuantizedEmbedding::fromEmbedding(std::shared_ptr EmbeddingModule, - int GroupSize, int Bits) { +std::shared_ptr QuantizedEmbedding::fromEmbedding( + const std::shared_ptr &EmbeddingModule, int GroupSize, + int Bits) { auto EmbeddingShape = EmbeddingModule->Parameters.at("weight").shape(); auto QuantizedModel = std::make_shared(QuantizedEmbedding( EmbeddingShape[0], EmbeddingShape[1], GroupSize, Bits)); @@ -39,8 +39,8 @@ QuantizedEmbedding::fromEmbedding(std::shared_ptr EmbeddingModule, return QuantizedModel; } std::shared_ptr -QuantizedLinear::fromLinear(std::shared_ptr LinearModule, int GroupSize, - int Bits) { +QuantizedLinear::fromLinear(const std::shared_ptr &LinearModule, + int GroupSize, int Bits) { auto LinearShape = LinearModule->Parameters.at("weight").shape(); auto OutputDims = LinearShape[0]; auto InputDims = LinearShape[1]; diff --git a/mlx/quantized.h b/mlx/quantized.h index e54c9df..563357e 100644 --- a/mlx/quantized.h +++ b/mlx/quantized.h @@ -28,8 +28,8 @@ class QuantizedEmbedding : public Embedding { } mx::array forward(mx::array Input) override; static std::shared_ptr - fromEmbedding(std::shared_ptr EmbeddingModule, int GroupSize = 64, - int Bits = 4); + fromEmbedding(const std::shared_ptr &EmbeddingModule, + int GroupSize = 64, int Bits = 4); }; class QuantizedLinear : public Linear { @@ -52,7 +52,7 @@ class QuantizedLinear : public Linear { } mx::array forward(mx::array Input) override; static std::shared_ptr - fromLinear(std::shared_ptr LinearModule, int GroupSize = 64, + fromLinear(const std::shared_ptr &LinearModule, int GroupSize = 64, int Bits = 4); }; diff --git a/model/converter.cpp b/model/converter.cpp index 0496e5c..bdb414e 100644 --- a/model/converter.cpp +++ b/model/converter.cpp @@ -6,7 +6,7 @@ #include std::unordered_map -weightsToMlx(std::string WeightPath) { +weightsToMlx(const std::string &WeightPath) { const std::filesystem::path Path(WeightPath); if (std::filesystem::is_directory(Path)) { std::unordered_map Loaded; @@ -33,7 +33,7 @@ weightsToMlx(std::string WeightPath) { } std::unordered_map -llamaToMlxllm(std::string WeightPath) { +llamaToMlxllm(const std::string &WeightPath) { std::unordered_map ModelWeights; auto Weight = weightsToMlx(WeightPath); for (auto &[K, V] : Weight) { diff --git a/model/converter.h b/model/converter.h index 3f74ad0..1aa9c8c 100644 --- a/model/converter.h +++ b/model/converter.h @@ -3,7 +3,8 @@ #include "mlx/mlx.h" #include -std::unordered_map weightsToMlx(std::string WeightPath); +std::unordered_map +weightsToMlx(const std::string &WeightPath); std::unordered_map -llamaToMlxllm(std::string WeightPath); \ No newline at end of file +llamaToMlxllm(const std::string &WeightPath); \ No newline at end of file diff --git a/model/llm/transformer.cpp b/model/llm/transformer.cpp index e778b11..08a05f0 100644 --- a/model/llm/transformer.cpp +++ b/model/llm/transformer.cpp @@ -16,12 +16,12 @@ namespace llm { -mx::array RMSNorm::forward(mx::array Input) { +mx::array RMSNorm::forward(const mx::array &Input) { return mx::fast::rms_norm(Input, 1.0 + Parameters.at("weight"), Eps); } std::tuple> -Attention::forward(mx::array Input, std::optional Mask, +Attention::forward(const mx::array &Input, std::optional Mask, std::optional> KVCache) { const auto &[B, L, D] = std::tie(Input.shape()[0], Input.shape()[1], Input.shape()[2]); @@ -66,7 +66,7 @@ Attention::forward(mx::array Input, std::optional Mask, {Keys, Values}}; } -mx::array MLP::forward(mx::array Input) { +mx::array MLP::forward(const mx::array &Input) { if (Gemma) { return std::dynamic_pointer_cast(Submodules["down_proj"]) ->forward( @@ -85,7 +85,7 @@ mx::array MLP::forward(mx::array Input) { std::tuple> TransformerBlock::forward( - mx::array Input, std::optional Mask, + const mx::array &Input, std::optional Mask, std::optional> KVCachePar) { mx::array NormOutput({}); if (!Gemma) { @@ -176,7 +176,7 @@ Transformer::forward( std::tuple>>> -Transformer::stepGenerate(mx::array Input, std::optional Temp) { +Transformer::stepGenerate(const mx::array &Input, std::optional Temp) { // Reshape Input to input[:, None] auto ReshapeDim = Input.shape(); ReshapeDim.insert(ReshapeDim.begin(), 1); @@ -198,7 +198,7 @@ Transformer::stepGenerate(mx::array Input, std::optional Temp) { std::tuple>>> Transformer::nextStepGenerate( - mx::array Y, std::optional Temp, + const mx::array &Y, std::optional Temp, std::optional>> KVCachePar) { // Reshape Y to y[:, None] auto ReshapeDim = Y.shape(); @@ -220,7 +220,7 @@ enum AnserSataus { GO, }; -AnserSataus answerSataus(std::string Text, std::string End) { +AnserSataus answerSataus(const std::string &Text, const std::string &End) { if (endsWith(Text, End)) { return STOP; } diff --git a/model/llm/transformer.h b/model/llm/transformer.h index 9542816..818f13f 100644 --- a/model/llm/transformer.h +++ b/model/llm/transformer.h @@ -26,7 +26,7 @@ class RMSNorm : public nn::Module { RMSNorm(int Dims, float Eps = 1e-5) : Eps(Eps) { registerParameter("weight", mx::ones({Dims})); } - mx::array forward(mx::array Input); + mx::array forward(const mx::array &Input); }; class Attention : public nn::Module { @@ -77,7 +77,7 @@ class Attention : public nn::Module { RopeTheta, RopeScale))); } std::tuple> - forward(mx::array Input, std::optional Mask = {}, + forward(const mx::array &Input, std::optional Mask = {}, std::optional> KVCache = {}); }; class MLP : public nn::Module { @@ -92,7 +92,7 @@ class MLP : public nn::Module { registerModule("up_proj", std::make_shared( nn::Linear(Dim, HiddenDim, false))); } - mx::array forward(mx::array Input); + mx::array forward(const mx::array &Input); }; class TransformerBlock : public nn::Module { bool Gemma; @@ -124,7 +124,7 @@ class TransformerBlock : public nn::Module { } } std::tuple> - forward(mx::array Input, std::optional Mask = {}, + forward(const mx::array &Input, std::optional Mask = {}, std::optional> KVCachePar = {}); }; class Transformer : public nn::Module { @@ -215,10 +215,10 @@ class Transformer : public nn::Module { KVCachePar = {}); std::tuple>>> - stepGenerate(mx::array Input, std::optional Temp = 0.0); + stepGenerate(const mx::array &Input, std::optional Temp = 0.0); std::tuple>>> - nextStepGenerate(mx::array Y, std::optional Temp = 0.0, + nextStepGenerate(const mx::array &Y, std::optional Temp = 0.0, std::optional>> KVCachePar = {}); LLMOutput generate(const std::string &Prompt, const BasePrompt &ModelPrompt, diff --git a/model/mllama/language.cpp b/model/mllama/language.cpp index 65460a9..abe6e8f 100644 --- a/model/mllama/language.cpp +++ b/model/mllama/language.cpp @@ -107,7 +107,7 @@ mx::array MllamaTextCrossAttention::forward( const mx::array &HiddenStates, const std::optional &CrossAttentionStates, const std::optional &AttentionMask, - std::shared_ptr Cache) { + const std::shared_ptr &Cache) { auto Shape = HiddenStates.shape(); int BatchSize = Shape[0]; int QLen = Shape[1]; @@ -179,7 +179,7 @@ MllamaTextSelfAttention::MllamaTextSelfAttention(const TextConfig &Config, mx::array MllamaTextSelfAttention::forward(const mx::array &X, const std::optional &Mask, - std::shared_ptr Cache) { + const std::shared_ptr &Cache) { auto Shape = X.shape(); int BatchSize = Shape[0]; int QLen = Shape[1]; @@ -260,7 +260,7 @@ MllamaSelfAttentionDecoderLayer::MllamaSelfAttentionDecoderLayer( mx::array MllamaSelfAttentionDecoderLayer::forward( const mx::array &HiddenStates, const std::optional &Mask, - std::shared_ptr Cache) { + const std::shared_ptr &Cache) { mx::array Residual = HiddenStates; mx::array Normed = std::dynamic_pointer_cast(Submodules["input_layernorm"]) @@ -298,7 +298,7 @@ mx::array MllamaCrossAttentionDecoderLayer::forward( const mx::array &HiddenStates, const mx::array &CrossAttentionStates, const std::optional &AttentionMask, const std::optional &FullTextRowMaskedOutMask, - std::shared_ptr Cache) { + const std::shared_ptr &Cache) { mx::array Residual = HiddenStates; mx::array Normed = std::dynamic_pointer_cast(Submodules["input_layernorm"]) diff --git a/model/mllama/language.h b/model/mllama/language.h index 67fb70b..fb12f16 100644 --- a/model/mllama/language.h +++ b/model/mllama/language.h @@ -45,7 +45,7 @@ class MllamaTextCrossAttention : public nn::Module { forward(const mx::array &HiddenStates, const std::optional &CrossAttentionStates = std::nullopt, const std::optional &AttentionMask = std::nullopt, - std::shared_ptr Cache = nullptr); + const std::shared_ptr &Cache = nullptr); private: TextConfig Config; @@ -65,7 +65,7 @@ class MllamaTextSelfAttention : public nn::Module { mx::array forward(const mx::array &X, const std::optional &Mask = std::nullopt, - std::shared_ptr Cache = nullptr); + const std::shared_ptr &Cache = nullptr); private: TextConfig Config; @@ -93,7 +93,7 @@ class MllamaSelfAttentionDecoderLayer : public nn::Module { mx::array forward(const mx::array &HiddenStates, const std::optional &Mask = std::nullopt, - std::shared_ptr Cache = nullptr); + const std::shared_ptr &Cache = nullptr); private: int HiddenSize; @@ -108,7 +108,7 @@ class MllamaCrossAttentionDecoderLayer : public nn::Module { const mx::array &HiddenStates, const mx::array &CrossAttentionStates, const std::optional &AttentionMask = std::nullopt, const std::optional &FullTextRowMaskedOutMask = std::nullopt, - std::shared_ptr Cache = nullptr); + const std::shared_ptr &Cache = nullptr); private: int HiddenSize; diff --git a/model/vlm_base.cpp b/model/vlm_base.cpp index cf93f3a..159592b 100644 --- a/model/vlm_base.cpp +++ b/model/vlm_base.cpp @@ -319,7 +319,7 @@ mx::array createAdditiveCausalMask(int N, int Offset) { } std::optional createAttentionMask( - mx::array H, + const mx::array &H, std::optional>> Cache) { int T = H.shape()[1]; std::optional Mask = std::nullopt; diff --git a/model/vlm_base.h b/model/vlm_base.h index dd41c61..58c7d0a 100644 --- a/model/vlm_base.h +++ b/model/vlm_base.h @@ -163,7 +163,7 @@ class LanguageModel : public mlx::core::nn::Module { }; std::optional createAttentionMask( - mx::array H, + const mx::array &H, std::optional>> = std::nullopt); mx::array createAdditiveCausalMask(int N, int Offset = 0); diff --git a/model/whisper/decoding.cpp b/model/whisper/decoding.cpp index 754470c..84a2bef 100644 --- a/model/whisper/decoding.cpp +++ b/model/whisper/decoding.cpp @@ -33,7 +33,7 @@ float compressionRatio(const std::string &Text) { // Language detection implementation std::pair>> -detectLanguage(std::shared_ptr Model, const mx::array &Mel, +detectLanguage(const std::shared_ptr &Model, const mx::array &Mel, std::shared_ptr Tokenizer) { if (!Tokenizer) { @@ -175,8 +175,9 @@ GreedyDecoder::finalize(const mx::array &Tokens, const mx::array &SumLogprobs) { } // SuppressBlank implementation -SuppressBlank::SuppressBlank(std::shared_ptr Tokenizer, - int SampleBegin, int NVocab) +SuppressBlank::SuppressBlank( + const std::shared_ptr &Tokenizer, int SampleBegin, + int NVocab) : SampleBegin(SampleBegin), Mask(mx::zeros({NVocab}, mx::float32)) { Name = "SuppressBlank"; std::vector MaskVec(NVocab, 0.0f); @@ -401,7 +402,7 @@ std::vector MaximumLikelihoodRanker::rank( } // DecodingTask implementation - Constructor and helper methods -DecodingTask::DecodingTask(std::shared_ptr Model, +DecodingTask::DecodingTask(const std::shared_ptr &Model, const DecodingOptions &Options) : Model(Model), Options(verifyOptions(Options)) { @@ -860,7 +861,7 @@ std::vector DecodingTask::run(const mx::array &Mel) { // Main decode function std::variant> -decode(std::shared_ptr Model, const mx::array &Mel, +decode(const std::shared_ptr &Model, const mx::array &Mel, const DecodingOptions &Options) { auto MelArray = Mel; if (Mel.ndim() == 2) { diff --git a/model/whisper/decoding.h b/model/whisper/decoding.h index b7da515..8e05fbf 100644 --- a/model/whisper/decoding.h +++ b/model/whisper/decoding.h @@ -17,7 +17,7 @@ class Whisper; float compressionRatio(const std::string &Text); std::pair>> -detectLanguage(std::shared_ptr Model, const mx::array &Mel, +detectLanguage(const std::shared_ptr &Model, const mx::array &Mel, std::shared_ptr Tokenizer = nullptr); struct DecodingOptions { @@ -123,7 +123,7 @@ class LogitFilter { class SuppressBlank : public LogitFilter { public: - SuppressBlank(std::shared_ptr Tokenizer, int SampleBegin, + SuppressBlank(const std::shared_ptr &Tokenizer, int SampleBegin, int NVocab); mx::array apply(const mx::array &Logits, const mx::array &Tokens) override; @@ -155,7 +155,8 @@ class ApplyTimestampRules : public LogitFilter { class DecodingTask { public: - DecodingTask(std::shared_ptr Model, const DecodingOptions &Options); + DecodingTask(const std::shared_ptr &Model, + const DecodingOptions &Options); std::vector run(const mx::array &Mel); private: @@ -185,7 +186,7 @@ class DecodingTask { }; std::variant> -decode(std::shared_ptr Model, const mx::array &Mel, +decode(const std::shared_ptr &Model, const mx::array &Mel, const DecodingOptions &Options = DecodingOptions()); } // namespace whisper diff --git a/model/whisper_transcribe.cpp b/model/whisper_transcribe.cpp index 12926db..b612350 100644 --- a/model/whisper_transcribe.cpp +++ b/model/whisper_transcribe.cpp @@ -229,7 +229,7 @@ std::optional getEnd(const std::vector &Segments) { // Decoding functions DecodingResult -decodeWithFallback(std::shared_ptr Model, +decodeWithFallback(const std::shared_ptr &Model, const mx::array &MelSegment, const DecodingOptions &DecodeOptions, const std::vector &Temperatures, @@ -285,7 +285,7 @@ decodeWithFallback(std::shared_ptr Model, // Word-level timestamp functions void addWordTimestamps(std::vector &Segments, - std::shared_ptr Model, + const std::shared_ptr &Model, std::unique_ptr &Tokenizer, const mx::array &Mel, int NumFrames, const std::string &PrependPunctuations, @@ -379,7 +379,8 @@ nextWordsSegment(const std::vector &Segments) { // Main transcribe function TranscribeResult transcribe(const std::variant &Audio, - std::shared_ptr Model, std::optional Verbose, + const std::shared_ptr &Model, + std::optional Verbose, std::variant> Temperature, std::optional CompressionRatioThreshold, std::optional LogprobThreshold, diff --git a/model/whisper_transcribe.h b/model/whisper_transcribe.h index 818d0e0..316eafc 100644 --- a/model/whisper_transcribe.h +++ b/model/whisper_transcribe.h @@ -77,7 +77,7 @@ std::optional getEnd(const std::vector &Segments); // Decoding functions DecodingResult -decodeWithFallback(std::shared_ptr Model, +decodeWithFallback(const std::shared_ptr &Model, const mx::array &MelSegment, const DecodingOptions &DecodeOptions, const std::vector &Temperatures, @@ -88,7 +88,7 @@ decodeWithFallback(std::shared_ptr Model, // Word-level timestamp functions void addWordTimestamps(std::vector &Segments, - std::shared_ptr Model, + const std::shared_ptr &Model, std::unique_ptr &Tokenizer, const mx::array &Mel, int NumFrames, const std::string &PrependPunctuations, @@ -104,7 +104,7 @@ nextWordsSegment(const std::vector &Segments); // Main transcribe function TranscribeResult transcribe(const std::variant &Audio, - std::shared_ptr Model, + const std::shared_ptr &Model, std::optional Verbose = std::nullopt, std::variant> Temperature = std::vector{0.0f, 0.2f, 0.4f, 0.6f, 0.8f, 1.0f},